
    IZja                     z   d Z ddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z ddlmZ ddlmZmZmZmZmZmZmZ d	d
lmZmZ d	dlmZmZmZmZmZmZm Z m!Z!  ej"        e#          Z$ ed          Z% e& ee'          j(        dz  dz            Z)h dZ*h dZ+da,ddhZ-h dZ. ed          Z/h dZ0h dZ1h dZ2h dZ3g dZ4dddddZ5d Z6dNd!ed"e7d#dfd$Z8dOd!ed&e7d#dfd'Z9d(ed#efd)Z:d(ed#efd*Z;dNd+ed,e7d#e&fd-Z<d(ed#efd.Z=d+ed#efd/Z>d0ed1ed#e&fd2Z?d3ed+ed#dfd4Z@d+ed#efd5ZAd6ed7e&dz  d#efd8ZBd6ed#e7fd9ZCd!ed&e7d#e&fd:ZDd!ed;eEe&         d&e7d#dfd<ZFd=edz  d&e7d#e&fd>ZGd?d@dAd0ed&e7dBe&d@e&d#e&f
dCZHd+ed#efdDZId+ed#e&fdEZJdFed+ed#efdGZKd!ed#dfdHZLd!ed#dfdIZMd!ed#efdJZNdKed#dfdLZOd!ed#dfdMZPdS )PzE
All functions related to XML generation, processing and validation.
    N)unescape)version)StringIO)dumps)Path)DTDElement
SubElement	XMLParser_Element
fromstringtostring   )Document	Extractor)is_element_in_itemis_first_element_in_itemis_in_table_cellis_last_element_in_cellis_last_element_in_itemsanitizesanitize_treetext_chars_testtrafilaturadataztei_corpus.dtd>   pabhilbdeldivrefrowbodycellcodeheaditemlistquotetablegraphic>   rendroletypetarget	renditionr   r   >   r   r   r)   r*   r+   T)remove_blank_text>   r   r   r#   r'   r)   r*   r+   r,   >   r   r    r"   r%   r&   r'   r(   >	   r   r    r"   r#   r%   r'   r(   r)   r,   >   r%   r(   noter*   figure)sitenametitleauthordateurlhostnamedescription
categoriestagslicenseidfingerprintlanguagez***__`)z#bz#iz#uz#ti  element	keep_tailreturnc                     |                                  }|dS |rJ| j        rC|                                 }||j        pd| j        z   |_        n|j        pd| j        z   |_        |                    |            dS )z
    Removes this element from the tree, including its children and
    text. The tail text is joined to the previous element or parent.
    N )	getparenttailgetprevioustextremove)rE   rF   parentpreviouss       [/Users/jameslopez/projects/MentorCore/.venv/lib/python3.11/site-packages/trafilatura/xml.pydelete_elementrR   V   s    
   F~ AW\ A&&((!;,"<FKK%]0bGL@HM
MM'    Finclude_formattingc                 D   |                                  }|dS t          | |          }| j        
|| j        z  }|                                 }||j        r|j         d| n||_        n |j        |j         d| |_        n||_        |                    |            dS )zAMerge element with its parent and convert formatting to markdown.N )rJ   replace_element_textrK   rL   rM   rN   )rE   rT   rO   	full_textrP   s        rQ   merge_with_parentrY   i   s      F~$W.@AAI|W\!	""$$H:B-V8=669666Y		 22y22
MM'rS   treec                 6   |                      d          D ]}t          |          dk    rmt          |j                  du rWt          |j                  du rA|                                }|+|j        dk    r |j        dk    r|                    |           | S )z"Remove text elements without text.rB   r   FNr,   r&   )iterlenr   rM   rK   rJ   tagrN   )rZ   rE   rO   s      rQ   remove_empty_elementsr_   ~   s    99S>> ' 'w<<1!>!>%!G!GO\c\hLiLimrLrLr&&((F !gkY&>&>6:QWCWCWg&&&KrS   c                    t          |                     d                    D ][}|                    ddd          D ]A}|j        |j        k    r/|                                j        t
          vrt          |           B\| S )z/Prevent nested tags among a fixed list of tags.z.//head | .//code | .//pr&   r'   r   )reversedxpathiterdescendantsr^   rJ   NESTING_WHITELISTrY   )rZ   elemsubelems      rQ   strip_double_tagsrg      s    $>??@@ + +++FFC@@ 	+ 	+G{dh&&7+<+<+>+>+BJ[+[+[!'***	+ KrS   docmetawith_metadatac                 p    |r fd j         D             }|                    |                    d          |                    d          |                    d          d                    |                    d          pg           d                    |                    d          pg           t	          |                    d          d	
          d           |                    d          }ndt	           j        d	
          i} j        }t	          |d	
          |d<   t          |d	          S )z0Build JSON output based on extracted informationc                 4    i | ]}|t          |d           S )N)getattr).0slotrh   s     rQ   
<dictcomp>z%build_json_output.<locals>.<dictcomp>   s'    WWWTdGGT488WWWrS   r9   r5   r;   ;r<   r=   r$   F)rT   )sourcezsource-hostnameexcerptr<   r=   rM   commentsbodyrM   comments)ensure_ascii)	__slots__updatepopjoinxmltotxtr$   rs   
json_dumps)rh   ri   
outputdictrs   s   `   rQ   build_json_outputr}      s2    ,WWWWWEVWWW
$..//#->>*#=#=%>>-88!hhz~~l'C'C'IrJJ!7!7!=2>> !7!7ERRR 		
 		
 		
 "~~n55hw|NNNO
+%luMMMJzju5555rS   c                     |                      d          D ])}|j        t          vr|j                                         *| S )zRemove unnecessary attributes.rB   )r\   r^   WITH_ATTRIBUTESattribclear)rZ   re   s     rQ   clean_attributesr      sB    		#    8?**KKrS   c                    t          d          }t          ||            d| j        _        |                    t          | j                             d| j        _        |                    t          | j                             |S )z4Build XML output tree based on extracted informationdocmainrt   )r	   add_xml_metar$   r^   appendr   rs   rh   outputs     rQ   build_xml_outputr      st    U^^F!!!GL MM"7<00111)G
MM"7#788999MrS   documentoptionsc                    t          | j                   t          | j                   |j        dk    rt          nt
          } ||           }t          |          }t          t          |d          t                    }|j        dk    r5|j
        r.t                              dt          |          |j                   t          |dd                                          S )z9Make sure the XML output is conform and valid if requiredxmlunicode)encodingxmlteizTEI validation result: %s %sT)pretty_printr   )rg   r$   r_   formatr   build_tei_outputr   r   r   CONTROL_PARSERtei_validationLOGGERdebugvalidate_teirq   strip)r   r   funcoutput_trees       rQ   control_xml_outputr      s    hm$$$(-(((&~66<LD$x..K,,KXkIFFFWWK ~!!g&<!3\+5N5NPWP^___KdYGGGMMOOOrS   r   c                     t           D ]U}t          ||d          }|r@|                     |t          |t                    r|nd                    |                     VdS )z-Add extracted metadata to the XML output treeNrp   )META_ATTRIBUTESrl   set
isinstancestrry   )r   rh   	attributevalues       rQ   r   r      sl    $ X X	D11 	XJJy:eS+A+A"V%%sxxPUWWWX XrS   c                 N    t          |           }t          || j                  }|S )z8Build TEI-XML output tree based on extracted information)write_teitree	check_teir9   r   s     rQ   r   r      s)     7##F vw{++FMrS   xmldocr9   c                    |                      d          D ]}d|_        |                    dd           |                                }|6t	          |          dk    r't          |          }|                    ||           |}|j        dk    rt          |           |                     d          D ]=}|j	        r4|j	        
                                rd|j	        dc|_        |_        |_	        >|                     d	          D ]}|j        t          vr1t                              d
|j        |           t          |           A|j        t           v rt#          |           n)|j        dk    rt%          |           t'          |           d |j        D             D ]>}t                              d||j        |           |j                            |           ?| S )zCCheck if the resulting XML file is conform and scrub remaining tagsr'   r   r/   headerNr   r   z.//text/body//div/lbz.//text/body//*z"not a TEI element, removing: %s %sr!   c                 $    g | ]}|t           v|S  )TEI_VALID_ATTRS)rm   as     rQ   
<listcomp>zcheck_tei.<locals>.<listcomp>
  s"    MMMA_4L4L!4L4L4LrS   z0not a valid TEI attribute, removing: %s in %s %s)r\   r^   r   rJ   r]   _tei_handle_complex_headreplace_move_element_one_level_upfindallrK   r   rM   TEI_VALID_TAGSr   warningrY   TEI_REMOVE_TAIL_handle_unwanted_tails!_handle_text_content_of_div_nodes_wrap_unwanted_siblings_of_divr   rx   )r   r9   re   rO   new_elemr   s         rQ   r   r      s    F## - -"""!!>t99q==/55HNN4***D:&t,,,566 B B9 	B** 	B-0$)T*DHdi011 ' '8>)) NN?3OOOd###8&&"4((((X-d333*4000 NMT[MMM 	' 	'INNMyZ^ZbdghhhKOOI&&&&	' MrS   c                     t           t          t                    a t                               |           }|du r*t                              dt           j        j                   |S )zUCheck if an XML document is conform to the guidelines of the Text Encoding InitiativeNFznot a valid TEI document: %s)TEI_DTDr   
TEI_SCHEMAvalidater   r   	error_log
last_error)r   results     rQ   r   r     sP     j//f%%F5w7H7STTTMrS   c                 n   | j         pd}|rX| j         rP| j        dv r|                                }n1| j        dk    rN	 t          |                     d          d                   }n# t
          t          f$ r d}Y nw xY wd|z   d| }n| j        d	k    rd
| d
}n| j        dk    r=|                     d          }|t          v rt          |          | t          |          }n~| j        dk    rsd|v s|                     d          rT|                     d          D ]7}| d|j	        pd }|
                                                    |           8d| d}nd| d}| j        dk    rm|rJd| d}|                     d          }|r	| d| d}nEt                              d|| j                   |}n!t                              d|| j                   | j        dk    r*|                                }|rt          |           s| d}t!          |           rt#          |           sd| }|S )zeDetermine element text based on just the text of the element. One must deal with the tail separately.rI   )articler)   r+   r'   r-   r      #rV   r    z~~r   r&   
z.//lbz```
z
```
rD   r"   []r0   ()zmissing link attribute: %s %s'zempty link: %s %sr%   z- )rM   r^   r   intget	TypeError
ValueErrorHI_FORMATTINGrb   rK   rJ   rN   r   r   r   r   r   r   )rE   rT   	elem_textnumberr-   r   	link_textr0   s           rQ   rW   rW     s   "I -gl -;666!))II[F""W[[00344z*   <55)55II[E!!*Y***II[D  ;;v&&D}$$,T2TIT}T?RTT	[F""y  GMM'$:$: !--00 . .B#, ? ?2 ? ?ILLNN))"----6I666		,	,,,	{e 		K(I(((I[[**F &(446444		?GN[[[%		NN.	7>JJJ{fOO%%	 	(4W== 	($I  (( %1A'1J1J %$$$	s    (A) )A?>A?
returnlistc           	         | j         dk    r)|                                 |                    d           | j        r#|                    t	          | |                     | j        rF| j         dk    r;t          |           r,|                    | j                                                   | D ]}t          |||           | j        s| j         dk    r| 	                    dd           d| 	                    dd           }|                    d	|                                 d
| 	                    dd           d           | j        r/|                    d| j                                                    n$| j         t          v r| j         dk    rt          |                     d                    }| 	                    d          p| 	                    d          }|r|                                sd}n"t          t          |          t                     }||k     r|                    d||z
  z   d           |                     d          r|                    dd|z   d           n.|                    d           n| j         dk    r| j         dk    rdS | j         t          v rI|                     d          s4t#          |           s%|                    |r| j         dk    rdnd           nS| j         dk    r|                    d           n2| j         t$          vr$t'          |           s|                    d           t          |           }| j        rO|sM|                    t#          |           s| j         dk    r| j                                        n| j                   t)          |           r|s|                    d           dS dS dS )zYRecursively convert a LXML element and its children to a flattened string representation.r%   Nz| r,   r6   rI   rV   altz![z](srcr   r#   z.//cellcolspanspanr   |r   z./cell[@role='head']z
|z---|r(   zancestor::cellu   
␤
z | r)   )r^   rL   r   rM   rW   rK   r   r   process_elementr   NEWLINE_ELEMSr]   rb   isdigitminr   MAX_TABLE_WIDTHr   SPECIAL_FORMATTINGr   r   )	rE   r   rT   childrM   
cell_count	span_infomax_span
is_in_cells	            rQ   r   r   U  s   {f!4!4!6!6!>$| M.w8JKKLLL| 0y005Eg5N5N0 	',,,../// ? ?z+=>>>>< ;)##kk'2..IIUB1G1GIIDL4::<<LL7;;ub3I3ILLLMMM| >!!"<gl&8&8&:&:"<"<===[M)){e## y!9!9::
#KK	22Igkk&6I6I	  D	(9(9(;(; D HH"3y>>?CCH((%%:0E)F&J&J&JKKK==!788 C%%&AFX,=&A&A&ABBB!!$''''[F""w{f'<'< F
 {m##GMM:J,K,K#TfgnToTo#*<_PUAUAU,,[_````			%    	.	.	.7Nw7W7W	.# "'**J| zJ z2DW2M2MxQXQ\`fQfQf',,,...lslxyyy w''  
  $       rS   	xmloutputc                     | dS g }t          | ||           t          t          d                    |          d          pd          S )zLConvert to plain text format and optionally preserve formatting as markdown.NrI   T)r   r   r   ry   )r   rT   r   s      rQ   rz   rz     sN    rJIz+=>>>HRWWZ00$77=2>>>rS   	null)delimr   r   c                   t          | j        |          p}t          | j        |          p}t                      }t	          j        ||t          j                  }|                    fd| j        | j	        | j
        | j        | j        | j        | j        ||| j        | j        fD                        |                                S )zAConvert the internal XML document representation to a CSV string.)	delimiterquotingc                     g | ]}|r|n	S r   r   )rm   dr   s     rQ   r   zxmltocsv.<locals>.<listcomp>  s2     	
 	
 	
 AA	
 	
 	
rS   )rz   r$   rs   r   csvwriterQUOTE_MINIMALwriterowr9   r?   r@   r:   r6   imager8   r>   pagetypegetvalue)r   rT   r   r   posttextcommentstextr   outputwriters      `    rQ   xmltocsvr     s     '9::BdHH13EFFN$L ZZF:fs?PQQQL 	
 	
 	
 	
 $! !	
 	
 	
  $ ??rS   c                    t          dd          }t          ||            t          |d          }t          |d          }t          | j                  }d|_        |                    dd           |                    |           t          | j                  }d|_        |                    dd	           |                    |           |S )
z6Bundle the extracted post and comments into a TEI treeTEIzhttp://www.tei-c.org/ns/1.0)xmlnsrM   r$   r!   r/   entryrt   )	r	   write_fullheaderr
   r   r$   r^   r   r   rs   )rh   teidoctextelemtextbodypostbodyrs   s         rQ   r   r     s    U"?@@@FVW%%%&&))H(F++H--HHLLL!!!OOH#G$899LLVZ(((OOL!!!MrS   c                     | j         r,| j        r%| j                                         d| j          d}nZ| j         p| j        pd}t                              t
          j                  r&|dk    r t                              d| j                   |S )z5Construct a publisher string to include in TEI headerz (r   zN/Azno publisher for URL %s)	r:   r5   r   r   isEnabledForloggingWARNINGr   r9   )rh   	publishers     rQ   _define_publisher_stringr    s     CG, C'--//FF73CFFF		$A(8AE	w// 	CI4F4FNN4gkBBBrS   r   c                    t          | d          }t          |d          }t          |d          }|j        t          |dd          _        |j        r|j        t          |d          _        t          |d          }t	          |          }|j        r@|t          |d	          _        t          |d
          }|j        t          |d          _        nt          |d           t          |d          }|j        r|j        t          |dd          _        |j        t          |dd          _        t          |d          }	t          |	d          }
d                    t          d|j
        |j        g                    }|s t                              d|j                   d                    t          d|j        |g                    |
_        |t          |	dd          _        t          |	d          }t          |d          }|j        t          |dd          _        |j        r|j        t          |d          _        t          |d          }|t          |d	          _        |j        rt          |dd|j                   |j        t          |d          _        t          |d          }t          |d          }|j        t          |d          _        |j        s|j        rt          |d          }t          |d          }|j        r/d                    |j                  t          |d d!          _        |j        r/d                    |j                  t          |d d"          _        t          |d#          }|j        t          |dd$          _        t          |d%          }t          |d&          }t          |d't&          d()          }d(t          |d*          _        t          |dd+,           |S )-z+Write TEI header based on gathered metadata	teiHeaderfileDesc	titleStmtr6   r   )r/   r7   publicationStmtr  availabilityr   	notesStmtr3   r?   r@   
sourceDescbiblz, Nzno sigle for URL %ssiglebiblFullptrURL)r/   r0   r8   profileDescabstract	textClasskeywords,termr<   r=   creationdownloadencodingDescappInfoapplicationTrafilatura)r   identlabelz$https://github.com/adbar/trafilatura)r0   )r
   r6   rM   r7   r  r>   r?   r@   ry   filterr5   r8   r   r   r9   r;   r<   r=   filedatePKG_VERSION)r   rh   r   filedescbib_titlestmtpublicationstmt_apublisher_stringr  	notesstmt
sourcedescsource_biblr  biblfullpublicationstmtprofiledescr  	textclassr  r  encodingdescappinfor   s                         rQ   r   r     s    ,,F&*--Hx55M;B=J}gF3338~ B3:>
=(++0"8->??/88 +:J
$k227!"3^DD-4_
<%%** 	$c***8[11Iz C8?

9f40005=D=PJy&}555:Hl33JZ00KIIfTG$4gl#CDDEEE ;,gk:::yyw}e.D!E!EFFK8=Jz60005*j11Hx55M;B=J}gF3338~ B3:>
=(++0 +<==O4DJ,,1{ K?EgkJJJJ/6|J'',V]33K+z22H%,%8Jx" TW\ T{K88	i44 	`CF88GL^C_C_Jxl;;;@< 	T=@XXgl=S=SJxf555:+z22H9@9IJxj1116fn55Ly11GWm[P]^^^K,9J{G$$){E*PQQQQMrS   c                    | j         r| j                                         rt          |           dk    rH| d         j        dk    r7| j          d| d         j         pd                                 | d         _         n1t	          d          }| j         |_         |                     d|           d| _         | j        r| j                                        rt          |           dk    rH| d         j        dk    r7| d         j         pd d| j                                         | d         _         n0t	          d          }| j        |_         |                     |           d| _        dS dS dS )z@Wrap loose text in <div> within <p> elements for TEI conformity.r   r   rV   rI   N)rM   r   r]   r^   r	   insertrK   r   )rE   	new_childs     rQ   r   r   -  sa   | **,, w<<!
# 5 5!(GG
0E2GGMMOOGAJOOI$\INNN1i(((| **,, w<<!3 6 6")"+"2"8bII7<IIOOQQGBKI$\INNN9%%%   rS   c                    | j         r| j                                         nd| _         | j         sdS | j        dk    r5d                    t	          d| j        | j         g                    | _        n]t          d          }| j         |_        |                                 }|,|                    |	                    |           dz   |           d| _         dS )z Handle tail on p and ab elementsNr   rV   r   )
rK   r   r^   ry   r$  rM   r	   rJ   r6  index)rE   new_siblingrO   s      rQ   r   r   B  s    +2<A7<%%'''TGL< {cxxtglGL-I J JKKcll"<""$$MM&,,w//!3[AAAGLLLrS   c                    t          d| j                  }| j        r| j                                        nd|_        |                                 D ]}|j        dk    rjt          |          dk    s|j        rCt          |          dk    s|d         j        rt          |d           |j        |d         _        j|j        |_        w|	                    |           | j        r| j                                        nd}|r||_        |S )z0Convert certain child elements to <ab> and <lb>.r   )r   Nr   r   r5  r   )
r	   r   rM   r   iterchildrenr^   r]   rK   r
   r   )rE   new_elementr   rK   s       rQ   r   r   S  s   $w~666K/6|Ew|))+++K%%'' 
& 
&9;!##{'7#{##q((KO,@({D111',zB$$#(:  u%%%%#*<97<TD  rS   div_elementc                    t          d          }d}|                                 }|dS |                                 D ]}|j        dk    r nx|j        t          v r-|p|                    |          }|                    |           J|r:t          |          dk    r'|                    ||           t          d          }d}|r+t          |          dk    r|                    ||           dS dS dS )z=Wrap unwanted siblings of a div element in a new div element.r!   Nr   )	r	   rJ   itersiblingsr^   TEI_DIV_SIBLINGSr9  r   r]   r6  )r>  r:  new_sibling_indexrO   siblings        rQ   r   r   h  s   %..K""$$F~++-- ) );%E;*** 1 JV\\'5J5Jw'''' ! )S%5%5%9%9/===%enn$(! 6S--22'555556 622rS   c                    |                                  }||                                 nd}||dS t          d          }|                    t          |                                                      |                    |                    |          dz   |            | j        r| j                                        nd}|r||_	        d| _        |j        r|j                                        nd}|r||_        d|_        t          |          dk    s|j	        s|j        r,|                    |                    |           dz   |           t          |          dk    r|j	        s|                    |           dS dS dS )z
    Fix TEI compatibility issues by moving certain p-elems up in the XML tree.
    There is always a n+2 nesting for p-elements with the minimal structure ./TEI/text/body/p
    Nr   r   r   )rJ   r	   extendr)   r@  r6  r9  rK   r   rM   r]   rN   )rE   rO   grand_parentr   rK   s        rQ   r   r     s   
   F)/);6##%%%L~-s||HOOD--//00111**622Q6@@@#*<97<TD "(+76;4D 
8}}qHMX]L..w77!;XFFF
6{{aF##### rS   )T)F)Q__doc__r   r  htmlr   importlib.metadatar   ior   jsonr   r{   pathlibr   
lxml.etreer   r	   r
   r   r   r   r   settingsr   r   utilsr   r   r   r   r   r   r   r   	getLogger__name__r   r&  r   __file__rO   r   r   r   r   r   rA  r   r   r   r   rd   r   r   r   boolrR   rY   r_   rg   r}   r   r   r   r   r   r   r   rW   r)   r   rz   r   r   r  r   r   r   r   r   r   r   rS   rQ   <module>rT     s    


        & & & & & &       $ $ $ $ $ $       Z Z Z Z Z Z Z Z Z Z Z Z Z Z Z Z Z Z ) ) ) ) ) ) ) )	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 
	8	$	$gm$$ Sh&/2BBCC
  & BAA
+888 T222OOOIII XXX???     3d#>> H      & x T d    *	 	X 	 	 	 	H     6 6x 6 6 6 6 6 608     h 8    P PI P# P P P P&X XH X X X X Xh 8    'h 'S4Z 'H ' ' ' 'T d    3( 3 3 3 3 3 3lD X D 49 D RV D [_ D  D  D  D N	?4 	?T 	?c 	? 	? 	? 	? LP]c   x T S WZ hk    @8     &h 3    DX D DX D D D DNx D    *H     "h 8    *6 6T 6 6 6 62$ $T $ $ $ $ $ $rS   