IZj'< dZddlZddlmZddlmZmZddlmZm Z m Z m Z m Z m Z ddlmZddlmZdd lmZmZmZmZmZdd lmZmZmZdd lmZmZeje Z!d d d d dddddddd Z"de"#DZ$hdZ%gdZ&dededefdZ'dYdede(defdZ)dZdede*e de+defd Z,d!e*ede-e.e.e.e*e(ffd"Z/dZd#ed$e(d%e+de-e+e*e(ffd&Z0d#ede+fd'Z1 d[d(ed)e(d*e+d%e+def d+Z2 d\d-e ded.e+d/e+de dzf d0Z3d-e dede dzfd1Z4d-e ddfd2Z5d-e ddfd3Z6d$e(dzde+fd4Z7d-e ddfd5Z8d-e ddfd6Z9d-e ddfd7Z:d-e ddfd8Z;id9e5d:e5d;e5de8d?e8d@e8dAe8dBe9dCe9dDe6dEe6dFe6dGe:dHe:dIe:dJe;iZd;dOdEdDdPdBdQdRdSdT Z?de de fdUZ@dZdVedWe+de(fdXZAdS)^z* Functions to process nodes in HTML code. N)deepcopy)fix_relative_urls get_base_url)Element SubElementXPath_Element strip_tagstostring) HtmlElement)duplicate_test)CUT_EMPTY_ELEMSMANUALLY_CLEANEDMANUALLY_STRIPPEDDocument Extractor)is_image_element textfiltertrim)META_ATTRIBUTESdelete_element#iz#bz#uz#tz#subz#sup) emibstrongukbdsampttvarsubsupci|]\}}|| Sr&).0kvs f/Users/jameslopez/projects/MentorCore/.venv/lib/python3.11/site-packages/trafilatura/htmlprocessing.py r+(s>>>TQAq>>>>figuresourcepicture){z("z('z treeoptionsreturnctjtj}}|js|gdn|dD] }d|_ |jr!d|D}|dt|||j dkrh| dSt|}|D])}| |D]}t|*| d|}n,|D])}| |D]}t|*t||j S)z/Prune the tree by discarding unwanted elements.)tabletdthtrz.//figure[descendant::table]divc$g|] }|tv |Sr&)PRESERVE_IMG_CLEANINGr'es r* z!tree_cleaning..;s#TTTqQ>S5S5S5S5S5Sr,imgrecallz.//p)rcopyrtablesextendxpathtagimagesremover focusfindriterr prune_html)r1r2 cleaning_liststripping_listelemtcopy expressionelements r* tree_cleaningrR/s%5$9$;$;=N=S=U=U>M >8889999JJ=>>  DDHH~%TTMTTT e$$$t^$$$}  TYYv%6%6%B' ( (J99Z00 ( (w'''' ( 99V   $D( ( (J99Z00 ( (w'''' ( dGM * **r,balancedrHc|dk}|dD]!}|jtvrt||"|S)zADelete selected empty elements to save space and processing time. precisionz-.//processing-instruction()|.//*[not(node())]) keep_tail)rDrErr)r1rHtailsrQs r*rKrKRsN [ E::MNN55 ;/ ) ) 7e 4 4 4 4 Kr,Fnodelist with_backupc|r0t|}t|}|D]}||D]}t| |r.t|}||dz kr|n|S|S)z2Prune the HTML tree by removing unwanted sections.)len text_contentrr)r1rXrYold_lenbackuprPsubtreenew_lens r*prune_unwanted_nodesrb\s d''))**$$$ !z$'' $ $G 7 # # # #  $ 9d''))**1,,tt&8 Kr, links_xpathcdd|DD}ttt|}td|D}t|t|||fS)zCollect heuristics on link textcg|]}||Sr&r&r<s r*r>z%collect_link_info..ts R R RAPQ Ra R R Rr,c3XK|]%}t|V&dSN)rr])r'rNs r* z$collect_link_info..ts6LL$t002233LLLLLLr,c3&K|] }|dkdV dS) r Nr&)r'lengths r*rhz$collect_link_info..ws&<<6 Q <Remove sections which are rich in links (probably boilerplate).//refFz .//graphicNr rjdrg?Tp<i,u8list link text/total: %s/%s – short elems/total: %s/%s皙?) findallrIr\rr]rEgetnextrrLOGGERdebug) rQrsrtrcro len_threshold link_textlimitlenelemlenlinklenelemnumrqs r*link_density_testr{s//(++K by||L!!-byF ;1-63 Q446677 y>>M ) )c)nns4yy3.N.N8O{c **222 ??   $HHH$iiG/@/M/M,*f a<<<  F         Ws] " "w{{zG7Kc7Q7Q<  &=r,c@|d}|sdStt|}|dkrdSt |\}}}}|dkrdSt d|||dkr |d|zkn|d |zkS) z=Remove tables which are rich in links (probably boilerplate).rvFrTztable link text: %s / total: %sir{g?)r|r\rr]rrr~r)rQrcrrr_s r*link_density_test_tablesrs//(++K u$w++--..//G}}u.{;;GWa!||t LL2GWEEE&-nn7S7] " "'C'M:QQr,r`tagname backtrackingcg}|rdnd}|rdnd}||D]}t|}t|||\} } | s4|rG| rEdt |cxkr|kr+nZt ||kr||t |D]}t||S)z{Determine the link density of elements with respect to their length, and remove the elements identified as boilerplate.rrwr r) rJrr]rr\appenddictfromkeysr) r`rrrt deletionsrdepth_thresholdrNelemtextresulttemplists r*delete_by_link_densityrs I*3CCM*1aaO W%%##))++,,,T8_MM  #l #x #AH 4U4U4U4U 4U4U4U4U4UZ]^bZcZcgvZvZv   T " " " i((t Nr,TrN comments_fixpreserve_spacescR|jdkrt|r|S|jdks!t|dkr|js |jsdS|s*|jdkr|st |jpd|_|S|js;t|dkr(|jdc|_|_|r|jdkrd|_|s=t |jpd|_|jrt |jpd|_|jst |s|jrt||rdS|S)z3Convert, format, and probe potential text elements.graphicdonerNlbrx) rErr\rstailrrdedupr)rNr2rrs r*handle_textnodersL x9!1$!7!7  x6c$ii1nnTYntynt DH,, 0TY/4DI 9Ta $y" 49  DH,,DH 0OO+t 9 0TY/4DI 9D))gmtU\@]@]t Kr,c|jdks!t|dkr|js |jsdSt |jpdt |jpdc|_|_|jdkr"|js|jr|jdc|_|_|js|jr(t |s|jrt||rdS|S)zBConvert, format, and probe potential text elements (light format).rrNr)rEr\rsrrrrr)rNr2s r* process_noders x6c$ii1nnTYntynt  ??2dDOO4KtDIty x4 di#y$ 49 yDI d     .w2O2O 4 Kr,c|d|jd|_d}|dddD]O}|jdvr=|dt|jd||jdkr|dz }d |_Pd S) zGConvert
    and
      to and underlying
    1. elements to .rendrlr dddtli)rr-itemN)setrErJstr)rNrsubelems r* convert_listsrsHHVTXDH A99T4.. ;, & & KK3w{#3#3 9 9a 9 9 : : :{d""Q r,cd}|jdkrst|dkr|djdkrd}|d}|r d}|D]}|jt |jrd}|rdnd |_d S) z?Convert quoted elements while accounting for nested structures.Fprer rspanTz#.//span[starts-with(@class,'hljs')]codequoteN)rEr\rDattribclear_is_code_blockrs)rN code_flag code_elemsrs r*convert_quotesrsI x5 t99>>d1gkV33IZZ EFF  'I% ' '$$&&&& $) $ $ I"/vvDHHHr,c0|sdStD] }||vrdS dS)z2Check if the element text is part of a code block.FT)CODE_INDICATORS)rs indicators r*rr2s: u$   44  5r,c||j|d|jd|_dS)z$Add head tags and delete attributes.rheadN)rrrrErNs r*convert_headingsr<s7KHHVTXDHHHr,cd|_dS)zConvert
      and
      to rN)rErs r*convert_line_breaksrCs DHHHr,c@d|_|dddS)z7Convert , , to delr overstrikeN)rErrs r*convert_deletionsrHs#DHHHV\"""""r,cRd|_|dD] }d|_ dS)zHandle details and summary.r9summaryrN)rErJ)rNrs r*convert_detailsrNs7DH99Y'' r,dlolulh1h2h3h4h5h6brhr blockquoterqrsstrikedetailsbase_urlcd|_|d}|j|r*|rt ||}|d|dSdS)z7Replace link tags and href attributes, delete the rest.refhreftargetN)rEgetrrrr)rNrrs r* convert_linkrlsqDH XXf  FK #  9&x88F 6""""" ##r,urlc||js>d}|jr|dz }||D] }d|_ t |dn:|ot |}|ddD]}t|||jrv|t D]H}|j | dt|jd|_In$t |gt R|t D]}t|j||jr|dD] }d|_ |jrt!|dD]}t!|d}t#|D]}|||r5|st+||S) zBSimplify markup and convert relevant HTML tags to an XML standard.z).//*[self::div or self::li or self::p]//az |.//table//ararhir?r)linksrBrDrEr rrJr formattingREND_TAG_MAPPINGkeysrrr CONVERSIONSrFrlreversedaddnextr]stripr) r1r2r xpath_exprrNrrgraphicsrs r* convert_tagsrxsV = )@ > ) . (JJJz**  DDHH4,<,,IIc5)) ) )D x ( ( ( (3II.335566  D K      HHV-dh7 8 8 8DHH  42*//112222 +**,,--$$DHd####~(IIe$$ ! !D DHH = (DIIe,,-- ( ( 3 344'11))GKK(((((C$4$4$6$6$<$<$>$>("3''' Kr,rc^dt|ddddS)Nhrrr )intrrs r*rs.>S&$!7!7!;<<>>r,rrcDt|ddS)Nrr)HTML_TAG_MAPPINGrrs r*rrs'(>(>?r,) rlrrrrrr?rrc|tD]}tt|j}t |r|||_n||_|jdkr0|d|jdd|j d|_td}| ||S)zConvert XML to simplified HTML.rrrrbodyhtml) rJHTML_CONVERSIONSrrrEcallablerrpoprrr)r1rN conversionroots r*convert_to_htmlrs *//1122   %c$(mm4 J   "!z$''DHH!DH 8s?? HHVT[__Xr:: ; ; ; ; K      DH 6??DKK Kr,document with_metadatact|j}|rTtd}tD]'}t ||x}rt |d||(|d|t|ddS)z1Convert the document to HTML and return a string.rmeta)namecontentrTunicode) pretty_printencoding) rrrrgetattrrinsertr r)rr html_treerrvalues r*build_html_outputr s ..I"v# C CD$///u C4dEBBBBD!!! ID9 E E E K K M MMr,)rS)F)FF)TFrg)B__doc__loggingrArcourlan.urlutilsrr lxml.etreerrrr r r lxml.htmlr deduplicationrsettingsrrrrrutilsrrrxmlrr getLogger__name__r~ritemsrr;rrRrrKrlboolrbtuplerrrrrrrrrrrrrrrrrrrrr r&r,r*rsz<<<<<<<<QQQQQQQQQQQQQQQQ!!!!!!))))))655555555500000000  8 $ $          ?>%5%;%;%=%=>>>777--- + +i +K + + + +F[k{d5kPTal(9k"9 3S$s) #$9999&&{&#&&Y^_ceijmen_nYo&&&&RRkRdRRRR,!      :! '' ' '' ' _ ''''Tx)4(  T     00d0000&t8h4 #H##### (t-- -             . >   !" #$% . #{ #cDj #T # # # #..{.Y.S4Z.S^....d     > >    ? ?  (x( N N N N# N N N N N Nr,