
    IZjM                     F   d Z 	 ddlZdZn# e$ r dZY nw xY wddlZddlZ	 ddlZdZn# e$ r dZY nw xY wddlm	Z	 ddl
mZ ddlmZmZmZmZ ddlmZ 	 ddlZdZn# e$ r dZY nw xY w	 ddlZdZn# e$ r dZY nw xY w	 ddlZdZn# e$ r dZY nw xY w	 dd	lmZ n# e$ r dZY nw xY wdd
lmZ ddlmZ ddl m!Z!m"Z"m#Z# ddl$m%Z% erddl&m'Z'm(Z(  ej)        e*          Z+ddhZ, ej-        dej.                  Z/ ej-        dej.                  Z0 ej-        d          Z1 ej-        d          Z2 e"ddddd          Z3 ej-        dej4        ej5        z            Z6 ej-        d          Z7 ej-        d          Z8h dZ9ddhZ:dZ; ej-        d          Z< ej-        d ej=                  Z>d!e?d"e?fd#Z@d$e?d"eAfd%ZBd&e?d"eCeD         fd'ZEd!e?eDz  d"eDfd(ZFd)eDd"eAfd*ZGd+eDd)eDd"eDfd,ZHd-eDd"e!dz  fd.ZId-ed"e!dz  fd/ZJ e	d01          d2eDd"eDfd3            ZKd4eDd"eDfd5ZLdcd4eDd7ed8         d"eDfd9ZM e	d:1          ddd;eDd<eAd=eAd"eDdz  fd>            ZNddd?eDd<eAd=eAd"eDdz  fd@ZOdAed"efdBZP e	d:1          d4eDd"eDfdC            ZQdDed"eAfdEZRdFeDdz  d"eAfdGZSdHedIeTd"efdJZUdKeTdLdMd"eAfdNZVdedAe!dOeDdPeAd"eAfdQZWdReDdSeDd"eDdz  fdTZXdReDdSeDdOeDdUdVd"eYeAdVf         f
dWZZdDed"eAfdXZ[d4eDdz  d"eAfdYZ\dZed[ed"dfd\Z]d]ed"eAfd^Z^d]ed"eAfd_Z_dDed"eAfd`Z`dDed"eAfdaZadDed"eAfdbZbdS )fzj
Module bundling functions related to HTML and text processing,
content filtering and language detection.
    NTF)	lru_cache)islice)TYPE_CHECKINGAnyLiteralcast	normalize)detect)
from_bytes)_Element)HtmlElement
HTMLParser
fromstring)HTTPResponse   )Document	Extractorutf-8utf_8z^< ?! ?DOCTYPE[^>]*/[^<]*>z(<html.*?)\s*/>z(<!--.*?-->|<[^>]*>)z([\x00-\x08\x0b\x0c\x0e-\x1f\ufffe\uffff])collect_idsdefault_doctypeencodingremove_comments
remove_pisz(?<![p{P}>])\n)flagsz^https?://|/+$z3[^\s]+\.(avif|bmp|gif|hei[cf]|jpe?g|png|webp)(\b|$)>   phitdrefcellheaditemquotecodepre)zhttp-equiv="content-language"zproperty="og:locale"z
([a-z]{2})z\W*(Drucken|E-?Mail|Facebook|Flipboard|Google|Instagram|Linkedin|Mail|PDF|Pinterest|Pocket|Print|QQ|Reddit|Twitter|WeChat|WeiBo|Whatsapp|Xing|Mehr zum Thema:?|More on this.{,8}$)$filecontentreturnc                 R   t          | t                    s| S t          rM| dd         dk    r?	 t          j        |           S # t
          $ r t                              d           Y nw xY wt          rR| dd         dk    rD	 t          j        |           S # t          j
        $ r t                              d           Y nw xY wt          r*	 t          j        |           S # t          j        $ r Y nw xY wt          r*	 t          j        |           S # t          j        $ r Y nw xY w| S )z
    Don't trust response headers and try to decompress a binary string
    with a cascade of installed packages. Use magic numbers when available.
    N   s   zinvalid GZ file   s   (/zinvalid ZSTD file)
isinstancebytesHAS_GZIPgzip
decompress	ExceptionLOGGERwarningHAS_ZSTD	zstandard	ZstdError
HAS_BROTLIbrotlierrorHAS_ZLIBzlib)r'   s    ]/Users/jameslopez/projects/MentorCore/.venv/lib/python3.11/site-packages/trafilatura/utils.pyhandle_compressed_filer=   j   sv   
 k5))   .KO66	.?;/// 	. 	. 	.NN,-----	.  0KO':::	0'444" 	0 	0 	0NN./////	0  	$[111| 	 	 	D	  	?;///z 	 	 	D	 sG   A $A)(A)B )CCC! !C32C3>D D$#D$datac                 V    	 |                      d           n# t          $ r Y dS w xY wdS )zLSimple heuristic to determine if a bytestring uses standard unicode encodingzUTF-8FT)decodeUnicodeDecodeError)r>   s    r<   isutf8rB      sC    G   uu4s    
&&bytesobjectc                    t          |           rdgS g }t          >t          |           d         }|'|                    |                                           t	          |           dk     rt          |           }n1t          | dd         | dd         z             pt          |           }t	          |          dk    r|                    d |D                        d	 |D             S )
z> "Read all input or first chunk and return a list of encodingsr   Nr   i'  i  ixr   c                     g | ]	}|j         
S  )r   ).0rs     r<   
<listcomp>z#detect_encoding.<locals>.<listcomp>   s    >>>q
>>>    c                 $    g | ]}|t           v|S rF   )UNICODE_ALIASES)rG   gs     r<   rI   z#detect_encoding.<locals>.<listcomp>   s"    ;;;!!?":":A":":":rJ   )rB   cchardet_detectappendlowerlenr   extend)rC   guessescchardet_guessdetection_resultss       r<   detect_encodingrV      s     k yG"(55jA%NN>//11222
;%&{33&{5D5'9K<O'OPPkT^_jTkTk
!!>>,=>>>???;;w;;;;rJ   c                 4   t          | t                    r| S d}t          |           } t          |           D ]M}	 |                     |          } n4# t
          t          f$ r  t                              d|           d}Y Jw xY w|pt          | dd          S )zCheck if the bytestring could be GZip and eventually decompress it,
    guess bytestring encoding and try to decode to Unicode string.
    Resort to destructive conversion otherwise.Nzwrong encoding detected: %sr   replace)r   errors)	r,   strr=   rV   r@   LookupErrorrA   r2   r3   )r'   htmltextguessed_encodings      r<   decode_filer^      s     +s## H )55K+K88  	"))*:;;H
 E	 /0 	 	 	NN8:JKKKHHH	 Ks;KKKKs   A.BB	beginningc                 
    d| vS )zOAssess if the object is proper HTML (awith a corresponding tag or declaration).htmlrF   )r_   s    r<   is_dubious_htmlrb      s    ""rJ   
htmlstringc                    t                               d|           } d|v r<|                     d          \  }}}t                              d|d          dz   |z   } t	          t          |                                                     D ]E\  }}d|v r4|                    d          rt                              d| d          }  n	|d	k    r nF| S )
z>Repair faulty HTML strings to make then palatable for libxml2. doctype
r   )countz<htmlz/>z\1>   )	INVALID_XML_CHARSsub	partitionDOCTYPE_TAG	enumerateiter
splitlinesendswithFAULTY_HTML)rc   r_   	firstline_restilines          r<   repair_faulty_htmlrx      s    "&&r:66JI'11$77	1d __R!_<<tCdJ
T*"7"7"9"9::;;  4d??t}}T22?$1EEJEq55E rJ   
htmlobjectc                     d}	 t          |                     dd          t                    }n2# t          $ r%}t                              d|           Y d}~nd}~ww xY w|S )z!Try to pass bytes to LXML parser.Nutf8surrogatepassparserzlxml parser bytestring %s)r   encodeHTML_PARSERr1   r2   r9   )ry   treeerrs      r<   fromstring_bytesr      sx    D7*++FODD[YYY 7 7 70#666666667Ks   */ 
AAAc                 8   t          | t                    r| S t          | t                    st          | d          r| j        } t          | t
          t          f          st          dt          |                     d}t          |           } | dd         
                                }t          |          }t          | |          } d}	 t          | t                    }nN# t          $ r t!          |           }d}Y n1t"          $ r%}t$                              d|           Y d}~nd}~ww xY w|t)          |          d	k     r|st!          |           }|A|du r=t)          |          d
k     r*t$                              dt)          |                     d}|S )zLoad object given as input and validate its type
    (accepted: lxml.html tree, trafilatura/urllib3 response, bytestring and string)
    r>   zincompatible input typeN2   Fr}   Tzlxml parsing failed: %sr   ri   z9parsed tree length: %s, wrong data type or not valid HTML)r,   r   r   hasattrr>   r-   rZ   	TypeErrortyper^   rP   rb   rx   r   r   
ValueErrorr   r1   r2   r9   rQ   )ry   r   r_   
check_flagfallback_parser   s         r<   	load_htmlr      s   
 *k** *l++ %wz6/J/J %_
j5#,// E14
3C3CDDDDZ((J3B3%%''I ++J#J	::JN5*[999   
++ 5 5 5.444444445 	D		A~
++ J$..3t99q==PRUVZR[R[\\\Ks   C# #D. 	D.	D))D.i @  )maxsizecharc                 Z    |                                  s|                                 r| ndS )z3Return a character if it belongs to certain classesre   )isprintableisspace)r   s    r<   return_printables_and_spacesr     s,     ##%%??44R?rJ   stringc                 ~    |                                  r| S d                    t          t          |                     S )z6Prevent non-printable and XML invalid character errorsre   )r   joinmapr   r   s    r<   remove_control_charactersr     s:      7733V<<===rJ   NFCunicodeform)r   NFDNFKCNFKDc                 "    t          ||           S )z;Normalize the given string to the specified unicode format.r	   )r   r   s     r<   normalize_unicoder   "  s    [&)))rJ   i   rw   preserve_spacetrailing_spacec                    t          |                     dd                              dd                              dd                    }|st          t                              d|                    }t          t          t          j        |                    rd}nV|rT| d	                                         rdnd
}| d                                         rdnd
}d
	                    |||g          }|S )zjRemove HTML space entities, then discard incompatible unicode
    and invalid XML characters on line levelz&#13;z&#10;rg   z&nbsp;     Nr   re   )
r   rX   trimLINES_TRIMMINGrk   allr   rZ   r   r   )rw   r   r   new_linespace_beforespace_afters         r<   line_processingr   '  s     )gt)D)D)L)LWVZ)[)[)c)cdlnv)w)wxxH 
F **4::;;s3;))** 	FHH 	F"&q'//"3"3;33L!%b!1!1!3!3;##KwwhDEEHOrJ   textc                     |rt          | d          S 	 d                    t          dfd|                                 D                                                     dd          S # t
          $ r Y dS w xY w)z<Convert text and discard incompatible and invalid charactersTrg   Nc              3   8   K   | ]}t          |          V  d S )N)r   )rG   rw   r   s     r<   	<genexpr>zsanitize.<locals>.<genexpr>C  s.      &k&kQUt^'L'L&k&k&k&k&k&krJ   u   ␤re   )r   r   filterrp   rX   AttributeError)r   r   r   s    ` r<   sanitizer   <  s      ;t^T:::yy&k&k&k&kY]YhYhYjYj&k&k&kllmmuub
 
 	
    tts   AA+ +
A98A9r   c                 
   |                                  D ]}|                                }||j        nd}|j        t          v p|t          v }|j        t          v p
|t          v p|}|j        D ]P}d|v rJ|j        |         r#|                    dd          d         | j        vr|j                            |           Q|j	        rt          |j	        ||          |_	        |j        rt          |j        ||          |_        | S )z?Trims spaces, removes control characters and normalizes unicodeNre   :r   r   )ro   	getparenttagSPACING_PROTECTEDFORMATTING_PROTECTEDattribsplitnsmappopr   r   tail)r   elemparent
parent_tagr   r   	attributes          r<   sanitize_treer   J  s   		 L L!!#)#5VZZ2
 %66Y*HY:Y%99qZK_=_qcq  	/ 	/Ii{9- /a1H1H1KSWS]1]1]KOOI...9 	L NNKKDI9 	L NNKKDIKrJ   c                     	 d                     |                                                                           S # t          t          f$ r Y dS w xY w)z/Remove unnecessary spaces within a text string.r   re   )r   r   stripr   r   r   s    r<   r   r   b  sT    xx''--///I&   rrs   8; AAelementc                     dD ]*}|                      |d          }t          |          r dS +| j                                        D ],\  }}|                    d          rt          |          r dS -dS )z*Check if an element is a valid img element)data-srcsrcre   Tr   F)getis_image_filer   items
startswith)r   attrr   values       r<   is_image_elementr   l  s    #  kk$## 	44	 #>//11 	 	KD%z** }U/C/C tt5rJ   imagesrcc                 ~    | t          |           dk    rdS t          t                              |                     S )zCheck if the observed string corresponds to a valid image extension.
    Use a length threshold and apply a regex on the content.Ni    F)rQ   boolIMAGE_EXTENSIONsearch)r   s    r<   r   r   z  s;     3x==4//u&&x00111rJ   iterablenc              #      K   t          |           }t          t          ||                    x}r%|V  t          t          ||                    x}#dS dS )zChunk data into smaller pieces.N)ro   tupler   )r   r   iteratorbatchs       r<   make_chunksr     sp       H~~H!,,--
-%  !,,--
-%     rJ   my_lenoptionsr   c                     | |j         k     r"t                              d|j                   dS | |j        k    r#t                              d| |j                   dS dS )z=Check if the document length is within acceptable boundaries.ztoo small/incorrect for URL %sFztoo large: length %s for URL %sT)min_file_sizer2   r9   urlmax_file_size)r   r   s     r<   is_acceptable_lengthr     s]    %%%5w{CCCu%%%6LLLu4rJ   target_languagestrictc                    t           D ]Y}|                     d| d          }|r<t          fd|D                       r dS t                              d|            dS Z|rP|                     d          }|r9t          fd|D                       rdS t                              d	           dS t                              d
           dS )zoCheck HTML meta-elements for language information and split
    the result in case there are several languages.z	.//meta[@z][@content]c              3      K   | ]F}t                               |                    d d                                                    v V  GdS )contentre   NRE_HTML_LANGr   r   rP   rG   r   r   s     r<   r   z"check_html_lang.<locals>.<genexpr>  sQ      ll^b?l&8&8)R9P9P9V9V9X9X&Y&YYllllllrJ   Tz%s lang attr failedFz//html[@lang]c              3      K   | ]F}t                               |                    d d                                                    v V  GdS )langre   Nr   r   s     r<   r   z"check_html_lang.<locals>.<genexpr>  sQ      ii[_?l&8&8&"9M9M9S9S9U9U&V&VViiiiiirJ   zHTML lang failedzNo relevant lang elements found)TARGET_LANG_ATTRSfindallanyr2   debugxpath)r   r   r   r   elemss    `   r<   check_html_langr     s    "  ::::;; 	llllfklllll ttLL.55555		  

?++ 	iiiichiiiii tLL+,,,5
LL23334rJ   	temp_texttemp_commentsc                     t           du rLt          |           t          |          k    rt          j        |           nt          j        |          \  }}nt                              d           d}|S )zARun external component (if installed) for language identificationTz3Language detector not installed, skipping detectionN)LANGID_FLAGrQ   	py3langidclassifyr2   r3   )r   r   resultrt   s       r<   language_classifierr     sj    d58^^c-FXFX5X5XI&y111^g^pq~^^	LMMMMrJ   docmetar   c                     |Qt          | |          |_        |j        5|j        |k    r*t                              d|j        |j                   d|fS d|fS )zFFilter text based on language detection and store relevant informationNzwrong language: %s %sTF)r   languager2   r3   r   )r   r   r   r   s       r<   language_filterr    sd     ".y-HH 'G,<,O,ONN2G4DgkRRR= '>rJ   c                     | j         | j        n| j         }| pL|                                p8t          t	          t
          j        |                                                    S )zFilter out unwanted text)r   r   r   r   r   	RE_FILTERmatchrp   )r   testtexts     r<   
textfilterr    sT    &|3w||H<a8++--aS(J]J]J_J_5`5`1a1aarJ   c                 J    t          |           o|                                  S )zJDetermine if a string is only composed of spaces and/or control characters)r   r   r   s    r<   text_chars_testr    s"     <<0 0 000rJ   	dest_elemsrc_elemc                 v    |                                 D ]#}|                     ||j        |                    $dS )z0Copy attributes from src element to dest elementN)keyssetr   )r	  r
  keys      r<   copy_attributesr    sA    }} 1 1c8?3/00001 1rJ   r   c                     |                                  dS | }|#|j        dk    rdS |                                 }|#dS )z+Check whether an element is in a table cellNFr!   T)r   r   )r   currents     r<   is_in_table_cellr    sT    ~~u#G

;&  4##%% 
 5rJ   c                    t          |           sdS | j        dk    r#|                                 }| p|d         | k    S t          t          |                                           }|                                }| p|d         | k    S )z:Check whether an element is the last element in table cellFr!   r   )r  r   getchildrenr   r   r   )r   childrenr   s      r<   is_last_element_in_cellr    s    D!! ux6##%%|3x|t33h 0 011%%''|3x|t33rJ   c                 T    | }|#|j         dk    rdS |                                }|#dS )z=Check whether an element is a list item or within a list itemNr#   TF)r   r   )r   r  s     r<   is_element_in_itemr    s?    &G

;&  4##%% 
 5rJ   c                     | j         dk    r	| j        rdS | }d}|$|j         dk    r|}n|                                }|$|dS |j        sdS dS )z:Check whether an element is the first element in list itemr#   TNF)r   r   r   )r   r  item_ancestors      r<   is_first_element_in_itemr     s{    {ft&GM

;&  #M##%%	 
 u t5rJ   c                     t          |           sdS | j        dk    r%t          |                                           dk    S |                                 }|dS |j        dk    S )z9Check whether an element is the last element in list itemFr#   r   NT)r  r   rQ   r  getnext)r   next_elements     r<   is_last_element_in_itemr    sm    g&& u {f7&&(())Q..??$$Lt6))rJ   )r   )FF)F)c__doc__r/   r.   ImportErrorloggingrer;   r:   	functoolsr   	itertoolsr   typingr   r   r   r   unicodedatar
   r8   r7   r5   r4   r   r   cchardetr   rN   charset_normalizerr   
lxml.etreer   	lxml.htmlr   r   r   urllib3.responser   settingsr   r   	getLogger__name__r2   rL   compileIrm   rr   HTML_STRIP_TAGSrj   r   UNICODE	MULTILINEr   URL_BLACKLIST_REGEXr   r   r   r   r   
IGNORECASEr  r-   r=   r   rB   listrZ   rV   r^   rb   rx   r   r   r   r   r   r   r   r   r   r   r   intr   r   r   r   r   r  r  r  r  r  r  r  r  r  rF   rJ   r<   <module>r9     s   
KKKHH   HHH  				KKKHH   HHH              4 4 4 4 4 4 4 4 4 4 4 4 ! ! ! ! ! !MMMJJ   JJJHH   HHHKK   KKK2222222   OOO * ) ) ) ) )       9 9 9 9 9 9 9 9 9 9 * ) ) ) ) ) .--------		8	$	$G$bj5rt<<bj+RT22"*455BJJKK  jUEGeivz{{{-RZ",5NOOO bj!233  "*STTPPP UO  N rz-(( BJG -	  	" "% " " " "J 4    < <49 < < < <0LUS[ LS L L L L2#s #t # # # #
3 3 3    " t);    (# (+"4 ( ( ( (V 5@s @s @ @ @ @
>c >c > > > >* *c *8T0U *be * * * *
 4 # t T ^adh^h    ( 3  d WZ]aWa     X    0 4     h 4    2C$J 24 2 2 2 2# # #     { t     +  T VZ    03 s sTz    s 3  Wa fklpr|l|f}    "b bT b b b b1C$J 14 1 1 1 11x 18 1 1 1 1 1	8 	 	 	 	 	4( 4t 4 4 4 4 T    h 4    (*X *$ * * * * * *s`    ( 22A A%$A%)A0 0A:9A:>B BBB B$#B$