
    IZjlJ                        d Z ddlZddlZddlZddlmZ ddlmZ ddlm	Z	 ddl
mZmZmZmZmZ ddlmZ ddlmZ dd	lmZmZ d
dlmZ d
dlmZmZmZmZ d
dlmZm Z  d
dl!m"Z"m#Z#m$Z$m%Z% d
dl&m'Z'm(Z(m)Z)m*Z*m+Z+ dgZ, ej-        e.          Z/ ej-        d          0                    ej1                    ej2        d          Z3 ej2        d          Z4 ej2        d          Z5 ej2        d          Z6 ej2        d          Z7 ej2        dej8                  Z9h dZ:h dZ;h dZ<h dZ=h dZ>ddhZ?h dZ@d d!hZAd"d#hZBh d$ZCd%d&d'd(d(d(d)d*ZDd+d,hZEg d-ZFd.eGd/eGfd0ZHd1eGd2eIeG         d/eGdz  fd3ZJd4ed5ed/efd6ZKd4ed/eLeGeGdz  f         fd7ZMd4ed/efd8ZNdOd4ed:eOe         d;ePd/eGdz  fd<ZQd4ed/eReGeGdz  eGdz  f         fd=ZSd4ed/eGdz  fd>ZTd4ed/eGdz  fd?ZUdPd4ed@eGdz  d/eGdz  fdAZVd4ed/eGdz  fdBZWdCeGd4ed/eOeG         fdDZXdQdFedGeYd/eGdz  fdHZZd4ed/eGdz  fdIZ[	 	 	 	 dRdKeeGz  d@eGdz  dLeLeGe	f         dz  dMeYd2eIeG         dz  d/efdNZ\dS )SzH
Module bundling all functions needed to scrape metadata from webpages.
    N)deepcopy)unescape)Any)extract_domainget_base_urlis_valid_urlnormalize_urlvalidate_url)	find_date)XPath)HtmlElementtostring   )prune_unwanted_nodes)extract_jsonextract_json_parse_errornormalize_authorsnormalize_json)Documentset_date_params)HTML_STRIP_TAGSline_processing	load_htmltrim)AUTHOR_DISCARD_XPATHSAUTHOR_XPATHSCATEGORIES_XPATHSTAGS_XPATHSTITLE_XPATHSr   htmldatez$https?://(?:www\.|w[0-9]+\.)?([^/]+)z("(?:\\"|[^"])*")|\su5   ^(.+)?\s+[–•·—|⁄*⋆~‹«<›»>:-]\s+(.+)$z["\']z=/(by-nc-nd|by-nc-sa|by-nc|by-nd|by-sa|by|zero)/([1-9]\.[0-9])zT(cc|creative commons) (by-nc-nd|by-nc-sa|by-nc|by-nd|by-sa|by|zero) ?([1-9]\.[0-9])?>   
dc.creator
dc:creatordcsext.authoratc-metaauthordc.creator.autparsely-authordcterms.creatorsailthru.authordcterms.creator.autshareaholic:article_author_namebylauthorauthorscreator	rbauthorscitation_authorarticle:author>   dc.descriptiondc:descriptiondcterms.abstractdcterms.descriptiontwitter:descriptionsailthru.descriptiondescription>
   dc.publisherdc:publisherdcterms.publishersailthru.publisher	copyright	publisher	rbpubnamecitation_journal_titletwitter:sitearticle:publisher>   parsely-tagsdcterms.subjectshareaholic:keywordstagskeywordscitation_keywords>   dc.titledcterms.titleparsely-titletwitter:titlesailthru.titleshareaholic:titletitlerbtitlefb_titleheadlinecitation_title	rbmainurltwitter:url>   twitter:imagetwitter:image:srcimageog:imageog:image:urlog:image:secure_urlr,   r1   rA   zapplication-name>   
http-equivcharsetpropertyrO   r8   sitenamerX   pagetype)zog:titlezog:descriptionzog:site_namerY   rZ   r[   zog:typez	og:authorzog:article:author)z.//head//link[@rel="canonical"]z.//head//basez6.//head//link[@rel="alternate"][@hreflang="x-default"]rF   returnc                     t          t          |                     }|sdS t                              d|          } d                    t          d|                     d                              S )z!Remove special characters of tags z, N)r   r   CLEAN_META_TAGSsubjoinfiltersplit)rF   trimmeds     `/Users/jameslopez/projects/MentorCore/.venv/lib/python3.11/site-packages/trafilatura/metadata.pynormalize_tagsrk      s_    8D>>""G rsG,,D99VD$**T"2"233444    r-   author_blacklistc                     d D             fd|                      d          D             }|r(d                    |                              d          S dS )z:Check if the authors string correspond to expected values.c                 6    h | ]}|                                 S  )lower).0as     rj   	<setcomp>z check_authors.<locals>.<setcomp>   s     <<<a		<<<rl   c                     g | ]>}|                                                                 v*|                                 ?S rp   )striprq   )rr   r,   rm   s     rj   
<listcomp>z!check_authors.<locals>.<listcomp>   s@    uuuffllnnFZFZF\F\dtFtFt6<<>>FtFtFtrl   ;z; N)rh   rf   rv   )r-   rm   new_authorss    ` rj   check_authorsrz      sj    <<+;<<<uuuuc0B0BuuuK 2yy%%++D1114rl   treemetadatac                 4   |                      d          D ]}|j        s
t          t                              d|j                            }	 t          j        |          }t          ||          }]# t
          j        $ r t          ||          }Y ~w xY w|S )z,Parse and extract metadata from JSON-LD datazK.//script[@type="application/ld+json" or @type="application/settings+json"]z\1)
xpathtextr   JSON_MINIFYre   jsonloadsr   JSONDecodeErrorr   )r{   r|   elemelement_textschemas        rj   extract_meta_jsonr      s    

hii H Hy 	%kooeTY&G&GHH	HZ--F#FH55HH# 	H 	H 	H/hGGHHH	HOs   $A33BBc                    t                               d          }|                     d          D ]}|                    d          |                    d          }}|re|                                sQ|t
          v r||t
          |         <   \|dk    rt          |          r||d<   w|t          v rt          d|          |d<   |S )	zESearch meta tags following the OpenGraph guidelines (https://ogp.me/))rO   r,   urlr8   r_   rX   r`   z+.//head/meta[starts-with(@property, "og:")]r^   contentzog:urlr   Nr,   )	dictfromkeysr~   getisspaceOG_PROPERTIESr   	OG_AUTHORr   )r{   resultr   property_namer   s        rj   extract_opengraphr      s    ]]effF 

HII 	D 	D!%*!5!5txx	7J7Jw 	D7??,, 	D--7>}]344(**|G/D/D* 'u)++#4T7#C#Cx  Mrl   c           	      (   t                                          t          |                     }t          |j        |j        |j        |j        |j        |j	        f          r|S g d}}| 
                    d          D ]t          j        d                    dd                                                    }|sAdj        v r                    dd                                          }|                    d          r|dk    r#|                    t'          |                     |t(          v rt+          |j        |          |_        |dk    r|j        p||_        |t,          v r|j	        p||_	        d	j        v r                    d	d                                          }|t.          v rt+          |j        |          |_        Z|t0          v r|j        p||_        s|t2          v r|j        p||_        |t4          v r|j        p||_        |t,          v r|j	        p||_	        |t6          v sd
|v r|}|dk    r|j        st9          |          r	||_        |t:          v r"|                    t'          |                     !dj        v rv                    dd                                          }|dk    rt+          |j        |          |_        t|dk    r|j        p||_        |dk    r|j        p||_        t          fdt<          D                       r=t>                               dtC          dd                                                     |j        p||_        ||_"        |S )z)Search meta tags for relevant informationNz.//head/meta[@content]rc   r   r^   og:zarticle:tagrB   nameztwitter:app:namerU   itempropr,   r8   rR   c              3   *   K   | ]}|j         vV  d S N)attrib)rr   keyr   s     rj   	<genexpr>zexamine_meta.<locals>.<genexpr>  s*      >>CDK'>>>>>>rl   zunknown attribute: %sFunicode)pretty_printencoding)#r   	from_dictr   allrO   r,   r   r8   r_   rX   iterfindr   re   r   rv   r   rq   
startswithappendrk   PROPERTY_AUTHORr   METANAME_IMAGEMETANAME_AUTHORMETANAME_TITLEMETANAME_DESCRIPTIONMETANAME_PUBLISHERTWITTER_ATTRSr   METANAME_TAG
EXTRA_METALOGGERdebugr   rF   )	r{   r|   rF   backup_sitenamecontent_attrproperty_attr	name_attritemprop_attrr   s	           @rj   examine_metar      s    zz##$5d$;$;<<H NOL N	
	 	 
 /D 677 A A&*2txx	2/F/FGGMMOO 	 $$ HHZ44::<<M''.. --N<889999/11"3HO\"R"R"555$,$5$E!!.00!)!?<t{"",,2244IO++"3HO\"R"Rn,,!)!?<222'/';'K|$$000$,$5$E!!n,,!)!?<m++/AY/N/N".m++HL+\R^E_E_++l**N<889994;&& HHZ44::<<M(("3HO\"R"R-//'/';'K|$$*,,!)!?< >>>>:>>>>> 	LL'EIFFFLLNN   !)<_HHMOrl      expressions	len_limitc                 Z   |D ]} ||           }|D ][}t          d                    |                                                    }|r#dt          |          cxk     r|k     r	n U|c c S \t          |          dk    r)t                              d|t          |                     dS )zExtract meta information    r   z#more than one invalid result: %s %sN)r   rf   itertextlenr   r   )r{   r   r   
expressionresultsr   r   s          rj   extract_metainfor   ,  s     " Z Z
*T"" 	 	D388DMMOO4455G 1s7||7777i77777w<<!LL>
CPWLLYYY4rl   c                    d}|                      d          }|Nt          |                                          }t                              |          x}r||d         |d         fS t
                              d           |ddfS )z2Extract text segments out of main <title> element.rc   z.//head//titleNr   r   zno main title found)findr   text_contentHTMLTITLE_REGEXmatchr   r   )r{   rO   title_elementr   s       rj   examine_title_elementr   ;  s     EII.//M ]//1122#))%0005 	-%(E!H,,
LL&'''$rl   c                 L   |                      d          }t          |          dk    r+t          |d                                                   }|r|S t	          | t
                    pd}|r|S t          |           \  }}}||fD ]}|rd|vr|c S |r't          |d                                                   S 	 t          |                     d          d                                                   }n*# t          $ r t          
                    d           Y nw xY w|pdS )	zExtract the document titlez.//h1r   r   rc   .z.//h2zno h2 title foundN)findallr   r   r   r   r   r   r~   
IndexErrorr   r   )r{   
h1_resultsrO   firstsecondts         rj   extract_titler   I  sG    g&&J
:!Z]//1122 	LT<006BE 066E5&V_   	AHHH 2JqM..00111*TZZ((+88::;; * * *()))))*=Ds   =:C8 8$DDc                     t          t          |           t                    }t          |t          d          }|rt          d|          }|S )zExtract the document author(s)x   )r   N)r   r   r   r   r   r   )r{   subtreer,   s      rj   extract_authorr   e  sH    "8D>>3HIIGg}DDDF 1"400Mrl   default_urlc                 .   t           D ]9}|                     |          }||j                            d          nd}|r n:|r|                    d          r|                     d          D ]{}|                    d          p|                    d          pd}|                    d          s|                    d	          r#t          |j        d
                   }|r||z   } n||r't          |          \  }}|r|rt          |          nd}|p|S )z'Extract the URL from the canonical linkNhref/z.//head//meta[@content]r   r^   rc   r   ztwitter:r   )	URL_SELECTORSr   r   r   r   r   r   r
   r	   )	r{   r   selectorelementr   attrtypebase_urlvalidation_result
parsed_urls	            rj   extract_urlr   o  sO   !  ))H%%,3,?gn  (((T 	E	  s~~c"" }}%>?? 	 	G{{6**Kgkk*.E.EKH""5)) X-@-@-L-L 'y(ABB "S.CE  V(4S(9(9%:+<UUmJ'''QU+rl   c                 X    t          |           ^}}t          d |D             d          S )z=Extract the name of a site from the main title (if it exists)c              3   &   K   | ]}|d |v |V  dS )r   Nrp   )rr   parts     rj   r   z#extract_sitename.<locals>.<genexpr>  s+      @@$4@C4KKKKKK@@rl   N)r   next)r{   _partss      rj   extract_sitenamer     s2    %d++IA@@%@@@$GGGrl   metatypec                    g }d|                      d          z   dz   | dk    rt          nt          }|D ]0}|                    fd ||          D                        |r n1| dk    r:|s8|                    d          D ]"}|                    |j        d                    #d t                              d	 |D                       D             S )
z!Find category and tag informationr   yz(?:y|ies|s)?/categoryc              3      K   | ]8}t          j        |j        d                    "|                                V  9dS )r   N)researchr   r   )rr   r   regexprs     rj   r   z#extract_catstags.<locals>.<genexpr>  sK      ppt	RY[_[fgm[nHoHopt((**pppppprl   zR.//head//meta[@property="article:section" or contains(@name, "subject")][@content]r   c                     g | ]}||S rp   rp   )rr   rs     rj   rw   z$extract_catstags.<locals>.<listcomp>  s    TTT!RSTATTTrl   c              3   8   K   | ]}|t          |          V  d S r   )r   )rr   xs     rj   r   z#extract_catstags.<locals>.<genexpr>  s/      $N$NAA$N_Q%7%7$N$N$N$N$N$Nrl   )	rstripr   r   extendr~   r   r   r   r   )r   r{   r   xpath_expressioncatexprr   r   s         @rj   extract_catstagsr     s   GHOOC(((?:G,4
,B,B((#  ppppwwt}}pppppp 	E	 :gzz"vww 	6 	6GNN7>)45555
 UTt}}$N$N$N$N$NNNTTTTrl   Fr   strictc                 L   t                               |                     dd                    }|r&d|d                                          d|d          S t	          |                                           }|r*|r&t                              |          }|r|d         ndS |S dS )	zkProbe a link for identifiable free license cues.
    Parse the href attribute first and then the link text.r   rc   zCC r   r   r   r   N)LICENSE_REGEXr   r   upperr   r   TEXT_LICENSE_REGEX)r   r   r   r   s       rj   parse_license_elementr     s       VR!8!899E 32U1X^^%%22a222$$&&''D  	/&--d33E$.588$.4rl   c                     |                      d          D ]}t          |d          }||c S |                     d          D ]}t          |d          }||c S dS )z:Search the HTML code for license information and parse it.z.//a[@rel="license"][@href]F)r   Nz[.//footer//a[@href]|.//div[contains(@class, "footer") or contains(@id, "footer")]//a[@href]T)r   r   r~   )r{   r   r   s      rj   extract_licenser     s     << =>>  &wu===MMM  ::{||  &wt<<<MMM 4rl   Tfilecontentdate_config	extensivec                 J   |pt                      }|ri |nt          |          }t          |           }|t                      S t	          |          }|j        rd|j        vrd|_        	 t          ||          }n2# t          $ r%}t          	                    d|           Y d}~nd}~ww xY w|j
        st          |          |_
        |j        r|rt          |j        |          |_        |j        st          |          |_        |j        r|rt          |j        |          |_        |j        st          ||          |_        |j        rt!          |j        d          |_        |j        |d<   t%          |fi ||_        |j        st+          |          |_        |j        rt-          |j        t.                    r|j        d         |_        n3t-          |j        t0                    rt3          |j                  |_        |j                            d          |_        |j        rFd	|j        vr=|j        d                                         s|j        
                                |_        n5|j        r.t8                              |j                  }|r|d
         |_        |j        st?          d|          |_        |j         st?          d|          |_         tC          |          |_"        |#                    d          |_$        |%                                 |S )aR  Main process for metadata extraction.

    Args:
        filecontent: HTML code as string or parsed tree.
        default_url: Previously known URL of the downloaded document.
        date_config: Provide extraction parameters to htmldate as dict().
        extensive: Use extensive search for date extraction.
        author_blacklist: Provide a blacklist of Author Names as set() to filter out authors.

    Returns:
        A trafilatura.settings.Document containing the extracted metadata information.
        The Document class has .as_dict() method that will return a copy as a dict.
    Nr   z%error in JSON metadata extraction: %sT)fastr   r   @r   r   r   tagmax_date)&setr   r   r   r   r,   r   	Exceptionr   warningrO   r   rz   r   r   r   r   hostnamer   dater_   r   
isinstancelistr   strlstripisupperMETA_URLr   
categoriesr   rF   r   licenser   filedateclean_and_trim)	r  r   r  r  rm   r{   r|   errmymatchs	            rj   extract_metadatar    sK   * (0355%0P/[//oi6P6PK [!!D|zz D!!H  3ho55E$T844 E E E>DDDDDDDDE > -&t,,  K+ K'9IJJ? /(.. K+ K'9IJJ < 6"455 | D*8<dCCC "Kd22k22HM  3,T22 +h'.. 	7 ( 1! 4H)400 	7 #H$5 6 6H$-44S99 	:H,=!=!=hFWXYFZFbFbFdFd!= ( 1 7 7 9 9H	 +.... 	+ '
H  A.z4@@ = 6(55 't,,H $
33HOs   ,A= =
B,B''B,)r   r   )F)NNTN)]__doc__r   loggingr   copyr   htmlr   typingr   courlanr   r   r   r	   r
   r    r   
lxml.etreer   	lxml.htmlr   r   htmlprocessingr   json_metadatar   r   r   r   settingsr   r   utilsr   r   r   r   xpathsr   r   r   r   r   __all__	getLogger__name__r   setLevelWARNINGcompiler  r   r   rd   r   Ir   r   r   r   r   r   METANAME_URLr   r   r   r   r   r   r   r  rk   r	  rz   r   r   r   r   r  intr   tupler   r   r   r   r   r   boolr   r   r  rp   rl   rj   <module>r3     sY      				                                            + + + + + + + + 0 0 0 0 0 0            0 / / / / / / / D D D D D D D D D D D D              ,		8	$	$  *   & &w 7 7 72:=>>bj011"*UVV"*X&&
[\\RZ[D  
  &           ]+   -.!34 322
 #"  -.	~~~5 5 5 5 5 53 #c( sTz    K 8     K DcDj,A    *]{ ]x ] ] ] ]@ ; T%[ S [^ae[e    

3d
C$J&'    d
    8 t     k d
 cDj    6H; H3: H H H HUs U+ U$s) U U U U* ;  t    $+ #*    " #)-(,h hs"hth c3h$&h 	h
 #hoh h h h h h hrl   