
    IZjC                        d Z ddlZddlZddlZddlmZmZ ddlmZm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlZddlZddlmZ ddlmZ ddlmZmZ ddlm Z m!Z!m"Z"m#Z#m$Z$ 	 ddl%m&Z& ej'        (                    d          Z)n# e*$ r dZ)Y nw xY w	 ddl+Z+ e+j,                    Z-e-.                    e+j/        e+j0                   e-.                    e+j/        e+j1                   dZ2n# e*$ r dZ2Y nw xY w ej3        e4          Z5 ej6        ej7        j8                   da9da:da;dedej<        ez  fdZ=dNdZ>ej?        @                    d          ZAe rdeAd         vreAdxx         dz  cc<   d ed          z   d z   ZBeBeAd!<   g d"ZCh d#ZD G d$ d%          ZEd&edeFeGeH         dz  eHdz  f         fd'ZIdOd&ed(eJeHeHf         dz  deJeHeHf         fd)ZKd&edej?        jL        fd*ZMdPd&ed+eNdej<        ez  fd,ZOd-eHd+eNd.eNd&edeEdz  f
d/ZPd-eHd0eEd1edeNfd2ZQd-eHd0eEd3eNd1edeEeHz  dz  f
d4ZRdedfd-eHd+eNd&ed1edz  deHdz  f
d5ZSddded6d-eHd3eNd+eNd.eNd&edeEdz  fd7ZTd-eHdeNfd8ZUd-eHdeNfd9ZVd-eHdeNfd:ZW	 	 	 	 	 dQd;eGeH         d<eXeH         dz  d=eHdz  d>edz  d?eNd@eNdefdAZYdRd>edCeZdeFeGeH         ef         fdDZ[	 dSdFeGeH         dGe\dHeeHgef         dIe\deeFeHef         ddf         f
dJZ]	 dOdFeGeH         dGe\d1edz  deeFeHeHf         ddf         fdKZ^	 dOdFeGeH         dGe\d1edz  deeFeHeEf         ddf         fdLZ_d-eHd+eNd.eNd&edeEdz  f
dMZ`dS )TzG
All functions needed to steer and execute downloads of web documents.
    N)Callable	Generator)ThreadPoolExecutoras_completed)ConfigParser)partial)version)BytesIO)sleep)Any)UrlStore)redirection_test   )DEFAULT_CONFIG	Extractor)HAS_ZSTDURL_BLACKLIST_REGEXdecode_fileis_acceptable_lengthmake_chunks)SOCKSProxyManager
http_proxyTFargsreturnc                  x    t           rt          nt          j        }t           r	dt           ini }d|d<    |di || S )zCConfigure urllib3 download pool according to user-defined settings.	proxy_url2   	num_pools )	PROXY_URLr   urllib3PoolManager)r   manager_classmanager_argss      a/Users/jameslopez/projects/MentorCore/.venv/lib/python3.11/site-packages/trafilatura/downloads.pycreate_poolr&   @   sN    )2K%%8KM/8@K++bL "L=00<04000    curlpycurl.Curlc                 b    t           r'|                     t          j        t                      dS dS )zBRoute the pycurl request through PROXY_URL when one is configured.N)r    setoptpycurl	PRE_PROXY)r(   s    r%   _apply_curl_proxyr.   H   s1     1F$i000001 1r'   )accept_encodingzstdzaccept-encodingz,zstdztrafilatura/trafilaturaz( (+https://github.com/adbar/trafilatura)
User-Agent)i  i  i  i  i  i  i  i  i	  i
  i  i  i  i  i  i  iV  >   #   6   :   ;   <   @   B   M   R   S   [   c                       e Zd ZdZg dZdedededdfdZde	fd	Z
defd
Zdeeef         ddfdZde	ddfdZdeeef         fdZdS )Responsez5Store information gathered in a HTTP response object.dataheadershtmlstatusurlrA   rD   rE   r   Nc                 L    || _         d | _        d | _        || _        || _        d S Nr@   )selfrA   rD   rE   s       r%   __init__zResponse.__init__p   s)    	.2 $	r'   c                     | j         d uS rG   )rA   rH   s    r%   __bool__zResponse.__bool__w   s    y$$r'   c                 8    | j         pt          | j                  S rG   )rC   r   rA   rK   s    r%   __repr__zResponse.__repr__z   s    y2K	222r'   
headerdictc                 L    d |                                 D             | _        dS )z#Store response headers if required.c                 >    i | ]\  }}|                                 |S r   )lower).0kvs      r%   
<dictcomp>z*Response.store_headers.<locals>.<dictcomp>   s&    DDDA		1DDDr'   N)itemsrB   )rH   rO   s     r%   store_headerszResponse.store_headers}   s*     ED1A1A1C1CDDDr'   decodec                 R    |r"| j         rt          | j                   | _        dS dS dS )z9Decode the bytestring in data and store a string in html.N)rA   r   rC   )rH   rY   s     r%   decode_datazResponse.decode_data   s<     	/di 	/#DI..DIII	/ 	/ 	/ 	/r'   c                 *      fd j         D             S )z,Convert the response object to a dictionary.c                 2    i | ]}|t          |          S r   )getattr)rS   attrrH   s     r%   rV   z$Response.as_dict.<locals>.<dictcomp>   s%    EEEdgdD))EEEr'   )	__slots__rK   s   `r%   as_dictzResponse.as_dict   s    EEEEdnEEEEr'   )__name__
__module____qualname____doc__r`   bytesintstrrI   boolrL   rN   dictrX   r[   ra   r   r'   r%   r?   r?   k   s       ;;<<<IU C c d    %$ % % % %3# 3 3 3 3ES#X E4 E E E E
/$ /4 / / / /
Fc3h F F F F F Fr'   r?   configc                     |                      ddd                                          }|r|                                nd}|                      dd          pd}||fS )zARead and extract HTTP header strings from the configuration file.DEFAULTUSER_AGENTS )fallbackNCOOKIE)getstrip
splitlines)rk   myagents
agent_listmycookies       r%   _parse_configrx      sg     zz)]Rz@@FFHHH*2<$$&&&J zz)X..6$Hxr'   rB   c                     | t           k    r4t          |           \  }}i }|rt          j        |          |d<   |r||d<   |pt          S )z1Internal function to decide on user-agent string.r2   Cookie)r   rx   randomchoiceDEFAULT_HEADERS)rk   rB   ru   rw   s       r%   _determine_headersr~      s]    *622( 	<$*M($;$;GL! 	) (GH%o%r'   c           	          t           sht          j                            |                     dd          |                     dd          d|                     dd          dz  t
                    a t           S )z5Define a retry strategy according to the config file.rm   MAX_REDIRECTSr   DOWNLOAD_TIMEOUT   )totalredirectconnectbackoff_factorstatus_forcelist)RETRY_STRATEGYr!   utilRetrygetintFORCE_STATUSrk   s    r%   _get_retry_strategyr      so      	
 ++--	?;;]]9o>>!==4FGG!K) , 
 
 r'   no_sslc                     |rt           nt          }|sFt          |                     dd          |rdnt	          j                    |rdnd          }|r|a n|a|S )zXCreate a urllib3 pool manager according to options in the config file and HTTPS setting.rm   r   N	CERT_NONECERT_REQUIRED)timeoutca_certs	cert_reqs)NO_CERT_POOL	HTTP_POOLr&   r   certifiwhere)rk   r   pools      r%   _initiate_poolr      sy     "0<<yD MM)-?@@#8TT%+@kk
 
 
  	LLIKr'   rE   with_headersc                 0   	 t          ||          }|                    d| t          |          t          |          d          }t	                      }|                    dd          }|                    d          D ]9}|                    |           t          |          |k    rt          d          :|
                                 t          t          |          |j        |                                          }	|r|	                    |j                   |	S # t"          j        j        $ r0 t(                              d	|            t-          | d
||          cY S t.          $ r&}
t(                              d| |
           Y d}
~
nd}
~
ww xY wdS )zPInternal function to robustly send a request (SSL or not) and return its result.)r   GETF)rB   retriespreload_contentrm   MAX_FILE_SIZEi   zMAX_FILE_SIZE exceededzretrying after SSLError: %sTzdownload error: %s %sN)r   requestr~   r   	bytearrayr   streamextendlen
ValueErrorrelease_connr?   rf   rD   geturlrX   rB   r!   
exceptionsSSLErrorLOGGERwarning_send_urllib_request	Exceptionerror)rE   r   r   rk   pool_managerresponserA   max_file_sizechunkresperrs              r%   r   r      s   8%fV<<<  ''&v..'//! ( 
 
 {{iAA__U++ 	; 	;EKK4yy=(( !9::: ) dX_hoo6G6GHH 	1x/000& E E E4c:::#C|VDDDDD 8 8 8,c3777777778 4s   DD! !AF$	F-FFr   optionsc                     t          |j        p|j        pd          }|j        dk    r#t                              d|j        |            dS t          ||          sdS dS )z2Check if the response conforms to formal criteria.ro      z!not a 200 response: %s for URL %sFT)r   rC   rA   rD   r   r   r   )rE   r   r   lentests       r%   _is_suitable_responser      sd    (-68=6B77G#8(/3OOOu11 u4r'   rY   c                 >    t          | ||          r|r|j        n|S dS )z:Internal function to run safety checks on response result.N)r   rC   )rE   r   rY   r   s       r%   _handle_responser      s.     S(G44 5 &4x}}H44r'   c                     |r|j         n|}t          | d||          }|r1|j        r*|st          |          }t	          | ||          r|j        S dS )a  Downloads a web page and seamlessly decodes the response.

    Args:
        url: URL of the page to fetch.
        no_ssl: Do not try to establish a secure connection (to prevent SSLError).
        config: Pass configuration values for output control.
        options: Extraction options (supersedes config).

    Returns:
        Unicode string or None in case of failed downloads and invalid results.

    T)rY   r   rk   r   N)rk   fetch_responserA   r   r   rC   )rE   r   rk   r   r   s        r%   	fetch_urlr     st    $  '2W^^FFc$vfMMMH !HM ! 	/v...G h88 	!= 4r'   )rY   r   r   rk   c                    t           st          nt          }t                              d|             || |||          }|st                              d|            dS |                    |           |S )a  Downloads a web page and returns a full response object.

    Args:
        url: URL of the page to fetch.
        decode: Use html attribute to decode the data (boolean).
        no_ssl: Don't try to establish a secure connection (to prevent SSLError).
        with_headers: Keep track of the response headers.
        config: Pass configuration values for output control.

    Returns:
        Response object or None in case of failed downloads and invalid results.

    zsending request: %szrequest failed: %sN)
HAS_PYCURLr   _send_pycurl_requestr   debugr[   )rE   rY   r   r   rk   dl_functionr   s          r%   r   r      sz    * /9R&&>RK
LL&,,,{3f==H )3///t   Or'   c                    d}t          j                    }|                    t           j        |                     d                     |                    t           j        d           |                    t           j        d           |                    t           j        d           |                    t           j        d           t          |           	 |
                                 |                    t           j                  dk     }n:# t           j        $ r(}t                              d| |           d}Y d}~nd}~ww xY w|                                 |S )	z+Send a basic HTTP HEAD request with pycurl.Futf-8
   r   Ti  zpycurl HEAD error: %s %sN)r,   Curlr+   URLencodeCONNECTTIMEOUTSSL_VERIFYPEERSSL_VERIFYHOSTNOBODYr.   performgetinfoRESPONSE_CODEr   r   r   close)rE   page_existsr(   r   s       r%   _pycurl_is_live_pager   ?  s&   K;==DKK
CJJw//000KK%r***KK%q)))KK%q)))KKt$$$dll6#7883><   /c::: 	JJLLLs   7D E EEc                     	 t          |           }n4# t          $ r'}t                              d| |           Y d}~dS d}~ww xY wdS )zGUse courlan redirection test (based on urllib3) to send a HEAD request.zurllib3 HEAD error: %s %sNFT)r   r   r   r   )rE   _r   s      r%   _urllib3_is_live_pager   Z  s`    S!!   0#s;;;uuuuu 4s    
A>Ac                 T    t           rt          |           nd}|pt          |           S )zCSend a HTTP HEAD request without taking anything else into account.F)r   r   r   )rE   results     r%   is_live_pager   d  s.    *4?!#&&&%F/*3///r'   	inputlist	blacklist
url_filter	url_storecompressionverbosec                     |t          |d|          }t          t                              |                     } rfd| D             } rfd| D             } |                    |            |S )zMFilter, convert input URLs and add them to domain-aware processing dictionaryNF)
compressedstrictr   c                 B    g | ]}t          j        d |          v|S )ro   )r   sub)rS   ur   s     r%   
<listcomp>z*add_to_compressed_dict.<locals>.<listcomp>z  s2    ]]]1-@-DR-K-KS\-\-\Q-\-\-\r'   c                 L    g | ]t          fd D                        S )c              3       K   | ]}|v V  	d S rG   r   )rS   fr   s     r%   	<genexpr>z4add_to_compressed_dict.<locals>.<listcomp>.<genexpr>}  s'      0L0LAa0L0L0L0L0L0Lr'   )any)rS   r   r   s    @r%   r   z*add_to_compressed_dict.<locals>.<listcomp>}  s<    MMM1S0L0L0L0L0L0L0L-L-LMQMMMr'   )r   listrj   fromkeysadd_urls)r   r   r   r   r   r   s    ``   r%   add_to_compressed_dictr   k  s     E7SSS	T]]9--..I ^]]]]	]]]	 NMMMM	MMM	y!!!r'         @
sleep_timec                 n    	 |                      |d          }|s| j        rnt          |           1|| fS )zRDetermine threading strategy and draw URLs respecting domain-based back-off rules.Ti )
time_limitmax_urls)get_download_urlsdoner   )r   r   
bufferlists      r%   load_download_bufferr     sT    00JQV0WW
 	 	j	
 y  r'   '  r   download_threadsworker	chunksizec              #     K   t          |          5 t          | |          D ]A}fd|D             }t          |          D ] }||         |                                fV  !B	 ddd           dS # 1 swxY w Y   dS )z3Use a thread pool to perform a series of downloads.)max_workersc                 >    i | ]}                     |          |S r   )submit)rS   rE   executorr   s     r%   rV   z'_buffered_downloads.<locals>.<dictcomp>  s)    PPP3X__VS993PPPr'   N)r   r   r   r   )r   r   r   r   r   future_to_urlfuturer  s     `    @r%   _buffered_downloadsr    s       
(8	9	9	9 =X Y77 	= 	=EPPPPP%PPPM&}55 = =#F+V]]__<<<<<=	== = = = = = = = = = = = = = = = = =s   AA66A:=A:c                 P    t          t          |          }t          | ||          S )z3Download queue consumer, single- or multi-threaded.)r   )r   r   r  )r   r   r   r   s       r%   buffered_downloadsr    s)     Y000Fz+;VDDDr'   c                 p    |r|j         nt          }t          t          |          }t	          | ||          S )z7Download queue consumer, returns full Response objects.r   )rk   r   r   r   r  )r   r   r   rk   r   s        r%   buffered_response_downloadsr
    s9      ':W^^NF^F333Fz+;VDDDr'   c                 4   d t          |                                          D             }t          j                    }|                    t          j        |                     d                     |                    t          j        t                     |                    t          j	        |           |                    t          j
        d           |                    t          j        |                    dd                     |                    t          j        |                    dd                     |                    t          j        |                    dd                     |                    t          j        |                    dd                     |                    t          j        d           |du rA|                    t          j        d	           |                    t          j        d	           n1|                    t          j        t)          j                               |r3t-                      }|                    t          j        |j                   t3          |           	 |                                }n# t          j        $ rr}t8                              d
| |           |du rG|j        d	         t<          v r3t8                              d| |           tA          | d||          cY d}~S Y d}~dS d}~ww xY wtC          ||"                    t          j#                  |"                    t          j$                            }	|%                                 |ri }
|&                                '                    dd          (                                D ]I}d|vr|)                    dd          \  }}|*                                |
|*                                <   J|	+                    |
           |	S )zDExperimental function using libcurl and pycurl to speed up downloadsc                 "    g | ]\  }}| d | S )z: r   )rS   headercontents      r%   r   z(_send_pycurl_request.<locals>.<listcomp>  s*    ccc_VWV((w((cccr'   r   r   rm   r   r   r   Tr   zpycurl error: %s %sFzretrying after SSL error: %s %sNz
iso-8859-1replace)errors:),r~   rW   r,   r   r+   r   r   SHARE
CURL_SHARE
HTTPHEADERFOLLOWLOCATION	MAXREDIRSr   r   TIMEOUTMAXFILESIZENOSIGNALr   r   CAINFOr   r   r
   HEADERFUNCTIONwriter.   
perform_rbr   r   r   CURL_SSL_ERRORSr   r   r?   r   r   EFFECTIVE_URLr   getvaluerY   rt   splitrs   rX   )rE   r   r   rk   
headerlistr(   headerbytesbufferbytesr   r   respheaderslinenamevalues                 r%   r   r     sn   
 dc@RSY@Z@Z@`@`@b@bcccJ ;==DKK
CJJw//000KKj)))KK!:...KK%q)))KK &--	?"K"KLLLKK%v}}Y@R'S'STTTKKi9K L LMMMKK"FMM)_$M$MNNNKK###~~F)1---F)1----FM7=??333 >iiF);+<===doo''<   *C555
 U??sx{o==LL:CEEE'T<HHHHHHHH ttttt  Kf.B!C!CT\\RXRfEgEghhDJJLLL (((**11,y1QQ\\^^ 	6 	6D $**S!,,KD%(-K

%%;'''Ks   J L&A!LLL)r(   r)   r   NrG   )F)NNNFF)r   )r   )are   loggingosr{   collections.abcr   r   concurrent.futuresr   r   configparserr   	functoolsr   importlib.metadatar	   ior
   timer   typingr   r   r!   courlanr   courlan.networkr   settingsr   r   utilsr   r   r   r   r   urllib3.contrib.socksr   environrr   r    ImportErrorr,   	CurlSharer  r+   SH_SHARELOCK_DATA_DNSLOCK_DATA_SSL_SESSIONr   	getLoggerrb   r   disable_warningsr   InsecureRequestWarningr   r   r   r"   r&   r.   r   make_headersr}   
USER_AGENTr   r  r?   tupler   rh   rx   rj   r~   r   r   ri   r   r   r   r   r   r   r   r   r   setr   floatr   rg   r  r  r
  r   r   r'   r%   <module>rF     s     				  / / / / / / / / ? ? ? ? ? ? ? ? % % % % % %       & & & & & &                          , , , , , , / / / / / / / /             777777
|,,II   IIIMMM!!##J fov';<<<fov'CDDD JJ   JJJ 
	8	$	$  +B C C C	1 1 3c 9 1 1 1 11 1 1 1 ,++D+AA 2o.?@@@%&&&'1&&&ggm4447aa
 *   ( ?>>F F F F F F F FF ,  5cT1A3:1M+N        	& 	&| 	&d38nt6K 	&W[\_ad\dWe 	& 	& 	& 	& 1C      <  'BUX[B[    *!c !4 !t !\ !^fim^m ! ! ! !H	s 	h 	 	t 	 	 	 		 *.9B^d    ) $	 	  	
 	4Z   > )  	  	
   _   >c d    6s t    0c 0d 0 0 0 0 "&!!% Cy3x$ d
 $	
      0! !H !% !%PTUXPY[cPcJd ! ! ! ! 	= =S	== cUCZ = 	=
 uS#Xd*+= = = =" !%E ES	EE E uS#Xd*+	E E E E !%	E 	ES		E	E 	E uS(]#T4/0		E 	E 	E 	EJc J4 Jt J\ J^fim^m J J J J J Js%   . B BBAC3 3C=<C=