
    IZje              
       ,   d Z ddlZddlmZ ddlmZmZmZmZ ddl	m
Z
  ej        d          Z ej        d          Z ej        d          Z ej        d	ej                  Zd!dededeedz  edz  f         fdZ	 d"dedee         dz  dededz  fdZdeez  defdZdeez  defdZdeez  deeef         fdZdedeedz  ef         fdZdededefdZdee         dedz  dee         fdZd#dedededefdZdedee         defd ZdS )$zD
Functions related to URL manipulation and extraction of URL parts.
    N)unescape)SplitResulturljoinurlsplit
urlunsplit)get_tldz{(?:(?:f|ht)tp)s?://(?:[^/?#]{,63}\.)?([^/?#.]{4,63}\.[^/?#]{2,63}|\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|[0-9a-f:]{16,})(?:/|$)z(?<=\D):\d+z^www[0-9]*\.z(?:feed(?:burner|proxy))Furlfastreturnc                    | rt          | t                    sdS |rwt                              |           }|r[t                              d|d                             d          d                   }|                    d          d         }|r||fS t          | dd	          }|dS |j        t                              d|j
                  fS )
z0Cached function to extract top-level domain info)NN    @.r   T)	as_objectfail_silently)
isinstancestrDOMAIN_REGEXmatchSTRIP_PORT_REGEXsubsplitr   domainCLEAN_FLD_REGEXfld)r	   r
   domain_matchfull_domainclean_matchtldinfos         \/Users/jameslopez/projects/MentorCore/.venv/lib/python3.11/site-packages/courlan/urlutils.pyget_tldinfor#      s     jc** z 0#))#.. 	0*..r<?3H3H3M3Mb3QRRK%++C003K 0"K//cT>>>Gz>?..r7;????    	blacklistc                 f    |t                      }t          | |          \  }}|r
||vr||vr|ndS )z;Extract domain name information using top-level domain infoNr
   )setr#   )r	   r%   r
   r   r   s        r"   extract_domainr)   ,   sX     EE	%c555FK 	!22{)7S7S 	r$   c                     t          | t                    rt          t          |                     }n5t          | t                    r| }nt          dt          |                     |S )z3Parse a string or use urllib.parse object directly.zwrong input type:)r   r   r   r   r   	TypeErrortype)r	   
parsed_urls     r"   _parser.   <   s_    #s 8hsmm,,

	C	%	% 8

+T#YY777r$   c                 \    t          |           }|j        r|j        dz   }nd}||j        z   S )ziStrip URL of some of its parts to get base URL.
    Accepts strings and urllib.parse ParseResult objects.z://r   )r.   schemenetloc)r	   r-   r0   s      r"   get_base_urlr2   G   s<     J "U*J%%%r$   c                     t          |           }t          |          }t          dd|j        |j        |j        g          }|dk    rd}|r|st          d|            ||fS )zvDecompose URL in two parts: protocol + host/domain and path.
    Accepts strings and urllib.parse ParseResult objects.r   /zincomplete URL: )r.   r2   r   pathqueryfragment
ValueError)r	   r-   hostnamepathvals       r"   get_host_and_pathr;   R   s     JJ''H	R*"2J4GH G "}} 37 31C11222Wr$   c                 J    t          | d          }t          |           }||fS )zXConvenience function returning domain and host info (protocol + host/domain) from a URL.Tr'   )r)   r2   )r	   
domainnamebase_urls      r"   get_hostinfor?   b   s,    $///JC  Hxr$   baseurlc                    |                     d          r|S t          |           j        }t          |          }|j        |dfvr,|j        r|S t	          |                    d                    S t          | |          S )z8Prepend protocol and host information to relative links.{r   http)r0   )
startswithr   r1   r0   r   _replacer   )r@   r	   base_netloc	split_urls       r"   fix_relative_urlsrH   i   s    
~~c 
7##*KIR000 	J),,F,;;<<<7C   r$   	link_list	urlfilterc                     t          t          |                     S fd| D             }|sd | D             }t          t          |                    S )zDReturn a list of links corresponding to the given substring pattern.Nc                     g | ]}|v |	S  rM   ).0linkrJ   s     r"   
<listcomp>zfilter_urls.<locals>.<listcomp>~   s#    EEEd93D3DT3D3D3Dr$   c                 F    g | ]}t                               |          |S rM   )FEED_WHITELIST_REGEXsearch)rN   rO   s     r"   rP   zfilter_urls.<locals>.<listcomp>   s<     
 
 
*>*E*Ed*K*K

 
 
r$   )sortedr(   )rI   rJ   filtered_lists    ` r"   filter_urlsrV   y   st    c)nn%%%EEEEiEEEM 

 
&
 
 
 #m$$%%%r$   T	referenceignore_suffixc                 n    t          |d          \  }}t          | d          \  }}|r||k    S ||k    S )zjDetermine if a link leads to another host, takes a reference URL and
    a URL as input, returns a booleanTr'   )r#   )r	   rW   rX   stripped_refrefstripped_domainr   s          r"   is_externalr]      sO     $ID999L#)#D999OV /,..S=r$   rO   known_linksc                 Z   | sdS | |v rdS | d         dk    r|                      d          n| dz   }||v rdS |                     d          r_|                     d          rd| dd         z   nd| d	d         z   }|d         dk    r|                     d          n|dz   }||v s||v rdS dS )
zDCompare the link and its possible variants to the existing URL base.FTr   r4   rC   https   N   )rstriprD   )rO   r^   
slash_testprotocol_tests       r"   is_known_linkrf      s     u{t &*"X__S!!!$*J[  t v 
!%!9!9QFT!""Xwabb?Q 	
 R C''   %%%$ 	
 K'':+D+D45r$   )F)NF)T) __doc__rehtmlr   urllib.parser   r   r   r   tldr   compiler   r   r   IrR   r   booltupler#   r(   r)   r.   r2   r;   r?   rH   listrV   r]   rf   rM   r$   r"   <module>rq      s    
			       C C C C C C C C C C C C      rz  2:n-- "*_--!rz"=rtDD @ @S @ @sTz3:7M1N @ @ @ @* ?D 	SD7;4Z    k! k    &cK' &C & & & &3, sCx      c  eC$JO4        !s ! ! ! ! ! ! &49 &t &S	 & & & & S S       #c( t      r$   