
    IZja7                        d Z ddlZddlZddlmZ ddlmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZmZmZ d	d
lmZ  eedd          Ze e ed                    n ej                    pd	Zg dZ ee          dhz  Zddedz  dedz  defdZ e            ZdddddddddZdedededz  fdZ G d d          Z ddej!        d edz  de fd!Z"dd#e#de$eef         fd$Z% G d% d&          Z& e'ed'          Z(d(Z)d)Z*d*Z+d+Z,d,Z-h d-Z.g d.Z/g d/Z0 ed0          Z1 e2g d1          Z3i d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSi dTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtdui dvdwdxdydzd{d|d}d~dddddddddddddddddddddddddi dddddddddddddddddddddddddddddddddddddddddddd	Z4dS )z<
Listing a series of settings that are applied module-wide.
    N)ConfigParser)datetime)unescape)Path)Any)ElementXPath_Element   )line_processingsched_getaffinity)csvjsonhtmlmarkdowntxtxmlxmlteipythonfilenameconfigreturnc                    ||S | *t          t          t                    j        dz            } n0t          |                                           st          d          t                      }|                    |            |S )zE
    Use configuration object or read and parse a settings file.
    Nzsettings.cfgz$The given config file does not exist)strr   __file__parentis_fileFileNotFoundErrorr   read)r   r   s     `/Users/jameslopez/projects/MentorCore/.venv/lib/python3.11/site-packages/trafilatura/settings.py
use_configr!      s|     tH~~,~=>>(^^##%% H FGGG^^F
KKM    MIN_EXTRACTED_SIZEMIN_OUTPUT_SIZEMIN_OUTPUT_COMM_SIZEMIN_EXTRACTED_COMM_SIZEMIN_DUPLCHECK_SIZEMAX_REPETITIONSMAX_FILE_SIZEMIN_FILE_SIZE)min_extracted_sizemin_output_sizemin_output_comm_sizemin_extracted_comm_sizemin_duplcheck_sizemax_repetitionsmax_file_sizemin_file_sizeoptionc                     |                      d|d                                          }|                                rt          |          ndS )zJRead an optional positive integer setting; None when empty or non-numeric.DEFAULT )fallbackN)getstripisdigitint)r   r3   values      r    _get_optional_intr=   :   sB    JJy&2J66<<>>E23u:::d2r"   c            +       |   e Zd ZU dZg dZeed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   edddddddddddddddddddddede	de
de
de
de
de
de
de
de
de
de	dz  de	dz  de	dz  de
de
d e
d!ee	         dz  d"ee	         dz  d#ee	e	f         dz  f(d$Zde	dz  de	dz  d%dfd&Zd'e	d%dfd(Zded%dfd)ZdS )*	Extractorz0Defines a class to store all extraction options.)r   formatfastfocuscomments
formattinglinksimagestablesdeduplangr+   r,   r-   r.   r/   r0   r1   r2   max_tree_sizesourceurlwith_metadataonly_with_metadatatei_validationdate_paramsauthor_blacklisturl_blacklistr+   r,   r-   r.   r/   r0   r1   r2   r   FTN)r   output_formatrA   	precisionrecallrC   rD   rE   rF   rG   rH   rI   rL   rK   rM   rN   rO   rQ   rR   rP   r   rS   rA   rT   rU   rC   rD   rE   rF   rG   rH   rI   rL   rK   rM   rN   rO   rQ   rR   rP   c                   |                      ||           |                     |           |                     |pt                     || _        |rdn|rdnd| _        || _        |p
| j        dk    | _        || _	        |	| _
        |
| _        || _        || _        || _        || _        || _        |pt#                      | _        |pt#                      | _        |p|pt)          |          p|dk    | _        |p't-          | j                            dd                    | _        t5          | j        d          | _        d S )	NrU   rT   balancedr   r   r5   EXTENSIVE_DATE_SEARCHMAX_TREE_SIZE)_set_source_set_format_add_configDEFAULT_CONFIGrA   rB   rC   r@   rD   rE   rF   rG   rH   rI   rL   rN   rO   setrQ   rR   boolrM   set_date_paramsr   
getbooleanrP   r=   rJ   )selfr   rS   rA   rT   rU   rC   rD   rE   rF   rG   rH   rI   rL   rK   rM   rN   rO   rQ   rR   rP   s                        r    __init__zExtractor.__init__r   sP   0 	f%%%'''1>222	&,Z((2Z++PZ
& * GdkZ.G 
"" 
 $	"(:$2*:*Ccee'4'=#0#z4F#z$}J]J]#zanrzaz+6 ,
/K""9.EFF;
 ;
 *;4;)X)Xr"   r   c                 n    |p|}|o(|                     dd                              d          | _        dS )z)Set the source attribute in a robust way.zutf-8replaceN)encodedecoderK   )rb   rL   rK   s      r    rZ   zExtractor._set_source   s7    Rw	!B!B!I!I'!R!Rr"   chosen_formatc                     |t           vr7t          dd                    t          t                                          || _        dS )z;Store the format if supported and raise an error otherwise.z#Cannot set format, must be one of: z, N)SUPPORTED_FORMATSAttributeErrorjoinsortedr@   )rb   rh   s     r    r[   zExtractor._set_format   sF     111 !mtyyQWXiQjQjGkGk!m!mnnn#r"   c           	          t                                           D ]*\  }}t          | ||                    d|                     +|| _        dS )z&Store options loaded from config file.r5   N)CONFIG_MAPPINGitemssetattrgetintr   )rb   r   keyr<   s       r    r\   zExtractor._add_config   sQ    (..00 	@ 	@JCD#v}}Y>>????r"   )__name__
__module____qualname____doc__	__slots__r;   __annotations__r]   r   r   r_   r^   dictrc   rZ   r[   r\    r"   r    r?   r?   A   st        66" " "IJ     
  ." !##($,0)--1-.Y .Y .Y .Y 	.Y
 .Y .Y .Y .Y .Y .Y .Y .Y .Y Dj.Y 4Z.Y  d
!.Y" #.Y$ !%.Y& '.Y( c(T/).Y* 3x$+.Y, #s(^d*-.Y .Y .Y .Y`SsTz S3: S$ S S S S
$ $ $ $ $ $, 4      r"   r?   argsrL   c                    t          t          | j                  | j        | j        | j        | j        | j        | j        | j	        | j
        || j        | j        | j                  }dD ]!}t          ||t          | |                     "|S )z-Derive extractor configuration from CLI args.)r   )r   rS   rD   rT   rU   rC   rG   rH   rI   rL   rM   rN   rO   )rA   rF   rE   )r?   r!   config_filerS   rD   rT   rU   no_comments	no_tablesdeduplicatetarget_languagerM   rN   validate_teirq   getattr)r|   rL   optionsattrs       r    args_to_extractorr      s    4#3444(?.{!~!(2(  G , 4 4wtT223333Nr"   T	extensivec                 V    d| t          j                                        d          dS )z/Provide default parameters for date extraction.Tz%Y-%m-%d)original_dateextensive_searchmax_date)r   nowstrftime)r   s    r    r`   r`      s0     %LNN++J77  r"   c            ,          e Zd ZdZg dZdddddddddddddddddddddddedz  dedz  dedz  dedz  d	edz  d
edz  dedz  dee         dz  dee         dz  dedz  dedz  dedz  dedz  dedz  dedz  dedz  dedz  dedz  dedz  dedz  dedz  f*dZe	de
eef         dd fd            Zd dZde
eedz  f         fdZdS )!DocumentzZDefines a class to store all necessary data and metadata fields for extracted information.)titleauthorrL   hostnamedescriptionsitenamedate
categoriestagsfingerprintidlicensebodyrC   commentsbodyraw_texttextlanguageimagepagetypefiledateN)r   r   rL   r   r   r   r   r   r   r   idvallicense_valr   rC   r   r   r   r   r   r   r   r   r   rL   r   r   r   r   r   r   r   r   r   r   rC   r   r   r   r   r   r   r   c                p   || _         || _        || _        || _        || _        || _        || _        || _        |	| _        |
| _	        || _
        || _        ||nt          d          | _        || _        ||nt          d          | _        || _        || _        || _        || _        || _        || _        d S )Nr   )r   r   rL   r   r   r   r   r   r   r   r   r   r   r   rC   r   r   r   r   r   r   r   )rb   r   r   rL   r   r   r   r   r   r   r   r   r   r   rC   r   r   r   r   r   r   r   s                         r    rc   zDocument.__init__   s    2 "'
"("$,'2$, $	,6&*	'2##.&*&6ddGFOO	$,6B6NllT[\bTcTc$, $	$,!&
$,$,r"   datar   c                 p     |             }|                                 D ]\  }}t          |||           |S )z.Set a series of attributes using a dictionary.)rp   rq   )clsr   docrs   r<   s        r    	from_dictzDocument.from_dict  sD     cee**,, 	% 	%JCCe$$$$
r"   c                     | j         D ]t}t          | |          }t          |t                    rMt	          |          dk    r|dd         dz   }t          t          |                    }t          | ||           udS )z*Limit text length and trim the attributes.'  Ni'  u   …)rx   r   
isinstancer   lenr   r   rq   )rb   slotr<   s      r    clean_and_trimzDocument.clean_and_trim'  s    N 	+ 	+DD$''E%%% +u::%%!%4%L50E'88dE***	+ 	+r"   c                 *      fd j         D             S )z%Convert the document to a dictionary.c                 4    i | ]}|t          |d           S N)r   ).0r   rb   s     r    
<dictcomp>z$Document.as_dict.<locals>.<dictcomp>5  s'    KKKDgdD$//KKKr"   )rx   )rb   s   `r    as_dictzDocument.as_dict3  s    KKKKDNKKKKr"   )r   N)rt   ru   rv   rw   rx   r   listr
   rc   classmethodrz   r   r   r   r   r{   r"   r    r   r      s4       ``  I8 !!#"&#'+!%"& "& $#(,## ##/-- -- -- Tz-- d
	--
 4Z-- *-- 4Z-- *-- Dj-- I$-- 3i$-- 4Z-- Tz-- 4Z-- o--  *!--" o#--$ *%--& Dj'--( *)--* Tz+--, *---. */-- -- -- --^ T#s(^ 
    [
+ 
+ 
+ 
+Lc3:o. L L L L L Lr"   r      i   i     i@B r   >   bipqdddtemh1h2h3h4h5h6lidivpremainspanstrongarticlesection
blockquote)4asideembedfencedframefooterformheadiframemenuobjectscriptappletaudiocanvasfiguremappicturesvgvideoareablinkbuttondatalistdialogframeframesetfieldsetlinkinputinslabellegendmarqueemathmenuitemnavnoindexnoscriptoptgroupr3   outputparamprogressrprtrtcselectrK   styletracktextareatimeuse)abbracronymaddressbdibdobigciter   dfnfonthgroupimgr   markmetarubysmalltbodytemplatetfoottheadz[.//aside|.//div[contains(@class|@id, 'footer')]|.//fencedframe|.//footer|.//script|.//style)
r   codedelr   hilbr   r   r   quoteaf	Afrikaansan	AragonesearArabicazAzerbaijanibe
Belarusianbg	BulgarianbnBengalibrBretonbsBosniancaCatalancsCzechcyWelshdaDanishdeGermanelGreekenEnglisheo	EsperantoesSpanishetEstonianeuBasquefaPersianfiFinnishfrFrenchgaIrishglGalicianguGujaratiheHebrewr  HindihrCroatianhtHaitianhu	HungarianhyArmenianr   
Indonesianis	IcelandicitItalianjvJavanesekaGeorgiankkKazakhknKannadakoKoreankuKurdishkyKyrgyzlaLatinr  Luxembourgishlt
LithuanianlvLatvianmk
Macedonianml	MalayalammrMarathimsMalaymtMaltesenbNorwegian_BokmalneNepalinlDutchnnNorwegian_NynorsknoocOccitanplPolishpt
PortuguesequQuechuaroRomanianruRussianskSlovaksl	SloveniansqAlbaniansrSerbiansvSwedishswSwahiliTamilTeluguTagalogTurkish	UkrainianUrdu
VietnameseVolapukWalloon)	tatetltrukurvivowa)NNr   )T)5rw   argparseosconfigparserr   r   r   r   pathlibr   typingr   
lxml.etreer   r	   r
   utilsr   r   _get_affinityr   	cpu_count	CPU_COUNTSUPPORTED_FMT_CLIr^   rj   r   r!   r]   ro   r;   r=   r?   	Namespacer   r_   rz   r`   r   minPARALLEL_CORESLRU_SIZEMAX_FILES_PER_DIRECTORYFILENAME_LEN	MAX_LINKSMAX_SITEMAPS_SEENCUT_EMPTY_ELEMSMANUALLY_CLEANEDMANUALLY_STRIPPEDBASIC_CLEAN_XPATH	frozensetTAG_CATALOGJUSTEXT_LANGUAGESr{   r"   r    <module>r     s3     				 % % % % % %                         / / / / / / / / / / " " " " " " /66%2%>CCa  !!!\R\^^EXWX	 POO C)**hZ7  t L44G S_    "  /(28.($$	 	3l 3C 3C$J 3 3 3 3p p p p p p p pf H. S4Z 9    , t tCH~    aL aL aL aL aL aL aL aLJ Y##   	   :8 8 8 v   2 Ewxx ifffggO+O+O 	(O 	-	O
 	,O 	+O 	)O 	(O 	)O 	)O 	'O 	'O 	(O 	(O 	'O  	)!O" 	+#O O$ 	)%O& 	*'O( 	()O* 	)+O, 	)-O. 	(/O0 	'1O2 	*3O4 	*5O6 	(7O8 	'9O: 	*;O< 	)=O> 	+?O@ 	*AOB 	,COD 	+EO O OF 	)GOH 	*IOJ 	*KOL 	(MON 	)OOP 	(QOR 	)SOT 	(UOV 	'WOX 	/YOZ 	,[O\ 	)]O^ 	,_O` 	+aOb 	)cOd 	'eOf 	)gO O Oh 	
iOj 	(kOl 	'mOn 	
oOp 	
qOr 	)sOt 	(uOv 	,wOx 	)yOz 	*{O| 	)}O~ 	(O@ 	+AOB 	*COD 	)EOF 	)GOH 	)IO OJ 







[O O O   r"   