
    #Zj.z                     0   d dl mZ d dlmZ ddlmZmZ ddlmZ ddlm	Z	m
Z
mZmZ ddlZddlZddlZddlZddlZddlZd:de	e         d	e
e         d
efdZ ed          Zd Ze G d d                      Zd Z e            Z[ ej        d          Z ed          Z ed          Z  ed e D                       Z! eg d          Z"dd e#d           e#d           e#d          dZ$ddg e%dd           e%dd           Z& ed!          Z' ed"          Z(d#d$d%d&Z) eg d'          Z*d(d)d*Z+d+d,d-d.d/Z, ed0          Z- ed1          Z.d2Z/e/d          Z0e/d         Z1 ed3          Z2d4 Z3d5 Z4 G d6 d7          Z5 G d8 d9          Z6dS );   )data)MToken    )	dataclassreplace)	num2words)ListOptionalTupleUnionNtokensunkreturnc                 L   d | D             }d | D             }d | D             }|d }nNd}| D ]I}|j         j        r(|r&|d                                         s|j        r|dz  }||j        |n|j        z  }Jt	          d                    d | d d         D                       | d         j        z   t          | d 	          j        | d         j	        || d
         j
        | d         j        t	          j        | d
         j         j        d t          |          dk    rt          |          d
         nd |rt          |          nd d                    t!          d | D                                 | d
         j         j        d |v rd nt#          |                              S )Nc                 >    h | ]}|j         j        |j         j        S N)_stress.0tks     U/Users/jameslopez/projects/MentorCore/.venv/lib/python3.11/site-packages/misaki/en.py	<setcomp>zmerge_tokens.<locals>.<setcomp>   s%    FFFbbdk.Ebdk.E.E.E    c                 >    h | ]}|j         j        |j         j        S r   )r   currencyr   s     r   r   zmerge_tokens.<locals>.<setcomp>   s%    LLL""$-2K2K2K2Kr   c                 &    h | ]}|j         j        S  )r   ratingr   s     r   r   zmerge_tokens.<locals>.<setcomp>   s    +++bbdk+++r     c              3   4   K   | ]}|j         |j        z   V  d S r   text
whitespacer   s     r   	<genexpr>zmerge_tokens.<locals>.<genexpr>   s+      CCRWr},CCCCCCr   c                 >    t          d | j        D                       S )Nc              3   N   K   | ] }||                                 k    rd ndV  !dS )r      N)lowerr   cs     r   r'   z1merge_tokens.<locals>.<lambda>.<locals>.<genexpr>   s6      *W*W!QWWYY11A*W*W*W*W*W*Wr   )sumr%   )r   s    r   <lambda>zmerge_tokens.<locals>.<lambda>   s    s*W*Wrw*W*W*W'W'W r   keyr   r   c                 0    h | ]}|j         j        D ]}|S r   )r   	num_flags)r   r   r-   s      r   r   zmerge_tokens.<locals>.<setcomp>%   s)    %Q%Q%QB"$.%Q%QQa%Q%Q%Q%Qr   )is_headaliasr   r   r3   prespacer   )r%   tagr&   phonemesstart_tsend_tsr   )r   r6   isspacer8   r   joinr%   maxr7   r&   r9   r:   
Underscorer4   lenlistsortedmin)r   r   r   r   r   r8   r   s          r   merge_tokensrC      s   FFFFFFFLLLLLH++F+++F
{ 	D 	DBt}    (2,2F2F2H2H  R[  Cr{2CHHWWCCvcrc{CCCCCfRjoUWWXXX\":(#bz 

1IK'&)&kkQ&6&64<<??D&.8S]]]Dggf%Q%Q&%Q%Q%QRRSSAY[)6>>44s6{{
 
 
   r   u
   AIOQWYʤʧc                 <    | rt          d | D                       ndS )Nc              3   0   K   | ]}|t           v rd ndV  dS )r*   r   N)
DIPHTHONGSr,   s     r   r'   z stress_weight.<locals>.<genexpr>-   s.      77qAOOqq777777r   r   )r.   )pss    r   stress_weightrH   ,   s)    ;=D377B7777771Dr   c                   8    e Zd ZU dZee         ed<   dZeed<   dS )TokenContextNfuture_vowelF	future_to)__name__
__module____qualname__rK   r
   bool__annotations__rL   r   r   r   rJ   rJ   /   s6         #'L(4.'''Itr   rJ   c                  D     dd l                     d            fdS )Nr   u   ^['‘’]+|\p{Lu}(?=\p{Lu}\p{Ll})|(?:^-)?(?:\d?[,.]?\d)+|[-_]+|['‘’]{2,}|\p{L}*?(?:['‘’]\p{L})*?\p{Ll}(?=\p{Lu})|\p{L}+(?:['‘’]\p{L})*|[^-_\p{L}'‘’\d]|['‘’]+$c                 0                         |           S r   )findall)wordSUBTOKEN_REGEXregexs    r   r/   z'make_subtokenize_once.<locals>.<lambda>8   s    ~t<< r   )rW   compile)rV   rW   s   @@r   make_subtokenize_oncerY   5   s:    LLL]]  $Z  [  [N<<<<<=r   z\[([^\]]+)\]\(([^\)]*)\)u   ',-._‘’/u   ;:,.!?—…"“”c              #   "   K   | ]
}|d v|V  dS )u   "“”Nr   )r   ps     r   r'   r'   A   s,      EE1!92D2DQ2D2D2D2DEEr   ).,-LRB--RRB-``""'':$#NFP()i   i   )r^   r_   r`   ra   rb   '   -   A   [   a   {   u#   bdfhjklmnpstvwzðŋɡɹɾʃʒʤʧθu   AIOWYiuæɑəɛɪɹʊʌ)dollarcent)poundpence)eurorp   )rd      £u   €)stndrdthdotslash)r\   /percentandplusat)%&+@uH   AIOWYbdfhijklmnpstuvwzæðŋɑɔəɛɜɡɪɹɾʃʊʌʒʤʧˈˌθᵊᵻʔuD   AIQWYabdfhijklmnpstuvwzðŋɑɒɔəɛɜɡɪɹʃʊʌʒʤʧˈˌːθᵊu   ˌˈu    AIOQWYaiuæɑɒɔəɛɜɪʊʌᵻc                     d }| S |dk     r4                      t          d                               t          d          S |dk    s|dv rBt           v r9                      t          d                               t          t                    S |dv rUt           fdt          D                       r5t           fdt
          D                       r S  |t           z             S |dk    r2t           vr)t           v r                       t          t                    S |dk    rUt           fd	t          D                       r5t           fd
t
          D                       r S  |t           z             S  S )Nc                    t          t          |                     fdD             }|                                D ]\  }}|         \  }}|dz
  |f|<   d                    d t	                    D                       } | S )Nc           	      l    i | ]0\  }}|t           v |t          d  |d         D                       1S )c              3   2   K   | ]\  }}|t           v |V  d S r   )VOWELS)r   jvs      r   r'   z<apply_stress.<locals>.restress.<locals>.<dictcomp>.<genexpr>`   s*      AA$!QQ&[[A[[[[AAr   N)STRESSESnext)r   ir[   ipss      r   
<dictcomp>z2apply_stress.<locals>.restress.<locals>.<dictcomp>`   sJ    cccdaUVZbUbUbAtAA#abb'AAAAAUbUbUbr         ?r    c                     g | ]\  }}|S r   r   )r   r   r[   s      r   
<listcomp>z2apply_stress.<locals>.restress.<locals>.<listcomp>d   s    000DAqa000r   )r@   	enumerateitemsr<   rA   )rG   stressesr   r   r   sr   s         @r   restresszapply_stress.<locals>.restress^   s    9R==!!ccccccccNN$$ 	" 	"DAqq6DAq#gq\CFFWW00F3KK00011	r   r!   r    )r         )r   r   r   c              3       K   | ]}|vV  	d S r   r   r   r   rG   s     r   r'   zapply_stress.<locals>.<genexpr>l   s'      &E&Eqq{&E&E&E&E&E&Er   c              3       K   | ]}|vV  	d S r   r   r   r   rG   s     r   r'   zapply_stress.<locals>.<genexpr>m   '      ++qq{++++++r   r   c              3       K   | ]}|vV  	d S r   r   r   s     r   r'   zapply_stress.<locals>.<genexpr>r   s'      ::AARK::::::r   c              3       K   | ]}|vV  	d S r   r   r   s     r   r'   zapply_stress.<locals>.<genexpr>s   r   r   )r   PRIMARY_STRESSSECONDARY_STRESSallr   r   )rG   r   r   s   `  r   apply_stressr   ]   s      ~		"zz."--556FKKK	2&I--.B2F2Fzz*B//77HXYYY	;		3&E&E&E&EH&E&E&E#E#E	++++F+++++ 	Ix(2-...	1r116F"6L6Lzz*N;;;	!:::::::::++++F+++++ 	Ix+,,,Ir   c                 F    t          t          j        d|                     S )Nz^[0-9]+$)rP   rematchr%   s    r   is_digitr   x   s    d++,,,r   c                       e Zd Zed             Zd Zd Zd Zed             Zd Z	d Z
d Zd	 Zd
 Zd Zd Zd Zd Zed             Zd Zd Zed             Zed             Zd ZdS )Lexiconc                    i }|                                  D ]\  }}t          |          dk     r||                                k    r0||                                k    r|||                                <   a||                                                                k    r|||                                <   i || S )Nr*   )r   r?   r+   
capitalize)dekr   s       r   grow_dictionaryzLexicon.grow_dictionary|   s     GGII 	! 	!DAq1vvzzAGGII~~&&()Aallnn%aggii**,,,, !''))!zqzr   c                    || _         d| _        i | _        i | _        t          j                            t          |rdnd d          5 }t          	                    t          j        |                    | _        d d d            n# 1 swxY w Y   t          j                            t          |rdnd d          5 }t          	                    t          j        |                    | _        d d d            n# 1 swxY w Y   t          d | j                                        D                       sJ |rt          nt          | j                                        D ]}t!          |t"                    r&t          fd|D                       s
J |            =d|v s
J |            |                                D ])}|%t          fd	|D                       s
J |            *d S )
N)r   r*   gbusz
_gold.jsonz_silver.jsonc              3   j   K   | ].}t          |t                    pt          |t                    V  /d S r   )
isinstancestrdict)r   r   s     r   r'   z#Lexicon.__init__.<locals>.<genexpr>   s;      ZZ:a%%<At)<)<ZZZZZZr   c              3       K   | ]}|v V  	d S r   r   r   r-   vocabs     r   r'   z#Lexicon.__init__.<locals>.<genexpr>   s'      22!1:222222r   DEFAULTc              3       K   | ]}|v V  	d S r   r   r   s     r   r'   z#Lexicon.__init__.<locals>.<genexpr>   s'      +B+B1AJ+B+B+B+B+B+Br   )britishcap_stressesgoldssilvers	importlib	resources	open_textr   r   r   jsonloadr   valuesGB_VOCABUS_VOCABr   r   )selfr   rvsr   r   s        @r   __init__zLexicon.__init__   s{   $
 **4G4MDD1Y1Y1YZZ 	?^_ 001>>DJ	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? 	? **4G4MDD1[1[1[\\ 	A`a"2249Q<<@@DL	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	A 	AZZdjFWFWFYFYZZZZZZZZ#1*##%% 	F 	FB"c"" F2222r2222266B6626 B F FA9+B+B+B+B+B+B+B(B(B99A99BF	F 	Fs$   
2BBB?2C==DDc                       fd|D             }d |v rdS t          d                    |          d          }|                    t          d          }t                              |          dfS )Nc                     g | ]B}|                                 j                            |                                          CS r   )isalphar   getupper)r   r-   r   s     r   r   z#Lexicon.get_NNP.<locals>.<listcomp>   s<    EEEAEdjnnQWWYY''EEEr   NNr    r   r      )r   r<   rsplitr   r   )r   rU   rG   s   `  r   get_NNPzLexicon.get_NNP   sp    EEEEEEE2:::"''"++q))YY'++""2&&))r   c                 V   |dk    r,|t           v r#|                     t           |         d d|          S |t          v r#|                     t          |         d d |          S d|                    d          v rw|                    dd                                          rOt          t          |                    d          t                              dk     r| 	                    |          S |dv r|dk    rd	nd
dfS |dv rP|
                    d          r| 	                    |          S |j        |dk    s|r|dk    r| j        d         dfS dS |dv r2|dk    r*|
                    d          r| 	                    |          S dS |dk    r|dk    rt           ddfS |dv r t                              |          dk    rdS |dv s
|dk    r"|dv r| j        d         ddd|j                 dfS |d v s|d!k    r#|d"k    r|j        |d!k    rt           nd}|d#z   dfS |d$v s|d%k    r|dk    r|j        d&k    rd'nd(dfS |d!k    r-t#          j        d)|          r|                     d*d d |          S |d+v r5|d,v r|j        r| j        d-         d.         dfS | j        d-         d/         dfS d0S )1NADDr   r\   r    r0   r   )aADTu   ɐu   ˈA   )amAmAMNNr   r   )u   ɐmr   )anAnANr   )u   ɐnr   IPRP)byByBYADV)u   bˈIr   toToTOr   INr   u   təu   tʊ)NFT)inInr   NNPu   ɪn)theTheTHETu   ðiu   ðəz
(?i)vs\.?$versus)usedUsedUSED)VBDJJr   r   r   r   )ADD_SYMBOLSlookupSYMBOLSstripr   r   r?   r=   splitr   
startswithrK   r   r   r   get_parent_tagr   r   r   rL   )r   rU   r7   r   ctxs        r   get_special_casezLexicon.get_special_case   s!   %<<DK//;;{40$cBBBW__;;wt}dD#>>>DJJsOO##S"(=(=(E(E(G(G#CPSTXT^T^_bTcTcilPmPmPmLnLnqrLrLr<<%%%Z$;;44E144'''~~d## +||D)))!)TT\\V\QR

z$'**8'''t||t 4 4|||D)))8S[[SE\\&)))1,,'''G,B,B3,G,G5,P,P9\!!ddllsl7J7J*T*5FFsGWXZ[[[\!!ddllse||'*'7'?3$;;^^TVFE>1$$^###++,4455&!CCD[[RXmT::[;;xtS999---m###z&)%0!33:f%i0!33zr   c                    | | S |                      d          rdS |                      d          rdS |                      d          s|                      d          rdS |                      d          s|                      d          rdS | S )	NVBVERBr   NOUNr   RBADJr   )r   )r7   s    r   r  zLexicon.get_parent_tag   s    ;J^^D!! 	6^^D!! 	6^^E"" 	cnnT&:&: 	5^^E"" 	cnnT&:&: 	5
r   c                 ~   || j         v s|t          v s	|| j        v rdS |                                rt	          d |D                       sdS t          |          dk    rdS ||                                k    r|                                | j         v rdS |dd          |dd                                          k    S )NTc              3   B   K   | ]}t          |          t          v V  d S r   ordLEXICON_ORDSr,   s     r   r'   z#Lexicon.is_known.<locals>.<genexpr>   s-      *P*Pa3q66\+A*P*P*P*P*P*Pr   Fr   )r   r   r   r   r   r?   r   r+   )r   rU   r7   s      r   is_knownzLexicon.is_known   s    4:DDL4H4H4 	s*P*P4*P*P*P'P'P 	5YY!^^4TZZ\\!!djjlldj&@&@4ABBx48>>++++r   c                 ,   d }||                                 k    r#|| j        vr|                                }|dk    }| j                            |          d}}||s| j                            |          d}}t          |t                    rJ|r|j        d|v rd}n||vrt          	                    |          }|                    ||d                   }||r't          |vr|                     |          \  }}|||fS t          ||          |fS )Nr   r   r   Noner   )r   r   r+   r   r   r   r   rK   r   r  r   r   r   )r   rU   r7   r   r  is_NNPrG   r   s           r   r   zLexicon.lookup   s%   4::<<D
$:$:::<<DE\FZ^^D))1F:f:))$//Bb$ 	, 2s'/FbLLB,,S11R	]++B:&:^2%=%=d++JB~6z!B''//r   c                 h    |sd S |d         dv r|dz   S |d         dv r|| j         rdndz   dz   S |dz   S )Nr!   u   ptkfθr   u
   szʃʒʧʤ   ɪ   ᵻz)r   r   stems     r   _sz
Lexicon._s   s]     	B4"X!!#:"X%%4<:44U;cAAczr   c                    t          |          dk     s|                    d          sdS |                    d          s)|                     |d d         |          r|d d         }n|                    d          s=t          |          dk    rS|                    d          r>|                    d	          s)|                     |d d
         |          r|d d
         }nYt          |          dk    rD|                    d	          r/|                     |d d         dz   |          r|d d         dz   }ndS |                     ||||          \  }}|                     |          |fS )Nr   r   r   ssr!   'sr   esiesy)r?   endswithr  r   r  r   rU   r7   r   r  r  r   s          r   stem_szLexicon.stem_s  s   t99q==c 2 2=:}}T"" 	t}}T#2#Y'D'D 	9DDmmD!! 	c$ii!mmd8K8KmTXTaTabgThThmoso|o|  ~B  CF  DF  CF  ~G  IL  pM  pMm9DDYY]]t}}U33]d3B3iPSmUX8Y8Y]9s?DD:{{4fc::fwwt}}f$$r   c                    |sd S |d         dv r|dz   S |d         dk    r|| j         rdndz   dz   S |d         dk    r|dz   S | j         st          |          dk     r|dz   S |d	         t          v r|d d         d
z   S |dz   S )Nr!   u
   pkfθʃsʧtr   r  r  r*   u   ɪdr  u   ɾᵻdu   ᵻdr   r?   US_TAUSr  s     r   _edzLexicon._ed  s     	(4"X%%#:"X__4<:44U;cAA"X__#:\ 	(SYY]]%<"X  9x''f}r   c                    t          |          dk     s|                    d          sdS |                    d          s)|                     |d d         |          r|d d         }nht          |          dk    rS|                    d          r>|                    d          s)|                     |d d         |          r|d d         }ndS |                     ||||          \  }}|                     |          |fS )	Nr   r   r   ddr!   edeedr  )r?   r"  r  r   r)  r#  s          r   stem_edzLexicon.stem_ed   s    t99q==c 2 2=:}}T"" 	t}}T#2#Y'D'D 	9DDYY]]t}}T22]4==;O;O]TXTaTabfgjhjgjbkmpTqTq]9DD:{{4fc::fxx~~v%%r   c                     |sd S | j         r|d         dv rd S n;t          |          dk    r(|d         dk    r|d         t          v r|d d         dz   S |dz   S )Nr!   u   əːr   r&  r  u   ɾɪŋu   ɪŋr'  r  s     r   _ingzLexicon._ing,  sz      	(4\ 	(Bx6!!t "YY]]tBx348w3F3F9x''f}r   c                 6   t          |          dk     s|                    d          sdS t          |          dk    r)|                     |d d         |          r|d d         }n|                     |d d         dz   |          r|d d         dz   }nSt          |          dk    r>t          j        d|          r)|                     |d d         |          r|d d         }ndS |                     ||||          \  }}|                     |          |fS )N   ingr   r   r   z ([bcdgklmnprstvxz])\1ing$|cking$)r?   r"  r  r   searchr   r0  r#  s          r   stem_ingzLexicon.stem_ing;  s   t99q==e 4 4=:t99q==T]]49c::=9DD]]49S=#.. 	9s?DDYY]]ry)LdSS]X\XeXefjknlnknfoqtXuXu]9DD:{{4fc::fyy&&r   c                   
 |                      |          \  }}|||fS |                                
t          |          dk    r|                    dd                                          r||                                k    rdk    st          |          dk    r|| j        vr|| j        vr||                                k    s(|dd          |dd                                          k    rD
| j        v s9
| j        v s0t          
fd| j	        | j
        | j        fD                       r
}|                     |          r|                     |          S |                    d          rD|                     |d d         d	z             r#|                     |d d         d	z             S |                    d          r>|                     |d d
                   r |                     |d d
                   S | 	                    |          \  }}|||fS | 
                    |          \  }}|||fS |                     |dn          \  }	}|	|	|fS dS )Nr   'r    r      c              3   @   K   | ]} |          d          V  dS )r   Nr   )r   fnr  r   r7   wls     r   r'   z#Lexicon.get_word.<locals>.<genexpr>S  sI       : :022sFC((+: : : : : :r   zs'r  r  r!   r   r   )r  r+   r?   r   r   r   r   r   anyr$  r.  r6  r  r   r"  )r   rU   r7   r   r  rG   r   r  r)  r0  r<  s     ```     @r   get_wordzLexicon.get_wordI  s   **4fcBB
F>v:ZZ\\t99q==T\\#r22::<<=AUAU5LLCIIMM$*$$T\)A)ADJJLL  DHQRR0@0@$@$@$*dl 2 2c : : : : : : :7;{DLRVR_6`: : : 7 7 2 D==s## 	<;;tS&#666]]4   	<T]]49t3CS%I%I 	<;;tCRCy4/fcBBB]]3 	<DMM$ss)S$A$A 	<;;tCRCy#vs;;;[[sFC88
F>v:ll4fc::V?;}}T3v~6SVWWf<zr   c                     d| vrdS |                      d          dk    rdS |                     d          d         }t          |          dk     pt          |          dhk    S )Nr\   Tr   Fr   r   )countr   r?   set)rU   centss     r   is_currencyzLexicon.is_currencyi  s`    d??4ZZ__q  5

3"5zzA~2Us!22r   c           	          t          j        d|          }|r|                                nd }|r|d t          |                    n|}g |                    d          r5                                         dd d d                      |dd          }d+ fd	t          |          r4|t          v r+ t          t          |          d	          d
           nUsVt          |          dk    rC|t          vr:t          |          r+ t          t          |          d	          d
           n|sd|vr|                    dd          }|d         dk    st          |          dk    rfd|D              nt          |          dk    r|                    d          st |d                    |d         dk    r@                                         dd dd                       |d         d           n- |dd          d           n |           n	|                    d          dk    s|sd}|                    dd                              d          D ]c}|sn\|d         dk    s4t          |          dk    r0t!          d |dd          D                       rfd|D              n ||           d}dn]|t          v rmt"                              |          rRd t'          |                    dd                              d          t          |                   D             }t          |          dk    r9|d         d         dk    r|d d         }n|d         d         dk    r
|dd          }t)          |          D ]\  }	\  }}
|	dk    r+                                         dd d d                       ||	dk                                   t+          |          dk    r!|
dk    r                     |
dz   d d d           n                     |
d d d                      nt          |          rt          t          |          d	          }nd|vr>t          t          |                    dd                    |t          v rdnd	          }ni|                    dd          }|d         dk    r+d d!                    d" |dd          D                       z   }nt          t1          |                    } |d
           st3          d#d$||           d%S d!                    d& D                       t5          d' D                       c}|d(v r                               |fS |d)v r                               |fS |d*k    r                               |fS |fS ),Nz[a-z']+$-minusr   TFc           	         t          j        d|r| nt          t          |                               }t	          |          D ]\  }}|dk    sdv ro|r9|dk    r3t          |          dk    r |dk    rdv r                    d           J                                        |d |d	k    rd
nd d                      ~|dk    r(dv r$r"d         d         dz   d         d         fd<   d S )Nz[^a-z]+r}   r   r   r   oner   )u   ər   pointr  nr!   u   ən)r   r   r   intr   r?   appendr   )	numfirstescapesplitsr   wr3   resultr   s	         r   
extend_numz&Lexicon.get_number.<locals>.extend_numz  s   Xj*P##Ys3xx=P=PQQF!&)) H H1::	!1!1 `aCKK!OOU

sV_O_O_i0000dkk!Tg22SWY]&^&^____%ZZC9$4$4$4"(*Q-%"7A!GF2JH Hr   ordinal)r   )rO  r   yearr\   r]   r    r   0r   c                 *    g | ]} |d           S FrN  r   r   rJ  rS  s     r   r   z&Lexicon.get_number.<locals>.<listcomp>  s(    999AU+++999r   00Or  r*   rY  c              3   "   K   | ]
}|d k    V  dS )rV  Nr   r   rJ  s     r   r'   z%Lexicon.get_number.<locals>.<genexpr>  s&      <W<W!Q#X<W<W<W<W<W<Wr   c                 *    g | ]} |d           S rX  r   rZ  s     r   r   z&Lexicon.get_number.<locals>.<listcomp>  s(    ===AZZ///===r   c                 >    g | ]\  }}|rt          |          nd |fS )r   )rK  )r   rM  units      r   r   z&Lexicon.get_number.<locals>.<listcomp>  sB      B  B  B	T#,c#hhh1d3  B  B  Br   r}   rr   r   cardinalzpoint r"   c              3   N   K   | ] }t          t          |                    V  !d S r   )r   rK  r^  s     r   r'   z%Lexicon.get_number.<locals>.<genexpr>  s0      .S.SQyQ/@/@.S.S.S.S.S.Sr   u   ❌zTODO:NUMr   c              3       K   | ]	\  }}|V  
d S r   r   )r   r[   r   s      r   r'   z%Lexicon.get_number.<locals>.<genexpr>  s&      !7!71!!7!7!7!7!7!7r   c              3       K   | ]	\  }}|V  
d S r   r   )r   r   r   s      r   r'   z%Lexicon.get_number.<locals>.<genexpr>  s&      <R<R41aQ<R<R<R<R<R<Rr   )r   r  )r,  'dr3  )TF)r   r5  groupr?   r   rL  r   r   ORDINALSr   rK  
CURRENCIESr   r"  r@  r   r=  r   rC  zipr   absr$  r<   floatprintrB   r  r)  r0  )r   rU   r   r4   r3   suffixrM  rN  pairsr   ra  r   rS  rR  s   `   `       @@r   
get_numberzLexicon.get_numberr  s   ;--#)3t&,6tMc&kk\M""$??3 	MM$++gtT4@@AAA8D		H 		H 		H 		H 		H 		H 		H 		H D>> 2	*f00JyTy999$GGGGG 0	*CIINNxz/I/IhW[nn/IJyTv666tDDDDD .	*S__,,sB''C1v}}C19999S99999SQs||D'9'9
3q6"""q6S==MM$++c4T"B"BCCCJs1vU33333Js122we44444
3ZZ__q   E||C,,22377   1Vs]]s3xx1}}<W<WsSTSUSUw<W<W<W9W9W}=========Js%0000 ##(;(;D(A(A# B  BSVY[]I^I^IdIdehIiIikuv~k  FA  FA  B  B  BE5zzA~~8A;!##!"1"IEE1Xa[A%%!!""IE"+E"2"2 U U;Cq55MM$++eT4"F"FGGG
3ad++++SUV[_cj[j[jdkk$s(D$EEEptp{p{  }A  CG  IM  OS  qT  qT  U  U  U  U	U ~~ 	2 Tz:::D T\\#r%:%:!;!;VW_M_M_		eoppp||C,,7c>>#chh.S.S$qrr(.S.S.S&S&SSDD$U4[[11DJtD)))) 	%T8444:!7!7!7!7!777<R<R6<R<R<R9R9R[  776??F**|##88F##V++u__99V$$f,,v~r   c                     |s|S t                               |          }|r'|                     |d         dz   d d d           d         nd }|r| d| n|S )Nr   r   r"   )ri  r   r$  )r   rG   r   s      r   append_currencyzLexicon.append_currency  sm     	I>>(++HPZ4;;x{3dDAA!DDVZ%-5"!!x!!!25r   c                     |                                  s| S t          j        |           }|t          |          k    rt	          t          |                    n| S r   )isdigitunicodedatanumericrK  r   )r-   rJ  s     r   numeric_if_neededzLexicon.numeric_if_needed  sK    yy{{ 	H""3q66kks3q66{{{q0r   c                    t          d | D                       rdS ddddgt          dR }|D ]1}|                     |          r| d t          |                    }  n2t          fdt	          |           D                       S )	Nc              3   6   K   | ]}t          |           V  d S r   r   r,   s     r   r'   z$Lexicon.is_number.<locals>.<genexpr>  s*      --18A;;------r   Fr3  rf  r,  r  r   c              3   `   K   | ](\  }}t          |          p|d v po|dk    o|dk    V  )dS )z,.r   rE  Nrz  )r   r   r-   r4   s      r   r'   z$Lexicon.is_number.<locals>.<genexpr>  sP      mmUYUVXY8A;;P!t)P0OAF0OqCxmmmmmmr   )r   rh  r"  r?   r   )rU   r4   suffixesr   s    `  r   	is_numberzLexicon.is_number  s    ------- 	54t<h<<< 	 	A}}Q Hc!ffWH~ mmmm]fgk]l]lmmmmmmr   c                    |j         j        |j        n|j         j                            t	          d          d                              t	          d          d          }t          j        d|          }d                    d |D                       }||                                k    rd n/| j	        t          ||                                k                       }|                     ||j        ||          \  }}|:t          |                     ||j         j                  |j         j                  |fS t$                              ||j         j                  rU|                     ||j         j        |j         j        |j         j                  \  }}t          ||j         j                  |fS t/          d |D                       sdS dS )	Ni   r8  i   NFKCr    c              3   J   K   | ]}t                               |          V  d S r   )r   rw  r,   s     r   r'   z#Lexicon.__call__.<locals>.<genexpr>  s0      BBw0033BBBBBBr   c              3   B   K   | ]}t          |          t          v V  d S r   r  r,   s     r   r'   z#Lexicon.__call__.<locals>.<genexpr>  s-      ::SVV|+::::::r   r   )r   r5   r%   r   chrru  	normalizer<   r+   r   rK  r   r>  r7   r   rr  r   r   r   r}  r4   rp  r3   r   )r   r   r  rU   r   rG   r   s          r   __call__zLexicon.__call__  s   4:-24:FFs4yyRUVV^^_bcg_h_hjmnn$VT22wwBBTBBBBB--43DSQUQ[Q[Q]Q]I]E^E^3_]]4==
F> 4 4R G GUUW]]]tRT\22 	rt}bdlBDN[[JBBDK00&88::T::::: 	:
 zr   N)rM   rN   rO   staticmethodr   r   r   r  r  r  r   r  r$  r)  r.  r0  r6  r>  rC  rp  rr  rw  r}  r  r   r   r   r   r   {   s         \F F F&* * *$ $ $L   \	, 	, 	,0 0 0(  % % %   
& 
& 
&  ' ' '  @ 3 3 \3O O Ob6 6 6 1 1 \1 n n \n    r   r   c            	          e Zd ZddZed             Zdedee         fdZ	d	ee         dee         fd
Z
ed	ee         deeeee         f                  fd            Zed             Zed             Zddedeeee         f         fdZdS )G2PNF   ❓c                 B   || _         || _        d|rdnd }t          j                            |          st          j                            |           |rdnddg}t          j        ||          | _        t          |          | _
        |r|nd | _        || _        d S )Nen_core_web_trfsmtransformertok2vectagger)enable)versionr   spacyutil
is_packageclidownloadr   nlpr   lexiconfallbackr   )r   r  r  r   r  r   name
componentss           r   r   zG2P.__init__  s    6s4ee66z$$T** 	%It$$$'*9mm	8D
:d:666w''$,6$r   c                    d}g }i }d}|                                  } t                              |           D ]}|| ||                                         z  }|                    | ||                                                                                    |                    d          }t          ||d d         dv rdndd                    rt          |          }n|dv rd}n|dk    rd	}nt          |          dk    r?|d         d
k    r3|d         d
k    r'|d         |dd          
                    d
          z   }nTt          |          dk    r?|d         dk    r3|d         dk    r'|d         |dd          
                    d          z   }nd }|||t          |          <   ||                    d          z  }|                    |                    d                     |                                }|t          |           k     r<|| |d          z  }|                    | |d                                                     |||fS )Nr    r   r*   r   )rE  r   )z0.5z+0.5r   z-0.5r   r{   r!   re   )lstrip
LINK_REGEXfinditerstartextendr   rg  r   rK  r?   rstriprL  end)r%   rR  r   featureslast_endmfs          r   
preprocesszG2P.preprocess  s@   {{}}$$T** 	 	Ad8AGGII-..FMM$x		1288::;;;

Aq!u
22!!::;<< FFo%%fQ!!"aD1QRR5<<,,,Q!!"aD1QRR5<<,,,}()V%aggajj FMM!''!**%%%uuwwHHc$iid899o%FMM$xyy///11222vx''r   r%   r   c                    |                      |          }d |D             }|s|S t          j        j                            |d |D                       }|                                D ]W\  }}t          |t                    s%t          |t                    s|dv sJ ||f            t          t          j        |j        j        |k              d                   D ]\  }	}
|
t          |          k    rt          |t                    s|||
         j        _        A|                    d          rQ|	dk    ||
         j        _        |	dk    r|                    d          nd||
         _        d||
         j        _        |                    d          r%|                    d          ||
         j        _        Y|S )	Nc                 |    g | ]9}t          |j        |j        |j        t          j        d dd                    :S )Tr    F)r4   r3   r6   )r%   r7   r&   r   )r   r%   tag_whitespace_r>   r   s     r   r   z G2P.tokenize.<locals>.<listcomp>   s[         !bg".UKKK
 
 
   r   c                     g | ]	}|j         
S r   r   r   s     r   r   z G2P.tokenize.<locals>.<listcomp>&  s    >`>`>`2rw>`>`>`r   )r   r   r   r{   r    r2  re   )r  r  training	Alignmentfrom_stringsr   r   r   rK  r   npwherey2xr   r?   r   r   r   r4   r  r8   r   r3   )r   r%   r   r  docmutable_tokensalignr   r   r   r   s              r   tokenizezG2P.tokenize  s   hhtnn      	"!!(55f>`>`Q_>`>`>`aaNN$$ 	B 	BDAqa%%WAs););WqK?O?O?ORSUVQW?O?OO!"(59>Q+>"?"?"BCC B B1N++++!!S)) 	B12N1%'..\\#&& B23q&N1%'/BCq&&#bN1%.12N1%'.. \\#&& B45HHSMMN1%'1B r   r   c                     g }|D ]Q}|r6|j         j        s*t          |                                |g| j                  n|}|                    |           R|S )Nr   )r   r4   rC   popr   rL  )r   r   rR  r   s       r   	fold_leftzG2P.fold_left8  sh     	 	BCIfRTRVR^fvzz||R0dh????dfBMM"r   c           	         g }d }t          |           D ]\  }j        j        (j        !fdt	          j                  D             }ng}j        |d         _        t          |          D ]n\  }}|j        j        |j        n|j        dk    r*|j        t          v r|j        }d|_        d|j        _	        n||j        dk    r|j        dv rd|_        d	|j        _	        nS|j        t          v rrt          d
 |j        D                       sTt                              |j        d                    d |j        D                                 |_        d|j        _	        n|[|j        dk    rd }n|dz   t          |          k    r6|dz   t          |           k    s| |dz            j        dk    r||j        _        nvd|cxk     rt          |          dz
  k     rYn nV|j        dk    rK||dz
           j        d         ||dz            j        d         z                                   rd|j        _        |j        j        |j        |                    |           |rWt'          |d         t(                    r<|d         d         j        s)d|j        _        |d                             |           O|                    |j        r|n|g           pd |D             S )Nc                     g | ]?}t          |d t          j        dj        j        j        j        d                    @S )r    TF)r4   r3   r   r6   )r%   r&   r   )r   r   r>   r   r3   r   )r   r&  tokens     r   r   z"G2P.retokenize.<locals>.<listcomp>E  se     4 4 4  b'@QZ_ZaZhsxyyy   4 4 4r   r!   rd   r    r   rc   )rE  u   –u   —r   c              3   t   K   | ]3}d t          |                                          cxk    odk    nc V  4dS )rm   z   N)r  r+   r,   s     r   r'   z!G2P.retokenize.<locals>.<genexpr>V  sM      5c5cVWbC		NN6Q6Q6Q6Qc6Q6Q6Q6Q5c5c5c5c5c5cr   c              3   ,   K   | ]}|t           v |V  d S r   )PUNCTSr,   s     r   r'   z!G2P.retokenize.<locals>.<genexpr>W  s.      HkHkq_`dj_j_j_j_j_j_jHkHkr   CDr   r   2r   Fc                 r    g | ]4}t          |t                    rt          |          d k    r|d         n|5S )r   r   )r   r@   r?   )r   rQ  s     r   r   z"G2P.retokenize.<locals>.<listcomp>i  s=    RRRq
1d++BA!!RRRr   )r   r   r5   r8   subtokenizer%   r&   r7   ri  r   
PUNCT_TAGSr   PUNCT_TAG_PHONEMESr   r<   r?   r   r   rL  r   r@   r4   )r   wordsr   r   tksr   r   r  s          @r   
retokenizezG2P.retokenize?  s   !&)) %	@ %	@HAuw}$)?4 4 4 4 'uz224 4 4
 g!&!1CG"3 @ @24:)R[-DVs]]rw*'<'<!wH"$BK"#BDKKVs]]rw,'>'>"'BK"#BDKKVz))#5c5c[][b5c5c5c2c2c)"4"8"8HkHkTVT[HkHkHkAkAk"l"lBK"#BDKK )v~~#'1CacS[[.@.@F1Q3KOW[D[D[(0''''SXXaZ'''''BGsNNAaCb@QRUVWXYVYRZR_`aRb@b?k?k?m?mN!%BDJ4:)R[-DLL$$$$ @z%)T:: @59R=C[ @#(BDL"I$$R((((LLr}!>2$????9@: SRERRRRr   c                     | j         }|rt          d |D             |          n|}|j        dv p|j        dk    o|j        dv }t	          ||          S )Nc              3      K   | ]Et          fd t          t          t          fD                       /t          v rdnt          v V  FdS )c              3       K   | ]}|v V  	d S r   r   )r   r   r-   s     r   r'   z.G2P.token_context.<locals>.<genexpr>.<genexpr>n  sK        ZR  ZRefZ[_`Z`  ZR  ZR  ZR  ZR  ZR  ZRr   N)r=  r   
CONSONANTSNON_QUOTE_PUNCTSr,   s    @r   r'   z$G2P.token_context.<locals>.<genexpr>n  s        S  S1VY  ZR  ZR  ZR  ZRkqs}  @P  kQ  ZR  ZR  ZR  WR  WR  Sa#333dd!v+  S  S  S  S  S  Sr   r   r   r   )rK   rL   )rK   r   r%   r7   rJ   )r  rG   r  vowelrL   s        r   token_contextzG2P.token_contextk  s      _a  l  S  SPR  S  S  S  UZ  [  [  [  glJ,.d5:3E3c%)WcJc	)DDDDr   c                 J   d                     d | d d         D                       | d         j        z   }d|v p d|v pt          d |D                       dk    }t          |           D ]\  }}|j        o|t          |           dz
  k    r'|j        t
          v r|j        |_        d|j        _        It          d	 |j        D                       rd|_        d|j        _        {|d
k    r||j        _	        |rd S d t          |           D             }t          |          dk    r`t          | |d
         d                  j                  dk    r6|d         d         }t          | |         j        d          | |         _        d S t          |          dk     s0t          d |D                       t          |          dz   dz  k    rd S t          |          d t          |          dz           }|D ],\  }}}t          | |         j        d          | |         _        -d S )Nr    c              3   4   K   | ]}|j         |j        z   V  d S r   r$   r   s     r   r'   z%G2P.resolve_tokens.<locals>.<genexpr>t  s+      EE2rw.EEEEEEr   r!   r"   r{   c                 r    h | ]4}|t           v|                                rd nt          |          rdnd5S )r   r   r*   )SUBTOKEN_JUNKSr   r   r,   s     r   r   z%G2P.resolve_tokens.<locals>.<setcomp>u  sk      6P  6P  6Pjkwx  AO  xO  xO199;;6eaaRZ[\R]R]MdQQcd  xO  xO  xOr   r   r   c              3   (   K   | ]}|t           v V  d S r   r  r,   s     r   r'   z%G2P.resolve_tokens.<locals>.<genexpr>{  s'      >>n,>>>>>>r   r   c                 h    g | ]/\  }}|j         t          |j         v t          |j                   |f0S r   )r8   r   rH   )r   r   r   s      r   r   z&G2P.resolve_tokens.<locals>.<listcomp>  sU      A  A  AV[VWY[tvt  ANbk1=3M3MqQ  A  A  Ar   r*   r   c              3   "   K   | ]
\  }}}|V  d S r   r   )r   br   s      r   r'   z%G2P.resolve_tokens.<locals>.<genexpr>  s(      $>$>71aQ$>$>$>$>$>$>r   )r<   r%   r?   r   r8   r  r   r   r   r6   r   r.   rA   )r   r%   r6   r   r   indicesr   s          r   resolve_tokenszG2P.resolve_tokensr  sp   wwEEEEEEEr
W$;  U#+  U  6P  6Pos  6P  6P  6P  2Q  2Q  TU  2Uv&& 		) 		)EAr{"Fa''BG7G,G,G"$'BK"#BDKK>>bg>>>>> $"$BK"#BDKQ ( 	F A  A_hio_p_p  A  A  Aw<<1VGAJqM%:%?!@!@A!E!E
1A!-fQi.@$!G!GF1IF\\A$>$>g$>$>$>!>!>3w<<PQ>VWBW!W!WF//"23w<<?"23 	H 	HGAq!!-fQi.@$!G!GF1I	H 	Hr   Tc                     |dk    rt           j        n|}|r ||          n|g i f\  }}}                     |||          }                     |          }t                               |          }t                      }t          t          t          |                              D ]\  }}t          |t                    s|j
        7                     t          ||j                  |          \  |_
        |_        |j
        = j        6                     t          ||j                            \  |_
        |_        t                               ||j
        |          }dt#          |          }	}d}
||	k     r t%          d |||	         D                       rd }nt'          |||	                   }|dn                     ||          \  }}|^|||         _
        |||         j        _        ||dz   |	         D ]}d|_
        ||_        t                               |||          }|}	d}na|dz   |	k     r|dz  }nR|	dz  }	||	         }|j
        <t)          d	 |j        D                       rd|_
        d
|j        _        n
 j        d}
n	d}||	k      |
rt'          |          }                     |          \  |d         _
        |d         j        _        t-          dt#          |                    D ]1}d||         _
        |d         j        j        ||         j        _        2t                               |            fd|D             } j        dk    r@|D ]=}|j
        r4|j
                            dd                              dd          |_
        >d                     fd|D                       }||fS )NT)r   r   Fc              3   D   K   | ]}|j         j        d up|j        d uV  d S r   )r   r5   r8   r   s     r   r'   zG2P.__call__.<locals>.<genexpr>  s7      aaRrtz-HD1Haaaaaar   r   r   r    c              3   (   K   | ]}|t           v V  d S r   r  r,   s     r   r'   zG2P.__call__.<locals>.<genexpr>  s'      DDqqN2DDDDDDr   r   c                 j    g | ]/}t          |t                    rt          |j                   n|0S )r  )r   r@   rC   r   r   r   r   s     r   r   z G2P.__call__.<locals>.<listcomp>  s=    bbbUWJr44H4HP,rtx0000bbbbr   z2.0u   ɾTu   ʔr&  c              3   R   K   | ]!}|j         j        n|j         |j        z   V  "d S r   )r8   r   r&   r  s     r   r'   zG2P.__call__.<locals>.<genexpr>  s:      mm`bbk&9$((r{bm[mmmmmmr   )r  r  r  r  r  rJ   reversedr@   r   r   r8   r  r   r   r   r  r  r?   r=  rC   r   r%   ranger  r  r<   )r   r%   r  r   r  r  r   rQ  leftrightshould_fallbackr   rG   r   xr   rR  s   `                r   r  zG2P.__call__  s   '1T'9'9S^^z
5?!SD!1!1!1dBPR^fhtVX66''''nnT)F"3"34455 -	& -	&DAqa&& :%+/<<QS8I8I8I3+O+O(AJ:%$-*C+/==ac9J9J9J+K+K(AJ''QZ;;SVV%D#O,,aaSTUYZ_U_S`aaaaa 5BB%aU
m44B-/Z\\T\\"c=R=R
F>')AdG$'-AdGI$tAve|_ * *%'
#)++CR88C EDDAX%%AIDDQJE5B{*DDBGDDDDD "*,BK*+BDKK!]6.2O!D7 ,,8  &!!__/3}}R/@/@,!qtv}q#a&&)) 2 2A$&AaDM$%aDFMAaDFMM2 ""1%%%%bbbb[abbb<5   T T; T"$+"5"5dC"@"@"H"Hs"S"SBKmmmmflmmmmmv~r   )NFFNr  )T)rM   rN   rO   r   r  r  r   r	   r   r  r  r   r  r  r  r   r  r   r   r   r  r    sQ       
 
 
 
 ( ( \(@S tF|    6V f     )S4< )SDvtF|7K1L,M )S )S )S \)SV E E \E H H \H4; ;S ;eCf<M6N ; ; ; ; ; ;r   r  r   )7r    r   r  r   dataclassesr   r   r   typingr	   r
   r   r   importlib.resourcesr   r   numpyr  r   r  ru  r   rC   	frozensetrF   rH   rJ   rY   r  rX   r  r  r  r  r  r  r  r  r  r  r(  ri  rh  r   r   r   r   r   r   r   r   r   r   r   r  r   r   r   <module>r     sh               * * * * * * * *       / / / / / / / / / / / /          				      f HSM V    < Y|$$
E E E        
> > > $#%% RZ344
>**	(	)	)9EEEEEEE  YQQQRR
!3SSYYSSYYUXUXY]U^U^__ B8r28r38Y<==

)-
.
. 

 

 9---..g&&eT
:
:9_``9[\\!A; 	5	6	6  6- - -r r r r r r r rhY Y Y Y Y Y Y Y Y Yr   