+
    &jw0                        ^ RI t ^ RIt^ RIt^ RIHtHt ^ RIHtHt ^ RI	H
t
HtHtHt ^ RIt/ RRbRRbR	R
bRRbRRbRRbRRbRRbRRbRRbRRbRRbRRbRR bR!R"bR#R$bR%R&b/ R'R(bR)R*bR+R,bR-R.bR/R0bR1R2bR3R4bR5R6bR7R8bR9R:bR;R<bR=R>bR?R@bRARBbRCRDbRERFbRGRHbC/ RIRJbRKRLbRMRNbRORPbRQRRbRSRTbRURVbRWRXbRYRZbR[R\bR]R^bR_R`bRaRbbRcRdbReRfbRgRhbRiRjbC/ RkRlbRmRnbRoRpbRqRrbRsRtbRuRvbRwRxbRyRzbR{R|bR}R~bRRbRRbRRbRRbRRbRRbRRbC/ RRbRRbRRbRRbRRbRRbRRbRRbRRbRRbRRbRRbRRbRRbRRbRRbRRbCRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR/Ct/ ]P!                  4        U Uu/ uF  w  rWbK	  	  upp CRRRRRRRRRRRRRRRR7RR7RRRRRR/Ct] ! R R4      4       t]! RR7      RR R ll4       t]! RR7      R^cRRRR/R R ll4       tR# u upp i )    N)	dataclassfield)cached_property	lru_cache)DictListOptionalTupleenenglishzhchinesedegermanesspanishrurussiankokoreanfrfrenchjajapanesept
portuguesetrturkishplpolishcacatalannldutchararabicsvswedishititalianid
indonesianhihindififinnishvi
vietnamesehehebrewuk	ukrainianelgreekmsmalaycsczechroromaniandadanishhu	hungariantatamilno	norwegianththaiururduhrcroatianbg	bulgarianlt
lithuanianlalatinmimaoriml	malayalamcywelshskslovaktetelugufapersianlvlatvianbnbengalisrserbianazazerbaijanisl	slovenianknkannadaetestonianmk
macedonianbrbretoneubasqueis	icelandichyarmeniannenepalimn	mongolianbsbosniankkkazakhsqalbanianswswahiliglgalicianmrmarathipapunjabisisinhalakmkhmersnshonayoyorubasosomaliaf	afrikaansococcitankageorgianbe
belarusiantgtajiksdsindhigugujaratiamamharicyiyiddishlolaouzuzbekfofaroesehtzhaitian creolepspashtotkturkmennnnynorskmtmaltesesasanskritlbluxembourgishmymyanmarbotibetantltagalogmgmalagasyasassamesetttatarhawhawaiianlnlingalahahausababashkirjwjavanesesu	sundaneseyue	cantoneseburmese	valencianflemishhaitianletzeburgeschpushtopanjabi	moldavianmoldovan	sinhalese	castilianmandarinc                   B  a  ] tR t^t o RtRtRtR/t]! ]	R7      t
R tR tV 3R lR ltV 3R lR	 lt]V 3R
 lR l4       t]V 3R lR l4       t]V 3R lR l4       t]V 3R lR l4       t]V 3R lR l4       t]V 3R lR l4       t]V 3R lR l4       t]V 3R lR l4       t]V 3R lR l4       t]V 3R lR l4       tR t]V 3R lR  l4       t]V 3R! lR" l4       t]V 3R# lR$ l4       t]V 3R% lR& l4       tV 3R' lR( ltV 3R) lR* lt V 3R+ lR, lt!V 3R- lt"R.t#V t$R# )0	TokenizerzIA thin wrapper around `tiktoken` providing quick access to special tokensN)default_factoryc                r   V P                   P                   F,  pV P                   P                  V4      pW P                  V&   K.  	  V P                  R ,          pV P                  R,          pV P                  R,          p\	        \
        P                  4       4      RV P                   pV.pV P                  e9   VP                  V^,           VP                  V P                  4      ,           4       V P                  e'   V P                  R8X  d   TMTpVP                  V4       \	        V4      V n        R# )<|startoftranscript|><|translate|><|transcribe|>N
transcribe)encodingspecial_tokens_setencode_single_tokenspecial_tokenstuple	LANGUAGESkeysnum_languageslanguageappendindextasksot_sequence)	selfspecialspecial_tokensot	translater   langsr   
task_tokens	   &        m/Users/jameslopez/projects/CWCArchive/cwc-podcast/.venv/lib/python3.14/site-packages/mlx_whisper/tokenizer.py__post_init__Tokenizer.__post_init__   s    }}77G MM==gFM+8( 8 &&'>?,,_=	--.>?
inn&'(<$*<*<=u==$a%++dmm*D DE99 ,0II,Ej9J
+!,/    c                <    V P                   P                  ! V3/ VB # N)r   encode)r   textkwargss   &&,r   r   Tokenizer.encode   s    }}##D3F33r   c                6   < V ^8  d   QhRS[ S[,          RS[/#    	token_idsreturnr   intstr)format__classdict__s   "r   __annotate__Tokenizer.__annotate__   s     9 9S	 9 9r   c                    V Uu. uF  q3V P                   8  g   K  VNK  	  ppV P                  P                  ! V3/ VB # u upi r   )timestamp_beginr   decode)r   r  r  ts   &&, r   r  Tokenizer.decode   sC     )F	11E1E-EQQ		F}}##I888 Gs
   AAc                6   < V ^8  d   QhRS[ S[,          RS[/# r  r	  )r  r  s   "r   r  r     s     9 9S	 9 9r   c                <    V P                   P                  ! V3/ VB # )z
Timestamp tokens are above other special tokens' id range and are ignored by `decode()`.
This method decodes given tokens with timestamps tokens annotated, e.g. "<|1.08|>".
)r   r  )r   r  r  s   &&,r   decode_with_timestamps Tokenizer.decode_with_timestamps   s    
 }}##I888r   c                    < V ^8  d   QhRS[ /# r  r  r
  )r  r  s   "r   r  r     s     ' 'S 'r   c                .    V P                   P                  # r   )r   	eot_tokenr   s   &r   eotTokenizer.eot   s    }}&&&r   c                    < V ^8  d   QhRS[ /# r  r  )r  r  s   "r   r  r     s     5 5C 5r   c                (    V P                   R ,          # )r   r   r  s   &r   r   Tokenizer.transcribe   s    ""#344r   c                    < V ^8  d   QhRS[ /# r  r  )r  r  s   "r   r  r     s     4 43 4r   c                (    V P                   R ,          # )r   r#  r  s   &r   r   Tokenizer.translate       ""?33r   c                    < V ^8  d   QhRS[ /# r  r  )r  r  s   "r   r  r     s     < <S <r   c                (    V P                   R ,          # )r   r#  r  s   &r   r   Tokenizer.sot   s    ""#:;;r   c                    < V ^8  d   QhRS[ /# r  r  )r  r  s   "r   r  r     s     4 4 4r   c                (    V P                   R ,          # )<|startoflm|>r#  r  s   &r   sot_lmTokenizer.sot_lm   r(  r   c                    < V ^8  d   QhRS[ /# r  r  )r  r  s   "r   r  r     s     6 6# 6r   c                (    V P                   R ,          # )<|startofprev|>r#  r  s   &r   sot_prevTokenizer.sot_prev   s    ""#455r   c                    < V ^8  d   QhRS[ /# r  r  )r  r  s   "r   r  r     s     3 33 3r   c                (    V P                   R ,          # )<|nospeech|>r#  r  s   &r   	no_speechTokenizer.no_speech   s    "">22r   c                    < V ^8  d   QhRS[ /# r  r  )r  r  s   "r   r  r     s     7 7s 7r   c                (    V P                   R ,          # )<|notimestamps|>r#  r  s   &r   no_timestampsTokenizer.no_timestamps   s    ""#566r   c                    < V ^8  d   QhRS[ /# r  r  )r  r  s   "r   r  r     s     / / /r   c                (    V P                   R ,          # )z<|0.00|>r#  r  s   &r   r  Tokenizer.timestamp_begin   s    "":..r   c                    < V ^8  d   QhRS[ /# r  r  )r  r  s   "r   r  r     s     5 5 5r   c                j    V P                   f   \        R4      hV P                  V P                   4      # )zGReturns the token id corresponding to the value of the `language` fieldz6This tokenizer does not have language token configured)r   
ValueErrorto_language_tokenr  s   &r   language_tokenTokenizer.language_token   s/     == UVV%%dmm44r   c                t    V P                   P                  R V R2R4      ;p'       d   V# \        RV R24      h)<||>Nz	Language z not found in tokenizer.)r   getKeyError)r   r   tokens   && r   rF  Tokenizer.to_language_token   sC    ''++b
",=tDD5DL8*,DEFFr   c                0   < V ^8  d   QhRS[ S[,          /# r  r
   r
  )r  r  s   "r   r  r     s     3 3U3Z 3r   c                    . pV P                   P                  4        F2  w  r#VP                  R 4      \        9   g   K!  VP	                  V4       K4  	  \        V4      RV P                   # z<|>N)r   itemsstripr   r   r   r   )r   resultrN  token_ids   &   r   all_language_tokensTokenizer.all_language_tokens   sY    #2288:OE{{5!Y.h'  ; V}1t1122r   c                0   < V ^8  d   QhRS[ S[,          /# r  )r
   r  )r  r  s   "r   r  r     s     X XE#J Xr   c                   a  \         ;QJ d!    . V 3R  lS P                   4       F  NK  	  5# ! V 3R  lS P                   4       4      # )c              3   f   <"   T F&  pSP                  V.4      P                  R 4      x  K(  	  R# 5irS  )r  rU  ).0_lr   s   & r   	<genexpr>/Tokenizer.all_language_codes.<locals>.<genexpr>   s,     W>VT[["&,,U33>Vs   .1)r   rX  r  s   fr   all_language_codesTokenizer.all_language_codes   s1    uWd>V>VWuWuWd>V>VWWWr   c                0   < V ^8  d   QhRS[ S[,          /# r  rQ  )r  r  s   "r   r  r     s     E EU3Z Er   c                b    \        \        V P                  4      V P                  .,           4      # r   )r   listr   r>  r  s   &r   #sot_sequence_including_notimestamps-Tokenizer.sot_sequence_including_notimestamps   s&    T$++,0B0B/CCDDr   c                0   < V ^8  d   QhRS[ S[,          /# r  rQ  )r  r  s   "r   r  r     s     !% !%5: !%r   c                   \        R4      pVRP                  4       ,          p\        R4      p\        ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       g   Q hV P                  P                  R4      ^ ,          V P                  P                  R4      ^ ,          0pV\        V4      ,            Fw  pV P                  P                  V4      V P                  P                  R	V,           4      3 F3  p\        V4      ^8X  g	   WB9   g   K  VP                  V^ ,          4       K5  	  Ky  	  \        \        V4      4      # )
uM  
Returns the list of tokens to suppress in order to avoid any speaker tags or non-speech
annotations, to prevent sampling texts that are not actually spoken in the audio, e.g.

- ♪♪♪
- ( SPEAKING FOREIGN LANGUAGE )
- [DAVID] Hey there,

keeping basic punctuations like commas, periods, question marks, exclamation points, etc.
u#   "#()*+/:;<=>@[\]^_`{|}~「」『』uK   << >> <<< >>> -- --- -( -[ (' (" (( )) ((( ))) [[ ]] {{ }} ♪♪ ♪♪♪u   ♩♪♫♬♭♮♯c              3   `   "   T F$  pR \        V4      u;8*  ;'       d    R8*  Mu x  K&  	  R# 5i)i@&  i&  N)ord)r]  cs   & r   r_  .Tokenizer.non_speech_tokens.<locals>.<genexpr>	  s$     E}!6SV--v--}s   ,.FTz -z ' )
re  splitsetallr   r   lenaddr   sorted)r   symbolsmiscellaneousrV  symboltokenss   &     r   non_speech_tokensTokenizer.non_speech_tokens   s    =>Z``b	
 34sE}EsssE}EEEEE --&&t,Q/1E1Ed1KA1NO] 333F$$V,$$S6\2 v;!#v'>JJvay) 4 VF^$$r   c                0   < V ^8  d   QhRS[ S[,          /# r  rx  r   r
  )r  r  s   "r   r  r    s     3 349 3r   c                h    V P                   R9   d   V P                  V4      # V P                  V4      # )r   >   r   r   r   rG   r   r   )r   split_tokens_on_unicodesplit_tokens_on_spaces)r   rx  s   &&r   split_to_word_tokensTokenizer.split_to_word_tokens  s3    ==AA //77**622r   c                0   < V ^8  d   QhRS[ S[,          /# r|  r}  )r  r  s   "r   r  r     s     " "d3i "r   c                V   V P                  V4      pR p. p. p. p^ pV F  pVP                  V4       V P                  V4      p	W99  g&   W'V	P                  V4      ,           ,          V8X  g   KP  VP                  V	4       VP                  V4       . pV\        V	4      ,          pK  	  WE3# )u   �)r  r   r   rr  )
r   rx  decoded_fullreplacement_charwordsword_tokenscurrent_tokensunicode_offsetrN  decodeds
   &&        r   r  !Tokenizer.split_tokens_on_unicode   s    226:#E!!%(11.AG !/?O1P PQ#$ W%"">2!##g,.  !!r   c                0   < V ^8  d   QhRS[ S[,          /# r|  r}  )r  r  s   "r   r  r  9  s     " "T#Y "r   c                   V P                  V4      w  r#. p. p\        W#4       F  w  rgV^ ,          V P                  8  pVP                  R4      p	VP	                  4       \
        P                  9   p
V'       g!   V	'       g   V
'       g   \        V4      ^ 8X  d%   VP                  V4       VP                  V4       K  VR,          V,           VR&   VR,          P                  V4       K  	  WE3# )r   rn  )
r  zipr  
startswithrU  stringpunctuationrr  r   extend)r   rx  subwordssubword_tokens_listr  r  subwordsubword_tokensr   
with_spacer  s   &&         r   r   Tokenizer.split_tokens_on_spaces9  s    (,(D(DV(L%'*8'I#G$Q'4883G ++C0J!--/V-?-??K*s5zQW%"">2!"I/b	B&&~6 (J !!r   c                   < V ^8  d   Qh/ S[ P                  ;R&   S[;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[S[3,          ;R&   # )r  r   r   r   r   r   r   )tiktokenEncodingr
  r	   r  r
   r   )r  r  s   "r   r  r     sm      	 
   sm"  3-  *!  cN@ r   )r    )%__name__
__module____qualname____firstlineno____doc__r   r   r   r   dictr   r   r   r  r  r   r  r   r   r   r/  r4  r9  r>  r  rG  rF  rX  ra  rf  ry  r  r  r  __annotate_func____static_attributes____classdictcell__)r  s   @r   r   r      s    S #HD!L%*4%@N0&49 99 9 ' ' 5 5 4 4 < < 4 4 6 6 3 3 7 7 / / 5 5G 3 3 X X E E !% !%F3 3" "2" "i  r   r   )maxsizec                0    V ^8  d   QhR\         R\        /# )r  namer   )r  r
  )r  s   "r   r  r  M  s     ! !s !C !r   c           
      
   \         P                  P                  \         P                  P                  \        4      R V  R24      p\        V4      ;_uu_ 4       pR V 4        UUu/ uF%  w  rE\        P                  ! V4      \        V4      bK'  	  pppRRR4       \        X4      p/ pRR.\        \        P                  4       4      RV  U	u. uF	  p	RV	 R2NK  	  up	ORNR	NR
NRNRNRN\        R4       U
u. uF  p
RV
R,          R R2NK  	  up
OpV F  pWxV&   V^,          pK  	  \        P                  ! \         P                  P!                  V4      VRVVR7      # u uppi   + '       g   i     L; iu up	i u up
i )assetsz	.tiktokenc              3   T   "   T F  q'       g   K  VP                  4       x  K   	  R # 5ir   )ro  )r]  lines   & r   r_  get_encoding.<locals>.<genexpr>R  s     E

s   	((Nz<|endoftext|>r   rJ  rK  r   r   r.  r3  r8  r=  i  g{Gz?z.2fzJ's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+)r  explicit_n_vocabpat_strmergeable_ranksr   )ospathjoindirname__file__openbase64	b64decoder
  rr  re  r   r   ranger  r  basename)r  r   
vocab_pathfidrN  rankranksn_vocabr   langispecialss   &&          r   get_encodingr  L  s   bggooh7dV9CUVJ	j		S  FE
E U#SY.E 	 
 

 %jGN 	 %))9$:>M$J	K$JDBtfB-$J	K 		
 	 	 	 	 	 */t	5ABq4xnB
	5H  'u1  WWj) a% 1
 
	 
L 
6s*   E(&+E"
E(E;9F "E((E8	r   r   r   c          
          V ^8  d   QhR\         R\        R\        \        ,          R\        \        ,          R\        /# )r  multilingualr   r   r   r  )boolr
  r	   r  r   )r  s   "r   r  r  r  sD        sm	
 3- r   c                   VeC   VP                  4       pV\        9  d(   V\        9   d   \        V,          pM\        RV 24      hV '       d   RpT;'       g    RpT;'       g    RpMRpR pR p\	        WAR7      p\        WQW#R7      # )NzUnsupported language: r  r   r   gpt2)r  r   )r   r   r   r   )lowerr   TO_LANGUAGE_CODErE  r  r   )r  r   r   r   encoding_namer   s   &$$$  r   get_tokenizerr  q  s     >>#9$+++H5 #9(!DEE&##t##|LH r   )r  c   )r  r  r  dataclassesr   r   	functoolsr   r   typingr   r   r	   r
   r  r   rT  r  r   r  r  )coder   s   00r   <module>r     s    	  ( 0 . . e)e)e 	(e 	)	e
 	)e 	(e 	(e 	*e 	,e 	)e 	(e 	)e 	'e 	(e 	)e  	)!e" 	,#e$ 	'%e& 	)'e( 	,)e* 	(+e, 	+-e. 	'/e0 	'1e2 	'3e4 	*5e6 	(7e8 	+9e: 	';e< 	+=e> 	&?e@ 	&AeB 	*CeD 	+EeF 	,GeH 	'IeJ 	'KeL 	+MeN 	'OeP 	(QeR 	(SeT 	)UeV 	)WeX 	)YeZ 	)[e\ 	-]e^ 	+_e` 	)aeb 	*ced 	,eef 	(geh 	(iej 	+kel 	*men 	(oep 	+qer 	)set 	(uev 	*wex 	)yez 	*{e| 	)}e~ 	)e@ 	)AeB 	'CeD 	'EeF 	(GeH 	(IeJ 	+KeL 	)MeN 	*OeP 	,QeR 	'SeT 	(UeV 	*WeX 	)YeZ 	)[e\ 	%]e^ 	'_e` 	)aeb 	
ced 	(eef 	)geh 	)iej 	)kel 	*/)))**'	:)')*+	;Ie	P,5OO,=>,=.$x~,=>tttTdt " C" C" C"L 4! !H 4  #	
  { ?s   *G