+
    &j                    ~    ^ RI Ht ^ RIt^ RIt^ RItR R ltR R ltRR R lltRR	 R
 lltR R lt	RR R llt
R# )    )annotationsNc                    V ^8  d   QhRRRR/# )   blobpathstrreturnbytes )formats   "e/Users/jameslopez/projects/CWCArchive/cwc-podcast/.venv/lib/python3.14/site-packages/tiktoken/load.py__annotate__r      s     ) ) ) )    c                   R V 9  d2   \        V R^ R7      ;_uu_ 4       pVP                  4       uuRRR4       # V P                  R4      '       d2   ^ RIpVP	                  V 4      pVP                  4        VP                  #  ^ RIpTP                  T 4      #   + '       g   i     Ln; i  \         d   p\        R4      ThRp?ii ; i)z://rb	bufferingNOblobfile is not installed. Please install it by running `pip install blobfile`.)zhttp://zhttps://)
openread
startswithrequestsgetraise_for_statuscontentblobfileImportError
read_bytes)r   fr   respr   es   &     r   	read_filer!      s    H(DA..!668 /. 233||H%||
 x((# /.  ]
	s#   BB* B'	*C5CCc               $    V ^8  d   QhRRRRRR/# )r   datar	   expected_hashr   r   boolr
   )r   s   "r   r   r      s!     ( (U (3 (4 (r   c                R    \         P                  ! V 4      P                  4       pW!8H  # N)hashlibsha256	hexdigest)r#   r$   actual_hashs   && r   
check_hashr,      s"    ..&002K''r   c               $    V ^8  d   QhRRRRRR/# )r   r   r   r$   
str | Noner   r	   r
   )r   s   "r   r   r   #   s!     3 3s 3: 3 3r   c                "   R pR\         P                  9   d   \         P                  R,          pMaR\         P                  9   d   \         P                  R,          pM4^ RIp\         P                  P	                  VP                  4       R4      pRpVR8X  d   \        V 4      # \        P                  ! V P                  4       4      P                  4       p\         P                  P	                  W54      p\         P                  P                  V4      '       d_   \        VR^ R7      ;_uu_ 4       pVP                  4       pRRR4       Ve   \        XV4      '       d   X#  \         P                  ! V4       \        V 4      p	V'       d$   \        W4      '       g   \#        R	V  R
V R24      h^ RIp
 \         P&                  ! VR R7       VR,           \)        V
P+                  4       4      ,           R,           p\        VR4      ;_uu_ 4       pVP-                  V	4       RRR4       \         P.                  ! W4       V	#   + '       g   i     EL; i  \          d     Li ; i  + '       g   i     LM; i  \          d    T'       d   h  T	# i ; i)TTIKTOKEN_CACHE_DIRDATA_GYM_CACHE_DIRNzdata-gym-cacheF r   r   z'Hash mismatch for data downloaded from z (expected z<). This may indicate a corrupted download. Please try again.)exist_ok.z.tmpwb)osenvirontempfilepathjoin
gettempdirr!   r(   sha1encoder*   existsr   r   r,   removeOSError
ValueErroruuidmakedirsr   uuid4writerename)r   r$   user_specified_cache	cache_dirr8   	cache_key
cache_pathr   r#   contentsrB   tmp_filenames   &&          r   read_file_cachedrM   #   s   rzz)JJ34			+JJ34	GGLL!4!4!68HI	$B""X__./99;Ii3J	ww~~j!!*da00A668D 1 Jt]$C$CK	IIj! "HZ@@5hZ{=/ ZH I
 	

 	
I-!C'#djjl*;;fD,%%GGH &
		,+ O? 100  		 &%     OsO   /H>!I 2AI6 I#I6 >I	I I #I3	.I6 6J	JJc               0    V ^8  d   QhRRRRRRRRRRR	R
/# )r   vocab_bpe_filer   encoder_json_filevocab_bpe_hashr.   encoder_json_hashclobber_one_byte_tokensr%   r   dict[bytes, int]r
   )r   s   "r   r   r   Y   sD     7 777 7 "	7
 "7 7r   c                  a \        R4       Uu. uF8  p\        V4      P                  4       '       g   K$  \        V4      R8w  g   K6  VNK:  	  ppV Uu/ uF  p\        V4      VbK  	  upo^ p\        R4       F:  pWV9  g   K  VP                  V4       VS\        RV,           4      &   V^,          pK<  	  \	        V4      R8X  g   Q h\        W4      P                  4       pVP                  R4      ^R	  U	u. uF  p	\        V	P                  4       4      NK  	  p
p	R V3R llp\        V4       UUu/ uF  w  r\        V.4      VbK  	  ppp?\	        V4      pV
 F%  w  rW}V! V4      V! V4      ,           &   V^,          pK'  	  ^ RIpVP                  \        W4      4      pVP                  4        UUu/ uF  w  ppV! V4      VbK  	  pppVP                  RR4       VP                  RR4       V'       d(   V F!  p\	        V4      ^8X  g   K  VV,          VV&   K#  	  VV8X  g   Q hV# u upi u upi u up	i u uppi u uppi )
r    
c                    V ^8  d   QhRRRR/# )r   valuer   r   r	   r
   )r   s   "r   r   5data_gym_to_mergeable_bpe_ranks.<locals>.__annotate__p   s     > >s >u >r   c                .   < \        V3R  lV  4       4      # )c              3  6   <"   T F  pSV,          x  K  	  R # 5ir'   r
   ).0bdata_gym_byte_to_bytes   & r   	<genexpr>Kdata_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gym.<locals>.<genexpr>q   s     =u!*1--us   )r	   )rY   r_   s   &r   decode_data_gym8data_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gymp   s    =u===r   Ns   <|endoftext|>s   <|startoftext|>   )rangechrisprintableappendlenrM   decodesplittuple	enumerater	   jsonloadsitemspop)rO   rP   rQ   rR   rS   r^   rank_to_intbytenvocab_bpe_contents	merge_str
bpe_mergesrb   i	bpe_ranksfirstsecondro   encoder_jsonkvencoder_json_loadedr_   s   &&&&&                @r   data_gym_to_mergeable_bpe_ranksr   Y   s/    #(+X+QQ1C1C1Eq#a&TW-qq+OX0?@1SVQY@	A4[#""1%34!#dQh-0FA	 
 4''' *.IPPR<N<T<TUY<Z[\]_<`a<`y%	)*<`Ja> >
 ,5_+EF+E41sQ+EIF 	IA#FG/%(?6+BBC	Q $ 
 ::./@TUL=I=O=O=QR=QTQ?1-q0=QR,d3.5$A1v{215	! % ++++_ Y@ b G Ss(   !H+H+H+H07"H52H:5I c               $    V ^8  d   QhRRRRRR/# )r   ry   rT   tiktoken_bpe_filer   r   Noner
   )r   s   "r   r   r      s'     	Q 	Q!1 	Qc 	Qd 	Qr   c           	         ^ RI pTP                  TR4      ;_uu_ 4       p\        T P	                  4       R R7       FV  w  rVTP                  \        P                  ! T4      R,           \        T4      P                  4       ,           R,           4       KX  	  RRR4       R#   \         d   p\        R4      ThRp?ii ; i  + '       g   i     R# ; i)r   Nr   r5   c                    V ^,          # )   r
   )xs   &r   <lambda>#dump_tiktoken_bpe.<locals>.<lambda>   s    1Q4r   )key       
)
r   r   BlobFilesortedrq   rE   base64	b64encoder   r=   )ry   r   r   r    r   tokenranks   &&     r   dump_tiktoken_bper      s    
 
		,d	3	3q!)//"3HKEGGF$$U+d2SY5E5E5GG%OP I 
4	3	  ]
	 
4	3	3s#   B! A7C !B=,B88B= C	c               $    V ^8  d   QhRRRRRR/# )r   r   r   r$   r.   r   rT   r
   )r   s   "r   r   r      s"       Z Sc r   c                   \        W4      p/ pVP                  4        FB  pV'       g   K   VP                  4       w  rV\        V4      V\        P
                  ! V4      &   KD  	  V#   \         d   p\        R T: RT  24      ThRp?ii ; i)zError parsing line z in N)rM   
splitlinesrl   intr   	b64decode	ExceptionrA   )r   r$   rK   retliner   r   r    s   &&      r   load_tiktoken_bper      s     1AH
C##%	[**,KE+.t9C  '( & J  	[24($?P>QRSYZZ	[s   4A&&B	1BB	r'   )NNF)
__future__r   r   r(   r6   r!   r,   rM   r   r   r   r
   r   r   <module>r      s7    "   	),(
3l7t	Q r   