+
    &j%                     t    R t ^ RIHt ^ RIt^ RIt^ RIt ! R R4      tRR R lltRR R lltR	 R
 lt	R t
R# )zJThis is an educational implementation of the byte pair encoding algorithm.)annotationsNc                  |    ] tR t^tR R ltRR R lltR R ltR R ltR	 R
 lt]	R R l4       t
]	R 4       tRtR# )SimpleBytePairEncodingc               $    V ^8  d   QhRRRRRR/# )   pat_strstrmergeable_ranksdict[bytes, int]returnNone )formats   "m/Users/jameslopez/projects/CWCArchive/cwc-podcast/.venv/lib/python3.14/site-packages/tiktoken/_educational.py__annotate__#SimpleBytePairEncoding.__annotate__   s"     + +3 +9I +d +    c                   Wn         W n        VP                  4        UUu/ uF  w  r4WCbK	  	  uppV n        \        P
                  ! V4      V n        R# u uppi )zCreates an Encoding object.N)r   r	   items_decoderregexcompile_pat)selfr   r	   token_bytestokens   &$$  r   __init__SimpleBytePairEncoding.__init__   sL     .FUF[F[F]^F]0B+F]^MM'*	 _s   Ac               $    V ^8  d   QhRRRRRR/# )r   textr   	visualise
str | Noner   	list[int]r   )r   s   "r   r   r      s!      3 : Y r   c                    V P                   P                  V4      p. pV F<  pVP                  R4      p\        V P                  WbR7      pVP                  V4       K>  	  V# )zHEncodes a string into tokens.

>>> enc.encode("hello world")
[388, 372]
utf-8)r    )r   findallencode
bpe_encoder	   extend)r   r   r    wordstokensword
word_bytesword_tokenss   &&&     r   r&   SimpleBytePairEncoding.encode   sZ     		!!$'DW-J$T%9%9:[KMM+&	 
 r   c                    V ^8  d   QhRRRR/# )r   r*   r"   r   bytesr   )r   s   "r   r   r   '   s     B B9 B Br   c                :   a  RP                  V 3R lV 4       4      # )zVDecodes a list of tokens into bytes.

>>> enc.decode_bytes([388, 372])
b'hello world'
r   c              3  J   <"   T F  pSP                   V,          x  K  	  R # 5iNr   ).0r   r   s   & r   	<genexpr>6SimpleBytePairEncoding.decode_bytes.<locals>.<genexpr>-   s     A&e,,&s    #)joinr   r*   s   f&r   decode_bytes#SimpleBytePairEncoding.decode_bytes'   s     xxA&AAAr   c                    V ^8  d   QhRRRR/# )r   r*   r"   r   r   r   )r   s   "r   r   r   /   s     	K 	KY 	K3 	Kr   c                F    V P                  V4      P                  RRR7      # )u   Decodes a list of tokens into a string.

Decoded bytes are not guaranteed to be valid UTF-8. In that case, we replace
the invalid bytes with the replacement character "�".

>>> enc.decode([388, 372])
'hello world'
r$   replaceerrors)r:   decoder9   s   &&r   rA   SimpleBytePairEncoding.decode/   s%       (//	/JJr   c                    V ^8  d   QhRRRR/# )r   r*   r"   r   list[bytes]r   )r   s   "r   r   r   :   s     : :) : :r   c                P    V Uu. uF  q P                   V,          NK  	  up# u upi )zDecodes a list of tokens into a list of bytes.

Useful for visualising how a string is tokenised.

>>> enc.decode_tokens_bytes([388, 372])
[b'hello', b' world']
r4   )r   r*   r   s   && r   decode_tokens_bytes*SimpleBytePairEncoding.decode_tokens_bytes:   s&     399&e$$&999s   #c               $    V ^8  d   QhRRRRRR/# )r   training_datar   
vocab_sizeintr   r   )r   s   "r   r   r   E   s&     X XS Xc XC Xr   c                4    \        WVR7      p\        W#R7      # )z#Train a BPE tokeniser on some data!)datarJ   r   r   r	   )	bpe_trainr   )rI   rJ   r   r	   s   &&& r   trainSimpleBytePairEncoding.trainD   s     $W^_%gWWr   c                	    \        V \        4      '       d   \        P                  ! V 4      p \	        V P
                  V P                  R 7      # )rN   )
isinstancer   tiktokenget_encodingr   _pat_str_mergeable_ranks)encodings   &r   from_tiktoken$SimpleBytePairEncoding.from_tiktokenJ   s=    h$$,,X6H%%%x7P7P
 	
r   )r   r   r	   r   Ncolour)__name__
__module____qualname____firstlineno__r   r&   r:   rA   rF   staticmethodrP   rY   __static_attributes__r   r   r   r   r      sG    + B	K: X X
 
 
r   r   c               (    V ^8  d   QhRRRRRRRR/# )	r   r	   r
   inputr0   r    r!   r   r"   r   )r   s   "r   r   r   S   s,     ! !%!.3!@J!!r   c                V   V Uu. uF  p\        V.4      NK  	  pp V'       d&   VR9   d   \        V4       MVR8X  d   \        V4       RpRp\        \	        VRR VR,          4      4       FA  w  rxV P                  V^ ,          V^,          ,           4      p	V	f   K1  Ve	   W8  g   K=  TpT	pKC  	  Vf   M?Vf   Q hVRV WE,          WE^,           ,          ,           .,           WE^,           R ,           pK  V'       d   \        4        V U
u. uF  qV
,          NK  	  pp
V# u upi u up
i )TsimpleN   NNr\   color)r0   visualise_tokensprint	enumeratezipget)r	   rd   r    bpartsmin_idxmin_rankipairrankpartr*   s   &&&         r   r'   r'   S   s$    "''AUA3ZE'
// 'h&e  U3BZr!;<GA"&&tAwa'89DX%5	 = """ hw5>EA+4F#F"GG%Z[P[P]J^^056d##F6M= (: 7s   D!D&c          
     ,    V ^8  d   QhRRRRRRRRRR	/# )
r   rM   r   rJ   rK   r   r    r!   r   r
   r   )r   s   "r   r   r   w   s;     B B
BB),B9CBBr   c           
     ^  a VR8  d   \        R4      h/ p\        R4       F  pWT\        V.4      &   K  	  \        P                  ! W 4       UUu. uF-  qfP                  R4       Uu. uF  p\        V.4      NK  	  upNK/  	  ppp\        V4      V8  Ed   \        P                  ! 4       oV F5  p	\        V	RR V	R,          4       F  p
SV
;;,          ^,          uu&   K  	  K7  	  \        SV3R lR7      pV^ ,          V^,          ,           p\        V4      pWV&   . pV F  p. p^ pV\        V4      ^,
          8  d[   We,          We^,           ,          3V8X  d   VP                  V4       V^,          pKO  VP                  We,          4       V^,          pKq  V\        V4      ^,
          8X  d   VP                  We,          4       VP                  V4       K  	  TpV'       g   EKd  \        RV^ ,           RV^,           24       \        R	V R
\        V4       R24       VR9   d8   \        R4       \        VR,           UUu. uF  qf F  qNK  	  K  	  upp4       M-VR8X  d'   \        R4       VR,           F  p\        V4       K  	  \        R4       EK  V# u upi u uppi u uppi )r   z;vocab_size must be at least 256, so we can encode all bytesr$   Nrg   c                   < SV ,          # r3   r   )xstatss   &r   <lambda>bpe_train.<locals>.<lambda>   s	    E!Hr   )keyz The current most common pair is z + zSo we made z our zth tokenz9Now the first fifty words in our training data look like::N2   Nrf   z:Now the first twenty words in our training data look like::N   N
   rk   ri   )
ValueErrorranger0   r   r%   r&   lencollectionsCounterro   maxappendrm   rl   )rM   rJ   r   r    ranksru   r+   rq   r)   piecerv   most_common_pairr   r   	new_wordsnew_wordr}   s   &&&&            @r   rO   rO   w   si    DVWWE4[eQCj  @E}}W?[ ?[t[[121s12?[ 
  
 e*z
!##%EE#2Jb	2dq  3  u*<=&q),<Q,??E
"k 	DHAc$i!m#GTa%[)-==OOK0FAOODG,FACIM!(X&   945Ea5H4IM]^_M`LabcK}E#e*XFG//QR E#J!QJDD5%D%J!QRh&RS!#JJD$K '$KLc 	3 V "Rs   J#&J;J#J)
J#c                    V ^8  d   QhRRRR/# )r   token_valuesrD   r   r   r   )r   s   "r   r   r      s      ; 4 r   c                   R
 Uu. uF	  pRV R2NK  	  ppV  Uu. uF  q3P                  RRR7      NK  	  pp^ pRpV Fo  pW%\        V4      ,          ,          pW8X  d(   W%^,           \        V4      ,          ,          pW8w  g   Q hTpV\        V4      ,          p\        W,           RR7       Kq  	  \        R	4       R# u upi u upi )   z[48;5;mr$   r>   r?   N )endz[0m)r         M   P   D      )rA   r   rm   )	r   ru   
backgroundr|   unicode_token_valuesrunning_length
last_colorr   rj   s	   &        r   rl   rl      s    /OP/O!L1%/OJP JVVAHHWYH?VNJ%C
O;< 2c*oEFE&&&
#e*$em$ & 
+! Q Ws
   B=Cc                    R p \        \        4      ;_uu_ 4       pVP                  4       pRRR4       \        P	                  XRV R7      p\        R4       VP                  R4      pVP                  V4      R8X  g   Q hVP                  V4      R8X  g   Q hVP                  V4      RR.8X  g   Q hV#   + '       g   i     L; i)	zN's|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+NiX  )rJ   r   zJThis is the sequence of merges performed in order to encode 'hello world':zhello worlds   hello worlds   hellos    world)
open__file__readr   rP   rm   r&   rA   r:   rF   )gpt2_patternfrM   encr*   s        r   train_simple_encodingr      s    ]  
h1vvx 
 !
&
&t\
&
RC	
VWZZ&F::f...F#~555""6*x.CCCCJ 
s   B33C	r[   )__doc__
__future__r   r   r   rT   r   r'   rO   rl   r   r   r   r   <module>r      s8    P "   D
 D
N!HBJ(r   