+
    QV-jI  ã                   óˆ   € ^ RI HtHtHtHtHtHt ^RIHtH	t	H
t
HtHt ^RIHt ^RIHtHtHtHt ^RIHt  ! R R]4      tR# )	é    )ÚDictÚIteratorÚListÚOptionalÚTupleÚUnion)Ú
AddedTokenÚ	TokenizerÚdecodersÚpre_tokenizersÚtrainers)ÚBPE)ÚBertNormalizerÚ	LowercaseÚSequenceÚunicode_normalizer_from_str)ÚBaseTokenizerc                   ó°   a a€ ] tR t^	t oRtRV3R lV 3R lllt]V3R lR	 l4       tR
^R.R. RR3V3R lR lltR
^R.R. RRR3V3R lR llt	Rt
VtV ;t# )ÚCharBPETokenizera´  Original BPE Tokenizer

Represents the BPE algorithm, as introduced by Rico Sennrich
(https://arxiv.org/abs/1508.07909)

The defaults settings corresponds to OpenAI GPT BPE tokenizers and differs from the original
Sennrich subword-nmt implementation by the following options that you can deactivate:
    - adding a normalizer to clean up the text (deactivate with `bert_normalizer=False`) by:
        * removing any control characters and replacing all whitespaces by the classic one.
        * handle chinese chars by putting spaces around them.
        * strip all accents.
    - spitting on punctuation in addition to whitespaces (deactivate it with
      `split_on_whitespace_only=True`)
Nú<unk>ú</w>Tc                ó  <€ V ^8„  d   QhRS[ S[S[S[S[S[3,          3,          ,          RS[ S[S[S[S[S[S[3,          ,          3,          ,          RS[S[S[3,          RS[RS[ S[,          RS[	RS[ S[,          RS[	R	S[	/	# )
é   ÚvocabÚmergesÚ	unk_tokenÚsuffixÚdropoutÚ	lowercaseÚunicode_normalizerÚbert_normalizerÚsplit_on_whitespace_only)
r   r   Ústrr   Úintr   r   r	   ÚfloatÚbool)ÚformatÚ__classdict__s   "€Úz/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/tokenizers/implementations/char_level_bpe.pyÚ__annotate__ÚCharBPETokenizer.__annotate__   s´   ø€ ÷ A0ñ A0á™™c¡4©©S¨¥>Ð1Õ2Õ3ðA0ñ ™™s¡D©©s±C¨x­Õ$9Ð9Õ:Õ;ðA0ñ ™™j˜Õ)ð	A0ñ
 ðA0ñ ™%•ðA0ñ ðA0ñ %¡S�MðA0ñ ðA0ñ #'ñA0ó    c
                ó"  <€ Ve(   Ve$   \        \        VVV\        V4      VR7      4      p
M\        \        \        V4      WTR7      4      p
V
P                  \        V4      4      e   V
P	                  \        V4      .4       . pV'       d   V\        V4      .,          pV'       d   V\        RR7      .,          pV'       d   V\        4       .,          p\        V4      ^ 8”  d0   \        V4      ^8”  d   \        V4      V
n
        MV^ ,          V
n
        V	'       d   \        P                  ! 4       V
n        M\        P                  ! 4       V
n        \        P                   ! VR7      V
n        RRRVR	VR
VRVRVRVRV	/p\$        SV `M  W¬4       R # )N)r   r   Úend_of_word_suffix)r   r   r.   F)r   )r   Úmodelr   r   r   r   r   r    r!   r"   )r
   r   r#   Útoken_to_idÚadd_special_tokensr   r   r   Úlenr   Ú
normalizerr   ÚWhitespaceSplitÚpre_tokenizerÚBertPreTokenizerr   Ú
BPEDecoderÚdecoderÚsuperÚ__init__)Úselfr   r   r   r   r   r   r    r!   r"   Ú	tokenizerÚnormalizersÚ
parametersÚ	__class__s   &&&&&&&&&&   €r)   r:   ÚCharBPETokenizer.__init__   si  ø€ ð Ò Ò!3Ü!ÜØØØ#Ü! )›nØ'-ôó‰Iô "¤#´°I³ÈÔ"kÓlˆIà× Ñ ¤ Y£Ó0Ò<Ø×(Ñ(¬#¨i«.Ð)9Ô:ð ˆçØÔ7Ð8JÓKÐLÕLˆKçØœN°UÔ;Ð<Õ<ˆKçØœI›K˜=Õ(ˆKô ˆ{Ó˜aÔÜ�;Ó !Ô#Ü'/°Ó'<�	Õ$à'2°1¥~�	Ô$ç#Ü&4×&DÒ&DÓ&FˆIÕ#ä&4×&EÒ&EÓ&GˆIÔ#ä$×/Ò/°vÔ>ˆ	Ôð �UØ˜Ø�fØ�wØ˜Ø Ð"4Ø˜Ø&Ð(@ð	
ˆ
ô 	‰Ñ˜Ö/r,   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   Úvocab_filenameÚmerges_filename)r#   )r'   r(   s   "€r)   r*   r+   ]   s   ø€ ÷ 9ñ 9¡#ð 9¹ñ 9r,   c                óJ   € \         P                  ! W4      w  r4\        W43/ VB # )N)r   Ú	read_filer   )rB   rC   Úkwargsr   r   s   &&,  r)   Ú	from_fileÚCharBPETokenizer.from_file\   s"   € äŸš nÓF‰ˆÜ Ñ8°Ñ8Ð8r,   i0u  iè  c                ó¶   <€ V ^8„  d   QhRS[ S[S[S[,          3,          RS[RS[RS[S[ S[S[3,          ,          RS[RS[S[,          RS[S[,          RS[/# )	r   ÚfilesÚ
vocab_sizeÚmin_frequencyÚspecial_tokensÚlimit_alphabetÚinitial_alphabetr   Úshow_progress)r   r#   r   r$   r	   r   r&   )r'   r(   s   "€r)   r*   r+   a   s~   ø€ ÷ 6ñ 6á‘S™$™s�)�^Õ$ð6ñ ð6ñ ð	6ñ
 ™U¡3©
 ?Õ3Õ4ð6ñ ð6ñ ™s�)ð6ñ ™•ð6ñ ñ6r,   c	           
     óª   € \         P                  ! VVVVVVVR7      p	\        V\        4      '       d   V.pV P                  P                  WR7       R# )z%Train the model using the given files©rK   rL   rM   rN   rO   r.   rP   )ÚtrainerN)r   Ú
BpeTrainerÚ
isinstancer#   Ú
_tokenizerÚtrain)
r;   rJ   rK   rL   rM   rN   rO   r   rP   rS   s
   &&&&&&&&& r)   rW   ÚCharBPETokenizer.traina   sT   € ô ×%Ò%Ø!Ø'Ø)Ø)Ø-Ø%Ø'ô
ˆô �eœS×!Ò!Ø�GˆEØ�‰×Ñ˜eÐÖ5r,   c                óì   <€ V ^8„  d   QhRS[ S[S[,          S[S[S[,          ,          3,          RS[RS[RS[S[ S[S[3,          ,          RS[RS[S[,          RS[S[,          RS[R	S[S[,          /	# )
r   ÚiteratorrK   rL   rM   rN   rO   r   rP   Úlength)r   r   r#   r$   r   r	   r   r&   )r'   r(   s   "€r)   r*   r+   {   s–   ø€ ÷ 
ñ 
á™¡�¡x±¹µÕ'>Ð>Õ?ð
ñ ð
ñ ð	
ñ
 ™U¡3©
 ?Õ3Õ4ð
ñ ð
ñ ™s�)ð
ñ ™•ð
ñ ð
ñ ™•ñ
r,   c
           
     ó|   € \         P                  ! VVVVVVVR7      p
V P                  P                  VV
V	R7       R# )z(Train the model using the given iteratorrR   )rS   r[   N)r   rT   rV   Útrain_from_iterator)r;   rZ   rK   rL   rM   rN   rO   r   rP   r[   rS   s   &&&&&&&&&& r)   r]   Ú$CharBPETokenizer.train_from_iterator{   sK   € ô ×%Ò%Ø!Ø'Ø)Ø)Ø-Ø%Ø'ô
ˆð 	�‰×+Ñ+ØØØð 	,ö 	
r,   © )	NNr   r   NFNTF)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r:   ÚstaticmethodrG   rW   r]   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r?   r(   s   @@r)   r   r   	   sy   ù‡ € ñ÷A0õ A0ðF ÷9ó ð9ð  ØØ8?°yØ"Ø&(Ø &Ø"÷6ò 6ð:  ØØ8?°yØ"Ø&(Ø &Ø"Ø $÷
÷ 
ò 
r,   r   N)Útypingr   r   r   r   r   r   Ú r	   r
   r   r   r   Úmodelsr   r=   r   r   r   r   Úbase_tokenizerr   r   r_   r,   r)   Ú<module>rm      s+   ðß ?× ?ç HÕ HÝ ß ZÓ ZÝ )ôM
�}ö M
r,   