+
    QV-j°  ã                   óˆ   € ^ RI HtHtHtHtHtHt ^ RIHtH	t	H
t
HtHtHt ^ RIHt ^ RIHtHtHt ^RIHt  ! R R]4      tR# )	é    )ÚDictÚIteratorÚListÚOptionalÚTupleÚUnion)Ú
AddedTokenÚ	TokenizerÚdecodersÚpre_tokenizersÚ
processorsÚtrainers)ÚBPE)Ú	LowercaseÚSequenceÚunicode_normalizer_from_str)ÚBaseTokenizerc                   ó    a a€ ] tR t^
t oRtRV3R lV 3R lllt]V3R lR l4       tR^R. 3V3R	 lR
 lltR^R. R3V3R lR llt	Rt
VtV ;t# )ÚByteLevelBPETokenizerzbByteLevelBPETokenizer

Represents a Byte-level BPE as introduced by OpenAI with their GPT-2 model
Nc                ó  <€ V ^8„  d   QhRS[ S[S[S[S[S[3,          3,          ,          RS[ S[S[S[S[S[S[3,          ,          3,          ,          RS[RS[RS[ S[,          RS[ S[,          RS[ S[,          RS[ S[,          R	S[/	# )
é   ÚvocabÚmergesÚadd_prefix_spaceÚ	lowercaseÚdropoutÚunicode_normalizerÚcontinuing_subword_prefixÚend_of_word_suffixÚtrim_offsets)	r   r   Ústrr   Úintr   r   ÚboolÚfloat)ÚformatÚ__classdict__s   "€Úz/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/tokenizers/implementations/byte_level_bpe.pyÚ__annotate__Ú"ByteLevelBPETokenizer.__annotate__   s¨   ø€ ÷ 80ñ 80á™™c¡4©©S¨¥>Ð1Õ2Õ3ð80ñ ™™s¡D©©s±C¨x­Õ$9Ð9Õ:Õ;ð80ñ ð	80ñ
 ð80ñ ™%•ð80ñ %¡S�Mð80ñ $,©C¥=ð80ñ %¡S�Mð80ñ ñ80ó    c
                ól  <€ Ve3   Ve/   \        \        TTTT;'       g    RT;'       g    RR7      4      p
M\        \        4       4      p
. pV'       d   V\        V4      .,          pV'       d   V\        4       .,          p\	        V4      ^ 8”  d0   \	        V4      ^8”  d   \        V4      V
n        MV^ ,          V
n        \        P                  ! VR7      V
n	        \        P                  ! 4       V
n        \        P                  ! V	R7      V
n        RRRVRVR	VR
VRVRVRV	/p\        SV `=  W¬4       R # )NÚ )r   r   r   )r   )r    ÚmodelÚByteLevelBPEr   r   r   r   r   r   r    )r
   r   r   r   Úlenr   Ú
normalizerr   Ú	ByteLevelÚpre_tokenizerr   Údecoderr   Úpost_processorÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   r    Ú	tokenizerÚnormalizersÚ
parametersÚ	__class__s   &&&&&&&&&&   €r'   r6   ÚByteLevelBPETokenizer.__init__   s/  ø€ ð Ò Ò!3Ü!ÜØØØ#Ø.G×.MÐ.MÈ2Ø'9×'?Ð'?¸Rôó‰Iô "¤#£%Ó(ˆIð ˆçØÔ7Ð8JÓKÐLÕLˆKçØœI›K˜=Õ(ˆKô ˆ{Ó˜aÔÜ�;Ó !Ô#Ü'/°Ó'<�	Õ$à'2°1¥~�	Ô$ä"0×":Ò":ÐL\Ô"]ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ#-×#7Ò#7À\Ô#Rˆ	Ô ð �^ØÐ 0Ø˜Ø�wØ Ð"4Ø'Ð)BØ Ð"4Ø˜Lð	
ˆ
ô 	‰Ñ˜Ö/r*   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   Úvocab_filenameÚmerges_filename)r!   )r%   r&   s   "€r'   r(   r)   K   s   ø€ ÷ >ñ >¡#ð >¹ñ >r*   c                óJ   € \         P                  ! W4      w  r4\        W43/ VB # )N)r   Ú	read_filer   )r>   r?   Úkwargsr   r   s   &&,  r'   Ú	from_fileÚByteLevelBPETokenizer.from_fileJ   s"   € äŸš nÓF‰ˆÜ$ UÑ=°fÑ=Ð=r*   i0u  Tc                ó„   <€ V ^8„  d   QhRS[ S[S[S[,          3,          RS[RS[RS[RS[S[ S[S[3,          ,          /# )r   ÚfilesÚ
vocab_sizeÚmin_frequencyÚshow_progressÚspecial_tokens)r   r!   r   r"   r#   r	   )r%   r&   s   "€r'   r(   r)   O   sX   ø€ ÷ 6ñ 6á‘S™$™s�)�^Õ$ð6ñ ð6ñ ð	6ñ
 ð6ñ ™U¡3©
 ?Õ3Õ4ñ6r*   c                óÞ   € \         P                  ! VVVV\        P                  P	                  4       R7      p\        V\        4      '       d   V.pV P                  P                  WR7       R# )z%Train the model using the given files©rG   rH   rI   rJ   Úinitial_alphabet)ÚtrainerN)	r   Ú
BpeTrainerr   r1   ÚalphabetÚ
isinstancer!   Ú
_tokenizerÚtrain)r7   rF   rG   rH   rI   rJ   rN   s   &&&&&& r'   rS   ÚByteLevelBPETokenizer.trainO   s]   € ô ×%Ò%Ø!Ø'Ø'Ø)Ü+×5Ñ5×>Ñ>Ó@ô
ˆô �eœS×!Ò!Ø�GˆEØ�‰×Ñ˜eÐÖ5r*   c                óº   <€ V ^8„  d   QhRS[ S[S[,          S[S[S[,          ,          3,          RS[RS[RS[RS[S[ S[S[3,          ,          RS[S[,          /# )r   ÚiteratorrG   rH   rI   rJ   Úlength)r   r   r!   r"   r#   r   r	   r   )r%   r&   s   "€r'   r(   r)   d   sp   ø€ ÷ 
ñ 
á™¡�¡x±¹µÕ'>Ð>Õ?ð
ñ ð
ñ ð	
ñ
 ð
ñ ™U¡3©
 ?Õ3Õ4ð
ñ ™•ñ
r*   c                ó°   € \         P                  ! VVVV\        P                  P	                  4       R7      pV P
                  P                  VVVR7       R# )z(Train the model using the given iteratorrL   )rN   rW   N)r   rO   r   r1   rP   rR   Útrain_from_iterator)r7   rV   rG   rH   rI   rJ   rW   rN   s   &&&&&&& r'   rY   Ú)ByteLevelBPETokenizer.train_from_iteratord   sT   € ô ×%Ò%Ø!Ø'Ø'Ø)Ü+×5Ñ5×>Ñ>Ó@ô
ˆð 	�‰×+Ñ+ØØØð 	,ö 	
r*   © )	NNFFNNNNF)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r6   ÚstaticmethodrC   rS   rY   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r;   r&   s   @@r'   r   r   
   sa   ù‡ € ñ÷
80õ 80ðt ÷>ó ð>ð  ØØ"Ø79÷6ò 6ð0  ØØ"Ø79Ø $÷
÷ 
ò 
r*   r   N)Útypingr   r   r   r   r   r   Ú
tokenizersr	   r
   r   r   r   r   Útokenizers.modelsr   Útokenizers.normalizersr   r   r   Úbase_tokenizerr   r   r[   r*   r'   Ú<module>rj      s+   ðß ?× ?ç \× \Ý !ß SÑ Så )ôp
˜Mö p
r*   