+
    QV-j‰  ã                   ó|   € ^ RI HtHtHtHtHtHt ^ RIHtH	t	H
t
HtHt ^ RIHt ^ RIHt ^RIHt  ! R R]4      tR# )	é    )ÚDictÚIteratorÚListÚOptionalÚTupleÚUnion)Ú
AddedTokenÚ	TokenizerÚdecodersÚpre_tokenizersÚtrainers)ÚBPE)ÚNFKC)ÚBaseTokenizerc                   ó¬   a a€ ] tR t^
t oRtRV3R lV 3R lllt]V3R lR l4       tR	^R.R
. R3V3R lR lltR	^R.R
. RR3V3R lR llt	Rt
VtV ;t# )ÚSentencePieceBPETokenizerzjSentencePiece BPE Tokenizer

Represents the BPE algorithm, with the pretokenization used by SentencePiece
Nú<unk>Tc                ó  <€ V ^8„  d   QhRS[ S[S[S[S[S[3,          3,          ,          RS[ S[S[S[S[S[S[3,          ,          3,          ,          RS[S[S[3,          RS[RS[RS[ S[	,          RS[ S[,          /# )é   ÚvocabÚmergesÚ	unk_tokenÚreplacementÚadd_prefix_spaceÚdropoutÚfuse_unk)
r   r   Ústrr   Úintr   r   r	   ÚboolÚfloat)ÚformatÚ__classdict__s   "€Ú}/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/tokenizers/implementations/sentencepiece_bpe.pyÚ__annotate__Ú&SentencePieceBPETokenizer.__annotate__   s•   ø€ ÷ 0ñ 0á™™c¡4©©S¨¥>Ð1Õ2Õ3ð0ñ ™™s¡D©©s±C¨x­Õ$9Ð9Õ:Õ;ð0ñ ™™j˜Õ)ð	0ñ
 ð0ñ ð0ñ ™%•ð0ñ ™4•.ñ0ó    c           
     ó¾  <€ Ve   Ve   \        \        WWcVR7      4      pM\        \        WcVR7      4      pVP                  \        V4      4      e   VP	                  \        V4      .4       \        4       Vn        V'       d   RMRp	\        P                  ! WIR7      Vn	        \        P                  ! WIR7      Vn        RRRVRVR	VR
V/p
\        SV `5  WŠ4       R # )N)r   r   r   ÚalwaysÚnever)r   Úprepend_schemeÚmodelÚSentencePieceBPEr   r   r   r   )r
   r   Útoken_to_idr   Úadd_special_tokensr   Ú
normalizerr   Ú	MetaspaceÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   Ú	tokenizerr*   Ú
parametersÚ	__class__s   &&&&&&&&   €r#   r4   Ú"SentencePieceBPETokenizer.__init__   sÌ   ø€ ð Ò Ò!3Ü!¤# e¸WÐdlÔ"mÓn‰Iä!¤#¨gÐU]Ô"^Ó_ˆIà× Ñ ¤ Y£Ó0Ò<Ø×(Ñ(¬#¨i«.Ð)9Ô:ä#›vˆ	Ôß%5™¸7ˆÜ"0×":Ò":À{Ô"rˆ	ÔÜ$×.Ò.¸;Ôfˆ	Ôð Ð'Ø˜Ø˜;ØÐ 0Ø�wð
ˆ
ô 	‰Ñ˜Ö/r&   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   Úvocab_filenameÚmerges_filename)r   )r!   r"   s   "€r#   r$   r%   2   s   ø€ ÷ Bñ B¡#ð B¹ñ Br&   c                óJ   € \         P                  ! W4      w  r4\        W43/ VB # )N)r   Ú	read_filer   )r;   r<   Úkwargsr   r   s   &&,  r#   Ú	from_fileÚ#SentencePieceBPETokenizer.from_file1   s"   € äŸš nÓF‰ˆÜ(¨ÑA¸&ÑAÐAr&   i0u  iè  c                ó    <€ V ^8„  d   QhRS[ S[S[S[,          3,          RS[RS[RS[S[ S[S[3,          ,          RS[RS[S[,          RS[/# )r   ÚfilesÚ
vocab_sizeÚmin_frequencyÚspecial_tokensÚlimit_alphabetÚinitial_alphabetÚshow_progress)r   r   r   r   r	   r   )r!   r"   s   "€r#   r$   r%   6   sp   ø€ ÷ 6ñ 6á‘S™$™s�)�^Õ$ð6ñ ð6ñ ð	6ñ
 ™U¡3©
 ?Õ3Õ4ð6ñ ð6ñ ™s�)ð6ñ ñ6r&   c           	     ó¨   € \         P                  ! VVVVVVR7      p\        V\        4      '       d   V.pV P                  P                  WR7       R# )z%Train the model using the given files©rD   rE   rF   rG   rH   rI   )ÚtrainerN)r   Ú
BpeTrainerÚ
isinstancer   Ú
_tokenizerÚtrain)	r5   rC   rD   rE   rF   rG   rH   rI   rL   s	   &&&&&&&& r#   rP   ÚSentencePieceBPETokenizer.train6   sQ   € ô ×%Ò%Ø!Ø'Ø)Ø)Ø-Ø'ô
ˆô �eœS×!Ò!Ø�GˆEØ�‰×Ñ˜eÐÖ5r&   c                óÖ   <€ V ^8„  d   QhRS[ S[S[,          S[S[S[,          ,          3,          RS[RS[RS[S[ S[S[3,          ,          RS[RS[S[,          RS[RS[S[,          /# )	r   ÚiteratorrD   rE   rF   rG   rH   rI   Úlength)r   r   r   r   r   r	   r   r   )r!   r"   s   "€r#   r$   r%   N   sˆ   ø€ ÷ 
ñ 
á™¡�¡x±¹µÕ'>Ð>Õ?ð
ñ ð
ñ ð	
ñ
 ™U¡3©
 ?Õ3Õ4ð
ñ ð
ñ ™s�)ð
ñ ð
ñ ™•ñ
r&   c	           	     óz   € \         P                  ! VVVVVVR7      p	V P                  P                  VV	VR7       R# )z(Train the model using the given iteratorrK   )rL   rT   N)r   rM   rO   Útrain_from_iterator)
r5   rS   rD   rE   rF   rG   rH   rI   rT   rL   s
   &&&&&&&&& r#   rV   Ú-SentencePieceBPETokenizer.train_from_iteratorN   sH   € ô ×%Ò%Ø!Ø'Ø)Ø)Ø-Ø'ô
ˆð 	�‰×+Ñ+ØØØð 	,ö 	
r&   © )NNr   u   â–�TNF)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r4   Ústaticmethodr@   rP   rV   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r8   r"   s   @@r#   r   r   
   ss   ù‡ € ñ÷
0õ 0ðB ÷Bó ðBð  ØØ8?°yØ"Ø&(Ø"÷6ò 6ð6  ØØ8?°yØ"Ø&(Ø"Ø $÷
÷ 
ò 
r&   r   N)Útypingr   r   r   r   r   r   Ú
tokenizersr	   r
   r   r   r   Útokenizers.modelsr   Útokenizers.normalizersr   Úbase_tokenizerr   r   rX   r&   r#   Ú<module>rg      s(   ðß ?× ?ç PÕ PÝ !Ý 'å )ô]
 ö ]
r&   