+
    QV-j�  ã                   óŒ   € ^ RI HtHtHtHtHt ^ RIHtHtH	t	H
t
 ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^RIHt  ! R R	]4      tR
# )é    )ÚDictÚIteratorÚListÚOptionalÚUnion)Ú
AddedTokenÚ	TokenizerÚdecodersÚtrainers)Ú	WordPiece)ÚBertNormalizer)ÚBertPreTokenizer)ÚBertProcessing)ÚBaseTokenizerc                   ó´   a a€ ] tR t^t oRtRV3R lV 3R lllt]V3R lR l4       tR	^R
. . RORR3V3R lR lltR	^R
. . RORRR3V3R lR llt	Rt
VtV ;t# )ÚBertWordPieceTokenizerzBert WordPiece TokenizerNTú##c                ó  <€ V ^8„  d   QhRS[ S[S[S[S[S[3,          3,          ,          RS[S[S[3,          RS[S[S[3,          RS[S[S[3,          RS[S[S[3,          RS[S[S[3,          RS[RS[R	S[ S[,          R
S[RS[/# )é   ÚvocabÚ	unk_tokenÚ	sep_tokenÚ	cls_tokenÚ	pad_tokenÚ
mask_tokenÚ
clean_textÚhandle_chinese_charsÚstrip_accentsÚ	lowercaseÚwordpieces_prefix)r   r   Ústrr   Úintr   Úbool)ÚformatÚ__classdict__s   "€Úz/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/tokenizers/implementations/bert_wordpiece.pyÚ__annotate__Ú#BertWordPieceTokenizer.__annotate__   sÎ   ø€ ÷ @0ñ @0á™™c¡4©©S¨¥>Ð1Õ2Õ3ð@0ñ ™™j˜Õ)ð@0ñ ™™j˜Õ)ð	@0ñ
 ™™j˜Õ)ð@0ñ ™™j˜Õ)ð@0ñ ™#™z˜/Õ*ð@0ñ ð@0ñ #ð@0ñ  ¡•~ð@0ñ ð@0ñ ñ@0ó    c                óx  <€ Ve!   \        \        V\        V4      R7      4      pM\        \        \        V4      R7      4      pVP                  \        V4      4      e   VP	                  \        V4      .4       VP                  \        V4      4      e   VP	                  \        V4      .4       VP                  \        V4      4      e   VP	                  \        V4      .4       VP                  \        V4      4      e   VP	                  \        V4      .4       VP                  \        V4      4      e   VP	                  \        V4      .4       \        VVV	V
R7      Vn        \        4       Vn        Vez   VP                  \        V4      4      pVf   \        R4      hVP                  \        V4      4      pVf   \        R4      h\        \        V4      V3\        V4      V34      Vn        \        P                  ! VR7      Vn        RRRVR	VR
VRVRVRVRVRV	RV
RV/p\        SV `=  WÏ4       R # )N)r   )r   r   r   r   z%sep_token not found in the vocabularyz%cls_token not found in the vocabulary)ÚprefixÚmodelÚBertWordPiecer   r   r   r   r   r   r   r   r   r    )r	   r   r!   Útoken_to_idÚadd_special_tokensr   Ú
normalizerr   Úpre_tokenizerÚ	TypeErrorr   Úpost_processorr
   ÚdecoderÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   r   r   r    Ú	tokenizerÚsep_token_idÚcls_token_idÚ
parametersÚ	__class__s   &&&&&&&&&&&&    €r&   r6   ÚBertWordPieceTokenizer.__init__   s	  ø€ ð ÒÜ!¤)¨E¼SÀ»^Ô"LÓM‰Iä!¤)´c¸)³nÔ"EÓFˆIð × Ñ ¤ Y£Ó0Ò<Ø×(Ñ(¬#¨i«.Ð)9Ô:Ø× Ñ ¤ Y£Ó0Ò<Ø×(Ñ(¬#¨i«.Ð)9Ô:Ø× Ñ ¤ Y£Ó0Ò<Ø×(Ñ(¬#¨i«.Ð)9Ô:Ø× Ñ ¤ Y£Ó0Ò<Ø×(Ñ(¬#¨i«.Ð)9Ô:Ø× Ñ ¤ Z£Ó1Ò=Ø×(Ñ(¬#¨j«/Ð):Ô;ä-Ø!Ø!5Ø'Øô	 
ˆ	Ôô #3Ó"4ˆ	ÔàÒØ$×0Ñ0´°Y³Ó@ˆLØÒ#ÜÐ GÓHÐHØ$×0Ñ0´°Y³Ó@ˆLØÒ#ÜÐ GÓHÐHä'5´s¸9³~À|Ð6TÔWZÐ[dÓWeÐgsÐVtÓ'uˆIÔ$Ü$×.Ò.Ð6GÔHˆ	Ôð �_Ø˜Ø˜Ø˜Ø˜Ø˜*Ø˜*Ø"Ð$8Ø˜]Ø˜ØÐ!2ð
ˆ
ô 	‰Ñ˜Ö/r)   c                ó    <€ V ^8„  d   QhRS[ /# )r   r   )r!   )r$   r%   s   "€r&   r'   r(   R   s   ø€ ÷ 7ñ 7™ñ 7r)   c                óF   € \         P                  ! V 4      p \        V 3/ VB # )N)r   Ú	read_filer   )r   Úkwargss   &,r&   Ú	from_fileÚ BertWordPieceTokenizer.from_fileQ   s"   € ä×#Ò# EÓ*ˆÜ% eÑ6¨vÑ6Ð6r)   i0u  iè  c                ó¦   <€ V ^8„  d   QhRS[ S[S[S[,          3,          RS[RS[RS[RS[S[,          RS[S[ S[S[3,          ,          RS[RS[/# )	r   ÚfilesÚ
vocab_sizeÚmin_frequencyÚlimit_alphabetÚinitial_alphabetÚspecial_tokensÚshow_progressr    )r   r!   r   r"   r   r#   )r$   r%   s   "€r&   r'   r(   V   sz   ø€ ÷ 6ñ 6á‘S™$™s�)�^Õ$ð6ñ ð6ñ ð	6ñ
 ð6ñ ™s�)ð6ñ ™U¡3©
 ?Õ3Õ4ð6ñ ð6ñ ñ6r)   c	           
     óª   € \         P                  ! VVVVVVVR7      p	\        V\        4      '       d   V.pV P                  P                  WR7       R# )z%Train the model using the given files©rF   rG   rH   rI   rJ   rK   Úcontinuing_subword_prefix)ÚtrainerN)r   ÚWordPieceTrainerÚ
isinstancer!   Ú
_tokenizerÚtrain)
r7   rE   rF   rG   rH   rI   rJ   rK   r    rO   s
   &&&&&&&&& r&   rS   ÚBertWordPieceTokenizer.trainV   sT   € ô& ×+Ò+Ø!Ø'Ø)Ø-Ø)Ø'Ø&7ô
ˆô �eœS×!Ò!Ø�GˆEØ�‰×Ñ˜eÐÖ5r)   c                óÜ   <€ V ^8„  d   QhRS[ S[S[,          S[S[S[,          ,          3,          RS[RS[RS[RS[S[,          RS[S[ S[S[3,          ,          RS[RS[R	S[S[,          /	# )
r   ÚiteratorrF   rG   rH   rI   rJ   rK   r    Úlength)r   r   r!   r"   r   r   r#   r   )r$   r%   s   "€r&   r'   r(   v   s’   ø€ ÷ !
ñ !
á™¡�¡x±¹µÕ'>Ð>Õ?ð!
ñ ð!
ñ ð	!
ñ
 ð!
ñ ™s�)ð!
ñ ™U¡3©
 ?Õ3Õ4ð!
ñ ð!
ñ ð!
ñ  ™•ñ!!
r)   c
           
     ó|   € \         P                  ! VVVVVVVR7      p
V P                  P                  VV
V	R7       R# )z(Train the model using the given iteratorrM   )rO   rW   N)r   rP   rR   Útrain_from_iterator)r7   rV   rF   rG   rH   rI   rJ   rK   r    rW   rO   s   &&&&&&&&&& r&   rY   Ú*BertWordPieceTokenizer.train_from_iteratorv   sK   € ô( ×+Ò+Ø!Ø'Ø)Ø-Ø)Ø'Ø&7ô
ˆð 	�‰×+Ñ+ØØØð 	,ö 	
r)   © )Nú[UNK]ú[SEP]ú[CLS]ú[PAD]ú[MASK]TTNTr   )r_   r\   r^   r]   r`   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r6   ÚstaticmethodrB   rS   rY   Ú__static_attributes__Ú__classdictcell__Ú__classcell__)r<   r%   s   @@r&   r   r      s|   ù‡ € Ù"÷@0õ @0ðD ÷7ó ð7ð  ØØ"Ø&(ò8
ð #Ø!%÷6ò 6ðF  ØØ"Ø&(ò8
ð #Ø!%Ø $÷!!
÷ !
ò !
r)   r   N)Útypingr   r   r   r   r   Ú
tokenizersr   r	   r
   r   Útokenizers.modelsr   Útokenizers.normalizersr   Útokenizers.pre_tokenizersr   Útokenizers.processorsr   Úbase_tokenizerr   r   r[   r)   r&   Ú<module>rq      s.   ðß 8Õ 8ç @Ó @Ý 'Ý 1Ý 6Ý 0å )ôK
˜]ö K
r)   