+
    QV-jj  ã                   óf   € ^ RI HtHtHtHtHt ^ RIHt ^ RIH	t	 ^ RI
Ht  ! R R4      tR R ltR	# )
é    )ÚRegexÚ	TokenizerÚdecodersÚpre_tokenizersÚ
processors)ÚBPE)Úbytes_to_unicode)ÚPreTrainedTokenizerFastc                   óX   a € ] tR t^t o RtR
R ltV 3R lR ltR tV 3R lR ltR	t	V t
R# )ÚMistralConverterz
A general tiktoken converter.
Nc                ó6   € Wn         W n        W0n        W@n        R # )N)ÚvocabÚpatternÚadd_prefix_spaceÚadditional_special_tokens)Úselfr   r   r   r   Úkwargss   &&&&&,Úr/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/transformers/integrations/mistral.pyÚ__init__ÚMistralConverter.__init__   s   € ð Œ
ØŒØ 0ÔØ)BÖ&ó    c                ó    <€ V ^8„  d   QhRS[ /# )é   r   ©Ústr)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚMistralConverter.__annotate__   s   ø€ ÷ ñ ±Sñ r   c                ó`  aa€ Vo\        4       oV3R  lp. p/ p\        SP                  4       4       F·  w  pw  rVWPP                  9  dœ   WAV! V4      &   \	        V4      ^8X  d   K4  . p\        ^\	        V4      4       F?  pVRV WXR r©V	S9   g   K  V
S9   g   K  Wš,           S9   g   K,  VP                  WšV34       KA  	  \        VV3R lRR7      pVP                  V4       K³  WAV&   K¹  	  \        VR RR7      pV Uu. uF   q²! V^ ,          4      V! V^,          4      3NK"  	  ppW3# u upi )c           	      óŽ   <€ R P                  V P                  R4       Uu. uF  pS\        V4      ,          NK  	  up4      # u upi )Ú zlatin-1)ÚjoinÚdecodeÚord)ÚbÚcharÚbyte_encoders   & €r   Útoken_bytes_to_stringÚOMistralConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_string   s;   ø€ Ø—7‘7ÀÇÁÈÔ@SÓTÑ@S¸˜L¬¨T«×3Ð3Ñ@SÑTÓUÐUùÒTs    ANc                 óD   <€ SV ^ ,          ,          SV ^,          ,          3# )r   © )ÚxÚ	bpe_rankss   &€r   Ú<lambda>ÚBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>-   s   ø€ °Y¸qÀ½tµ_ÀiÐPQÐRSÕPTÅoÑ4Vr   F)ÚkeyÚreversec                 ó   € V ^,          # )r   r,   )Úvals   &r   r/   r0   1   s   € °°A¶r   )	r	   Ú	enumerateÚitemsr   ÚlenÚrangeÚappendÚsortedÚextend)r   r   r)   ÚmergesÚidxÚtokenÚrankÚlocalÚindexÚpiece_lÚpiece_rr4   r.   r(   s   &&          @@r   Úextract_vocab_merges_from_modelÚ0MistralConverter.extract_vocab_merges_from_model   s1  ù€ Øˆ	Ü'Ó)ˆõ	Vð ˆØˆÜ"+¨I¯O©OÓ,=Ö">ÑˆC‘�%Ø×:Ñ:Ô:Ø69Ñ+¨EÓ2Ñ3Ü�u“: ”?ÙØ�Ü" 1¤c¨%£jÖ1�EØ',¨V¨e }°e¸F°m˜WØ )Ö+°¸9Ö0DÈ'ÕJ[Ð`iÖIiØŸ™ g¸Ð%=Ö>ñ 2ô ˜uÔ*VÐ`eÔf�Ø—‘˜eÖ$à"�e“ñ #?ô ˜Ñ$6ÀÔFˆÙ\bÓcÑ\bÐUXÐ(¨¨Q­Ó0Ñ2GÈÈAÍÓ2OÓPÑ\bˆÐcØˆ}Ðùò ds   Ä &D+c                óÆ   € V P                  V P                  4      w  r\        \        WR R7      4      p\	        VP
                  R4      '       d   RVP
                  n        V# )F)Úfuse_unkÚignore_mergesT)rD   r   r   r   ÚhasattrÚmodelrH   )r   Úvocab_scoresr<   Ú	tokenizers   &   r   rL   ÚMistralConverter.tokenizer5   sM   € Ø#×CÑCÀDÇJÁJÓOÑˆÜœc ,ÀÔGÓHˆ	Ü�9—?‘? O×4Ò4Ø,0ˆI�O‰OÔ)ØÐr   c                ó    <€ V ^8„  d   QhRS[ /# )r   Úreturn)r   )r   r   s   "€r   r   r   <   s   ø€ ÷ ñ ™9ñ r   c                ó–  € V P                  4       p\        P                  ! \        P                  ! \	        V P
                  4      R RR7      \        P                  ! V P                  RR7      .4      Vn        \        P                  ! 4       Vn
        VP                  V P                  4       \        P                  ! RR7      Vn        V# )ÚisolatedF)ÚbehaviorÚinvert)r   Ú	use_regex)Útrim_offsets)rL   r   ÚSequenceÚSplitr   r   Ú	ByteLevelr   Úpre_tokenizerr   ÚdecoderÚadd_special_tokensr   r   Úpost_processor)r   rL   s   & r   Ú	convertedÚMistralConverter.converted<   s™   € Ø—N‘NÓ$ˆ	Ü"0×"9Ò"9ä×$Ò$¤U¨4¯<©<Ó%8À:ÐV[Ô\Ü×(Ò(¸$×:OÑ:OÐ[`Ôaðó#
ˆ	Ôô %×.Ò.Ó0ˆ	ÔØ×$Ñ$ T×%CÑ%CÔDä#-×#7Ò#7ÀUÔ#Kˆ	Ô àÐr   )r   r   r   r   )Nzs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+FN)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   rD   rL   r]   Ú__static_attributes__Ú__classdictcell__)r   s   @r   r   r      s)   ø‡ € ñôC÷ð ò6÷ö r   r   c                ó$   € V ^8„  d   QhR\         /# )r   Útokenizer_filer   )r   s   "r   r   r   L   s   € ÷ -ñ -¬Sñ -r   c                ó\  € ^ RI Hp ^ RIHp VP	                  V 4      pVP
                  P                  P                  p\        VP
                  P                  P                  R R7      pV Uu. uF  qfR,          NK  	  pp\        V4       UUu/ uF  w  r†WhbK	  	  p	ppV	P                  V4       T	pVP
                  P                  P                  P                  p
\        \        WGV
R7      P!                  4       R7      pVP#                  RV/4       R	VP$                  P&                  R
VP(                  P&                  RVP*                  P&                  RVP,                  P&                  /pVP/                  4        F  w  rÞWç9   g   K  VP#                  WÞ/4       K!  	  V# u upi u uppi )z1Convert a "tekken" tokenizer to a fast Tokenizer.)ÚSpecialTokens)ÚMistralTokenizerc                 ó   € V R ,          # )r?   r,   )r-   s   &r   r/   Ú*convert_tekken_tokenizer.<locals>.<lambda>X   s   € ÐmnÐouÖmvr   )r1   Ú	token_str)r   r   r   )Útokenizer_objectr   Ú	bos_tokenÚ	eos_tokenÚ	pad_tokenÚ	unk_token)Ú%mistral_common.tokens.tokenizers.baseri   Ú(mistral_common.tokens.tokenizers.mistralrj   Ú	from_fileÚinstruct_tokenizerrL   Ú_tekken_token2id_nospecialr:   Ú_all_special_tokensr5   ÚupdateÚ_modelÚ_pat_strr
   r   r]   r[   ÚbosÚvalueÚeosÚpadÚunkr6   )rg   ri   rj   Úmistral_tokenizerr   Úsorted_tokensr>   Úall_specialr=   Úspecials_tokensr   rL   Ú
MAP_SPECALÚspecial_keyÚspecial_tokens   &              r   Úconvert_tekken_tokenizerrˆ   L   s†  € õ DÝIð )×2Ñ2°>ÓBÐð ×0Ñ0×:Ñ:×UÑU€EÜÐ,×?Ñ?×IÑI×]Ñ]ÑcvÔw€MÙ3@ÓA±=¨%˜×%Ð%±=€KÐAä4=¸kÔ4JÔKÑ4J¡j c�u’zÑ4J€OÑKà×Ñ˜5Ô!Ø€Eð  ×2Ñ2×<Ñ<×CÑC×LÑL€Gô (Ü)ØÈô
ç
‰)‹+ô€Ið × Ñ Ð"=¸{Ð!KÔLð 	�]×&Ñ&×,Ñ,Ø�]×&Ñ&×,Ñ,Ø�]×&Ñ&×,Ñ,Ø�]×&Ñ&×,Ñ,ð	€Jð '1×&6Ñ&6Ö&8Ñ"ˆØÖ'Ø×(Ñ(¨+Ð)EÖFñ '9ð ÐùòA BùãKs   Á.F#ÂF(N)Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr   Ú#transformers.convert_slow_tokenizerr	   Ú*transformers.tokenization_utils_tokenizersr
   r   rˆ   r,   r   r   Ú<module>r�      s&   ðß MÕ MÝ !å @Ý N÷Añ A÷H-r   