+
    QV-jÉ  ã                   óF   € ^ RI Ht ^ RIHt ^ RIHt ^ RIHtHt R R lt	R# )é    )ÚPath)ÚAny)ÚTikTokenConverter)ÚTIKTOKEN_VOCAB_FILEÚTOKENIZER_FILEc                ó0   € V ^8„  d   QhR\         R\        /# )é   ÚencodingÚ
output_dir)r   Ústr)Úformats   "Ús/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/transformers/integrations/tiktoken.pyÚ__annotate__r      s   € ÷ -)ñ -)¤sð -)¼ñ -)ó    c                ó²  € \        V4      pVP                  RR7       VR,          \        ,          pV\        ,          pVP                  P                  RRR7       \        VP                  4       4      p\        VP                  4       4      p ^ RIHp ^ RI	H
p \        V \
        4      '       d	   V! V 4      p V! V P                  V4       \!        Y@P"                  T P$                  R7      P'                  4       p
T
P)                  T4       R
#   \         d>   p\        T4      p	RT	P                  4       9   d   \        R4      Th\        R	4      ThR
p?ii ; i)a—  
Converts given `tiktoken` encoding to `PretrainedTokenizerFast` and saves the configuration of converted tokenizer
on disk.

Args:
    encoding (`str` or `tiktoken.Encoding`):
        Tokenizer from `tiktoken` library. If `encoding` is `str`, the tokenizer will be loaded with
        `tiktoken.get_encoding(encoding)`.
    output_dir (`str`):
        Save path for converted tokenizer configuration file.
T)Úexist_okÚtiktoken)Úparentsr   )Úget_encoding)Údump_tiktoken_bpeÚblobfilezY`blobfile` is required to save a `tiktoken` file. Install it with `pip install blobfile`.zY`tiktoken` is required to save a `tiktoken` file. Install it with `pip install tiktoken`.N)Ú
vocab_fileÚpatternÚextra_special_tokens)r   Úmkdirr   r   Úparentr   Úabsoluter   r   Útiktoken.loadr   Ú
isinstanceÚ_mergeable_ranksÚImportErrorÚlowerÚ
ValueErrorr   Ú_pat_strÚ_special_tokensÚ	convertedÚsave)r
   r   Ú	save_fileÚtokenizer_fileÚsave_file_absoluteÚoutput_file_absoluter   r   ÚeÚ	error_msgÚ	tokenizers   &&         r   Úconvert_tiktoken_to_fastr/      s8  € ô �jÓ!€JØ×Ñ˜dÐÔ#à˜ZÕ'Ô*=Õ=€IØ¤.Õ0€Nð ×Ñ×Ñ 4°$ÐÔ7ä˜Y×/Ñ/Ó1Ó2ÐÜ˜~×6Ñ6Ó8Ó9ÐðÝ)Ý3ä�h¤×$Ò$Ù# HÓ-ˆHá˜(×3Ñ3Ð5GÔHô "Ø%×/@Ñ/@ÐW_×WoÑWoôç�iƒkð ð ‡N�NÐ'Ö(øô ô Ü˜“Fˆ	Ø˜Ÿ™Ó*Ô*ÜØkóàðô Øgó
àð	ûðús   Â=D ÄEÄ8EÅEN)
Úpathlibr   Útypingr   Ú#transformers.convert_slow_tokenizerr   Ú*transformers.tokenization_utils_tokenizersr   r   r/   © r   r   Ú<module>r5      s   ðÝ Ý å Aß Z÷-)r   