+
    TV-j¡Q  ã                   óF  € ^ RI t ^ RIt^ RIt^ RIHt ^ RIHt ^ RIHtHtH	t	H
t
 ^ RIHtHt  ! R R4      t ! R R	]4      t ! R
 R]4      t ! R R]4      tR t ! R R4      t ! R R]4      t]P*                  ! R]R7       R tR tR tR tR tRR R lltR R ltR# )é    N)Úpartial)ÚJSONDecodeError)ÚAnyÚDictÚListÚOptional)ÚAutoTokenizerÚPreTrainedTokenizerFastc                   óJ   a € ] tR t^t o RtRtR tR tR t]	R 4       t
RtV tR# )	ÚStreamingDetokenizera  The streaming detokenizer interface so that we can detokenize one token at a time.

Example usage is as follows:

    detokenizer = ...

    # Reset the tokenizer state
    detokenizer.reset()

    for token in generate(...):
        detokenizer.add_token(token.item())

        # Contains the whole text so far. Some tokens may not be included
        # since it contains whole words usually.
        detokenizer.text

        # Contains the printable segment (usually a word) since the last
        # time it was accessed
        detokenizer.last_segment

        # Contains all the tokens added so far
        detokenizer.tokens

    # Make sure that we detokenize any remaining tokens
    detokenizer.finalize()

    # Now detokenizer.text should match tokenizer.decode(detokenizer.tokens)
c                ó   € \        4       h©N©ÚNotImplementedError©Úselfs   &Úg/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/tokenizer_utils.pyÚresetÚStreamingDetokenizer.reset+   ó   € Ü!Ó#Ð#ó    c                ó   € \        4       hr   r   ©r   Útokens   &&r   Ú	add_tokenÚStreamingDetokenizer.add_token.   r   r   c                ó   € \        4       hr   r   r   s   &r   ÚfinalizeÚStreamingDetokenizer.finalize1   r   r   c                óZ   € V P                   pWP                  R p\        V4      V n        V# )zTReturn the last segment of readable text since last time this property was accessed.N)ÚtextÚoffsetÚlen)r   r!   Úsegments   &  r   Úlast_segmentÚ!StreamingDetokenizer.last_segment4   s+   € ð �y‰yˆØ—{‘{�}Ð%ˆÜ˜$“iˆŒØˆr   )r"   N)r!   Útokensr"   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú	__slots__r   r   r   Úpropertyr%   Ú__static_attributes__Ú__classdictcell__©Ú__classdict__s   @r   r   r      s4   ø‡ € ñð: -€Iò$ò$ò$ð ñó ör   r   c                   óL   a € ] tR t^=t o RtR tR tR tR t]	R 4       t
RtV tR# )	ÚNaiveStreamingDetokenizerzûNaiveStreamingDetokenizer relies on the underlying tokenizer
implementation and should work with every tokenizer.

Its complexity is O(T^2) where T is the longest line since it will
repeatedly detokenize the same tokens until a new line is generated.
c                ój   € Wn         V P                   P                  ^ .4       V P                  4        R# )r   N)Ú
_tokenizerÚdecoder   )r   Ú	tokenizers   &&r   Ú__init__Ú"NaiveStreamingDetokenizer.__init__E   s$   € Ø#ŒØ�‰×Ñ ˜sÔ#Ø�
‰
Žr   c                óL   € ^ V n         . V n        RV n        . V n        RV n        R# ©r   Ú N)r"   r'   Ú_textÚ_current_tokensÚ_current_textr   s   &r   r   ÚNaiveStreamingDetokenizer.resetJ   s'   € ØˆŒØˆŒØˆŒ
Ø!ˆÔØˆÖr   c                ór   € V P                   P                  V4       V P                  P                  V4       R # r   )r?   Úappendr'   r   s   &&r   r   Ú#NaiveStreamingDetokenizer.add_tokenQ   s(   € Ø×Ñ×#Ñ# EÔ*Ø�‰×Ñ˜5Ö!r   c                óš   € V ;P                   V P                  P                  V P                  4      ,          un         . V n        R V n        R# )r=   N)r>   r6   r7   r?   r@   r   s   &r   r   Ú"NaiveStreamingDetokenizer.finalizeU   s5   € Ø�
Š
�d—o‘o×,Ñ,¨T×-AÑ-AÓBÕB�
Ø!ˆÔØˆÖr   c                ó–  € V P                   '       d®   V P                  P                  V P                   4      V n        V P                  P	                  R 4      '       gO   V P                  P
                  '       dG   \        V P                  4      ^ 8”  d-   V P                  R,          R8X  d   V P                  RR V n        V P                  '       d]   V P                  R,          R8X  dE   V ;P                  V P                  ,          un        V P                   P                  4        RV n        V P                  V P                  ,           # )õ   ï¿½Ú NÚ
r=   éÿÿÿÿ)	r?   r6   r7   r@   ÚendswithÚclean_up_tokenization_spacesr#   r>   Úclearr   s   &r   r!   ÚNaiveStreamingDetokenizer.textZ   sî   € à××ÐØ!%§¡×!7Ñ!7¸×8LÑ8LÓ!MˆDÔØ×!Ñ!×*Ñ*¨8×4Ò4Ø—‘×<×<Ð<Ü˜×*Ñ*Ó+¨aÔ/Ø×&Ñ& rÕ*¨cÔ1à%)×%7Ñ%7¸¸Ð%<�Ô"Ø××Ð $×"4Ñ"4°RÕ"8¸DÔ"@Ø�JŠJ˜$×,Ñ,Õ,�JØ× Ñ ×&Ñ&Ô(Ø!#ˆDÔØ�z‰z˜D×.Ñ.Õ.Ð.r   )r@   r?   r>   r6   r"   r'   N)r(   r)   r*   r+   r,   r9   r   r   r   r.   r!   r/   r0   r1   s   @r   r4   r4   =   s2   ø‡ € ñòò
 ò"ò ð
 ñ/ó ö/r   r4   c                   óJ   a € ] tR t^kt o RtR	R ltR tR
R ltR tR t	Rt
V tR# )ÚSPMStreamingDetokenizerz¡A streaming detokenizer for SPM models.

It adds tokens to the text if the next token starts with the special SPM
underscore which results in linear complexity.
c                óÐ  € W n         R P                  4       V n        R.\        VP                  P                  4       4      ^,           ,          V n        VP                  P                  4        Fe  w  r4VP                  R4      '       d-   \        \        VR,          ^4      .4      V P                  V&   KH  VP                  4       V P                  V&   Kg  	  V P                  4        R# )õ   â–�r=   z<0x:é   é   NN)Ú
trim_spaceÚencodeÚ_sepÚmaxÚvocabÚvaluesÚtokenmapÚitemsÚ
startswithÚbytesÚintr   )r   r8   rV   ÚvalueÚtokenids   &&&  r   r9   Ú SPMStreamingDetokenizer.__init__r   s¦   € Ø$ŒØ—O‘OÓ%ˆŒ	ð ˜¤ I§O¡O×$:Ñ$:Ó$<Ó =ÀÕ AÕBˆŒØ'Ÿo™o×3Ñ3Ö5‰NˆEØ×Ñ ×&Ò&ä).´°E¸#µJÀÓ0CÐ/DÓ)E�—‘˜gÓ&à).¯©«�—‘˜gÓ&ñ 6ð 	�
‰
Žr   c                ó>   € ^ V n         RV n        RV n        . V n        R# )r   r   r=   N©r"   Ú
_unflushedr!   r'   r   s   &r   r   ÚSPMStreamingDetokenizer.reset�   s   € ØˆŒØˆŒØˆŒ	ØˆŽr   c                óz  € V P                   P                  V P                  R 4      P                  RR4      pV'       g   VP	                  R4      '       d   R# V P
                  '       g2   V P                  '       d    V'       d   V^ ,          R8X  d
   VR,          pV ;P
                  V,          un        RV n         R# )ó    úutf-8ÚreplacerH   NrI   ºé   NNr   )rf   rk   rX   r7   rL   r!   rV   )r   Úforcer!   s   && r   Ú
_try_flushÚ"SPMStreamingDetokenizer._try_flush‡   sx   € Ø�‰×&Ñ& t§y¡y°$Ó7×>Ñ>¸wÈ	ÓRˆß˜Ÿ™ x×0Ò0ÙØ�y�yˆy˜TŸ_Ÿ_˜_·¸$¸q½'ÀS¼.Ø˜•8ˆDØ�	Š	�TÕ�	ØˆŽr   c                ó´   € V P                   P                  V4       V P                  V,          pV ;P                  V,          un        V P	                  4        R # r   )r'   rC   r\   rf   ro   )r   r   Úvs   && r   r   Ú!SPMStreamingDetokenizer.add_token�   s:   € Ø�‰×Ñ˜5Ô!Ø�M‰M˜%Õ ˆØ�Š˜1Õ�Ø�‰Ör   c                ó8   € V P                  R R7       RV n        R# )T)rn   r   N)ro   rf   r   s   &r   r   Ú SPMStreamingDetokenizer.finalize–   s   € Ø�‰˜dˆÔ#ØˆŽr   )rX   rf   r"   r!   r\   r'   rV   N)T)F)r(   r)   r*   r+   r,   r9   r   ro   r   r   r/   r0   r1   s   @r   rQ   rQ   k   s(   ø‡ € ñôòôò÷ð r   rQ   c                   ó`   a € ] tR t^›t o RtRtRtR tR tR t	R t
R tR t]R	 4       tR
tV tR# )ÚBPEStreamingDetokenizerz—A streaming detokenizer for OpenAI style BPE models.

It adds tokens to the text if the next token starts with a space similar to
the SPM detokenizer.
Nc                ó  € VP                   V n        R .\        VP                  4      ,          V n        VP                  P                  4        F  w  r#W P                  V&   K  	  V P                  4        V P                  4        R # r   )rM   Úclean_spacesr#   rZ   r\   r]   r   Úmake_byte_decoder)r   r8   ra   rb   s   &&  r   r9   Ú BPEStreamingDetokenizer.__init__¥   sg   € Ø%×BÑBˆÔð ˜¤ Y§_¡_Ó!5Õ5ˆŒØ'Ÿo™o×3Ñ3Ö5‰NˆEØ%*�M‰M˜'Ó"ñ 6ð 	�
‰
Œð 	×ÑÖ r   c                ó>   € ^ V n         RV n        RV n        . V n        R# r<   re   r   s   &r   r   ÚBPEStreamingDetokenizer.reset³   s   € ØˆŒØˆŒØˆŒ	ØˆŽr   c                óð   € \        4       pV FU  pV P                  P                  VR 4      pV'       d   VP                  V4       K:  VP	                  \        VR4      4       KW  	  VP                  RR4      # )Frj   rk   )Ú	bytearrayÚ_byte_decoderÚgetrC   Úextendr_   r7   )r   ÚseqÚbarrÚcÚress   &&   r   Ú_decode_bytesÚ%BPEStreamingDetokenizer._decode_bytes¹   s`   € Ü‹{ˆÛˆAØ×$Ñ$×(Ñ(¨¨EÓ2ˆCßØ—‘˜CÖ à—‘œE ! WÓ-Ö.ñ ð �{‰{˜7 IÓ.Ð.r   c                ó  € \        V4      ^ 8X  d   V# V^ ,          R8w  d   V# V P                  '       g
   VR,          # V P                  '       d2   VR,          P                  V P                  4      '       d
   VR,          # V# )r   rI   rl   )r#   r!   ry   r^   Ú_space_matches©r   Úcurrent_texts   &&r   Ú_maybe_trim_spaceÚ)BPEStreamingDetokenizer._maybe_trim_spaceÃ   ss   € Üˆ|Ó Ô!ØÐØ˜!�_ Ô#ØÐØ——�Ø Õ#Ð#Ø××Ð <°Õ#3×#>Ñ#>¸t×?RÑ?R×#SÒ#SØ Õ#Ð#ØÐr   c                ó  € V P                   P                  V4       V\        V P                  4      8  d   V P                  V,          MR pV ;P                  V,          un        V P                  V P                  4      pVP                  R4      '       gk   \        V4      ^8X  d(   V P                  P                  V^ ,          4      ^ 8X  g2   V ;P                  V P                  V4      ,          un	        RV n        R# R# R# )Ú!rH   r=   N)r'   rC   r#   r\   rf   r‡   rL   r€   r�   r!   r�   )r   r   rr   r!   s   &&  r   r   Ú!BPEStreamingDetokenizer.add_tokenÎ   s¸   € Ø�‰×Ñ˜5Ô!Ø$)¬C°·±Ó,>Ô$>ˆD�M‰M˜%Ö ÀCˆØ�Š˜1Õ�Ø×!Ñ! $§/¡/Ó2ˆð �}‰}˜X×&Ò&Ü�‹F�aŒK˜D×.Ñ.×2Ñ2°1°Qµ4Ó8¸BÔ>à�IŠI˜×/Ñ/°Ó5Õ5�IØ ˆDŽOñ ?ñ 'r   c                óÄ   a € \        V 3R  lS P                   4       4      P                  RR4      pS ;P                  S P	                  V4      ,          un        RS n        R# )c              3   óJ   <"  € T F  pSP                   V,          x € K  	  R # 5ir   )r€   )Ú.0r…   r   s   & €r   Ú	<genexpr>Ú3BPEStreamingDetokenizer.finalize.<locals>.<genexpr>Ý   s   øé € Ð PÁ¸1 ×!3Ñ!3°A×!6Ò!6Ãùs   ƒ #rj   rk   r=   N)r   rf   r7   r!   r�   r‹   s   f r   r   Ú BPEStreamingDetokenizer.finalizeÜ   sJ   ø€ Ü Ô PÀÇÂÓ PÓP×WÑWØØó
ˆð 	�	Š	�T×+Ñ+¨LÓ9Õ9�	ØˆŽr   c           	     óì  € V P                   e   R# / p^ \        R4      \        R4      ^,           \        R4      \        R4      ^,           \        R4      \        R4      ^,           .p^ p\        \        W"R,          4      4       Ff  w  pw  rVV^,          ^ 8X  d2   \	        WV4       F   pWq\        R	V,           4      &   V^,          pK"  	  KG  \	        WV4       F  pWq\        V4      &   K  	  Kh  	  Wn         R# )
zQSee https://github.com/openai/gpt-2/blob/master/src/encoder.py for the rationale.Nr�   Ú~ô   Â¡ô   Â¬ô   Â®ô   Ã¿rl   é   )r€   ÚordÚ	enumerateÚzipÚrangeÚchr)ÚclsÚchar_to_bytesÚlimitsÚnÚiÚstartÚstopÚbs   &       r   rz   Ú)BPEStreamingDetokenizer.make_byte_decoderä   s×   € ð ×ÑÒ(ÙàˆàÜ�‹HÜ�‹H�q�LÜ�‹IÜ�‹I˜�MÜ�‹IÜ�‹I˜�Mð
ˆð ˆÜ )¬#¨f¸RµjÓ*AÖ BÑˆA‰}�Ø�1�u˜ŒzÜ˜uÖ+�AØ34¤# d¨Q¥h£-Ñ0Ø˜•F’Aó ,ô ˜uÖ+�AØ,-¤# a£&Ó)ó ,ñ !Cð *Ör   )rf   ry   r"   r!   r\   r'   )	Ú.Ú?r�   Ú,zn'tz'mz'sz'vez're)r(   r)   r*   r+   r,   r€   rŠ   r9   r   r‡   r�   r   r   Úclassmethodrz   r/   r0   r1   s   @r   rw   rw   ›   sH   ø‡ € ñð €MØJ€Nò!òò/ò	ò!òð ñ*ó ö*r   rw   c           
      ó$  € V P                  4       pRR.pV F)  w  r4W19   g   K  WA9   g   K  VVW,          3W,          33u # 	  RV9   dF   RV9   d?   RpRpVV\        V P                  VRR7      4      \        V P                  VRR7      4      3# R# )	ú<think>z
<|channel>z
<channel|>z<|channel>thoughtF©Úadd_special_tokens)r²   z</think>)z<longcat_think>z</longcat_think>)NNNN)Ú	get_vocabÚtuplerW   )r8   rZ   ÚTHINK_TOKENSÚthink_startÚ	think_ends   &    r   Ú_infer_thinkingrº      s¼   € Ø×ÑÓ!€EàØ/ð€Ló #/ÑˆØÖ IÖ$6àØØÕ#Ð%ØÕ!Ð#ð	ò ñ #/ð �uÔ °Ô!6Ø)ˆØ ˆ	àØÜ�)×"Ñ" ;À5Ð"ÓIÓJÜ�)×"Ñ" 9ÀÐ"ÓGÓHð	
ð 	
ð $Ð#r   c                   ó„  a a€ ] tR tRt oRt]RRRRR3R ltRR/R ltV3R lR	 ltR R
 lt	R!R lt
R!R ltR!R ltR!R lt]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       t]R 4       tR tV 3R ltRtVt V ;t!# )"ÚTokenizerWrapperi  zžA wrapper that combines an HF tokenizer and a detokenizer.

Accessing any attribute other than the ``detokenizer`` is forwarded to the
huggingface tokenizer.
Nc                óÂ  € Wn         W n        Ve   \        V4      MVP                  0V n        \        V4      w  V n        V n        V n        V n	        W@n
        VP                  R J;'       g    VR JV n        Wpn        WPn        W`n        R V n        R V n        VeE   \%        VP'                  VRR7      4      V n        \%        VP'                  VRR7      4      V n        R # R # )NFr³   )r6   Ú_detokenizer_classÚsetÚeos_token_idÚ_eos_token_idsrº   Ú_think_startÚ
_think_endÚ_think_start_tokensÚ_think_end_tokensÚ_chat_templateÚchat_templateÚhas_chat_templateÚ_tool_parserÚ_tool_call_startÚ_tool_call_endÚ_tool_call_start_tokensÚ_tool_call_end_tokensr¶   rW   )r   r8   Údetokenizer_classÚeos_token_idsrÇ   Útool_call_startÚtool_call_endÚtool_parsers   &&&&&&&&r   r9   ÚTokenizerWrapper.__init__&  s÷   € ð $ŒØ"3Ôð Ò(ô �Ôà×(Ñ(Ð)ð 	Ôô ˜IÓ&ñ	
ØÔØŒOØÔ$ØÔ"ð ,Ôà×#Ñ#¨4Ð/×LÐL°=ÈÐ3Lð 	Ôð (ÔØ /ÔØ+ÔØ'+ˆÔ$Ø%)ˆÔ"ØÒ&Ü+0Ø× Ñ  ÀUÐ ÓKó,ˆDÔ(ô */Ø× Ñ  À5Ð ÓIó*ˆDÖ&ñ	 'r   ÚtokenizeTc               ó  € R V9  d   V P                   VR &   V P                  e:   V P                  ! V/ VB pV'       d   V P                  P                  VRR7      pV# RVR&   V P                  P                  ! VRV/VB # )Úenable_thinkingFr³   Úreturn_dictrÔ   )Úhas_thinkingrÆ   r6   rW   Úapply_chat_template)r   rÔ   ÚargsÚkwargsÚouts   &$*, r   rÙ   Ú$TokenizerWrapper.apply_chat_templateO  s†   € Ø FÔ*Ø(,×(9Ñ(9ˆFÐ$Ñ%à×ÑÒ*Ø×%Ò% tÐ6¨vÑ6ˆCßØ—o‘o×,Ñ,¨SÀUÐ,ÓK�ØˆJà %ˆˆ}ÑØ�‰×2Ò2°DÐVÀ8ÐVÈvÑVÐVr   c                ó    <€ V ^8„  d   QhRS[ /# )é   r   )Ústr)Úformatr2   s   "€r   Ú__annotate__ÚTokenizerWrapper.__annotate__\  s   ø€ ÷ 
*ñ 
*¡3ñ 
*r   c                óÖ   € R p \        V4      pVf   \        RV R24      hV P                  P                  V4       R #   \         d    T P                  P                  T4      p LXi ; i)NÚ'z#' is not a token for this tokenizer)r`   Ú
ValueErrorr6   Úconvert_tokens_to_idsrÁ   Úadd)r   r   Útoken_ids   && r   Úadd_eos_tokenÚTokenizerWrapper.add_eos_token\  sn   € Øˆð	DÜ˜5“zˆHð ÒÜ˜q  Ð'JÐKÓLÐLà×Ñ×Ñ Ö)øô ô 	DØ—‘×<Ñ<¸UÓCŠHð	Dús   „? ¿&A(Á'A(c           
     ó  aaa€ T;'       g    ^ pT;'       g    \        S4      pV'       d%   \        V\        S4      ,
          V^,
          R4      M!\        W4\        S4      ,
          ^,           4      pV Fˆ  oSS,          S^ ,          8X  g   K  \        ;QJ d4    VVV3R l\        ^\        S4      4       4       F  '       d   K   RM)	  RM%! VVV3R l\        ^\        S4      4       4       4      '       g   K†  Su # 	  R# )r   c              3   óX   <"  € T F  pSSV,           ,          SV,          8H  x € K!  	  R # 5ir   © )r”   Újr¨   Úsequencer'   s   & €€€r   r•   Ú)TokenizerWrapper._find.<locals>.<genexpr>r  s$   øé € ÐUÑ=T¸�v˜a !�e•}¨°­Ö3Ó=Tùs   ƒ'*FTrK   )r#   r¢   Úall)r   r'   rð   r©   ÚendÚreverseÚ
outer_loopr¨   s   &ff&&& @r   Ú_findÚTokenizerWrapper._findh  s¹   ú€ Ø—
�
˜ˆØ× Ð ”S˜“[ˆ÷ ô �#œ˜H›Õ% u¨q¥y°"Ô5ä�u¤C¨£MÕ1°AÕ5Ó6ð 	ó
 ˆAØ�a�y˜H Q�KÖ'ß“3ÖU¼UÀ1ÄcÈ(ÃmÔ=TÓU—3—3’3ÖU¼UÀ1ÄcÈ(ÃmÔ=TÓU×UÔUØ’Hñ ð ˆ	r   c                ó<   € V P                  WP                  W#R 7      # ©)r©   ró   ©rö   rÄ   ©r   r'   r©   ró   s   &&&&r   Úfind_think_startÚ!TokenizerWrapper.find_think_startv  s   € Ø�z‰z˜&×":Ñ":À%ˆzÓQÐQr   c                ó>   € V P                  WP                  W#R R7      # ©T)r©   ró   rô   rú   rû   s   &&&&r   Úrfind_think_startÚ"TokenizerWrapper.rfind_think_starty  s%   € Ø�z‰zØ×,Ñ,°EÈDð ó 
ð 	
r   c                ó<   € V P                  WP                  W#R 7      # rù   ©rö   rÅ   rû   s   &&&&r   Úfind_think_endÚTokenizerWrapper.find_think_end~  s   € Ø�z‰z˜&×"8Ñ"8ÀˆzÓOÐOr   c                ó>   € V P                  WP                  W#R R7      # rÿ   r  rû   s   &&&&r   Úrfind_think_endÚ TokenizerWrapper.rfind_think_end�  s%   € Ø�z‰zØ×*Ñ*°%È$ð ó 
ð 	
r   c                ó   € V P                   R J# r   ©rÂ   r   s   &r   rØ   ÚTokenizerWrapper.has_thinking†  s   € à× Ñ ¨Ð,Ð,r   c                ó   € V P                   # r   r
  r   s   &r   r¸   ÚTokenizerWrapper.think_startŠ  ó   € à× Ñ Ð r   c                ó’   € V P                   f   R # \        V P                   4      ^8”  d   \        R4      hV P                   ^ ,          # )Nz0The start thinking sequence is more than 1 token)rÄ   r#   ræ   r   s   &r   Úthink_start_idÚTokenizerWrapper.think_start_idŽ  sA   € à×#Ñ#Ò+ÙÜˆt×'Ñ'Ó(¨1Ô,ÜÐOÓPÐPØ×'Ñ'¨Õ*Ð*r   c                ó   € V P                   # r   )rÄ   r   s   &r   Úthink_start_tokensÚ#TokenizerWrapper.think_start_tokens–  s   € à×'Ñ'Ð'r   c                ó   € V P                   # r   )rÃ   r   s   &r   r¹   ÚTokenizerWrapper.think_endš  s   € à�‰Ðr   c                ó’   € V P                   f   R # \        V P                   4      ^8”  d   \        R4      hV P                   ^ ,          # )Nz.The end thinking sequence is more than 1 token)rÅ   r#   ræ   r   s   &r   Úthink_end_idÚTokenizerWrapper.think_end_idž  sA   € à×!Ñ!Ò)ÙÜˆt×%Ñ%Ó&¨Ô*ÜÐMÓNÐNØ×%Ñ% aÕ(Ð(r   c                ó   € V P                   # r   )rÅ   r   s   &r   Úthink_end_tokensÚ!TokenizerWrapper.think_end_tokens¦  s   € à×%Ñ%Ð%r   c                ó   € V P                   R J# r   ©rÊ   r   s   &r   Úhas_tool_callingÚ!TokenizerWrapper.has_tool_callingª  s   € à×$Ñ$¨DÐ0Ð0r   c                ó   € V P                   # r   r  r   s   &r   rÐ   Ú TokenizerWrapper.tool_call_start®  s   € à×$Ñ$Ð$r   c                ó   € V P                   # r   )rÌ   r   s   &r   Útool_call_start_tokensÚ'TokenizerWrapper.tool_call_start_tokens²  s   € à×+Ñ+Ð+r   c                ó   € V P                   # r   )rË   r   s   &r   rÑ   ÚTokenizerWrapper.tool_call_end¶  s   € à×"Ñ"Ð"r   c                ó   € V P                   # r   )rÍ   r   s   &r   Útool_call_end_tokensÚ%TokenizerWrapper.tool_call_end_tokensº  s   € à×)Ñ)Ð)r   c                ó   € V P                   # r   )rÉ   r   s   &r   rÒ   ÚTokenizerWrapper.tool_parser¾  r  r   c                ó$   € V P                  V 4      # )z'
Get a stateful streaming detokenizer.
)r¾   r   s   &r   ÚdetokenizerÚTokenizerWrapper.detokenizerÂ  s   € ð
 ×&Ñ& tÓ,Ð,r   c                óÊ   € VR 8X  d   V P                   # VR8X  d   V P                  # VP                  R4      '       d   V P                  V4      # \	        V P
                  V4      # )r.  rÏ   Ú_)Ú_detokenizerrÁ   r^   Ú__getattribute__Úgetattrr6   )r   Úattrs   &&r   Ú__getattr__ÚTokenizerWrapper.__getattr__É  s[   € Ø�=Ô Ø×$Ñ$Ð$Ø�_Ô$Ø×&Ñ&Ð&Ø�_‰_˜S×!Ò!Ø×(Ñ(¨Ó.Ð.ä˜4Ÿ?™?¨DÓ1Ð1r   c                ó  <€ VR9   d<   VR 8X  d   \        R4      hVR8X  d!   Ve   \        V4      M	\        4       V n        R# R# VP                  R4      '       d   \        SV `  W4       R# \        V P                  W4       R# )r.  rÏ   zCannot set the detokenizer.Nr1  >   r.  rÏ   )ÚAttributeErrorr¿   rÁ   r^   ÚsuperÚ__setattr__Úsetattrr6   )r   r5  ra   Ú	__class__s   &&&€r   r;  ÚTokenizerWrapper.__setattr__Ó  sl   ø€ ØÐ3Ô3Ø�}Ô$Ü$Ð%BÓCÐCØ˜Ô(Ø49Ò4E¤c¨%¤jÌ3Ë5�Ö#ñ )à�_‰_˜S×!Ò!Ü‰GÑ Ö,ä�D—O‘O TÖ1r   )rÆ   r¾   rÁ   rÃ   rÅ   rÂ   rÄ   r6   rË   rÍ   rÊ   rÌ   rÉ   rÈ   )NNF©NN)"r(   r)   r*   r+   r,   r4   r9   rÙ   rê   rö   rü   r   r  r  r.   rØ   r¸   r  r  r¹   r  r  r  rÐ   r$  rÑ   r)  rÒ   r.  r6  r;  r/   r0   Ú__classcell__©r=  r2   s   @@r   r¼   r¼     s~  ù‡ € ñð 4ØØØØØô'ðRW°$ô W÷
*ð 
*ôôRô
ô
Pô
ð
 ñ-ó ð-ð ñ!ó ð!ð ñ+ó ð+ð ñ(ó ð(ð ñó ðð ñ)ó ð)ð ñ&ó ð&ð ñ1ó ð1ð ñ%ó ð%ð ñ,ó ð,ð ñ#ó ð#ð ñ*ó ð*ð ñ!ó ð!ð ñ-ó ð-ò2÷	2õ 	2r   r¼   c                   ór   a a€ ] tR tRt oRtV 3R ltR tR tV 3R ltV 3R lt	V 3R lt
V 3R	 ltR
tVtV ;t# )ÚNewlineTokenizeriß  zBA tokenizer that replaces newlines with <n> and <n> with new line.c                ó(   <€ \         SV `  ! V/ VB  R # r   )r:  r9   ©r   rÚ   rÛ   r=  s   &*,€r   r9   ÚNewlineTokenizer.__init__â  s   ø€ Ü‰Ò˜$Ð) &Ô)r   c                ó&   € VP                  R R4      # )rJ   ú<n>©rk   ©r   r!   s   &&r   Ú_preprocess_textÚ!NewlineTokenizer._preprocess_textå  s   € Ø�|‰|˜D %Ó(Ð(r   c                ó&   € VP                  R R4      # )rH  rJ   rI  rJ  s   &&r   Ú_postprocess_textÚ"NewlineTokenizer._postprocess_textè  s   € Ø�|‰|˜E 4Ó(Ð(r   c                óD   <€ \         SV `  ! V P                  V4      3/ VB # r   )r:  rW   rK  )r   r!   rÛ   r=  s   &&,€r   rW   ÚNewlineTokenizer.encodeë  s"   ø€ Ü‰wŠ~˜d×3Ñ3°DÓ9ÑD¸VÑDÐDr   c                ól   <€ \         SV `  ! V Uu. uF  q0P                  V4      NK  	  up3/ VB # u upi r   )r:  Úencode_batchrK  )r   ÚtextsrÛ   Útr=  s   &&, €r   rS  ÚNewlineTokenizer.encode_batchî  s4   ø€ Ü‰wÒ#ÁuÓ$MÁuÀ!×%:Ñ%:¸1Ö%=ÁuÑ$MÑXÐQWÑXÐXùÒ$Ms   �1c                óB   <€ V P                  \        SV `  ! V/ VB 4      # r   )rN  r:  r7   rE  s   &*,€r   r7   ÚNewlineTokenizer.decodeñ  s"   ø€ Ø×%Ñ%¤e¡g¢n°dÐ&E¸fÑ&EÓFÐFr   c                ón   <€ \         SV `  ! V/ VB pV Uu. uF  q@P                  V4      NK  	  up# u upi r   )r:  Úbatch_decoderN  )r   rÚ   rÛ   ÚdecodedÚdr=  s   &*,  €r   rZ  ÚNewlineTokenizer.batch_decodeô  s8   ø€ Ü‘'Ò&¨Ð7°Ñ7ˆÙ3:Ó;±7¨a×&Ñ& qÖ)±7Ñ;Ð;ùÒ;s   –2rî   )r(   r)   r*   r+   r,   r9   rK  rN  rW   rS  r7   rZ  r/   r0   r@  rA  s   @@r   rC  rC  ß  s3   ù‡ € ÙLõ*ò)ò)õEõYõG÷<õ <r   rC  )Úfast_tokenizer_classc                 ó"  a a€ \        S 4      \        S4      8w  d   R # \        S \        4      '       d\   \        S 4      \        S4      8H  ;'       d=    \        ;QJ d     V V3R lS  4       F  '       d   K   R # 	  R# ! V V3R lS  4       4      # \        S \
        4      '       dh   \        S 4      \        S4      8H  ;'       dI    \        ;QJ d&    R \        S S4       4       F  '       d   K   R # 	  R# ! R \        S S4       4       4      # S S8H  # )Fc              3   óp   <"  € T F+  qS9   ;'       d    \        SV,          SV,          4      x € K-  	  R # 5ir   ©Ú_match)r”   ÚkÚar«   s   & €€r   r•   Ú_match.<locals>.<genexpr>   s-   øé € Ð'QÉqÈ!¨Q©×(EÐ(E´6¸!¸A½$ÀÀ!ÅÓ3EÔ(EËqùs   ƒ6“#6Tc              3   ó<   "  € T F  w  r\        W4      x € K  	  R # 5ir   ra  )r”   ÚaiÚbis   &  r   r•   re    s   é € Ð'OÁY¹6¸2¬¨r¯¨ÃYùs   ‚)ÚtypeÚ
isinstanceÚdictr#   rò   Úlistr¡   )rd  r«   s   ffr   rb  rb  ü  sÁ   ù€ ÜˆAƒw”$�q“'ÔÙÜ�!”T×ÒÜ�1‹vœ˜Q›Ñ×QÐQ§C£CÕ'QÉqÓ'Q§C¤CÐQ¢CÐQ CÕ'QÉqÓ'QÓ$QÐQÜ�!”T×ÒÜ�1‹vœ˜Q›Ñ×OÐO§C£CÑ'OÄSÈÈAÄYÓ'O§C¤CÐO¢CÐO CÑ'OÄSÈÈAÄYÓ'OÓ$OÐOà�‰6€Mr   c                 óT   € R RRR RRRR/RR/R R	/R R
/R RRRR^R^ /./p\        W4      # )ri  ÚSequenceÚdecodersÚReplaceÚpatternÚStringrS   ÚcontentrI   ÚByteFallbackÚFuseÚStripr©   rª   ra  ©ÚdecoderÚ_target_descriptions   & r   Ú_is_spm_decoderrz    s^   € à�
ØØ�Y 	¨H°eÐ+<¸iÈÐMØ�^Ð$Ø�VÐØ�W˜i¨¨g°q¸&À!ÐDð	
ðÐô Ð%Ó/Ð/r   c           	      óB   € R RRR RRRR/RR/R R	/R R
/./p\        W4      # )ri  rn  ro  rp  rq  rr  rS   rs  rI   rt  ru  ra  rw  s   & r   Ú_is_spm_decoder_no_spacer|    sJ   € à�
ØØ�Y 	¨H°eÐ+<¸iÈÐMØ�^Ð$Ø�VÐð
ðÐô Ð%Ó/Ð/r   c                 ó\   € \        V \        4      ;'       d    V P                  R R4      R8H  # )ri  NÚ	ByteLevel)rj  rk  r�   )rx  s   &r   Ú_is_bpe_decoderr     s'   € Ü�gœtÓ$×QÐQ¨¯©°V¸TÓ)BÀkÑ)QÐQr   c                ó  € \        V \        4      '       g   R# RV 9   d   R# RV 9   d
   RV 9   d   R# RV 9   d   R# R	V 9   d   R
# RV 9   d   R# RV 9   d   R# RV 9   g   RV 9   d   R# RV 9   d   R# RV 9   d   R# RV 9   d
   RV 9   d   R# R# )z;Attempt to auto-infer a tool parser from the chat template.Nz<minimax:tool_call>Ú
minimax_m2z<|tool_call>z<tool_call|>Úgemma4z<start_function_call>Úfunction_gemmaz<longcat_tool_call>Úlongcatz	<arg_key>Úglm47z<|tool_list_start|>Úpythonicz<tool_call>\n<function=z<tool_call>
<function=Úqwen3_coderz<|tool_calls_section_begin|>Úkimi_k2z[TOOL_CALLS]Úmistralz<tool_call>ztool_call.nameÚ
json_tools)rj  rà   )rÇ   s   &r   Ú_infer_tool_parserr‹  $  sš   € ä�m¤S×)Ò)ÙØ	 -Ô	/ÙØ	˜=Ô	(¨^¸}Ô-LÙØ	  MÔ	1ÙØ	 -Ô	/ÙØ	˜Ô	%ÙØ	 -Ô	/Ùà" mÔ3Ø$¨Ô5áØ	'¨=Ô	8ÙØ	˜=Ô	(ÙØ	˜-Ô	'Ð,<ÀÔ,MÙÙr   c                óh   € V ^8„  d   QhR\         \        \        \        3,          ,          R\        /# )rß   Útokenizer_config_extraÚreturn)r   r   rà   r   r¼   )rá   s   "r   râ   râ   B  s/   € ÷ Fñ Fä$¤T¬#¬s¨(¥^Õ4ðFô ñ	Fr   c           
     óH  € \         pV R,          pVP                  4       '       d¦   \        VRRR7      ;_uu_ 4       p \        P                  ! V4      p RRR4       RX9   dh   \        TR,          4      '       d   \        pMH\        TR,          4      '       d   \        \        RR	7      pM\        TR,          4      '       d   \        p\        V\        4      '       d   V.pV R
,          pRp	\         P"                  ! V 3/ T;'       g    / B p
V
P$                  pVP'                  RR4      ;p'       d$   \(        P*                  ! RV 24      P,                  p	VP'                  R\/        V
P0                  4      4      pVeC   \(        P*                  ! RV 24      pVP2                  pVP4                  pVP6                  pWÛR&   MRpRpRp\9        V
VVV	VVVR7      #   \
         d'   p\        RTP                  TP                  4      hRp?ii ; i  + '       g   i     ELÄ; i)z½Load a huggingface tokenizer and try to infer the type of streaming
detokenizer to use.

Note, to use a fast streaming tokenizer, pass a local file path rather than
a Hugging Face repo ID.
ztokenizer.jsonÚrrj   )ÚencodingzFailed to parse tokenizer.jsonNrx  F)rV   ztokenizer_config.jsonÚchat_template_typezmlx_lm.chat_templates.Útool_parser_typezmlx_lm.tool_parsers.)rÏ   rÇ   rÒ   rÐ   rÑ   )r4   ÚexistsÚopenÚjsonÚloadr   ÚdocÚposrz  rQ   r|  r   r  rw   rj  r`   r	   Úfrom_pretrainedÚinit_kwargsr�   Ú	importlibÚimport_modulerÙ   r‹  rÇ   Úparse_tool_callrÐ   rÑ   r¼   )Ú
model_pathr�  rÏ   rÎ   Útokenizer_fileÚfidÚtokenizer_contentÚeÚtokenizer_config_filerÇ   r8   Útokenizer_configr’  r“  Útool_modulerÒ   rÐ   rÑ   s   &&&               r   r—  r—  B  s  € ô 2ÐàÐ"2Õ2€Nà×Ñ×ÒÜ�. #°×8Õ8¸CðVÜ$(§I¢I¨c£NÑ!÷ 9ð Ð)Ô)ÜÐ0°Õ;×<Ò<Ü$;Ñ!Ü)Ð*;¸IÕ*F×GÒGÜ$+Ô,CÐPUÔ$VÑ!Ü Ð!2°9Õ!=×>Ò>Ü$;Ð!ä�-¤×%Ò%Ø&˜ˆà&Ð)@Õ@ÐØ€Mä×-Ò-ØñØ-×3Ð3°ñ€Ið !×,Ñ,Ðà-×1Ñ1Ð2FÈÓNÐNÐÖNÜ!×/Ò/Ø$Ð%7Ð$8Ð9ó
ç
Ñ
ð 	ð (×+Ñ+ØÔ.¨y×/FÑ/FÓGóÐð Ò#Ü×-Ò-Ð0DÐEUÐDVÐ.WÓXˆØ!×1Ñ1ˆØ%×5Ñ5ˆØ#×1Ñ1ˆØ/?Ð+Ò,àˆØˆØˆäØØØ#Ø#ØØ'Ø#ôð øôW #ô VÜ%Ð&FÈÏÉÈqÏuÉuÓUÐUûðVú÷ 9×8Ð8ús(   ¼H¾GÇHÇ'!HÈHÈHÈH!	c                óH   € V ^8„  d   QhR\         R\        R\        R\         /# )rß   rð   ÚbosÚeosrŽ  )r   r`   )rá   s   "r   râ   râ   ‹  s.   € ÷ Gñ GœDð G¤sð G´ð G¼ñ Gr   c                 ó^   € V ^ ,          V8w  d   T MV R,          pVR,          V8X  d   VRR # T# )r   rl   NrK   rî   )rð   r¨  r©  Úremoved_boss   &&& r   Úno_bos_or_eosr¬  ‹  s5   € Ø& q�k¨SÔ0‘(°h¸rµl€KØ*¨2�°#Ô5ˆ;�s˜ÐÐF¸;ÐFr   r?  )rœ  r–  ÚwarningsÚ	functoolsr   r   Útypingr   r   r   r   Útransformersr	   r
   r   r4   rQ   rw   rº   r¼   rC  Úregisterrb  rz  r|  r  r‹  r—  r¬  rî   r   r   Ú<module>r²     s®   ðÛ Û Û Ý Ý  ß ,Ó ,ç ?÷/ñ /ôd+/Ð 4ô +/ô\-Ð2ô -ô`b*Ð2ô b*òJ$÷>}2ñ }2ô@<Ð.ô <ð4 × Ò Ð)Ð@PÕ Qòò
0ò	0òRò÷<F÷RGr   