
    i=G                     `   d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZmZm	Z	m
Z
 d dlmZmZ  G d d      Z G d d	e      Z G d
 de      Z G d de      Z G d d      Z G d de      Z ej(                  de       d Zd Zd Zd Zd Z	 	 dde
eeef      defdZde	dedede	fdZy)     N)partial)JSONDecodeError)AnyDictListOptional)AutoTokenizerPreTrainedTokenizerFastc                   6    e Zd ZdZdZd Zd Zd Zed        Z	y)StreamingDetokenizerab  The streaming detokenizer interface so that we can detokenize one token at a time.

    Example usage is as follows:

        detokenizer = ...

        # Reset the tokenizer state
        detokenizer.reset()

        for token in generate(...):
            detokenizer.add_token(token.item())

            # Contains the whole text so far. Some tokens may not be included
            # since it contains whole words usually.
            detokenizer.text

            # Contains the printable segment (usually a word) since the last
            # time it was accessed
            detokenizer.last_segment

            # Contains all the tokens added so far
            detokenizer.tokens

        # Make sure that we detokenize any remaining tokens
        detokenizer.finalize()

        # Now detokenizer.text should match tokenizer.decode(detokenizer.tokens)
    )texttokensoffsetc                     t               NNotImplementedErrorselfs    k/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/tokenizer_utils.pyresetzStreamingDetokenizer.reset+       !##    c                     t               r   r   r   tokens     r   	add_tokenzStreamingDetokenizer.add_token.   r   r   c                     t               r   r   r   s    r   finalizezStreamingDetokenizer.finalize1   r   r   c                 \    | j                   }|| j                  d }t        |      | _        |S )zTReturn the last segment of readable text since last time this property was accessed.N)r   r   len)r   r   segments      r   last_segmentz!StreamingDetokenizer.last_segment4   s-     yyt{{}%$ir   N)
__name__
__module____qualname____doc__	__slots__r   r   r   propertyr#    r   r   r   r      s1    : -I$$$  r   r   c                   8    e Zd ZdZd Zd Zd Zd Zed        Z	y)NaiveStreamingDetokenizera  NaiveStreamingDetokenizer relies on the underlying tokenizer
    implementation and should work with every tokenizer.

    Its complexity is O(T^2) where T is the longest line since it will
    repeatedly detokenize the same tokens until a new line is generated.
    c                 j    || _         | j                   j                  dg       | j                          y )Nr   )
_tokenizerdecoder   )r   	tokenizers     r   __init__z"NaiveStreamingDetokenizer.__init__E   s&    #s#

r   c                 J    d| _         g | _        d| _        g | _        d| _        y Nr    )r   r   _text_current_tokens_current_textr   s    r   r   zNaiveStreamingDetokenizer.resetJ   s'    
!r   c                 p    | j                   j                  |       | j                  j                  |       y r   )r6   appendr   r   s     r   r   z#NaiveStreamingDetokenizer.add_tokenQ   s(    ##E*5!r   c                     | xj                   | j                  j                  | j                        z  c_         g | _        d| _        y )Nr4   )r5   r.   r/   r6   r7   r   s    r   r   z"NaiveStreamingDetokenizer.finalizeU   s5    

doo,,T-A-ABB
!r   c                 :   | j                   r| j                  j                  | j                         | _        | j                  j	                  d      s@| j                  j
                  r>t        | j                        dkD  r&| j                  d   dk(  r| j                  d d | _        | j                  rR| j                  d   dk(  r@| xj                  | j                  z  c_        | j                   j                          d| _        | j                  | j                  z   S )N   �r    
r4   )	r6   r.   r/   r7   endswithclean_up_tokenization_spacesr!   r5   clearr   s    r   r   zNaiveStreamingDetokenizer.textZ   s    !%!7!78L8L!MD!!**84<<**+a/&&r*c1%)%7%7%<"$"4"4R"8D"@JJ$,,,J  &&(!#DzzD....r   N)
r$   r%   r&   r'   r1   r   r   r   r)   r   r*   r   r   r,   r,   =   s/    
 " 
 / /r   r,   c                   2    e Zd ZdZddZd Zd	dZd Zd Zy)
SPMStreamingDetokenizerzA streaming detokenizer for SPM models.

    It adds tokens to the text if the next token starts with the special SPM
    underscore which results in linear complexity.
    c                    || _         dj                         | _        dgt        |j                  j                               dz   z  | _        |j                  j                         D ]Z  \  }}|j                  d      r't        t        |dd d      g      | j                  |<   >|j                         | j                  |<   \ | j                          y )N   ▁r4      z<0x         )
trim_spaceencode_sepmaxvocabvaluestokenmapitems
startswithbytesintr   )r   r0   rK   valuetokenids        r   r1   z SPMStreamingDetokenizer.__init__r   s    $OO%	 IOO$:$:$< = AB'oo335NE7&).E!AJ0C/D)Eg&).g& 6 	

r   c                 <    d| _         d| _        d| _        g | _        y )Nr   r   r4   r   
_unflushedr   r   r   s    r   r   zSPMStreamingDetokenizer.reset   s    	r   c                    | j                   j                  | j                  d      j                  dd      }|s|j	                  d      ry | j
                  s| j                  r|r|d   dk(  r|dd  }| xj
                  |z  c_        d| _         y )	N    utf-8replacer<   r   r>   rG   r   )rZ   r^   rM   r/   r@   r   rK   )r   forcer   s      r   
_try_flushz"SPMStreamingDetokenizer._try_flush   ss    &&tyy$7>>w	Rx0yyT__$q'S.8D		T	r   c                     | j                   j                  |       | j                  |   }| xj                  |z  c_        | j	                          y r   )r   r9   rQ   rZ   r`   )r   r   vs      r   r   z!SPMStreamingDetokenizer.add_token   s:    5!MM% 1r   c                 6    | j                  d       d| _        y )NT)r_   r   )r`   rZ   r   s    r   r   z SPMStreamingDetokenizer.finalize   s    d#r   N)T)F)	r$   r%   r&   r'   r1   r   r`   r   r   r*   r   r   rD   rD   k   s     r   rD   c                   L    e Zd ZdZdZdZd Zd Zd Zd Z	d Z
d	 Zed
        Zy)BPEStreamingDetokenizerzA streaming detokenizer for OpenAI style BPE models.

    It adds tokens to the text if the next token starts with a space similar to
    the SPM detokenizer.
    N)	.?!,zn'tz'mz'sz'vez'rec                    |j                   | _        d gt        |j                        z  | _        |j                  j                         D ]  \  }}|| j                  |<    | j                          | j                          y r   )rA   clean_spacesr!   rO   rQ   rR   r   make_byte_decoder)r   r0   rV   rW   s       r   r1   z BPEStreamingDetokenizer.__init__   sk    %BB Y__!55'oo335NE7%*DMM'" 6 	

 	 r   c                 <    d| _         d| _        d| _        g | _        y r3   rY   r   s    r   r   zBPEStreamingDetokenizer.reset   s    	r   c                     t               }|D ]M  }| j                  j                  |d      }|r|j                  |       3|j	                  t        |d             O |j                  dd      S )NFr]   r^   )	bytearray_byte_decodergetr9   extendrT   r/   )r   seqbarrcress        r   _decode_bytesz%BPEStreamingDetokenizer._decode_bytes   s`    {A$$((E2CC E!W-.  {{7I..r   c                     t        |      dk(  r|S |d   dk7  r|S | j                  s|dd  S | j                  r#|dd  j                  | j                        r|dd  S |S )Nr   r>   rG   )r!   r   rk   rS   _space_matchesr   current_texts     r   _maybe_trim_spacez)BPEStreamingDetokenizer._maybe_trim_space   sq    |!!_###<#3#>#>t?R?R#S##r   c                    | j                   j                  |       |t        | j                        k  r| j                  |   nd}| xj                  |z  c_        | j                  | j                        }|j                  d      s\t        |      dk(  r!| j                  j                  |d         dk(  s,| xj                  | j                  |      z  c_	        d| _        y y y )Nrh   r<   rG   r       r4   )r   r9   r!   rQ   rZ   rw   r@   rp   rq   r   r|   )r   r   rb   r   s       r   r   z!BPEStreamingDetokenizer.add_token   s    5!$)C,>$>DMM% C1!!$//2 }}X&FaKD..221Q48B>II//55I DO ? 'r   c                      t         fd j                  D              j                  dd      } xj                   j	                  |      z  c_        d _        y )Nc              3   <   K   | ]  }j                   |     y wr   )rp   ).0ru   r   s     r   	<genexpr>z3BPEStreamingDetokenizer.finalize.<locals>.<genexpr>   s      P1!3!3A!6s   r]   r^   r4   )ro   rZ   r/   r   r|   rz   s   ` r   r   z BPEStreamingDetokenizer.finalize   sJ      P PPWW
 			T++L99	r   c           	         | j                   yi }dt        d      t        d      dz   t        d      t        d      dz   t        d      t        d	      dz   g}d}t        t        ||dd             D ]W  \  }\  }}|d
z  dk(  r(t	        ||      D ]  }||t        d|z         <   |dz  } 9t	        ||      D ]  }||t        |      <    Y || _         y)zQSee https://github.com/openai/gpt-2/blob/master/src/encoder.py for the rationale.Nr   rh   ~rG      ¡   ¬   ®   ÿ      )rp   ord	enumerateziprangechr)clschar_to_byteslimitsnistartstopbs           r   rl   z)BPEStreamingDetokenizer.make_byte_decoder   s     (HHqLIIMIIM
  )#ffQRj*A BA}t1uzud+A34M#dQh-0FA , ud+A,-M#a&) , !C *r   )r$   r%   r&   r'   rp   ry   r1   r   rw   r|   r   r   classmethodrl   r*   r   r   re   re      sE     MJN!/	! * *r   re   c                        e Zd ZdZedddddfdZdddZdefdZe	d	        Z
e	d
        Ze	d        Ze	d        Ze	d        Ze	d        Ze	d        Ze	d        Ze	d        Ze	d        Zd Z fdZ xZS )TokenizerWrapperzA wrapper that combines an HF tokenizer and a detokenizer.

    Accessing any attribute other than the ``detokenizer`` is forwarded to the
    huggingface tokenizer.
    Nc                    || _         || _        |t        |      n|j                  h| _        d | _        d | _        d | _        d | _        || _	        |j                  d uxs |d u| _        || _        || _        || _        |j                         }ddg}	|	D ]2  \  }
}|
|v s||v s|
| _        || _        ||
   | _        ||   | _         n |r||vs|r||vrd | _        d | _        d | _        y y y )N)z<think>z</think>)z<longcat_think>z</longcat_think>)r.   _detokenizer_classseteos_token_id_eos_token_ids_think_start
_think_end_think_start_id_think_end_id_chat_templatechat_templatehas_chat_template_tool_parser_tool_call_start_tool_call_end	get_vocab)r   r0   detokenizer_classeos_token_idsr   tool_call_starttool_call_endtool_parserrO   THINK_TOKENSthink_start	think_ends               r   r1   zTokenizerWrapper.__init__  s8    $"3 ( (() 	
 !#!+##4/L=3L 	 ( /+##%#3
 '3"Ke#	U(:$/!"+',['9$%*9%5" '3 u <m58$(D!"&D $D	 9Mr   T)tokenizec                    | j                   3 | j                   |i |}|r| j                  j                  |d      }|S d|d<    | j                  j                  |d|i|S )NF)add_special_tokensreturn_dictr   )r   r.   rL   apply_chat_template)r   r   argskwargsouts        r   r   z$TokenizerWrapper.apply_chat_template:  sq    *%$%%t6v6Coo,,SU,KJ %}2t22DV8VvVVr   r   c                     d }	 t        |      }|t        d| d      | j                  j                  |       y # t        $ r | j                  j                  |      }Y Sw xY w)N'z#' is not a token for this tokenizer)rU   
ValueErrorr.   convert_tokens_to_idsr   add)r   r   token_ids      r   add_eos_tokenzTokenizerWrapper.add_eos_tokenD  sn    	D5zH q'JKLL)  	D<<UCH	Ds   < $A#"A#c                     | j                   d uS r   r   r   s    r   has_thinkingzTokenizerWrapper.has_thinkingP  s      ,,r   c                     | j                   S r   r   r   s    r   r   zTokenizerWrapper.think_startT         r   c                     | j                   S r   )r   r   s    r   think_start_idzTokenizerWrapper.think_start_idX  s    ###r   c                     | j                   S r   )r   r   s    r   r   zTokenizerWrapper.think_end\  s    r   c                     | j                   S r   )r   r   s    r   think_end_idzTokenizerWrapper.think_end_id`  s    !!!r   c                     | j                   d uS r   r   r   s    r   has_tool_callingz!TokenizerWrapper.has_tool_callingd  s    $$D00r   c                     | j                   S r   r   r   s    r   r   z TokenizerWrapper.tool_call_starth  s    $$$r   c                     | j                   S r   )r   r   s    r   r   zTokenizerWrapper.tool_call_endl  s    """r   c                     | j                   S r   )r   r   s    r   r   zTokenizerWrapper.tool_parserp  r   r   c                 $    | j                  |       S )z7
        Get a stateful streaming detokenizer.
        )r   r   s    r   detokenizerzTokenizerWrapper.detokenizert  s    
 &&t,,r   c                     |dk(  r| j                   S |dk(  r| j                  S |j                  d      r| j                  |      S t	        | j
                  |      S )Nr   r   _)_detokenizerr   rS   __getattribute__getattrr.   )r   attrs     r   __getattr__zTokenizerWrapper.__getattr__{  sX    = $$$_$&&&__S!((..4??D11r   c                     |dv r3|dk(  rt        d      |dk(  r|t        |      n	t               | _        y y |j                  d      rt        |   ||       y t        | j                  ||       y )N>   r   r   r   zCannot set the detokenizer.r   r   )AttributeErrorr   r   rS   super__setattr__setattrr.   )r   r   rV   	__class__s      r   r   zTokenizerWrapper.__setattr__  sm    33}$$%BCC(494Ec%j35# )__S!Ge,DOOT51r   )r$   r%   r&   r'   r,   r1   r   strr   r)   r   r   r   r   r   r   r   r   r   r   r   r   __classcell__r   s   @r   r   r      s    41%f 37 W
*3 
* - - ! ! $ $   " " 1 1 % % # # ! ! - -2	2 	2r   r   c                   V     e Zd ZdZ fdZd Zd Z fdZ fdZ fdZ	 fdZ
 xZS )	NewlineTokenizerzBA tokenizer that replaces newlines with <n> and <n> with new line.c                 $    t        |   |i | y r   )r   r1   r   r   r   r   s      r   r1   zNewlineTokenizer.__init__  s    $)&)r   c                 &    |j                  dd      S )Nr?   <n>r^   r   r   s     r   _preprocess_textz!NewlineTokenizer._preprocess_text  s    ||D%((r   c                 &    |j                  dd      S )Nr   r?   r   r   s     r   _postprocess_textz"NewlineTokenizer._postprocess_text  s    ||E4((r   c                 B    t        |   | j                  |      fi |S r   )r   rL   r   )r   r   r   r   s      r   rL   zNewlineTokenizer.encode  s"    w~d33D9DVDDr   c                 h    t        |   |D cg c]  }| j                  |       c}fi |S c c}w r   )r   encode_batchr   )r   textsr   tr   s       r   r   zNewlineTokenizer.encode_batch  s6    w#u$Mu!T%:%:1%=u$MXQWXX$Ms   /c                 @    | j                  t        |   |i |      S r   )r   r   r/   r   s      r   r/   zNewlineTokenizer.decode  s"    %%egnd&Ef&EFFr   c                 j    t        |   |i |}|D cg c]  }| j                  |       c}S c c}w r   )r   batch_decoder   )r   r   r   decodeddr   s        r   r   zNewlineTokenizer.batch_decode  s:    '&773:;7a&&q)7;;;s   0)r$   r%   r&   r'   r1   r   r   rL   r   r/   r   r   r   s   @r   r   r     s1    L*))EYG< <r   r   )fast_tokenizer_classc                 F    t               t              k7  ryt         t              r.t               t              k(  xr t	         fd D              S t         t
              r5t               t              k(  xr t	        d t               D              S  k(  S )NFc              3   N   K   | ]  }|v xr t        |   |           y wr   _match)r   kar   s     r   r   z_match.<locals>.<genexpr>  s-     'Qq!Q(E6!A$!3E(Eqs   "%c              3   :   K   | ]  \  }}t        ||        y wr   r   )r   aibis      r   r   z_match.<locals>.<genexpr>  s     'OY62rr2Ys   )type
isinstancedictr!   alllistr   )r   r   s   ``r   r   r     s    Aw$q'!T1vQQC'Qq'Q$QQ!T1vQOC'OSAY'O$OO6Mr   c           	      J    ddddiddddidd	id
ddddgd}t        ||       S )NSequenceReplaceStringrF   r>   r  patterncontentr  ByteFallbackFuseStriprG   r   )r  r  r   r   r  decodersr   decoder_target_descriptions     r   _is_spm_decoderr    sM    He+<M^$Vq!D	
 %w//r   c                 >    ddddiddddidd	igd
}t        ||       S )Nr	  r
  r  rF   r>   r  r  r  r  r  r   r  s     r   _is_spm_decoder_no_spacer    sA    He+<M^$V
 %w//r   c                 P    t        | t              xr | j                  dd       dk(  S )Nr  	ByteLevel)r  r  rq   )r  s    r   _is_bpe_decoderr    s$    gt$QVT)Bk)QQr   c                     t        | t              syd| v ryd| v ryd| v ryd| v ry	d
| v ryd| v sd| v ryd| v ryd| v ryd| v rd| v ryy)z;Attempt to auto-infer a tool parser from the chat template.Nz<minimax:tool_call>
minimax_m2z<start_function_call>function_gemmaz<longcat_tool_call>longcatz	<arg_key>glm47z<|tool_list_start|>pythonicz<tool_call>\n<function=z<tool_call>
<function=qwen3_coderz<|tool_calls_section_begin|>kimi_k2z[TOOL_CALLS]mistralz<tool_call>ztool_call.name
json_tools)r  r   )r   s    r   _infer_tool_parserr'    s    mS)	-	/	 M	1	-	/		%	-	/"m3$5	'=	8	=	(	-	',<,Mr   tokenizer_config_extrareturnc           	         t         }| dz  }|j                         r{t        |dd      5 }	 t        j                  |      }	 ddd       dv rIt        |d         rt        }n4t        |d         rt        t        d	      }nt        |d         rt        }t        |t              r|g}| d
z  }d}	t!        j"                  | fi |xs i }
|
j$                  }|j'                  dd      x}r"t)        j*                  d|       j,                  }	|j'                  dt/        |
j0                              }|Bt)        j*                  d|       }|j2                  }|j4                  }|j6                  }||d<   nd}d}d}t9        |
|||	|||      S # t
        $ r&}t        d|j                  |j                        d}~ww xY w# 1 sw Y   xY w)zLoad a huggingface tokenizer and try to infer the type of streaming
    detokenizer to use.

    Note, to use a fast streaming tokenizer, pass a local file path rather than
    a Hugging Face repo ID.
    ztokenizer.jsonrr]   )encodingzFailed to parse tokenizer.jsonNr  F)rK   ztokenizer_config.jsonchat_template_typezmlx_lm.chat_templates.tool_parser_typezmlx_lm.tool_parsers.)r   r   r   r   r   )r,   existsopenjsonloadr   docposr  rD   r  r   r  re   r  rU   r	   from_pretrainedinit_kwargsrq   	importlibimport_moduler   r'  r   parse_tool_callr   r   r   )
model_pathr(  r   r   tokenizer_filefidtokenizer_contentetokenizer_config_filer   r0   tokenizer_configr-  r.  tool_moduler   r   r   s                     r   r2  r2    s    2"22N.#8CV$(IIcN! 9 ))0;<$;!)*;I*FG$+,CPU$V! !29!=>$;!-%&&)@@M---3I !,,-112FNNN!//$%7$89


 	 (++.y/F/FG #--0DEUDV.WX!11%55#11/?+,##'# W # V%&FquuUUV 98s(   G F	F=!F88F==G  G
sequenceboseosc                 >    | d   |k7  r| n| dd  }|d   |k(  r|d d S |S )Nr   rG   r=   r*   )rB  rC  rD  removed_boss       r   no_bos_or_eosrG  ;  s7    &qkS0(hqrlK*2#5;sF;Fr   )NN)r7  r1  warnings	functoolsr   r   typingr   r   r   r   transformersr	   r
   r   r,   rD   re   r   r   registerr   r  r  r  r'  r   r2  rU   rG  r*   r   r   <module>rM     s          , , ?/ /d+/ 4 +/\-2 -`b*2 b*JN2 N2b<. <4   )@P Q
0	0R< 8<F$T#s(^4F 	FRGD Gs G G Gr   