+
    UV-jê>  ã                   ó
  € ^ RI t ^ RIt^ RIHt ^ RIHt ^ RIHt ^ RIHtHt ^ RI	H
t
 RtR t ! R	 R
4      t ! R R]4      t ! R R]4      t ! R R]4      t ! R R4      t ! R R4      tR tR tR tR tR tR/ 3R ltR# )é    N)Úcopy)Úpartial)ÚJSONDecodeError)ÚListÚOptional)ÚAutoTokenizeru   ï¿½c                 óD   € V '       d   V ^ ,          R8X  d
   V R,          # V # )r   Ú ºé   NN© )Úxs   &Úh/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/tokenizer_utils.pyÚ_remove_spacer      s   € ßˆQˆq�T�SŒ[Ø��uˆØ€Hó    c                   ó\   a € ] tR t^t o RtR	tR t. 3V 3R lR lltR t]	R 4       t
RtV tR# )
ÚStreamingDetokenizera  The streaming detokenizer interface so that we can detokenize one token at a time.

Example usage is as follows:

    detokenizer = ...

    # Reset the tokenizer state
    detokenizer.reset()

    for token in generate(...):
        detokenizer.add_token(token.item())

        # Contains the whole text so far. Some tokens may not be included
        # since it contains whole words usually.
        detokenizer.text

        # Contains the printable segment (usually a word) since the last
        # time it was accessed
        detokenizer.last_segment

        # Contains all the tokens added so far
        detokenizer.tokens

    # Make sure that we detokenize any remaining tokens
    detokenizer.finalize()

    # Now detokenizer.text should match tokenizer.decode(detokenizer.tokens)
c                ó   € \        4       h©N©ÚNotImplementedError©Úselfs   &r   ÚresetÚStreamingDetokenizer.reset3   ó   € Ü!Ó#Ð#r   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# ©é   Úskip_special_token_ids©r   Úint)ÚformatÚ__classdict__s   "€r   Ú__annotate__Ú!StreamingDetokenizer.__annotate__6   s   ø€ ÷ $ñ $±t¹Cµyñ $r   c                ó   € \        4       hr   r   ©r   Útokenr    s   &&&r   Ú	add_tokenÚStreamingDetokenizer.add_token6   r   r   c                ó   € \        4       hr   r   r   s   &r   ÚfinalizeÚStreamingDetokenizer.finalize9   r   r   c                ó’   € V P                   pV'       d3   VR,          \        8w  d!   WP                  R p\        V4      V n        V# R# )zTReturn the last segment of readable text since last time this property was accessed.NÚ éÿÿÿÿ)ÚtextÚREPLACEMENT_CHARÚoffsetÚlen)r   r2   Úsegments   &  r   Úlast_segmentÚ!StreamingDetokenizer.last_segment<   s=   € ð �y‰yˆß�D˜•HÔ 0Ô0ØŸ;™;˜=Ð)ˆGÜ˜d›)ˆDŒKØˆNÙr   )r4   N)r2   Útokensr4   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú	__slots__r   r*   r-   Úpropertyr7   Ú__static_attributes__Ú__classdictcell__©r$   s   @r   r   r      s@   ø‡ € ñð: -€Iò$ð DF÷ $ò $ò$ð ñó ör   r   c                   ót   a € ] tR t^Gt o RtR tR tR t. 3V 3R lR lltR t	]
R 4       t]
R	 4       tR
tV tR# )ÚNaiveStreamingDetokenizerzûNaiveStreamingDetokenizer relies on the underlying tokenizer
implementation and should work with every tokenizer.

Its complexity is O(T^2) where T is the longest line since it will
repeatedly detokenize the same tokens until a new line is generated.
c                ój   € Wn         V P                   P                  ^ .4       V P                  4        R# )r   N)Ú
_tokenizerÚdecoder   )r   Ú	tokenizers   &&r   Ú__init__Ú"NaiveStreamingDetokenizer.__init__O   s$   € Ø#ŒØ�‰×Ñ ˜sÔ#Ø�
‰
Žr   c                ó8   € \        V 4      ! V P                  4      # r   )ÚtyperG   r   s   &r   Ú__copy__Ú"NaiveStreamingDetokenizer.__copy__T   s   € Ü�DŒz˜$Ÿ/™/Ó*Ð*r   c                óL   € ^ V n         . V n        RV n        . V n        RV n        R# ©r   r0   N)r4   Ú_tokensÚ_textÚ_current_tokensÚ_current_textr   s   &r   r   ÚNaiveStreamingDetokenizer.resetW   s'   € ØˆŒØˆŒØˆŒ
Ø!ˆÔØˆÖr   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# r   r!   )r#   r$   s   "€r   r%   Ú&NaiveStreamingDetokenizer.__annotate__^   s   ø€ ÷ +ñ +±t¹Cµyñ +r   c                óL   € W9   d   R # V P                   P                  V4       R # r   )rT   Úappendr(   s   &&&r   r*   Ú#NaiveStreamingDetokenizer.add_token^   s   € ØÔ*ÙØ×Ñ×#Ñ# EÖ*r   c                óä   € V P                   P                  V P                  4       V ;P                  V P                  P                  V P                  4      ,          un        . V n        R V n        R# )r0   N)rR   ÚextendrT   rS   rG   rH   rU   r   s   &r   r-   Ú"NaiveStreamingDetokenizer.finalizec   sM   € Ø�‰×Ñ˜D×0Ñ0Ô1Ø�
Š
�d—o‘o×,Ñ,¨T×-AÑ-AÓBÕB�
Ø!ˆÔØˆÖr   c                óÚ  € V P                   '       d+   V P                  P                  V P                   4      V n        V P                  '       d‚   V P                  R,          R8X  dj   V P                  P                  V P                   4       V ;P                  V P                  ,          un        V P                   P                  4        RV n        V P                  V P                  ,           # )r   Ú
r0   r1   )rT   rG   rH   rU   rR   r]   rS   Úclearr   s   &r   r2   ÚNaiveStreamingDetokenizer.texti   s¤   € à××ÐØ!%§¡×!7Ñ!7¸×8LÑ8LÓ!MˆDÔØ××Ð $×"4Ñ"4°RÕ"8¸DÔ"@Ø�L‰L×Ñ × 4Ñ 4Ô5Ø�JŠJ˜$×,Ñ,Õ,�JØ× Ñ ×&Ñ&Ô(Ø!#ˆDÔØ�z‰z˜D×.Ñ.Õ.Ð.r   c                ó   € V P                   # r   )rR   r   s   &r   r9   Ú NaiveStreamingDetokenizer.tokenst   s   € à�|‰|Ðr   )rU   rT   rS   rG   rR   r4   N)r:   r;   r<   r=   r>   rJ   rN   r   r*   r-   r@   r2   r9   rA   rB   rC   s   @r   rE   rE   G   sW   ø‡ € ñòò
+ò ð DF÷ +ò +ò
 ð ñ/ó ð/ð ñó ör   rE   c                   óX   a € ] tR t^yt o RtR
R ltR tR t. 3V 3R lR lltR t	Rt
V tR	# )ÚSPMStreamingDetokenizerae  A streaming detokenizer for SPM models.

It adds tokens to the text if the next token starts with the special SPM
underscore which results in linear complexity.

Handles UTF-8 byte tokens (like <0xE5><0xA4><0xA2>) by accumulating them
and decoding as UTF-8 bytes, which is necessary for multi-byte characters
like Chinese that may not be in the vocabulary.
c                ó4  € W n         R .\        VP                  4      ,          V n        R.\        VP                  4      ,          V n        ^ .\        VP                  4      ,          V n        VP                  P                  4        F}  w  r4W0P                  V&   VP                  R4      '       g   K,  \        V4      ^8¼  g   K>  V^,          R8X  g   KN  RV P                  V&   \        VR,          ^4      V P
                  V&   K  	  V P                  4        R # )NFz<0xÚ>T:é   é   N)
Ú
trim_spacer5   ÚvocabÚtokenmapÚis_byte_tokenÚ
byte_valueÚitemsÚ
startswithr"   r   ©r   rI   rk   ÚvalueÚtokenids   &&&  r   rJ   Ú SPMStreamingDetokenizer.__init__„   sÍ   € Ø$Œð ˜¤ Y§_¡_Ó!5Õ5ˆŒØ#˜W¤s¨9¯?©?Ó';Õ;ˆÔØ˜#¤ I§O¡OÓ 4Õ4ˆŒà'Ÿo™o×3Ñ3Ö5‰NˆEØ%*�M‰M˜'Ñ"à×Ñ ×&Ô&¬3¨u«:¸®?¸uÀQ½xÈ3¾Ø.2�×"Ñ" 7Ñ+Ü+.¨u°S­z¸2Ó+>�—‘ Ó(ñ 6ð 	�
‰
Žr   c                ó\   € ^ V n         RV n        \        4       V n        RV n        . V n        R# rQ   )r4   Ú
_unflushedÚ	bytearrayÚ_byte_bufferr2   r9   r   s   &r   r   ÚSPMStreamingDetokenizer.reset•   s(   € ØˆŒØˆŒÜ%›KˆÔØˆŒ	ØˆŽr   c                ó@  € V P                   '       dG    V P                   P                  R4      pV ;P                  V,          un        \	        4       V n         R# R#   \         d8    T ;P                  T P                   P                  RRR7      ,          un         LTi ; i)z?Decode accumulated bytes as UTF-8 and append to unflushed text.úutf-8Úreplace©ÚerrorsN)ry   rH   rw   ÚUnicodeDecodeErrorrx   )r   Údecodeds   & r   Ú_flush_bytesÚ$SPMStreamingDetokenizer._flush_bytesœ   s€   € à××ÐðWØ×+Ñ+×2Ñ2°7Ó;�Ø—’ 7Õ*•ô !*£ˆDÖñ øô &ô Wà—’ 4×#4Ñ#4×#;Ñ#;¸GÈIÐ#;Ó#VÕV—ðWús   ”4A Á?BÂBc                ó0   <€ V ^8„  d   QhRS[ S[,          /# r   r!   )r#   r$   s   "€r   r%   Ú$SPMStreamingDetokenizer.__annotate__§   s   ø€ ÷ !ñ !±t¹Cµyñ !r   c                ó\  € W9   d   R # V P                   V,          '       d/   V P                  P                  V P                  V,          4       R # V P	                  4        V P
                  V,          pV'       d™   V^ ,          R8X  d‹   V P                  '       g   V P                  '       g5   V ;P                  V P                  P                  RR4      ,          un        M*\        V P                  P                  RR4      4      V n        W0n        R # V ;P                  V,          un        R # )Nõ   â–�r
   )rn   ry   rZ   ro   r‚   rm   r2   rk   rw   r}   r   )r   r)   r    Úvs   &&& r   r*   Ú!SPMStreamingDetokenizer.add_token§   sÆ   € ØÔ*Ùà×Ñ˜e×$Ô$à×Ñ×$Ñ$ T§_¡_°UÕ%;Ô<Ùð 	×ÑÔà�M‰M˜%Õ ˆß��1•˜Ô!Ø�y�yˆy §§ Ø—	’	˜TŸ_™_×4Ñ4°X¸sÓCÕC–	ä)¨$¯/©/×*AÑ*AÀ(ÈCÓ*PÓQ�”	ØŽOà�OŠO˜qÕ �Or   c                ó8  € V P                  4        V P                  '       g   V P                  '       g5   V ;P                  V P                  P	                  R R4      ,          un        M*\        V P                  P	                  R R4      4      V n        RV n        R# )r‡   r
   r0   N)r‚   r2   rk   rw   r}   r   r   s   &r   r-   Ú SPMStreamingDetokenizer.finalize½   sb   € à×ÑÔà�9�9ˆ9˜DŸOŸO˜OØ�IŠI˜Ÿ™×0Ñ0°¸3Ó?Õ?ŽIä% d§o¡o×&=Ñ&=¸hÈÓ&LÓMˆDŒIØˆŽr   )	ry   rw   ro   rn   r4   r2   rm   r9   rk   N)T)r:   r;   r<   r=   r>   rJ   r   r‚   r*   r-   rA   rB   rC   s   @r   rf   rf   y   s4   ø‡ € ñôò"ò	,ð DF÷ !ò !÷,ð r   rf   c                   óf   a € ] tR t^Èt o RtRtR
R ltR t. 3V 3R lR lltR t	]
R 4       tR	tV tR# )ÚBPEStreamingDetokenizerz—A streaming detokenizer for OpenAI style BPE models.

It adds tokens to the text if the next token starts with a space similar to
the SPM detokenizer.
Nc                óø   € W n         R .\        VP                  4      ,          V n        VP                  P	                  4        F  w  r4W0P                  V&   K  	  V P                  4        V P                  4        R # r   )rk   r5   rl   rm   rp   r   Úmake_byte_decoderrr   s   &&&  r   rJ   Ú BPEStreamingDetokenizer.__init__Ñ   s^   € Ø$Œð ˜¤ Y§_¡_Ó!5Õ5ˆŒØ'Ÿo™o×3Ñ3Ö5‰NˆEØ%*�M‰M˜'Ó"ñ 6ð 	�
‰
Œð 	×ÑÖ r   c                ó>   € ^ V n         RV n        RV n        . V n        R# rQ   )r4   rw   r2   r9   r   s   &r   r   ÚBPEStreamingDetokenizer.resetß   s   € ØˆŒØˆŒØˆŒ	ØˆŽr   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# r   r!   )r#   r$   s   "€r   r%   Ú$BPEStreamingDetokenizer.__annotate__å   s   ø€ ÷ !ñ !±t¹Cµyñ !r   c                óÜ  a € W9   d   R # S P                   V,          pS P                  V^ ,          ,          ^ 8X  d˜   \        V 3R lS P                   4       4      P	                  R4      pS P
                  '       g   S P                  '       g   S ;P
                  V,          un        M"S ;P
                  \        V4      ,          un        VS n        R # S ;P                  V,          un        R # )Nc              3   óJ   <"  € T F  pSP                   V,          x € K  	  R # 5ir   ©Ú_byte_decoder©Ú.0Úcr   s   & €r   Ú	<genexpr>Ú4BPEStreamingDetokenizer.add_token.<locals>.<genexpr>ë   ó"   øé € ð %Ù/>¨!�×"Ñ" 1×%Ò%«ùó   ƒ #r|   )rm   r˜   rx   rw   rH   r2   rk   r   )r   r)   r    rˆ   Úcurrent_texts   f&&  r   r*   Ú!BPEStreamingDetokenizer.add_tokenå   s¡   ø€ ØÔ*ÙØ�M‰M˜%Õ ˆà×Ñ˜a �dÕ# rÔ)Ü$ô %Ø/3¯ªó%ó ç‰f�W‹oð ð �y�yˆy §§ Ø—	’	˜\Õ)–	à—	’	œ]¨<Ó8Õ8•	ØˆDŽOà�OŠO˜qÕ �Or   c                ó˜  a €  \        V 3R  lS P                   4       4      P                  RRR7      pS P                  '       g   S P                  '       g   S ;P                  V,          un        M#S ;P                  \        V4      ,          un         RS n        R#   \        \        3 d    \        RS P                   24        L4i ; i)c              3   óJ   <"  € T F  pSP                   V,          x € K  	  R # 5ir   r—   r™   s   & €r   rœ   Ú3BPEStreamingDetokenizer.finalize.<locals>.<genexpr>ø   rž   rŸ   r|   Úignorer~   z!Warning: could not decode bytes: r0   N)	rx   rw   rH   r2   rk   r   r€   ÚKeyErrorÚprint)r   r    s   f r   r-   Ú BPEStreamingDetokenizer.finalizeö   sŸ   ø€ ð		IÜ$ô %Ø/3¯ªó%ó ç‰f�W XˆfÓ.ð ð �y�yˆy §§ Ø—	’	˜\Õ)–	à—	’	œ]¨<Ó8Õ8–	ð ˆŽøô #¤HÐ-ô 	IÜÐ5°d·o±oÐ5FÐGÖHð	Iús$   ƒAB ÁB ÁB Á1"B Â)C	ÃC	c           	     óì  € V P                   e   R# / p^ \        R4      \        R4      ^,           \        R4      \        R4      ^,           \        R4      \        R4      ^,           .p^ p\        \        W"R,          4      4       Ff  w  pw  rVV^,          ^ 8X  d2   \	        WV4       F   pWq\        R	V,           4      &   V^,          pK"  	  KG  \	        WV4       F  pWq\        V4      &   K  	  Kh  	  Wn         R# )
zQSee https://github.com/openai/gpt-2/blob/master/src/encoder.py for the rationale.NÚ!Ú~ô   Â¡ô   Â¬ô   Â®ô   Ã¿r   é   )r˜   ÚordÚ	enumerateÚzipÚrangeÚchr)ÚclsÚchar_to_bytesÚlimitsÚnÚiÚstartÚstopÚbs   &       r   r�   Ú)BPEStreamingDetokenizer.make_byte_decoder  s×   € ð ×ÑÒ(ÙàˆàÜ�‹HÜ�‹H�q�LÜ�‹IÜ�‹I˜�MÜ�‹IÜ�‹I˜�Mð
ˆð ˆÜ )¬#¨f¸RµjÓ*AÖ BÑˆA‰}�Ø�1�u˜ŒzÜ˜uÖ+�AØ34¤# d¨Q¥h£-Ñ0Ø˜•F’Aó ,ô ˜uÖ+�AØ,-¤# a£&Ó)ó ,ñ !Cð *Ör   )rw   r4   r2   rm   r9   rk   )F)r:   r;   r<   r=   r>   r˜   rJ   r   r*   r-   Úclassmethodr�   rA   rB   rC   s   @r   r�   r�   È   sE   ø‡ € ñð €Mô!òð DF÷ !ò !ò"ð ñ*ó ö*r   r�   c                   óº   a € ] tR tRt o RtR tV 3R lR ltV 3R lR ltV 3R lR	 ltV 3R
 lR lt	V 3R lR lt
V 3R lR ltV 3R lR ltV 3R lR ltRtV tR# )Ú_ServerTokenStreameri  zCEmit server text deltas per token while buffering incomplete UTF-8.c                óR  € Wn         W n        R V n        RV n        R V n        RV n        RV n        \        V\        4      '       do   RV n        VP                  V n        VP                  V n        VP                  V n        VP                  V n        \        P                   ! R4      ! R4      V n        R # \        V\"        4      '       d^   RV n        VP                  V n        VP                  V n        VP$                  V n        \        P                   ! R4      ! R4      V n        R # R # )NFÚdelegateÚspmr|   r}   Úbpe)rG   Ú_detokenizerÚ_decoderÚ_startedÚ	_tokenmapÚ_modeÚ_trim_spaceÚ
isinstancerf   rm   rk   rn   Ú_is_byte_tokenro   Ú_byte_valueÚcodecsÚgetincrementaldecoderr�   r˜   )r   rI   Údetokenizers   &&&r   rJ   Ú_ServerTokenStreamer.__init__"  sî   € Ø#ŒØ'ÔØˆŒØˆŒØˆŒØˆŒ
Ø ˆÔä�kÔ#:×;Ò;ØˆDŒJØ(×1Ñ1ˆDŒNØ*×5Ñ5ˆDÔØ"-×";Ñ";ˆDÔØ*×5Ñ5ˆDÔÜ"×8Ò8¸ÔAÀ)ÓLˆDŽMÜ˜Ô%<×=Ò=ØˆDŒJØ(×1Ñ1ˆDŒNØ*×5Ñ5ˆDÔØ!,×!:Ñ!:ˆDÔÜ"×8Ò8¸ÔAÀ)ÓLˆDŽMñ >r   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   r2   Úreturn©Ústr)r#   r$   s   "€r   r%   Ú!_ServerTokenStreamer.__annotate__9  s   ø€ ÷ ñ ™sð ¡sñ r   c                óª   € V'       d<   V P                   '       g*   V P                  '       d   V^ ,          R8X  d
   VR,          pV'       d   RV n         V# )r   r
   r   T)rÈ   rË   )r   r2   s   &&r   Ú
_emit_textÚ_ServerTokenStreamer._emit_text9  s:   € ß˜ŸŸ˜¨$×*:×*:Ð*:¸tÀA½wÈ#¼~Ø˜•8ˆDßØ ˆDŒMØˆr   c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r   r)   rÔ   ©r"   rÖ   )r#   r$   s   "€r   r%   r×   @  s   ø€ ÷ ñ ¡cð ©cñ r   c                ó`   €  V P                   P                  V.4      #   \         d     R # i ; i)r0   )rG   rH   Ú	Exception©r   r)   s   &&r   Ú_fallback_decodeÚ%_ServerTokenStreamer._fallback_decode@  s0   € ð	Ø—?‘?×)Ñ)¨5¨'Ó2Ð2øÜô 	Úð	ús   ‚ ž-¬-c                ó&   <€ V ^8„  d   QhRS[ RS[/# rÜ   rÝ   )r#   r$   s   "€r   r%   r×   F  s   ø€ ÷ ,ñ ,©ð ,±ñ ,r   c                ón  €  V P                   V,          '       d;   V P                  V,          pV P                  P                  \	        V34      R R7      # V P
                  V,          pTf   T P                  T4      # TP                  RR4      #   \        \        3 d    T P                  T4      u # i ; i)F©Úfinalr‡   r
   )
rÍ   rÎ   rÇ   rH   ÚbytesrÉ   Ú
IndexErrorÚ	TypeErrorrá   r}   )r   r)   ro   rs   s   &&  r   Ú_spm_text_for_tokenÚ(_ServerTokenStreamer._spm_text_for_tokenF  s§   € ð	0Ø×"Ñ" 5×)Ô)Ø!×-Ñ-¨eÕ4�
Ø—}‘}×+Ñ+¬E°:°-Ó,@ÈÐ+ÓNÐNØ—N‘N 5Õ)ˆEð Š=Ø×(Ñ(¨Ó/Ð/Ø�}‰}˜X sÓ+Ð+øô œIÐ&ô 	0Ø×(Ñ(¨Ó/Ò/ð	0ús   ‚B ›9B ÁB Â"B4Â3B4c                ó&   <€ V ^8„  d   QhRS[ RS[/# rÜ   rÝ   )r#   r$   s   "€r   r%   r×   S  s   ø€ ÷ >ñ >©ð >±ñ >r   c                óP  a €  S P                   V,          pTf   S P                  T4      #  \	        T 3R lT 4       4      pS P                  P                  TRR7      #   \        \        3 d    S P                  T4      u # i ; i  \
         d    S P                  T4      u # i ; i)Nc              3   óJ   <"  € T F  pSP                   V,          x € K  	  R # 5ir   r—   r™   s   & €r   rœ   Ú;_ServerTokenStreamer._bpe_text_for_token.<locals>.<genexpr>]  s   øé € ÐE¹u¸! × 2Ñ 2°1× 5Ò 5»uùrŸ   Frå   )rÉ   rè   ré   rá   rç   r¦   rÇ   rH   )r   r)   rs   Útoken_bytess   f&  r   Ú_bpe_text_for_tokenÚ(_ServerTokenStreamer._bpe_text_for_tokenS  s§   ø€ ð	0Ø—N‘N 5Õ)ˆEð Š=Ø×(Ñ(¨Ó/Ð/ð	0ÜÔE¹uÓEÓEˆKð �}‰}×#Ñ# K°uÐ#Ó=Ð=øô œIÐ&ô 	0Ø×(Ñ(¨Ó/Ò/ð	0ûô ô 	0Ø×(Ñ(¨Ó/Ò/ð	0ús"   ƒA ¬B Á"BÂBÂB%Â$B%c                ó&   <€ V ^8„  d   QhRS[ RS[/# rÜ   rÝ   )r#   r$   s   "€r   r%   r×   b  s   ø€ ÷ Añ A¡3ð A©3ñ Ar   c                óì   € V P                   R 8X  d!   V P                  V P                  V4      4      # V P                   R8X  d!   V P                  V P                  V4      4      # V P	                  VRR7      # )rÄ   rÅ   N)Úfinish_reason)rÊ   rÙ   rê   rñ   Ú_fallback_advancerà   s   &&r   Ú_decode_tokenÚ"_ServerTokenStreamer._decode_tokenb  sd   € Ø�:‰:˜ÔØ—?‘? 4×#;Ñ#;¸EÓ#BÓCÐCØ�:‰:˜ÔØ—?‘? 4×#;Ñ#;¸EÓ#BÓCÐCØ×%Ñ% e¸4Ð%Ó@Ð@r   c                ó<   <€ V ^8„  d   QhRS[ RS[S[,          RS[/# ©r   r)   rõ   rÔ   ©r"   r   rÖ   )r#   r$   s   "€r   r%   r×   i  s&   ø€ ÷ .ñ .¡sð .¹8ÁC½=ð .ÉSñ .r   c                ó  € VR 8X  d1   V P                   P                  4        V P                   P                  # V P                   P                  V4       Ve   V P                   P                  4        V P                   P                  # )r¼   )rÆ   r-   r*   r7   )r   r)   rõ   s   &&&r   rö   Ú&_ServerTokenStreamer._fallback_advancei  sp   € Ø˜FÔ"Ø×Ñ×&Ñ&Ô(ð
 × Ñ ×-Ñ-Ð-ð ×Ñ×'Ñ'¨Ô.ØÒ(Ø×!Ñ!×*Ñ*Ô,Ø× Ñ ×-Ñ-Ð-r   c                ó<   <€ V ^8„  d   QhRS[ RS[S[,          RS[/# rú   rû   )r#   r$   s   "€r   r%   r×   r  s&   ø€ ÷ 	ñ 	™Sð 	±¹#µð 	Á3ñ 	r   c           	     ó8  € V P                   R 8X  d   V P                  W4      # . pVR8w  d!   VP                  V P                  V4      4       Ve<   VP                  V P	                  V P
                  P                  RRR7      4      4       RP                  V4      # )rÃ   r¼   r   Trå   r0   )rÊ   rö   rZ   r÷   rÙ   rÇ   rH   Újoin)r   r)   rõ   Úpartss   &&& r   ÚadvanceÚ_ServerTokenStreamer.advancer  s€   € Ø�:‰:˜Ô#Ø×)Ñ)¨%Ó?Ð?àˆØ˜FÔ"Ø�L‰L˜×+Ñ+¨EÓ2Ô3ØÒ$Ø�L‰L˜Ÿ™¨¯©×)=Ñ)=¸cÈÐ)=Ó)NÓOÔPØ�w‰w�u‹~Ðr   c                ó    <€ V ^8„  d   QhRS[ /# )r   rÔ   rÕ   )r#   r$   s   "€r   r%   r×   }  s   ø€ ÷ Fñ F™#ñ Fr   c                óÜ   € V P                   R 8X  d1   V P                  P                  4        V P                  P                  # V P	                  V P
                  P                  RRR7      4      # )rÃ   r   Trå   )rÊ   rÆ   r-   r7   rÙ   rÇ   rH   r   s   &r   r-   Ú_ServerTokenStreamer.finalize}  sU   € Ø�:‰:˜Ô#Ø×Ñ×&Ñ&Ô(Ø×$Ñ$×1Ñ1Ð1Ø�‰˜tŸ}™}×3Ñ3°C¸tÐ3ÓDÓEÐEr   )
r˜   rÎ   rÇ   rÆ   rÍ   rÊ   rÈ   rG   rÉ   rË   N)r:   r;   r<   r=   r>   rJ   rÙ   rá   rê   rñ   r÷   rö   r  r-   rA   rB   rC   s   @r   rÁ   rÁ     sb   ø‡ € ÙMòM÷.ð ÷ð ÷,ð ,÷>ð >÷Að A÷.ð .÷	ð 	÷Fö Fr   rÁ   c                   ó6   a € ] tR tRt o Rt]3R ltR tRtV t	R# )ÚTokenizerWrapperi„  zžA wrapper that combines an HF tokenizer and a detokenizer.

Accessing any attribute other than the ``detokenizer`` is forwarded to the
huggingface tokenizer.
c                ó,   € Wn         V! V4      V n        R # r   )rG   rÆ   )r   rI   Údetokenizer_classs   &&&r   rJ   ÚTokenizerWrapper.__init__‹  s   € Ø#ŒÙ-¨iÓ8ˆÖr   c                óT   € VR 8X  d   V P                   # \        V P                  V4      # )rÑ   )rÆ   ÚgetattrrG   )r   Úattrs   &&r   Ú__getattr__ÚTokenizerWrapper.__getattr__�  s'   € Ø�=Ô Ø×$Ñ$Ð$ä˜4Ÿ?™?¨DÓ1Ð1r   )rÆ   rG   N)
r:   r;   r<   r=   r>   rE   rJ   r  rA   rB   rC   s   @r   r  r  „  s   ø‡ € ñð 5Nô 9÷2ð 2r   r  c                óP   € \        V P                  4      pVP                  4        V# )z@Return an isolated, reset streaming detokenizer for a processor.)r   rÑ   r   )Ú	processorrÑ   s   & r   Úmake_streaming_detokenizerr  –  s$   € ä�y×,Ñ,Ó-€KØ×ÑÔØÐr   c                 ó"  a a€ \        S 4      \        S4      8w  d   R # \        S \        4      '       d\   \        S 4      \        S4      8H  ;'       d=    \        ;QJ d     V V3R lS  4       F  '       d   K   R # 	  R# ! V V3R lS  4       4      # \        S \
        4      '       dh   \        S 4      \        S4      8H  ;'       dI    \        ;QJ d&    R \        S S4       4       F  '       d   K   R # 	  R# ! R \        S S4       4       4      # S S8H  # )Fc              3   óp   <"  € T F+  qS9   ;'       d    \        SV,          SV,          4      x € K-  	  R # 5ir   ©Ú_match)rš   ÚkÚar½   s   & €€r   rœ   Ú_match.<locals>.<genexpr>¡  s-   øé € Ð'QÉqÈ!¨Q©×(EÐ(E´6¸!¸A½$ÀÀ!ÅÓ3EÔ(EËqùs   ƒ6“#6Tc              3   ó<   "  € T F  w  r\        W4      x € K  	  R # 5ir   r  )rš   ÚaiÚbis   &  r   rœ   r  £  s   é € Ð'OÁY¹6¸2¬¨r¯¨ÃYùs   ‚)rM   rÌ   Údictr5   ÚallÚlistr³   )r  r½   s   ffr   r  r  �  sÁ   ù€ ÜˆAƒw”$�q“'ÔÙÜ�!”T×ÒÜ�1‹vœ˜Q›Ñ×QÐQ§C£CÕ'QÉqÓ'Q§C¤CÐQ¢CÐQ CÕ'QÉqÓ'QÓ$QÐQÜ�!”T×ÒÜ�1‹vœ˜Q›Ñ×OÐO§C£CÑ'OÄSÈÈAÄYÓ'O§C¤CÐO¢CÐO CÑ'OÄSÈÈAÄYÓ'OÓ$OÐOà�‰6€Mr   c                 óT   € R RRR RRRR/RR/R R	/R R
/R RRRR^R^ /./p\        W4      # )rM   ÚSequenceÚdecodersÚReplaceÚpatternÚStringr‡   Úcontentr
   ÚByteFallbackÚFuseÚStripr»   r¼   r  ©ÚdecoderÚ_target_descriptions   & r   Ú_is_spm_decoderr.  ¨  s^   € à�
ØØ�Y 	¨H°eÐ+<¸iÈÐMØ�^Ð$Ø�VÐØ�W˜i¨¨g°q¸&À!ÐDð	
ðÐô Ð%Ó/Ð/r   c           	      óB   € R RRR RRRR/RR/R R	/R R
/./p\        W4      # )rM   r"  r#  r$  r%  r&  r‡   r'  r
   r(  r)  r  r+  s   & r   Ú_is_spm_decoder_no_spacer0  µ  sJ   € à�
ØØ�Y 	¨H°eÐ+<¸iÈÐMØ�^Ð$Ø�VÐð
ðÐô Ð%Ó/Ð/r   c                 ó\   € \        V \        4      ;'       d    V P                  R R4      R8H  # )rM   NÚ	ByteLevel)rÌ   r  Úget)r,  s   &r   Ú_is_bpe_decoderr4  Á  s'   € Ü�gœtÓ$×QÐQ¨¯©°V¸TÓ)BÀkÑ)QÐQr   Tc                óv  € \         pV R,          pVP                  4       '       d¤   \        VR4      ;_uu_ 4       p \        P                  ! V4      p RRR4       RX9   dh   \        TR,          4      '       d   \        pMH\        TR,          4      '       d   \        \        RR7      pM\        TR,          4      '       d   \        pV'       d"   \        \        P                   ! V 3/ VB V4      # V#   \
         d'   p\        RTP                  TP                  4      hRp?ii ; i  + '       g   i     LÝ; i)z½Load a huggingface tokenizer and try to infer the type of streaming
detokenizer to use.

Note, to use a fast streaming tokenizer, pass a local file path rather than
a Hugging Face repo ID.
ztokenizer.jsonÚrzFailed to parse tokenizer.jsonNr,  F)rk   )rE   ÚexistsÚopenÚjsonÚloadr   ÚdocÚposr.  rf   r0  r   r4  r�   r  r   Úfrom_pretrained)Ú
model_pathÚreturn_tokenizerÚtokenizer_config_extrar
  Útokenizer_fileÚfÚtokenizer_contentÚes   &&&     r   Úload_tokenizerrE  Å  s	  € ô 2ÐàÐ"2Õ2€NØ×Ñ×ÒÜ�. #×&Ô&¨!ðVÜ$(§I¢I¨a£LÑ!÷ 'ð Ð)Ô)ÜÐ0°Õ;×<Ò<Ü$;Ñ!Ü)Ð*;¸IÕ*F×GÒGÜ$+Ô,CÐPUÔ$VÑ!Ü Ð!2°9Õ!=×>Ò>Ü$;Ð!çÜÜ×)Ò)¨*ÑOÐ8NÑOØó
ð 	
ð
 !Ð øô# #ô VÜ%Ð&FÈÏÉÈqÏuÉuÓUÐUûðVú÷ '×&ús(   ºD(¼C4Ã4D%Ã?!D Ä D%Ä%D(Ä(D8	)rÏ   r9  r   Ú	functoolsr   r   Útypingr   r   Útransformersr   r3   r   r   rE   rf   r�   rÁ   r  r  r  r.  r0  r4  rE  r   r   r   Ú<module>rI     s�   ðÛ Û Ý Ý Ý  ß !å &àÐ ò÷1ñ 1ôh/Ð 4ô /ôdLÐ2ô Lô^T*Ð2ô T*÷nbFñ bF÷J2ñ 2ò$òò
0ò	0òRð 15ÈRö !r   