+
    QV-j ƒ  ã            "       ó¸  € R t ^ RIHt ^ RIt^ RIHtHtHtHtH	t	 ^ RI
HtHt ^RIHt ^RIHtHtHtHtHt ^RIHt ^RIHt ]P2                  ! ]4      t/ R	R
RRR/bRRRRRRRRRRRRRRRRRRR R!R!/
bR"RRRRRRRRRRRRRRRRRR R!R!/
bR#RRRRRRRRRRRRRRRRRR R!R!/
bR$RRRRRRRRRRRRRRRRRR R!R!R%R&R'R(/bR)RRRRRRRRRRRRRRRRRR R!R!R%R*R'R(R+R+/bR,RRRRRRRRRRRRRRRRRR R!R!R-R./bR/RRRRRRRRRRRRRRRRRR R!R!/
bR0RRRRRRRRRRRRR1RRRRRRR R!R!R%R&R'R(/bR2RRRRRRRRRRRRRRRRRR R!R!/
bR3R4R5R6R7R8R9R:R;/bR<RRRRRRRRRRRRRRRRRR R!R!/
bR=RR>RRRR?R!R!R@RA/bRBRRCRRDRRERRFR1RGRRHRRR@RARIRJRKRKR!R!/bRLRRRRRRRRRRRRRRR@RMR!R!/	bRNRR>RRORRPRRRR?R@RA/bRQRRRRRRRRRRRRR@RR/bRSR!R!RRRRRRARRRTRURVRWRXRYRZR/	R[RRRRRRRRRRRRRRRRRR\R!R!/
R]RRRRRRRRRRRRR1RRRRRRR R^R+R!R!/R_RRRRRRRRRRRRR1RRRRRRR R^R+R!R!/R`RRRRRRRRRRRRR1RaRRRRRR R^R+R!R!/RbRRCRRDRRERRFR1RGRRHRRR@RARIRJRKRKR!R!/RcRRRRRRRRRRRRRRRRRR R!R!/
Rd/ RRbRRbRRbRRbRRebRRbRRbRRbR1RbRfRbRR bR%R*bR'R(bRgRbR!R!bRhRib/CtR3RjRkRlRmRnRoRpRqRrRsR4R5R6R7R8R9R:R;RtRu/
RvRwRwRjRR4R5R6R7R8R9R:R;//tR0RxRy/RdRzRy//tR{ t ! R| R}4      t  ! R~ R]4      t! ! R€ R�]4      t" ! R‚ Rƒ]4      t# ! R„ R…]4      t$ ! R† R‡]4      t% ! Rˆ R‰]4      t&/ R]!bR#]"bR$]"bR/]"bR0]"bR<]#bR=]$bR2]$bRL]$bRN]$bRQ]$bRB]%bRS]$bR[]$bR]]&bRŠ]&bR‹]&bRb]%Rc]!RŒ]!Rd]"/Ct'R� RŽ lt(R# )�z
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
)ÚarrayN)Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigram)Ú
AddedToken)ÚGemmaConverterÚGPT2ConverterÚLlamaConverterÚQwen2ConverterÚT5Converter)Úlogging)ÚtqdmÚgeneralÚarchitectureÚ
model_typeÚnameÚ_model_name_or_pathÚllamaÚcontext_lengthÚmax_position_embeddingsÚblock_countÚnum_hidden_layersÚfeed_forward_lengthÚintermediate_sizeÚembedding_lengthÚhidden_sizezrope.dimension_countÚhead_dimzrope.freq_baseÚ
rope_thetazattention.head_countÚnum_attention_headszattention.head_count_kvÚnum_key_value_headsz attention.layer_norm_rms_epsilonÚrms_norm_epsÚ
vocab_sizeÚmistralÚqwen2Ú	qwen2_moeÚexpert_countÚnum_expertsÚexpert_used_countÚnum_experts_per_tokÚgpt_ossÚnum_local_expertsÚsliding_windowÚlfm2zshortconv.l_cacheÚconv_L_cacheÚqwen3Ú	qwen3_moezattention.key_lengthÚfalconÚ	tokenizerzggml.bos_token_idÚbos_token_idzggml.eos_token_idÚeos_token_idzggml.unknown_token_idÚunk_token_idzggml.padding_token_idÚpad_token_idÚphi3ÚbloomÚn_layerÚn_headzattention.layer_norm_epsilonÚlayer_norm_epsilonÚt5Ún_positionsÚ
num_layersÚd_ffÚd_modelÚd_kvÚ	num_headsz attention.relative_buckets_countÚrelative_attention_num_bucketsÚdecoder_start_token_idÚstablelmÚlayer_norm_epsÚgpt2Ún_ctxÚn_embdÚ
starcoder2Únorm_epsilonÚmambazssm.conv_kernelÚconv_kernelzssm.state_sizeÚ
state_sizezssm.time_step_rankÚtime_step_rankzssm.inner_sizeÚnemotronÚnorm_epsÚgemma2zattention.sliding_windowÚgemma3Úgemma4Úglobal_head_dimÚumt5ÚdeciÚ
minimax_m2Ú
rotary_dimzattention.value_lengthÚexpert_feed_forward_lengthÚexpert_gating_funcÚscoring_funcz
ggml.modelÚtokenizer_typezggml.tokensÚtokenszggml.scoresÚscoreszggml.token_typeÚ
token_typezggml.mergesÚmergeszggml.add_space_prefixÚadd_prefix_spaceÚtokenizer_configÚchat_templateÚnorm_topk_probTÚuse_routing_biasc                 óä  € \        V\        4      '       g   V.p\        V4      ^8X  d   V^ ,          pRpMV^ ,          ^	8w  d   \        R4      hVw  rVR9   d   \	        V ^ ,          4      p V # VR9   d   \        V ^ ,          4      p V # V^8X  d   \        V ^ ,          4      p V # V^8X  d4   \        R\        V 4      4      P                  4       P                  4       p V # V^	8X  d   \        W4      p V # )é   NzPReceived multiple types, therefore expected the first type to indicate an array.ÚB)é    rk   é   é   é   é   é
   é   )é   é   )Ú
isinstanceÚlistÚlenÚ
ValueErrorÚintÚfloatÚboolr   ÚtobytesÚdecodeÚ_gguf_parse_value)Ú_valueÚ	data_typeÚarray_data_types   && Úo/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/transformers/integrations/ggml.pyr   r   ~  só   € Ü�i¤×&Ò&Ø�Kˆ	Ü
ˆ9ƒ~˜ÔØ˜a•Lˆ	Ø‰à�Q�<˜1ÔÜÐoÓpÐpØ%.Ñ"ˆ	àÐ.Ô.Ü�V˜A•Y“ˆð €Mð 
�gÔ	Ü�v˜a•yÓ!ˆð €Mð 
�aŒÜ�f˜Q•i“ˆð
 €Mð	 
�aŒÜ�sœD ›LÓ)×1Ñ1Ó3×:Ñ:Ó<ˆð €Mð 
�aŒÜ" 6Ó;ˆØ€Mó    c                   ó&   a € ] tR tRt o R tRtV tR# )ÚGGUFTokenizerSkeletoni–  c                óü  a€ VP                  4        F  w  r#\        WV4       K  	  \        V R 4      '       EgI   \        V R4      '       d   \        V R4      '       g   \        R4      hV P                  pV P
                  p\        V4       UUu/ uF  w  rgWuV,          bK  	  uppo\        P                  R4       . p\        SP                  4       4       Fq  w  rš. p\        ^\        V	4      4       F.  pV	RV WœR ríWÔ9   g   K  Wä9   g   K  VP                  WÞV
34       K0  	  \        VV3R lRR7      pVP                  V4       Ks  	  \        VR	 RR7      pV Uu. uF  qÿ^ ,          V^,          3NK  	  ppW€n        M~V P                   U	u. uF  p	\!        V	P#                  R
4      4      NK  	  up	V n        \        V R4      '       g4   \        \        V P                  4      4       Uu. uF  pRNK  	  upV n        \        V R4      '       g   . V n        \        V R4      '       g   RV n        \        V R4      '       d$   V P&                  f   V P(                  V n        R# R# R# u uppi u upi u up	i u upi )rd   ra   rb   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.z:Merges were not in checkpoint, building merges on the fly.Nc                 óD   <€ SV ^ ,          ,          SV ^,          ,          3# )rm   © )ÚxÚvocabs   &€rƒ   Ú<lambda>Ú0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>¬  s   ø€ °U¸1¸Q½4µ[À%ÈÈ!ÍÅ+Ñ4Nr„   T)ÚkeyÚreversec                 ó   € V ^,          # )rn   r‰   )Úvals   &rƒ   rŒ   r�   ®  s   € °C¸¶Fr„   Ú Úadded_tokensr8   Úunknown_token_id)ÚitemsÚsetattrÚhasattrry   ra   rb   Ú	enumerateÚloggerÚwarningr   Úrangerx   ÚappendÚsortedÚextendrd   ÚtupleÚsplitr“   r8   r”   )ÚselfÚdict_ÚkÚvra   rb   ÚiÚtrd   ÚmergeÚpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rr‘   Ú_r‹   s   &&               @rƒ   Ú__init__ÚGGUFTokenizerSkeleton.__init__—  s  ø€ Ø—K‘K–M‰DˆAÜ�D˜QÖñ "ô �t˜X×&Ó&Ü˜4 ×*Ò*´'¸$À×2IÒ2IÜ Øróð ð —[‘[ˆFØ—[‘[ˆFÜ.7¸Ô.?Ô@Ñ.?¡d a�Q˜q�	’\Ñ.?Ò@ˆEä�N‰NÐWÔXØˆFÜ&*¨5¯;©;«=Ö&9Ñ"�Ø�Ü" 1¤c¨%£jÖ1�EØ',¨V¨e }°e¸F°m˜WØÖ(¨WÖ->ØŸ™ g¸Ð%DÖEñ 2ô ˜uÔ*NÐX\Ô]�Ø—‘˜eÖ$ñ ':ô ˜FÑ(:ÀDÔIˆFÙ28Ó9±&¨3˜1•v˜s 1�vÓ&±&ˆFÐ9Ø �Kà@DÇÂÓLÁ°uœ5 §¡¨SÓ!1Ö2ÁÑLˆDŒKÜ˜4 ×*Ò*Ü-2´3°t·{±{Ó3CÔ-DÓEÑ-D¨›tÑ-DÑE�”ä�t˜^×,Ò,Ø "ˆDÔä�t˜^×,Ò,Ø $ˆDÔô �4Ð+×,Ò,°×1BÑ1BÒ1JØ $× 5Ñ 5ˆDÖñ 2KÑ,ùó7 Aùò :ùò MùâEs   ÂI)ÅI/Æ#I4Ç-I9)r“   rd   rb   r8   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r®   Ú__static_attributes__Ú__classdictcell__©Ú__classdict__s   @rƒ   r†   r†   –  s   ø‡ € ÷'6ð '6r„   r†   c                   óD   a € ] tR tRt o R tR tR tR tR tR t	Rt
V tR	# )
ÚGGUFLlamaConverteriÁ  c                ó”   € \        V4      V n        V P                  V n        / V n        \	        V P                  R R4      R8g  V n        R# )r`   r   N)r†   ÚprotoÚoriginal_tokenizerÚadditional_kwargsÚgetattrÚis_llama_3_tokenizer©r¡   Útokenizer_dicts   &&rƒ   r®   ÚGGUFLlamaConverter.__init__Â  s>   € Ü*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÔÜ$+¨D¯J©JÐ8HÈ'Ó$RÐV]Ñ$]ˆÖ!r„   c                óT   € \        \        VP                  VP                  4      4      # ©N©rw   Úzipra   rb   ©r¡   r»   s   &&rƒ   r‹   ÚGGUFLlamaConverter.vocabÈ  ó   € Ü”C˜Ÿ™ e§l¡lÓ3Ó4Ð4r„   c                ó   € VP                   # rÄ   ©rd   rÇ   s   &&rƒ   rd   ÚGGUFLlamaConverter.mergesË  ó   € Ø�|‰|Ðr„   c                ó*  € V P                  V P                  4      pV P                  V P                  4      p\        V4       UUUu/ uF
  w  pw  rVWTbK  	  ppppVP                  e   VP
                  VP                  ,          MR p\        VRR 4      e   VP
                  VP                  ,          MR p	\        VRR 4      e   VP
                  VP                  ,          MR p
\        \        VVVRRR7      4      p. p\        V P                  R4      '       ge   Ve   VP                  \        VRRR7      4       V	e   VP                  \        V	RRR7      4       V
e   VP                  \        V
RRR7      4       M‰\        P                  ! \        P                  ! V P                  P                   4      ^8H  4      ^ ,          pV F;  pVP                  \        V P                  P
                  V,          RRR7      4       K=  	  \#        V4      ^ 8w  d   VP%                  V4       \#        V P                  P&                  4      ^ 8w  dB   TP)                  V P                  P&                   Uu. uF  p\        VRRR7      NK  	  up4       W€P*                  R&   W�P*                  R	&   W P*                  R
&   V P,                  '       d?   R V P*                  R&   RV P*                  R&   RV P*                  R&   RV P.                  n        V# u upppi u upi )Nr6   r7   T)Ú	unk_tokenÚfuse_unkÚbyte_fallbackrc   F©Ú
normalizedÚspecialrÏ   Ú	eos_tokenÚ	bos_tokenre   Úclean_up_tokenization_spacesÚlegacy)r‹   r»   rd   r˜   r8   ra   r¾   r6   r   r   r—   rœ   r
   ÚnpÚwherer   rc   rx   Úadd_special_tokensr“   Ú
add_tokensr½   r¿   r¼   rØ   )r¡   r»   Úvocab_scoresrd   r¥   ÚwordÚ_scoreÚ	bpe_vocabrÏ   rÖ   rÕ   r5   Úspecial_tokensÚspecial_tokens_idxÚidxÚadded_tokens   &&              rƒ   r5   ÚGGUFLlamaConverter.tokenizerÎ  s¤  € Ø—z‘z $§*¡*Ó-ˆØ—‘˜TŸZ™ZÓ(ˆÜ6?ÀÔ6MÕNÑ6MÑ!2 ¡N T�T’WÑ6Mˆ	ÒNà8=×8JÑ8JÒ8V�E—L‘L ×!3Ñ!3Ö4Ð\`ˆ	Ü8?ÀÀ~ÐW[Ó8\Ò8h�E—L‘L ×!3Ñ!3Ö4Ðnrˆ	Ü8?ÀÀ~ÐW[Ó8\Ò8h�E—L‘L ×!3Ñ!3Ö4Ðnrˆ	äÜØØØ#ØØ"ôó
ˆ	ð ˆä�t—z‘z <×0Ò0ØÒ$Ø×%Ñ%¤j°ÀuÐVZÔ&[Ô\àÒ$Ø×%Ñ%¤j°ÀuÐVZÔ&[Ô\àÒ$Ø×%Ñ%¤j°ÀuÐVZÔ&[Ô\øô "$§¢¬"¯(ª(°4·:±:×3HÑ3HÓ*IÈQÑ*NÓ!OÐPQÕ!RÐã)�Ø×%Ñ%¤j°·±×1BÑ1BÀ3Õ1GÐTYÐcgÔ&hÖiñ *ô ˆ~Ó !Ô#Ø×(Ñ(¨Ô8äˆt�z‰z×&Ñ&Ó'¨1Ô,Ø× Ñ Ø]a×]gÑ]g×]tÒ]tÓuÑ]tÈk”˜K°EÀ5×IÑ]tÑuôð /8×Ñ˜{Ñ+Ø.7×Ñ˜{Ñ+Ø.7×Ñ˜{Ñ+à×$×$Ð$Ø9=ˆD×"Ñ"Ð#5Ñ6ØEIˆD×"Ñ"Ð#AÑBà/4ˆD×"Ñ" 8Ñ,Ø-2ˆD×#Ñ#Ô*àÐùôk OùòN vs   ÁL	É/Lc                ód  € \         P                  ! 4       \         P                  ! 4       \         P                  ! R R4      .pV P                  '       d"   V\         P
                  ! RRRR7      .,          pV'       d!   V\         P                  ! R^R7      .,          p\         P                  ! V4      # )õ   â–�r’   FT©re   Útrim_offsetsÚ	use_regex©ÚcontentÚleft)r   ÚByteFallbackÚFuseÚReplacer¿   Ú	ByteLevelÚStripÚSequence©r¡   Úreplacementre   Úsequences   &&& rƒ   ÚdecoderÚGGUFLlamaConverter.decoder  s†   € ä×!Ò!Ó#Ü�MŠM‹OÜ×Ò˜U CÓ(ð
ˆð ×$×$Ð$Øœ×+Ò+¸UÐQVÐbfÔgÐhÕhˆHçØœŸš°¸!Ô<Ð=Õ=ˆHÜ× Ò  Ó*Ð*r„   c                ó&  € V P                  V P                  4      pV P                  V P                  4      pVe   W!n        RpRp\        V P                  R4      '       d   V P                  P
                  pV P                  W44      pVe   WQn        V P                  W44      Vn        V P                  4       pV'       d   Wan        V P                  '       d:   \        P                  ! RRRR7      Vn        \        P                  ! . 4      Vn        V# )Nrç   Tre   Frè   )r5   r»   Ú
normalizerr—   r¼   re   Úpre_tokenizerr÷   Úpost_processorr¿   r   rñ   r   ró   )r¡   r5   rú   rõ   re   rû   rü   s   &      rƒ   Ú	convertedÚGGUFLlamaConverter.converted  sì   € à—N‘N 4§:¡:Ó.ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÒ!Ø#-Ô àˆØÐÜ�4×*Ñ*Ð,>×?Ò?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;ÓIˆØÒ$Ø&3Ô#à ŸL™L¨ÓGˆ	ÔØ×,Ñ,Ó.ˆßØ'5Ô$ð ×$×$Ð$Ü&4×&>Ò&>Ø!&°UÀdô'ˆIÔ#ô
 $/×#7Ò#7¸Ó#;ˆIÔ àÐr„   )r½   r¿   r¼   r»   N©r°   r±   r²   r³   r®   r‹   rd   r5   r÷   rý   r´   rµ   r¶   s   @rƒ   r¹   r¹   Á  s*   ø‡ € ò^ò5òò8òt+÷!ð !r„   r¹   c                   óD   a a€ ] tR tRt oR tV3R lV 3R lltRtVtV ;t# )ÚGGUFQwen2Converteri:  c                ó4   € \        V4      V n        / V n        R # rÄ   ©r†   r¼   r½   rÀ   s   &&rƒ   r®   ÚGGUFQwen2Converter.__init__;  ó   € Ü"7¸Ó"GˆÔØ!#ˆÖr„   c                ó    <€ V ^8„  d   QhRS[ /# ©rn   Úreturn©r   )Úformatr·   s   "€rƒ   Ú__annotate__ÚGGUFQwen2Converter.__annotate__?  s   ø€ ÷ ñ ™9ñ r„   c           
     ó2  <€ \        V P                  P                  4       UUu/ uF  w  rW!bK	  	  pppV P                  P                  p\        SV `  W44      pVP                  \        R RRR7      \        RRRR7      \        RRRR7      .4       V# u uppi )ú<|endoftext|>FTrÒ   z<|im_start|>z
<|im_end|>)r˜   r¼   ra   rd   Úsuperrý   rÛ   r
   ©r¡   r¥   rÞ   r‹   rd   r5   Ú	__class__s   &     €rƒ   rý   ÚGGUFQwen2Converter.converted?  s“   ø€ Ü(1°$×2IÑ2I×2PÑ2PÔ(QÔRÑ(Q™W˜Q�’Ñ(QˆÑRØ×(Ñ(×/Ñ/ˆÜ‘GÑ% eÓ4ˆ	à×$Ñ$ä˜?°uÀdÔKÜ˜>°eÀTÔJÜ˜<°EÀ4ÔHðô	
ð Ðùó Ss   ¤B©r½   r¼   ©	r°   r±   r²   r³   r®   rý   r´   rµ   Ú__classcell__©r  r·   s   @@rƒ   r  r  :  s   ù‡ € ò$÷÷ ó r„   r  c                   óP   a € ] tR tRt o R tR tR tR tR tV 3R lR lt	R	t
V tR
# )ÚGGUFPhi3ConverteriN  c                óV   € \        V4      V n        V P                  V n        / V n        R # rÄ   ©r†   r»   r¼   r½   rÀ   s   &&rƒ   r®   ÚGGUFPhi3Converter.__init__O  s"   € Ü*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÖr„   c                óT   € \        \        VP                  VP                  4      4      # rÄ   rÅ   rÇ   s   &&rƒ   r‹   ÚGGUFPhi3Converter.vocabT  rÉ   r„   c                ó   € VP                   # rÄ   rË   rÇ   s   &&rƒ   rd   ÚGGUFPhi3Converter.mergesW  rÍ   r„   c                ó0  € V P                  V P                  4      pV P                  V P                  4      p\        V4       UUUu/ uF
  w  pw  rVWTbK  	  pppp\	        \        Ws4      4      pVP                  \        R RRRRR7      \        RRRR7      \        RRRRR7      \        RRRRR7      \        R	RRRR7      \        R
RRRR7      \        RRRRR7      \        RRRRR7      \        RRRRR7      \        RRRRR7      \        RRRRR7      \        RRRRR7      .4       VP                  e   VP                  VP                  ,          MRV P                  R&   VP                  e   VP                  VP                  ,          MRV P                  R&   VP                  e   VP                  VP                  ,          MRV P                  R&   VP                  e   VP                  VP                  ,          MRV P                  R&   V# u upppi )ú</s>TF)ÚrstripÚlstriprÓ   rÔ   r  rÒ   z<|assistant|>)r"  rÓ   rÔ   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>NrÏ   rÕ   rÖ   Ú	pad_token)r‹   r»   rd   r˜   r   r   rÛ   r
   r8   ra   r½   r7   r6   r9   )	r¡   r»   rÝ   rd   r¥   rÞ   rß   rà   r5   s	   &&       rƒ   r5   ÚGGUFPhi3Converter.tokenizerZ  sú  € Ø—z‘z $§*¡*Ó-ˆØ—‘˜TŸZ™ZÓ(ˆÜ6?ÀÔ6MÕNÑ6MÑ!2 ¡N T�T’WÑ6Mˆ	ÒNäœc )Ó4Ó5ˆ	à×$Ñ$ä˜6¨$°uÈÐX\Ô]Ü˜?°uÀdÔKÜ˜?°4ÀEÐSWÔXÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[Ü˜<°ÀÐPTÔUÜ˜9¨T¸eÈTÔRÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[Ü˜:¨d¸uÈdÔSðô	
ð$ 16×0BÑ0BÒ0NˆE�L‰L˜×+Ñ+Ö,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÒ0NˆE�L‰L˜×+Ñ+Ö,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÒ0NˆE�L‰L˜×+Ñ+Ö,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÒ0NˆE�L‰L˜×+Ñ+Ö,ÐTXð 	×Ñ˜{Ñ+ð ÐùôE Os   ÁHc                óþ   € \         P                  ! 4       \         P                  ! 4       \         P                  ! VR 4      .pV'       d!   V\         P                  ! R ^R7      .,          p\         P
                  ! V4      # )r’   rë   )r   rî   rï   rð   rò   ró   rô   s   &&& rƒ   r÷   ÚGGUFPhi3Converter.decoder�  s\   € ä×!Ò!Ó#Ü�MŠM‹OÜ×Ò˜[¨#Ó.ð
ˆ÷ ØœŸš°¸!Ô<Ð=Õ=ˆHÜ× Ò  Ó*Ð*r„   c                ó    <€ V ^8„  d   QhRS[ /# r  r	  )r
  r·   s   "€rƒ   r  ÚGGUFPhi3Converter.__annotate__Œ  s   ø€ ÷ 
ñ 
™9ñ 
r„   c                óÔ   € V P                  V P                  4      pR pRp\        V P                  R4      '       d   V P                  P                  pV P                  W#4      Vn        V# )rç   Tre   )r5   r»   r—   r¼   re   r÷   )r¡   r5   rõ   re   s   &   rƒ   rý   ÚGGUFPhi3Converter.convertedŒ  s\   € Ø—N‘N 4§:¡:Ó.ˆ	àˆØÐÜ�4×*Ñ*Ð,>×?Ò?Ø#×6Ñ6×GÑGÐà ŸL™L¨ÓGˆ	ÔàÐr„   ©r½   r¼   r»   Nrÿ   r¶   s   @rƒ   r  r  N  s)   ø‡ € ò$ò
5òò%òN	+÷
ö 
r„   r  c                   óD   a a€ ] tR tRt oR tV3R lV 3R lltRtVtV ;t# )ÚGGUFGPTConverteri™  c                ó4   € \        V4      V n        / V n        R # rÄ   r  rÀ   s   &&rƒ   r®   ÚGGUFGPTConverter.__init__š  r  r„   c                ó    <€ V ^8„  d   QhRS[ /# r  r	  )r
  r·   s   "€rƒ   r  ÚGGUFGPTConverter.__annotate__ž  s   ø€ ÷ ñ ™9ñ r„   c                óÂ   <€ \        V P                  P                  4       UUu/ uF  w  rW!bK	  	  pppV P                  P                  p\        SV `  W44      pV# u uppi rÄ   )r˜   r¼   ra   rd   r  rý   r  s   &     €rƒ   rý   ÚGGUFGPTConverter.convertedž  sZ   ø€ Ü(1°$×2IÑ2I×2PÑ2PÔ(QÔRÑ(Q™W˜Q�’Ñ(QˆÑRØ×(Ñ(×/Ñ/ˆÜ‘GÑ% eÓ4ˆ	ØÐùó Ss   ¤Ar  r  r  s   @@rƒ   r.  r.  ™  s   ù‡ € ò$÷÷ ó r„   r.  c                   óJ   a € ] tR tRt o R tR tR tR tV 3R lR ltRt	V t
R	# )
ÚGGUFT5Converteri¥  c                óÞ   € R .VR&   \        V4      V n        \        V P                  P                  4       UUu/ uF  w  r#W2bK	  	  uppV n        V P                  V n        / V n        R# u uppi ©z
dummy textrd   N)r†   r»   r˜   ra   Útoken2idr¼   r½   )r¡   rÁ   r¤   r£   s   &&  rƒ   r®   ÚGGUFT5Converter.__init__¦  s_   € à$0 >ˆ�xÑ ä*¨>Ó:ˆŒ
Ü*3°D·J±J×4EÑ4EÔ*FÔGÑ*F¡$ !˜šÑ*FÒGˆŒØ"&§*¡*ˆÔØ!#ˆÖùó Hs   ¹A)c                óT   € \        \        VP                  VP                  4      4      # rÄ   rÅ   rÇ   s   &&rƒ   r‹   ÚGGUFT5Converter.vocab¯  rÉ   r„   c                ó(  € \        V P                  R R4      '       du   . p\        V P                  RR4      '       d    V\        P                  ! RR7      .,          pV\        P                  ! RRR7      .,          p\        P
                  ! V4      # R# )rØ   Tre   rç   )Úprependr’   )Úpatternrì   N)r¾   r¼   r   ÚPrependrð   ró   )r¡   r»   rö   s   && rƒ   rú   ÚGGUFT5Converter.normalizer²  sx   € Ü�4×*Ñ*¨H°d×;Ò;ØˆHÜ�t×.Ñ.Ð0BÀD×IÒIØœ[×0Ò0¸Ô?Ð@Õ@�Øœ×,Ò,°SÀ%ÔHÐIÕIˆHÜ×'Ò'¨Ó1Ð1Ùr„   c                ód   € \         P                  ! R R.. RORV P                  R,          3.R7      # )ú$Ar!  )ÚsingleÚpairrá   )rC  r!  z$Br!  )r   ÚTemplateProcessingr9  )r¡   s   &rƒ   rü   ÚGGUFT5Converter.post_processor»  s5   € Ü×,Ò,Ø˜&�>Ú-à˜Ÿ™ vÕ.Ð/ðô
ð 	
r„   c                ó    <€ V ^8„  d   QhRS[ /# r  r	  )r
  r·   s   "€rƒ   r  ÚGGUFT5Converter.__annotate__Ä  s   ø€ ÷ ñ ™9ñ r„   c                óæ  € V P                  V P                  4      p\        \        VV P                  P                  R R7      4      pV P                  V P                  4      pVe   W2n        RpRp\        V P                  R4      '       d   V P                  P                  pV P                  WE4      pVe   Wbn	        V P                  WE4      Vn
        V P                  4       pV'       d   Wrn        V# )F©Úunk_idrÑ   rç   Tre   )r‹   r»   r   r	   r8   rú   r—   r¼   re   rû   r÷   rü   )r¡   rÝ   r5   rú   rõ   re   rû   rü   s   &       rƒ   rý   ÚGGUFT5Converter.convertedÄ  sÔ   € Ø—z‘z $§*¡*Ó-ˆÜÜØØ—z‘z×.Ñ.Ø#ôó
ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÒ!Ø#-Ô àˆØÐÜ�4×*Ñ*Ð,>×?Ò?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;ÓIˆØÒ$Ø&3Ô#à ŸL™L¨ÓGˆ	ÔØ×,Ñ,Ó.ˆßØ'5Ô$àÐr„   )r½   r¼   r»   r9  N)r°   r±   r²   r³   r®   r‹   rú   rü   rý   r´   rµ   r¶   s   @rƒ   r6  r6  ¥  s#   ø‡ € ò$ò5òò
÷ö r„   r6  c                   óJ   a € ] tR tRt o R tR tR tR tV 3R lR ltRt	V t
R	# )
ÚGGUFGemmaConverteriä  c                ób   € R .VR&   \        V4      V n        V P                  V n        / V n        R# r8  r  rÀ   s   &&rƒ   r®   ÚGGUFGemmaConverter.__init__å  s.   € à$0 >ˆ�xÑ ä*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÖr„   c                ód  € \        \        VP                  VP                  4      4      p. pV F~  w  rEVR 8X  d   VP	                  RV34       K!  RV9   dE   \        VP                  4       4      ^ 8X  d'   R\        V4      ,          pVP	                  We34       Kl  VP	                  WE34       K€  	  V# )z<0x09>Ú	r’   rç   )rw   rÆ   ra   rb   rœ   rx   Ústrip)r¡   r»   Úoriginal_vocabÚupdated_vocabÚtokenÚscoreÚunderscoress   &&     rƒ   r‹   ÚGGUFGemmaConverter.vocabí  s–   € Üœc %§,¡,°·±Ó=Ó>ˆØˆã*‰LˆEØ˜Ô Ø×$Ñ$ d¨E ]Ö3Ø˜”¤# e§k¡k£mÓ"4¸Ô"9Ø#¤c¨%£jÕ0�Ø×$Ñ$ kÐ%9Ö:à×$Ñ$ e ^Ö4ñ +ð Ðr„   c                ó0   € \         P                  ! R R4      # )r’   rç   )r   rð   rÇ   s   &&rƒ   rú   ÚGGUFGemmaConverter.normalizerü  s   € Ü×"Ò" 3¨Ó.Ð.r„   c                óþ   € \         P                  ! R R4      \         P                  ! 4       \         P                  ! 4       .pV'       d!   V\         P                  ! R^R7      .,          p\         P
                  ! V4      # )rç   r’   rë   )r   rð   rî   rï   rò   ró   rô   s   &&& rƒ   r÷   ÚGGUFGemmaConverter.decoderÿ  s\   € ä×Ò˜U CÓ(Ü×!Ò!Ó#Ü�MŠM‹Oð
ˆ÷ ØœŸš°¸!Ô<Ð=Õ=ˆHÜ× Ò  Ó*Ð*r„   c                ó    <€ V ^8„  d   QhRS[ /# r  r	  )r
  r·   s   "€rƒ   r  ÚGGUFGemmaConverter.__annotate__
  s   ø€ ÷ ñ ™9ñ r„   c                ó¾  € V P                  V P                  4      p\        \        VV P                  P                  V P
                  R 7      4      pV P                  V P                  4      pVe   W2n        RpRp\        V P                  R4      '       d   V P                  P                  pV P                  WE4      Vn
        V P                  WE4      pVe   Wbn        V# )rK  rç   Tre   )r‹   r»   r   r	   r8   Úhandle_byte_fallbackrú   r—   r¼   re   r÷   rû   )r¡   rÝ   r5   rú   rõ   re   rû   s   &      rƒ   rý   ÚGGUFGemmaConverter.converted
  sÃ   € Ø—z‘z $§*¡*Ó-ˆÜÜØØ—z‘z×.Ñ.Ø"×7Ñ7ôó
ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÒ!Ø#-Ô àˆØÐÜ�4×*Ñ*Ð,>×?Ò?Ø#×6Ñ6×GÑGÐà ŸL™L¨ÓGˆ	ÔØ×*Ñ*¨;ÓIˆØÒ$Ø&3Ô#àÐr„   r,  N)r°   r±   r²   r³   r®   r‹   rú   r÷   rý   r´   rµ   r¶   s   @rƒ   rO  rO  ä  s#   ø‡ € ò$òò/ò	+÷ö r„   rO  Úgemma3_textÚgemma4_textÚdecilmc                óR   € V ^8„  d   QhR\         R\        \        \        3,          /# )rn   r   r  )ÚstrrŸ   r   Údict)r
  s   "rƒ   r  r  >  s$   € ÷ 7ñ 7¬ð 7ÄÄyÔRVÀÕAWñ 7r„   c                óf   € T p\         V,          ! V4      pVP                  4       pWCP                  3# )a  
Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

Args:
    architecture (`str`): The model architecture derived from gguf file.
    transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
        Instance of a slow tokenizer to convert in the backend tokenizer for
        [`~tokenization_utils_base.PreTrainedTokenizerFast`].

Return:
    A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
    [`~tokenization_utils_base.PreTrainedTokenizerFast`]
)ÚGGUF_TO_FAST_CONVERTERSrý   r½   )r   rÁ   Útokenizer_class_nameÚ	converterÚfast_tokenizers   &&   rƒ   Úconvert_gguf_tokenizerro  >  s7   € ð (ÐÜ'Ð(<Ö=¸nÓM€IØ×(Ñ(Ó*€NØ×6Ñ6Ð6Ð6r„   ))Ú__doc__r   ÚnumpyrÙ   Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr   r	   Ú r
   Úconvert_slow_tokenizerr   r   r   r   r   Úutilsr   Úutils.loggingr   Ú
get_loggerr°   r™   ÚGGUF_CONFIG_MAPPINGÚGGUF_TOKENIZER_MAPPINGÚGGUF_CONFIG_DEFAULTS_MAPPINGr   r†   r¹   r  r  r.  r6  rO  rk  ro  r‰   r„   rƒ   Ú<module>r|     ss	  ðñõ
 ã ß SÕ Sß *å ß oÕ oÝ Ý  ð 
×	Ò	˜HÓ	%€ðqØØ˜ØÐ%ððqð
 ØÐ3ØÐ*ØÐ2Ø˜Mà 
Ø˜,ØÐ 5Ø!Ð#8Ø*¨NØ�lððqð$ ØÐ3ØÐ*ØÐ2Ø˜Mà 
Ø˜,ØÐ 5Ø!Ð#8Ø*¨NØ�lðð%qð> ØÐ3ØÐ*ØÐ2Ø˜MØ Ø˜,ØÐ 5Ø!Ð#8Ø*¨NØ�lðð?qðV ØÐ3ØÐ*ØÐ2Ø˜MØ Ø˜,ØÐ 5Ø!Ð#8Ø*¨NØ�lØ˜ØÐ2ððWqðr ØÐ3ØÐ*ØÐ2Ø˜MØ Ø˜,ØÐ 5Ø!Ð#8Ø*¨NØ�lØÐ+ØÐ2ØÐ*ððsqðP ØÐ3ØÐ*ØÐ2Ø˜MØ Ø˜,ØÐ 5Ø!Ð#8Ø*¨NØ�lØ˜^ððQqðj ØÐ3ØÐ*ØÐ2Ø˜MØ Ø˜,ØÐ 5Ø!Ð#8Ø*¨NØ�lððkqðB ØÐ3ØÐ*ØÐ2Ø˜MØ Ø˜,Ø 
ØÐ 5Ø!Ð#8Ø*¨NØ�lØ˜ØÐ2ððCqð` ØÐ3ØÐ*ØÐ2Ø˜MØ Ø˜,ØÐ 5Ø!Ð#8Ø*¨NØ�lððaqðx Ø˜^Ø˜^Ø Ø ð	ðyqðD ØÐ3ØÐ*ØÐ2Ø˜MØ Ø˜,ØÐ 5Ø!Ð#8Ø*¨NØ�lððEqð\ Ø�yØ˜MØ Ø�lØ&Ð(<ðð]qðj 	Ø˜-Ø�|Ø˜vØ˜IØ Ø Ø!Ð#8Ø&Ð(<Ø*Ð,LØ Ð":Ø�lððkqðD ØÐ3ØÐ*ØÐ2Ø˜MØ ØÐ 5Ø!Ð#8Ø&Ð(8Ø�lð
ðEqðZ Ø�yØ˜'Ø˜HØÐ4Ø Ø&Ð(<ðð[qðj ØÐ*ØÐ3Ø˜MØÐ2ØÐ 5Ø!Ð#8Ø&¨ððkqð| Ø�lØÐ3Ø˜MØ*Ð,@ØÐ*Ø˜=Ø˜,ØÐ.ØÐ-ð
ð ØÐ3ØÐ*ØÐ2Ø˜MØ Ø˜,ØÐ 5Ø!Ð#8Ø*¨JØ�lðð ØÐ3ØÐ*ØÐ2Ø˜MØ Ø˜,ð 	 
ØÐ 5Ø!Ð#8Ø*¨NØ"Ð$4Ø�lðð  ØÐ3ØÐ*ØÐ2Ø˜MØ Ø˜,ð 	 
ØÐ 5Ø!Ð#8Ø*¨NØ"Ð$4Ø�lðð  ØÐ3ØÐ*ØÐ2Ø˜MØ Ø˜$ð 	Ð 1ØÐ 5Ø!Ð#8Ø*¨NØ"Ð$4Ø�lðð" Ø˜-Ø�|Ø˜vØ˜IØ Ø Ø!Ð#8Ø&Ð(<Ø*Ð,LØ Ð":Ø�lðð ØÐ3ØÐ*ØÐ2Ø˜MØ Ø˜,ØÐ 5Ø!Ð#8Ø*¨NØ�lðð ð ØÐ3ðàÐ*ðð 	Ð2ðð 	˜Mð	ð
 	 ðð 	˜,ðð 	Ð 5ðð 	"Ð#8ðð 	 
ðð 	! $ðð 	+¨Nðð 	Ð+ðð 	Ð2ðð 	% dðð 	�lðð  	˜nð!ñqÐ ðh	 ØÐ&Ø�xØ�xØ˜<Ø�xØ˜^Ø˜^Ø Ø ØÐ!3ðð Ø˜Ø�lØ˜^Ø˜^Ø Ø ððÐ ð2 ð 	˜$ð	ð ð 	˜Dð	ð Ð ò ÷0(6ñ (6ôVv˜ô vôr˜ô ô(H˜ô HôV	�}ô 	ô<�kô <ô~>˜ô >ðBØÐðàÐðð Ð#ðð Ðð	ð
 Ð#ðð Ððð Ððð Ððð Ð ðð Ððð Ð"ðð 	ˆ/ðð Ððð Ð ðð Ð ðð  Ð%ð!ð" Ð%ð#ð$ ˆOØ
ÐØÐ ØÐ$ñ+Ð ÷27r„   