+
    QV-jO1 ã            	       óP  € R t ^ RIt^ RIHt ^ RIHt ^ RIHt ^ RIH	t	H
t
HtHtHtHtHt ^ RIHtHtHt ^ RIHt ^RIHtHtHtHt ^R	IHt ]P8                  ! ]4      t. R¥Ot]. R¦O,           t R§R
 lt!R R lt"R¨R R llt# ! R R4      t$ ! R R]$4      t%R R lt& ! R R4      t' ! R R]'4      t( ! R R]'4      t) ! R R]'4      t* ! R R]'4      t+ ! R R ]'4      t, ! R! R"]'4      t- ! R# R$]'4      t. ! R% R&]'4      t/ ! R' R(]'4      t0 ! R) R*]'4      t1 ! R+ R,]'4      t2 ! R- R.]'4      t3 ! R/ R0]34      t4 ! R1 R2]34      t5 ! R3 R4]34      t6 ! R5 R6]34      t7 ! R7 R8]34      t8 ! R9 R:]34      t9 ! R; R<]34      t: ! R= R>]34      t; ! R? R@]34      t< ! RA RB]34      t= ! RC RD]34      t> ! RE RF]34      t? ! RG RH]34      t@ ! RI RJ]34      tA ! RK RL]34      tB ! RM RN]34      tC ! RO RP]'4      tD ! RQ RR]34      tE ! RS RT]'4      tF ! RU RV]'4      tG ! RW RX]'4      tH ! RY RZ]34      tI ! R[ R\]34      tJ ! R] R^]34      tK ! R_ R`]'4      tL ! Ra Rb]34      tM ! Rc Rd]34      tN ! Re Rf]34      tORg tP ! Rh Ri4      tQ ! Rj Rk4      tR/ Rl]4bRm]0bRn]5bRo](bRp]EbRq]HbRr]6bRs]FbRt]-bRu](bRv]2bRw]7bRx](bRy](bRz](bR{](bR|](b/ R}]4bR~]*bR]-bR€].bR�](bR‚](bRƒ]0bR„]<bR…]0bR†]0bR‡](bRˆ]LbR‰]8bRŠ]9bR‹]+bRŒ](bR�]0bC/ RŽ]:bR�],bR�]AbR‘]/bR’]>bR“]?bR”]0bR•]1bR–];bR—](bR˜]BbR™]CbRš]DbR›]<bRœ]=bR�])bRž]IbCRŸ]KR ]KR¡]JR¢]K/CtSR©R£ R¤ lltTR# )ªzù
Utilities to convert slow tokenizers in their fast tokenizers counterparts.

All the conversions are grouped here to gather SentencePiece dependencies outside of the fast tokenizers files and
allow to make our dependency on SentencePiece optional.
N)Ú
Collection)Ú	lru_cache)Úversion)Ú
AddedTokenÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramÚ	WordPiece)Útqdm)Úis_protobuf_availableÚis_sentencepiece_availableÚloggingÚrequires_backends)ÚPROTOBUF_IMPORT_ERRORc                 ó>  € \        4       '       d	   ^ RIHp V# \        4       '       dX   ^ RIp\
        P                  ! VP                  P                  4      \
        P                  ! R4      8  d	   ^ RI	Hp V# ^ RI	H
p V# \        \        P                  ! V 4      4      h)é    )Úsentencepiece_model_pb2Nz4.0.0)Úsentencepiece_model_pb2_new)r   Úsentencepiecer   r   Úgoogle.protobufr   ÚparseÚprotobufÚ__version__Útransformers.utilsr   ÚImportErrorr   Úformat)Úerror_messager   Úgoogles   &  Út/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/transformers/convert_slow_tokenizer.pyÚimport_protobufr$   _   sr   € Ü!×#Ò#Ý9à&Ð&Ü×ÒÛä�=Š=˜Ÿ™×4Ñ4Ó5¼¿ºÀgÓ8NÔNÝBð 'Ð&õ bØ&Ð&äÔ/×6Ò6°}ÓEÓFÐFó    c                ó0   € V ^8„  d   QhR\         R\        /# )é   Úadd_prefix_spaceÚreturn)ÚboolÚstr)r    s   "r#   Ú__annotate__r,   p   s   € ÷ ñ ¬$ð Äsñ r%   c                 óL   € V '       d   R p\        VRR4      '       g   RpV# RpV# )ÚalwaysÚlegacyTÚfirstÚnever)Úgetattr)r(   Úoriginal_tokenizerÚprepend_schemes   && r#   Ú_get_prepend_schemer5   p   s4   € ßØ!ˆÜÐ)¨8°T×:Ò:Ø$ˆNð Ðð !ˆØÐr%   c                óH   € V ^8„  d   QhR\         \        ,          R,          /# )r'   Úskip_tokensN)r   r+   )r    s   "r#   r,   r,   z   s   € ÷ ñ ´jÄµoÈÕ6Lñ r%   c                 ó  a € Ve   \        V4      M	\        4       pVR JpV'       d   \        V4      MS p. pVP                  4        Fˆ  w  rVWR9   d   K  . p\        ^\	        V4      4       F>  pVR V WXR  r©W’9   g   W¢9   d   K  V	S 9   g   K"  V
S 9   g   K+  VP                  WšV34       K@  	  \        VV 3R lR7      pVP                  V4       KŠ  	  \        VR VR7      pV Uu. uF  q»^ ,          V^,          3NK  	  ppV# u upi )Nc                 óD   <€ SV ^ ,          ,          SV ^,          ,          3# ©r   © )ÚxÚvocabs   &€r#   Ú<lambda>Ú!generate_merges.<locals>.<lambda>Š   s   ø€ ¨U°1°Qµ4­[¸%ÀÀ!Å½+Ñ,Fr%   ©Úkeyc                 óZ   € V ^,          \        V ^ ,          4      \        V ^,          4      3# ©r'   )Úlen©Úvals   &r#   r>   r?   �   s   € ¨S°­V´S¸¸Q½³[Ä#ÀcÈ!ÅfÃ+Ñ,Nr%   ©rA   Úreverse)ÚsetÚdictÚitemsÚrangerD   ÚappendÚsortedÚextend)r=   Úvocab_scoresr7   rH   ÚmergesÚmergeÚpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rrF   s   f&&         r#   Úgenerate_mergesrX   z   s  ø€ Ø&1Ò&=”#�kÔ"Ä3Ã5€KØ $Ð&€Gß)0”4˜Ô%°e€Là€FØ*×0Ñ0Ö2ÑˆØÔÙØˆÜ˜1œc %›jÖ)ˆEØ$ V e˜}¨e°F¨m�WØÔ%¨Ô)?ÙØ˜%Ö G¨uÖ$4Ø—‘˜g°Ð<Ö=ñ *ô �uÔ"FÔGˆØ�‰�eÖñ 3ô �FÑ NÐX_Ô`€FÙ*0Ó1©& 3�1�v�s˜1•vÓ©&€FÐ1Ø€Mùò 2s   Ã%Dc                   óH   a € ] tR t^’t o RtV 3R lR ltV 3R lR ltRtV tR# )ÚSentencePieceExtractorzd
Extractor implementation for SentencePiece trained models. https://github.com/google/sentencepiece
c                ó    <€ V ^8„  d   QhRS[ /# )r'   Úmodel©r+   )r    Ú__classdict__s   "€r#   r,   Ú#SentencePieceExtractor.__annotate__—   s   ø€ ÷ 
ñ 
™cñ 
r%   c                ó  € \        V R 4       \        V R4       \        4       pVP                  4       p\        VR4      ;_uu_ 4       pVP	                  VP                  4       4       RRR4       W0n        R#   + '       g   i     L; i)r   r   ÚrbN)r   r$   Ú
ModelProtoÚopenÚParseFromStringÚreadÚproto)Úselfr\   Ú	model_pb2ÚmÚfs   &&   r#   Ú__init__ÚSentencePieceExtractor.__init__—   sd   € Ü˜$ Ô0Ü˜$ 
Ô+ô $Ó%ˆ	à× Ñ Ó"ˆÜ�%˜×Ô !Ø×Ñ˜aŸf™f›hÔ'÷ àŽ
÷ ×ús   Á A7Á7B	c                ó\   <€ V ^8„  d   QhRS[ S[S[S[3,          S[S[ ,          3,          /# ©r'   r)   ©ÚtuplerJ   r+   ÚintÚlist)r    r^   s   "€r#   r,   r_   £   s*   ø€ ÷ !ñ !©u±T¹#¹s¸(µ^ÁTÉ%Å[Ð5PÕ/Qñ !r%   c                ó¦  € V P                   P                  P                   Vf2   ^ RIHpHp V P                   P                  P                  ^8X  d   TMTpV P                   P                   Uu. uF  qUP                  VP                  3NK  	  ppVP                  R8w  d)   V P                   P                  P                  VR&   WbR&   M9^RIHp \        V4       UU	U
u/ uF
  w  pw  ršW˜bK  	  pp	pp
V! V4      pWbR&   W²R&   \        V P                   P                  4       UUu. uF3  w  rÍVP                  R9   g   K  WÍP                  VP                  ^8H  3NK5  	  ppp\        VR R	7       UUUu. uF  w  rÏp\!        VR
VR7      NK  	  upppVR&   \#        V P                   P$                  RR4      VR&   V# u upi u up
p	pi u uppi u upppi )úª
By default will return vocab and merges with respect to their order, by sending `vocab_scores` we're going to
order the merges with respect to the piece scores instead.
N)r   r   r   Úunk_idr=   )rX   rQ   c                 ó   € V ^ ,          # r:   r;   ©r<   s   &r#   r>   Ú0SentencePieceExtractor.extract.<locals>.<lambda>Á   s   € ÈQÈqÎTr%   r@   F©Ú
normalizedÚspecialÚadditional_special_tokensÚprecompiled_charsmapÚ_spm_precompiled_charsmap©é   é   )rf   Útrainer_specru   Útokenizers.modelsr   r   Ú
model_typeÚpiecesÚpieceÚscoreÚ__name__Útokenization_utils_baserX   Ú	enumerateÚtyperN   r   r2   Únormalizer_spec)rg   r„   Úkwargsr   r   r†   r=   rX   ÚiÚwordr‡   rQ   ÚidÚpÚspm_added_tokensÚtokenr{   s   &&,              r#   ÚextractÚSentencePieceExtractor.extract£   s¡  € ð
 	�
‰
×Ñ×&Ò&ØÒß6à$(§J¡J×$;Ñ$;×$FÑ$FÈ!Ô$K™ÐQTˆJØ9=¿¹×9JÒ9JÓKÑ9J°—+‘+˜uŸ{™{Ó+Ñ9JˆÐKà×Ñ %Ô'Ø#Ÿz™z×6Ñ6×=Ñ=ˆF�8ÑØ#�7ŠOå@ä5>¸uÔ5EÕFÑ5EÑ!1 ¡M T�T’WÑ5EˆEÒFÙ$ UÓ+ˆFØ#�7‰OØ%�8Ñô ENÈdÏjÉj×N_ÑN_ÔD`ÔuÑD`¹5¸2Ðde×djÑdjÐntÑdtÔ6˜R§¡¨!¯&©&°A©+Ó6ÑD`ÐÑuô '-Ð-=Á>Õ&Rõ/
á&RÑ"�˜7ô �u¨¸×@Ù&Ró/
ˆÐ*Ñ+ô /6°d·j±j×6PÑ6PÐRhÐjnÓ.oˆÐ*Ñ+Øˆùò/ Lùô Gùó vùô/
s   Á. F:ÃF?Ä$GÄ>GÅ2G©rf   N)	rˆ   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rk   r”   Ú__static_attributes__Ú__classdictcell__©r^   s   @r#   rZ   rZ   ’   s   ø‡ € ñ÷
ð 
÷!ö !r%   rZ   c                   ó6   a € ] tR t^Çt o RV 3R lR lltRtV tR# )ÚGemmaSentencePieceExtractorNc                ó\   <€ V ^8„  d   QhRS[ S[S[S[3,          S[S[ ,          3,          /# rn   ro   )r    r^   s   "€r#   r,   Ú(GemmaSentencePieceExtractor.__annotate__È   s*   ø€ ÷ ñ ©E±$±s¹C°xµ.Á$ÁuÅ+Ð2MÕ,Nñ r%   c                óä   € V P                   p\        VP                  4       4       Uu/ uF  q2P                  V4      VbK  	  ppRV9  d   VP	                  R4      VR&   \        WA4      pWE3# u upi )rt   Ú	ú<0x09>)ÚsprL   ÚGetPieceSizeÚid_to_pieceÚgetrX   )rg   rP   r¥   rU   r=   rQ   s   &&    r#   r”   Ú#GemmaSentencePieceExtractor.extractÈ   so   € ð
 �W‰WˆÜ;@ÀÇÁÓARÔ;SÓTÑ;S°%—‘ Ó&¨Ò-Ñ;SˆÐTð �uÔØŸ)™) HÓ-ˆE�$‰KÜ  Ó5ˆØˆ}Ðùò Us   ¨A-r;   ©N)rˆ   r—   r˜   r™   r”   r›   rœ   r�   s   @r#   rŸ   rŸ   Ç   s   ø‡ € ÷÷ ð r%   rŸ   c                ó0   € V ^8„  d   QhR\         R\        /# )r'   r†   r)   )r+   r*   )r    s   "r#   r,   r,   Ø   s   € ÷ Iñ Iœcð I¤dñ Ir%   c                 óŽ   € \        V 4      ^8  ;'       g1    V R,          R8g  ;'       g    V R,          P                  4       '       * # )r'   Ú,éÿÿÿÿéþÿÿÿ)rD   Úisdigit)r†   s   &r#   Úcheck_number_commar±   Ø   s9   € Üˆu‹:˜‰>×HÐH˜U 2�Y¨#Ñ-×HÐH°U¸2µY×5FÑ5FÓ5HÔ1HÐHr%   c                   ó8   a € ] tR t^Üt o R tV 3R lR ltRtV tR# )Ú	Converterc                ó   € Wn         R # rª   ©r3   )rg   r3   s   &&r#   rk   ÚConverter.__init__Ý   s   € Ø"4Ör%   c                ó    <€ V ^8„  d   QhRS[ /# rn   ©r   )r    r^   s   "€r#   r,   ÚConverter.__annotate__à   s   ø€ ÷ $ñ $™9ñ $r%   c                ó   € \        4       hrª   )ÚNotImplementedError©rg   s   &r#   Ú	convertedÚConverter.convertedà   s   € Ü!Ó#Ð#r%   rµ   N)rˆ   r—   r˜   r™   rk   r½   r›   rœ   r�   s   @r#   r³   r³   Ü   s   ø‡ € ò5÷$ö $r%   r³   c                   ó2   a € ] tR t^ät o V 3R lR ltRtV tR# )ÚBertConverterc                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   ÚBertConverter.__annotate__å   ó   ø€ ÷ #ñ #™9ñ #r%   c           	     ó|  € V P                   P                  p\        \        V\	        V P                   P
                  4      R 7      4      pRpRpRp\        V P                   R4      '       da   V P                   P                  P                  pV P                   P                  P                  pV P                   P                  P                  p\        P                  ! RVVVR7      Vn        \        P                  ! 4       Vn        \	        V P                   P"                  4      p\	        V P                   P$                  4      pV P                   P&                  pV P                   P(                  p	\*        P,                  ! V RV R2V RV RV R2Wh3Wy3.R	7      Vn        \0        P                  ! R
R7      Vn        V# )©Ú	unk_tokenFÚbasic_tokenizerT©Ú
clean_textÚhandle_chinese_charsÚstrip_accentsÚ	lowercaseú:0 $A:0 ú:0ú:0 $B:1 ú:1©ÚsingleÚpairÚspecial_tokensú##©Úprefix©r3   r=   r   r   r+   rÆ   ÚhasattrrÇ   Útokenize_chinese_charsrË   Údo_lower_caser	   ÚBertNormalizerÚ
normalizerr
   ÚBertPreTokenizerÚpre_tokenizerÚ	cls_tokenÚ	sep_tokenÚcls_token_idÚsep_token_idr   ÚTemplateProcessingÚpost_processorr   Údecoder©
rg   r=   Ú	tokenizerrÚ   rË   rÛ   ÚclsÚseprâ   rã   s
   &         r#   r½   ÚBertConverter.convertedå   ó„  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ò>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5àÐ#ØÐ#ðô$
ˆ	Ô ô %×.Ò.°dÔ;ˆ	ÔàÐr%   r;   N©rˆ   r—   r˜   r™   r½   r›   rœ   r�   s   @r#   rÀ   rÀ   ä   ó   ø‡ € ÷#ö #r%   rÀ   c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚSplinterConverteri  c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   ÚSplinterConverter.__annotate__  s   ø€ ÷ .ñ .™9ñ .r%   c           
     ó’  € V P                   P                  p\        \        V\	        V P                   P
                  4      R 7      4      pRpRpRp\        V P                   R4      '       da   V P                   P                  P                  pV P                   P                  P                  pV P                   P                  P                  p\        P                  ! RVVVR7      Vn        \        P                  ! 4       Vn        \	        V P                   P"                  4      p\	        V P                   P$                  4      p\	        V P                   P&                  4      pRp	V P                   P(                  p
V P                   P*                  pV P                   P,                  pV P                   P/                  R4      pV P                   P0                  R8X  d   V RV RV	 RV R	V R
2
pMV RV R	V RV	 RV R
2
p\2        P4                  ! V RV R2VWj3W{3WŒ3W�3.R7      Vn        \8        P                  ! RR7      Vn        V# )rÅ   FrÇ   TrÈ   Ú.ÚrightrÍ   Ú rÏ   rÐ   rÎ   rÑ   rÕ   rÖ   )r3   r=   r   r   r+   rÆ   rÙ   rÇ   rÚ   rË   rÛ   r	   rÜ   rÝ   r
   rÞ   rß   rà   rá   Úquestion_tokenrâ   rã   Úquestion_token_idÚconvert_tokens_to_idsÚpadding_sider   rä   rå   r   ræ   )rg   r=   rè   rÚ   rË   rÛ   ré   rê   ÚquestionÚdotrâ   rã   rø   Údot_token_idrÓ   s   &              r#   r½   ÚSplinterConverter.converted  s  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ò>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆÜ�t×.Ñ.×=Ñ=Ó>ˆØˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆØ ×3Ñ3×EÑEÐØ×.Ñ.×DÑDÀSÓIˆà×"Ñ"×/Ñ/°7Ô:Ø�U˜( 8 *¨A¨c¨U°!°C°5¸ÀÀÀRÐH‰Dà�U˜( 3 % x°¨z¸¸3¸%¸qÀÀÀRÐHˆDä#-×#@Ò#@Ø�U˜( 3 % rÐ*ØàÐ#ØÐ#ØÐ-ØÐ#ð	ô	$
ˆ	Ô ô %×.Ò.°dÔ;ˆ	ÔàÐr%   r;   Nrí   r�   s   @r#   rð   rð     s   ø‡ € ÷.ö .r%   rð   c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚFunnelConverteri=  c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   ÚFunnelConverter.__annotate__>  rÃ   r%   c           	     ó|  € V P                   P                  p\        \        V\	        V P                   P
                  4      R 7      4      pRpRpRp\        V P                   R4      '       da   V P                   P                  P                  pV P                   P                  P                  pV P                   P                  P                  p\        P                  ! RVVVR7      Vn        \        P                  ! 4       Vn        \	        V P                   P"                  4      p\	        V P                   P$                  4      pV P                   P&                  pV P                   P(                  p	\*        P,                  ! V RV R2V RV RV R2Wh3Wy3.R	7      Vn        \0        P                  ! R
R7      Vn        V# )rÅ   FrÇ   TrÈ   z:2 $A:0 rÎ   rÏ   rÐ   rÑ   rÕ   rÖ   rØ   rç   s
   &         r#   r½   ÚFunnelConverter.converted>  rì   r%   r;   Nrí   r�   s   @r#   r   r   =  rî   r%   r   c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚMPNetConverterid  c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   ÚMPNetConverter.__annotate__e  rÃ   r%   c                ó‚  € V P                   P                  p\        \        V\	        V P                   P
                  4      R 7      4      pRpRpRp\        V P                   R4      '       da   V P                   P                  P                  pV P                   P                  P                  pV P                   P                  P                  p\        P                  ! RVVVR7      Vn        \        P                  ! 4       Vn        \	        V P                   P"                  4      p\	        V P                   P$                  4      pV P                   P&                  pV P                   P(                  p	\*        P,                  ! V RV R2V RV RV RV R	2Wh3Wy3.R
7      Vn        \0        P                  ! RR7      Vn        V# )rÅ   FrÇ   TrÈ   rÍ   rÎ   z:0 rÏ   rÐ   rÑ   rÕ   rÖ   rØ   rç   s
   &         r#   r½   ÚMPNetConverter.convertede  sŠ  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ò>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   S¨¨¨X°c°U¸"Ð=àÐ#ØÐ#ðô$
ˆ	Ô ô %×.Ò.°dÔ;ˆ	ÔàÐr%   r;   Nrí   r�   s   @r#   r  r  d  rî   r%   r  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚOpenAIGPTConverteri‹  c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   ÚOpenAIGPTConverter.__annotate__Œ  s   ø€ ÷ ñ ™9ñ r%   c                ó  € V P                   P                  p\        V P                   P                  P	                  4       4      pV P                   P
                  p\        \        VVR \        V4      RRR7      4      pVP                  \        V4      4      e   VP                  \        V4      .4       \        P                  ! RR7      Vn        \        P                  ! 4       Vn        \"        P$                  ! RR7      Vn        V# )Nú</w>F)r=   rQ   ÚdropoutrÆ   Úend_of_word_suffixÚfuse_unkT)rÌ   ©Úsuffix)r3   Úencoderrr   Ú	bpe_ranksÚkeysrÆ   r   r   r+   Útoken_to_idÚadd_special_tokensr	   rÜ   rÝ   r
   rÞ   rß   r   Ú
BPEDecoderræ   ©rg   r=   rQ   rÆ   rè   s   &    r#   r½   ÚOpenAIGPTConverter.convertedŒ  sÔ   € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØÜ˜i›.Ø#)Øôó	
ˆ	ð × Ñ ¤ Y£Ó0Ò<Ø×(Ñ(¬#¨i«.Ð)9Ô:ä*×9Ò9ÀDÔIˆ	ÔÜ"0×"AÒ"AÓ"Cˆ	ÔÜ$×/Ò/°vÔ>ˆ	ÔàÐr%   r;   Nrí   r�   s   @r#   r  r  ‹  s   ø‡ € ÷ö r%   r  c                   ó6   a € ] tR tRt o RV 3R lR lltRtV tR# )ÚGPT2Converteri¦  Nc                ó„   <€ V ^8„  d   QhRS[ S[S[3,          R,          RS[S[S[S[3,          ,          R,          RS[/# ©r'   r=   NrQ   r)   ©rJ   r+   rq   rr   rp   r   )r    r^   s   "€r#   r,   ÚGPT2Converter.__annotate__§  sF   ø€ ÷ "ñ "™t¡C© H�~°Õ4ð "ÁTÉ%ÑPSÑUXÐPXÍ/ÕEZÐ]aÕEað "Ñmvñ "r%   c                óx  € V'       g   V P                   P                  pV'       g    \        V P                   P                  4      p\	        \        VVR RRRR7      4      p\        V P                   RR4      p\        P                  ! VR7      Vn	        \        P                  ! 4       Vn        \        V P                   RR4      '       dU   V P                   P                  pV P                   P                  p\        P                  ! V R2V R2WV3.R	7      Vn        V# \        P                  ! RR
7      Vn        V# )NÚ F©r=   rQ   r  Úcontinuing_subword_prefixr  r  r(   ©r(   Úadd_bos_tokenz:0 $A:0z:0 $A:0 $B:1rÑ   ©Útrim_offsets)r3   r  rr   r  r   r   r2   r
   Ú	ByteLevelrß   r   ræ   Ú	bos_tokenÚbos_token_idr   rä   rå   )rg   r=   rQ   rè   r(   Úbosr.  s   &&&    r#   r½   ÚGPT2Converter.converted§  s  € ßØ×+Ñ+×3Ñ3ˆEßÜ˜$×1Ñ1×;Ñ;Ó<ˆFäÜØØØØ*,Ø#%Øôó	
ˆ	ô # 4×#:Ñ#:Ð<NÐPUÓVÐÜ"0×":Ò":ÐL\Ô"]ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ�4×*Ñ*¨O¸U×CÒCØ×)Ñ)×3Ñ3ˆCØ×2Ñ2×?Ñ?ˆLÜ'1×'DÒ'DØ˜˜g�Ø�u˜LÐ)àÐ'ð ô(ˆIÔ$ð Ðô (2×';Ò';ÈÔ'OˆIÔ$ØÐr%   r;   ©NNrí   r�   s   @r#   r  r  ¦  s   ø‡ € ÷"÷ "ð "r%   r  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚHerbertConverteriÌ  c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   ÚHerbertConverter.__annotate__Í  ó   ø€ ÷ ñ ™9ñ r%   c           
     ó°  € R pRpV P                   P                  p\        V P                   P                  P	                  4       4      pW^ ,          ^ ,          9   d
   VR,          p\        \        VVRV P                   P                  VR7      4      p\        P                  ! RRR7      Vn
        \        P                  ! 4       Vn        \        P                  ! VR7      Vn        \"        P$                  ! V P                   P&                  V P                   P(                  3V P                   P*                  V P                   P,                  3R7      Vn        V# )	z	#version:r  ºé   NNN)r  rÆ   r  F)rÌ   rË   r  )rê   ré   )r3   r  rr   r  r  r   r   rÆ   r	   rÜ   rÝ   r
   rÞ   rß   r   r  ræ   r   ÚBertProcessingrá   rã   rà   râ   rå   )rg   Útokenizer_info_strÚtoken_suffixr=   rQ   rè   s   &     r#   r½   ÚHerbertConverter.convertedÍ  s  € Ø(ÐØˆà×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ¨¥¨1¥Ô-Ø˜B•ZˆFäÜØØØØ×1Ñ1×;Ñ;Ø#/ôó
ˆ	ô  +×9Ò9ÀEÐY^Ô_ˆ	ÔÜ"0×"AÒ"AÓ"Cˆ	ÔÜ$×/Ò/°|ÔDˆ	ÔÜ#-×#<Ò#<Ø×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYô$
ˆ	Ô ð
 Ðr%   r;   Nrí   r�   s   @r#   r3  r3  Ì  ó   ø‡ € ÷ö r%   r3  c                   ó6   a € ] tR tRt o RV 3R lR lltRtV tR# )ÚQwen2Converterië  Nc                ó„   <€ V ^8„  d   QhRS[ S[S[3,          R,          RS[S[S[S[3,          ,          R,          RS[/# r!  r"  )r    r^   s   "€r#   r,   ÚQwen2Converter.__annotate__ì  sF   ø€ ÷ (ñ (™t¡C© H�~°Õ4ð (ÁTÉ%ÑPSÑUXÐPXÍ/ÕEZÐ]aÕEað (Ñmvñ (r%   c                óT  € V'       g   V P                   P                  pV'       g.   \        V P                   P                  P	                  4       4      p\        \        VVR R RRRRR7      4      p\        P                  ! 4       Vn	        \        P                  ! \        P                  ! \        R4      RRR7      \        P                  ! \        V P                   RR4      RR7      .4      Vn        \"        P                  ! 4       Vn        \&        P                  ! RR	7      Vn        V# )
Nr%  F)r=   rQ   r  rÆ   r'  r  r  Úbyte_fallbackzn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+Úisolated©ÚbehaviorÚinvertr(   ©r(   Ú	use_regexr*  )r3   r  rr   r  r  r   r   r	   ÚNFCrÝ   r
   ÚSequenceÚSplitr   r,  r2   rß   r   ræ   r   rå   )rg   r=   rQ   rè   s   &&& r#   r½   ÚQwen2Converter.convertedì  s  € ßØ×+Ñ+×3Ñ3ˆEßÜ˜$×1Ñ1×;Ñ;×@Ñ@ÓBÓCˆFäÜØØØØØ*,Ø#%ØØ#ô	ó
ˆ	ô  +ŸšÓ0ˆ	Ôä"0×"9Ò"9ä×$Ò$Üð Nóð (Ø ôô ×(Ò(Ü%,¨T×-DÑ-DÐFXÐZ_Ó%`Ø#ôðó#
ˆ	Ôô  %×.Ò.Ó0ˆ	ÔÜ#-×#7Ò#7ÀUÔ#Kˆ	Ô àÐr%   r;   r1  rí   r�   s   @r#   r@  r@  ë  s   ø‡ € ÷(÷ (ð (r%   r@  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚRobertaConverteri  c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   ÚRobertaConverter.__annotate__  ó   ø€ ÷ ñ ™9ñ r%   c                óÚ  € V P                   pVP                  p\        VP                  P	                  4       4      p\        \        VVR RRRR7      4      p\        P                  ! VP                  R7      Vn
        \        P                  ! 4       Vn        \        P                  ! VP                  VP                   3VP"                  VP$                  3VP                  RR7      Vn        V# )Nr%  Fr&  r(  T©rê   ré   r(   r+  )r3   r  rr   r  r  r   r   r
   r,  r(   rß   r   ræ   r   ÚRobertaProcessingrá   rã   rà   râ   rå   ©rg   Úotr=   rQ   rè   s   &    r#   r½   ÚRobertaConverter.converted  sÄ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ò":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ#-×#?Ò#?Ø—‘˜rŸ™Ð/Ø—‘˜rŸ™Ð/Ø×0Ñ0Øô	$
ˆ	Ô ð Ðr%   r;   Nrí   r�   s   @r#   rP  rP    ó   ø‡ € ÷ö r%   rP  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚRoFormerConverteri4  c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   ÚRoFormerConverter.__annotate__5  rÃ   r%   c           	     ód  € ^RI Hp V P                  P                  p\	        \        V\        V P                  P                  4      R7      4      pRpRp\        V P                  R4      '       dA   V P                  P                  P                  pV P                  P                  P                  p\        P                  ! RRVVR7      Vn        \        P                   P#                  V! V4      4      Vn        \        V P                  P&                  4      p\        V P                  P(                  4      pV P                  P*                  pV P                  P,                  p	\.        P0                  ! V RV R2V RV R	V R
2Wh3Wy3.R7      Vn        \4        P
                  ! RR7      Vn        V# )r9  )ÚJiebaPreTokenizerrÅ   FrÇ   TrÈ   rÍ   rÎ   rÏ   rÐ   rÑ   rÕ   rÖ   )Ú"models.roformer.tokenization_utilsr`  r3   r=   r   r   r+   rÆ   rÙ   rÇ   rË   rÛ   r	   rÜ   rÝ   r
   ÚPreTokenizerÚcustomrß   rà   rá   râ   rã   r   rä   rå   r   ræ   )
rg   r`  r=   rè   rË   rÛ   ré   rê   râ   rã   s
   &         r#   r½   ÚRoFormerConverter.converted5  sx  € ÝIà×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	àˆØˆÜ�4×*Ñ*Ð,=×>Ò>Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!&Ø'Ø#ô	 
ˆ	Ôô #1×"=Ñ"=×"DÑ"DÑEVÐW\ÓE]Ó"^ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5àÐ#ØÐ#ðô$
ˆ	Ô ô %×.Ò.°dÔ;ˆ	ÔàÐr%   r;   Nrí   r�   s   @r#   r\  r\  4  rî   r%   r\  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚDebertaConverteri[  c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   ÚDebertaConverter.__annotate__\  r6  r%   c                óÜ  € V P                   pVP                  p\        VP                  P	                  4       4      p\        \        VVR RRRR7      4      p\        P                  ! VP                  R7      Vn
        \        P                  ! 4       Vn        \        P                  ! RRRV P                   P                  R4      3RV P                   P                  R4      3.R	7      Vn        V# )
Nr%  Fr&  r(  ú[CLS]:0 $A:0 [SEP]:0ú![CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1ú[CLS]ú[SEP]rÑ   )r3   r  rr   r  r  r   r   r
   r,  r(   rß   r   ræ   r   rä   rù   rå   rW  s   &    r#   r½   ÚDebertaConverter.converted\  sÕ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ò":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ#-×#@Ò#@Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðô$
ˆ	Ô ð Ðr%   r;   Nrí   r�   s   @r#   rf  rf  [  r>  r%   rf  c                   ó¢   a a€ ] tR tRt oRt]t/ t]RR l4       t	]
RR l4       tV 3R ltR tR tR tR	 tR
 tR tR tV3R lR ltRtVtV ;t# )ÚSpmConverteriz  Fc                óÒ  € V P                   P                  pV P                   P                  pV P                  P                  p\        V\        4      '       d&   \        \        TT;'       g    . VRVRR7      4      pMt\        V\        4      '       d]   V'       dU   \        V^ ,          \        \        ,          4      '       d-   \        \        VV P                   P                  VR7      4      pMR# \        P                  ! RR4      .pV'       d'   VP                  ^ \        P                   ! V4      4       \        P"                  ! V4      Vn        V'       d\   \&        P"                  ! \&        P                  ! RR4      \&        P(                  ! 4       \&        P*                  ! 4       .4      Vn        V# \&        P"                  ! \&        P                  ! RR4      .4      Vn        V# )zç
Similar to convert_from_spm method, but used only when there is no `model_type` class, i.e. there is no matching class in `TOKENIZERS_MAPPING` and we just create a tokenizer instead of extracting stuff from the sentencepiece file
TN)r=   rQ   rÆ   r  rD  r  )r=   ru   rD  rö   õ   â–�)r‚   rD  Ú	unk_piecerŒ   r}   Ú
isinstancerJ   r   r   rr   rp   r   ru   r	   ÚReplaceÚinsertÚPrecompiledrL  rÝ   r   ÚByteFallbackÚFuseræ   )rf   r=   rQ   rD  rs  r}   rè   Ú_normalizerss   &&&     r#   Úbuild_tokenizer_from_spm_protoÚ+SpmConverter.build_tokenizer_from_spm_proto  s}  € ð
 ×*Ñ*×8Ñ8ˆØ×&Ñ&×0Ñ0ˆ	Ø$×4Ñ4×IÑIÐô �eœT×"Ò"Ü!ÜØØ!Ÿ<˜< RØ'Ø!Ø"/Ø ôó	‰Iô ˜œt×$Ò$¯´:¸eÀA½hÌÔPTÍ×3UÒ3UÜ!ÜØØ ×-Ñ-×4Ñ4Ø"/ôó‰Iñ ô $×+Ò+¨C°Ó7Ð8ˆßØ×Ñ ¤;×#:Ò#:Ð;OÓ#PÔQÜ*×3Ò3°LÓAˆ	Ô÷ Ü (× 1Ò 1Ü×!Ò! %¨Ó-¬x×/DÒ/DÓ/FÌÏÊËÐXó!ˆIÔð Ðô !)× 1Ò 1´8×3CÒ3CÀEÈ3Ó3OÐ2PÓ QˆIÔàÐr%   c                ó   € Ve   WR&   V# )z‡
Hook used when converting directly from a SentencePiece model without a slow tokenizer instance.
By default, return kwargs unchanged.
r=   r;   )ré   r=   r�   s   &&,r#   Úconvert_from_spmÚSpmConverter.convert_from_spm¯  s   € ð ÒØ#�7‰OØˆr%   c                óæ  <€ \        V R 4       \        SV `  ! V!   \        4       pVP	                  4       p\        V P                  P                  R4      ;_uu_ 4       pVP                  VP                  4       4       RRR4       W0n
        V P                  P                  P                  '       d-   V P                  '       g   \        P                  ! R4       R# R# R#   + '       g   i     Lj; i)r   ra   Na  The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.)r   Úsuperrk   r$   rb   rc   r3   Ú
vocab_filerd   re   rf   r‚   rD  Úhandle_byte_fallbackÚwarningsÚwarn)rg   Úargsrh   ri   rj   Ú	__class__s   &*   €r#   rk   ÚSpmConverter.__init__¹  s°   ø€ Ü˜$ 
Ô+ä‰Ò˜$Òô $Ó%ˆ	à× Ñ Ó"ˆÜ�$×)Ñ)×4Ñ4°d×;Ô;¸qØ×Ñ˜aŸf™f›hÔ'÷ <àŒ
à�:‰:×"Ñ"×0×0Ð0¸×9R×9RÐ9RÜ�MŠMðeöñ :SÑ0÷	 <×;ús   Á C Ã C0	c                ón   € VP                    Uu. uF  q"P                  VP                  3NK  	  up# u upi rª   ©r…   r†   r‡   ©rg   rf   r†   s   && r#   r=   ÚSpmConverter.vocabÎ  s)   € Ø8=¿ºÓE¹¨u—‘˜eŸk™kÓ*¹ÑEÐEùÒEs   � 2c                ó.   € VP                   P                  # rª   )r‚   ru   ©rg   rf   s   &&r#   ru   ÚSpmConverter.unk_idÑ  s   € Ø×!Ñ!×(Ñ(Ð(r%   c                óž  € VP                   P                  pV P                  V4      pV^8X  d2   \        \	        VV P                  V4      V P                  R7      4      pM¤V^8X  d“   V P                  V P                  P                  4      P                  V4      w  rV\        V4       UUU	u/ uF
  w  pw  r‰W‡bK  	  p
ppp	\        \        V
VVP                   P                  RV P                  RR7      4      pM\        R4      h\        VP                  4       UUu. uFT  w  r¼VP                   R
9   g   K  Y¼P"                  VP                   ^8H  ;'       g    VP"                  V P$                  9   3NKV  	  pppTP'                  \)        VR R7       UUUu. uF  w  r¾p\+        VRVR	7      NK  	  uppp4       V# u up	ppi u uppi u upppi )r9  ©ru   rD  TN©rÆ   r  rD  r  z]You're trying to run a `Unigram` model but you're file was trained with a different algorithmc                 ó   € V ^ ,          # r:   r;   rw   s   &r#   r>   Ú(SpmConverter.tokenizer.<locals>.<lambda>   ó   € ÐQRÐSTÖQUr%   r@   Fry   r   )r‚   r„   r=   r   r   ru   rƒ  ÚSpmExtractorr3   r‚  r”   rŠ   r   rs  Ú	Exceptionr…   r‹   r†   rÔ   Ú
add_tokensrN   r   )rg   rf   r„   rP   rè   Ú_rQ   rŽ   r�   r‡   Ú	bpe_vocabr�   r‘   r’   r“   r{   s   &&              r#   rè   ÚSpmConverter.tokenizerÔ  s±  € Ø×'Ñ'×2Ñ2ˆ
Ø—z‘z %Ó(ˆà˜Œ?Ü!ÜØ ØŸ;™; uÓ-Ø"&×";Ñ";ôó‰Ið ˜1Œ_Ø×)Ñ)¨$×*AÑ*A×*LÑ*LÓM×UÑUÐVbÓc‰IˆAÜ9BÀ<Ô9PÕQÑ9PÑ%5 Q©¨˜šÑ9PˆIÒQÜ!ÜØØØ#×0Ñ0×:Ñ:Ø!Ø"&×";Ñ";Ø ôó	‰Iô Øoóð ô # 5§<¡<Ô0ô
á0‘�Ø�v‰v˜Ñô IˆR—‘˜!Ÿ&™& A™+×GÐG¨¯©°D×4GÑ4GÑ)GÓHÙ0ð 	ñ 
ð
 	×Ñô +1Ð1AÁ~Õ*Võá*VÑ&�B˜wô ˜5¨U¸G×DÙ*Vóô	
ð ÐùôC Rùó*
ùôs   Â,F;ÄGÄ6 GÅGÆGc                ó>  € VP                   P                  p\        P                  ! R RR7      \        P                  ! \        R4      R4      .pV'       g   \        P                  ! V4      # \        P                  ! \        P                  ! V4      .V,           4      # )FT)Úleftrõ   ú {2,}rr  )rŒ   r}   r	   ÚStripru  r   rL  rw  ©rg   rf   r}   rz  s   &&  r#   rÝ   ÚSpmConverter.normalizer  s{   € Ø$×4Ñ4×IÑIÐä×Ò 5°Ô5Ü×Ò¤ g£°Ó6ð
ˆ÷ $Ü×'Ò'¨Ó5Ð5ä×'Ò'¬×)@Ò)@ÐAUÓ)VÐ(WÐZfÕ(fÓgÐgr%   c                óZ   € \        W P                  4      p\        P                  ! WR 7      # ©©Úreplacementr4   )r5   r3   r
   Ú	Metaspace©rg   r¥  r(   r4   s   &&& r#   rß   ÚSpmConverter.pre_tokenizer  s$   € Ü,Ð-=×?VÑ?VÓWˆÜ×'Ò'°KÔ_Ð_r%   c                ó   € R # rª   r;   r¼   s   &r#   rå   ÚSpmConverter.post_processor  s   € Ùr%   c                óZ   € \        W P                  4      p\        P                  ! WR 7      # r£  )r5   r3   r   r¦  r§  s   &&& r#   ræ   ÚSpmConverter.decoder  s$   € Ü,Ð-=×?VÑ?VÓWˆÜ×!Ò!¨kÔYÐYr%   c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   ÚSpmConverter.__annotate__  s   ø€ ÷ ñ ™9ñ r%   c                ó�  € V P                  V P                  4      pV P                  V P                  4      pVe   W!n        RpRp\        V P                  R4      '       d   V P                  P
                  pV P                  W44      pVe   WQn        V P                  W44      Vn        V P                  4       pV'       d   Wan        V# )Nrr  Tr(   )	rè   rf   rÝ   rÙ   r3   r(   rß   ræ   rå   )rg   rè   rÝ   r¥  r(   rß   rå   s   &      r#   r½   ÚSpmConverter.converted  s¯   € Ø—N‘N 4§:¡:Ó.ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÒ!Ø#-Ô àˆØÐÜ�4×*Ñ*Ð,>×?Ò?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;ÓIˆØÒ$Ø&3Ô#à ŸL™L¨ÓGˆ	ÔØ×,Ñ,Ó.ˆßØ'5Ô$àÐr%   r–   rª   )rˆ   r—   r˜   r™   rƒ  rZ   r–  rÔ   Ústaticmethodr{  Úclassmethodr~  rk   r=   ru   rè   rÝ   rß   rå   ræ   r½   r›   rœ   Ú__classcell__)r‡  r^   s   @@r#   rp  rp  z  sx   ù‡ € Ø ÐØ)€LØ€Nàó-ó ð-ð^ óó ðõò*Fò)ò0òd	hò`òòZ÷÷ ð r%   rp  c                   ó2   a € ] tR tRt o R tR tR tRtV tR# )ÚAlbertConverteri5  c                óä   € VP                    Uu. uFU  p\        VP                  4      '       d   VP                  VP                  3MVP                  VP                  ^d,
          3NKW  	  up# u upi ©éd   ©r…   r±   r†   r‡   r‹  s   && r#   r=   ÚAlbertConverter.vocab6  óf   € ð Ÿšó
á%�ô +=¸U¿[¹[×*IÒ*IˆU�[‰[˜%Ÿ+™+Ñ&ÐPU×P[ÑP[Ð]b×]hÑ]hÐknÕ]nÐOoÒoÙ%ñ
ð 	
ùò 
ó   �AA-c                ó¶  € \         P                  ! R R4      \         P                  ! RR4      .pV P                  P                  '       gI   VP	                  \         P
                  ! 4       4       VP	                  \         P                  ! 4       4       V P                  P                  '       d%   VP	                  \         P                  ! 4       4       VP                  P                  pV'       d&   VP	                  \         P                  ! V4      4       VP	                  \         P                  ! \        R4      R4      4       \         P                  ! V4      # ©z``Ú"z''rž  rö   ©r	   ru  r3   Úkeep_accentsrM   ÚNFKDÚStripAccentsrÛ   Ú	LowercaserŒ   r}   rw  r   rL  ©rg   rf   Úlist_normalizersr}   s   &&  r#   rÝ   ÚAlbertConverter.normalizer<  óõ   € ä×Ò  cÓ*Ü×Ò  cÓ*ð
Ðð ×&Ñ&×3×3Ð3Ø×#Ñ#¤K×$4Ò$4Ó$6Ô7Ø×#Ñ#¤K×$<Ò$<Ó$>Ô?Ø×"Ñ"×0×0Ð0Ø×#Ñ#¤K×$9Ò$9Ó$;Ô<à$×4Ñ4×IÑIÐçØ×#Ñ#¤K×$;Ò$;Ð<PÓ$QÔRà×Ñ¤× 3Ò 3´E¸'³NÀCÓ HÔIÜ×#Ò#Ð$4Ó5Ð5r%   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # ©rj  rk  rl  rm  rÑ   ©r   rä   r3   rù   r¼   s   &r#   rå   ÚAlbertConverter.post_processorO  óR   € Ü×,Ò,Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðô
ð 	
r%   r;   N©	rˆ   r—   r˜   r™   r=   rÝ   rå   r›   rœ   r�   s   @r#   rµ  rµ  5  ó   ø‡ € ò
ò6÷&
ð 
r%   rµ  c                   ó,   a € ] tR tRt o R tR tRtV tR# )ÚBarthezConverteriZ  c                ó
   € ^pV# ©r€   r;   ©rg   rf   ru   s   && r#   ru   ÚBarthezConverter.unk_id[  ó   € ØˆØˆr%   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # ©z<s> $A </s>z<s> $A </s> </s> $B </s>ú<s>ú</s>rÑ   rË  r¼   s   &r#   rå   ÚBarthezConverter.post_processor_  óR   € Ü×,Ò,Ø Ø+à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r%   r;   N)rˆ   r—   r˜   r™   ru   rå   r›   rœ   r�   s   @r#   rÑ  rÑ  Z  s   ø‡ € ò÷
ð 
r%   rÑ  c                   óF   a € ] tR tRt o R tR tR t]RR l4       tRt	V t
R# )	ÚCamembertConverterij  c                ó¨   € . ROpY!P                   R,           Uu. uF  q3P                  VP                  3NK  	  up,          pVR.,          pV# u upi )ú
<s>NOTUSEDr8  )©rà  ç        ©ú<pad>râ  ©z</s>NOTUSEDrâ  ©ú<unk>râ  )ú<unk>NOTUSEDiœÿÿÿ©ú<mask>râ  rŠ  ©rg   rf   r=   r†   s   &&  r#   r=   ÚCamembertConverter.vocabk  sP   € ò
ˆð 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØ�/Ð"Õ"ˆØˆùò Ló   š Ac                ó   € ^# rÓ  r;   rŽ  s   &&r#   ru   ÚCamembertConverter.unk_idx  s   € ár%   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # rØ  rË  r¼   s   &r#   rå   Ú!CamembertConverter.post_processor|  rÜ  r%   Nc                ó6  € \        VP                  R R4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      pR	VR3R
VR3R.pVe"   VP                  \        V4      R,          4       VP	                  VR34       WbR&   V# )Ú	pad_tokenrä  rÆ   rç  Ú
mask_tokenrê  râ  r8  r=   rá  rå  )rè  ç      YÀ©r+   r¨   rO   rr   rM   )ré   r=   r�   ró  rÆ   rô  Ú
vocab_lists   &&,    r#   r~  Ú#CamembertConverter.convert_from_spm†  s    € ä˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð  Ø˜ÐØ Ø˜ÐØ$ð
ˆ
ð ÒØ×Ñœd 5›k¨"�oÔ.Ø×Ñ˜: sÐ+Ô,Ø$ˆw‰Øˆr%   r;   rª   ©rˆ   r—   r˜   r™   r=   ru   rå   r²  r~  r›   rœ   r�   s   @r#   rÞ  rÞ  j  s(   ø‡ € òòò
ð óó ör%   rÞ  c                   ó2   a € ] tR tRt o R tR tR tRtV tR# )ÚDebertaV2Converteriš  c                ó,  € . pV P                   P                  '       d'   VP                  \        P                  ! R R7      4       \        W P                   4      pVP                  \        P                  ! WR7      4       \        P                  ! V4      # )rE  )rG  r¤  )r3   Úsplit_by_punctrM   r
   ÚPunctuationr5   r¦  rL  )rg   r¥  r(   Úlist_pretokenizersr4   s   &&&  r#   rß   Ú DebertaV2Converter.pre_tokenizer›  so   € ØÐØ×"Ñ"×1×1Ð1Ø×%Ñ%¤n×&@Ò&@È*Ô&UÔVÜ,Ð-=×?VÑ?VÓWˆØ×!Ñ!¤.×":Ò":À{Ô"rÔsÜ×&Ò&Ð'9Ó:Ð:r%   c                óÞ  € . pV P                   P                  '       d%   VP                  \        P                  ! 4       4       VP                  \        P
                  ! 4       4       VP                  P                  pV'       d&   VP                  \        P                  ! V4      4       VP                  \        P                  ! \        R 4      R4      4       \        P                  ! V4      # )rž  rö   )r3   rÛ   rM   r	   rÄ  rŸ  rŒ   r}   rw  ru  r   rL  rÅ  s   &&  r#   rÝ   ÚDebertaV2Converter.normalizer£  s¨   € ØÐØ×"Ñ"×0×0Ð0Ø×#Ñ#¤K×$9Ò$9Ó$;Ô<Ø×Ñ¤× 1Ò 1Ó 3Ô4à$×4Ñ4×IÑIÐßØ×#Ñ#¤K×$;Ò$;Ð<PÓ$QÔRØ×Ñ¤× 3Ò 3´E¸'³NÀCÓ HÔIä×#Ò#Ð$4Ó5Ð5r%   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # rÊ  rË  r¼   s   &r#   rå   Ú!DebertaV2Converter.post_processor°  rÍ  r%   r;   N)	rˆ   r—   r˜   r™   rß   rÝ   rå   r›   rœ   r�   s   @r#   rû  rû  š  s   ø‡ € ò;ò6÷
ð 
r%   rû  c                   óF   a € ] tR tRt o R tR tR t]RR l4       tRt	V t
R# )	ÚMBartConverteri»  c                ó¾   € . ROpY!P                   R,           Uu. uF  q3P                  VP                  3NK  	  up,          pV. RO,          pVR.,          pV# u upi )rÙ  ºr€   NN©©rÙ  râ  rã  ©rÚ  râ  ræ  )©Úar_ARrâ  ©Úcs_CZrâ  ©Úde_DErâ  ©Úen_XXrâ  ©Úes_XXrâ  ©Úet_EErâ  ©Úfi_FIrâ  ©Úfr_XXrâ  ©Úgu_INrâ  ©Úhi_INrâ  ©Úit_ITrâ  ©Úja_XXrâ  ©Úkk_KZrâ  ©Úko_KRrâ  ©Últ_LTrâ  ©Úlv_LVrâ  ©Úmy_MMrâ  ©Úne_NPrâ  ©Únl_XXrâ  ©Úro_ROrâ  ©Úru_RUrâ  ©Úsi_LKrâ  ©Útr_TRrâ  ©Úvi_VNrâ  ©Úzh_CNrâ  ré  rŠ  rë  s   &&  r#   r=   ÚMBartConverter.vocab¼  sa   € ò
ˆð 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØò 
õ 	
ˆð6 	�/Ð"Õ"ˆØˆùò; Ls   š Ac                ó   € ^# rÓ  r;   rŽ  s   &&r#   ru   ÚMBartConverter.unk_idâ  ó   € Ùr%   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # )z$A </s> en_XXz$A $B </s> en_XXr  rÚ  rÑ   rË  r¼   s   &r#   rå   ÚMBartConverter.post_processorå  óR   € Ü×,Ò,Ø"Ø#à˜$×1Ñ1×GÑGÈÓPÐQØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r%   Nc                óà  € \        VP                  R R4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR	4      4      pVR
3VR
3VR
3VR
3.pVe"   VP                  \        V4      R,          4       VP                  R \         4       4       VP                  VR
34       W‚R&   V# )r-  rÙ  ró  rä  Ú	eos_tokenrÚ  rÆ   rç  rô  rê  râ  r  c              3   ó(   "  € T F  qR 3x € K
  	  R# 5i©râ  Nr;   ©Ú.0Ú	lang_codes   & r#   Ú	<genexpr>Ú2MBartConverter.convert_from_spm.<locals>.<genexpr>ÿ  s   é € ÐL¹O¨y cÕ*»Oùó   ‚r=   )r+   r¨   rO   rr   ÚMBART_LANGUAGESrM   ©	ré   r=   r�   r-  ró  rF  rÆ   rô  r÷  s	   &&,      r#   r~  ÚMBartConverter.convert_from_spmï  så   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÒØ×Ñœd 5›k¨"�oÔ.Ø×ÑÑL½OÓLÔLØ×Ñ˜: sÐ+Ô,Ø$ˆw‰Øˆr%   r;   rª   rù  r�   s   @r#   r  r  »  s)   ø‡ € ò$òLò
ð óó ör%   r  c                   óF   a € ] tR tRt o R tR tR t]RR l4       tRt	V t
R# )	ÚMBart50Converteri  c                óŠ  € . ROpY!P                   R,           Uu. uF  q3P                  VP                  3NK  	  up,          pV. RNRNRNRNRNRNR	NR
NRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNR NR!NR"NR#NR$NR%NR&NR'NR(NR)NR*NR+NR,NR-NR.NR/NR0NR1NR2NR3NR4NR5NR6N,          pVR7.,          pV# u upi )8rÙ  r  r	  r  r  r  r  r  r  r  r  r  r  r   r"  r$  r&  r(  r*  r,  r.  r0  r2  r4  r6  r8  r:  r<  )Úaf_ZArâ  )Úaz_AZrâ  )Úbn_INrâ  )Úfa_IRrâ  )Úhe_ILrâ  )Úhr_HRrâ  )Úid_IDrâ  )Úka_GErâ  )Úkm_KHrâ  )Úmk_MKrâ  )Úml_INrâ  )Úmn_MNrâ  )Úmr_INrâ  )Úpl_PLrâ  )Úps_AFrâ  )Úpt_XXrâ  )Úsv_SErâ  )Úsw_KErâ  )Úta_INrâ  )Úte_INrâ  )Úth_THrâ  )Útl_XXrâ  )Úuk_UArâ  )Úur_PKrâ  )Úxh_ZArâ  )Úgl_ESrâ  )Úsl_SIrâ  ré  rŠ  rë  s   &&  r#   r=   ÚMBart50Converter.vocab  sé  € ò
ˆð 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØð  R�.ð  R .ð  R°.ð  RÀ.ð  RÐR`ð  RÐbpð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sA	ð  Rð  C	Q	ð  Rð  S	a	ð  Rð  c	q	ð  Rð  s	A
ð  Rð  C
Q
ð  Rð  S
a
ð  Rð  c
q
ð  Rð  s
Að  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rð  Sað  Rð  cqð  Rð  sAð  Rð  CQð  Rõ  	RˆØ�/Ð"Õ"ˆØˆùò Ls   š C c                ó   € ^# rÓ  r;   rŽ  s   &&r#   ru   ÚMBart50Converter.unk_id  rA  r%   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # )zen_XX $A </s>zen_XX $A $B </s>r  rÚ  rÑ   rË  r¼   s   &r#   rå   ÚMBart50Converter.post_processor  rD  r%   Nc                óà  € \        VP                  R R4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR	4      4      pVR
3VR
3VR
3VR
3.pVe"   VP                  \        V4      R,          4       VP                  R \         4       4       VP                  VR
34       W‚R&   V# )rà   rÙ  ró  rä  rF  rÚ  rÆ   rç  rô  rê  râ  r  c              3   ó(   "  € T F  qR 3x € K
  	  R# 5irH  r;   rI  s   & r#   rL  Ú4MBart50Converter.convert_from_spm.<locals>.<genexpr>/  s   é € ÐNÑ<M¨y cÕ*Ó<MùrN  r=   )r+   r¨   rO   rr   ÚMBART50_LANGUAGESrM   )	ré   r=   r�   rà   ró  rF  rÆ   rô  r÷  s	   &&,      r#   r~  Ú!MBart50Converter.convert_from_spm  sæ   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÒØ×Ñœd 5›k¨"�oÔ.Ø×ÑÑNÕ<MÓNÔNØ×Ñ˜: sÐ+Ô,Ø$ˆw‰Øˆr%   r;   rª   rù  r�   s   @r#   rS  rS    s(   ø‡ € ò
òò
ð óó ör%   rS  c                   óF   a € ] tR tRt o R tR tR t]RR l4       tRt	V t
R# )	ÚNllbConverteri5  c                ó”   € . ROpY!P                   R,           Uu. uF  q3P                  VP                  3NK  	  up,          pV# u upi )rÙ  r  r	  rŠ  rë  s   &&  r#   r=   ÚNllbConverter.vocab6  óC   € ò
ˆð 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØˆùò Ló   š Ac                ó   € ^# rÓ  r;   rŽ  s   &&r#   ru   ÚNllbConverter.unk_id@  rA  r%   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # )zeng_Latn $A </s>zeng_Latn $A $B </s>Úeng_LatnrÚ  rÑ   rË  r¼   s   &r#   rå   ÚNllbConverter.post_processorC  sR   € Ü×,Ò,Ø%Ø&à˜T×4Ñ4×JÑJÈ:ÓVÐWØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r%   Nc                óÂ  € \        VP                  R R4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      pV^ V^V^V^/pVe[   \        V\        4      '       d   VP	                  4       MV UU	u. uF  w  r‰VNK	  	  up	pp
V
 F  pW·9   d   K  \        V4      W{&   K  	  WrR&   V# u up	pi )	r-  rÙ  ró  rä  rF  rÚ  rÆ   rç  r=   )r+   r¨   rt  rJ   r  rD   )ré   r=   r�   r-  ró  rF  rÆ   Úreordered_vocabÚtokr™  Útokensr“   s   &&,         r#   r~  ÚNllbConverter.convert_from_spmM  sÝ   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	ð �qØ�qØ�qØ�qð	
ˆð ÒÜ%/°´t×%<Ò%<�U—Z‘Z”\ÑUZÔB[ÑUZÉ6È3Ã3ÑUZÒB[ˆFÛ�ØÔ+ÙÜ),¨_Ó)=�Ó&ñ  ð *ˆw‰Øˆùó C\s   Â&Cr;   rª   rù  r�   s   @r#   r{  r{  5  s(   ø‡ € òòò
ð óó ör%   r{  c                   ó2   a € ] tR tRt o R tR tR tRtV tR# )ÚSeamlessM4TConverterid  c                ó”   € . ROpY!P                   R,           Uu. uF  q3P                  VP                  3NK  	  up,          pV# u upi )rä  r  )rã  ræ  r
  r  rŠ  rë  s   &&  r#   r=   ÚSeamlessM4TConverter.vocabe  r~  r  c                ó.   € V P                   P                  # rª   )r3   Úunk_token_idrŽ  s   &&r#   ru   ÚSeamlessM4TConverter.unk_ido  s   € Ø×&Ñ&×3Ñ3Ð3r%   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # )z__eng__ $A </s>z__eng__ $A $B </s>Ú__eng__rÚ  rÑ   rË  r¼   s   &r#   rå   Ú#SeamlessM4TConverter.post_processorr  sR   € Ü×,Ò,Ø$Ø%à˜D×3Ñ3×IÑIÈ)ÓTÐUØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r%   r;   N©	rˆ   r—   r˜   r™   r=   ru   rå   r›   rœ   r�   s   @r#   r‹  r‹  d  s   ø‡ € òò4÷
ð 
r%   r‹  c                   óF   a € ] tR tRt o R tR tR t]RR l4       tRt	V t
R# )	ÚXLMRobertaConverteri}  c                ó¨   € . ROpY!P                   R,           Uu. uF  q3P                  VP                  3NK  	  up,          pVR.,          pV# u upi )rÙ  r  r	  ré  rŠ  rë  s   &&  r#   r=   ÚXLMRobertaConverter.vocab~  sP   € ò
ˆð 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØ�/Ð"Õ"ˆØˆùò Lrí  c                ó
   € ^pV# rÓ  r;   rÔ  s   && r#   ru   ÚXLMRobertaConverter.unk_id‰  rÖ  r%   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # rØ  rË  r¼   s   &r#   rå   Ú"XLMRobertaConverter.post_processor�  rÜ  r%   Nc                ó¨  € \        VP                  R R4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR	4      4      pVR
3VR
3VR
3VR
3.pVe"   VP                  \        V4      R,          4       VP	                  VR
34       W‚R&   V# )r-  rÙ  ró  rä  rF  rÚ  rÆ   rç  rô  rê  râ  r  r=   rö  rP  s	   &&,      r#   r~  Ú$XLMRobertaConverter.convert_from_spm—  sÑ   € ä˜Ÿ
™
 ;°Ó6Ó7ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ™ L°(Ó;Ó<ˆ
ð ˜ÐØ˜ÐØ˜ÐØ˜Ðð	
ˆ
ð ÒØ×Ñœd 5›k¨"�oÔ.Ø×Ñ˜: sÐ+Ô,Ø$ˆw‰Øˆr%   r;   rª   rù  r�   s   @r#   r–  r–  }  s(   ø‡ € ò	òò
ð óó ör%   r–  c                   ó2   a € ] tR tRt o R tR tR tRtV tR# )ÚXLNetConverteri¬  c                óä   € VP                    Uu. uFU  p\        VP                  4      '       d   VP                  VP                  3MVP                  VP                  ^d,
          3NKW  	  up# u upi r·  r¹  r‹  s   && r#   r=   ÚXLNetConverter.vocab­  r»  r¼  c                ó¶  € \         P                  ! R R4      \         P                  ! RR4      .pV P                  P                  '       gI   VP	                  \         P
                  ! 4       4       VP	                  \         P                  ! 4       4       V P                  P                  '       d%   VP	                  \         P                  ! 4       4       VP                  P                  pV'       d&   VP	                  \         P                  ! V4      4       VP	                  \         P                  ! \        R4      R4      4       \         P                  ! V4      # r¾  rÀ  rÅ  s   &&  r#   rÝ   ÚXLNetConverter.normalizer³  rÈ  r%   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # )z$A:0 <sep>:0 <cls>:2z!$A:0 <sep>:0 $B:1 <sep>:1 <cls>:2z<sep>z<cls>rÑ   rË  r¼   s   &r#   rå   ÚXLNetConverter.post_processorÆ  rÍ  r%   r;   NrÎ  r�   s   @r#   r   r   ¬  rÏ  r%   r   c                   ó   € ] tR tRtRtR# )ÚReformerConverteriÑ  r;   N©rˆ   r—   r˜   r™   r›   r;   r%   r#   r¨  r¨  Ñ  ó   † Ûr%   r¨  c                   ó,   a € ] tR tRt o R tR tRtV tR# )ÚRemBertConverteriÕ  c                ó–  € \         P                  ! R R4      \         P                  ! RR4      \         P                  ! \        R4      R4      .pV P                  P                  '       gI   VP                  \         P                  ! 4       4       VP                  \         P                  ! 4       4       V P                  P                  '       d%   VP                  \         P                  ! 4       4       VP                  P                  pV'       d&   VP                  \         P                  ! V4      4       \         P                  ! V4      # r¾  )r	   ru  r   r3   rÁ  rM   rÂ  rÃ  rÛ   rÄ  rŒ   r}   rw  rL  rÅ  s   &&  r#   rÝ   ÚRemBertConverter.normalizer×  sê   € ä×Ò  cÓ*Ü×Ò  cÓ*Ü×Ò¤ g£°Ó4ð
Ðð
 ×&Ñ&×3×3Ð3Ø×#Ñ#¤K×$4Ò$4Ó$6Ô7Ø×#Ñ#¤K×$<Ò$<Ó$>Ô?Ø×"Ñ"×0×0Ð0Ø×#Ñ#¤K×$9Ò$9Ó$;Ô<à$×4Ñ4×IÑIÐçØ×#Ñ#¤K×$;Ò$;Ð<PÓ$QÔRä×#Ò#Ð$4Ó5Ð5r%   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # rÊ  rË  r¼   s   &r#   rå   ÚRemBertConverter.post_processorê  rÍ  r%   r;   N)rˆ   r—   r˜   r™   rÝ   rå   r›   rœ   r�   s   @r#   r¬  r¬  Õ  s   ø‡ € ò6÷&
ð 
r%   r¬  c                   ó   € ] tR tRtRtR# )ÚBertGenerationConverteriõ  r;   Nr©  r;   r%   r#   r²  r²  õ  rª  r%   r²  c                   óL   a € ] tR tRt o R t]R	R l4       tR tR tR t	Rt
V tR# )
ÚPegasusConverteriù  c                óª  € V P                   P                  R 3V P                   P                  R 3.pV P                   P                  e    W P                   P                  R 3.,          pV P                   P                  eO   V P                   P
                  V P                   P                  8  d    W P                   P                  R 3.,          pT\        ^V P                   P                  4       Uu. uF  pRV R2R3NK  	  up,          pY!P                  R,           Uu. uF  qDP                  VP                  3NK  	  up,          pV# u upi u upi )râ  ú<unk_Ú>ºr'   NNrõ  )r3   ró  rF  Úmask_token_sentrô  Úmask_token_idÚoffsetrL   r…   r†   r‡   )rg   rf   r=   rŽ   r†   s   &&   r#   r=   ÚPegasusConverter.vocabú  s'  € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 ×"Ñ"×2Ñ2Ò>Ø×.Ñ.×>Ñ>ÀÐDÐEÕEˆEð ×#Ñ#×.Ñ.Ò:Ø×'Ñ'×5Ñ5¸×8OÑ8O×8VÑ8VÔVà×.Ñ.×9Ñ9¸3Ð?Ð@Õ@ˆEà´%¸¸4×;RÑ;R×;YÑ;YÔ2ZÓ[Ñ2Z¨Q�U˜1˜#˜Q�< Ó(Ñ2ZÑ[Õ[ˆØ¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØˆùò \ùÚKs   Ã1EÄ  ENc           	     ó@  € \        VP                  R R4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      p\        VP                  RR4      4      pVR3VR3.pVR	8w  d   VP                  VR34       VR	8w  d   We8w  d   VP                  VR34       TP                  \	        ^VP                  R
^g4      4       Uu. uF  pRV R2R3NK  	  up4       Ve"   VP                  \        V4      R,          4       WrR&   V# u upi )ró  rä  rF  rÚ  rô  z<mask_1>r¹  z<mask_2>râ  ÚNoner»  r¶  r·  r¸  r=   rõ  )r+   r¨   rM   rO   rL   rr   )	ré   r=   r�   ró  rF  rô  r¹  r÷  rŽ   s	   &&,      r#   r~  Ú!PegasusConverter.convert_from_spm  s  € ä˜Ÿ
™
 ;°Ó8Ó9ˆ	Ü˜Ÿ
™
 ;°Ó7Ó8ˆ	Ü˜Ÿ™ L°*Ó=Ó>ˆ
Ü˜fŸj™jÐ):¸JÓGÓHˆð ˜ÐØ˜Ðð
ˆ
ð ˜ÔØ×Ñ˜z¨3Ð/Ô0Ø˜fÔ$¨Ô)FØ×Ñ˜°Ð4Ô5à×Ñ¼5ÀÀFÇJÁJÈxÐY\ÓD]Ô;^Ó_Ñ;^°a˜e A 3 a˜L¨&Ó1Ñ;^Ñ_Ô`ØÒØ×Ñœd 5›k¨"�oÔ.Ø$ˆw‰Øˆùò	 `s   ÃDc                ód   € VP                   P                  V P                  P                  ,           # rª   )r‚   ru   r3   r»  rŽ  s   &&r#   ru   ÚPegasusConverter.unk_id#  s%   € Ø×!Ñ!×(Ñ(¨4×+BÑ+B×+IÑ+IÕIÐIr%   c                ó¬   € \        W P                  4      p\        P                  ! \        P                  ! 4       \        P
                  ! WR 7      .4      # r£  )r5   r3   r
   rL  ÚWhitespaceSplitr¦  r§  s   &&& r#   rß   ÚPegasusConverter.pre_tokenizer&  sE   € Ü,Ð-=×?VÑ?VÓWˆÜ×&Ò&ä×.Ò.Ó0Ü×(Ò(°[Ô`ðó
ð 	
r%   c                óš   € V P                   P                  pWP                   P                  3.p\        P                  ! R V.R RV.VR7      # )ú$Aú$BrÑ   )r3   rF  Úeos_token_idr   rä   )rg   ÚeosrÔ   s   &  r#   rå   ÚPegasusConverter.post_processor/  sP   € Ø×%Ñ%×/Ñ/ˆà×)Ñ)×6Ñ6Ð7ð
ˆô ×,Ò,°T¸3°KÀtÈTÐSVÐFWÐhvÔwÐwr%   r;   rª   )rˆ   r—   r˜   r™   r=   r²  r~  ru   rß   rå   r›   rœ   r�   s   @r#   r´  r´  ù  s5   ø‡ € òð& óó ðò*Jò
÷xð xr%   r´  c                   ó@   a € ] tR tRt o R tR t]RR l4       tRtV t	R# )ÚT5Converteri7  c                ó  € V P                   P                  pVP                   Uu. uF  q3P                  VP                  3NK  	  ppT\        V^,
          RR4       Uu. uF  pRV R2R3NK  	  up,          pV# u upi u upi )r9  ú
<extra_id_r·  râ  r®   )r3   Ú
_extra_idsr…   r†   r‡   rL   )rg   rf   Únum_extra_idsr†   r=   rŽ   s   &&    r#   r=   ÚT5Converter.vocab8  s|   € Ø×/Ñ/×:Ñ:ˆØ9>¿ºÓF¹°—+‘+˜uŸ{™{Ó+¹ˆÐFØ´E¸-È!Õ:KÈRÐQSÔ4TÓUÑ4T¨q�Z ˜s !Ð$ cÓ*Ñ4TÑUÕUˆØˆùò GùÚUs   ¥ A:ÁA?c                ót   € \         P                  ! R R.. RORV P                  P                  R4      3.R7      # ©rÆ  rÚ  rÑ   )rÆ  rÚ  rÇ  rÚ  rË  r¼   s   &r#   rå   ÚT5Converter.post_processor>  ó=   € Ü×,Ò,Ø˜&�>Ú-à˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r%   Nc                ó   € VP                  R ^d4      p\        V^,
          RR4       Uu. uF	  pRV R2NK  	  ppVe   \        V4      M. pVP                  R V 4       4       VP	                  RV4       WbR&   V# u upi )Ú	extra_idsrÎ  r·  c              3   ó(   "  € T F  qR 3x € K
  	  R# 5irH  r;   )rJ  r“   s   & r#   rL  Ú/T5Converter.convert_from_spm.<locals>.<genexpr>L  s   é € ÐA±L¨5 #�,³LùrN  r|   r=   r®   )r¨   rL   rr   rO   Ú
setdefault)ré   r=   r�   r×  rŽ   Úextra_tokensr÷  s   &&,    r#   r~  ÚT5Converter.convert_from_spmG  s…   € à—J‘J˜{¨CÓ0ˆ	Ü38¸ÀQ½ÈÈBÔ3OÓPÑ3O¨a˜* Q C qÓ)Ñ3OˆÐPØ$)Ò$5”T˜%”[¸2ˆ
Ø×ÑÑA±LÓAÔAà×ÑÐ5°|ÔDØ$ˆw‰Øˆùò Qs   ©A;r;   rª   )
rˆ   r—   r˜   r™   r=   rå   r²  r~  r›   rœ   r�   s   @r#   rÌ  rÌ  7  s#   ø‡ € òò
ð óó ör%   rÌ  c                   ó&   a € ] tR tRt o R tRtV tR# )ÚUdopConverteriS  c                ót   € \         P                  ! R R.. RORV P                  P                  R4      3.R7      # rÓ  rË  r¼   s   &r#   rå   ÚUdopConverter.post_processorT  rÕ  r%   r;   N©rˆ   r—   r˜   r™   rå   r›   rœ   r�   s   @r#   rÞ  rÞ  S  s   ø‡ € ÷
ð 
r%   rÞ  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚWhisperConverteri^  c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   ÚWhisperConverter.__annotate___  s   ø€ ÷  ñ  ™9ñ  r%   c                óÌ  € V P                   P                  p\        V P                   P                  P	                  4       4      p\        \        VVR RRRR7      4      p\        P                  ! V P                   P                  R7      Vn
        \        P                  ! 4       Vn        V P                   P                  pV P                   P                  V4      pV P                   P                  pV P                   P                   pRP#                  V Uu. uF  qˆ R2NK	  	  up4      p	\$        P&                  ! V	 RV R2V	 RV R	2Wg3.\)        WT4      OR
7      Vn        V# u upi )Nr%  Fr&  r(  rö   rÎ   z $A:0 z $A:0 $B:1 rÐ   rÑ   )r3   r  rr   r  r  r   r   r
   r,  r(   rß   r   ræ   Úprefix_tokensÚconvert_ids_to_tokensrF  rÈ  Újoinr   rä   Úziprå   )
rg   r=   rQ   rè   Úprefix_token_idsÚprefixesrÉ  rÈ  r“   Úprefix_templates
   &         r#   r½   ÚWhisperConverter.converted_  sO  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ò":ÈD×LcÑLc×LtÑLtÔ"uˆ	ÔÜ$×.Ò.Ó0ˆ	Ôà×2Ñ2×@Ñ@ÐØ×*Ñ*×@Ñ@ÐAQÓRˆØ×%Ñ%×/Ñ/ˆØ×.Ñ.×;Ñ;ˆØŸ(™(¹hÓ#G¹h°U g¨R£L¹hÑ#GÓHˆÜ#-×#@Ò#@Ø%Ð& f¨S¨E°Ð4Ø#Ð$ K°¨u°BÐ7àÐ#ðä�XÓ0ðô$
ˆ	Ô ð Ðùò $Hs   ÄE!r;   Nrí   r�   s   @r#   rã  rã  ^  s   ø‡ € ÷ ö  r%   rã  c                   ó&   a € ] tR tRt o R tRtV tR# )ÚBigBirdConverteri‚  c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # rÊ  rË  r¼   s   &r#   rå   ÚBigBirdConverter.post_processorƒ  rÍ  r%   r;   Nrá  r�   s   @r#   rð  rð  ‚  s   ø‡ € ÷
ð 
r%   rð  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚCLIPConverteriŽ  c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   ÚCLIPConverter.__annotate__�  s   ø€ ÷ 'ñ '™9ñ 'r%   c                ó‚  € V P                   P                  p\        V P                   P                  P	                  4       4      pV P                   P
                  p\        \        VVR RRR\        V4      R7      4      p\        P                  ! \        P                  ! 4       \        P                  ! \        R4      R4      \        P                  ! 4       .4      Vn        \         P                  ! \         P"                  ! \        R4      RR	R
7      \         P$                  ! RR7      .4      Vn        \(        P$                  ! 4       Vn        \,        P.                  ! V P                   P0                  V P                   P2                  3V P                   P4                  V P                   P6                  3RRR7      Vn        V# )Nr%  r  F©r=   rQ   r  r'  r  r  rÆ   z\s+rö   z9's|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+ÚremovedTrF  r(  rU  )r3   r  rr   r  r  rÆ   r   r   r+   r	   rL  rK  ru  r   rÄ  rÝ   r
   rM  r,  rß   r   ræ   r   rV  rF  rÈ  r-  r.  rå   r  s   &    r#   r½   ÚCLIPConverter.converted�  sk  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØØ*,Ø#)ØÜ˜i›.ôó

ˆ	ô  +×3Ò3Ü�_Š_Ó¤× 3Ò 3´E¸&³MÀ3Ó GÌ×I^ÒI^ÓI`Ðaó 
ˆ	Ôô #1×"9Ò"9ä×$Ò$ÜÐZÓ[Ø&Øôô
 ×(Ò(¸%Ô@ðó	#
ˆ	Ôô %×.Ò.Ó0ˆ	Ôô $.×#?Ò#?Ø×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ"Øô	$
ˆ	Ô ð Ðr%   r;   Nrí   r�   s   @r#   rô  rô  Ž  s   ø‡ € ÷'ö 'r%   rô  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚLayoutLMv2Converteri¹  c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   Ú LayoutLMv2Converter.__annotate__º  rÃ   r%   c           	     ó|  € V P                   P                  p\        \        V\	        V P                   P
                  4      R 7      4      pRpRpRp\        V P                   R4      '       da   V P                   P                  P                  pV P                   P                  P                  pV P                   P                  P                  p\        P                  ! RVVVR7      Vn        \        P                  ! 4       Vn        \	        V P                   P"                  4      p\	        V P                   P$                  4      pV P                   P&                  pV P                   P(                  p	\*        P,                  ! V RV R2V RV RV R2Wh3Wy3.R	7      Vn        \0        P                  ! R
R7      Vn        V# )rÅ   FTrÇ   rÈ   rÍ   rÎ   rÏ   rÐ   rÑ   rÕ   rÖ   rØ   rç   s
   &         r#   r½   ÚLayoutLMv2Converter.convertedº  s„  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=×>Ò>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ò9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÒ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5àÐ#ØÐ#ðô$
ˆ	Ô ô %×.Ò.°dÔ;ˆ	ÔàÐr%   r;   Nrí   r�   s   @r#   rü  rü  ¹  rî   r%   rü  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚBlenderbotConverterià  c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   Ú BlenderbotConverter.__annotate__á  rS  r%   c                ó´  € V P                   pVP                  p\        VP                  P	                  4       4      p\        \        VVR RRRR7      4      p\        P                  ! VP                  R7      Vn
        \        P                  ! 4       Vn        \        P                  ! RVP                   R2VP                  VP                   3.R7      Vn        V# )Nr%  Fr&  r(  z$A:0 rÎ   )rÒ   rÔ   )r3   r  rr   r  r  r   r   r
   r,  r(   rß   r   ræ   r   rä   rF  rÈ  rå   rW  s   &    r#   r½   ÚBlenderbotConverter.convertedá  s½   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ò":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ò.Ó0ˆ	ÔÜ#-×#@Ò#@Ø˜2Ÿ<™<˜.¨Ð+à—‘˜rŸ™Ð/ðô$
ˆ	Ô ð Ðr%   r;   Nrí   r�   s   @r#   r  r  à  rZ  r%   r  c                   ó2   a € ] tR tRt o R tR tR tRtV tR# )ÚXGLMConverteriý  c                óª   € . ROpY!P                   R,           Uu. uF  q3P                  VP                  3NK  	  up,          pV. RO,          pV# u upi )rÙ  r  r	  ))z<madeupword0>râ  )z<madeupword1>râ  )z<madeupword2>râ  )z<madeupword3>râ  )z<madeupword4>râ  )z<madeupword5>râ  )z<madeupword6>râ  rŠ  rë  s   &&  r#   r=   ÚXGLMConverter.vocabþ  sT   € ò
ˆð 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØò  zõ  	zˆØˆùò Ls   š Ac                ó
   € ^pV# rÓ  r;   rÔ  s   && r#   ru   ÚXGLMConverter.unk_id	  rÖ  r%   c           	     ó¤   € \         P                  ! R RRV P                  P                  R4      3RV P                  P                  R4      3.R7      # )z</s> $Az</s> $A </s> </s> $BrÙ  rÚ  rÑ   rË  r¼   s   &r#   rå   ÚXGLMConverter.post_processor  sR   € Ü×,Ò,ØØ'à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r%   r;   Nr”  r�   s   @r#   r  r  ý  s   ø‡ € ò	ò÷
ð 
r%   r  c                   óP   a € ] tR tRt o Rt]tRR0t R tR t	R t
R tR	 tR
tV tR# )ÚGemmaConverteri  Tz<start_of_turn>z<end_of_turn>c                ó0   € \         P                  ! R R4      # ©rö   rr  )r	   ru  rŽ  s   &&r#   rÝ   ÚGemmaConverter.normalizer(  s   € Ü×"Ò" 3¨Ó.Ð.r%   c                óÖ  € V P                   P                  R 3V P                   P                  R 3V P                   P                  R 3.pY!P                  R,           Uu. uF  q3P
                  VP                  3NK  	  up,          p\        ;QJ d    R V 4       F  '       g   K   RM	  RM! R V 4       4      '       g%   \        R \        V4       4       R4      pVe   RW$&   V# u upi )râ  r  c              3   ó8   "  € T F  q^ ,          R8H  x € K  	  R# 5i)r   r£   Nr;   )rJ  r<   s   & r#   rL  Ú'GemmaConverter.vocab.<locals>.<genexpr>4  s   é € Ð/© A�Q•4˜4–<«ùs   ‚TFc              3   óJ   "  € T F  w  rV^ ,          R8X  g   K  Vx € K  	  R# 5i)r   r¤   Nr;   )rJ  rŽ   r<   s   &  r#   rL  r  5  s!   é € Ð"VÑ1A©¨ÀQÀqÅTÈXÑEU§1¢1Ó1Aùs   ‚#™
#N)r£   râ  )
r3   ró  rF  r-  r…   r†   r‡   ÚanyÚnextrŠ   )rg   rf   r=   r†   Úoverride_indexs   &&   r#   r=   ÚGemmaConverter.vocab+  sÈ   € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆ÷ ‹sÑ/©Ó/�s�sŠsÑ/©Ó/×/Ò/Ü!Ñ"V´¸5Ô1AÓ"VÐX\Ó]ˆNØÒ)Ø(3�Ñ%àˆùò Ls   Á C&c                ó0   € \         P                  ! R R4      # )rö   Úmerged_with_previous)r
   rM  ©rg   r¥  r(   s   &&&r#   rß   ÚGemmaConverter.pre_tokenizer;  s   € Ü×#Ò# CÐ)?Ó@Ð@r%   c                ó
   € ^pV# rÓ  r;   rÔ  s   && r#   ru   ÚGemmaConverter.unk_id>  rÖ  r%   c                óª   € \         P                  ! \         P                  ! R R4      \         P                  ! 4       \         P                  ! 4       .4      # )rr  rö   )r   rL  ru  rx  ry  r  s   &&&r#   ræ   ÚGemmaConverter.decoderB  s?   € Ü× Ò ä× Ò  ¨Ó,Ü×%Ò%Ó'Ü—’“ðó
ð 	
r%   r;   N)rˆ   r—   r˜   r™   rƒ  rŸ   r–  rÔ   rÝ   r=   rß   ru   ræ   r›   rœ   r�   s   @r#   r  r    s>   ø‡ € ØÐØ.€Là'¨Ð9€Nðò/òò Aò÷
ð 
r%   r  c                   óH   a € ] tR tRt o RtR tR tR tR tR t	R t
R	tV tR
# )ÚLlamaConverteriL  Tc                ó8  € V P                   P                  ^ 4      R3V P                   P                  ^4      R3V P                   P                  ^4      R3.pY!P                  R,           Uu. uF  q3P                  VP                  3NK  	  up,          pV# u upi )r   râ  r  )r3   rè  r…   r†   r‡   rë  s   &&  r#   r=   ÚLlamaConverter.vocabO  sŽ   € à×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCð
ˆð
 	¿,¹,ÀrÖ:JÓKÑ:J°—;‘; §¡Ó,Ñ:JÑKÕKˆØˆùò Ls   Á, Bc                ó
   € ^ pV# r:   r;   rÔ  s   && r#   ru   ÚLlamaConverter.unk_idX  rÖ  r%   c                óþ   € \         P                  ! R R4      \         P                  ! 4       \         P                  ! 4       .pV'       d!   V\         P                  ! R^R7      .,          p\         P
                  ! V4      # ©rr  rö   )Úcontentr�  ©r   ru  rx  ry  rŸ  rL  ©rg   r¥  r(   Úsequences   &&& r#   ræ   ÚLlamaConverter.decoder\  ó\   € ä×Ò˜U CÓ(Ü×!Ò!Ó#Ü�MŠM‹Oð
ˆ÷
 ØœŸš°¸!Ô<Ð=Õ=ˆHÜ× Ò  Ó*Ð*r%   c                ó(  € \        V P                  R R4      '       du   . p\        V P                  RR4      '       d    V\        P                  ! RR7      .,          pV\        P                  ! RRR7      .,          p\        P
                  ! V4      # R# )r/   Tr(   rr  )Úprependrö   )Úpatternr,  N)r2   r3   r	   ÚPrependru  rL  )rg   rf   r/  s   && r#   rÝ   ÚLlamaConverter.normalizerf  sx   € Ü�4×*Ñ*¨H°d×;Ò;ØˆHÜ�t×.Ñ.Ð0BÀD×IÒIØœ[×0Ò0¸Ô?Ð@Õ@�Øœ×,Ò,°SÀ%ÔHÐIÕIˆHÜ×'Ò'¨Ó1Ð1Ùr%   c                óš   € \        V P                  R R4      '       g.   \        W P                  4      p\        P                  ! WRR7      # R# )r/   TF©r¥  r4   ÚsplitN)r2   r3   r5   r
   r¦  r§  s   &&& r#   rß   ÚLlamaConverter.pre_tokenizero  s?   € Ü�t×.Ñ.°¸$×?Ò?Ü0Ð1A×CZÑCZÓ[ˆNÜ!×+Ò+¸ÐjoÔpÐpÙr%   c                ó   € R # rª   r;   r¼   s   &r#   rå   ÚLlamaConverter.post_processoru  s   € ár%   r;   N)rˆ   r—   r˜   r™   rƒ  r=   ru   ræ   rÝ   rß   rå   r›   rœ   r�   s   @r#   r%  r%  L  s.   ø‡ € ØÐòòò+òò÷ð r%   r%  c                   ó2   a € ] tR tRt o V 3R lR ltRtV tR# )ÚMarkupLMConverteriz  c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   ÚMarkupLMConverter.__annotate__{  s   ø€ ÷ "ñ "™9ñ "r%   c                óŒ  € V P                   pVP                  p\        VP                  P	                  4       4      p\        \        VVR RRRV P                   P                  R7      4      p\        P                  ! VP                  R7      Vn        \        P                  ! 4       Vn        \        V P                   P                  4      p\        V P                   P                   4      pV P                   P"                  pV P                   P$                  p\&        P(                  ! V RV 2V RV RV 2WW3Wh3.R7      Vn        V# )Nr%  Frø  r(  z $A z $B rÑ   )r3   r  rr   r  r  r   r   rÆ   r
   r,  r(   rß   r   ræ   r+   rà   rá   râ   rã   r   rä   rå   )	rg   rX  r=   rQ   rè   ré   rê   râ   rã   s	   &        r#   r½   ÚMarkupLMConverter.converted{  s(  € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%ØØ×1Ñ1×;Ñ;ôó

ˆ	ô #1×":Ò":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ò.Ó0ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ò#@Ø�U˜$˜s˜eÐ$Ø�5˜˜S˜E  c UÐ+àÐ#ØÐ#ðô$
ˆ	Ô ð Ðr%   r;   Nrí   r�   s   @r#   r>  r>  z  s   ø‡ € ÷"ö "r%   r>  c                   ó<   a € ] tR tRt o RtR tR tR tR tRt	V t
R# )	ÚMoshiConverteri   Tc                ó&  € \        V R 4       \        P                  W4       \        4       pVP	                  4       p\        VR4      ;_uu_ 4       pVP                  VP                  4       4       RRR4       W@n        R#   + '       g   i     L; i©r   ra   N©	r   r³   rk   r$   rb   rc   rd   re   rf   ©rg   r‚  r�   rh   ri   rj   s   &&,   r#   rk   ÚMoshiConverter.__init__£  sh   € Ü˜$ 
Ô+ä×Ñ˜4Ô,ô $Ó%ˆ	à× Ñ Ó"ˆÜ�*˜d×#Ô# qØ×Ñ˜aŸf™f›hÔ'÷ $àŽ
÷ $×#úó   Á B Â B	c                óþ   € VP                   P                  p\        P                  ! R R4      .pV'       g   \        P                  ! V4      # \        P                  ! \        P
                  ! V4      .V,           4      # r  )rŒ   r}   r	   ru  rL  rw  r   s   &&  r#   rÝ   ÚMoshiConverter.normalizer°  sg   € Ø$×4Ñ4×IÑIÐä×Ò  UÓ+ð
ˆ÷ $Ü×'Ò'¨Ó5Ð5ä×'Ò'¬×)@Ò)@ÐAUÓ)VÐ(WÐZfÕ(fÓgÐgr%   c                óþ   € \         P                  ! R R4      \         P                  ! 4       \         P                  ! 4       .pV'       d!   V\         P                  ! R^R7      .,          p\         P
                  ! V4      # r+  r-  r.  s   &&& r#   ræ   ÚMoshiConverter.decoderº  r1  r%   c                ó6   € R p\         P                  ! WRR7      # )r0   Fr8  )r
   r¦  r§  s   &&& r#   rß   ÚMoshiConverter.pre_tokenizerÄ  s   € Ø ˆÜ×'Ò'°KÐfkÔlÐlr%   r–   N)rˆ   r—   r˜   r™   rƒ  rk   rÝ   ræ   rß   r›   rœ   r�   s   @r#   rD  rD     s'   ø‡ € ØÐòòhò+÷mð mr%   rD  c                   óX   a € ] tR tRt o RtRR ltR tR tR tR t	R	 t
R
 tR tRtV tR# )ÚHeliumConverteriÉ  TNc                ó&  € \        V R 4       \        P                  W4       \        4       pVP	                  4       p\        VR4      ;_uu_ 4       pVP                  VP                  4       4       RRR4       W@n        R#   + '       g   i     L; irF  rG  rH  s   &&,   r#   rk   ÚHeliumConverter.__init__Ì  sf   € Ü˜$ 
Ô+ä×Ñ˜4Ô,ä#Ó%ˆ	à× Ñ Ó"ˆÜ�*˜d×#Ô# qØ×Ñ˜aŸf™f›hÔ'÷ $àŽ
÷ $×#úrJ  c                óp  € V P                  V4      p\        \        VV P                  V4      V P                  R 7      4      p\        VP                  4       UUu. uFT  w  rEVP                  R
9   g   K  YEP                  VP                  ^8H  ;'       g    VP                  V P                  9   3NKV  	  pppTP                  \        VR R7       UUUu. uF  w  rGp\        VRVRR7      NK  	  uppp4       VP                  \        RRRR7      .4       VP                  R^R	7       V# u uppi u upppi )r‘  c                 ó   € V ^ ,          # r:   r;   rw   s   &r#   r>   Ú+HeliumConverter.tokenizer.<locals>.<lambda>í  r•  r%   r@   FT)rz   r{   Úsingle_wordÚ
ry   rä  )ró  Úpad_idr   )r=   r   r   ru   rƒ  rŠ   r…   r‹   r†   rÔ   r˜  rN   r   Úenable_padding)	rg   rf   rP   rè   r�   r‘   r’   r“   r{   s	   &&       r#   rè   ÚHeliumConverter.tokenizerØ  s#  € Ø—z‘z %Ó(ˆÜÜØØ—{‘{ 5Ó)Ø"×7Ñ7ôó
ˆ	ô # 5§<¡<Ô0ô
á0‘�Ø�v‰v˜Ñô IˆR—‘˜!Ÿ&™& A™+×GÐG¨¯©°D×4GÑ4GÑ)GÓHÙ0ð 	ñ 
ð
 	×Ñô +1Ð1AÁ~Õ*Võá*VÑ&�B˜wô ˜5¨U¸GÐQU×VÙ*Vóô	
ð 	×Ñœj¨¸%ÈÔOÐPÔQØ× Ñ ¨7¸1Ð Ô=ØÐùó
ùôs   ÁD+Á4 D+ÂD+ÃD1c                óÀ   € . pVP                    FK  pVP                  R 8X  d   VRVP                  3.,          pK,  W#P                  VP                  3.,          pKM  	  V# )z<0x0A>rY  rŠ  rë  s   &&  r#   r=   ÚHeliumConverter.vocabô  sV   € ØˆØ—\”\ˆEØ�{‰{˜hÔ&Ø˜4 §¡Ð-Ð.Õ.’àŸ;™;¨¯©Ð4Ð5Õ5’ñ	 "ð
 ˆr%   c                ó
   € ^ pV# r:   r;   rÔ  s   && r#   ru   ÚHeliumConverter.unk_idý  rÖ  r%   c                óî   € \         P                  ! R R4      \         P                  ! 4       \         P                  ! 4       .pV\         P                  ! R^R7      .,          p\         P
                  ! V4      # r+  r-  r.  s   &&& r#   ræ   ÚHeliumConverter.decoder  sY   € ä×Ò˜U CÓ(Ü×!Ò!Ó#Ü�MŠM‹Oð
ˆð
 	”X—^’^¨C°aÔ8Ð9Õ9ˆÜ× Ò  Ó*Ð*r%   c                ó„   € \         P                  ! \         P                  ! R 4      \         P                  ! R R4      .4      # r  )r	   rL  r5  ru  rŽ  s   &&r#   rÝ   ÚHeliumConverter.normalizer
  s2   € Ü×#Ò#¤[×%8Ò%8¸Ó%=¼{×?RÒ?RÐSWÐY^Ó?_Ð$`ÓaÐar%   c                óZ   € \         P                  ! \         P                  ! R R4      .4      # )rY  Ú
contiguous)r
   rL  rM  r  s   &&&r#   rß   ÚHeliumConverter.pre_tokenizer  s#   € Ü×&Ò&¬×(<Ò(<¸TÀ<Ó(PÐ'QÓRÐRr%   c                ó>   € \         P                  ! R R.. ROR.R7      # )rÙ  rÆ  rÑ   )rÙ  rÆ  rÙ  rÇ  )rÙ  r9  )r   rä   r¼   s   &r#   rå   ÚHeliumConverter.post_processor  s/   € Ü×,Ò,àØðòð ðô
ð 	
r%   r–   rª   )rˆ   r—   r˜   r™   rƒ  rk   rè   r=   ru   ræ   rÝ   rß   rå   r›   rœ   r�   s   @r#   rR  rR  É  s:   ø‡ € ØÐô
òò8òò+òbòS÷
ð 
r%   rR  c                   ó4   a € ] tR tRt o RtRR ltR tRtV tR# )ÚParakeetConverteri"  TNc                ó2  € Wn         \        V R 4       \        P                  W4       \	        4       pVP                  4       p\        VR4      ;_uu_ 4       pVP                  VP                  4       4       RRR4       W@n	        R#   + '       g   i     L; irF  )
r‚  r   r³   rk   r$   rb   rc   rd   re   rf   )rg   r‚  r†  rh   ri   rj   s   &&*   r#   rk   ÚParakeetConverter.__init__%  sk   € Ø$Œä˜$ 
Ô+ä×Ñ˜4Ô,ä#Ó%ˆ	Ø× Ñ Ó"ˆÜ�*˜d×#Ô# qØ×Ñ˜aŸf™f›hÔ'÷ $àŽ
÷ $×#ús   Á BÂB	c                ó†  € V P                  V4      p\        V4       UUUu/ uF
  w  pw  rEWCbK  	  pppp\        Wb4      p\        \	        VVVP
                  P                  R V P                  RR7      4      p\        VP                  4       U	U
u. uFT  w  ršV
P                  R9   g   K  YšP                  V
P                  ^8H  ;'       g    V
P                  V P                  9   3NKV  	  pp	p
TP                  \        VR R7       U	UUu. uF  w  rœp\        VRVR7      NK  	  uppp	4       V# u upppi u up
p	i u uppp	i )TNr’  c                 ó   € V ^ ,          # r:   r;   rw   s   &r#   r>   Ú-ParakeetConverter.tokenizer.<locals>.<lambda>K  r•  r%   r@   Fry   r   )r=   rŠ   rX   r   r   r‚   rs  rƒ  r…   r‹   r†   rÔ   r˜  rN   r   )rg   rf   rP   rŽ   r�   r‡   rš  rQ   rè   r�   r‘   r’   r“   r{   s   &&            r#   rè   ÚParakeetConverter.tokenizer2  s3  € Ø—z‘z %Ó(ˆä5>¸|Ô5LÕMÑ5LÑ!1 ¡M T�T’WÑ5Lˆ	ÒMÜ  Ó9ˆÜÜØØØ×,Ñ,×6Ñ6ØØ"×7Ñ7Øôó	
ˆ	ô # 5§<¡<Ô0ô
á0‘�Ø�v‰v˜Ñô IˆR—‘˜!Ÿ&™& A™+×GÐG¨¯©°D×4GÑ4GÑ)GÓHÙ0ð 	ñ 
ð
 	×Ñô +1Ð1AÁ~Õ*Võá*VÑ&�B˜wô ˜5¨U¸G×DÙ*Vóô	
ð Ðùô5 Nùó
ùôs   ¡D/ÂD6Â* D6ÃD6Ä
D<)rf   r‚  rª   )	rˆ   r—   r˜   r™   rƒ  rk   rè   r›   rœ   r�   s   @r#   rk  rk  "  s   ø‡ € ØÐô÷ð r%   rk  c            	     ó>  € \        \        \        R4      \        R4      ^,           4      4      \        \        \        R4      \        R4      ^,           4      4      ,           \        \        \        R4      \        R4      ^,           4      4      ,           p V R,          p^ p\        R4       F=  pW09  g   K  V P                  V4       VP                  RV,           4       V^,          pK?  	  V Uu. uF  p\	        V4      NK  	  pp\        \        W4      4      # u upi )	a  
Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
characters the bpe code barfs on.

The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
tables between utf-8 bytes and unicode strings.
Ú!Ú~ô   Â¡ô   Â¬ô   Â®ô   Ã¿:NNNé   )rr   rL   ÚordrM   ÚchrrJ   rê  )ÚbsÚcsÚnÚbs       r#   Úbytes_to_unicoder€  R  sÞ   € ô 	ŒU”3�s“8œS ›X¨�\Ó*Ó+¬d´5¼¸T»ÄCÈÃIÐPQÅMÓ3RÓ.SÕSÔVZÔ[`ÔadÐeiÓajÔloÐptÓluÐxyÕlyÓ[zÓV{Õ{ð ð 
ˆA�€BØ	€AÜ�4Ž[ˆØŽ;Ø�I‰I�aŒLØ�I‰I�d˜Q•hÔØ��FŠAñ	 ñ
 Ó	™"�QŒ#ˆaŽ&™"€BÐ	Ü”�B“ÓÐùò 
s   Ã0Dc                   óX   a € ] tR tRt o RtRR ltV 3R lR ltR tV 3R lR	 ltR
t	V t
R# )ÚTikTokenConverterij  z
A general tiktoken converter.
Nc                ó’   € Wn         W n        W0n        \        V\        4      '       d   VP                  4       V n        R # TV n        R # rª   )r‚  r4  r(   rt  rJ   r  Úextra_special_tokens)rg   r‚  r4  r(   r„  r�   s   &&&&&,r#   rk   ÚTikTokenConverter.__init__o  sA   € ð %ŒØŒØ 0Ôä+5Ð6JÌD×+QÒ+QÐ ×%Ñ%Ó'ð 	Ö!ØWkð 	Ö!r%   c                ó    <€ V ^8„  d   QhRS[ /# ©r'   Útiktoken_urlr]   )r    r^   s   "€r#   r,   ÚTikTokenConverter.__annotate__~  s   ø€ ÷ ñ ¹Cñ r%   c                ój  aa€  ^ RI Hp T! T4      o\	        4       oT3R lp. p/ pSP                  4        Fž  w  rgYuT! T4      &   \        T4      ^8X  d   K!  . p\        ^\        T4      4       F?  p	TRT	 YiR rºT
S9   g   K  TS9   g   K  Y«,           S9   g   K,  TP                  Y«T34       KA  	  \        TT3R lRR7      pTP                  T4       K   	  \        TR RR7      pT Uu. uF   qÃ! T^ ,          4      T! T^,          4      3NK"  	  ppYT3#   \         d    \        R4      hi ; iu upi )	r   )Úload_tiktoken_bpezY`tiktoken` is required to read a `tiktoken` file. Install it with `pip install tiktoken`.c           	      óŽ   <€ R P                  V P                  R4       Uu. uF  pS\        V4      ,          NK  	  up4      # u upi ©r%  zlatin-1©ré  Údecoderz  ©r  ÚcharÚbyte_encoders   & €r#   Útoken_bytes_to_stringÚPTikTokenConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_string‰  s;   ø€ Ø—7‘7ÀÇÁÈÔ@SÓTÑ@S¸˜L¬¨T«×3Ð3Ñ@SÑTÓUÐUùÒTó    ANc                 óD   <€ SV ^ ,          ,          SV ^,          ,          3# r:   r;   )r<   r  s   &€r#   r>   ÚCTikTokenConverter.extract_vocab_merges_from_model.<locals>.<lambda>—  s   ø€ °¸1¸Q½4µÀ)ÈAÈaÍDÅ/Ñ0Rr%   FrG   c                 ó   € V ^,          # rC   r;   rE   s   &r#   r>   r—  ™  ó   € °°A¶r%   )Útiktoken.loadr‹  r—  Ú
ValueErrorr€  rK   rD   rL   rM   rN   rO   )rg   rˆ  r‹  r“  rQ   r=   r“   ÚrankrT   rU   rV   rW   rF   r  r’  s   &&           @@r#   Úextract_vocab_merges_from_modelÚ1TikTokenConverter.extract_vocab_merges_from_model~  sC  ù€ ð	Ý7ñ & lÓ3ˆ	Ü'Ó)ˆõ	Vð ˆØˆØ$Ÿ?™?Ö,‰KˆEØ26Ñ'¨Ó.Ñ/Ü�5‹z˜QŒÙØˆEÜ˜q¤# e£*Ö-�Ø#(¨¨% =°%¸°-˜Ø˜iÖ'¨G°yÖ,@ÀgÕFWÐ\eÖEeØ—L‘L '°DÐ!9Ö:ñ .ô ˜5Ô&RÐ\aÔbˆEØ�M‰M˜%Ö ñ -ô ˜Ñ$6ÀÔFˆÙ\bÓcÑ\bÐUXÐ(¨¨Q­Ó0Ñ2GÈÈAÍÓ2OÓPÑ\bˆÐcØˆ}Ðøô5 ô 	ÜØkóð ð	üò2 ds   „D Ã+&D0ÄD-c                óÆ   € V P                  V P                  4      w  r\        \        WR R7      4      p\	        VP
                  R4      '       d   RVP
                  n        V# ©F)r  Úignore_mergesT©r�  r‚  r   r   rÙ   r\   r¡  ©rg   rP   rQ   rè   s   &   r#   rè   ÚTikTokenConverter.tokenizer�  óM   € Ø#×CÑCÀDÇOÁOÓTÑˆÜœc ,ÀÔGÓHˆ	Ü�9—?‘? O×4Ò4Ø,0ˆI�O‰OÔ)ØÐr%   c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   r‰  ¤  s   ø€ ÷ ñ ™9ñ r%   c                óô  € V P                  4       p\        P                  ! \        P                  ! \	        V P
                  4      R RR7      \        P                  ! V P                  RR7      .4      Vn        \        P                  ! 4       Vn
        V P                  e8   TP                  V P                   Uu. uF  p\        VRRR7      NK  	  up4       \        P                  ! RR7      Vn        V# u upi )rE  FrF  rI  Try   r*  )rè   r
   rL  rM  r   r4  r,  r(   rß   r   ræ   r„  r  r   r   rå   )rg   rè   r“   s   &  r#   r½   ÚTikTokenConverter.converted¤  sÊ   € Ø—N‘NÓ$ˆ	Ü"0×"9Ò"9ä×$Ò$¤U¨4¯<©<Ó%8À:ÐV[Ô\Ü×(Ò(¸$×:OÑ:OÐ[`Ôaðó#
ˆ	Ôô %×.Ò.Ó0ˆ	Ôà×$Ñ$Ò0Ø×(Ñ(ØPT×PiÒPiÓjÑPiÀu”˜E¨e¸T×BÑPiÑjôô $.×#7Ò#7ÀUÔ#Kˆ	Ô àÐùò ks   Â9C5)r(   r„  r4  r‚  ©Nzs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+FN)rˆ   r—   r˜   r™   rš   rk   r�  rè   r½   r›   rœ   r�   s   @r#   r‚  r‚  j  s(   ø‡ € ñô
÷ð ò>÷ö r%   r‚  c                   óT   a € ] tR tRt o R
R ltV 3R lR ltR tV 3R lR ltR	tV t	R# )ÚMistralConverteri¸  Nc                ó’   € Wn         W n        W0n        \        V\        4      '       d   VP                  4       V n        R # TV n        R # rª   )r‚  r4  r(   rt  rJ   r  r|   )rg   r‚  r4  r(   r|   r�   s   &&&&&,r#   rk   ÚMistralConverter.__init__¹  sG   € ð %ŒØŒØ 0Ôô Ð3´T×:Ò:ð &×*Ñ*Ó,ð 	Ö&ð +ð 	Ö&r%   c                ó    <€ V ^8„  d   QhRS[ /# r‡  r]   )r    r^   s   "€r#   r,   ÚMistralConverter.__annotate__Ê  s   ø€ ÷ %ñ %¹Cñ %r%   c                ó®  aa€ ^ RI p^ RIp\        V P                  RRR7      ;_uu_ 4       pVP	                  V4      pRRR4       XR,          R,          V n        VR,           Uu. uF  p\        VR,          VR	,          R
7      NK   	  upV n        VR,          p\        4       o\        V3R l4       p. p	/ p
\        V P                  4       F  w  r¼WºVP                  &   K  	  V Uu. uF  qbP                  VR,          4      NK  	  pp\        V4      p\        V4       UUu/ uF  w  rìWÎbK	  	  uppo\        \        VRR7      4       F¢  w  rìWêV! V4      &   \        V4      ^8X  d   K!  . p\!        ^\        V4      4       FC  pVRV VVR ppVV9   g   K  VV9   g   K  VV,           V9   g   K/  VP#                  VVV34       KE  	  \%        VV3R lRR7      pV	P'                  V4       K¤  	  \%        V	R RR7      p	V	 Uu. uF!  pV! V^ ,          4      V! V^,          4      3NK#  	  p	pW©3#   + '       g   i     ELú; iu upi u upi u uppi u upi )r   NÚrzutf-8)ÚencodingÚconfigr4  rÔ   Ú	token_strÚ
is_control)r{   r=   c           	      óŽ   <€ R P                  V P                  R4       Uu. uF  pS\        V4      ,          NK  	  up4      # u upi r�  rŽ  r�  s   & €r#   r“  ÚOMistralConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_string×  s;   ø€ à—7‘7ÀÇÁÈÔ@SÓTÑ@S¸˜L¬¨T«×3Ð3Ñ@SÑTÓUÐUùÒTr•  Útoken_bytesz(Converting tekken.json to tokenizer.json)Údescc                 óD   <€ SV ^ ,          ,          SV ^,          ,          3# r:   r;   )r<   Útoken_to_ranks   &€r#   r>   ÚBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>ë  s   ø€ °¸qÀ½tÕ1DÀmÐTUÐVWÕTXÕFYÑ0Zr%   FrG   c                 ó   € V ^,          # rC   r;   rE   s   &r#   r>   r¼  í  r™  r%   )Úbase64Újsonrc   r‚  Úloadr4  r   r|   r€  r   rŠ   r,  Ú	b64decoderI   r   rD   rL   rM   rN   rO   )rg   rˆ  r¾  r¿  rj   ÚuntypedÚkr  r“  rQ   r=   Úidxr“   Úrank_setrœ  rT   rU   rV   rW   rF   r’  r»  s   &&                  @@r#   r�  Ú0MistralConverter.extract_vocab_merges_from_modelÊ  s0  ù€ ÛÛä�$—/‘/ 3°×9Õ9¸QØ—i‘i “lˆG÷ :à˜xÕ(¨Õ3ˆŒàIPÐQaÖIbó*
ÙIbÀAŒJ�q˜•~¨q°­×?ÑIbñ*
ˆÔ&ð ˜GÕ$ˆ	Ü'Ó)ˆä	ô	Vó 
ð	Vð ˆØˆÜ# D×$BÑ$BÖC‰JˆCØ#&�%—-‘-Ó ñ DáAJÓKÁ¸A×%Ñ% a¨Õ&6Ö7Áˆ	ÐKÜ�y“>ˆÜ8AÀ)Ô8LÔMÑ8L©¨˜šÑ8LÒMˆÜ$¤T¨)Ð:dÔ%eÖf‰KˆDØ26Ñ'¨Ó.Ñ/Ü�5‹z˜QŒÙØˆEÜ˜q¤# e£*Ö-�Ø#(¨¨% =°%¸¸°-˜�Ø˜hÖ&¨7°hÖ+>ÀGÈgÕDUÐZbÖCbØ—L‘L '¨7°DÐ!9Ö:ñ .ô ˜5Ô&ZÐdiÔjˆEØ�M‰M˜%Ö ñ gô ˜Ñ$6ÀÔFˆÙ\bÓcÑ\bÐUXÑ(¨¨Q­Ó0Ñ2GÈÈAÍÓ2OÓPÑ\bˆÐcØˆ}Ð÷C :×9Ð9üò*
ùò LùãMùò ds#   «H.Á%$IÃ' IÄ"IÈ'IÈ.H?	c                óÆ   € V P                  V P                  4      w  r\        \        WR R7      4      p\	        VP
                  R4      '       d   RVP
                  n        V# r   r¢  r£  s   &   r#   rè   ÚMistralConverter.tokenizerñ  r¥  r%   c                ó    <€ V ^8„  d   QhRS[ /# rn   r¸   )r    r^   s   "€r#   r,   r¯  ø  s   ø€ ÷ ñ ™9ñ r%   c                ó–  € V P                  4       p\        P                  ! \        P                  ! \	        V P
                  4      R RR7      \        P                  ! V P                  RR7      .4      Vn        \        P                  ! 4       Vn
        VP                  V P                  4       \        P                  ! RR7      Vn        V# )rE  FrF  rI  r*  )rè   r
   rL  rM  r   r4  r,  r(   rß   r   ræ   r˜  r|   r   rå   )rg   rè   s   & r#   r½   ÚMistralConverter.convertedø  s™   € Ø—N‘NÓ$ˆ	Ü"0×"9Ò"9ä×$Ò$¤U¨4¯<©<Ó%8À:ÐV[Ô\Ü×(Ò(¸$×:OÑ:OÐ[`Ôaðó#
ˆ	Ôô %×.Ò.Ó0ˆ	Ôà×Ñ˜T×;Ñ;Ô<Ü#-×#7Ò#7ÀUÔ#Kˆ	Ô àÐr%   )r(   r|   r4  r‚  r©  )
rˆ   r—   r˜   r™   rk   r�  rè   r½   r›   rœ   r�   s   @r#   r«  r«  ¸  s$   ø‡ € ô
÷"%ð %òN÷ö r%   r«  ÚAlbertTokenizerÚBartTokenizerÚBarthezTokenizerÚBertTokenizerÚBigBirdTokenizerÚBlenderbotTokenizerÚCamembertTokenizerÚCLIPTokenizerÚCodeGenTokenizerÚConvBertTokenizerÚDebertaTokenizerÚDebertaV2TokenizerÚDistilBertTokenizerÚDPRReaderTokenizerÚDPRQuestionEncoderTokenizerÚDPRContextEncoderTokenizerÚElectraTokenizerÚFNetTokenizerÚFunnelTokenizerÚGPT2TokenizerÚHerbertTokenizerÚLayoutLMTokenizerÚLayoutLMv2TokenizerÚLayoutLMv3TokenizerÚLayoutXLMTokenizerÚLongformerTokenizerÚLEDTokenizerÚLxmertTokenizerÚMarkupLMTokenizerÚMBartTokenizerÚMBart50TokenizerÚMPNetTokenizerÚMobileBertTokenizerÚMvpTokenizerÚNllbTokenizerÚOpenAIGPTTokenizerÚPegasusTokenizerÚQwen2TokenizerÚReformerTokenizerÚRemBertTokenizerÚRobertaTokenizerÚRoFormerTokenizerÚSeamlessM4TTokenizerÚSqueezeBertTokenizerÚT5TokenizerÚUdopTokenizerÚWhisperTokenizerÚXLMRobertaTokenizerÚXLNetTokenizerÚSplinterTokenizerÚXGLMTokenizerÚLlamaTokenizerÚCodeLlamaTokenizerÚGemmaTokenizerÚPhi3Tokenizerc                ó$   € V ^8„  d   QhR\         /# rn   r¸   )r    s   "r#   r,   r,   C  s   € ÷ $ñ $Ì)ñ $r%   c                óR  € V P                   P                  pV\        9   d,   V'       g$   \        V,          pV! V 4      P                  4       # V P                  P                  R4      '       d?   W n        \        P                  R4       \        V P                  4      P                  4       #  \        P                  R4       \        V P                  V P                  R7      P                  4       #   \         d+    \        R\        \        P                  4       4       24      hi ; i)a\  
Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

Args:
    transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
        Instance of a slow tokenizer to convert in the backend tokenizer for
        [`~tokenization_utils_base.PreTrainedTokenizerFast`].
   from_tiktoken (bool, optional): Whether to use the `tiktoken` library to convert the tokenizer instead of sentencepiece.
        Defaults to False.

Return:
    A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
    [`~tokenization_utils_base.PreTrainedTokenizerFast`]
ztekken.jsonz#Converting from Mistral tekken.jsonzConverting from Tiktoken)r‚  r„  zÌConverting from SentencePiece and Tiktoken failed, if a converter for SentencePiece is available, provide a model path with a SentencePiece tokenizer.model file.Currently available slow->fast converters: )r‡  rˆ   ÚSLOW_TO_FAST_CONVERTERSr½   r‚  Úendswithr3   ÚloggerÚinfor«  r‚  r„  r—  r›  rr   r  )Útransformer_tokenizerÚfrom_tiktokenÚtokenizer_class_nameÚconverter_classs   &&  r#   Úconvert_slow_tokenizerr  C  sþ   € ð  1×:Ñ:×CÑCÐØÔ6Ô6¿}Ü1Ð2FÕGˆÙÐ4Ó5×?Ñ?ÓAÐAØ	×	)Ñ	)×	2Ñ	2°=×	AÒ	AØ3HÔ0Ü�‰Ð9Ô:ÜÐ 5× @Ñ @ÓA×KÑKÓMÐMð	Ü�K‰KÐ2Ô3Ü$Ø0×;Ñ;Ø%:×%OÑ%Oô÷ ‰i‹kðøô ô 	Üð>ä>BÔCZ×C_ÑC_ÓCaÓ>bÐ=cðeóð ð	ús   Â-AC1 Ã15D&)r  r  r  r  r  r  r  r  r  r  r!  r#  r%  r'  r)  r+  r-  r/  r1  r3  r5  r7  r9  r;  r=  )rU  rV  rW  rX  rY  rZ  r[  r\  r]  r^  r_  r`  ra  rb  rc  rd  re  rf  rg  rh  ri  rj  rk  rl  rm  rn  ro  )r%  rª   )F)Urš   r„  Úcollections.abcr   Ú	functoolsr   Ú	packagingr   Ú
tokenizersr   r   r   r   r	   r
   r   rƒ   r   r   r   r   Úutilsr   r   r   r   Úutils.import_utilsr   Ú
get_loggerrˆ   r  rO  rx  r$   r5   rX   rZ   rŸ   r±   r³   rÀ   rð   r   r  r  r  r3  r@  rP  r\  rf  rp  rµ  rÑ  rÞ  rû  r  rS  r{  r‹  r–  r   r¨  r¬  r²  r´  rÌ  rÞ  rã  rð  rô  rü  r  r  r  r%  r>  rD  rR  rk  r€  r‚  r«  r  r  r;   r%   r#   Ú<module>r     s‹  ðñó Ý &Ý å ß f× fÑ fß 5Ñ 5Ý ç `Ó `Ý 5ð 
×	Ò	˜HÓ	%€ò€ð8 $ò 'õ Ð ô>Gõ"÷÷02ñ 2ôjÐ"8ô õ"I÷$ñ $ô$�Iô $ôN/˜	ô /ôd$�iô $ôN$�Yô $ôN˜ô ô6#�Iô #ôL�yô ô>)�Yô )ôX�yô ô:$˜	ô $ôN�yô ô>x�9ô xôv"
�lô "
ôJ
�|ô 
ô -˜ô -ô`
˜ô 
ôBG�\ô GôT-�|ô -ô`,�Lô ,ô^
˜<ô 
ô2,˜,ô ,ô^"
�\ô "
ôJ	˜ô 	ô
�|ô 
ô@	˜lô 	ô;x�|ô ;xô|�,ô ô8
�Lô 
ô!�yô !ôH	
�|ô 	
ô(�Iô (ôV$˜)ô $ôN˜)ô ô:
�Lô 
ô61
�\ô 1
ôh+�\ô +ô\#˜	ô #ôL&m�\ô &môRV
�lô V
ôr-˜ô -ò`÷0Kñ K÷\Mñ Mð`8Ø�ð8àÐ%ð8ð Ð(ð8ð �]ð	8ð
 Ð(ð8ð Ð.ð8ð Ð,ð8ð �]ð8ð ˜ð8ð ˜ð8ð Ð(ð8ð Ð,ð8ð ˜=ð8ð ˜-ð8ð " =ð8ð  ! -ð!8ð" ˜ñ#8ð$ �_ð%8ð& �ð'8ð( �]ð)8ð* Ð(ð+8ð, ˜ð-8ð. ˜=ð/8ð0 Ð+ð18ð2 Ð-ð38ð4 Ð+ð58ð6 Ð$ð78ð8 �}ð98ð: Ð*ð;8ð< �nð=8ð> Ð(ð?8ð@ �nðA8ðB ˜=ðC8ðD Ð$òE8ðF �]ðG8ðH Ð,ðI8ðJ Ð(ðK8ðL �nðM8ðN Ð*ðO8ðP Ð(ðQ8ðR Ð(ðS8ðT Ð*ðU8ðV Ð0ðW8ðX ˜MðY8ðZ �;ð[8ð\ �]ð]8ð^ Ð(ð_8ð` Ð.ða8ðb �nðc8ðd Ð*ðe8ðf �]ñg8ðh �nØ˜.Ø�nØ�^ño8Ð ÷v$ñ $r%   