+
    TV-jÙ ã                   ó~  € ^ RI t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RI	H
t
 ^ RIHt ^ RIHtHtHtHtHtHtHtHt ^ RIHt ^ RIHt ^ RIHt ^ RIHt ^RIHt ^R	I H!t!H"t"H#t#H$t$H%t%H&t&H't'H(t(H)t) ^R
I*H+t+ ^RI,H-t- ^RI.H/t/H0t0 Rt1^dt2Rt3Rt4Rt5^ t6Rt7Rt8^t9Rt:Rt;Rt<R t=R t>]P~                  ! ]P€                  ! 4       4      tA]P„                  RHR R ll4       tC]
 ! R R4      4       tDR tERRRRRRRRRRRR R!RR"^@R#^ R$RR%R/R& R' lltFR(^RRRRRRRRRR)R!RR"^@R#^ /	R* R+ lltGRIR, R- lltHRJR. R/ lltIRHR0 ltJRHR1 ltK]
 ! R2 R34      4       tLR4 tMR5 tNR6 tOR7 tPR8 tQ ! R9 R:4      tR ! R; R<4      tS ! R= R>4      tT ! R? R@4      tU]
 ! RA RB4      4       tVRKRC RD lltWRE tX]YRF8X  d   ]Z! RG4       ]X! 4        R# R# )Lé    N)Údeque)Ú	dataclass)Úpartial)ÚAnyÚCallableÚ	GeneratorÚListÚOptionalÚSequenceÚTupleÚUnion)Útree_reduce)ÚPreTrainedTokenizer©Úcache)	ÚArraysCacheÚBatchKVCacheÚBatchRotatingKVCacheÚ	CacheListÚKVCacheÚQuantizedKVCacheÚRotatingKVCacheÚTokenBufferÚload_prompt_cache)Úmake_sampler)ÚTokenizerWrapper)Ú#does_model_support_input_embeddingsÚloadÚhelloç        g      ð?z(mlx-community/Llama-3.2-3B-Instruct-4bitiˆ  c                 ó(   € V P                  4       R9  # )Úfalse)r"   Úf)Úlower)Ústrings   &Ú`/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/generate.pyÚstr2boolr'   ;   s   € Ø�<‰<‹> Ñ/Ð/ó    c                 óÂ  € \         P                  ! RR7      p V P                  R\        R\         R2RR7       V P                  RR	R
R7       V P                  R\        RR7       V P                  R\        RLRRR7       V P                  RRRR7       V P                  RR\
        RR7       V P                  RRRR7       V P                  RR\        \        RR7       V P                  R\        \        R R7       V P                  R!\        \        R"R7       V P                  R#\        \        R$R7       V P                  R%\        \        R&R7       V P                  R'\        \        R(R7       V P                  R)\        R*R+R7       V P                  R,\        \        R-R7       V P                  R.\        \        R/R7       V P                  R0R	R1R7       V P                  R2R	R3R7       V P                  R4R5RR67       V P                  R7\         R8R9R7       V P                  R:\        R;RR7       V P                  R<\        RR=R7       V P                  R>R?R	R@R7       V P                  RA\        RBRR7       V P                  RC\        RD^@R7       V P                  RERF\        \"        RG7       V P                  RH\        RIRR7       V P                  RJ\        RK^R7       V # )Mz&Set up and return the argument parser.zLLM inference script)Údescriptionz--modelz[The path to the local model directory or Hugging Face repo. If no model is specified, then z	 is used.N)ÚtypeÚhelpÚdefaultz--trust-remote-codeÚ
store_truez)Enable trusting remote code for tokenizer)Úactionr,   z--adapter-pathz9Optional path for the trained adapter weights and config.)r+   r,   z--extra-eos-tokenÚ+z:Add tokens in the list of eos tokens that stop generation.)r+   r-   Únargsr,   z--system-promptz.System prompt to be used for the chat template)r-   r,   z--promptz-pz;Message to be processed by the model ('-' reads from stdin)z--prefill-responsez1Prefill response to be used for the chat templatez--max-tokensz-mz$Maximum number of tokens to generate)r+   r-   r,   z--tempzSampling temperaturez--top-pzSampling top-pz--min-pzSampling min-pz--top-kzSampling top-kz--xtc-probabilityz5Probability of XTC sampling to happen each next tokenz--xtc-thresholdr    zDThresold the probs of each next token candidate to be sampled by XTCz--min-tokens-to-keepz*Minimum tokens to keep for min-p sampling.z--seedz	PRNG seedz--ignore-chat-templatez9Use the raw prompt without the tokenizer's chat template.z--use-default-chat-templatezUse the default chat templatez--chat-template-configz„Additional config for `apply_chat_template`. Should be a dictionary of string keys to values represented as a JSON decodable string.)r,   r-   z	--verboseTzTLog verbose output when 'True' or 'T' or only print the response when 'False' or 'F'z--max-kv-sizez$Set the maximum key-value cache sizez--prompt-cache-filez;A file containing saved KV caches to avoid recomputing themz--quantize-activationsz-qazSQuantize activations using the same quantization config as the corresponding layer.z	--kv-bitszFNumber of bits for KV cache quantization. Defaults to no quantization.z--kv-group-sizez%Group size for KV cache quantization.z--quantized-kv-startzLWhen --kv-bits is set, start quantizing the KV cache from this step onwards.)r,   r+   r-   z--draft-modelz,A model to be used for speculative decoding.z--num-draft-tokensz:Number of tokens to draft when using speculative decoding.© )ÚargparseÚArgumentParserÚadd_argumentÚstrÚDEFAULT_MODELÚDEFAULT_PROMPTÚintÚDEFAULT_MAX_TOKENSÚfloatÚDEFAULT_TEMPÚDEFAULT_TOP_PÚDEFAULT_MIN_PÚDEFAULT_TOP_KÚDEFAULT_XTC_PROBABILITYÚDEFAULT_MIN_TOKENS_TO_KEEPÚDEFAULT_SEEDr'   ÚDEFAULT_QUANTIZED_KV_START)Úparsers    r&   Úsetup_arg_parserrE   ?   s³  € ä×$Ò$Ð1GÔH€FØ
×ÑØÜð.Ü.;¨_¸IðGð ð ô ð ×ÑØØØ8ð ô ð
 ×ÑØÜØHð ô ð
 ×ÑØÜØØØIð ô ð ×ÑØØØ=ð ô ð
 ×ÑØØÜØJð	 ô ð ×ÑØØØ@ð ô ð
 ×ÑØØÜÜ"Ø3ð ô ð ×ÑØ”u¤lÐ9Oð ô ð ×ÑØœ¤}Ð;Kð ô ð ×ÑØœ¤}Ð;Kð ô ð ×ÑØœ¤]Ð9Ið ô ð ×ÑØÜÜ'ØDð	 ô ð ×ÑØÜØØSð	 ô ð ×ÑØÜÜ*Ø9ð	 ô ð ×ÑØÜÜØð	 ô ð ×ÑØ ØØHð ô ð
 ×ÑØ%ØØ,ð ô ð
 ×ÑØ ðIàð	 ô ð ×ÑØÜØØcð	 ô ð ×ÑØÜØ3Øð	 ô ð ×ÑØÜØØJð	 ô ð ×ÑØ ØØØbð	 ô ð ×ÑØÜØUØð	 ô ð ×ÑØÜØ4Øð	 ô ð ×ÑØð"äÜ*ð ô ð ×ÑØÜØ;Øð	 ô ð ×ÑØÜØIØð	 ô ð €Mr(   c                ó„   € V ^8„  d   QhR\         P                  R\        \        \        P
                  ,          ,          /# )é   ÚmodelÚstreams)ÚnnÚModuler
   r	   ÚmxÚStream)Úformats   "r&   Ú__annotate__rO   æ   s+   € ÷ $*ñ $*”r—y‘yð $*¬8´D¼¿¹µOÕ+Dñ $*r(   c              #  ó¢  "  € \         P                  P                  4       '       g    Rx € R# \        R V ^ 4      p\         P                  ! 4       R,          pVRV,          8”  d%   VR,          pVR,          p\        RV RV R24       \         P                  ! V4      p Rx € Ve!   V F  p\         P                  ! V4       K  	  M\         P                  ! 4        \         P                  ! V4       R#   i ; i  Te!   T F  p\         P                  ! T4       K  	  M\         P                  ! 4        \         P                  ! T4       i ; i5i)	zî
A context manager to temporarily change the wired limit.

Note, the wired limit should not be changed during an async eval.  If an
async eval could be running pass in the streams to synchronize with prior
to exiting the context manager.
Nc                 ój   € \        V\        P                  4      '       d   WP                  ,           # T # ©N)Ú
isinstancerL   ÚarrayÚnbytes)ÚaccÚxs   &&r&   Ú<lambda>Úwired_limit.<locals>.<lambda>õ   s"   € ¬Z¸¼2¿8¹8×-DÒ-D˜3§¡�>ÐMÈ#ÐMr(   Ú max_recommended_working_set_sizegÍÌÌÌÌÌì?z0[WARNING] Generating with a model that requires z6 MB which is close to the maximum recommended size of zƒ MB. This can be slow. See the documentation for possible work-arounds: https://github.com/ml-explore/mlx-lm/tree/main#large-modelsi   )rL   ÚmetalÚis_availabler   Údevice_infoÚprintÚset_wired_limitÚsynchronize)rH   rI   Úmodel_bytesÚmax_rec_sizeÚmodel_mbÚ
max_rec_mbÚ	old_limitÚss   &&      r&   Úwired_limitrg   å   s#  é € ô �8‰8× Ñ ×"Ò"ð	Ûáä!ÙMÈuÐVWó
ˆô —~’~Ó'Ð(JÕKˆØ˜˜|Õ+Ô+Ø" eÕ+ˆHØ%¨Õ.ˆJÜØBÀ8À*ð MEØEOÀLð QNðNôô ×&Ò& |Ó4ˆ	ð	*ÛàÒ"Û �AÜ—N’N 1Ö%ò !ô —’Ô Ü×Ò˜yÖ)øð1 ûð& Ò"Û �AÜ—N’N 1Ö%ò !ô —’Ô Ü×Ò˜yÕ)üs6   ‚%E¨C6 ¬A4EÂ!C; Â%AEÃ6C8Ã8EÃ;AEÅEc                   ó4   a € ] tR tRt o RtRtV 3R ltRtV tR# )ÚGenerationResponsei  a¶  
The output of :func:`stream_generate`.

Args:
    text (str): The next segment of decoded text. This can be an empty string.
    token (int): The next token.
    from_draft (bool): Whether the token was generated by the draft model.
    logprobs (mx.array): A vector of log probabilities.
    prompt_tokens (int): The number of tokens in the prompt.
    prompt_tps (float): The prompt processing tokens-per-second.
    generation_tokens (int): The number of generated tokens.
    generation_tps (float): The tokens-per-second for generation.
    peak_memory (float): The peak memory used so far in GB.
    finish_reason (str): The reason the response is being sent: "length", "stop" or `None`
Nc                ó¶   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[P                  ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[S[ ,          ;R
&   # )rG   ÚtextÚtokenÚlogprobsÚ
from_draftÚprompt_tokensÚ
prompt_tpsÚgeneration_tokensÚgeneration_tpsÚpeak_memoryÚfinish_reason)r6   r9   rL   rT   Úboolr;   r
   )rN   Ú__classdict__s   "€r&   rO   ÚGenerationResponse.__annotate__  s�   ø‡ ‚ ñ$ �Iñ% ñ& �Jñ' ñ( �h‰hÑñ) ñ* Ññ+ ñ, Ññ- ñ. Ññ/ ñ0 Ññ1 ñ2 Ññ3 ñ4 Ññ5 ñ6 ™C•=Ñ'ò7 r(   r2   )	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rt   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©rv   s   @r&   ri   ri     s   ø‡ € ñð2 $(€M÷7 ƒ r(   ri   c                 ó°   € Vf   R # \        V 4       F@  w  rE\        VR4      '       g   K  VP                  V8¼  g   K,  VP                  W#R7      W&   KB  	  R # )NÚto_quantized)Ú
group_sizeÚbits)Ú	enumerateÚhasattrÚoffsetr‚   )Úprompt_cacheÚquantized_kv_startÚkv_group_sizeÚkv_bitsÚeÚcs   &&&&  r&   Úmaybe_quantize_kv_cacherŽ   +  sF   € Ø‚ÙÜ˜,Ö'‰ˆÜ�1�n×%Ô%¨!¯(©(Ð6HÖ*HØŸn™n¸˜nÓTˆL‹Oó (r(   Ú
max_tokensé   ÚsamplerÚlogits_processorsÚmax_kv_sizerˆ   Úprefill_step_sizeé   r‹   rŠ   r‰   Úprompt_progress_callbackÚinput_embeddingsc                óî  € V ^8„  d   QhR\         P                  R\        P                  R\        R\
        \        \         P                  .\         P                  3,          ,          R\
        \        \        \         P                  \         P                  .\         P                  3,          ,          ,          R\
        \        ,          R\
        \        ,          R\        R	\
        \        ,          R
\        R\        R\
        \        \        \        .R3,          ,          R\
        \         P                  ,          R\        \        \         P                  \         P                  3,          RR3,          /# )rG   ÚpromptrH   r�   r‘   r’   r“   rˆ   r”   r‹   rŠ   r‰   r–   Nr—   Úreturn)rL   rT   rJ   rK   r9   r
   r   r	   r   r   r   )rN   s   "r&   rO   rO   3  s0  € ÷ cñ cÜ�H‰Hðcä�9‰9ðcô ð	cô
 ”h¤§¡˜z¬2¯8©8Ð3Õ4Õ5ðcô  ¤¤X¬r¯x©x¼¿¹Ð.BÄBÇHÁHÐ.LÕ%MÕ NÕOðcô œ#•ðcô œ3•-ðcô ðcô ”c�]ðcô ðcô ðcô '¤x´´c°
¸DÐ0@Õ'AÕBðcô œrŸx™xÕ(ðcô Œu”R—X‘XœrŸx™xÐ'Õ(¨$°Ð4Õ5ñcr(   c             #  óØ  aaaaaaa"  € Vek   \        S4      '       g   \        R4      h\        V 4      ^ 8”  d>   \        V 4      \        V4      8w  d%   \        R\        V4       R\        V 4       R24      hM\        V 4      ^ 8X  d   \        R4      hRoSf   \        P                  ! SVR7      oT;'       g    R p\
        P                  ! \        V
V	VR	7      oS;'       g    R
 oR VV3R lloRR VVVVVV3R lllp\        P                  ! \        4      ;_uu_ 4        Ve   \        V4      M
\        V 4      p^ pV! Wþ4       Wï,
          ^8”  d°   Wï,
          ^,
          p\        VV4      pS! V RV R,          Ve   VRV R,          MRR7       S! S4       \        P                  ! S Uu. uF  pVP                  NK  	  up4       VV,          pV! Wþ4       V VR p Ve   VVR MTp\        P                  ! 4        K¼  V! WR7      w  ppRRR4       \        P                  ! XX4       ^ p VV8w  d#   V! V4      w  pp\        P                  ! VV4       V^ 8X  d    \        P                  ! V4       V! XV4       VV8X  d   R# VP!                  4       V3x € VR,          ^ 8X  d   \        P                  ! 4        XXppV^,          pKŸ  u upi   + '       g   i     LÍ; i5i)a£  
A generator producing token ids based on the given prompt from the model.

Args:
    prompt (mx.array): The input prompt.
    model (nn.Module): The model to use for generation.
    max_tokens (int): The maximum number of tokens. Use``-1`` for an infinite
      generator. Default: ``256``.
    sampler (Callable[mx.array, mx.array], optional): A sampler for sampling a
      token from a vector of log probabilities. Default: ``None``.
    logits_processors (List[Callable[[mx.array, mx.array], mx.array]], optional):
      A list of functions that take tokens and logits and return the processed
      logits. Default: ``None``.
    max_kv_size (int, optional): Maximum size of the key-value cache. Old
      entries (except the first 4 tokens) will be overwritten.
    prompt_cache (List[Any], optional): A pre-computed prompt cache. Note, if
      provided, the cache will be updated in place.
    prefill_step_size (int): Step size for processing the prompt.
    kv_bits (int, optional): Number of bits to use for KV cache quantization.
      None implies no cache quantization. Default: ``None``.
    kv_group_size (int): Group size for KV cache quantization. Default: ``64``.
    quantized_kv_start (int): Step to begin using a quantized KV cache.
       when ``kv_bits`` is non-None. Default: ``0``.
    prompt_progress_callback (Callable[[int, int], None]): A call-back which takes the
       prompt tokens processed so far and the total number of prompt tokens.
    input_embeddings (mx.array, optional): Input embeddings to use instead of or in
      conjunction with prompt tokens. Default: ``None``.

Yields:
    Tuple[mx.array, mx.array]: One token and a vector of log probabilities.
Nz(Model does not support input embeddings.z8When providing input_embeddings, their sequence length (z0) must match the sequence length of the prompt (z), or the prompt must be empty.z=Either input_embeddings or prompt (or both) must be provided.)r“   c                  ó   € R # rR   r2   )Ú_s   *r&   rX   Úgenerate_step.<locals>.<lambda>y  s   € Átr(   ©r‰   rŠ   r‹   c                 ó2   € \         P                  ! V RR7      # ©é   ©Úaxiséÿÿÿÿ©rL   Úargmax©rW   s   &r&   rX   rž   ‚  ó   € ¤B§I¢I¨a°bÕ$9r(   c                ón   € V ^8„  d   QhR\         P                  R\        \         P                  ,          /# ©rG   Úinput_tokensr—   ©rL   rT   r
   )rN   s   "r&   rO   Ú#generate_step.<locals>.__annotate__„  s'   € ÷ ;ñ ;¤"§(¡(ð ;¼hÄrÇxÁxÕ>Pñ ;r(   c                 ó6   <€ Ve   S! V SVR7      # S! V SR7      # )N)r   r—   r   r2   )r¬   r—   rH   rˆ   s   &&€€r&   Ú_model_callÚ"generate_step.<locals>._model_call„  s-   ø€ ØÒ'ÙØ LÐCSôð ñ ˜¨\Ô:Ð:r(   c                ón   € V ^8„  d   QhR\         P                  R\        \         P                  ,          /# r«   r­   )rN   s   "r&   rO   r®   Œ  s'   € ÷ 0ñ 0œBŸH™Hð 0¼ÄÇÁÕ8Jñ 0r(   c                 óæ  <€ \         P                  ! \        4      ;_uu_ 4        S! V R ,          Ve
   VR ,          MR R7      pVR,          pS'       dA   \        V 4      ^ 8”  d1   Se   \         P                  ! SV .4      MT oS F  pV! SV4      pK  	  S	! S4       V\         P
                  ! VRR7      ,
          pS
! V4      pWTP                  ^ 4      3uuR R R 4       #   + '       g   i     R # ; i)N©r¬   r—   T)Úkeepdims©ºNNNr¥   r·   )rL   ÚstreamÚgeneration_streamÚlenÚconcatÚ	logsumexpÚsqueeze)r¬   r—   ÚlogitsÚ	processorrm   Úsampledr°   r’   rˆ   Úquantize_cache_fnr‘   Útokenss   &&    €€€€€€r&   Ú_stepÚgenerate_step.<locals>._stepŒ  s×   ø€ ô �YŠYÔ(×)Õ)Ù Ø)¨$Õ/à.>Ò.JÐ$ TÖ*ÐPTôˆFð ˜HÕ%ˆFç ¤S¨Ó%6¸Ô%:ð Ò)ô —I’I˜v |Ð4Ô5à%ð ó
 "3�IÙ& v¨vÓ6’Fñ "3ñ ˜lÔ+à¤§¢¨V¸dÔ CÕCˆHÙ˜hÓ'ˆGØ×,Ñ,¨QÓ/Ð/÷/ *×)×)Ó)ús   ¤B0CÃC0	r´   r�   rR   )r   Ú
ValueErrorrº   r   Úmake_prompt_cacheÚ	functoolsr   rŽ   rL   r¸   r¹   ÚminÚevalÚstateÚclear_cacheÚ
async_evalÚitem)r™   rH   r�   r‘   r’   r“   rˆ   r”   r‹   rŠ   r‰   r–   r—   rÃ   Útotal_prompt_tokensÚprompt_processed_tokensÚ	remainingÚn_to_processr�   Úyrm   ÚnÚnext_yÚnext_logprobsr°   rÁ   rÂ   s   &f$dd$d$$$$$$           @@@r&   Úgenerate_steprÖ   3  sÍ  þé € ð^ Ò#Ü2°5×9Ò9ÜÐGÓHÐHÜ�‹[˜1Œ_¤ V£´Ð4DÓ0EÔ!EÜØJÌ3ÐO_ÓK`ÐJað bAÜADÀVÃÀð N(ð(óð øô
 
ˆV‹˜Ô	ÜØKó
ð 	
ð €Fð ÒÜ×.Ò.ØØ#ô
ˆð
  8×LÐL¹OÐä!×)Ò)ÜØ-Ø#Øô	Ðð ×:Ð:Ñ9€G÷;ñ ;÷0÷ 0ô8 
�ŠÔ$×	%Õ	%à%5Ò%AŒCÐ Ô!ÄsÈ6Ã{ð 	ð #$ÐÙ Ð!8ÔNØ!Õ;¸aÔ?Ø,ÕFÈ!ÕKˆIÜÐ0°)Ó<ˆLÙØ# M \Ð2°4Õ8ð (Ò3ð % ] lÐ3°DÖ9àõñ ˜lÔ+Ü�GŠG¡lÓ3¡l �Q—W”W¡lÑ3Ô4Ø# |Õ3Ð#Ù$Ð%<ÔRØ˜L˜MÐ*ˆFð $Ò/ð !  Ñ/à%ð ô
 �NŠNÖá¨ÔS‰ˆˆ8÷; 
&ô> ‡M‚M�!�XÔØ	€AØ
Ø�
Œ?Ù$)¨!£HÑ!ˆF�MÜ�MŠM˜& -Ô0Ø�Œ6Ü�GŠG�AŒJÙ$Ð%8Ð:MÔNØ�
Œ?ÙØ�f‰f‹h˜Ð Ò Øˆs�7�aŒ<Ü�NŠNÔØ˜mˆ8ˆØ	ˆQ�Šùò9 4÷% 
&×	%üs@   ‰B0K*Â:*K*Ã%A K*Ä%BKÆ2KÇAKÈC K*ËKËK'	Ë"K*Únum_draft_tokensé   c                ó„  € V ^8„  d   QhR\         P                  R\        P                  R\        P                  R\        R\        R\
        \        \         P                  .\         P                  3,          ,          R\
        \        \        \         P                  \         P                  .\         P                  3,          ,          ,          R\
        \        ,          R	\        R
\
        \        ,          R\        R\        R\        \        \         P                  \         P                  \        3,          RR3,          /# )rG   r™   rH   Údraft_modelr×   r�   r‘   r’   rˆ   r”   r‹   rŠ   r‰   rš   N)rL   rT   rJ   rK   r9   r
   r   r	   r   r   r   ru   )rN   s   "r&   rO   rO   Ù  s  € ÷ u$ñ u$Ü�H‰Hðu$ä�9‰9ðu$ô —‘ðu$ô
 ðu$ô ðu$ô ”h¤§¡˜z¬2¯8©8Ð3Õ4Õ5ðu$ô  ¤¤X¬r¯x©x¼¿¹Ð.BÄBÇHÁHÐ.LÕ%MÕ NÕOðu$ô œ3•-ðu$ô ðu$ô ”c�]ðu$ô ðu$ô ðu$ô Œu”R—X‘XœrŸx™x¬Ð-Õ.°°dÐ:Õ;ñu$r(   c       	      #  ó  aaaaaaaaa a!"  € V P                  \        P                  4      pRo Vf.   \        P                  ! V4      o\        P                  ! S4      oM0VR\        VP                  4       oV\        VP                  4      R o\        P                  ! S4      '       gK   S Uu0 uF/  qÝP                  4       '       d   K  \        V4      P                  kK1  	  pp\        RV R24      hS;'       g    R o\        P                  ! \        VV
V	R7      o!VV3R loRVVV V!3R lloVV!3R lpVV3R	 lpVVV3R
 lp\        P                  ! \         4      ;_uu_ 4        V! SSV4      pV! VSV4      pRRR4       ^ p^ p^ p  \#        VV,
          V4      pV! XV4      pS e/   S RS P$                  VP$                  ,
          V,
          ^,            o \        P&                  ! VV.4      pS! VSVV^,           4      w  pp\        P(                  ! VV4       VP+                  4       pVP+                  4       p^ pVV8  dG   VV,          VV,          VV,          pppVV8w  d   M#V^,          pV^,          pVVR3x € VV8X  g   KL   VV8  d   V^,          pVV,          VV,          R3x € VV8X  d   M›\        P,                  ! VV,          .\        P                  4      pTpVV8X  d?   \        P&                  ! \        P,                  ! VRR \        P                  4      V.4      pS e   S R\/        VV,
          ^4      )  o V! VV4       EKÊ  V! VV4       R# u upi   + '       g   i     ELò; i  T! TT4       i ; i5i)a  
A generator producing token ids based on the given prompt from the model.

Args:
    prompt (mx.array): The input prompt.
    model (nn.Module): The model to use for generation.
    draft_model (nn.Module): The draft model for speculative decoding.
    num_draft_tokens (int, optional): The number of draft tokens for
      speculative decoding. Default: ``2``.
    max_tokens (int): The maximum number of tokens. Use``-1`` for an infinite
      generator. Default: ``256``.
    sampler (Callable[[mx.array], mx.array], optional): A sampler for sampling a
      token from a vector of log probabilities. Default: ``None``.
    logits_processors (List[Callable[[mx.array, mx.array], mx.array]], optional):
      A list of functions that take tokens and logits and return the processed
      logits. Default: ``None``.
    prompt_cache (List[Any], optional): A pre-computed prompt cache. Note, if
      provided, the cache will be updated in place. The cache must be trimmable.
    prefill_step_size (int): Step size for processing the prompt.
    kv_bits (int, optional): Number of bits to use for KV cache quantization.
      None implies no cache quantization. Default: ``None``.
    kv_group_size (int): Group size for KV cache quantization. Default: ``64``.
    quantized_kv_start (int): Step to begin using a quantized KV cache.
       when ``kv_bits`` is non-None. Default: ``0``.

Yields:
    Tuple[mx.array, mx.array, bool]: One token, a vector of log probabilities,
      and a bool indicating if the token was generated by the draft model
Nz<Speculative decoding requires a trimmable prompt cache (got z).c                 ó2   € \         P                  ! V RR7      # r¡   r¦   r¨   s   &r&   rX   Ú+speculative_generate_step.<locals>.<lambda>  r©   r(   rŸ   c                 óŒ   <€ S'       d   S F  pV! W4      pK  	  V\         P                  ! VRRR7      ,
          pS! V4      pWC3# )r¢   T©r¤   rµ   r¥   )rL   r¼   )rÂ   r¾   r¿   rm   rÒ   r’   r‘   s   &&   €€r&   Ú_process_and_sampleÚ6speculative_generate_step.<locals>._process_and_sample   sE   ø€ ßÛ.�	Ù" 6Ó2’ñ /ð œBŸLšL¨°bÀ4ÔHÕHˆÙ�HÓˆØˆ{Ðr(   c           
      ó†  <€ \         P                  ! \        4      ;_uu_ 4        V ! VR ,          VR7      pVRV) R 1R3,          pS! V4       S
'       d¹   . . reV^8”  d   VR V^,
          )  p\        V4       FX  pSe   \         P                  ! SV.4      MToS	! SVRVR3,          4      w  r(VP                  V4       VP                  V4       KZ  	  \         P                  ! V^ R7      \         P                  ! V^ R7      3uuR R R 4       # S	! R VP                  ^ 4      4      uuR R R 4       #   + '       g   i     R # ; i)Nr   r·   r£   )rL   r¸   r¹   ÚrangeÚconcatenateÚappendr½   )rH   r   rÒ   Ú	n_predictr¾   Úout_yÚout_logprobsÚirm   rà   r’   Úprev_tokensrÁ   s   &&&&     €€€€r&   rÃ   Ú(speculative_generate_step.<locals>._step)  s  ø€ Ü�YŠYÔ(×)Õ)Ù˜1˜T�7¨%Ô0ˆFØ˜A 	˜z™{¨AÐ-Õ.ˆFá˜eÔ$ß à&(¨"�|Ø˜q”=ØÐ,˜i¨!�mÐ,Ð-�AÜ˜yÖ)�Að 'Ò2ô Ÿš¨°QÐ'7Ô8àð  ñ
 #6°kÀ6È!ÈQÐPQÈ'Å?Ó"S‘K�AØ—L‘L ”OØ ×'Ñ'¨Ö1ñ *ô —~’~ e°!Ô4´b·n²nØ  qô7ð ÷' *Ò)ñ. +¨4°·±ÀÓ1BÓC÷/ *×)×)Ó)ús   ¤CD/ÄD/Ä/E 	c                 ó:  <€ VP                   ^8”  d„   \        SVP                   ^,
          4      pV ! VRV R,          VR7       S! V4       \        P                  ! V Uu. uF  qDP                  NK  	  up4       W#R p\        P
                  ! 4        K”  V# u upi )r¢   Nr   )ÚsizerÈ   rL   rÉ   rÊ   rË   )rH   r   rÒ   rÑ   r�   r”   rÁ   s   &&&  €€r&   Ú_prefillÚ+speculative_generate_step.<locals>._prefillC  s|   ø€ Ø�f‰f�qŒjÜÐ0°!·&±&¸1µ*Ó=ˆLÙ�!�M�\Ð" 4Õ(°Õ6Ù˜eÔ$Ü�GŠG¡eÓ,¡e —W”W¡eÑ,Ô-Ø�-Ð ˆAÜ�NŠNÖØˆùò -s   Á Bc                 óž   <€ \         P                  ! SW,
          4       \         P                  ! S\        W,
          ^,
          ^ 4      4       R# ©r¢   N)r   Útrim_prompt_cacheÚmax)Ú	num_draftÚ
num_acceptÚdraft_cacheÚmodel_caches   &&€€r&   Ú_rewind_cacheÚ0speculative_generate_step.<locals>._rewind_cacheM  s5   ø€ Ü×Ò ¨YÕ-CÔDÜ×Ò ¬S°Õ1GÈ!Õ1KÈQÓ-OÖPr(   c                 ó  <€ V^ 8X  d&   \         P                  ! . \         P                  4      # . p\        V4       F6  pS! SSV 4      w  r\         P                  ! V 4       VP                  V 4       K8  	  \         P                  ! V4      # ©r   )rL   rT   Úuint32rã   rÌ   rå   rä   )rÒ   rô   Úysr�   rÃ   rö   rÚ   s   &&  €€€r&   Ú_draft_generateÚ2speculative_generate_step.<locals>._draft_generateQ  sk   ø€ Ø˜Œ>Ü—8’8˜B¤§	¡	Ó*Ð*ØˆÜ�yÖ!ˆAÙ˜ k°1Ó5‰DˆAÜ�MŠM˜!ÔØ�I‰I�aŽLñ "ô �~Š~˜bÓ!Ð!r(   TF)r¢   r¥   )ÚastyperL   rü   r   rÆ   rº   ÚlayersÚcan_trim_prompt_cacheÚis_trimmabler+   rx   rÅ   rÇ   r   rŽ   r¸   r¹   rÈ   rí   rä   rÉ   ÚtolistrT   ró   )"r™   rH   rÚ   r×   r�   r‘   r’   rˆ   r”   r‹   rŠ   r‰   rÒ   r�   Útypesrî   rø   rþ   Údraft_yÚntoksrô   rÓ   Údraft_tokensrÂ   rm   ÚtnÚdtnÚlpnrà   rÃ   rö   r÷   rê   rÁ   s"   &&f$$dd$d$$$                @@@@@@r&   Úspeculative_generate_stepr  Ù  s5  ÿùé € ðZ 	�‰”b—i‘iÓ €AØ€Kð ÒÜ×-Ò-¨eÓ4ˆÜ×-Ò-¨kÓ:‰à"Ð#6¤S¨¯©Ó%6Ð7ˆØ"¤3 u§|¡|Ó#4Ð#6Ð7ˆä×&Ò& {×3Ò3Ù+6ÓO©; a¿n¹n×>NÔ!”�a“×!Ô!©;ˆÐOÜØNÈuÈgÐUWÐXó
ð 	
ð ×:Ð:Ñ9€Gä!×)Ò)ÜØ-Ø#Øô	Ðö÷Dò Dö4öQ÷"ô 
�ŠÔ$×	%Õ	%Ù˜;¨°QÓ7ˆÙ�U˜K¨Ó+ˆ÷ 
&ð €Eà€IØ	€Að+$ØÜ˜J¨Õ.Ð0@ÓAˆIÙ*¨7°IÓ>ˆLØÒ&Ø)Ð*U¨K×,<Ñ,<¸q¿v¹vÕ,EÈ	Õ,QÐTUÕ,UÐV�Ü—’  <Ð0Ó1ˆAÙ$ U¨K¸¸IÈ½MÓJÑˆF�HÜ�GŠG�F˜LÔ)Ø'×.Ñ.Ó0ˆLØ—]‘]“_ˆFØˆAØ�i”-Ø% a�y¨,°q­/¸8ÀA½;˜�C�Ø˜”9ØØ�Q•�Ø˜•
�Ø˜#˜t�mÒ#Ø˜JÖ&ØØ�zÔ!Ø˜•
�Ø˜Q•i ¨!¥¨eÐ3Ò3à˜
Ô"Øä—’˜& �)˜¤b§i¡iÓ0ˆAØˆGð
 �IŒ~ÜŸ.š.Ü—X’X˜l¨2¨3Ð/´·±Ó;¸WÐEó�ð Ò&Ø)Ð*B¬S°¸QµÀÓ-BÐ,BÐC�Ù˜) Q×'á�i Ö#ùòy P÷R 
&×	%Ð	%ûñf 	�i Õ#üs]   ŒBN
Â+N
Â/M#ÃM#Ã$N
Ã>A,N
Å*M(Å?N
ÆC<M< ÊC	M< ÍN
Í(M9	Í3	N
Í<NÎN
c                ó2  € V ^8„  d   QhR\         P                  R\        \        \        3,          R\        \
        \        P                  \        \        ,          3,          R\        R\        \         P                  ,          R\        \        RR3,          /# )rG   rH   Ú	tokenizerr™   r�   rÚ   rš   N)rJ   rK   r   r   r   r6   rL   rT   r	   r9   r
   r   ri   )rN   s   "r&   rO   rO   ‘  s†   € ÷ `
ñ `
Ü�9‰9ð`
äÔ(Ô*:Ð:Õ;ð`
ô ”#”r—x‘x¤¤c¥Ð*Õ+ð`
ô ð	`
ô
 œ"Ÿ)™)Õ$ð`
ô Ô! 4¨Ð-Õ.ñ`
r(   c              +  ó(  "  € \        V\        4      '       g   \        V4      p\        V\        P                  4      '       gu   \        V\        4      '       dI   VP
                  RJ ;'       g!    VP                  VP
                  4      '       * pVP                  W&R7      p\        P                  ! V4      pVP                  pW5R&   Vf)   VP                  RR4       \        W 3/ VB pR V 4       pM1VP                  RR4       VP                  RR4       \        W V3/ VB p\        V \        .4      ;_uu_ 4        \        P                  ! 4       p	\!        V4       Fû  w  p
w  r¼pV
^ 8X  dE   \        P                  ! 4       V	,
          pVP"                  V,          p\        P                  ! 4       p	W±P$                  9   d    M™VP'                  V4       V
^,           V8X  d    Mx\)        VP*                  VVVVP"                  XV
^,           V
^,           \        P                  ! 4       V	,
          ,          \        P,                  ! 4       R,          RR	7
      x € Ký  	  VP/                  4        \)        VP*                  XXXVP"                  XX
^,           V
^,           \        P                  ! 4       V	,
          ,          \        P,                  ! 4       R,          W±P$                  9   d   R
MRR	7
      x € RRR4       R#   + '       g   i     R# ; i5i)aW  
A generator producing text based on the given prompt from the model.

Args:
    model (nn.Module): The model to use for generation.
    tokenizer (PreTrainedTokenizer): The tokenizer.
    prompt (Union[str, mx.array, List[int]]): The input prompt string or
      integer tokens.
    max_tokens (int): The maximum number of tokens to generate.
      Default: ``256``.
    draft_model (Optional[nn.Module]): An optional draft model. If provided
      then speculative decoding is used. The draft model must use the same
      tokenizer as the main model. Default: ``None``.
    kwargs: The remaining options get passed to :func:`generate_step`.
      See :func:`generate_step` for more details.

Yields:
    GenerationResponse: An instance containing the generated text segment and
        associated metadata. See :class:`GenerationResponse` for details.
N©Úadd_special_tokensr�   r×   c              3   ó.   "  € T F  w  rWR 3x € K  	  R# 5i)FNr2   )Ú.0rl   rm   s   &  r&   Ú	<genexpr>Ú"stream_generate.<locals>.<genexpr>Á  s   é € ð 
Ù<K©¨ˆU˜eÕ$»Oùs   ‚r“   r–   ç    eÍÍA)
rk   rl   rm   rn   ro   rp   rq   rr   rs   rt   ÚstopÚlength)rS   r   rL   rT   r6   Ú	bos_tokenÚ
startswithÚencodeÚdetokenizerÚpoprÖ   r  rg   r¹   ÚtimeÚperf_counterr…   rí   Úeos_token_idsÚ	add_tokenri   Úlast_segmentÚget_peak_memoryÚfinalize)rH   r  r™   r�   rÚ   Úkwargsr  r  Útoken_generatorÚticrÓ   rl   rm   rn   Úprompt_timerp   s   &&&&&,          r&   Ústream_generater)  ‘  s–  é € ô8 �iÔ!1×2Ò2Ü$ YÓ/ˆ	ä�fœbŸh™h×'Ò'Ü�fœc×"Ò"à!*×!4Ñ!4¸Ð!<÷ "ð "ÀF×DUÑDUØ×#Ñ#óEô AÐð ×%Ñ% fÐ%ÓTˆFÜ—’˜&Ó!ˆà×'Ñ'€Kà%ˆ<ÑàÒØ�
‰
Ð% tÔ,Ü'¨Ñ@¸Ñ@ˆñ
Ù<Kó
‰ð 	�
‰
�= $Ô'Ø�
‰
Ð-¨tÔ4Ü3Ø˜;ñ
Ø*0ñ
ˆô 
�UÔ.Ð/×	0Õ	0Ü×ÒÓ!ˆÜ09¸/Ö0JÑ,ˆAÑ,� Ø�AŒvÜ"×/Ò/Ó1°CÕ7�Ø#Ÿ[™[¨;Õ6�
Ü×'Ò'Ó)�Ø×/Ñ/Ô/Ùà×!Ñ! %Ô(Ø�A•˜*Ô$Ùä$Ø ×-Ñ-ØØ!Ø%Ø$Ÿk™kØ%Ø"# a¥%Ø ! A¥¬$×*;Ò*;Ó*=ÀÕ*CÕDÜ×.Ò.Ó0°3Õ6Ø"ôô ñ 1Kð2 	×ÑÔÜ Ø×)Ñ)ØØØ!Ø Ÿ+™+Ø!Ø !�eØ �E¤d×&7Ò&7Ó&9¸CÕ&?Õ@Ü×*Ò*Ó,¨sÕ2Ø$)×-DÑ-DÔ$D™&È(ô
ò 	
÷9 
1×	0×	0Ò	0üs&   ‚A,LÁ/CLÄ>F6K>Ë4
LË>L	Ì		Lc          
      óÂ   € V ^8„  d   QhR\         P                  R\        \        \        3,          R\        \
        \        \        ,          3,          R\        R\
        /# )rG   rH   r  r™   Úverboserš   )	rJ   rK   r   r   r   r6   r	   r9   ru   )rN   s   "r&   rO   rO   ô  sW   € ÷ +ñ +Ü�9‰9ð+äÔ(Ô*:Ð:Õ;ð+ô ”#”tœC•y�.Õ!ð+ô ð	+ô 	ñ+r(   c                ó   € V'       d   \        R4       Rp\        WV3/ VB  F5  pV'       d   \        VP                  RRR7       WVP                  ,          pK7  	  V'       d›   \        4        \        R4       \        V4      ^ 8X  d   \        R4       R# \        RXP                   RVP
                  R R	24       \        R
VP                   RVP                  R R	24       \        RVP                  R R24       V# )a¼  
Generate a complete response from the model.

Args:
   model (nn.Module): The language model.
   tokenizer (PreTrainedTokenizer): The tokenizer.
   prompt (Union[str, List[int]]): The input prompt string or integer tokens.
   verbose (bool): If ``True``, print tokens and timing information.
       Default: ``False``.
   kwargs: The remaining options get passed to :func:`stream_generate`.
      See :func:`stream_generate` for more details.
Ú T)ÚendÚflushz!No text generated for this promptNzPrompt: ú	 tokens, ú.3fú tokens-per-seczGeneration: zPeak memory: ú GBz
==========)	r^   r)  rk   rº   ro   rp   rq   rr   rs   )rH   r  r™   r+  r%  rk   Úresponses   &&&&,  r&   Úgenerater5  ô  sî   € ÷& ÜˆhŒà€DÜ# E°fÑGÀÔGˆßÜ�(—-‘- R¨tÕ4Ø—‘ÕŠñ H÷
 ÜŒÜˆhŒÜˆt‹9˜Œ>ÜÐ5Ô6ÙÜØ�x×-Ñ-Ð.¨iØ×"Ñ" 3Ð' ð8ô	
ô 	Ø˜8×5Ñ5Ð6°iØ×&Ñ& sÐ+¨?ð<ô	
ô 	�˜h×2Ñ2°3Ð7°sÐ;Ô<Ø€Kr(   c           
      óÂ   € Vf   \        R V  4       4      p\        P                  ! V  Uu. uF$  p^ .V\        V4      ,
          ,          V,           NK&  	  up4      # u upi )Nc              3   ó8   "  € T F  p\        V4      x € K  	  R # 5irR   ©rº   ©r  Úps   & r&   r  Ú$_left_pad_prompts.<locals>.<genexpr>$  ó   é € Ð1© Aœ˜QŸ˜«ùó   ‚©ró   rL   rT   rº   ©ÚpromptsÚ
max_lengthr:  s   && r&   Ú_left_pad_promptsrB  "  sQ   € ØÒÜÑ1©Ó1Ó1ˆ
Ü�8Š8¹gÓF¹g¸�a�S˜J¬¨Q«Õ/Õ0°1×4Ð4¹gÑFÓGÐGùÒFs   «*Ac                 óÀ   € Vf   \        R V  4       4      p\        P                  ! V  Uu. uF#  q"^ .V\        V4      ,
          ,          ,           NK%  	  up4      # u upi )Nc              3   ó8   "  € T F  p\        V4      x € K  	  R # 5irR   r8  r9  s   & r&   r  Ú%_right_pad_prompts.<locals>.<genexpr>*  r<  r=  r>  r?  s   && r&   Ú_right_pad_promptsrF  (  sO   € ØÒÜÑ1©Ó1Ó1ˆ
Ü�8Š8¹gÓF¹g¸˜!˜ 
¬S°«VÕ 3Õ4×4Ð4¹gÑFÓGÐGùÒFs   «)Ac                   óL   a € ] tR tRt o Rt^ t^ t^ t^ t^ t	^ t
^ tV 3R ltRtV tR# )Ú
BatchStatsi.  aý  
An data object to hold generation stats.

Args:
    prompt_tokens (int): The number of prompt tokens processed.
    prompt_tps (float): The prompt processing tokens-per-second.
    prompt_time (float): The time in seconds spent in prompt processing.
    generation_tokens (int): The number of generated tokens.
    generation_tps (float): The tokens-per-second for generation.
    generation_time (float): The time in seconds spent in generation .
    peak_memory (float): The peak memory used so far in GB.
c                ón   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[ ;R&   S[;R&   S[;R&   S[;R&   # )rG   ro   rp   r(  rq   rr   Úgeneration_timers   )r9   r;   )rN   rv   s   "€r&   rO   ÚBatchStats.__annotate__.  s`   ø‡ ‚ ñ Ññ ñ  Ññ! ñ" Ññ# ñ$ Ññ% ñ& Ññ' ñ( Ññ) ñ* Ñò+ r(   r2   N)rx   ry   rz   r{   r|   ro   rp   r(  rq   rr   rJ  rs   r}   r~   r   r€   s   @r&   rH  rH  .  s7   ø‡ € ñð €MØ€JØ€KØÐØ€NØ€OØ€K÷+ ƒ r(   rH  c                óD  aa€ VV3R lo\        V R4      '       d)   V P                  4       pV Uu. uF  pS! V4      NK  	  up# Ve'   V P                   Uu. uF  p\        VS4      NK  	  up# V P                   Uu. uF  p\	        S4      NK  	  up# u upi u upi u upi )zO
Convert a list of regular caches into their corresponding
batch-aware caches.
c                 óÊ  <€ \        V 4      \        J d   \        S4      # \        V \        4      '       d   \
        P                  ! S4      V n        V # \        V \        4      '       d3   V P                  ^ 8”  d   \        R4      h\        V P                  S4      # \        V \        4      '       d   \        V3R lV P                   4       !  # \        \        V 4       R24      h)r   z2RotatingKVCache with keep tokens is not supported.c              3   ó4   <"  € T F  pS! V4      x € K  	  R # 5irR   r2   )r  Úsub_cÚto_batch_caches   & €r&   r  Ú6_make_cache.<locals>.to_batch_cache.<locals>.<genexpr>W  s   øé € ÐKÁ(¸™~¨e×4Ð4Ã(ùs   ƒz does not yet support batching)r+   r   r   rS   r   rL   rT   Úleft_paddingr   ÚkeeprÅ   r   Úmax_sizer   Úcaches)r�   rR  rP  s   &€€r&   rP  Ú#_make_cache.<locals>.to_batch_cacheL  sª   ø€ Ü�‹7”gÓÜ Ó-Ð-Ü˜œ;×'Ò'ÜŸXšX lÓ3ˆAŒNØˆHÜ˜œ?×+Ò+Ø�v‰v˜ŒzÜ Ð!UÓVÐVÜ'¨¯
©
°LÓAÐAÜ˜œ9×%Ò%ÜÔKÀ!Ç(Â(ÓKÑLÐLä¤ Q£˜yÐ(FÐGÓHÐHr(   Ú
make_cache)r†   rW  r  r   r   )rH   rR  r“   r   r�   r�   rP  s   &f&   @r&   Ú_make_cacherX  F  sž   ù€ öIô ˆu�l×#Ò#Ø× Ñ Ó"ˆÙ+0Ó1©5 a‘˜qÖ!©5Ñ1Ð1àÒ"àINÏÊóÙIUÀAÔ$ [°,Ö?Éñð ð 5:·L²LÓA±L¨q”˜\Ö*±LÑAÐAùò 2ùòùò Bs   °BÁBÁ<Bc           
      ó~  € . pV '       g   V# \        \        V ^ ,          4      4       FŒ  p\        V ^ ,          V,          R4      '       dG   TP                  V ^ ,          V,          P	                  V  Uu. uF  q3V,          NK  	  up4      4       Ki  \        \        V ^ ,          V,          4       R24      h	  V# u upi )r   Úmergez+ does not yet support batching with history)rã   rº   r†   rå   rZ  rÅ   r+   )rU  Úbatch_cacheré   r�   s   &   r&   Ú_merge_cachesr\  f  s¤   € Ø€KçØÐä”3�v˜a•y“>Ö"ˆÜ�6˜!•9˜Q•< ×)Ò)Ø×Ñ˜v a�y¨�|×1Ñ1ÁÓ2HÁ¸A°Q·4°4ÁÑ2HÓIÖJäÜ˜˜q�	 !�Ó%Ð&Ð&QÐRóð ñ	 #ð Ðùò 3Is   Á3B:c                 óx   € V '       g   V# V'       g   V # \        W4       F  w  r#VP                  V4       K  	  V # rR   )ÚzipÚextend)Úcache_aÚcache_bÚcaÚcbs   &&  r&   Ú_extend_cacherd  v  s3   € ßØˆßØˆÜ�gÖ'‰ˆØ
�	‰	�"Žñ (à€Nr(   c                óÀ  € / p\        V 4       F3  w  r#Tp V F  pVP                  V/ 4      pK  	  \        V4      V3VR&   K5  	  \	        4       pVP                  4        F#  w  rxVR8X  d   K  WR&   VP                  V4       K%  	  V'       d¤   VP                  4       p	V	P                  4        F}  w  rxVR9   d   K  VP                  V4       V	R,          p
Wz9  d   W¡Jd   V
R,          p
K  Wz9   d	   W§,          MTVR&   RV9  g   KZ  RVR,          9   g   Kj  VR,          R,          VR&   K  	  K«  V#   \         d     TP                  T/ 4      pT3T3TR&    EKN  i ; i)uu   Build an Aho-Corasick trie from the provided sequences

See https://en.wikipedia.org/wiki/Ahoâ€“Corasick_algorithm .
Ú	__match__Ú__fail__)rg  rf  )r…   Ú
setdefaultÚtupleÚ	TypeErrorr   Úitemsrå   Úpopleft)Ú	sequencesÚtrieÚidxÚseqÚnodeÚtokÚqueueÚkeyÚchildÚparentÚfails   &          r&   Ú_build_trierx  €  sU  € ð
 €DÜ˜iÖ(‰ˆØˆð	.Û�Ø—‘ s¨BÓ/’ñ ä!& s£¨SÐ 1ˆD�Óñ )ô ‹G€EØ—j‘j–l‰
ˆØ�+ÔÙØ ˆjÑØ�‰�UÖñ	 #÷
 Ø—‘“ˆØ Ÿ,™,ž.‰JˆCØÐ/Ô/ÙØ�L‰L˜ÔØ˜*Õ%ˆDØ”/ dÓ&6Ø˜JÕ'’Ø-0¬[ ¦	¸dˆE�*ÑØ %Ö'¨K¸5ÀÕ;LÖ,LØ%*¨:Õ%6°{Õ%C��kÓ"ó )ð €Køô/ ô 	.Ø—?‘? 3¨Ó+ˆDØ"% ¨ ˆD�Õð	.ús   –+D3Ä3%EÅEc                óN   € W 9  d   WJd   V R,          p K  W 9   d	   W,          p V # )z"One step in the Aho-Corasick trie.rg  r2   )rq  rn  rW   s   &&&r&   Ú
_step_trierz  ¦  s'   € à
Œ-˜DÓ,Ø�JÕŠØ„yØ�wˆØ€Kr(   c                   óN   a € ] tR tRt o Rt/ R3R ltR tR t]R 4       t	Rt
V tR	# )
ÚSequenceStateMachinei¯  a¢  A state machine that uses one Aho-Corasick trie per state to efficiently
track state across a generated sequence.

The transitions are provided as state -> [(sequence, new_state)].

Example:

    sm = SequenceStateMachine(
        transitions={
            "normal": [
                (think_start_tokens, "reasoning"),
                (tool_start_tokens, "tool"),
                (eos, None),
            ],
            "reasoning": [
                (think_end_tokens, "normal"),
                (eos, None),
            ],
            "tool": [
                (tool_end_tokens, None),
                (eos, None)
            ],
        },
        initial="normal"
    )
Únormalc                óø   € W n         / V n        VP                  4        F*  w  r4\        V!  w  rV\	        V4      V3V P                  V&   K,  	  V P                  '       g   \	        . 4      . 3V P                  V&   R # R # rR   )Ú_initialÚ_statesrk  r^  rx  )ÚselfÚtransitionsÚinitialÚsrcÚedgesrm  Údsts   &&&    r&   Ú__init__ÚSequenceStateMachine.__init__Ë  sm   € ØŒØˆŒØ%×+Ñ+Ö-‰JˆCÜ  %™[‰NˆIÜ!,¨YÓ!7¸Ð =ˆD�L‰L˜Óñ .ð �|�|ˆ|Ü%0°£_°bÐ$9ˆD�L‰L˜Ó!ñ r(   c                ó|   € \         P                  \        4      pV P                  Vn        V P                  Vn        V# rR   )ÚobjectÚ__new__r|  r  r€  )r�  ÚmemoÚnews   && r&   Ú__deepcopy__Ú!SequenceStateMachine.__deepcopy__Ô  s,   € Ü�n‰nÔ1Ó2ˆØ—}‘}ˆŒØ—l‘lˆŒØˆ
r(   c                óx   € V P                   V P                  V P                   ,          ^ ,          V P                  3# rû   ©r  r€  ©r�  s   &r&   Ú
make_stateÚSequenceStateMachine.make_stateÚ  s)   € Ø—‘˜tŸ|™|¨D¯M©MÕ:¸1Õ=¸t¿|¹|ÐLÐLr(   c                óð   € V w  r#p\        W4V,          ^ ,          V4      pRpVP                  R4      pVe<   V^ ,          pWB,          ^,          V^,          ,          pVe   WB,          ^ ,          MRpW#V3WR3# )r   Nrf  )rz  Úget)rÊ   rW   rf   rÓ   Ústatesrp  Úmatchs   &&     r&   r˜  ÚSequenceStateMachine.matchÝ  su   € à‰ˆˆfÜ�q �) A�,¨Ó*ˆàˆØ—‘�kÓ"ˆØÒØ˜•(ˆCØ•	˜!•˜U 1�XÕ&ˆAØ !¢�•	˜!–°4ˆAà�fˆ~˜sÐ%Ð%r(   r‘  N)rx   ry   rz   r{   r|   r‡  rŽ  r“  Ústaticmethodr˜  r~   r   r€   s   @r&   r|  r|  ¯  s5   ø‡ € ñð6 $&¨xô :òòMð ñ&ó ö&r(   r|  c                   óä   a € ] tR tRt o Rt] ! R R4      4       tRV 3R lR lltR tV 3R	 lR
 lt	R t
R tV 3R lR ltV 3R lR ltV 3R lR ltV 3R lR lt]RV 3R lR ll4       tRtV tR# )ÚPromptProcessingBatchiì  zî
A batch processor for prompt tokens with support for incremental processing.

This class handles batched prompt processing, managing KV caches and preparing
tokens for generation. It supports extending, filtering, and splitting batches.
c                   ó,   a € ] tR tRt o V 3R ltRtV tR# )ÚPromptProcessingBatch.Responseiô  c                óJ   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   # )rG   ÚuidÚprogressÚend_of_segmentÚend_of_prompt)r9   ri  ru   )rN   rv   s   "€r&   rO   Ú+PromptProcessingBatch.Response.__annotate__ô  s5   ø‡ ‚ á‰ñ ñ ‰ñ ñ Ññ	 ñ
 Ñò r(   r2   N©rx   ry   rz   r{   r}   r~   r   r€   s   @r&   ÚResponserž  ô  ó   ø‡ ‡ „ r(   r¦  Nc                óB  <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[S[S[,          ,          RS[S[S[S[,          ,          ,          RS[RS[S[S[S[P                  .S[P                  3,          ,          ,          RS[S[S[P                  .S[P                  3,          ,          RS[S[S[S[S[P                  S[P                  .S[P                  3,          ,          ,          ,          R	S[S[S[	,          ,          R
S[S[S[,          ,          /
# )rG   rH   ÚuidsrU  rÂ   r”   ÚsamplersÚfallback_samplerr’   Ústate_machinesr�   )
rJ   rK   r	   r9   r   r
   r   rL   rT   r|  )rN   rv   s   "€r&   rO   Ú"PromptProcessingBatch.__annotate__û  s  ø€ ÷ #
ñ #
á�y‰yð#
ñ ‘3�ið#
ñ ‘T™#•Y•ð	#
ñ
 ™™d¡3�i�Õ)ð#
ñ ð#
ñ ™4¡©"¯(©(¨±R·X±XÐ)=Õ >Õ?Õ@ð#
ñ #¡8©R¯X©X¨J¹¿¹Ð,@Õ#AÕBð#
ñ $Ù‘‘h¡§¡©"¯(©(Ð3±R·X±XÐ=Õ>Õ?Õ@õ
ð#
ñ !¡Ñ&:Õ!;Õ<ð#
ñ ™T¡#�YÕ'ñ#
r(   c                ó˜  € Wn         W n        \        V4      V n        Ve   TMV Uu. uF  p. NK  	  upV n        WPn        Ve   TM. V n        T;'       g    R V n        Ve   TM. V n        V	e   T	M\        4       .\        V4      ,          V n        V
e
   V
V n        R # \        .\        V P                  4      ,          V n        R # u upi )Nc                 ó2   € \         P                  ! V RR7      # r¡   r¦   r¨   s   &r&   rX   Ú0PromptProcessingBatch.__init__.<locals>.<lambda>  s   € ¼r¿yºyÈÐQSÕ?Tr(   )rH   r©  r\  rˆ   rÂ   r”   rª  r«  r’   r|  rº   r¬  r:   r�   )r�  rH   r©  rU  rÂ   r”   rª  r«  r’   r¬  r�   r�   s   &&&&&&&&&&& r&   r‡  ÚPromptProcessingBatch.__init__û  sÍ   € ð Œ
ØŒ	Ü)¨&Ó1ˆÔØ &Ò 2‘fÁTÓ8JÁTÀ»ÁTÑ8JˆŒà!2ÔØ$,Ò$8™¸bˆŒØ 0× UÐ UÑ5TˆÔà!2Ò!>ÑÀBð 	Ôð
 Ò)ñ ä&Ó(Ð)¬C°«IÕ5ð 	Ôð Ò%ð ð 	Žô %Ð%¬¨D¯I©I«Õ6ð 	Žùò 9Ks   §Cc                ó,   € \        V P                  4      # rR   ©rº   r©  r’  s   &r&   Ú__len__ÚPromptProcessingBatch.__len__   ó   € Ü�4—9‘9‹~Ðr(   c                ó6   <€ V ^8„  d   QhRS[ RS[S[,          /# ©rG   ro  rš   ©r9   r	   r   )rN   rv   s   "€r&   rO   r­  #  ó   ø€ ÷ ;ñ ;¡ð ;©©c­ñ ;r(   c                ó`   € V P                    Uu. uF  q"P                  V4      NK  	  up# u upi rR   ©rˆ   Úextract©r�  ro  r�   s   && r&   Úextract_cacheÚ#PromptProcessingBatch.extract_cache#  ó(   € Ø(,×(9Ò(9Ó:Ñ(9 1—	‘	˜#–Ñ(9Ñ:Ð:ùÒ:ó   �+c                óè  € \        V P                  4      '       g#   R .\        V P                  4      ,          V n        \        V P                  4      '       g#   R .\        V P                  4      ,          V n        \        VP                  4      '       d   VP                  MR .\        VP                  4      ,          p\        VP                  4      '       d   VP                  MR .\        VP                  4      ,          pV P                  P                  VP                  4       \        V P                  VP                  4      V n        V P                  P                  VP                  4       V P                  P                  V4       V P                  P                  V4       V P                  P                  VP                  4       V P                  P                  VP                  4       R # rR   )Úanyrª  rº   r©  r’   r_  rd  rˆ   rÂ   r�   r¬  )r�  Úbatchrª  r’   s   &&  r&   r_  ÚPromptProcessingBatch.extend&  sN  € Ü�4—=‘=×!Ò!Ø!˜F¤S¨¯©£^Õ3ˆDŒMÜ�4×)Ñ)×*Ò*Ø&* V¬c°$·)±)«nÕ%<ˆDÔ"Ü%(¨¯©×%8Ò%8�5—>’>¸t¸fÄsÈ5Ï:É:ÃÕ>Vˆô �5×*Ñ*×+Ò+ð ×#Ò#à�œ#˜eŸj™j›/Õ)ð 	ð 	�	‰	×Ñ˜Ÿ™Ô$Ü)¨$×*;Ñ*;¸U×=OÑ=OÓPˆÔØ�‰×Ñ˜5Ÿ<™<Ô(Ø�‰×Ñ˜XÔ&Ø×Ñ×%Ñ%Ð&7Ô8Ø�‰×Ñ˜u×/Ñ/Ô0Ø×Ñ×"Ñ" 5×#7Ñ#7Ö8r(   c                ó8  € V P                   P                  V P                   4      pV P                  Vn        \        V P                  4      Vn        \
        P                  ! V P                  4      Vn        \        V P                  4      Vn        V P                  Vn	        \        V P                  4      Vn
        V P                  Vn        \        V P                  4      Vn        \        V P                  4      Vn        \        V P                  4      Vn        V# rR   )Ú	__class__r‹  rH   Úlistr©  ÚcopyÚdeepcopyrˆ   rÂ   r”   rª  r«  r’   r¬  r�   )r�  Ú	new_batchs   & r&   Ú_copyÚPromptProcessingBatch._copy:  sÆ   € Ø—N‘N×*Ñ*¨4¯>©>Ó:ˆ	ØŸ*™*ˆ	ŒÜ˜dŸi™i›ˆ	ŒÜ!%§¢¨t×/@Ñ/@Ó!Aˆ	ÔÜ §¡Ó,ˆ	ÔØ&*×&<Ñ&<ˆ	Ô#Ü! $§-¡-Ó0ˆ	ÔØ%)×%:Ñ%:ˆ	Ô"Ü&*¨4×+AÑ+AÓ&Bˆ	Ô#Ü#'¨×(;Ñ(;Ó#<ˆ	Ô Ü# D§O¡OÓ4ˆ	ÔØÐr(   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# )rG   Úindices©r	   r9   )rN   rv   s   "€r&   rO   r­  H  s   ø€ ÷ ñ ™T¡#�Yñ r(   c           	     ó   € \        V4      p\        \        \        \        V P                  4      4      4      \        V4      ,
          4      pV P                  4       pV P                  V4       VP                  V4       V# rR   )ÚsortedÚsetrã   rº   r©  rÍ  Úfilter)r�  rÐ  Úindices_leftrÌ  s   &&  r&   ÚsplitÚPromptProcessingBatch.splitH  s[   € Ü˜“/ˆÜœc¤%¬¨D¯I©I«Ó"7Ó8¼3¸w»<ÕGÓHˆØ—J‘J“Lˆ	Ø�‰�LÔ!Ø×Ñ˜Ô!àÐr(   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# ©rG   rS  rÑ  )rN   rv   s   "€r&   rO   r­  Q  s   ø€ ÷ Iñ I™4¡�9ñ Ir(   c                ót  € V Uu. uF  q P                   V,          NK  	  upV n         V'       g   V P                  P                  4        M$V P                   F  pVP                  V4       K  	  V Uu. uF  q P                  V,          NK  	  upV n        \        V P                  4      '       d)   V Uu. uF  q P                  V,          NK  	  upV n        MR .\        V4      ,          V n        \        V P                  4      '       d)   V Uu. uF  q P                  V,          NK  	  upV n        M. .\        V4      ,          V n        V Uu. uF  q P                  V,          NK  	  upV n	        V Uu. uF  q P                  V,          NK  	  upV n
        R # u upi u upi u upi u upi u upi u upi rR   )r©  rˆ   ÚclearrÕ  rÂ   rÄ  rª  rº   r’   r�   r¬  ©r�  rS  ro  r�   s   &&  r&   rÕ  ÚPromptProcessingBatch.filterQ  sW  € Ù/3Ó4©t¨—Y‘Y˜s—^�^©tÑ4ˆŒ	ßØ×Ñ×#Ñ#Õ%à×&Ô&�Ø—‘˜–ñ 'á37Ó8±4¨C—{‘{ 3×'Ð'±4Ñ8ˆŒÜˆt�}‰}×ÒÙ;?Ó@¹4°CŸ]™]¨3×/Ð/¹4Ñ@ˆD�Mà!˜F¤S¨£YÕ.ˆDŒMÜˆt×%Ñ%×&Ò&ÙMQÓ%RÉTÀc×&<Ñ&<¸S×&AÐ&AÉTÑ%RˆDÕ"à&( T¬C°«IÕ%5ˆDÔ"Ù;?Ó@¹4°CŸ?™?¨3×/Ð/¹4Ñ@ˆŒÙCGÓHÁ4¸C×2Ñ2°3×7Ð7Á4ÑHˆÖùò! 5ùò 9ùâ@ùò &Sùò AùÚHs#   …FÁ3F!Â5F&ÄF+ÅF0Å7F5c                ó@   <€ V ^8„  d   QhRS[ S[ S[,          ,          /# ©rG   rÂ   rÑ  )rN   rv   s   "€r&   rO   r­  d  s   ø€ ÷ .ñ .™T¡$¡s¥)�_ñ .r(   c                ó”  € \        V P                  4      \        V4      8w  d   \        R4      hV'       g   R# \        V P                  V4       F  w  r#W#,          pK  	  V Uu. uF  p\        V4      NK  	  pp\        V4      pV Uu. uF  qvV,
          NK  	  pp\        V4      p	V	^ 8”  d3   \        WR7      pV P                   F  p
V
P                  WXR7       K  	  M\        P                  ! V4      pVP                  ^,          ^ 8”  d­   \        V P                  VP                  ^,          4      pV P                  VRRV13,          V P                  R7       \        P                  ! V P                   U
u. uF  qªP                   NK  	  up
4       \        P"                  ! 4        VRVR13,          pKÄ  V	^ 8”  dt   V P                   F  p
V
P%                  4        K  	  \        P                  ! V P                   U
u. uF  qªP                   NK  	  up
4       \        P"                  ! 4        R# R# u upi u upi u up
i u up
i )za
Process prompt tokens through the model.

Args:
    tokens: List of token sequences to process.
z3The batch length doesn't match the number of inputsN)rA  )ÚlengthsÚright_paddingr·   r   )rº   r©  rÅ   r^  rÂ   ró   rF  rˆ   ÚpreparerL   rT   ÚshaperÈ   r”   rH   rÉ   rÊ   rË   r$  )r�  rÂ   ÚstiÚtir:  râ  rA  ÚlÚpaddingÚmax_paddingr�   rÑ   s   &&          r&   r™   ÚPromptProcessingBatch.promptd  sÇ  € ô ˆt�y‰y‹>œS ›[Ô(ÜÐRÓSÐSçÙô ˜4Ÿ;™;¨Ö/‰GˆCØ�IŠCñ 0ñ $*Ó*¡6˜a”3�q–6¡6ˆÐ*Ü˜“\ˆ
Ù+2Ó3©7 a —>�>©7ˆÐ3Ü˜'“lˆð ˜Œ?Ü'¨ÔFˆFØ×&Ô&�Ø—	‘	 '�	ÖAò 'ô —X’X˜fÓ%ˆFð �l‰l˜1�o Ô!Ü˜t×5Ñ5°v·|±|ÀAµÓGˆLØ�J‰J�v˜a  , Ð.Õ/°t×7HÑ7HˆJÔIÜ�GŠG d×&7Ò&7Ó8Ñ&7 —W”WÑ&7Ñ8Ô9Ü�NŠNÔØ˜A˜|™}Ð,Õ-ŠFð ˜Œ?Ø×&Ô&�Ø—
‘
–ñ 'ä�GŠG d×&7Ò&7Ó8Ñ&7 —W”WÑ&7Ñ8Ô9Ü�NŠNÖñ	 ùò/ +ùâ3ùò  9ùò 9s   Á$H6Â	H;Å;I ÈIc                ó@   <€ V ^8„  d   QhRS[ S[ S[,          ,          /# rà  rÑ  )rN   rv   s   "€r&   rO   r­  ”  s   ø€ ÷ "ñ "™t¡D©¥I�ñ "r(   c                ó\  € \         ;QJ d    R V 4       F  '       g   K   RM	  RM! R V 4       4      '       d$   T P                  V Uu. uF  q"RR NK	  	  up4       \        P                  ! V Uu. uF  q"R,          NK  	  up4      p\	        V P
                  V P                  VV P                  V P                  V P                  V P                  V P                  V P                  V P                  4
      p. V n        . V n        . V n        . V n	        . V n        . V n        V# u upi u upi )zµ
Transition from prompt processing to generation.

Args:
    tokens: Final tokens for each sequence to start generation.

Returns:
    A GenerationBatch ready for token generation.
c              3   ó>   "  € T F  p\        V4      ^8„  x € K  	  R# 5irñ   r8  )r  Úts   & r&   r  Ú1PromptProcessingBatch.generate.<locals>.<genexpr>ž  s   é € Ð*¡6˜aŒs�1‹v˜Žz£6ùs   ‚TFNr¥   )rÄ  r™   rL   rT   ÚGenerationBatchrH   r©  rˆ   rÂ   rª  r«  r’   r¬  r�   )r�  rÂ   rï  Ú
last_tokenÚ
generations   &&   r&   r5  ÚPromptProcessingBatch.generate”  sõ   € ÷ ‹3Ñ*¡6Ó*�3�3Š3Ñ*¡6Ó*×*Ò*Ø�K‰K©Ó0© A˜3˜B›©Ñ0Ô1Ü—X’X©fÓ5©f¨ Ÿu˜u©fÑ5Ó6ˆ
ä$Ø�J‰JØ�I‰IØØ×ÑØ�K‰KØ�M‰MØ×!Ñ!Ø×"Ñ"Ø×ÑØ�O‰Oó
ˆ
ð ˆŒ	ØˆÔØˆŒØˆŒØ!#ˆÔØˆŒàÐùò/ 1ùÚ5s   Á
D$Á2D)c                ó€   <€ V ^8„  d   QhRS[ P                  RS[S[P                  .S[P                  3,          RS[/# )rG   rH   r«  r”   )rJ   rK   r   rL   rT   r9   )rN   rv   s   "€r&   rO   r­  ¹  sA   ø€ ÷ 
ñ 
á�y‰yð
ñ #¡B§H¡H :©r¯x©xÐ#7Õ8ð
ñ ñ	
r(   c                ó&   € V ! VVV. . . . . . . R 7
      # ))
rH   r«  r”   r©  rU  rÂ   rª  r’   r�   r¬  r2   )ÚclsrH   r«  r”   s   &&&&r&   ÚemptyÚPromptProcessingBatch.empty¸  s/   € ñ ØØ-Ø/ØØØØØ ØØô
ð 	
r(   )
r«  r’   r�   rH   r”   rˆ   rª  r¬  rÂ   r©  )Nr•   NNNNN)r•   )rx   ry   rz   r{   r|   r   r¦  r‡  r´  r¿  r_  rÍ  r×  rÕ  r™   r5  Úclassmethodrø  r~   r   r€   s   @r&   rœ  rœ  ì  sŒ   ø‡ € ñð ÷ð ó ð÷#
ò #
òJ÷;ð ;ò9ò(÷ð ÷Ið I÷&.ð .÷`"ð "ðH ÷
ñ 
ó ö
r(   rœ  c                   óÄ   a € ] tR tRt o Rt] ! R R4      4       tV 3R lR ltR tR t	V 3R	 lR
 lt
V 3R lR ltV 3R lR ltV 3R lR lt]V 3R lR l4       tRtV tR# )rñ  iÍ  zÜ
A batched token generator that manages multiple sequences in parallel.

This class handles the generation phase after prompt processing, managing
KV caches, sampling, and stop sequence detection for multiple sequences.
c                   ó,   a € ] tR tRt o V 3R ltRtV tR# )ÚGenerationBatch.ResponseiÕ  c                ó  <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[P                  ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[S[ ,          ,          ;R&   S[S[S[,          ,          ;R&   S[S[S[ ,          ,          ;R&   # )	rG   r   rl   rm   rt   Úcurrent_stateÚmatch_sequencerˆ   Ú
all_tokens)r9   rL   rT   r
   r6   r	   r   )rN   rv   s   "€r&   rO   Ú%GenerationBatch.Response.__annotate__Õ  s�   ø‡ ‚ á‰ñ ñ ‰
ñ ñ —(‘(Ññ	 ñ
  ¡•}Ñ$ñ ñ  ¡•}Ñ$ñ ñ !¡¡c¥Õ+Ñ+ñ ñ ™t¡C�yÕ)Ñ)ñ ñ ™T¡#�YÕ'Ñ'ò r(   r2   Nr¥  r€   s   @r&   r¦  rý  Õ  r§  r(   r¦  c                ó  <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[P
                  RS[S[,          RS[S[S[,          ,          RS[S[S[S[P
                  .S[P
                  3,          ,          ,          RS[S[P
                  .S[P
                  3,          RS[S[S[S[S[P
                  S[P
                  .S[P
                  3,          ,          ,          ,          R	S[S[	,          R
S[S[,          /
# )rG   rH   r©  Úinputsrˆ   rÂ   rª  r«  r’   r¬  r�   )
rJ   rK   r	   r9   rL   rT   r   r
   r   r|  )rN   rv   s   "€r&   rO   ÚGenerationBatch.__annotate__à  sð   ø€ ÷ (ñ (á�y‰yð(ñ ‘3�ið(ñ —‘ð	(ñ
 ™3•ið(ñ ‘T™#•Y•ð(ñ ™4¡©"¯(©(¨±R·X±XÐ)=Õ >Õ?Õ@ð(ñ #¡B§H¡H :©r¯x©xÐ#7Õ8ð(ñ $Ù‘‘h¡§¡©"¯(©(Ð3±R·X±XÐ=Õ>Õ?Õ@õ
ð(ñ Ñ1Õ2ð(ñ ™•Iñ(r(   c                óø  € Wn         W n        W@n        WPn        W`n        Wpn        W€n        W�n        W n        V P                  '       d9   \        V P                  4      \        V P                  4      8w  d   \        R 4      hV P                  '       d9   \        V P                  4      \        V P                  4      8w  d   \        R4      hRV n        . V n        W0n        . V n        V Uu. uF  p\        V4      NK  	  upV n        ^ .\        V P                  4      ,          V n        V	 Uu. uF  qÌP%                  4       NK  	  upV n        V P                  '       d   V P)                  4        R# R# u upi u upi )z(Insufficient number of samplers providedz1Insufficient number of logits_processors providedN)rH   r©  rˆ   rÂ   rª  r«  r’   r¬  r�   rº   rÅ   Ú_current_tokensÚ_current_logprobsÚ_next_tokensÚ_next_logprobsr   Ú_token_contextÚ_num_tokensr“  Ú_matcher_statesrÃ   )r�  rH   r©  r  rˆ   rÂ   rª  r«  r’   r¬  r�   rï  Úms   &&&&&&&&&&&  r&   r‡  ÚGenerationBatch.__init__à  s  € ð Œ
ØŒ	Ø(ÔØŒà ŒØ 0ÔØ!2ÔØ,ÔØ$Œà�=�=ˆ=œS §¡Ó/´3°t·y±y³>ÔAÜÐGÓHÐHØ×!×!Ð!¤c¨$×*@Ñ*@Ó&AÄSÈÏÉÃ^Ô&SÜÐPÓQÐQà#ˆÔØ!#ˆÔØ"ÔØ ˆÔÙ7=Ó>±v°!œ{¨1ž~±vÑ>ˆÔØ˜3¤ T§Y¡Y£Õ/ˆÔÙ8FÓG¹°1§¡¦¹ÑGˆÔà�9�9ˆ9Ø�J‰JŽLñ ùò	 ?ùâGs   Ã*E2Ä,E7c                ó,   € \        V P                  4      # rR   r³  r’  s   &r&   r´  ÚGenerationBatch.__len__
  r¶  r(   c                ó>  € V P                   P                  VP                   4       \        V P                  VP                  4      V n        V P                  P                  VP                  4       V P
                  P                  VP
                  4       V P                  P                  VP                  4       V P                  P                  VP                  4       V P                  P                  VP                  4       V P                  f$   VP                  V n	        VP                  V n
        MdVP                  eW   \        P                  ! V P                  VP                  .4      V n	        V P                  P                  VP                  4       V P                  f$   VP                  V n        VP                  V n        MdVP                  eW   \        P                  ! V P                  VP                  .4      V n        V P                  P                  VP                  4       V P                  P                  VP                  4       V P                   P                  VP                   4       V P"                  P                  VP"                  4       R# )z0Extend this batch with another generation batch.N)r©  r_  rd  rˆ   rÂ   rª  r’   r�   r¬  r  r  rL   rä   r	  r
  r  r  r  )r�  rÅ  s   &&r&   r_  ÚGenerationBatch.extend  sÞ  € à�	‰	×Ñ˜Ÿ™Ô$Ü)¨$×*;Ñ*;¸U×=OÑ=OÓPˆÔØ�‰×Ñ˜5Ÿ<™<Ô(Ø�‰×Ñ˜UŸ^™^Ô,Ø×Ñ×%Ñ% e×&=Ñ&=Ô>Ø�‰×Ñ˜u×/Ñ/Ô0Ø×Ñ×"Ñ" 5×#7Ñ#7Ô8Ø×ÑÒ'Ø#(×#8Ñ#8ˆDÔ Ø%*×%<Ñ%<ˆDÕ"Ø×"Ñ"Ò.Ü#%§>¢>Ø×%Ñ% u×'<Ñ'<Ð=ó$ˆDÔ ð ×"Ñ"×)Ñ)¨%×*AÑ*AÔBØ×ÑÒ$Ø %× 2Ñ 2ˆDÔØ"'×"6Ñ"6ˆDÕØ×ÑÒ+Ü "§¢°×0AÑ0AÀ5×CUÑCUÐ/VÓ WˆDÔØ×Ñ×&Ñ& u×';Ñ';Ô<Ø×Ñ×"Ñ" 5×#7Ñ#7Ô8Ø×Ñ×Ñ × 1Ñ 1Ô2Ø×Ñ×#Ñ# E×$9Ñ$9Ö:r(   c                ój   <€ V ^8„  d   QhRS[ S[S[,          S[S[P                  ,          3,          /# ©rG   rš   )r   r	   r9   rL   rT   )rN   rv   s   "€r&   rO   r  (  s*   ø€ ÷ :.ñ :.‘u™T¡#�Y©©R¯X©X­Ð6Õ7ñ :.r(   c           
     óŽ  € V P                   V n        V P                  V n        V P                  pV P	                  VR,          V P
                  R7      pVR,          p. p\        V P                  4      '       dÅ   \        V P                  4       UUu. uF  w  rEVP                  WV^,            4      NK!  	  ppp. p\        \        V P                  4      4       FH  pW'V^,            pV P                  V,           F  p	V	! W7,          V4      pK  	  VP                  V4       KJ  	  \        P                   ! V^ R7      pV\        P"                  ! VRRR7      ,
          p
\        V P$                  4      '       dŠ   . p\        \        V P                  4      4       FL  pV P$                  V,          ;'       g    V P&                  pV! W§V^,            4      pVP                  V4       KN  	  \        P                   ! V^ R7      pMV P'                  V
4      pWÐn         \)        V
4      V n        \        P*                  ! V P                   V P                  V4       \        P,                  ! WP                  4       VP/                  4       p\1        V P2                  V4       F  w  rïVP                  V4       K  	  WP                  3# u uppi )zX
Perform a single generation step.

Returns:
    Tuple of token list and logprobs list.
r   r£   Trß   )r·   Nr¥   r¶   )r	  r  r
  r  rH   rˆ   rÄ  r’   r…   r  Úupdate_and_fetchrã   rº   r©  rå   rL   rä   r¼   rª  r«  rÉ  rÌ   rÉ   r  r^  rÂ   )r�  r  r¾   Útoken_contextré   ÚtcÚprocessed_logitsrŒ   Úsample_logitsr¿   rm   Úall_samplesÚsample_samplerrÀ   ræ  rç  s   &               r&   rÃ   ÚGenerationBatch._step(  sN  € ð  $×0Ñ0ˆÔØ!%×!4Ñ!4ˆÔØ×%Ñ%ˆð —‘˜F 7�O°4×3DÑ3D�ÓEˆØ˜Õ!ˆð ˆÜˆt×%Ñ%×&Ò&ô ' t×':Ñ':Ô;ôá;‘E�Að ×#Ñ# F¨q°1­uÐ$5Ö6Ù;ð ñ ð  "ÐÜœ3˜tŸy™y›>Ö*�Ø &¨1¨q­5Ð 1�Ø!%×!7Ñ!7¸×!:Ð!:�IÙ$-¨mÕ.>ÀÓ$N’Mñ ";à ×'Ñ'¨Ö6ñ	 +ô
 —^’^Ð$4¸1Ô=ˆFð œBŸLšL¨°bÀ4ÔHÕHˆô ˆt�}‰}×ÒØˆKÜœ3˜tŸy™y›>Ö*�Ø!%§¡¨qÕ!1×!JÐ!J°T×5JÑ5J�Ù(¨°a¸!µeÐ)<Ó=�Ø×"Ñ" 7Ö+ñ +ô —n’n [°qÔ9‰Gà×+Ñ+¨HÓ5ˆGð $ÔÜ" 8›nˆÔÜ
�Š�d×'Ñ'¨×)<Ñ)<¸mÔLô
 	�Š�×.Ñ.Ô/Ø—‘“ˆÜ˜4Ÿ;™;¨Ö/‰GˆCØ�J‰J�rŽNñ 0à×-Ñ-Ð-Ð-ùóOs   Â%Kc                ó6   <€ V ^8„  d   QhRS[ RS[S[,          /# r¸  r¹  )rN   rv   s   "€r&   rO   r  d  rº  r(   c                ó`   € V P                    Uu. uF  q"P                  V4      NK  	  up# u upi rR   r¼  r¾  s   && r&   r¿  ÚGenerationBatch.extract_cached  rÁ  rÂ  c                ó0   <€ V ^8„  d   QhRS[ S[,          /# rÚ  rÑ  )rN   rv   s   "€r&   rO   r  g  s   ø€ ÷ Kñ K™4¡�9ñ Kr(   c                ó´  € V Uu. uF  q P                   V,          NK  	  upV n         V'       g   V P                  P                  4        M$V P                   F  pVP                  V4       K  	  V Uu. uF  q P                  V,          NK  	  upV n        \        V P                  4      '       d(   V Uu. uF  q P                  V,          NK  	  upV n        \        V P                  4      '       d(   V Uu. uF  q P                  V,          NK  	  upV n        V Uu. uF  q P                  V,          NK  	  upV n        V Uu. uF  q P                  V,          NK  	  upV n	        V'       d   V P                  V,          MRV n
        V Uu. uF  q P                  V,          NK  	  upV n        V Uu. uF  q P                  V,          NK  	  upV n        V Uu. uF  q P                  V,          NK  	  upV n        V Uu. uF  q P                  V,          NK  	  upV n        R# u upi u upi u upi u upi u upi u upi u upi u upi u upi u upi )z4Filter the batch to keep only the specified indices.N)r©  rˆ   rÜ  rÕ  rÂ   rÄ  rª  r’   r�   r¬  r	  r
  r  r  r  rÝ  s   &&  r&   rÕ  ÚGenerationBatch.filterg  sÞ  € á/3Ó4©t¨—Y‘Y˜s—^�^©tÑ4ˆŒ	ßØ×Ñ×#Ñ#Õ%à×&Ô&�Ø—‘˜–ñ 'á37Ó8±4¨C—{‘{ 3×'Ð'±4Ñ8ˆŒÜˆt�}‰}×ÒÙ;?Ó@¹4°CŸ]™]¨3×/Ð/¹4Ñ@ˆDŒMÜˆt×%Ñ%×&Ò&ÙMQÓ%RÉTÀc×&<Ñ&<¸S×&AÐ&AÉTÑ%RˆDÔ"Ù;?Ó@¹4°CŸ?™?¨3×/Ð/¹4Ñ@ˆŒÙCGÓHÁ4¸C×2Ñ2°3×7Ð7Á4ÑHˆÔç7;˜D×-Ñ-¨dÖ3ÀˆÔÙCGÓHÁ4¸C×2Ñ2°3×7Ð7Á4ÑHˆÔÙCGÓHÁ4¸C×2Ñ2°3×7Ð7Á4ÑHˆÔÙ=AÓB¹T°c×,Ñ,¨S×1Ð1¹TÑBˆÔÙEIÓJÁT¸c× 4Ñ 4°S× 9Ð 9ÁTÑJˆÖùò% 5ùò 9ùâ@ùâ%RùÚ@ùÚHùò IùÚHùÚBùÚJs;   …H(Á3H-Â5H2Ã7H7ÄH<ÅIÆIÆ5IÇIÈIc                ó0   <€ V ^8„  d   QhRS[ S[,          /# r  )r	   r¦  )rN   rv   s   "€r&   rO   r  }  s   ø€ ÷ ;ñ ;‘d™8•nñ ;r(   c                óØ  € V P                   '       g   . # V P                  4       w  r. p. p\        \        V P                   4      4       EFh  pRpRpV P                  V;;,          ^,          uu&   V P                  V,          V P
                  V,          8¼  d   RpV P                  V,          P                  V P                  V,          W,          4      w  V P                  V&   rxVe   Vf   RpVei   VP                  V P                  V P                   V,          W,          W%,          VVVV P                  V4      V P                  V,          R7      4       EK  VP                  V4       VP                  V P                  V P                   V,          W,          W%,          RVVRRR7      4       EKk  	  \        V4      \        V P                   4      8  d   V P                  V4       V# )zk
Generate the next batch of tokens.

Returns:
    List of Response objects for each sequence in the batch.
Nr  r  )r   rl   rm   rt   rÿ  r   rˆ   r  )r   rl   rm   rt   r   rÿ  rˆ   r  )r©  rÃ   rã   rº   r  r�   r¬  r˜  r  rå   r¦  r¿  rÂ   rÕ  )	r�  rÂ   rm   rS  Ú	responsesré   rt   r   rÿ  s	   &        r&   ÚnextÚGenerationBatch.next}  s¨  € ð �y�yˆyØˆIàŸ:™:›<ÑˆàˆØˆ	Ü”s˜4Ÿ9™9“~×&ˆAØ ˆMØ!ˆNà×Ñ˜Q× 1Õ$ÓØ×Ñ Õ" d§o¡o°aÕ&8Ô8Ø (�ð ×#Ñ# AÕ&×,Ñ,¨T×-AÑ-AÀ!Õ-DÀfÅiÓPñ CˆD× Ñ  Ñ# ^ð Ò)¨mÒ.CØ &�àÒ(Ø× Ñ Ø—M‘MØ ŸI™I a�LØ$�iØ!)¥Ø&3Ø&3Ø'5Ø%)×%7Ñ%7¸Ó%:Ø#'§;¡;¨q¥>ð "ó 	÷ð —‘˜A”Ø× Ñ Ø—M‘MØ ŸI™I a�LØ$�iØ!)¥Ø&*Ø'5Ø&3Ø%)Ø#'ð "ó 	÷ñ; 'ôT ˆt‹9”s˜4Ÿ9™9“~Ô%Ø�K‰K˜ÔàÐr(   c                óz   <€ V ^8„  d   QhRS[ P                  RS[S[P                  .S[P                  3,          /# )rG   rH   r«  )rJ   rK   r   rL   rT   )rN   rv   s   "€r&   rO   r  »  s7   ø€ ÷ 
ñ 
á�y‰yð
ñ #¡B§H¡H :©r¯x©xÐ#7Õ8ñ
r(   c                ón   € V ! VV. \         P                  ! . \         P                  R 7      . . . . . . R7
      # ))Údtype)
rH   r«  r©  r  rˆ   rÂ   rª  r’   r�   r¬  )rL   rT   rü   )r÷  rH   r«  s   &&&r&   rø  ÚGenerationBatch.emptyº  s>   € ñ ØØ-ØÜ—8’8˜B¤b§i¡iÔ0ØØØØ ØØô
ð 	
r(   )r  r  r  r
  r	  r  r  r«  r’   r�   rH   rˆ   rª  r¬  rÂ   r©  N)rx   ry   rz   r{   r|   r   r¦  r‡  r´  r_  rÃ   r¿  rÕ  r(  rú  rø  r~   r   r€   s   @r&   rñ  rñ  Í  sx   ø‡ € ñð ÷(ð (ó ð(÷(ð (òTò;÷6:.ð :.÷x;ð ;÷Kð K÷,;ð ;ðz ÷
ó ö
r(   rñ  c                   ó  a € ] tR tRt o RtR^€RRRRRRR^ R	^R
RRRRR/	V 3R lR llt]R 4       tR tR t	]
P                  R#R l4       tR$V 3R lR lltR$V 3R lR lltR tR tR tR%R lt]R 4       tV 3R lR ltR tR  tR! tR"tV tR# )&ÚBatchGeneratoriÎ  a?  
A batch generator implements continuous batching.

This class provides automatic management of prompt processing and generation
batches, handling the transition between the two.

It also allows for segmented prompt processing which guarantees that the
generator will stop at these boundaries when processing an input.
r�   Ústop_tokensNr‘   r’   Úcompletion_batch_sizeÚprefill_batch_sizer”   r•   r“   r¸   c                ól  <€ V ^8„  d   QhRS[ P                  RS[RS[S[S[S[,          ,          ,          RS[S[S[P                  .S[P                  3,          ,          RS[S[S[S[P                  S[P                  .S[P                  3,          ,          ,          RS[RS[RS[R	S[S[,          /	# )
rG   rH   r�   r0  r‘   r’   r1  r2  r”   r“   )	rJ   rK   r9   r
   r   r   rL   rT   r	   )rN   rv   s   "€r&   rO   ÚBatchGenerator.__annotate__Ù  s¼   ø€ ÷ 4)ñ 4)á�y‰yð4)ñ ð	4)ñ
 ™h¡x±¥}Õ5Õ6ð4)ñ ™(¡B§H¡H :©r¯x©xÐ#7Õ8Õ9ð4)ñ $Ù‘™2Ÿ8™8¡R§X¡XÐ.±·±Ð8Õ9Õ:õ
ð4)ñ  #ð4)ñ  ð4)ñ ð4)ñ ™c•]ñ4)r(   c       	        ó>  € Wn         W n        T;'       g    R  V n        T;'       g    . V n        ^ V n        W€n        Wpn        \        Wg4      V n        W�n	        T
;'       g    \        V n        \        V'       d   RV Uu. uF  q»R3NK  	  up/M/ RR7      V n        ^ V n        \        P!                  V P                   V P                  VR7      V n        \$        P!                  V P                   V P                  4      V n        \)        4       V n        . V n        ^ V n        ^ V n        ^ V n        ^ V n        \6        P8                  P;                  4       '       d8   \6        P<                  ! \6        P>                  ! 4       R,          4      V n         R# RV n         R# u upi )c                 ó2   € \         P                  ! V RR7      # r¡   r¦   r¨   s   &r&   rX   Ú)BatchGenerator.__init__.<locals>.<lambda>ë  s   € ¬R¯YªY°q¸rÕ-Br(   r}  N)rƒ  )r”   rZ   )!rH   r�   r‘   r’   Ú	uid_countr”   r2  ró   r1  r“   r¹   Ú_streamr|  Ú_default_state_machineÚ
_uid_countrœ  rø  Ú_prompt_batchrñ  Ú_generation_batchr   Ú_unprocessed_sequencesÚ_currently_processingÚ_prompt_tokens_counterÚ_prompt_time_counterÚ_gen_tokens_counterÚ_steps_counterrL   r[   r\   r_   r]   Ú_old_wired_limit)r�  rH   r�   r0  r‘   r’   r1  r2  r”   r“   r¸   rp  s   &&$$$$$$$$$ r&   r‡  ÚBatchGenerator.__init__Ù  s[  € ð  Œ
Ø$ŒØ×CÐCÑ#BˆŒØ!2×!8Ð!8°bˆÔØˆŒØ!2ÔØ"4ÔÜ%(Ð)>Ó%SˆÔ"Ø&Ôà×2Ð2Ô!2ˆŒä&:ß@KˆX©{Ó;©{¨˜d›©{Ñ;Ñ<ÐQSØô'
ˆÔ#ð ˆŒÜ2×8Ñ8Ø�J‰JØ�L‰LØ/ð 9ó 
ˆÔô
 "1×!6Ñ!6°t·z±zÀ4Ç<Á<Ó!PˆÔÜ&+£gˆÔ#Ø%'ˆÔ"à&'ˆÔ#Ø$%ˆÔ!Ø#$ˆÔ ØˆÔä�8‰8× Ñ ×"Ò"Ü$&×$6Ò$6Ü—’Ó Ð!CÕDó%ˆDÖ!ð %)ˆDÖ!ùò/ <s   Â F
c                ó   € V P                   # rR   )r9  r’  s   &r&   r¸   ÚBatchGenerator.stream  s   € à�|‰|Ðr(   c                ó´   € V P                   eJ   \        P                  ! V P                  4       \        P                  ! V P                   4       R V n         R # R # rR   )rD  rL   r`   r9  r_   r’  s   &r&   ÚcloseÚBatchGenerator.close  s?   € Ø× Ñ Ò,Ü�NŠN˜4Ÿ<™<Ô(Ü×Ò˜t×4Ñ4Ô5Ø$(ˆDÖ!ñ -r(   c                ó&   € V P                  4        R # rR   )rI  r’  s   &r&   Ú__del__ÚBatchGenerator.__del__  s   € Ø�
‰
Žr(   c              #  óF  "  € T;'       g    \        4       p^ V n        ^ V n        ^ V n        \        P
                  ! 4       p Vx € \        P
                  ! 4       pW2,
          pW@P                  ,
          pV;P                  V P                  ,          un        V;P                  V P                  ,          un        VP                  VP                  ,          Vn        V;P                  V P                  ,          un	        V;P                  V,          un
        VP                  VP                  ,          Vn        \        VP                  \        P                  ! 4       R,          4      Vn        R#   \        P
                  ! 4       pY2,
          pY@P                  ,
          pT;P                  T P                  ,          un        T;P                  T P                  ,          un        TP                  TP                  ,          Tn        T;P                  T P                  ,          un	        T;P                  T,          un
        TP                  TP                  ,          Tn        \        TP                  \        P                  ! 4       R,          4      Tn        i ; i5i)r   r  N)rH  r@  rA  rB  r  r  ro   r(  rp   rq   rJ  rr   ró   rs   rL   r#  )r�  Ústatsr'  ÚtocÚ
total_timeÚgen_times   &&    r&   rO  ÚBatchGenerator.stats  sä  é € à×%Ð%œ›ˆØ&'ˆÔ#Ø$%ˆÔ!Ø#$ˆÔ Ü×ÒÓ!ˆð	SØŠKä×#Ò#Ó%ˆCØ�ˆJØ!×$=Ñ$=Õ=ˆHØ×Ò 4×#>Ñ#>Õ>ÕØ×Ò ×!:Ñ!:Õ:ÕØ$×2Ñ2°U×5FÑ5FÕFˆEÔØ×#Ò# t×'?Ñ'?Õ?Õ#Ø×!Ò! XÕ-Õ!Ø#(×#:Ñ#:¸U×=RÑ=RÕ#RˆEÔ Ü # E×$5Ñ$5´r×7IÒ7IÓ7KÈcÕ7QÓ RˆEÖøô ×#Ò#Ó%ˆCØ�ˆJØ!×$=Ñ$=Õ=ˆHØ×Ò 4×#>Ñ#>Õ>ÕØ×Ò ×!:Ñ!:Õ:ÕØ$×2Ñ2°U×5FÑ5FÕFˆEÔØ×#Ò# t×'?Ñ'?Õ?Õ#Ø×!Ò! XÕ-Õ!Ø#(×#:Ñ#:¸U×=RÑ=RÕ#RˆEÔ Ü # E×$5Ñ$5´r×7IÒ7IÓ7KÈcÕ7QÓ RˆEÕüs   ‚?J!ÁE2 ÁD,J!Å2D,JÊJ!c                óì  <€ V ^8„  d   QhRS[ S[ S[,          ,          RS[S[ S[,          ,          RS[S[ S[ S[,          ,          ,          RS[S[ S[ S[,          ,          ,          RS[S[ S[S[P                  .S[P                  3,          ,          ,          RS[S[ S[ S[S[P                  S[P                  .S[P                  3,          ,          ,          ,          RS[S[ S[,          ,          /# )rG   r@  r�   rU  r  rª  r’   r¬  ©r	   r9   r
   r   r   rL   rT   r|  )rN   rv   s   "€r&   rO   r4  1  sÒ   ø€ ÷ 
ñ 
á‘d™3•i•ð
ñ ™T¡#�YÕ'ð
ñ ™™d¡3�i�Õ)ð	
ñ
 ™T¡$¡s¥)�_Õ-ð
ñ ™4¡©"¯(©(¨±R·X±XÐ)=Õ >Õ?Õ@ð
ñ $Ù‘‘h¡§¡©"¯(©(Ð3±R·X±XÐ=Õ>Õ?Õ@õ
ð
ñ !¡Ñ&:Õ!;Õ<ñ
r(   c           	     óZ   € T P                  V Uu. uF  qˆ.NK  	  upVVVVVV4      # u upi rR   )Úinsert_segments)	r�  r@  r�   rU  r  rª  r’   r¬  r:  s	   &&&&&&&& r&   ÚinsertÚBatchGenerator.insert1  s>   € ð ×#Ñ#Ù!Ó"™'�Q‹S™'Ñ"ØØØØØØó
ð 	
ùÚ"s   �(c                óü  <€ V ^8„  d   QhRS[ S[ S[ S[,          ,          ,          RS[S[ S[,          ,          RS[S[ S[ S[,          ,          ,          RS[S[ S[ S[,          ,          ,          RS[S[ S[S[P                  .S[P                  3,          ,          ,          RS[S[ S[ S[S[P                  S[P                  .S[P                  3,          ,          ,          ,          RS[S[ S[,          ,          /# )rG   Úsegmentsr�   rU  r  rª  r’   r¬  rU  )rN   rv   s   "€r&   rO   r4  G  s×   ø€ ÷ 0ñ 0á‘t™D¡�I•Õ'ð0ñ ™T¡#�YÕ'ð0ñ ™™d¡3�i�Õ)ð	0ñ
 ™T¡$¡s¥)�_Õ-ð0ñ ™4¡©"¯(©(¨±R·X±XÐ)=Õ >Õ?Õ@ð0ñ $Ù‘‘h¡§¡©"¯(©(Ð3±R·X±XÐ=Õ>Õ?Õ@õ
ð0ñ !¡Ñ&:Õ!;Õ<ñ0r(   c                ó®  € . pT;'       g    V P                   .\        V4      ,          pT;'       g    V U	u. uF  p	. NK  	  up	pT;'       g    R .\        V4      ,          pT;'       g    V P                  .\        V4      ,          pT;'       g    V P                  .\        V4      ,          pT;'       g    R .\        V4      ,          p\	        \        V4      4       F!  p
W:,          e   K  V P                  4       W:&   K#  	  \        VVVVVVV4       F´  w  r¼rÞppp\        V4      p\        VR,          4      ^8w  d+   VP                  VR,          RR  4       VR,          R R VR&   V P                  P                  V P                  W¼WÞVVV34       VP                  V P                  4       V ;P                  ^,          un
        K¶  	  V# u up	i )Nr¥   éþÿÿÿ)r�   rº   r’   r:  rã   Ú_make_new_cacher^  rÉ  rå   r>  r;  )r�  r[  r�   rU  r  rª  r’   r¬  r©  r�   ré   rp  r  r�   Úatrf   ÚlpÚsms   &&&&&&&&          r&   rW  ÚBatchGenerator.insert_segmentsG  s¹  € ð ˆà×DÐD D§O¡OÐ#4´s¸8³}Õ#Dˆ
Ø×9Ð9±Ó#9±¨1£B±Ñ#9ˆ
Ø×5Ð5 ˜v¬¨H«Õ5ˆØ-÷ 
ð 
Ø×#Ñ#Ð$¤s¨8£}Õ4ð 	ð (÷ 
ð 
Ø×(Ñ(Ð)¬C°«MÕ9ð 	ð ×1Ð1˜D˜6¤C¨£MÕ1ˆÜ”s˜8“}Ö%ˆAØ�yÔ Ø ×0Ñ0Ó2�“	ñ &ô ),ØØØØØØØö)
Ñ$ˆC�A˜1˜b "ô �s“)ˆCÜ�3�r•7‹|˜qÔ Ø—
‘
˜3˜r�7 2 3˜<Ô(Ø˜b�' # 2˜,��B‘Ø×'Ñ'×.Ñ.Ø—‘ #¨!°°B¸Ð;ôð �K‰K˜Ÿ™Ô(Ø�OŠO˜qÕ �Oñ#)
ð& ˆùòC $:s   ¸Gc                ó   € V P                   f!   \        P                  ! V P                  4      # \        P                  ! V P                  4       Uu. uF1  p\	        V\
        4      '       d   \        V P                   R7      MTNK3  	  up# u upi )N)rT  )r“   r   rÆ   rH   rS   r   r   )r�  Úcis   & r&   r^  ÚBatchGenerator._make_new_cachey  s   € Ø×ÑÒ#Ü×*Ò*¨4¯:©:Ó6Ð6ô ×-Ò-¨d¯j©jÔ9ó
ñ :�ô ˜b¤'×*Ò*ô  ¨×)9Ñ)9Õ:àòñ :ñ
ð 	
ùò 
s   Á7Bc                ón  € \        V4      p/ p\        V P                  P                  4       F  w  r4WA9   g   K  ^V3W$&   K  	  \        V P                  P                  4       F  w  r4WA9   g   K  ^V3W$&   K  	  \        V P
                  4       F"  w  r5V^ ,          V9   g   K  ^ V3W%^ ,          &   K$  	  V# )rG   )rÔ  r…   r=  r©  r<  r>  )r�  r©  Úresultsré   Úuid_irp  s   &&    r&   Ú
_find_uidsÚBatchGenerator._find_uids†  s¦   € Ü�4‹yˆØˆÜ! $×"8Ñ"8×"=Ñ"=Ö>‰HˆAØŽ}Ø"# Q �“ñ ?ô " $×"4Ñ"4×"9Ñ"9Ö:‰HˆAØŽ}Ø"# Q �“ñ ;ô   × ;Ñ ;Ö<‰FˆAØ�1�v˜Ž~Ø#$ a &�˜A�“ñ =ð ˆr(   c                ó¤  € / pV P                  V4      P                  4        Fª  w  pw  rEV^ 8X  d   V P                  V,          R,          W#&   K-  V^8X  d=   V P                  P	                  V4      V P                  P
                  V,          3W#&   Kp  V P                  P	                  V4      V P                  P
                  V,          3W#&   K¬  	  V# )r   :é   é   N)ri  rk  r>  r<  r¿  rÂ   r=  )r�  r©  rg  r   Ústagero  s   &&    r&   r¿  ÚBatchGenerator.extract_cache”  sº   € ØˆØ!%§¡°Ó!6×!<Ñ!<Ö!>ÑˆC‘�%Ø˜ŒzØ#×:Ñ:¸3Õ?ÀÕD�“Ø˜!”à×&Ñ&×4Ñ4°SÓ9Ø×&Ñ&×-Ñ-¨cÕ2ð �“ð ×*Ñ*×8Ñ8¸Ó=Ø×*Ñ*×1Ñ1°#Õ6ð �“ñ "?ð ˆr(   c           	     ó0  a€ / pV'       d   V P                  V4      p\        \        \        V P                  4      4      4      \        \        \        V P
                  4      4      4      \        \        \        V P                  4      4      4      3oV P                  V4      P                  4        F  w  rESV,          P                  V4       K  	  \        S^ ,          4      \        V P                  4      8  d.   \        V3R l\        V P                  4       4       4      V n        \        S^,          4      \        V P
                  4      8  dj   V P
                  P                  \        S^,          4      4       \        V P                  4       UUu. uF  w  rgVS^,          9   g   K  VNK  	  uppV n        \        S^,          4      \        V P                  4      8  d,   V P                  P                  \        S^,          4      4       V# u uppi )r   c              3   óL   <"  € T F  w  rVS^ ,          9   g   K  Vx € K  	  R# 5i)r   Nr2   )r  ré   rW   rS  s   &  €r&   r  Ú(BatchGenerator.remove.<locals>.<genexpr>³  s$   øé € ð 0ÙD‘d�aÈÈTÐRSÍWÉ—’ÓDùs   ƒ$š
$)r¿  rÔ  rã   rº   r>  r<  r=  ri  ÚvaluesÚremover   r…   rÕ  rÓ  r?  )	r�  r©  Úreturn_prompt_cachesrU  rn  ro  ré   rW   rS  s	   &&&     @r&   rt  ÚBatchGenerator.remove¥  s“  ø€ ØˆßØ×'Ñ'¨Ó-ˆFô ””c˜$×5Ñ5Ó6Ó7Ó8Ü””c˜$×,Ñ,Ó-Ó.Ó/Ü””c˜$×0Ñ0Ó1Ó2Ó3ð
ˆð
 Ÿ/™/¨$Ó/×6Ñ6Ö8‰JˆEØ��K×Ñ˜sÖ#ñ 9ô ˆt�A�w‹<œ#˜d×9Ñ9Ó:Ô:Ü*/ô 0Ü'¨×(CÑ(CÔDó0ó +ˆDÔ'ô ˆt�A�w‹<œ#˜d×0Ñ0Ó1Ô1Ø×Ñ×%Ñ%¤f¨T°!­W£oÔ6ä'¨×(BÑ(BÔCô*ÙC‘d�aÀqÈDÐQRÍGÁ|—�ÑCò*ˆDÔ&ô ˆt�A�w‹<œ#˜d×4Ñ4Ó5Ô5Ø×"Ñ"×)Ñ)¬&°°aµ«/Ô:àˆùó*s   ÆHÆ,Hc                óò   € \        R  V P                   4       4      pV\        R V P                  P                   4       4      ,          pV\        R V P                  P                   4       4      ,          pV# )c              3   óV   "  € T F  q^,           F  q"P                   x € K  	  K!  	  R# 5i)rl  N©rU   )r  r:  r�   s   &  r&   r  Ú5BatchGenerator.prompt_cache_nbytes.<locals>.<genexpr>Â  s    é € ÐPÑ&A ÈQÏ4È4Àa—H–HÉ4‘HÓ&Aùs   ‚')c              3   ó8   "  € T F  qP                   x € K  	  R # 5irR   ry  ©r  r�   s   & r&   r  rz  Ã  s   é € ÐGÑ'F !—X–XÓ'Fùr=  c              3   ó8   "  € T F  qP                   x € K  	  R # 5irR   ry  r|  s   & r&   r  rz  Ä  s   é € ÐKÑ'J !—X–XÓ'Jùr=  )Úsumr>  r<  rˆ   r=  )r�  Útotals   & r&   Úprompt_cache_nbytesÚ"BatchGenerator.prompt_cache_nbytesÀ  s_   € äÑP d×&AÒ&AÓPÓPˆØ”ÑG t×'9Ñ'9×'FÒ'FÓGÓGÕGˆØ”ÑK t×'=Ñ'=×'JÒ'JÓKÓKÕKˆØˆr(   c                ó    <€ V ^8„  d   QhRS[ /# )rG   rÓ   )r9   )rN   rv   s   "€r&   rO   r4  Ç  s   ø€ ÷  
ñ  
™Sñ  
r(   c                ó¨  € . p. p. p. p. p. p. p\        V4       EF  p	V P                  P                  4       p
VP                  V
^ ,          4       VP                  V
^,          4       VP                  V
^,          4       VP                  V
^,          4       VP                  V
^,          4       VP                  V
^,          4       VP                  V
^,          4       V P                  P                  V
^,          ^ \        R V
^,           4       4      .4       EK  	  \        V P                  VVVV P                  VV P                  VVVR7
      # )r   c              3   ó8   "  € T F  p\        V4      x € K  	  R # 5irR   r8  )r  rf   s   & r&   r  Ú-BatchGenerator._make_batch.<locals>.<genexpr>Ù  s   é € Ð$A±[°¤S¨§V V³[ùr=  )
rH   r©  rU  rÂ   r”   rª  r«  r’   r¬  r�   )
rã   r>  rl  rå   r?  r~  rœ  rH   r”   r‘   )r�  rÓ   r©  rU  rÂ   rª  r’   r�   r¬  r�   Úsequences   &&         r&   Ú_make_batchÚBatchGenerator._make_batchÇ  s*  € ØˆØˆØˆØˆØÐØˆ
ØˆÜ�q—ˆAØ×2Ñ2×:Ñ:Ó<ˆHØ�K‰K˜ �Ô$Ø�M‰M˜( 1�+Ô&Ø�M‰M˜( 1�+Ô&Ø�O‰O˜H Q�KÔ(Ø×$Ñ$ X¨a¥[Ô1Ø×Ñ˜h q�kÔ*Ø×!Ñ! (¨1¥+Ô.Ø×&Ñ&×-Ñ-Ø˜!•˜a¤Ñ$A°X¸a¶[Ó$AÓ!AÐB÷ñ ô %Ø—*‘*ØØØØ"×4Ñ4ØØ!Ÿ\™\Ø/Ø)Ø!ô
ð 	
r(   c           	     ó‚	  € . p. p\        V P                  4      ^ 8”  dƒ   V P                  P                  4       pV ;P                  \        V4      ,          un        V ;P                  ^,          un        V P                  R,          ^ 8X  d   \
        P                  ! 4        \        V P                  4      V P                  8¼  d   W!3# \        V P                  \        V P                  4      ,
          V P                  \        V P                  4      ,
          \        V P                  4      4      pV^ 8”  d+   V P                  P                  V P                  V4      4       . p. p\        V P                  4       FY  w  rgV^ ,          p\        V4      ^8X  d+   \        V^ ,          4      ^8X  d   VP!                  V4       KH  VP!                  V4       K[  	  V'       Ed   V Uu. uF#  q`P                  V,          ^ ,          ^ ,          NK%  	  p	pV Uu. uF$  q`P                  V,          ^,          3^,          NK&  	  p
pV Uu. uF  q`P                  V,          NK  	  upV n        V P                  P#                  V4      P%                  V	4      p\        V
4       F=  w  rlVP!                  \&        P)                  VP*                  V,          VRR4      4       K?  	  V P                  P                  V4       . p\        V P                  4       EF  w  rg\&        P)                  V P                  P*                  V,          ^ RR4      pV^ ,          p\        \        V^ ,          4      V P,                  4      pVP!                  V^ ,          RV 4       V^ ,          VR V^ &   \        V^ ,          4      ^ 8X  d   VP/                  ^ 4       RVn        V^;;,          \        VR,          4      ,          uu&   V^,          V^,          3Vn        VP!                  V4       EK  	  V ;P4                  \7        R V 4       4      ,          un        \8        P:                  ! 4       pV P                  P=                  V4       \8        P:                  ! 4       pV ;P>                  VV,
          ,          un        W!3# u upi u upi u upi )r   rØ   TFNc              3   ó8   "  € T F  p\        V4      x € K  	  R # 5irR   r8  r9  s   & r&   r  Ú'BatchGenerator._next.<locals>.<genexpr>/  s   é € Ð*C¹7°a¬3¨q¯6¨6»7ùr=  r¥   ) rº   r=  r(  rB  rC  rL   rË   r1  rÈ   r2  r<  r>  r_  r‡  r…   r?  rå   r×  r5  rœ  r¦  r©  r”   r  r¢  r¡  r@  r~  r  r  r™   rA  )r�  Úgeneration_responsesÚprompt_responsesrÓ   rS  r×  ré   rp  r[  Úlast_inputsr¡  Ú	gen_batchr:  r@  r4  r'  rP  s   &                r&   Ú_nextÚBatchGenerator._nexté  sÝ  € Ø!ÐØÐô ˆt×%Ñ%Ó&¨Ô*Ø#'×#9Ñ#9×#>Ñ#>Ó#@Ð Ø×$Ò$¬Ð,@Ó(AÕAÕ$Ø×Ò 1Õ$ÕØ×"Ñ" SÕ(¨AÔ-Ü—’Ô ô ˆt×%Ñ%Ó&¨$×*DÑ*DÔDØ#Ð9Ð9ô Ø×#Ñ#¤c¨$×*<Ñ*<Ó&=Õ=Ø×&Ñ&¬¨T×-CÑ-CÓ)DÕDÜ�×+Ñ+Ó,ó
ˆð
 ˆqŒ5Ø×Ñ×%Ñ% d×&6Ñ&6°qÓ&9Ô:ð ˆØˆÜ × :Ñ :Ö;‰FˆAØ˜1•vˆHÜ�8‹} Ô!¤c¨(°1­+Ó&6¸!Ô&;Ø—‘˜Q–à—‘˜A–ñ <÷ ˆ5ÙHMÓNÉÀ1×5Ñ5°aÕ8¸Õ;¸A×>Ð>ÉˆKÐNÙINÓOÉÀA×3Ñ3°AÕ6°qÕ9Ð;¸a×?Ð?ÉˆHÐOÙQUÓ)VÑQUÈA×*DÑ*DÀQ×*GÐ*GÑQUÑ)VˆDÔ&Ø×*Ñ*×0Ñ0°Ó7×@Ñ@ÀÓMˆIÜ! (Ö+‘�Ø ×'Ñ'Ü)×2Ñ2Ø!Ÿ™ qÕ)ØØØó	öñ ,ð ×"Ñ"×)Ñ)¨)Ô4ð ˆÜ × :Ñ :×;‰FˆAÜ,×5Ñ5Ø×"Ñ"×'Ñ'¨Õ*¨A¨u°eóˆHð ˜1•vˆHÜ”C˜ �Ó$ d×&<Ñ&<Ó=ˆAØ�N‰N˜8 A�; r¨˜?Ô+Ø" 1�+ a b˜/ˆH�Q‰KÜ�8˜A•;Ó 1Ô$Ø—‘˜Q”Ø*.�Ô'Ø��F”c˜' "�+Ó&Õ&‹FØ!$ Q¥¨¨Q­Ð 0ˆHÔØ×#Ñ# H×-ñ <ð  	×#Ò#¤sÑ*C¹7Ó*CÓ'CÕCÕ#Ü×ÒÓ!ˆØ×Ñ×!Ñ! 'Ô*Ü×ÒÓ!ˆØ×!Ò! S¨3¥YÕ.Õ!àÐ5Ð5ùòO OùÚOùÚ)Vs   Ç$)R2È*R7ÉR<c                ó°   € \         P                  ! V P                  4      ;_uu_ 4        V P                  4       uuRRR4       #   + '       g   i     R# ; i)zo
Get the next batch of responses.

Returns:
    Tuple of prompt processing responses and generation responses.
N©rL   r¸   r9  r�  r’  s   &r&   r(  ÚBatchGenerator.next7  s/   € ô �YŠY�t—|‘|×$Õ$Ø—:‘:“<÷ %×$×$Ó$ús   ©AÁA	c                óÞ   € \         P                  ! V P                  4      ;_uu_ 4         V P                  4       w  rV'       g   V'       d   K%  VuuRRR4       #   + '       g   i     R# ; i)zz
Return only generated tokens ignoring batch generation responses.

Returns:
    List of GenerationBatch.Response objects
Nr“  )r�  r�  rŒ  s   &  r&   Únext_generatedÚBatchGenerator.next_generatedA  sD   € ô �YŠY�t—|‘|×$Õ$ØØ9=¿¹»Ñ6Ð ß+×0@ÙØ+÷ %×$×$Ó$ús   ©#AÁAÁA,	)r?  r:  rB  r=  rD  r<  rA  r@  rC  r9  r;  r>  r1  r’   r“   r�   rH   r2  r”   r‘   r8  rR   )NNNNNN©F)rx   ry   rz   r{   r|   r‡  Úpropertyr¸   rI  rL  Ú
contextlibÚcontextmanagerrO  rX  rW  r^  ri  r¿  rt  r€  r‡  r�  r(  r–  r~   r   r€   s   @r&   r/  r/  Î  s  ø‡ € ñð4)ð ð	4)ð
 :>ð4)ð =Að4)ð ð4)ð &(ð4)ð #$ð4)ð "&ð4)ð &*ð4)ð ÷4)ò 4)ðl ñó ðò)òð ×ÑóSó ðS÷(
ò 
÷,0ò 0òd
òòô"ð6 ñó ð÷ 
ð  
òDL6ò\ ÷,ð ,r(   r/  c                   ó0   a € ] tR tRt o RtV 3R ltRtV tR# )ÚBatchResponseiP  z°
A data object to hold a batch generation response.

Args:
    texts: (List[str]): The generated text for each prompt.
    stats (BatchStats): Statistics about the generation.
c                ó~   <€ V ^8„  d   Qh/ S[ S[,          ;R&   S[;R&   S[S[ S[ S[,          ,          ,          ;R&   # )rG   ÚtextsrO  rU  )r	   r6   rH  r
   r   )rN   rv   s   "€r&   rO   ÚBatchResponse.__annotate__P  s=   ø‡ ‚ ñ ‘�9Ññ ñ Ññ ñ ‘T™$™s�)•_Õ%Ñ%ò r(   r2   N)rx   ry   rz   r{   r|   r}   r~   r   r€   s   @r&   r�  r�  P  s   ø‡ € ñ÷ ƒ r(   r�  c                ó  € V ^8„  d   QhR\         \         \        ,          ,          R\        \         \         \        ,          ,          ,          R\        \        \         \        ,          3,          R\
        R\
        R\        /# )rG   r@  Úprompt_cachesr�   r+  ru  rš   )r	   r9   r
   r   r   ru   r�  )rN   s   "r&   rO   rO   _  sr   € ÷ L/ñ L/ô ”$”s•)�_ðL/ô œD¤¤c¥�OÕ,ð	L/ô
 ”cœ4¤�9�nÕ%ðL/ô ðL/ô ðL/ô ñL/r(   c           
     ó  € \        V 3RVP                   Uu. uF  qˆ.NK  	  up/VB p	\        V4      p
^ pV'       d   \        RV
 R2RR7       \	        V\
        4      '       d   V.\        V4      ,          pV	P                  W$VR7      pV Uu/ uF  qÝ. bK  	  pp/ pV	P                  4       ;_uu_ 4       pV	P                  4       ;p'       dŸ   V F–  pVP                  eG   V'       d   VP                  VVP                  &   V'       d   V^,          p\        RV R	V
 R2RR7       VP                  R
8w  g   Kj  VVP                  ,          P                  VP                  4       K˜  	  K¶   RRR4       V	P                  4        V'       d   \        RV R	V
 24       V Uu. uF  qÑP                  Wí,          4      NK  	  ppV'       d   V Uu. uF  qÓV,          NK  	  upMRpV'       di   \        RXP                    RVP"                  R R24       \        RVP$                   RVP&                  R R24       \        RVP(                  R R24       \+        VXV4      # u upi u upi   + '       g   i     EL; iu upi u upi )a8  
Generate responses for the given batch of prompts.

Args:
   model (nn.Module): The language model.
   tokenizer (PreTrainedTokenizer): The tokenizer.
   prompts (List[List[int]]): The input prompts.
   prompt_caches (List[List[Any]], optional): Pre-computed prompt-caches
      for each input prompt. Note, unlike ``generate_step``, the caches
      won't be updated in-place.
   verbose (bool): If ``True``, print tokens and timing information.
      Default: ``False``.
   max_tokens (Union[int, List[int]): Maximum number of output tokens. This
      can be per prompt if a list is provided.
   return_prompt_caches (bool): Return the prompt caches in the batch
      responses. Default: ``False``.
   kwargs: The remaining options get passed to :obj:`BatchGenerator`.
      See :obj:`BatchGenerator` for more details.
r0  z'[batch_generate] Finished processing 0/z ...Ú)r.  )rU  Nz%[batch_generate] Finished processing Ú/r  z[batch_generate] Prompt: r0  r1  r2  z[batch_generate] Generation: z[batch_generate] Peak memory: r3  )r/  r   rº   r^   rS   r9   rX  rO  r–  rt   rˆ   r   rå   rl   rI  Údecodero   rp   rq   rr   rs   r�  )rH   r  r@  r¢  r�   r+  ru  r%  rï  ÚgenÚnum_samplesÚfinr©  r   rg  rO  r'  ÚrrŸ  rU  s   &&&&&&&,            r&   Úbatch_generater«  _  sX  € ô< Øñà"+×"9Ò"9Ó:Ñ"9˜Q“SÑ"9Ñ:ðð ñ€Cô
 �g“,€KØ
€CßÜÐ7¸°}ÀDÐIÈtÕTä�*œc×"Ò"Ø �\¤C¨£LÕ0ˆ
à�:‰:�g°-ˆ:Ó@€DÙ"&Ó'¡$˜3�BŠw¡$€GÐ'Ø€MØ	�‰�Œ˜Ø×-Ñ-Ó/Ð/ˆiÖ/Û�Ø—?‘?Ò.ß+Ø/0¯~©~˜ a§e¡eÑ,ßØ˜q�˜ÜØCÀCÀ5ÈÈ+ÈÐVZÐ[Ø $õð —?‘? fÖ,Ø˜AŸE™E•N×)Ñ)¨!¯'©'Ö2ó ð 0÷ 
ð ‡I�I„KßÜÐ5°c°U¸!¸K¸=ÐIÔJñ 8<Ó<±t°×Ñ˜g�lÖ+±t€EÐ<ß5I©DÓ1©D S˜C× Ð ©DÒ1Èt€FßÜØ'¨×(;Ñ(;Ð'<¸IÀe×FVÑFVÐWZÐE[Ð[jÐkô	
ô 	Ø+¨E×,CÑ,CÐ+DÀIØ×#Ñ# CÐ(¨ð9ô	
ô 	Ð.¨u×/@Ñ/@ÀÐ.EÀSÐIÔJÜ˜  vÓ.Ð.ùòY ;ùò (ç	�ˆüò& =ùÚ1s;   —I#ÂI(Â6I-ÃI-Ã* I-Ä+I-Ä;2I-Æ#JÇJÉ-I>	c                  ó¶	  € \        4       p V P                  4       pVP                  e*   \        P                  P                  VP                  4       VP
                  R JpV'       dŸ   \        VP
                  RR7      w  r4\        V^ ,          \        4      '       di   VP                  e.   VP                  V^ ,          P                  8w  d   \        R4      hVP                  V^ ,          P                  8w  d   \        R4      hV'       g   / M\        P                  ! XR,          4      pVP                   '       d   RMR VR&   VP"                  pV'       d6   Vf   XR,          pM'VXR,          8w  d   \        RV R	VR,           R
24      hT;'       g    \$        p\'        VVP(                  VRVP*                  /R7      w  rxVP,                   F  p	VP/                  V	4       K  	  / p
VP0                  e!   \        P                  ! VP0                  4      p
VP2                  P5                  RR4      P5                  RR4      pVR8X  d   \6        P8                  P;                  4       MTpVP<                  '       gò   VP>                  '       dà   VP@                  e   RRRVP@                  /.pM. pVPC                  RRRV/4       VPD                  R JpV'       d    VPC                  RRRVPD                  /4       VPF                  ! V3RRRVRV'       * /V
B pV'       d:   RVR!,          R&   VPG                  VRVV'       * R7      pW¾PI                  R4      R  pVPK                  VRR7      pMVPK                  V4      pVPL                  e@   \'        VPL                  4      w  ppVPN                  VPN                  8w  d   \        R4      hMR p\Q        VPR                  VPT                  VPV                  VPX                  VPZ                  VP\                  VP^                  VPK                  R4      \a        VPb                  4      ,           R7      p\e        TTTVPf                  VPh                  TVPj                  V'       d   XMR VP                  VP                  VPl                  VVPn                  R 7      pVPh                  '       g   \q        V4       R # R # )"NT)Úreturn_metadatazF--kv-bits does not match the kv cache loaded from --prompt-cache-file.zL--kv-group-size does not match the kv cache loaded from --prompt-cache-file.Útokenizer_configÚtrust_remote_coderH   zProviding a different model (z-) than that used to create the prompt cache (z) is an error.Úquantize_activations)Úadapter_pathr®  Úmodel_configz\nÚ
z\tÚ	Ú-ÚroleÚsystemÚcontentÚuserÚ	assistantÚtokenizeFÚcontinue_final_messageÚadd_generation_promptz<query>)r»  r¼  r½  r  z5Draft model tokenizer does not match model tokenizer.)Útop_kÚxtc_probabilityÚxtc_thresholdÚxtc_special_tokens)
r�   r+  r‘   r“   rˆ   r‹   rŠ   r‰   rÚ   r×   r¥   )9rE   Ú
parse_argsÚseedrL   ÚrandomÚprompt_cache_filer   rS   r   r‹   r„   rÅ   rŠ   rƒ   ÚjsonÚloadsr¯  rH   r7   r   r±  r°  Úextra_eos_tokenÚadd_eos_tokenÚchat_template_configr™   ÚreplaceÚsysÚstdinÚreadÚignore_chat_templateÚhas_chat_templateÚsystem_promptrå   Úprefill_responseÚapply_chat_templateÚindexr  rÚ   Ú
vocab_sizer   ÚtempÚtop_pÚmin_pÚmin_tokens_to_keepr¾  r¿  rÀ  rÉ  r   r5  r�   r+  r“   r‰   r×   r^   )rD   ÚargsÚusing_cacherˆ   Úmetadatar®  Ú
model_pathrH   r  Ú	eos_tokenÚtemplate_kwargsr™   ÚmessagesÚhas_prefillÚtest_promptrÚ   Údraft_tokenizerr‘   r4  s                      r&   Úmainrä  ®  sk  € ÜÓ€FØ×ÑÓ€Dà‡y�yÒÜ
�	‰	�‰�t—y‘yÔ!ð ×(Ñ(°Ð4€KßÜ!2Ø×"Ñ"Ø ô"
Ñˆô �l 1•oÔ'7×8Ò8Ø�|‰|Ò'¨D¯L©L¸LÈ½O×<PÑ<PÔ,PÜ Ø\óð ð ×!Ñ! \°!¥_×%?Ñ%?Ô?Ü Øbóð ÷ ‰¤4§:¢:¨hÐ7IÕ.JÓ#Kð ð 59×4J×4JÐ4J©DÐPTÐÐ(Ñ)à—‘€JßØÒØ! 'Õ*‰JØ˜8 GÕ,Ô,ÜØ/°
¨|ð <4Ø4<¸WÕ4EÐ3Fð Gðóð ð
 ×,Ð,œ}€JäØØ×&Ñ&Ø)Ø,¨d×.GÑ.GÐHô	Ñ€Eð ×)Ô)ˆ	Ø×Ñ 	Ö*ñ *ð €OØ× Ñ Ò,ÜŸ*š* T×%>Ñ%>Ó?ˆà�[‰[× Ñ  ¨Ó-×5Ñ5°e¸TÓB€FØ!'¨3¤ŒS�Y‰Y�^‰^Ô°F€FØ×$×$Ð$¨×)D×)DÐ)DØ×ÑÒ)Ø ¨9°d×6HÑ6HÐIÐJ‰HàˆHØ�‰˜ ¨°FÐ;Ô<à×+Ñ+°4Ð7ˆßØ�O‰O˜V [°)¸T×=RÑ=RÐSÔTØ×.Ò.Øñ
àð
ð $/ð
ð '2¤/ð	
ð
 ñ
ˆ÷ Ø&/ˆH�R�L˜Ñ#Ø#×7Ñ7ØØØ'2Ø*5¤oð	 8ó ˆKð ×-Ñ-¨iÓ8Ð:Ð;ˆFØ×!Ñ! &¸UÐ!ÓC‰à×!Ñ! &Ó)ˆà×ÑÒ#Ü'+¨D×,<Ñ,<Ó'=Ñ$ˆ�_Ø×%Ñ%¨×)=Ñ)=Ô=ÜÐTÓUÐUð >ð ˆÜØ�	‰	Ø�
‰
Ø�
‰
Ø×ÑØ�j‰jØ×,Ñ,Ø×(Ñ(Ø$×+Ñ+¨DÓ1´D¸×9PÑ9PÓ4QÕQô	€Gô ØØØØ—?‘?Ø—‘ØØ×$Ñ$ß%0‘\°dØ—‘Ø×(Ñ(Ø×2Ñ2ØØ×.Ñ.ô€Hð �<�<ˆ<ÜˆhŽñ r(   Ú__main__zƒCalling `python -m mlx_lm.generate...` directly is deprecated. Use `mlx_lm.generate...` or `python -m mlx_lm generate ...` instead.rR   )r�   Nr˜  )Né€   FF)[r3   rš  rÊ  rÇ   rÆ  rÌ  r  Úcollectionsr   Údataclassesr   r   Útypingr   r   r   r	   r
   r   r   r   Úmlx.coreÚcorerL   Úmlx.nnrJ   Ú	mlx.utilsr   Útransformersr   Úmodelsr   Úmodels.cacher   r   r   r   r   r   r   r   r   Úsample_utilsr   Útokenizer_utilsr   Úutilsr   r   r8   r:   r<   r=   r>   r?   r@   ÚDEFAULT_XTC_THRESHOLDrA   rB   r7   rC   r'   rE   Únew_thread_local_streamÚdefault_devicer¹   r›  rg   ri   rŽ   rÖ   r  r)  r5  rB  rF  rH  rX  r\  rd  rx  rz  r|  rœ  rñ  r/  r�  r«  rä  rx   r^   r2   r(   r&   Ú<module>r÷     s½  ðó Û Û Û Û Û 
Û Ý Ý !Ý ÷	÷ 	ó 	õ Ý Ý !Ý ,å ÷
÷ 
õ 
õ 'Ý -ß <à€ØÐ Ø€Ø€Ø€Ø€ØÐ ØÐ ØÐ Ø€Ø:€Ø!Ð ò0ò_ðF ×.Ò.¨r×/@Ò/@Ó/BÓCÐ ð ×Ñö$*ó ð$*ðN ÷(ð (ó ð(ò:Uðcð ð	cð
 9=ðcð SWðcð "&ðcð #'ðcð "ðcð "ðcð ðcð  ðcð FJðcð ,0÷cðLu$ð
 ðu$ð ðu$ð 9=ðu$ð SWðu$ð #'ðu$ð !ðu$ð "ðu$ð ðu$ð  ÷u$÷p`
÷F+ô\HôHð ÷ð ó ðò.Bò@ò ò#òL÷:&ñ :&÷z^
ñ ^
÷B~
ñ ~
÷B,ñ ,ðD ÷&ð &ó ð&÷L/ò^zðz ˆzÔÙ	ð	Pôñ 	†Fñ r(   