+
    UV-j&à ã            8      ó  € ^ RI Ht ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RI	H
t
 ^ RIHt ^ RIHtHtHtHtHtHtHt ^ RIHt ^ RIHt ^ RIHtHt ^ RIHt ^RIHt  ^R	I!H"t" ^R
I#H$t$ ^RI%H&t& ^RI'H(t(H)t)H*t*H+t+H,t, ^RI-H.t.H/t/ ^RI0H1t1H2t2 ^RI3H4t4H5t5H6t6H7t7H8t8H9t9 ]Pt                  ! R4      t;Rt<Rt=Rt>^ t?Rt@^tARtB^ tC^tD^2tER R ltFRRRRRRRRRRRR/R R lltGR R ltHR  R! ltIR" R# ltJ ! R$ R%4      tKR& R' ltLR( tMR)]<R*]=R+RR,]AR-RR.]AR/RR0]AR1]>R2]@R3]?R4RRRR5RR6RR7]4R8]5R9]6R:RR;RR<]BRRRR=R>RR?RR@RRARRBRC/RD RE lltN] ! RF RG4      4       tORmRH ltPRmRI ltQ0 RnmtRRotSRJ RK ltTRL RM ltURN RO ltVRP RQ ltWRR RS ltXRT tYR^@]53RU ltZ] ! RV RW4      4       t[] ! RX RY4      4       t\] ! RZ R[4      4       t]R\RR]R/R^ R_ llt^ ! R` Ra4      t_ ! Rb Rc4      t` ! Rd Re4      ta ! Rf Rg4      tbRpRh Ri lltcRj tdRqRk Rl llteR# )ré    )ÚannotationsN)Ú	Generator)Ú	dataclass)ÚAnyÚCallableÚDictÚListÚOptionalÚTupleÚUnion)Úmake_logits_processorsÚmake_sampler)Útqdm)Úapc)Úcache)Úapply_chat_template)Útop_p_sampling)Úmake_speculative_prompt_cacheÚrun_speculative_roundsÚrun_speculative_server_roundsÚspeculative_hidden_stateÚspeculative_prefill_kwargs)ÚBatchTurboQuantKVCacheÚturboquant_enabled)Úgroup_images_by_shapeÚprepare_inputs)ÚDEFAULT_KV_GROUP_SIZEÚDEFAULT_KV_QUANT_SCHEMEÚDEFAULT_QUANTIZED_KV_STARTÚgeneration_streamÚmaybe_quantize_kv_cacheÚwired_limitúmlx_vlm.generatei   ç        ç      ð?c               ó   € V ^8„  d   QhRR/# ©é   ÚreturnÚbool© )Úformats   "Úd/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/generate/ar.pyÚ__annotate__r.   7   s   € ÷ 4ñ 4˜tñ 4ó    c                ó.   € \        \        V R V 4      4      # )Úenabled)r*   Úgetattr)Úpolicys   &r-   Ú_policy_enabledr4   7   s   € Ü”˜ 	¨6Ó2Ó3Ð3r/   Ú	input_idsÚinputs_embedsÚprompt_cacheÚdraft_modelÚ
draft_kindÚprefill_kwargsc               ó   € V ^8„  d   QhRR/# r'   r+   )r,   s   "r-   r.   r.   ;   s   € ÷ #ñ #ð 
ñ#r/   c               ón  € T;'       g    / pV .p\        V R R4      pVe   W€Jd   VP                  V4       V F:  p	\        V	RR4      p
\        V
4      '       g   K#  \        V
! VVVVVVR7      4      u # 	  \        ;QJ d    R V 4       F  '       g   K   RM	  RM! R V 4       4      '       d   R# VRJ # )Úlanguage_modelNÚchunked_prefill_policy©r5   r6   r7   r8   r9   r:   c              3  ó<   "  € T F  p\        VR R4      x € K  	  R# 5i)Úno_chunked_prefillFN)r2   )Ú.0Ú	candidates   & r-   Ú	<genexpr>Ú+_chunked_prefill_enabled.<locals>.<genexpr>Y   s   é € Ð
WÉJ¸yŒ7�9Ð2°E×:Ð:ËJùs   ‚TF)r2   ÚappendÚcallabler4   Úany)Úmodelr5   r6   r7   r8   r9   r:   Ú
candidatesr=   rC   r3   s   &$$$$$$    r-   Ú_chunked_prefill_enabledrK   ;   s¾   € ð $×)Ð) r€NØ�€JÜ˜UÐ$4°dÓ;€NØÒ! nÓ&AØ×Ñ˜.Ô)ãˆ	Ü˜Ð$<¸dÓCˆÜ�F×ÔÜ"ÙØ'Ø"/Ø!-Ø +Ø)Ø#1ôó	ò 	ñ  ÷ ƒsÑ
WÉJÓ
W‡s‡s‚sÑ
WÉJÓ
W×WÒWÙð ˜$ÐÐr/   c               ó   € V ^8„  d   QhRR/# ©r(   r)   Úintr+   )r,   s   "r-   r.   r.   a   s   € ÷ 1ñ 1¨ñ 1r/   c                 óÚ   € \         P                  P                  R 4      p V f   \        #  \	        ^ \        V 4      4      #   \         d     \        P                  RT 4       \        u # i ; i)Ú!MLX_VLM_BATCH_CACHE_EVAL_INTERVALz5Ignoring invalid MLX_VLM_BATCH_CACHE_EVAL_INTERVAL=%r)	ÚosÚenvironÚgetÚ!DEFAULT_BATCH_CACHE_EVAL_INTERVALÚmaxrN   Ú
ValueErrorÚloggerÚwarning)Úraws    r-   Ú_get_batch_cache_eval_intervalrZ   a   s[   € Ü
�*‰*�.‰.Ð<Ó
=€CØ
‚{Ü0Ð0ð1Ü�1”c˜#“hÓÐøÜô 1Ü�‰ÐNÐPSÔTÜ0Ò0ð1ús   «A  Á 'A*Á)A*c               ó(   € V ^8„  d   QhRRRRRRRR/# )r(   ÚseedrN   Úrow_idÚpositionr)   r+   )r,   s   "r-   r.   r.   l   s(   € ÷ ñ ˜ð  cð °Sð ¸Sñ r/   c                óP  € \        V 4      R ,          R,          pV\        V4      ^,           R,          ,           R,          pV\        V4      ^,           R,          ,          R,          pW3^,	          ,          pVR,          R,          pW3^,	          ,          p\        VR,          4      # )l   ¹yn< l   ÿÿ l   kJ× l   5.e i-5ë)rN   )r\   r]   r^   Úxs   &&& r-   Ú_position_seedra   l   s�   € Ü	ˆT‹�ZÕ	 :Õ-€AØ	
Œc�&‹k˜A�o Õ+Õ	+¨zÕ9€AØ	
Œs�8‹}˜qÕ  JÕ.Õ	/°:Õ=€AØˆb�…L€AØ	
ˆZ�˜:Õ%€AØˆb�…L€AÜˆq�:�~ÓÐr/   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r(   r\   rN   Úrow_idsú	List[int]Ú	positionsr)   úmx.arrayr+   )r,   s   "r-   r.   r.   v   s(   € ÷ ñ ˜ð  yð ¸Yð È8ñ r/   c                óÂ   € \         P                  ! \        W4       UUu. uF.  w  r4\         P                  P	                  \        WV4      4      NK0  	  upp4      # u uppi ©N)ÚmxÚstackÚzipÚrandomÚkeyra   )r\   rc   re   ÚrowÚposs   &&&  r-   Ú_position_keysrp   v   sP   € Ü�8Š8ô   Ô3ô	
á3‘�ô �I‰I�M‰Mœ.¨°CÓ8Ö9Ù3ò	
óð ùó	
s   Ÿ4A
c                  óV   € ] tR t^tRtR R ltR R ltR R ltR R	 ltR
 R lt	Rt
R# )Ú_PositionedTargetSamplerzFSampler with stateless target draws keyed by generated-token position.c               ó$   € V ^8„  d   QhRRRRRR/# )r(   ÚtemperatureÚfloatÚtop_pr\   rN   r+   )r,   s   "r-   r.   Ú%_PositionedTargetSampler.__annotate__‚   s!   € ÷ ñ  uð °Uð À#ñ r/   c               	óf   € \        V4      V n        \        V4      V n        \        V4      V n        R # rh   )ru   rt   rv   rN   r\   )Úselfrt   rv   r\   s   &$$$r-   Ú__init__Ú!_PositionedTargetSampler.__init__‚   s%   € Ü  Ó-ˆÔÜ˜5“\ˆŒ
Ü˜“IˆŽ	r/   c               ó    € V ^8„  d   QhRRRR/# )r(   Úlogprobsrf   r)   r+   )r,   s   "r-   r.   rw   ‡   s   € ÷ Hñ H ð H¨hñ Hr/   c                	óô   € V P                   ^ 8”  d2   V P                   R8  d!   \        WP                   V P                  4      # \        P                  P                  V^V P                  ,          ,          4      # )r   r%   )rv   r   rt   ri   rl   Úcategorical)ry   r}   s   &&r-   Ú__call__Ú!_PositionedTargetSampler.__call__‡   sT   € Ø�:‰:˜Œ>˜dŸj™j¨3Ô.Ü! (¯J©J¸×8HÑ8HÓIÐIÜ�y‰y×$Ñ$ X°°T×5EÑ5EÕ1EÕ%FÓGÐGr/   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r(   r}   rf   rc   rd   re   r)   r+   )r,   s   "r-   r.   rw   Œ   s8   € ÷ Iñ IàðIð ð	Ið
 ðIð 
ñIr/   c               	óœ  € VP                   ^ ,          \        V4      8w  g   \        V4      \        V4      8w  d   \        R4      h\        V P                  W#4      pV P
                  ^ 8”  d:   V P
                  R8  d)   \        P                  ! V P                  RR7      ! W4      # \        P                  ! V P                  RR7      ! W4      # )r   z5row_ids and positions must match logprobs batch size.r%   )Úin_axes©r   r   )
ÚshapeÚlenrV   rp   r\   rv   ri   ÚvmapÚ_sample_top_p_oneÚ_sample_one)ry   r}   rc   re   Úkeyss   &&$$ r-   Úsample_targetÚ&_PositionedTargetSampler.sample_targetŒ   s’   € ð �>‰>˜!Õ¤ G£Ô,´°G³ÄÀIÃÔ0NÜÐTÓUÐUÜ˜dŸi™i¨Ó<ˆØ�:‰:˜Œ>˜dŸj™j¨3Ô.Ü—7’7˜4×1Ñ1¸6ÕBÀ8ÓRÐRÜ�wŠw�t×'Ñ'°Õ8¸ÓHÐHr/   c               ó$   € V ^8„  d   QhRRRRRR/# ©r(   r}   rf   rm   r)   r+   )r,   s   "r-   r.   rw   š   s&   € ÷ Qñ Q Hð Q°8ð QÀñ Qr/   c                	ót   € \         P                  P                  V^V P                  ,          ,          VR7      # )é   ©rm   )ri   rl   r   rt   )ry   r}   rm   s   &&&r-   rŠ   Ú$_PositionedTargetSampler._sample_oneš   s,   € Ü�y‰y×$Ñ$ X°°T×5EÑ5EÕ1EÕ%FÈCÐ$ÓPÐPr/   c               ó$   € V ^8„  d   QhRRRRRR/# r�   r+   )r,   s   "r-   r.   rw   �   s&   € ÷ Vñ V¨(ð V¸ð VÀhñ Vr/   c                	ó”  € VP                   \        P                  8X  d    VP                  \        P                  4      p\        P
                  ! WP                  ,          RR7      p\        P                  ! VRR7      p\        P                  ! W4RR7      p\        P                  ! VRR7      p\        P                  ! V^V P                  ,
          8„  V\        P                  ! V4      4      p\        P                  P                  \        P                  ! V4      VR7      p\        P                  ! WHR,          RR7      ^ ,          # )r‘   ©Úaxisr’   éÿÿÿÿ).N)Údtyperi   Úbfloat16ÚastypeÚfloat32Úsoftmaxrt   ÚargsortÚtake_along_axisÚcumsumÚwhererv   Ú
zeros_likerl   r   Úlog)	ry   r}   rm   ÚprobsÚsorted_indicesÚsorted_probsÚcumulative_probsÚ	top_probsÚsampled_poss	   &&&      r-   r‰   Ú*_PositionedTargetSampler._sample_top_p_one�   sß   € Ø�>‰>œRŸ[™[Ô(Ø—‘¤r§z¡zÓ2ˆHÜ—
’
˜8×&6Ñ&6Õ6¸RÔ@ˆÜŸš E°Ô3ˆÜ×)Ò)¨%ÀbÔIˆÜŸ9š9 \¸Ô;ÐÜ—H’HØ˜q 4§:¡:�~Ñ-ØÜ�MŠM˜,Ó'ó
ˆ	ô
 —i‘i×+Ñ+¬B¯FªF°9Ó,=À3Ð+ÓGˆÜ×!Ò! .¸iÕ2HÈrÔRÐSTÕUÐUr/   )r\   rt   rv   N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__rz   r€   rŒ   rŠ   r‰   Ú__static_attributes__r+   r/   r-   rr   rr      s(   † ÙPõõ
Hõ
IõQ÷Vñ Vr/   rr   c               ó   € V ^8„  d   QhRR/# )r(   ÚnameÚstrr+   )r,   s   "r-   r.   r.   ­   s   € ÷ Uñ U Cñ Ur/   c                ól   € \         P                  P                  R 4      pV'       d   \        W V4      # T# )r#   )ÚsysÚmodulesrS   r2   )r²   ÚfallbackÚgenerate_modules   && r-   Ú_generate_module_overrider¹   ­   s)   € Ü—k‘k—o‘oÐ&8Ó9€Oß7FŒ7�?¨(Ó3ÐTÈHÐTr/   c                óJ  € V f   R # \        V \        \        34      '       gK   \        V 4      R9   d;   \        ;QJ d    R V  4       F  '       d   K   RM	  RM! R V  4       4      '       g   \        R4      h\        V 4      ^8X  d   V ^ ,          V ^ ,          3# \        V 4      # )Nc              3  óD   "  € T F  p\        V4      \        J x € K  	  R # 5irh   )ÚtyperN   )rB   Úvalues   & r-   rD   Ú)normalize_resize_shape.<locals>.<genexpr>¸   s   é € Ð7± u”�U“œsÕ"³ùs   ‚ FTz)resize_shape must contain 1 or 2 integers)r‘   r(   )Ú
isinstancer³   Úbytesr‡   ÚallrV   Útuple)Úvaluess   &r-   Únormalize_resize_shaperÄ   ²   s{   € Ø‚~Ùä�v¤¤U˜|×,Ò,Ü�‹K˜6Ô!ß‹CÑ7±Ó7�C�CŠCÑ7±Ó7×7Ò7äÐDÓEÐEÜ%(¨£[°AÔ%5ˆF�1�I�v˜a•yÐ!ÐH¼5À»=ÐHr/   Ú
max_tokensrt   Úrepetition_penaltyÚrepetition_context_sizeÚpresence_penaltyÚpresence_context_sizeÚfrequency_penaltyÚfrequency_context_sizerv   Úmin_pÚtop_kÚ
logit_biasÚmax_kv_sizeÚkv_bitsÚkv_group_sizeÚkv_quant_schemeÚquantized_kv_startÚsamplerÚlogits_processorsÚprefill_step_sizeÚdflashÚdraft_block_sizeÚprompt_cache_checkpointÚprompt_cache_checkpoint_lenr\   ÚverboseFc          >     ó”   € V ^8„  d   QhRRRRRRRRR	R
RRRR
RRRR
RRRRRRRRRRRRRRRR
RRRRRRRRR R!R"RR#R$R%RR&RR'R(R)RR*RR+R,R-R./# )/r(   r5   rf   rI   ú	nn.ModulerÅ   rN   rt   ru   rÆ   zOptional[float]rÇ   úOptional[int]rÈ   rÉ   rÊ   rË   rv   rÌ   rÍ   rÎ   zOptional[Dict[int, float]]r7   úOptional[List[Any]]rÏ   rÐ   rÑ   rÒ   r³   rÓ   rÔ   ú(Optional[Callable[[mx.array], mx.array]]rÕ   ú8Optional[List[Callable[[mx.array, mx.array], mx.array]]]rÖ   r8   úOptional[nn.Module]r9   rØ   rÙ   z*Optional[Callable[[int, List[Any]], None]]rÚ   r\   rÛ   r*   r)   z0Generator[Tuple[mx.array, mx.array], None, None]r+   )r,   s   "r-   r.   r.   ¾   sg  € ÷ Tñ TØðTàðTð ðTð ðTð (ðTð +ðTð &ðTð )ðTð 'ðTð *ðTð ðTð ðTð  ð!Tð" +ð#Tð$ &ð%Tð& ð'Tð( ð)Tð* ð+Tð, ð-Tð. ð/Tð0 6ð1Tð2 Pð3Tð4 %ð5Tð6 %ð7Tð8 ð9Tð: $ð;Tð< Hð=Tð> "/ð?Tð@ ðATðB ðCTðF 6ñGTr/   c             +  ó\  aaaa a8a9a:a;a<a="  € \         P                  ! \        R\        4      VVVVR7      o:SRJ ;'       d    V^ 8H  p!SfK   Ve-   V^ 8”  d&   V\        8X  d   V\
        8X  d   \        VVVR7      oM\        R\        4      ! VVVVR7      o\        R\        4      ! VVVVV	V
V4      o9Ve   S9P                  V4       T p"\        P                  ! . V P                  R7      o=^ o<S P                  R	R4      p#Sf#   \        P                  ! SP                   VR
7      oRo8/ o;Veo   ^RIHp$ V$! SVV4       \'        VV4      o;\)        SR4      '       d   SP                   MSp%\)        V%R4      '       d   RV%n        \)        V%R4      '       d   RV%n        RV V8VV9VV:VV;V<V=3
R llp&\        P.                  ! \0        4      ;_uu_ 4        SP2                  ! W3RV/S B p'V'P4                  p(S P7                  V'P9                  4       P;                  4        U)U*u/ uF  w  p)p*V)R8w  g   K  V*f   K  V)V*bK  	  up*p)4       S p+S;'       d   / S CS;Cp+Ve   \=        SV V(SVVV+R7      '       g   RpVe   Ve   \?        V4      MRp,Rp-VRJ;'       d    V(P@                  ^,          V8„  ;'       g4    V,RJ;'       d(    ^ T,u;8  ;'       d    V(P@                  ^,          8  Mu p.VEe‹   V.'       Ed‚   V(P@                  ^,          p/^ p0\C        V/RRV'       * R7      ;_uu_ 4       p1V(P@                  ^,          ^8”  Ed!   \E        VV(P@                  ^,          ^,
          4      p2V,e'   V-'       g   V0V,8  d   V0V2,           V,8”  d
   V,V0,
          p2SP                   ! R RV RRV213,          RV(RRV213,          RSRV2/S B  S:! S4       \        PF                  ! S U3u. uF  p3V3PH                  NK  	  up34       V0V2,          p0V,e   V-'       g   V0V,8X  d   V! V0S4       Rp-V(RV2R13,          p(V RV2R13,          p \        PJ                  ! 4        V1P7                  V24       EK9   RRR4       V RR!R13,          p V&! V V(R7      w  p"p4RRR4       \        PL                  ! V"X44       Ve"   \O        SVSV V"V4S8VVSVV!R7       Rj  x€L
  R# ^ p5 V5V8w  d*   V&! V"R,          4      w  p6p7\        PL                  ! V6V74       V5^ 8X  d   \        PF                  ! V"4       V5V8X  d   R# V"PQ                  4       V43x € V5R,          ^ 8X  d   \        PJ                  ! 4        V#e   V#PS                  X64      p6X6X7p4p"V5^,          p5K²  u up*p)i u up3i   + '       g   i     EL.; i  + '       g   i     EL ; i Lí5i)"a¢  
A generator producing token ids based on the given prompt from the model.

Args:
    input_ids (mx.array): The input prompt token ids.
    model (nn.Module): The model to use for generation.
    pixel_values: The pixel values for vision models (optional).
    mask: The attention mask (optional).
    max_tokens (int): Maximum number of tokens to generate.
    temperature (float): The temperature for sampling, if 0 the argmax is used.
    repetition_penalty (float, optional): The penalty factor for repeating
      tokens.
    repetition_context_size (int, optional): The number of tokens to
      consider for repetition penalty.
    presence_penalty (float, optional): Additive penalty for tokens that
      already appeared in recent generated context.
    presence_context_size (int, optional): The number of tokens to
      consider for presence penalty.
    frequency_penalty (float, optional): Additive penalty scaled by token
      frequency in recent generated context.
    frequency_context_size (int, optional): The number of tokens to
      consider for frequency penalty.
    top_p (float, optional): Nucleus sampling, higher means model considers
      more less likely words.
    min_p (float, optional): Minimum probability threshold relative to the
      highest-probability token.
    top_k (int, optional): Restrict sampling to the top-k tokens.
    logit_bias (dictionary, optional): Additive logit bias.
    prompt_cache (list, optional): Pre-existing KV cache for the prompt.
    max_kv_size (int, optional): Maximum KV cache size.
    kv_bits (float, optional): Number of bits for KV cache quantization.
    kv_group_size (int): Group size for uniform KV cache quantization.
    kv_quant_scheme (str): KV cache quantization backend.
    quantized_kv_start (int): Start index for quantized KV cache.
    sampler (Callable[mx.array, mx.array], optional): A sampler for sampling a
      token from a vector of log probabilities.
    logits_processors (List[Callable[[mx.array, mx.array], mx.array]], optional):
      A list of functions that take tokens and logits and return the processed
      logits.
    prefill_step_size (int): Number of tokens to process per prefill step.
      Chunked prefill processes prompts in smaller chunks to reduce peak
      memory usage.
    draft_model (nn.Module, optional): A drafter for speculative decoding.
      When set, the decode loop is replaced by the drafter's speculative
      loop (e.g. DFlash block-diffusion). VLM prefill with image/audio
      is supported via the same ``get_input_embeddings`` path the normal
      decoder uses; decode itself is text-only. ``temperature`` and
      ``sampler`` are respected; ``logprobs`` is always ``None`` on the
      speculative path.
    draft_block_size (int, optional): Override the drafter's configured
      block size.

Yields:
    Generator[Tuple[mx.array, mx.array], None, None]: A generator producing
      one token and a vector of log probabilities.
r!   )rÓ   rÑ   rÐ   rÒ   N)rt   rv   r\   r   )Útemprv   rÌ   rÍ   r   ©r™   Úthinking_budget_criteria)rÏ   )Úvalidate_drafter_compatibilityr=   Ú_position_idsÚ_rope_deltasc                óà  <
€ SpS'       d   / SCSCp\         P                  ! \        4      ;_uu_ 4        R V9   d   S	P                  ! R	RS/VB pMS	P                  ! V 3RVRS/VB pVoVP                  R,          p\        S
4      ^ 8”  dI   \        V 4      ^ 8”  d9   \         P                  ! SV P                  4       .4      oS
 F  pV! SV4      pK  	  S! S4       V\         P                  ! VR
RR7      ,
          p\        SV^ .VP                  ^ ,          ,          \        \        SSVP                  ^ ,          ,           4      4      R7      p SVP                  ^ ,          ,          oVP                  e   RVP                  /oMVP                  e   RVP                  /oM/ oYP                  ^ ,          ^8X  d   VP                  ^ 4      MT3uuRRR4       #   + '       g   i     R# ; i)Údecoder_input_idsr   r6   T©r—   Úkeepdims©rc   re   NÚcross_attention_statesÚencoder_outputsr+   r˜   ©ºNNNr˜   rò   )ri   Ústreamr    r=   Úlogitsr‡   ÚconcatÚflattenÚ	logsumexpÚ_sample_with_positionsr†   ÚlistÚrangerï   rð   Úsqueeze)Úyr6   Ústep_kwargsÚoutputsrô   Ú	processorr}   ÚkwargsÚlast_outputsrI   Ú
processorsr7   Úquantize_cache_fnrÔ   Ú"speculative_prefill_capture_kwargsÚtarget_sample_positionÚtokenss   &&     €€€€€€€€€€r-   Ú_stepÚgenerate_step.<locals>._stepd  sÎ  ø€ ð ˆß-ØJ˜VÐJÐ'IÐJˆKä�YŠYÔ(×)Õ)Ø" kÔ1Ø×.Ò.ñ Ø&ðà!ñ‘ð
  ×.Ò.Øñà"/ðð 'ðð "ñ	�ð #ˆLØ—^‘^ HÕ-ˆFä�:‹ Ô"¤s¨1£v°¤zÜŸš F¨A¯I©I«KÐ#8Ó9�ã!+�IÙ& v¨vÓ6’Fñ ",ñ ˜lÔ+à¤§¢¨V¸"ÀtÔ LÕLˆHÜ&ØØØ˜˜hŸn™n¨QÕ/Õ/ÜÜØ.Ø.°·±ÀÕ1BÕBóóô	
ˆAð # h§n¡n°QÕ&7Õ7Ð"à×-Ñ-Ò9Ø2°G×4RÑ4RÐS‘Ø×(Ñ(Ò4Ø+¨W×-DÑ-DÐE‘à�à¯^©^¸AÕ->À!Ô-C�h×&Ñ& qÔ)ÈÐQ÷] *×)×)Ó)ús   ´FGÇG-	Úmaskr6   r?   FÚPrefillÚtok)ÚtotalÚdescÚunitÚdisableÚinputsrò   r   Ún_to_processT)r6   )r9   rÅ   rÔ   rØ   Úsampler_is_greedyé   rh   r+   r˜   )*Ú	functoolsÚpartialr¹   r!   ÚDEFAULT_MIN_PÚDEFAULT_TOP_Krr   r   r   Úextendri   Úarrayr™   Úpopr   Úmake_prompt_cacher=   Úspeculative.draftersrç   r   Úhasattrrè   ré   ró   r    Úget_input_embeddingsr6   ÚupdateÚto_dictÚitemsrK   rN   r†   r   ÚminÚevalÚstateÚclear_cacheÚ
async_evalr   ÚitemÚapply_forced_token)>r5   rI   Úpixel_valuesr	  rÅ   rt   rÆ   rÇ   rÈ   rÉ   rÊ   rË   rv   rÌ   rÍ   rÎ   r7   rÏ   rÐ   rÑ   rÒ   rÓ   rÔ   rÕ   rÖ   r8   r9   rØ   rÙ   rÚ   r\   rÛ   r   r  rü   ræ   rç   Úlmr  Úembedding_outputr6   ÚkÚvÚpolicy_kwargsÚcheckpoint_lenÚcheckpoint_doneÚshould_chunkÚtotal_tokensÚprocessed_tokensÚpbarr  Úcr}   ÚnÚnext_yÚnext_logprobsr  r  r  r  r  r  s>   &f&&$$$$$$$$$$$$d$$$$$d$$$$$$$$$l                       @@@@@@r-   Úgenerate_stepr9  ¾   sö  ÿùé € ôz "×)Ò)Ü!Ð";Ô=TÓUØ-Ø#ØØ'ôÐð   4˜×<Ð<¨K¸1Ñ,<ÐØ‚àÒØ˜a”ØœÔ&ØœÔ&ä.Ø'ØØô‰Gô 0°ÄÔMØ ØØØô	ˆGô +Ø Ô"8ôð 	ØØØØØØó
€Jð Ò$Ø×ÑÐ+Ô,à€AÜ�XŠX�b 	§¡Ô0€FØÐà%Ÿz™zÐ*DÀdÓKÐð ÒÜ×.Ò.Ø× Ñ Ø#ô
ˆð €LØ)+Ð&ØÒÝIá& u¨k¸:ÔFÜ-GØ˜ó.
Ð*ô &-¨UÐ4D×%EÒ%EˆU×!Ò!È5ˆÜ�2�×'Ò'Ø#ˆBÔÜ�2�~×&Ò&Ø"ˆBŒO÷5R÷ 5Rð 5Rôn 
�ŠÔ$×	%Õ	%à ×5Ò5Øñ
Ø*.ð
Ø28ñ
Ðð )×6Ñ6ˆà�‰ð -×4Ñ4Ó6×<Ñ<Ô>ôá>‘D�A�qØ˜Ñ'ô à,-ô ��1’Ù>òô	
ð ˆß-ØL˜vÐLÐ)KÐLˆMØÒ(Ô1IØØØ'Ø%Ø#Ø!Ø(÷2
ó 2
ð !%Ðð 'Ò2Ø+Ò7ô Ð+Ô,ð ð	 	ð  ˆà TÐ)×XÐX¨m×.AÑ.AÀ!Õ.DÐGXÑ.X÷
ð 
ð  $Ð&×VÐV¨1¨~×+VÔ+VÀ×@SÑ@SÐTUÕ@VÔ+Vð 	ð
 Ó(¯\¨\à(×.Ñ.¨qÕ1ˆLØ ÐÜØ"¨¸ÈGÌ÷õ àØ#×)Ñ)¨!Õ,¨qÕ0Ü#&Ð'8¸-×:MÑ:MÈaÕ:PÐSTÕ:TÓ#U�Là&Ò2ß /Ø,¨~Ô=Ø,¨|Õ;¸nÔLà'5Ð8HÕ'H˜Ø×(Ò(ñ Ø(¨¨M¨\¨MÐ)9Õ:ðà&3°A°}¸°}Ð4DÕ&Eðð +ðð &2ð	ð
 !òñ & lÔ3Ü—G’G©lÓ;©l¨˜QŸWœW©lÑ;Ô<Ø$¨Õ4Ð$à&Ò2ß /Ø,°Ô>á/Ð0@À,ÔOØ*.˜Ø$1°!°\±]Ð2BÕ$C�MØ )¨!¨\©]Ð*:Õ ;�IÜ—N’NÔ$Ø—K‘K ×-ð; 1÷ðD " ! R¡S &Õ)ˆIá˜I°]ÔC‰ˆˆ8÷a 
&ôd ‡M‚M�!�XÔð ÒÜ)ØØØØØØØØ!Ø!ØØ-Ø/ô
÷ 	
ð 	
ñ 	à	€AØ
Ø�
Œ?Ù$)¨!¨D­'£NÑ!ˆF�MÜ�MŠM˜& -Ô0Ø�Œ6Ü�GŠG�AŒJØ�
Œ?Ùà�f‰f‹h˜Ð Ò Øˆs�7�aŒ<Ü�NŠNÔà#Ò/Ø-×@Ñ@ÀÓHˆFØ˜mˆ8ˆØ	ˆQ�Šùó]ùòn <÷)÷ ð ú÷Y 
&×	%Ð	%úñl	
ùs¨   ŒGV,ÇAVÈU7È,U7È2U7È9VÉAVÊVÊ/VÊ;VË
&VË11VÌ"A	V	Í,A*V	ÏU=Ï+V	ÐAV	Ñ"VÑ?=V,Ò<V*Ò=B:V,Õ7VÕ=V	ÖVÖ	VÖV'	Ö!
V,c                  óˆ   € ] tR tRt$ RtR]R&   R]R&   R]R&   R	]R
&   R	]R&   R	]R&   R]R&   R]R&   RtR]R&   RtR]R&   RtR# )ÚBatchGenerationResulti  ak  
Result of batch generation with optional image size tracking.

Attributes:
    texts: Generated text for each sample
    tokens: Last generated token for each sample
    logprobs: Log probabilities for each sample
    prompt_tokens: Number of prompt tokens per sample
    generation_tokens: Number of generated tokens per sample
    total_tokens: Total tokens (prompt + generation) per sample
    prompt_tps: Prompt tokens per second per sample
    generation_tps: Generation tokens per second per sample
    peak_memory: Peak memory usage in GB
    image_sizes: Original (height, width) for each image (for tracking)
ú	List[str]ÚtextsúList[Optional[int]]r  zList[Optional[List[float]]]r}   rd   Úprompt_tokensÚgeneration_tokensr2  zList[float]Ú
prompt_tpsÚgeneration_tpsr$   ru   Úpeak_memoryNúOptional[List[Tuple[int, int]]]Úimage_sizesr+   )	r«   r¬   r­   r®   r¯   Ú__annotations__rC  rE  r°   r+   r/   r-   r;  r;    sN   ‡ ñð  ÓØÓØ)Ó)ØÓØ Ó ØÓØÓØÓØ€K�ÓØ37€KÐ0×7r/   r;  c           
     óÂ   € Vf   \        R V  4       4      p\        P                  ! V  Uu. uF$  p^ .V\        V4      ,
          ,          V,           NK&  	  up4      # u upi )Nc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irh   ©r‡   ©rB   Úps   & r-   rD   Ú$_left_pad_prompts.<locals>.<genexpr>5  ó   é € Ð1© Aœ˜QŸ˜«ùó   ‚)rU   ri   r  r‡   ©ÚpromptsÚ
max_lengthrK  s   && r-   Ú_left_pad_promptsrR  3  sQ   € ØÒÜÑ1©Ó1Ó1ˆ
ä�8Š8¹gÓF¹g¸�a�S˜J¬¨Q«Õ/Õ0°1×4Ð4¹gÑFÓGÐGùÒFs   «*Ac                óÔ   € Vf   \        R V  4       4      p\        P                  ! V  Uu. uF-  p\        V4      ^ .V\	        V4      ,
          ,          ,           NK/  	  up4      # u upi )Nc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irh   rI  rJ  s   & r-   rD   Ú%_right_pad_prompts.<locals>.<genexpr><  rM  rN  )rU   ri   r  rù   r‡   rO  s   && r-   Ú_right_pad_promptsrV  :  sU   € ØÒÜÑ1©Ó1Ó1ˆ
ä�8Š8ÁGÓLÁG¸q”T˜!“W ˜s j´3°q³6Õ&9Õ:×:Ð:ÁGÑLÓMÐMùÒLs   «3A%c               ó(   € V ^8„  d   QhRRRRRRRR/# )r(   r-  rf   Úrow_idxrN   Ú
batch_sizer)   r+   )r,   s   "r-   r.   r.   Q  s(   € ÷ ñ ˜ð ¨Cð ¸Sð ÀXñ r/   c                óZ   € V P                   ^ ,          V8X  d   WV^,            # V R,          # )r   :Nr‘   N)r†   )r-  rX  rY  s   &&&r-   Ú_prompt_kwarg_rowr[  Q  s*   € Ø‡w�wˆq…z�ZÔØ˜7 Q�;Ð'Ð'ØˆR�5€Lr/   c               ó$   € V ^8„  d   QhRRRRRR/# )r(   Úprompt_kwargsÚdictrY  rN   r)   z
List[dict]r+   )r,   s   "r-   r.   r.   W  s!   € ÷ ñ °ð À#ð È*ñ r/   c                ó°  € V^8:  d   T ;'       g    / .# \        V4       Uu. uF  p/ NK  	  ppT ;'       g    / P                  4        F„  w  rE\        V\        P                  4      '       dS   VP
                  ^ 8”  dB   VP                  ^ ,          ^8¼  d*   \        V4       F  p\        WVV4      W6,          V&   K  	  Kw  V F  pWWV&   K	  	  K†  	  V# u upi )aŽ  Normalize batched prompt kwargs into one dict per batch row.

``model.get_input_embeddings()`` commonly returns batch-sized tensors
(notably ``inputs_embeds``). ``BatchGenerator.insert()`` stores prompt
kwargs per sequence, so passing the same batched dict for every row causes
the prompt builder to concatenate those batched tensors ``batch_size``
times, effectively squaring the batch dimension.
)rú   r!  r¿   ri   r  Úndimr†   r[  )r]  rY  Ú_Úrowsr,  r-  Úirn   s   &&      r-   Ú_split_prompt_kwargs_per_rowrd  W  s½   € ð �Q„Ø×#Ð# Ð$Ð$ä˜jÔ)Ó*Ñ)�1‹BÑ)€DÐ*Ø×$Ð$ "×+Ñ+Ö-‰ˆÜ�aœŸ™×"Ò" q§v¡v°¤z°a·g±g¸aµjÀA´oÜ˜:Ö&�Ü.¨q°ZÓ@�•˜“
ó 'ó �Ø�A“ó ñ .ð €Kùò +s   ¢Cc               ó(   € V ^8„  d   QhRRRRRRRR/# )	r(   rm   r³   r-  rf   Úsequence_lengthrN   r)   r*   r+   )r,   s   "r-   r.   r.   n  s,   € ÷ ñ Ø	ðØðØ,/ðà	ñr/   c                ó~   € V \         9   ;'       d.    VP                  ^8¬  ;'       d    VP                  ^,          V8H  # )r(   )Ú_SEQUENCE_ALIGNED_PROMPT_KWARGSr`  r†   )rm   r-  rf  s   &&&r-   Ú!_is_sequence_aligned_prompt_kwargri  n  sA   € ð 	Ô.Ñ.÷ 	*ð 	*Ø�F‰F�a‰K÷	*ð 	*à�G‰G�A�J˜/Ñ)ðr/   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r(   r-  rf   Útarget_lengthrN   Úleftr*   r)   r+   )r,   s   "r-   r.   r.   x  s,   € ÷ 	)ñ 	)Øð	)Ø #ð	)Ø.2ð	)àñ	)r/   c               ó@  € WP                   ^,          ,
          pV^ 8:  d   V # V P                   ^ ,          V3\        V P                   R,          4      ,           p\        P                  ! W@P                  R7      pV'       d   WP.MW.p\        P
                  ! V^R7      # )r‘   :r(   NNrå   r–   )r†   rÂ   ri   Úzerosr™   Úconcatenate)r-  rk  rl  ÚpadÚ	pad_shapeÚpad_vÚpartss   &&$    r-   Ú"_pad_sequence_aligned_prompt_kwargrt  x  st   € ð Ÿ'™' !�*Õ
$€CØ
ˆa„xØˆØ—‘˜•˜SÐ!¤E¨!¯'©'°"­+Ó$6Õ6€IÜ�HŠH�Y§g¡gÔ.€EßˆU‰J Q J€EÜ�>Š>˜% aÔ(Ð(r/   c               ó$   € V ^8„  d   QhRRRRRR/# )r(   Úprompt_kwargs_listúList[Optional[dict]]r5   úList[List[int]]r)   zTuple[mx.array, dict]r+   )r,   s   "r-   r.   r.   „  s&   € ÷ /(ñ /(Ø,ð/(àð/(ð ñ/(r/   c                ó¨  € V Uu. uF  p\        V4      NK  	  pp\        V4      p. pRpRp\        W4       Fµ  w  r‰V'       d   VP                  R4      f   \	        R4      hVR,          p
V
P
                  pV
P                  R,          pW”8  dL   \        P                  ! V
P                  ^ ,          WI,
          V3VR7      p\        P                  ! Wº.^R7      p
VP                  V
4       K·  	  \        P                  ! V^ R7      p/ p/ p\        V 4      p\        \        W4      4       Fè  w  pw  r‰V'       g   K  VP                  4        FÂ  w  ppVR8X  g   V\        9   d   K  \        V\        P                  4      '       d{   VP                   ^ 8”  dj   VP                  ^ ,          ^8¼  dR   \#        VVV4      p\%        VVV	4      '       d   \'        VVRR7      pVP)                  V. 4      P                  V4       K´  VV9  g   K½  VVV&   KÄ  	  Kê  	  VP                  4        F!  w  pp\        P                  ! V^ R7      VV&   K#  	  WÍ3# u upi )	z>Batch per-row prompt kwargs for a left-padded prefill forward.Nr6   zinputs_embeds is requiredrå   r–   T©rl  r˜   )r‡   rU   rk   rS   rV   r™   r†   ri   rn  ro  rF   Ú	enumerater!  ÚAPC_PRIVATE_PROMPT_KEYSr¿   r  r`  r[  ri  rt  Ú
setdefault)rv  r5   ÚidsÚlengthsrQ  Ú
row_embedsÚembed_dtypeÚ	embed_dimÚkwÚlengthÚembedsrp  r6   Úmerged_kwargsÚper_row_keysrY  rc  r,  r-  Úrow_vÚvss   &&                   r-   Ú_merge_prefill_prompt_kwargsrŠ  „  s  € ñ
 $-Ó-¡9˜CŒs�3Žx¡9€GÐ-Ü�W“€Jà!#€JØ€KØ€IÜÐ,Ö6‰
ˆß�R—V‘V˜OÓ,Ò4ÜÐ8Ó9Ð9Ø�OÕ$ˆØ—l‘lˆØ—L‘L Õ$ˆ	ØÔÜ—(’(Ø—‘˜a• *Õ"5°yÐAØ!ôˆCô —^’^ S M¸Ô:ˆFØ×Ñ˜&Ö!ñ 7ô —N’N :°AÔ6€Mà€MØ€LÜÐ'Ó(€JÜ$¤SÐ);Ó%EÖF‰ˆ‰<ˆBßÙØ—H‘H–J‰DˆAˆqØ�OÔ# qÔ,CÔ'CÙÜ˜!œRŸX™X×&Ò&¨1¯6©6°A¬:¸!¿'¹'À!½*È¼/Ü)¨!¨Q°
Ó;�Ü4°Q¸¸v×FÒFÜ>Ø˜z°ô�Eð ×'Ñ'¨¨2Ó.×5Ñ5°eÖ<Ø˜-Ö'Ø#$�˜aÓ ó ñ Gð ×#Ñ#Ö%‰ˆˆ2ÜŸ>š>¨"°1Ô5ˆ�aÓñ &ð Ð'Ð'ùòU .s   …Ic                óÆ  € V '       g   V# V'       g   V # . p\        W4       F»  w  r4\        VR4      '       g9   \        VP                  R4      '       d   VP                  P                  V.4      p\        VR4      '       g9   \        VP                  R4      '       d   VP                  P                  V.4      pVP	                  V4       VP                  V4       K½  	  V# )z6Extend cache_a with cache_b along the batch dimension.Úleft_paddingÚmerge)rk   r  Ú	__class__r�  r  rF   )Úcache_aÚcache_bÚextendedÚcaÚcbs   &&   r-   Ú_extend_cacher”  ¶  s¬   € çØˆßØˆØ€HÜ�gÖ'‰ˆÜ�r˜>×*Ò*¬w°r·|±|ÀW×/MÒ/MØ—‘×#Ñ# R DÓ)ˆBÜ�r˜>×*Ò*¬w°r·|±|ÀW×/MÒ/MØ—‘×#Ñ# R DÓ)ˆBØ
�	‰	�"ŒØ�‰˜Öñ (ð €Or/   c           
     óv  aaaa
aa€ SRJ;'       d    \        SV4      oVVV3R lo
RV
VVV3R llo\        V R4      '       dT   V P                  4       p\        V4      p\	        V4       UUu. uF   w  rxS! Y†^8”  d   Wv^,
          8  MRR7      NK"  	  upp# Sea   \        V P
                  4      p\        V4       Uu. uF5  pWv^,
          8  g   V^8:  d	   S
! S4      M\        P                  ! S4      NK7  	  up# V P
                   U	u. uF  p	\        P                  ! S4      NK  	  up	# u uppi u upi u up	i )uÊ  
Convert a list of regular caches into their corresponding
batch-aware caches.

When *kv_bits* is set, a quantized batch cache is used instead of
``BatchKVCache`` so that KV states are quantized on-the-fly during
generation, reducing memory usage for long sequences.

*kv_quant_scheme* selects the quantization backend:
- ``"uniform"`` â†’ ``BatchQuantizedKVCache`` (``mx.quantize``)
- ``"turboquant"`` or fractional *kv_bits* â†’ ``BatchTurboQuantKVCache``
Nc                ór   <€ S'       d   \        V SR 7      # \        P                  ! V S\        S4      R7      # ))Úbits)Ú
group_sizer—  )r   r   ÚBatchQuantizedKVCacherN   )ÚlprÐ   rÑ   Ú	use_turbos   &€€€r-   Ú_make_quant_cacheÚ&_make_cache.<locals>._make_quant_cacheÜ  s1   ø€ ßÜ)¨"°7Ô;Ð;Ü×*Ò*Ø˜=¬s°7«|ô
ð 	
r/   Tc                ó  <€ \        V \        P                  4      '       d+   Se   V'       d	   S! S4      # \        P                  ! S4      # \        V \        P                  4      '       d+   Se   V'       d	   S! S4      # \        P                  ! S4      # \        V \        P
                  4      '       d+   Se   V'       d	   S! S4      # \        P                  ! S4      # \        V \        P                  4      '       d   \        P                  ! S4      V n	        V # \        V \        P                  4      '       d>   V P                  ^ 8”  d   \        R4      h\        P                  ! V P                  S4      # \        V \        P                  4      '       d)   \        P                  ! V3R lV P                    4       !  # \        V \"        4      '       d   \        P                  ! V3R lV  4       !  # \        \%        V 4       R24      h)Nz2RotatingKVCache with keep tokens is not supported.c              3  ó4   <"  € T F  pS! V4      x € K  	  R # 5irh   r+   ©rB   Úsub_cÚto_batch_caches   & €r-   rD   Ú6_make_cache.<locals>.to_batch_cache.<locals>.<genexpr>ø  s   øé € Ð$QÉ¸u¡^°E×%:Ð%:Ëùó   ƒc              3  ó4   <"  € T F  pS! V4      x € K  	  R # 5irh   r+   r   s   & €r-   rD   r£  ú  s   øé € Ð$JÉ¸u¡^°E×%:Ð%:Ëùr¤  z does not yet support batching)r¿   r   ÚKVCacheÚBatchKVCacheÚChunkedKVCacheÚSimpleKVCacheÚArraysCacheri   r  rŒ  ÚRotatingKVCacheÚkeeprV   ÚBatchRotatingKVCacheÚmax_sizeÚ	CacheListÚcachesrÂ   r¼   )r5  Úquantizerœ  rÐ   rŒ  r¢  s   &&€€€€r-   r¢  Ú#_make_cache.<locals>.to_batch_cacheã  sr  ø€ Ü�aœŸ™×'Ò'ØÒ"§xÙ(¨Ó6Ð6Ü×%Ò% lÓ3Ð3Ü˜œ5×/Ñ/×0Ò0ØÒ"§xÙ(¨Ó6Ð6Ü×%Ò% lÓ3Ð3Ü˜œ5×.Ñ.×/Ò/ØÒ"§xÙ(¨Ó6Ð6Ü×%Ò% lÓ3Ð3Ü˜œ5×,Ñ,×-Ò-ÜŸXšX lÓ3ˆAŒNØˆHÜ˜œ5×0Ñ0×1Ò1Ø�v‰v˜ŒzÜ Ð!UÓVÐVÜ×-Ò-¨a¯j©j¸,ÓGÐGÜ˜œ5Ÿ?™?×+Ò+Ü—?’?Ô$QÈÏÊÓ$QÑRÐRÜ˜œ5×!Ò!Ü—?’?Ô$JÉÓ$JÑKÐKä¤ Q£˜yÐ(FÐGÓHÐHr/   Ú
make_cache)r±  )T)	r   r  r³  r‡   r{  Úlayersrú   r   r§  )rI   rŒ  rÐ   rÑ   rÒ   Úmodel_cacher6  rc  r5  ra  rœ  r¢  r›  s   &fff&     @@@r-   Ú_make_cacher¶  Ç  s3  ý€ ð& ˜tÐ#×TÐTÔ(:¸7ÀOÓ(T€I÷
÷Iò Iô6 ˆu�l×#Ò#Ø×&Ñ&Ó(ˆÜ�Óˆô " +Ô.ô
á.‘�ñ ˜1¸´U¨°­Eª	À×FÙ.ò
ð 	
ð
 ÒÜ�E—L‘LÓ!ˆAô ˜qœóñ "�Að ˜q�5”y A¨¤Fñ & lÔ3ä×+Ò+¨LÓ9ò:ñ "ñð ð ;@¿,º,ÓG¹,°Q”×"Ò" <Ö0¹,ÑGÐGùó
ùòùò Hs   Á-&D+Â=;D1Ä	D6c                  ó~   € ] tR tRt$ Rt^ tR]R&   ^ tR]R&   ^ tR]R&   ^ t	R]R&   ^ t
R]R	&   ^ tR]R
&   ^ tR]R&   RtR# )Ú
BatchStatsi  aý  
An data object to hold generation stats.

Args:
    prompt_tokens (int): The number of prompt tokens processed.
    prompt_tps (float): The prompt processing tokens-per-second.
    prompt_time (float): The time in seconds spent in prompt processing.
    generation_tokens (int): The number of generated tokens.
    generation_tps (float): The tokens-per-second for generation.
    generation_time (float): The time in seconds spent in generation .
    peak_memory (float): The peak memory used so far in GB.
rN   r?  ru   rA  Úprompt_timer@  rB  Úgeneration_timerC  r+   N)r«   r¬   r­   r®   r¯   r?  rF  rA  r¹  r@  rB  rº  rC  r°   r+   r/   r-   r¸  r¸    sP   ‡ ñð €M�3ÓØ€J�ÓØ€K�ÓØÐ�sÓØ€N�EÓØ€O�UÓØ€K�×r/   r¸  c                  ó>   € ] tR tRt$ RtR]R&   R]R&   RtR]R	&   R
tR# )ÚBatchResponsei,  a…  
An data object to hold a batch generation response.

Args:
    texts: (List[str]): The generated text for each prompt.
    stats (BatchStats): Statistics about the generation.
    image_sizes: (Optional[List[Tuple[int, int]]]): Original (height, width)
        for each image. Useful for tracking which images produced which responses
        and for debugging padding/batching behavior.
r<  r=  r¸  ÚstatsNrD  rE  r+   )r«   r¬   r­   r®   r¯   rF  rE  r°   r+   r/   r-   r¼  r¼  ,  s    ‡ ñ	ð ÓØÓØ37€KÐ0×7r/   r¼  c                  óZ   € ] tR tRt$ RtR]R&   R]R&   RtR]R&   RtR]R	&   ^ tR]R
&   Rt	R# )ÚPromptProgressi>  z>Per-request prompt processing metrics for continuous batching.rN   Úuidr?  r$   ru   rA  r¹  Úcached_tokensr+   N)
r«   r¬   r­   r®   r¯   rF  rA  r¹  rÁ  r°   r+   r/   r-   r¿  r¿  >  s.   ‡ áHà	ƒHØÓØ€J�ÓØ€K�ÓØ€M�3×r/   r¿  rc   re   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR/# )	r(   rÔ   úCallable[[mx.array], mx.array]r}   rf   rc   úOptional[List[int]]re   r)   r+   )r,   s   "r-   r.   r.   I  s:   € ÷ 
ñ 
Ø+ð
àð
ð !ð	
ð
 #ð
ð ñ
r/   c               ór   € \        V R R4      p\        V4      '       d   Ve   Ve   V! WVR7      # V ! V4      # )rŒ   Nrî   ©r2   rG   )rÔ   r}   rc   re   rŒ   s   &&$$ r-   rø   rø   I  s?   € ô ˜G _°dÓ;€MÜ�×Ò 7Ò#6¸9Ò;PÙ˜XÀ)ÔLÐLÙ�8ÓÐr/   c                  ó¾   € ] tR tRtRt] ! R R4      4       tRR R lltR	 tR
 t	RR/R R llt
R R ltR tR tR R ltR R ltR R lt]RR R ll4       tRtR# )ÚGenerationBatchiV  a  
Batched token generator with double-buffered pipelining.

Manages the generation phase after prompt processing, with KV caches,
sampling, and stop detection for multiple sequences. Uses async_eval
to overlap GPU computation with CPU processing (decode-ahead pattern).
c                  óN   € ] tR tRt$ R]R&   R]R&   R]R&   R]R&   R	tR
]R&   RtR	# )ÚGenerationBatch.Responsei_  rN   rÀ  Útokenru   Útoken_logprobúOptional[str]Úfinish_reasonNz!Optional[List[Tuple[int, float]]]Útop_logprobsr+   )r«   r¬   r­   r®   rF  rÏ  r°   r+   r/   r-   ÚResponserÊ  _  s#   ‡ à‹Ø‹
ØÓØ$Ó$Ø:>ˆÐ7×>r/   rÐ  FNc               óD   € V ^8„  d   QhRRRRRRRRR	R
RRRRRRRRRRRR/# )r(   rI   rÝ   Úuidsrd   r  rf   r7   ú	List[Any]rÔ   rÃ  rÅ   Útop_logprobs_krN   Úgreedy_samplingr*   Útoken_contextzOptional[List[List[int]]]rÕ   úHOptional[List[Optional[List[Callable[[mx.array, mx.array], mx.array]]]]]ræ   rß   r+   )r,   s   "r-   r.   ÚGenerationBatch.__annotate__g  sx   € ÷ )!ñ )!àð)!ð ð)!ð ð	)!ð
  ð)!ð 0ð)!ð ð)!ð ð)!ð ð)!ð 1ð)!ð
ð)!ð #6ñ)!r/   c                	óê  € Wn         \        VR V4      V n        W n        W@n        WPn        W`n        Wpn        ^ .\        V4      ,          V n	        RV n
        W€n        W�n        T;'       g    . V n        T;'       g    . V n        T
;'       g    .  Uu. uF  p\        V4      NK  	  upV n        V P#                  4        RV n        RV n        W0n        RV n        RV n        RV n        RV n        R# u upi )r=   TN)rI   r2   Ú_language_modelrÒ  r7   rÔ   Ústop_criteriarÅ   r‡   Ú_num_tokensÚcompute_logprobsrÔ  rÕ  rÕ   ræ   rù   rÖ  Ú_ensure_token_contextÚ_current_tokensÚ_current_lpsÚ_next_tokensÚ	_next_lpsÚ_next_top_idxÚ_next_top_lpré   )ry   rI   rÒ  r  r7   rÔ   rÛ  rÅ   rÔ  rÕ  rÖ  rÕ   ræ   Úctxs   &&&&&&&&&&&&& r-   rz   ÚGenerationBatch.__init__g  sí   € ð" Œ
Ü& uÐ.>ÀÓFˆÔØŒ	Ø(ÔØŒØ*ÔØ$ŒØ˜3¤ T£�?ˆÔØ $ˆÔØ,ÔØ.ÔØ!2×!8Ð!8°bˆÔØ(@×(FÐ(FÀBˆÔ%Ø4A×4GÐ4GÀRÐ4GÓIÑ4G¨Cœd 3žiÑ4GÑIˆÔØ×"Ñ"Ô$à#ˆÔØ ˆÔØ"ÔØˆŒØ!ˆÔØ ˆÔð !ˆÖùò Js   ÂC0c                	ó,   € \        V P                  4      # rh   ©r‡   rÒ  ©ry   s   &r-   Ú__len__ÚGenerationBatch.__len__’  ó   € Ü�4—9‘9‹~Ðr/   c                	ó€   € V P                    Uu. uF#  p\        VR 4      '       g   K  VP                  NK%  	  up# u upi ©r$  ©r7   r  r$  ©ry   r5  s   & r-   Úcache_statesÚGenerationBatch.cache_states•  ó1   € Ø!%×!2Ò!2ÓJÑ!2˜A´g¸aÀ×6I”�—”Ñ!2ÑJÐJùÒJó   �;¨;Úforcec               ó   € V ^8„  d   QhRR/# )r(   rõ  r*   r+   )r,   s   "r-   r.   rØ  ˜  s   € ÷ 	Fñ 	F¨dñ 	Fr/   c               	ó€  € V'       gI   V P                   '       d   \        V P                   4      '       g   V P                   '       g   . V n        R # \        V P                  4      \        V P                  4      8  df   \        V P                  4      \        V P                  4      ,
          pV P                  P                  \        V4       Uu. uF  p. NK  	  up4       R # \        V P                  4      \        V P                  4      8”  d*   V P                  R \        V P                  4       V n        R # R # u upi rh   )rÕ   rH   rÖ  r‡   rÒ  r  rú   )ry   rõ  Úmissingra  s   &$  r-   rÞ  Ú%GenerationBatch._ensure_token_context˜  sÚ   € ß˜$×0×0Ð0´S¸×9OÑ9O×5PÒ5PØ×)×)Ð)Ø%'�Ô"ÙÜˆt×!Ñ!Ó"¤S¨¯©£^Ô3Ü˜$Ÿ)™)“n¤s¨4×+=Ñ+=Ó'>Õ>ˆGØ×Ñ×%Ñ%´5¸´>Ó&B±>¨a£r±>Ñ&BÖCÜ�×#Ñ#Ó$¤s¨4¯9©9£~Ô5Ø!%×!3Ñ!3Ð4D´c¸$¿)¹)³nÐ!EˆDÖñ 6ùò 'Cs   ÃD;c               ó    € V ^8„  d   QhRRRR/# )r(   r  rf   Ú
fwd_kwargsr^  r+   )r,   s   "r-   r.   rØ  £  s   € ÷ ñ ¨(ð Àñ r/   c           	     	ó  € V P                   '       dQ   V P                  '       g?   V P                  ^ 8”  g.   V P                  '       d   \	        V P                  4      '       d   R# \        V P                  RR4      p\        V4      '       g   R# V P                  ! VR,          3RV P                  RRRR/VB pVP                  R,          pV! V4      pVf   R# VP                  ^8X  d"   VP                  ^,          ^8X  d
   VR	,          pV# )
r   NÚspeculative_argmax_from_hiddenr   Úreturn_hiddenTÚskip_logits©rò   Nr˜   )rò   r   )rÕ  rÝ  rÔ  rÕ   rH   r2   rÚ  rG   r7   Úhidden_statesr`  r†   )ry   r  rû  Úargmax_from_hiddenÚoutputÚhiddenÚsampleds   &&&    r-   Ú_greedy_argmax_stepÚ#GenerationBatch._greedy_argmax_step£  s  € à×$×$Ð$Ø×$×$Ð$Ø×"Ñ" QÔ&Ø×&×&Ð&¬3¨t×/EÑ/E×+FÒ+Fáä$Ø× Ñ Ð"BÀDó
Ðô Ð*×+Ò+Ùà×%Ò%Ø�7�Oñ
à×#Ñ#ð
ð ð
ð ð	
ð
 ñ
ˆð ×%Ñ% bÕ)ˆÙ$ VÓ,ˆØŠ?ÙØ�<‰<˜1Ô §¡¨qÕ!1°QÔ!6Ø˜d•mˆGØˆr/   c           
     ó
  € V P                   V n        V P                  V n        V P                  p/ pV P                  e   V P                  VR&   V P                  W4      pVef   W0n         RV n        RV n        RV n        \        P                  ! V P                   4       \        P                  ! V4       VP                  4       RRR3# V P                  ! VR,          3RV P                  /VB p\        VR4      '       d   VP                  MTpVR,          pV P                   '       EdL   \#        V P                   4      '       Ed0   VP                  4       pV P%                  4        \'        V4       F'  w  rxV P(                  V,          P+                  V4       K)  	  . p	\-        VP.                  ^ ,          4       FŸ  pWWV^,            p
V P                   V,          ;'       g    . pV F]  p\        VR4      '       d   VP1                  Wg,          V
4      p
K/  V! \        P2                  ! V P(                  V,          4      V
4      p
K_  	  V	P+                  V
4       K¡  	  \        P4                  ! V	^ R7      pV\        P6                  ! VRRR7      ,
          p\9        V P:                  T^ .\=        V P>                  4      ,          V P@                   Uu. uF  qî^,           NK  	  upR	7      pW0n         V P                  pV P                  pV P                   .pV PB                  '       dR   V\        PD                  ! VP.                  ^ ,          4      V3,          V n        VP+                  V P                  4       MRV n        V PF                  pV^ 8”  d‹   \        PH                  ! VRR7      pVR
V) R13,          R
RRR13,          PK                  \        PL                  4      p\        PN                  ! VVRR7      pVV n        VV n        VPQ                  VV.4       MRV n        RV n        \        P                  ! V!   V P                  e‘   WP                  .pVe   VPQ                  VV.4       \        P                  ! V!   Ve   VP                  4       MRpVe   VP                  4       MRpVP                  4       V P                  P                  4       VV3# \        P                  ! V4       VP                  4       RRR3# u upi )z2Perform one generation step with double buffering.NÚrope_deltasr   rô   Úprocess_last_tokenr–   Trì   rî   .r   r˜   rñ   ))rá  rß  râ  rà  ré   r  rã  rä  ri   r&  r#  ÚtolistrÚ  r7   r  rô   rÕ   rH   rÞ  r{  rÖ  rF   rú   r†   r
  r  ro  r÷   rø   rÔ   r‡   rÒ  rÜ  rÝ  ÚarangerÔ  rž   r›   Úint32rŸ   r  )ry   r  rû  r  r  rô   Úlast_tokensrc  rË  Úprocessed_logitsÚsample_logitsr  rÿ   r}   r6  Úprev_top_idxÚprev_top_lpÚeval_targetsr,  Úsort_idxÚtop_idxÚtop_lpÚto_evalÚtop_idx_listÚtop_lp_lists   &                        r-   r  ÚGenerationBatch._stepÁ  sC  € à#×0Ñ0ˆÔØ ŸN™NˆÔØ×%Ñ%ˆàˆ
Ø×ÑÒ(Ø(,×(9Ñ(9ˆJ�}Ñ%à×*Ñ*¨6Ó>ˆØÒØ 'ÔØ!ˆDŒNØ!%ˆDÔØ $ˆDÔÜ�MŠM˜$×+Ñ+Ô,Ü�GŠG�FŒOØ—=‘=“? D¨$°Ð4Ð4à×%Ò%Ø�7�Oñ
Ø#'×#4Ñ#4ð
Ø8Bñ
ˆô #*¨&°(×";Ò";�—’ÀˆØ˜Õ!ˆà×!×!Ñ!¤c¨$×*@Ñ*@×&AÓ&AØ Ÿ-™-›/ˆKØ×&Ñ&Ô(Ü% kÖ2‘�Ø×"Ñ" 1Õ%×,Ñ,¨UÖ3ñ 3ð  "ÐÜ˜6Ÿ<™<¨�?Ö+�Ø &¨1¨q­5Ð 1�Ø!×3Ñ3°AÕ6×<Ð<¸"�
Û!+�IÜ˜yÐ*>×?Ò?Ø(1×(DÑ(DØ'�N¨Mó)šñ )2ÜŸHšH T×%7Ñ%7¸Õ%:Ó;¸]ó)šñ ",ð !×'Ñ'¨Ö6ñ ,ô —^’^Ð$4¸1Ô=ˆFàœBŸLšL¨°bÀ4ÔHÕHˆÜ(Ø�L‰LØØ�Cœ#˜dŸi™i›.Õ(Ø&*×&6Ò&6Ó7Ñ&6 ˜1—u�uÑ&6Ñ7ô	
ˆð $ÔØ×)Ñ)ˆØ×'Ñ'ˆà×)Ñ)Ð*ˆØ× × Ð Ø%¤b§i¢i°·±¸aÕ0@Ó&AÀ7Ð&JÕKˆDŒNØ×Ñ §¡Õ/à!ˆDŒNà×ÑˆØˆqŒ5ä—z’z (°Ô4ˆHØ˜s Q B¡C˜xÕ(¨©d°¨d¨Õ3×:Ñ:¼2¿8¹8ÓDˆGÜ×'Ò'¨°'ÀÔCˆFØ!(ˆDÔØ &ˆDÔØ×Ñ ¨&Ð 1Õ2à!%ˆDÔØ $ˆDÔä
�Š�|Ò$à×ÑÒ(Ø×0Ñ0Ð1ˆGØÒ'Ø—‘ ¨kÐ:Ô;Ü�GŠG�WÒØ4@Ò4L˜<×.Ñ.Ô0ÐRVˆLØ2=Ò2I˜+×,Ñ,Ô.ÈtˆKà—‘“Ø×!Ñ!×(Ñ(Ó*ØØð	ð ô �GŠG�FŒOØ—=‘=“? D¨$°Ð4Ð4ùòY 8s   ËTc           	     ó~  aa€ . oVV3R loS! V P                   V P                  V P                  V P                  V P                  V P
                  V P                  34       V P                   F  p S! VP                  4       K  	  S'       d   \        P                  ! S!   R# R#   \        \        3 d     KN  i ; i)zBMaterialize lazy decode outputs before mutating batch-owned state.c                óÌ   <€ \        V \        P                  4      '       d   SP                  V 4       R # \        V \        \
        34      '       d   V  F  pS! V4       K  	  R # R # rh   )r¿   ri   r  rF   rù   rÂ   )r½   r'  Úappend_arraysÚtargetss   & €€r-   r  Ú:GenerationBatch._eval_pending_state.<locals>.append_arrays(  sH   ø€ Ü˜%¤§¡×*Ò*Ø—‘˜uÖ%Ü˜E¤D¬% =×1Ò1Û!�DÙ! $Ö'ó "ñ 2r/   N)rß  rà  rá  râ  rã  rä  ré   r7   r$  ÚAttributeErrorÚ	TypeErrorri   r#  )ry   r5  r  r  s   & @@r-   Ú_eval_pending_stateÚ#GenerationBatch._eval_pending_state$  s«   ù€ àˆö	(ñ 	à×$Ñ$Ø×!Ñ!Ø×!Ñ!Ø—‘Ø×"Ñ"Ø×!Ñ!Ø×!Ñ!ðô
	
ð ×"Ô"ˆAðÙ˜aŸg™gÖ&ñ #÷ Ü�GŠG�WÔñ øô #¤IÐ.ô Úðús   Á1B'Â'B<Â;B<c               ó   € V ^8„  d   QhRR/# )r(   Úotherz'GenerationBatch'r+   )r,   s   "r-   r.   rØ  C  s   € ÷ CXñ CXÐ-ñ CXr/   c                óì	  € \        V P                  4      ^ 8H  pV'       g;   \        VP                  4      ^ 8”  d!   V P                  4        VP                  4        V P                  ;'       d    \	        V P                  4      pVP                  ;'       d    \	        VP                  4      pV'       g	   V'       d7   V P                  \        V4      R7       VP                  \        V4      R7       V P                  P                  VP                  4       \        V P                  VP                  4      V n	        V P                  P                  VP                  4       V P                  P                  VP                  4       V P                  P                  VP                  4       V P                  P                  VP                  4       V P                  P                  VP                  4       V P                  4        V P                  f$   VP                  V n        VP                  V n        MŒVP                  e   \         P"                  ! V P                  VP                  .4      V n        V P                  e@   VP                  e2   \         P"                  ! V P                  VP                  .4      V n        V P$                  fG   VP$                  V n        VP&                  V n        VP(                  V n        VP*                  V n        EMWVP$                  EeI   \         P"                  ! V P$                  VP$                  .4      V n        V P&                  e@   VP&                  e2   \         P"                  ! V P&                  VP&                  .4      V n        V P(                  e¯   VP(                  e¡   V P(                  P,                  R,          VP(                  P,                  R,          8X  dd   \         P"                  ! V P(                  VP(                  .4      V n        \         P"                  ! V P*                  VP*                  .4      V n        MRV n        RV n        V'       d   VP.                  V n        R# V P.                  RJ VP.                  RJ 8w  d   \1        R4      hV P.                  e4   \         P"                  ! V P.                  VP.                  .4      V n        R# R# )z0Extend this batch with another generation batch.)rõ  Nzbextend() mixes MRoPE and non-MRoPE batches; both sides must carry rope_deltas or neither side may.r˜   )r‡   rÒ  r"  rÕ   rH   rÞ  r*   r  r”  r7   rÅ   rÜ  rÖ  ræ   rß  rà  ri   ro  rá  râ  rã  rä  r†   ré   ÚRuntimeError)ry   r%  Úself_was_emptyÚself_has_processorsÚother_has_processorss   &&   r-   r  ÚGenerationBatch.extendC  s½  € ä˜TŸY™Y›¨1Ñ,ˆß¤# e§j¡j£/°AÔ"5Ø×$Ñ$Ô&Ø×%Ñ%Ô'à"×4Ñ4×TÐT¼¸T×=SÑ=SÓ9TÐØ$×6Ñ6×WÐW¼3¸u×?VÑ?VÓ;WÐß×"6Ø×&Ñ&¬TÐ2FÓ-GÐ&ÔHØ×'Ñ'¬dÐ3FÓ.GÐ'ÔHà�	‰	×Ñ˜Ÿ™Ô$Ü)¨$×*;Ñ*;¸U×=OÑ=OÓPˆÔØ�‰×Ñ˜u×/Ñ/Ô0Ø×Ñ×Ñ × 1Ñ 1Ô2Ø×Ñ×!Ñ! %×"5Ñ"5Ô6Ø×Ñ×%Ñ% e×&=Ñ&=Ô>Ø×%Ñ%×,Ñ,¨U×-KÑ-KÔLØ×"Ñ"Ô$à×ÑÒ'Ø#(×#8Ñ#8ˆDÔ Ø %× 2Ñ 2ˆDÕØ×"Ñ"Ò.Ü#%§>¢>Ø×%Ñ% u×'<Ñ'<Ð=ó$ˆDÔ ð × Ñ Ò,°×1CÑ1CÒ1OÜ$&§N¢NØ×&Ñ&¨×(:Ñ(:Ð;ó%�Ô!ð ×ÑÒ$Ø %× 2Ñ 2ˆDÔØ"Ÿ_™_ˆDŒNØ!&×!4Ñ!4ˆDÔØ %× 2Ñ 2ˆDÖØ×ÑÓ+Ü "§¢°×0AÑ0AÀ5×CUÑCUÐ/VÓ WˆDÔØ�~‰~Ò)¨e¯o©oÒ.IÜ!#§¢°·±ÀÇÁÐ0QÓ!R�”ð ×"Ñ"Ò.Ø×'Ñ'Ò3Ø×&Ñ&×,Ñ,¨RÕ0°E×4GÑ4G×4MÑ4MÈbÕ4QÔQä%'§^¢^Ø×'Ñ'¨×)<Ñ)<Ð=ó&�Ô"ô %'§N¢NØ×&Ñ&¨×(:Ñ(:Ð;ó%�Õ!ð &*�Ô"Ø$(�Ô!çØ %× 2Ñ 2ˆDÖØ×Ñ 4Ð'¨U×-?Ñ-?À4Ð-GÔHÜð9óð ð ×ÑÒ*Ü "§¢°×0AÑ0AÀ5×CUÑCUÐ/VÓ WˆDÖñ +r/   c               ó   € V ^8„  d   QhRR/# ©r(   r¬  rd   r+   )r,   s   "r-   r.   rØ  ˆ  s   € ÷ )@ñ )@˜9ñ )@r/   c                ó²  € \        V4      \        V P                  4      8  d   V P                  4        V Uu. uF  q P                  V,          NK  	  upV n        V Uu. uF  q P                  V,          NK  	  upV n        V Uu. uF  q P                  V,          NK  	  upV n        V P
                  '       d(   V Uu. uF  q P
                  V,          NK  	  upV n        V P                  '       d(   V Uu. uF  q P                  V,          NK  	  upV n        V P                  '       d(   V Uu. uF  q P                  V,          NK  	  upV n        V'       gc   V P                  P                  4        RV n
        RV n        RV n        RV n        RV n        RV n        RV n        . V n        . V n        . V n        R# \"        P$                  ! V\"        P&                  4      pV P                   F  pVP)                  V4       K  	  V P                  e   V P                  V,          V n        V P                  e   V P                  V,          V n        V P                  e1   V P                  V,          V n        V P                  V,          V n        V P                   e   V P                   V,          V n        R# R# u upi u upi u upi u upi u upi u upi )z4Filter the batch to keep only the specified indices.N)r‡   rÒ  r"  rÅ   rÜ  rÖ  rÕ   ræ   r7   Úclearrß  rà  rá  râ  rã  rä  ré   ri   r  r  Úfilter)ry   r¬  ÚidxÚkeep_arrr5  s   &&   r-   r0  ÚGenerationBatch.filterˆ  sQ  € äˆt‹9”s˜4Ÿ9™9“~Ô%Ø×$Ñ$Ô&á/3Ó4©t¨—Y‘Y˜s—^�^©tÑ4ˆŒ	Ù;?Ó@¹4°CŸ?™?¨3×/Ð/¹4Ñ@ˆŒÙ=AÓB¹T°c×,Ñ,¨S×1Ð1¹TÑBˆÔØ××ÐÙEIÓ!JÁT¸c×"4Ñ"4°S×"9Ð"9ÁTÑ!JˆDÔØ×!×!Ð!ÙMQÓ%RÉTÀc×&<Ñ&<¸S×&AÐ&AÉTÑ%RˆDÔ"Ø×(×(Ð(á>Bó-Ù>B°s×-Ñ-¨c×2Ð2¹dñ-ˆDÔ)÷ Ø×Ñ×#Ñ#Ô%Ø#'ˆDÔ Ø $ˆDÔØ $ˆDÔØ!ˆDŒNØ!%ˆDÔØ $ˆDÔØ $ˆDÔØ!#ˆDÔØ%'ˆDÔ"Ø,.ˆDÖ)ä—x’x ¤b§h¡hÓ/ˆHØ×&Ô&�Ø—‘˜Ö"ñ 'à× Ñ Ò,Ø$(×$5Ñ$5°hÕ$?�Ô!Ø�~‰~Ò)Ø!%§¡°Õ!9�”Ø×!Ñ!Ò-Ø%)×%7Ñ%7¸Õ%A�Ô"Ø$(×$5Ñ$5°hÕ$?�Ô!Ø× Ñ Ò,Ø$(×$5Ñ$5°hÕ$?�Ö!ñ -ùòG 5ùÚ@ùÚBùâ!Jùâ%Rùò-s#   ¸J;ÁK ÂKÂ?K
Ã8KÄ1Kc               ó   € V ^8„  d   QhRR/# )r(   r)   zList[Response]r+   )r,   s   "r-   r.   rØ  ³  s   € ÷ ;ñ ;�nñ ;r/   c                óh  € V P                   '       g   . # V P                  4       w  rr4. p. pRp\        \        V P                   4      4       EFÞ  pRp	V P                  V;;,          ^,          uu&   W,          p
V\        V P
                  4      8  dÕ   V P
                  V,          eÀ   V P
                  V,          pV! V
4       Vf;   \        P                  ! V P                  4       V P                  P                  4       pVP                  \        P                  ! Wx,          .\        P                  R7      4      p\        VP                  4       4      pW×V,          8w  d   W×V&   V P                  V
4      '       d   Rp	M+V P                  V,          V P                   V,          8¼  d   Rp	V	f   VP#                  V4       RpVe"   \%        \'        W8,          WH,          4      4      pTP#                  T P)                  V P                   V,          T
Ve	   W(,          MRV	VR7      4       EKá  	  VeL   \        P                  ! V\        P                  R7      V n        \        P*                  ! V P                  4       \        V4      \        V P                   4      8  d   V P-                  V4       V# )z"Generate the next batch of tokens.Nrå   Ústopr„  r$   )rÀ  rË  rÌ  rÎ  rÏ  )rÒ  r  rú   r‡   rÜ  ræ   ri   r#  rá  r  r(  r  r  rN   r'  rÛ  rÅ   rF   rù   rk   rÐ  r&  r0  )ry   r  Úlp_listr  r  r¬  Ú	responsesÚforced_next_tokensrc  rÎ  r  Úcriteriar7  Ú
next_tokenr  s   &              r-   ÚnextÚGenerationBatch.next³  s   € à�y�yˆyØˆIà59·Z±Z³\Ñ2ˆ˜àˆØˆ	Ø!ÐÜ”s˜4Ÿ9™9“~×&ˆAØ ˆMØ×Ñ˜Q× 1Õ$ÓØ•)ˆCà”C˜×5Ñ5Ó6Ô6Ø×1Ñ1°!Õ4Ò@à×8Ñ8¸Õ;�Ù˜”Ø%Ò-Ü—G’G˜D×-Ñ-Ô.Ø)-×):Ñ):×)AÑ)AÓ)CÐ&Ø!×4Ñ4Ü—H’HÐ0Õ3Ð4¼B¿H¹HÔEó�ô ! §¡£Ó/�
Ø°AÕ!6Ô6Ø,6 qÑ)à×!Ñ! #×&Ò&Ø &‘Ø×!Ñ! !Õ$¨¯©¸Õ(:Ô:Ø (�àÒ$Ø—‘˜A”àˆFØÒ'Üœc ,¥/°;µ>ÓBÓC�à×ÑØ—‘ØŸ	™	 !�ØØ07Ò0C '¦*ÈØ"/Ø!'ð ó ÷ñA 'ðT Ò)Ü "§¢Ð);Ä2Ç8Á8Ô LˆDÔÜ�MŠM˜$×+Ñ+Ô,äˆt‹9”s˜4Ÿ9™9“~Ô%Ø�K‰K˜ÔàÐr/   c               ó   € V ^8„  d   QhRR/# )r(   rÕ  r*   r+   )r,   s   "r-   r.   rØ  ñ  s   € ÷  ñ  ð ñ r/   c                óX  € V P                  V 4      pWn        \        VRV4      Vn        . Vn        . Vn        W'n        W7n        . Vn        . Vn	        WGn
        WWn        Wgn        . Vn        . Vn        . Vn        RVn        RVn        RVn        RVn        RVn        RVn        RVn        V# )z!Create an empty generation batch.r=   N)Ú__new__rI   r2   rÚ  rÒ  r7   rÔ   rÛ  rÅ   rÜ  rÝ  rÔ  rÕ  rÖ  rÕ   ræ   rß  rà  rá  râ  rã  rä  ré   )ÚclsrI   rÔ   rÛ  rÝ  rÔ  rÕ  Úbatchs   &&&&&&& r-   ÚemptyÚGenerationBatch.emptyð  s¹   € ð —‘˜CÓ ˆØŒÜ '¨Ð/?ÀÓ GˆÔØˆŒ
ØˆÔØŒØ+ÔØˆÔØˆÔØ!1ÔØ-ÔØ /ÔØ ˆÔØ"$ˆÔØ)+ˆÔ&Ø $ˆÔØ!ˆÔØ!ˆÔØˆŒØ"ˆÔØ!ˆÔØ!ˆÔØˆr/   )rà  rß  rÚ  râ  rá  rã  rä  rÜ  ré   rÝ  rÕ  rÕ   rÅ   rI   r7   rÔ   rÛ  ræ   rÖ  rÔ  rÒ  )r   FNNN)Tr   F)r«   r¬   r­   r®   r¯   r   rÐ  rz   rê  rñ  rÞ  r  r  r"  r  r0  r<  ÚclassmethodrC  r°   r+   r/   r-   rÈ  rÈ  V  sy   † ñð ÷?ð ?ó ð?÷)!òVòKð	F°U÷ 	Fõò<a5òFõ>CXõJ)@õV;ðz ö ó ô r/   rÈ  c                  óº   € ] tR tRtRtRt]P                  tRRR]P                  RR/R	 R
 llt
R tR tR R ltR R ltR tR R ltR R ltR tR R ltRtR# )ÚSpeculativeGenerationBatchi  z>GenerationBatch-compatible wrapper for server-side MTP decode.TrØ   NÚtoken_dtyperÕ  Fc               óP   € V ^8„  d   QhRRRRRRRRRR	R
RRRRRRR	RRRR	RRRRRR/# )r(   rI   rÝ   r8   r9   r³   rÒ  rd   Úfirst_tokensrf   r7   rÓ  rÔ   rÃ  rÅ   r  Úshared_kv_statesúOptional[dict]r?  rØ   rÞ   rH  zmx.DtyperÕ  r*   r+   )r,   s   "r-   r.   Ú'SpeculativeGenerationBatch.__annotate__  s–   € ÷ &!ñ &!àð&!ð ð&!ð ð	&!ð
 ð&!ð ð&!ð  ð&!ð 0ð&!ð ð&!ð ð&!ð )ð&!ð  ð&!ð (ð&!ð  ð!&!ð" ñ#&!r/   c               	ó~  € Wn         W n        W0n        \        V4      V n        \        V4      V n        WPn        W`n        Wpn        W€n	        \        V	4      V n
        W n        W°n        WÀn        WÐn        Wàn        Wðn        ^ .\#        V4      ,          V n        R.\#        V4      ,          V n        RV n        RV n        R# )r   FN)rI   r8   r9   rù   rÒ  Ú	_all_uidsrJ  r7   rÔ   rÛ  rÅ   r  rK  r?  rØ   rH  rÕ  r‡   rÜ  Ú	_finishedÚ_sent_firstÚ_rounds_iter)ry   rI   r8   r9   rÒ  rJ  r7   rÔ   rÛ  rÅ   r  rK  r?  rØ   rH  rÕ  s   &&&&&&&&&&&&&$$$r-   rz   Ú#SpeculativeGenerationBatch.__init__  s¤   € ð& Œ
Ø&ÔØ$ŒÜ˜“JˆŒ	Ü˜d›ˆŒØ(ÔØ(ÔØŒØ*ÔÜ˜zÓ*ˆŒØŒØ 0ÔØ*ÔØ 0ÔØ&ÔØ.ÔØ˜3¤ T£�?ˆÔØ˜¤3 t£9Õ,ˆŒØ ˆÔØ ˆÖr/   c                	ó:   € \        R  V P                   4       4      # )c              3  ó.   "  € T F  q'       * x € K  	  R # 5irh   r+   )rB   Údones   & r-   rD   Ú5SpeculativeGenerationBatch.__len__.<locals>.<genexpr>C  s   é € Ð7© —8‘8«ùs   ‚)ÚsumrP  ré  s   &r-   rê  Ú"SpeculativeGenerationBatch.__len__B  s   € ÜÑ7¨¯ªÓ7Ó7Ð7r/   c                	ó˜   € \        V P                  V P                  4       UUu. uF  w  rV'       d   K  VNK  	  uppV n        R # u uppi rh   )rk   rO  rP  rÒ  )ry   rÀ  rV  s   &  r-   Ú_refresh_uidsÚ(SpeculativeGenerationBatch._refresh_uidsE  s8   € ä!$ T§^¡^°T·^±^Ô!Dô
Ù!D‘I�CÌD�CˆCÑ!Dò
ˆŽ	ùó 
s
   ¤AµAc               ó   € V ^8„  d   QhRR/# )r(   r%  z'SpeculativeGenerationBatch'r+   )r,   s   "r-   r.   rM  J  s   € ÷ Tñ TÐ8ñ Tr/   c                	ó†   € \        V 4      ^ 8X  d(   V P                  P                  VP                  4       R# \        R4      h)r   Nz5Cannot extend an active speculative generation batch.)r‡   Ú__dict__r  r'  )ry   r%  s   &&r-   r  Ú!SpeculativeGenerationBatch.extendJ  s2   € Üˆt‹9˜Œ>Ø�M‰M× Ñ  §¡Ô0ÙÜÐRÓSÐSr/   c               ó   € V ^8„  d   QhRR/# r-  r+   )r,   s   "r-   r.   rM  P  s   € ÷ ñ ˜9ñ r/   c                	óÜ   € V Uu0 uF  q P                   V,          kK  	  pp\        V P                  4       F  w  rEWS9  g   K  R V P                  V&   K  	  V P	                  4        R# u upi )TN)rÒ  r{  rO  rP  r[  )ry   r¬  r1  Ú	keep_uidsrc  rÀ  s   &&    r-   r0  Ú!SpeculativeGenerationBatch.filterP  sZ   € Ù/3Ó4©t¨—Y‘Y˜s—^�^©tˆ	Ð4Ü §¡Ö/‰FˆAØÖ#Ø$(�—‘˜qÓ!ñ 0ð 	×ÑÖùò	 5s   …A)c                	ó€   € V P                    Uu. uF#  p\        VR 4      '       g   K  VP                  NK%  	  up# u upi rî  rï  rð  s   & r-   rñ  Ú'SpeculativeGenerationBatch.cache_statesW  ró  rô  c               ó$   € V ^8„  d   QhRRRRRR/# )r(   rn   rN   rË  r)   rÍ  r+   )r,   s   "r-   r.   rM  Z  s!   € ÷ ñ  #ð ¨cð °mñ r/   c                	óŽ   € V P                  V4      '       d   R # V P                  V,          V P                  V,          8¼  d   R# R# )r6  r„  N)rÛ  rÜ  rÅ   )ry   rn   rË  s   &&&r-   Ú_finish_reasonÚ)SpeculativeGenerationBatch._finish_reasonZ  s8   € Ø×Ñ˜e×$Ò$ÙØ×Ñ˜CÕ  D§O¡O°CÕ$8Ô8ÙÙr/   c               ó$   € V ^8„  d   QhRRRRRR/# )r(   r8  úList[GenerationBatch.Response]Útok_listr>  r)   ÚNoner+   )r,   s   "r-   r.   rM  a  s(   € ÷ ñ à1ðð &ðð 
ñ	r/   c           
     	ór  € \        V4       F§  w  r4Ve   V P                  V,          '       d   K$  \        V4      pV P                  V;;,          ^,          uu&   V P	                  W44      pVe   RV P                  V&   VP                  V P                  V P                  V,          VRVR7      4       K©  	  R # )NTr$   ©rÀ  rË  rÌ  rÎ  )r{  rP  rN   rÜ  ri  rF   rÐ  rO  )ry   r8  rm  rn   rË  rÎ  s   &&&   r-   Ú_append_token_responsesÚ2SpeculativeGenerationBatch._append_token_responsesa  s¢   € ô
 $ HÖ-‰JˆCØŠ} §¡¨s× 3Ô 3ÙÜ˜“JˆEØ×Ñ˜S×! QÕ&Ó!Ø ×/Ñ/°Ó;ˆMØÒ(Ø&*�—‘˜sÑ#Ø×ÑØ—‘ØŸ™ sÕ+ØØ"%Ø"/ð	 ó öó .r/   c                	óè  a € S P                   e   R # V 3R lp\        S P                  S P                  S P                  S P
                  S P                  S P                  S P                  '       d   \        S P                  4      M^ S P                  S P                  S P                  VS P                  S P                  R S P                  ^ .\!        S P"                  4      ,          R7      S n         R # )Nc                óº   <€ SP                   V ,          ;'       gA    SP                  V4      ;'       g(    SP                  V ,          SP                  V ,          8¬  # rh   )rP  rÛ  rÜ  rÅ   )Úseq_idxÚtoken_idry   s   &&€r-   Ú
stop_checkÚ<SpeculativeGenerationBatch._start_rounds.<locals>.stop_check{  sV   ø€ à—‘˜wÕ'÷ Ið IØ×%Ñ% hÓ/÷Ið Ià×#Ñ# GÕ,°·±ÀÕ0HÑHðr/   )r9   Úfirst_bonusrÅ   rÔ   rØ   rH  rw  rÕ  rK  Úeos_token_idsr?  rc   )rR  r   rI   r8   r7   r  r9   rJ  rÅ   rU   rÔ   rØ   rH  rÕ  rK  r?  r‡   rO  )ry   rw  s   f r-   Ú_start_roundsÚ(SpeculativeGenerationBatch._start_roundsw  s¹   ø€ Ø×ÑÒ(Ùõ	ô :Ø�J‰JØ×ÑØ×ÑØ�K‰KØ—‘Ø×)Ñ)Ø/3¯¯¨”s˜4Ÿ?™?Ô+ÀAØ—L‘LØ!×2Ñ2Ø×(Ñ(Ø!Ø ×0Ñ0Ø!×2Ñ2ØØ×,Ñ,Ø�Cœ#˜dŸn™nÓ-Õ-ô!
ˆÖr/   c               ó   € V ^8„  d   QhRR/# )r(   r)   rl  r+   )r,   s   "r-   r.   rM  •  s   € ÷ 8ñ 8Ð4ñ 8r/   c                	ó0  € \        V 4      ^ 8X  d   . # . pV P                  '       Eg   RV n        \        P                  ! V P                  4       \        V P                  P                  4       4       F£  w  r#V P                  V,          '       d   K   \        V4      pV P                  V;;,          ^,          uu&   V P                  W#4      pVe   RV P                  V&   VP                  V P                  V P                  V,          VRVR7      4       K¥  	  V P                  4        V# V P                  4         \!        V P"                  4      w  rVT P'                  Y4       \)        T\*        4      '       dl   \        TP-                  R^ 4      4      ^,           \        TP-                  R^4      4      8  d,    \!        T P"                  4      w  rVT P'                  Y4       K�  T P                  4        T#   \$         d‚    \        T P                  4       FS  w  r'T'       d   K  RT P                  T&   TP                  T P                  T P                  T,          RRRR7      4       KU  	  T P                  4        Tu # i ; i  \$         d     K¯  i ; i)r   TNr$   rp  r„  Ú	round_posÚ	round_len)r‡   rQ  ri   r#  rJ  r{  r  rP  rN   rÜ  ri  rF   rÐ  rO  r[  r{  r<  rR  ÚStopIterationrq  r¿   r^  rS   )ry   r8  rn   rË  rÎ  rm  Ú
round_metarV  s   &       r-   r<  ÚSpeculativeGenerationBatch.next•  s?  € Üˆt‹9˜Œ>ØˆIà46ˆ	Ø××ÑØ#ˆDÔÜ�GŠG�D×%Ñ%Ô&Ü'¨×(9Ñ(9×(@Ñ(@Ó(BÖC‘
�Ø—>‘> #×&Ô&ÙÜ˜E›
�Ø× Ñ  ×%¨Õ*Ó%Ø $× 3Ñ 3°CÓ ?�Ø Ò,Ø*.�D—N‘N 3Ñ'Ø× Ñ Ø—M‘MØ ŸN™N¨3Õ/Ø#Ø&)Ø&3ð	 "ó öñ Dð  ×ÑÔ ØÐà×ÑÔð	Ü#'¨×(9Ñ(9Ó#:Ñ ˆHð  	×$Ñ$ YÔ9Ü˜¤T×*Ò*¬sØ�N‰N˜;¨Ó*ó0
àõ0ä�J—N‘N ;°Ó2Ó3ô04ðÜ'+¨D×,=Ñ,=Ó'>Ñ$�ð ×(Ñ(¨Ö=à×ÑÔØÐøô5 ô 	Ü& t§~¡~Ö6‘	�ß‘tØ*.�D—N‘N 3Ñ'Ø×$Ñ$ØŸ™Ø $§¡¨sÕ 3Ø"&Ø*-Ø*2ð	 &ó öñ 7ð ×ÑÔ ØÒð	ûô* !ô Úðús+   Ä<G7 Æ;J Ç7*JÈ&AJÊJÊJÊJ)rO  rP  rÜ  rR  rQ  rØ   r9   r8   rJ  rÕ  r  rÅ   rI   r7   r?  rÔ   rK  rÛ  rH  rÒ  )r«   r¬   r­   r®   r¯   Úis_speculativerÈ  rÐ  ri   r  rz   rê  r[  r  r0  rñ  ri  rq  r{  r<  r°   r+   r/   r-   rG  rG    sq   † ÙHà€NØ×'Ñ'€Hð&!ð +/ð&!ð  !#§¡ð!&!ð" !&÷#&!òP8ò
õ
TõòKõõò,
÷<8ñ 8r/   rG  c                  ó  € ] tR tRtRtRR]R]]RRRRRRRRRR3R R lltR t	R t
R	 tR
 R ltR R ltR R ltR R ltR R ltR"R R lltR R ltR R ltR R ltR#R R llt]R 4       t]R R  l4       tR!tR# )$ÚPromptProcessingBatchiÐ  a  
Handles VLM prompt processing with inputs_embeds and chunked prefill.

Processes prompt tokens incrementally (one chunk per step) to allow
interleaving with generation for continuous batching. Transitions to
a GenerationBatch when prompt processing is complete.
NFc          *     ól   € V ^8„  d   QhRRRRRRRRRR	R
RRRRRRRRRRRRRRRRRRRRRRRR R!R"RR#RR$R%/# )&r(   rI   rÝ   rÒ  rd   r5   rx  rÅ   r6   rf   r]  r^  rÕ   r×  ræ   rß   rÖ   rÞ   rÑ   rN   rÒ   r³   Ú
warm_cacheÚapc_metaúOptional[List[dict]]Úapc_managerúOptional['_apc.APCManager']Úright_pad_per_rowrÄ  Úsuffix_lensÚapc_moderÍ  r8   râ   r9   rØ   rÕ  r*   r+   )r,   s   "r-   r.   Ú"PromptProcessingBatch.__annotate__Ù  só   € ÷ a.ñ a.àða.ð ða.ð #ð	a.ð
 ða.ð  ða.ð ða.ð
ða.ð #6ða.ð )ða.ð ða.ð ða.ð  (ð!a.ð" 'ð#a.ð$ 1ð%a.ð& /ð'a.ð( )ð)a.ð*  ð+a.ð, )ð-a.ð. "ð/a.ð0 (ð1a.ð2 ñ3a.r/   c           
     	óÄ
  a
aa€ Wn         W n        \        V4      V n        W@n        W�n        VV n        VV n        VV n        VV n	        V Uu. uF  p\        V4      NK  	  pp\        V4      pVV n        T;'       g    TV n        V  Ve'   ^ .\        V4      ,          p\        VVR7      V n        M*V Uu. uF  pVV,
          NK  	  pp\!        VVR7      V n        \        V4      V n        \%        V4      V n        ^ V n        T;'       g    . V n        T;'       g    . V n        V P*                  '       d7   \/        V P*                  4      '       d   V Uu. uF  p\        V4      NK  	  upM. V n        WPn        T;'       g    / V n        . V n        V P4                  '       dñ   V P2                  eã   V P2                  P8                  ^ ,          pV P2                  P8                  ^,          pV P4                  P;                  4        FŠ  w  pp\=        V\>        P@                  4      '       g   K(  VPB                  ^8¼  g   K;  VP8                  ^ ,          V8X  g   KU  VP8                  ^,          V8X  g   Ko  V P6                  PE                  V4       KŒ  	  T;'       g    . V n#        Wðn$        VV n%        RV n&        RV n'        . V n(        . V n)        \U        V4       FÕ  w  p p!R p"^ p#V \        V PF                  4      8  dm   V PF                  V ,          eX   V PF                  V ,          PW                  R4      p"\Y        V PF                  V ,          PW                  R4      ;'       g    ^ 4      p#V PP                  PE                  V"e   \        V"4      MT!4       V PR                  PE                  V#4       K×  	  Ve   WÐn-        MˆVe*   Ve&   \]        VV\        V4      VV
VV3R lR7      V n-        M[\        V4      ^8X  d7   Vf3   S
f/   \_        VR4      '       d   \`        Pb                  ! V4      V n-        M\e        VVS
SSR	7      V n-        Vey   \/        V4      '       dh   V PZ                   FW  p$\g        V$R
R 4      p%\i        V%4      '       g#   RV n&        V Pk                  4        \m        R4      hV%! VV P                  R7       KY  	  V P
                  e„   \o        V P4                  4      p&Ve    Ve   V&Pq                  \s        VV4      4       \u        V P                   V P                  V P2                  V PZ                  VVV&R7      '       g
   R V n        R # R # R # u upi u upi u upi )N)rQ  Tr$   Úfull_input_idsÚ
prefix_lenc                ó$   <€ \        V VSSSR 7      # )©rÐ   rÑ   rÒ   )r¶  )r*  rš  rÐ   rÑ   rÒ   s   &&€€€r-   Ú<lambda>Ú0PromptProcessingBatch.__init__.<locals>.<lambda>F  s   ø€ ¬+ØØØ#Ø"/Ø$3õ+r/   )r9   rY  rŒ  r³  r³  r•  ÚprepareFz8APC mixed prefill requires a prompt cache with prepare())Úright_paddingr  r?   );rI   rÒ  rù   Ú_prompt_uidsrÅ   rÖ   r8   r9   rØ   rÕ  r‡   rU   Ú_right_pad_per_rowÚ_suffix_lensrV  Ú
_input_idsrR  Ú_left_padding_per_rowrX  Ú_total_prompt_tokensÚ_processed_prompt_columnsrÕ   ræ   rH   Ú_token_contextÚ_inputs_embedsÚ_prompt_kwargsÚ_prompt_length_aware_keysr†   r!  r¿   ri   r  r`  rF   Ú	_apc_metaÚ_apc_managerÚ	_apc_modeÚ_apc_harvest_enabledÚ_prompt_time_sÚ_prompt_tokens_per_rowÚ_cached_tokens_per_rowr{  rS   rN   r7   r   r  r   r  r¶  r2   rG   Ú_release_apc_meta_blocksr'  r^  r  r   rK   )'ry   rI   rÒ  r5   rÅ   r6   r]  rÕ   ræ   rÖ   rÐ   rÑ   rÒ   rˆ  r‰  r‹  r�  rŽ  r�  r8   r9   rØ   rÕ  r~  r  rQ  rŒ  ÚlÚprompt_batchÚ
prompt_lenr,  r-  r1  Ú
suffix_lenr’  r“  r5  r˜  r.  s'   &&&&&&&&&&fff&&&&&&&&&&                r-   rz   ÚPromptProcessingBatch.__init__Ù  s´  ú€ ð6 Œ
ØŒ	Ü  ›JˆÔØ$ŒØ!2ÔØ&ˆÔØ$ˆŒØ 0ˆÔØ.ˆÔá'0Ó1¡y ”3�s–8¡yˆÐ1Ü˜“\ˆ
ð #4ˆÔØ'×2Ð2¨7ˆÔÙàÒ(ð ˜3¤ Y£Õ/ˆLÜ0°ÀzÔRˆD�Oá4;Ó<±G¨q˜J¨ŸN˜N±GˆLÐ<Ü/°	ÀjÔQˆDŒOÜ%)¨,Ó%7ˆÔ"Ü$'¨£LˆÔ!Ø)*ˆÔ&à!2×!8Ð!8°bˆÔØ(@×(FÐ(FÀBˆÔ%ð ×%×%Ð%¬#¨d×.DÑ.D×*EÒ*Eñ #,Ó,¡)˜3ŒT�#ŽY¡)Ò,àð 	Ôð
 ,ÔØ+×1Ð1¨rˆÔØ46ˆÔ&Ø××Ð 4×#6Ñ#6Ò#BØ×.Ñ.×4Ñ4°QÕ7ˆLØ×,Ñ,×2Ñ2°1Õ5ˆJØ×+Ñ+×1Ñ1Ö3‘��1ä˜q¤"§(¡(×+Ô+ØŸ™ !žØŸ™ �
 lÖ2ØŸ™ �
 jÖ0à×2Ñ2×9Ñ9¸!Ö<ñ 4ð "Ÿ˜ RˆŒØ'ÔØ!ˆŒØ$(ˆÔ!Ø!ˆÔØ13ˆÔ#Ø13ˆÔ#Ü(¨Ö1‰OˆC�Ø!ˆNØˆJØ”S˜Ÿ™Ó(Ô(¨T¯^©^¸CÕ-@Ò-LØ!%§¡°Õ!4×!8Ñ!8Ð9IÓ!J�Ü  §¡°Õ!4×!8Ñ!8¸Ó!F×!KÐ!KÈ!ÓL�
Ø×'Ñ'×.Ñ.Ø'5Ò'A”�NÔ#Àzôð ×'Ñ'×.Ñ.¨zÖ:ñ  2ð Ò!Ø *ÕØÒ$¨Ò)?Ü =ØØ%Ü˜y›>Ø)öô!ˆDÕô �	‹N˜aÔØ!Ò)Ø’Ü˜˜|×,Ò,ä %× 7Ò 7¸Ó >ˆDÕä +ØØØØ+Ø /ô!ˆDÔð Ò(¬SÐ1B×-CÒ-CØ×&Ô&�Ü! ! Y°Ó5�Ü ×(Ò(Ø05�DÔ-Ø×1Ñ1Ô3Ü&ØRóð ñ Ð&7À×ARÑAR×Sñ 'ð ×!Ñ!Ò-Ü  ×!4Ñ!4Ó5ˆMØÒ&¨:Ò+AØ×$Ñ$Ü.¨z¸;ÓGôô ,Ø—
‘
ØŸ/™/Ø"×1Ñ1Ø!×.Ñ.Ø'Ø%Ø,÷ó ð *.�Ö&ññ .ùò[ 2ùò" =ùò -s   ÁUÂ5UÅUc                	ó,   € \        V P                  4      # rh   rè  ré  s   &r-   rê  ÚPromptProcessingBatch.__len__|  rì  r/   c                	ó®   € V P                   f   R # V P                   F4  pVf   K	  V P                   P                  VP                  R. 4      4       K6  	  R # )NÚ
apc_blocks)r¦  r¥  ÚreleaserS   )ry   Úmetas   & r-   r¬  Ú.PromptProcessingBatch._release_apc_meta_blocks  sD   € Ø×ÑÒ$ÙØ—N”NˆDØÔØ×!Ñ!×)Ñ)¨$¯(©(°<ÀÓ*DÖEó #r/   c                óÚ   € V P                   e   V P                  f   V P                  4       RJ# V P                  4       e   R# V P                   P                  ^,          V P                  8„  # )z:True if prompt needs chunked processing before generate().NT)r¢  rÖ   Ú_next_apc_checkpoint_columnr†   ré  s   &r-   Úneeds_processingÚ&PromptProcessingBatch.needs_processing†  sa   € à×ÑÒ&¨$×*@Ñ*@Ò*HØ×3Ñ3Ó5¸TÐAÐAØ×+Ñ+Ó-Ò9ÙØ×"Ñ"×(Ñ(¨Õ+¨d×.DÑ.DÑDÐDr/   c               ó$   € V ^8„  d   QhRRRRRR/# )r(   Ú	batch_idxrN   r·  r^  r)   rÞ   r+   )r,   s   "r-   r.   r�  Ž  s)   € ÷ Fñ FØðFØ$(ðFà	ñFr/   c                	ó¦  € \        VP                  R 4      ;'       g    ^ 4      pV P                  R8w  g   V^ 8:  g   VP                  R4      '       d   R# \        VP                  R^ 4      ;'       g    ^ 4      pW48:  d   RVR&   R# V P                  e$   W4,
          pWPP                  V,          8¼  d   R# V# V P
                  V,          V,           # )r/  Úexactr0  Nr“  T)rN   rS   r§  r›  rœ  rž  )ry   r¾  r·  r/  r“  Úsuffix_checkpoints   &&&   r-   Ú_apc_checkpoint_column_for_metaÚ5PromptProcessingBatch._apc_checkpoint_column_for_metaŽ  s½   € ô ˜TŸX™XÐ&6Ó7×<Ð<¸1Ó=ˆà�N‰N˜gÔ%Ø Ô"Ø�x‰xÐ)×*Ò*áÜ˜Ÿ™ ,°Ó2×7Ð7°aÓ8ˆ
ØÔ'Ø&*ˆDÐ"Ñ#ÙØ×"Ñ"Ò.Ø .Õ ;ÐØ ×$5Ñ$5°iÕ$@Ô@ÙØ$Ð$Ø×)Ñ)¨)Õ4°~ÕEÐEr/   c               ó   € V ^8„  d   QhRR/# )r(   r)   rÞ   r+   )r,   s   "r-   r.   r�  £  s   € ÷ ñ ¨]ñ r/   c                	óš  € V P                   e2   V P                  R8w  g!   V P                  '       d   V P                  f   R # V P                  pWP                  P
                  ^,          ,           pR p\        V P                  4       F?  w  rEVf   K  V P                  WE4      pVe   Wa8:  g   Wb8¼  d   K.  Vf   TM
\        W64      pKA  	  V# )NrÀ  )	r¦  r§  r¥  r¢  r   r†   r{  rÂ  r"  )ry   ÚstartÚendÚnext_colr¾  r·  Úcols   &      r-   rº  Ú1PromptProcessingBatch._next_apc_checkpoint_column£  s°   € à×ÑÒ%Ø�~‰~ Ô(Ø—>—>�>Ø×"Ñ"Ò*áØ×.Ñ.ˆØ×)Ñ)×/Ñ/°Õ2Õ2ˆØ"&ˆÜ(¨¯©Ö8‰OˆIØŠ|ÙØ×6Ñ6°yÓGˆCØŠ{˜cœl¨c¬jÙØ&Ò.‘s´C¸Ó4FŠHñ  9ð ˆr/   c               ó    € V ^8„  d   QhRRRR/# )r(   r¾  rN   r)   r+   )r,   s   "r-   r.   r�  ·  s   € ÷ Qñ Q°Cð Q¸Cñ Qr/   c                	ó¤  € V P                   V,          p\        VP                  R ^ 4      ;'       g    ^ 4      pV P                  e:   \	        V P
                  V,          \        ^ V P                  4      4      pW4,           # V P                  V P                  V,          ,
          pV\	        V P
                  V,          \        ^ V4      4      ,           # )r“  )	r¥  rN   rS   r›  r"  rœ  rU   r   rž  )ry   r¾  r·  r“  Úsuffix_doneÚ	real_dones   &&    r-   Ú_row_real_tokens_processedÚ0PromptProcessingBatch._row_real_tokens_processed·  s°   € Ø�~‰~˜iÕ(ˆÜ˜Ÿ™ ,°Ó2×7Ð7°aÓ8ˆ
Ø×"Ñ"Ò.ÜØ×!Ñ! )Õ,Ü�A�t×5Ñ5Ó6óˆKð Õ+Ð+à×*Ñ*¨T×-GÑ-GÈ	Õ-RÕRð 	ð œC × 1Ñ 1°)Õ <¼cÀ!ÀYÓ>OÓPÕPÐPr/   c               ó    € V ^8„  d   QhRRRR/# )r(   r¾  rN   r)   rß   r+   )r,   s   "r-   r.   r�  Å  s   € ÷ Rñ R°Sð RÐ=Pñ Rr/   c                	óº   € V^ 8X  d5   \        V P                  4      ^8X  d   V P                  f   V P                  # \        P
                  ! V P                  V4      # ©r   )r‡   rÒ  r›  r7   Ú_apcÚextract_prompt_cache_from_batch)ry   r¾  s   &&r-   Ú_apc_prompt_cache_for_storeÚ1PromptProcessingBatch._apc_prompt_cache_for_storeÅ  sJ   € ð ˜Œ>œc $§)¡)›n°Ô1°d×6MÑ6MÒ6UØ×$Ñ$Ð$Ü×3Ò3°D×4EÑ4EÀyÓQÐQr/   c               ó   € V ^8„  d   QhRR/# ©r(   r)   rn  r+   )r,   s   "r-   r.   r�  Ì  s   € ÷ +ñ +¨dñ +r/   c           	     	óð  € V P                   e   V P                  R8w  d   R # \        V P                  4       F»  w  rVe   VP	                  R4      '       d   K"  \        VP	                  R4      ;'       g    ^ 4      pV^ 8:  d   KO  V P                  V4      V8w  d   Kg  V P                  V4      pVf   K~  V P                   P                  VR,          R V VVP	                  R^ 4      R7       RVR&   K½  	  R # )NrÀ  r0  r/  r’  Ú
extra_hash©rÛ  T)	r¦  r§  r{  r¥  rS   rN   rÏ  rÖ  Ústore_exact_cache)ry   r¾  r·  r/  r7   s   &    r-   Ú_store_apc_exact_checkpointsÚ2PromptProcessingBatch._store_apc_exact_checkpointsÌ  så   € Ø×ÑÒ$¨¯©¸'Ô(AÙÜ(¨¯©Ö8‰OˆIØŠ|˜tŸx™xÐ(9×:Ò:ÙÜ  §¡Ð*:Ó!;×!@Ð!@¸qÓAˆNØ Ô"ÙØ×.Ñ.¨yÓ9¸^ÔKÙØ×;Ñ;¸IÓFˆLØÒ#ÙØ×Ñ×/Ñ/ØÐ%Õ& ¨Ð7ØØŸ8™8 L°!Ó4ð 0ô ð
 '+ˆDÐ"Ó#ó!  9r/   c               ó    € V ^8„  d   QhRRRR/# )r(   r6  rÞ   r)   r^  r+   )r,   s   "r-   r.   r�  á  s   € ÷ ñ ¨ð À$ñ r/   c                	óÆ   € Ve   V P                   '       g   V P                  # \        V P                  4      pV P                    F  pW#,          RR V1R3,          W#&   K  	  V# )Nrò   .)r¤  r£  r^  )ry   r6  Úoutr,  s   &&  r-   Ú_prompt_kwargs_for_stepÚ-PromptProcessingBatch._prompt_kwargs_for_stepá  s[   € ØŠ9˜D×:×:Ð:Ø×&Ñ&Ð&Ü�4×&Ñ&Ó'ˆØ×/Ô/ˆAØ•V˜A˜r ˜r 3˜JÕ'ˆC‹Fñ 0àˆ
r/   c               ó   € V ^8„  d   QhRR/# rM   r+   )r,   s   "r-   r.   r�  é  s   € ÷ ñ ˜Sñ r/   c           
     ó  € V P                  4       '       g   ^ # V P                  ;'       g    V P                  P                  ^,          p\	        WP                  P                  ^,          ^,
          4      pV P                  4       pVe   \	        W#V P                  ,
          4      pV^ 8:  d   ^ # V P                  V4      pV P                  ! V P                  RRV13,          3RV P                  RV P                  RRV13,          RV/VB  \        P                  ! V P                   Uu. uF  qUP                  NK  	  up4       V ;P                  V,          un        V P                  4        V P                  RVR13,          V n        V P                  RVR13,          V n	        V P                   F/  pV P                   V,          RVR1R3,          V P                   V&   K1  	  \        P"                  ! 4        V# u upi )z:Process one chunk of the prompt. Returns tokens processed.Nrò   r   r6   r  .)r»  rÖ   r¢  r†   r"  rº  r   rã  rI   r�  r7   ri   r#  r$  rÞ  r¤  r£  r%  )ry   Ústepr6  Úcheckpoint_colr]  r5  r,  s   &      r-   Úprompt_stepÚ!PromptProcessingBatch.prompt_stepé  s¹  € à×$Ñ$×&Ò&Ùà×%Ñ%×EÐE¨×)<Ñ)<×)BÑ)BÀ1Õ)EˆÜ�×)Ñ)×/Ñ/°Õ2°QÕ6Ó7ˆØ×9Ñ9Ó;ˆØÒ%Ü�A¨×(FÑ(FÕFÓGˆAØ�Œ6ÙØ×4Ñ4°QÓ7ˆØ�
Š
Ø�O‰O˜A˜r ˜r˜EÕ"ñ	
à×#Ñ#ð	
ð ×-Ñ-¨a°°!°¨eÕ4ð	
ð ð		
ð
 ò	
ô 	�Š $×"3Ò"3Ó4Ñ"3˜Q—”Ñ"3Ñ4Ô5Ø×&Ò&¨!Õ+Õ&Ø×)Ñ)Ô+Ø"×1Ñ1°!°Q±R°%Õ8ˆÔØŸ/™/¨!¨Q©R¨%Õ0ˆŒØ×/Ô/ˆAØ%)×%8Ñ%8¸Õ%;¸A¸q¹rÀ3¸JÕ%GˆD×Ñ Ó"ñ 0ä
�ŠÔØˆùò 5s   Ä,G>c               ó    € V ^8„  d   QhRRRR/# ©r(   Ú	elapsed_sru   r)   rn  r+   )r,   s   "r-   r.   r�    s   € ÷ :ñ :¨Eð :°dñ :r/   c                	ó^   € V ;P                   \        R \        V4      4      ,          un         R# )r$   N)r©  rU   ru   )ry   rí  s   &&r-   Úrecord_prompt_timeÚ(PromptProcessingBatch.record_prompt_time  s   € Ø×Òœs 3¬¨iÓ(8Ó9Õ9×r/   c               ó   € V ^8„  d   QhRR/# ©r(   r)   zList[PromptProgress]r+   )r,   s   "r-   r.   r�  
  s   € ÷ 
ñ 
Ð!5ñ 
r/   c                	ó
  € V P                   ^ 8:  d   . # \        V P                  V P                  V P                  4       UUUu. uF0  w  rp\        VVW P                   ,          V P                   VR7      NK2  	  uppp# u upppi )r   )rÀ  r?  rA  r¹  rÁ  )r©  rk   rš  rª  r«  r¿  )ry   rÀ  r?  rÁ  s   &   r-   Úprompt_progressÚ%PromptProcessingBatch.prompt_progress
  sˆ   € Ø×Ñ !Ô#ØˆIô 69Ø×!Ñ!Ø×+Ñ+Ø×+Ñ+ô6õ
ñ6Ñ1� Mô ØØ+Ø(×+>Ñ+>Õ>Ø ×/Ñ/Ø+÷ñ6ó
ð 	
ùô 
s   Á6A>c               ó   € V ^8„  d   QhRR/# )r(   r)   rÈ  r+   )r,   s   "r-   r.   r�    s   € ÷ cñ cà	ñcr/   c                óZ  € \        V P                  4      pV P                  e>   V P                  e0   VP	                  \        V P                  V P                  4      4       V P                  ! V P                  3RV P                  RV P                  /VB p\        VR4      '       d   VP                  MTpV P                  eæ   \        V P                  4      '       dË   VP                  ^,          p\        P                   ! V P                   U	u. uF  q˜^,
          V	,
          NK  	  up	\        P"                  R7      R ,          p
\        P$                  ! W§P                  ^ ,          ^VP                  R!,          34      p
\        P&                  ! Wz^R7      P)                  ^4      pM	VR",          pV P*                  '       dÉ   \        V P*                  4      '       d®   . p\-        VP                  ^ ,          4       Fs  pW|V^,            pV P*                  V,          ;'       g    . pV F1  pV! \        P                   ! V P.                  V,          4      V4      pK3  	  VP1                  V4       Ku  	  \        P2                  ! V^ R7      pV\        P4                  ! VR!RR7      ,
          p\7        VV^ .\9        V P:                  4      ,          ^ .\9        V P:                  4      ,          R	7      p\        P<                  ! V4       V P                  ev   \        V P                  4      '       d[   V P                   FJ  p\?        VR
R4      p\A        V4      '       g#   RV n!        V PE                  4        \G        R4      hV! 4        KL  	  \H        PK                  \L        PN                  4      '       dè   \P        PR                  PU                  R4      '       dÃ   V P                  '       d   V P                  ^ ,          MRpVe˜   \?        VRR4      p\?        VRR4      p\H        PW                  R\?        VRR4      \        VR4      '       d   VPY                  4       MT\        VR4      '       d   VPY                  4       MTV P                  V PZ                  4       V P                  eÝ   V P                  eÏ   \]        V P                  V P                  V P                  \_        V P:                  4      TV P                  TT\_        V P`                  4      \c        V P                  V4      V P                  R8X  d   VPd                  MRV P                  V Pf                  V P                  Ph                  V Pj                  R7      pRpM¤\m        V P                  \_        V P:                  4      TV P                  TT\_        V P`                  4      TV Pj                  V P.                   Uu. uF  p\_        V4      NK  	  up\_        V P*                  4      \_        V Pn                  4      R7      pVVn8        V'       dL   \s        V\l        4      '       d6   V\        Pt                  ! VP                  ^ ,          4      V3,          Vn;        V^ 8”  d�   \s        V\l        4      '       dy   Tp\        Px                  ! VR!R7      pVRV) R13,          RRRR!13,          P{                  \        P"                  4      p\        P&                  ! VVR!R7      pVVn>        VVn?        \?        V P                  RV P                  4      pV P�                  V\9        VP:                  4      4      pVEe1   VP‚                  ^ 8X  d   VP…                  ^^4      pMVP‚                  ^8X  d
   VR#,          pVP                  ^ ,          pVP                  ^ ,          V8w  dÁ   VP                  ^ ,          ^8X  d,   \        P$                  ! VVVP                  ^,          34      pM}VP                  ^ ,          V8  da   VVP                  ^ ,          ,
          p \        P2                  ! V\        P$                  ! VR!R V VP                  ^,          34      .^ R7      pMVRV pVVnC        . p!V P                   F  p V!P1                  VPˆ                  4       K!  	  V.p"V!'       d   V"P1                  V!4       V'       d2   \s        V\l        4      '       d   V"P1                  VPv                  4       V^ 8”  d>   \s        V\l        4      '       d(   V"P�                  VP|                  VP~                  .4       Ve   V"P1                  V4       \        P�                  ! V"!   V P’                  Ee`   V P”                  '       EdM   V PB                  '       Ed:    \—        V P”                  4       EF  w  p#p$V$f   K  V P˜                  R8X  dx   V P›                  V#4      p%V%e6   V P’                  P�                  V$R,          V%V$PU                  R^ 4      R7       V P’                  PŸ                  V$PU                  R. 4      4       K•  \         P¢                  ! V P’                  V P                  V#V$R,          V$PU                  R^ 4      V$PU                  R^ 4      R7      p&V P’                  PŸ                  V$PU                  R. 4      V&,           4       EK!  	  . V n        . V n        . V n        . V n        . V nJ        V# u up	i u upi   \Š        \Œ        3 d     EK—  i ; i  \¤         de   p'\H        PW                  RT'4       T P”                   F4  p$T$f   K	  T P’                  PŸ                  T$PU                  R. 4      4       K6  	   Rp'?'L²Rp'?'ii ; i)$z7Process final tokens and transition to GenerationBatch.Nr   r6   rô   rå   r–   Trì   rî   ÚfinalizeFz9APC mixed prefill requires a prompt cache with finalize()Ú	APC_DEBUGÚoffsetrŒ  z\post-prefill cache[0]: _idx=%s offset=%s left_padding=%s right_pad_per_row=%s suffix_lens=%sÚ_idxr  Úmtp)rI   r8   r9   rÒ  rJ  r7   rÔ   rÛ  rÅ   r  rK  r?  rØ   rH  rÕ  )rI   rÒ  r  r7   rÔ   rÛ  rÅ   rÔ  rÕ  rÖ  rÕ   ræ   .r=   rÀ  r’  rÛ  rÜ  rµ  r“  )rÛ  Úskip_first_n_tokensz-APC harvest failed during batched prefill: %s)rò   NNr˜   rñ   r   )Sr^  r£  r8   r9   r  r   rI   r�  r7   r¢  r  rô   r›  rH   r†   ri   r  r  Úbroadcast_torŸ   rû   rÕ   rú   r¡  rF   ro  r÷   rø   r‡   rÒ  r&  r2   rG   r¨  r¬  r'  rW   ÚisEnabledForÚloggingÚDEBUGrQ   rR   rS   rX   r  rœ  rG  rù   rÅ   r   rK  rØ   r™   rÕ  rÈ  ræ   rÝ  r¿   r  râ  rž   r›   rã  rä  Ú_capture_rope_deltasr`  Úreshaperé   r$  r   r!  r  r#  r¦  r¥  r{  r§  rÖ  rÝ  r¶  rÔ  Úharvest_blocks_from_batch_cacheÚ	Exception)(ry   rÔ   rÛ  rÝ  rÔ  Úcall_kwargsr  rô   ÚseqrK  Úlast_idxr  rc  r  r  rÿ   r}   rJ  r5  rø  Úc0Úoffrš  Ú	gen_batchrå  r,  r  r  r  r=   r	  Útarget_brp  rñ  r  r¾  r·  r7   Ú
new_blocksÚes(   &&&&&                                   r-   ÚgenerateÚPromptProcessingBatch.generate  st	  € ô ˜4×.Ñ.Ó/ˆØ×ÑÒ'¨D¯O©OÒ,GØ×ÑÜ*¨4¯?©?¸D×<LÑ<LÓMôð —’Ø�O‰Oñ
à×#Ñ#ð
ð ×-Ñ-ð
ð ñ	
ˆô #*¨&°(×";Ò";�—’ÀˆØ×"Ñ"Ò.´3°t×7NÑ7N×3OÒ3Oà—,‘,˜q•/ˆCÜ—x’xØ&*×&=Ò&=Ó>Ñ&= �q•˜1—�Ñ&=Ñ>ÄbÇhÁhôàõˆHô —’ x·,±,¸qµ/À1ÀfÇlÁlÐSUÕFVÐ1WÓXˆHÜ×'Ò'¨¸qÔA×IÑIÈ!ÓL‰Fà˜HÕ%ˆFØ×!×!Ð!¤c¨$×*@Ñ*@×&AÒ&AØ!ÐÜ˜6Ÿ<™<¨�?Ö+�Ø &¨1¨q­5Ð 1�Ø!×3Ñ3°AÕ6×<Ð<¸"�
Û!+�IÙ$-ÜŸš ×!4Ñ!4°QÕ!7Ó8¸-ó%’Mñ ",ð !×'Ñ'¨Ö6ñ ,ô —^’^Ð$4¸1Ô=ˆFàœBŸLšL¨°bÀ4ÔHÕHˆÜ-ØØØ�Cœ#˜dŸi™i›.Õ(Ø�cœC §	¡	›NÕ*ô	
ˆô 	�Š�lÔ#ð ×"Ñ"Ò.´3°t×7NÑ7N×3OÒ3OØ×&Ô&�Ü" 1 j°$Ó7�Ü ×)Ò)Ø05�DÔ-Ø×1Ñ1Ô3Ü&ØSóð ñ –
ñ 'ô ×ÑœwŸ}™}×-Ò-´"·*±*·.±.À×2MÒ2MØ)-×):×):Ð):�×"Ñ" 1Ö%ÀˆBØŠ~Ü˜b (¨DÓ1�Ü˜R °Ó6�Ü—‘ØrÜ˜B ¨Ó-Ü$+¨C°×$:Ò$:�C—J‘J”LÀÜ#*¨2¨x×#8Ò#8�B—I‘I”K¸bØ×+Ñ+Ø×%Ñ%ôð ×ÑÒ'¨D¯O©OÒ,GÜ2Ø—j‘jØ ×,Ñ,ØŸ?™?Ü˜$Ÿ)™)“_Ø)Ø!×.Ñ.ØØ+Ü §¡Ó0Ü/°·±ÀÓHà/3¯©À%Ô/G�F×+Ò+ÈTà"Ÿo™oØ!%×!6Ñ!6Ø ŸO™O×1Ñ1Ø $× 4Ñ 4ô#ˆIð&  %Ñä'Ø—j‘jÜ˜$Ÿ)™)“_Ø#Ø!×.Ñ.ØØ+Ü §¡Ó0Ø-Ø $× 4Ñ 4Ø48×4GÒ4GÓHÑ4G¨Sœt CžyÑ4GÑHÜ"& t×'=Ñ'=Ó">Ü)-¨d×.KÑ.KÓ)LôˆIð &6ˆ	Ô"ç¤
¨9´o× FÒ FØ"*Ü—	’	˜,×,Ñ,¨QÕ/Ó0°,Ð>õ#ˆIÔð
 ˜AÔ¤*¨Y¼×"HÒ"HØˆAÜ—z’z (°Ô4ˆHØ˜s Q B¡C˜xÕ(¨©d°¨d¨Õ3×:Ñ:¼2¿8¹8ÓDˆGÜ×'Ò'¨°'ÀÔCˆFØ&-ˆIÔ#Ø%+ˆIÔ"ä  §¡Ð-=¸t¿z¹zÓJˆØ×/Ñ/°ÄÀIÇNÁNÓ@SÓTˆØÓ"à×Ñ 1Ô$Ø)×1Ñ1°!°QÓ7‘Ø×!Ñ! QÔ&Ø)¨'Õ2�ð $×)Ñ)¨!Õ,ˆHØ× Ñ  Õ# xÔ/Ø×$Ñ$ QÕ'¨1Ô,Ü"$§/¢/Ø# h°×0AÑ0AÀ!Õ0DÐ%Eó#‘Kð !×&Ñ& qÕ)¨HÔ4Ø" [×%6Ñ%6°qÕ%9Õ9�CÜ"$§.¢.à'ÜŸOšOØ +¨B¨CÐ 0Ø!$ k×&7Ñ&7¸Õ&:Ð ;óðð ô	#‘Kð #.¨i¨xÐ"8�KØ%0ˆIÔ"ð
 ˆØ×"Ô"ˆAðØ×#Ñ# A§G¡GÖ,ñ #ð
 %�~ˆßØ×Ñ Ô-ß¤
¨9´o× FÒ FØ×Ñ 	× 3Ñ 3Ô4Ø˜AÔ¤*¨Y¼×"HÒ"HØ×Ñ ×!8Ñ!8¸)×:PÑ:PÐ QÔRØÒ"Ø×Ñ Ô,Ü
�Š�Òð ×ÑÓ)Ø——‘Ø×)×)Ñ)ðNÜ'0°·±×'@‘O�I˜tØ’|Ù Ø—~‘~¨Ô0Ø'+×'GÑ'GÈ	Ó'R˜Ø'Ò3Ø ×-Ñ-×?Ñ?Ø $Ð%5Õ 6Ø ,Ø+/¯8©8°LÀ!Ó+Dð @ô ð
 ×)Ñ)×1Ñ1°$·(±(¸<ÈÓ2LÖMä%)×%IÒ%IØ ×-Ñ-Ø ×-Ñ-Ø%Ø Ð!1Õ2Ø'+§x¡x°¸aÓ'@Ø04·±¸ÀqÓ0Iô&˜
ð ×)Ñ)×1Ñ1Ø ŸH™H \°2Ó6¸ÕC÷ñ+ (Að> ˆŒ	ØˆÔØ ˆÔØ!#ˆÔØˆŒØÐùò] ?ùòl Iøô| #¤IÐ.ô Ûðûô\ ô NÜ—‘ÐNÐPQÔRà ŸNœN�DØÔ'Ø×)Ñ)×1Ñ1°$·(±(¸<ÈÓ2LÖMö +ûðNús=   Äl×$lâ$l"æ;D8l; ì"l8ì7l8ì;n*í&n%í1/n%î%n*c                	ó   € V P                   # rh   )rŸ  ré  s   &r-   Útotal_prompt_tokensÚ)PromptProcessingBatch.total_prompt_tokens  s   € à×(Ñ(Ð(r/   c               ó   € V ^8„  d   QhRR/# )r(   ÚBrN   r+   )r,   s   "r-   r.   r�    s   € ÷ ñ °ñ r/   c           	     	ó°  € \        V R 4      '       g   R# V P                  pVf)   \        P                  ! V^3\        P                  R7      # VP
                  ^ 8X  d   VP                  ^^4      pMVP
                  ^8X  d
   VR,          pVP                  ^ ,          ^8X  d    V^8”  d   \        P                  ! W!^34      pVP                  ^ ,          V8w  d}   VP                  ^ ,          V8”  d   VRV pV# WP                  ^ ,          ,
          p\        P                  ! V\        P                  ! VRR W2P                  ^,          34      .^ R7      pV# )ré   Nrå   r–   r   r˜   )
r  ré   ri   rn  r  r`  r  r†   rþ  ro  )r=   r  r	  rp  s   &&  r-   r  Ú*PromptProcessingBatch._capture_rope_deltas  s4  € ä�~ ~×6Ò6ÙØ$×1Ñ1ˆØÒÜ—8’8˜Q ˜F¬"¯(©(Ô3Ð3Ø×Ñ˜qÔ Ø%×-Ñ-¨a°Ó3‰KØ×Ñ Ô"Ø% gÕ.ˆKà×Ñ˜QÕ 1Ô$¨¨Q¬ÜŸ/š/¨+¸1°vÓ>ˆKØ×Ñ˜QÕ 1Ô$Ø× Ñ  Õ# aÔ'Ø)¨"¨1˜o�ð Ðð ×+Ñ+¨AÕ.Õ.�Ü Ÿnšnà#ÜŸš¨°B°CÐ(8¸3×@QÑ@QÐRSÕ@TÐ:UÓVðð ô�ð Ðr/   )r¨  r¦  r¥  r§  r«  r�  r¢  rž  r   r£  r¤  r©  rª  rš  r›  rœ  r¡  rŸ  rØ   r9   r8   rÕ  rÕ   rÅ   rI   rÖ   r7   ræ   rÒ  rh   )Tr   )r«   r¬   r­   r®   r¯   ÚDEFAULT_PREFILL_STEP_SIZEr   r   rz   rê  r¬  r»  rÂ  rº  rÏ  rÖ  rÞ  rã  ré  rï  rô  r  Úpropertyr  Ústaticmethodr  r°   r+   r/   r-   r†  r†  Ð  s°   † ñð$ Ø8<Ø+DØØ2Ø6Ø*.Ø)-Ø37Ø15Ø+/Ø"&Ø+/Ø$(Ø*.Ø %÷3a.òFòFòEõFõ*õ(QõRõ+÷*õõ<:õ
÷$cðJ ñ)ó ð)ð ôó ôr/   r†  c            (      óœ  € ] tR tRtRtR]RRRRR]R]R	]R
RRRR]	R]
R]RRR^ RRRRRRRRRRRRRR/R R llt]tR R ltR R ltR  R! ltR" R# ltR$ R% ltR& R' ltR( R) ltR* R+ ltR, R- lt]R. 4       tR/ tR0 tRBR1 R2 lltR3 R4 lt]R5 4       t]R6 4       t]R7 4       t R8 t!]"R9 R: l4       t#]"R; R< l4       t$R= R> lt%R? t&R@ t'RAt(R# )CÚBatchGeneratori"  a  
Continuous batching with separate prompt processing and generation phases.

next() returns (prompt_responses, generation_responses) where:
- prompt_responses contains completed prompt-batch timing stats
- generation_responses is a list of GenerationBatch.Response objects
rÅ   Ústop_tokensNrÔ   Úcompletion_batch_sizeÚprefill_batch_sizerÖ   r7   rÐ   rÑ   rÒ   rÓ   rÝ  TrÔ  rÕ   ró   r‹  r8   r9   rØ   rÕ  Fc          "     ó\   € V ^8„  d   QhRRRRRRRRRRR	R
RRRRRRRRRRRRRRRRRRRR
RR/# )r(   rÅ   rN   r  zOptional[set]rÔ   rà   r  r  rÖ   rÞ   rÑ   rÒ   r³   rÓ   rÝ  r*   rÔ  rÕ   rá   r‹  rŒ  r8   râ   r9   rÍ  rØ   rÕ  r+   )r,   s   "r-   r.   ÚBatchGenerator.__annotate__+  sÜ   € ÷ YKñ YKð
 ðYKð #ðYKð :ðYKð  #ðYKð  ðYKð )ðYKð ðYKð ðYKð  ðYKð  ð!YKð" ð#YKð$
ð%YKð, 1ð-YKð. )ð/YKð0 "ð1YKð2 (ð3YKð4 ñ5YKr/   c          	     	óš  € Wn         W0n        W n        W n        W°n        WÀn        WÐn        Wàn        Wðn        T;'       g    . V n	        VV n
        VV n        VV n        T;'       g    VR J V n        V P                  e   R pRp^ pRV n        ^ V n        R V n        Ve   V
e   R pVe,   \        P                   ! V4      V n        V P                  f   R pVV n        \%        VR4      '       d   VP&                  MTV n        T;'       g    R V n        ^ V n        W€n        Wpn        W`n        T;'       g    \2        V n        V P&                  P6                  P9                  V4       \:        P=                  V P                   V P(                  V P&                  P6                  V P                  V P                  V P                  R7      V n        R V n         . V n!        ^ V n"        ^ V n#        ^ V n$        ^ V n%        \M        4       V n'        \P        PR                  ! 4       V n*        V PT                  PW                  \Y        WP4                  .4      4       R # )NFÚ	tokenizerc                ó2   € \         P                  ! V RR7      # )r‘   r–   r˜   )ri   Úargmax©r`   s   &r-   r–  Ú)BatchGenerator.__init__.<locals>.<lambda>h  s   € ¬R¯YªY°q¸rÕ-Br/   )rÝ  rÔ  rÕ  )-rI   rÅ   rÿ   rÐ   rÑ   rÒ   rÓ   rÝ  rÔ  rÕ   r8   r9   rØ   rÕ  r�  rÔ  Úmodel_apc_moder‹  r  r#  rÔ   Ú	uid_countrÖ   r  r  r    Ú_streamÚstopping_criteriaÚadd_eos_token_idsrÈ  rC  Ú_generation_batchÚ_prompt_batchÚ_unprocessed_sequencesÚ_prompt_tokens_counterÚ_prompt_time_counterÚ_gen_tokens_counterÚ_steps_counterrZ   Ú_cache_eval_intervalÚ
contextlibÚ	ExitStackÚ_wire_stackÚenter_contextr"   )ry   rI   rÿ   rÅ   r  rÔ   r  r  rÖ   r7   rÐ   rÑ   rÒ   rÓ   rÝ  rÔ  rÕ   ró   r‹  r8   r9   rØ   rÕ  s   &&&$$$$$$$$$$$$$$$$$$$$r-   rz   ÚBatchGenerator.__init__+  s  € ð8 Œ
Ø$ŒØ"ŒØŒØ*ÔØ.ÔØ"4ÔØ 0ÔØ,ÔØ!2×!8Ð!8°bˆÔØ&ˆÔØ$ˆŒØ 0ˆÔØ.×AÐA°'¸T°/ˆÔØ×ÑÒ'ØˆKØ$ÐØˆNØ$)ˆDÔ!Ø"#ˆDÔð ˆŒØÒ" wÒ':ØˆKØÒ"Ü ×/Ò/°Ó6ˆDŒMØ�}‰}Ò$Ø"�Ø&ˆÔä#*¨9°k×#BÒ#BˆI×ÒÈ	ð 	Œð ×CÐCÑ#BˆŒØˆŒØ!2ÔØ"4ÔØ%:Ô"à×2Ð2Ô!2ˆŒà�‰×(Ñ(×:Ñ:¸;ÔGä!0×!6Ñ!6Ø�J‰JØ�L‰LØ�N‰N×,Ñ,Ø!×2Ñ2Ø×.Ñ.Ø ×0Ñ0ð "7ó "
ˆÔð ?CˆÔØ&(ˆÔ#à&'ˆÔ#Ø$%ˆÔ!Ø#$ˆÔ ØˆÔÜ$BÓ$DˆÔ!ä%×/Ò/Ó1ˆÔØ×Ñ×&Ñ&¤{°5¿<¹<¸.Ó'IÖJr/   c               ó    € V ^8„  d   QhRRRR/# )r(   r]  r^  r)   rN   r+   )r,   s   "r-   r.   r!  ‹  s   € ÷ 4ñ 4¨Tð 4°cñ 4r/   c                óø   € V P                   f   ^ # Vf   / pVP                  R4      pVf*   VP                  R4      p\        P                  ! VRR7      pVP                  R4      p\        P                  ! WB4      # )zSalt for the APC hash chain.NÚ_apc_image_hashr)  )r)  Ú	image_refÚ_apc_tenant)r‹  rS   rÔ  Úhash_image_payloadÚtenant_scoped_hash)ry   r]  Úimgr)  Útenants   &&   r-   Ú_apc_extra_hashÚBatchGenerator._apc_extra_hash‹  su   € à×ÑÒ#ÙØÒ ØˆMØ×ÑÐ 1Ó2ˆØŠ;Ø(×,Ñ,¨^Ó<ˆLÜ×)Ò)°|ÈtÔTˆCØ×"Ñ" =Ó1ˆÜ×&Ò& vÓ3Ð3r/   c               ó   € V ^8„  d   QhRR/# )r(   r)   zset[int]r+   )r,   s   "r-   r.   r!  ˜  s   € ÷ Hñ H hñ Hr/   c                	óV   € \         P                  ! V P                  P                  4      # rh   )rÔ  Ú multimodal_token_ids_from_configrI   Úconfigré  s   &r-   Ú_apc_media_token_idsÚ#BatchGenerator._apc_media_token_ids˜  s   € Ü×4Ò4°T·Z±Z×5FÑ5FÓGÐGr/   c               ó    € V ^8„  d   QhRRRR/# ©r(   Úids_listrd   r)   rN   r+   )r,   s   "r-   r.   r!  ›  s   € ÷ $ñ $°Ið $À#ñ $r/   c                	óp   € \         P                  ! WP                  4       4      p\        ^ V^,
          4      # rÓ  )rÔ  Úmedia_safe_prefix_minrI  rU   )ry   rM  Úsafe_mins   && r-   Ú_apc_safe_prefix_lookup_minÚ*BatchGenerator._apc_safe_prefix_lookup_min›  s,   € Ü×-Ò-¨h×8QÑ8QÓ8SÓTˆÜ�1�h •lÓ#Ð#r/   c               ó$   € V ^8„  d   QhRRRRRR/# ©r(   rM  rd   r“  rN   r)   r*   r+   )r,   s   "r-   r.   r!  Ÿ  s"   € ÷ 
ñ 
°ð 
Èð 
ÐPTñ 
r/   c                	óN   € \         P                  ! VVV P                  4       4      # rh   )rÔ  Úprefix_leaves_text_only_suffixrI  ©ry   rM  r“  s   &&&r-   Ú_apc_suffix_is_text_onlyÚ'BatchGenerator._apc_suffix_is_text_onlyŸ  s'   € Ü×2Ò2ØØØ×%Ñ%Ó'ó
ð 	
r/   c               ó$   € V ^8„  d   QhRRRRRR/# rT  r+   )r,   s   "r-   r.   r!  ¦  s$   € ÷ 
ñ 
Ø!ð
Ø/2ð
à	ñ
r/   c                	óN   € \         P                  ! VVV P                  4       4      # rh   )rÔ  Úprefix_contains_media_tokensrI  rW  s   &&&r-   Ú_apc_prefix_has_media_tokensÚ+BatchGenerator._apc_prefix_has_media_tokens¦  s)   € ô ×0Ò0ØØØ×%Ñ%Ó'ó
ð 	
r/   c               ó    € V ^8„  d   QhRRRR/# rL  r+   )r,   s   "r-   r.   r!  ¯  s   € ÷ 
ñ 
°)ð 
Àñ 
r/   c                	óþ   € V P                   e   \        V RR4      R8w  d   ^ # \        P                  ! V\	        V4      V P                   P
                  ,
          V P                  4       \	        V4      ^,
          R7      # )Nr�  ÚblockrÀ  )Úmax_prefix_tokens)r‹  r2   rÔ  Ú%adjust_prefix_to_text_suffix_boundaryr‡   Úexact_cache_guard_tokensrI  )ry   rM  s   &&r-   Ú_apc_exact_checkpoint_lenÚ(BatchGenerator._apc_exact_checkpoint_len¯  sg   € Ø×ÑÒ#¤w¨t°ZÀÓ'IÈWÔ'TÙÜ×9Ò9ØÜ�‹M˜D×,Ñ,×EÑEÕEØ×%Ñ%Ó'Ü! (›m¨aÕ/ô	
ð 	
r/   c               ó   € V ^8„  d   QhRR/# )r(   r)   rL  r+   )r,   s   "r-   r.   r!  ¹  s   € ÷ _ñ _¨ñ _r/   c                óJ  € V P                   f   R# Vw  r#rErgV'       d   \        V4      ^8  d   R# V P                  V4      pT P                  T;'       g    / 4      p	\	        V RR4      p
V
R8X  dl   V P                   P                  VV	VR7      w  r¼VeF   V^ 8”  d?   V\        V4      8  d/   V P                  W<4      '       g   R# R. RVRVR	V	R
\        V4      /# R# V P                   P                  W9R7      w  rÞV^ 8”  d7   V P                  W>4      '       d    V P                   P                  V4       . p^ pRp^ pV\        V4      8  d*   V P                   P                  VV	\        Wè4      R7      w  r¼Rp^ p\        Wì4      \        V4      8  d9   V P                   P                  VV	\        WìV4      \        Wì4      ^ 8„  R7      w  ppV\        Wì4      8”  dc   V\        V4      8  dS   V'       d   V P                   P                  V4       V P                  VV4      '       g   R# R. RVRVR	V	R
\        V4      /# WÎ8”  db   V\        V4      8  dR   V'       d   V P                   P                  V4       V P                  W<4      '       g   R# R. RVRVR	V	R
\        V4      /# V^ 8”  dX   V\        V4      8  dH   V P                  W>4      '       g   V P                   P                  V4       R# RVRVR	V	R
\        V4      /# V'       d   V P                   P                  V4       R# )z‚Look up an APC prefix for ``sequence``. Returns dict with matched
blocks + suffix metadata when there is a usable hit, else None.
Nr�  ra  rÀ  )rÛ  Úmin_prefix_tokensÚmatched_blocksrˆ  r“  rÛ  r’  rÜ  )rÛ  ri  Úallow_memory_overlap)r‹  r‡   rQ  rC  r2   Úlookup_exact_cacherX  rù   Úlookup_prefixr]  r¶  rU   Úlookup_prefix_disk_cache)ry   ÚsequencerÀ  rM  Úmax_toksr]  Úlpsr:  Úsafe_lookup_minrÛ  r�  Úexact_cacheÚexact_prefix_lenÚmatchedr“  rˆ  Údisk_prefix_lens   &&               r-   Ú_apc_pick_forÚBatchGenerator._apc_pick_for¹  s?  € ð ×ÑÒ#ÙØ@HÑ=ˆ�x°ßœ3˜x›=¨1Ô,ÙØ×:Ñ:¸8ÓDˆØ×)Ñ)¨-×*=Ð*=¸2Ó>ˆ
Ü˜4 ¨WÓ5ˆØ�wÔØ,0×,<Ñ,<×,OÑ,OØØ%Ø"1ð -Pó -Ñ)ˆKð Ò'Ø$ qÔ(Ø$¤s¨8£}Ô4à×4Ñ4°X×PÒPÙà$ bØ  +Ø Ð"2Ø  *Ø$¤d¨8£nðð ñ Ø"×.Ñ.×<Ñ<Øð =ó 
Ñˆð ˜Œ>˜d×?Ñ?À×UÒUØ×Ñ×$Ñ$ WÔ-ØˆGØˆJØˆØÐØœ˜H›Ô%Ø,0×,<Ñ,<×,OÑ,OØØ%Ü"% jÓ"Bð -Pó -Ñ)ˆKð
 ˆ
ØˆÜˆzÓ,¬s°8«}Ô<Ø*.×*:Ñ*:×*SÑ*SØØ%Ü"% jÀOÓ"TÜ%(¨Ó%FÈÑ%Jð	 +Tó +Ñ'ˆJ˜ð œSØó
ô 
à¤ H£Ô-ßØ× Ñ ×(Ñ(¨Ô1Ø×0Ñ0°¸?×KÒKÙà  "Ø˜jØ˜oØ˜jØ ¤$ x£.ðð ð Ô(Ð-=ÄÀHÃÔ-MßØ× Ñ ×(Ñ(¨Ô1Ø×0Ñ0°×LÒLÙà  "Ø˜kØÐ.Ø˜jØ ¤$ x£.ðð ð ˜Œ>˜j¬3¨x«=Ô8Ø×0Ñ0°×FÒFØ× Ñ ×(Ñ(¨Ô1Ùà  'Ø˜jØ˜jØ ¤$ x£.ð	ð ÷ Ø×Ñ×$Ñ$ WÔ-Ùr/   c               ó    € V ^8„  d   QhRRRR/# )r(   Ú	sequenceszList[tuple]r)   z!Optional['PromptProcessingBatch']r+   )r,   s   "r-   r.   r!  	  s    € ÷ U
ñ U
Ø$ðU
à	*ñU
r/   c                ó  € V P                   f   R# V Uu. uF  q P                  V4      NK  	  pp\        ;QJ d    R V 4       F  '       g   K   RM	  RM! R V 4       4      pV'       g   R# V Uu. uF  q"^ ,          NK  	  ppV Uu. uF  p\        V^,          4      NK  	  ppV Uu. uF  q"^,          NK  	  ppV Uu. uF  q"^,          NK  	  ppV Uu. uF  q"^,          NK  	  p	pV Uu. uF  q"^,          NK  	  p
pV Uu. uF  q»'       d
   VR,          M^ NK  	  pp\	        \        V4      4       Uu. uF  qÖV,          WÍ,          R NK  	  ppV Uu. uF  p\        V4      NK  	  pp\        V4      pV Uu. uF  pVV,
          NK  	  pp. p\        V4       FÇ  w  ppVe   VP                  R4      f   \        R4      hVR,          pVRWÍ,          R1R3,          pVV,          pV^ 8”  db   \        P                  ! VP                  ^ ,          VVP                  R,,          3VP                  R	7      p\        P                  ! VV.^R
7      pVP                  V4       KÉ  	  \        P                  ! V^ R
7      p/ p/ p\        V4      p\        V4       EF  w  ppV'       g   K  \        Wm,          4      pWÍ,          pVV,          pVP!                  4        FÖ  w  pp VR8X  g   VV P"                  9   d   K   \%        V \        P&                  4      '       d‰   V P(                  ^ 8”  dx   V P                  ^ ,          ^8¼  d`   \+        V VV4      p!\-        VV!V4      '       d   V!RVR1R3,          p!\/        V!VRR7      p!VP1                  V. 4      P                  V!4       KÈ  VV9  g   KÑ  V VV&   KØ  	  EK   	  VP!                  4        F!  w  pp"\        P                  ! V"^ R
7      VV&   K#  	  \3        V RR4      p#V#R8X  dX   V Uu. uF*  pVe
   VR,          MV P4                  P7                  4       NK,  	  p$p\8        P:                  ! V$V4      w  p%p&V%f   R# My\=        V P4                  R4      '       d$   \        V P4                  P7                  4       4      M\        V P4                  P>                  4      p'\8        P@                  ! VV'R7      w  p%p&\	        \        V4      4       Uu. uF’  pRWm,          RWÍ,          RW=,          '       d   W=,          R,          M T PC                  W�,          ;'       g    / 4      RW=,          '       d   W=,          P                  R. 4      M. RV PE                  Wm,          4      /NK”  	  p(p\G        R\H        4      p)V)! R-/ RV P4                  bRVbRVbRVbRVbRVbRV	bRV
bR V PJ                  bR!V PL                  bR"V PN                  bR#V PP                  bRV%bR$V(bR%V P                   bR&VbR'VbRV#bR(\3        V R(R4      bR)\3        V R)R4      bR*\3        V R*R4      bR+\3        V R+R4      b # u upi u upi u upi u upi u upi u upi u upi u upi u upi u upi u upi u upi u upi ).a˜  Build a multi-row PromptProcessingBatch admitting ``sequences``.

Each row is independently looked up in APC. Warm rows have their
suffixes prefilled against pre-populated K/V; cold rows prefill from
scratch in the same batch. Right-padding aligns RoPE positions
across rows with different prefix/suffix lengths.

Returns ``None`` if APC is disabled (in which case the caller should
use the cold-only path).
Nc              3  ó(   "  € T F  qR Jx € K
  	  R # 5irh   r+   rJ  s   & r-   rD   Ú;BatchGenerator._build_mixed_prompt_batch.<locals>.<genexpr>+	  s   é € Ð4©e¨ •}«eùs   ‚TFr“  r6   z4APC mixed prefill requires precomputed inputs_embedsrò   rå   r–   .rz  r�  ra  rÀ  rˆ  r³  )Ú
num_layersr’  rÛ  rµ  rj  r/  r†  rI   rÒ  r5   rÅ   r]  rÕ   ræ   rÖ   rÐ   rÑ   rÒ   r‰  r‹  r�  rŽ  r8   r9   rØ   rÕ  r˜   r+   ))r‹  rw  rH   rù   rú   r‡   rU   r{  rS   rV   ri   rn  r†   r™   ro  rF   r!  Ú_APC_PRIVATE_KEYSr¿   r  r`  r[  ri  rt  r}  r2   rI   r³  rÔ  Ú!make_warm_batch_exact_cache_multir  r´  Úmake_warm_batch_kv_cache_multirC  re  r¹   r†  rÖ   rÐ   rÑ   rÒ   )*ry   rz  ÚsÚpicksÚany_warmrÒ  Úfull_idsÚmax_tokens_listrv  rÕ   ræ   rK  Úprefix_lensrc  Úsuffix_ids_listrŽ  Úmax_suffix_lenr�  Úsuffix_embeds_per_rowrƒ  ÚfullÚsuffrp  Úpad_embr6   r†  r‡  rY  Úfull_lenr“  Ú	right_padr,  r-  rˆ  r‰  r�  Ú
row_cachesrˆ  ra  r~  r‰  Úprompt_batch_clss*   &&                                        r-   Ú_build_mixed_prompt_batchÚ(BatchGenerator._build_mixed_prompt_batch	  s~  € ð ×ÑÒ#ÙáFOÓ&PÁiÀ×'9Ñ'9¸!Ö'<ÁiˆÐ&Pß“3Ñ4©eÓ4—3—3’3Ñ4©eÓ4Ó4ˆßÙá'Ó(™i˜�!—�™iˆÐ(Ù(1Ó2©	 1”D˜˜1�–J©	ˆÐ2Ù)2Ó3© A˜QŸ4˜4©ˆÐ3Ù,5Ó6©I q Ÿd˜d©IÐÐ6Ù+4Ó5©9 a˜qŸT˜T©9ÐÐ5Ù2;Ó#<±)¨Q a§D D±)Ð Ð#<ñ =BÓB¹E°q®!�q˜–°Ò2¹EˆÐBÜBGÌÈIËÔBWÓXÑBW¸Q A�; {¥~Ð'7Ó8ÑBWˆÐXÙ'6Ó7¡ !”s˜1–v¡ˆÐ7ä˜[Ó)ˆÙ9DÓE¹°A˜^¨a×/Ð/¹ÐÐEð 13ÐÜÐ1Ö2‰EˆAˆrØŠz˜RŸV™V OÓ4Ò<Ü Ð!WÓXÐXØ�oÕ&ˆDØ˜˜;�>Ñ+¨QÐ.Õ/ˆDØ# AÕ&ˆCØ�QŒwÜŸ(š(Ø—Z‘Z •] C¨¯©°B­Ð8ÀÇ
Á
ô�ô —~’~ t¨W o¸AÔ>�Ø!×(Ñ(¨Ö.ñ 3ô ŸšÐ'<À1ÔEˆð !ˆØˆÜÐ+Ó,ˆ
ÜÐ1×2‰EˆAˆrßÙÜ˜8�;Ó'ˆHØ$�ˆJØ)¨!Õ,ˆIØŸ™ž
‘��1Ø˜Ô'¨1°×0FÑ0FÔ+FÙÜ˜a¤§¡×*Ò*¨q¯v©v¸¬z¸a¿g¹gÀa½jÈA¼oÜ-¨a°°JÓ?�EÜ8¸¸EÀ8×LÒLØ % a¨©°cÐ&9Õ :˜Ü BØ!Ø*Ø!&ô!˜ð
 !×+Ñ+¨A¨rÓ2×9Ñ9¸%Ö@Ø˜mÖ+Ø'(�M !Ó$ô #ñ 3ð* "×'Ñ'Ö)‰EˆAˆrÜ!Ÿ~š~¨b°qÔ9ˆM˜!Óñ *ô ˜4 ¨WÓ5ˆØ�wÔñ óá�Að $%¢=��,–°d·j±j×6KÑ6KÓ6MÒMÙð ð ô !×BÒBØØó‰MˆJ˜ð Ò!Ùð "ô ˜4Ÿ:™: |×4Ò4ô �D—J‘J×)Ñ)Ó+Ô,ä˜Ÿ™×*Ñ*Ó+ð ô
 !×?Ò?Ø *ô‰MˆJ˜ô  œ3˜y›>Ô*ó
ñ +�ð ! (¥+Ø˜k�nØà—x”xð •H˜\Ö*à×-Ñ-Ð.@Õ.C×.IÐ.IÀrÓJàÀEÇHÄH˜e�hŸl™lÐ+;¸RÔ@ÐRTØ  $×"@Ñ"@ÀÅÓ"Mó
ñ +ð 	ð 
ô 5Ø#Ô%:ó
Ðñ  ò 
Ø—*’*ð
áð
ñ &ð
ñ 'ð	
ñ
 (ð
ñ (ð
ñ 0ð
ñ &>ð
ð #×4Ò4ð
ð —L’Lð
ð ×,Ò,ð
ð !×0Ò0ð
ñ "ð
ñ ð
ð ×(Ò(ð
ñ  0ð!
ñ" $ð#
ñ$ ð%
ô&    m°TÔ:ð'
ô( ˜t \°4Ô8ð)
ô* % TÐ+=¸tÔDð+
ô, $ DÐ*;¸UÔCð-
ð 	
ùò] 'Qùò
 )ùÚ2ùÚ3ùÚ6ùÚ5ùÚ#<ùò CùÚXùÚ7ùò Fùòjùò*
sU   •YÁ2YÂ	YÂ*YÃYÃY Ã/Y%ÄY*Ä9Y/ÅY4Å>Y9Ð0Y>Ó-AZÔ:AZc               ó$   € V ^8„  d   QhRRRRRR/# )r(   Úinput_ids_listrx  rv  rw  r)   zOptional[List[Optional[dict]]]r+   )r,   s   "r-   r.   r!  ±	  s(   € ÷ ñ à'ðð 1ðð 
(ñ	r/   c                ó   € V P                   f   R# . p\        W4       F\  w  rET P                  T;'       g    / 4      pVP                  R\	        V4      R^ RVR. RV P                  \	        V4      4      /4       K^  	  V# )zlBuild per-row harvest metadata for a cold-prefill batch so the
produced K/V are added to APC after prefill.
Nr’  r“  rÛ  rµ  r/  )r‹  rk   rC  rF   rù   re  )ry   r•  rv  r·  rM  rƒ  rÛ  s   &&&    r-   Ú_build_apc_meta_for_coldÚ'BatchGenerator._build_apc_meta_for_cold±	  s„   € ð ×ÑÒ#ÙØ%'ˆÜ ÖC‰LˆHØ×-Ñ-¨b¯h¨h°BÓ7ˆJØ�K‰Kà$¤d¨8£nØ  !Ø  *Ø  "Ø$ d×&DÑ&DÄTÈ(Ã^Ó&Tðöñ Dð ˆr/   c                	ó   € V P                   # rh   )r*  ré  s   &r-   ró   ÚBatchGenerator.streamÉ	  s   € à�|‰|Ðr/   c                	óh   € V P                   e$   V P                   P                  4        R V n         R # R # rh   )r7  Úcloseré  s   &r-   rœ  ÚBatchGenerator.closeÍ	  s-   € Ø×ÑÒ'Ø×Ñ×"Ñ"Ô$Ø#ˆDÖñ (r/   c                	ó&   € V P                  4        R # rh   )rœ  ré  s   &r-   Ú__del__ÚBatchGenerator.__del__Ò	  s   € Ø�
‰
Žr/   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	r(   rÅ   zUnion[List[int], int, None]r]  rŠ  rÕ   r×  ræ   rß   r+   )r,   s   "r-   r.   r!  Õ	  s4   € ÷ (ñ (ð 0ð(ð ,ð	(ð

ð(ð #6ñ(r/   c           	     	óæ  € . pVe   \        V\        4      '       d(   T;'       g    V P                  .\        V4      ,          pVf   / .\        V4      ,          pVf   V P                  .\        V4      ,          pM$\        V4      \        V4      8w  d   \        R4      hVf   R .\        V4      ,          pM$\        V4      \        V4      8w  d   \        R4      h\        VVVVV4       Fd  w  rxršpV P                  P                  V P                  WxWšV34       VP                  V P                  4       V ;P                  ^,          un	        Kf  	  \        V P                  R R7      V n        V# )Nz1Insufficient number of logits_processors providedz8Insufficient number of thinking_budget_criteria providedc                ó&   € \        V ^,          4      # )r‘   rI  r&  s   &r-   r–  Ú'BatchGenerator.insert.<locals>.<lambda>û	  s   € ´s¸1¸Q½4´yr/   r’   )r¿   rN   rÅ   r‡   rÕ   rV   rk   r/  rF   r)  Úsorted)ry   rP  rÅ   r]  rÕ   ræ   rÒ  rK  Úmrƒ  rš  Útcs   &&&&&&      r-   ÚinsertÚBatchGenerator.insertÕ	  sG  € ð ˆàÒ¤¨J¼×!<Ò!<Ø$×7Ð7¨¯©Ð8¼3¸w»<ÕGˆJàÒ Ø˜D¤3 w£<Õ/ˆMØÒ$Ø!%×!7Ñ!7Ð 8¼3¸w»<Õ GÑÜÐ"Ó#¤s¨7£|Ô3ÜÐPÓQÐQØ#Ò+Ø(, v´°G³Õ'<Ñ$ÜÐ)Ó*¬c°'«lÔ:ÜÐWÓXÐXä #ØØØØØ$ö!
ÑˆA�"˜"ð ×'Ñ'×.Ñ.°·±ÀÀbÈbÐ/QÔRØ�K‰K˜Ÿ™Ô'Ø�NŠN˜aÕ�Nñ!
ô '-Ø×'Ñ'Ñ-@ô'
ˆÔ#ð ˆr/   c               ó   € V ^8„  d   QhRR/# r'   r+   )r,   s   "r-   r.   r!  ÿ	  s   € ÷ ñ ˜Tñ r/   c           	     ó²  € \         P                  ! V P                  4      ;_uu_ 4        \        V P                  4       F9  w  pw  p    pW18X  g   K  V P                  P                  V4        RRR4       R# 	  V P                  eˆ   WP                  P                  9   dn   \        V P                  P                  4      ^8X  dJ   . V P                  n        . V P                  n	        RV n        \         P                  ! 4         RRR4       R# WP                  P                  9   d‹   V P                  P                  P                  V4      p\        \        V P                  P                  4      4       Uu. uF  q"V8w  g   K  VNK  	  ppV P                  P                  V4        RRR4       R#  RRR4       R# u upi   + '       g   i     R# ; i)z(Remove a sequence from the batch by uid.NTF)ri   ró   r*  r{  r/  r  r.  rÒ  r‡   r7   r%  r-  Úindexrú   r0  )ry   rÀ  rc  Úseq_uidra  r1  r¬  s   &&     r-   ÚremoveÚBatchGenerator.removeÿ	  sp  € ä�YŠY�t—|‘|×$Õ$ä/8¸×9TÑ9TÖ/UÑ+�Ñ+�G˜Q  1 a¨Ø–>Ø×/Ñ/×3Ñ3°AÔ6Ø÷ %Ñ$á/Uð ×!Ñ!Ò-°#×9KÑ9K×9PÑ9PÔ2PÜ�t×)Ñ)×.Ñ.Ó/°1Ô4Ø.0�D×&Ñ&Ô+Ø68�D×&Ñ&Ô3Ø)-�DÔ&Ü—N’NÔ$Ø÷ %Ñ$ð" ×,Ñ,×1Ñ1Ô1Ø×,Ñ,×1Ñ1×7Ñ7¸Ó<�Ü#(¬¨T×-CÑ-C×-HÑ-HÓ)IÔ#JÓWÑ#J˜aÐSVÉhŸ˜Ñ#J�ÐWØ×&Ñ&×-Ñ-¨dÔ3Ø÷+ %Ñ$ð. ÷/ %Ñ$ùò& X÷' %×$Ð$ús=   ©&GÁGÁ:BGÄA)GÅ:G ÆG ÆGÆ5GÇ GÇG	c                ó   € V P                   # )z1Backward-compatible alias for server flush logic.)r/  ré  s   &r-   Úunprocessed_promptsÚ"BatchGenerator.unprocessed_prompts
  s   € ð ×*Ñ*Ð*r/   c                ó^   € \        V P                  4      ^ 8„  ;'       g    V P                  RJ# )z5True if there are prompts waiting or being processed.N)r‡   r/  r.  ré  s   &r-   Úhas_pending_promptsÚ"BatchGenerator.has_pending_prompts
  s.   € ô �4×.Ñ.Ó/°!Ñ3×UÐU°t×7IÑ7IÐQUÐ7UÐUr/   c                óž   € \        V P                  4      ^ 8„  ;'       g/    V P                  RJ;'       g    \        V P                  4      ^ 8„  # )z$True if there is any remaining work.N)r‡   r-  r.  r/  ré  s   &r-   Úhas_workÚBatchGenerator.has_work$
  sQ   € ô �×&Ñ&Ó'¨!Ñ+÷ 4ð 4Ø×!Ñ!¨Ð-÷4ð 4ä�4×.Ñ.Ó/°!Ñ3ð	
r/   c                ó,  € \        4       pV P                  Vn        V P                  Vn        V P                  ^ 8”  d   V P                  V P                  ,          M^ Vn        V P                  Vn        \        P                  ! 4       R,          Vn
        V# )z$Return accumulated batch statistics.ç    eÍÍA)r¸  r0  r?  r1  r¹  rA  r2  r@  ri   Úget_peak_memoryrC  )ry   r½  s   & r-   r½  ÚBatchGenerator.stats-
  s�   € ä“ˆØ"×9Ñ9ˆÔØ ×5Ñ5ˆÔð ×(Ñ(¨1Ô,ð ×'Ñ'¨$×*CÑ*CÖCàð 	Ôð
 #'×":Ñ":ˆÔÜ×.Ò.Ó0°3Õ6ˆÔØˆr/   c               ó    € V ^8„  d   QhRRRR/# rì  r+   )r,   s   "r-   r.   r!  <
  s   € ÷  ñ  ¸5ð  ÀTñ  r/   c                	óV   € \        V R R4      p\        V4      '       d   V! V4       R# R# )rï  NrÆ  )r®  rí  Úrecorders   && r-   Ú_record_prompt_batch_timeÚ(BatchGenerator._record_prompt_batch_time;
  s)   € ä˜<Ð)=¸tÓDˆÜ�H×ÒÙ�YÖñ r/   c               ó   € V ^8„  d   QhRR/# rò  r+   )r,   s   "r-   r.   r!  B
  s   € ÷ ñ Ð0Dñ r/   c                	óP   € \        V R R4      p\        V4      '       d   V! 4       # . # )rô  NrÆ  )r®  Úprogresss   & r-   Ú_prompt_batch_progressÚ%BatchGenerator._prompt_batch_progressA
  s)   € ä˜<Ð):¸DÓAˆÜ�H×ÒÙ“:ÐØˆ	r/   c               ó   € V ^8„  d   QhRR/# rÙ  r+   )r,   s   "r-   r.   r!  H
  s   € ÷ 5ñ 5°Tñ 5r/   c                	ó€   € \        V P                  4      ^ 8X  d	   Wn        R# V P                  P                  V4       R# )r   N)r‡   r-  r  )ry   r  s   &&r-   Ú_extend_generation_batchÚ'BatchGenerator._extend_generation_batchH
  s/   € Üˆt×%Ñ%Ó&¨!Ô+Ø%.Ö"à×"Ñ"×)Ñ)¨)Ö4r/   c           
     	óÚ  € . p. p\        V P                  4      ^ 8”  Ed%   V P                  P                  4       pV ;P                  \        V4      ,          un        V ;P                  ^,          un        V P
                  ^ 8”  d¿   V P                  V P
                  ,          ^ 8X  d�   \        V P                  RR4      p\        V4      '       d   \        P                  ! V! 4       4       MC\        P                  ! V P                  P                   Uu. uF  qUP                  NK  	  up4       \        P                  ! 4        \        V P                  RR4      '       d   \        V P                  4      ^ 8”  d   W23# \        V P                  4      V P                  8¼  d   W23# V P                  Ee¿   V P                  P                  4       '       d�   \         P"                  ! 4       pV P                  P%                  4       p\         P"                  ! 4       V,
          pV ;P&                  V,          un        V P)                  V P                  V4       V ;P*                  V,          un        W23# \         P"                  ! 4       pV P                  P-                  V P.                  V P0                  P2                  V P4                  V P6                  R7      p	\         P"                  ! 4       V,
          pV ;P&                  V,          un        V P)                  V P                  V4       V P9                  V P                  4      pV P;                  V	4       RV n        \        P                  ! 4        W23# \        V P                  4      p
V P                  V
,
          pV P<                  '       Ed   W°P>                  8¼  Ed
   \A        V P>                  \        V P<                  4      4      pV P<                  RV p\B        PE                  \F        PH                  4      '       dP   \J        PL                  PO                  R4      '       d+   \B        PQ                  RV\        V P<                  4      4       V PS                  V4      pVEeí   V P<                  VR V n        WÐn        V ;P*                  V P                  PT                  ,          un        V P                  P                  4       '       d„   \         P"                  ! 4       pV P                  P%                  4       p\         P"                  ! 4       V,
          pV ;P&                  V,          un        V P)                  V P                  V4       W23# \         P"                  ! 4       pV P                  P-                  V P.                  V P0                  P2                  V P4                  V P6                  R7      p	\         P"                  ! 4       V,
          pV ;P&                  V,          un        V P)                  V P                  V4       V P9                  V P                  4      pV P;                  V	4       RV n        \        P                  ! 4        W23# V P<                  VR V n        V Uu. uF  qÿ^ ,          NK  	  ppV Uu. uF  qÿ^,          NK  	  ppV Uu. uF  qÿ^,          NK  	  ppV Uu. uF  qÿ^,          NK  	  ppV Uu. uF  qÿ^,          NK  	  ppV Uu. uF  qÿ^,          NK  	  pp\W        VV4      w  ppV PY                  VV4      p\[        R\\        4      pV! R/ R	V P^                  bR
VbRVbRVbRVbRVbRVbRVbRV P`                  bRV Pb                  bRV Pd                  bRV Pf                  bRVbRV Ph                  bRV Pj                  bR\        V RR4      bR\        V RR4      bR\        V RR4      bR\        V RR4      b V n        V ;P*                  V P                  PT                  ,          un        V P                  P                  4       '       d„   \         P"                  ! 4       pV P                  P%                  4       p\         P"                  ! 4       V,
          pV ;P&                  V,          un        V P)                  V P                  V4       W23# \         P"                  ! 4       pV P                  P-                  V P.                  V P0                  P2                  V P4                  V P6                  R7      p	\         P"                  ! 4       V,
          pV ;P&                  V,          un        V P)                  V P                  V4       V P9                  V P                  4      pV P;                  V	4       RV n        \        P                  ! 4        W23# W23# u upi u upi u upi u upi u upi u upi u upi )r   rñ  Nr„  F)rÝ  rÔ  rù  zAPC admit n=%d (pending=%d)r†  rI   rÒ  r5   rÅ   r6   r]  rÕ   ræ   rÖ   rÐ   rÑ   rÒ   r‰  r‹  r�  r8   r9   rØ   rÕ  r+   )6r‡   r-  r<  r2  r3  r4  r2   rG   ri   r#  r7   r$  r%  r  r.  r»  ÚtimeÚperf_counterré  r1  rÀ  r0  r  rÔ   r#  r+  rÝ  rÔ  rÅ  rÉ  r/  r  r"  rW   rÿ  r   r  rQ   rR   rS   rX   r’  r  rŠ  r—  r¹   r†  rI   rÖ   rÐ   rÑ   rÒ   r‹  r�  )ry   r   Úgeneration_responsesÚprompt_responsesrñ  r5  Úticr6  Úelapsedr  Ú
num_activeÚ
num_to_addrz  ÚmixedÚnstepr‚  rÒ  r5   r†  rv  rÕ   ræ   r6   r†  r‰  r‘  s   &,                        r-   Ú_nextÚBatchGenerator._nextN
  sl  € Ø!ÐØÐô ˆt×%Ñ%Ó&¨Õ*Ø#'×#9Ñ#9×#>Ñ#>Ó#@Ð Ø×$Ò$¬Ð,@Ó(AÕAÕ$Ø×Ò 1Õ$Õà×)Ñ)¨AÔ-Ø×'Ñ'¨$×*CÑ*CÕCÀqÔHä& t×'=Ñ'=¸~ÈtÓT�Ü˜L×)Ò)Ü—G’G™L›NÕ+ä—G’G¨d×.DÑ.D×.QÒ.QÓRÑ.Q¨ŸWœWÑ.QÑRÔSÜ—’Ô ô �D×*Ñ*Ð,<¸e×DÒDÜ�D×*Ñ*Ó+¨aÔ/à#Ð9Ð9äˆt×%Ñ%Ó&¨$×*DÑ*DÔDØ#Ð9Ð9à×ÑÓ)Ø×!Ñ!×2Ñ2×4Ò4Ü×'Ò'Ó)�Ø×&Ñ&×2Ñ2Ó4�Ü×+Ò+Ó-°Õ3�Ø×)Ò)¨WÕ4Õ)Ø×.Ñ.¨t×/AÑ/AÀ7ÔKØ×+Ò+¨qÕ0Õ+Ø'Ð=Ð=ä×#Ò#Ó%ˆCØ×*Ñ*×3Ñ3Ø—‘Ø—‘×0Ñ0Ø!%×!6Ñ!6Ø#×2Ñ2ð	 4ó ˆIô ×'Ò'Ó)¨CÕ/ˆGØ×%Ò%¨Õ0Õ%Ø×*Ñ*¨4×+=Ñ+=¸wÔGØ#×:Ñ:¸4×;MÑ;MÓNÐØ×)Ñ)¨)Ô4Ø!%ˆDÔÜ�NŠNÔØ#Ð9Ð9ä˜×/Ñ/Ó0ˆ
Ø×/Ñ/°*Õ<ˆ
Ø×&×&Ñ&¨:×9PÑ9PÕ+Pô
 �D×+Ñ+¬S°×1LÑ1LÓ-MÓNˆAØ×3Ñ3°B°QÐ7ˆIÜ×"Ñ"¤7§=¡=×1Ò1´b·j±j·n±nÀ[×6QÒ6QÜ—‘Ø1ØÜ˜×3Ñ3Ó4ôð
 ×2Ñ2°9Ó=ˆEØÓ Ø.2×.IÑ.IÈ!È"Ð.M�Ô+Ø%*Ô"Ø×+Ò+¨t×/AÑ/A×/UÑ/UÕUÕ+Ø×%Ñ%×6Ñ6×8Ò8Ü×+Ò+Ó-�CØ ×.Ñ.×:Ñ:Ó<�EÜ"×/Ò/Ó1°CÕ7�GØ×-Ò-°Õ8Õ-Ø×2Ñ2°4×3EÑ3EÀwÔOð  (Ð=Ð=ô ×+Ò+Ó-�CØ $× 2Ñ 2× ;Ñ ;ØŸ™ØŸ™×8Ñ8Ø)-×)>Ñ)>Ø'+×':Ñ':ð	 !<ó !�Iô #×/Ò/Ó1°CÕ7�GØ×-Ò-°Õ8Õ-Ø×2Ñ2°4×3EÑ3EÀwÔOØ'+×'BÑ'BÀ4×CUÑCUÓ'VÐ$Ø×1Ñ1°)Ô<Ø)-�DÔ&Ü—N’NÔ$Ø'Ð=Ð=à*.×*EÑ*EÀaÀbÐ*IˆDÔ'á"+Ó,¡)˜Q�a—D�D¡)ˆDÐ,Ù'0Ó1¡y !˜1Ÿ˜¡yˆIÐ1Ù-6Ó7©Y¨ Ÿt˜t©YˆOÐ7Ù09Ó!:±	¨1 A§$ $±	ÐÐ!:Ù/8Ó 9©y¨! 1§ ©yÐÐ 9Ù6?Ó'@±i°¨!¯¨±iÐ$Ð'@ä+GØ" Ió,Ñ(ˆM˜=ð
 ×4Ñ4°YÐ@RÓSˆHä8Ø'Ô)>ó Ðñ "2ò "Ø—j’jð"áð"ñ $ð"ñ +ð	"ñ
 ,ð"ñ ,ð"ñ #4ð"ñ *Bð"ð #'×"8Ò"8ð"ð Ÿšð"ð #×0Ò0ð"ð !%× 4Ò 4ð"ñ "ð"ð !×,Ò,ð"ð Ÿšð"ô  $ D¨-¸Ô>ð!"ô" # 4¨°tÔ<ð#"ô$ ")¨Ð/AÀ4Ô!Hð%"ô& !(¨Ð.?ÀÔ Gð'"ˆDÔð* ×'Ò'¨4×+=Ñ+=×+QÑ+QÕQÕ'à×!Ñ!×2Ñ2×4Ò4Ü×'Ò'Ó)�Ø×&Ñ&×2Ñ2Ó4�Ü×+Ò+Ó-°Õ3�Ø×)Ò)¨WÕ4Õ)Ø×.Ñ.¨t×/AÑ/AÀ7ÔKð" $Ð9Ð9ô ×'Ò'Ó)�Ø ×.Ñ.×7Ñ7Ø—L‘LØ—N‘N×4Ñ4Ø%)×%:Ñ%:Ø#'×#6Ñ#6ð	 8ó �	ô ×+Ò+Ó-°Õ3�Ø×)Ò)¨WÕ4Õ)Ø×.Ñ.¨t×/AÑ/AÀ7ÔKØ#'×#>Ñ#>¸t×?QÑ?QÓ#RÐ Ø×-Ñ-¨iÔ8Ø%)�Ô"Ü—’Ô à#Ð9Ð9àÐ5Ð5ùòg Sùòf -ùÚ1ùÚ7ùÚ!:ùÚ 9ùÚ'@s*   Äg
ÚgÚgÚ6gÛgÛ$g#Û;g(c                	ó´   € \         P                  ! V P                  4      ;_uu_ 4        V P                  ! R/ VB uuR R R 4       #   + '       g   i     R # ; i)Nr+   )ri   ró   r*  rÖ  )ry   r   s   &,r-   r<  ÚBatchGenerator.nextô
  s3   € Ü�YŠY�t—|‘|×$Õ$Ø—:’:Ñ' Ñ'÷ %×$×$Ó$ús   ©AÁA	) r4  r2  r-  r.  r1  r0  r3  r*  r/  r7  r‹  r�  r  rÝ  rØ   r9   r8   rÕ  rÐ   rÑ   rÒ   rÕ   rÅ   rI   r  rÖ   rÿ   rÓ   rÔ   r#  rÔ  r)  )NNNN))r«   r¬   r­   r®   r¯   ÚDEFAULT_MAX_TOKENSÚDEFAULT_COMPLETION_BATCH_SIZEÚDEFAULT_PREFILL_BATCH_SIZEr  r   r   r   rz   r|  r  rC  rI  rQ  rX  r]  re  rw  r’  r—  r  ró   rœ  rŸ  r¨  r®  r±  r´  r·  r½  r  rÀ  rÅ  rÉ  rÖ  r<  r°   r+   r/   r-   r  r  "  sá  † ñðYKð
 -ðYKð &*ðYKð =AðYKð &CðYKð #=ðYKð ,EðYKð ðYKð ðYKð 3ðYKð  7ðYKð #=ðYKð  "&ð!YKð"  ð#YKð( ð)YKð* ð+YKð, 48ð-YKð. ,0ð/YKð0 %)ð1YKð2 +/ð3YKð4 !&÷5YKð| 0Ðõ4õHõ$õ
õ
õ
õ_õBU
õnð0 ñó ðò$ò
÷(õTð6 ñ+ó ð+ð ñVó ðVð ñ
ó ð
òð ô ó ð ð
 ôó ðõ5òd6öL(r/   r  c               ó4   € V ^8„  d   QhRRRRRRRRRR	R
R	RR	/# )r(   ÚimageszUnion[str, List[str]]ÚaudiosrP  r<  rÅ   úUnion[int, List[int]]rÛ   r*   Úgroup_by_shapeÚtrack_image_sizesr+   )r,   s   "r-   r.   r.   ù
  s[   € ÷ Nñ Nð "ðNð "ð	Nð
 ðNð &ðNð ðNð ðNð ñNr/   c	                óš  € ^ RI Hp
 ^RIHp VP                  P                  4        \        VR4      '       d   VP                  MTp\        V\        4      '       d   V.pVf   \        WVRWV3/ V	B w  rÞ\        WÞ4      # \        VR4      '       d   VP                  MRp. p. pV F¢  p\        V\        4      '       d   V! VRV4      pM!\        VV
P                  4      '       d   TpMTpVP                  V4       \        VR4      '       d*   VP                  VP                  VP                  34       K‘  VP                  R4       K¤  	  V'       d@   \!        V4      ^8”  d0   \#        V4      w  ppV'       d   \%        R\!        V4       R24       MSV'       d'   V^ ,          P                  V^ ,          P                  3MRpVV/pV\'        \)        \!        V4      4      4      /pR.\!        V4      ,          pR.\!        V4      ,          p\+        4       pVP-                  4        EFæ  w  ppV Uu. uF  pVV,          NK  	  ppV Uu. uF  pVV,          NK  	  ppV Uu. uF  pVV,          NK  	  pp\        V\&        4      '       d   V Uu. uF  pVV,          NK  	  ppMTp\/        V	4      p V P1                  R	4      p!V!e{   \        V!\&        4      '       de   V!'       d;   \2        ;QJ d    R
 V! 4       F  '       d   K   RM	  RM! R
 V! 4       4      '       d   V!V R	&   MV Uu. uF  pV!V,          NK  	  upV R	&   \        V VVVV3/ V B w  p"p#\5        V4       F  w  p$p%V"V$,          VV%&   VV$,          VV%&   K   	  V;P6                  V#P6                  ,          un        V;P8                  V#P8                  ,          un        V;P:                  V#P:                  ,          un        V;P<                  V#P<                  ,          un        EKé  	  \'        \)        \!        V4      \!        V4      4      4      p&V&'       Ed¤   V& Uu. uF  pVV,          NK  	  pp\        V\&        4      '       d   V& Uu. uF  pVV,          NK  	  ppMTp\/        V	4      p V P1                  R	4      p!V!e{   \        V!\&        4      '       de   V!'       d;   \2        ;QJ d    R V! 4       F  '       d   K   RM	  RM! R V! 4       4      '       d   V!V R	&   MV& Uu. uF  pV!V,          NK  	  upV R	&   \        V VVRV3/ V B w  p"p#\5        V&4       F  w  p$p%V"V$,          VV%&   K  	  V;P6                  V#P6                  ,          un        V;P8                  V#P8                  ,          un        V;P:                  V#P:                  ,          un        V;P<                  V#P<                  ,          un        \>        P@                  ! 4        VP8                  ^ 8”  d#   VP6                  VP8                  ,          Vn!        VP<                  ^ 8”  d#   VP:                  VP<                  ,          Vn"        \>        PF                  ! 4       R,          Vn$        V'       d�   \%        R\!        V4       R24       \%        RVP6                   RVPB                  R R24       \%        RVP:                   RVPD                  R R24       \%        RVPH                  R R24       \        VV4      p'V'       d   VV'n%        V'# u upi u upi u upi u upi u upi u upi u upi u upi )aÉ  
Generate responses for the given batch of prompts with variable-sized images.

This function implements the transformers-style approach to batching:
1. Group images with the same shape for efficient batch processing
2. Process each group as a batch (no padding waste within groups)
3. Track original image sizes for proper attention masking
4. Restore results to original batch order

Key insight: Instead of padding all images to the same spatial dimensions
(which wastes computation and may hurt accuracy), we group same-sized
images together so there's zero padding within each group.

Args:
   model (nn.Module): The language model.
   processor (PreTrainedTokenizer): The tokenizer/processor.
   images (Union[str, List[str]]): Images (paths, URLs, or PIL images).
   audios (Union[str, List[str]]): Audio files (not yet supported for batching).
   prompts (List[str]): The input prompts.
   max_tokens (Union[int, List[int]]): Maximum number of output tokens. This
      can be per prompt if a list is provided.
   verbose (bool): If ``True``, print tokens and timing information.
   group_by_shape (bool): If ``True``, group same-shaped images for efficient
      batch processing.
   track_image_sizes (bool): If ``True``, track and return original image sizes.
   kwargs: The remaining options get passed to :obj:`BatchGenerator`.
      See :obj:`BatchGenerator` for more details.

Returns:
    BatchResponse with generated texts, statistics, and optionally image_sizes.
)ÚImage)Úprocess_imager#  NÚimage_processorÚheightz[batch_generate] Found z unique image shapesrÕ   c              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irh   ©rG   rJ  s   & r-   rD   Ú!batch_generate.<locals>.<genexpr>o  ó   é € Ð+SÑAR¸A¬H°Q¯K¨KÓARùrN  FTc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irh   ré  rJ  s   & r-   rD   rê  –  rë  rN  rº  z%[batch_generate] Finished processing z samplesz[batch_generate] Prompt: z	 tokens, z.3fz tokens-per-secz[batch_generate] Generation: z[batch_generate] Peak memory: z GBr…   )&ÚPILrä  Úutilsrå  ÚdetokenizerÚresetr  r#  r¿   r³   Ú_generate_batchr¼  ræ  rF   rç  Úwidthr‡   r   Úprintrù   rú   r¸  r!  r^  rS   rÁ   r{  r?  r¹  r@  rº  ri   r%  rA  rB  r»  rC  rE  )(rI   rÿ   rÞ  rß  rP  rÅ   rÛ   rá  râ  r   rä  rå  r#  r=  r½  ræ  Úprocessed_imagesÚimage_sizes_originalrA  Úpil_imgÚgrouped_imagesÚgrouped_indicesr†   Ú	all_textsÚall_image_sizesÚtotal_statsÚindicesrc  Úgroup_imagesÚgroup_promptsÚgroup_sizesÚgroup_max_tokensÚgroup_kwargsrÕ   Úchunk_textsÚchunk_statsÚjÚorig_idxÚtext_only_indicesÚresponses(   &&&&&&&&&,                              r-   Úbatch_generater  ù
  sÀ  € õV å%à×Ñ×ÑÔ!Ü'.¨y¸+×'FÒ'F�	×#Ò#ÈI€Iô �&œ#×ÒØ�ˆð ‚~Ü&Ø˜g t¨Zñ
ØDJñ
‰ˆô ˜UÓ*Ð*ô &-¨YÐ8I×%JÒ%Jˆ	×!Ò!ÐPTð ð ÐØÐÛˆÜ�cœ3×ÒÙ# C¨¨Ó?‰GÜ˜˜UŸ[™[×)Ò)Ø‰GàˆGØ×Ñ Ô(ä�7˜H×%Ò%Ø ×'Ñ'¨¯©¸¿¹Ð(GÖHà ×'Ñ'¨Ö/ñ ÷ œ#Ð.Ó/°!Ô3Ü*?Ð@PÓ*QÑ'ˆ˜çÜÐ+¬C°Ó,?Ð+@Ð@TÐUÔVø÷
  ð ˜aÕ ×'Ñ'Ð)9¸!Õ)<×)BÑ)BÑCàð 	ð
  Ð!1Ð2ˆØ ¤$¤u¬SÐ1AÓ-BÓ'CÓ"DÐEˆð �œ˜W›Õ%€IØ�fœs 7›|Õ+€OÜ“,€Kà)×/Ñ/×1‰ˆˆwá5<Ó=±W°Ð(¨×+Ð+±WˆÐ=Ù-4Ó5©W¨˜ Ÿ˜©WˆÐ5Ù8?Ó@¹°1Ð+¨A×.Ð.¹ˆÐ@ô �j¤$×'Ò'Ù7>Ó?±w°! 
¨1§ ±wÐÐ?Ðà)Ðä˜F“|ˆØ(×,Ñ,Ð-@ÓAÐØÒ(¬ZÐ8IÌ4×-PÒ-Pß$¯«Ñ+SÑARÓ+S¯¯ªÑ+SÑARÓ+S×(SÒ(SØ4E�Ð0Ò1ñ 3:ó5Ù29¨QÐ% a×(Ð(±'ñ5�Ð0Ñ1ô
 $3ØØØØØñ$
ð ñ$
Ñ ˆ�[ô % WÖ-‰KˆAˆxØ"-¨a¥.ˆI�hÑØ(3°A­ˆO˜HÓ%ñ .ð
 	×!Ò! [×%>Ñ%>Õ>Õ!Ø×Ò ;×#:Ñ#:Õ:ÕØ×%Ò%¨×)FÑ)FÕFÕ%Ø×#Ò# {×'BÑ'BÕB×#Ð#ñS 2ôV œU¤3Ð'7Ó#8¼#¸g»,ÓGÓHÐßÐÙ->Ó?Ñ->¨˜ Ÿ˜Ñ->ˆÐ?Ü�j¤$×'Ò'Ù7HÓIÑ7H°! 
¨1§ Ñ7HÐÐIÐà)Ðä˜F“|ˆØ(×,Ñ,Ð-@ÓAÐØÒ(¬ZÐ8IÌ4×-PÒ-Pß$¯«Ñ+SÑARÓ+S¯¯ªÑ+SÑARÓ+S×(SÒ(SØ4E�Ð0Ò1ñ 3Dó5Ù2C¨QÐ% a×(Ð(Ñ2Cñ5�Ð0Ñ1ô $3ØØØØØñ$
ð ñ$
Ñ ˆ�[ô %Ð%6Ö7‰KˆAˆxØ"-¨a¥.ˆI�hÓñ 8ð 	×!Ò! [×%>Ñ%>Õ>Õ!Ø×Ò ;×#:Ñ#:Õ:ÕØ×%Ò%¨×)FÑ)FÕFÕ%Ø×#Ò# {×'BÑ'BÕBÕ#ä‡N‚NÔð ×Ñ Ô"Ø!,×!:Ñ!:¸[×=TÑ=TÕ!TˆÔØ×"Ñ" QÔ&à×)Ñ)¨K×,GÑ,GÕGð 	Ô"ô !×0Ò0Ó2°SÕ8€KÔçÜÐ5´c¸'³l°^À8ÐLÔMÜØ'¨×(AÑ(AÐ'BÀ)ÈK×LbÑLbÐcfÐKgÐgvÐwô	
ô 	Ø+¨K×,IÑ,IÐ+JÈ)Ø×)Ñ)¨#Ð.¨oð?ô	
ô 	Ð.¨{×/FÑ/FÀsÐ.KÈ3ÐOÔPä˜Y¨Ó4€HßØ.ˆÔØ€OùòK >ùÚ5ùÚ@ùò  @ùò5ùò6 @ùâIùò5s0   È1\%É	\*É!\/Ê\4Ì(\9Ñ\>Ñ3]Ô]c                ó„   € \        V R 4      '       d   V P                  4       # \        P                  ! R\        ^R7       V # )Úclonez°Sharing logits processor across batch entries because it does not implement clone(). Stateful logits processors should implement clone() to avoid shared state across sequences.)Ú
stacklevel)r  r
  ÚwarningsÚwarnÚRuntimeWarning)rÿ   s   &r-   Ú _clone_or_share_logits_processorr  Ê  s=   € Üˆy˜'×"Ò"Ø�‰Ó Ð Ü‡M‚Mð	2ô 	Øõð Ðr/   c          
     ó,   € V ^8„  d   QhRRRRRRRRR	R
/# )r(   rP  r<  rÞ  r	   rÅ   rà  rÛ   r*   r)   zTuple[List[str], BatchStats]r+   )r,   s   "r-   r.   r.   ×  sE   € ÷ Añ Að ðAð ð	Að
 &ðAð ðAð "ñAr/   c                ó�	  € \        VR 4      '       d   VP                  MTp\        V4      pVP                  RR4      p	\	        \        V4      4       U
u. uF  p
Y£e   \        V4      M^ 8  d   ^M^ NK  	  pp
\        V4       U
Uu. uF$  w  r¬\        VV P                  VWº,          R7      NK&  	  pp
pV P                  P                  R9   d   \        VRR4      RJ MRp\        VP                  RR4      4      p\        V P                  RR4      p\        VVRVVVVRR	7      pVP                  R
R4      pVP                  RR4      pVP                  RR4      pVP                  4        UUu/ uF  w  ppVR9  g   K  VVbK  	  pppV P                  ! VV3RV/VB p/ VCVP                  4       CpVP                  R\         4      e‰   \#        V4      pVP                  R4      pVP                  R4      pVe    Ve   VP%                  \'        VV4      4       \)        V VVP*                  VVVR7      '       g   VP                  RR4       RVR&   \-        V P.                  V3RVRVRR/VB pV	'       dp   \0        ;QJ d    R V	 4       F  '       d   K   RM	  RM! R V	 4       4      '       d6   \	        V4       UUu. uF  pV	 Uu. uF  p\3        V4      NK  	  upNK   	  p	ppVP5                  VP7                  4       V\9        VV4      V	R7      pV U u/ uF  p V . bK  	  p!p \:        P<                  ! 4       p"VP>                  '       d^   VPA                  4       w  pp#V# FB  p$V$PB                  R8w  g   K  V!V$PD                  ,          PG                  V$PH                  4       KD  	  Ko  \:        P<                  ! 4       V",
          p%VPK                  4        VPL                  p&. p'V F_  p V&PO                  4        V!V ,           F  p(V&PQ                  V(4       K  	  V&PS                  4        V'PG                  V&PT                  4       Ka  	  VPW                  4       p)V%V)PX                  ,
          V)n-        V)PZ                  ^ 8”  d#   V)P\                  V)PZ                  ,          V)n/        V'V)3# u up
i u upp
i u uppi u upi u uppi u up i )r#  rÕ   N)Ú
num_imagesÚchat_templateTÚresize_shapeÚimage_token_indexF)rÞ  ÚaudiorP  r  r  Úadd_special_tokensÚpad_to_uniform_sizer5   r)  Úattention_maskr	  rÖ   r8   r9   )r5   r6   r8   r9   r:   r  r  rÝ  c              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irh   ré  )rB   rÿ   s   & r-   rD   Ú"_generate_batch.<locals>.<genexpr>1  s   é € ð !Ù-> 	Œ�×ÐÓ->ùrN  )r]  rÕ   r6  )Úgemma3Úgemma3nÚgemma4Úgemma4_unified)r5   r)  r  )0r  r#  r‡   r  rú   r{  r   rH  Ú
model_typer2   rÄ   r   rS   r!  r  r   r  r^  r  r   rK   r6   r  r=   rÁ   r  r¨  r  rd  rÌ  rÍ  r·  r<  rÎ  rÀ  rF   rË  rœ  rï  rð  Ú	add_tokenrø  Útextr½  r¹  rº  r@  rB  )*rI   rÿ   rP  rÞ  rÅ   rÛ   r   r#  rY  rÕ   rc  Únum_images_listrK  Úformatted_promptsr  r  r  r  r5   r)  r	  r,  r-  Údata_kwargsr+  Ú
gen_kwargsr.  r8   r9   Úgenra  rÒ  rÀ  ÚresultsrÐ  rÎ  ÚrÚ
total_timerï  r=  Útr½  s*   &&&&&&,                                   r-   rñ  rñ  ×  sÎ  € ô (/¨y¸+×'FÒ'F�	×#Ò#ÈI€IÜ�W“€JØŸ
™
Ð#6¸Ó=Ðô ”s˜7“|Ô$óá$ˆAð Ò!3”#�f”+¸Ô;‰ÀÒBÙ$ð ð ô ˜gÔ&ôñ '‰DˆAô 	ØØ�L‰LØØ&Õ)÷		
ñ 'ð ñ ð �<‰<×"Ñ"Ð&WÔWô 	�	˜?¨DÓ1°TÑ9àð ô *¨&¯*©*°^ÀTÓ*JÓK€LÜ §¡Ð.AÀ4ÓHÐäØØØØ!Ø+Ø!Ø-Ø!ô	€Fð —
‘
˜;¨Ó-€IØ—:‘:˜n¨dÓ3€LØ�:‰:Ð&¨Ó-€Dð —L‘L”Nôá"‰DˆAˆqØÐCÑCô 	ˆˆ1ŠÙ"ð ñ ð ×1Ò1Ø�<ñØ&*ðØ.9ñÐð ?�KÐ>Ð#3×#;Ñ#;Ó#=Ð>€Jà‡z�zÐ%Ô'@ÓAÒMÜ˜ZÓ(ˆØ—j‘j Ó/ˆØ—Z‘Z Ó-ˆ
ØÒ" zÒ'=Ø× Ñ Ô!;¸JÈÓ!TÔUÜ'ØØØ*×8Ñ8Ø#Ø!Ø(÷
ó 
ð �J‰JÐ*¨DÔ1Ø*.ˆFÐ&Ñ'ô Ø×ÑØñð &ðð )ð	ð
 ðð ñ€C÷ ŸS›Sñ !Ù->ó!ŸSŸSšSñ !Ù->ó!÷ ò ô
 ˜:Ô&ô
á&�ñ ;LÓLÑ:K°QÔ-¨aÖ0Ñ:KÔLÙ&ð 	ñ 
ð
 �:‰:Ø×ÑÓØÜ2°:¸zÓJØ+ð	 ó €Dñ #'Ó'¡$˜3ˆs�BŠw¡$€GÐ'ä
×
Ò
Ó
€CØ
�,�,ˆ,Ø"%§(¡(£*ÑˆÐÛ%ˆAØ�‰ &Ö(Ø˜Ÿ™•×%Ñ% a§g¡gÖ.ó &ô ×"Ò"Ó$ sÕ*€Jà‡I�I„Kà×'Ñ'€KØ€EÛˆØ×ÑÔØ˜—�ˆAØ×!Ñ! !Ö$ñ à×ÑÔØ�‰�[×%Ñ%Ö&ñ ð �I‰I‹K€EØ&¨×):Ñ):Õ:€EÔØ×Ñ˜qÔ Ø$×6Ñ6¸×9NÑ9NÕNˆÔØ�%ˆ<ÐùògùóùóBùòV Mùó
ùò (s6   Á"R'Â*R,Å-R2Å>R2Ê?	R=ËR8ËR=ÌSÒ8R=rh   >
   Úpos_hwÚposition_idsr  Útoken_type_idsÚper_layer_inputsÚvisual_pos_masksÚmm_token_type_idsÚdecoder_inputs_embedsÚdeepstack_visual_embedsÚfull_text_row_masked_out_mask)r>  r<  )NNNé€   FTT)Néd   F)fÚ
__future__r   r5  r  r   rQ   rµ   rÌ  r  Úcollections.abcr   Údataclassesr   Útypingr   r   r   r	   r
   r   r   Úmlx.coreÚcoreri   Úmlx.nnÚnnÚmlx_lm.sample_utilsr   r   r   Ú r   rÔ  Úmodelsr   Úprompt_utilsr   Úsample_utilsr   Úspeculative.utilsr   r   r   r   r   Ú
turboquantr   r   rî  r   r   Úcommonr   r   r   r    r!   r"   Ú	getLoggerrW   rÚ  ÚDEFAULT_TEMPERATUREÚDEFAULT_TOP_Pr  r  ÚDEFAULT_REPETITION_CONTEXT_SIZEr  rÛ  rÜ  rT   r4   rK   rZ   ra   rp   rr   r¹   rÄ   r9  r;  rR  rV  rh  r|  r[  rd  ri  rt  rŠ  r”  r¶  r¸  r¼  r¿  rø   rÈ  rG  r†  r  r  r  rñ  r+   r/   r-   Ú<module>rK     sz  ðÝ "ã Û Û Û 	Û 
Û Û Ý %Ý !ß D× DÑ Då Ý ß DÝ å Ý Ý .Ý )÷õ ÷ Dß 9÷÷ ð 
×	Ò	Ð-Ó	.€àÐ ØÐ Ø€Ø€Ø€Ø"$Ð Ø Ð Ø "Ð ØÐ Ø$&Ð !õ4ð#ð ð#ð ð	#ð
 ð#ð ð#ð ð#ð ÷#õL1õõ÷+Vñ +Võ\Uò
	IðTð )ðTð -ðTð +/ðTð .MðTð )-ðTð ,KðTð *.ðTð -LðTð !ðTð !ðTð  ð!Tð" .2ð#Tð$ )-ð%Tð& "&ð'Tð(  $ð)Tð* /ð+Tð, 3ð-Tð. 9ð/Tð0 9=ð1Tð2 SWð3Tð4 (Að5Tð6 (,ð7Tð8 ð9Tð: '+ð;Tð< KOð=Tð> 26ð?Tð@ ðATðB ÷CTðn
 ÷8ð 8ó ð8ô:HôNò#Ð ð =Ð õõõ.õ	)õ/(òdð( ØØ+ôJHðZ ÷ð ó ðð. ÷8ð 8ó ð8ð" ÷ð ó ðð
ð $(ð	
ð
 &*÷
÷{ñ {÷|yñ y÷xO	ñ O	÷dT(ñ T(÷nNòb
÷Añ Ar/   