+
    TV-jJä  ã                   ó  € ^ RI t ^ RIHt ^ RIHt ^ RIHtHtHtH	t	 ^ RI
Ht ^ RIHt ^ RIHtHtHtHt ^RIHt R/R R llt/ 3R	 R
 lltR0R ltR R ltR R ltR R lt ! R R4      t ! R R]4      t ! R R]4      t ! R R]4      t ! R R]4      t  ! R R]4      t! ! R R]4      t" ! R  R!]4      t#R" t$ ! R# R$]4      t% ! R% R&]4      t& ! R' R(4      t'] ! R) R*4      4       t( ! R+ R,4      t) ! R- R.4      t*R# )1é    N)Údeque)Ú	dataclass)ÚAnyÚDictÚListÚOptional)Útree_flattenÚtree_mapÚtree_reduceÚtree_unflatten©Úcreate_causal_maskc                ó|   € V ^8„  d   QhR\         P                  R\        \        ,          R\        \
        ,          /# )é   ÚmodelÚmax_kv_sizeÚreturn)ÚnnÚModuler   Úintr   r   )Úformats   "Úd/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/cache.pyÚ__annotate__r      s2   € ÷ 6ñ 6Ü�9‰9ð6äœ#•ð6ô 
Œ#…Yñ6ó    c                ó  € \        V R4      '       d   V P                  4       # \        V P                  4      pVe'   \	        V4       Uu. uF  p\        V^R7      NK  	  up# \	        V4       Uu. uF  p\        4       NK  	  up# u upi u upi )a©  
Construct the model's cache for use in generation.

This function will defer the cache construction to the model if it has a
``make_cache`` method, otherwise it will make a default KV cache.

Args:
    model (nn.Module): The language model.
    max_kv_size (Optional[int]): If provided and the model does not have a
        ``make_cache`` method, a ``RotatingKVCache`` is used with a maximum
        size of ``max_kv_size``
Ú
make_cache)Úmax_sizeÚkeep)Úhasattrr   ÚlenÚlayersÚrangeÚRotatingKVCacheÚKVCache)r   r   Ú
num_layersÚ_s   &&  r   Úmake_prompt_cacher'      s‡   € ô  ˆu�l×#Ò#Ø×ÑÓ!Ð!ä�U—\‘\Ó"€JØÒäCHÈÔCTó
ÙCT¸aŒO [°q×9ÑCTñ
ð 	
ô $)¨Ô#4Ó5Ñ#4˜a”–	Ñ#4Ñ5Ð5ùò	
ùò 6s   Á	BÁ/B
c                ót   € V ^8„  d   QhR\         R\        \        ,          R\        \         \         3,          /# )r   Ú	file_nameÚcacheÚmetadata)Ústrr   r   r   )r   s   "r   r   r   +   s-   € ÷ ?ñ ?¤ð ?¬T´#­Yð ?Ä$ÄsÌCÀxÅ.ñ ?r   c                óZ  € V Uu. uF  q3P                   NK  	  ppV Uu. uF  q3P                  NK  	  pp\        \        V4      4      pV Uu. uF  p\	        V4      P
                  NK  	  ppWRV.p\        \        V4      4      p\        P                  ! WV4       R# u upi u upi u upi )zê
Save a pre-computed prompt cache to a file.

Args:
    file_name (str): The ``.safetensors`` file name.
    cache (List[Any]): The model state.
    metadata (Dict[str, str]): Optional metadata to save along with model
        state.
N)ÚstateÚ
meta_stateÚdictr	   ÚtypeÚ__name__ÚmxÚsave_safetensors)r)   r*   r+   ÚcÚ
cache_dataÚ
cache_infoÚcache_classesÚcache_metadatas   &&&     r   Úsave_prompt_cacher:   +   s‘   € ñ $)Ó)¡5˜a—'”'¡5€JÐ)Ù(-Ó.© 1—,”,©€JÐ.Ü”l :Ó.Ó/€JÙ/4Ó5©u¨!”T˜!“W×%Ô%©u€MÐ5Ø ¨MÐ:€NÜœ, ~Ó6Ó7€NÜ×Ò˜	¨~Ö>ùò *ùÚ.ùâ5s   …BŸB#ÁB(c           	     óp  € \         P                  ! V RR7      w  r#\        \        VP	                  4       4      4      p\        \        VP	                  4       4      4      pVw  rEp\        WbV4       UUU	u. uF&  w  rxp	\        4       V,          P                  W‰4      NK(  	  p
ppp	V'       d   W¥3# V
# u up	ppi )a.  
Load a prompt cache from a file.

Args:
    file_name (str): The ``.safetensors`` file name.
    return_metadata (bool): Whether or not to return metadata.
        Default: ``False``.

Returns:
    List[Any] or Tuple[List[Any], Dict[str, str]]: The prompt cache and
        the metadata if requested.
T)Úreturn_metadata)r3   Úloadr   ÚlistÚitemsÚzipÚglobalsÚ
from_state)r)   r<   Úarraysr9   Úinfor+   Úclassesr5   r.   r/   r*   s   &&         r   Úload_prompt_cacherF   >   s¤   € ô  ŸWšW YÀÔEÑ€FÜœD §¡£Ó0Ó1€FÜ#¤D¨×)=Ñ)=Ó)?Ó$@ÓA€NØ,Ñ€D�Gô %(¨¸Ô$>õá$>Ñ ˆA�jô 	‹	�!�×Ñ Ö2Ù$>ð 
ò ÷ ØˆÐØ€Lùôs   Á4,B1c                óF   € V ^8„  d   QhR\         \        ,          R\        /# )r   r*   r   )r   r   Úbool)r   s   "r   r   r   X   s   € ÷ 0ñ 0¤¤c¥ð 0¬tñ 0r   c                ój   € \         ;QJ d    R V  4       F  '       d   K   R# 	  R# ! R V  4       4      # )z(
Check if model's cache can be trimmed.
c              3   ó@   "  € T F  qP                  4       x € K  	  R # 5i©N©Úis_trimmable©Ú.0r5   s   & r   Ú	<genexpr>Ú(can_trim_prompt_cache.<locals>.<genexpr>\   s   é € Ð/© A�~‰~×Ð«ùó   ‚FT)Úall©r*   s   &r   Úcan_trim_prompt_cacherU   X   s-   € ÷ ‹3Ñ/©Ó/�3Œ3Ð/Š3Ð/ˆ3Ñ/©Ó/Ó/Ð/r   c                óh   € V ^8„  d   QhR\         \        ,          R\        R\         \        ,          /# )r   r*   Ú
num_tokensr   )r   r   r   )r   s   "r   r   r   _   s)   € ÷ 2ñ 2œT¤#�Yð 2´Cð 2¼DÄ½Iñ 2r   c                ó    € \        V 4      '       d   \        V 4      ^ 8X  d   ^ # V  Uu. uF  q"P                  V4      NK  	  up^ ,          # u upi )aF  
Trim the model's cache by the given number of tokens.

This function will trim the cache if possible (in-place) and return the
number of tokens that were trimmed.

Args:
    cache (List[Any]): The model's cache.
    num_tokens (int): The number of tokens to trim.

Returns:
    (int): The number of tokens that were trimmed.
)rU   r    Útrim)r*   rW   r5   s   && r   Útrim_prompt_cacherZ   _   s@   € ô ! ×'Ò'¬3¨u«:¸¬?ÙÙ(-Ó.© 1�F‰F�:Ö©Ñ.¨qÕ1Ð1ùÒ.s   ¨Ac          	      ó^   € V ^8„  d   QhR\         R\         R\        R\        \         ,          /# )r   ÚNÚoffsetÚreturn_arrayÚwindow_size)r   rH   r   )r   s   "r   r   r   r   s0   € ÷ 
ñ 
Ü
ð
Üð
Ü'+ð
Ü:BÄ3½-ñ
r   c                 ód   € Ve   \        WVR7      # V ^8X  d   R # V'       d   \        WVR7      # R# )N©r_   Úcausalr   )r\   r]   r^   r_   s   &&&&r   Úcreate_attention_maskrc   r   s4   € ð ÒÜ! !¸ÔEÐEØ	
ˆaŒÙß	Ü! !¸ÔEÐEár   c                   óº   a € ] tR t^t o ]R 4       t]P                  R 4       t]R 4       t]P                  R 4       tR tR t	]R 4       t
R t]R	 4       tR
tV tR# )Ú
_BaseCachec                ó   € . # rK   © ©Úselfs   &r   r.   Ú_BaseCache.state€   s   € àˆ	r   c                ó8   € Ve   V'       d   \        R4      hR # R # )Nz,This cache has no state but a state was set.©Ú
ValueError©ri   Úvs   &&r   r.   rj   „   s   € àŠ=ŸQÜÐKÓLÐLñ ‰=r   c                ó   € R # )Ú rg   rh   s   &r   r/   Ú_BaseCache.meta_state‰   s   € ár   c                ó8   € Ve   V'       d   \        R4      hR # R # )Nz6This cache has no meta_state but a meta_state was set.rl   rn   s   &&r   r/   rr   �   s   € àŠ=ŸQÜÐUÓVÐVñ ‰=r   c                ó   € R # ©Frg   rh   s   &r   rM   Ú_BaseCache.is_trimmable’   s   € Ùr   c                ó   € ^ # )z³
Return the size (i.e. sequence length) of the cache.

Not every cache is required to implement this, in which case the size
will always be 0 (though the cache may not be empty).
rg   rh   s   &r   ÚsizeÚ_BaseCache.size•   s   € ñ r   c                ó   € \        R4      h)z&Return the size of this cache in bytesz%Cache sub-class must implement nbytes©ÚNotImplementedErrorrh   s   &r   ÚnbytesÚ_BaseCache.nbytesž   s   € ô "Ð"IÓJÐJr   c                ó   € \        R4      h)z&
Return if the cache is empty or not.
z$Cache sub-class must implement this.r{   rh   s   &r   ÚemptyÚ_BaseCache.empty£   s   € ô "Ð"HÓIÐIr   c                ó@   € V P                  V 4      pWn        W#n        V# rK   )Ú__new__r.   r/   )Úclsr.   r/   Úobjs   &&& r   rB   Ú_BaseCache.from_state©   s!   € ð �k‰k˜#ÓˆØŒ	Ø#ŒØˆ
r   rg   N)r2   Ú
__module__Ú__qualname__Ú__firstlineno__Úpropertyr.   Úsetterr/   rM   rx   r}   r€   ÚclassmethodrB   Ú__static_attributes__Ú__classdictcell__©Ú__classdict__s   @r   re   re      s›   ø‡ € Øñó ðð ‡\�\ñMó ðMð ñó ðð ×ÑñWó ðWòòð ñKó ðKòJð ñó ör   re   c                   óŒ   a € ] tR t^²t o RtR tR t]R 4       t]P                  R 4       tR t
R tR tR	 t]R
 4       tRtV tR# )ÚConcatenateKVCachea  ConcatenateKVCache the simplest KV cache implementation.

Can be used as a mock KV cache or when large blocks are being processed at
a time in which case KVCache isn't necessarily faster. Consider using the
KVCache with a larger step size before using this cache.
c                ó0   € R V n         R V n        ^ V n        R # rK   ©ÚkeysÚvaluesr]   rh   s   &r   Ú__init__ÚConcatenateKVCache.__init__º   ó   € ØˆŒ	ØˆŒØˆŽr   c                óP  € V P                   f   Wn         W n        MR\        P                  ! V P                   V.RR7      V n         \        P                  ! V P                  V.RR7      V n        V P                   P                  R,          V n        V P                   V P                  3# )N©Úaxiséþÿÿÿ)r•   r–   r3   ÚconcatenateÚshaper]   ©ri   r•   r–   s   &&&r   Úupdate_and_fetchÚ#ConcatenateKVCache.update_and_fetch¿   ss   € Ø�9‰9ÒØŒIØ �KäŸš¨¯	©	°4Ð'8¸rÔBˆDŒIÜŸ.š.¨$¯+©+°vÐ)>ÀRÔHˆDŒKØ—i‘i—o‘o bÕ)ˆŒà�y‰y˜$Ÿ+™+Ð%Ð%r   c                ó2   € V P                   V P                  3# rK   ©r•   r–   rh   s   &r   r.   ÚConcatenateKVCache.stateÊ   s   € à�y‰y˜$Ÿ+™+Ð%Ð%r   c                óh   € Vw  V n         V n        V P                   P                  R,          V n        R# )r   Nr�   ©r•   r–   rŸ   r]   rn   s   &&r   r.   r¥   Î   s$   € à!"ÑˆŒ	�4”;Ø—i‘i—o‘o bÕ)ˆŽr   c                ó   € R # ©Trg   rh   s   &r   rM   ÚConcatenateKVCache.is_trimmableÓ   ó   € Ùr   c                ód   € \        V P                  V4      pV ;P                  V,          un        V# rK   ©Úminr]   ©ri   Úns   &&r   rY   ÚConcatenateKVCache.trimÖ   ó%   € Ü�—‘˜QÓˆØ�Š�qÕ�Øˆr   c                ó0   € \        VR V P                  /VB # ©r]   ©rc   r]   ©ri   ÚargsÚkwargss   &*,r   Ú	make_maskÚConcatenateKVCache.make_maskÛ   ó   € Ü$ dÐI°4·;±;ÐIÀ&ÑIÐIr   c                ó   € V P                   R J # rK   ©r•   rh   s   &r   r€   ÚConcatenateKVCache.emptyÞ   ó   € Ø�y‰y˜DÐ Ð r   c                ó„   € V P                   f   ^ # V P                   P                  V P                  P                  ,           # rK   ©r•   r}   r–   rh   s   &r   r}   ÚConcatenateKVCache.nbytesá   ó.   € à�9‰9ÒÙØ�y‰y×Ñ $§+¡+×"4Ñ"4Õ4Ð4r   ©r•   r]   r–   N)r2   r‡   rˆ   r‰   Ú__doc__r—   r¡   rŠ   r.   r‹   rM   rY   r¹   r€   r}   r�   rŽ   r�   s   @r   r’   r’   ²   si   ø‡ € ñòò
	&ð ñ&ó ð&ð ‡\�\ñ*ó ð*òòò
Jò!ð ñ5ó ö5r   r’   c                   óÐ   a € ] tR t^èt o RtRV 3R lR lltR t]R 4       t]P                  R 4       t]R 4       t
]
P                  R 4       t
R	 tR
 tR tR t]R 4       tRtV tR# )ÚQuantizedKVCacheé   c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   Ú
group_sizeÚbits©r   )r   r�   s   "€r   r   ÚQuantizedKVCache.__annotate__ë   s   ø€ ÷ ñ ¡3ð ±3ñ r   c                óH   € R V n         R V n        ^ V n        Wn        W n        R # rK   )r•   r–   r]   rÊ   rË   ©ri   rÊ   rË   s   &&&r   r—   ÚQuantizedKVCache.__init__ë   s!   € ØˆŒ	ØˆŒØˆŒØ$ŒØŽ	r   c                óZ  a aaaa€ SP                   w  r4rVVP                   R,          pS P                  oS P                  e2   SV,           S P                  ^ ,          P                   R	,          8”  Ed&   ^\        P                  P
                  ,          S P                  ,          oS P                  V,           ^,
          S P                  ,          S P                  ,          pW4V3oVVV V3R lp	V3R lp
S P                  e|   SS P                  ,          ^ 8w  d4   \        V3R lS P                  S P                  34      w  S n        S n	        \        V
S P                  S P                  34      w  S n        S n	        MV	! V4      V	! V4      uS n        S n	        S ;P                  V,          un        \        P                  ! SS P                  S P                  R7      o\        P                  ! VS P                  S P                  R7      p\        \        S P                  4      4       FZ  pSV,          S P                  V,          RSS P                  1R3&   W+,          S P                  V,          RSS P                  1R3&   K\  	  \        V 3R lS P                  S P                  34      # )
é   c                 óF  <€ \         P                  ! . SOV S,          N5\         P                  R 7      \         P                  ! . SOV SP                  ,          N5SP                  R 7      \         P                  ! . SOV SP                  ,          N5SP                  R 7      3# )©Údtype)r3   ÚzerosÚuint32rÊ   rÕ   )ÚdimÚ
el_per_intr•   ri   rŸ   s   &€€€€r   Ú
init_quantÚ5QuantizedKVCache.update_and_fetch.<locals>.init_quantü   s{   ø€ ä—H’HÐ8˜uÐ8 c¨ZÕ&7Ñ8ÄÇ	Á	ÔJÜ—H’HÐ=˜uÐ= c¨T¯_©_Õ&<Ñ=ÀTÇZÁZÔPÜ—H’HÐ=˜uÐ= c¨T¯_©_Õ&<Ñ=ÀTÇZÁZÔPðð r   c                 ó¦   <€ \         P                  ! . SOV P                  R,          N5V P                  R7      p\         P                  ! W.RR7      # )rÒ   rÔ   r›   éÿÿÿÿr�   )r3   rÖ   rŸ   rÕ   rž   )ÚxÚnew_xrŸ   s   & €r   Úexpand_quantÚ7QuantizedKVCache.update_and_fetch.<locals>.expand_quant  s=   ø€ ÜŸšÐ!6 5Ð!6¨!¯'©'°"­+Ñ!6¸a¿g¹gÔF�Ü—~’~ q j°rÔ:Ð:r   c                 ó    <€ V R RS1R3,          # ©.NºNNNrg   )rÞ   Úprevs   &€r   Ú<lambda>Ú3QuantizedKVCache.update_and_fetch.<locals>.<lambda>
  s   ø€  ! C¨¨$¨° MÖ"2r   ©rÊ   rË   .rä   c                 ó4   <€ V R RSP                   1R3,          # rã   r´   ©rÞ   ri   s   &€r   ræ   rç     s   ø€  ! C¨¨4¯;©;¨¸Ð$9Ö":r   rÝ   r�   )rŸ   r]   r•   r3   r×   rx   rË   Ústepr
   r–   ÚquantizerÊ   r"   r    )ri   r•   r–   ÚBÚ
n_kv_headsÚ	num_stepsÚ
k_head_dimÚ
v_head_dimÚ	new_stepsrÚ   rà   ÚirÙ   rå   rŸ   s   ff&         @@@r   r¡   Ú!QuantizedKVCache.update_and_fetchò   sï  ü€ Ø/3¯z©zÑ,ˆ�yØ—\‘\ "Õ%ˆ
Ø�{‰{ˆà�9‰9Ò ¨	Õ!1°T·Y±Y¸qµ\×5GÑ5GÈÕ5KÕ KØœRŸY™YŸ^™^Õ+¨t¯y©yÕ8ˆJØŸ™ YÕ.°Õ2°t·y±yÕ@À4Ç9Á9ÕLˆIØ IÐ.ˆE÷ð õ;ð �y‰yÒ$Ø˜$Ÿ)™)Õ# qÔ(Ü-5Ü2°T·Y±YÀÇÁÐ4Ló.Ñ*�D”I˜tœ{ô *2Ø  4§9¡9¨d¯k©kÐ":ó*Ñ&�”	˜4�;ñ *4°JÓ)?ÁÈJÓAWÐ&�”	˜4œ;à�Š�yÕ �ä�{Š{˜4¨D¯O©OÀ$Ç)Á)ÔLˆÜ—’˜V°·±ÀdÇiÁiÔPˆÜ”s˜4Ÿ9™9“~Ö&ˆAØ7;¸AµwˆD�I‰I�a�L˜˜d T§[¡[Ð0°!Ð3Ñ4Ø9?½ˆD�K‰K˜�N˜3  t§{¡{Ð 2°AÐ5Ó6ñ 'ô Ô:¸T¿Y¹YÈÏÉÐ<TÓUÐUr   c                óæ   a € S P                   S P                  ^ ,          P                  ^,          8X  d   S P                  S P                  3# \	        V 3R lS P                  S P                  34      # )r   c                 ó4   <€ V R RSP                   1R3,          # rã   r´   rê   s   &€r   ræ   Ú(QuantizedKVCache.state.<locals>.<lambda>#  s   ø€ ˜!˜C  4§;¡; °Ð1Ö2r   )r]   r•   rŸ   r–   r
   rh   s   fr   r.   ÚQuantizedKVCache.state  sT   ø€ à�;‰;˜$Ÿ)™) A�,×,Ñ,¨QÕ/Ô/Ø—9‘9˜dŸk™kÐ)Ð)äÜ2°T·Y±YÀÇÁÐ4Lóð r   c                ó$   € Vw  V n         V n        R # rK   r¤   rn   s   &&r   r.   rø   &  ó   € à!"ÑˆŒ	�4–;r   c                óv   € \        \        \        V P                  V P                  V P
                  34      4      # rK   )ÚtupleÚmapr,   r]   rÊ   rË   rh   s   &r   r/   ÚQuantizedKVCache.meta_state*  s'   € ä”Sœ˜tŸ{™{¨D¯O©O¸T¿Y¹YÐGÓHÓIÐIr   c                óL   € \        \        V4      w  V n        V n        V n        R # rK   )rý   r   r]   rÊ   rË   rn   s   &&r   r/   rþ   .  s   € ä25´c¸1³+Ñ/ˆŒ�T”_ d¦ir   c                ó   € R # r©   rg   rh   s   &r   rM   ÚQuantizedKVCache.is_trimmable2  r«   r   c                ód   € \        V P                  V4      pV ;P                  V,          un        V# rK   r­   r¯   s   &&r   rY   ÚQuantizedKVCache.trim5  r²   r   c                ó0   € \        VR V P                  /VB # r´   rµ   r¶   s   &*,r   r¹   ÚQuantizedKVCache.make_mask:  r»   r   c                ó   € V P                   R J # rK   r½   rh   s   &r   r€   ÚQuantizedKVCache.empty=  r¿   r   c                óJ   € \        R  V P                  V P                  3^ 4      # )c                 ó&   € WP                   ,           # rK   ©r}   )ÚarÞ   s   &&r   ræ   Ú)QuantizedKVCache.nbytes.<locals>.<lambda>B  s
   € ¨¯H©H®r   )r   r•   r–   rh   s   &r   r}   ÚQuantizedKVCache.nbytes@  s    € äÑ4°t·y±yÀ$Ç+Á+Ð6NÐPQÓRÐRr   )rË   rÊ   r•   r]   r–   N)é@   é   )r2   r‡   rˆ   r‰   rë   r—   r¡   rŠ   r.   r‹   r/   rM   rY   r¹   r€   r}   r�   rŽ   r�   s   @r   rÇ   rÇ   è   s¢   ø‡ € Ø€D÷ò ò)VðV ñó ðð ‡\�\ñ#ó ð#ð ñJó ðJð ×Ññ>ó ð>òòò
Jò!ð ñSó öSr   rÇ   c                   ó¸   a € ] tR tRt o RtR tR tR t]R 4       t	]	P                  R 4       t	R tR	 tRV 3R
 lR lltR t]R 4       tR t]R 4       tRtV tR# )r$   iE  rÈ   c                ó0   € R V n         R V n        ^ V n        R # rK   r”   rh   s   &r   r—   ÚKVCache.__init__H  r™   r   c                ó  € V P                   pV P                  e;   W1P                  ^,          ,           V P                  P                  ^,          8”  Ed‰   VP                  w  rErgVP                  ^,          pV P                  VP                  ^,          ,           ^,
          V P                  ,          p	WEW�P                  ,          V3p
WEW�P                  ,          V3p\        P
                  ! W¡P                  4      p\        P
                  ! W²P                  4      pV P                  e¥   W0P                  ,          ^ 8w  d;   V P                  RR V1R3,          V n        V P                  RR V1R3,          V n        \        P                  ! V P                  V.^R7      V n        \        P                  ! V P                  V.^R7      V n        MWÍuV n        V n        V ;P                   VP                  ^,          ,          un         WP                  RW0P                   1R3&   W P                  RW0P                   1R3&   V P                  RR V P                   1R3,          V P                  RR V P                   1R3,          3# ©N.rä   r›   )	r]   r•   rŸ   rë   r3   rÖ   rÕ   r–   rž   ©ri   r•   r–   rå   rí   rî   r&   rð   rñ   Ún_stepsÚk_shapeÚv_shapeÚnew_kÚnew_vs   &&&           r   r¡   ÚKVCache.update_and_fetchM  sÔ  € Ø�{‰{ˆØ�9‰9Ò ¯
©
°1­Õ!5¸¿¹¿¹ÈÕ9KÕ KØ+/¯:©:Ñ(ˆA˜1ØŸ™ a�ˆJØ—y‘y 4§:¡:¨a¥=Õ0°1Õ4¸¿¹ÕBˆGØ g·	±	Õ&9¸:ÐFˆGØ g·	±	Õ&9¸:ÐFˆGÜ—H’H˜W§j¡jÓ1ˆEÜ—H’H˜W§l¡lÓ3ˆEØ�y‰yÒ$ØŸ)™)Õ# qÔ(Ø $§	¡	¨#¨u°¨u°a¨-Õ 8�D”IØ"&§+¡+¨c°5°D°5¸!¨mÕ"<�D”KÜŸNšN¨D¯I©I°uÐ+=ÀAÔF�”	Ü Ÿnšn¨d¯k©k¸5Ð-AÈÔJ�•à).Ð&�”	˜4œ;à�Š�t—z‘z !•}Õ$�Ø04�	‰	�#�tŸk™kÐ)¨1Ð,Ñ-Ø28�‰�C˜§¡Ð+¨QÐ.Ñ/Ø�y‰y˜˜m §¡˜m¨QÐ.Õ/°·±¸SÀ-ÀDÇKÁKÀ-ÐQRÐ=RÕ1SÐSÐSr   c                ó   € V P                   # rK   r´   rh   s   &r   rx   ÚKVCache.sizee  s   € Ø�{‰{Ðr   c                ó  € V P                   V P                  P                  ^,          8X  d   V P                  V P                  3# V P                  RRV P                   1R3,          V P                  RRV P                   1R3,          3# ©r   .Nrä   ©r]   r•   rŸ   r–   rh   s   &r   r.   ÚKVCache.stateh  óp   € à�;‰;˜$Ÿ)™)Ÿ/™/¨!Õ,Ô,Ø—9‘9˜dŸk™kÐ)Ð)ð —	‘	˜#˜} §¡˜}¨aÐ/Õ0Ø—‘˜C  4§;¡; °Ð1Õ2ðð r   c                óh   € Vw  V n         V n        V P                   P                  ^,          V n        R# ©r   Nr§   rn   s   &&r   r.   r!  r  ó$   € à!"ÑˆŒ	�4”;Ø—i‘i—o‘o aÕ(ˆŽr   c                ó   € R # r©   rg   rh   s   &r   rM   ÚKVCache.is_trimmablew  r«   r   c                ód   € \        V P                  V4      pV ;P                  V,          un        V# rK   r­   r¯   s   &&r   rY   ÚKVCache.trimz  r²   r   c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[/# ©r   rÊ   rË   r   ©r   rÇ   )r   r�   s   "€r   r   ÚKVCache.__annotate__  s#   ø€ ÷ ñ ¡sð ±sð ÑCSñ r   c                óø   € \        WR 7      pV P                  Vn        V P                  eO   \        P                  ! V P                  WR 7      Vn        \        P                  ! V P
                  WR 7      Vn        V# )rè   )rÇ   r]   r•   r3   rì   r–   )ri   rÊ   rË   Úquant_caches   &&& r   Úto_quantizedÚKVCache.to_quantized  s[   € Ü&°*ÔHˆØ!Ÿ[™[ˆÔØ�9‰9Ò Ü!Ÿ{š{¨4¯9©9ÀÔWˆKÔÜ!#§¢Ø—‘¨
ô"ˆKÔð Ðr   c                ó0   € \        VR V P                  /VB # r´   rµ   r¶   s   &*,r   r¹   ÚKVCache.make_mask‰  r»   r   c                ó,   € \         P                  V4      # rK   )ÚBatchKVCacheÚmerge©r&   Úcachess   &&r   r6  ÚKVCache.mergeŒ  s   € ä×!Ñ! &Ó)Ð)r   c                ó   € V P                   R J # rK   r½   rh   s   &r   r€   ÚKVCache.empty�  r¿   r   c                ó„   € V P                   f   ^ # V P                   P                  V P                  P                  ,           # rK   rÁ   rh   s   &r   r}   ÚKVCache.nbytes“  rÃ   r   rÄ   N©r  é   )r2   r‡   rˆ   r‰   rë   r—   r¡   rx   rŠ   r.   r‹   rM   rY   r0  r¹   rŒ   r6  r€   r}   r�   rŽ   r�   s   @r   r$   r$   E  s�   ø‡ € Ø€Dòò
Tò0ð ñó ðð ‡\�\ñ)ó ð)òò÷
ò òJð ñ*ó ð*ò!ð ñ5ó ö5r   r$   c                   ó  a € ] tR tRt o RtRR ltRR ltR tR tR t	R	 t
R
 t]R 4       t]P                  R 4       t]R 4       t]P                  R 4       tR tR tRV 3R lR lltRV 3R lR llt]R 4       tR t]R 4       tRtV tR# )r#   iš  rÈ   c                óV   € W n         R V n        R V n        ^ V n        Wn        ^ V n        R # rK   )r   r•   r–   r]   r   Ú_idx)ri   r   r   s   &&&r   r—   ÚRotatingKVCache.__init__�  s(   € ØŒ	ØˆŒ	ØˆŒØˆŒØ ŒØˆŽ	r   Nc                óâ   € . pV^ 8”  d8   VRRV P                   1R3,          VRWP                   ,           R1R3,          .pMV.pVe   VP                  V4       \        P                  ! V^R7      # ©r   .Nrä   r›   )r   Úappendr3   rž   )ri   Ú	trim_sizero   rF  Úto_cats   &&&& r   Ú_trimÚRotatingKVCache._trim¥  sl   € ØˆØ�qŒ=Ø˜˜[˜tŸy™y˜[¨!Ð+Õ,¨a°°YÇÁÕ5JÑ5LÈaÐ0OÕ.PÐQ‰Fà�SˆFØÒØ�M‰M˜&Ô!Ü�~Š~˜f¨1Ô-Ð-r   c                ó~  € V P                   VP                  ^,          8X  d   V# V P                   V P                  8  dh   \        P                  ! VRRV P
                  1R3,          VRV P                   R1R3,          VRV P
                  V P                   1R3,          .^R7      # VRRV P                   1R3,          # )zI
Rearrange the cache into temporal order, slicing off the end if unused.
.Nrä   r›   )rB  rŸ   r]   r3   rž   r   rn   s   &&r   Ú_temporal_orderÚRotatingKVCache._temporal_order¯  s¬   € ð �9‰9˜Ÿ™ �
Ô"ØˆHØ�Y‰Y˜Ÿ™Ô$Ü—>’>à�c˜;˜TŸY™Y˜;¨Ð)Õ*Ø�c˜4Ÿ9™9™;¨Ð)Õ*Ø�c˜4Ÿ9™9 t§y¡yÐ0°!Ð3Õ4ðð
 ôð ð �S˜+˜DŸI™I˜+ qÐ(Õ)Ð)r   c                ó�  € V P                   f   Wn         W n        MÈV P                  V P                   4      V n         V P                  V P                  4      V n        V P                   P                  ^,          V n        V P                  V P
                  ,
          ^,           pV P                  W0P                   V4      V n         V P                  W0P                  V4      V n        V ;P                  VP                  ^,          ,          un        V P                   P                  ^,          V n        V P                   V P                  3# rK   )r•   r–   rL  rŸ   rB  r   rI  r]   )ri   r•   r–   rG  s   &&& r   Ú_update_concatÚRotatingKVCache._update_concatÁ  sÚ   € Ø�9‰9ÒØŒIØ �Kð ×,Ñ,¨T¯Y©YÓ7ˆDŒIØ×.Ñ.¨t¯{©{Ó;ˆDŒKØŸ	™	Ÿ™¨Õ*ˆDŒIð Ÿ	™	 D§M¡MÕ1°AÕ5ˆIØŸ
™
 9¯i©i¸Ó>ˆDŒIØŸ*™* Y·±¸VÓDˆDŒKØ�Š�t—z‘z !•}Õ$�Ø—I‘I—O‘O AÕ&ˆŒ	Ø�y‰y˜$Ÿ+™+Ð%Ð%r   c                óL  € VP                   w  r4rVV P                  pV P                  eO   WpP                  P                   ^,          8¼  Ed$   V P                  P                   ^,          V P                  8  dø   VP                   ^,          p\	        V P
                  V P                  V,
          4      p	W4W–3p
W4W˜3p\        P                  ! W¡P                  4      p\        P                  ! W²P                  4      pV P                  eT   \        P                  ! V P                  V.^R7      V n        \        P                  ! V P                  V.^R7      V n
        MWÍuV n        V n
        Wpn        V P                  P                   ^,          V P                  ,
          pV^ 8”  dR   V P                  WàP                  4      V n        V P                  WàP                  4      V n
        V P                  V n        V P                  V P                  8X  d   V P                  V n        WP                  RV P                  V P                  V,           1R3&   W P                  RV P                  V P                  V,           1R3&   V ;P                  V,          un        V ;P                  V,          un        V P                  V P                  8  dE   V P                  RR V P                  1R3,          V P                  RR V P                  1R3,          3# V P                  V P                  3# )Nr›   .rä   )rŸ   r]   r•   r   r®   rë   r3   rÖ   rÕ   rž   r–   rB  rI  r   ©ri   r•   r–   rí   rî   ÚSrð   rå   rñ   Únew_sizer  r  r  r  rG  s   &&&            r   Ú_update_in_placeÚ RotatingKVCache._update_in_placeÕ  s;  € ð (,§z¡zÑ$ˆ�qØ�{‰{ˆØ�9‰9ÒØ—I‘I—O‘O AÕ&Õ&¨4¯9©9¯?©?¸1Õ+=ÀÇÁÔ+MàŸ™ a�ˆJÜ˜4Ÿ9™9 d§m¡m°dÕ&:Ó;ˆHØ hÐ;ˆGØ hÐ;ˆGÜ—H’H˜W§j¡jÓ1ˆEÜ—H’H˜W§l¡lÓ3ˆEØ�y‰yÒ$ÜŸNšN¨D¯I©I°uÐ+=ÀAÔF�”	Ü Ÿnšn¨d¯k©k¸5Ð-AÈÔJ�•à).Ð&�”	˜4œ;ØŒIð —I‘I—O‘O AÕ&¨¯©Õ6ˆ	Ø�qŒ=ØŸ
™
 9¯i©iÓ8ˆDŒIØŸ*™* Y·±Ó<ˆDŒKØŸ™ˆDŒIð �9‰9˜Ÿ™Ô%ØŸ	™	ˆDŒIð 8<�	‰	�#�t—y‘y 4§9¡9¨q¥=Ð0°!Ð3Ñ4Ø9?�‰�C˜Ÿ™ T§Y¡Y°¥]Ð2°AÐ5Ñ6Ø�Š�qÕ�Ø�	Š	�Q��	ð �;‰;˜Ÿ™Ô&Ø—9‘9˜S - D§K¡K -°Ð2Õ3°T·[±[ÀÀmÈÏÉÀmÐUVÐAVÕ5WÐWÐWØ�y‰y˜$Ÿ+™+Ð%Ð%r   c                óv   € VP                   ^,          ^8X  d   V P                  W4      # V P                  W4      # ©r   ©rŸ   rU  rO  r    s   &&&r   r¡   Ú RotatingKVCache.update_and_fetch   ó4   € Ø�:‰:�a�=˜AÔØ×(Ñ(¨Ó6Ð6Ø×"Ñ" 4Ó0Ð0r   c                óB   € \        V P                  V P                  4      # rK   )r®   r]   r   rh   s   &r   rx   ÚRotatingKVCache.size  s   € Ü�4—;‘; §¡Ó.Ð.r   c                ó  € V P                   V P                  P                  ^,          8  dE   V P                  RRV P                   1R3,          V P                  RRV P                   1R3,          3# V P                  V P                  3# r  r   rh   s   &r   r.   ÚRotatingKVCache.state  sj   € à�;‰;˜Ÿ™Ÿ™¨Õ+Ô+Ø—9‘9˜S - D§K¡K -°Ð2Õ3°T·[±[ÀÀmÈÏÉÀmÐUVÐAVÕ5WÐWÐWà—9‘9˜dŸk™kÐ)Ð)r   c                ó$   € Vw  V n         V n        R # rK   r¤   rn   s   &&r   r.   r_    rú   r   c           	     óŒ   € \        \        \        V P                  V P                  V P
                  V P                  34      4      # rK   )rü   rý   r,   r   r   r]   rB  rh   s   &r   r/   ÚRotatingKVCache.meta_state  s-   € ä”Sœ˜tŸy™y¨$¯-©-¸¿¹ÀdÇiÁiÐPÓQÓRÐRr   c                óX   € \        \        V4      w  V n        V n        V n        V n        R # rK   )rý   r   r   r   r]   rB  rn   s   &&r   r/   rb    s#   € ä;>ÜØó<
Ñ8ˆŒ	�4”= $¤+¨t®yr   c                ó4   € V P                   V P                  8  # rK   )r]   r   rh   s   &r   rM   ÚRotatingKVCache.is_trimmable  s   € Ø�{‰{˜TŸ]™]Ñ*Ð*r   c                ó–   € \        V P                  V4      pV ;P                  V,          un        V ;P                  V,          un        V# rK   )r®   r]   rB  r¯   s   &&r   rY   ÚRotatingKVCache.trim!  s2   € Ü�—‘˜QÓˆØ�Š�qÕ�Ø�	Š	�Q��	Øˆr   c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[/# r+  r,  )r   r�   s   "€r   r   ÚRotatingKVCache.__annotate__'  s(   ø€ ÷ Fñ F¡sð F±sð FÑCSñ Fr   c                ó   € \        R 4      h)z RotatingKVCache Quantization NYIr{   rÏ   s   &&&r   r0  ÚRotatingKVCache.to_quantized'  s   € Ü!Ð"DÓEÐEr   c                ó<   <€ V ^8„  d   QhRS[ RS[S[ ,          RS[/# ©r   r\   r_   r^   ©r   r   rH   )r   r�   s   "€r   r   ri  *  s)   ø€ ÷ ñ ÙðÙ#+©C¥=ðÙGKñr   c                óZ  € V^8”  db   T;'       g    V P                   p\        V P                   ^,
          V P                  4      pWA,           V8”  g	   V'       d   \        WVR7      # R# Vf   R# V P                  V8¼  d¬   V P                   V8”  d™   V P                  pWPP                   8¼  d   ^ pV P                  V P                   8  d   V P                  ^,           pMV P                   p\
        P                  ! V4      Wb,
          8¬  p\
        P                  ! Wu^,           R7      pV# R# R# )rÒ   ra   rb   N)Úshift)r   r®   r]   r   rB  r3   ÚarangeÚroll)ri   r\   r_   r^   r]   ÚidxÚ	mask_sizeÚmasks   &&&&    r   r¹   ÚRotatingKVCache.make_mask*  så   € ð ˆqŒ5Ø%×6Ð6¨¯©ˆKÜ˜Ÿ™¨Õ*¨D¯K©KÓ8ˆFØ�z˜KÔ'¯<Ü)¨!ÀÔMÐMáàÒ"Ùà�{‰{˜kÔ)¨d¯m©m¸kÔ.IØ—i‘i�ØŸ-™-Ô'Ø�CØ—;‘; §¡Ô.Ø $§¡¨a¥‘Ià $§¡�IÜ—y’y Ó+°	Õ0GÑH�Ü—w’w˜t°­7Ô3�Ø�ñ /JÑ)r   c                ó,   € \         P                  V4      # rK   )ÚBatchRotatingKVCacher6  r7  s   &&r   r6  ÚRotatingKVCache.mergeD  s   € ä#×)Ñ)¨&Ó1Ð1r   c                ó   € V P                   R J # rK   r½   rh   s   &r   r€   ÚRotatingKVCache.emptyH  r¿   r   c                ó„   € V P                   f   ^ # V P                   P                  V P                  P                  ,           # rK   rÁ   rh   s   &r   r}   ÚRotatingKVCache.nbytesK  rÃ   r   )rB  r   r•   r   r]   r–   ©r   rK   r>  ©NF)r2   r‡   rˆ   r‰   rë   r—   rI  rL  rO  rU  r¡   rx   rŠ   r.   r‹   r/   rM   rY   r0  r¹   rŒ   r6  r€   r}   r�   rŽ   r�   s   @r   r#   r#   š  s×   ø‡ € Ø€Dôô.ò*ò$&ò()&òV1ò
/ð ñ*ó ð*ð ‡\�\ñ#ó ð#ð ñSó ðSð ×Ññ
ó ð
ò+ò÷Fò F÷ò ð4 ñ2ó ð2ò!ð ñ5ó ö5r   r#   c                   óø   a a€ ] tR tRt oV 3R ltRV3R lR llt]R 4       tR tR t	]R 4       t
]
P                  R	 4       t
R
 tR tR tRR ltR tR tV3R lR lt]R 4       tR t]R 4       tRtVtV ;t# )ÚArraysCacheiR  c                óB   <€ \         SV `  V 4      pR Vn        R Vn        V# rK   )Úsuperrƒ   Úleft_paddingÚlengths)r„   r·   r¸   ÚinstanceÚ	__class__s   &*, €r   rƒ   ÚArraysCache.__new__S  s&   ø€ Ü‘7‘? 3Ó'ˆØ $ˆÔØˆÔØˆr   c                ó@   <€ V ^8„  d   QhRS[ S[S[,          ,          /# ©r   r„  )r   r   r   )r   r�   s   "€r   r   ÚArraysCache.__annotate__Y  s   ø€ ÷ 7ñ 7©8±D¹µIÕ+>ñ 7r   c                ón   € R .V,          V n         V'       d   \        P                  ! V4      V n        R # R # rK   )r*   r3   Úarrayr„  )ri   rx   r„  s   &&&r   r—   ÚArraysCache.__init__Y  s(   € Ø�V˜d•]ˆŒ
ßÜ "§¢¨Ó 6ˆDÖñ r   c                óî   € V P                    F  pVf   K	  VP                  ^ ,          u # 	  V P                  e   V P                  P                  # V P                  e   V P                  P                  # ^# rK   )r*   rŸ   r„  rx   r…  ©ri   r5   s   & r   Ú
batch_sizeÚArraysCache.batch_size^  s`   € à—”ˆAØŒ}Ø—w‘w˜q•zÒ!ñ ð ×ÑÒ(Ø×$Ñ$×)Ñ)Ð)Ø�\‰\Ò%Ø—<‘<×$Ñ$Ð$ár   c                ó"   € W P                   V&   R # rK   rT   )ri   rs  Úvalues   &&&r   Ú__setitem__ÚArraysCache.__setitem__j  s   € Ø�
‰
�3‹r   c                ó(   € V P                   V,          # rK   rT   ©ri   rs  s   &&r   Ú__getitem__ÚArraysCache.__getitem__m  s   € Ø�z‰z˜#�Ðr   c                ó   € V P                   # rK   rT   rh   s   &r   r.   ÚArraysCache.statep  s   € à�z‰zÐr   c                ó   € Wn         R # rK   rT   rn   s   &&r   r.   rœ  t  s   € àŽ
r   c                ó  € V P                    Uu. uF  q"e	   W!,          MRNK  	  upV n         V P                  e   V P                  V,          V n        V P                  e   V P                  V,          V n        R# R# u upi ©ú>
In-place filter to keep just the given indices in the cache.
N©r*   r„  r…  ©ri   Úbatch_indicesr5   s   && r   ÚfilterÚArraysCache.filterx  sn   € ð LPÏ:Ê:ÓVÉ:Àaª-�aÖ&¸TÒAÉ:ÑVˆŒ
Ø×ÑÒ(Ø $× 1Ñ 1°-Õ @ˆDÔØ�<‰<Ò#ØŸ<™<¨Õ6ˆDŽLñ $ùò Ws   �A=c                óZ  aa€ V P                   oVP                   oVV3R lp\        V P                  VP                  4       UUu. uF  w  r4V! W44      NK  	  uppV n        V! V P                  VP                  4      V n        V! V P                  VP                  4      V n        R# u uppi )ú2
In-place extend this cache with the other cache.
c                 ó`  <€ R ;r#V e   V P                   pV P                  pVe   VP                   pVP                  pVf   R # V f(   \        P                  ! S3VR,          ,           VR7      p Vf(   \        P                  ! S3VR,          ,           VR7      p\        P                  ! W.4      # )N:rÒ   NNrÔ   )rŸ   rÕ   r3   rÖ   rž   )r  ÚbrŸ   rÕ   Úa_batchÚb_batchs   &&  €€r   ÚcatÚArraysCache.extend.<locals>.catŠ  s’   ø€ Ø Ð ˆEØŠ}ØŸ™�ØŸ™�ØŠ}ØŸ™�ØŸ™�àŠ}ÙàŠyÜ—H’H˜g˜Z¨%°­)Õ3¸5ÔA�ØŠyÜ—H’H˜g˜Z¨%°­)Õ3¸5ÔA�ä—>’> 1 &Ó)Ð)r   N)r‘  r@   r*   r„  r…  )ri   Úotherr¬  r5   Úorª  r«  s   &&   @@r   ÚextendÚArraysCache.extend‚  s„   ù€ ð
 —/‘/ˆØ×"Ñ"ˆö	*ô& -0°·
±
¸E¿K¹KÔ,HÔIÑ,H¡D A‘c˜!–iÑ,HÒIˆŒ
Ù × 1Ñ 1°5×3EÑ3EÓFˆÔÙ˜4Ÿ<™<¨¯©Ó7ˆŽùó Js   ÁB'c                óž   € \        \        V P                  4      4      pV P                   Uu. uF  q3W^,            NK  	  upVn        V# u upi )rÒ   )r�  r    r*   )ri   rs  r*   r5   s   &&  r   ÚextractÚArraysCache.extract¡  s?   € ÜœC §
¡
›OÓ,ˆØ15·²Ó<±¨A˜ Q�wÓ'±Ñ<ˆŒØˆùò =s   ­A
c                ó<   € \         P                  ! V4      V n        R # rK   )r3   r�  r…  )ri   r…  r¸   s   &&,r   ÚprepareÚArraysCache.prepare¦  s   € Ü—x’x Ó(ˆŽr   c                ó"   € R V n         R V n        R # rK   ©r…  r„  rh   s   &r   ÚfinalizeÚArraysCache.finalize©  s   € ØˆŒØ ˆÖr   c                ó¦   € V P                   e   V ;P                   V,          un         V P                  e   V ;P                  V,          un        R # R # rK   r¹  )ri   r\   s   &&r   ÚadvanceÚArraysCache.advance­  s<   € Ø�<‰<Ò#Ø�LŠL˜AÕ�LØ×ÑÒ(Ø×Ò Õ"×ñ )r   c                ó    <€ V ^8„  d   QhRS[ /# )r   r\   rÌ   )r   r�   s   "€r   r   r‹  ³  s   ø€ ÷ ñ ™3ñ r   c                óê   € V P                   e,   \        P                  ! V4      pW P                   R,          8¬  # V P                  e,   \        P                  ! V4      pW P                  R,          8  # R # )N©rä   N)r„  r3   rq  r…  )ri   r\   Úposs   && r   r¹   ÚArraysCache.make_mask³  s[   € Ø×ÑÒ(Ü—)’)˜A“,ˆCØ×+Ñ+¨GÕ4Ñ4Ð4Ø�\‰\Ò%Ü—)’)˜A“,ˆCØŸ™ gÕ.Ñ.Ð.ár   c                ó„  a€ \        V^ ,          P                  4      p\        V4      pV ! V4      p\        ;QJ d    R V 4       F  '       d   K   RM	  RM! R V 4       4      '       d&   \        P                  ! ^ .V,          4      Vn        V# \        V4       F¡  o\        \        V3R lV 4       4      4      p\        VP                  4      pW6^ &   \        P                  ! WeP                  4      VS&   \        V4       F5  pW,          S,          f   K  W,          S,          VS,          Ww^,           % K7  	  K£  	  V# )r   c              3   ó@   "  € T F  qP                  4       x € K  	  R # 5irK   )r€   rN   s   & r   rP   Ú$ArraysCache.merge.<locals>.<genexpr>Ä  s   é € Ð)¡&˜Q�w‰w�yˆy£&ùrR   FTc              3   óN   <"  € T F  qS,          f   K  VS,          x € K  	  R # 5irK   rg   )rO   r5   Úes   & €r   rP   rÆ  É  s   øé € ÐH©V¨¸µtœt˜q Ÿtšt«Vùs   ƒ%”%)r    r*   rS   r3   r�  r„  r"   ÚnextÚiterr>   rŸ   rÖ   rÕ   )	r„   r8  Ún_staterí   r*   Úc_initrŸ   ró   rÈ  s	   &&      @r   r6  ÚArraysCache.merge½  sð   ø€ ä�f˜Q•i—o‘oÓ&ˆÜ�‹KˆÙ�G“ˆ÷ ‹3Ñ)¡&Ó)�3�3Š3Ñ)¡&Ó)×)Ò)Ü!#§¢¨1¨#°­'Ó!2ˆEÔØˆLä�w–ˆAÜœ$ÔH©VÓHÓHÓIˆFÜ˜Ÿ™Ó&ˆEØ�!‰HÜ—x’x §|¡|Ó4ˆE�!‰HÜ˜1–X�Ø•9˜Q•<Ò'ÙØ&,¥i°¥l��a•˜ �UÒ#ó ñ  ð ˆr   c                ó,   € V P                   ^ ,          RJ # ©r   NrT   rh   s   &r   r€   ÚArraysCache.emptyÓ  s   € Ø�z‰z˜!�} Ð$Ð$r   c                ó:   € \        R  V P                   4       4      # )c              3   óD   "  € T F  qf   K  VP                   x € K  	  R # 5irK   r
  rN   s   & r   rP   Ú%ArraysCache.nbytes.<locals>.<genexpr>Ø  s   é € ÐA¡Z ”8�1—8–8£Zùs   ‚ Œ )Úsumr*   rh   s   &r   r}   ÚArraysCache.nbytesÖ  s   € äÑA T§Z¢ZÓAÓAÐAr   r¡  rK   )r2   r‡   rˆ   r‰   rƒ   r—   rŠ   r‘  r•  r™  r.   r‹   r¤  r°  r³  r¶  rº  r½  r¹   rŒ   r6  r€   r}   r�   rŽ   Ú__classcell__)r‡  r�   s   @@r   r�  r�  R  s»   ù‡ € õ÷7ò 7ð
 ñ	ó ð	ò òð ñó ðð ‡\�\ñó ðò7ò8ò>ô
)ò!ò#÷ð ð ñó ðò*%ð ñBó ÷Bð Br   r�  c                   óÀ   a € ] tR tRt o RtR tR tR t]R 4       t	]	P                  R 4       t	R tR	 t]R
 4       t]P                  R 4       tR t]R 4       tRtV tR# )ÚChunkedKVCacheiÛ  rÈ   c                óJ   € R V n         R V n        ^ V n        Wn        ^ V n        R # rK   )r•   r–   r]   Ú
chunk_sizeÚstart_position)ri   rÚ  s   &&r   r—   ÚChunkedKVCache.__init__Þ  s$   € ØˆŒ	ØˆŒØˆŒØ$ŒØˆÖr   c                ó¬  € V P                   eÆ   V P                   P                  ^,          V P                  8¼  d˜   V ;P                  V P                   P                  ^,          V P                  ,
          ,          un        V P                   RV P                  ) R 1R3,          V n         V P                  RV P                  ) R 1R3,          V n        R # R # R # )N.rä   )r•   rŸ   rÚ  rÛ  r–   rh   s   &r   Úmaybe_trim_frontÚChunkedKVCache.maybe_trim_frontå  s–   € à�9‰9Ò  T§Y¡Y§_¡_°QÕ%7¸4¿?¹?Ô%JØ×Ò 4§9¡9§?¡?°1Õ#5¸¿¹Õ#GÕGÕØŸ	™	 #¨¯©Ð'7Ñ'9¸1Ð"<Õ=ˆDŒIØŸ+™+ c¨D¯O©OÐ+;Ñ+=¸qÐ&@ÕAˆDŽKñ &KÑ r   c                ó  € V P                   V P                  ,
          pV P                  e;   W1P                  ^,          ,           V P                  P                  ^,          8”  Ed‰   VP                  w  rErgVP                  ^,          pV P                  VP                  ^,          ,           ^,
          V P                  ,          p	WEW�P                  ,          V3p
WEW�P                  ,          V3p\
        P                  ! W¡P                  4      p\
        P                  ! W²P                  4      pV P                  e¥   W0P                  ,          ^ 8w  d;   V P                  RR V1R3,          V n        V P                  RR V1R3,          V n        \
        P                  ! V P                  V.^R7      V n        \
        P                  ! V P                  V.^R7      V n        MWÍuV n        V n        V ;P                   VP                  ^,          ,          un         V P                   V P                  ,
          pWP                  RW>1R3&   W P                  RW>1R3&   V P                  RR V1R3,          V P                  RR V1R3,          3# r  )
r]   rÛ  r•   rŸ   rë   r3   rÖ   rÕ   r–   rž   )ri   r•   r–   rå   rí   rî   r&   rð   rñ   r  r  r  r  r  Úends   &&&            r   r¡   ÚChunkedKVCache.update_and_fetchì  sÞ  € Ø�{‰{˜T×0Ñ0Õ0ˆØ�9‰9Ò ¯
©
°1­Õ!5¸¿¹¿¹ÈÕ9KÕ KØ+/¯:©:Ñ(ˆA˜1ØŸ™ a�ˆJØ—y‘y 4§:¡:¨a¥=Õ0°1Õ4¸¿¹ÕBˆGØ g·	±	Õ&9¸:ÐFˆGØ g·	±	Õ&9¸:ÐFˆGÜ—H’H˜W§j¡jÓ1ˆEÜ—H’H˜W§l¡lÓ3ˆEØ�y‰yÒ$ØŸ)™)Õ# qÔ(Ø $§	¡	¨#¨u°¨u°a¨-Õ 8�D”IØ"&§+¡+¨c°5°D°5¸!¨mÕ"<�D”KÜŸNšN¨D¯I©I°uÐ+=ÀAÔF�”	Ü Ÿnšn¨d¯k©k¸5Ð-AÈÔJ�•à).Ð&�”	˜4œ;à�Š�t—z‘z !•}Õ$�Ø�k‰k˜D×/Ñ/Õ/ˆØ&*�	‰	�#�t�x Ð"Ñ#Ø(.�‰�C˜˜ 1Ð$Ñ%Ø�y‰y˜˜d˜s˜d A˜Õ&¨¯©°C¸¸#¸¸q°LÕ(AÐAÐAr   c                ó  € V P                   V P                  P                  ^,          8X  d   V P                  V P                  3# V P                  RRV P                   1R3,          V P                  RRV P                   1R3,          3# r  r   rh   s   &r   r.   ÚChunkedKVCache.state  r"  r   c                óh   € Vw  V n         V n        V P                   P                  ^,          V n        R# r$  r§   rn   s   &&r   r.   rä    r%  r   c                ó   € R # r©   rg   rh   s   &r   rM   ÚChunkedKVCache.is_trimmable  r«   r   c                ó†   € \        V P                  V P                  ,
          V4      pV ;P                  V,          un        V# rK   )r®   r]   rÛ  r¯   s   &&r   rY   ÚChunkedKVCache.trim  s0   € Ü�—‘˜d×1Ñ1Õ1°1Ó5ˆØ�Š�qÕ�Øˆr   c                ó`   € \        \        \        V P                  V P                  34      4      # rK   )rü   rý   r,   rÚ  rÛ  rh   s   &r   r/   ÚChunkedKVCache.meta_state  s#   € ä”Sœ˜tŸ™°×0CÑ0CÐDÓEÓFÐFr   c                ó@   € \        \        V4      w  V n        V n        R # rK   )rý   r   rÚ  rÛ  rn   s   &&r   r/   rë     s   € ä/2´3¸«{Ñ,ˆŒ˜Ö,r   c                ó   € V P                   R J # rK   r½   rh   s   &r   r€   ÚChunkedKVCache.empty$  r¿   r   c                ó„   € V P                   f   ^ # V P                   P                  V P                  P                  ,           # rK   rÁ   rh   s   &r   r}   ÚChunkedKVCache.nbytes'  rÃ   r   )rÚ  r•   r]   rÛ  r–   N)r2   r‡   rˆ   r‰   rë   r—   rÞ  r¡   rŠ   r.   r‹   rM   rY   r/   r€   r}   r�   rŽ   r�   s   @r   rØ  rØ  Û  sš   ø‡ € Ø€Dò òBòBð2 ñó ðð ‡\�\ñ)ó ð)òòð
 ñGó ðGð ×Ññ;ó ð;ò!ð ñ5ó ö5r   rØ  c                   óú   a € ] tR tRt o R tR tR tR t]R 4       t	]	P                  R 4       t	]R 4       t]P                  R	 4       tR
 tR t]R 4       tR tR tR tR tR t]R 4       t]R 4       tRtV tR# )Ú	CacheListi.  c                ó   € Wn         R # rK   ©r8  )ri   r8  s   &*r   r—   ÚCacheList.__init__/  s   € ØŽr   c                ó(   € V P                   V,          # rK   rô  r˜  s   &&r   r™  ÚCacheList.__getitem__2  s   € Ø�{‰{˜3ÕÐr   c                ó’   € \         ;QJ d&    R  V P                   4       F  '       d   K   R# 	  R# ! R  V P                   4       4      # )c              3   ó@   "  € T F  qP                  4       x € K  	  R # 5irK   rL   rN   s   & r   rP   Ú)CacheList.is_trimmable.<locals>.<genexpr>6  s   é € Ð9©[¨—>‘>×#Ð#«[ùrR   FT)rS   r8  rh   s   &r   rM   ÚCacheList.is_trimmable5  s3   € ß‹sÑ9¨T¯[ª[Ó9�sŒsÐ9ŠsÐ9ˆsÑ9¨T¯[ª[Ó9Ó9Ð9r   c                óN   € V P                    F  pVP                  V4      pK  	  X# rK   )r8  rY   )ri   r°   r5   Úms   &&  r   rY   ÚCacheList.trim8  s"   € Ø—”ˆAØ—‘�q“	ŠAñ àˆr   c                óV   € V P                    Uu. uF  qP                  NK  	  up# u upi rK   )r8  r.   r�  s   & r   r.   ÚCacheList.state=  s    € à!%§¢Ó-¡˜A—”¡Ñ-Ð-ùÒ-s   �&c                óP   € \        V P                  V4       F  w  r#W2n        K  	  R # rK   )r@   r8  r.   )ri   ro   r5   Úss   &&  r   r.   r   A  s   € ä˜Ÿ™ QÖ'‰DˆAØŽGó (r   c                ó¾   € V P                    Uu. uF  p\        V4      P                  NK  	  upV P                    Uu. uF  qP                  NK  	  up3# u upi u upi rK   )r8  r1   r2   r/   r�  s   & r   r/   ÚCacheList.meta_stateF  sL   € ð (,§{¢{Ó3¡{ !ŒT�!‹W×Ô¡{Ñ3Ø#'§;¢;Ó/¡;˜a�\Œ\¡;Ñ/ð
ð 	
ùÚ3ùÚ/s
   �A½Ac                ó^   € \        V P                  V^,          4       F  w  r#W2n        K  	  R# )rÒ   N)r@   r8  r/   )ri   ro   r5   rý  s   &&  r   r/   r  M  s"   € ä˜Ÿ™ Q q¥TÖ*‰DˆAØŽLó +r   c                óN   € V P                    F  pVP                  V4       K  	  R# rŸ  )r8  r¤  r¢  s   && r   r¤  ÚCacheList.filterR  s   € ð —”ˆAØ�H‰H�]Ö#ó r   c                óz   € \        V P                  VP                  4       F  w  r#VP                  V4       K  	  R# )r§  N)r@   r8  r°  )ri   r®  r5   r¯  s   &&  r   r°  ÚCacheList.extendY  s*   € ô ˜Ÿ™ U§\¡\Ö2‰DˆAØ�H‰H�QŽKó 3r   c           	     ó  a€ V ! 4       p\         ;QJ dA    . V3R  l\        \        S^ ,          P                  4      4       4       F  NK  	  5Vn        V# ! V3R  l\        \        S^ ,          P                  4      4       4       4      Vn        V# )c              3   óÀ   <"  € T FN  pS^ ,          P                   V,          P                  S Uu. uF  q"P                   V,          NK  	  up4      x € KP  	  R# u upi 5irÏ  )r8  r6  )rO   ró   r5   r8  s   &  €r   rP   Ú"CacheList.merge.<locals>.<genexpr>c  sR   øé € ð 
á1�ð �1�I×Ñ˜QÕ×%Ñ%¹FÓ&C¹F°q§x¡x°§{ {¹FÑ&C×DÐDÛ1ùò 'Dùs   ƒ,A¯A
Á
A)rü   r"   r    r8  )r„   r8  r*   s   &f r   r6  ÚCacheList.merge`  ss   ø€ á“ˆß”uô 
äœ3˜v a�y×/Ñ/Ó0Ô1ó
—uˆŒð ˆð	 ô 
äœ3˜v a�y×/Ñ/Ó0Ô1ó
ó 
ˆŒð ˆr   c                ó>   a€ \        V3R  lV P                   4       !  # )c              3   óD   <"  € T F  qP                  S4      x € K  	  R # 5irK   )r³  )rO   r5   rs  s   & €r   rP   Ú$CacheList.extract.<locals>.<genexpr>j  s   øé € Ð?±;¨aŸ9™9 SŸ>˜>³;ùs   ƒ )rò  r8  r˜  s   &fr   r³  ÚCacheList.extracti  s   ø€ ÜÔ?°4·;²;Ó?Ñ@Ð@r   c                óP   € V P                    F  pVP                  ! R/ VB  K  	  R # )Nrg   )r8  r¶  )ri   r¸   r5   s   &, r   r¶  ÚCacheList.preparel  s   € Ø—”ˆAØ�IŠIÑ˜Ôó r   c                óL   € V P                    F  pVP                  4        K  	  R # rK   )r8  rº  r�  s   & r   rº  ÚCacheList.finalizep  s   € Ø—”ˆAØ�J‰JŽLó r   c                ó:   € \        R  V P                   4       4      # )c              3   ó@   "  € T F  qP                  4       x € K  	  R # 5irK   )rx   rN   s   & r   rP   Ú!CacheList.size.<locals>.<genexpr>u  s   é € Ð1¡[ —6‘6—8�8£[ùrR   )Úmaxr8  rh   s   &r   rx   ÚCacheList.sizet  s   € ÜÑ1 T§[¢[Ó1Ó1Ð1r   c                óD   € V P                   ^ ,          P                  4       # r~  )r8  r€   rh   s   &r   r€   ÚCacheList.emptyw  s   € Ø�{‰{˜1�~×#Ñ#Ó%Ð%r   c                ó:   € \        R  V P                   4       4      # )c              3   ó8   "  € T F  qP                   x € K  	  R # 5irK   r
  rN   s   & r   rP   Ú#CacheList.nbytes.<locals>.<genexpr>|  s   é € Ð1¡[ —8–8£[ùó   ‚)rÔ  r8  rh   s   &r   r}   ÚCacheList.nbytesz  s   € äÑ1 T§[¢[Ó1Ó1Ð1r   c           	     óÄ   € V P                  V 4      p\        V.VO5!   UUUu. uF&  w  rEp\        4       V,          P                  WF4      NK(  	  upppVn        V# u upppi rK   )rƒ   r@   rA   rB   r8  )r„   r.   r/   r…   r  r5   rý  s   &&&    r   rB   ÚCacheList.from_state~  sW   € à�k‰k˜#Óˆä9<¸UÐ9PÀZÔ9Põ
Ù9P©g¨a°AŒG‹I�a�L×#Ñ# AÖ)Ñ9Pó
ˆŒ
ð ˆ
ùô
s   £,Arô  N)r2   r‡   rˆ   r‰   r—   r™  rM   rY   rŠ   r.   r‹   r/   r¤  r°  rŒ   r6  r³  r¶  rº  rx   r€   r}   rB   r�   rŽ   r�   s   @r   rò  rò  .  sÓ   ø‡ € òò ò:òð
 ñ.ó ð.ð ‡\�\ñó ðð ñ
ó ð
ð ×Ññó ðò$òð ñó ðòAò òò2ò&ð ñ2ó ð2ð ñó ör   rò  c                 ó
  € V P                   V,          pRRV P                  V,
          ,          ,           pVRR p\        P                  ! V4      V,          W,          ,
          V,          p\        P                  ! WVR7      pV# ).Nr›   ).rK   rÝ   )rŸ   Úndimr3   rq  Útake_along_axis)rÞ   Úshiftsrœ   r°   Úexpand_shiftsÚexpand_indicesrs  Úrolleds   &&&     r   Údynamic_rollr+  ‡  sh   € Ø	�‰��€AØ˜W¨¯©°­Õ6Õ6€MØ" 3 BÐ'€NÜ�9Š9�Q‹<˜Õ'¨&Õ*?Õ?À1Õ
D€CÜ×Ò ¨TÔ2€FØ€Mr   c                   óì   a € ] tR tRt o RtV 3R lR ltR tRRRRR	R/R
 ltR t]	R 4       t
]
P                  R 4       t
R tR tRV 3R lR lltR tR tR t]R 4       tR tR t]	R 4       tRtV tR# )r5  i�  rÈ   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# rŠ  ©r   r   )r   r�   s   "€r   r   ÚBatchKVCache.__annotate__“  s   ø€ ÷ #ñ #¡T©#¥Yñ #r   c                óÔ   € RV n         RV n        \        P                  ! V4      V n        \        P                  ! V Uu. uF  q") NK  	  up4      V n        ^ V n        RV n        R# u upi )a5  
The BatchKV cache expects inputs to be left-padded.

E.g. the following prompts:

    [1, 3, 5]
    [7]
    [2, 6, 8, 9]

Should be padded like so:

    [0, 1, 3, 5]
    [0, 0, 0, 7]
    [2, 6, 8, 9]

And ``left_padding`` specifies the amount of padding for each.
In this case, ``left_padding = [1, 3, 0]``.
N)r•   r–   r3   r�  r„  r]   rB  Ú_right_padding)ri   r„  Úls   && r   r—   ÚBatchKVCache.__init__“  sV   € ð& ˆŒ	ØˆŒÜŸHšH \Ó2ˆÔÜ—h’h©LÓ9©L q£©LÑ9Ó:ˆŒØˆŒ	à"ˆÖùò  :s   ¾A%c                óZ  € V P                   pV P                  e;   W1P                  ^,          ,           V P                  P                  ^,          8”  Ed‰   VP                  w  rErgVP                  ^,          pV P                  VP                  ^,          ,           ^,
          V P                  ,          p	WEW�P                  ,          V3p
WEW�P                  ,          V3p\        P
                  ! W¡P                  4      p\        P
                  ! W²P                  4      pV P                  e¥   W0P                  ,          ^ 8w  d;   V P                  RR V1R3,          V n        V P                  RR V1R3,          V n        \        P                  ! V P                  V.^R7      V n        \        P                  ! V P                  V.^R7      V n        MWÍuV n        V n        V ;P                  VP                  ^,          ,          un	        V ;P                   VP                  ^,          ,          un         WP                  RW0P                   1R3&   W P                  RW0P                   1R3&   V P                  RR V P                   1R3,          V P                  RR V P                   1R3,          3# r  )
rB  r•   rŸ   rë   r3   rÖ   rÕ   r–   rž   r]   r  s   &&&           r   r¡   ÚBatchKVCache.update_and_fetch®  sé  € Ø�y‰yˆØ�9‰9Ò ¯
©
°1­Õ!5¸¿¹¿¹ÈÕ9KÕ KØ+/¯:©:Ñ(ˆA˜1ØŸ™ a�ˆJØ—y‘y 4§:¡:¨a¥=Õ0°1Õ4¸¿¹ÕBˆGØ g·	±	Õ&9¸:ÐFˆGØ g·	±	Õ&9¸:ÐFˆGÜ—H’H˜W§j¡jÓ1ˆEÜ—H’H˜W§l¡lÓ3ˆEØ�y‰yÒ$ØŸ)™)Õ# qÔ(Ø $§	¡	¨#¨u°¨u°a¨-Õ 8�D”IØ"&§+¡+¨c°5°D°5¸!¨mÕ"<�D”KÜŸNšN¨D¯I©I°uÐ+=ÀAÔF�”	Ü Ÿnšn¨d¯k©k¸5Ð-AÈÔJ�•à).Ð&�”	˜4œ;à�Š�t—z‘z !•}Õ$�Ø�	Š	�T—Z‘Z •]Õ"�	Ø.2�	‰	�#�tŸi™iÐ'¨Ð*Ñ+Ø06�‰�C˜§	¡	Ð)¨1Ð,Ñ-Ø�y‰y˜˜k §	¡	˜k¨1Ð,Õ-¨t¯{©{¸3ÀÀ$Ç)Á)ÀÈQÐ;NÕ/OÐOÐOr   r„  Nr…  Úright_paddingc               ó6  € Veb   V P                   e   \        R4      h\        P                  ! V4      pV ;P                  V,          un        V ;P
                  V,          un        Ve0   \        V4      ^ 8”  d   \        P                  ! V4      V n        R # R # R # )Nz7Left padding can only be added to an empty BatchKVCache)r•   rm   r3   r�  r„  r]   r  r1  ©ri   r„  r…  r6  s   &$$$r   r¶  ÚBatchKVCache.prepareÇ  s}   € ØÒ#Ø�y‰yÒ$Ü ØMóð ô Ÿ8š8 LÓ1ˆLØ×Ò Õ-ÕØ�KŠK˜<Õ'�KàÒ$¬¨]Ó);¸aÔ)?Ü"$§(¢(¨=Ó"9ˆDÖñ *@Ñ$r   c                ó@  € V P                   e�   V P                   p\        V P                  VR,          ^R7      V n        \        V P                  VR,          ^R7      V n        V ;P                  V,          un        V ;P
                  V,          un        R V n         R # R # ©Nr›   rÁ  )r1  r+  r•   r–   r]   r„  )ri   Úpaddings   & r   rº  ÚBatchKVCache.finalizeÔ  sw   € Ø×ÑÒ*Ø×)Ñ)ˆGÜ$ T§Y¡Y°¸Õ0@ÀqÔIˆDŒIÜ& t§{¡{°G¸GÕ4DÈ1ÔMˆDŒKØ�KŠK˜7Õ"�KØ×Ò Õ(ÕØ"&ˆDÖñ +r   c                ó  € V P                   V P                  r!V P                  VP                  ^,          8  d1   VRRV P                  1R3,          pVRRV P                  1R3,          pWV P                  V P
                  3# r  )r•   r–   rB  rŸ   r]   r„  ©ri   Úkro   s   &  r   r.   ÚBatchKVCache.stateÝ  sn   € à�y‰y˜$Ÿ+™+ˆ1Ø�9‰9�q—w‘w˜q•zÔ!Ø�#�{˜Ÿ™�{ AÐ%Õ&ˆAØ�#�{˜Ÿ™�{ AÐ%Õ&ˆAØ�T—[‘[ $×"3Ñ"3Ð3Ð3r   c                ó€   € Vw  V n         V n        V n        V n        V P                   P                  ^,          V n        R# r$  )r•   r–   r]   r„  rŸ   rB  rn   s   &&r   r.   rA  å  s-   € àABÑ>ˆŒ	�4”; ¤¨TÔ->Ø—I‘I—O‘O AÕ&ˆŽ	r   c                ó   € R # r©   rg   rh   s   &r   rM   ÚBatchKVCache.is_trimmableê  r«   r   c                ó–   € \        V P                  V4      pV ;P                  V,          un        V ;P                  V,          un        V# rK   )r®   rB  r]   r¯   s   &&r   rY   ÚBatchKVCache.trimí  s2   € Ü�—	‘	˜1ÓˆØ�	Š	�Q��	Ø�Š�qÕ�Øˆr   c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   r\   r^   )r   rH   )r   r�   s   "€r   r   r/  ó  s   ø€ ÷ 
ñ 
™3ð 
©dñ 
r   c                óJ   € \        V3R V P                  RV P                  /VB # )r]   r„  )r   rB  r„  )ri   r\   r^   r¸   s   &&&,r   r¹   ÚBatchKVCache.make_maskó  s4   € Ü!Øñ
Ø—i‘ið
Ø.2×.?Ñ.?ð
ØCIñ
ð 	
r   c                ó8  € V P                   e1   V P                   V,          V n         V P                  V,          V n        V P                  V,          V n        V P                  V,          V n        V P                  P	                  4       P                  4       pV^ 8”  d}   V P                   e;   V P                   RVR1R3,          V n         V P                  RVR1R3,          V n        V ;P                  V,          un        V ;P                  V,          un        R# R# )r   N.rä   )r•   r–   r]   r„  r®   ÚitemrB  )ri   r£  Úmin_left_pads   && r   r¤  ÚBatchKVCache.filterø  sØ   € ð �9‰9Ò ØŸ	™	 -Õ0ˆDŒIØŸ+™+ mÕ4ˆDŒKØ—k‘k -Õ0ˆŒØ ×-Ñ-¨mÕ<ˆÔð ×(Ñ(×,Ñ,Ó.×3Ñ3Ó5ˆØ˜!ÔØ�y‰yÒ$Ø ŸI™I c¨<©=¸!Ð&;Õ<�”	Ø"Ÿk™k¨#¨|©}¸aÐ*?Õ@�”Ø�IŠI˜Õ%�IØ×Ò Õ-×ñ r   c           	     ó6  aaaa	a
€ V P                   fs   VP                   fe   \        P                  ! V P                  VP                  .4      V n        \        P                  ! V P                  VP                  .4      V n        R# \        V P                  VP                  4      o	^ ;r#V P                   e9   V P                   P                  w  popoV P                  P                  ^,          oVP                   e9   VP                   P                  w  popoVP                  P                  ^,          o\        W#4      o
VVVV	V
3R lp\        \        P                  \        V! V 4      V! V4      3!  4      w  V n         V n        V n        V n        S	V n        R# )r§  Nc                 óŒ  <€ V P                   V P                  r!Vfn   V P                  P                  ^ ,          p\        P
                  ! . 4      P                  VS	^ S4      p\        P
                  ! . 4      P                  VS	^ S
4      pSV P                  ,
          pSVP                  ^,          ,
          V,
          pV^ 8  d   VRR V1R3,          pVRR V1R3,          p^ pV^ 8w  g   V^ 8w  d4   RRWE3R.p\        P                  ! W4      p\        P                  ! W&4      pV P                  V,           pWV P                  V3# ©N.rä   )r   r   )
r•   r–   r]   rŸ   r3   r�  ÚreshaperB  Úpadr„  )r5   r@  ro   ÚBcÚleftÚrightrR  r„  ÚDÚHÚMÚmax_idxr   s   &       €€€€€r   rR  Ú BatchKVCache.extend.<locals>.pad   s  ø€ Ø—6‘6˜1Ÿ8™8ˆqØŠyØ—X‘X—^‘^ AÕ&�Ü—H’H˜R“L×(Ñ(¨¨Q°°1Ó5�Ü—H’H˜R“L×(Ñ(¨¨Q°°1Ó5�Ø˜QŸV™VÕ#ˆDØ˜qŸw™w q�zÕ)¨DÕ0ˆEØ�qŒyØ�c˜6˜E˜6 1�nÕ%�Ø�c˜6˜E˜6 1�nÕ%�Ø�Ø�qŒy˜E QœJØ˜v¨ }°fÐ=�Ü—F’F˜1“N�Ü—F’F˜1“N�ØŸ>™>¨DÕ0ˆLØ˜Ÿ™ <Ð/Ð/r   )r•   r3   rž   r„  r]   r  rB  rŸ   r–   rý   r@   ©ri   r®  ÚL1ÚL2rí   rR  rV  rW  rX  rY  r   s   &&    @@@@@r   r°  ÚBatchKVCache.extend  s5  ü€ ð �9‰9Ò §¡Ò!3Ü "§¢°×0AÑ0AÀ5×CUÑCUÐ/VÓ WˆDÔÜŸ.š.¨$¯+©+°u·|±|Ð)DÓEˆDŒKÙä�d—i‘i §¡Ó,ˆØˆˆØ�9‰9Ò ØŸ)™)Ÿ/™/‰KˆAˆq�"�aØ—‘×!Ñ! !Õ$ˆAØ�:‰:Ò!ØŸ*™*×*Ñ*‰KˆAˆq�"�aØ—‘×"Ñ" 1Õ%ˆAÜ�r“;ˆ÷	0ñ 	0ô& BEÜ�N‰NœC¡# d£)©S°«ZÐ!8Ñ9óB
Ñ>ˆŒ	�4”; ¤¨TÔ->ð ˆŽ	r   c                ó¨  € \        4       pV P                  V,          P                  4       p\        P                  ! V P
                  W^,           1RW0P                  13,          4      Vn        \        P                  ! V P                  W^,           1RW0P                  13,          4      Vn        VP
                  P                  ^,          Vn	        V# )rÒ   rä   )
r$   r„  rK  r3   Ú
contiguousr•   rB  r–   rŸ   r]   )ri   rs  r*   r<  s   &&  r   r³  ÚBatchKVCache.extract8  s—   € Ü“	ˆØ×#Ñ# CÕ(×-Ñ-Ó/ˆÜ—]’] 4§9¡9¨S¸µ7¨]¸A¸wÏÉÐ?RÐ-RÕ#SÓTˆŒ
Ü—}’} T§[¡[°¸Qµw°ÀÀ7ÏYÉYÐCVÐ1VÕ%WÓXˆŒØ—z‘z×'Ñ'¨Õ*ˆŒØˆr   c                ó  € V Uu. uF  q"P                  4       NK  	  pp\        V4      pV^ 8X  d   \        ^ .\        V4      ,          4      # V Uu. uF  qTV,
          NK  	  pp\        V4      p\        R V 4       4      p\        R V 4       4      p	\        R V 4       4      p
\	        \        R V 4       4      4      p\        P                  ! WxWI3VR7      p\        P                  ! WxWJ3VR7      p\        \        Wa4      4       Fœ  w  pw  ròVP                  f   K  VP                  RRVP                  1R3,          WÎV^,           1RWÿVP                  ,           13&   VP                  RRVP                  1R3,          WÞV^,           1RWÿVP                  ,           13&   Kž  	  V ! V4      pVVn
        VVn        V;P                  VP                  ^,          ,          un        VP                  ^,          Vn        V# u upi u upi )	r   c              3   óz   "  € T F1  qP                   f   K  VP                   P                  ^,          x € K3  	  R # 5irK   ©r•   rŸ   rN   s   & r   rP   Ú%BatchKVCache.merge.<locals>.<genexpr>K  ó$   é € ÐF© A·6±6”�—‘—‘˜Q—’«ùó   ‚;–%;c              3   óz   "  € T F1  qP                   f   K  VP                   P                  ^,          x € K3  	  R # 5irK   rd  rN   s   & r   rP   re  L  ó$   é € ÐG©& Q·F±F”�—‘—‘˜a—’«&ùrg  c              3   óz   "  € T F1  qP                   f   K  VP                   P                  ^,          x € K3  	  R # 5irK   ©r–   rŸ   rN   s   & r   rP   re  M  ó'   é € ÐK©F q·h±hÔ"�—‘—‘ ×"Ò"«Fùrg  c              3   ól   "  € T F*  qP                   f   K  VP                   P                  x € K,  	  R # 5irK   ©r•   rÕ   rN   s   & r   rP   re  N  ó   é € ÐJ©V¨·v±v”|�q—v‘v—|–|«Vùó   ‚4–4rÔ   N.rä   )rx   r  r5  r    rÉ  rÊ  r3   rÖ   Ú	enumerater@   r•   r]   r–   rŸ   rB  )r„   r8  r5   r…  Ú
max_lengthr2  r<  rí   rW  ÚDkÚDvÚdtr•   r–   ró   Úpr*   s   &&               r   r6  ÚBatchKVCache.merge@  s¹  € á%+Ó,¡V —6‘6–8¡VˆÐ,Ü˜“\ˆ
ð ˜Œ?Ü  ¤c¨&£kÕ 1Ó2Ð2á+2Ó3©7 a —>�>©7ˆÐ3Ü�‹KˆÜÑF©ÓFÓFˆÜÑG©&ÓGÓGˆÜÑK©FÓKÓKˆÜ”$ÑJ©VÓJÓJÓKˆä�xŠx˜˜zÐ.°bÔ9ˆÜ—’˜1 Ð0¸Ô;ˆÜ"¤3 wÓ#7Ö8‰IˆA‰v�Ø�v‰vŠ~ÙØ34·6±6¸#¸zÀÇÁ¸zÈ1Ð:LÕ3MˆD�Q˜•U�˜A˜q q§x¡x¥<Ð/Ð/Ñ0Ø56·X±X¸cÀ:ÀQÇXÁXÀ:ÈqÐ>PÕ5QˆF�q˜1•u�9˜a ¨¯©¥\Ð!1Ð1Ó2ñ	 9ñ �G“ˆØˆŒ
ØˆŒØ�Š˜Ÿ
™
 1�Õ%�Ø—Z‘Z •]ˆŒ
àˆùò9 -ùò 4s   …HÁHc                ó   € V P                   # rK   )rB  rh   s   &r   rx   ÚBatchKVCache.size`  s   € Ø�y‰yÐr   c                ó   € V P                   R J # rK   r½   rh   s   &r   r€   ÚBatchKVCache.emptyc  r¿   r   c                ó„   € V P                   f   ^ # V P                   P                  V P                  P                  ,           # rK   rÁ   rh   s   &r   r}   ÚBatchKVCache.nbytesf  rÃ   r   )rB  r1  r•   r„  r]   r–   ru   )r2   r‡   rˆ   r‰   rë   r—   r¡   r¶  rº  rŠ   r.   r‹   rM   rY   r¹   r¤  r°  r³  rŒ   r6  rx   r€   r}   r�   rŽ   r�   s   @r   r5  r5  �  s»   ø‡ € Ø€D÷#ð #ò6Pð2: dð :°Dð :Èô :ò'ð ñ4ó ð4ð ‡\�\ñ'ó ð'òò÷
ò 
ò
.ò&+òZð ñó ðò>ò!ð ñ5ó ö5r   r5  c                   óR  a € ] tR tRt o RtV 3R lR ltR"R ltR tR tR	 t	R
 t
RRRRRR/R ltR t]R 4       t]P                  R 4       t]R 4       t]P                  R 4       tR tR tR#V 3R lR lltR$V 3R lR lltR tR tR t]R 4       tR tR t]R  4       tR!tV tR# )%rx  im  rÈ   c                ó0   <€ V ^8„  d   QhRS[ S[,          /# rŠ  r.  )r   r�   s   "€r   r   Ú!BatchRotatingKVCache.__annotate__p  s   ø€ ÷ ñ ©t±C­yñ r   c                óü   € R V n         R V n        \        P                  ! V4      V n        \        P                  ! V Uu. uF  q3) NK  	  up4      V n        Wn        ^ V n        ^ V n        RV n	        R V n
        R # u upi r  )r•   r–   r3   r�  r„  r]   r   rB  Ú_offsetÚrotatedÚ_lengths)ri   r   r„  r2  s   &&& r   r—   ÚBatchRotatingKVCache.__init__p  sh   € ØˆŒ	ØˆŒäŸHšH \Ó2ˆÔÜ—h’h©LÓ9©L q£©LÑ9Ó:ˆŒà ŒØˆŒ	ØˆŒØˆŒð ˆŽùò  :s   ¾A9Nc                ój   € V^ 8”  d   VRVR1R3,          pVe   \         P                  ! W#.^R7      # V# rE  )r3   rž   )ri   rG  ro   rF  s   &&&&r   rI  ÚBatchRotatingKVCache._trim€  s:   € Ø�qŒ=Ø�#�y‘z 1Ð$Õ%ˆAØÒÜ—>’> 1 +°AÔ6Ð6Øˆr   c                óL  € V P                   '       d’   \        P                  ! V P                  V P                  ) ^R7      V n        \        P                  ! V P
                  V P                  ) ^R7      V n        V P                  P                  ^,          V n        RV n         R# R# )z*
Rearrange the cache into temporal order.
r›   FN)rƒ  r3   rr  r•   rB  r–   rŸ   rh   s   &r   rL  Ú$BatchRotatingKVCache._temporal_order‡  si   € ð �<�<ˆ<ÜŸš §	¡	¨D¯I©I¨:¸AÔ>ˆDŒIÜŸ'š' $§+¡+°·	±	¨zÀÔBˆDŒKØŸ	™	Ÿ™¨Õ*ˆDŒIØ ˆDŽLñ	 r   c                ób  € V P                   f   Wn         W n        EMÊV P                  4        V P                   P                  ^,          V P                  8”  dO   V P                   RR V P                  1R3,          V n         V P                  RR V P                  1R3,          V n        V P
                  e­   \        P                  ! ^ V P                  V P
                  ,
          4      p\        V P                   VR,          ^R7      V n         \        V P                  VR,          ^R7      V n        V ;P                  V,          un
        V ;P                  V,          un        V P                  V P                  ,
          ^,           pV^ 8”  d   V ;P                  V,          un
        V P                  W@P                   V4      V n         V P                  W@P                  V4      V n        V ;P                  VP                  ^,          ,          un        V ;P                  VP                  ^,          ,          un        V P                   P                  ^,          V n        \        P                  ! V P                   V P                  V P                  34      V n         V P                   V P                  3# )N.rä   r›   rÁ  )r•   r–   rL  rŸ   rB  r„  r3   Úmaximumr]   r+  r„  r   rI  r‚  Údepends)ri   r•   r–   rr  rG  s   &&&  r   rO  Ú#BatchRotatingKVCache._update_concat‘  sÌ  € Ø�9‰9ÒØŒIØ ŽKð × Ñ Ô"ð �y‰y�‰˜qÕ! D§I¡IÔ-Ø ŸI™I c¨;¨T¯Y©Y¨;¸Ð&9Õ:�”	Ø"Ÿk™k¨#¨{°·±¨{¸AÐ*=Õ>�”ð �}‰}Ò(Ü—z’z ! T§[¡[°4·=±=Õ%@ÓA�Ü(¨¯©°D¸µMÈÔJ�”	Ü*¨4¯;©;¸¸W½ÈAÔN�”Ø×!Ò! TÕ)Õ!Ø—’˜tÕ#•ð Ÿ	™	 D§M¡MÕ1°AÕ5ˆIØ˜1Œ}Ø×!Ò! YÕ.Õ!ØŸ
™
 9¯i©i¸Ó>ˆDŒIØŸ*™* Y·±¸VÓDˆDŒKØ�Š�t—z‘z !•}Õ$�Ø�Š˜Ÿ
™
 1�Õ%�Ø—I‘I—O‘O AÕ&ˆŒ	ô —J’J˜tŸy™y¨4×+<Ñ+<¸d¿k¹kÐ*JÓKˆŒ	à�y‰y˜$Ÿ+™+Ð%Ð%r   c                óª  € V P                   e   \        R4      hVP                  w  r4rVV P                  pV P                  eO   WpP                  P                  ^,          8¼  Ed$   V P                  P                  ^,          V P
                  8  dø   VP                  ^,          p\        V P                  V P
                  V,
          4      p	W4W–3p
W4W˜3p\        P                  ! W¡P                  4      p\        P                  ! W²P                  4      pV P                  eT   \        P                  ! V P                  V.^R7      V n        \        P                  ! V P                  V.^R7      V n        MWÍuV n        V n        Wpn        V P                  P                  ^,          V P
                  ,
          pV^ 8”  dk   V P                  WàP                  4      V n        V P                  WàP                  4      V n        V P
                  V n        V ;P                  V,          un        V P                  V P
                  8X  d   RV n        ^ V n        V P                   '       d   V ;P                  V,          un        WP                  RV P                  V P                  V,           1R3&   W P                  RV P                  V P                  V,           1R3&   V ;P                  V,          un        V ;P"                  V,          un        V ;P                  V,          un        \        P$                  ! V P                  V P                  V P"                  34      V n        V P                  V P
                  8  dE   V P                  RR V P                  1R3,          V P                  RR V P                  1R3,          3# V P                  V P                  3# )NzFfinalize() should be called before deocoding with BatchRotatingKVCacher›   T.rä   )r„  ÚRuntimeErrorrŸ   r‚  r•   r   r®   rë   r3   rÖ   rÕ   rž   r–   rB  rI  r„  rƒ  r]   rŒ  rR  s   &&&            r   rU  Ú%BatchRotatingKVCache._update_in_place¸  s¾  € Ø�=‰=Ò$ÜØXóð ð (,§z¡zÑ$ˆ�qØ�|‰|ˆØ�9‰9ÒØ—I‘I—O‘O AÕ&Õ&¨4¯9©9¯?©?¸1Õ+=ÀÇÁÔ+MàŸ™ a�ˆJÜ˜4Ÿ9™9 d§m¡m°dÕ&:Ó;ˆHØ hÐ;ˆGØ hÐ;ˆGÜ—H’H˜W§j¡jÓ1ˆEÜ—H’H˜W§l¡lÓ3ˆEØ�y‰yÒ$ÜŸNšN¨D¯I©I°uÐ+=ÀAÔF�”	Ü Ÿnšn¨d¯k©k¸5Ð-AÈÔJ�•à).Ð&�”	˜4œ;ØŒIð —I‘I—O‘O AÕ&¨¯©Õ6ˆ	Ø�qŒ=ØŸ
™
 9¯i©iÓ8ˆDŒIØŸ*™* Y·±Ó<ˆDŒKØŸ™ˆDŒIØ×Ò Õ*Õð �9‰9˜Ÿ™Ô%ØˆDŒLØˆDŒIØ�<�<ˆ<Ø×Ò Õ"Õð 8<�	‰	�#�t—y‘y 4§9¡9¨q¥=Ð0°!Ð3Ñ4Ø9?�‰�C˜Ÿ™ T§Y¡Y°¥]Ð2°AÐ5Ñ6Ø�Š˜Õ�Ø�Š�qÕ�Ø�	Š	�Q��	ô —J’J˜tŸy™y¨4×+<Ñ+<¸d¿k¹kÐ*JÓKˆŒ	ð �<‰<˜$Ÿ-™-Ô'à—	‘	˜#˜~ §¡˜~¨qÐ0Õ1Ø—‘˜C  4§<¡< °Ð2Õ3ðð ð �y‰y˜$Ÿ+™+Ð%Ð%r   c                óv   € VP                   ^,          ^8X  d   V P                  W4      # V P                  W4      # rX  rY  r    s   &&&r   r¡   Ú%BatchRotatingKVCache.update_and_fetchó  r[  r   r„  r…  r6  c               óX  € Veb   V P                   e   \        R4      h\        P                  ! V4      pV ;P                  V,          un        V ;P
                  V,          un        VeA   \        V4      ^ 8”  d/   \        P                  ! V4      V P
                  ,           V n        R # R # R # )Nz?Left padding can only be added to an empty BatchRotatingKVCache)r•   rm   r3   r�  r„  r]   r  r„  r8  s   &$$$r   r¶  ÚBatchRotatingKVCache.prepareø  s…   € ØÒ#Ø�y‰yÒ$Ü ØUóð ô Ÿ8š8 LÓ1ˆLØ×Ò Õ-ÕØ�KŠK˜<Õ'�KàÒ$¬¨]Ó);¸aÔ)?ÜŸHšH WÓ-°·±Õ;ˆDŽMñ *@Ñ$r   c                óŒ  € V P                   e¶   \        P                  ! ^ V P                  V P                   ,
          4      p\	        V P
                  VR,          ^R7      V n        \	        V P                  VR,          ^R7      V n        V ;P                  V,          un        V ;P                  V,          un        R V n         R # R # r;  )r„  r3   r‹  r]   r+  r•   r–   r„  )ri   rr  s   & r   rº  ÚBatchRotatingKVCache.finalize  s„   € Ø�=‰=Ò$Ü—:’:˜a §¡¨t¯}©}Õ!<Ó=ˆDÜ$ T§Y¡Y°°WµÀAÔFˆDŒIÜ& t§{¡{°D¸µMÈÔJˆDŒKØ×Ò Õ%ÕØ�KŠK˜4Õ�KØ ˆDŽMñ %r   c                ó  € V P                   V P                  r!V P                  VP                  ^,          8  d0   VRRV P                  1R3,          VRRV P                  1R3,          r!WV P                  V P
                  3# r  )r•   r–   r‚  rŸ   r]   r„  r?  s   &  r   r.   ÚBatchRotatingKVCache.state  sk   € à�y‰y˜$Ÿ+™+ˆ1Ø�<‰<˜!Ÿ'™' !�*Ô$Ø�S˜.˜DŸL™L˜.¨!Ð+Õ,¨a°°^°t·|±|°^ÀQÐ0FÕ.GˆqØ�T—[‘[ $×"3Ñ"3Ð3Ð3r   c                ó<   € Vw  V n         V n        V n        V n        R # rK   ©r•   r–   r]   r„  rn   s   &&r   r.   r˜    s   € àABÑ>ˆŒ	�4”; ¤¨TÖ->r   c           	     óŒ   € \        \        \        V P                  V P                  V P
                  V P                  34      4      # rK   )rü   rý   r,   r   r‚  rB  rƒ  rh   s   &r   r/   ÚBatchRotatingKVCache.meta_state  s-   € ä”Sœ˜tŸ}™}¨d¯l©l¸D¿I¹IÀtÇ|Á|ÐTÓUÓVÐVr   c                óˆ   € \        \        VR ,          4      w  V n        V n        V n        \        V^,          4      V n        R# ):Né   NN)rý   r   r   r‚  rB  rH   rƒ  rn   s   &&r   r/   rœ    s4   € ä14ÜØˆb�Eó2
Ñ.ˆŒ�t”| T¤Yô ˜A˜a�D“zˆŽr   c                ó4   € V P                   V P                  8  # rK   )r‚  r   rh   s   &r   rM   Ú!BatchRotatingKVCache.is_trimmable%  s   € Ø�|‰|˜dŸm™mÑ+Ð+r   c                óÈ   € \        V P                  V4      pV ;P                  V,          un        V ;P                  V,          un        V ;P                  V,          un        V# rK   )r®   r‚  rB  r]   r¯   s   &&r   rY   ÚBatchRotatingKVCache.trim(  s@   € Ü�—‘˜aÓ ˆØ�Š˜Õ�Ø�	Š	�Q��	Ø�Š�qÕ�Øˆr   c                ó,   <€ V ^8„  d   QhRS[ RS[ RS[/# r+  r,  )r   r�   s   "€r   r   r€  /  s(   ø€ ÷ Kñ K¡sð K±sð KÑCSñ Kr   c                ó   € \        R 4      h)z%BatchRotatingKVCache Quantization NYIr{   rÏ   s   &&&r   r0  Ú!BatchRotatingKVCache.to_quantized/  s   € Ü!Ð"IÓJÐJr   c                ó<   <€ V ^8„  d   QhRS[ RS[S[ ,          RS[/# rm  rn  )r   r�   s   "€r   r   r€  2  s)   ø€ ÷ ñ ÙðÙ#+©C¥=ðÙGKñr   c                óJ  € V P                   pT;'       g    V P                  p\        V P                  ^,
          V P                  4      p\        P
                  ! WQ,           4      pV'       d   \        P
                  ! WUV,           4      MTpVR,          pVR,          pWv8¬  pW‡Wb,           8  ,          pV P                  V P                  ,
          \        V^8„  4      ,           ;p	^ 8”  d	   WI,
          pV^8H  ;'       d.    V P                  ;'       g    V P                  V P                  8¬  p
V
'       d
   V^,
          pW†\        P                  ! VR4      8¬  ,          pV
'       d>   V P                  pW°P                  8¼  d   ^ p\        P                  ! W‹^,           RR7      pV# )rÒ   N)rp  rœ   rÁ  )rÒ   r   rž  rÝ   )r„  r   r®   r‚  r3   rq  rB  r   rƒ  Úexpand_dimsrr  )ri   r\   r_   r^   r„  r]   ÚrindsÚlindsru  rG  rƒ  rs  s   &&&&        r   r¹   ÚBatchRotatingKVCache.make_mask2  s:  € ð ×(Ñ(ˆØ!×2Ð2 T§]¡]ˆÜ�T—]‘] QÕ&¨¯©Ó5ˆÜ—	’	˜&�*Ó%ˆß17”—	’	˜&¨1¥*Ô-¸UˆØ�g•ˆØ�d•ˆØ‰~ˆØ˜Õ+Ñ+Õ+ˆØŸ™ T§]¡]Õ2´S¸¸Q¹³ZÕ?Ð?ˆIÀ1ÔDØ'Õ3ˆLà�q‘&×IÐI˜dŸl™l×HÐH¨d¯i©i¸4¿=¹=Ñ.HˆßØ'¨!Õ+ˆLà¤§¢¨|¸YÓ GÑGÕHˆçØ—)‘)ˆCØ—m‘mÔ#Ø�Ü—7’7˜4¨Q¥w°RÔ8ˆDàˆr   c                óâ   € V P                   e1   V P                   V,          V n         V P                  V,          V n        V P                  V,          V n        V P                  V,          V n        R# rŸ  rš  )ri   r£  s   &&r   r¤  ÚBatchRotatingKVCache.filterO  sQ   € ð �9‰9Ò ØŸ	™	 -Õ0ˆDŒIØŸ+™+ mÕ4ˆDŒKØ—k‘k -Õ0ˆŒØ ×-Ñ-¨mÕ<ˆÖr   c           	     ó,  aaaa	a
€ V P                   fs   VP                   fe   \        P                  ! V P                  VP                  .4      V n        \        P                  ! V P                  VP                  .4      V n        R# V P
                  VP
                  8w  g   V P                  VP                  8w  d!   V P                  4        VP                  4        \        V P                  VP                  4      o	^ ;r#V P                   e9   V P                   P                  w  popoV P                  P                  ^,          oVP                   e9   VP                   P                  w  popoVP                  P                  ^,          o\        W#4      o
VVVV	V
3R lp\        \        P                  \        V! V 4      V! V4      3!  4      w  V n         V n
        V n        V n        S	V n        \        V P                  VP                  4      V n        R# )r§  Nc                 óŒ  <€ SV P                   ,
          pV P                  V P                  r2Vfn   V P                  P                  ^ ,          p\
        P                  ! . 4      P                  VS	^ S4      p\
        P                  ! . 4      P                  VS	^ S
4      pSVP                  ^,          ,
          V,
          pV^ 8  d   VRR V1R3,          pVRR V1R3,          p^ pV^ 8w  g   V^ 8w  d4   RRW3R.p\
        P                  ! W&4      p\
        P                  ! W64      pV P                  V,           pW#V P                  V3# rP  )
rB  r•   r–   r]   rŸ   r3   r�  rQ  rR  r„  )r5   rT  r@  ro   rS  rU  rR  r„  rV  rW  rX  rY  r   s   &       €€€€€r   rR  Ú(BatchRotatingKVCache.extend.<locals>.padp  s  ø€ Ø˜QŸV™VÕ#ˆDØ—6‘6˜1Ÿ8™8ˆqØŠyØ—X‘X—^‘^ AÕ&�Ü—H’H˜R“L×(Ñ(¨¨Q°°1Ó5�Ü—H’H˜R“L×(Ñ(¨¨Q°°1Ó5�Ø˜qŸw™w q�zÕ)¨DÕ0ˆEØ�qŒyØ�c˜6˜E˜6 1�nÕ%�Ø�c˜6˜E˜6 1�nÕ%�Ø�Ø�qŒy˜E QœJØ˜v¨ }°fÐ=�Ü—F’F˜1“N�Ü—F’F˜1“N�ØŸ>™>¨DÕ0ˆLØ˜Ÿ™ <Ð/Ð/r   )r•   r3   rž   r„  r]   rƒ  rB  rL  r  rŸ   r–   rý   r@   r‚  r[  s   &&    @@@@@r   r°  ÚBatchRotatingKVCache.extendY  s‚  ü€ ð �9‰9Ò §¡Ò!3Ü "§¢°×0AÑ0AÀ5×CUÑCUÐ/VÓ WˆDÔÜŸ.š.¨$¯+©+°u·|±|Ð)DÓEˆDŒKÙà�L‰L˜EŸM™MÔ)¨d¯i©i¸5¿:¹:Ô.EØ× Ñ Ô"Ø×!Ñ!Ô#ä�d—i‘i §¡Ó,ˆØˆˆØ�9‰9Ò ØŸ)™)Ÿ/™/‰KˆAˆq�"�aØ—‘×!Ñ! !Õ$ˆAØ�:‰:Ò!ØŸ*™*×*Ñ*‰KˆAˆq�"�aØ—‘×"Ñ" 1Õ%ˆAÜ�r“;ˆ÷	0ñ 	0ô& BEÜ�N‰NœC¡# d£)©S°«ZÐ!8Ñ9óB
Ñ>ˆŒ	�4”; ¤¨TÔ->ð ˆŒ	Ü˜4Ÿ<™<¨¯©Ó7ˆŽr   c                óò  € \         P                  ! V P                  V P                  4       \	        V P
                  4      p\        ^ V P                  P                  4       V,          4      pV P                  P                  4       V,          pV P                  W^,            Vn        V P                  W^,            Vn	        V P                  Vn
        V P                  '       dx   \         P                  ! VP                  V P                  ) ^R7      Vn        \         P                  ! VP                  V P                  ) ^R7      Vn	        V P
                  Vn
        \         P                  ! VP                  RRW2P                  13,          4      Vn        \         P                  ! VP                  RRW2P                  13,          4      Vn	        WBn        VP                  P                  ^,          Vn
        V# )r   r›   rä   )r3   Úevalr„  r]   r#   r   r  Útolistr•   r–   rB  rƒ  rr  r`  rŸ   )ri   rs  r*   r<  r]   s   &&   r   r³  ÚBatchRotatingKVCache.extract‰  sP  € Ü
�Š�×!Ñ! 4§;¡;Ô/Ü §¡Ó.ˆÜ�a˜×*Ñ*×1Ñ1Ó3°CÕ8Ó9ˆØ—‘×#Ñ#Ó% cÕ*ˆØ—Y‘Y˜s¨1¥WÐ-ˆŒ
Ø—{‘{ 3¨q­Ð1ˆŒØ—Y‘YˆŒ
Ø�<�<ˆ<ÜŸš §¡¨d¯i©i¨Z¸aÔ@ˆEŒJÜŸ7š7 5§<¡<°$·)±)°À!ÔDˆEŒLØŸ™ˆEŒJÜ—]’] 5§:¡:¨a°°G¿j¹jÐ4HÐ.HÕ#IÓJˆŒ
Ü—}’} U§\¡\°!°Q¸Ç*Á*Ð8LÐ2LÕ%MÓNˆŒØŒØ—Z‘Z×%Ñ% aÕ(ˆŒ
Øˆr   c                ó:  a€ \         ;QJ d    V3R  lS 4       F  '       d   K   RM	  RM! V3R  lS 4       4      '       g   \        R4      hS Uu. uF  q"P                  NK  	  ppS Uu. uF  q"P                  4       NK  	  pp\	        V4      pV^ 8X  d,   V ! S^ ,          P
                  ^ .\        S4      ,          4      # V Uu. uF  qeV,
          NK  	  pp\        S4      p\	        R S 4       4      p	\	        R S 4       4      p
\	        R S 4       4      p\        \        R S 4       4      4      p\        P                  ! W‰WZ3VR7      p\        P                  ! W‰W[3VR7      p\        \        WtS4      4       F—  w  pw  prbVP                  f   K  VP                  VP                  4      R
V) R	1R3,          WßV^,           1RVVV,           13&   VP                  VP                  4      R
V) R	1R3,          WïV^,           1RVVV,           13&   K™  	  V ! S^ ,          P
                  V4      pVVn        VVn        \        P                   ! V4      Vn        VP"                  ^,          Vn        VP"                  ^,          Vn        V# u upi u upi u upi )c              3   ób   <"  € T F$  qP                   S^ ,          P                   8H  x € K&  	  R# 5irÏ  )r   )rO   r5   r8  s   & €r   rP   Ú-BatchRotatingKVCache.merge.<locals>.<genexpr>�  s"   øé € ÐD¹V¸—:‘: ¨¥×!3Ñ!3Ö3»Vùs   ƒ,/FTzEBatchRotatingKVCache can only merge caches with the same maximum sizec              3   óz   "  € T F1  qP                   f   K  VP                   P                  ^,          x € K3  	  R # 5irK   rd  rN   s   & r   rP   r¸  ¬  rf  rg  c              3   óz   "  € T F1  qP                   f   K  VP                   P                  ^,          x € K3  	  R # 5irK   rd  rN   s   & r   rP   r¸  ­  ri  rg  c              3   óz   "  € T F1  qP                   f   K  VP                   P                  ^,          x € K3  	  R # 5irK   rk  rN   s   & r   rP   r¸  ®  rl  rg  c              3   ól   "  € T F*  qP                   f   K  VP                   P                  x € K,  	  R # 5irK   rn  rN   s   & r   rP   r¸  ¯  ro  rp  rÔ   N.rä   )rS   rm   r]   rx   r  r   r    rÉ  rÊ  r3   rÖ   rq  r@   r•   rL  r–   r�  rŸ   rB  r‚  )r„   r8  r5   Úoffsetsr…  rr  r2  r<  rí   rW  rs  rt  ru  r•   r–   ró   rv  r*   s   &f                r   r6  ÚBatchRotatingKVCache.merge›  s6  ø€ ç‹sÔD¹VÓD�s�sŠsÔD¹VÓD×DÒDÜØWóð ñ &,Ó,¡V —8”8¡VˆÐ,Ù%+Ó,¡V —6‘6–8¡VˆÐ,Ü˜“\ˆ
ð ˜Œ?Ù�v˜a•y×)Ñ)¨A¨3´°V³Õ+<Ó=Ð=á+2Ó3©7 a —>�>©7ˆÐ3Ü�‹KˆÜÑF©ÓFÓFˆÜÑG©&ÓGÓGˆÜÑK©FÓKÓKˆÜ”$ÑJ©VÓJÓJÓKˆä�xŠx˜˜zÐ.°bÔ9ˆÜ—’˜1 Ð0¸Ô;ˆÜ%¤c¨'¸FÓ&CÖD‰LˆA‰y��1Ø�v‰vŠ~ÙØ,-×,=Ñ,=¸a¿f¹fÓ,EÀcÈAÈ2É3ÐPQÀkÕ,RˆD�Q˜•U�˜A˜q 1 q¥5˜yÐ(Ñ)Ø./×.?Ñ.?ÀÇÁÓ.IÈ#ÐPQÈrÉsÐTUÈ+Õ.VˆF�q˜1•u�9˜a  Q¨¥U Ð*Ó+ñ	 Eñ �F˜1•I×&Ñ&¨Ó0ˆØˆŒ
ØˆŒÜ—x’x Ó(ˆŒØ—Z‘Z •]ˆŒ
ØŸ
™
 1�ˆŒàˆùò= -ùÚ,ùò 4s   ÁJÁ+JÃJc                óB   € \        V P                  V P                  4      # rK   )r®   r‚  r   rh   s   &r   rx   ÚBatchRotatingKVCache.sizeÂ  s   € Ü�4—<‘< §¡Ó/Ð/r   c                ó   € V P                   R J # rK   r½   rh   s   &r   r€   ÚBatchRotatingKVCache.emptyÅ  r¿   r   c                ó„   € V P                   f   ^ # V P                   P                  V P                  P                  ,           # rK   rÁ   rh   s   &r   r}   ÚBatchRotatingKVCache.nbytesÈ  rÃ   r   )	rB  r„  r‚  r•   r„  r   r]   rƒ  r–   rK   r>  r  )r2   r‡   rˆ   r‰   rë   r—   rI  rL  rO  rU  r¡   r¶  rº  rŠ   r.   r‹   r/   rM   rY   r0  r¹   r¤  r°  r³  rŒ   r6  rx   r€   r}   r�   rŽ   r�   s   @r   rx  rx  m  s  ø‡ € Ø€D÷ð ô ò!ò%&òN9&òv1ð
< dð <°Dð <Èô <ò!ð ñ4ó ð4ð ‡\�\ñCó ðCð ñWó ðWð ×Ññ"ó ð"ò,ò÷Kò K÷ò ò:=ò.8ò`ð$ ñ$ó ð$òL0ò!ð ñ5ó ö5r   rx  c                   óZ   a € ] tR tRt o RtRt. 3R ltR t]R 4       t	]R 4       t
RtV tR	# )
ÚTokenBufferiÏ  z”A simple token buffer that can be efficiently appended to in a similar
fashion to the KVCache.

Perhaps these could share some logic in the future.
rÈ   c                ó|   € \         P                  ! V\         P                  R 7      V n        \	        V4      V n        R# )rÔ   N)r3   r�  Úint32Ú_bufferr    Ú_size)ri   Útokenss   &&r   r—   ÚTokenBuffer.__init__Ø  s#   € Ü—x’x ¬b¯h©hÔ7ˆŒÜ˜“[ˆŽ
r   c                óö  € V P                   pV\        V4      ,           pW0P                  ,           ^,
          V P                  ,          V P                  ,          pW@P                  P                  8”  df   \
        P                  ! V P                  \
        P                  ! W@P                  P                  ,
          \
        P                  R7      .4      V n        WP                  W#% W0n         V P                  RV # )rÒ   rÔ   N)	rÊ  r    rë   rÉ  rx   r3   rž   rÖ   rÈ  )ri   rË  Ústartrá  rT  s   &&   r   r¡   ÚTokenBuffer.update_and_fetchÜ  s§   € Ø—
‘
ˆØ”c˜&“kÕ!ˆàŸ9™9•_ qÕ(¨T¯Y©YÕ6¸$¿)¹)ÕCˆØ—l‘l×'Ñ'Ô'ÜŸ>š>Ø—‘œrŸxšx¨·<±<×3DÑ3DÕ(DÌBÏHÉHÔUÐVóˆDŒLð #)�‰�UÐØŒ
à�|‰|˜D˜SÐ!Ð!r   c                ó   € V P                   # rK   )rÉ  rh   s   &r   r.   ÚTokenBuffer.stateê  s   € à�|‰|Ðr   c                ó4   € V P                   R V P                   # rK   ©rÉ  rÊ  rh   s   &r   rË  ÚTokenBuffer.tokensî  s   € à�|‰|˜L˜dŸj™jÐ)Ð)r   rÓ  N)r2   r‡   rˆ   r‰   rÅ   rë   r—   r¡   rŠ   r.   rË  r�   rŽ   r�   s   @r   rÆ  rÆ  Ï  sF   ø‡ € ñð €Dà ô !ò"ð ñó ðð ñ*ó ö*r   rÆ  c                   ó,   a € ] tR tRt o V 3R ltRtV tR# )ÚPromptTrieResultió  c                ó¶   <€ V ^8„  d   Qh/ S[ ;R&   S[S[S[,          ,          ;R&   S[S[S[,          ,          ;R&   S[S[S[,          ,          ;R&   S[;R&   # )r   r   ÚexactÚshorterÚlongerÚcommon_prefix)r   r   r   r   )r   r�   s   "€r   r   ÚPromptTrieResult.__annotate__ó  s^   ø‡ ‚ á�Jñ ñ ‘D™•IÕÑñ ñ ‘d™3•iÕ Ñ ñ	 ñ
 ‘T™#•YÕÑñ ñ Ñò r   rg   N©r2   r‡   rˆ   r‰   Ú__annotate_func__r�   rŽ   r�   s   @r   rÖ  rÖ  ó  ó   ø‡ ‡ „ r   rÖ  c                   ó€   a € ] tR tRt o R tV 3R lR ltV 3R lR ltV 3R lR ltV 3R	 lR
 ltV 3R lR lt	Rt
V tR# )Ú
PromptTrieiü  c                ó   € / V n         R # rK   ©Ú_trierh   s   &r   r—   ÚPromptTrie.__init__ý  s	   € ØˆŽ
r   c                ó<   <€ V ^8„  d   QhRS[ RS[S[,          RS[ /# )r   r   rË  r”  ©r   r   r   )r   r�   s   "€r   r   ÚPromptTrie.__annotate__   s&   ø€ ÷ ñ ™ð ¡d©3¥ið ¹ñ r   c                óÌ   € WP                   9  d   / V P                   V&   V P                   V,          pV F  pWT9  d   / WE&   WE,          pK  	  VP                  R R4      pW4R &   V# )Ú	__value__N)rä  Úget)ri   r   rË  r”  ÚcurrentÚtokrå   s   &&&&   r   ÚaddÚPromptTrie.add   se   € ØŸ
™
Ô"Ø "ˆD�J‰J�uÑà—*‘*˜UÕ#ˆÛˆCØÔ!Ø!�‘Ø•lŠGñ ð �{‰{˜;¨Ó-ˆØ$�ÑØˆr   c                ó6   <€ V ^8„  d   QhRS[ RS[S[,          /# ©r   r   rË  rç  )r   r�   s   "€r   r   rè    s   ø€ ÷ $ñ $™ð $¡d©3¥iñ $r   c                ó\   € V P                   V,          pV F  pW4,          pK  	  VR ,          # ©rê  rã  )ri   r   rË  rì  rí  s   &&&  r   rë  ÚPromptTrie.get  s,   € Ø—*‘*˜UÕ#ˆÛˆCØ•lŠGñ à�{Õ#Ð#r   c                ó6   <€ V ^8„  d   QhRS[ RS[S[,          /# rñ  rç  )r   r�   s   "€r   r   rè    s   ø€ ÷ ñ ™ð ¡d©3¥iñ r   c                ó^  € V P                   V,          .pV F"  pVP                  VR,          V,          4       K$  	  VR,          P                  R4      p\        \	        V4      ^ R4       F>  pW6,          pW6^,
          ,          pW&^,
          ,          p\	        V4      ^ 8”  d    V# W„ K@  	  V# )rÒ   rê  rÝ   )rä  rF  Úpopr"   r    )	ri   r   rË  Úpathrí  r”  ró   ÚnodeÚparents	   &&&      r   r÷  ÚPromptTrie.pop  s˜   € Ø—
‘
˜5Õ!Ð"ˆÛˆCØ�K‰K˜˜R� �Ö&ñ à�R•—‘˜[Ó)ˆÜ”s˜6“{ A rÖ*ˆAØ•7ˆDØ˜a�%•[ˆFØ˜Q�•-ˆCÜ�4‹y˜1Œ}Øàˆð ’ñ +ð ˆr   c                ó6   <€ V ^8„  d   QhRS[ RS[S[,          /# rñ  rç  )r   r�   s   "€r   r   rè  !  s   ø€ ÷ ñ ¡#ð ©t±C­yñ r   c                ó¸   € . pV P                   V,          p\        V4       F5  w  rVR V9   d"   VP                  WTP                  R 4      34       WF,          pK7  	  V# ró  )rä  rq  rF  r÷  )ri   r   rË  r–   rì  ró   rí  s   &&&    r   Úpop_prefixesÚPromptTrie.pop_prefixes!  sT   € ØˆØ—*‘*˜UÕ#ˆÜ Ö'‰FˆAØ˜gÔ%Ø—‘˜q§+¡+¨kÓ":Ð;Ô<Ø•lŠGñ (ð ˆr   c                ó<   <€ V ^8„  d   QhRS[ RS[S[,          RS[/# )r   r   rË  r   )r   r   r   rÖ  )r   r�   s   "€r   r   rè  *  s,   ø€ ÷ *Mñ *M™Cð *M©©c­ð *MÑ7Gñ *Mr   c                ó(  € WP                   9  d   \        VR R R ^ 4      # V P                   V,          pV'       g   RV9   d   \        V. R R ^ 4      # Rp^ pV\        V4      8  d1   W%,          V9   d$   W2V,          ,          pRV9   d   TpV^,          pK@  V\        V4      ^,
          u;8X  d   ^ 8¼  d   M M\        WR R ^ 4      # R pV^ 8”  d   VR V^,            pR pTpV^ 8”  d¢   R p	V. 3.p
V
'       d‡   V
P                  4       w  r;RV9   d$   V	e   \        V4      \        V	4      8  d   Tp	KB  KD  V	e   \        V4      \        V	4      8  g   Kc  V F#  pV
P	                  W<,          W¼.,           34       K%  	  KŽ  VR V V	,           p\        VR WgV4      # )Nrê  rÝ   )rä  rÖ  r    r÷  rF  )ri   r   rË  rì  Ú
last_indexÚindexrÙ  rÚ  rÛ  ÚbestÚstackÚextrarí  s   &&&          r   ÚsearchÚPromptTrie.search*  sƒ  € ØŸ
™
Ô"Ü# E¨4°°t¸QÓ?Ð?à—*‘*˜UÕ#ˆç˜+¨Ô0Ü# E¨2¨t°T¸1Ó=Ð=ð ˆ
ØˆØ”c˜&“kÔ! f¥m°wÔ&>Ø U�mÕ,ˆGØ˜gÔ%Ø"�
Ø�Q�JŠEð œ˜V› q�Ö-¨A×-Ü# E°4¸¸qÓAÐAð ˆØ˜Œ>ØÐ-˜z¨A�~Ð.ˆGð ˆØˆØ�1Œ9ØˆDØ˜r�]�OˆEßØ!&§¡£‘�Ø 'Ô)Ø’|¤s¨5£z´C¸³IÔ'=Ø$šñ (>à’\¤S¨£Z´#°d³)Ö%;Û&˜ØŸ™ g¥l°E¸EµMÐ%BÖCó  'à˜F˜U�^ dÕ*ˆFÜ  t¨W¸mÓLÐLr   rã  N)r2   r‡   rˆ   r‰   r—   rî  rë  r÷  rþ  r  r�   rŽ   r�   s   @r   rá  rá  ü  s>   ø‡ € ò÷ð ÷$ð $÷ð ÷ð ÷*Mö *Mr   rá  c                   óÎ   a € ] tR tRt o ] ! R R4      4       t ! R R4      tRV 3R lR lltR t]	R	 4       t
V 3R
 lR ltRR/V 3R lR lltRRRR/V 3R lR lltR tRtV tR# )ÚLRUPromptCacheiW  c                   ó,   a € ] tR tRt o V 3R ltRtV tR# )ÚLRUPromptCache.CacheEntryiX  c                óN   <€ V ^8„  d   Qh/ S[ S[,          ;R&   S[;R&   S[;R&   # )r   Úprompt_cacher}   Ú
cache_type)r   r   r   r,   )r   r�   s   "€r   r   Ú&LRUPromptCache.CacheEntry.__annotate__X  s,   ø‡ ‚ á™3•iÑñ ñ ‰ñ ñ ‰ò	 r   rg   NrÝ  r�   s   @r   Ú
CacheEntryr  X  rß  r   r  c                   óp   a € ] tR tRt o . RO3V 3R lR lltR tRV 3R lR lltV 3R lR ltR	 tR
t	V t
R# )ÚLRUPromptCache.CacheOrderi^  c                ó0   <€ V ^8„  d   QhRS[ S[,          /# )r   Úordering)r   r,   )r   r�   s   "€r   r   Ú&LRUPromptCache.CacheOrder.__annotate___  s   ø€ ÷ 	8ñ 	8¡T©#¥Yñ 	8r   c                óZ   € Wn         V Uu/ uF  q"\        4       bK  	  upV n        R # u upi rK   )Ú	_orderingr   Ú_lrus)ri   r  r@  s   && r   r—   Ú"LRUPromptCache.CacheOrder.__init___  s$   € Ø%ŒNÙ.6Ó7©h¨œU›Wš*©hÑ7ˆDŽJùÒ7s   ‹(c                óV   € \        R  V P                  P                  4        4       4      # )c              3   ó8   "  € T F  p\        V4      x € K  	  R # 5irK   )r    )rO   Úlrus   & r   rP   Ú4LRUPromptCache.CacheOrder.__len__.<locals>.<genexpr>d  s   é € Ð?Ñ+> C”s˜3—x�xÓ+>ùr   )rÔ  r  r–   rh   s   &r   Ú__len__Ú!LRUPromptCache.CacheOrder.__len__c  s    € ÜÑ?¨4¯:©:×+<Ñ+<Ô+>Ó?Ó?Ð?r   c                ó<   <€ V ^8„  d   QhRS[ RS[S[ ,          RS[/# )r   r   rË  r  )r   r   r,   )r   r�   s   "€r   r   r  f  s&   ø€ ÷ 	;ñ 	;™cð 	;©4±­9ð 	;Á#ñ 	;r   c                óL   € V P                   V,          P                  W34       R # rK   )r  rF  ©ri   r   rË  r  s   &&&&r   ÚpushÚLRUPromptCache.CacheOrder.pushf  s   € Ø�J‰J�zÕ"×)Ñ)¨5¨/Ö:r   c                ó6   <€ V ^8„  d   QhRS[ RS[S[ ,          /# rñ  )r   r   )r   r�   s   "€r   r   r  i  s   ø€ ÷ 	ñ 	¡ð 	©T±#­Yñ 	r   c                óš   € V P                    F(  p V P                  V,          P                  W34        R # 	  R #   \         d     K<  i ; irK   )r  r  Úremoverm   r#  s   &&& r   r(  Ú LRUPromptCache.CacheOrder.removei  sE   € Ø"Ÿnœn�
ðØ—J‘J˜zÕ*×1Ñ1°5°/ÔBÚó -øô "ô Úðús   ‘#;»A
Á	A
c                ó~  € ^ pV^,           \        V P                  4      8  dŒ   V P                  V P                  V,          ,          pV P                  V P                  V^,           ,          ,          pV'       d*   \        V4      \        V4      8¼  d   VP                  4       # V^,          pK¬  XP                  4       # r~  )r    r  r  Úpopleft)ri   ró   Úlru_aÚlru_bs   &   r   r÷  ÚLRUPromptCache.CacheOrder.popq  s�   € ØˆAØ�a•%œ#˜dŸn™nÓ-Ô-ØŸ
™
 4§>¡>°!Õ#4Õ5�ØŸ
™
 4§>¡>°!°aµ%Õ#8Õ9�ßœS ›Z¬3¨u«:Ô5Ø Ÿ=™=›?Ð*Ø�Q•’Ø—=‘=“?Ð"r   )r  r  N)Ú	assistantÚuserÚsystem)r/  )r2   r‡   rˆ   r‰   r—   r  r$  r(  r÷  r�   rŽ   r�   s   @r   Ú
CacheOrderr  ^  s6   ø‡ € Ú1P÷ 	8ò 	8ò	@÷	;ò 	;÷	ð 	÷	#ð 	#r   r2  c                ó&   <€ V ^8„  d   QhRS[ RS[ /# )r   r   Ú	max_bytesrÌ   )r   r�   s   "€r   r   ÚLRUPromptCache.__annotate__{  s   ø€ ÷ Dñ D¡ð D±cñ Dr   c                óÜ   € Wn         W n        \        4       V n        \        P                  4       V n        ^ V n        V P                  P                   Uu/ uF  q3^ bK  	  upV n	        R# u upi rÏ  )
r   r4  rá  rä  r
  r2  Ú_lruÚ_n_bytesr  Ú_n_bytes_by_type)ri   r   r4  r@  s   &&& r   r—   ÚLRUPromptCache.__init__{  sT   € Ø ŒØ"ŒÜ“\ˆŒ
Ü"×-Ñ-Ó/ˆŒ	ØˆŒØ/3¯y©y×/BÒ/BÓ CÑ/B¨! A¢Ñ/BÑ CˆÖùÒ Cs   ÁA)c                ó,   € \        V P                  4      # rK   )r    r7  rh   s   &r   r  ÚLRUPromptCache.__len__ƒ  s   € Ü�4—9‘9‹~Ðr   c                ó   € V P                   # rK   )r8  rh   s   &r   r}   ÚLRUPromptCache.nbytes†  s   € à�}‰}Ðr   c                ó6   <€ V ^8„  d   QhRS[ RS[S[,          /# rñ  rç  )r   r�   s   "€r   r   r5  Š  s   ø€ ÷ ñ ©ð ±d¹3µiñ r   c                óº  € V P                   P                  W4      pVP                  eS   V P                   P                  VP                  VP                  4      p\
        P                  ! VP                  4      . 3# VP                  e   \        VP                  4      M^ pVP                  eÐ   VP                  V8”  d¿   V P                   P                  VP                  VP                  4      p\        VP                  4      '       dt   \
        P                  ! VP                  4      p\        \        V4      ^,
          VP                  4      p\        VP                  4      V,
          p\        Wh4       WbVR  3# V^ 8”  dU   V P                   P                  VP                  VP                  4      p\
        P                  ! VP                  4      W%R  3# R V3# rK   )rä  r  rØ  rë  r   ÚcopyÚdeepcopyr  rÙ  r    rÚ  rÛ  rU   r®   rZ   )	ri   r   rË  ÚresultÚcache_entryÚshort_lengthr*   ÚprefixÚnum_to_trims	   &&&      r   Úfetch_nearest_cacheÚ"LRUPromptCache.fetch_nearest_cacheŠ  sX  € Ø—‘×"Ñ" 5Ó1ˆØ�<‰<Ò#ØŸ*™*Ÿ.™.¨¯©°v·|±|ÓDˆKÜ—=’= ×!9Ñ!9Ó:¸BÐ>Ð>à.4¯n©nÒ.H”s˜6Ÿ>™>Ô*ÈaˆØ�=‰=Ò$¨×)=Ñ)=ÀÔ)LØŸ*™*Ÿ.™.¨¯©°v·}±}ÓEˆKÜ$ [×%=Ñ%=×>Ò>ÜŸš k×&>Ñ&>Ó?�ÜœS ›[¨1�_¨f×.BÑ.BÓC�Ü! &§-¡-Ó0°6Õ9�Ü! %Ô5Ø V W˜oÐ-Ð-à˜!ÔØŸ*™*Ÿ.™.¨¯©°v·~±~ÓFˆKÜ—=’= ×!9Ñ!9Ó:¸FÀ=Ð<QÐQÐQà�Vˆ|Ðr   r  r/  c                óR   <€ V ^8„  d   QhRS[ RS[S[,          RS[S[ ,          RS[/# )r   r   rË  r  r  )r   r   r   r,   )r   r�   s   "€r   r   r5     sA   ø€ ÷ )Dñ )Dáð)Dñ ‘S•	ð)Dñ ™3•ið	)Dñ ñ)Dr   c               óN  € \         P                  V\        R  V 4       4      V4      pV ;P                  VP                  ,          un        V P
                  V;;,          VP                  ,          uu&   V P                  P                  WV4      pVer   V ;P                  VP                  ,          un        V P
                  VP                  ;;,          VP                  ,          uu&   V P                  P                  W4       V P                  P                  WV4       \        V4      '       d™   V P                  P                  W4       Fy  w  ruV ;P                  VP                  ,          un        V P
                  VP                  ;;,          VP                  ,          uu&   V P                  P                  WRV 4       K{  	  \        V P                  4      V P                  8”  dŽ   V P                  P!                  4       w  rV P                  P!                  W4      pV ;P                  VP                  ,          un        V P
                  VP                  ;;,          VP                  ,          uu&   V P                  V P"                  8”  d�   V P                  P!                  4       w  rV P                  P!                  W4      pV ;P                  VP                  ,          un        V P
                  VP                  ;;,          VP                  ,          uu&   Kª  R# )c              3   ó8   "  € T F  qP                   x € K  	  R # 5irK   r
  rN   s   & r   rP   Ú.LRUPromptCache.insert_cache.<locals>.<genexpr>ª  s   é € Ð=±¨1Ÿhžh³ùr   N)r
  r  rÔ  r8  r}   r9  rä  rî  r  r7  r(  r$  rU   rþ  r    r   r÷  r4  )ri   r   rË  r  r  Úentryrå   Ú
prefix_lens   &&&&$   r   Úinsert_cacheÚLRUPromptCache.insert_cache   s  € ô ×)Ñ)Øœ#Ñ=±Ó=Ó=¸zó
ˆð
 	�Š˜Ÿ™Õ%�Ø×Ñ˜j×)¨U¯\©\Õ9Ó)Ø�z‰z�~‰~˜e¨UÓ3ˆØÒØ�MŠM˜TŸ[™[Õ(�MØ×!Ñ! $§/¡/×2°d·k±kÕAÓ2Ø�I‰I×Ñ˜UÔ+Ø�	‰	�‰�u jÔ1ô ! ×.Ò.Ø%)§Z¡Z×%<Ñ%<¸UÖ%KÑ!�
Ø—’ §¡Õ-•Ø×%Ñ% e×&6Ñ&6×7¸5¿<¹<ÕGÓ7Ø—	‘	× Ñ  ¨{°
Ð(;Ö<ñ &Lô ˆt�y‰y‹>˜DŸM™MÔ)Ø ŸI™IŸM™M›O‰MˆEØ—J‘J—N‘N 5Ó1ˆEØ�MŠM˜UŸ\™\Õ)�MØ×!Ñ! %×"2Ñ"2×3°u·|±|ÕCÓ3Ø�m‰m˜dŸn™nÔ,Ø ŸI™IŸM™M›O‰MˆEØ—J‘J—N‘N 5Ó1ˆEØ�MŠM˜UŸ\™\Õ)�MØ×!Ñ! %×"2Ñ"2×3°u·|±|ÕCÕ3ñ	 -r   Ún_sequencesNÚn_bytesc                óF   <€ V ^8„  d   QhRS[ S[,          RS[ S[,          /# )r   rR  rS  )r   r   )r   r�   s   "€r   r   r5  Ë  s)   ø€ ÷ Dñ DÙ&¡s�mðDÙ=EÁc½]ñDr   c               óà  € Ve   \        ^ V4      MRpVe   \        ^ V4      MRp\        V P                  4      V8”  d�   V P                  P                  4       w  r4V P                  P                  W44      pV ;P
                  VP                  ,          un        V P                  VP                  ;;,          VP                  ,          uu&   K©  V P
                  V8”  d�   V P                  P                  4       w  r4V P                  P                  W44      pV ;P
                  VP                  ,          un        V P                  VP                  ;;,          VP                  ,          uu&   K   R # )Nì            )	r  r    r7  r÷  rä  r8  r}   r9  r  )ri   rR  rS  r   rË  rN  s   &$$   r   Útrim_toÚLRUPromptCache.trim_toË  sô   € ð .9Ò-D”c˜!˜[Ô)È'ˆØ%,Ò%8”#�a˜”/¸gˆä�$—)‘)‹n˜{Ô*Ø ŸI™IŸM™M›O‰MˆEØ—J‘J—N‘N 5Ó1ˆEØ�MŠM˜UŸ\™\Õ)�MØ×!Ñ! %×"2Ñ"2×3°u·|±|ÕCÕ3Ø�m‰m˜gÔ%Ø ŸI™IŸM™M›O‰MˆEØ—J‘J—N‘N 5Ó1ˆEØ�MŠM˜UŸ\™\Õ)�MØ×!Ñ! %×"2Ñ"2×3°u·|±|ÕCÕ3ñ	 &r   c                ó¾   € / pV P                   P                   F@  pR \        V P                   P                  V,          4      RV P                  V,          /W&   KB  	  V# )rR  rS  )r7  r  r    r  r9  )ri   rC  r  s   &  r   Ústats_by_typeÚLRUPromptCache.stats_by_typeÜ  sS   € ØˆØŸ)™)×-Ô-ˆJàœs 4§9¡9§?¡?°:Õ#>Ó?Ø˜4×0Ñ0°Õ<ð"ˆFÓñ .ð
 ˆr   )r7  r8  r9  rä  r4  r   )é
   rV  )r2   r‡   rˆ   r‰   r   r  r2  r—   r  rŠ   r}   rH  rP  rW  rZ  r�   rŽ   r�   s   @r   r
  r
  W  s•   ø‡ € Ø÷ð ó ð÷
#ñ #÷:Dò Dòð ñó ð÷ð ð,)Dð &÷)Dò )DðVDØ.2ðDØMQ÷Dò D÷"ð r   r
  rK   ru   )+rA  Úcollectionsr   Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorer3   Úmlx.nnr   Ú	mlx.utilsr	   r
   r   r   Úbaser   r'   r:   rF   rU   rZ   rc   re   r’   rÇ   r$   r#   r�  rØ  rò  r+  r5  rx  rÆ  rÖ  rá  r
  rg   r   r   Ú<module>re     s"  ðó Ý Ý !ß ,Ó ,å Ý ß IÓ Iå $÷6ð8 TV÷ ?ô&õ40õ2õ&
÷0ñ 0ôf35˜ô 35ôlZS�zô ZSôzR5ˆjô R5ôju5�jô u5ôpFB�*ô FBôRP5�Zô P5ôfV�
ô VòrôZ5�:ô Z5ôz_5˜:ô _5÷D!*ñ !*ðH ÷ð ó ð÷XMñ XM÷vLó Lr   