+
    UV-jÀ= ã                  ó@  € R t ^ RIHt ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RI	t	^ RI
t
^ RIHt ^ RIHt ^ RIHt ^ RIHtHtHtHtHtHt ^ RIHt ^ RIt]P:                  ! R4      t^tRt ^ t!R	 R
 lt"R R lt#R R lt$RwR R llt%R R lt&R R lt'R R lt(RR/R R llt)R R lt*R R lt+R R lt,R  R! lt-RxR" R# llt.R$ R% lt/R& R' lt0R( R) lt1R* R+ lt2R, R- lt3R.R/R/ R0 llt4] ! R1 R24      4       t5] ! R3 R44      4       t6]! R5R67       ! R7 R84      4       t7]! R5R67       ! R9 R:4      4       t8]! R5R67       ! R; R<4      4       t9]! R5R67       ! R= R>4      4       t:] ! R? R@4      4       t;RA RB lt<RC RD lt=RE RF lt>RG RH lt?RI RJ lt@RK RL ltARM RN ltBRO RP ltCRQ RR ltDRS RT ltERU RV ltF ! RW RX4      tG ! RY RZ4      tHRyR[ R\ lltIR] R^ ltJR_ R` ltKRa Rb ltLRc Rd ltMRe Rf ltNRg Rh ltORi Rj ltPRk Rl ltQRm^ Rn^ /Ro Rp lltRRq Rr ltSRs Rt ltTRyRu Rv lltUR# )zu¾  Automatic Prefix Caching (APC) for mlx-vlm.

Hash-based, block-level KV cache reuse across requests. The KV cache is split
into fixed-size blocks (default 16 tokens). Each fully-filled block is
identified by a chained hash::

    block_hash[i] = H(block_hash[i-1], tuple(tokens[i*bs:(i+1)*bs]), extra_hash[i])

``extra_hash[i]`` carries multimodal context (e.g. an image content hash) so
identical token IDs with different images don't collide. ``H`` defaults to
Python's built-in ``hash`` (fast, deterministic within a single process). Set
``APC_HASH=sha256`` to opt into a stable cryptographic hash (~100-200 ns/tok
overhead).

Eviction is LRU with reference counting: blocks are kept alive while
``ref_cnt > 0`` and the free queue is a doubly-linked list embedded in
``APCBlock`` for O(1) move-to-tail. All blocks are pre-allocated as a pool
to avoid Python object churn. When ``APC_DISK_PATH`` is configured, full
blocks are also written to a shard-based SSD tier and can be restored after
process restart through a direct-read prompt-cache path.

Numerical note: APC itself is *exact*. The K/V tensors stored in the block
pool are byte-identical to what a fresh prefill would produce â€” the cache
introduces no approximation, it just retains tensors. However, cold-vs-warm
runs of the same prompt can produce slightly different logits because of
**batch non-invariance** in the attention kernel: a long Q (cold prefill,
e.g. 60 tokens) and a short Q (warm-start suffix, e.g. 13 tokens against
47 cached tokens) trigger different tile shapes / reduction orders inside
flash-attention, and floating-point matmul is non-associative. The
Thinking Machines analysis (2025) and Microsoft Research's LLM-42 paper
give the formal treatment. The same drift happens without prefix caching
any time dynamic batching changes the batch composition between two
identical requests â€” APC just makes it visible by giving a clean
cold/warm contrast. Warm-to-warm runs *are* deterministic: identical
prompts repeated under APC always produce identical text. For
bit-equivalent cold==warm, you need batch-invariant RMSNorm / matmul /
attention kernels (vLLM's ``--enable-batch-invariance``, SGLang with
FlashInfer/FA3), not a different cache design.
)ÚannotationsN)ÚOrderedDict)Ú	dataclass)ÚPath)ÚAnyÚIterableÚListÚOptionalÚSequenceÚTuplezmlx_vlm.apci   c               ó   € V ^8„  d   QhRR/# ©é   ÚreturnÚbool© )Úformats   "Ú\/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/apc.pyÚ__annotate__r   B   s   € ÷ Bñ B˜$ñ Bó    c                 ód   € \         P                  P                  R R4      P                  4       R8H  # )ÚAPC_HASHÚfastÚsha256)ÚosÚenvironÚgetÚlowerr   r   r   Ú_hash_use_sha256r   B   s%   € Ü�:‰:�>‰>˜* fÓ-×3Ñ3Ó5¸ÑAÐAr   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   ÚparentÚintÚtokensúTuple[int, ...]Úextrar   r   )r   s   "r   r   r   F   s(   € ÷ )ñ )˜ð ) oð )¸cð )Àcñ )r   c                ó
  € \        4       '       dè   \        P                  ! 4       pVP                  \	        V R,          4      P                  ^R4      4       VP                  \        P                  ! V\        P                  R7      P                  4       4       VP                  \	        VR,          4      P                  ^R4      4       \        P                  VP                  4       R,          RRR7      # \        WV34      # )zChain hash for a single block.Úlittle©ÚdtypeºNé   NT©Úsignedì   ÿÿÿÿ )r   Úhashlibr   Úupdater!   Úto_bytesÚnpÚasarrayÚint32ÚtobytesÚ
from_bytesÚdigestÚhash)r    r"   r$   Úhs   &&& r   Ú_hash_tokensr9   F   s°   € ä×ÒÜ�NŠNÓˆØ	�‰”�V˜}Õ-Ó.×7Ñ7¸¸8ÓDÔEØ	�‰”—’˜F¬"¯(©(Ô3×;Ñ;Ó=Ô>Ø	�‰”�U˜mÕ,Ó-×6Ñ6°q¸(ÓCÔDÜ�~‰~˜aŸh™h›j¨�n¨h¸tˆ~ÓDÐDÜ� Ð'Ó(Ð(r   c               ó    € V ^8„  d   QhRRRR/# )r   Úvaluesr!   r   r   )r   s   "r   r   r   Q   s   € ÷ Añ A˜cð A cñ Ar   c                 óú   € \         P                  ! 4       pV  F4  pVP                  \        VR,          4      P	                  ^R4      4       K6  	  \        P                  VP                  4       R,          RRR7      # )é   r&   r)   Tr+   r-   )r.   r   r/   r!   r0   r5   r6   )r;   r8   Úvalues   *  r   Ú_stable_int_hashr?   Q   sZ   € Ü�ŠÓ€AÛˆØ	�‰”�U˜mÕ,Ó-×6Ñ6°q¸(ÓCÖDñ ä�>‰>˜!Ÿ(™(›* R�.¨(¸4ˆ>Ó@Ð@r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   ÚtenantúOptional[str]Úpayload_hashr!   r   r   )r   s   "r   r   r   X   s&   € ÷ 	Añ 	A˜}ð 	A¸Cð 	AÈñ 	Ar   c                ó¸  € V '       g   \        V4      # \        V 4      P                  R4      p\        P                  ! 4       pVP                  \        V4      P                  ^R4      4       VP                  V4       VP                  \        VR,          4      P                  ^R4      4       \         P                  VP                  4       R,          RRR7      # )z5Stable APC salt for tenant-scoped multimodal context.úutf-8r&   r)   Tr+   r-   )
r!   ÚstrÚencoder.   r   r/   Úlenr0   r5   r6   )rA   rC   Útenant_bytesr8   s   &&  r   Útenant_scoped_hashrJ   X   sœ   € çÜ�<Ó Ð Ü�v“;×%Ñ% gÓ.€LÜ�ŠÓ€AØ‡H�HŒS�Ó×'Ñ'¨¨8Ó4Ô5Ø‡H�Hˆ\ÔØ‡H�HŒS� Õ/Ó0×9Ñ9¸!¸XÓFÔGÜ�>‰>˜!Ÿ(™(›* R�.¨(¸4ˆ>Ó@Ð@r   c               ó    € V ^8„  d   QhRRRR/# )r   Úxúmx.arrayr   r   )r   s   "r   r   r   d   s   € ÷ 5ñ 5�xð 5 Hñ 5r   c                ól   € \         P                  ! \         P                  ! W P                  R7      4      # )z;Materialize ``x`` into a fresh MLX-owned contiguous buffer.r'   )ÚmxÚ
contiguousÚarrayr(   )rL   s   &r   Ú_copy_mlx_arrayrR   d   s   € ä�=Š=œŸš !¯7©7Ô3Ó4Ð4r   c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R
/# )r   ÚkeysrM   r;   Úoffsetr!   Úmin_capacity_tokensúOptional[int]Ústepr   úTuple[mx.array, mx.array]r   )r   s   "r   r   r   i   sD   € ÷ ñ Ø
ðàðð ð	ð
 'ðð ðð ñr   c               óv  € Ve   W28:  d   W3# \        V4      pV^ 8”  d   WT,           ^,
          V,          V,          pWPP                  ^,          8:  d   W3# WPP                  ^,          ,
          p. V P                  R,          OVNV P                  ^,          N5p. VP                  R,          OVNVP                  ^,          N5p\        P                  ! V \        P                  ! WpP
                  R7      .^R7      p \        P                  ! V\        P                  ! W�P
                  R7      .^R7      pW3# )NºNr   Nr'   ©Úaxis)r!   ÚshaperO   ÚconcatenateÚzerosr(   )	rT   r;   rU   rV   rX   ÚcapacityÚ
pad_tokensÚk_shapeÚv_shapes	   &&$$$    r   Ú_pad_kv_for_capacityre   i   sü   € ð Ò"Ð&9Ô&CØˆ|ÐÜÐ&Ó'€HØˆa„xØ•_ qÕ(¨TÕ1°TÕ9ˆØ—:‘:˜a•=Ô Øˆ|ÐØŸJ™J q�MÕ)€JØ:�—
‘
˜2•Ð: 
Ð:¨D¯J©J°q­MÑ:€GØ>�—‘˜RÕ Ð> *Ð>¨f¯l©l¸1­oÑ>€GÜ�>Š>˜4¤§¢¨'¿¹Ô!DÐEÈAÔN€DÜ�^Š^˜V¤R§X¢X¨g¿\¹\Ô%JÐKÐRSÔT€FØˆ<Ðr   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	r   Úcr   rV   rW   Úeval_targetsúList[mx.array]r   úOptional[Any]r   )r   s   "r   r   r   €   s8   € ÷ `ñ `Ø
ð`ð 'ð`ð !ð	`ð
 ñ`r   c               ó  € ^ RI Hp \        WP                  4      '       Ed   \	        V 4      ! 4       p\        \        V R^ 4      ;'       g    ^ 4      pV P                  eÂ   V P                  e´   V^ 8”  d­   \        V P                  RRV1R3,          4      p\        V P                  RRV1R3,          4      p\        \        V R\        \	        V 4      RR4      4      ;'       g    ^ 4      p\        VVVVVR7      w  rgWdn        Wtn        WTn        VP                  Wg.4       V# \        WP                  4      '       dþ   \	        V 4      ! \        \        V R	4      4      \        \        V R
^ 4      4      R7      p\        \        V R^ 4      ;'       g    ^ 4      Vn        \        \        V R^ 4      ;'       g    ^ 4      Vn        V P                  ej   V P                  e\   \        V P                  4      Vn        \        V P                  4      Vn        VP                  VP                  VP                  .4       V# \        WP                  4      '       dé   \	        V 4      ! \        \        V R4      4      R7      p\        \        V R^ 4      ;'       g    ^ 4      Vn        \        \        V R^ 4      ;'       g    ^ 4      Vn        V P                  ej   V P                  e\   \        V P                  4      Vn        \        V P                  4      Vn        VP                  VP                  VP                  .4       V# \        WP"                  4      '       Ed   VP#                  \%        V P                  4      4      p. Vn        V P                   F[  p	V	f   VP                  P'                  R4       K$  \        V	4      p
VP                  P'                  V
4       VP'                  V
4       K]  	  V P(                  e6   \        V P(                  4      Vn        VP'                  VP(                  4       V P*                  e6   \        V P*                  4      Vn        VP'                  VP*                  4       V# \        WP,                  4      '       dt   V P.                   Uu. uF  p\1        VVVR7      NK  	  p
p\2        ;QJ d    R V
 4       F  '       g   K   RM	  RM! R V
 4       4      '       d   R# VP,                  ! V
!  # \        V \4        4      '       de   V  Uu. uF  p\1        VVVR7      NK  	  p
p\2        ;QJ d    R V
 4       F  '       g   K   RM	  RM! R V
 4       4      '       d   R# \5        V
4      # R# u upi u upi )zEDeep-copy one prompt-cache entry, preserving its concrete cache kind.©ÚcacherU   N.ºNNNrX   é   ©rU   rV   rX   Úmax_sizeÚkeep©rq   rr   Ú_idxÚ
chunk_size©ru   Ústart_position©rV   rh   c              3  ó(   "  € T F  qR J x € K
  	  R # 5i©Nr   ©Ú.0Úsub_cs   & r   Ú	<genexpr>Ú-_clone_cache_entry_for_apc.<locals>.<genexpr>Ï   ó   é € Ð1©& ˜�}«&ùó   ‚TFc              3  ó(   "  € T F  qR J x € K
  	  R # 5irz   r   r{   s   & r   r~   r   Ü   r€   r�   )Úmlx_lm.modelsrm   Ú
isinstanceÚKVCacheÚtyper!   ÚgetattrrT   r;   rR   re   rU   ÚextendÚRotatingKVCachert   ÚChunkedKVCacherw   ÚArraysCacherH   ÚappendÚleft_paddingÚlengthsÚ	CacheListÚcachesÚ_clone_cache_entry_for_apcÚanyÚtuple)rg   rV   rh   Úlm_cacheÚoutÚoffrT   r;   rX   ÚstateÚcopiedr}   s   &$$         r   r‘   r‘   €   sd  € õ 0ä�!×%Ñ%×&Ó&Ü�1Œg‹iˆÜ”'˜!˜X qÓ)×.Ð.¨QÓ/ˆØ�6‰6Ò !§(¡(Ò"6¸3À¼7Ü" 1§6¡6¨#¨t°¨t°Q¨,Õ#7Ó8ˆDÜ$ Q§X¡X¨c°4°C°4¸¨lÕ%;Ó<ˆFÜ”w˜q &¬'´$°q³'¸6À3Ó*GÓH×MÐMÈAÓNˆDÜ/ØØØØ$7Øô‰LˆDð ŒHØŒJØŒJØ×Ñ  Ô/Øˆ
ä�!×-Ñ-×.Ò.Ü�1ŒgÜœ  JÓ/Ó0Ü”W˜Q ¨Ó*Ó+ô
ˆô œ  H¨aÓ0×5Ð5°AÓ6ˆŒ
Ü”w˜q &¨!Ó,×1Ð1°Ó2ˆŒØ�6‰6Ò !§(¡(Ò"6Ü& q§v¡vÓ.ˆCŒHÜ(¨¯©Ó2ˆCŒJØ×Ñ §¡¨3¯:©:Ð 6Ô7Øˆ
ä�!×,Ñ,×-Ò-Ü�1Œg¤¤W¨Q°Ó%=Ó!>Ô?ˆÜœ  H¨aÓ0×5Ð5°AÓ6ˆŒ
Ü ¤¨Ð,<¸aÓ!@×!EÐ!EÀAÓFˆÔØ�6‰6Ò !§(¡(Ò"6Ü& q§v¡vÓ.ˆCŒHÜ(¨¯©Ó2ˆCŒJØ×Ñ §¡¨3¯:©:Ð 6Ô7Øˆ
ä�!×)Ñ)×*Ó*Ø×"Ñ"¤3 q§w¡w£<Ó0ˆØˆŒ	Ø—W”WˆEØŠ}Ø—	‘	× Ñ  Ô&ÙÜ$ UÓ+ˆFØ�I‰I×Ñ˜VÔ$Ø×Ñ Ö'ñ ð �>‰>Ò%Ü.¨q¯~©~Ó>ˆCÔØ×Ñ × 0Ñ 0Ô1Ø�9‰9Ò Ü)¨!¯)©)Ó4ˆCŒKØ×Ñ §¡Ô,Øˆ
ä�!×'Ñ'×(Ò(ð Ÿšó
ñ "�ô 'ØØ$7Ø)÷ñ
 "ð 	ð 
÷ ‹3Ñ1©&Ó1�3�3Š3Ñ1©&Ó1×1Ò1ÙØ×!Ò! 6Ñ*Ð*ä�!”U×Òñ ó
ñ �ô 'ØØ$7Ø)÷ñ
 ð 	ð 
÷ ‹3Ñ1©&Ó1�3�3Š3Ñ1©&Ó1×1Ò1ÙÜ�V‹}Ðáùò3
ùò
s   Ò"VÔ!VrV   c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úprompt_cacheúSequence[Any]rV   rW   r   úOptional[List[Any]]r   )r   s   "r   r   r   ã   s(   € ÷ ñ Øðð 'ðð ñ	r   c               ó¨   € . p. pV  F)  p\        VVVR 7      pVf    R# VP                  V4       K+  	  V'       d   \        P                  ! V4       V# )rx   N)r‘   rŒ   rO   Úeval)rš   rV   rh   r•   rg   r˜   s   &$    r   Ú_clone_prompt_cache_for_apcrŸ   ã   sW   € ð
 $&€LØ€CÛˆÜ+ØØ 3Ø%ô
ˆð
 Š>ÚØ�
‰
�6Öñ ÷ Ü
�Š�ÔØ€Jr   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   Ú
layer_keysúSequence[mx.array]Úlayer_valuesÚ
prefix_lenr!   r   rœ   r   )r   s   "r   r   r   ø   s0   € ÷ ñ Ø"ðà$ðð ðð ñ	r   c                ó¼  € ^ RI Hp V^ 8:  g   \        V 4      \        V4      8w  d   R# . p. p\        W4       F‚  w  rgV! 4       p\	        VRRV1R3,          4      Vn        \	        VRRV1R3,          4      Vn        W(n        VP                  VP
                  VP                  .4       VP                  V4       K„  	  V'       d   \        P                  ! V4       V# )zCDeep-copy layer-major K/V tensors into compact ``KVCache`` entries.©r…   N.rn   )Úmlx_lm.models.cacher…   rH   ÚziprR   rT   r;   rU   rˆ   rŒ   rO   rž   )	r¡   r£   r¤   r…   rh   r•   ÚkÚvrg   s	   &&&      r   Ú#_clone_layer_major_kv_cache_for_apcr«   ø   s¼   € õ ,à�Q„œ#˜j›/¬S°Ó->Ô>ÙØ#%€LØ€CÜ�JÖ-‰ˆÙ‹IˆÜ   3¨¨¨°QÐ#6Õ!7Ó8ˆŒÜ" 1 S¨+¨:¨+°qÐ%8Õ#9Ó:ˆŒØŒØ×Ñ˜QŸV™V Q§X¡XÐ.Ô/Ø�
‰
�1Žñ .÷ Ü
�Š�ÔØ€Jr   c               ó    € V ^8„  d   QhRRRR/# ©r   rg   r   r   r   r   )r   s   "r   r   r     s   € ÷ ñ  sð ¨tñ r   c                óì  € ^ RI Hp \        V VP                  VP                  VP
                  VP                  34      '       d   R# \        WP                  4      '       dI   \        ;QJ d&    R V P                   4       F  '       d   K   R# 	  R# ! R V P                   4       4      # \        V \        4      '       d5   \        ;QJ d    R V  4       F  '       d   K   R# 	  R# ! R V  4       4      # R# )é    rl   Tc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   ©Ú_cache_entry_supports_exact_apcr{   s   & r   r~   Ú2_cache_entry_supports_exact_apc.<locals>.<genexpr>  s   é € ÐPÁx¸eÔ2°5×9Ð9Ãxùó   ‚Fc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   r±   r{   s   & r   r~   r³      s   é € ÐIÁq¸eÔ2°5×9Ð9Ãqùr´   )rƒ   rm   r„   r…   r‰   rŠ   r‹   r�   Úallr�   r“   ©rg   r”   s   & r   r²   r²     s¾   € Ý/äØ	à×ÑØ×$Ñ$Ø×#Ñ#Ø× Ñ ð		
÷ò ñ Ü�!×'Ñ'×(Ò(ß‹sÑPÀqÇxÂxÓP�sŒsÐPŠsÐPˆsÑPÀqÇxÂxÓPÓPÐPÜ�!”U×Òß‹sÑIÁqÓI�sŒsÐIŠsÐIˆsÑIÁqÓIÓIÐIÙr   c               ó    € V ^8„  d   QhRRRR/# r­   r   )r   s   "r   r   r   $  s   € ÷ +ñ + sð +¨tñ +r   c                ó8   € ^ RI Hp \        WP                  4      # )r¯   rl   )rƒ   rm   r„   r…   r·   s   & r   Ú_cache_entry_supports_block_apcrº   $  s   € Ý/ä�a×)Ñ)Ó*Ð*r   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   Ú	token_idsúSequence[int]Ú
extra_hashr!   Ú
block_sizer   r   )r   s   "r   r   r   *  s/   € ÷ Añ A˜mð A¸ð AÈ#ð AÐRUñ Ar   c                ór  € \         P                  ! 4       pVP                  \        VR,          4      P	                  ^R4      4       VP                  \        V4      P	                  ^RRR7      4       \
        P                  ! V  Uu. uF  p\        V4      NK  	  up\
        P                  R7      pVP                  \        VP                  4      P	                  ^RRR7      4       VP                  VP                  4       4       \        P                  VP                  4       R,          RRR7      # u upi )r=   r&   Fr+   r'   r)   Tr-   )r.   r   r/   r!   r0   r1   r2   r3   Úsizer4   r5   r6   )r¼   r¾   r¿   r8   ÚtÚarrs   &&&   r   Ú_sequence_hashrÄ   *  sÖ   € Ü�ŠÓ€AØ‡H�HŒS�˜}Õ-Ó.×7Ñ7¸¸8ÓDÔEØ‡H�HŒS�‹_×%Ñ% a¨¸%Ð%Ó@ÔAÜ
�*Š*¡iÓ0¡i ”c˜!–f¡iÑ0¼¿¹Ô
A€CØ‡H�HŒS�—‘‹]×#Ñ# A x¸Ð#Ó>Ô?Ø‡H�HˆS�[‰[‹]ÔÜ�>‰>˜!Ÿ(™(›* R�.¨(¸4ˆ>Ó@Ð@ùò 1s   ÂD4c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úpixel_valuesúOptional[mx.array]Ú	image_refr   r   r!   r   )r   s   "r   r   r   4  s&   € ÷ !=ñ !=Ø$ð!=àð!=ð 	ñ!=r   c                ó  € V eˆ    \         P                  ! V 4      P                  \         P                  RR7      p\        P
                  ! VP                  4       4      P                  4       p\        P                  VR,          RRR7      # Vf   ^ # \        V\        \        34      '       d(   \        pV F  p\        V\!        VR7      4      pK  	  V# \        V\"        4      '       dS   \        P
                  ! VP%                  R4      4      P                  4       p\        P                  VR,          RRR7      # \        V\&        4      '       dB   \        P                  \        P
                  ! V4      P                  4       R,          RRR7      # \        P
                  ! \)        V4      P%                  R4      4      P                  4       p\        P                  VR,          RRR7      #   \         d     ELqi ; i)	zÖStable content hash of an image payload.

Prefers hashing the actual ``pixel_values`` tensor (so resize/transform
differences invalidate the cache). Falls back to hashing the source
identifier (path / URL / repr).
F)Úcopyr)   r&   Tr+   )rÈ   rE   )r1   r2   ÚastypeÚfloat16r.   r   r4   r6   r!   r5   Ú	Exceptionr„   Úlistr“   ÚSEED_PARENT_HASHr?   Úhash_image_payloadrF   rG   ÚbytesÚrepr)rÆ   rÈ   rÃ   r6   r8   Úits   &&    r   rÐ   rÐ   4  s‡  € ð Òð	Ü—*’*˜\Ó*×1Ñ1´"·*±*À5Ð1ÓIˆCÜ—^’^ C§K¡K£MÓ2×9Ñ9Ó;ˆFÜ—>‘> &¨¥*¨h¸t�>ÓDÐDð ÒÙÜ�)œd¤E˜]×+Ò+ÜˆÛˆBÜ  Ô$6ÀÔ$DÓEŠAñ àˆÜ�)œS×!Ò!Ü—’ 	× 0Ñ 0°Ó 9Ó:×AÑAÓCˆÜ�~‰~˜f R�j¨(¸4ˆ~Ó@Ð@Ü�)œU×#Ò#Ü�~‰~Ü�NŠN˜9Ó%×,Ñ,Ó.¨rÕ2°HÀTð ó 
ð 	
ô �^Š^œD ›O×2Ñ2°7Ó;Ó<×CÑCÓE€FÜ�>‰>˜& �* h°tˆ>Ó<Ð<øô% ô 	Úð	ús   †BG/ Ç/G>Ç=G>c               ó    € V ^8„  d   QhRRRR/# )r   Úconfigr   r   zset[int]r   )r   s   "r   r   r   X  s   € ÷ ñ ¨Sð °Xñ r   c                ó„   € \        4       pR F/  p\        WR4      pVf   K  VP                  \        V4      4       K1  	  V# )z?Return token IDs that represent media placeholders in a prompt.N)Úimage_token_idÚimage_token_indexÚvideo_token_idÚvideo_token_index)Úsetr‡   Úaddr!   )rÕ   ÚidsÚattrÚtoken_ids   &   r   Ú multimodal_token_ids_from_configrà   X  s@   € ä“E€Cóˆô ˜6¨Ó.ˆØÔØ�G‰G”C˜“MÖ"ñð €Jr   c               ó$   € V ^8„  d   QhRRRRRR/# )r   r¼   r½   Úmedia_token_idsúIterable[int]r   zTuple[Tuple[int, int], ...]r   )r   s   "r   r   r   g  s&   € ÷ ñ Øðà"ðð !ñr   c                óB  € V Uu0 uF  p\        V4      kK  	  ppV'       g   R# . pRp\        V 4       F9  w  rb\        V4      V9   d   Vf   TpK  K  Vf   K%  VP                  WV34       RpK;  	  Ve   VP                  V\        V 4      34       \	        V4      # u upi )zIReturn contiguous media-token spans as half-open ``(start, end)`` ranges.Nr   )r!   Ú	enumeraterŒ   rH   r“   )r¼   râ   rß   Ú	media_idsÚspansÚstartÚidxs   &&     r   Úmedia_token_spansrê   g  s    € ñ
 0?Ó?© 8”�X–©€IÐ?ßØˆ	à#%€EØ€EÜ" 9Ö-‰ˆÜˆx‹=˜IÔ%ØŠ}Ø’ñ àÔØ�L‰L˜%˜Ô&ØŠEñ .ð ÒØ�‰�eœS ›^Ð,Ô-Ü�‹<Ðùò @s   …Bc               ó$   € V ^8„  d   QhRRRRRR/# )r   r¼   r½   râ   rã   r   r!   r   )r   s   "r   r   r   ~  s&   € ÷ -ñ -Øð-à"ð-ð 	ñ-r   c                óP   € \        W4      pV'       g   ^ # \        R V 4       4      # )a  Minimum prefix length that leaves a text-only suffix.

APC restore paths consume full prompt-level image/video feature tensors. Until
media-feature slicing is model-aware, restored prefixes must include every
media placeholder token so the suffix can be embedded as text-only.
c              3  ó*   "  € T F	  w  rVx € K  	  R # 5irz   r   )r|   Ú_startÚends   &  r   r~   Ú(media_safe_prefix_min.<locals>.<genexpr>‹  s   é € Ð,¡e‘{�v�s£eùó   ‚)rê   Úmax)r¼   râ   rç   s   && r   Úmedia_safe_prefix_minró   ~  s%   € ô ˜iÓ9€EßÙÜÑ,¡eÓ,Ó,Ð,r   c               ó(   € V ^8„  d   QhRRRRRRRR/# ©	r   r¼   r½   r¤   r!   râ   rã   r   r   r   )r   s   "r   r   r   Ž  s6   € ÷ Pñ PØðPàðPð #ðPð 
ñ	Pr   c                ó0   € \        V4      \        W4      8¬  # rz   )r!   ró   )r¼   r¤   râ   s   &&&r   Úprefix_leaves_text_only_suffixr÷   Ž  s   € ô
 ˆz‹?Ô3°IÓOÑOÐOr   c               ó(   € V ^8„  d   QhRRRRRRRR/# rõ   r   )r   s   "r   r   r   –  s6   € ÷ 
Rñ 
RØð
Ràð
Rð #ð
Rð 
ñ	
Rr   c                ó"  a€ V Uu0 uF  p\        V4      kK  	  upoS'       d   V^ 8:  d   R# \        \        V4      \        V 4      4      p\        ;QJ d"    V3R lV RV  4       F  '       g   K   R# 	  R# ! V3R lV RV  4       4      # u upi )zCReturn whether the prefix itself contains media placeholder tokens.Fc              3  ó@   <"  € T F  p\        V4      S9   x € K  	  R # 5irz   ©r!   )r|   rß   ræ   s   & €r   r~   Ú/prefix_contains_media_tokens.<locals>.<genexpr>   s   øé € ÐQÑ:P¨hŒs�8‹} 	Ö)Ó:Pùó   ƒNT)r!   ÚminrH   r’   )r¼   r¤   râ   rß   Ú
prefix_endræ   s   &&&  @r   Úprefix_contains_media_tokensr   –  sz   ø€ ñ 0?Ó?© 8”�X–©Ñ?€Iß˜
 aœÙÜ”S˜“_¤c¨)£nÓ5€Jß‹3ÔQ¸)ÀKÀZÑ:PÓQ�3Œ3ÐQŠ3ÐQˆ3ÔQ¸)ÀKÀZÑ:PÓQÓQÐQùò	 @s   †BÚmax_prefix_tokensc          
     ó,   € V ^8„  d   QhRRRRRRRRR	R/# )
r   r¼   r½   Údesired_prefix_lenr!   râ   rã   r  rW   r   r   )r   s   "r   r   r   £  s:   € ÷ ñ Øðàðð #ðð
 %ðð 	ñr   c               ó¾   € Vf   \        V 4      ^,
          M
\        V4      pV^ 8:  d   ^ # \        ^\        V4      4      p\        V\        W4      4      pWd8”  d   ^ # V# )z•Move an APC prefix forward until its suffix contains no media tokens.

Returns ``0`` when no useful safe prefix exists within ``max_prefix_tokens``.
)rH   r!   rò   ró   )r¼   r  râ   r  Úmax_lenÚdesiredr¤   s   &&&$   r   Ú%adjust_prefix_to_text_suffix_boundaryr  £  sa   € ð 0Ò7ŒˆI‹˜Ö¼SÐARÓ=Sð ð �!„|ÙÜ�!”SÐ+Ó,Ó-€GÜ�WÔ3°IÓOÓP€JØÔÙØÐr   c                  ó²   € ] tR tRt$ RtR]R&   RtR]R&   ]tR]R&   Rt	R	]R
&   ^ t
R]R&   ^ tR]R&   RtR]R&   RtR]R&   RtR]R&   RtR]R&   RtR]R&   RtR# )ÚAPCBlockiº  zBOne fixed-size KV block. Holds per-layer K/V slabs once committed.r!   Úblock_idNrW   Ú
block_hashÚparent_hashr#   r¼   r¾   Úref_cntzOptional[List[mx.array]]rT   r;   ç        ÚfloatÚ	last_usedzOptional['APCBlock']ÚprevÚnextr   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__annotations__r  rÏ   r  r¼   r¾   r  rT   r;   r  r  r  Ú__static_attributes__r   r   r   r	  r	  º  sr   ‡ áLàƒMØ $€J�Ó$Ø'€K�Ó'Ø!#€IˆÓ#Ø€J�ÓØ€GˆSÓØ%)€DÐ
"Ó)Ø'+€FÐ$Ó+Ø€IˆuÓØ!%€DÐ
Ó%Ø!%€DÐ
×%r   r	  c                  óD   € ] tR tRt$ RtR]R&   R]R&   R]R&   R	]R
&   RtR# )ÚAPCExactCacheEntryiË  z<Exact-prefix prompt-cache snapshot for custom cache layouts.r#   r¼   r!   r¾   ú	List[Any]rš   r  r  r   N©r  r  r  r  r  r  r  r   r   r   r  r  Ë  s   ‡ áFàÓØƒOØÓØ×r   r  T)Úfrozenc                  óX   € ] tR tRt$ RtR]R&   R]R&   R]R&   R]R&   R	]R
&   R	]R&   RtR# )Ú_DiskBlockSnapshotiÕ  z@Immutable view of an APC block for the asynchronous disk writer.r!   r  r  r¾   r#   r¼   ri   rT   r;   r   Nr  r   r   r   r   r   Õ  s'   ‡ áJàƒOØÓØƒOØÓØ
ÓØ×r   r   c                  óN   € ] tR tRt$ RtR]R&   R]R&   R]R&   R]R&   R]R	&   R
tR# )Ú_DiskLayerMajorBlockiá  z7Per-block metadata for a direct layer-major disk write.r!   r  r  r¾   r#   r¼   Úsource_block_idxr   Nr  r   r   r   r"  r"  á  s!   ‡ áAàƒOØÓØƒOØÓØ×r   r"  c                  ób   € ] tR tRt$ RtR]R&   R]R&   R]R&   R]R	&   R
]R&   R]R&   R]R&   RtR# )Ú_DiskLayerMajorSnapshotiì  z7Direct shard snapshot from the live per-layer KV cache.úList[_DiskLayerMajorBlock]Úblocksri   r¡   r£   r!   r¿   rF   Ústore_idÚsegment_indexÚsegment_countr   Nr  r   r   r   r%  r%  ì  s-   ‡ áAà&Ó&ØÓØ Ó ØƒOØƒMØÓØ×r   r%  c                  ó@   € ] tR tRt$ R]R&   R]R&   R]R&   R]R&   R	tR
# )Ú_DiskExactCacheSnapshotiù  r!   Ú
cache_hashr#   r¼   r¾   r  rš   r   N)r  r  r  r  r  r  r   r   r   r,  r,  ù  s   ‡ àƒOØÓØƒOØ×r   r,  c                  ó¾   € ] tR tRt$ ^ tR]R&   ^ tR]R&   ^ tR]R&   ^ tR]R&   ^ t	R]R&   ^ t
R]R&   ^ tR]R	&   ^ tR]R
&   ^ tR]R&   ^ tR]R&   ^ tR]R&   R R ltRtR# )ÚAPCStatsi  r!   ÚhitsÚmissesÚmatched_tokensÚserved_tokensÚ	evictionsÚstoresÚ	pool_usedÚ	disk_hitsÚdisk_writesÚ
exact_hitsÚexact_storesc               ó$   € V ^8„  d   QhRRRRRR/# )r   Ú
num_blocksr!   r¿   r   Údictr   )r   s   "r   r   ÚAPCStats.__annotate__  s!   € ÷ 
ñ 
 3ð 
°Cð 
¸Dñ 
r   c                	óŒ  € V P                   V P                  ,           pV^ 8”  d   V P                   V,          MRpRVRVRV P                  RV P                  RV P                  RV P                   RV P                  R	VR
V P
                  RV P                  RV P                  RV P                  RV P                  RV P                  /# )r¯   r  r¿   r<  r6  Úlookups_hitÚlookups_missr2  r3  Útoken_hit_rater4  r5  r7  r8  r9  r:  )r2  r3  r6  r0  r1  r4  r5  r7  r8  r9  r:  )Úselfr<  r¿   ÚdenomÚhit_rates   &&&  r   ÚsnapshotÚAPCStats.snapshot  s´   € Ø×#Ñ# d×&8Ñ&8Õ8ˆØ27¸!´)�4×&Ñ&¨Ö.Àˆà˜*Ø˜*Ø˜Ÿ™Ø˜4Ÿ9™9Ø˜DŸK™KØ˜d×1Ñ1Ø˜T×/Ñ/Ø˜hØ˜Ÿ™Ø�d—k‘kØ˜Ÿ™Ø˜4×+Ñ+Ø˜$Ÿ/™/Ø˜D×-Ñ-ð
ð 	
r   r   N)r  r  r  r  r0  r  r1  r2  r3  r4  r5  r6  r7  r8  r9  r:  rF  r  r   r   r   r/  r/    sw   ‡ à€Dˆ#ƒMØ€FˆCƒOØ€N�CÓØ€M�3ÓØ€IˆsÓØ€FˆCƒOØ€IˆsÓØ€IˆsÓØ€K�ÓØ€J�ÓØ€L�#Ó÷
ñ 
r   r/  c               ó    € V ^8„  d   QhRRRR/# )r   ÚnamerF   r   r   )r   s   "r   r   r   $  s   € ÷ ñ ˜#ð  #ñ r   c                óv   € \         P                  ! RRV 4      P                  R4      ;'       g    RpVR,          # )zFSanitize a model identifier into a filesystem-friendly directory name.z[^A-Za-z0-9._-]+Ú_Údefault:Né€   N)ÚreÚsubÚstrip)rI  Úsafes   & r   Ú_safe_namespacerR  $  s2   € ä�6Š6Ð% s¨DÓ1×7Ñ7¸Ó<×IÐIÀ	€DØ��:Ðr   c               ó    € V ^8„  d   QhRRRR/# )r   Úpathr   r   z Optional[Tuple[dict, dict, int]]r   )r   s   "r   r   r   *  s   € ÷ ñ  4ð Ð,Lñ r   c                ó0  €  \        V R4      ;_uu_ 4       pVP                  ^4      p\        V4      ^8  d    RRR4       R# \        P	                  VR4      pV^ 8:  g   VR8”  d    RRR4       R# VP                  V4      p\        V4      V8  d    RRR4       R#  RRR4       \
        P                  ! X4      p\        VP                  R/ 4      ;'       g    / 4      pWV^X,           3#   + '       g   i     LV; i  \        \        3 d     R# i ; i)zÁRead a safetensors header without touching tensor payload bytes.

Returns ``(tensor_entries, metadata, data_start)`` where ``data_start`` is
the absolute file offset of the tensor data buffer.
ÚrbNr&   Ú__metadata__i   )ÚopenÚreadrH   r!   r5   ÚjsonÚloadsr=  ÚpopÚOSErrorÚ
ValueError)rT  ÚfÚ
head_bytesÚheader_sizeÚheader_bytesÚheaderÚmetadatas   &      r   Ú_read_safetensors_headerre  *  sø   € ðÜ�$˜×Ô ØŸ™ ›ˆJÜ�:‹ Ô"Ø÷ Ñô Ÿ.™.¨°XÓ>ˆKð ˜aÔ ;Ð1AÔ#AØ÷ Ñð Ÿ6™6 +Ó.ˆLÜ�<Ó  ;Ô.Ø÷ Ñð /÷ ô —’˜LÓ)ˆÜ˜Ÿ
™
 >°2Ó6×<Ð<¸"Ó=ˆØ  [¥Ð0Ð0÷ ×ûô ”ZÐ ô ÚðúsK   ‚D  –#C-¹D  Á%C-Á(D  Á2"C-ÂD  Â;D  ÃD  Ã-C=	Ã8D  Ä DÄDc               ó    € V ^8„  d   QhRRRR/# )r   rT  r   r   zOptional[dict]r   )r   s   "r   r   r   D  s   € ÷ 	ñ 	 Tð 	¨nñ 	r   c                ó6   € \        V 4      pVf   R# V^,          # )z¶Read only the ``__metadata__`` dict from a safetensors file, without
mmap'ing the tensor payload. Used to populate the disk index on init.

Returns ``None`` on any read/parse error.
N)re  )rT  rc  s   & r   Ú_read_safetensors_metadatarh  D  s    € ô & dÓ+€FØ‚~ÙØ�!�9Ðr   c               ó    € V ^8„  d   QhRRRR/# )r   r^   r½   r   r!   r   )r   s   "r   r   r   P  s   € ÷ ñ �-ð  Cñ r   c                ó@   € ^pV  F  pV\        V4      ,          pK  	  V# ©r=   rû   )r^   r•   Údims   &  r   Ú_numelrm  P  s#   € Ø
€CÛˆØŒs�3‹x�Šñ à€Jr   c               ó   € V ^8„  d   QhRR/# )r   r(   rF   r   )r   s   "r   r   r   W  s   € ÷ ñ  3ñ r   c                ó<  € V R8X  d6   \         P                  ! R4      \        P                  \        P                  3# R\         P                  ! R4      \        P
                  R3R\         P                  ! R4      \        P                  R3/pVP                  V 4      # )zEReturn ``(numpy_dtype, mlx_dtype, bitcast_to)`` for supported dtypes.ÚBF16z<u2ÚF16z<f2NÚF32z<f4)r1   r(   rO   Úuint16Úbfloat16rÌ   Úfloat32r   )r(   Úmappings   & r   Ú_safetensors_dtype_inforw  W  sn   € à�„Ü�xŠx˜‹¤§	¡	¬2¯;©;Ð6Ð6à”—’˜“¤§¡¨TÐ2Ø”—’˜“¤§¡¨TÐ2ð€Gð �;‰;�uÓÐr   c               ó    € V ^8„  d   QhRRRR/# )r   Úentryr=  r   z*Optional[Tuple[int, int, Tuple[int, ...]]]r   )r   s   "r   r   r   b  s   € ÷ ñ Øðà/ñr   c                óò  €  V R ,          w  r\         ;QJ d    . R V R,           4       F  NK  	  5M! R V R,           4       4      p\        \        V R,          4      4      pVf   R# Vw  p p\        V4      \        V4      8  d   R# \	        V4      VP
                  ,          \        V4      \        V4      ,
          8w  d   R# \        V4      \        V4      V3#   \        \        \        3 d     R# i ; i)Údata_offsetsc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   rû   ©r|   rL   s   & r   r~   Ú-_safetensors_tensor_bounds.<locals>.<genexpr>g  s   é € Ð5¡n ”c˜!—f�f£nùr´   r^   r(   N)	r“   rw  rF   r!   rm  ÚitemsizeÚKeyErrorÚ	TypeErrorr^  )ry  rè   rï   r^   Ú
dtype_infoÚnp_dtyperK  s   &      r   Ú_safetensors_tensor_boundsr„  b  sÈ   € ðØ˜>Õ*‰
ˆß”Ñ5 e¨G¦nÓ5—‘Ñ5 e¨G¦nÓ5Ó5ˆÜ,¬S°°wµÓ-@ÓAˆ
ØÒÙØ#‰ˆ�!�QÜˆs‹8”c˜%“jÔ ÙÜ�%‹=˜8×,Ñ,Õ,´°C³¼3¸u»:Õ0EÔEÙÜ�5‹zœ3˜s›8 UÐ*Ð*øÜ”i¤Ð,ô Úðús$   ‚A$C Á(C Â	:C ÃC ÃC6Ã5C6c               ó    € V ^8„  d   QhRRRR/# )r   ry  r=  r   rÇ   r   )r   s   "r   r   r   u  s   € ÷ ñ °$ð Ð;Mñ r   c                ó6  € \        V4      pVf   R # Vw   r4\        \        VR,          4      4      pVf   R # Vw  rgp\        P                  ! W\        V4      R7      P                  V4      p	\        P                  ! W—R7      p
Ve   V
P                  V4      p
V
# )Nr(   )r(   Úcountr'   )
r„  rw  rF   r1   Ú
frombufferrm  ÚreshaperO   rQ   Úview)Úbufry  ÚboundsrK  r^   r‚  rƒ  Ú	mlx_dtypeÚ
bitcast_torÃ   r•   s   &&         r   Ú!_mlx_array_from_safetensors_bytesr�  u  s‰   € Ü'¨Ó.€FØ‚~ÙØ�K€A€qÜ(¬¨U°7­^Ó)<Ó=€JØÒÙØ&0Ñ#€H˜Ü
�-Š-˜´6¸%³=Ô
A×
IÑ
IÈ%Ó
P€CÜ
�(Š(�3Ô
(€CØÒØ�h‰h�zÓ"ˆØ€Jr   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	r   rT  r   Ú
data_startr!   ry  r=  r   rÇ   r   )r   s   "r   r   r   …  s2   € ÷ Eñ EØ
ðEØðEØ(,ðEàñEr   c                ó~  € \        V4      pVf   R # Vw  rEp \        V R4      ;_uu_ 4       pVP                  W,           4       VP                  WT,
          4      p\	        V4      WT,
          8w  d    R R R 4       R #  R R R 4       \        \        X4      T4      #   + '       g   i     L%; i  \
         d     R # i ; i)NrV  )r„  rX  ÚseekrY  rH   r]  r�  Ú
memoryview)	rT  r‘  ry  rŒ  rè   rï   rK  r_  Úraws	   &&&      r   Ú_read_safetensors_tensorr–  …  s¤   € ô (¨Ó.€FØ‚~ÙØ�M€E�ðÜ�$˜×Ô Ø�F‰F�:Õ%Ô&Ø—&‘&˜�Ó%ˆCÜ�3‹x˜3�;Ô&Ø÷	 Ñð '÷ ô -¬Z¸«_¸eÓDÐD÷ ×ûô
 ô Úðús5   ˜B- ¬ABÁ2B- Á=B- ÂB*	Â%B- Â*B- Â-B<Â;B<c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R
/# )r   rT  r   r‘  r!   ry  r=  Úaxis0_startÚ	axis0_endr   zOptional[Tuple[bytes, dict]]r   )r   s   "r   r   r   —  sD   € ÷ $ñ $Ø
ð$àð$ð ð$ð ð	$ð
 ð$ð "ñ$r   c                óþ  € \        V4      pVf   R # Vw  rgpV'       g   R # \        V4      p\        V4      pV^ 8  g   WC8  g   WH^ ,          8”  d   R # \        \        VR,          4      4      p	V	f   R # V	w  p
 p\	        VR,          4      V
P
                  ,          pWcV,          ,           pWdV,          ,           p \        V R4      ;_uu_ 4       pVP                  W,           4       VP                  WÜ,
          4      p\        V4      WÜ,
          8w  d    R R R 4       R #  R R R 4       \        T4      pYC,
          .TR,          OTR&   ^ YÜ,
          .TR&   XT3#   + '       g   i     LA; i  \         d     R # i ; i)Nr(   ºr=   NNrV  r^   r{  )r„  r!   rw  rF   rm  r  rX  r“  rY  rH   r]  r=  )rT  r‘  ry  r˜  r™  rŒ  rè   rK  r^   r‚  rƒ  Ú	row_bytesÚ
byte_startÚbyte_endr_  r•  Úsliced_entrys   &&&&&            r   Ú#_read_safetensors_axis0_slice_bytesr   —  s_  € ô (¨Ó.€FØ‚~ÙØ�O€EˆeßÙÜ�kÓ"€KÜ�I“€IØ�Q„˜)Ô1°YÀqÅÔ5IÙÜ(¬¨U°7­^Ó)<Ó=€JØÒÙØ�N€Hˆa�Ü�u˜R•yÓ! H×$5Ñ$5Õ5€IØ yÕ0Õ0€JØ 9Õ,Õ,€HðÜ�$˜×Ô Ø�F‰F�:Õ*Ô+Ø—&‘&˜Õ.Ó/ˆCÜ�3‹x˜8Õ0Ô0Ø÷	 Ñð 1÷ ô ˜“;€LØ&Õ4ÐA°u¸RµyÐA€L�ÑØ$% xÕ'<Ð#=€L�Ñ Ø�ÐÐ÷ ×ûô
 ô Úðús7   Â<E- ÃAEÄE- Ä!E- ÅE*	Å%E- Å*E- Å-E<Å;E<c               ó0   € V ^8„  d   QhRRRRRRRRRRR	R
/# )r   rT  r   r‘  r!   ry  r=  r˜  r™  r   rÇ   r   )r   s   "r   r   r   ¾  sL   € ÷ Lñ LØ
ðLàðLð ðLð ð	Lð
 ðLð ñLr   c                óZ   € \        WW#V4      pVf   R # Vw  rg\        \        V4      V4      # rz   )r   r�  r”  )rT  r‘  ry  r˜  r™  Úslicedr•  rŸ  s   &&&&&   r   Ú_read_safetensors_axis0_slicer¤  ¾  s8   € ô 1Ø˜%¨ió€Fð ‚~ÙØÑ€CÜ,¬Z¸«_¸lÓKÐKr   c               ó   € V ^8„  d   QhRR/# )r   r   rW   r   )r   s   "r   r   r   Î  s   € ÷ "ñ "˜ñ "r   c                 ó$  €  ^ RI p \        V P                  4       P                  4      #   \         d     Mi ; i ^ RIpTP                  R.RR7      P                  R4      pRp^ p^ pTP                  4        Fõ  pRT9   d>   \        P                  ! RT4      pT'       d   \        TP                  ^4      4      pKE  KG  TP                  R	4      '       dA   \        TP                  R
4      ^,          P                  4       P                  R4      4      pKž  TP                  R4      '       g   K·  \        TP                  R
4      ^,          P                  4       P                  R4      4      pK÷  	  YE,           T,          #   \         d     R# i ; i)uý   Best-effort reading of currently-available system RAM. Returns
``None`` when we can't tell, in which case the caller should treat the
answer as "don't know â€” proceed".

Uses ``psutil`` when available; falls back to ``vm_stat`` on macOS.
Never raises.
NÚvm_statg      ð?)ÚtimeoutrE   i @  zpage size ofzpage size of (\d+) byteszPages free:Ú:Ú.zPages inactive:)Úpsutilr!   Úvirtual_memoryÚ	availablerÍ   Ú
subprocessÚcheck_outputÚdecodeÚ
splitlinesrN  ÚsearchÚgroupÚ
startswithÚsplitrP  Úrstrip)r«  r®  r•   Ú	page_sizeÚ
free_pagesÚinactive_pagesÚlineÚms           r   Ú_free_ram_bytesr¼  Î  sI  € ðÛä�6×(Ñ(Ó*×4Ñ4Ó5Ð5øÜô ÙðúðÛà×%Ñ% y k¸3Ð%Ó?×FÑFÀwÓOˆØˆ	Øˆ
ØˆØ—N‘NÖ$ˆDØ Ô%ä—I’IÐ9¸4Ó@�ßÜ # A§G¡G¨A£J£’Iñ à—‘ ×/Ò/Ü  §¡¨C£°Õ!3×!9Ñ!9Ó!;×!BÑ!BÀ3Ó!GÓH’
Ø—‘Ð!2×3Ô3Ü!$ T§Z¡Z°£_°QÕ%7×%=Ñ%=Ó%?×%FÑ%FÀsÓ%KÓ!L’ñ %ð Õ+¨yÕ8Ð8øÜô Úðús2   ‚&) ©7¶7»A%F  Â!4F  ÃAF  Ä/AF  Æ FÆFc                  ó„  € ] tR tRtRtRtRtRt]! ]4      ^ ,           t	]! ]4      ^ ,           t
RtRzR R	 llt]R
 R l4       t]R R l4       t]R R l4       tR R ltR R ltR R ltR R lt]R R l4       t]R R l4       tR R ltR R lt]R  R! l4       t]R" R# l4       t]R$ R% l4       t]R& R' l4       tR( R) ltR* R+ lt R, R- lt!R. R/ lt"R0 R1 lt#R2^ R3RR4^ /R5 R6 llt$R7R8R9R/R: R; llt%R< R= lt&R> R? lt'R7R8/R@ RA llt(R7R8/RB RC llt)RD RE lt*RF RG lt+RH RI lt,RJRK/RL RM llt-RN RO lt.RP RQ lt/RR RS lt0RJRT/RU RV llt1RW RX lt2RY RZ lt3R[ R\ lt4R] R^ lt5R_ R` lt6Ra Rb lt7Rc Rd lt8Re Rf lt9]Rg Rh l4       t:]Ri Rj l4       t;Rk Rl lt<Rm Rn lt=Ro Rp lt>Rq Rr lt?Rs Rt lt@Ru Rv ltARw Rx ltBRytCR# ){ÚDiskBlockStoreió  uû  Persistent SSD-backed cold tier for APC blocks.

Layout: one or more segment shards per ``store_kv_blocks`` call. Each
shard is a single ``.safetensors`` file under ``<root>/<namespace>/``
containing a bounded run of that call's blocks. Co-locating
chain-contiguous blocks in segment files
makes restoration O(few sequential reads) rather than O(num_blocks
random reads) â€” the difference is roughly two orders of magnitude on
Apple NVMe for ~200-block prefixes.

Shard naming: ``shard_<32-hex>.safetensors``. The 32-hex stem is the
SHA-256 prefix of the contained block hashes; identical stores
naturally dedup. Inside the file:

  * tensor ``k{layer}`` / ``v{layer}`` per layer in runtime KV layout
  * metadata: ``block_hashes`` (csv), ``num_layers``, ``block_size``,
    and ``b{idx}_meta`` (JSON) per block carrying parent_hash,
    extra_hash, and the token tuple

The in-memory index ``hash â†’ (shard_path, block_idx)`` is rebuilt on
init by scanning shards (cheap, just reads safetensors headers).
Shard mmap'd handles are kept in a small LRU cache so siblings within
a single restore don't re-mmap the same file.

Writes go through a single background worker so the prefill hot path
isn't blocked. Eviction is at segment-shard granularity (drop one
bounded file).
z.safetensorsÚshard_Úexact_gÍÌÌÌÌÌì?Nc               ó(   € V ^8„  d   QhRRRRRRRR/# )	r   Úrootr   Ú	namespacerF   Únum_workersr!   Ú	max_bytesrW   r   )r   s   "r   r   ÚDiskBlockStore.__annotate__  s8   € ÷ Sñ SàðSð ðSð ð	Sð
 !ñSr   c           	     	ó®  € \        V4      \        V4      ,          V n        V P                  P                  R R R7       W@n        ^ V n        \        P                  ! RR7      V n        \        P                  ! 4       V n        / V n        \        P                  ! 4       V n        / V n        / V n        \        P"                  ! 4       V n        \&        P(                  P+                  RR4      P-                  4       V n        V P.                  R9  d(   \0        P3                  RV P.                  4       RV n        \5        4       V n        \        P                  ! 4       V n        \;        \&        P(                  P+                  R^4      4      V n        \;        \?        \&        P(                  P+                  RR	4      4      R,          4      V n         \C        ^\;        \&        P(                  P+                  R
R4      4      4      V n"        \C        ^ \;        \&        P(                  P+                  RR4      4      4      V n#        \5        4       V n$        \        P                  ! 4       V n%        \;        \&        P(                  P+                  R^4      4      V n&        V PO                  4       pV'       d"   \0        PQ                  RVV P                  4       V PS                  4       V n*        \W        \C        ^V4      4       Uu. uF)  p\        PX                  ! V PZ                  R RV 2R7      NK+  	  upV n.        V P\                   F  pVP_                  4        K  	  R# u upi )T©ÚparentsÚexist_oki   )ÚmaxsizeÚAPC_DISK_READ_MODEÚdirectz5APC disk: unknown APC_DISK_READ_MODE=%r; using directÚAPC_DISK_HEADER_CACHEÚAPC_DISK_DIRECT_MAX_OVERREAD_MBÚ8ÚAPC_DISK_SHARD_MAX_BLOCKSÚ256ÚAPC_DISK_RESTORE_CLEAR_EVERYÚ1ÚAPC_DISK_MMAP_CACHEz5APC disk: removed %d orphaned partial file(s) from %sz	apc-disk-)ÚtargetÚdaemonrI  N)rÍ  Úmmapi   )0r   rR  ÚdirÚmkdirrÅ  r4  ÚqueueÚQueueÚ_qÚ	threadingÚEventÚ_stopÚ
_in_flightÚLockÚ_in_flight_lockÚ_indexÚ_exact_indexÚRLockÚ_index_lockr   r   r   r   Ú
_read_modeÚloggerÚwarningr   Ú_header_cacheÚ_header_cache_lockr!   Ú_header_cache_maxr  Ú_direct_max_overread_bytesrò   Ú_shard_max_blocksÚ_restore_clear_everyÚ_mmap_cacheÚ_mmap_cache_lockÚ_mmap_cache_maxÚ_cleanup_partialsÚinfoÚ_rebuild_indexÚ_disk_bytesÚrangeÚThreadÚ_writer_loopÚ_workersrè   )rC  rÂ  rÃ  rÄ  rÅ  Ú	n_orphansÚirÂ   s   &&&&&   r   Ú__init__ÚDiskBlockStore.__init__  su  € ô ˜“:¤°	Ó :Õ:ˆŒØ�‰�‰˜t¨dˆÔ3Ø"ŒØˆŒÜ$Ÿ{š{°4Ô8ˆŒÜ—_’_Ó&ˆŒ
ð 79ˆŒÜ(Ÿ~š~Ó/ˆÔà35ˆŒà-/ˆÔÜ$Ÿ?š?Ó,ˆÔô
 Ÿ*™*Ÿ.™.Ð)=¸xÓH×NÑNÓPˆŒØ�?‰?Ð"4Ô4Ü�N‰NØGØ—‘ôð 'ˆDŒOô KVË-ˆÔÜ"+§.¢.Ó"2ˆÔÜ!$¤R§Z¡Z§^¡^Ð4KÈQÓ%OÓ!PˆÔÜ*-Ü”"—*‘*—.‘.Ð!BÀCÓHÓIÈWÕUó+
ˆÔ'ô "%ØŒs”2—:‘:—>‘>Ð"=¸uÓEÓFó"
ˆÔô %(ØŒs”2—:‘:—>‘>Ð"@À#ÓFÓGó%
ˆÔ!ô DOÃ=ˆÔÜ )§¢Ó 0ˆÔÜ"¤2§:¡:§>¡>Ð2GÈÓ#KÓLˆÔà×*Ñ*Ó,ˆ	ßÜ�K‰KØGØØ—‘ôð  ×.Ñ.Ó0ˆÔô œ3˜q +Ó.Ô/ó	
ñ 0�ô ×ÒØ×(Ñ(°¸iÈÀs¸O÷ñ 0ñ	
ˆŒð —”ˆAØ�G‰GŽIó ùò
s   Ë6/Mc               ó    € V ^8„  d   QhRRRR/# ©r   rT  r   r   r   r   )r   s   "r   r   rÆ  q  ó   € ÷ :ñ : tð :°ñ :r   c                	ó  € VP                   pVP                  V P                  4      '       g   R # V\        V P                  4      R p\        V4      ^ 8w  d   R # \        ;QJ d    R V 4       F  '       d   K   R # 	  R# ! R V 4       4      # )FNc              3  ó*   "  € T F	  qR 9   x € K  	  R# 5i©Ú0123456789abcdefNr   ©r|   rg   s   & r   r~   Ú5DiskBlockStore._is_canonical_shard.<locals>.<genexpr>x  ó   é € Ð9±D¨qÐ*Ö*³Dùrñ   T)Ústemr´  ÚSHARD_PREFIXrH   r¶   ©ÚclsrT  r
  Úrests   &&  r   Ú_is_canonical_shardÚ"DiskBlockStore._is_canonical_shardp  ór   € à�y‰yˆØ�‰˜s×/Ñ/×0Ò0ÙØ”C˜×(Ñ(Ó)Ð+Ð,ˆÜˆt‹9˜Œ?Ùß‹sÑ9±DÓ9�sŒsÐ9ŠsÐ9ˆsÑ9±DÓ9Ó9Ð9r   c               ó    € V ^8„  d   QhRRRR/# r  r   )r   s   "r   r   rÆ  {  r  r   c                	ó  € VP                   pVP                  V P                  4      '       g   R # V\        V P                  4      R p\        V4      ^ 8w  d   R # \        ;QJ d    R V 4       F  '       d   K   R # 	  R# ! R V 4       4      # )FNc              3  ó*   "  € T F	  qR 9   x € K  	  R# 5ir  r   r  s   & r   r~   Ú5DiskBlockStore._is_canonical_exact.<locals>.<genexpr>‚  r	  rñ   T)r
  r´  ÚEXACT_PREFIXrH   r¶   r  s   &&  r   Ú_is_canonical_exactÚ"DiskBlockStore._is_canonical_exactz  r  r   c               ó    € V ^8„  d   QhRRRR/# r  r   )r   s   "r   r   rÆ  …  s   € ÷ Nñ N¨Dð N°Tñ Nr   c                	óV   € V P                  V4      ;'       g    V P                  V4      # rz   )r  r  )r  rT  s   &&r   Ú_is_canonical_store_fileÚ'DiskBlockStore._is_canonical_store_file„  s&   € à×&Ñ& tÓ,×MÐM°×0GÑ0GÈÓ0MÐMr   c               ó    € V ^8„  d   QhRRRR/# )r   Úshard_idrF   r   r   r   )r   s   "r   r   rÆ  ˆ  s   € ÷ 5ñ 5 Cð 5¨Dñ 5r   c                	óD   € V P                   V V P                   2,          # rz   )rÙ  ÚSUFFIX)rC  r  s   &&r   Ú_shard_pathÚDiskBlockStore._shard_pathˆ  s   € Ø�x‰x˜X˜J t§{¡{ mÐ4Õ4Ð4r   c               ó    € V ^8„  d   QhRRRR/# )r   rT  r   r   ÚNoner   )r   s   "r   r   rÆ  ‹  s   € ÷ /ñ /¨ð /°$ñ /r   c           	     	óè  € V P                   ;_uu_ 4        V P                  P                  4        UUUu. uF  w  pw  r4W18X  g   K  VNK  	  ppppV F  pV P                  V K  	  V P                  P                  4        UUu. uF  w  r#W18X  g   K  VNK  	  pppV F  pV P                  V K  	  R R R 4       V P                  ;_uu_ 4        V P
                  P                  VR 4       R R R 4       V P                  ;_uu_ 4        V P                  P                  VR 4       R R R 4       R # u upppi u uppi   + '       g   i     L‘; i  + '       g   i     Lk; i  + '       g   i     R # ; irz   )	rç  rä  Úitemsrå  rò  rñ  r\  rì  rë  )rC  rT  r8   ÚsprK  ÚstaleÚstale_exacts   &&     r   Ú_drop_index_for_pathÚ#DiskBlockStore._drop_index_for_path‹  s  € Ø××ÓØ)-¯©×):Ñ):Ô)<ÕKÑ)<™:˜1™g˜rÀÁ
—Q�QÑ)<ˆEÒKÛ�Ø—K‘K ’Nñ à*.×*;Ñ*;×*AÑ*AÔ*CÔRÑ*C¡ ÀrÁzŸ1˜1Ñ*CˆKÑRÛ �Ø×%Ñ% aÒ(ñ !÷ ð ×"×"Ó"Ø×Ñ× Ñ   tÔ,÷ #à×$×$Ó$Ø×Ñ×"Ñ" 4¨Ô.÷ %Ñ$ùô Lùó S÷	 ×ú÷ #×"úç$×$Ð$úsL   •D:´D-ÁD-Á7D:Â
D4
ÂD4
ÂD:ÃEÄE Ä-D:Ä:E
	ÅE	Å E1	c               ó   € V ^8„  d   QhRR/# ©r   r   r$  r   )r   s   "r   r   rÆ  ˜  s   € ÷ ñ ¸ñ r   c                	ó  € V P                   ;_uu_ 4        V P                  P                  4        V P                  P                  4        R R R 4       V P                  ;_uu_ 4        V P
                  P                  4        R R R 4       V P                  ;_uu_ 4        V P                  P                  4        R R R 4       ^ V n        R #   + '       g   i     L‡; i  + '       g   i     Lc; i  + '       g   i     L?; irz   )	rç  rä  Úclearrå  rò  rñ  rì  rë  r÷  ©rC  s   &r   Ú'_clear_disk_state_after_external_deleteÚ6DiskBlockStore._clear_disk_state_after_external_delete˜  sž   € Ø××ÓØ�K‰K×ÑÔØ×Ñ×#Ñ#Ô%÷ ð ×"×"Ó"Ø×Ñ×"Ñ"Ô$÷ #à×$×$Ó$Ø×Ñ×$Ñ$Ô&÷ %àˆÖ÷ ×ú÷ #×"úç$×$ús#   •5C	Á&CÂC/Ã	C	ÃC,	Ã/C?	c               ó   € V ^8„  d   QhRR/# r   r   )r   s   "r   r   rÆ  ¢  s   € ÷ ñ ˜Tñ r   c                	ó  € V P                   P                  4       p V P                   P                  R R R7       T'       g   T P                  4        R #   \         d-   p\        P                  RT P                   T4        Rp?R# Rp?ii ; i)TrÈ  z1APC disk: failed to create cache directory %s: %sNF)rÙ  ÚexistsrÚ  r]  ré  rê  r1  )rC  ÚexistedÚes   &  r   Ú_ensure_dirÚDiskBlockStore._ensure_dir¢  sm   € Ø—(‘(—/‘/Ó#ˆð	Ø�H‰H�N‰N 4°$ˆNÔ7÷ Ø×8Ñ8Ô:Ùøô ô 	Ü�N‰NØCÀTÇXÁXÈqôõ ûð		ús   œA ÁB
Á!BÂB
c               ó    € V ^8„  d   QhRRRR/# )r   Úblock_hashesr½   r   rF   r   )r   s   "r   r   rÆ  °  s   € ÷ Dñ D Mð D°cñ Dr   c                	óò   € \         P                  ! 4       pV  F4  pVP                  \        VR,          4      P	                  ^R4      4       K6  	  \
        P                   VP                  4       R,           2# ©r=   r&   :Né    Nr-   )r.   r   r/   r!   r0   r¾  r  Ú	hexdigest)r;  r8   Úbhs   &  r   Ú_shard_id_forÚDiskBlockStore._shard_id_for¯  s]   € ä�NŠNÓˆÛˆBØ�H‰H”S˜˜}Õ-Ó.×7Ñ7¸¸8ÓDÖEñ ä ×-Ñ-Ð.¨q¯{©{«}¸SÕ/AÐ.BÐCÐCr   c               ó    € V ^8„  d   QhRRRR/# )r   r-  r!   r   rF   r   )r   s   "r   r   rÆ  ·  s   € ÷ Dñ D #ð D¨#ñ Dr   c                	óà   € \         P                  ! 4       pVP                  \        V R,          4      P	                  ^R4      4       \
        P                   VP                  4       R,           2# r=  )r.   r   r/   r!   r0   r¾  r  r?  )r-  r8   s   & r   Ú_exact_id_forÚDiskBlockStore._exact_id_for¶  sS   € ä�NŠNÓˆØ	�‰”�Z =Õ1Ó2×;Ñ;¸A¸xÓHÔIÜ ×-Ñ-Ð.¨q¯{©{«}¸SÕ/AÐ.BÐCÐCr   c               ó   € V ^8„  d   QhRR/# ©r   r   r!   r   )r   s   "r   r   rÆ  ¼  s   € ÷ ñ  3ñ r   c                óˆ  € V P                  4       '       g   ^ # ^ pV P                  P                  RV P                   24       FL  pVP	                  4       '       d   V P                  V4      '       d   K2   VP                  4        V^,          pKN  	  V#   \         d"   p\        P                  RY#4        Rp?Ky  Rp?ii ; i)z¬Delete anything in the dir that isn't a canonical shard (left
over from a crashed write, or files from an older block-per-file
layout that this class no longer recognises).Ú*z)APC disk: failed to remove partial %s: %sN)
r8  rÙ  Úglobr   Úis_filer  Úunlinkr]  ré  rê  )rC  ÚnÚpr7  s   &   r   rô  Ú DiskBlockStore._cleanup_partials¼  s    € ð ×Ñ×!Ò!ÙØˆØ—‘—‘  4§;¡; -Ð0Ö1ˆAØ—9‘9—;’; $×"?Ñ"?À×"BÒ"BÙðRØ—‘”
Ø�Q•’ñ 2ð ˆøô ô RÜ—‘ÐJÈA×QÒQûðRús   Á6BÂCÂ B<Â<Cc               ó   € V ^8„  d   QhRR/# rH  r   )r   s   "r   r   rÆ  Í  s   € ÷ -ñ - ñ -r   c           	     ó`  € V P                  4       '       g   ^ # ^ pV P                  ;_uu_ 4        V P                  P                  4        V P                  P                  4        V P
                  P                  RV P                   24       EF,  pV P                  V4      '       g   K   WP                  4       P                  ,          p\        T4      pTf*   \        P                  RT4        TP                  4        Kv  T P!                  T4      '       d-    \#        TP%                  RR4      4      pY P                  T&   K¹  TP%                  RR4      pT'       g   KÕ   TP+                  R4       Uu. uF  qf'       g   K  \#        T4      NK  	  pp\-        T4       F  w  r‰Y(3T P                  T	&   K  	  EK/  	  RRR4       V#   \         d     EKJ  i ; i  \         d     Lçi ; i  \&        \(        3 d     EKt  i ; iu upi   \(         d     EKŒ  i ; i  + '       g   i     T# ; i)uG  Scan shards, populate ``_index``, return total bytes on disk.

Uses a header-only safetensors read so each shard scan touches only
the file's leading few KB â€” no MLX array construction, no mmap of
the tensor payload. On a disk with hundreds of cached shards this
keeps server-startup overhead and Python heap growth minimal.
rJ  Nz'APC disk: shard %s unreadable, droppingr-  Ú r;  Ú,)r8  rç  rä  r/  rå  rÙ  rK  r   r  ÚstatÚst_sizer]  rh  ré  rê  rM  r  r!   r   r�  r^  rµ  rå   )
rC  ÚtotalrO  rd  r-  Ú
hashes_csvrL   r;  ré   r@  s
   &         r   rö  ÚDiskBlockStore._rebuild_indexÍ  sä  € ð ×Ñ×!Ò!ÙØˆØ××ÓØ�K‰K×ÑÔØ×Ñ×#Ñ#Ô%Ø—X‘X—]‘] Q t§{¡{ mÐ#4×5�Ø×4Ñ4°Q×7Ò7ÙðØŸV™V›X×-Ñ-Õ-�Eô 6°aÓ8�ØÒ#Ü—N‘NÐ#LÈaÔPðØŸ™œ
ñ Ø×+Ñ+¨A×.Ò.ð!Ü%(¨¯©°lÀBÓ)GÓ%H˜
ð 56×%Ñ% jÑ1ÙØ%Ÿ\™\¨.¸"Ó=�
ß!ÙðØ4>×4DÑ4DÀSÔ4IÓ#OÑ4I¨qËQ¤F¤C¨¦FÑ4I�LÐ#Oô  )¨Ö6‘G�CØ'( h�D—K‘K “Oô  7ñ; 6÷ ðD ˆøô5 ô Ûðûô #ô Ùðûô &¤zÐ2ô !Û ð!üò $PøÜ!ô Ûðú÷= ÖðD ˆúsº   ¯A:HÂ* GÃ
%HÃ0GÄ HÄG+Ä5)HÅHÅ"H	Å5	H
ÆH
ÆH	Æ)HÇG	ÇHÇG	ÇHÇG(	Ç%HÇ'G(	Ç(HÇ+H	Ç<HÈ H	ÈHÈH	È	H	ÈHÈH	ÈHÈH-	c               ó   € V ^8„  d   QhRR/# rH  r   )r   s   "r   r   rÆ  ý  s   € ÷  ñ  ˜Cñ  r   c                	ó   € V P                   # rz   )r÷  r0  s   &r   Ú
disk_bytesÚDiskBlockStore.disk_bytesü  s   € à×ÑÐr   c               ó   € V ^8„  d   QhRR/# rH  r   )r   s   "r   r   rÆ    s   € ÷ $ñ $ Cñ $r   c                	ó’   € V P                   ;_uu_ 4        \        V P                  4      uuR R R 4       #   + '       g   i     R # ; irz   )rç  rH   rä  r0  s   &r   Únum_blocks_indexedÚ!DiskBlockStore.num_blocks_indexed   s)   € à××ÓÜ�t—{‘{Ó#÷ ××Óúó	   •5µA	c               ó   € V ^8„  d   QhRR/# rH  r   )r   s   "r   r   rÆ    s   € ÷ *ñ * 3ñ *r   c                	ó’   € V P                   ;_uu_ 4        \        V P                  4      uuR R R 4       #   + '       g   i     R # ; irz   )rç  rH   rå  r0  s   &r   Únum_exact_indexedÚ DiskBlockStore.num_exact_indexed  s+   € à××ÓÜ�t×(Ñ(Ó)÷ ××Óúrb  c               ó   € V ^8„  d   QhRR/# r   r   )r   s   "r   r   rÆ    s   € ÷ +ñ + tñ +r   c                	ó    € V P                   R 8H  # )rÍ  )rè  r0  s   &r   Úload_returns_detachedÚ$DiskBlockStore.load_returns_detached
  s   € à�‰ (Ñ*Ð*r   c               ó   € V ^8„  d   QhRR/# rH  r   )r   s   "r   r   rÆ    s   € ÷ [ñ [˜cñ [r   c                óh  a a€ S P                  4       '       g   ^ # S P                  e   S P                  S P                  8:  d   ^ # \        S P                  S P                  ,          4      pS P
                  ;_uu_ 4        \        S P                  P                  4       4      pRRR4       S P                  ;_uu_ 4        X Uu0 uF/  q3S P                  9   g   K  S P                  V,          ^ ,          kK1  	  ppVP                  V 3R lV 4       4       RRR4       . pS P                  P                  RS P                   24       F­  pS P                  V4      '       d   VX9   d   K#   VP!                  4       p\%        T4      p^ p	\'        T4      p
T
e/   T
P)                  RT4      p \        T
P)                  RR4      4      p	TP/                  RTP0                  RTP2                  R	TRTRT	/4       K¯  	  / oV F8  pVR,          p\5        VR,          SP)                  WËR,          4      4      SV&   K:  	  VP7                  V3R
 lR7       ^ pV Fu  pS P                  V8:  d    Mc\        VR,          4      pVR	,          p VP9                  4        S ;P                  T,          un        T^,          pS P?                  T4       Kw  	  V'       dT   S ;P@                  V,          un         \:        PC                  RVS P                  R,          S P                  R,          4       V#   + '       g   i     EL�; iu upi   + '       g   i     EL5; i  \"         d     EK  i ; i  \*        \,        3 d    ^ p	 EL¹i ; i  \"         d#   p\:        P=                  RYo4        Rp?EKX  Rp?ii ; i)zÊEvict segment shards until under the low watermark.

Stores are ordered by last-used time; segments within the same store
are evicted tail-first so a partially-retained store still has a
useful prefix.
Nc              3  ón   <"  € T F*  qSP                   9   g   K  SP                   V,          x € K,  	  R # 5irz   )rå  )r|   r8   rC  s   & €r   r~   Ú.DiskBlockStore._maybe_evict.<locals>.<genexpr>"  s1   øé € ð #Ù.>¨Àt×GXÑGXÑBXÔ$�×!Ñ! !×$Ò$Ó.>ùs   ƒ5š5rJ  r(  r)  Ú0r  rÁ   rT  c                óZ   <€ SV R ,          ,          \        V R,          4      ) V R,          3# )r(  r)  r  rû   )rg   Ústore_last_useds   &€r   Ú<lambda>Ú-DiskBlockStore._maybe_evict.<locals>.<lambda>K  s+   ø€ Ø  *¥Õ.Ü�Q�Õ'Ó(Ð(Ø�+•ñr   )Úkeyz APC disk: failed to evict %s: %sz8APC disk: evicted %d shard(s); now %.1f MB / %.1f MB capg    €„.A)"r8  rÅ  r÷  r!   Ú_EVICT_LOW_WATERMARKrã  rÛ   rá  rT   rç  rä  r/   rÙ  rK  r   r  rU  r]  rF   rh  r   r�  r^  rŒ   Úst_mtimerV  rþ   ÚsortrM  ré  rê  r*  r4  rõ  )rC  rÖ  Úin_flight_hashesr8   Úin_flight_pathsÚ
candidatesrO  Ústr(  r)  rd  Ú	candidateÚsidÚevictedrÁ   r7  rq  s   f               @r   Ú_maybe_evictÚDiskBlockStore._maybe_evict  s:  ù€ ð ×Ñ×!Ò!ÙØ�>‰>Ò! T×%5Ñ%5¸¿¹Ô%GÙÜ�T—^‘^ d×&?Ñ&?Õ?Ó@ˆð ×!×!Ó!Ü" 4§?¡?×#7Ñ#7Ó#9Ó:Ð÷ "à××Óá+;óÙ+; aÀDÇKÁKÑ?OÔ!�—‘˜A•˜q×!Ð!Ñ+;ð ð ð ×"Ñ"ô #Ù.>ó#ô ÷	 ð ,.ˆ
Ø—‘—‘  4§;¡; -Ð0Ö1ˆAØ×0Ñ0°×3Ò3°q¸OÔ7KÙðØ—V‘V“X�ô ˜1“vˆHØˆMÜ1°!Ó4ˆHØÒ#Ø#Ÿ<™<¨
°HÓ=�ð&Ü$'¨¯©°_ÀcÓ(JÓ$K�Mð ×Ñð
   §¡Ø˜BŸJ™JØ˜AØ Ø# ]ð	öñ! 2ð8 -/ˆÛ#ˆIØ˜JÕ'ˆCÜ#&Ø˜+Õ&Ø×#Ñ# C°;Õ)?Ó@ó$ˆO˜CÓ ñ $ð 	�‰ôð 	ô 	
ð ˆÛ#ˆIØ×Ñ 6Ô)ÙÜ�y Õ(Ó)ˆDØ˜&Õ!ˆAðØ—‘”
ð ×Ò Õ$ÕØ�q�LˆGà×%Ñ% aÖ(ñ $÷ Ø�NŠN˜gÕ%�NÜ�K‰KØJØØ× Ñ  3Õ&Ø—‘ Õ$ô	ð ˆ÷[ "×!Ð!üò÷ ×Ðûô ô Ûðûô "¤:Ð.ô &Ø$%“Mð&ûôJ ô Ü—‘ÐAÀ1ÔHÞûðúsl   Â $L*Ã MÃL>ÃL>Ã:MÅ-MÆ,M*ÊNÌ*L;	Ì>MÍM	ÍM'Í&M'Í*NÎ NÎN1ÎN,Î,N1c               ó   € V ^8„  d   QhRR/# ©r   Ú
shard_pathr   r   )r   s   "r   r   rÆ  l  s   € ÷ ñ ¨Tñ r   c                ó   € V P                  4       '       g   R# VP                  4       '       g   V P                  V4       R# V P                  ;_uu_ 4        V P                  P                  V4      pVe(   V P                  P                  V4       VuuRRR4       #  RRR4       \        V4      pVf*   \        P                  RV4       V P                  V4       R# V P                  ;_uu_ 4        W0P                  V&   V P                  P                  V4       \        V P                  4      V P                  8”  d   V P                  P                  RR7       KB   RRR4       V#   + '       g   i     LÓ; i  + '       g   i     T# ; i)z:Return parsed safetensors header info for a shard, cached.Nz(APC disk shard header read failed for %sF©Úlast)r8  r5  r*  rì  rë  r   Úmove_to_endre  ré  rê  rH   rí  Úpopitem)rC  rƒ  ÚcachedÚparseds   &&  r   Ú_open_shard_headerÚ!DiskBlockStore._open_shard_headerl  s=  € à×Ñ×!Ò!ÙØ× Ñ ×"Ò"Ø×%Ñ% jÔ1ÙØ×$×$Ó$Ø×'Ñ'×+Ñ+¨JÓ7ˆFØÒ!Ø×"Ñ"×.Ñ.¨zÔ:Ø÷	 %Ò$à!÷ %ô
 *¨*Ó5ˆØŠ>Ü�N‰NÐEÀzÔRØ×%Ñ% jÔ1ÙØ×$×$Ó$Ø-3×Ñ˜zÑ*Ø×Ñ×*Ñ*¨:Ô6Ü�d×(Ñ(Ó)¨D×,BÑ,BÔBØ×"Ñ"×*Ñ*°Ð*Ö6ð C÷ %ð
 ˆ÷ %×$ú÷ %Ö$ð
 ˆús   Á<E)Ã2A,E<Å)E9	Å<F	c               ó   € V ^8„  d   QhRR/# r‚  r   )r   s   "r   r   rÆ  …  s   € ÷ ñ  dñ r   c                óø  € V P                  4       '       g   R# VP                  4       '       g   V P                  V4       R# V P                  ;_uu_ 4        V P                  P                  V4      pVe(   V P                  P                  V4       VuuRRR4       #  RRR4        \        P                  ! \        V4      RR7      w  r4 \        P                  ! TR4       \!        T4      \!        T4      3pT P                  ;_uu_ 4        Y`P                  T&   T P                  P                  T4       \#        T P                  4      T P$                  8”  d   T P                  P'                  RR7       KB   RRR4       T#   + '       g   i     Lí; i  \         d3   p\        P                  RY4       T P                  T4        Rp?R# Rp?ii ; i  \         d     ELi ; i  + '       g   i     T# ; i)z=Return (arrays_dict, file_metadata) for a shard, mmap-cached.NT)Úreturn_metadataz%APC disk shard load failed for %s: %sFr…  )r8  r5  r*  rò  rñ  r   r‡  rO   ÚloadrF   rÍ   ré  rê  r   Úutimer]  r=  rH   ró  rˆ  )rC  rƒ  r‰  Úarraysrd  r7  Úbundles   &&     r   Ú_open_shardÚDiskBlockStore._open_shard…  s’  € à×Ñ×!Ò!ÙØ× Ñ ×"Ò"Ø×%Ñ% jÔ1ÙØ×"×"Ó"Ø×%Ñ%×)Ñ)¨*Ó5ˆFØÒ!Ø× Ñ ×,Ñ,¨ZÔ8Ø÷	 #Ò"à!÷ #ð
	Ü!Ÿwšw¤s¨:£ÈÔMÑˆFð	Ü�HŠH�Z Ô&ô �v“,¤ X£Ð/ˆØ×"×"Ó"Ø+1×Ñ˜ZÑ(Ø×Ñ×(Ñ(¨Ô4Ü�d×&Ñ&Ó'¨$×*>Ñ*>Ô>Ø× Ñ ×(Ñ(¨eÐ(Ö4ð ?÷ #ð
 ˆ÷- #×"ûô ô 	Ü�N‰NÐBÀJÔRØ×%Ñ% jÔ1Ýûð	ûô ô 	Úð	ú÷ #Ö"ð
 ˆúsC   Á<FÂ'#F ÃG ÄA,G(ÆF	ÆGÆ!'GÇGÇG%Ç$G%Ç(G9	c               ó    € V ^8„  d   QhRRRR/# )r   r  r!   r   r   r   )r   s   "r   r   rÆ  ¥  s   € ÷ -ñ -˜cð - dñ -r   c                	ó„   € V P                   ;_uu_ 4        WP                  9   uuR R R 4       #   + '       g   i     R # ; irz   )rç  rä  )rC  r  s   &&r   ÚhasÚDiskBlockStore.has¥  s'   € Ø××ÓØ§¡Ñ,÷ ××Óúó   •.®?	c               ó    € V ^8„  d   QhRRRR/# )r   r-  r!   r   r   r   )r   s   "r   r   rÆ  ©  s   € ÷ 3ñ 3 Cð 3¨Dñ 3r   c                	ó„   € V P                   ;_uu_ 4        WP                  9   uuR R R 4       #   + '       g   i     R # ; irz   )rç  rå  )rC  r-  s   &&r   Ú	has_exactÚDiskBlockStore.has_exact©  s)   € Ø××ÓØ×!2Ñ!2Ñ2÷ ××Óúrš  r¾   r  Úmin_prefix_tokensc          
     ó,   € V ^8„  d   QhRRRRRRRRRR	/# )
r   r¼   r½   r¾   r!   r  rW   rŸ  r   zOptional[Tuple[int, int]]r   )r   s   "r   r   rÆ  ­  s<   € ÷ +ñ +à ð+ð ð	+ð
 )ð+ð ð+ð 
#ñ+r   c               	óÖ  € \         ;QJ d    . R  V 4       F  NK  	  5M! R  V 4       4      p\        V4      ^,
          pVe   V^ 8”  d   \        V\        V4      4      pWd8:  d   R# V P                  ;_uu_ 4        \        V P                  P                  4       4      pRRR4       RpX EF  w  ršV P                  V
4      pVf   K  Vw  rÍpVP                  R4      R8w  d   K:   \        VP                  RR4      4      p\         ;QJ d3    . R VP                  RR4      P                  R	4       4       F  NK  	  5M,! R VP                  RR4      P                  R	4       4       4      p\        T4      pYò8w  g   TT8:  g   TT8”  g   TRT T8w  d   Kë  Te   TT^,          8”  g   Kÿ  \        T	4      T3pEK  	  V#   + '       g   i     EL(; i  \        \        3 d     EK<  i ; i)
c              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   rû   ©r|   rÂ   s   & r   r~   Ú3DiskBlockStore.find_exact_prefix.<locals>.<genexpr>µ  ó   é € Ð6©I qœC ŸF˜F«Iùr´   NÚlayoutÚexact_cache_v1r¾   ro  c              3  óJ   "  € T F  q'       g   K  \        V4      x € K  	  R # 5irz   rû   r}  s   & r   r~   r¤  É  ó   é € ð &Ù$L˜qÓPQ”F”C˜—F�FÓ$Lùó   ‚	#�#r¼   rS  rT  )r“   rH   rþ   r!   rç  rÎ   rå  r&  r‹  r   rµ  r�  r^  )rC  r¼   r¾   r  rŸ  Útoken_tupler  ÚentriesÚbestr-  rT  rŠ  Ú_tensor_entriesrd  Ú_data_startÚstored_extraÚstored_tokensr¤   s   &&$$$             r   Úfind_exact_prefixÚ DiskBlockStore.find_exact_prefix­  s½  € ÷ ”eÑ6©IÓ6—e‘eÑ6©IÓ6Ó6ˆÜ�kÓ" QÕ&ˆØÒ(Ð->ÀÔ-BÜ˜'¤3Ð'8Ó#9Ó:ˆGØÔ'Ùà××ÓÜ˜4×,Ñ,×2Ñ2Ó4Ó5ˆG÷ ð +/ˆÜ 'ÑˆJØ×,Ñ,¨TÓ2ˆFØŠ~ÙØ5;Ñ2ˆO {Ø�|‰|˜HÓ%Ð)9Ô9ÙðÜ" 8§<¡<°¸cÓ#BÓC�ß %¤ñ &Ø$,§L¡L°¸bÓ$A×$GÑ$GÈÔ$Ló&§¡ñ &Ø$,§L¡L°¸bÓ$A×$GÑ$GÈÔ$Ló&ó !�ô
 ˜]Ó+ˆJàÔ*ØÐ!2Ô2Ø Ô'Ø˜{ 
Ð+¨}Ô<áØŠ|˜z¨D°­GÖ3Ü˜J›¨Ð4“ñ/ !(ð0 ˆ÷9 ×Ðûô  œzÐ*ô Ûðús   Á;$F>Ã&BGÆ>G	ÇG(Ç'G(Úwait_in_flight_msr  rV   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	r   r-  r!   r´  r  rV   rW   r   ú0Optional[Tuple[Tuple[int, ...], int, List[Any]]]r   )r   s   "r   r   rÆ  Ú  s2   € ÷ 
ñ 
àð
ð !ð	
ð
 +ð
ð 
:ñ
r   c               	óN  € V P                   ;_uu_ 4        V P                  P                  V4      pR R R 4       Xf    V^ 8”  d“   V P                  ;_uu_ 4        V P                  P                  V4      pR R R 4       XeW   VP                  VR,          4      '       d9   V P                   ;_uu_ 4        V P                  P                  V4      pR R R 4       Vf   R # V P                  WCR7      #   + '       g   i     LÅ; i  + '       g   i     L•; i  + '       g   i     LN; i)Nç     @�@©rV   )rç  rå  r   rã  rá  ÚwaitÚ_load_exact_cache_file)rC  r-  r´  rV   rT  Úevs   &&$$  r   Úload_exact_cacheÚDiskBlockStore.load_exact_cacheÚ  sÛ   € ð ××ÓØ×$Ñ$×(Ñ(¨Ó4ˆD÷ àŠ<Ø  1Ô$Ø×)×)Ó)ØŸ™×,Ñ,¨ZÓ8�B÷ *à’> b§g¡gÐ.?À&Õ.H×&IÒ&IØ×)×)Ó)Ø#×0Ñ0×4Ñ4°ZÓ@˜÷ *àŠ|ÙØ×*Ñ*Øð +ó 
ð 	
÷ ×ú÷ *×)ú÷ *×)ús#   •C.ÁDÂ2DÃ.C>	ÄD	ÄD$	c               ó$   € V ^8„  d   QhRRRRRR/# )r   rT  r   rV   rW   r   r¶  r   )r   s   "r   r   rÆ  ð  s(   € ÷ ,3ñ ,3àð,3ð +ð	,3ð
 
:ñ,3r   c               	ó  € V P                  V4      pVf   R # Vw  rEpVP                  R4      R8w  d   R #  \        ;QJ d3    . R VP                  RR4      P                  R4       4       F  NK  	  5M,! R VP                  RR4      P                  R4       4       4      p\	        VP                  RR4      4      p\	        VP                  R	R4      4      p	T	^ 8:  d   R # . p
. p\        T	4       F6  pT P                  TTTTR
T 2TTR7      pTf    R # T
P                  T4       K8  	  T'       d   \        P                  ! T4        \        P                  ! TR 4       YxT
3#   \
        \        3 d     R # i ; i  \         d     L)i ; i)Nr¦  r§  c              3  óJ   "  € T F  q'       g   K  \        V4      x € K  	  R # 5irz   rû   r}  s   & r   r~   Ú8DiskBlockStore._load_exact_cache_file.<locals>.<genexpr>ý  s   é € ð Ù H˜1ËA””�A—�Ó Hùrª  r¼   rS  rT  r¾   ro  Únum_entriesrg   rx   )r‹  r   r“   rµ  r!   r�  r^  rø  Ú_load_exact_cache_entryrŒ   rO   rž   r   r‘  r]  )rC  rT  rV   rŠ  Útensor_entriesrd  r‘  r¼   r¾   Ú	n_entriesrš   rh   rý  Úloadeds   &&$           r   r»  Ú%DiskBlockStore._load_exact_cache_fileð  sŒ  € ð ×(Ñ(¨Ó.ˆØŠ>ÙØ/5Ñ,ˆ *Ø�<‰<˜Ó!Ð%5Ô5Ùð	ßœñ Ø (§¡¨[¸"Ó =× CÑ CÀCÔ HóŸ™ñ Ø (§¡¨[¸"Ó =× CÑ CÀCÔ Hóó ˆIô ˜XŸ\™\¨,¸Ó<Ó=ˆJÜ˜HŸL™L¨¸Ó<Ó=ˆIð ˜Œ>Ùà"$ˆØ')ˆÜ�yÖ!ˆAØ×1Ñ1ØØØØØ�A�3�Ø$7Ø)ð 2ó ˆFð Š~ÚØ×Ñ Ö'ñ "÷ Ü�GŠG�LÔ!ð	Ü�HŠH�T˜4Ô ð  lÐ2Ð2øô5 œ:Ð&ô 	Úð	ûô0 ô 	Ùð	ús*   ¶
E" ÁBE" ÅE: Å"E7Å6E7Å:FÆFc               ó8   € V ^8„  d   QhRRRRRRRRRR	R
RRRRR/# )r   rT  r   rÅ  r=  rd  r‘  r!   ÚprefixrF   rV   rW   rh   ri   r   rj   r   )r   s   "r   r   rÆ    sd   € ÷ ]ñ ]àð]ð ð]ð ð	]ð
 ð]ð ð]ð +ð]ð %ð]ð 
ñ]r   c               	ó–  € ^ RI Hp VP                  V R24      p	V	R8X  EdD   VP                  V R2R4      R8X  d7   VP                  4       p
 \	        VP                  V R2R4      4      V
n        V
# VP                  V R24      pVP                  V R	24      pVe   Vf   R
# \        WV4      p\        WV4      pVe   Vf   R
#  \	        VP                  V R2\        VP                  ^,          4      4      4      p\	        VP                  V R2R4      4      p\        TTTTTR7      w  rÞTP                  4       p
YÚn        Yên        Yún        TP                  YÞ.4       T
# V	R8X  Ed!    \	        VP                  V R2R4      4      p\	        W5 R2,          4      p\	        VP                  V R2R4      4      p\	        VP                  V R2R4      4      pTP!                  TTR7      p
TT
n        TT
n        TP                  T R2R4      R8X  d   T
# TP                  T R24      pTP                  T R	24      pTe   Tf   R
# \        YT4      p\        YT4      pTe   Tf   R
# YÚn        Yên        TP                  YÞ.4       T
# V	R8X  Ed    \	        W5 R2,          4      p\	        VP                  V R2R4      4      p\	        VP                  V R2R4      4      pTP%                  TR7      p
TT
n        TT
n        TP                  T R2R4      R8X  d   T
# TP                  T R24      pTP                  T R	24      pTe   Tf   R
# \        YT4      p\        YT4      pTe   Tf   R
# YÚn        Yên        TP                  YÞ.4       T
# V	R8X  Edy    \	        VP                  V R2R4      4      pTP)                  TR7      p
. p\+        T4       F…  pTP                  T RT R2R4      R8X  d   TP-                  R
4       K3  TP                  T RT 24      pTf    R
# \        YT4      pTf    R
# TP-                  T4       TP-                  T4       K‡  	  TT
n        TP                  T R24      pTe=   \        YT4      T
n        T
P.                  f   R
# TP-                  T
P.                  4       TP                  T R24      pTe=   \        YT4      T
n        T
P0                  f   R
# TP-                  T
P0                  4       T
# V	R!9   d‹    \	        VP                  V R2R4      4      p. p\+        T4       F8  pT P3                  TTTTT RT 2TTR 7      pTf    R
# TP-                  T4       K:  	  T	R8X  d   TP4                  ! T!  # \7        T4      # R
#   \        \        3 d    ^ T
n         T
# i ; i  \        \        3 d     R
# i ; i  \        \        \        3 d     R
# i ; i  \        \        \        3 d     R
# i ; i  \        \        3 d     R
# i ; i  \        \        3 d     R
# i ; i)"r¯   rl   Ú_kindÚkvÚ_emptyro  rÔ  Ú_offsetÚ_kÚ_vNÚ_steprÒ  rp   Úrotating_kvÚ_keepÚ	_max_sizert   rs   Ú
chunked_kvÚ_chunk_sizeÚ_start_positionrv   r’  Ú_size)rÁ   Ú_sÚ_noneÚ_left_paddingÚ_lengthsÚ
cache_listÚ_erx   )rÞ  r“   )rƒ   rm   r   r…   r!   rU   r�  r^  r–  rF   r^   re   rT   r;   rˆ   r€  r‰   rt   rŠ   rw   r‹   rø  rŒ   r�   rŽ   rÄ  r�   r“   ) rC  rT  rÅ  rd  r‘  rÊ  rV   rh   r”   Úkindrg   Úk_entryÚv_entryr©   rª   r–   rX   rr   rq   rU   ré   ru   rw   rÁ   ÚstatesÚjry  r—   Úlp_entryÚlengths_entryrÇ  r}   s    &&&&&&$$                        r   rÄ  Ú&DiskBlockStore._load_exact_cache_entry  sz  € õ 	4à�|‰|˜v˜h eÐ,Ó-ˆØ�4�<Ø�|‰|˜v˜h fÐ-¨sÓ3°sÔ:Ø×$Ñ$Ó&�ð!Ü" 8§<¡<°6°(¸'Ð0BÀCÓ#HÓI�A”Hð �Ø$×(Ñ(¨F¨8°2¨Ó7ˆGØ$×(Ñ(¨F¨8°2¨Ó7ˆGØŠ '¢/ÙÜ(¨¸7ÓCˆAÜ(¨¸7ÓCˆAØŠy˜AšIÙðÜ˜(Ÿ,™,¨&¨°Ð'9¼3¸q¿w¹wÀq½z»?ÓKÓL�Ü˜8Ÿ<™<¨6¨(°%Ð(8¸%Ó@ÓA�ô (ØØØØ$7Øô‰DˆAð × Ñ Ó"ˆAØŒFØŒHØŒHØ×Ñ  Ô'ØˆHà�=Õ ðÜ˜8Ÿ<™<¨6¨(°%Ð(8¸#Ó>Ó?�Ü˜x¨(°)Ð(<Õ=Ó>�Ü˜XŸ\™\¨V¨H°GÐ*<¸cÓBÓC�Ü˜(Ÿ,™,¨&¨° ¸Ó<Ó=�ð ×(Ñ(°(ÀÐ(ÓFˆAØˆAŒHØˆAŒFØ�|‰|˜v˜h fÐ-¨sÓ3°sÔ:Ø�Ø$×(Ñ(¨F¨8°2¨Ó7ˆGØ$×(Ñ(¨F¨8°2¨Ó7ˆGØŠ '¢/ÙÜ(¨¸7ÓCˆAÜ(¨¸7ÓCˆAØŠy˜AšIÙØŒFØŒHØ×Ñ  Ô'ØˆHà�<ÕðÜ  ¨H°KÐ*@Õ!AÓB�
Ü˜XŸ\™\¨V¨H°GÐ*<¸cÓBÓC�Ü!$ X§\¡\°V°H¸OÐ2LÈcÓ%RÓ!S�ð ×'Ñ'°:Ð'Ó>ˆAØˆAŒHØ-ˆAÔØ�|‰|˜v˜h fÐ-¨sÓ3°sÔ:Ø�Ø$×(Ñ(¨F¨8°2¨Ó7ˆGØ$×(Ñ(¨F¨8°2¨Ó7ˆGØŠ '¢/ÙÜ(¨¸7ÓCˆAÜ(¨¸7ÓCˆAØŠy˜AšIÙØŒFØŒHØ×Ñ  Ô'ØˆHà�8ÕðÜ˜8Ÿ<™<¨6¨(°%Ð(8¸#Ó>Ó?�ð ×$Ñ$¨$Ð$Ó/ˆAØ/1ˆFÜ˜4–[�Ø—<‘< 6 (¨"¨Q¨C¨uÐ 5°sÓ;¸sÔBØ—M‘M $Ô'ÙØ&×*Ñ*¨f¨X°R¸°sÐ+;Ó<�Ø’=ÚÜ0°À5ÓI�Ø’=ÚØ—‘˜eÔ$Ø×#Ñ# EÖ*ñ !ð ˆAŒGØ%×)Ñ)¨V¨H°MÐ*BÓCˆHØÒ#Ü!9¸$ÈHÓ!U�”Ø—>‘>Ò)ÙØ×#Ñ# A§N¡NÔ3Ø*×.Ñ.°&°¸Ð/BÓCˆMØÒ(Ü4°TÀ}ÓU�”	Ø—9‘9Ò$ÙØ×#Ñ# A§I¡IÔ.ØˆHàÐ*Ô*ðÜ˜8Ÿ<™<¨6¨(°%Ð(8¸#Ó>Ó?�ð ˆFÜ˜4–[�Ø×4Ñ4ØØ"ØØØ�h˜b  Ð$Ø(;Ø!-ð 5ó �ð ’=ÚØ—‘˜eÖ$ñ !ð �|Ô#Ø×)Ò)¨6Ñ2Ð2Ü˜“=Ð áøôU "¤:Ð.ô !Ø �A•HØ�ð!ûô œzÐ*ô Úðûô, œi¬Ð4ô Úðûô2 œi¬Ð4ô Úðûô. œzÐ*ô ÚðûôD œzÐ*ô Úðúso   Á#V* ÃAW	 Å/A.W! ÊAW> Î X ÔX3 Ö*W×W×	W×W×!W;×:W;×>XØXØX0Ø/X0Ø3YÙYc               ó$   € V ^8„  d   QhRRRRRR/# )r   r  r!   r´  r  r   ú5Optional[Tuple[List[mx.array], List[mx.array], dict]]r   )r   s   "r   r   rÆ  ½  s$   € ÷ 8ñ 8Øð8Ø5:ð8à	>ñ8r   c               ó˜  € V P                   ;_uu_ 4        V P                  P                  V4      pRRR4       Xf    V^ 8”  d“   V P                  ;_uu_ 4        V P                  P                  V4      pRRR4       XeW   VP                  VR,          4      '       d9   V P                   ;_uu_ 4        V P                  P                  V4      pRRR4       Vf   R# Vw  rVV P                  R8X  d   V P                  WV4      # V P                  WV4      #   + '       g   i     Lê; i  + '       g   i     Lº; i  + '       g   i     Ls; i)zÜRead one block. Returns (keys, values, per-block metadata) or None.

Per-block metadata is decoded from the shard's ``b{idx}_meta`` JSON
entry and includes ``token_ids``, ``parent_hash``, ``extra_hash``,
``block_hash``.
Nr¸  rØ  )	rç  rä  r   rã  rá  rº  rè  Ú
_load_mmapÚ_load_direct)rC  r  r´  ry  r¼  rƒ  Ú	block_idxs   &&$    r   r�  ÚDiskBlockStore.load½  sì   € ð ××ÓØ—K‘K—O‘O JÓ/ˆE÷ àŠ=Ø  1Ô$Ø×)×)Ó)ØŸ™×,Ñ,¨ZÓ8�B÷ *à’> b§g¡gÐ.?À&Õ.H×&IÒ&IØ×)×)Ó)Ø $§¡§¡°
Ó ;˜÷ *àŠ}ÙØ %Ñˆ
Ø�?‰?˜fÔ$Ø—?‘? :Ó9Ð9Ø× Ñ  Ó7Ð7÷ ×ú÷ *×)ú÷ *×)ús#   •DÁD&Â2D9ÄD#	Ä&D6	Ä9E		c               ó$   € V ^8„  d   QhRRRRRR/# )r   r;  r½   r´  r  r   ú;List[Optional[Tuple[List[mx.array], List[mx.array], dict]]]r   )r   s   "r   r   rÆ  Ö  s$   € ÷ 1ñ 1Ø)ð1ØAFð1à	Dñ1r   c               óx  € V'       g   . # V P                   R8X  d"   V Uu. uF  q0P                  W2R7      NK  	  up# . pV Fé  pV P                  ;_uu_ 4        V P                  P	                  V4      pRRR4       Xfš   V^ 8”  d“   V P
                  ;_uu_ 4        V P                  P	                  V4      pRRR4       XeW   VP                  VR,          4      '       d9   V P                  ;_uu_ 4        V P                  P	                  V4      pRRR4       VP                  V4       Kë  	  R.\        V4      ,          p^ pV\        V4      8  d¢   WH,          pVf   V^,          pK'  V^ ,          p	V^,           p
V
\        V4      8  d*   WJ,          e    WJ,          ^ ,          V	8X  d   V
^,          p
K9  \        WŠ4       Uu. uF  q´V,          ^,          NK  	  ppV P                  Wœ4      pW×WŠ% T
pK±  V# u upi   + '       g   i     EL�; i  + '       g   i     ELa; i  + '       g   i     EL; iu upi )zôRead multiple blocks, preserving order.

In direct mode, consecutive requests from the same shard are coalesced
into larger byte-range reads. In mmap mode, fall back to one-at-a-time
loads so the old comparison path stays simple and unchanged.
rØ  )r´  Nr¸  )rè  r�  rç  rä  r   rã  rá  rº  rŒ   rH   rø  Ú_load_direct_many)rC  r;  r´  r8   r¬  ry  r¼  r•   rý  rƒ  rä  r©   Úblock_indicesrÇ  s   &&$           r   Ú	load_manyÚDiskBlockStore.load_manyÖ  sà  € ÷ ØˆIØ�?‰?˜fÔ$áKWóÙKWÀa—	‘	˜!�	ÖAÉ<ñð ð 57ˆÛˆAØ×!×!Ó!ØŸ™Ÿ™¨Ó*�÷ "àŠ}Ð!2°QÔ!6Ø×)×)Ó)ØŸ™×,Ñ,¨QÓ/�B÷ *à’> b§g¡gÐ.?À&Õ.H×&IÒ&IØ×)×)Ó)Ø $§¡§¡°Ó 2˜÷ *à�N‰N˜5Ö!ñ ð MQÈ6ÔTWØóU
õ L
ˆð ˆØ”#�g“,ÔØ•JˆEØŠ}Ø�Q•�ÙØ˜q�ˆJØ�A•ˆAà”C˜“LÔ Ø•JÒ*Ø•J˜q•M ZÔ/à�Q•’Ü49¸!´KÓ@±K¨q Q�Z¨Ÿ]˜]±KˆMÐ@Ø×+Ñ+¨JÓFˆFØ�ˆHØŠAØˆ
ùòK÷ "×!Ð!ú÷ *×)Ð)ú÷ *×)Ð)üò* As/    G6ÁG;ÂHÃ5H#ÇH7Ç;HÈH È#H4c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úfile_metadatar=  rí  r!   r   r   )r   s   "r   r   rÆ  	  s!   € ÷ ñ °Dð ÀSð ÈTñ r   c                	ó^  € VP                  R V R24      pV'       g   / #  \        P                  ! V4      p\        VP                  R4      \        4      '       d#   RP                  R VR,           4       4      VR&   RV9   d   \        VR,          4      VR&   V#   \         d    / u # i ; i)ÚbÚ_metar¼   rT  c              3  óJ   "  € T F  p\        \        V4      4      x € K  	  R # 5irz   ©rF   r!   r£  s   & r   r~   Ú8DiskBlockStore._decode_block_metadata.<locals>.<genexpr>  s   é € ð 3Ù)@ A”Cœ˜A›—K�KÓ)@ùó   ‚!#r¾   )r   rZ  r[  r„   rÎ   ÚjoinrF   rÍ   )rC  r÷  rí  Úblock_meta_strÚ
block_metas   &&&  r   Ú_decode_block_metadataÚ%DiskBlockStore._decode_block_metadata	  s«   € Ø&×*Ñ*¨Q¨y¨k¸Ð+?Ó@ˆßØˆIð	ÜŸš NÓ3ˆJô ˜*Ÿ.™.¨Ó5´t×<Ò<Ø*-¯(©(ñ 3Ø)3°KÖ)@ó3ó +�
˜;Ñ'ð ˜zÔ)Ü+.¨z¸,Õ/GÓ+H�
˜<Ñ(ØÐøÜô 	ØŠIð	ús   ¡A:B ÂB,Â+B,c               ó$   € V ^8„  d   QhRRRRRR/# ©r   rƒ  r   rí  r!   r   ré  r   )r   s   "r   r   rÆ    s)   € ÷ Sñ SØðSØ+.ðSà	>ñSr   c                	óÒ  € V P                  V4      pVf   R # Vw  rE \        VP                  RR4      4      p \        T4       Uu. uF  qtRT RT 2,          NK  	  pp\        T4       Uu. uF  qtRT RT 2,          NK  	  p	pY‰T P                  YR4      3#   \        \        3 d     R # i ; iu upi u upi   \         d"   p
\        P                  RY4        R p
?
R # R p
?
ii ; i)NÚ
num_layersro  rù  rÐ  rÑ  ú$APC disk shard %s missing tensor: %s)
r”  r!   r   r�  r^  rø  r€  ré  rê  r  )rC  rƒ  rí  r“  r’  r÷  r  ÚlrT   r;   r7  s   &&&        r   rë  ÚDiskBlockStore._load_mmap  sü   € ð ×!Ñ! *Ó-ˆØŠ>ÙØ &Ñˆð	Ü˜]×.Ñ.¨|¸SÓAÓBˆJð	Ü:?À
Ô:KÓLÑ:K°Q˜Q˜y˜k¨¨A¨3Ð/×0Ð0Ñ:KˆDÐLÜ<AÀ*Ô<MÓNÑ<M°q˜q  ¨2¨a¨SÐ1×2Ð2Ñ<MˆFÐNð ˜T×8Ñ8¸ÓRÐRÐRøô œ:Ð&ô 	Úð	üò MùÚNøÜô 	Ü�N‰NÐAÀ:ÔQÝûð	úsF   �B ¹B: ÁB0ÁB: Á,B5ÂB: ÂB-Â,B-Â0
B: Â:C&ÃC!Ã!C&c               ó$   € V ^8„  d   QhRRRRRR/# r  r   )r   s   "r   r   rÆ  .  s$   € ÷ -ñ -Øð-Ø+.ð-à	>ñ-r   c                	óL   € V P                  W.4      pV'       d
   V^ ,          # R# )r¯   N)rò  )rC  rƒ  rí  rÇ  s   &&& r   rì  ÚDiskBlockStore._load_direct.  s'   € ð ×'Ñ'¨
°KÓ@ˆß"ˆv�a�yÐ,¨Ð,r   Úpreserve_capacityFc               ó(   € V ^8„  d   QhRRRRRRRR/# )	r   rƒ  r   ró  r½   r  r   r   ú;Optional[Tuple[List[mx.array], List[mx.array], List[dict]]]r   )r   s   "r   r   rÆ  4  s9   € ÷ J&ñ J&àðJ&ð %ðJ&ð
  ðJ&ð 
EñJ&r   c          
     	ó‚  € V'       g   R # V^ ,          p\        V4      \        \        WD\        V4      ,           4      4      8w  d   R # V P                  V4      pVf   R # Vw  rgpVP	                  R4      p	V	R8X  d   V P                  WWxV4      # V	R9  d   R #  \        VP	                  RR4      4      p
\        VP	                  RR4      4      pT
^ 8:  g   T^ 8:  d   R # YK,          pT\        T4      T,          ,           p\        TP	                  RR4      P                  R	4       Uu. uF  qî'       g   K  TNK  	  up4      pT^ 8„  ;'       d#    T^ 8H  ;'       d    T\        T4      ,           T8¬  pT'       d   T	R8X  d   T'       d   R MTp. p. p\        T
4       F“  pTP	                  R
T 24      pTP	                  RT 24      pTe   Tf    R # \        YT4      p\        YT4      pTe   Tf    R # TP                  TRTT1R3,          4       TP                  TRTT1R3,          4       K•  	  T Uu. uF  pT P                  TT4      NK  	  pp \        P                  ! TR 4       \         P"                  ! TT,           4       TTT3#   \        \        3 d     R # i ; iu upi u upi   \         d     LQi ; i)Nr¦  Útoken_major_v2Úlayer_major_v2r  ro  r¿   r;  rS  rT  r©   rª   .rn   ©Úlayer_major_v1r  )rÎ   rø  rH   r‹  r   Ú_load_token_major_segmentr!   r�  r^  rµ  r–  rŒ   r  r   r‘  r]  rO   rž   )rC  rƒ  ró  r  Ú	start_idxrŠ  rÅ  r÷  r‘  r¦  r  r¿   Útoken_startÚ	token_endrL   Úshard_n_blocksÚrequested_to_shard_endÚ	slice_endrT   r;   r	  rá  râ  r©   rª   ré   rd  s   &&&$                       r   Ú_load_layer_major_segmentÚ(DiskBlockStore._load_layer_major_segment4  så  € ÷ ÙØ! !Õ$ˆ	Ü�Ó¤$Ü�)¬¨]Ó);Õ;Ó<ó#
ô 
ñ Ø×(Ñ(¨Ó4ˆØŠ>ÙØ4:Ñ1ˆ zØ×"Ñ" 8Ó,ˆØÐ%Ô%Ø×1Ñ1Ø¨MÀ}óð ð Ð=Ô=Ùð	Ü˜]×.Ñ.¨|¸SÓAÓBˆJÜ˜]×.Ñ.¨|¸SÓAÓBˆJð ˜Œ?˜j¨AœoÙàÕ,ˆØ¤# mÓ"4°zÕ"AÕAˆ	ÜØ%×)Ñ)¨.¸"Ó=×CÑCÀCÔHÓNÑH�1ËA�QˆQÑHÑNó
ˆð ˜QÑ÷ Að AØ˜Q‘÷Að AàœC Ó.Õ.°.Ñ@ð 	÷ "ØÐ.Ô.ß*ñ	 ð ð 	ð  "ˆØ!#ˆÜ�zÖ"ˆAØ$×(Ñ(¨1¨Q¨C¨Ó1ˆGØ$×(Ñ(¨1¨Q¨C¨Ó1ˆGØŠ '¢/ÚÜ(¨ÀÓIˆAÜ(¨ÀÓIˆAØŠy˜AšIÚØ�K‰K˜˜#˜{¨9Ð4°aÐ7Õ8Ô9Ø�M‰M˜!˜C ¨YÐ!6¸Ð9Õ:Ö;ñ #ñ HUó
ÙGTÀˆD×'Ñ'¨°sÖ;Á}ð 	ð 
ð	Ü�HŠH�Z Ô&ô 	�Š��v•ÔØ�V˜XÐ%Ð%øô] œ:Ð&ô 	Úð	üò Oùò<
øô
 ô 	Ùð	ús6   Â6J Ä(	J&Ä6J&È7J+ÉJ0 ÊJ#Ê"J#Ê0J>Ê=J>c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úsegmentsú Sequence[Tuple[Path, List[int]]]r  r   r   r  r   )r   s   "r   r   rÆ  €  s.   € ÷ D&ñ D&à2ðD&ð  ð	D&ð
 
EñD&r   c               ó(  a a%€ V'       g   R# . p. pRpRp\        V4      ^,
          p\        V4       EFø  w  pw  ršV
'       g    R# V
^ ,          p\        V
4      \        \        W»\        V
4      ,           4      4      8w  d    R# S P	                  V	4      pVf    R# Vw  po%pS%P                  R4      pVR9  d    R#  \        S%P                  RR4      4      p\        S%P                  RR4      4      pT^ 8:  g   T^ 8:  d    R# Tf   TpTpMTT8w  g   TT8w  d    R# TT,          pT\        T
4      T,          ,           p\        S%P                  RR4      P                  R	4       Uu. uF  pT'       g   K  TNK  	  up4      pT^ 8„  ;'       d#    T^ 8H  ;'       d    T\        T
4      ,           T8¬  pT'       d   Y‡8X  d   TR8X  d   T'       d   RMTpTP                  T	TS%TTT\        T
4      34       TP                  T%T 3R
 lT
 4       4        \        P                  ! T	R4       EKû  	  Vf   R# . p. p\        V4       EF•  p. p. pV F�  w  p	ppppppVP                  RV 24      pVP                  RV 24      p Ve   V f     R# \        WžV4      p!\        WžV 4      p"V!e   V"f     R# VP                  V!RVV1R3,          4       VP                  V"RVV1R3,          4       KŸ  	  \        V4      ^8X  d
   V^ ,          M\         P"                  ! V^R7      p#\        V4      ^8X  d
   V^ ,          M\         P"                  ! V^R7      p$\         P$                  ! V#V$4       VP                  V#4       VP                  V$4       ???#?$S P&                  ^ 8”  g   EK^  V^,           S P&                  ,          ^ 8X  g   EK€  \         P(                  ! 4        EK˜  	  VVV3#   \        \        3 d      R# i ; iu upi   \         d     EKÞ  i ; i)a•  Load layer-major segments without holding all segment tensors.

The older restore path first read every segment's K/V for every layer,
then concatenated all layers at once. For long prefixes this doubled
peak MLX memory: segment tensors plus final KVCache tensors. This
routine reads all segments for one layer, emits that layer's final K/V,
clears temporary allocator state, and moves to the next layer.
Nr¦  r  r  ro  r¿   r;  rS  rT  c              3  óH   <"  € T F  pSP                  SV4      x € K  	  R # 5irz   ©r  ©r|   ré   r÷  rC  s   & €€r   r~   ÚMDiskBlockStore._load_layer_major_prefix_segments_layerwise.<locals>.<genexpr>Ô  ó$   øé € ð ÙKXÀC�×+Ñ+¨M¸3×?Ð?Ë=ùó   ƒ"r©   rª   .rn   r\   r  )rH   rå   rÎ   rø  r‹  r   r!   r�  r^  rµ  rŒ   rˆ   r   r‘  r]  r–  rO   r_   rž   rð  Úclear_cache)&rC  r   r  Úsegment_infosrd  r  Úblock_size_refÚlast_segment_idxÚsegment_idxrƒ  ró  r  rŠ  rÅ  r‘  r¦  Úshard_layersr¿   r  r  rL   r  r  r  rT   r;   Ú	layer_idxÚk_partsÚv_partsÚ_file_metadataÚ_block_indicesrá  râ  r©   rª   Úk_outÚv_outr÷  s&   f&$                                  @r   Ú+_load_layer_major_prefix_segments_layerwiseÚ:DiskBlockStore._load_layer_major_prefix_segments_layerwise€  s  ù€ ÷ Ùð ð 	ð  "ˆØ$(ˆ
Ø(,ˆÜ˜x›=¨1Õ,Ðä8AÀ(×8KÑ4ˆKÑ4˜*ß ÚØ% aÕ(ˆIÜ�MÓ"¤dÜ�i¬S°Ó-?Õ!?Ó@ó'ô ò Ø×,Ñ,¨ZÓ8ˆFØŠ~ÚØ8>Ñ5ˆN˜M¨:Ø"×&Ñ& xÓ0ˆFØÐAÔAÚðÜ" =×#4Ñ#4°\À3Ó#GÓH�Ü  ×!2Ñ!2°<ÀÓ!EÓF�
ð ˜qÔ  J°!¤OÚØÒ!Ø)�
Ø!+‘Ø Ô+¨z¸^Ô/KÚà# jÕ0ˆKØ#¤c¨-Ó&8¸:Õ&EÕEˆIÜ Ø)×-Ñ-¨n¸bÓA×GÑGÈÔLÓRÑL�qÔPQ—�ÑLÑRóˆNð  Ñ"÷ Eð EØ ‘N÷Eð Eà¤ MÓ 2Õ2°nÑDð #÷ &Ø#Ô7ØÐ"2Ô2ß.ñ ð ð ð × Ñ àØ"Ø!ØØØÜ˜Ó'ðô
ð �O‰Oõ ÙKXóô ðÜ—’˜ T×*ñ 9LðF ÒÙà!ˆØ!#ˆÜ˜z×*ˆIØ&(ˆGØ&(ˆGó ñØØØØØØØà(×,Ñ,¨q°°¨_Ó=�Ø(×,Ñ,¨q°°¨_Ó=�Ø’? g¢oÛÜ,¨ZÀWÓM�Ü,¨ZÀWÓM�Ø’9 ¢	ÛØ—‘˜q  k°)Ð&;¸QÐ!>Õ?Ô@Ø—‘˜q  k°)Ð&;¸QÐ!>Õ?Ö@ñ ô #& g£,°!Ô"3�G˜A–J¼¿ºÈÐVWÔ9XˆEÜ"% g£,°!Ô"3�G˜A–J¼¿ºÈÐVWÔ9XˆEÜ�GŠG�E˜5Ô!Ø�K‰K˜ÔØ�M‰M˜%Ô Ø˜ %¨à×)Ñ)¨A×-Ø •] d×&?Ñ&?Õ?À1×Dä—’× ñC +ðF �V˜XÐ%Ð%øôs œzÐ*ô Ûðüò SøôB ô Ûðús0   Â>6O#Å)
O<
Å8O<
ÈPÏ#O9Ï8O9ÐPÐPc               ó0   € V ^8„  d   QhRRRRRRRRRR	R
R/# )r   rƒ  r   rÅ  r=  r÷  r‘  r!   ró  r½   r   r  r   )r   s   "r   r   rÆ    sG   € ÷ 1&ñ 1&àð1&ð ð1&ð ð	1&ð
 ð1&ð %ð1&ð 
Eñ1&r   c           	     	óR  €  \        VP                  R R4      4      p\        VP                  RR4      4      pT^ 8:  g   T^ 8:  d   R# T^ ,          pY‡,          p	T	\	        T4      T,          ,           p
TP                  R4      pTP                  R4      pTe   Tf   R# \        YY¹T
4      p\        YYÉT
4      pTe   Tf   R# \	        TP                  4      ^8w  g   \	        TP                  4      ^8w  d   R# TP                  ^,          T8w  g   TP                  ^,          T8w  d   R# \        T4       Uu. uF$  p\        P                  ! TRTR3,          R4      NK&  	  pp\        T4       Uu. uF$  p\        P                  ! TRTR3,          R4      NK&  	  ppT Uu. uF  pT P                  TT4      NK  	  pp \        P                  ! TR4       \        P                  ! YÞ.4       TTT3#   \        \        3 d     R# i ; iu upi u upi u upi   \         d     LPi ; i)	r  ro  r¿   NÚk_allÚv_allrn   .)r=   r   r¯   é   )r!   r   r�  r^  rH   r¤  r^   rø  rO   Ú	transposer  r   r‘  r]  rž   )rC  rƒ  rÅ  r÷  r‘  ró  r  r¿   r  r  r  rá  râ  r:  r;  r	  rT   r;   ré   rd  s   &&&&&&              r   r  Ú(DiskBlockStore._load_token_major_segment  s  € ð	Ü˜]×.Ñ.¨|¸SÓAÓBˆJÜ˜]×.Ñ.¨|¸SÓAÓBˆJð ˜Œ?˜j¨AœoÙà! !Õ$ˆ	ØÕ,ˆØ¤# mÓ"4°zÕ"AÕAˆ	Ø ×$Ñ$ WÓ-ˆØ ×$Ñ$ WÓ-ˆØŠ?˜gšoÙä-Ø G¸)ó
ˆô .Ø G¸)ó
ˆð Š=˜EšMÙÜˆu�{‰{Ó˜qÔ ¤C¨¯©Ó$4¸Ô$9ÙØ�;‰;�q�>˜ZÔ'¨5¯;©;°q­>¸ZÔ+GÙäFKÈJÔFWÓXÑFWÀ”—’˜U 1 a¨ 9Õ-¨|Ö<ÑFWˆÐXäBGÈ
ÔBSó
ÙBS¸QŒB�LŠL˜˜q ! S˜yÕ)¨<Ö8ÑBSð 	ð 
ñ HUó
ÙGTÀˆD×'Ñ'¨°sÖ;Á}ð 	ð 
ð	Ü�HŠH�Z Ô&ô 	�Š��ÔØ�V˜XÐ%Ð%øôM œ:Ð&ô 	Úð	üò4 Yùò
ùò
øô
 ô 	Ùð	ús5   ‚6G1 Ä7*H	Å0*HÆ HÆ>H Ç1HÈHÈH&È%H&c               ó    € V ^8„  d   QhRRRR/# )r   r   r!  r   r  r   )r   s   "r   r   rÆ  9  s    € ÷ T&ñ T&Ø8ðT&à	DñT&r   c                óâ  a a3€ V'       g   R# RpRpRpRpRpRp^ p\        4       p	\        4       p
. pV EFæ  w  rÍS P                  V4      pVf    R# Vw  po3pS3P                  R4      R8w  d    R#  \        S3P                  RR4      4      p\        S3P                  RR4      4      pT^ 8:  g   T^ 8:  d    R# Tf   TpTpMTT8w  g   TT8w  d    R# TP                  R4      pTP                  R4      pTe   Tf    R# T^ ,          pTT,          pT\        T4      T,          ,           p\        TTTTT4      p\        TTTTT4      pTe   Tf    R# Tw  ppTw  pp\        ;QJ d    . R	 TR
,           4       F  NK  	  5M! R	 TR
,           4       4      p\        ;QJ d    . R TR
,           4       F  NK  	  5M! R TR
,           4       4      p\        T4      ^8w  g   \        T4      ^8w  d    R# T^,          T8w  g   T^,          T8w  d    R# T^ ,          T^ ,          8w  d    R# Tf8   TR,          pTR,          p\        TR,          4      p\        TR,          4      pMMTTR,          8w  g=   TTR,          8w  g/   T\        TR,          4      8w  g   T\        TR,          4      8w  d    R# T	P                  T4       T
P                  T4       TT^ ,          ,          pTP                  T3T 3R lT 4       4        \        P                  ! TR4       EKé  	  Ve   Ve   Ve   Ve   Ve   V^ 8:  d   R# \        V4      p \        V4      p!V e   V!f   R# V w  p"p#p$V!w  p%p&p' \        P                   ! V	V"R7      P#                  V.VO54      p(\        P                   ! V
V%R7      P#                  V.VO54      p)Rp*T^,           T*,           ^,
          T*,          T*,          p+. p,. p-\%        T4       EF   p.\        P&                  ! T^,          T^,          T+T^,          3T"R7      p/\        P&                  ! T^,          T^,          T+T^,          3T%R7      p0T(RT.R3,          P)                  ^^^ ^4      T/RRT1R3&   T)RT.R3,          P)                  ^^^ ^4      T0RRT1R3&   T,P+                  \,        P.                  ! T/T#R7      4       T-P+                  \,        P.                  ! T0T&R7      4       EK  	  T$e!   T, U1u. uF  p1T1P1                  T$4      NK  	  p,p1T'e!   T- U2u. uF  p2T2P1                  T'4      NK  	  p-p2T$e   T, U1u. uF  p1\3        T14      NK  	  p,p1T'e   T- U2u. uF  p2\3        T24      NK  	  p-p2\,        P4                  ! T,T-,           4       T,T-T3#   \        \
        3 d      R# i ; i  \         d     EK”  i ; i  \
         d     R# i ; iu up1i u up2i u up1i u up2i )zûFast path for token-major shards.

Concatenate raw token-major byte ranges before constructing MLX arrays.
This avoids a first-request MLX compile of 72 per-layer concatenations
when a prefix spans a common-prefix shard plus a request-specific shard.
Nr¦  r  r  ro  r¿   r:  r;  c              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   rû   r}  s   & r   r~   ÚCDiskBlockStore._load_token_major_prefix_segments.<locals>.<genexpr>u  ó   é € ÐDÑ,C qœC ŸF˜FÓ,Cùr´   r^   c              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   rû   r}  s   & r   r~   rB  v  rC  r´   r›  r(   c              3  óH   <"  € T F  pSP                  SV4      x € K  	  R # 5irz   r$  r%  s   & €€r   r~   rB  �  r'  r(  r'   ro   rn   .)Ú	bytearrayr‹  r   r!   r�  r^  rH   r   r“   rF   rˆ   r   r‘  r]  rw  r1   rˆ  r‰  rø  r`   r=  rŒ   rO   rQ   rŠ  rR   rž   )4rC  r   r  r+  Úk_tail_shapeÚv_tail_shapeÚk_dtypeÚv_dtypeÚtotal_tokensÚk_bufÚv_bufrd  rƒ  ró  rŠ  rÅ  r‘  r.  r¿   rá  râ  r  r  r  Úk_slicedÚv_slicedÚk_rawÚk_sliced_entryÚv_rawÚv_sliced_entryrc   rd   Úk_dtype_infoÚv_dtype_infoÚ
k_np_dtypeÚk_mlx_dtypeÚk_bitcast_toÚ
v_np_dtypeÚv_mlx_dtypeÚv_bitcast_toÚk_npÚv_npÚkv_stepra   rT   r;   r	  Úk_layerÚv_layerr©   rª   r÷  s4   f&                                                 @r   Ú!_load_token_major_prefix_segmentsÚ0DiskBlockStore._load_token_major_prefix_segments9  sÀ  ù€ ÷ Ùà$(ˆ
Ø(,ˆØ26ˆØ26ˆØ!%ˆØ!%ˆØˆÜ“ˆÜ“ˆØ!ˆä)1Ñ%ˆJØ×,Ñ,¨ZÓ8ˆFØŠ~ÚØ8>Ñ5ˆN˜M¨:Ø× Ñ  Ó*Ð.>Ô>ÚðÜ" =×#4Ñ#4°\À3Ó#GÓH�Ü  ×!2Ñ!2°<ÀÓ!EÓF�
ð ˜qÔ  J°!¤OÚØÒ!Ø)�
Ø!+‘Ø Ô+¨z¸^Ô/KÚà$×(Ñ(¨Ó1ˆGØ$×(Ñ(¨Ó1ˆGØŠ '¢/ÚØ% aÕ(ˆIØ# jÕ0ˆKØ#¤c¨-Ó&8¸:Õ&EÕEˆIÜ:Ø˜J¨°¸ióˆHô ;Ø˜J¨°¸ióˆHð Ò 8Ò#3ÚØ$,Ñ!ˆE�>Ø$,Ñ!ˆE�>ß”eÑD¨N¸7Ö,CÓD—e‘eÑD¨N¸7Ö,CÓDÓDˆGß”eÑD¨N¸7Ö,CÓD—e‘eÑD¨N¸7Ö,CÓDÓDˆGÜ�7‹|˜qÔ ¤C¨£L°AÔ$5ÚØ�q�z˜ZÔ'¨7°1­:¸Ô+CÚØ�q�z˜W Q�ZÔ'ÚØÒ#Ø& r�{�Ø& r�{�Ü˜n¨WÕ5Ó6�Ü˜n¨WÕ5Ó6‘à ¨¥Ô+Ø 7¨2¥;Ô.Øœc .°Õ"9Ó:Ô:Øœc .°Õ"9Ó:Ô:âà�L‰L˜ÔØ�L‰L˜ÔØ˜G A�JÕ&ˆLØ�O‰Oõ ÙKXóô ðÜ—’˜ T×*ñC *2ðL ÒØÒ#ØÒ#ØŠØŠØ˜qÔ áä.¨wÓ7ˆÜ.¨wÓ7ˆØÒ <Ò#7ÙØ0<Ñ-ˆ
�K Ø0<Ñ-ˆ
�K ð	Ü—=’= ¨jÔ9×AÑAØÐ- Ñ-óˆDô —=’= ¨jÔ9×AÑAØÐ- Ñ-óˆDð ˆØ! AÕ%¨Õ/°!Õ3¸Õ?À7ÕJˆØ!ˆØ!#ˆÜ�z×"ˆAÜ—h’hØ˜a• ,¨q¥/°8¸\È!½_ÐMØ ôˆGô —h’hØ˜a• ,¨q¥/°8¸\È!½_ÐMØ ôˆGð .2°!°Q¸°)­_×-FÑ-FÀqÈ!ÈQÐPQÓ-RˆG�C˜˜,˜¨Ð)Ñ*Ø-1°!°Q¸°)­_×-FÑ-FÀqÈ!ÈQÐPQÓ-RˆG�C˜˜,˜¨Ð)Ñ*Ø�K‰KœŸš °Ô<Ô=Ø�M‰Mœ"Ÿ(š( 7°+Ô>×?ñ #ð Ò#Ù26Ó7±$¨Q�A—F‘F˜<Ö(±$ˆDÐ7ØÒ#Ù4:Ó;±F¨q�a—f‘f˜\Ö*±FˆFÐ;ØÒ#Ù04Ó5±¨1”O AÖ&±ˆDÐ5ØÒ#Ù28Ó9±&¨Q”o aÖ(±&ˆFÐ9Ü
�Š��v•ÔØ�V˜XÐ%Ð%øôg œzÐ*ô Ûðûôp ô Ûðûô4 ô 	Úð	üò2 8ùâ;ùâ5ùâ9sO   Á16VÌV8Í(AW ÔWÔ%W"Õ	W'Õ'W,ÖV5Ö4V5Ö8W×W×W×WTc               ó$   € V ^8„  d   QhRRRRRR/# )r   r;  r½   r  r   r   r  r   )r   s   "r   r   rÆ  Ï  s)   € ÷ u&ñ u&Ø)ðu&ØAEðu&à	Dñu&r   c               óÞ	  € V'       g   R# \         P                  P                  RR4      P                  4       R9   p\        P
                  ! 4       p. pV FS  pV P                  ;_uu_ 4        V P                  P                  V4      pRRR4       Xf    R# VP                  V4       KU  	  . pV Fy  w  ršV'       d9   VR,          ^ ,          V	8w  g$   VR,          ^,          R,          ^,           V
8w  d   VP                  Wš.34       KZ  VR,          ^,          P                  V
4       K{  	  \        P
                  ! 4       pV P                  V4      p\        P
                  ! 4       pVeE   V'       d;   \        R\        V4       R\        V4       RWÛ,
          R RWÔ,
          R R	2	R
R7       V# \        P
                  ! 4       pV P                  VVR7      p\        P
                  ! 4       pVeG   V'       d=   \        R\        V4       R\        V4       RVV,
          R RVV,
          R R	2	R
R7       V# \        P
                  ! 4       p. p\        V4      ^,
          p\        V4       F`  w  pw  p	pTP                  T P                  T	TT;'       d-    VV8H  ;'       d     \        V4      ;'       d    V^ ,          ^ 8H  R7      4       Kb  	  \        P
                  ! 4       p\         ;QJ d    R V 4       F  '       g   K   R
M	  RM! R V 4       4      '       d   R# V^ ,          w  ppp\        V4      pV^ 8X  g   \        V4      V8w  d   R# . p. p. pV F=  pVw  pp p!\        V4      V8w  g   \        V 4      V8w  d    R# VP#                  V!4       K?  	  \        P
                  ! 4       p"\%        V4       F�  p#TP                  \&        P(                  ! V Uu. uF  pV^ ,          V#,          NK  	  up^R7      4       TP                  \&        P(                  ! V Uu. uF  pV^,          V#,          NK  	  up^R7      4       K�  	  \&        P*                  ! VV,           4       \        P
                  ! 4       p$V'       dH   \        R\        V4       R\        V4       RVV,
          R RV$V",
          R RV$V,
          R R	2R
R7       VVV3#   + '       g   i     ELR; iu upi u upi )aQ  Load a cached prefix directly as per-layer K/V tensors.

Handles prefixes that span several layer-major shards. Returns
``(keys, values, per_block_metadata)`` where each key/value tensor
covers the full requested prefix for one layer. This is the warm-disk
fast path: 72 tensors for a Qwen3-VL-4B prefix instead of 209 * 72
block slabs.
NÚAPC_DISK_TRACErS  zAPC_DISK_TRACE restore blocks=z
 segments=z raw_token_major=z.3fzs total=ÚsT)Úflush©r  z layerwise=c              3  ó(   "  € T F  qR J x € K
  	  R # 5irz   r   )r|   Úsegs   & r   r~   Ú9DiskBlockStore.load_layer_major_prefix.<locals>.<genexpr>"  s   é € Ð6¡o˜s�d�{£oùr�   Fr\   z load=zs concat_eval=)rÔ  ÚtrueÚyeséÿÿÿÿ)r   r   r   r   ÚtimeÚperf_counterrç  rä  rŒ   ra  ÚprintrH   r6  rå   r  r   r’   rˆ   rø  rO   r_   rž   )%rC  r;  r  ÚtraceÚtrace_t0r¬  r8   ry  r   rƒ  rí  Útrace_raw_t0Úraw_token_majorÚtrace_raw_t1Útrace_layerwise_t0Ú	layerwiseÚtrace_layerwise_t1Útrace_load_t0Úloaded_segmentsr,  r-  ró  Útrace_load_t1Ú
first_keysÚfirst_valuesrK  r  rT   r;   rd  rj  Úseg_keysÚ
seg_valuesÚseg_metadataÚtrace_concat_t0r	  Útrace_concat_t1s%   &&$                                  r   Úload_layer_major_prefixÚ&DiskBlockStore.load_layer_major_prefixÏ  s   € ÷ ÙÜ—
‘
—‘Ð/°Ó4×:Ñ:Ó<Ð@TÑTˆÜ×$Ò$Ó&ˆà*,ˆÛˆAØ×!×!Ó!ØŸ™Ÿ™¨Ó*�÷ "àŠ}ÚØ�N‰N˜5Ö!ñ ð 24ˆÛ%,Ñ!ˆJçØ˜B•< •? jÔ0Ø˜B•< •? 2Õ&¨Õ*¨iÔ7à—‘ ¨[Ð 9Ö:à˜•˜Q•×&Ñ& yÖ1ñ &-ô ×(Ò(Ó*ˆØ×@Ñ@ÀÓJˆÜ×(Ò(Ó*ˆØÒ&ßÜðÜ! ,Ó/Ð0°
¼3¸x»=¸/ð J'Ø'3Õ'BÀ3Ð&Gð HØ)Õ4°SÐ9¸ð<ð õð #Ð"ä!×.Ò.Ó0ÐØ×DÑDØØ/ð Eó 
ˆ	ô "×.Ò.Ó0ÐØÒ ßÜðÜ! ,Ó/Ð0°
¼3¸x»=¸/ð J!Ø!3Ð6HÕ!HÈÐ Mð NØ/°(Õ:¸3Ð?¸qðBð õð Ðä×)Ò)Ó+ˆØˆÜ˜x›=¨1Õ,ÐÜ8AÀ(Ö8KÑ4ˆKÑ4˜* mØ×"Ñ"Ø×.Ñ.ØØ!à)÷ 2ð 2Ø'Ð+;Ñ;÷2ð 2ä  Ó/÷2ð 2ð *¨!Õ,°Ñ1ð /ó 	öñ 9Lô ×)Ò)Ó+ˆß‹3Ñ6¡oÓ6�3�3Š3Ñ6¡oÓ6×6Ò6Ùà&5°aÕ&8Ñ#ˆ
�L !Ü˜“_ˆ
Ø˜Œ?œc ,Ó/°:Ô=Ùà!ˆØ!#ˆØ!ˆÛ"ˆCØ14Ñ.ˆH�j ,Ü�8‹} 
Ô*¬c°*«oÀÔ.KÚØ�O‰O˜LÖ)ñ	 #ô ×+Ò+Ó-ˆÜ�zÖ"ˆAØ�K‰KœŸš¹_Ó'M¹_°c¨¨A­¨q¯	¨	¹_Ñ'MÐTUÔVÔWØ�M‰MÜ—’±_ÓE±_¨c  A¥ q§	 	±_ÑEÈAÔNöñ #ô
 	�Š��v•ÔÜ×+Ò+Ó-ˆßÜðÜ˜lÓ+Ð,¨J´s¸8³}°oð FØ%¨Õ5°cÐ:ð ;Ø.°Õ@ÀÐEð FØ(¨8Õ3°CÐ8¸ð	;ð
 õð �V˜XÐ%Ð%÷G "×!Ð!üòh (NùâEs   Á,SÏS%Ð!S*ÓS"c               ó(   € V ^8„  d   QhRRRRRRRR/# )	r   rÅ  r=  r  r!   ró  r½   rƒ  r   r   )r   s   "r   r   rÆ  F  s2   € ÷ "ñ "àð"ð ð"ð %ð	"ð
 ñ"r   c           
     	ór  € . p^ pV F«  p\        V4       F™  pR F�  p	RV RV	 V 2p
VP                  V
4      pVf   \        P                  RWJ4          R# \	        V4      pVf   \        P                  RVV
4          R# Vw  rÞpVP                  WzW½V34       WnV,
          ,          pK’  	  K›  	  K­  	  WV3# )r¯   rù  rK  Nr  z4APC disk shard %s has unsupported/corrupt tensor: %s)r©   rª   )rø  r   ré  rê  r„  rŒ   )rC  rÅ  r  ró  rƒ  ÚspecsÚtotal_bytesrí  r	  ÚsuffixrI  ry  rŒ  rè   rï   rK  s   &&&&&           r   Ú_collect_direct_specsÚ$DiskBlockStore._collect_direct_specsF  sÎ   € ð ˆØˆÛ&ˆIÜ˜:Ö&�Û(�FØ˜y˜k¨¨6¨(°1°#Ð6�DØ*×.Ñ.¨tÓ4�EØ’}ÜŸ™ØBÀJôô  $Ü7¸Ó>�FØ’~ÜŸ™ØRØ&Ø ôô
  $Ø$*‘M�E Ø—L‘L )°5ÀÐ!EÔFØ¨¥;Õ.’Kó% )ó 'ñ 'ð* Ð!Ð!r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   rƒ  r   ró  r½   r   rð  r   )r   s   "r   r   rÆ  f  s)   € ÷ ~ñ ~Øð~Ø/<ð~à	Dñ~r   c                	ód
  € V'       g   . # V P                  V4      pVf   R .\        V4      ,          # Vw  rEp \        VP                  RR4      4      pTP                  R4      R9   Ed    T Uu. uF4  p\        \        P                  ! TRT R2,          4      R,          4      NK6  	  p	pT P                  T	RR7      p
T
f   R .\        T4      ,          # T
w  r¼p. p \        TP                  R	R4      4      p\        T4       Fd  w  ppTT,          pTT,           pTP                  T Uu. uF  pTR
TT1R3,          NK  	  upT Uu. uF  pTR
TT1R3,          NK  	  upT34       Kf  	  T# T P                  YGY!4      pTf   R .\        T4      ,          # Tw  ppT'       g&   T Uu. uF  p. . T P                  TT4      3NK  	  up# \        R T 4       4      p\!        R T 4       4      pTT,
          p\        T4      ^8”  dX   TTT P"                  ,           8”  d@   \        T4      ^,          pT P%                  YR T 4      T P%                  YTR  4      ,           #  \'        TR4      ;_uu_ 4       pTP)                  TT,           4       TP+                  T4      p\        T4      T8w  d   R .\        T4      ,          uuR R R 4       # \-        T4      p T U!U"UUu/ uF  w  p!p"p!ppT"T TT,
          TT,
           bK  	  p#pp"p!pR R R 4       T U!U"U%u/ uF  w  p!p"p% p!T"T%bK  	  p&p"p!p%. pT FÌ  p. p'. p(Rp)\5        T4       Fu  p*RT RT* 2p+RT RT* 2p,\7        X#T+,          T&T+,          4      p\7        T#T,,          T&T,,          4      pTe   Tf   Rp) M&T'P                  T4       T(P                  T4       Kw  	  T)'       d'   TP                  T'T(T P                  TT4      34       K»  TP                  R 4       KÎ  	   \8        P:                  ! TR 4       T#   \        \
        3 d    R .\        T4      ,          u # i ; iu upi   \        \        \
        \        P                  3 d    R .\        T4      ,          u # i ; i  \        \
        3 d    R .\        T4      ,          u # i ; iu upi u upi u upi u uppp"p!i   + '       g   i     ELÉ; i  \.         d5   p$\0        P3                  RTT$4       R .\        T4      ,          u R p$?$# R p$?$ii ; iu up%p"p!i   \.         d     T# i ; i)Nr  ro  r¦  rù  rú  r  Frh  r¿   .rn   c              3  ó.   "  € T F  w    rqx € K  	  R # 5irz   r   )r|   rK  rè   s   &  r   r~   Ú3DiskBlockStore._load_direct_many.<locals>.<genexpr>¡  s   é € Ð=±uÑ"3 ! Q¨°!œ³uùs   ‚c              3  ó0   "  € T F  w     rVx € K  	  R # 5irz   r   )r|   rK  rï   s   &  r   r~   r�  ¢  s   é € Ð7±™o˜a  A q•c³ùs   ‚rV  z&APC disk direct read failed for %s: %sTrÐ  rÑ  )r  r  r  )r‹  rH   r!   r   r�  r^  rZ  r[  r€  ÚJSONDecodeErrorr„  rå   rŒ   r‹  r  rþ   rò   rî  rò  rX  r“  rY  r”  r]  ré  rê  rø  r�  r   r‘  )-rC  rƒ  ró  rŠ  rÅ  r÷  r‘  r  ré   r;  rÇ  r¡   r£   Ú	metadatasr•   r¿   rý  Úmdrè   rï   r©   rª   Ú	collectedrˆ  r‰  rí  Ú	min_startÚmax_endÚspanÚmidr_  ÚslabrŠ  rK  rI  Úraw_by_namer7  ry  Úentries_by_namerT   r;   Úokr	  Úk_nameÚv_names-   &&&                                          r   rò  Ú DiskBlockStore._load_direct_manyf  ss  € ÷ ØˆIØ×(Ñ(¨Ó4ˆØŠ>Ø�6œC Ó.Õ.Ð.Ø4:Ñ1ˆ zð	/Ü˜]×.Ñ.¨|¸SÓAÓBˆJð ×Ñ˜XÓ&ð +
õ 
ð
3ñ  -ó á,˜ô œŸ
š
 =°1°S°E¸°Õ#@ÓAÀ,ÕOÖPÙ,ð ð  ð ×1Ñ1°,ÐRWÐ1ÓXˆFØŠ~Ø�v¤ MÓ 2Õ2Ð2Ø28Ñ/ˆJ iØˆCð3Ü  ×!2Ñ!2°<ÀÓ!EÓF�
ô # 9Ö-‘��2Ø˜J��Ø˜jÕ(�Ø—
‘
á7AÓB±z°!˜˜3  c 	¨1Ð,×-Ð-±zÑBÙ7CÓD±|°!˜˜3  c 	¨1Ð,×-Ð-±|ÑDØðöñ .ð ˆJà×.Ñ.Ø¨ó
ˆ	ð ÒØ�6œC Ó.Õ.Ð.Ø&Ñˆˆ{ßñ "/óá!.�Ið �R˜×4Ñ4°]ÀIÓNÓOÙ!.ñð ô
 Ñ=±uÓ=Ó=ˆ	ÜÑ7±Ó7Ó7ˆØ˜Õ"ˆä�Ó Ô"Ø�{ T×%DÑ%DÕDÔDä�mÓ$¨Õ)ˆCØ×)Ñ)Ø¨$¨3Ð/óà×&Ñ& zÀÀÐ3FÓGõHð Hð	/Ü�j $×'Ô'¨1ð
 —‘�z IÕ-Ô.Ø—v‘v˜d“|�Ü�t“9 Ô$Ø ˜6¤C¨Ó$6Õ6÷ (Ò'ô " $Ó'�ñ 38öá27Ñ.˜˜4  E¨3ð ˜$˜u yÕ0°3¸µ?ÐCÒCÙ27ð ó ÷ (ñ$ DIÕIÁ5Ñ+?¨1¨d°E¸1¸a˜4 š;Á5ˆÒIØKMˆÛ&ˆIØ#%ˆDØ%'ˆFØˆBÜ˜:Ö&�Ø˜Y˜K r¨!¨Ð-�Ø˜Y˜K r¨!¨Ð-�Ü5Ø Õ'¨¸Õ)@ó�ô 6Ø Õ'¨¸Õ)@ó�ð ’9 ¢	Ø�BÙØ—‘˜A”Ø—‘˜aÖ ñ '÷ Ø—
‘
àØØ×3Ñ3°MÀ9ÓMðöð —
‘
˜4Ö ñ7 'ð<	Ü�HŠH�Z Ô&ð ˆ
øôg œ:Ð&ô 	/Ø�6œC Ó.Õ.Ò.ð	/üò øô œi¬´T×5IÑ5IÐJô 3Ø�v¤ MÓ 2Õ2Ò2ð3ûô œzÐ*ô 3Ø�v¤ MÓ 2Õ2Ò2ð3üò CùÚDùòùõ8÷ (×'Ð'ûô ô 	/Ü�N‰NÐCÀZÐQRÔSØ�6œC Ó.Õ.Õ.ûð	/üô JøôD ô 	ØØˆ
ð	úsÌ   ¹P Á,Q Á0:P?Â*Q ÃR Ä+R,
ÅR1Æ(R6ÉS É2ASÊ>
S Ë	SË#R;Ë>SÌS ÌTÏ<T  Ð$P<Ð;P<Ð?Q Ñ8Q?Ñ>Q?Ò$R)Ò(R)Ò;SÓS	ÓS ÓS ÓTÓ")TÔTÔTÔ T/Ô.T/c               ó    € V ^8„  d   QhRRRR/# )r   r'  zList['APCBlock']r   r$  r   )r   s   "r   r   rÆ  æ  s   € ÷ 1ñ 1Ð!1ð 1°dñ 1r   c                óœ  € V'       g   R# . pV EF  pVP                   e   VP                  e   VP                  f   K0  TP                  \	        \        VP                   4      \        VP                  4      \        VP                  4      \        ;QJ d    . R VP                   4       F  NK  	  5M! R VP                   4       4      \        VP                  4      \        VP                  4      R7      4       \        V4      V P                  8¼  g   EK
  V P                  V4       . pEK   	  V'       g   R# V P                  V4       R# )z–Schedule segment-shard writes containing ``blocks``. Returns
immediately; the writer thread does the safetensors save + atomic
rename + index update.
Nc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   rû   r£  s   & r   r~   Ú,DiskBlockStore.save_batch.<locals>.<genexpr>÷  s   é € Ð#@±K¨q¤C¨§F F³Kùr´   )r  r  r¾   r¼   rT   r;   )r  rT   r;   rŒ   r   r!   r  r¾   r“   r¼   rÎ   rH   rï  Ú_enqueue_block_snapshots)rC  r'  Ú	snapshotsrù  s   &&  r   Ú
save_batchÚDiskBlockStore.save_batchæ  sè   € ÷
 Ùà.0ˆ	ÜˆAØ�|‰|Ò# q§v¡v¢~¸¿¹Ò9IÙØ×ÑÜ"Ü" 1§<¡<Ó0Ü # A§M¡MÓ 2Ü" 1§<¡<Ó0ß#œeÑ#@°A·K²KÓ#@Ÿe™eÑ#@°A·K²KÓ#@Ó@Ü˜aŸf™f›Ü §¡›>ôô	ô �9‹~ ×!7Ñ!7×7Ø×-Ñ-¨iÔ8Ø“	ñ ÷  Ùà×%Ñ% iÖ0r   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR	/# )
r   r-  r!   r¼   r½   r¾   rš   r›   r   r$  r   )r   s   "r   r   rÆ  	  s<   € ÷ /ñ /àð/ð !ð/ð ð	/ð
 $ð/ð 
ñ/r   c                óø   € \         ;QJ d    . R V 4       F  NK  	  5M! R V 4       4      pV'       d	   V'       g   R# \        \        V4      V\        V4      \        V4      R7      pV P	                  V4       R# )z«Schedule an exact prompt-cache snapshot write.

Exact snapshots are used for custom cache layouts that cannot be
reconstructed from independently concatenated K/V blocks.
c              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   rû   r£  s   & r   r~   Ú2DiskBlockStore.save_exact_cache.<locals>.<genexpr>	  r¥  r´   N)r-  r¼   r¾   rš   )r“   r,  r!   rÎ   Ú_enqueue_exact_snapshot)rC  r-  r¼   r¾   rš   r«  rF  s   &&&&&  r   Úsave_exact_cacheÚDiskBlockStore.save_exact_cache	  s]   € ÷ ”eÑ6©IÓ6—e‘eÑ6©IÓ6Ó6ˆß§,ÙÜ*Ü˜:“Ø!Ü˜:“Ü˜lÓ+ô	
ˆð 	×$Ñ$ XÖ.r   c          
     ó,   € V ^8„  d   QhRRRRRRRRRR	/# )
r   r'  r&  r¡   r¢   r£   r¿   r!   r   r$  r   )r   s   "r   r   rÆ  	  s<   € ÷ #ñ #à*ð#ð 'ð#ð )ð	#ð
 ð#ð 
ñ#r   c                óŒ  € V'       d   V'       d	   V'       g   R# \        V4      p\        V4      pV Uu. uF  qwP                  NK  	  ppV P                  V4      p	\        V4      V P                  ,           ^,
          V P                  ,          p
\        ^ \        V4      V P                  4       F†  p\        WW°P                  ,            4      pV Uu. uF  qwP                  NK  	  pp\        VVV\        V4      V	W°P                  ,          V
R7      pV P                  V P                  V4      WÞ4       Kˆ  	  R# u upi u upi )zÄSchedule a layer-major shard write directly from a KV cache.

This avoids building thousands of per-block MLX tensors when the
caller only needs durable disk persistence for future warm restores.
N)r'  r¡   r£   r¿   r(  r)  r*  )	rÎ   r  rA  rH   rï  rø  r%  r!   Ú_enqueue_shard)rC  r'  r¡   r£   r¿   Úshared_layer_keysÚshared_layer_valuesrù  Úall_block_hashesr(  r*  rè   Úchunkr;  rF  s   &&&&&          r   Úsave_layer_major_blocksÚ&DiskBlockStore.save_layer_major_blocks	  s  € ÷ ŸZ¯|ÙÜ  Ó,ÐÜ" <Ó0ÐÙ28Ó9±&¨QŸLœL±&ÐÐ9Ø×%Ñ%Ð&6Ó7ˆä�‹K˜$×0Ñ0Õ0°1Õ4Ø×#Ñ#õ$ˆô ˜1œc &›k¨4×+AÑ+AÖBˆEÜ˜¨×0FÑ0FÕ(FÐGÓHˆEÙ27Ó8±%¨QŸLœL±%ˆLÐ8Ü.ØØ,Ø0Ü˜z›?Ø!Ø#×'=Ñ'=Õ=Ø+ôˆHð ×ÑØ×"Ñ" <Ó0°,öó Cùò :ùò 9s   µD<ÃEc               ó    € V ^8„  d   QhRRRR/# )r   r¦  úList[_DiskBlockSnapshot]r   r$  r   )r   s   "r   r   rÆ  @	  s   € ÷ 
ñ 
Ð2Jð 
Ètñ 
r   c                	óš   € V Uu. uF  q"P                   NK  	  ppV P                  V P                  V4      V\        V4      4       R # u upi rz   )r  r²  rA  rÎ   )rC  r¦  rù  r;  s   &&  r   r¥  Ú'DiskBlockStore._enqueue_block_snapshots@	  sA   € Ù.7Ó8©i¨Ÿœ©iˆÐ8Ø×ÑØ×Ñ˜|Ó,¨l¼DÀ»Oö	
ùò 9s   …Ac               ó    € V ^8„  d   QhRRRR/# )r   rF  r,  r   r$  r   )r   s   "r   r   rÆ  F	  s   € ÷ Wñ WÐ0Gð WÈDñ Wr   c                	ó<  € \        VP                  4      pV P                  V4      pV P                  V4      pVP	                  4       '       d;   V P
                  ;_uu_ 4        V P                  P                  W$4       R R R 4       R # \        P                  ! 4       pV P                  ;_uu_ 4        WPP                  V&   R R R 4        V P                  P                  W2.W34       R #   + '       g   i     R # ; i  + '       g   i     LE; i  \        P                   dw    T P                  ;_uu_ 4        T P                  P!                  TR 4       R R R 4       M  + '       g   i     M; iTP#                  4        \$        P'                  R4        R # i ; i)Nz8APC disk write queue full; dropping exact-cache snapshot)r!   r-  rE  r!  r5  rç  rå  Ú
setdefaultrÞ  rß  rã  rá  rÝ  Ú
put_nowaitrÛ  ÚFullr\  rÛ   ré  rê  )rC  rF  r-  r  rT  r¼  s   &&    r   r­  Ú&DiskBlockStore._enqueue_exact_snapshotF	  s  € Ü˜×,Ñ,Ó-ˆ
Ø×%Ñ% jÓ1ˆØ×Ñ Ó)ˆØ�;‰;�=Š=Ø×!×!Ó!Ø×!Ñ!×,Ñ,¨ZÔ>÷ "áä�_Š_ÓˆØ×!×!Ó!Ø*,�O‰O˜JÑ'÷ "ð	WØ�G‰G×Ñ ¨,¸ÐEÖF÷ "Ö!áú÷ "×!ûô �z‰zô 	WØ×%×%Ó%Ø—‘×#Ñ# J°Ô5÷ &×%×%Ð%úà�F‰FŒHÜ�N‰NÐU×Vð		WúsB   Á"C)Â1C=Ã	D Ã)C:	Ã=D	Ä)FÄ9EÅ	FÅE/Å*-FÆFc               ó(   € V ^8„  d   QhRRRRRRRR/# )	r   r  rF   r;  r½   Úpayloadr   r   r$  r   )r   s   "r   r   rÆ  Z	  s2   € ÷ ñ àðð $ðð ð	ð
 
ñr   c           	     	óœ  € V P                  V4      pVP                  4       '       dZ   V P                  ;_uu_ 4        \        V4       F+  w  rVV P                  P                  \        V4      WE34       K-  	  R R R 4       R # \        P                  ! 4       pV P                  ;_uu_ 4        V F  pWpP                  \        V4      &   K  	  R R R 4        V P                  P                  V\        V4      W734       R #   + '       g   i     R # ; i  + '       g   i     LN; i  \        P                   d“    T P                  ;_uu_ 4        T F(  pT P                  P!                  \        T4      R 4       K*  	  R R R 4       M  + '       g   i     M; iTP#                  4        \$        P'                  R\)        T4      4        R # i ; i)Nz8APC disk write queue full; dropping shard with %d blocks)r!  r5  rç  rå   rä  r¿  r!   rÞ  rß  rã  rá  rÝ  rÀ  rÎ   rÛ  rÁ  r\  rÛ   ré  rê  rH   )rC  r  r;  rÄ  rT  ré   r  r¼  s   &&&&    r   r²  ÚDiskBlockStore._enqueue_shardZ	  sD  € ð ×Ñ Ó)ˆà�;‰;�=Š=Ø×!×!Ó!ä'0°Ö'>‘O�CØ—K‘K×*Ñ*¬3¨z«?¸T¸KÖHñ (?÷ "ñ ä�_Š_ÓˆØ×!×!Ó!Û*�
Ø35—‘¤ J£Ó0ñ +÷ "ð
	Ø�G‰G×Ñ ¬$¨|Ó*<¸gÐJÖK÷ "Ö!ñ ú÷ "×!ûô
 �z‰zô 	Ø×%×%Ó%Û".�JØ—O‘O×'Ñ'¬¨J«¸Ö>ñ #/÷ &×%×%Ð%úð �F‰FŒHÜ�N‰NØJÜ�LÓ!÷ð	úsA   ¼;C=Â*!DÃ'D$ Ã=D	ÄD!	Ä$)GÅ/FÅ<	GÆFÆ7GÇ
Gc               ó$   € V ^8„  d   QhRRRRRR/# )r   r¡   ri   r£   r   z%Tuple[List[mx.array], List[mx.array]]r   )r   s   "r   r   rÆ  z	  s&   € ÷ *ñ *Ø"ð*à$ð*ð 
/ñ*r   c                	ó†  € V '       g   W3# \        V ^ ,          P                  ^,          4      pRpV^,           V,           ^,
          V,          V,          pWB,
          pV^ 8:  d   W3# . p. p\        W4       EFD  w  r‰\        VP                  4      ^8w  g   \        V	P                  4      ^8w  d%   VP	                  V4       VP	                  V	4       K^  . VP                  R,          OVNVP                  ^,          N5p
. V	P                  R,          OVNV	P                  ^,          N5pVP	                  \
        P                  ! V\
        P                  ! W¨P                  R7      .^R7      4       VP	                  \
        P                  ! V	\
        P                  ! W¹P                  R7      .^R7      4       EKG  	  Wg3# )r¯   ro   r[   r'   r\   )	r!   r^   r¨   rH   rŒ   rO   r_   r`   r(   )r¡   r£   rK  r^  ra   rb   Úpadded_keysÚpadded_valuesr©   rª   Úk_pad_shapeÚv_pad_shapes   &&          r   Ú_pad_layer_major_arraysÚ&DiskBlockStore._pad_layer_major_arraysy	  sn  € ÷
 ØÐ+Ð+Ü˜: a�=×.Ñ.¨qÕ1Ó2ˆØˆØ! AÕ%¨Õ/°!Õ3¸Õ?À7ÕJˆØÕ,ˆ
Ø˜Œ?ØÐ+Ð+à&(ˆØ(*ˆÜ˜
×1‰DˆAÜ�1—7‘7‹|˜qÔ ¤C¨¯©£L°AÔ$5Ø×"Ñ" 1Ô%Ø×$Ñ$ QÔ'ÙØ@˜AŸG™G B�KÐ@¨Ð@°Q·W±W¸QµZÑ@ˆKØ@˜AŸG™G B�KÐ@¨Ð@°Q·W±W¸QµZÑ@ˆKØ×ÑÜ—’ ¤2§8¢8¨K¿w¹wÔ#GÐHÈqÔQôð × Ñ Ü—’ ¤2§8¢8¨K¿w¹wÔ#GÐHÈqÔQ÷ñ 2ð Ð)Ð)r   c               ó    € V ^8„  d   QhRRRR/# )r   Úindicesr½   r   zList[Tuple[int, int]]r   )r   s   "r   r   rÆ  ™	  s   € ÷ ñ  Mð Ð6Kñ r   c                	ó  € V '       g   . # . p\        V ^ ,          4      ;r#V R,           F;  p\        V4      pWS^,           8X  d   TpK  VP                  W#^,           34       T;r#K=  	  VP                  W#^,           34       V# )r¯   r›  )r!   rŒ   )rÐ  Úrangesrè   r  Úidx_rawré   s   &     r   Ú_contiguous_rangesÚ!DiskBlockStore._contiguous_ranges˜	  s€   € çØˆIØ(*ˆÜ˜7 1�:“Ð&ˆØ˜r—{�{ˆGÜ�g“,ˆCØ˜Q•hŒØ�ÙØ�M‰M˜5¨¥(Ð+Ô,ØÐˆE‘Dñ #ð 	�‰�u Q�hÐ'Ô(Øˆr   c          
     ó,   € V ^8„  d   QhRRRRRRRRR	R
/# )r   rg   r   rÊ  rF   r’  zdict[str, mx.array]rd  údict[str, str]r   r   r   )r   s   "r   r   rÆ  ¨	  sC   € ÷ Tñ TàðTð ðTð $ð	Tð
 !ðTð 
ñTr   c                	ó4
  a aaa€ ^ RI Hp \        WP                  4      '       dè   \	        \        VR^ 4      ;'       g    ^ 4      pRSS R2&   \        V4      SS R2&   \        \	        \        VR\        \        V4      RR4      4      ;'       g    ^ 4      4      SS R2&   VP                  e   VP                  e   V^ 8:  d   R
SS R2&   R# VP                  RR	V1R3,          SS R2&   VP                  RR	V1R3,          SS R2&   R# \        WP                  4      '       Ed   RSS R2&   \        \	        \        VR^ 4      ;'       g    ^ 4      4      SS R2&   \        \	        \        VR4      4      4      SS R2&   \        \	        \        VR^ 4      ;'       g    ^ 4      4      SS R2&   \        \	        \        VR^ 4      ;'       g    ^ 4      4      SS R2&   VP                  e   VP                  f   R
SS R2&   R# VP                  SS R2&   VP                  SS R2&   R# \        WP                  4      '       d×   RSS R2&   \        \	        \        VR4      4      4      SS R2&   \        \	        \        VR^ 4      ;'       g    ^ 4      4      SS R2&   \        \	        \        VR^ 4      ;'       g    ^ 4      4      SS R2&   VP                  e   VP                  f   R
SS R2&   R# VP                  SS R2&   VP                  SS R2&   R# \        WP                  4      '       d¨   RSS R2&   \        \        VP                  4      4      SS R2&   \        VP                  4       F   w  rxVf   R
SS RV R2&   K  VSS RV 2&   K"  	  VP                  e   VP                  SS R 2&   VP                   e   VP                   SS R!2&   R# \        WP"                  4      '       d“   R"SS R2&   \        \        VP$                  4      4      SS R2&   \&        ;QJ d5    VVVV 3R# l\        VP$                  4       4       F  '       d   K   R$# 	  R# ! VVVV 3R# l\        VP$                  4       4       4      # \        V\(        4      '       du   R%SS R2&   \        \        V4      4      SS R2&   \&        ;QJ d+    VVVV 3R& l\        V4       4       F  '       d   K   R$# 	  R# ! VVVV 3R& l\        V4       4       4      # R$# )'r¯   rl   rU   rÍ  rÌ  rÏ  rX   ro   rÒ  NrÔ  rÎ  T.rn   rÐ  rÑ  rÓ  rr   rÔ  rq   rÕ  rt   rÖ  ru   r×  rw   rØ  r’  rÙ  rÚ  rÛ  rÜ  rÝ  rÞ  c              3  óZ   <"  € T F   w  rSP                  VS R V 2SS4      x € K"  	  R# 5i©rß  N©Ú_snapshot_exact_cache_entry©r|   rä  r}   r’  rd  rÊ  rC  s   &  €€€€r   r~   Ú=DiskBlockStore._snapshot_exact_cache_entry.<locals>.<genexpr>ë	  sB   øé € ð ñ !4‘H�Að ×0Ñ0Ø˜f˜X R¨ sÐ+¨V°X÷ð ó !4ùó   ƒ(+Fr“   c              3  óZ   <"  € T F   w  rSP                  VS R V 2SS4      x € K"  	  R# 5irÚ  rÛ  rÝ  s   &  €€€€r   r~   rÞ  õ	  sB   øé € ð ñ !-‘H�Að ×0Ñ0Ø˜f˜X R¨ sÐ+¨V°X÷ð ó !-ùrß  )rƒ   rm   r„   r…   r!   r‡   rF   r†   rT   r;   r‰   rŠ   r‹   rH   rå   r�   rŽ   r�   r�   r¶   r“   )	rC  rg   rÊ  r’  rd  r”   r–   rä  r—   s	   f&fff    r   rÜ  Ú*DiskBlockStore._snapshot_exact_cache_entry¨	  s÷  û€ õ 	4ä�a×)Ñ)×*Ò*Ü”g˜a ¨1Ó-×2Ð2°Ó3ˆCØ)-ˆH˜�x˜uÐ%Ñ&Ü+.¨s«8ˆH˜�x˜wÐ'Ñ(Ü),Ü”G˜A˜v¤w¬t°A«w¸ÀÓ'DÓE×JÐJÈÓKó*ˆH˜�x˜uÐ%Ñ&ð �v‰vŠ~ §¡Ò!1°S¸A´XØ.1�˜F˜8 6Ð*Ñ+ÙØ$%§F¡F¨3°°°°a¨<Õ$8ˆF�f�X˜R�=Ñ!Ø$%§H¡H¨S°$°3°$¸¨\Õ$:ˆF�f�X˜R�=Ñ!Ùä�a×1Ñ1×2Ó2Ø)6ˆH˜�x˜uÐ%Ñ&Ü),¬S´¸¸FÀAÓ1F×1KÐ1KÈ!Ó-LÓ)MˆH˜�x˜uÐ%Ñ&Ü-0´´W¸QÀ
Ó5KÓ1LÓ-MˆH˜�x˜yÐ)Ñ*Ü+.¬s´7¸1¸hÈÓ3J×3OÐ3OÈaÓ/PÓ+QˆH˜�x˜wÐ'Ñ(Ü(+¬C´¸¸6À1Ó0E×0JÐ0JÈÓ,KÓ(LˆH˜�x˜t�_Ñ%Ø�v‰vŠ~ §¡Ò!1Ø.1�˜F˜8 6Ð*Ñ+ÙØ$%§F¡FˆF�f�X˜R�=Ñ!Ø$%§H¡HˆF�f�X˜R�=Ñ!Ùä�a×0Ñ0×1Ò1Ø)5ˆH˜�x˜uÐ%Ñ&Ü/2´3´w¸qÀ,Ó7OÓ3PÓ/QˆH˜�x˜{Ð+Ñ,Ü+.¬s´7¸1¸hÈÓ3J×3OÐ3OÈaÓ/PÓ+QˆH˜�x˜wÐ'Ñ(Ü36Ü”G˜AÐ/°Ó3×8Ð8°qÓ9ó4ˆH˜�x˜Ð/Ñ0ð �v‰vŠ~ §¡Ò!1Ø.1�˜F˜8 6Ð*Ñ+ÙØ$%§F¡FˆF�f�X˜R�=Ñ!Ø$%§H¡HˆF�f�X˜R�=Ñ!Ùä�a×-Ñ-×.Ò.Ø)1ˆH˜�x˜uÐ%Ñ&Ü),¬S°·±«\Ó):ˆH˜�x˜uÐ%Ñ&Ü% a§g¡gÖ.‘�Ø’=Ø69�H ˜x r¨!¨¨EÐ2Ó3à/4�F˜f˜X R¨ sÐ+Ó,ñ	 /ð
 �~‰~Ò)Ø34·>±>�˜&˜ Ð/Ñ0Ø�y‰yÒ$Ø./¯i©i�˜&˜ Ð*Ñ+Ùä�a×+Ñ+×,Ò,Ø)5ˆH˜�x˜uÐ%Ñ&Ü),¬S°·±«]Ó);ˆH˜�x˜uÐ%Ñ&ß“3÷ ô !*¨!¯(©(Ô 3ó	—3”3ð ’3ð �3÷ ô !*¨!¯(©(Ô 3ó	ó ð ô �aœ×ÒØ)0ˆH˜�x˜uÐ%Ñ&Ü),¬S°«V«ˆH˜�x˜uÐ%Ñ&ß“3÷ ô !*¨!¤ó	—3”3ð ’3ð �3÷ ô !*¨!¤ó	ó ð ñ r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   rT  r   rF  r,  r   ú	List[int]r   )r   s   "r   r   rÆ  þ	  s(   € ÷ "*ñ "*àð"*ð *ð"*ð 
ñ	"*r   c                	ó.  € V P                  4       '       g   \        R V P                   24      hRRR\        \	        VP
                  4      4      R\        \	        VP                  4      4      RRP                  R VP                   4       4      R\        \        VP                  4      4      R	V P                  VP
                  4      /p/ p\        VP                  4       F/  w  rVV P                  VR
V 2WC4      '       d   K#  \        RV 24      h	  V'       g   . # \        P                   ! \#        VP%                  4       4      4       \&        P(                  ! 4        R\*        P,                  ! 4        2pVP.                  VP0                   RV V P2                   2,          p\        P4                  ! \        V4      WCR7       \&        P6                  ! W�4        V ;P8                  VP;                  4       P<                  ,          un        V P>                  ;_uu_ 4        WP@                  \	        VP
                  4      &   RRR4       V PC                  4        \	        VP
                  4      .#   \         d     Lqi ; i  + '       g   i     LG; i)ú&APC disk cache directory unavailable: r¦  r§  r-  r¾   r¼   rT  c              3  óJ   "  € T F  p\        \        V4      4      x € K  	  R # 5irz   rü  r£  s   & r   r~   Ú=DiskBlockStore._write_exact_cache_snapshot.<locals>.<genexpr>	
  s   é € Ð!JÑ7I°!¤#¤c¨!£f§+ +Ó7Iùrþ  rÃ  r(  rg   z'unsupported exact-cache entry at index Ú-rª  ©rd  N)"r8  r]  rÙ  rF   r!   r-  r¾   rÿ  r¼   rH   rš   rE  rå   rÜ  r^  rO   rž   rÎ   r;   r   ÚgetpidrÞ  Ú	get_identr    r
  r   Úsave_safetensorsÚreplacer÷  rU  rV  rç  rå  r  )	rC  rT  rF  rd  r’  rý  rg   ÚtagÚtmps	   &&&      r   Ú_write_exact_cache_snapshotÚ*DiskBlockStore._write_exact_cache_snapshotþ	  s  € ð
 ×Ñ×!Ò!ÜÐBÀ4Ç8Á8À*ÐMÓNÐNàÐ&Øœ#œc (×"5Ñ"5Ó6Ó7Øœ#œc (×"5Ñ"5Ó6Ó7Ø˜Ÿ™Ñ!J°x×7IÒ7IÓ!JÓJØœ3œs 8×#8Ñ#8Ó9Ó:Ø˜×*Ñ*¨8×+>Ñ+>Ó?ð$
ˆð ')ˆÜ˜h×3Ñ3Ö4‰DˆAØ×3Ñ3°A¸¸1¸#°wÀ×QÔQÜ Ð#JÈ1È#Ð!NÓOÐOñ 5÷ ØˆIä
�Š”�V—]‘]“_Ó%Ô&Ü—’“�˜Qœy×2Ò2Ó4Ð5Ð6ˆØ�k‰k˜tŸy™y˜k¨¨3¨%°·±¨}Ð=Õ=ˆÜ
×ÒœC ›H fÕ@Ü
�
Š
�3Ôð	Ø×Ò §	¡	£× 3Ñ 3Õ3Õð ××ÓØ:>×Ñœc (×"5Ñ"5Ó6Ñ7÷ à×ÑÔÜ�H×'Ñ'Ó(Ð)Ð)øô ô 	Ùð	úç×ús   Ç1I3 È#"JÉ3JÊ JÊJ	c               ó$   € V ^8„  d   QhRRRRRR/# )r   rT  r   rF  r%  r   rã  r   )r   s   "r   r   rÆ  "
  s(   € ÷ -.ñ -.àð-.ð *ð-.ð 
ñ	-.r   c                	ó²  € VP                   pV'       g   . # \        VP                  4      \        VP                  4      8w  d   \	        R 4      h/ pVP
                  VR&   \        \        VP                  4      4      VR&   \        \        VP                  4      4      VR&   \        V4       F†  w  rV\        P                  ! R\        VP                  4      R\        VP                  4      R\        VP                  4      RVP                    Uu. uF  p\        V4      NK  	  up/4      VRV R	2&   Kˆ  	  T P#                  V Uu. uF  qfP$                  NK  	  up4      p. p	. p
\        VP&                  4      p\)        VP                  VP                  4       Fß  w  rÍV UUu. uF  w  rïVR
Wë,          Wû,          1R3,          NK!  	  pppV UUu. uF  w  rïVR
Wë,          Wû,          1R3,          NK!  	  pppT	P+                  \        V4      ^8X  d
   V^ ,          M\,        P.                  ! V^R7      4       T
P+                  \        V4      ^8X  d
   V^ ,          M\,        P.                  ! V^R7      4       Ká  	  V P1                  Wš4      w  ršV P3                  WWIW«4       V Uu. uF  qfP                  NK  	  up# u upi u upi u uppi u uppi u upi )z3layer-major disk snapshot has mismatched K/V layersr(  r)  r*  r  r  r¾   r¼   rù  rú  .rn   r\   )r'  rH   r¡   r£   r^  r(  rF   r!   r)  r*  rå   rZ  Údumpsr  r  r¾   r¼   rÔ  r#  r¿   r¨   rŒ   rO   r_   rÍ  Ú_save_layer_major_shard)rC  rT  rF  r'  rd  ré   rù  rÂ   rÒ  r¡   r£   ÚbsÚk_srcÚv_srcrè   rï   r0  r1  s   &&&               r   Ú_write_layer_major_snapshotÚ*DiskBlockStore._write_layer_major_snapshot"
  sy  € ð
 —‘ˆßØˆIÜˆx×"Ñ"Ó#¤s¨8×+@Ñ+@Ó'AÔAÜÐRÓSÐSà#%ˆØ'×0Ñ0ˆ�ÑÜ$'¬¨H×,BÑ,BÓ(CÓ$Dˆ�Ñ!Ü$'¬¨H×,BÑ,BÓ(CÓ$Dˆ�Ñ!Ü Ö'‰FˆCÜ'+§z¢zà ¤# a§l¡lÓ"3Ø!¤3 q§}¡}Ó#5Ø ¤# a§l¡lÓ"3Ø°!·+²+Ó!>±+¨Q¤# a¦&±+Ñ!>ð	ó(ˆH�q˜˜˜U�^Ó$ñ (ð ×(Ñ(ÁfÓ)MÁfÀ×*<Ô*<ÁfÑ)MÓNˆØ%'ˆ
Ø')ˆÜ�×$Ñ$Ó%ˆÜ × 3Ñ 3°X×5JÑ5JÖK‰LˆEÙNTÔUÉfÁ
À�u˜S %¥*¨s­xÐ"7¸Ð:×;Ð;ÉfˆGÑUÙNTÔUÉfÁ
À�u˜S %¥*¨s­xÐ"7¸Ð:×;Ð;ÉfˆGÑUØ×ÑÜ! '›l¨aÔ/�˜–
´R·^²^ÀGÐRSÔ5Tôð ×ÑÜ! '›l¨aÔ/�˜–
´R·^²^ÀGÐRSÔ5Töñ Lð $(×#?Ñ#?Øó$
Ñ ˆ
ð 	×$Ñ$Ø˜(°ô	
ñ '-Ó-¡f —”¡fÑ-Ð-ùò1 "?ùò *Nùó
 VùÛUùò .s   ÄJ>ÅKÆ$%KÇ%KÊ'Kc               ó$   € V ^8„  d   QhRRRRRR/# )r   rT  r   r'  rº  r   rã  r   )r   s   "r   r   rÆ  Q
  s(   € ÷ &.ñ &.àð&.ð )ð&.ð 
ñ	&.r   c                	óš  € / pV'       d6   V^ ,          P                   '       d   \        V^ ,          P                   4      M^ p\        V4       F¤  w  rVVP                   e   VP                  f   K#  \        P
                  ! R\        VP                  4      R\        VP                  4      R\        VP                  4      RVP                   Uu. uF  p\        V4      NK  	  up/4      VRV R2&   K¦  	  . p. p	\        V4       F±  p
TP                  \        P                  ! V Uu. uF%  qfP                   f   K  VP                   V
,          NK'  	  up^R7      4       T	P                  \        P                  ! V Uu. uF%  qfP                  f   K  VP                  V
,          NK'  	  up^R7      4       K³  	  V P                  W‰4      w  r‰V'       d6   V^ ,          P                  '       d   \        V^ ,          P                  4      M^ pV P!                  WW8W›4       V Uu. uF  qfP                  NK  	  up# u upi u upi u upi u upi )r¯   r  r  r¾   r¼   rù  rú  r\   )rT   rH   rå   r;   rZ  rô  r!   r  r  r¾   r¼   rø  rŒ   rO   r_   rÍ  rõ  )rC  rT  r'  rd  r  ré   rù  rÂ   r¡   r£   r	  r¿   s   &&&         r   Ú_write_block_snapshotÚ$DiskBlockStore._write_block_snapshotQ
  s×  € ð
 $&ˆß,2°v¸aµy·~·~°~”S˜ �Ÿ™Ô(È1ˆ
Ü Ö'‰FˆCØ�v‰vŠ~ §¡Ò!1ÙÜ'+§z¢zà ¤# a§l¡lÓ"3Ø!¤3 q§}¡}Ó#5Ø ¤# a§l¡lÓ"3Ø°!·+²+Ó!>±+¨Q¤# a¦&±+Ñ!>ð	ó(ˆH�q˜˜˜U�^Ó$ñ (ð &(ˆ
Ø')ˆÜ�zÖ"ˆAØ×ÑÜ—’Ù(.ÓE© 1·&±&”Y�Q—V‘V˜A—Y�Y©ÑEÈAôôð
 ×ÑÜ—’Ù*0ÓI©& Q·H±H”[�Q—X‘X˜a—[�[©&ÑIÐPQôöñ #ð $(×#?Ñ#?Øó$
Ñ ˆ
÷ 28¸FÀ1½I×<O×<OÐ<O”S˜ �×,Ñ,Ô-ÐUVˆ
Ø×$Ñ$Ø˜(°ô	
ñ '-Ó-¡f —”¡fÑ-Ð-ùò/ "?ùò Fùò
 Jùò .s$   ÃH9Ä&H>Ä:H>Å=IÆIÈ"Ic               ó4   € V ^8„  d   QhRRRRRRRRR	RR
RRR/# )r   rT  r   r'  r›   rd  r×  r¡   ri   r£   r¿   r!   r   r$  r   )r   s   "r   r   rÆ  y
  sP   € ÷ !ñ !àð!ð ð!ð !ð	!ð
 #ð!ð %ð!ð ð!ð 
ñ!r   c                	óP  € V P                  4       '       g   \        R V P                   24      h/ p\        \	        WE4      4       F  w  pw  ršW—RV 2&   W§RV 2&   K  	  RVR&   RP                  R V 4       4      VR&   \        \        V4      4      VR&   \        \        V4      4      VR	&   \        P                  ! \        VP                  4       4      4       \        P                  ! 4        R
\        P                   ! 4        2pVP"                  VP$                   RV V P&                   2,          p\        P(                  ! \        V4      WsR7       \        P*                  ! WÁ4        V ;P,                  VP/                  4       P0                  ,          un        V P2                  ;_uu_ 4        \        V4       F(  w  rÞW3V P4                  \        VP6                  4      &   K*  	  RRR4       V P9                  4        R#   \         d     Lsi ; i  + '       g   i     L3; i)rå  r©   rª   r  r¦  rT  c              3  ó^   "  € T F#  p\        \        VP                  4      4      x € K%  	  R # 5irz   )rF   r!   r  ©r|   rù  s   & r   r~   Ú9DiskBlockStore._save_layer_major_shard.<locals>.<genexpr>‰
  s!   é € Ð+SÉFÀq¬C´°A·L±LÓ0A×,BÐ,BËFùs   ‚+-r;  r  r¿   rè  rª  ré  N)r8  r]  rÙ  rå   r¨   rÿ  rF   rH   r!   rO   rž   rÎ   r;   r   rê  rÞ  rë  r    r
  r   rì  rí  r÷  rU  rV  rç  rä  r  r  )rC  rT  r'  rd  r¡   r£   r¿   r’  r	  r©   rª   rî  rï  ré   rù  s   &&&&&&&        r   rõ  Ú&DiskBlockStore._save_layer_major_shardy
  sµ  € ð ×Ñ×!Ò!ÜÐBÀ4Ç8Á8À*ÐMÓNÐNØ&(ˆÜ"¤3 zÓ#@ÖA‰IˆA‰v�Ø�Q�q�c�7‰OØ�Q�q�c�7‹Oñ Bð .ˆ�ÑØ#&§8¡8Ñ+SÉFÓ+SÓ#Sˆ�Ñ Ü!$¤S¨£_Ó!5ˆ�ÑÜ!$¤S¨£_Ó!5ˆ�ÑÜ
�Š”�V—]‘]“_Ó%Ô&ô —’“�˜Qœy×2Ò2Ó4Ð5Ð6ˆØ�k‰k˜tŸy™y˜k¨¨3¨%°·±¨}Ð=Õ=ˆÜ
×ÒœC ›H fÕ@Ü
�
Š
�3Ôð	Ø×Ò §	¡	£× 3Ñ 3Õ3Õð ××ÓÜ# FÖ+‘�Ø26°�—‘œC §¡Ó-Ó.ñ ,÷ ð 	×ÑÖøô ô 	Ùð	úç×ús   Å-1H Æ28HÈHÈHÈH%	c               ó   € V ^8„  d   QhRR/# r-  r   )r   s   "r   r   rÆ  œ
  s   € ÷ ñ ˜dñ r   c           
     	ó  €  V P                   P                  4       pVf   V P                   P                  4        R# Vw  r#rEV P                  V4      p \	        V\
        4      '       d   V P                  Wd4      pM:\	        V\        4      '       d   V P                  Wd4      pMV P                  Wd4      p V P                  ;_uu_ 4        V F(  pV P                  P                  \!        V4      R4       K*  	  RRR4       VP#                  4        V P                   P                  4        RpRpEK0    \         d!   p\        P                  RYg4        Rp?L¢Rp?ii ; i  + '       g   i     Lo; i  T P                  ;_uu_ 4        T F(  pT P                  P                  \!        T4      R4       K*  	  RRR4       M  + '       g   i     M; iTP#                  4        T P                   P                  4        RpRpi ; i)TNz%APC disk shard save failed for %s: %s)rÝ  r   Ú	task_doner!  r„   r,  rð  r%  rù  rý  rÍ   ré  rê  rã  rá  r\  r!   rÛ   )	rC  Úitemr  r;  rÄ  r¼  rT  r7  r  s	   &        r   rú  ÚDiskBlockStore._writer_loopœ
  s†  € ØØ—7‘7—;‘;“=ˆDØŠ|Ø—‘×!Ñ!Ô#ÙØ26Ñ/ˆH GØ×#Ñ# HÓ-ˆDðÜ˜gÔ'>×?Ò?Ø#'×#CÑ#CÀDÓ#R‘LÜ Ô)@×AÒAØ#'×#CÑ#CÀDÓ#R‘Là#'×#=Ñ#=¸dÓ#L‘Lð ×)×)Ó)Û&2˜
ØŸ™×+Ñ+¬C°
«O¸TÖBñ '3÷ *ð —‘”Ø—‘×!Ñ!Ô#ð
 �Ø“øô ô QÜ—‘ÐFÈ×PÑPûðQú÷ *×)û�T×)×)Ó)Û&2˜
ØŸ™×+Ñ+¬C°
«O¸TÖBñ '3÷ *×)×)Ð)úð —‘”Ø—‘×!Ñ!Ô#ð
 �Ø‘úsl   Á'D1 Á:E2 Á;'D1 Â"E2 Â#D1 Â4E2 Ã	/EÄ1EÄ<EÅE2 ÅEÅE2 ÅE/	Å2HÆ/F?	Æ6	HÆ?GÇ
7Hc               ó   € V ^8„  d   QhRR/# r-  r   )r   s   "r   r   rÆ  º
  s   € ÷ 	%ñ 	%�tñ 	%r   c                	ó  € V P                   P                  4        V P                   F  pV P                  P	                  R 4       K   	  V P                   F  pVP                  4        K  	  V P                  ;_uu_ 4        V P                  P                  4        R R R 4       V P                  ;_uu_ 4        V P                  P                  4        R R R 4       R #   + '       g   i     LI; i  + '       g   i     R # ; irz   )rà  rÛ   rû  rÝ  Úputrÿ  rì  rë  r/  rò  rñ  )rC  rK  rÂ   s   &  r   ÚcloseÚDiskBlockStore.closeº
  s¡   € Ø�
‰
�‰ÔØ—”ˆAØ�G‰G�K‰K˜Öñ à—”ˆAØ�F‰FŽHñ à×$×$Ó$Ø×Ñ×$Ñ$Ô&÷ %à×"×"Ó"Ø×Ñ×"Ñ"Ô$÷ #Ñ"÷ %×$úç"×"Ð"ús   Â CÂ7C/ÃC,	Ã/D 	)rî  r÷  rå  rë  rì  rí  rá  rã  rä  rç  rñ  rò  ró  rÝ  rè  rð  rï  rà  rû  rÙ  r4  rÅ  )rL  r=   N)Dr  r  r  r  r  r   r  r  rH   ÚSHARD_STEM_LENÚEXACT_STEM_LENru  rþ  Úclassmethodr  r  r  r!  r*  r1  r8  ÚstaticmethodrA  rE  rô  rö  Úpropertyr\  r`  re  ri  r  r‹  r”  r˜  r�  r²  r½  r»  rÄ  r�  rô  r  rë  rì  r  r6  r  ra  r„  r‹  rò  r§  r®  r·  r¥  r­  r²  rÍ  rÔ  rÜ  rð  rù  rý  rõ  rú  r  r  r   r   r   r¾  r¾  ó  sh  † ñð: €FØ€LØ€LÙ˜Ó&¨Õ+€NÙ˜Ó&¨Õ+€Nð Ð÷Sðl ô:ó ð:ð ô:ó ð:ð ôNó ðNõ5õ/õõð ôDó ðDð ôDó ðDõ
õ"-ð^ ô ó ð ð ô$ó ð$ð ô*ó ð*ð ô+ó ð+õ[õ|õ2õ@-õ3ð+ð ð	+ð
 ,0ð+ð "#÷+ðZ
ð $'ð	
ð
 .2÷
õ,,3õ\]ð~8Ø=@÷8ð21ØIL÷1õfõ$Sõ&-ðJ&ð
 #(÷J&õXD&õL1&õfT&ðlu&ØHL÷u&õn"õ@~õ@1õ</õ.#õJ
õWõ(ð> ô*ó ð*ð< ôó ðõTõl"*õH-.õ^&.õP!õF÷<	%ñ 	%r   r¾  c                  ó  € ] tR tRtRt]]R3R R lltR R ltR R	 lt	R
 R lt
R R ltR R ltR R ltR'R R lltR^ /R R lltR(R R lltR)R R lltR^ R^ /R R lltR R ltR  R! ltR" R# ltR$ R% ltR&tR# )*Ú
APCManageriÆ
  z2Block pool, hash table, LRU free queue, and stats.Nc               ó$   € V ^8„  d   QhRRRRRR/# )r   r<  r!   r¿   ÚdiskzOptional['DiskBlockStore']r   )r   s   "r   r   ÚAPCManager.__annotate__É
  s-   € ÷ D
ñ D
àðD
ð ðD
ð )ñ	D
r   c           	     	ó†  € W n         Wn        \        V4       Uu. uF  p\        VR 7      NK  	  upV n        RV n        RV n        V P                   F  pV P                  V4       K  	  / V n        \        4       V n
        \        4       V n        \        P                  ! 4       V n        W0n        \#        ^ \%        \&        P(                  P+                  RR4      4      4      V n        \#        ^\%        \&        P(                  P+                  RR4      4      4      V n        \%        \1        \&        P(                  P+                  RR4      4      R,          4      V n        \#        ^\%        \&        P(                  P+                  RR	4      4      4      V n        \#        ^\%        \&        P(                  P+                  R
R4      4      4      V n        \#        ^ \%        \&        P(                  P+                  RR4      4      4      V n        \#        ^ \%        \&        P(                  P+                  RR4      4      4      V n        R# u upi ))r
  NÚAPC_EXACT_CACHE_ENTRIESÚ2ÚAPC_EXACT_PREFIX_GUARD_TOKENSÚ16ÚAPC_DISK_MIN_FREE_RAM_GBz2.0ÚAPC_DISK_EVAL_BLOCK_CHUNKrÐ  ÚAPC_DISK_LOAD_BLOCK_CHUNKrÒ  ÚAPC_MAX_POOL_TENSORSÚ450000Ú!APC_LAYER_MAJOR_MEMORY_MIN_TOKENSÚ50000é   @)r¿   r<  rø  r	  ÚpoolÚ
_free_headÚ
_free_tailÚ
_free_pushÚ
hash_tabler   Ú_exact_cacher/  ÚstatsrÞ  ræ  Úlockr  rò   r!   r   r   r   Ú_exact_cache_maxÚexact_cache_guard_tokensr  Ú_disk_min_free_ram_bytesÚ_disk_eval_block_chunkÚ_disk_load_block_chunkÚ_max_pool_tensorsÚ_layer_major_memory_min_tokens)rC  r<  r¿   r  rý  rù  s   &&&&  r   rþ  ÚAPCManager.__init__É
  s¬  € ð %ŒØ$ŒÜCHÈÔCTÓ$UÑCT¸a¤X°q×%9ÑCTÑ$UˆŒ	Ø.2ˆŒØ.2ˆŒØ—”ˆAØ�O‰O˜AÖñ à/1ˆŒÜDOÃMˆÔÜ“ZˆŒ
Ü—O’OÓ%ˆŒ	ØŒ	Ü #ØŒs”2—:‘:—>‘>Ð";¸SÓAÓBó!
ˆÔô ),ØŒs”2—:‘:—>‘>Ð"AÀ4ÓHÓIó)
ˆÔ%ô ),Ü”"—*‘*—.‘.Ð!;¸UÓCÓDÈÕPó)
ˆÔ%ô '*ØŒs”2—:‘:—>‘>Ð"=¸sÓCÓDó'
ˆÔ#ô '*ØŒs”2—:‘:—>‘>Ð"=¸uÓEÓFó'
ˆÔ#ô "%ØŒs”2—:‘:—>‘>Ð"8¸(ÓCÓDó"
ˆÔô /2ØŒs”2—:‘:—>‘>Ð"EÀwÓOÓPó/
ˆÖ+ùòu %Vs   šH>c               ó    € V ^8„  d   QhRRRR/# ©r   rù  r	  r   r$  r   )r   s   "r   r   r    s   € ÷ "ñ "˜Hð "¨ñ "r   c                	óÀ   € V P                   Vn        R Vn        V P                   e   WP                   n        MWn        Wn         \        P                  ! 4       Vn        R # rz   )r(  r  r  r'  ro  r  ©rC  rù  s   &&r   r)  ÚAPCManager._free_push  s?   € Ø—‘ˆŒØˆŒØ�?‰?Ò&Ø#$�O‰OÕ àŒOØŒÜ—i’i“kˆŽr   c               ó    € V ^8„  d   QhRRRR/# r7  r   )r   s   "r   r   r    s   € ÷ 	ñ 	˜hð 	¨4ñ 	r   c                	ó  € VP                   e   VP                  VP                   n        MVP                  V n        VP                  e   VP                   VP                  n         MVP                   V n        R ;Vn         Vn        R # rz   )r  r  r'  r(  r9  s   &&r   Ú_free_removeÚAPCManager._free_remove  sX   € Ø�6‰6ÒØŸ&™&ˆA�F‰F�KàŸf™fˆDŒOØ�6‰6ÒØŸ&™&ˆA�F‰F�KàŸf™fˆDŒOØÐˆŒ�–r   c               ó   € V ^8„  d   QhRR/# )r   r   zOptional[APCBlock]r   )r   s   "r   r   r  &  s   € ÷ ñ Ð.ñ r   c                	óŠ  € V P                   pVf   R # V P                  V4       VP                  ed   V P                  P	                  VP                  4      VJ d;   V P                  VP                   V P
                  ;P                  ^,          un        R Vn        RVn        \        Vn	        ^ Vn
        R Vn        R Vn        V# ©Nr   )r'  r=  r  r*  r   r,  r4  r¼   rÏ   r  r¾   rT   r;   r9  s   & r   Ú
_evict_lruÚAPCManager._evict_lru&  s™   € Ø�O‰OˆØŠ9ÙØ×Ñ˜!ÔØ�<‰<Ò#¨¯©×(;Ñ(;¸A¿L¹LÓ(IÈQÓ(NØ—‘ §¡Ð-Ø�J‰J× Ò  AÕ%Õ ØˆŒØˆŒÜ(ˆŒØˆŒØˆŒØˆŒØˆr   c               ó    € V ^8„  d   QhRRRR/# )r   rù  r	  r   r   )r   s   "r   r   r  6  s   € ÷ ñ  8ð °ñ r   c                	ó|   € VP                   ^ 8X  d   V P                  V4       V;P                   ^,          un         V# ©r¯   )r  r=  r9  s   &&r   Ú_acquire_existingÚAPCManager._acquire_existing6  s-   € Ø�9‰9˜Œ>Ø×Ñ˜aÔ Ø	�	Š	�Q��	Øˆr   c               ó    € V ^8„  d   QhRRRR/# r7  r   )r   s   "r   r   r  <  s   € ÷ ñ ˜hð ¨4ñ r   c                	óŽ   € V;P                   ^,          un         VP                   ^ 8:  d   ^ Vn         V P                  V4       R# R# ©r=   N)r  r)  r9  s   &&r   Ú_release_oneÚAPCManager._release_one<  s2   € Ø	�	Š	�Q��	Ø�9‰9˜Œ>ØˆAŒIØ�O‰O˜AÖñ r   c               ó    € V ^8„  d   QhRRRR/# )r   r'  zIterable[APCBlock]r   r$  r   )r   s   "r   r   r  B  s   € ÷ %ñ %Ð0ð %°Tñ %r   c                	óœ   € V P                   ;_uu_ 4        V F  pV P                  V4       K  	  R R R 4       R #   + '       g   i     R # ; irz   )r-  rL  )rC  r'  rù  s   && r   ÚreleaseÚAPCManager.releaseB  s/   € Ø�Y�Y‹YÛ�Ø×!Ñ! !Ö$ñ ÷ �Y�YŠYús	   •:ºA	c          
     ó,   € V ^8„  d   QhRRRRRRRRRR	/# )
r   r¼   r½   r¾   r!   r  rW   rŸ  r   úTuple[Optional[List[Any]], int]r   )r   s   "r   r   r  H  sC   € ÷ H(ñ H(à ðH(ð ðH(ð )ð	H(ð
 ðH(ð 
)ñH(r   c           	     óâ  € V P                   pV P                  ^ 8:  d   Vf   R	# \        ;QJ d    . R V 4       F  NK  	  5M! R V 4       4      p\        V4      ^,
          pVe   V^ 8”  d   \	        V\        V4      4      pWt8:  d   R	# Rp^ p	V P                  ;_uu_ 4        Rp
RpV P                  ^ 8”  dð   V P                  P                  4        Fs  w  rÍ\        VP                  4      pVP                  V8w  g   Wä8:  g   Wç8”  d   K9  VRV VP                  8w  d   KO  Ve   V\        VP                  4      8”  g   Ko  Tp
TpKu  	  Ve[   V
eW   V P                  P                  V
4       \        P                  ! 4       Vn        \        VP                  4      p	VP                  pRRR4       VRJ;'       d    W—8  pV'       db   V P                  ^ 8”  dQ   \!        4       pVeC   VV P                  8  d2   \"        P%                  RVR
,          V P                  R
,          4       RpV'       Ed¸   VEe³   VP'                  VVV\)        WI4      R7      pVEe�   Vw  ppVP+                  V\        V4      ^,           R7      pVEec   Vw  pppVV8X  EdU   \        V4      V8X  EdD   VRV V8X  Ed9   V P                  ^ 8”  Edz   \-        V4      pVEej   \/        VW P0                  4      pV P                  ;_uu_ 4        V P2                  ;P4                  ^,          un        V P2                  ;P6                  ^,          un        V P2                  ;P8                  ^,          un        V P2                  ;P:                  V,          un        VV P                  9  d–   \=        V\        V4      V\        P                  ! 4       R7      V P                  V&   V P                  P                  V4       \        V P                  4      V P                  8”  d   V P                  P?                  RR7       KB  RRR4       VV3# V P                  ;_uu_ 4        V P2                  ;P4                  ^,          un        V P2                  ;P6                  ^,          un        V P2                  ;P8                  ^,          un        V P2                  ;P:                  V,          un        RRR4       VV3# Vf   R	# \-        V\        V4      ^,           R7      pVf   R	# V P                  ;_uu_ 4        V P2                  ;P4                  ^,          un        V P2                  ;P8                  ^,          un        V P2                  ;P:                  V	,          un        RRR4       VV	3#   + '       g   i     ELý; i  + '       g   i     TT3# ; i  + '       g   i     TT3# ; i  + '       g   i     TT	3# ; i)a  Return an exact-prefix prompt-cache snapshot for custom caches.

Mixed architectures such as Nemotron-H use recurrent SSM state in
addition to attention KV. That state is not block-concatenable, so the
safe reuse unit is an exact prompt-cache snapshot at a prefix boundary.
Nc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   rû   r£  s   & r   r~   Ú0APCManager.lookup_exact_cache.<locals>.<genexpr>X  r¥  r´   z=APC: skipping exact disk restore (free RAM %.1f GB < %.1f GB)F)r¾   r  rŸ  r¹  ©r¼   r¾   rš   r  r…  ©Nr¯   r%  ) r  r.  r“   rH   rþ   r!   r-  r+  r&  r¼   r¾   r‡  ro  r  rš   r0  r¼  ré  rõ  r²  rò   r½  rŸ   rÄ   r¿   r,  r9  r7  r0  r2  r  rˆ  )rC  r¼   r¾   r  rŸ  r  r«  r  Úsource_cacher¤   Úbest_keyÚ
best_entryrt  ry  Úcandidate_lenÚcan_try_diskÚfree_nowÚ
disk_matchr-  Údisk_prefix_lenrÇ  r±  Ústored_extra_hashrš   Ústorage_copyÚpromote_keys   &&&&&                     r   Úlookup_exact_cacheÚAPCManager.lookup_exact_cacheH  së  € ð �y‰yˆØ× Ñ  AÔ%¨$ª,ØˆNß”eÑ6©IÓ6—e‘eÑ6©IÓ6Ó6ˆÜ�kÓ" QÕ&ˆØÒ(Ð->ÀÔ-BÜ˜'¤3Ð'8Ó#9Ó:ˆGØÔ'ØˆNà,0ˆØˆ
Ø�Y�Y‹YØ&*ˆHØ7;ˆJØ×$Ñ$ qÔ(Ø"&×"3Ñ"3×"9Ñ"9Ö";‘J�CÜ$'¨¯©Ó$8�Mà×(Ñ(¨JÔ6Ø(Ô=Ø(Ô2á Ø" > MÐ2°e·o±oÔEÙ Ø!Ò)¨]¼SÀ×AUÑAUÓ=VÖ-VØ#&˜Ø%*š
ñ #<ð Ò)¨hÒ.BØ×%Ñ%×1Ñ1°(Ô;Ü+/¯9ª9«;�JÔ(Ü!$ Z×%9Ñ%9Ó!:�JØ#-×#:Ñ#:�L÷- ð0  4Ð'×@Ð@¨JÑ,@ˆß˜D×9Ñ9¸AÔ=Ü&Ó(ˆHØÒ#¨°4×3PÑ3PÔ(PÜ—‘ØVØ Õ(Ø×1Ñ1°WÕ=ôð
  %�çˆ<˜DÓ,Ø×/Ñ/ØØ%Ø"3Ü"%Ð&7Ó"Dð	 0ó ˆJð Ó%Ø.8Ñ+�
˜OØ×.Ñ.ØÜ(+¨KÓ(8¸1Õ(<ð /ó �ð Ó%ØEKÑB�MÐ#4°là)¨ZÕ7Ü Ó.°/ÕAØ'Ð(8¨Ð9¸]ÕJð  ×0Ñ0°1Õ4Ü+FÀ|Ó+T˜LØ+Ó7Ü.<Ø$1°:¿¹ó/" ð &*§Y§Y£YØ$(§J¡J×$9Ò$9¸QÕ$>Õ$9Ø$(§J¡J×$8Ò$8¸AÕ$=Õ$8Ø$(§J¡J§O¢O°qÕ$8¥OØ$(§J¡J×$=Ò$=ÀÕ$PÕ$=Ø'2¸$×:KÑ:KÔ'Kä,>Ø:GÜ;>¸z»?Ø=IÜ:>¿)º)»+ô	-.ð )-×(9Ñ(9¸+Ñ(Fð )-×(9Ñ(9×(EÑ(EÀkÔ(Rä,/°×0AÑ0AÓ,BØ.2×.CÑ.Cô-Dð -1×,=Ñ,=×,EÑ,EÈ5Ð,EÖ,Q÷' &/ð( (4°_Ð'DÐ DØ!ŸYŸY›YØ ŸJ™J×1Ò1°QÕ6Õ1Ø ŸJ™J×0Ò0°AÕ5Õ0Ø ŸJ™JŸOšO¨qÕ0�OØ ŸJ™J×5Ò5¸ÕHÕ5÷	 'ð
  ,¨_Ð<Ð<àÒØˆNÜ2ØÜ # KÓ 0°1Õ 4ô
ˆð ÒØˆNØ�Y�Y‹YØ�J‰J×!Ò! QÕ&Õ!Ø�J‰J�OŠO˜qÕ �OØ�J‰J×%Ò%¨Õ3Õ%÷ ð ˜ZÐ'Ð'÷_ �YˆYú÷N &/¦Yð( (4°_Ð'DÐ Dúß&žYð
  ,¨_Ð<Ð<ú÷ ŽYð ˜ZÐ'Ð'ús;   Â"BVÅA&VË<D3V/ÑBWÔ%A*WÖV,	Ö/W	×W	×W.	r¾   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	r   r¼   r½   rš   r›   r¾   r!   r   r   r   )r   s   "r   r   r  Ò  s2   € ÷ (ñ (à ð(ð $ð(ð
 ð(ð 
ñ(r   c          	     óÊ  € V P                   ^ 8:  d   V P                  e	   V'       g   R# \        ;QJ d    . R V 4       F  NK  	  5M! R V 4       4      p\        V4      pVf   R# \	        WCV P
                  4      pRpV P                  ;_uu_ 4        V P                   ^ 8”  d˜   \        V\        V4      V\        P                  ! 4       R7      V P                  V&   V P                  P                  V4       \        V P                  4      V P                   8”  d   V P                  P                  RR7       KB  RpRRR4       V P                  e`    V P                  P                  WdW54       V P                  ;_uu_ 4        V P                  ;P                   ^,          un        RRR4       RpV'       dC   V P                  ;_uu_ 4        V P                  ;P(                  ^,          un        RRR4       R# R#   + '       g   i     LÉ; i  + '       g   i     Lq; i  \"         d!   p\$        P'                  RT4        Rp?L˜Rp?ii ; i  + '       g   i     R# ; i)z:Store a full prompt-cache snapshot for exact-prefix reuse.NFc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   rû   r£  s   & r   r~   Ú/APCManager.store_exact_cache.<locals>.<genexpr>Ü  r¥  r´   rW  r…  Tz)APC exact disk save scheduling failed: %s)r.  r  r“   rŸ   rÄ   r¿   r-  r  r!   ro  r+  r‡  rH   rˆ  r®  r,  r8  rÍ   ré  rê  r:  )	rC  r¼   rš   r¾   r«  r˜   rt  Ústoredr7  s	   &&&$     r   Ústore_exact_cacheÚAPCManager.store_exact_cacheÒ  s³  € ð ×!Ñ! QÔ&¨4¯9©9Ò+<ÇYÙß”eÑ6©IÓ6—e‘eÑ6©IÓ6Ó6ˆÜ,¨\Ó:ˆØŠ>ÙÜ˜[°d·o±oÓFˆØˆØ�Y�Y‹YØ×$Ñ$ qÔ(Ü);Ø)Ü" :›Ø!'Ü"Ÿiši›kô	*�×!Ñ! #Ñ&ð ×!Ñ!×-Ñ-¨cÔ2Ü˜$×+Ñ+Ó,¨t×/DÑ/DÔDØ×%Ñ%×-Ñ-°5Ð-Ö9Ø�÷ ð �9‰9Ò ðOØ—	‘	×*Ñ*¨3¸ZÔPØ—Y—Y“YØ—J‘J×*Ò*¨aÕ/Õ*÷ à�÷ Ø——“Ø—
‘
×'Ò'¨1Õ,Õ'÷ áÙ÷1 �Yú÷ —Yûô ô OÜ—‘ÐJÈA×NÑNûðOú÷ –áúsI   ÂB)G=Å0H# Æ$HÆ'
H# Ç$IÇ=H	ÈH 	ÈH# È#IÈ.I	É	IÉI"	c               ó0   € V ^8„  d   QhRRRRRRRRRR	R
R/# )r   r¼   r½   r¾   r!   r  rW   rŸ  Úallow_memory_overlapr   r   rS  r   )r   s   "r   r   r  ü  sN   € ÷ X*ñ X*à ðX*ð ðX*ð )ð	X*ð
 ðX*ð #ðX*ð 
)ñX*r   c           
     óJ  € V P                   pVf   R
# V P                  ;_uu_ 4        V P                  ^ 8”  dZ   \        4       pVeL   WpP                  8  d<   \        P                  RVR,          V P                  R,          4       R
uuRRR4       # \        V4      V P                  ,          pVe.   V^ 8”  d'   \        V\        V4      V P                  ,          4      p\        p	. p
. p\        V4       EF  p\        ;QJ d?    . R WV P                  ,          V^,           V P                  ,            4       F  NK  	  5M8! R WV P                  ,          V^,           V P                  ,            4       4      p\        W�V4      pV P                  P                  V4      pV'       g$   Ve    VP                   V8X  d   R
u uuRRR4       # VP#                  V4      '       g    M)V
P%                  V4       VP%                  V4       Tp	EK  	  V
'       g   R
uuRRR4       # \        V
4      V P                  ,          pVV8:  d   R
uuRRR4       #  RRR4       VP'                  X
4      pVf   R
# Vw  ppp\        V4      \        X4      8w  d   R
# \)        VV4       Fž  w  pp \        ;QJ d3    . R VP                  RR4      P+                  R4       4       F  NK  	  5M,! R VP                  RR4      P+                  R4       4       4      p\        VP                  RR	4      4      pTT8w  g
   TT8w  g   Kœ  R
u # 	  \1        VVX4      pV P                  ;_uu_ 4        V P2                  ;P4                  \        V
4      ,          un        V P2                  ;P6                  ^,          un        V P2                  ;P8                  V,          un        RRR4       VV3#   + '       g   i     EL—; i  \,        \.        3 d    R
u u # i ; i  + '       g   i     TT3# ; i)a	  Return a ready prompt cache from a layer-major disk shard.

This is the warm-disk fast path. It deliberately does not promote
individual APCBlock slabs into the memory pool; it restores the prefix
as one per-layer K/V tensor set, matching what generation consumes.
NzDAPC: skipping disk prompt-cache restore (free RAM %.1f GB < %.1f GB)c              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   rû   r£  s   & r   r~   Ú6APCManager.lookup_prefix_disk_cache.<locals>.<genexpr>   ó   é € ð áW˜ô ˜—F�FÛWùr´   c              3  óJ   "  € T F  q'       g   K  \        V4      x € K  	  R # 5irz   rû   r}  s   & r   r~   rq  B  r©  rª  r¼   rS  rT  r¾   ro  rX  r%  )r  r-  r0  r¼  ré  rõ  rH   r¿   rþ   r!   rÏ   rø  r“   r9   r*  r   r¼   r˜  rŒ   r„  r¨   rµ  r�  r^  Úmake_warm_kv_cache_from_layersr,  r7  r0  r2  )rC  r¼   r¾   r  rŸ  rn  r  r^  Ún_fullr    r;  Úchunksrý  r¶  r8   Úb_memr2  rÇ  rT   r;   r“  rd  r±  r°  Ú
warm_caches   &&&&&&                   r   Úlookup_prefix_disk_cacheÚ#APCManager.lookup_prefix_disk_cacheü  sv  € ð �y‰yˆØŠ<ØˆNØ�Y�Y‹YØ×,Ñ,¨qÔ0Ü*Ó,�ØÒ'¨H×7TÑ7TÔ,TÜ—K‘Kð7à  GÕ,Ø×5Ñ5¸ÕAô	ð #÷ ŠYô ˜“^ t§¡Õ6ˆFØ Ò,Ð1BÀQÔ1FÜ˜V¤SÐ):Ó%;¸t¿¹Õ%NÓO�Ü%ˆFØ&(ˆLØ,.ˆFÜ˜6—]�ßœñ à&¨4¯?©?Õ':¸aÀ!½eÀtÇÁÕ=VÑWóŸ™ñ à&¨4¯?©?Õ':¸aÀ!½eÀtÇÁÕ=VÑWóó �ô ! °
Ó;�ð Ÿ™×+Ñ+¨AÓ.�ç,ØÒ)ØŸ™¨5Ô0à"‘N÷A ŠYðB —x‘x —{’{ÙØ×#Ñ# AÔ&Ø—‘˜eÔ$Ø“ñ' #÷*  Ø÷Q ŠYôR ! Ó.°·±Õ@ˆNØÐ!2Ô2Ø÷W ŠYðT 3÷U ðZ ×-Ñ-¨lÓ;ˆØŠ>ØˆNØ"(Ñˆˆf�iÜˆy‹>œS ›[Ô(ØˆNÜ" 6¨9Ö5‰OˆE�8ðß %¤ñ &Ø$,§L¡L°¸bÓ$A×$GÑ$GÈÔ$Ló&§¡ñ &Ø$,§L¡L°¸bÓ$A×$GÑ$GÈÔ$Ló&ó !�ô  # 8§<¡<°¸cÓ#BÓC�ð  Ô%¨¸Ö)CØ’ñ  6ô 4°D¸&À.ÓQˆ
ð
 �Y�Y‹YØ�J‰J× Ò ¤C¨Ó$5Õ5Õ Ø�J‰J�OŠO˜qÕ �OØ�J‰J×%Ò%¨Õ7Õ%÷ ð ˜>Ð)Ð)÷O �YˆYûôr œzÐ*ô Ø”ðú÷ ŽYð ˜>Ð)Ð)úsK   §A O ÂD/O ÇO Ç#2O ÈO È"$O Ê%BO4Í!A3PÏ O1	Ï4PÐPÐP"	c               ó$   € V ^8„  d   QhRRRRRR/# )r   r¼   r½   r¾   r!   r   zTuple[List[APCBlock], int]r   )r   s   "r   r   r  V  s$   € ÷ +ñ +Ø&ð+Ø47ð+à	#ñ+r   c           
     ó¸  € V P                   ;_uu_ 4        \        V4      V P                  ,          p. p\        p\	        V4       Få  p\
        ;QJ d?    . R WV P                  ,          V^,           V P                  ,            4       F  NK  	  5M8! R WV P                  ,          V^,           V P                  ,            4       4      p\        WWV4      pV P                  P                  V4      p	V	e   V	P                  V8w  d    M&VP                  V P                  V	4      4       TpKç  	  \        V4      V P                  ,          p
V
^ 8”  dH   V P                  ;P                  ^,          un        V P                  ;P                  V
,          un        M#V P                  ;P                  ^,          un        WJ3uuRRR4       #   + '       g   i     R# ; i)zíWalk the hash chain over ``token_ids``; return acquired matched
blocks and matched_token_count. Caller must release the blocks.

This memory-only path stops at the first block that is not already
present in the in-process APCBlock pool.
c              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   rû   r£  s   & r   r~   Ú+APCManager.lookup_prefix.<locals>.<genexpr>d  rr  r´   N)r-  rH   r¿   rÏ   rø  r“   r9   r*  r   r¼   rŒ   rG  r,  r0  r2  r1  )rC  r¼   r¾   ru  Úmatchedr    rý  r¶  r8   rw  r2  s   &&&        r   Úlookup_prefixÚAPCManager.lookup_prefixV  sR  € ð �Y�Y‹YÜ˜“^ t§¡Õ6ˆFØ&(ˆGÜ%ˆFÜ˜6–]�ßœñ à&¨4¯?©?Õ':¸aÀ!½eÀtÇÁÕ=VÑWóŸ™ñ à&¨4¯?©?Õ':¸aÀ!½eÀtÇÁÕ=VÑWóó �ô ! °
Ó;�ØŸ™×+Ñ+¨AÓ.�Ø’= E§O¡O°uÔ$<ÙØ—‘˜t×5Ñ5°eÓ<Ô=Ø’ñ #ô ! ›\¨D¯O©OÕ;ˆNØ Ô!Ø—
‘
—’ 1Õ$•Ø—
‘
×)Ò)¨^Õ;Ö)à—
‘
×!Ò! QÕ&Õ!ØÐ*÷- �Y�Y‹Yús   •F(GÇG	Úskip_first_n_tokensc               ó0   € V ^8„  d   QhRRRRRRRRRRR	R
/# )r   r¼   r½   r¡   ri   r£   r¾   r!   r‚  r   úList[APCBlock]r   )r   s   "r   r   r  w  sN   € ÷ Pñ Pà ðPð #ðPð %ð	Pð ðPð !ðPð 
ñPr   c               ó  € V P                   ;_uu_ 4        \        V4      V P                  ,          pWPP                  ,          pW`P                  ,          p\        ^ \        V4      V P                  ,
          4      p	\        VW�P                  ,          V P                  ,          4      p
. p. p\        V4      \        V4      ,           p\        ;QJ d    . R VRV
  4       F  NK  	  5M! R VRV
  4       4      pRpV P                  ^ 8”  Ed   V P                  ^ 8”  dò   W P                  8¼  dâ   \        VVV
4      pVeÑ   \        WäV P                  4      p\        V\        V4      V\        P                  ! 4       R7      V P                  V&   V P                  P                  V4       \        V P                  4      V P                  8”  d   V P                  P!                  RR7       KB  V P"                  ;P$                  ^,          un        Rp\&        p\)        V4       F”  p\        ;QJ d@    . R VVV P                  ,          V^,           V P                  ,            4       F  NK  	  5M9! R VVV P                  ,          V^,           V P                  ,            4       4      p\+        VVV4      pK–  	  \)        Wv4       EFL  p\        ;QJ d@    . R VVV P                  ,          V^,           V P                  ,            4       F  NK  	  5M9! R VVV P                  ,          V^,           V P                  ,            4       4      p\+        VVV4      pV P,                  e\   V P,                  P/                  V4      '       g;   VP1                  \3        \        V4      \        V4      \        V4      VVR	7      4       V'       d   TpEK  V P4                  P7                  V4      pVe9   VP8                  V8X  d(   V P;                  V4      pVP1                  V4       TpEKb  V P<                  ^ 8”  dg   V^ 8”  d`   \        V P4                  4      ^,           V,          V P<                  8”  d.   \>        PA                  R
VV4       V P,                  f    EM}TpEKÙ  V PC                  4       pVf.   \>        PA                  RVV4       V P,                  f    EM<TpEK  VV P                  ,          pVV P                  ,           pV Uu. uF  p\E        VRVV1R3,          4      NK  	  ppV Uu. uF  p\E        VRVV1R3,          4      NK  	  pp\F        PH                  ! VV,           4       VVn%        VVn&        VVn        VVn'        VVn(        VVn)        ^Vn*        VV P4                  V&   VP1                  V4       V P"                  ;PV                  ^,          un+        V P"                  ;PX                  V P                  ,          un,        TpEKO  	  V P,                  e\   V'       dT    V P,                  P[                  WÂW0P                  4       V P"                  ;P\                  \        V4      ,          un.        \c        R V Pd                   4       4      V P"                  n3        VuuRRR4       # u upi u upi   \^         d!   p\>        Pa                  RT4        Rp?LgRp?ii ; i  + '       g   i     R# ; i)z³Slice ``layer_keys`` / ``layer_values`` into block_size chunks and
store any new full blocks beyond ``skip_first_n_tokens``.

Returns newly acquired blocks (caller must release).
c              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   rû   r£  s   & r   r~   Ú-APCManager.store_kv_blocks.<locals>.<genexpr>“  s   é € ÐVÑ0U¨1¤ A§ Ó0Uùr´   NFrW  r…  Tc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   rû   r£  s   & r   r~   r‡  ¯  rr  r´   c              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   rû   r£  s   & r   r~   r‡  ¶  rr  r´   )r  r  r¾   r¼   r#  zCAPC pool tensor limit reached; skipping memory store at block %d/%dz8APC pool exhausted; skipping memory store at block %d/%d.rn   z#APC disk save scheduling failed: %sc              3  óD   "  € T F  qP                   f   K  ^x € K  	  R # 5irz   ©r  r}  s   & r   r~   r‡    ó   é € Ð&X±)¨Q¿|¹|§q¢q³)ùó   ‚ –
 )4r-  rH   r¿   rò   r/  rþ   r“   r4  r.  r«   rÄ   r  r!   ro  r+  r‡  rˆ  r,  r:  rÏ   rø  r9   r  r˜  rŒ   r"  r*  r   r¼   rG  r3  ré  ÚdebugrB  rR   rO   rž   r  r  r¾   rT   r;   r  r5  r3  r·  r8  rÍ   rê  Úsumr&  r6  ) rC  r¼   r¡   r£   r¾   r‚  ru  Ú	skip_fullÚfull_prefix_tokensÚguarded_prefix_tokensÚlayer_major_prefix_tokensÚ
new_blocksÚdisk_blocksÚper_block_tensorsr«  Úlayer_major_storedr˜   rt  r    rý  r¶  r8   ÚexistingÚacquiredrù  rè   rï   r©   Úk_slabsrª   Úv_slabsr7  s    &&&&$$                          r   Ústore_kv_blocksÚAPCManager.store_kv_blocksw  s¾  € ð �Y�Y‹YÜ˜“^ t§¡Õ6ˆFØ+¯©Õ>ˆIØ!'¯/©/Õ!9ÐÜ$'Ø”3�y“> D×$AÑ$AÕAó%Ð!ô ),Ø"Ø&¯/©/Õ9¸T¿_¹_ÕLó)Ð%ð *,ˆJØ68ˆKÜ # J£´#°lÓ2CÕ CÐßœ%ÑV°	Ð:TÐ;TÑ0UÓVŸ%™%ÑV°	Ð:TÐ;TÑ0UÓVÓVˆKØ!&Ðà×3Ñ3°aÕ7Ø×)Ñ)¨AÔ-Ø-×1TÑ1TÔTä<ØØ Ø-ó�ð
 Ò%Ü(¨À$Ç/Á/ÓR�CÜ-?Ø"-Ü#& z£?Ø%+Ü"&§)¢)£+ô	.�D×%Ñ% cÑ*ð ×%Ñ%×1Ñ1°#Ô6Ü˜d×/Ñ/Ó0°4×3HÑ3HÔHØ×)Ñ)×1Ñ1°uÐ1Ö=Ø—J‘J×+Ò+¨qÕ0Õ+Ø)-Ð&Ü%ˆFä˜9Ö%�ßœñ à& q¨4¯?©?Õ':¸aÀ!½eÀtÇÁÕ=VÑWóŸ™ñ à& q¨4¯?©?Õ':¸aÀ!½eÀtÇÁÕ=VÑWóó �ô & f¨e°ZÓ@’ñ &ô ˜9×-�ßœñ à& q¨4¯?©?Õ':¸aÀ!½eÀtÇÁÕ=VÑWóŸ™ñ à& q¨4¯?©?Õ':¸aÀ!½eÀtÇÁÕ=VÑWóó �ô ! ¨°
Ó;�Ø—9‘9Ò(°·±·±¸q×1AÒ1AØ×&Ñ&Ü,Ü'*¨1£vÜ(+¨F«Ü'*¨:£Ø&+Ø-.ôô÷ &Ø�FÚØŸ?™?×.Ñ.¨qÓ1�ØÒ'¨H×,>Ñ,>À%Ô,GØ#×5Ñ5°hÓ?�HØ×%Ñ% hÔ/Ø�FÚà×*Ñ*¨QÔ.Ø)¨AÔ-Ü˜TŸ_™_Ó-°Õ1Ð5FÕFØ×,Ñ,ô-ô —L‘Lð)àØô	ð —y‘yÒ(ÚØ�FÚØ—O‘OÓ%�Ø’9Ü—L‘LØRØØôð
 —y‘yÒ(ÚØ�FÚØ˜DŸO™OÕ+�Ø˜dŸo™oÕ-�ñ
 KUÓUÉ*ÀQœ?¨1¨S°%¸°)¸QÐ->Õ+?Ö@É*�ÐUÙJVÓWÉ,ÀQœ?¨1¨S°%¸°)¸QÐ->Õ+?Ö@É,�ÐWÜ—’˜ 'Õ)Ô*Ø �”Ø &�”Ø#�”Ø)�”Ø �”Ø"�”Ø�”	Ø%&�—‘ Ñ"Ø×!Ñ! !Ô$Ø—
‘
×!Ò! QÕ&Õ!Ø—
‘
×(Ò(¨D¯O©OÕ;Õ(Ø“ñQ .ðR �y‰yÒ$¯ðMØ—I‘I×5Ñ5Ø#°¿¹ôð —J‘J×*Ò*¬c°+Ó.>Õ>Õ*ô $'Ñ&X°$·)²)Ó&XÓ#XˆD�J‰JÔ Ø÷E ŠYùòT VùÚWøô( !ô MÜ—N‘NÐ#HÈ!×LÑLûðMú÷ �YˆYúsd   •M2[-ÎA[-Ï
D>[-Ô Z5
Ô([-Ô. Z:
ÕC[-Ø,AZ?Ù>,[-Ú5
[-Ú?[*Û
[%Û [-Û%[*Û*[-Û-[>	c               ó   € V ^8„  d   QhRR/# )r   r   r=  r   )r   s   "r   r   r  	  s   € ÷ ñ  ñ r   c           	     	ó  a € S P                   ;_uu_ 4        \        R  S P                   4       4      S P                  n        S P                  P                  S P                  S P                  4      pS P                  eÑ   S P                  P                  VR&   S P                  P                  VR&   S P                  P                  VR&    \        V 3R lS P                  P                  P                  RS P                  P                   24       4       4      VR&   S P                  P                   VR&   S P                  P"                  VR	&   VuuRRR4       #   \         d	    R
TR&    LPi ; i  + '       g   i     R# ; i)c              3  óD   "  € T F  qP                   f   K  ^x € K  	  R # 5irz   r‹  r}  s   & r   r~   Ú,APCManager.stats_snapshot.<locals>.<genexpr>  rŒ  r�  Nr\  Údisk_max_bytesÚdisk_evictionsc              3  ón   <"  € T F*  pSP                   P                  V4      '       g   K&  ^x € K,  	  R# 5irK  )r  r  )r|   rO  rC  s   & €r   r~   r¡    s,   øé € ð -á!K˜AØŸ9™9×=Ñ=¸a×@÷ šÛ!Kùs   ƒ#5«
5rJ  Ú
disk_filesÚdisk_blocks_indexedÚdisk_exact_indexedrn  )r-  r�  r&  r,  r6  rF  r<  r¿   r  r\  rÅ  r4  rÙ  rK  r   r]  r`  re  )rC  Úsnaps   f r   Ústats_snapshotÚAPCManager.stats_snapshot	  s.  ø€ Ø�Y�Y‹YÜ#&Ñ&X°$·)²)Ó&XÓ#XˆD�J‰JÔ Ø—:‘:×&Ñ& t§¡¸¿¹ÓHˆDØ�y‰yÒ$Ø%)§Y¡Y×%9Ñ%9��\Ñ"Ø)-¯©×)<Ñ)<�Ð%Ñ&Ø)-¯©×)<Ñ)<�Ð%Ñ&ð,Ü),ô -à!%§¡§¡×!3Ñ!3°a¸¿	¹	×8HÑ8HÐ7IÐ4JÔ!Kó-ó *�D˜Ñ&ð /3¯i©i×.JÑ.J�Ð*Ñ+Ø-1¯Y©Y×-HÑ-H�Ð)Ñ*Ø÷% ŠYøô ô ,Ø)+�D˜Ó&ð,ú÷ �YˆYús1   –B5E2ÃAEÄ3E2ÅE/Å,E2Å.E/Å/E2Å2F	c               ó   € V ^8„  d   QhRR/# r-  r   )r   s   "r   r   r    s   € ÷ $ñ $˜Tñ $r   c                	ó†   € V P                   ;_uu_ 4        \        4       V n        R R R 4       R #   + '       g   i     R # ; irz   )r-  r/  r,  r0  s   &r   Úreset_statsÚAPCManager.reset_stats  s    € Ø�Y�Y‹YÜ!›ˆDŒJ÷ �Y�YŠYús	   •/¯A 	c               ó   € V ^8„  d   QhRR/# r-  r   )r   s   "r   r   r  "  s   € ÷ $ñ $�tñ $r   c                	óþ  € V P                   ;_uu_ 4        V P                   FF  pR Vn        RVn        \        Vn        ^ Vn        R Vn        R Vn        ^ Vn	        R ;Vn
        Vn        KH  	  V P                  P                  4        R ;V n        V n        V P                   F  pV P!                  V4       K  	  V P"                  P                  4        \%        4       V n        R R R 4       R #   + '       g   i     R # ; irA  )r-  r&  r  r¼   rÏ   r  r¾   rT   r;   r  r  r  r*  r/  r'  r(  r)  r+  r/  r,  r9  s   & r   r/  ÚAPCManager.clear"  s¹   € Ø�Y�Y‹YØ—Y”Y�Ø#�”Ø �”Ü 0�”Ø �”Ø�”Ø�”Ø�”	Ø"&Ð&�”˜žñ ð �O‰O×!Ñ!Ô#Ø04Ð4ˆDŒO˜dœoØ—Y”Y�Ø—‘ Ö"ñ à×Ñ×#Ñ#Ô%Ü!›ˆDŒJ÷ �Y�YŠYús   •CC+Ã+C<	c               ó   € V ^8„  d   QhRR/# r-  r   )r   s   "r   r   r  4  s   € ÷ ñ �tñ r   c                óZ   € V P                   e   V P                   P                  4        R# R# )z3Best-effort shutdown: close the disk writer thread.N)r  r  r0  s   &r   r  ÚAPCManager.close4  s   € à�9‰9Ò Ø�I‰I�O‰OÖñ !r   )r1  r2  r0  r+  r.  r'  r(  r4  r3  r¿   r  r/  r*  r-  r<  r&  r,  )r¯   Nr¯   )r¯   Nr¯   FrF  )r  r  r  r  r  ÚDEFAULT_NUM_BLOCKSÚDEFAULT_BLOCK_SIZErþ  r)  r=  rB  rG  rL  rP  rd  rk  ry  r€  rœ  r©  r­  r/  r  r  r   r   r   r  r  Æ
  s�   † Ù<ð -Ø,Ø+/÷	D
õN"õ	õõ õõ%÷H(ðT(ð
 ÷(÷TX*÷t+ðBPð ðPð $%÷Põdõ*$õ$÷$ñ r   r  c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úmatched_blocksr„  rV   rW   r   r  r   )r   s   "r   r   r   :  s&   € ÷ *ñ *Ø"ð*à&ð*ð ñ*r   c                óœ  € ^ RI Hp V '       g   . # \        V ^ ,          P                  4      p. p\	        R V  4       4      pV! 4       p\        \        VR\        VRR4      4      4      pTpVe@   \        V\        V4      4      pW…8”  d%   V^ 8”  d   W‡,           ^,
          V,          V,          p\        V4       EF„  p	V  U
u. uF  qªP                  V	,          NK  	  pp
V  U
u. uF  qªP                  V	,          NK  	  pp
\        P                  ! V^R7      p\        P                  ! V^R7      pW…8”  dÓ   W…,
          p. VP                  R,          OVNVP                  ^,          N5p. VP                  R,          OVNVP                  ^,          N5p\        P                  ! V\        P                  ! VVP                  R7      .^R7      p\        P                  ! V\        P                  ! VVP                  R7      .^R7      pV	^ 8X  d   TMV! 4       pVVn        VVn	        VVn        VP!                  V4       EK‡  	  V# u up
i u up
i )zžStitch matched blocks into per-layer ``KVCache`` instances pre-filled
with the cached prefix's K/V state. Used by the single-stream
``stream_generate`` path.
r¦   c              3  óh   "  € T F(  qP                   ^ ,          P                  R,          x € K*  	  R# 5i©r¯   Néþÿÿÿ©rT   r^   r  s   & r   r~   Ú%make_warm_kv_cache.<locals>.<genexpr>H  ó"   é € ÐA±.¨Q—V‘V˜A•Y—_‘_ R×(Ò(³.ùó   ‚02rX   ro   r\   r[   r'   )r§   r…   rH   rT   r�  r!   r‡   rò   rø  r;   rO   r_   r^   r`   r(   rU   rŒ   )r¸  rV   r…   r  r•   r¤   Ú
step_prober^  ra   r/  rù  ÚksÚvsÚmerged_kÚmerged_vrb   rË  rÌ  rg   s   &&                 r   Úmake_warm_kv_cacherÆ  :  sñ  € õ ,çØˆ	Ü�^ AÕ&×+Ñ+Ó,€JØ€CÜÑA±.ÓAÓA€JÙ“€JÜ”'˜* f¬g°g¸vÀsÓ.KÓLÓM€GØ€HØÒ&Ü�z¤3Ð':Ó#;Ó<ˆØÔ  W¨q¤[Ø!Õ+¨aÕ/°GÕ;¸wÕFˆHÜ˜:×&ˆ	Ù)7Ó8© A�f‰f�Y×Ð©ˆÐ8Ù+9Ó:©> a�h‰h�y×!Ð!©>ˆÐ:Ü—>’> "¨1Ô-ˆÜ—>’> "¨1Ô-ˆØÔ Ø!Õ.ˆJØN˜HŸN™N¨2Õ.ÐN°
ÐN¸H¿N¹NÈ1Õ<MÑNˆKØN˜HŸN™N¨2Õ.ÐN°
ÐN¸H¿N¹NÈ1Õ<MÑNˆKÜ—~’~Øœ2Ÿ8š8 K°x·~±~ÔFÐGÈaôˆHô —~’~Øœ2Ÿ8š8 K°x·~±~ÔFÐGÈaôˆHð $ qœ.‰J©g«iˆØˆŒØˆŒØˆŒØ�
‰
�1�ñ' 'ð( €Jùò' 9ùÚ:s   ÃIÃ"I	c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   r¡   ri   r£   r¤   r!   r   r  r   )r   s   "r   r   r   g  s0   € ÷ ñ Øðà ðð ðð ñ	r   c                ó¼   € ^ RI Hp . p\        W4       F/  w  rVV! 4       pWWn        Wgn        W'n        VP                  V4       K1  	  \        P                  ! 4        V# )zFBuild ``KVCache`` objects from already-concatenated disk-restored K/V.r¦   )	r§   r…   r¨   rT   r;   rU   rŒ   rO   r)  )r¡   r£   r¤   r…   r•   r©   rª   rg   s   &&&     r   rt  rt  g  sO   € õ ,à€CÜ�JÖ-‰ˆÙ‹IˆØŒØŒØŒØ�
‰
�1Žñ .ô ‡N‚NÔØ€Jr   c               ó    € V ^8„  d   QhRRRR/# )r   r¸  r„  r   r  r   )r   s   "r   r   r   z  s   € ÷ ñ Ø"ðàñr   c                óB  € ^ RI Hp V '       g   . # \        V ^ ,          P                  4      p\	        R V  4       4      p. p\        V4       FÅ  pV  Uu. uF  qfP                  V,          NK  	  ppV  Uu. uF  qfP                  V,          NK  	  pp\        P                  ! V^R7      p	\        P                  ! V^R7      p
V! ^ .R7      pV	V
\        P                  ! V.4      \        P                  ! ^ .4      3Vn
        VP                  V4       KÇ  	  V# u upi u upi )zîStitch matched blocks into per-layer single-row ``BatchKVCache``
instances pre-filled with the cached prefix's K/V state. Used by the
batched continuous-batching path; the resulting cache list can be
``extend()``-ed into a running batch.
©ÚBatchKVCachec              3  óh   "  € T F(  qP                   ^ ,          P                  R,          x € K*  	  R# 5ir»  r½  r  s   & r   r~   Ú+make_warm_batch_kv_cache.<locals>.<genexpr>‡  r¿  rÀ  r\   ©r�   )r§   rÌ  rH   rT   r�  rø  r;   rO   r_   rQ   r—   rŒ   )r¸  rÌ  r  r¤   r•   r/  rù  rÂ  rÃ  rÄ  rÅ  rg   s   &           r   Úmake_warm_batch_kv_cacherÐ  z  sø   € õ 1çØˆ	Ü�^ AÕ&×+Ñ+Ó,€JÜÑA±.ÓAÓA€JØ€CÜ˜:Ö&ˆ	Ù)7Ó8© A�f‰f�Y×Ð©ˆÐ8Ù+9Ó:©> a�h‰h�y×!Ð!©>ˆÐ:Ü—>’> "¨1Ô-ˆÜ—>’> "¨1Ô-ˆÙ q cÔ*ˆð ØÜ�HŠH�j�\Ó"Ü�HŠH�a�S‹Mð	
ˆŒð 	�
‰
�1Žñ 'ð €Jùò 9ùÚ:s   ÁDÁ4Dc               ó$   € V ^8„  d   QhRRRRRR/# )r   ÚpickszList[Optional[dict]]r  r!   r   zTuple[List[Any], int]r   )r   s   "r   r   r   š  s+   € ÷ Qñ QØðQàðQð ñQr   c           
     ó0  € ^ RI Hp \        V 4      pV  Uu. uF  qD'       d
   VR,          M^ NK  	  ppV'       d   \        V4      M^ pV^ 8X  d   . ^ 3# R R lp\	        R V  4       4      pV! V^ 4      w  ršV	P
                  ^,          pV	P
                  R	,          pV	P                  p. p\        V4       EFÌ  p. p. pV  EF  pVfW   VP                  \        P                  ! ^W¶V3VR7      4       VP                  \        P                  ! ^W¶V3VR7      4       K^  V! VV4      w  ppVVR,          ,
          pV^ 8”  dm   \        P                  ! ^VVV3VR7      p\        P                  ! ^VVV3VR7      p\        P                  ! VV.^R7      p\        P                  ! VV.^R7      pVP                  V4       VP                  V4       EK  	  \        P                  ! V^ R7      p\        P                  ! V^ R7      pV Uu. uF  pVV,
          NK  	  ppV Uu. uF  pVNK  	  ppV! ^ .V,          R7      pVV\        P                  ! V4      \        P                  ! V4      3Vn        VP                  V4       EKÏ  	  Wæ3# u upi u upi u upi )
a  Build a multi-row ``BatchKVCache`` list for mixed warm / cold prefill.

``picks`` is per-row, with each entry being ``None`` (cold) or a dict
with key ``matched_blocks`` (list of APCBlock) and ``prefix_len``.

Returns ``(cache_list, max_prefix)`` where ``max_prefix`` is the cache's
``_idx`` after warm-init (= max prefix_len across rows).

For row ``i``:
  * left_padding[i] = max_prefix - prefix_len[i]
  * keys[i, :, left_padding[i]:max_prefix, :] = concatenated block K
  * keys[i, :, :left_padding[i], :] = zeros (will be hidden by mask)
rË  r¤   c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úpickr=  r/  r!   r   rY   r   )r   s   "r   r   Ú4make_warm_batch_kv_cache_multi.<locals>.__annotate__³  s'   € ÷ 	Fñ 	F˜Dð 	F¨Sð 	FÐ5Nñ 	Fr   c                ó¸  € V P                  R 4      pVeB   W!,          pV R,          pVP                  RRV1R3,          VP                  RRV1R3,          3# V R,          pV Uu. uF  qfP                  V,          NK  	  ppV Uu. uF  qfP                  V,          NK  	  pp\        P                  ! V^R7      \        P                  ! V^R7      3# u upi u upi )rx  Nr¤   .rn   r¸  r\   )r   rT   r;   rO   r_   )	rÕ  r/  rx  rg   r¤   r'  rù  rÂ  rÃ  s	   &&       r   Úlayer_tensorsÚ5make_warm_batch_kv_cache_multi.<locals>.layer_tensors³  sË   € Ø—X‘X˜lÓ+ˆ
ØÒ!ØÕ%ˆAØ˜lÕ+ˆJØ—6‘6˜#˜{ 
˜{¨AÐ-Õ.°·±¸¸k¸z¸kÈ1Ð9LÕ0MÐMÐMØÐ&Õ'ˆÙ)/Ó0© A�f‰f�Y×Ð©ˆÐ0Ù+1Ó2©6 a�h‰h�y×!Ð!©6ˆÐ2Ü�~Š~˜b qÔ)¬2¯>ª>¸"À1Ô+EÐEÐEùò 1ùÚ2s   Á$CÂCc              3  ó8   "  € T F  q'       g   K  Vx € K  	  R # 5irz   r   )r|   rO  s   & r   r~   Ú1make_warm_batch_kv_cache_multi.<locals>.<genexpr>¿  s   é € Ð(™U˜£a—!’!›Uùs   ‚	�
r'   r\   rÏ  rn  )r§   rÌ  rH   rò   r  r^   r(   rø  rŒ   rO   r`   r_   rQ   r—   )rÒ  r  rÌ  ÚBrO  Úprefix_lensÚ
max_prefixrØ  ÚsampleÚsample_krK  Ú
n_kv_headsÚhead_dimr(   r•   r/  Úrow_keysÚ
row_valuesrÕ  Úwarm_kÚwarm_vÚlpÚpad_kÚpad_vrÄ  rÅ  Úplr�   rU   rg   s   &&                            r   Úmake_warm_batch_kv_cache_multirë  š  s[  € õ" 1äˆE‹
€AÙ8=Ó>¹°1¦a�1�\–?¨QÒ.¹€KÐ>ß%0”�[Ô!°a€JØ�Q„Ø�1ˆuˆõ	Fô Ñ(™UÓ(Ó(€FÙ ¨Ó*�K€HØ—‘ Õ"€JØ�~‰~˜bÕ!€HØ�N‰N€Eà€CÜ˜:×&ˆ	ð $&ˆØ%'ˆ
ÜˆDØŠ|à—‘Ü—H’H˜a ¸ÐBÈ%ÔPôð ×!Ñ!Ü—H’H˜a ¸ÐBÈ%ÔPôñ Ù*¨4°Ó;‰NˆF�FØ˜d <Õ0Õ0ˆBØ�AŒvÜŸš ! Z°°XÐ!>ÀeÔL�ÜŸš ! Z°°XÐ!>ÀeÔL�ÜŸš¨°¨¸aÔ@�ÜŸš¨°¨¸aÔ@�Ø�O‰O˜FÔ#Ø×Ñ˜f×%ñ% ô& —>’> (°Ô3ˆÜ—>’> *°1Ô5ˆá2=Ó>±+¨B˜
 RŸ˜±+ˆÐ>Ù*Ó+™{˜“"™{ˆÐ+Ù q c¨A¥gÔ.ˆàØÜ�HŠH�VÓÜ�HŠH�\Ó"ð	
ˆŒð 	�
‰
�1�ñI 'ðJ ˆ?Ðùò{ ?ùòf ?ùÚ+s   –J	ÈJÈJc               ó$   € V ^8„  d   QhRRRRRR/# )r   rL   r   r•   ri   r   r$  r   )r   s   "r   r   r   î  s!   € ÷ *ñ *˜#ð * Nð *°tñ *r   c                óÐ   € \        V \        P                  4      '       d   VP                  V 4       R # \        V \        \
        34      '       d   V  F  p\        W!4       K  	  R # R # rz   )r„   rO   rQ   rŒ   rÎ   r“   Ú_collect_mx_arrays)rL   r•   r  s   && r   rî  rî  î  sF   € Ü�!”R—X‘X×ÒØ�
‰
�1ŽÜ	�Aœœe�}×	%Ò	%ÛˆDÜ˜tÖ)ó ñ 
&r   c               ó$   € V ^8„  d   QhRRRRRR/# ©r   r¬  r›   rÝ  r½   r   r   r   )r   s   "r   r   r   ö  s&   € ÷ ñ Øðàðð 	ñr   c           	     ór  € ^ RI Hp \        V ^ ,          P                  4      pVP                  V4      p. p\	        V4       Fç  pV  Uu. uF  qwP                  V,          NK  	  pp\        R V 4       R4      p	V	f   VP                  R4       KN  . p
V Fj  pVfL   V
P                  \        P                  ! RV	P                  R,          ,           V	P                  R7      4       KR  V
P                  VR,          4       Kl  	  VP                  \        P                  ! V
^ R7      4       Ké  	  WTn        V# u upi )	r¯   rl   c              3  ó0   "  € T F  qf   K  Vx € K  	  R # 5irz   r   )r|   rf  s   & r   r~   Ú._merge_arrays_cache_entries.<locals>.<genexpr>  s   é € Ð:¡&˜Q—q’q£&ùs   ‚Œ
Nr›  r'   :Nr=   Nr\   rk  )rƒ   rm   rH   r‹   rø  r  rŒ   rO   r`   r^   r(   r_   )r¬  rÝ  r”   rÁ   r•   Úmerged_statesÚ	state_idxry  rã  rß  Úrowsr—   s   &&          r   Ú_merge_arrays_cache_entriesr÷  ö  sý   € õ 0äˆw�q�z×ÑÓ €DØ
×
Ñ
˜tÓ
$€CØ.0€MÜ˜4–[ˆ	Ù6=Ó>±g¨U—+‘+˜i×(Ð(±gˆÐ>ÜÑ:¡&Ó:¸DÓAˆØŠ>Ø× Ñ  Ô&ÙØˆÛˆEØŠ}Ø—‘œBŸHšH T¨F¯L©L¸Õ,<Õ%<ÀFÇLÁLÔQÖRà—‘˜E "�IÖ&ñ	 ð
 	×ÑœRŸ^š^¨D°qÔ9Ö:ñ !ð „IØ€Jùò ?s   ÁD4c               ó$   € V ^8„  d   QhRRRRRR/# rð  r   )r   s   "r   r   r     s&   € ÷ 'ñ 'Øð'àð'ð 	ñ'r   c                ó~  a€ ^ RI Ho V '       g   R# V ^ ,          p\        ;QJ d    V3R lV  4       F  '       d   K   RM	  RM! V3R lV  4       4      '       d   SP                  P	                  V 4      # \        ;QJ d    V3R lV  4       F  '       d   K   RM	  RM! V3R lV  4       4      '       d   SP                  P	                  V 4      # \        ;QJ d    V3R lV  4       F  '       d   K   RM	  RM! V3R lV  4       4      '       d   SP
                  P	                  V 4      # \        ;QJ d    V3R lV  4       F  '       d   K   RM	  RM! V3R lV  4       4      '       d   \        W4      # \        ;QJ d    V3R	 lV  4       F  '       d   K   RM	  RM! V3R	 lV  4       4      '       d¦   \        \        VP                  4      4       UUu. uF/  p\        V  Uu. uF  qDP                  V,          NK  	  upV4      NK1  	  ppp\        ;QJ d    R
 V 4       F  '       g   K   RM	  RM! R
 V 4       4      '       d   R# SP                  ! V!  # \        ;QJ d    R V  4       F  '       d   K   RM	  RM! R V  4       4      '       d’   \        \        V4      4       UUu. uF%  p\        V  Uu. uF  qDV,          NK  	  upV4      NK'  	  ppp\        ;QJ d    R V 4       F  '       g   K   RM	  RM! R V 4       4      '       d   R# SP                  ! V!  # R# u upi u uppi u upi u uppi )r¯   rl   Nc              3  óP   <"  € T F  p\        VSP                  4      x € K  	  R # 5irz   )r„   r…   ©r|   rg   r”   s   & €r   r~   Ú-_merge_exact_cache_entries.<locals>.<genexpr>  s!   øé € Ð
<±G¨qŒ:�a˜×)Ñ)×*Ð*³Gùó   ƒ#&FTc              3  óP   <"  € T F  p\        VSP                  4      x € K  	  R # 5irz   )r„   rŠ   rû  s   & €r   r~   rü    s!   øé € Ð
C¹7°aŒ:�a˜×0Ñ0×1Ð1»7ùrý  c              3  óP   <"  € T F  p\        VSP                  4      x € K  	  R # 5irz   )r„   r‰   rû  s   & €r   r~   rü    s!   øé € Ð
D¹G°qŒ:�a˜×1Ñ1×2Ð2»Gùrý  c              3  óP   <"  € T F  p\        VSP                  4      x € K  	  R # 5irz   )r„   r‹   rû  s   & €r   r~   rü    s!   øé € Ð
@¹°1Œ:�a˜×-Ñ-×.Ð.»ùrý  c              3  óP   <"  € T F  p\        VSP                  4      x € K  	  R # 5irz   )r„   r�   rû  s   & €r   r~   rü  !  s!   øé € Ð
>±g°Œ:�a˜×+Ñ+×,Ð,³gùrý  c              3  ó(   "  € T F  qR J x € K
  	  R # 5irz   r   r  s   & r   r~   rü  )  ó   é € Ð)¡&˜Q�D�y£&ùr�   c              3  óB   "  € T F  p\        V\        4      x € K  	  R # 5irz   )r„   r“   r  s   & r   r~   rü  ,  s   é € Ð
1© AŒ:�aœ×Ð«ùs   ‚c              3  ó(   "  € T F  qR J x € K
  	  R # 5irz   r   r  s   & r   r~   rü  4  r  r�   )rƒ   rm   r¶   rÌ  ÚmergeÚBatchRotatingKVCacher÷  rø  rH   r�   Ú_merge_exact_cache_entriesr’   r�   )r¬  rÝ  Úfirstrý  ry  Úmergedr”   s   &&    @r   r  r    sU  ø€ õ 0çÙØ�A�J€Eß
ƒsÔ
<±GÓ
<‡s‡s‚sÔ
<±GÓ
<×<Ò<Ø×$Ñ$×*Ñ*¨7Ó3Ð3ß
ƒsÔ
C¹7Ó
C‡s‡s‚sÔ
C¹7Ó
C×CÒCØ×$Ñ$×*Ñ*¨7Ó3Ð3ß
ƒsÔ
D¹GÓ
D‡s‡s‚sÔ
D¹GÓ
D×DÒDØ×,Ñ,×2Ñ2°7Ó;Ð;ß
ƒsÔ
@¹Ó
@‡s‡s‚sÔ
@¹Ó
@×@Ò@Ü*¨7Ó@Ð@ß
ƒsÔ
>±gÓ
>‡s‡s‚sÔ
>±gÓ
>×>Ò>ô œ3˜uŸ|™|Ó,Ô-ô
ñ
 .�ô	 'Ù.5Ó6©g U—‘˜a—�©gÑ6Øöñ .ð 	ñ 
÷ ‹3Ñ)¡&Ó)�3�3Š3Ñ)¡&Ó)×)Ò)ÙØ×!Ò! 6Ñ*Ð*ß
ƒsÑ
1©Ó
1‡s‡s‚sÑ
1©Ó
1×1Ò1ô œ3˜u›:Ô&ô
ñ
 '�ô	 'Ù'.Ó/¡w˜e�q—�¡wÑ/Øöñ 'ð 	ñ 
÷ ‹3Ñ)¡&Ó)�3�3Š3Ñ)¡&Ó)×)Ò)ÙØ×!Ò! 6Ñ*Ð*Ùùò' 7ùó
ùò 0ùó
s0   ÇL.Ç&L)ÈL.Ê-L9Ê;L4ËL9Ì)L.Ì4L9c               ó$   € V ^8„  d   QhRRRRRR/# )r   Ú
row_cacheszSequence[Sequence[Any]]rÝ  r½   r   rS  r   )r   s   "r   r   r   :  s&   € ÷ 7ñ 7Ø'ð7àð7ð %ñ7r   c                óN  a€ V '       g   . ^ 3# \        V 4      \        V4      8w  d   R# \        V ^ ,          4      o\        ;QJ d    V3R lV  4       F  '       g   K   RM	  RM! V3R lV  4       4      '       d   R# . p\        S4       F>  p\        V  Uu. uF  qDV,          NK  	  upV4      pVf   Ru # VP	                  V4       K@  	  . pV F  p\        VP                  V4       K  	  V'       d   \        P                  ! V4       Y!'       d   \        V4      3# ^ 3# u upi )z?Merge single-row exact-cache snapshots into batch-aware caches.c              3  ó@   <"  € T F  p\        V4      S8g  x € K  	  R # 5irz   )rH   )r|   ÚrowrÃ  s   & €r   r~   Ú4make_warm_batch_exact_cache_multi.<locals>.<genexpr>E  s   øé € Ð
9©j sŒ3ˆs‹8�{Ö"«jùrý   TFrX  )
rH   r’   rø  r  rŒ   rî  r—   rO   rž   rò   )	r  rÝ  r•   Ú	entry_idxr  r
  rh   rg   rÃ  s	   &&      @r   Ú!make_warm_batch_exact_cache_multir  :  s÷   ø€ ÷ Ø�1ˆuˆÜ
ˆ:ƒœ#˜kÓ*Ô*ØˆÜ�j •mÓ$€Kß
ƒsÔ
9©jÓ
9‡s‡s‚sÔ
9©jÓ
9×9Ò9Øˆà€CÜ˜;Ö'ˆ	Ü+Ù'1Ó2¡z ��^ˆ^¡zÑ2Øó
ˆð Š>ØŠNØ�
‰
�6Öñ (ð $&€LÛˆÜ˜1Ÿ7™7 LÖ1ñ çÜ
�Š�ÔØ¦K”�KÓ Ð6Ð6°QÐ6Ð6ùò 3s   ÂD"
c               ó$   € V ^8„  d   QhRRRRRR/# )r   Úbatch_cachesr›   Ú	batch_idxr!   r   rœ   r   )r   s   "r   r   r   Z  s&   € ÷ ñ Øðàðð ñr   c                óü   € . p. pV  FS  p\        VRR4      p\        V4      '       g    R# V! V4      pVP                  V4       \        VP                  V4       KU  	  V'       d   \
        P                  ! V4       V# )zDExtract one row from batch-aware caches as single-row cache objects.ÚextractN)r‡   ÚcallablerŒ   rî  r—   rO   rž   )r  r  r•   rh   rg   r  Ú	extracteds   &&     r   Úextract_prompt_cache_from_batchr  Z  sn   € ð €CØ#%€LÛˆÜ˜!˜Y¨Ó-ˆÜ˜× Ò ÚÙ˜IÓ&ˆ	Ø�
‰
�9ÔÜ˜9Ÿ?™?¨LÖ9ñ ÷ Ü
�Š�ÔØ€Jr   r¾   r‚  c               ó4   € V ^8„  d   QhRRRRRRRRR	RR
RRR/# )r   Úapc_managerz'APCManager'r  r  r  r!   Úfull_token_idsr½   r¾   r‚  r   r„  r   )r   s   "r   r   r   n  sN   € ÷ (ñ (Øð(àð(ð ð(ð "ð	(ð ð(ð ð(ð ñ(r   c               óØ  € . p. pV F¸  p\        VRR4      p	\        VRR4      p
\        VRR4      p\        VRR4      pV	e	   V
e   Vf   . u # Ve"    \        WÂ,          P                  4       4      pM^ pVP	                  W’V^,           1RWÛ1R3,          4       VP	                  W¢V^,           1RWÛ1R3,          4       Kº  	  V P                  VVVVVR7      #   \         d    ^ p Lsi ; i)z Slice one row out of a batched KV cache and store its full blocks.

Used at the end of prompt prefill in continuous-batching mode to add
the new prefix to APC.
rT   Nr;   rt   r�   rn   )r¾   r‚  )r‡   r!   r  rÍ   rŒ   rœ  )r  r  r  r  r¾   r‚  r¡   r£   rg   rT   r;   ré   r�   rç  s   &&&&$$        r   Úharvest_blocks_from_batch_cacher  n  s  € ð "$€JØ#%€LÛˆÜ�q˜& $Ó'ˆÜ˜˜H dÓ+ˆÜ�a˜ Ó&ˆÜ˜q .°$Ó7ˆØŠ<˜6š>¨Sª[ØŠIàÒ#ðÜ˜Õ0×5Ñ5Ó7Ó8‘ð ˆBà×Ñ˜$¨9°q­=Ð8¸!¸R¸VÀQÐFÕGÔHØ×Ñ˜F¨y¸1­}Ð#<¸aÀÀÈÐ#JÕKÖLñ# ð$ ×&Ñ&ØØØØØ/ð 'ó ð øô ô Ø’ðús   ÁCÃC)Ã(C)c               ó    € V ^8„  d   QhRRRR/# )r   Úlanguage_modelr   r   rB   r   )r   s   "r   r   r   ™  s   € ÷ ñ  3ð ¨=ñ r   c                ó„  € \        V R4      '       g   R#  V P                  4       pT'       d=   \        ;QJ d    R T 4       F  '       d   K   RM	  RM! R T 4       4      '       d   R# T'       d=   \        ;QJ d    R T 4       F  '       d   K   RM	  RM! R T 4       4      '       d   R# R#   \         d     R# i ; i)	a7  Return the APC strategy supported by ``language_model``.

``"block"`` is the normal block-level KV path. ``"exact"`` is a
conservative whole-prefix snapshot path for custom mixed cache layouts
such as hybrid SSM/attention models, where recurrent state cannot be
reconstructed by concatenating K/V blocks alone.
Ú
make_cacheÚblockNc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   )rº   r  s   & r   r~   Ú!model_apc_mode.<locals>.<genexpr>§  ó   é € ÐUÉÀ1Ô;¸A×>Ð>Ëùr´   FTc              3  ó8   "  € T F  p\        V4      x € K  	  R # 5irz   r±   r  s   & r   r~   r&  ©  r'  r´   Úexact)Úhasattrr#  rÍ   r¶   )r!  rš   s   & r   Úmodel_apc_moder+  ™  sŒ   € ô �> <×0Ò0ÙðØ%×0Ñ0Ó2ˆ÷ Ÿ›ÑUÉÓUŸŸšÑUÉÓU×UÒUÙßŸ›ÑUÉÓUŸŸšÑUÉÓU×UÒUÙÙøô ô Úðús   –B0 Â0B?Â>B?c               ó    € V ^8„  d   QhRRRR/# )r   r!  r   r   r   r   )r   s   "r   r   r   ®  s   € ÷ 6ñ 6 sð 6¨tñ 6r   c                ó   € \        V 4      R J# rz   )r+  )r!  s   &r   Úmodel_supports_apcr.  ®  s   € Ü˜.Ó)°Ð5Ð5r   c               ó    € V ^8„  d   QhRRRR/# )r   Úmodel_namespacerB   r   zOptional[APCManager]r   )r   s   "r   r   r   ²  s   € ÷ ,Oñ ,O˜mð ,OÐ7Kñ ,Or   c           	     ó*  € \         P                  P                  RR4      R9  d   R# \        \         P                  P                  R\        4      4      p\        \         P                  P                  R\
        4      4      pRp\         P                  P                  R4      pV'       dü   T ;'       g!    \         P                  P                  RR	4      p\        \         P                  P                  R
^ 4      4      pV^ 8”  d   \        VR,          4      MRp\        \         P                  P                  RR4      4      p \        \        V4      P                  4       VVVR7      pV'       d   VR R2MRp	\        P                  RVP                  VV	VP                  4       \        P                  RTT\!        4       '       d   RMR\#        V4      4       \%        W!VR7      #   \         d!   p
\        P                  RT
4        Rp
?
LgRp
?
ii ; i)a  Build an APCManager from env vars when ``APC_ENABLED=1``, else None.

When ``APC_DISK_PATH`` is set, also wires up the shard-based SSD tier.
The disk read path defaults to direct file reads so restored K/V tensors
are MLX-owned buffers rather than mmap-backed safetensors views.
ÚAPC_ENABLEDro  rÔ  NÚAPC_BLOCK_SIZEÚAPC_NUM_BLOCKSÚAPC_DISK_PATHÚAPC_DISK_NAMESPACErL  ÚAPC_DISK_MAX_GBÚAPC_DISK_WORKERS)rÃ  rÄ  rÅ  z.1fz GBÚ	unboundedz1APC disk tier at %s (ns=%s, cap=%s, read_mode=%s)z(APC disk tier disabled (init failed): %sz<APC enabled (block_size=%d, num_blocks=%d, hash=%s, disk=%s)r   r   )r<  r¿   r  )rÔ  rl  ÚTruerm  r%  )r   r   r   r!   r¶  rµ  r  r¾  r   Ú
expanduserré  rõ  rÙ  rè  rÍ   rê  r   r   r  )r0  r¿   r<  r  Ú	disk_pathÚnsÚmax_gbrÅ  ÚworkersÚcap_strr7  s   &          r   Úfrom_envrA  ²  s—  € ô 
‡z�z‡~�~�m SÓ)Ð1MÔMÙÜ”R—Z‘Z—^‘^Ð$4Ô6HÓIÓJ€JÜ”R—Z‘Z—^‘^Ð$4Ô6HÓIÓJ€Jà%)€DÜ—
‘
—‘˜Ó/€IßØ×OÐO¤§
¡
§¡Ð/CÀYÓ OˆÜ”r—z‘z—~‘~Ð&7¸Ó;Ó<ˆØ/5¸¬z”C˜ 'Õ*Ô+¸tˆ	Ü”b—j‘j—n‘nÐ%7¸Ó=Ó>ˆð	JÜ!Ü�Y“×*Ñ*Ó,ØØ#Ø#ô	ˆD÷ -6˜ ˜ CÑ(¸;ˆGÜ�K‰KØCØ—‘ØØØ—‘ôô ‡K�KØFØØÜ$×&Ò&‰¨FÜˆT‹
ôô  ÈÔNÐNøô ô 	JÜ�N‰NÐEÀq×IÑIûð	Jús   ÅA#G' Ç'HÇ2HÈHrF  )NNrz   )Vr  Ú
__future__r   r.   rZ  Úloggingr   rÛ  rN  rÞ  ro  Úcollectionsr   Údataclassesr   Úpathlibr   Útypingr   r   r   r	   r
   r   Úmlx.coreÚcorerO   Únumpyr1   Ú	getLoggerré  r¶  rµ  rÏ   r   r9   r?   rJ   rR   re   r‘   rŸ   r«   r²   rº   rÄ   rÐ   rà   rê   ró   r÷   r   r  r	  r  r   r"  r%  r,  r/  rR  re  rh  rm  rw  r„  r�  r–  r   r¤  r¼  r¾  r  rÆ  rt  rÐ  rë  rî  r÷  r  r  r  r  r+  r.  rA  r   r   r   Ú<module>rL     sW  ðñ&õP #ã Û Û Û 	Û Û 	Û Û Ý #Ý !Ý ß A× Aå Û à	×	Ò	˜=Ó	)€àÐ ØÐ ØÐ õBõ)õA÷	Aõ5õ
õ.`ðFð *.÷õ*õ0õ(+õA÷!=õHõõ.-õ Põ
Rðð
 (,÷ð. ÷&ð &ó ð&ð  ÷ð ó ðñ �$Ô÷ð ó ðñ �$Ô÷ð ó ðñ �$Ô÷	ð 	ó ð	ñ �$Ô÷ð ó ðð ÷
ð 
ó ð
õDõõ4	õõõõ&õ Eõ$$õNLõ "÷JP%ñ P%÷f>q	ñ q	÷h*õZõ&õ@Qõh*õõ4'õT7õ@ð((ð ð(ð  !÷(õVõ*6÷,Oñ ,Or   