+ QV-jOaR6t-0t^RIHtHt^RIHt^RIt^RIHt^RI H t H t H t H t HtHt] !4'd^RIHt] !RRR 7t]P(!]4t/t]^k/t]^k!R R ]4t!R R ]4t!RR]4t!RR]4t!RR]4t!RR]4t!RR]4t!RR]4t !RR] 4t!!RR] 4t"!RR]4t#!R R!]#4t$!R"R#]$]4t%]PMR$]R%]R&]R']$R(]$R)]$R*]$R+]%/4!R,R-4t'!R.R/]'4t(!R0R1]'4t)!R2R3]'4t*!R4R5]'4t+])t,R#)7)ABCabstractmethod)IterableNPreTrainedConfig)is_hqq_availableis_optimum_quanto_availableis_quanto_greateris_torch_greater_or_equalis_torchdynamo_compilinglogging) Quantizerz2.7T accept_devcaa]tRt^+toRtRtRtV3RltRtRt ] V3RlRl4t ] V3R lR l4t ] V3R lR l4t ] V3R lRl4t] V3RlRl4tRtRtV3RlRltV3RlRltV3RltRtVtV;t#)CacheLayerMixinz0Base, abstract class for a single layer's cache.FNc <\SV`!R/VBVPPRR4pVeE\ 4PR4pVe\ W4'd V\ V&R#V\V&R#R#) layer_typeN StaticLayer)super__init_subclass____dict__getglobals issubclassLAYER_TYPE_STATIC_CACHE_MAPPINGLAYER_TYPE_CACHE_MAPPING)clskwargsr static_base __class__s&, i/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/transformers/cache_utils.pyr!CacheLayerMixin.__init_subclass__4sj !+F+\\%%lD9  !!)-- 6K&:c+G+G>A/ ;7:(4 "c0RVnRVnRVnR#NF)keysvaluesis_initializedselfs&r#__init__CacheLayerMixin.__init__>s)- +/ #r%c0VPP #Nr"__name__r+s&r#__repr__CacheLayerMixin.__repr__C..))*+r%cR<V^8dQhRS[PRS[PRR/# key_states value_statesreturnNtorchTensor)format __classdict__s"r# __annotate__CacheLayerMixin.__annotate__Gs!ddelld%,,d[_dr%cR#r0rr,r9r:s&&&r#lazy_initialization#CacheLayerMixin.lazy_initializationFsadr%c<V^8dQhRS[PRS[PRS[S[PS[P3,/#r8r9r:r;r=r>tuple)r?r@s"r#rArBJs?00,,06;ll0 u||U\\) *0r%cR#r0rr,r9r:argsr s&&&*,r#updateCacheLayerMixin.updateIs-0r%c<<V^8dQhRS[RS[S[S[3,/#r8 query_lengthr;intrJ)r?r@s"r#rArBOsGG3G5c?Gr%cR#r0r)r,rRs&&r#get_mask_sizesCacheLayerMixin.get_mask_sizesNsDGr%c <V^8dQhRS[/#r8r;rT)r?r@s"r#rArBRs(((r%cR#r0rr+s&r#get_seq_lengthCacheLayerMixin.get_seq_lengthQ%(r%c <V^8dQhRS[/#rYrZ)r?r@s"r#rArBUs--S-r%cR#r0rr+s&r#get_max_cache_shape#CacheLayerMixin.get_max_cache_shapeTs*-r%c VP'dGVPPRRR7VnVPPRRR7VnR#R#z(Offload this layer's data to CPU device.cpuT non_blockingN)r*r(tor)r+s&r#offloadCacheLayerMixin.offloadWsC     U >DI++..T.BDK r%c ,VP'dVPPVP8wd[VPPVPRR7VnVPPVPRR7VnR#R#R#zcIn case of layer offloading, this allows to move the data back to the layer's device ahead of time.TrfN)r*r(devicerhr)r+s&r#prefetchCacheLayerMixin.prefetch]sd    499#3#3t{{#B T[[t DDI++..4.HDK$C r%c<V^8dQhRR/#r8r;Nr)r?r@s"r#rArBcs / /t /r%c @VP'd5VPP4VPP4\ VR4'dF\ VP \4'd ^VnR#VP P4R#R#)4Resets the cache values while preserving the objectscumulative_lengthN)r*r(zero_r)hasattr isinstancertrTr+s&r#resetCacheLayerMixin.resetcsl     IIOO  KK    4, - -$00#66)*&&&,,. .r%c8<V^8dQhRS[PRR/#r8beam_idxr;Nr= LongTensor)r?r@s"r#rArBps%WWe&6&6W4Wr%c DVP4^8dVPP^VPVPP44VnVP P^VPVP P44VnR#R#)z,Reorders this layer's cache for beam search.N)r\r( index_selectrhrmr)r,r|s&&r# reorder_cacheCacheLayerMixin.reorder_cachepsn    1 $ ..q(++dii>N>N2OPDI++221hkk$++BTBT6UVDK %r%c4<V^8dQh/S[R,;R&#)r8Nrstr)r?r@s"r#rArB+sd !r%)rtr*r(r))r2 __module__ __qualname____firstlineno____doc__is_compileablerrr-r3rrErNrVr\rarirnrxr__annotate_func____static_attributes____classdictcell__ __classcell__r"r@s@@r#rr+s:N"J;$ ,dd00GG((--C I / /WWKr%rcaa]tRt^wtoRtRtRV3RlV3RllltV3RlRltV3RlRltV3R lR lt V3R lR lt V3R lRlt V3RlRlt V3RlRlt V3RlRltRtVtV;t#) DynamicLayerz A cache layer that grows dynamically as more tokens are generated. This is the default for generative models. It stores the key and value states as tensors of shape `[batch_size, num_heads, seq_len, head_dim]`. Fc.<V^8dQhRS[R,/#r8configNr)r?r@s"r#rADynamicLayer.__annotate__/$6r%c$<\SV`4R#r0rr-r,rr"s&&r#r-DynamicLayer.__init__ r%cR<V^8dQhRS[PRS[PRR/#r7r<)r?r@s"r#rArs+##ell#%,,#[_#r%c"VPVPuVnVn\P!.VPVPR7Vn\P!.VPVPR7VnRVnR#dtypermTN)rrmr=tensorr(r)r*rDs&&&r#rE DynamicLayer.lazy_initializations^","2"2J4E4E DKLL4::dkkJ ll2TZZ L "r%c<V^8dQhRS[PRS[PRS[S[PS[P3,/#rHrI)r?r@s"r#rArs?&&,,&6;ll& u||U\\) *&r%c VP'gVPW4\P!VPV.RR7Vn\P!VP V.RR7VnVPVP 3#)1 Update the key and value caches in-place, and return the necessary keys and value states. Args: key_states (`torch.Tensor`): The new key states to cache. value_states (`torch.Tensor`): The new value states to cache. Returns: tuple[`torch.Tensor`, `torch.Tensor`]: The key and value states. dim)r*rEr=catr(r)rLs&&&*,r#rNDynamicLayer.updatesg"""  $ $Z >IItyy*52> iil ;D yy$++%%r%c<<V^8dQhRS[RS[S[S[3,/#rQrS)r?r@s"r#rAr#$$3$5c?$r%c :^pVP4V,pW23#)zDReturn the length and offset of the cache, used to generate the mask)r\r,rR kv_offset kv_lengths&& r#rVDynamicLayer.get_mask_sizess# '')L8 ##r%c <V^8dQhRS[/#rYrZ)r?r@s"r#rArs###r%c VP'd VPP4^8Xd^#VPPR,#)1Returns the sequence length of the cached states.r)r*r(numelshaper+s&r#r\DynamicLayer.get_seq_lengths6"""diioo&71&<yyr""r%c <V^8dQhRS[/#rYrZ)r?r@s"r#rArsSr%c R#)zeReturns the maximum sequence length of the cache object. DynamicLayer does not have a maximum length.rr+s&r#ra DynamicLayer.get_max_cache_shapes r%c$<V^8dQhRS[RR/#r8 max_lengthr;NrZ)r?r@s"r#rArs 7 7s 7t 7r%c V^8d!VP4\V4, pVP4V8:dR#VPRRV1R3,VnVPRRV1R3,VnR#)z Crop the past key values up to a new `max_length` in terms of tokens. `max_length` can also be negative to remove `max_length` tokens. N.NNN)r\absr(r)r,rs&&r#cropDynamicLayer.cropsl >,,.Z@J    J . IIc;J;12 kk#{ {A"56 r%c$<V^8dQhRS[RR/#r8repeatsr;NrZ)r?r@s"r#rArsHHsHtHr%c VP4^8dGVPPV^R7VnVPPV^R7VnR#R#)z8Repeat the cache `repeats` times in the batch dimension.rN)r\r(repeat_interleaver)r,rs&&r#batch_repeat_interleave$DynamicLayer.batch_repeat_interleavesN    1 $ 33G3CDI++77Q7GDK %r%c8<V^8dQhRS[PRR/#r8indicesr;Nr<)r?r@s"r#rArs44ELL4T4r%c VP4^8d7VPVR3,VnVPVR3,VnR#R#)z *"2"22"66 ))TYY $;D!IIt{{L&ArJ#Aq4+>+>*>*B*Da$GH '1t/B/B.BQ.F.H!(KL 11r%c<<V^8dQhRS[RS[S[S[3,/#rQrS)r?r@s"r#rArs# $ $3 $5c? $r%c VPVP8p\VPVP, ^,^4pV'dVP^, V,pWC3#VPV,pWC3#zNReturn the length and offset of the cache, used to generate the attention mask)rtrmax)r,rRis_fullrrs&& r#rV(DynamicSlidingWindowLayer.get_mask_sizessx((D,?,??..1D1DDqH!L ++a/,>I##..=I##r%c <V^8dQhRS[/#rYrZ)r?r@s"r#rAr &&&r%c VP#rrtr+s&r#r\(DynamicSlidingWindowLayer.get_seq_length %%%r%c <V^8dQhRS[/#rYrZ)r?r@s"r#rAr s##S#r%c VP#z+Return the maximum cache shape of the cacherr+s&r#ra-DynamicSlidingWindowLayer.get_max_cache_shape s"""r%c$<V^8dQhRS[RR/#rrZ)r?r@s"r#rArs 5 5s 5t 5r%c <VP4VP8d \R4h\SV`V4VP P R,VnR#)z Crop the past key values up to a new `max_length` in terms of tokens. `max_length` can also be negative to remove `max_length` tokens. zCannot `crop` a `DynamicSlidingWindowLayer` after it has seen more tokens than itssliding window (otherwise some states are lost)Nr)r\rrrrr(rrt)r,rr"s&&r#rDynamicSlidingWindowLayer.cropsR    D$7$7 7B   Z !%!4r%)rrtr(rr)NN)r2rrrrrr-rErNrVr\rarrrrrs@@r#rrs` J Z ZRR22: $ $&&## 5 5 5r%rcaa]tRtRtoRtRtRV3RlV3RllltV3RlRltV3RlR ltV3R lt V3R lt V3R lV3R llt V3RlV3Rllt V3RlV3Rllt V3RlV3RlltV3RlV3RlltRtVtV;t#)DynamicIndexedLayeriag A cache layer that extends `DynamicLayer` with an extra indexer key cache for Dynamic Sparse Attention (DSA) models (e.g. GLM MoE DSA, DeepSeek V32). The main K/V cache stores tensors of shape `[batch_size, num_heads, seq_len, head_dim]` (inherited). The indexer key cache stores a tensor of shape `[batch_size, seq_len, index_head_dim]` (3D, single-head). deepseek_sparse_attentionc.<V^8dQhRS[R,/#rr)r?r@s"r#rA DynamicIndexedLayer.__annotate__+s22/$62r%cB<\SV`V4RVnRVnR#r')rr- indexer_keysis_indexer_initializedrs&&r#r-DynamicIndexedLayer.__init__+s   15,1#r%c8<V^8dQhRS[PRR/#r8indexer_key_statesr;Nr<)r?r@s"r#rAr0s++ell+t+r%cVPVPuVnVn\P !.VPVPR7VnRVnR#r)rrm indexer_dtypeindexer_devicer=rrrr,r s&&r#lazy_initialization_indexer/DynamicIndexedLayer.lazy_initialization_indexer0sJ2D2J2JL^LeLe/D/!LL43E3EdNaNab&*#r%cN<V^8dQhRS[PRS[P/#r8r r;r<)r?r@s"r#rAr5s# ! ! !%,, !r%c VP'gVPV4\P!VPV.^R7VnVP#)a0 Update the indexer key cache by concatenation, and return the full indexer keys. Args: indexer_key_states (`torch.Tensor`): New indexer keys, shape `[batch_size, seq_len, index_head_dim]`. Returns: `torch.Tensor`: The full cached indexer keys, shape `[batch_size, total_len, index_head_dim]`. r)rrr=rrrs&&r#update_indexer"DynamicIndexedLayer.update_indexer5sK***  , ,-? @!IIt'8'8:L&MSTU   r%c<\SV`4VP'd%VPP RRR7VnR#R#)reTrfN)rrirrrhr,r"s&r#riDynamicIndexedLayer.offloadDs<   & & & $ 1 1 4 4U 4 ND  'r%c<\SV`4VP'dVVPPVP8wd/VPP VPRR7VnR#R#R#)TrfN)rrnrrrmrhrs&r#rnDynamicIndexedLayer.prefetchIs\   & & &4+<+<+C+Ct{{+R $ 1 1 4 4T[[t 4 TD ,S &r%c<V^8dQhRR/#rqr)r?r@s"r#rArNs&&t&r%c<\SV`4VP'dVPP 4R#R#r0)rrxrrrurs&r#rxDynamicIndexedLayer.resetNs/    & & &    # # % 'r%c8<V^8dQhRS[PRR/#r{r})r?r@s"r#rArSs%iie&6&6i4ir%c<\SV`V4VP'dhVPP 4^8dGVPP ^VP VPP44VnR#R#R#rN)rrrrrrrhrm)r,r|r"s&&r#r!DynamicIndexedLayer.reorder_cacheSsk h'  & & &4+<+<+B+B+Dq+H $ 1 1 > >q(++dN_N_NfNfBg hD ,I &r%c$<V^8dQhRS[RR/#rrZ)r?r@s"r#rArXsDDsDtDr%cz<\SV`V4VP'd VPP 4^8XdR#V^8dTM,VPP ^,\ V4, pVPP ^,V8d VPRRV1R3,VnR#R#)rNr)rrrrrrr)r,r effectiver"s&& r#rDynamicIndexedLayer.cropXs  Z ***d.?.?.E.E.G1.L ",/Jt7H7H7N7Nq7QTWXbTc7c    " "1 % 1 $ 1 1!ZiZ2B CD  2r%c$<V^8dQhRS[RR/#rrZ)r?r@s"r#rAr`sTTsTtTr%c<\SV`V4VP'dFVPP 4^8d%VPP V^R7VnR#R#R#)rrN)rrrrrr)r,rr"s&&r#r+DynamicIndexedLayer.batch_repeat_interleave`sZ '0  & & &4+<+<+B+B+Dq+H $ 1 1 C CGQR C SD ,I &r%c8<V^8dQhRS[PRR/#rr<)r?r@s"r#rAres#@@ELL@T@r%c<\SV`V4VP'd>VPP 4^8dVPVR3,VnR#R#R#)r.N)rrrrr)r,rr"s&&r#r(DynamicIndexedLayer.batch_select_indicesesR $W-  & & &4+<+<+B+B+Dq+H $ 1 1'3, ?D ,I &r%)rrrrr0)r2rrrrrr-rrrirnrxrrrrrrrrs@@r#rrs{-J22 ++ ! !O U && ii DDTT @@@r%rcaa]tRtRtoRtRtRtV3RlV3RlltV3RlRltV3R lR lt V3R lR lt V3R lRlt V3RlRlt Rt VtV;t#)rikar A static cache layer that stores the key and value states as static tensors of shape `[batch_size, num_heads, max_cache_len), head_dim]`. It lazily allocates its full backing tensors, and then mutates them in-place. Built for `torch.compile` support. Args: max_cache_len (`int`): Maximum number of tokens that can be stored, used for tensor preallocation. TFc <V^8dQhRS[/#r8 max_cache_lenrZ)r?r@s"r#rAStaticLayer.__annotate__xs>>c>r%ct<\SV`4Wn\P!^.\ R7VnR#)rrN)rr-r1r=rrTrtr,r1r"s&&r#r-StaticLayer.__init__xs) *!&qc!=r%cR<V^8dQhRS[PRS[PRR/#r7r<)r?r@s"r#rAr2~s+(#(#ell(#%,,(#[_(#r%c VPVPuVnVnVPR,wVnVnVPR,VnVPR,Vn\P!VPVPVPVP 3VPVPR7Vn \P!VPVPVPVP 3VPVPR7Vn VPPVP4Vn \4'g|\PP!VP4\PP!VP4\PP!VP4RVnR#)a Lazy initialization of the keys and values tensors. This allows to get all properties (dtype, device, num_heads in case of TP etc...) at runtime directly, which is extremely practical as it avoids moving devices, dtypes etc later on for each `update` (which could break the static dynamo addresses as well). If this is unwanted, one can call `early_initialization(...)` on the Cache directly, which will call this function ahead-of-time (this is required for `torch.export` for example). Note that for `compile`, as we internally don't compile the prefill, this is guaranteed to have been called already when compiling. If compiling the prefill as well, e.g. calling `model.compile(...)` before `generate` with a static cache, it is still supported in general, but without guarantees depending on the compilation options (e.g. cuda graphs, i.e. `mode="reduce-overhead"` is known to fail). But it will in general work correctly, and prefill should not be compiled anyway for performances! Nr8NrTNr)rrmrmax_batch_size num_heads v_head_dim k_head_dimr=zerosr1r(r)rtrhr _dynamomark_static_addressr*rDs&&&r#rEStaticLayer.lazy_initialization~sR#-"2"2J4E4E DK.8.>.>r.B+T^&,,R0$**2.KK  $..$2D2Ddoo V**;;  kk  $..$2D2Ddoo V**;;  "&!7!7!:!:4;;!G ()) MM - -dii 8 MM - -dkk : MM - -d.D.D E"r%c<V^8dQhRS[PRS[PRS[S[PS[P3,/#rHrI)r?r@s"r#rAr2s? & &,, &6;ll & u||U\\) * &r%c VP'gVPW4VPR,p\P!WPP R7VP ,pVP PV4VPP^Wa4VPP^Wb4VPVP3# \d&YPRRT3&Y PRRT3&LGi;i)rrmrr) r*rErr=arangermrtadd_r( index_copy_r)NotImplementedError)r,r9r:rMr rcache_positions&&&*, r#rNStaticLayer.updates"""  $ $Z >$$R( i DtG]G]] ##I. = II ! !!^ @ KK # #A~ D yy$++%% # =.8IIaN* +0> (#(#T & &D$$ DD"""r%rcaa]tRtRtoRtRtV3RlV3RlltV3RlRltV3RlR ltV3R lR lt V3R lt R t Vt V;t #)StaticSlidingWindowLayeria A static cache layer that stores the key and value states as static tensors of shape `[batch_size, num_heads, min(max_cache_len, sliding_window), head_dim]`. It lazily allocates its full backing tensors, and then mutates them in-place. Built for `torch.compile` support. Args: max_cache_len (`int`): Maximum number of tokens that can be stored, used for tensor preallocation. sliding_window (`int`): The size of the sliding window. Tc&<V^8dQhRS[RS[/#)r8r1rrZ)r?r@s"r#rA%StaticSlidingWindowLayer.__annotate__s''c'3'r%cL<\W!4p\SV` VR7^VnR#)rLN)minrr-cumulative_length_int)r,r1reffective_max_cache_lenr"s&&& r#r-!StaticSlidingWindowLayer.__init__s'"%n"D '>?%&"r%c<V^8dQhRS[PRS[PRS[S[PS[P3,/#rHrI)r?r@s"r#rArWsDM2M2,,M26;llM2 u||U\\) *M2r%c VP'gVPW4VPR,pVPpW`P8pV;PV, unV'Ed/VPR,^8XdVP P RRR7pVPP RRR7p \P!R.\VPR7p WRRV 3&W)RRV 3&VP PV4VPPV 4VP VP3#\P!VP R ,V3RR7p \P!VPR ,V3RR7p EM%We,VP8dqV^8XdTp Tp EM\P!VP RRRV1R3,V3RR7p \P!VPRRRV1R3,V3RR7p M\P!WPPR7VP,p VP P!^W4VPP!^W4VPP%V4VP VP3#VP PV RRVP)R1R3,4VPPV RRVP)R1R3,4W3# \"d&YP RRT 3&Y PRRT 3&Li;i) r)dimsrrrNrCrr)rr:NNr)r*rErrZr1r(rollr)r=rrTrmcopy_rrDrtrFrGrE)r,r9r:rMr rcurrent_lengthrnew_keys new_valuesindexrrrHs&&&*, r#rNStaticSlidingWindowLayer.updates"""  $ $Z >$$R( 33 $6$66 ""i/" 7#q(99>>"2>6![[--br-:  bTT[[I(2Au%*61a;' ) !!*-yy$++--#())TYY{-CZ,PVX"Y$)IIt{{;/G.V\^$_!  '$*<*< <"",$0!"'))TYYq!_n_a7O-PR\,]ce"f$)IIt{{1a.RS;S/TVb.cik$l!#\\)KKH4KaKaaN A %%aD ''>H  " " ' ' 299dkk) ) 1t/A/A.A.CQ(FGH +Aq43E3E2E2G,JKL11' A2< !Q./4@ Aq.01 As8L//-MMc<<V^8dQhRS[RS[S[S[3,/#rQrS)r?r@s"r#rArW=s#$$3$5c?$r%c .VPpVPVP8p\VPV, ^,^4pV'dW!,^, pWT3#VPV,V8dVPV,pWT3#TpWT3#r)r1rZr)r,rRrrrrs&& r#rV'StaticSlidingWindowLayer.get_mask_sizes=s++,,0B0BB22^CaGK &59I##  ' ', 6 G22\AI ##'I##r%c <V^8dQhRS[/#rYrZ)r?r@s"r#rArWOs***r%c VP#rrZr+s&r#r\'StaticSlidingWindowLayer.get_seq_lengthOs)))r%c2<\SV`4^VnR#r")rrxrZrs&r#rxStaticSlidingWindowLayer.resetSs  %&"r%rm)r2rrrrrr-rNrVr\rxrrrrs@@r#rUrUsF J'' M2M2^$$$**''r%rUc~aa]tRtRtoRtV3RlV3RlltV3RlRltV3RlRltV3R lV3R lltR t Vt V;t #) StaticIndexedLayeriXa A `StaticLayer` with an additional statically-allocated indexer key cache for Dynamic Sparse Attention (DSA) models (e.g. GLM MoE DSA, DeepSeek V32). This is the static, `torch.compile`-friendly counterpart of `DynamicIndexedLayer`: the indexer key buffer is preallocated once and mutated in-place. The main K/V cache is inherited from `StaticLayer` (`[batch_size, num_heads, max_cache_len, head_dim]`). The indexer key cache stores a tensor of shape `[batch_size, max_cache_len, index_head_dim]` (3D, single-head). c <V^8dQhRS[/#r0rZ)r?r@s"r#rAStaticIndexedLayer.__annotate__bsFFcFr%c<\SV`VR7RVnRVn\P !^.\ R7VnR#)rLNFr)rr-rrr=rrTindexer_cumulative_lengthr4s&&r#r-StaticIndexedLayer.__init__bs: }515,1#*/qc)E&r%c8<V^8dQhRS[PRR/#r r<)r?r@s"r#rArtjs + +ell +t +r%c VPVPuVnVnVPwr#p\ P !W PV3VPVPR7VnVPPVP4Vn \4'gS\ PPVP4\ PPVP4RVnR#r)rrmrrrr=r=r1rrvrhr r>r?r)r,r r9_index_head_dims&& r#r.StaticIndexedLayer.lazy_initialization_indexerjs2D2J2JL^LeLe/D/,>,D,D)>!KK // @$$&&  *.)G)G)J)J4K^K^)_&')) MM - -d.?.? @ MM - -d.L.L M&*#r%cN<V^8dQhRS[PRS[P/#rr<)r?r@s"r#rArtys#!!!%,,!r%c VP'gVPV4VP^,p\P!W P R7VP ,pVP PV4VPP^W14VP# \dYPRT3&TP#i;i)a Update the indexer key cache in-place at the current positions, and return the full static buffer. Args: indexer_key_states (`torch.Tensor`): New indexer keys, shape `[batch_size, seq_len, index_head_dim]`. Returns: `torch.Tensor`: The full static indexer key cache, shape `[batch_size, max_cache_len, index_head_dim]`. Unfilled positions are masked out downstream by the indexer's attention mask, exactly as the main `StaticLayer` returns its full preallocated K/V. rCr) rrrr=rDrrvrErrFrG)r,r seq_lenrHs&& r#r!StaticIndexedLayer.update_indexerys***  , ,-? @$**1-g6I6IJTMkMkk &&++G4 F    ) )!^ P    # F3E  a/ 0    FsB--CCc<V^8dQhRR/#rqr)r?r@s"r#rArts33t3r%c<\SV`4VP'd7VPP 4VP P 4R#R#r0)rrxrrrurvrs&r#rxStaticIndexedLayer.resetsA    & & &    # # %  * * 0 0 2 'r%)rvrrrr) r2rrrrr-rrrxrrrrs@@r#rrrrXs9FF + +!!6333r%rrcaa]tRtRtoRtR V3RlV3RllltV3RlRlt]R4t]R4t V3R lR lt R t Vt V;t #) QuantizedLayeria A quantized layer similar to what is described in the [KIVI: A Tuning-Free Asymmetric 2bit Quantization for KV Cache paper](https://huggingface.co/papers/2402.02750). It allows the model to generate longer sequence length without allocating too much memory for the key and value caches by applying quantization. The cache has two types of storage, one for original precision and one for the quantized cache. A `residual length` is set as a maximum capacity for the original precision cache. When the length goes beyond maximum capacity, the original precision cache is discarded and moved into the quantized cache. The quantization is done per-channel with a set `q_group_size` for both Keys and Values, in contrast to what was described in the paper. c 8<V^8dQhRS[RS[RS[RS[RS[/#r8nbitsaxis_key axis_value q_group_sizeresidual_lengthrZ)r?r@s"r#rAQuantizedLayer.__annotate__s=#### #  #  #r%cn<\SV`4WnW nW0nW@nWPn^VnR#r")rr-rrrrrrtr,rrrrrr"s&&&&&&r#r-QuantizedLayer.__init__s3   $(.!"r%c<V^8dQhRS[PRS[PRS[S[PS[P3,/#rHrI)r?r@s"r#rArs?#0#0,,#06;ll#0 u||U\\) *#0r%c :V;PVPR,, unVP'guVPW4VP VP 4VP R7VnVP VP 4VPR7Vn W3#VPVP4pVPVP4p\P!WPPV.RR7p\P!W`PV.RR7pVPP4^8XdVPPR,^,VP 8dVP VP 4VP R7VnVP VP 4VPR7Vn \P"!.VP$VP&R7Vn \P"!.VP$VP&R7VnWx3#\P!VPV.RR7Vn \P!VPV.RR7VnWx3#)r)axisrrr)rtrr*rE _quantize contiguousr_quantized_keysr_quantized_values _dequantizer=rr(r)rrrrrm) r,r9r:rMr dequant_keysdequant_valueskeys_to_returnvalues_to_returns &&&*, r#rNQuantizedLayer.updates *"2"22"66"""  $ $Z >#'>>*2G2G2IPTP]P]>#^D %)^^L4K4K4MTXTcTc^%dD "+ +''(<(<= ))$*@*@AL))Z#HbQ 99nkk<%PVXY 99==?a DIIOOB$7!$;t?S?S$S#'>>.2K2K2MTXTaTa>#bD %)^^4D4O4O4QX\XgXg^%hD " Rz/?/? HYHYZDI,,r1A1A*J[J[\DK // 499j"9rBDI))T[[,$?RHDK//r%cR#r0r)r,rrs&&&r#rQuantizedLayer._quantizes'*r%cR#r0r)r,q_tensors&&r#rQuantizedLayer._dequantizer^r%c <V^8dQhRS[/#rYrZ)r?r@s"r#rArrr%c VP#rrr+s&r#r\QuantizedLayer.get_seq_lengthrr%) rrrrrtr(rrrr)rr@)r2rrrrr-rNrrrr\rrrrs@@r#rrsL ## #0#0J**((&&&r%rcNaa]tRtRtoRV3RlV3RllltRtRtRtVtV;t #)QuantoQuantizedLayeric 8<V^8dQhRS[RS[RS[RS[RS[/#rrZ)r?r@s"r#rA!QuantoQuantizedLayer.__annotate__s=)()()()( )(  )(  )(r%c<\S V`VVVVVR7\4'g \R4h\ RRR7'd ^RIHpHpHpM \R4hVPR 9d\RVP 24hVPR 9d\RVP 24hVPR 9d\R VP 24hVP^8XdTMTVn V!4VnR #) rrrrrzYou need to install optimum-quanto in order to use KV cache quantization with optimum-quanto backend. Please install it via with `pip install optimum-quanto`z0.2.5Tr) MaxOptimizerqint2qint4ziYou need optimum-quanto package version to be greater or equal than 0.2.5 to use `QuantoQuantizedLayer`. zA`nbits` for `quanto` backend has to be one of [`2`, `4`] but got zE`axis_key` for `quanto` backend has to be one of [`0`, `-1`] but got zG`axis_value` for `quanto` backend has to be one of [`0`, `-1`] but got N)r8r)rr)rr-r ImportErrorr optimum.quantorrrrrrrqtype optimizer) r,rrrrrrrrr"s &&&&&& r#r-QuantoQuantizedLayer.__init__s !%+  +,,T w4 8 8 A A{  ::V #`aeakak`lmn n == 'deiererdstu u ??' )YZ^ZiZiYjk #jjAoU5 %r%c^RIHpVPWPW P4wrEV!WPW$WPP4pV#)r)quantize_weight)rrrrr)r,rrrscale zeropointqtensors&&& r#rQuantoQuantizedLayer._quantizes?2>>&**dDUDUV!&**d9N_N_`r%c"VP4#r0) dequantize)r,rs&&r#r QuantoQuantizedLayer._dequantizes!!##r%)rrr r2rrrr-rrrrrrs@@r#rrs)()(V$$r%rcNaa]tRtRtoRV3RlV3RllltRtRtRtVtV;t #)HQQQuantizedLayeric 8<V^8dQhRS[RS[RS[RS[RS[/#rrZ)r?r@s"r#rAHQQQuantizedLayer.__annotate__s=!&!&!&!& !&  !&  !&r%cr<\SV`VVVVVR7\4'g \R4hVPR9d\ RVP 24hVP R9d\ RVP 24hVPR9d\ RVP 24h\Vn R#)rzYou need to install `HQQ` in order to use KV cache quantization with HQQ backend. Please install it via with `pip install hqq`zM`nbits` for `HQQ` backend has to be one of [`1`, `2`, `3`, `4`, `8`] but got zA`axis_key` for `HQQ` backend has to be one of [`0`, `1`] but got zC`axis_value` for `HQQ` backend has to be one of [`0`, `1`] but got N)r`r8r)rr`) rr-rrrrrr HQQQuantizer quantizerrs&&&&&&r#r-HQQQuantizedLayer.__init__s !%+   !!@  ::_ ,_`d`j`j_kl  == &`aeanan`opq q ??& (bcgcrcrbstu u%r%c VPPVVVPPVPPVP VP R7wr4VPPVR&VPPW4VPPR7VR,PVP4VR&VR,PVP4VR&W43#))rrm compute_dtyper group_sizer)metarmrzero) rquantizer(rmrrrcudarh)r,rrrrs&&& r#rHQQQuantizedLayer._quantizeBs// 99##))//**(( 0 !% _ Gtyy7G7GHW ((8W F|w~~6V }r%cDVwr#VPPW#4pV#r0)rr)r,r quant_tensorrrs&& r#rHQQQuantizedLayer._dequantizeQs#$ **<> r%)rrrrs@@r#rrs!&!&F r%rca]tRtRtoRtRtRtRt]RV3RlRll4t ]V3R lR l4t ]V3R lR l4t R t Rt V3RlRltV3RlRltV3RlRltRtVtR#)LinearAttentionCacheLayerMixiniWzABase, abstract class for a linear attention single layer's cache.TcLRVnRVnRVnRVnRVnR#r') conv_statesrecurrent_statesis_conv_states_initializedis_recurrent_states_initializedhas_previous_stater+s&r#r-'LinearAttentionCacheLayerMixin.__init__]s*0459*/'/4,"'r%c0VPP #r0r1r+s&r#r3'LinearAttentionCacheLayerMixin.__repr__dr5r%Ncn<V^8dQhRS[PR,RS[PR,RR/#r8rNrr;r<)r?r@s"r#rA+LinearAttentionCacheLayerMixin.__annotate__hs8 <<$.INX\I\ r%cR#r0rr,rrs&&&r#rE2LinearAttentionCacheLayerMixin.lazy_initializationgsr%cN<V^8dQhRS[PRS[P/#r8rr;r<)r?r@s"r#rArmsOOU\\OellOr%cR#r0r)r,rs&&r#update_conv_state0LinearAttentionCacheLayerMixin.update_conv_statelsLOr%cN<V^8dQhRS[PRS[P/#r8rr;r<)r?r@s"r#rArpsYYu||Y Yr%cR#r0r)r,rs&&r#update_recurrent_state5LinearAttentionCacheLayerMixin.update_recurrent_stateosVYr%c VP'd#VPPRRR7VnVP'd%VPPRRR7VnR#R#rd)rrrhrrr+s&r#ri&LinearAttentionCacheLayerMixin.offloadrs\  * * *#//225t2LD   / / /$($9$9$<$r?rrrrs&&&r#rE(LinearAttentionLayer.lazy_initializations  "&1&7&79K9K #DJ 9D9J9J19M{O`O`acOd 6D !6$// ::VZVaVabD +-- 11$2B2BC.2D +  '$)$4$45EZZ`d`k`k$lD !+-- 11$2G2GH37D 0 (r%cN<V^8dQhRS[PRS[P/#rr<)r?r@s"r#rArs#  U\\   r%c (VP'gVPVR7VP'g/VPP V4RVnVP#VP R,pW0P 8d>VPP VRVP )R13,4VP#VPPV)RR7pWRRV)R13&VPP V4VP#)z Update the linear attention cache in-place, and return the necessary conv states. Args: conv_states (`torch.Tensor`): The new conv states to cache. Returns: `torch.Tensor`: The updated conv states. )rT.N)shiftsr_rr)rrErrrbrr ra)r,rr num_new_tokensnew_conv_statess&&, r#r&LinearAttentionLayer.update_conv_states...  $ $ $ =&&&    " "; /&*D #)..r2N!6!66  &&{39N9N8N8P3P'QR  #'"2"2"7"7UW"7"X:E1~o&6 67  &&7r%cN<V^8dQhRS[PRS[P/#rr<)r?r@s"r#rArs&%%u||%RWR^R^%r%c VP'gVPVR7VPPV4VP#)z Update the linear attention cache in-place, and return the necessary ssm states. Args: smm_states (`torch.Tensor`): The new ssm states to cache. Returns: `torch.Tensor`: The updated ssm states. )r)rrErrb)r,rr s&&,r#r+LinearAttentionLayer.update_recurrent_statesC333  $ $6F $ G ##$45$$$r%) r rrmrrrrr9rr0r) r2rrrr-rErrrrrrs@@r#rrs388,  B%%%r%rcpa]tRtRtoRtR V3RlRlltV3RlRltV3RlR ltV3R lR ltR t Vt R#)$LinearAttentionAndFullAttentionLayeriFNc.<V^8dQhRS[R,/#rr)r?r@s"r#rA1LinearAttentionAndFullAttentionLayer.__annotate__s,,/$6,r%cZ\PV4\PV4R#r0)rr-r)r,rs&&r#r--LinearAttentionAndFullAttentionLayer.__init__sd#%%d+r%c<V^8dQhRR/#rqr)r?r@s"r#rArsEEdEr%c\V4^8Xd)\V4^8Xd\P!V.VO5!\V4^8Xd,\V4^8Xd\P!V3/VBR#R#R#)r8N)lenrrEr)r,rMr s&*,r#rE8LinearAttentionAndFullAttentionLayer.lazy_initializations] t9>c&kQ.  , ,T 9D 9 t9>c&kQ. 4 4T DV D/>r%c<V^8dQhRR/#rqr)r?r@s"r#rArs!!t!r%cZ\PV4\PV4R#r0)rrxrr+s&r#rx*LinearAttentionAndFullAttentionLayer.resets""4(4 r%c4<V^8dQhRS[P/#rr})r?r@s"r#rArs33e&6&63r%c Z\PW4\PW4R#r)rrrrs&&r#r2LinearAttentionAndFullAttentionLayer.reorder_caches**4:""42r%rr0) r2rrrrr-rErxrrrrs@r#rrs4N,,EE!!33r%rfull_attentionsliding_attentionchunked_attentionmambaconvlinear_attentionmoehybridca]tRtRtoRtR2V3RlRlltRtR3V3RlRlltR3V3R lR lltV3R lR lt V3R lRlt V3RlRlt V3RlRlt V3RlRlt R4V3RlRlltR5V3RlRlltV3RlRltR4V3RlRlltRtV3RlRltV3R lR!ltV3R"lR#ltV3R$lR%lt]V3R&lR'l4t]V3R(lR)l4t]V3R*lR+l4t]V3R,lR-l4t]V3R.lR/l4tR0tR1tVtR#)6Cacheias A `Cache` is mostly a list of `CacheLayerMixin` objects, one per model layer. It serves as a container for the Cache of each layer. Args: layers (`Optional`, *optional*): A list of pre-created `CacheLayerMixin` or `LinearAttentionCacheLayerMixin`. If omitted (`None`), then `layer_class_to_replicate` will be used. layer_class_to_replicate (`type[CacheLayerMixin | LinearAttentionCacheLayerMixin]`, *optional*): Only used if `layers` is omitted (`None`), in which case it will be used as the base class for each layer, and the layers will be added lazily as soon as `update` is called with a `layer_idx` greater than the current list of layers. offloading (`bool`, *optional*, defaults to `False`): Whether to perform offloading of the layers to `cpu`, to save GPU memory. offload_only_non_sliding (`bool`, *optional*, defaults to `True`): If `offloading` is `True`, this further decides if only the non-sliding layers will be offloaded (because usually the sliding layers are small in size, so there is no need to offload them, and skipping it is faster). Nc<V^8dQhRS[S[S[,,R,RS[S[S[,,R,RS[RS[/#)r8layersNlayer_class_to_replicate offloadingoffload_only_non_sliding)listrrtypebool)r?r@s"r#rACache.__annotate__'sZrr_'EEFMr#'9W'W"X[_"_r r #' rr%c@VeVe \R4hVfVf \R4hVeTM.VnW nW0nVP'dMW@n\ 'd\ P!4M\ PP4Vn R#R#)NaYou can construct a Cache either from a list `layers` of all the predefined `CacheLayer`, or from a `layer_class_to_replicate`, in which case the Cache will append a new layer corresponding to `layer_class_to_replicate` for each new call to `update` with an idx not already in the Cache.z_You should provide exactly one of `layers` or `layer_class_to_replicate` to initialize a Cache.) rr2r3r4only_non_sliding#_is_torch_greater_or_equal_than_2_7r=Streamrprefetch_stream)r,r2r3r4r5s&&&&&r#r-Cache.__init__'s  ":"Fq  >6>q !' 2f (@%$ ???$< !5X5X5<<>^c^h^h^o^o^qD  r%cNVPP RVP R2#)z(layers=))r"r2r2r+s&r#r3Cache.__repr__?s$..))*(4;;-qAAr%c&<V^8dQhRS[RS[/#r8 layer_idxr;rTr8)r?r@s"r#rAr9Bs..#..r%c V'd'WPVRPR4,pMV\VP48dTM^p\ 'd VP M(\PPVP 4;_uu_4VPV,P4RRR4R# \dTPPR4pLi;i +'giR#;i)a Prefetch a given layer on its device. If `only_non_sliding` is True, it will try to prefetch only the layers which are non-sliding. If the `layer_idx` is outside the range, this will circle back to the first layers. Note that we use a non-default stream for this, to avoid blocking. NF) rrfrrr2r<r>r=rstreamrnr,rEr;s&&&r#rnCache.prefetchBs  9% (C(I(I%(PP &/T[[1A%A qI&I%HT ! !ejjN_N_`d`t`tNu u u KK " + + -v u  9 OO11%8  9 v u us$C"C.&C+*C+. C? c&<V^8dQhRS[RS[/#rDrF)r?r@s"r#rAr9Vs----r%c V'dVPV,'g$VPV,P4R#R#)z Offload a given `layer_idx`. If `only_non_sliding` is True, it will offload `layer_idx` only if it is a non-sliding layer. Note that we do it on the default stream, so that we ensure all earlier computation in the layer's `update` methods are finished. N)rr2rirIs&&&r#ri Cache.offloadVs0 !T__Y%?%? KK " * * ,&@r%c <V^8dQhRS[PRS[PRS[RS[S[PS[P3,/#)r8r9r:rEr;)r=r>rTrJ)r?r@s"r#rAr9_sG  ,, 6;ll OR  u||U\\) * r%c HVPeF\VP4V8:d,VPPVP44KEVP'df\ P PVP4PVP4VPV^,VP4VPV,P!W.VO5/VBwrgVP'dVPW0P4Wg3#)a Updates the cache with the new `key_states` and `value_states` for the layer `layer_idx`. Parameters: key_states (`torch.Tensor`): The new key states to cache. value_states (`torch.Tensor`): The new value states to cache. layer_idx (`int`): The index of the layer to cache the states for. Return: A tuple containing the updated key and value states. )r3rr2appendr4r=rdefault_streamrm wait_streamr>rnr;rNri)r,r9r:rErMr r(r)s&&&&*, r#rN Cache.update_s$  ( ( 4dkk"i/ ""4#@#@#BC ??? JJ % %j&7&7 8 D DTEYEY Z MM)a-)>)> ?{{9-44Z_PT_X^_  ??? LL$9$9 :|r%cT<V^8dQhRS[PRS[RS[P/#)r8rrEr;r=r>rT)r?r@s"r#rAr9s-U\\cX]XdXdr%c \VPV,\4'g \R4hVPV,P!V3/VBpV#)a# Updates the cache with the new `conv_states` for the layer `layer_idx`. Parameters: conv_states (`torch.Tensor`): The new conv states to cache. layer_idx (`int`): The index of the layer to cache the states for. Return: `torch.Tensor`: The updated conv states. ?Cannot call `update_conv_state` on a non-LinearAttention layer!)rwr2rrr)r,rrEr s&&&,r#rCache.update_conv_statesK$++i02PQQ^_ _kk),>>{UfU r%cT<V^8dQhRS[PRS[RS[P/#)r8rrEr;rU)r?r@s"r#rAr9s.  u|| PS bgbnbn r%c \VPV,\4'g \R4hVPV,P!V3/VBpV#)a% Updates the cache with the new `recurrent_states` for the layer `layer_idx`. Parameters: smm_states (`torch.Tensor`): The new ssm states to cache. layer_idx (`int`): The index of the layer to cache the states for. Return: `torch.Tensor`: The updated ssm states. rW)rwr2rrr)r,rrEr s&&&,r#rCache.update_recurrent_statesN$++i02PQQ^_ _;;y1HHIYd]cdr%cT<V^8dQhRS[PRS[RS[P/#)r8r rEr;rU)r?r@s"r#rAr9s2III#IRWR^R^Ir%c  \VPV,R4'g7\RV R\VPV,4P R24hVPV,P V4#)aa Updates the indexer key cache for layer `layer_idx`. Parameters: indexer_key_states (`torch.Tensor`): The new indexer key states to cache, shape `[batch_size, seq_len, index_head_dim]`. layer_idx (`int`): The index of the layer to cache the states for. Return: `torch.Tensor`: The updated indexer key states (full cache). rz&Cannot call `update_indexer` on layer z which is a zY; it has no indexer key cache (expected a `DynamicIndexedLayer` or `StaticIndexedLayer`).)rvr2rr7r2r)r,r rEs&&&r#rCache.update_indexerstt{{9-/?@@8 < I./889:NO  {{9%445GHHr%c <V^8dQhRS[RS[S[S[,,RS[S[S[,,RS[PRS[P/#)r8 batch_sizer:head_dimrrm)rTr6r=rrm)r?r@s"r#rAr9s\!F!F!Fc?!FS / !F {{ !F  !Fr%c \V\4'dV.\V4,p\V\4'dV.\V4,p\V4\VP48wd/\ R\V4 R\VP4 R24h\V4\VP48wd/\ R\V4 R\VP4 R24h\ VPW#4F2wrgp\ P!W^V3WER7p VPW4K4 R#)z Initialize all the layers in advance (it's otherwise lazily initialized on the first `update` call). This is useful for our `export` recipes, as `export` needs everything in advance. z,`num_head` was provided as a list of length z, but the Cache currently has z layersz,`head_dim` was provided as a list of length rN) rwrTrr2rzipr=r=rE) r,r`r:rarrmlayerlayer_num_headslayer_head_dimfake_kv_tensors &&&&&& r#early_initializationCache.early_initializations% i % %" c$i/I h $ $ zCI-H y>S- ->s9~>NNlmpquq|q|m}l~FG  x=C , ,>s9~>NNlmpquq|q|m}l~FG 7:$++y6[ 2EN#[[*q.)YafvN  % %n E 7\r%c&<V^8dQhRS[RS[/#r8rEr;rZ)r?r@s"r#rAr9s777C7r%c |aV\SP48d^#\SPV,\4'g?V^8wd\ RV R24h\ V3Rl\ \S4444pSPV,P4# \d \ R4hi;i)z=Returns the sequence length of the cache for the given layer.z+You called `get_seq_length` on layer index R, but this layer is a LinearAttention layer, which does not track sequence length.c3z<"TF0p\SPV,\4'gK,VxK2 R#5ir0rwr2r.0idxr,s& r# 'Cache.get_seq_length..) r0@Jt{{[^O_apDq0@); ;z{`get_seq_length` can only be called on Attention layers, and the current Cache seem to only contain LinearAttention layers.) rr2rwrrnextrange StopIterationr\r,rEsf&r#r\Cache.get_seq_lengths DKK( ($++i0/BBA~ A)M66  rc$i0@ rr {{9%4466 !  . s 'B$$B;c4<V^8dQhRS[R,RS[/#)r8rENr;rF)r?r@s"r#rAr9s99C$J9$9r%c aVeV\SP48dR#Vf3\V3Rl\\S4^, RR444pM6\ SPV,\4'g\ RV R24hSPV,P# \d \ R4hi;i)zYReturns whether the LinearAttention layer at index `layer_idx` has previous state or not.Fc3z<"TF0p\SPV,\4'gK,VxK2 R#5ir0)rwr2rrps& r#rs+Cache.has_previous_state..s.!;!$++c"24RSC;rvz`has_previous_state` can only be called on LinearAttention layers, and the current Cache seem to only contain Attention layers.z/You called `has_previous_state` on layer index zJ, but this layer is an Attention layer, which does not support calling it.r) rr2rwrxryrrwrrrzsf&r#rCache.has_previous_states  Y#dkk2B%B    !$SY]B;! DKK 24RSSA)M//  {{9%888!  5 s 0B++CcB<V^8dQhRS[RS[RS[S[S[3,/#r8rRrEr;rS)r?r@s"r#rAr9s/CC3C3C5c?Cr%c aV\SP48dV^3#\SPV,\4'g?V^8wd\ RV R24h\ V3Rl\ \S4444pSPV,PV4# \d \ R4hi;i)z Return a tuple (kv_length, kv_offset) corresponding to the length and offset that will be returned for the given layer at `layer_idx`. The masks are then prepared according to the given lengths (kv_length, kv_offset) and patterns for each layer. z+You called `get_mask_sizes` on layer index rmc3z<"TF0p\SPV,\4'gK,VxK2 R#5ir0rorps& r#rs'Cache.get_mask_sizes..(rurvz{`get_mask_sizes` can only be called on Attention layers, and the current Cache seem to only contain LinearAttention layers.) rr2rwrrrwrxryrVr,rRrEsf&&r#rVCache.get_mask_sizess DKK( (? "$++i0/BBA~ A)M66  rc$i0@ rr {{9%44\BB !  . s 'B''B>c&<V^8dQhRS[RS[/#rkrZ)r?r@s"r#rAr91s<;;e&6&6;r%c \\VP44F%pVPV,PV4K' R#)z!Reorder the cache for beam searchN)rxrr2r)r,r|rEs&& r#rCache.reorder_cache>s1s4;;/0I KK " 0 0 :1r%c <V^8dQhRS[/#rrZ)r?r@s"r#rAr9Cs44s4r%c \\VP44F%pVPV,PV4K' R#)z"Crop the cache to the given lengthN)rxrr2r)r,rrEs&& r#r Cache.cropCs1s4;;/0I KK " ' ' 31r%c <V^8dQhRS[/#r8rrZ)r?r@s"r#rAr9HsDDsDr%c \\VP44F%pVPV,PV4K' R#)zRepeat and interleave the cacheN)rxrr2r)r,rrEs&& r#rCache.batch_repeat_interleaveHs1s4;;/0I KK " : :7 C1r%c4<V^8dQhRS[P/#r8rr<)r?r@s"r#rAr9MsAAELLAr%c \\VP44F%pVPV,PV4K' R#)zSelect indices from the cacheN)rxrr2r)r,rrEs&& r#rCache.batch_select_indicesMs1s4;;/0I KK " 7 7 @1r%c <V^8dQhRS[/#rYrZ)r?r@s"r#rAr9Ssr%c VPUu.uFqPNK pp\\V44^8d\ RV 24hV^,#uupi)z*Return the maximum batch size of the cachez0Max batch size is not consistent across layers: )r2r9rsetrr,rdr)s& r#r9Cache.max_batch_sizeRsU59KK@K5&&K@ s6{ a OPVxXY YayAsAc <V^8dQhRS[/#rYrZ)r?r@s"r#rAr9[ssr%c jVPUu.uFqPNK pp\V4#uupi)z,Return the maximum cache length of the cache)r2r1rrs& r#r1Cache.max_cache_lenZs048;;?;%%%;?6{@s0c <V^8dQhRS[/#rYr8)r?r@s"r#rAr9asBBBr%c \VP4^8XdR#\;QJd&RVP4F 'dK R# R#!RVP44#)z&Return whether the cache is compilableFc38"TFqPxK R#5ir0)rrqrds& r#rs'Cache.is_compileable..fsA[E''[Trr2allr+s&r#rCache.is_compileable`sI t{{ q sAT[[AssAsAsAT[[AAAr%c <V^8dQhRS[/#rYr)r?r@s"r#rAr9is[[[r%c \VP4^8;'dI\;QJd&RVP4F 'dK R# R#!RVP44#)z,Return whether the cache data is initializedc38"TFqPxK R#5ir0)r*rs& r#rs'Cache.is_initialized..ks+ZkU,@,@krFTrr+s&r#r*Cache.is_initializedhsK4;;!#ZZ+Zdkk+ZZZ+Zdkk+Z(ZZr%c0<V^8dQhRS[S[,/#rY)r6r8)r?r@s"r#rAr9nsNNDJNr%c ZVPUu.uFp\VRR4NK up#uupi)z9Return whether the layers of the cache are sliding windowrF)r2rr,rds& r#rCache.is_slidingms+BFM|U3MMMs(c ,\VP4#)z> This value corresponds to the number of layers in the model. )rr2r+s&r#__len__ Cache.__len__rs 4;;r%)r3r2r4r;r>)NNFT)Trr0) r2rrrrr-r3rnrirNrrrrhr\rrVrarxrrrrpropertyr9r1rr*rrrrrs@r#r0r0s7&rr0B..(--  D(  (II*!F!FF770994CC<<<+ ;; 44 DD AA  BB[[NN  r%r0cLaa]tRtRtoRtRV3RlV3RllltRtRtVtV;t #) DynamicCachei{a A cache that grows dynamically as more tokens are generated. This is the default for generative models. It stores the key and value states as a list of `CacheLayer`, one for each layer. The expected shape for each tensor in the `CacheLayer`s is `[batch_size, num_heads, seq_len, head_dim]`. If a config is passed, it will additionally check for sliding or hybrid cache structure, greatly reducing the memory requirement of the cached tensors to `[batch_size, num_heads, min(seq_len, sliding_window), head_dim]`. See `Cache` for details on common methods that are implemented by all cache classes. Args: ddp_cache_data (`Iterable[tuple[torch.Tensor, torch.Tensor]]`, *optional*): It was originally added for compatibility with `torch.distributed` (DDP). In a nutshell, it is `map(gather_map, zip(*caches))`, i.e. each item in the iterable contains the key and value states for a layer gathered across replicas by torch.distributed (shape=[global batch size, num_heads, seq_len, head_dim]). Note: it needs to be the 1st arg as well to work correctly config (`PreTrainedConfig`, *optional*): The config of the model for which this Cache will be used. If passed, it will be used to check for sliding or hybrid layer structure, greatly reducing the memory requirement of the cached tensors to `[batch_size, num_heads, min(seq_len, sliding_window), head_dim]`. offloading (`bool`, *optional*, defaults to `False`): Whether to perform offloading of the layers to `cpu`, to save GPU memory. offload_only_non_sliding (`bool`, *optional*, defaults to `False`): If `offloading` is `True`, this further decides if only the non-sliding layers will be offloaded (because usually the sliding layers are small in size, so there is no need to offload them, and skipping it is faster). Example: ```python >>> from transformers import AutoTokenizer, AutoModelForCausalLM, DynamicCache >>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2-0.5B-Instruct") >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct") >>> inputs = tokenizer(text="My name is Qwen2", return_tensors="pt") >>> # Prepare a cache class and pass it to model's forward >>> past_key_values = DynamicCache(config=model.config) >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True) >>> outputs.past_key_values # access cache filled with key/values from generation ``` c<V^8dQhRS[S[S[PR,R3,,R,RS[R,RS[RS[/#)r8ddp_cache_dataN.rr4r5)rrJr=r>rr8)r?r@s"r#rADynamicCache.__annotate__s[9v9v u||d':C'?!@ADH9v!4'9v 9v #' 9vr%c<.pVeVPRR7p\VRR4;'g\VRR4p\VRR4pVfG.p\VP4F+p VeVP R4KVP R4K- \ VR4'dVRVP )pVF4p \PV \4p VP V !V44K6 Ve\V4FwrVfl\V 4^8Xd V ^,MRpVe4V^,P4pVP \VR 74MVP \44W\,PV ^,V ^,4wp K \V4^8Xd\SV`A\VVR 7R#\SV`AWSVR 7R#) NTdecoderrr layer_typesr(r'num_kv_shared_layersr)r3r4r5r2r4r5)get_text_configrrxnum_hidden_layersrPrvrrrr enumerateritemrrNrr-)r,rrr4r5r2decoder_configrrrzr cache_clsrEkv_and_optional_slidingsliding_window_tensorr"s&&&&& r#r-DynamicCache.__init__s  #33D3AN$^5EtLPW 6QN".-FK" ~??@A%1#**+>?#**+;< A ~'=>>)*P^-P-P,PQ ) 488\R  i78*  %6?6O2 >KNNeJfjkJk,CA,Fqu),8)>q)A)F)F)H &?~&^_ ln5(//0G0JLcdeLfg17P" v;!  G )5%)A   G F\t  ur%c#"VPF)pVPVP\VRR43xK+ R#5i)rN)r2r(r)rrs& r#__iter__DynamicCache.__iter__s3[[E**ellGE;SUY,ZZ Z!s<>r)NNFF) r2rrrrr-rrrrrs@@r#rr{s$(T9v9vv[[r%rcFaa]tRtRtoRtRV3RlV3RllltRtVtV;t#) StaticCacheiaD Static Cache class to be used with `torch.compile(model)` and `torch.export()`. It will check the `config` for potential hybrid cache structure, and initialize each layer accordingly. See `Cache` for details on common methods that are implemented by all cache classes. Args: config (`PreTrainedConfig`): The config of the model for which this Cache will be used. It will be used to check for sliding or hybrid layer structure, and initialize each layer accordingly. max_cache_len (`int`): The maximum number of tokens that this Cache should hold. offloading (`bool`, *optional*, defaults to `False`): Whether to perform offloading of the layers to `cpu`, to save GPU memory. offload_only_non_sliding (`bool`, *optional*, defaults to `True`): If `offloading` is `True`, this further decides if only the non-sliding layers will be offloaded (because usually the sliding layers are small in size, so there is no need to offload them, and skipping it is faster). Example: ```python >>> from transformers import AutoTokenizer, AutoModelForCausalLM, StaticCache >>> model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-2-7b-chat-hf") >>> tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-chat-hf") >>> inputs = tokenizer(text="My name is Llama", return_tensors="pt") >>> # Prepare a cache class and pass it to model's forward >>> # Leave empty space for 10 new tokens, which can be used when calling forward iteratively 10 times to generate >>> max_generated_length = inputs.input_ids.shape[1] + 10 >>> past_key_values = StaticCache(config=model.config, max_cache_len=max_generated_length) >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True) >>> outputs.past_key_values # access cache filled with key/values from generation StaticCache() ``` c2<V^8dQhRS[RS[RS[RS[/#)r8rr1r4r5)rrTr8)r?r@s"r#rAStaticCache.__annotate__s93r3r 3r3r 3r #' 3rr%c <VPRR7p\VRR4pVf\VRR4e&\VP4Uu.uFpRNK ppMX\VRR4e&\VP4Uu.uFpRNK ppM$\VP4Uu.uFpRNK pp\VR ^4pV^8dVRV)p\P 4U U u0uF@wr\ V \4'gK\V \4'gK5V R8wgK>V kKB p p p .p VFp V R8Xd\W!PR 7pMoW9d\W!PR 7pMRV R9d \4pM@V \9d\V ,!VR 7pM V R 8Xd\VR 7pM \!VR 7pV P#V4K \$SV`MWVR 7R#uupiuupiuupiuup p i)TrrNrr(rr)r'r)r1rrLrr)r*r+r,r-)rrrxrritemsrwr7rrrUrrrrrrrrPrr-)r,rr1r4r5r rrzrnamersliding_layer_typesr2rrdr"s&&&&&, r#r-StaticCache.__init__s'''5fmT:  v/6BJv?W?W9XY9XA/9X Y&v/EqI ! #%&<(<'<=K6;;= = #t$ )3C9R)S X\`sXs D=  %J001"/@[@[20}]r]rsKK,.>>7 CR_`::*G#-@ MM% )&, XpqM]\Y  s* G'? G,$ G1"G6G6G6"G6r)FT r2rrrrr-rrrrs@@r#rrs$N3r3r3rr%rcFaa]tRtRtoRtRV3RlV3RllltRtVtV;t#)QuantizedCacheiDa2 A quantizer cache similar to what is described in the [KIVI: A Tuning-Free Asymmetric 2bit Quantization for KV Cache paper](https://huggingface.co/papers/2402.02750). It allows the model to generate longer sequence length without allocating too much memory for keys and values by applying quantization. The cache has two types of storage, one for original precision and one for the quantized cache. A `residual length` is set as a maximum capacity for the original precision cache. When the length goes beyond maximum capacity, the original precision cache is discarded and moved into the quantized cache. The quantization is done per-channel with a set `q_group_size` for both keys and values, in contrast to what was described in the paper. See `Cache` for details on common methods that are implemented by all cache classes. Args: backend (`str`): The quantization backend to use. One of `("quanto", "hqq"). config (`PreTrainedConfig`): The config of the model for which this Cache will be used. nbits (`int`, *optional*, defaults to 4): The number of bits for quantization. axis_key (`int`, *optional*, defaults to 0): The axis on which to quantize the keys. axis_value (`int`, *optional*, defaults to 0): The axis on which to quantize the values. q_group_size (`int`, *optional*, defaults to 64): Quantization is done per-channel according to a set `q_group_size` for both keys and values. residual_length (`int`, *optional*, defaults to 128): Maximum capacity for the original precision cache cD<V^8dQhRS[RS[RS[RS[RS[RS[RS[/#)r8backendrrrrrr)rrrT)r?r@s"r#rAQuantizedCache.__annotate__csQ(((!( (  (  (((r%c <VR8Xd\pMVR8Xd\pM\RV R24hVPRR7p\ VP 4U u.uF p V!W4WVV4NK p p \ S V`V R7R#uup i)quantohqqzUnknown quantization backend ``Tr)r2N)rrrrrxrrr-) r,rrrrrrr layer_classrzr2r"s &&&&&&&& r#r-QuantizedCache.__init__cs h .K  +K=gYaHI I'''56334 4 ? S4   '  sA=rrrrs@@r#rrDs<(((r%rca]tRtRtoRtV3RlRltRtV3RlRltRtRV3R lR llt R t V3R lR lt V3RlRlt V3RlRlt V3RlRltV3RlRltV3RlRltV3RlRlt]R4t]V3RlRl4tRtVtR#) EncoderDecoderCachei|aQ Base, abstract class for all encoder-decoder caches. Can be used to hold combinations of self-attention and cross-attention caches. See `Cache` for details on common methods that are implemented by all cache classes. Args: caches (`Iterable`): Usually an iterable of length 2, containing 2 `Cache` objects, the first one for self-attention, the second one for cross-attention. Can optionally also be an iterable of length 1, containing a `tuple[tuple[torch.Tensor]]` (usually used for compatibility with torch dp and ddp). Example: ```python >>> from transformers import AutoProcessor, AutoModelForCausalLM, DynamicCache, EncoderDecoderCache >>> model = AutoModelForCausalLM.from_pretrained("openai/whisper-small") >>> processor = AutoProcessor.from_pretrained("openai/whisper-small") >>> inputs = processor(audio=YOUR-AUDIO, return_tensors="pt") >>> # Prepare cache classes for encoder and decoder and pass it to model's forward >>> self_attention_cache = DynamicCache(config=self.config) >>> cross_attention_cache = DynamicCache(config=self.config) >>> past_key_values = EncoderDecoderCache(self_attention_cache, cross_attention_cache) >>> outputs = model(**inputs, past_key_values=past_key_values, use_cache=True) >>> outputs.past_key_values # access cache filled with key/values from generation EncoderDecoderCache() ``` c<V^8dQhRR/#rqr)r?r@s"r#rA EncoderDecoderCache.__annotate__shh4hr%c \V4^8Xd..r2V^,Fp\V4^8Xd3VPVR,4VPVR,4KE\V4^8Xd3VPVR,4VPVR,4K\R\V4: RV: 24h \V4Vn\V4VnM\V4^8Xd\ V^,\4'd\ V^,\4'g4\R\V^,4: R\V^,4: 24hV^,VnV^,VnM\R \V4 24h/Vn \\VP 44F7p\VP PV4^84VPV&K9 R #) r`:NrN:rNNr8:r8NNz$Expected len(combined_cache_data) = z% to be 4 or 6. combined_cache_data = z;One of the two arguments is not a Cache: type(caches[0]) = z, type(caches[1]) = zExpected 1 or 2 arguments, got N)rrPrrself_attention_cachecross_attention_cacherwr0 TypeErrorr7 is_updatedrxr8r\)r,cachesself_attention_cache_datacross_attention_cache_datacombined_cache_datarEs&* r#r-EncoderDecoderCache.__init__s v;! DF'A'-ayy#*+q0-445H5LM.556I"6MN,-2-445H5LM.556I"6MN$'L5H1I0MMt^q]u%vww(1)55N(OD %)56P)QD & [A fQi//z&)U7S7S"^DQWXYQZOK__tbfgmnogpbqau vww(.q D %)/D &>s6{mLM Ms4#=#=>?I)-d.H.H.W.WXa.bef.f)gDOOI &@r%c# t"\VPVP4FwrW,xK R#5i)zuReturns tuples of style (self_attn_k, self_attn_v, self_attn_sliding, cross_attn_k, cross_attn_v, cross_attn_sliding)N)rcrr)r,self_attention_layercross_attention_layers& r#rEncoderDecoderCache.__iter__s1;>t?X?XZ^ZtZt;u 7 &> >W>W=XXp))*! - r%c ,\VP4#)z Support for backwards-compatible `past_key_values` length, e.g. `len(past_key_values)`. This value corresponds to the number of layers in the model. )rrr+s&r#rEncoderDecoderCache.__len__s 4,,--r%c&<V^8dQhRS[RS[/#rkrZ)r?r@s"r#rArsCCCCCr%c 8VPPV4#)zYReturns the sequence length of the cached states. A layer index can be optionally passed.)rr\rzs&&r#r\"EncoderDecoderCache.get_seq_lengths((77 BBr%cVPP4VPP4VPFpRVPV&K R#)FN)rrxrrrzs& r#rxEncoderDecoderCache.resetsB !!'') ""((*I).DOOI &)r%c4<V^8dQhRS[P/#rr})r?r@s"r#rArrr%c rVPPV4VPPV4R#r)rrrrs&&r#r!EncoderDecoderCache.reorder_caches* !!//9 ""00:r%c <V^8dQhRS[/#)r8methodr)r?r@s"r#rArs#r%c \VP\4'd!\VP\4'gF\ RV RVPP 4 RVPP 4 R24hR#)rz)` is only defined for dynamic cache, got z" for the self attention cache and z for the cross attention cache.N)rwrrrr__str__)r,rs&&r#check_dynamic_cache'EncoderDecoderCache.check_dynamic_caches} t00, ? ?455|DDF8DTE^E^EfEfEhDij''+'A'A'I'I'K&LLkm Er%c <V^8dQhRS[/#)r8maximum_lengthrZ)r?r@s"r#rArs7737r%c VPVPP4VPPV4R#)z Crop the past key values up to a new `maximum_length` in terms of tokens. `maximum_length` can also be negative to remove `maximum_length` tokens. This is used in assisted decoding and contrastive search (on the Hub). N)r rr2r)r,rs&&r#rEncoderDecoderCache.crops0   !3!34 !!&&~6r%c <V^8dQhRS[/#rrZ)r?r@s"r#rArsDDsDr%c VPVPP4VPPV4VPPV4R#)zaRepeat the cache `repeats` times in the batch dimension. Used in contrastive search (on the Hub).N)r rr2rrrs&&r#r+EncoderDecoderCache.batch_repeat_interleavesD   !=!=!F!FG !!99'B ""::7Cr%c4<V^8dQhRS[P/#rr<)r?r@s"r#rArsAAELLAr%c VPVPP4VPPV4VPPV4R#)zeOnly keep the `indices` in the batch dimension of the cache. Used in contrastive search (on the Hub).N)r rr2rrrs&&r#r(EncoderDecoderCache.batch_select_indicessD   !:!:!C!CD !!66w? ""77@r%c <V^8dQhRS[/#rYrZ)r?r@s"r#rArs??S?r%c 6VPP4#)zKReturns the maximum sequence length (i.e. max capacity) of the cache object)rrar+s&r#ra'EncoderDecoderCache.get_max_cache_shapes((<<>>r%cB<V^8dQhRS[RS[RS[S[S[3,/#rrS)r?r@s"r#rArs/QQ3Q3Q5c?Qr%c8VPPW4#r0)rrVrs&&&r#rV"EncoderDecoderCache.get_mask_sizess((77 PPr%c.VPP#r0)rrr+s&r#rEncoderDecoderCache.is_slidings((333r%c <V^8dQhRS[/#rYr)r?r@s"r#rArs888r%c.VPP#r0)rrr+s&r#r"EncoderDecoderCache.is_compileables((777r%)rrrNr)r2rrrrr-rr3rr\rxrr rrrrarVrrrrrrs@r#rr|s@hh<?   .CC/ ;; 77DD AA ??QQ4488r%rcV^8dQh/^\9d\\\3,;R&^\9d\\\3,;R&#)r8rr)__conditional_annotations__dictrr7)r?s"r#rArAs@##B/.$sDy/.C$N65c4i5O$r%).r#abcrrcollections.abcrr=configuration_utilsrutilsrr r r r r hqq.core.quantizerrr< get_loggerr2loggerrrrrrrrrUrrrrrrrrrNr0rrrrSlidingWindowCacherA)r#s@r#r-s##$ 1;&?RV&W#   H %-/. 465IWcIWXN4?N4bT5 T5nI@,I@Xk"/k"\|'{|'~@3@3FI&\I&X4$>4$n66r; S; |I%9I%X3+?3>, 66 %$0 #6&e e P h[5h[V[r%[r|5(U5(pL8%L8`!r%