+
    TV-j¨]  ã                   óÄ  € ^ RI Ht ^ RIHt ^ RIHtHtHtHt ^ RI	H
t ^ RIHt ^RIHtHtHt ^RIHtHtHt ^RIHt ^RIHt ] ! R	 R
]4      4       t ! R R]P6                  4      t]! ]P:                  RR7      R 4       t]! ]P:                  RR7      R 4       t]! ]P:                  RR7      R 4       t  ! R R]P6                  4      t! ! R R]P6                  4      t" ! R R]P6                  4      t# ! R R]P6                  4      t$ ! R R]P6                  4      t% ! R R]P6                  4      t& ! R R]P6                  4      t' ! R  R!]P6                  4      t(R# )"é    )Ú	dataclass)Úpartial)ÚAnyÚDictÚListÚOptionalN)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCacheÚRotatingKVCacheÚ
_BaseCache)Úinitialize_rope)Ú	SwitchGLUc                   ó®   a € ] tR t^t o RtRt^#tRt^tRt	Rt
RtRtRtRt^tR	t^t^ tRtR
tRtR	tRt^tRtR
tRtRtR
tR	tR	tR	t R	t!Rt"R t#V 3R lt$Rt%V t&R	# )Ú	ModelArgsÚgemma4_texti   i   é   i   ç      Ð?ç�íµ ÷Æ°>i   NFç      ð?i   g      >@Tc           	     ó,  € V P                   f   RRRRRRR/RRR	RR
RR//V n         V P                  fa   R.V P                  ^,
          ,          R.,           pWP                  \	        V4      ,          ^,           ,          R V P                   V n        R # R # )NÚfull_attentionÚpartial_rotary_factorr   Ú
rope_thetag    €„.AÚ	rope_typeÚproportionalÚsliding_attentionr   ç     ˆÃ@Údefault)Úrope_parametersÚlayer_typesÚsliding_window_patternÚnum_hidden_layersÚlen)ÚselfÚpatterns   & Új/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/gemma4_text.pyÚ__post_init__ÚModelArgs.__post_init__2   s³   € Ø×ÑÒ'à Ø+¨TØ  )Ø ð#ð
 $Ø+¨SØ  'Ø ð&ð$ˆDÔ ð ×ÑÒ#Ø*Ð+¨t×/JÑ/JÈQÕ/NÕOØ ðSõ ˆGð !(×+AÑ+AÄSÈÃ\Õ+QÐTUÕ+UÕ VØ(�$×(Ñ(ð ˆDÖñ	 $ó    c                óþ  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[S[,          ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[,          ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[S[ ,          ,          ;R&   S[;R&   # ) é   Ú
model_typeÚhidden_sizer$   Úintermediate_sizeÚnum_attention_headsÚhead_dimÚglobal_head_dimÚglobal_partial_rotary_factorÚrms_norm_epsÚ
vocab_sizeÚvocab_size_per_layer_inputÚnum_key_value_headsÚnum_global_key_value_headsÚnum_kv_shared_layersÚpad_token_idÚhidden_size_per_layer_inputÚrope_traditionalr   r!   Úsliding_windowr#   Úmax_position_embeddingsÚattention_k_eq_vÚfinal_logit_softcappingÚuse_double_wide_mlpÚenable_moe_blockÚnum_expertsÚtop_k_expertsÚmoe_intermediate_sizer"   Útie_word_embeddings)ÚstrÚintÚfloatr   Úboolr   r   )ÚformatÚ__classdict__s   "€r(   Ú__annotate__ÚModelArgs.__annotate__   sµ  ø‡ ‚ áÑ#ñ ñ Ññ ñ Ññ	 ñ
 Ñ!ñ ñ Ñ ñ ñ Ññ ñ Ññ ñ #(Ñ.ñ ñ Ññ ñ Ññ ñ !$Ñ,ñ ñ Ñ ñ ñ !)©¥Ñ4ñ ñ Ñ"ñ ñ  Ññ! ñ" "%Ñ*ñ# ñ$ Ñ"ñ% ñ& !Ñ&ñ' ñ( ™d•^Ñ*ñ) ñ* Ññ+ ñ,  Ñ#ñ- ñ. !Ñ)ñ/ ñ0 Ñ"ñ1 ñ2 #Ñ)ñ3 ñ4 Ñ$ñ5 ñ6 Ñ"ñ7 ñ8 ™#•Ñ%ñ9 ñ: ™C•=Ñ'ñ; ñ< $¡C�=Ñ/ñ= ñ> ™$™s�)Õ$Ñ+ñ? ñ@ Ñ$òA r+   )r"   r!   )'Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r.   r/   r$   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r   r!   r>   r#   r?   r@   rA   rB   rC   rD   rE   rF   r"   rG   r)   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©rM   s   @r(   r   r      sÀ   ø‡ € à#€JØ€KØÐØ!ÐØ ÐØ€HØ€OØ*.Ð Ø€LØ€JØ&,ÐØ ÐØ04ÐØ "ÐØ€LØ'*ÐØ"ÐØ#&ÐØ&*€OØ€NØ"#ÐØ#)ÐØ"ÐØ%)ÐØ $ÐØ"ÐØ!%€KØ#'€MØ+/ÐØ'+€KØ $Ðò÷E ƒ r+   r   c                   óX   a a€ ] tR t^It oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )ÚRMSNormNoScalez RMSNorm without learnable scale.c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r-   ÚdimÚeps)rI   rJ   )rL   rM   s   "€r(   rN   ÚRMSNormNoScale.__annotate__L   s   ø€ ÷ ñ ™Cð ¡eñ r+   c                ó0   <€ \         SV `  4        W n        R # ©N)ÚsuperÚ__init__r\   )r&   r[   r\   Ú	__class__s   &&&€r(   ra   ÚRMSNormNoScale.__init__L   s   ø€ Ü‰ÑÔØŽr+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r-   ÚxÚreturn©ÚmxÚarray)rL   rM   s   "€r(   rN   r]   P   s#   ø€ ÷ 3ñ 3™"Ÿ(™(ð 3¡r§x¡xñ 3r+   c                óX   € \         P                  P                  VR V P                  4      # r_   )ri   ÚfastÚrms_normr\   ©r&   rf   s   &&r(   Ú__call__ÚRMSNormNoScale.__call__P   s   € Ü�w‰w×Ñ  4¨¯©Ó2Ð2r+   ©r\   )r   ©
rP   rQ   rR   rS   Ú__doc__ra   ro   rU   rV   Ú__classcell__©rb   rM   s   @@r(   rY   rY   I   s!   ù‡ € Ù*÷õ ÷3÷ 3ð 3r+   rY   T)Ú	shapelessc                 óH   € \         P                  ! W,          4      V ,          # r_   )ri   Útanh)Úsoftcaprf   s   &&r(   Úlogit_softcaprz   T   s   € ä�7Š7�1•;Ó 'Õ)Ð)r+   c                 óZ   € V \         P                  ! VR4      ,           ^V,          ,
          # ©é   éÿÿÿÿ)ri   Úexpand_dims)Úx2Úy2Úxys   &&&r(   Ú_complete_squarerƒ   Y   s!   € à”—’˜r 2Ó&Õ&¨¨R­Õ/Ð/r+   c                 ó<   € \         P                  ! V 4      V,          # r_   )ÚnnÚgelu_approx)Úgaterf   s   &&r(   Úgeglurˆ   ^   s   € ä�>Š>˜$Ó !Õ#Ð#r+   c                   óT   a a€ ] tR t^ct oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚMLPc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r-   ÚconfigÚ	layer_idx©r   rI   )rL   rM   s   "€r(   rN   ÚMLP.__annotate__d   s   ø€ ÷ Tñ T™yð T±Sñ Tr+   c                óÜ  <€ \         SV `  4        VP                  VP                  ,
          pY#u;8¬  ;'       d    ^ 8„  Mu pVP                  ;'       d    TpVP
                  V'       d   ^M^,          p\        P                  ! VP                  VRR7      V n	        \        P                  ! WaP                  RR7      V n
        \        P                  ! VP                  VRR7      V n        R# )r   F©ÚbiasN)r`   ra   r$   r:   rB   r0   r…   ÚLinearr/   Ú	gate_projÚ	down_projÚup_proj)r&   r�   rŽ   Úfirst_kv_shared_layer_idxÚis_kv_shared_layerÚuse_double_wider0   rb   s   &&&    €r(   ra   ÚMLP.__init__d   s³   ø€ Ü‰ÑÔà×$Ñ$ v×'BÑ'BÕBð 	"ð '×GÔGÀaÔGÐØ ×4Ñ4×KÐKÐ9KˆØ"×4Ñ4¿_¹ÐRSÕTÐäŸš 6×#5Ñ#5Ð7HÈuÔUˆŒÜŸšÐ#4×6HÑ6HÈuÔUˆŒÜ—y’y ×!3Ñ!3Ð5FÈUÔSˆŽr+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# re   rh   )rL   rM   s   "€r(   rN   r�   q   s'   ø€ ÷ Iñ I™"Ÿ(™(ð I¡r§x¡xñ Ir+   c                ót   € V P                  \        V P                  V4      V P                  V4      4      4      # r_   )r–   rˆ   r•   r—   rn   s   &&r(   ro   ÚMLP.__call__q   s)   € Ø�~‰~œe D§N¡N°1Ó$5°t·|±|ÀA³ÓGÓHÐHr+   )r–   r•   r—   )r   ©	rP   rQ   rR   rS   ra   ro   rU   rV   rt   ru   s   @@r(   rŠ   rŠ   c   s#   ù‡ € ÷Tõ T÷I÷ Ið Ir+   rŠ   c                   óT   a a€ ] tR t^ut oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚRouterz@Expert router: norm -> scale -> project -> top-k -> renormalize.c                ó    <€ V ^8„  d   QhRS[ /# ©r-   r�   ©r   )rL   rM   s   "€r(   rN   ÚRouter.__annotate__x   s   ø€ ÷ 3ñ 3™yñ 3r+   c                ó~  <€ \         SV `  4        Wn        VP                  V n        \
        P                  ! VP                  VP                  R R7      V n	        \        P                  ! VP                  34      V n        \        P                  ! VP                  34      V n        VP                  R,          V n        R# )Fr’   Nç      à¿)r`   ra   r�   r5   r\   r…   r”   r/   rD   Úprojri   ÚonesÚscaleÚper_expert_scaleÚ
_root_size©r&   r�   rb   s   &&€r(   ra   ÚRouter.__init__x   s…   ø€ Ü‰ÑÔØŒØ×&Ñ&ˆŒÜ—I’I˜f×0Ñ0°&×2DÑ2DÈ5ÔQˆŒ	Ü—W’W˜f×0Ñ0Ð2Ó3ˆŒ
Ü "§¢¨×);Ñ);Ð(=Ó >ˆÔØ ×,Ñ,¨dÕ2ˆŽr+   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r-   rf   rh   )rL   rM   s   "€r(   rN   r¥   �   s   ø€ ÷ ,ñ ,™"Ÿ(™(ñ ,r+   c                óä  € \         P                  P                  WP                  V P                  ,          V P
                  4      pV P                  V4      p\         P                  ! W P                  P                  ) RR7      pVRV P                  P                  ) R13,          p\         P                  ! W#RR7      p\         P                  ! VRR7      pW@P                  V,          ,          pW43# )r}   )ÚkthÚaxis.N©r²   r~   )ri   rl   rm   rª   r¬   r\   r¨   Úargpartitionr�   rE   Útake_along_axisÚsoftmaxr«   )r&   rf   Úexpert_scoresÚtop_k_indicesÚtop_k_weightss   &&   r(   ro   ÚRouter.__call__�   sµ   € Ü�G‰G×Ñ˜Q§
¡
¨T¯_©_Õ <¸d¿h¹hÓGˆàŸ	™	 !›ˆäŸšØ§¡× 9Ñ 9Ð9Àô
ˆð & c¨D¯K©K×,EÑ,EÐ+EÑ+GÐ&GÕHˆä×*Ò*¨=ÈbÔQˆÜŸ
š
 =°rÔ:ˆØ%×(=Ñ(=¸mÕ(LÕLˆàÐ+Ð+r+   )r¬   r�   r\   r«   r¨   rª   rr   ru   s   @@r(   r¡   r¡   u   s!   ù‡ € ÙJ÷3ó 3÷,÷ ,ð ,r+   r¡   c                   ó*   a € ] tR t^’t o RtR tRtV tR# )ÚGeGLUz0GELU-gated linear unit activation for SwitchGLU.c                ó   € \        W!4      # r_   )rˆ   )r&   rf   r‡   s   &&&r(   ro   ÚGeGLU.__call__•   s   € Ü�T‹~Ðr+   © N)rP   rQ   rR   rS   rs   ro   rU   rV   rW   s   @r(   r¼   r¼   ’   s   ø‡ € Ù:÷ð r+   r¼   c                   óT   a a€ ] tR t^™t oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚExpertsz*Sparse MoE using SwitchGLU with gather_mm.c                ó    <€ V ^8„  d   QhRS[ /# r£   r¤   )rL   rM   s   "€r(   rN   ÚExperts.__annotate__œ   s   ø€ ÷ 	
ñ 	
™yñ 	
r+   c                óš   <€ \         SV `  4        \        VP                  VP                  VP
                  \        4       R R7      V n        R# )F)Ú
input_dimsÚhidden_dimsrD   Ú
activationr“   N)r`   ra   r   r/   rF   rD   r¼   Ú
switch_glur­   s   &&€r(   ra   ÚExperts.__init__œ   s=   ø€ Ü‰ÑÔä#Ø×)Ñ)Ø×4Ñ4Ø×*Ñ*Ü“wØô
ˆŽr+   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r-   rf   r¸   r¹   rg   rh   )rL   rM   s   "€r(   rN   rÃ   §   s=   ø€ ÷ ñ Ù—‘ðÙ*,¯(©(ðÙCEÇ8Á8ðá	�‰ñr+   c                ó€   € \         P                  ! VR4      pV P                  W4      pWE,          P                  R4      # )r}   r~   éþÿÿÿ)ri   r   rÈ   Úsum)r&   rf   r¸   r¹   ÚwÚys   &&&&  r(   ro   ÚExperts.__call__§   s2   € ô �NŠN˜=¨"Ó-ˆØ�O‰O˜AÓ-ˆà•�{‰{˜2‹Ðr+   )rÈ   rr   ru   s   @@r(   rÁ   rÁ   ™   s!   ù‡ € Ù4÷	
ó 	
÷÷ ð r+   rÁ   c                   óT   a a€ ] tR t^°t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# rŒ   r�   )rL   rM   s   "€r(   rN   ÚAttention.__annotate__±   s   ø€ ÷ 4
ñ 4
™yð 4
±Sñ 4
r+   c                óÎ  <€ \         SV `  4        Wn        W n        VP                  V,          V n        V P
                  R 8H  V n        W!P                  VP                  ,
          8  V n	        V P
                  R8X  d1   \        VR4      '       d   VP                  '       d   VP                  MVP                  V n        VP                  pVP                  V n        VP                   ;'       d    V P                  '       * V n        V P"                  '       d!   VP$                  e   VP$                  V n        MVP(                  V n        RV n        \,        P.                  ! W0P                  V P                  ,          RR7      V n        V P                  '       dƒ   \,        P.                  ! W0P&                  V P                  ,          RR7      V n        V P"                  '       g9   \,        P.                  ! W0P&                  V P                  ,          RR7      V n        \,        P.                  ! V P                  V P                  ,          VRR7      V n        \,        P8                  ! V P                  VP:                  R7      V n        V P                  '       dX   \,        P8                  ! V P                  VP:                  R7      V n        \A        V P                  VP:                  R7      V n!        V P                  '       d   R MRpVPD                  PG                  V/ 4      pVPG                  RR	4      p\I        V P                  VPJ                  VVVPL                  R
7      V n'        R# )r   r   r3   Nr   Fr’   rq   r   r   )ÚdimsÚtraditionalÚbaseÚscaling_configr?   )(r`   ra   r�   rŽ   r"   Ú
layer_typeÚ
is_slidingr$   r:   Úhas_kvÚhasattrr3   r2   r/   r1   Ún_headsr@   Ú
use_k_eq_vr9   Ú
n_kv_headsr8   rª   r…   r”   Úq_projÚk_projÚv_projÚo_projÚRMSNormr5   Úq_normÚk_normrY   Úv_normr!   Úgetr   r=   r?   Úrope)r&   r�   rŽ   r[   Ú	layer_keyÚrope_paramsr   rb   s   &&&    €r(   ra   ÚAttention.__init__±   s[  ø€ Ü‰ÑÔØŒØ"ŒØ ×,Ñ,¨YÕ7ˆŒØŸ/™/Ð-@Ñ@ˆŒØ×":Ñ":¸V×=XÑ=XÕ"XÑXˆŒð �‰Ð"2Ô2Ü˜Ð 1×2Ò2Ø×&×&Ð&ð ×"Ò"ð —‘ð 	Œð × Ñ ˆØ×1Ñ1ˆŒð !×1Ñ1×IÐI¸$¿/¹/Ô6IˆŒØ�?�?ˆ?˜v×@Ñ@ÒLØ$×?Ñ?ˆD�Oà$×8Ñ8ˆDŒOàˆŒ
ä—i’i §\¡\°D·M±MÕ%AÈÔNˆŒØ�;�;ˆ;ÜŸ)š) C¯©¸4¿=¹=Õ)HÈuÔUˆDŒKØ—?—?�?Ü ŸišiØŸ™¨4¯=©=Õ8¸uô�”ô —i’i §¡¨t¯}©}Õ <¸cÈÔNˆŒä—j’j §¡°F×4GÑ4GÔHˆŒØ�;�;ˆ;ÜŸ*š* T§]¡]¸×8KÑ8KÔLˆDŒKÜ(¨¯©¸F×<OÑ<OÔPˆDŒKð ,0¯?¯?¨?Ñ'Ð@Pˆ	Ø×,Ñ,×0Ñ0°¸BÓ?ˆØ —_‘_ \°7Ó;ˆ
Ü#Ø—‘Ø×/Ñ/ØØ&Ø$*×$BÑ$Bô
ˆŽ	r+   c                óº   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[S[,          RS[S[,          RS[ P                  /# )r-   rf   ÚmaskÚcacheÚ	shared_kvÚoffsetrg   ©ri   rj   r   r   Útuple)rL   rM   s   "€r(   rN   rÔ   ç   sd   ø€ ÷ -;ñ -;á�8‰8ð-;ñ ‘r—x‘xÕ ð-;ñ ™�}ð	-;ñ
 ™E•?ð-;ñ ™•ð-;ñ 
�‰ñ-;r+   c           	     ó:  € VP                   w  rgpV P                  V4      P                  WgV P                  V P                  4      p	V P                  V	4      p	Ve   Vw  r«EM-V P                  '       g   \        RV P                   R24      hV P                  V4      P                  WgV P                  V P                  4      p
T
pV P                  '       g7   V P                  V4      P                  WgV P                  V P                  4      pVe!   \        P                  ! VP                  4      M^ pV P!                  V
4      p
V
P#                  ^ ^^^4      p
V P%                  W¥R7      p
V P'                  V4      pVP#                  ^ ^^^4      pV	P#                  ^ ^^^4      p	V P%                  W•R7      p	Ve   VP)                  W«4      w  r«\+        WšW³V P,                  VR7      pVP#                  ^ ^^^4      P                  WgR4      pV P/                  V4      W«3V3# )NzLayer z/ is a KV-shared layer but received no shared_kv)rò   )rð   rª   rï   r~   )Úshaperá   ÚreshaperÞ   r2   ræ   rÜ   Ú
ValueErrorrŽ   râ   rà   rß   rã   ri   rj   rò   rç   Ú	transposerê   rè   Úupdate_and_fetchr   rª   rä   )r&   rf   rï   rð   rñ   rò   ÚBÚLÚ_ÚqueriesÚkeysÚvaluesÚoutputs   &&&&&&       r(   ro   ÚAttention.__call__ç   sÓ  € ð —'‘'‰ˆˆaà—+‘+˜a“.×(Ñ(¨¨t¯|©|¸T¿]¹]ÓKˆØ—+‘+˜gÓ&ˆàÒ Ø$‰LˆD‘&Ø——�ÜØ˜Ÿ™Ð(Ð(WÐXóð ð —;‘;˜q“>×)Ñ)¨!°·±ÀÇÁÓOˆDØˆFØ—?—?�?ØŸ™ Q›×/Ñ/°°d·o±oÀtÇ}Á}ÓU�à/4Ò/@”R—X’X˜eŸl™lÔ+ÀaˆFà—;‘;˜tÓ$ˆDØ—>‘> ! Q¨¨1Ó-ˆDØ—9‘9˜T�9Ó1ˆDà—[‘[ Ó(ˆFØ×%Ñ% a¨¨A¨qÓ1ˆFà×#Ñ# A q¨!¨QÓ/ˆØ—)‘)˜G�)Ó3ˆàÒØ ×1Ñ1°$Ó?‰LˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆà�{‰{˜6Ó" T N°FÐ:Ð:r+   )r�   rÜ   r2   rÛ   rç   râ   rŽ   rÚ   rÞ   rà   rä   ræ   rá   rê   rª   rß   rè   rã   ©NNNNrŸ   ru   s   @@r(   rÒ   rÒ   °   s   ù‡ € ÷4
ó 4
÷l-;÷ -;ò -;r+   rÒ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚDecoderLayeri  c                ó&   <€ V ^8„  d   QhRS[ RS[/# rŒ   r�   )rL   rM   s   "€r(   rN   ÚDecoderLayer.__annotate__  s   ø€ ÷ 3*ñ 3*™yð 3*±Sñ 3*r+   c                ó²  <€ \         SV `  4        Wn        W n        VP                  V,          V n        \        W4      V n        \        W4      V n	        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  VP                  R 7      V n        VP$                  V n        V P&                  '       d´   \)        V4      V n        \-        V4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  VP                  R 7      V n        VP6                  V n        V P6                  '       d—   \        P8                  ! VP                  V P6                  RR7      V n        \        P8                  ! V P6                  VP                  RR7      V n        \        P                  ! VP                  VP                  R 7      V n        MRV n        RV n        RV n        \@        PB                  ! R4      V n"        R# )rq   Fr’   N)r}   )#r`   ra   r�   rŽ   r"   rÚ   rÒ   Ú	self_attnrŠ   Úmlpr…   rå   r/   r5   Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormrC   Ú
enable_moer¡   ÚrouterrÁ   ÚexpertsÚpost_feedforward_layernorm_1Úpost_feedforward_layernorm_2Úpre_feedforward_layernorm_2r<   r”   Úper_layer_input_gateÚper_layer_projectionÚpost_per_layer_input_normri   r©   Úlayer_scalar)r&   r�   rŽ   rb   s   &&&€r(   ra   ÚDecoderLayer.__init__  s  ø€ Ü‰ÑÔØŒØ"ŒØ ×,Ñ,¨YÕ7ˆŒÜ" 6Ó5ˆŒÜ�vÓ)ˆŒÜ!Ÿzšz¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(*¯
ª
Ø×Ñ F×$7Ñ$7ô)
ˆÔ%ô *,¯ªØ×Ñ F×$7Ñ$7ô*
ˆÔ&ô +-¯*ª*Ø×Ñ F×$7Ñ$7ô+
ˆÔ'ð
 !×1Ñ1ˆŒØ�?�?ˆ?Ü  ›.ˆDŒKÜ" 6›?ˆDŒLÜ02·
²
Ø×"Ñ"¨×(;Ñ(;ô1ˆDÔ-ô 13·
²
Ø×"Ñ"¨×(;Ñ(;ô1ˆDÔ-ô 02¯zªzØ×"Ñ"¨×(;Ñ(;ô0ˆDÔ,ð
 ,2×+MÑ+MˆÔ(Ø×+×+Ð+Ü(*¯	ª	Ø×"Ñ" D×$DÑ$DÈ5ô)ˆDÔ%ô )+¯	ª	Ø×0Ñ0°&×2DÑ2DÈ5ô)ˆDÔ%ô .0¯ZªZØ×"Ñ"¨×(;Ñ(;ô.ˆDÕ*ð )-ˆDÔ%Ø(,ˆDÔ%Ø-1ˆDÔ*ô ŸGšG D›MˆÖr+   c                óä   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[S[ P                  ,          RS[S[,          RS[S[,          RS[ P                  /# )r-   rf   rï   rð   Úper_layer_inputrñ   rò   rg   ró   )rL   rM   s   "€r(   rN   r  M  sw   ø€ ÷ 8$ñ 8$á�8‰8ð8$ñ ‘r—x‘xÕ ð8$ñ ™�}ð	8$ñ
 "¡"§(¡(Õ+ð8$ñ ™E•?ð8$ñ ™•ð8$ñ 
�‰ñ8$r+   c                ó¢  € TpV P                  V4      pV P                  W‚W5VR 7      w  r…pV P                  V4      pWx,           pTpV P                  '       d„   V P	                  V4      p	V P                  V	4      p	V P                  V	4      p	V P                  V4      w  r«V P                  V4      pV P                  WÊV4      pV P                  V4      pWœ,           pM"V P	                  V4      pV P                  V4      pV P                  V4      pWx,           pV P                  eŠ   V P                  e|   V P                  en   Vej   TpV P                  V4      p\        P                   ! V4      p\"        P$                  ! WÔ4      pV P                  V4      pV P                  V4      pW},           pV P&                  e   W€P&                  ,          pW…V3# ))rñ   rò   )r  r	  r  r  r  r
  r  r  r  r  r  r  r  r  r  r…   r†   ri   Úmultiplyr  )r&   rf   rï   rð   r  rñ   rò   ÚresidualÚhÚh1r¸   r¹   Úh2r‡   s   &&&&&&&       r(   ro   ÚDecoderLayer.__call__M  s°  € ð ˆà× Ñ  Ó#ˆØ#Ÿ~™~Ø�U¸ð  .ó  
Ñˆ�fð ×)Ñ)¨!Ó,ˆØ�Lˆàˆà�?�?ˆ?Ø×/Ñ/°Ó2ˆBØ—‘˜"“ˆBØ×2Ñ2°2Ó6ˆBà+/¯;©;°q«>Ñ(ˆMØ×1Ñ1°!Ó4ˆBØ—‘˜b°Ó?ˆBØ×2Ñ2°2Ó6ˆBà•‰Aà×.Ñ.¨qÓ1ˆAØ—‘˜“ˆAà×+Ñ+¨AÓ.ˆØ�Lˆð ×%Ñ%Ò1Ø×)Ñ)Ò5Ø×.Ñ.Ò:ØÒ+àˆHØ×,Ñ,¨QÓ/ˆDÜ—>’> $Ó'ˆDÜ—;’;˜tÓ5ˆDØ×,Ñ,¨TÓ2ˆDØ×1Ñ1°$Ó7ˆDØ•ˆAà×ÑÒ(Ø×%Ñ%Õ%ˆAà˜VÐ#Ð#r+   )r�   r  r  r<   r  rŽ   r  rÚ   r
  r  r  r  r  r  r  r  r  r  r  r	  )NNNNNrŸ   ru   s   @@r(   r  r    s   ù‡ € ÷3*ó 3*÷j8$÷ 8$ò 8$r+   r  c                   ó†   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRV3R lR lltR tRV3R	 lR
 lltRt	Vt
V ;t# )ÚGemma4TextModeliˆ  c                ó    <€ V ^8„  d   QhRS[ /# r£   r¤   )rL   rM   s   "€r(   rN   ÚGemma4TextModel.__annotate__‰  s   ø€ ÷ 1Nñ 1N™yñ 1Nr+   c                ó8  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        VP                  V n        VP                  V n        \        P                  ! VP                  VP                  4      V n        VP                  R ,          V n        \        VP                  4       Uu. uF  p\        WR7      NK  	  upV n        \        P                   ! VP                  VP"                  R7      V n        VP&                  V n        V P&                  '       dî   \        P                  ! VP(                  VP                  VP&                  ,          4      V n        VP&                  R ,          V n        RV n        VP                  R,          V n        \        P2                  ! VP                  VP                  VP&                  ,          RR7      V n        \        P                   ! VP&                  VP"                  R7      V n        M#RV n        RV n        RV n        RV n        RV n        \9        \        \;        V P                  4      4      4      V n        VP>                  ^ 8”  dŸ   \;        V P                  4      pW1P>                  ,
          p/ p\        V4       F"  pW%V P                  V,          P@                  &   K$  	  \        WC4       F3  pWPP                  V,          P@                  ,          V P<                  V&   K5  	  R# R# u upi )g      à?)rŽ   rq   Fr’   Nr§   gÍ;fž æ?)!r`   ra   r�   r6   r>   Úwindow_sizer#   r$   r…   Ú	Embeddingr/   Úembed_tokensÚembed_scaleÚranger  Úlayersrå   r5   Únormr<   r7   Úembed_tokens_per_layerÚembed_tokens_per_layer_scaleÚper_layer_input_scaleÚper_layer_projection_scaler”   Úper_layer_model_projectionÚper_layer_projection_normÚlistr%   Úprevious_kvsr:   rÚ   )r&   r�   ÚiÚNÚMÚkvs_by_typeÚjrb   s   &&     €r(   ra   ÚGemma4TextModel.__init__‰  sl  ø€ Ü‰ÑÔØŒØ ×+Ñ+ˆŒØ!×0Ñ0ˆÔØ&,×&CÑ&CˆÔ#Ø!'×!9Ñ!9ˆÔäŸLšL¨×):Ñ):¸F×<NÑ<NÓOˆÔØ!×-Ñ-¨sÕ2ˆÔä7<¸V×=UÑ=UÔ7Vó
Ù7V°!ŒL˜×-Ñ7Vñ
ˆŒô —J’J˜v×1Ñ1°v×7JÑ7JÔKˆŒ	ð ,2×+MÑ+MˆÔ(Ø×+×+Ð+Ü*,¯,ª,Ø×1Ñ1Ø×(Ñ(¨6×+MÑ+MÕMó+ˆDÔ'ð 17×0RÑ0RÐTWÕ0WˆDÔ-Ø)2ˆDÔ&Ø.4×.@Ñ.@À$Õ.FˆDÔ+Ü.0¯iªiØ×"Ñ"Ø×(Ñ(¨6×+MÑ+MÕMØô/ˆDÔ+ô
 .0¯ZªZØ×2Ñ2¸×8KÑ8Kô.ˆDÕ*ð +/ˆDÔ'Ø)-ˆDÔ&Ø.2ˆDÔ+Ø.2ˆDÔ+Ø-1ˆDÔ*ô !¤¤s¨4¯;©;Ó'7Ó!8Ó9ˆÔØ×&Ñ&¨Ô*Ü�D—K‘KÓ ˆAØ×/Ñ/Õ/ˆAØˆKÜ˜1–X�Ø9:˜DŸK™K¨�N×5Ñ5Ó6ñ ä˜1–[�Ø'2·;±;¸qµ>×3LÑ3LÕ'M�×!Ñ! !Ó$ó !ñ +ùòA
s   Â9Lc                óˆ   <€ V ^8„  d   QhRS[ S[P                  ,          RS[ S[P                  ,          RS[P                  /# )r-   Ú	input_idsÚinput_embeddingsrg   )r   ri   rj   )rL   rM   s   "€r(   rN   r&  ¼  s?   ø€ ÷ &
ñ &
á™BŸH™HÕ%ð&
ñ #¡2§8¡8Õ,ð&
ñ 
�‰ñ	&
r+   c                óü  € VfŸ   Vf   \        R4      hV P                  P                  P                  4       P	                  R4      pVP                  4       P	                  R4      p\        VVV P                  P                  V4      4      p\        P                  ! VR4      pV P                  V4      pW`P                  ,          p\        P                  ! VRV P                  P                  V P                  34      # )Nz=input_embeddings must be provided when input_ids are omitted.r~   )ÚRuntimeErrorr*  ÚweightÚsquarerÍ   rƒ   Ú	as_linearri   Úargminr/  r0  Ú	unflattenr�   r$   r<   )r&   r>  r?  Únorms_embeddingÚnorms_inputÚdistanceÚresults   &&&    r(   Ú_get_per_layer_inputsÚ%Gemma4TextModel._get_per_layer_inputs¼  sä   € ð
 ÒØÒ'Ü"ØSóð ð #×/Ñ/×6Ñ6×=Ñ=Ó?×CÑCÀBÓGˆOØ*×1Ñ1Ó3×7Ñ7¸Ó;ˆKÜ'ØØØ×!Ñ!×+Ñ+Ð,<Ó=óˆHô Ÿ	š	 (¨BÓ/ˆIà×,Ñ,¨YÓ7ˆØ×;Ñ;Õ;ˆÜ�|Š|ØØØ�[‰[×*Ñ*¨D×,LÑ,LÐMó
ð 	
r+   c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[ P                  /# )r-   r?  Úper_layer_inputsrg   ©ri   rj   r   )rL   rM   s   "€r(   rN   r&  ä  s?   ø€ ÷ Vñ VáŸ(™(ðVñ #¡2§8¡8Õ,ðVñ 
�‰ñ	Vr+   c                ó  € V P                  V4      pW0P                  ,          p\        P                  ! VRV P                  P
                  V P                  34      pV P                  V4      pVf   V# W2,           V P                  ,          # r|   )	r3  r2  ri   rF  r�   r$   r<   r4  r1  )r&   r?  rN  r  s   &&& r(   Ú_project_per_layer_inputsÚ)Gemma4TextModel._project_per_layer_inputsä  s†   € ð
  $×>Ñ>Ð?OÓPÐØ3×6UÑ6UÕUÐÜ!Ÿ|š|Ø ØØ�[‰[×*Ñ*¨D×,LÑ,LÐMó 
Ðð
  $×=Ñ=Ð>RÓSÐàÒ#Ø'Ð'à$Õ7¸4×;UÑ;UÕUÐUr+   c                óF  € / p. p\        V P                  V4       F‚  w  rVVP                  V9  dL   VP                  R 8X  d   \        W4      VR &   M+VP                  R8X  d   \        WV P                  R7      VR&   VP                  W5P                  ,          4       K„  	  V# )r   r   )r(  )Úzipr-  rÚ   r
   r(  Úappend)r&   r  rð   rï   ÚmasksÚlÚcs   &&&    r(   Ú_make_masksÚGemma4TextModel._make_masks÷  s�   € ØˆØˆÜ˜Ÿ™ UÖ+‰DˆAØ�|‰| 4Ô'Ø—<‘<Ð#3Ô3Ü-BÀ1Ó-H�DÐ)Ò*Ø—\‘\Ð%8Ô8Ü0EØ¨$×*:Ñ*:ô1�DÐ,Ñ-ð �L‰L˜Ÿl™lÕ+Ö,ñ ,ð ˆr+   c                óˆ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          /# ©r-   Úinputsr?  rN  rO  )rL   rM   s   "€r(   rN   r&    s?   ø€ ÷ :ñ :á—‘ð:ñ #¡2§8¡8Õ,ð	:ñ
 #¡2§8¡8Õ,ñ:r+   c           
     ó^  € Vf   V P                  V4      pTpWPP                  ,          pV P                  '       d'   Vf   V P                  W4      pV P	                  WT4      pVe5   \        V P                  4       UUu. uF  w  rgVRRVR3,          NK  	  pppMR .\        V P                  4      ,          pVf   R .\        V P                  4      ,          pM4VR .\        V P                  4      \        V4      ,
          ,          ,           pV P                  WR4      pR.\        V P                  4      ,          p	\        \        V P                  VVV P                  V4      4       F-  w  p
w  r¼rÞpWž,          w  ppV! VVVVVVR7      w  pppVV3Wš&   K/  	  V P                  V4      # u uppi )N:NNN)r  rñ   rò   )NN)r*  r+  r<   rK  rQ  Ú	enumerater-  r%   rY  rT  r6  r.  )r&   r]  rð   r?  rN  r  r7  rý   rV  ÚintermediatesÚidxÚlayerrX  rï   Úprev_idxr  Úkvsrò   s   &&&&&             r(   ro   ÚGemma4TextModel.__call__  s¶  € ð Ò#Ø#×0Ñ0°Ó8ÐØˆØ× Ñ Õ ˆð ×+×+Ð+ØÒ'Ø#'×#=Ñ#=¸fÓ#WÐ Ø#×=Ñ=¸aÓRÐØÒ'ä9BÀ4Ç;Á;Ô9Oô Ù9O±°Ð   A q¨! ×,Ð,Ñ9Oð ñ  Ðð !%˜v¬¨D¯K©KÓ(8Õ8Ðð Š=Ø�FœS §¡Ó-Õ-‰Eà˜T˜F¤c¨$¯+©+Ó&6¼¸U»Õ&CÕDÕDˆEð × Ñ  Ó*ˆØ%˜¬¨T¯[©[Ó)9Õ9ˆÜ@IÜØ—‘ØØØ×!Ñ!Ø óöA
Ñ<ˆCÑ<�%˜D¨Oð (Õ1‰KˆC�á"ØØØØ /ØØô‰NˆAˆs�Fð #& v ˆMÓñ)A
ð, �y‰y˜‹|ÐùóO s   Á>F))r�   r+  r*  r/  r0  r<   r-  r.  r$   r1  r3  r4  r2  r6  r#   r6   r(  r_   r  )rP   rQ   rR   rS   ra   rK  rQ  rY  ro   rU   rV   rt   ru   s   @@r(   r$  r$  ˆ  s=   ù‡ € ÷1Nó 1N÷f&
ò &
÷PVò Vò&÷:÷ :ò :r+   r$  c                   ó    a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltR t]R 4       t]R 4       t	]R	 4       t
]R
 4       tR tRtVtV ;t# )ÚModeliB  c                ó    <€ V ^8„  d   QhRS[ /# )r-   Úargsr¤   )rL   rM   s   "€r(   rN   ÚModel.__annotate__C  s   ø€ ÷ Tñ T™Yñ Tr+   c                óB  <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        VP                  V n        VP                  V n        V P                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# R# )Fr’   N)r`   ra   ri  r.   r$  ÚmodelrA   rG   r…   r”   r/   r6   Úlm_head)r&   ri  rb   s   &&€r(   ra   ÚModel.__init__C  su   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ$ TÓ*ˆŒ
Ø'+×'CÑ'CˆÔ$Ø#'×#;Ñ#;ˆÔ Ø×'×'Ð'ÜŸ9š9 T×%5Ñ%5°t·±ÈUÔSˆDŽLñ (r+   c                óˆ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          /# r\  rO  )rL   rM   s   "€r(   rN   rj  M  s?   ø€ ÷ ñ á—‘ðñ #¡2§8¡8Õ,ð	ñ
 #¡2§8¡8Õ,ñr+   c                ó
  € V P                  VVVVR 7      pV P                  '       d'   V P                   P                  P                  V4      pMV P	                  V4      pV P
                  e   \        V P
                  V4      pV# ))rð   r?  rN  )rl  rG   r*  rD  rm  rA   rz   )r&   r]  rð   r?  rN  Úouts   &&&&& r(   ro   ÚModel.__call__M  s|   € ð �j‰jØØØ-Ø-ð	 ó 
ˆð ×#×#Ð#Ø—*‘*×)Ñ)×3Ñ3°CÓ8‰Cà—,‘,˜sÓ#ˆCØ×'Ñ'Ò3Ü × <Ñ <¸cÓBˆCØˆ
r+   c           
     ó  a€ / pVP                  4        Fè  w  op\        ;QJ d    V3R  lR 4       F  '       g   K   RM	  RM! V3R  lR 4       4      '       d   KH  SP                  R4      '       dU   SP                  R4      p\	        \
        P                  \
        P                  ! V^RR7      4      w  rVWRV R2&   WbV R2&   K³  SP                  R4      '       d   SP                  R	4      pW2V R
2&   Kä  W2S&   Kê  	  V# )c              3   ó.   <"  € T F
  pVS9   x € K  	  R # 5ir_   r¿   )Ú.0ÚsÚks   & €r(   Ú	<genexpr>Ú!Model.sanitize.<locals>.<genexpr>e  s    øé € ð 	ñ�Að �Q–óùs   ƒTFz.experts.gate_up_projz.gate_up_projr³   z.switch_glu.gate_proj.weightz.switch_glu.up_proj.weightz.experts.down_projz
.down_projz.switch_glu.down_proj.weight)zself_attn.rotary_embÚ	input_maxÚ	input_minÚ
output_maxÚ
output_minrÌ   )ÚitemsÚanyÚendswithÚremovesuffixÚmapri   Ú
contiguousÚsplit)r&   ÚweightsÚ	sanitizedÚvrØ   r‡   Úuprw  s   &&     @r(   ÚsanitizeÚModel.sanitizeb  sô   ø€ Øˆ	Ø—M‘M–O‰DˆAˆqß‹sô 	ñó	�s�sŠsô 	ñó	÷ 	ò 	ñ à�z‰zÐ1×2Ò2Ø—~‘~ oÓ6�ÜœrŸ}™}¬b¯hªh°q¸!À"Ô.EÓF‘�ØCG˜T˜FÐ">Ð?Ñ@ØAC˜T˜FÐ"<Ð=Ñ>Ùà�z‰zÐ.×/Ò/Ø—~‘~ lÓ3�ØCD˜T˜FÐ">Ð?Ñ@Ùà�a‹Lñ3 $ð6 Ðr+   c                ó   € R  pV# )c                 ó@   € V P                  R 4      '       d   R^@R^/# R# )zrouter.projÚ
group_sizeÚbitsT)r€  )Úpathrý   s   &&r(   Ú	predicateÚ(Model.quant_predicate.<locals>.predicateƒ  s#   € Ø�}‰}˜]×+Ò+Ø$ b¨&°!Ð4Ð4Ùr+   r¿   )r&   r�  s   & r(   Úquant_predicateÚModel.quant_predicate�  s   € ò	ð
 Ðr+   c                ó.   € V P                   P                  # r_   )rl  r-  ©r&   s   &r(   r-  ÚModel.layersŠ  s   € à�z‰z× Ñ Ð r+   c                ó.   € V P                   P                  # r_   )ri  r2   r•  s   &r(   r2   ÚModel.head_dimŽ  s   € à�y‰y×!Ñ!Ð!r+   c                ó.   € V P                   P                  # r_   )ri  r8   r•  s   &r(   rà   ÚModel.n_kv_heads’  s   € à�y‰y×,Ñ,Ð,r+   c                ój  € V P                   P                  V P                   P                  ,
          p. p\        V4       Fp  pV P                   P                  V,          R 8X  d   VP                  \        4       4       K@  VP                  \        V P                   P                  ^ R7      4       Kr  	  V# )r   )Úmax_sizeÚkeep)	ri  r$   r:   r,  r"   rU  r   r   r>   )r&   Úfirst_kv_sharedÚcachesr7  s   &   r(   Ú
make_cacheÚModel.make_cache–  s…   € ØŸ)™)×5Ñ5¸¿	¹	×8VÑ8VÕVˆØˆÜ�Ö'ˆAØ�y‰y×$Ñ$ QÕ'Ð+;Ô;Ø—‘œg›iÖ(à—‘Ü#Ø!%§¡×!9Ñ!9Øôöñ	 (ð ˆr+   )ri  rA   rm  rl  r.   rG   )NNN)rP   rQ   rR   rS   ra   ro   r‰  Úpropertyr’  r-  r2   rà   r   rU   rV   rt   ru   s   @@r(   rg  rg  B  sz   ù‡ € ÷Tó T÷ò ò*ð> ñó ðð ñ!ó ð!ð ñ"ó ð"ð ñ-ó ð-÷ò r+   rg  ))Údataclassesr   Ú	functoolsr   Útypingr   r   r   r   Úmlx.coreÚcoreri   Úmlx.nnr…   rØ   r	   r
   r   rð   r   r   r   Ú
rope_utilsr   Úswitch_layersr   r   ÚModulerY   Úcompilerz   rƒ   rˆ   rŠ   r¡   r¼   rÁ   rÒ   r  r$  rg  r¿   r+   r(   Ú<module>r­     sA  ðõ "Ý ß ,Ó ,å Ý ç TÑ Tß 7Ñ 7Ý 'Ý $ð ô5�ó 5ó ð5ôp3�R—Y‘Yô 3ñ 	ˆ�‰˜tÔ$ñ*ó %ð*ñ 	ˆ�‰˜tÔ$ñ0ó %ð0ñ 	ˆ�‰˜tÔ$ñ$ó %ð$ôIˆ"�)‰)ô Iô$,ˆR�Y‰Yô ,ô:ˆB�I‰Iô ôˆb�i‰iô ô.d;�—	‘	ô d;ôNn$�2—9‘9ô n$ôbw�b—i‘iô wôtaˆB�I‰Iö ar+   