+
    TV-jrS  ã                   ó¬  € ^ RI t ^ RIHt ^ RIHt ^ RIHtHtHtH	t	 ^ RI
Ht ^ RIHt ^ RIHtHt ^RIHtHtHt ^RIHtHt ] ! R R	]4      4       t] ! R
 R]4      4       t ! R R]P6                  4      t ! R R]P6                  4      t ! R R]P6                  4      t]! ]P>                  RR7      R 4       t  ! R R]P6                  4      t! ! R R]P6                  4      t" ! R R]P6                  4      t#]! ]P>                  RR7      R 4       t$ ! R R]P6                  4      t% ! R R]P6                  4      t& ! R  R!]P6                  4      t'R# )"é    N)Ú	dataclass)Úpartial)ÚAnyÚDictÚListÚOptional)Útree_flattenÚtree_unflatten)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCacheÚRotatingKVCachec                   ó0   a € ] tR t^t o RtV 3R ltRtV tR# )Ú
TextConfigNc                óv  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[ ,          ;R&   S[S[,          ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[,          ;R&   # )é   Ú
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚhead_dimÚrms_norm_epsÚ
vocab_sizeÚnum_key_value_headsÚnum_kv_shared_layersÚvocab_size_per_layer_inputÚsliding_windowÚmax_position_embeddingsÚrope_local_base_freqÚ
rope_thetaÚfinal_logit_softcappingÚlayer_typesÚactivation_sparsity_patternÚhidden_size_per_layer_inputÚaltup_num_inputsÚaltup_coef_clipÚaltup_correct_scaleÚaltup_active_idxÚlaurel_rankÚrope_scaling)ÚstrÚintÚfloatr   Úboolr   r   )ÚformatÚ__classdict__s   "€Úf/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/gemma3n.pyÚ__annotate__ÚTextConfig.__annotate__   sQ  ø‡ ‚ á�Oñ ñ Ññ ñ Ññ	 ñ
 Ññ ñ Ññ ñ �Mñ ñ Ññ ñ �Oñ ñ Ññ ñ Ññ ñ !$Ñ#ñ ñ Ññ ñ !Ñ ñ ñ  Ññ ñ  Ññ! ñ" #Ñ"ñ# ñ$ ‘c•Ññ% ñ& "&¡e¥Ñ,ñ' ñ( "%Ñ$ñ) ñ* Ññ+ ñ, Ññ- ñ. Ññ/ ñ0 Ññ1 ñ2 Ññ3 ñ4 ™4•.Ñ'ò5 ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r,   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©r2   s   @r3   r   r      s   ø‡ € ð4 $(€L÷5 ƒ r6   r   c                   ó,   a € ] tR t^-t o V 3R ltRtV tR# )Ú	ModelArgsc                ó2   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   # )r   r   Útext_config)r-   Údict)r1   r2   s   "€r3   r4   ÚModelArgs.__annotate__-   s   ø‡ ‚ á�Oñ ñ Ñò r6   r7   N)r8   r9   r:   r;   r<   r=   r>   r?   s   @r3   rA   rA   -   s   ø‡ ‡ „ r6   rA   c                   óH   a a€ ] tR t^3t oRV3R lV 3R llltR tRtVtV ;t# )Ú
RMSNoScalec                ó    <€ V ^8„  d   QhRS[ /# )r   Úeps)r/   )r1   r2   s   "€r3   r4   ÚRMSNoScale.__annotate__4   s   ø€ ÷ ñ ™Eñ r6   c                ó0   <€ \         SV `  4        Wn        R # ©N)ÚsuperÚ__init__rI   )ÚselfrI   Ú	__class__s   &&€r3   rN   ÚRMSNoScale.__init__4   s   ø€ Ü‰ÑÔØŽr6   c                óX   € \         P                  P                  VR V P                  4      # rL   )ÚmxÚfastÚrms_normrI   )rO   Úxs   &&r3   Ú__call__ÚRMSNoScale.__call__8   s   € Ü�w‰w×Ñ  4¨¯©Ó2Ð2r6   ©rI   )gñhãˆµøä>©	r8   r9   r:   r;   rN   rW   r=   r>   Ú__classcell__©rP   r2   s   @@r3   rG   rG   3   s   ù‡ € ÷õ ÷3ò 3r6   rG   c                   óT   a a€ ] tR t^<t oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚGemma3nLaurelBlockz Learned Augmented Residual Layerc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úconfig©r   )r1   r2   s   "€r3   r4   ÚGemma3nLaurelBlock.__annotate__?   s   ø€ ÷ 
ñ 
™zñ 
r6   c                óÒ  <€ \         SV `  4        Wn        \        P                  ! V P                  P
                  V P                  P                  R R7      V n        \        P                  ! V P                  P                  V P                  P
                  R R7      V n        \        P                  ! V P                  P
                  V P                  P                  R7      V n        R# ©F©Úbias©ÚdimsrI   N)rM   rN   ra   ÚnnÚLinearr   r+   Úlinear_leftÚlinear_rightÚRMSNormr   Úpost_laurel_norm©rO   ra   rP   s   &&€r3   rN   ÚGemma3nLaurelBlock.__init__?   sš   ø€ Ü‰ÑÔØŒäŸ9š9Ø�K‰K×#Ñ# T§[¡[×%<Ñ%<À5ô
ˆÔô ŸIšIØ�K‰K×#Ñ# T§[¡[×%<Ñ%<À5ô
ˆÔô !#§
¢
Ø—‘×(Ñ(Ø—‘×(Ñ(ô!
ˆÖr6   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r   rV   Úreturn©rS   Úarray)r1   r2   s   "€r3   r4   rc   N   s#   ø€ ÷ #ñ #™"Ÿ(™(ð #¡r§x¡xñ #r6   c                óx   € V P                  V4      pV P                  V4      pV P                  V4      pW,           # rL   )rl   rm   ro   )rO   rV   Úlaurel_xÚnormed_laurel_xs   &&  r3   rW   ÚGemma3nLaurelBlock.__call__N   s;   € Ø×#Ñ# AÓ&ˆØ×$Ñ$ XÓ.ˆØ×/Ñ/°Ó9ˆØÕ"Ð"r6   )ra   rl   rm   ro   )
r8   r9   r:   r;   Ú__doc__rN   rW   r=   r>   r[   r\   s   @@r3   r^   r^   <   s!   ù‡ € Ù*÷
ó 
÷#÷ #ð #r6   r^   c                   óT   a a€ ] tR t^Ut oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma3nAttentionc                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# ©r   ra   Ú	layer_idxÚis_kv_shared_layer©r   r.   r0   )r1   r2   s   "€r3   r4   ÚGemma3nAttention.__annotate__V   s"   ø€ ÷ 
ñ 
™zð 
±cð 
Étñ 
r6   c                óÊ  <€ \         SV `  4        VP                  V,          R 8H  V n        VP                  pVP
                  ;V n        pVP                  ;V n        pWV,          V n	        VP                  ;V n
        pW n        RV n        \        P                  ! WEV,          RR7      V n        \        P                  ! WFV,          RR7      V n        \        P                  ! WFV,          RR7      V n        \        P                  ! WW,          VRR7      V n        \        P&                  ! VP                  VP(                  R7      V n        \        P&                  ! VP                  VP(                  R7      V n        \/        VP(                  R7      V n        W0n        \        P4                  ! TRV P                  '       d   VP6                  MVP8                  R7      V n        R# )Úsliding_attentionç      ð?Frf   rh   rY   )ÚtraditionalÚbaseN)rM   rN   r$   Ú
is_slidingr   r   Ún_headsr   Ú
n_kv_headsÚrepeatsr   r€   Úscalerj   rk   Úq_projÚk_projÚv_projÚo_projrn   r   Úq_normÚk_normrG   Úv_normr�   ÚRoPEr!   r"   Úrope)	rO   ra   r€   r�   ÚdimrŠ   r‹   r   rP   s	   &&&&    €r3   rN   ÚGemma3nAttention.__init__V   sV  ø€ Ü‰ÑÔØ ×,Ñ,¨YÕ7Ð;NÑNˆŒà× Ñ ˆØ!'×!;Ñ!;Ð;ˆŒ�wØ'-×'AÑ'AÐAˆŒ˜*ØÕ,ˆŒØ#)§?¡?Ð2ˆŒ˜Ø"ŒàˆŒ
ä—i’i ¨xÕ%7¸eÔDˆŒÜ—i’i °(Õ%:ÀÔGˆŒÜ—i’i °(Õ%:ÀÔGˆŒÜ—i’i Õ 2°C¸eÔDˆŒä—j’j f§o¡o¸6×;NÑ;NÔOˆŒÜ—j’j f§o¡o¸6×;NÑ;NÔOˆŒÜ  V×%8Ñ%8Ô9ˆŒà"4Ôä—G’GØØà/3¯¯¨�×+Ò+ÀF×DUÑDUô	
ˆŽ	r6   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# )r   rV   ÚmaskÚcachert   ©rS   rv   r   r   )r1   r2   s   "€r3   r4   rƒ   v   sH   ø€ ÷ *#ñ *#á�8‰8ð*#ñ ‘r—x‘xÕ ð*#ñ ™�}ð	*#ñ
 
�‰ñ*#r6   c           	     ó¢  € VP                   w  rEpV P                  V4      pVP                  WERV P                  4      pV P	                  V4      p^ pV P
                  '       d    Ve   VP                  w  ršVP                  pMÛVe   VP                  pV P                  V4      P                  WERV P                  4      p	V P                  V	4      p	V	P                  ^ ^^^4      p	V P                  W˜R7      p	V P                  V4      P                  WERV P                  4      p
V P                  V
4      p
V
P                  ^ ^^^4      p
Ve   VP                  Wš4      w  ršVP                  ^ ^^^4      pV P                  WxR7      p\        WyW£V P                   VR7      pVP                  ^ ^^^4      P                  WER4      pV P#                  V4      # )é   )Úoffset)r›   r�   rš   éÿÿÿÿ)ÚshaperŽ   Úreshaper   r’   r�   ÚstaterŸ   r�   r“   Ú	transposer–   r�   r”   Úupdate_and_fetchr   r�   r‘   )rO   rV   rš   r›   ÚBÚLÚ_ÚqueriesrŸ   ÚkeysÚvaluesÚoutputs   &&&&        r3   rW   ÚGemma3nAttention.__call__v   sœ  € ð —'‘'‰ˆˆaà—+‘+˜a“.ˆØ—/‘/ !¨¨D¯M©MÓ:ˆØ—+‘+˜gÓ&ˆàˆØ×"×"Ð" uÒ'8à Ÿ;™;‰LˆDØ—\‘\‰Fð Ò ØŸ™�Ø—;‘;˜q“>×)Ñ)¨!°°D·M±MÓBˆDØ—;‘;˜tÓ$ˆDØ—>‘> ! Q¨¨1Ó-ˆDØ—9‘9˜T�9Ó1ˆDà—[‘[ “^×+Ñ+¨A°"°d·m±mÓDˆFØ—[‘[ Ó(ˆFØ×%Ñ% a¨¨A¨qÓ1ˆFàÒ Ø$×5Ñ5°dÓC‘�à×#Ñ# A q¨!¨QÓ/ˆØ—)‘)˜G�)Ó3ˆä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆà�{‰{˜6Ó"Ð"r6   )r   r�   r‰   r“   r�   r€   rŠ   r‹   r‘   r’   rŽ   rŒ   r–   r�   r”   r�   ©NNrZ   r\   s   @@r3   r}   r}   U   s   ù‡ € ÷
ó 
÷@*#÷ *#ò *#r6   r}   T)Ú	shapelessc                 ó  € \         P                  ! V RRR7      p\         P                  ! V RRR7      pW#VP                  VP                  4      ,          ,           p\
        P                  ! \         P                  ! ^ W,
          4      4      # )rž   T©ÚaxisÚkeepdimsr    )rS   ÚmeanÚstdÚastypeÚdtyperj   Úgelu_approxÚmaximum)ÚinputsÚstd_multiplierÚinputs_meanÚ
inputs_stdÚcutoff_xs   &&   r3   Ú	gelu_topkr¿   £   sa   € ä—'’'˜& r°DÔ9€KÜ—’˜ R°$Ô7€JØ¨.×*?Ñ*?À
×@PÑ@PÓ*QÕQÕQ€HÜ�>Š>œ"Ÿ*š* Q¨Õ(9Ó:Ó;Ð;r6   c                   óT   a a€ ] tR t^«t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚMLPc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   ra   r€   )r   r.   )r1   r2   s   "€r3   r4   ÚMLP.__annotate__¬   s   ø€ ÷ ñ ™zð ±cñ r6   c                ó$  <€ \         SV `  4        Wn        VP                  V n        \	        VP
                  \        4      '       d   VP
                  V,          MVP
                  V n        \        P                  ! V P                  V P
                  R R7      V n	        \        P                  ! V P                  V P
                  R R7      V n
        \        P                  ! V P
                  V P                  R R7      V n        VP                  e   VP                  V,          V n        MRV n        V P                  ^ 8”  dQ   \        P                  ! R4      \         P"                  ! ^V P                  ,          ^,
          4      ,          V n        R# R# )Frf   Nç        ç       @)rM   rN   ra   r   Ú
isinstancer   Úlistrj   rk   Ú	gate_projÚup_projÚ	down_projr%   Úactivation_sparsityÚmathÚsqrtrS   ÚerfinvÚ_std_multiplier)rO   ra   r€   rP   s   &&&€r3   rN   ÚMLP.__init__¬   s)  ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔô ˜&×2Ñ2´D×9Ò9ð ×$Ñ$ YÖ/à×)Ñ)ð 	Ôô
 Ÿš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒØ×-Ñ-Ò9Ø'-×'IÑ'IÈ)Õ'TˆDÕ$à'*ˆDÔ$Ø×#Ñ# aÔ'Ü#'§9¢9¨S£>´B·I²IØ�D×,Ñ,Õ,¨qÕ0ó5õ $ˆDÖ ñ (r6   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   rV   ru   )r1   r2   s   "€r3   r4   rÃ   Á   s   ø€ ÷ ñ ™"Ÿ(™(ñ r6   c                óò   € V P                  V4      pV P                  R 8”  d   \        W P                  4      pM\        P
                  ! V4      pV P                  V4      pV P                  W4,          4      pV# )rÅ   )rÉ   rÌ   r¿   rÐ   rj   r¸   rÊ   rË   )rO   rV   rÉ   ÚactivationsrÊ   rË   s   &&    r3   rW   ÚMLP.__call__Á   s`   € Ø—N‘N 1Ó%ˆ	Ø×#Ñ# cÔ)Ü# I×/CÑ/CÓD‰KäŸ.š.¨Ó3ˆKØ—,‘,˜q“/ˆØ—N‘N ;Õ#8Ó9ˆ	ØÐr6   )rÐ   rÌ   ra   rË   rÉ   r   r   rÊ   )r   rZ   r\   s   @@r3   rÁ   rÁ   «   s   ù‡ € ÷õ ÷*÷ ð r6   rÁ   c                   óx   a a€ ] tR t^Ìt oRtV3R lV 3R lltV3R lR ltV3R lR ltV3R lR	 ltR
t	Vt
V ;t# )ÚGemma3nAltUpzAlternating Updates (AltUp)c                ó    <€ V ^8„  d   QhRS[ /# r`   rb   )r1   r2   s   "€r3   r4   ÚGemma3nAltUp.__annotate__Ï   s   ø€ ÷ 
ñ 
™zñ 
r6   c                óÌ  <€ \         SV `  4        Wn        \        P                  ! V P                  P
                  34      V n        \        P                  ! V P                  P                  V P                  P                  R R7      V n
        \        P                  ! V P                  P                  V P                  P                  ^,          R R7      V n        \        P                  ! V P                  P
                  V P                  P                  R R7      V n        \        P                  ! V P                  P
                  V P                  P                  R7      V n        R# re   )rM   rN   ra   rS   Úzerosr   Úcorrect_output_scalerj   rk   r'   Úcorrection_coefsÚprediction_coefsÚmodality_routerrn   r   Úrouter_normrp   s   &&€r3   rN   ÚGemma3nAltUp.__init__Ï   sì   ø€ Ü‰ÑÔØŒä$&§H¢H¨d¯k©k×.EÑ.EÐ-GÓ$HˆÔ!Ü "§	¢	Ø�K‰K×(Ñ(¨$¯+©+×*FÑ*FÈUô!
ˆÔô !#§	¢	Ø�K‰K×(Ñ(¨$¯+©+×*FÑ*FÈÕ*IÐPUô!
ˆÔô  "ŸyšyØ�K‰K×#Ñ# T§[¡[×%AÑ%AÈô 
ˆÔô Ÿ:š:Ø—‘×(Ñ(Ø—‘×(Ñ(ô
ˆÖr6   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rs   ru   )r1   r2   s   "€r3   r4   rÙ   â   s#   ø€ ÷ ñ ©2¯8©8ð ¹¿¹ñ r6   c                óð   € V P                  V4      V P                  P                  R,          ,          pV P                  V4      P	                  \
        P                  4      p\
        P                  ! V4      # )r†   g      ð¿)rà   ra   r   rß   r¶   rS   Úfloat32Útanh)rO   rV   Úrouter_inputsÚrouteds   &&  r3   Úcompute_router_modalitiesÚ&Gemma3nAltUp.compute_router_modalitiesâ   sS   € Ø×(Ñ(¨Ó+¨t¯{©{×/FÑ/FÈÕ/LÕMˆØ×%Ñ% mÓ4×;Ñ;¼B¿J¹JÓGˆÜ�wŠw�v‹Ðr6   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rs   ru   )r1   r2   s   "€r3   r4   rÙ   ç   s#   ø€ ÷ +ñ +™Ÿ™ð +¡b§h¡hñ +r6   c                ó®  € V P                  WP                  P                  ,          4      pV P                  P                  P                  \        P                  4      V P                  n        V P                  P                  ee   \        P                  ! V P                  P                  V P                  P                  ) V P                  P                  4      V P                  n        V P                  V4      P                  ! . VP                  R R OV P                  P                  NV P                  P                  N5!  P                  ^ ^^^4      pVP                  \        P                  4      pVP                  ^^^^ 4      p\        P                  ! WS4      pVP                  ^^ ^^4      pWd,          pVP                  VP                  4      # )Nr    )rè   ra   r*   rÞ   Úweightr¶   rS   rä   r(   Úclipr¢   r¡   r'   r¤   Úmatmulr·   )rO   rV   Ú
modalitiesÚ	all_coefsÚx_upÚ
x_permutedÚpredictionss   &&     r3   ÚpredictÚGemma3nAltUp.predictç   sw  € Ø×3Ñ3°A·k±k×6RÑ6RÕ4SÓTˆ
à'+×'<Ñ'<×'CÑ'C×'JÑ'JÌ2Ï:É:Ó'Vˆ×ÑÔ$à�;‰;×&Ñ&Ò2Ü+-¯7ª7Ø×%Ñ%×,Ñ,Ø—‘×,Ñ,Ð,Ø—‘×+Ñ+ó,ˆD×!Ñ!Ô(ð ×!Ñ! *Ó-ß‰Wðð à×!Ñ! # 2Ð&ðð —‘×,Ñ,ðð —‘×,Ñ,ó	÷ ‰Y�q˜!˜Q Ó"ð 	ð �x‰xœŸ
™
Ó#ˆØ—^‘^ A q¨!¨QÓ/ˆ
Ü—i’i 
Ó6ˆØ!×+Ñ+¨A¨q°!°QÓ7ˆØÕˆØ×!Ñ! !§'¡'Ó*Ð*r6   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   ró   Ú	activatedru   )r1   r2   s   "€r3   r4   rÙ     s#   ø€ ÷ 1ñ 1¡2§8¡8ð 1¹¿¹ñ 1r6   c                ó°  € V P                  V4      pV P                  P                  P                  \        P
                  4      V P                  n        V P                  P                  ee   \        P                  ! V P                  P                  V P                  P                  ) V P                  P                  4      V P                  n        V P                  V4      R,           pWP                  P                  ,          pW%,
          pVP                  ^^ 4      pVR ,          VR,          ,          pWq,          pVP                  VP                  4      # )Nr†   ).N)rè   rÝ   rì   r¶   rS   rä   ra   r(   rí   r*   Úmoveaxisr·   )rO   ró   r÷   rï   rð   Úactive_xÚ
innovationÚ	correcteds   &&&     r3   ÚcorrectÚGemma3nAltUp.correct  s  € Ø×3Ñ3°IÓ>ˆ
à'+×'<Ñ'<×'CÑ'C×'JÑ'JÌ2Ï:É:Ó'Vˆ×ÑÔ$à�;‰;×&Ñ&Ò2Ü+-¯7ª7Ø×%Ñ%×,Ñ,Ø—‘×,Ñ,Ð,Ø—‘×+Ñ+ó,ˆD×!Ñ!Ô(ð ×)Ñ)¨*Ó5¸Õ;ˆ	ØŸ{™{×;Ñ;Õ<ˆØÕ)ˆ
à×&Ñ& q¨!Ó,ˆ	Ø˜tÕ$ y°Õ';Õ;ˆ	ØÕ ˆ	à×Ñ 	§¡Ó0Ð0r6   )ra   rÜ   rÝ   rß   rÞ   rà   )r8   r9   r:   r;   r{   rN   rè   rô   rý   r=   r>   r[   r\   s   @@r3   r×   r×   Ì   s5   ù‡ € Ù%÷
ó 
÷&ð ÷
+ð +÷:1÷ 1ð 1r6   r×   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma3nDecoderLayeri  c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# r   r‚   )r1   r2   s   "€r3   r4   Ú Gemma3nDecoderLayer.__annotate__  s"   ø€ ÷ '
ñ '
™zð '
±cð '
Étñ '
r6   c                óì  <€ \         SV `  4        Wn        VP                  V n        W n        \        WV4      V n        \        WR 7      V n        \        P                  ! V P                  VP                  R7      V n        \        P                  ! V P                  VP                  R7      V n        \        P                  ! V P                  VP                  R7      V n        \        P                  ! V P                  VP                  R7      V n        V P                  P                   V n        VP"                  V n        \%        V4      V n        \)        V4      V n        \        P,                  ! V P                  V P"                  RR7      V n        \        P,                  ! V P"                  V P                  RR7      V n        \        P                  ! V P                  VP                  R7      V n        R# ))r€   rY   Frf   N)rM   rN   ra   r   r€   r}   Ú	self_attnrÁ   Úmlprj   rn   r   Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormr‰   r&   r×   Úaltupr^   Úlaurelrk   Úper_layer_input_gateÚper_layer_projectionÚpost_per_layer_input_norm)rO   ra   r€   r�   rP   s   &&&&€r3   rN   ÚGemma3nDecoderLayer.__init__  s{  ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ"ŒÜ)¨&Ð=OÓPˆŒÜ�vÔ3ˆŒÜ!ŸzšzØ×ÑØ×#Ñ#ô 
ˆÔô
 )+¯
ª
Ø×ÑØ×#Ñ#ô)
ˆÔ%ô *,¯ªØ×ÑØ×#Ñ#ô*
ˆÔ&ô +-¯*ª*Ø×ÑØ×#Ñ#ô+
ˆÔ'ð Ÿ.™.×3Ñ3ˆŒà+1×+MÑ+MˆÔ(ä! &Ó)ˆŒ
Ü(¨Ó0ˆŒÜ$&§I¢IØ×Ñ˜d×>Ñ>ÀUô%
ˆÔ!ô %'§I¢IØ×,Ñ,¨d×.>Ñ.>ÀUô%
ˆÔ!ô *,¯ªØ×ÑØ×#Ñ#ô*
ˆÖ&r6   c          	      óž   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[S[ P                  ,          /# )r   rV   rš   r›   Úper_layer_inputrœ   )r1   r2   s   "€r3   r4   r  E  sM   ø€ ÷ ,%ñ ,%á�8‰8ð,%ñ ‘r—x‘xÕ ð,%ñ ™�}ð	,%ñ
 "¡"§(¡(Õ+ñ,%r6   c                ól  € V P                   P                  V4      pWPP                  P                  ,          pV P	                  V4      pV P                  V4      pV P                  VVV4      p	V P                  V	4      p	Wi,           p
W¨,           R,          pV P                  V4      pV P                  V4      pV P                  V4      pW¾,           pV P                   P                  W_4      pVV P                  P                  ,          pV P                  P                  '       d   VV P                   P                  ,          pV P                  V4      p\        P                   ! V4      p\"        P$                  ! VV4      pV P'                  V4      pV P)                  V4      pVR,          V,           VR&   V# )rÆ   ºrž   NNçÍ;fž æ?)r
  rô   ra   r*   r  r  r  r  r  r  r	  rý   r)   rÜ   r  rj   r¸   rS   Úmultiplyr  r  )rO   rV   rš   r›   r  ró   Úactive_predictionÚactive_prediction_normedÚlaurel_outputÚattnÚ
attn_gatedÚattn_laurelÚ	attn_normÚattn_ffwÚattn_ffw_normÚattn_ffw_laurel_gatedÚcorrected_predictionsÚfirst_predictions   &&&&&             r3   rW   ÚGemma3nDecoderLayer.__call__E  s„  € ð —j‘j×(Ñ(¨Ó+ˆØ'¯©×(DÑ(DÕEÐà#'×#7Ñ#7Ð8IÓ#JÐ ØŸ™Ð$<Ó=ˆà�~‰~Ø$ØØó
ˆð ×,Ñ,¨TÓ2ˆà&Õ-ˆ
Ø!Õ1°iÕ@ˆà×2Ñ2°;Ó?ˆ	Ø—8‘8˜IÓ&ˆØ×7Ñ7¸ÓAˆØ +Õ ;ÐØ $§
¡
× 2Ñ 2°;Ó VÐà0°·±×1MÑ1MÕNÐØ�;‰;×*×*Ð*Ø/°$·*±*×2QÑ2QÕQÐà×4Ñ4Ð5EÓFÐÜŸ>š>Ð*:Ó;ÐäŸ;š;Ð'7¸ÓIÐà×4Ñ4Ð5EÓFÐØ×9Ñ9Ð:JÓKÐà$9¸"Õ$=Ð@PÕ$PÐ˜bÑ!à$Ð$r6   )r
  ra   r   r&   r  r‰   r  r€   r  r  r  r  r	  r  r  r  ©NNNrZ   r\   s   @@r3   r   r     s   ù‡ € ÷'
ó '
÷R,%÷ ,%ò ,%r6   r   c                 óN   € \         P                  ! W,          4      pW ,          pV# rL   )rS   rå   )ÚsoftcaprV   Úouts   && r3   Úlogit_softcapr'  t  s   € ä
�'Š'�!•+Ó
€CØ
�-€CØ€Jr6   c                   ó~   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltV3R lR ltV3R lR	 ltR
 tRt	Vt
V ;t# )ÚLanguageModeli{  c                ó    <€ V ^8„  d   QhRS[ /# r`   rb   )r1   r2   s   "€r3   r4   ÚLanguageModel.__annotate__|  s   ø€ ÷ LSñ LS™zñ LSr6   c           	     óŠ  <€ \         S	V `  4        Wn        VP                  V n        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  VP                  ,
          V n
        \        P                  ! VP
                  VP                  4      V n        \        VP                  4       Uu. uF  p\        VVW P                  8¬  R 7      NK  	  upV n        \        P                  ! VP                  VP                  VP                  ,          4      V n        \        P$                  ! VP                  VP                  VP                  ,          RR7      V n        \        P(                  ! VP                  VP*                  R7      V n        \        ^V P                  P.                  4       Uu. uF0  p\        P$                  ! VP                  VP                  RR7      NK2  	  upV n        \        ^V P                  P.                  4       Uu. uF0  p\        P$                  ! VP                  VP                  RR7      NK2  	  upV n        \        P(                  ! VP                  VP*                  R7      V n        VP6                  P9                  R4      V n        VP6                  P9                  R4      V n        VP>                  V n        VP6                  RV P                   p\A        V4      ^,
          VRRR	1,          P9                  R4      ,
          p\A        V4      ^,
          VRRR	1,          P9                  R4      ,
          p. V n!        \E        V P                  P6                  4       F†  w  rxWpP                  8  d   V PB                  PG                  V4       K2  VR8X  d   V PB                  PG                  V4       KV  VR8X  d   V PB                  PG                  V4       Kz  \I        RV 24      h	  R# u upi u upi u upi )
)ra   r€   r�   Frf   rh   rY   r…   Úfull_attentionNúUnknown layer type: r    )%rM   rN   ra   r   r&   r   r   r   r#   r   Úfirst_kv_shared_layer_idxrj   Ú	EmbeddingÚembed_tokensÚranger   ÚlayersÚembed_tokens_per_layerrk   Úper_layer_model_projectionrn   r   Úper_layer_projection_normr'   Úaltup_projectionsÚaltup_unembed_projectionsÚnormr$   ÚindexÚfirst_sliding_idxÚfirst_full_idxr   ÚlenÚlayer_idx_to_cache_idxÚ	enumerateÚappendÚNotImplementedError)
rO   ra   r€   r¨   Úconcrete_layersÚshared_full_idxÚshared_sliding_idxÚiÚ
layer_typerP   s
   &&       €r3   rN   ÚLanguageModel.__init__|  s‰  ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ+1×+MÑ+MˆÔ(Ø ×+Ñ+ˆŒØ*0×*KÑ*KˆÔ'Ø!'×!9Ñ!9ˆÔØ'-×'EÑ'EˆÔ$à×$Ñ$ v×'BÑ'BÕBð 	Ô&ô ŸLšL¨×):Ñ):¸F×<NÑ<NÓOˆÔô # 6×#;Ñ#;Ô<ó
ñ =�	ô  ØØ#Ø#,×0NÑ0NÑ#N÷ñ
 =ñ
ˆŒô ')§l¢lØ×-Ñ-Ø×$Ñ$ v×'IÑ'IÕIó'
ˆÔ#ô
 +-¯)ª)Ø×ÑØ×$Ñ$ v×'IÑ'IÕIØô+
ˆÔ'ô *,¯ªØ×3Ñ3Ø×#Ñ#ô*
ˆÔ&ô ˜1˜dŸk™k×:Ñ:Ô;ó"
á;�ô �IŠI�f×(Ñ(¨&×*<Ñ*<À5×IÙ;ñ"
ˆÔô ˜1˜dŸk™k×:Ñ:Ô;ó*
á;�ô �IŠI�f×(Ñ(¨&×*<Ñ*<À5×IÙ;ñ*
ˆÔ&ô
 —J’JØ×ÑØ×#Ñ#ô
ˆŒ	ð
 "(×!3Ñ!3×!9Ñ!9Ð:MÓ!NˆÔØ$×0Ñ0×6Ñ6Ð7GÓHˆÔØ$×3Ñ3ˆÔà ×,Ñ,Ð-M¨t×/MÑ/MÐNˆä�Ó  1Õ$ ±t¸°tÕ'<×'BÑ'BÐCSÓ'TÕTð 	ô �Ó  1Õ$ ±t¸°tÕ'<×'BÑ'BÐCVÓ'WÕWð 	ð ')ˆÔ#Ü& t§{¡{×'>Ñ'>Ö?‰MˆAØ×1Ñ1Ô1Ø×+Ñ+×2Ñ2°1Ö5àÐ!1Ô1Ø×/Ñ/×6Ñ6°ÖGØÐ#6Ô6Ø×/Ñ/×6Ñ6Ð7IÖJä-Ð0DÀZÀLÐ.QÓRÐRó @ùòk
ùò2"
ùò
*
s   Ã%#P6Ç'6P;É6Q c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r   rº   Úinput_embeddingsru   )r1   r2   s   "€r3   r4   r+  Ê  s+   ø€ ÷ Bñ Bá—‘ðBñ Ÿ(™(ñ	Br6   c                ó  € Vf+   V P                  V4      V P                  R,          ,          pMTpV P                  V4      pV P                  WE4      pVf   R .\	        V P
                  4      ,          p\        VW P                  ,          4      p\        VW P                  ,          V P                  R7      pTp\        P                  ! V^,          R	RR7      R,          p	V.p
T
P                  V P                   Uu. uF
  q»! V4      NK  	  up4       \        P                  ! V
^ R7      p\        P                  ! VR,          ^,          R	RR7      R,          pVR,          V	\        P                  ! V\        P                   ! VP"                  4      P$                  4      ,          ,          VR&   \'        V P
                  4       Fa  w  rÞVRRVR3,          pV P(                  P*                  V,          R8H  pV'       d   TpMTpV! VVW P,                  V,          ,          V4      pKc  	  \        P                  ! V^ ,          ^,          R	RR7      R,          p	\'        V P.                  4       F#  w  rÛV! WM^,           ,          4      WM^,           &   K%  	  \        P                  ! VR,          ^,          R	RR7      R,          pVR,          V	\        P                  ! V\        P                   ! VP"                  4      P$                  4      ,          ,          VR&   \        P                  ! V^ R7      pV P1                  V4      pV P                   P3                  V4      pV P4                  e   \7        V P4                  V4      pV# u upi )
Nç      à?)Úwindow_sizeTr±   )r²   r  :NNNr-  r    )r1  r   Úget_per_layer_inputsÚproject_per_layer_inputsr=  r3  r   r<  r;  r   rS   r´   Úextendr7  Ústackr¹   Úfinfor·   Úminr?  ra   r$   r>  r8  r9  Ú	as_linearr#   r'  )rO   rº   r›   rJ  ÚhÚper_layer_inputsÚglobal_maskÚsliding_window_maskÚh0Útarget_magnitudeÚh_listÚprojÚmagsrE  Úlayerr  Ú	is_globalrš   r&  s   &&&&               r3   rW   ÚLanguageModel.__call__Ê  sÔ  € ð Ò#Ø×!Ñ! &Ó)¨T×-=Ñ-=¸sÕ-BÕC‰Aà ˆAà×4Ñ4°VÓ<ÐØ×8Ñ8¸ÓMÐàŠ=Ø�FœS §¡Ó-Õ-ˆEä+ØØ×%Ñ%Õ&ó
ˆô 4ØØ×(Ñ(Õ)Ø×+Ñ+ô
Ðð
 ˆô Ÿ7š7 2 q¥5¨r¸DÔAÀSÕHÐà�ˆØ�‰¨D×,BÒ,BÓCÑ,B D�t˜B–xÑ,BÑCÔDÜ�HŠH�V !Ô$ˆÜ�wŠw�q˜•u •z¨°TÔ:¸cÕAˆØ�"•Ð)¬B¯JªJ°t¼R¿XºXÀbÇhÁhÓ=O×=SÑ=SÓ,TÕTÕUˆˆ"‰Ü! $§+¡+Ö.‰HˆAØ.¨q°!°Q¸¨zÕ:ˆOàŸ™×/Ñ/°Õ2Ð6FÑFˆIçØ"‘à*�áØØØ×1Ñ1°!Õ4Õ5Øó	ŠAñ /ô$ Ÿ7š7 1 Q¥4¨1¥9°2ÀÔEÈÕLÐÜ  ×!?Ñ!?Ö@‰GˆAÙ˜A !�e�H“~ˆA�!�e‹Hñ Aä�wŠw�q˜•u •z¨°TÔ:¸cÕAˆØ�"•Ð)¬B¯JªJ°t¼R¿XºXÀbÇhÁhÓ=O×=SÑ=SÓ,TÕTÕUˆˆ"‰ä�GŠG�A˜AÔˆà�i‰i˜‹lˆØ×Ñ×)Ñ)¨#Ó.ˆØ×'Ñ'Ò3Ü × <Ñ <¸cÓBˆCØˆ
ùòE Ds   Ã=Nc                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Ú	input_idsrt   ru   )r1   r2   s   "€r3   r4   r+    s#   ø€ ÷ 

ñ 

©b¯h©hð 

¹2¿8¹8ñ 

r6   c                ó0  € WP                   8  p\        P                  ! W!\        P                  ! V4      4      pV P	                  V4      V P
                  R ,          ,          pVP                  ! . VP                  OV P                  NV P
                  N5!  # )rL  )	r   rS   ÚwhereÚ
zeros_liker4  r&   r¢   r¡   r   )rO   rb  Úper_layer_inputs_maskÚtokensÚresults   &&   r3   rN  Ú"LanguageModel.get_per_layer_inputs  s‹   € Ø )×,KÑ,KÑ KÐÜ—’Ð/¼B¿MºMÈ)Ó<TÓUˆØ×,Ñ,¨VÓ4Ø×,Ñ,¨cÕ1õ
ˆð �~Š~ð 
Ø�_‰_ð
à×"Ñ"ð
ð ×,Ñ,ó
ð 	
r6   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r   Úinputs_embedsrV  rt   ru   )r1   r2   s   "€r3   r4   r+    s:   ø€ ÷ Gñ Gá—x‘xðGñ Ÿ(™(ðGñ 
�‰ñ	Gr6   c                ó,  € V P                  V4      V P                  R,          ,          pVP                  ! . VP                  RR OV P                  P
                  NV P                  P                  N5!  pV P                  V4      pW2,           R,          # )rL  Ng      à¿r    r  )r5  r   r¢   r¡   ra   r   r&   r6  )rO   rk  rV  r  s   &&& r3   rO  Ú&LanguageModel.project_per_layer_inputs  s˜   € ð
  $×>Ñ>¸}ÓMØ×Ñ˜dÕ"õ 
Ðð  4×;Ò;ð  
Ø× Ñ   "Ð%ð 
à�K‰K×)Ñ)ð 
ð �K‰K×3Ñ3ó 
Ðð
  $×=Ñ=Ð>RÓSÐØ$Õ7¸IÕFÐFr6   c                ó,  € . pV P                   P                  R V P                    Fj  pVR8X  d   VP                  \	        4       4       K%  VR8X  d3   VP                  \        V P                   P                  ^ R7      4       K^  \        RV 24      h	  V# )Nr-  r…   )Úmax_sizeÚkeepr.  )ra   r$   r/  r@  r   r   r   rA  )rO   ÚcachesrF  s   &  r3   Ú
make_cacheÚLanguageModel.make_cache*  s†   € ØˆØŸ+™+×1Ñ1Ð2R°D×4RÑ4RÓSˆJØÐ-Ô-Ø—‘œg›iÖ(ØÐ2Ô2Ø—‘Ü#¨T¯[©[×-GÑ-GÈaÔPöô *Ð,@ÀÀÐ*MÓNÐNñ Tð ˆr6   )r7  r8  ra   r1  r4  r#   r<  r/  r;  r   r&   r>  r3  r9  r   r5  r6  r   r   r   r#  )r8   r9   r:   r;   rN   rW   rN  rO  rr  r=   r>   r[   r\   s   @@r3   r)  r)  {  sA   ù‡ € ÷LSó LS÷\Bò B÷H

ð 

÷Gð G÷ ò r6   r)  c                   óZ   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltR tRtVtV ;t	# )	ÚGemma3ni8  c                ó    <€ V ^8„  d   QhRS[ /# ©r   Úargs©rA   )r1   r2   s   "€r3   r4   ÚGemma3n.__annotate__9  s   ø€ ÷ Tñ T™Yñ Tr6   c                ó~   <€ \         SV `  4        \        \        P	                  VP
                  4      4      V n        R # rL   )rM   rN   r)  r   Ú	from_dictrC   Úlanguage_model©rO   rx  rP   s   &&€r3   rN   ÚGemma3n.__init__9  s,   ø€ Ü‰ÑÔÜ+¬J×,@Ñ,@À×AQÑAQÓ,RÓSˆÖr6   c                ó^   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          /# rI  ©rS   rv   r   )r1   r2   s   "€r3   r4   rz  =  s,   ø€ ÷ 
ñ 
á—‘ð
ñ #¡2§8¡8Õ,ñ	
r6   c                ó(   € V P                  WVR 7      # ©)r›   rJ  ©r}  ©rO   rº   r›   rJ  s   &&&&r3   rW   ÚGemma3n.__call__=  s"   € ð ×"Ñ"ØÐ2Bð #ó 
ð 	
r6   c                ó6   € V P                   P                  4       # rL   )r}  rr  ©rO   s   &r3   rr  ÚGemma3n.make_cacheG  s   € Ø×"Ñ"×-Ñ-Ó/Ð/r6   r„  r®   )
r8   r9   r:   r;   rN   rW   rr  r=   r>   r[   r\   s   @@r3   ru  ru  8  s%   ù‡ € ÷Tó T÷
ò 
÷0ò 0r6   ru  c                   óp   a a€ ] tR tRt oV3R lV 3R lltR
V3R lR lltR t]R 4       tR t	R	t
VtV ;t# )ÚModeliK  c                ó    <€ V ^8„  d   QhRS[ /# rw  ry  )r1   r2   s   "€r3   r4   ÚModel.__annotate__L  s   ø€ ÷ *ñ *™Yñ *r6   c                ór   <€ \         SV `  4        Wn        \        V4      V n        VP
                  V n        R # rL   )rM   rN   rx  ru  Úmodelr   r~  s   &&€r3   rN   ÚModel.__init__L  s)   ø€ Ü‰ÑÔØŒ	Ü˜T“]ˆŒ
ØŸ/™/ˆŽr6   c                ó^   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          /# rI  r�  )r1   r2   s   "€r3   r4   r�  R  s0   ø€ ÷ Rñ Rá—‘ðRñ #¡2§8¡8Õ,ñ	Rr6   c                ó(   € V P                  WVR 7      # rƒ  )r�  r…  s   &&&&r3   rW   ÚModel.__call__R  s   € ð �z‰z˜&Ð@PˆzÓQÐQr6   c                ó²   € \        \        VP                  4       4      4      pR F  pVR,          P                  VR4       K  	  \	        \        V4      4      # )Úvision_towerr�  N)r•  Úaudio_towerÚembed_audioÚembed_vision)r
   rÈ   ÚitemsÚpoprD   r	   )rO   ÚweightsÚks   && r3   ÚsanitizeÚModel.sanitizeZ  sG   € Ü ¤ g§m¡m£oÓ!6Ó7ˆÛOˆAØ�GÕ× Ñ   DÖ)ñ Pä”L Ó)Ó*Ð*r6   c                óB   € V P                   P                  P                  # rL   )r�  r}  r3  rˆ  s   &r3   r3  ÚModel.layers`  s   € à�z‰z×(Ñ(×/Ñ/Ð/r6   c                ó6   € V P                   P                  4       # rL   )r�  rr  rˆ  s   &r3   rr  ÚModel.make_cached  s   € Ø�z‰z×$Ñ$Ó&Ð&r6   )rx  r�  r   r®   )r8   r9   r:   r;   rN   rW   r�  Úpropertyr3  rr  r=   r>   r[   r\   s   @@r3   r‹  r‹  K  s>   ù‡ € ÷*ó *÷Rò Rò+ð ñ0ó ð0÷'ò 'r6   r‹  )(rÍ   Údataclassesr   Ú	functoolsr   Útypingr   r   r   r   Úmlx.coreÚcorerS   Úmlx.nnrj   Ú	mlx.utilsr	   r
   rˆ   r   r   r   r›   r   r   r   rA   ÚModulerG   r^   r}   Úcompiler¿   rÁ   r×   r   r'  r)  ru  r‹  r7   r6   r3   Ú<module>r­     s9  ðó Ý !Ý ß ,Ó ,å Ý ß 2ç TÑ Tß +ð ô(�ó (ó ð(ð8 ô�ó ó ðô
3�—‘ô 3ô#˜Ÿ™ô #ô2K#�r—y‘yô K#ñ\ 	ˆ�‰˜tÔ$ñ<ó %ð<ôˆ"�)‰)ô ôBL1�2—9‘9ô L1ô^V%˜"Ÿ)™)ô V%ñr 	ˆ�‰˜tÔ$ñó %ðôz�B—I‘Iô zôz0ˆb�i‰iô 0ô&'ˆB�I‰Iö 'r6   