+
    UV-jÏ%  ã                   óž  € ^ RI Ht ^ RIHtHt ^ RIHt ^ RIH	t	 ^RI
HtHtHt ^RIHtHt ^RIHt  ! R R]	P&                  4      t ! R	 R
]	P&                  4      t ! R R]	P&                  4      t]! ]P.                  RR7      RR l4       t ! R R]	P&                  4      t ! R R]	P&                  4      t ! R R]	P&                  4      tR# )é    )Úpartial)ÚAnyÚOptionalN)ÚLanguageModelOutputÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCacheÚRotatingKVCache©Ú
TextConfigc                   óH   a a€ ] tR t^t oRV3R lV 3R llltR tRtVtV ;t# )ÚRMSNormc                ó&   <€ V ^8„  d   QhRS[ RS[/# )é   ÚdimsÚeps)ÚintÚfloat)ÚformatÚ__classdict__s   "€Úo/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/gemma3/language.pyÚ__annotate__ÚRMSNorm.__annotate__   s   ø€ ÷ ñ ™Sð ¡uñ ó    c                óh   <€ \         SV `  4        \        P                  ! V34      V n        W n        R # ©N)ÚsuperÚ__init__ÚmxÚonesÚweightr   )Úselfr   r   Ú	__class__s   &&&€r   r   ÚRMSNorm.__init__   s%   ø€ Ü‰ÑÔÜ—g’g˜t˜gÓ&ˆŒØŽr   c                óz   € \         P                  P                  VR V P                  ,           V P                  4      # )g      ð?)r   ÚfastÚrms_normr!   r   ©r"   Úxs   &&r   Ú__call__ÚRMSNorm.__call__   s(   € Ü�w‰w×Ñ  3¨¯©Õ#4°d·h±hÓ?Ð?r   )r   r!   )gñhãˆµøä>©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r*   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r#   r   s   @@r   r   r      s   ù‡ € ÷õ ÷
@ò @r   r   c                   óT   a a€ ] tR t^t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r   ÚconfigÚ	layer_idx©r   r   )r   r   s   "€r   r   ÚAttention.__annotate__   s   ø€ ÷ 
ñ 
™zð 
±cñ 
r   c                ót  <€ \         SV `  4        VP                  pVP                  ;V n        pVP
                  ;V n        pWE,          V n        VP                  ;V n        pW n	        VP                  R,          V n        \        P                  ! W4V,          RR7      V n        \        P                  ! W5V,          RR7      V n        \        P                  ! W5V,          RR7      V n        \        P                  ! WF,          VRR7      V n        \%        WaP&                  R7      V n        \%        WaP&                  R7      V n        V^,           VP,                  ,          ^ 8g  V n        \        P0                  ! TVP2                  V P.                  '       d   VP4                  MVP6                  R7      V n        R# )ç      à?F©Úbias)r   r   )ÚtraditionalÚbaseNg      à¿)r   r   Úhidden_sizeÚnum_attention_headsÚn_headsÚnum_key_value_headsÚ
n_kv_headsÚrepeatsÚhead_dimr:   Úquery_pre_attn_scalarÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚo_projr   Úrms_norm_epsÚq_normÚk_normÚsliding_window_patternÚ
is_slidingÚRoPEÚrope_traditionalÚrope_local_base_freqÚrope_global_base_freqÚrope)r"   r9   r:   ÚdimrE   rG   rI   r#   s   &&&    €r   r   ÚAttention.__init__   sD  ø€ Ü‰ÑÔà× Ñ ˆØ!'×!;Ñ!;Ð;ˆŒ�wØ'-×'AÑ'AÐAˆŒ˜*ØÕ,ˆŒØ#)§?¡?Ð2ˆŒ˜Ø"Œà×1Ñ1°4Õ7ˆŒ
ä—i’i ¨xÕ%7¸eÔDˆŒÜ—i’i °(Õ%:ÀÔGˆŒÜ—i’i °(Õ%:ÀÔGˆŒÜ—i’i Õ 2°C¸eÔDˆŒä 8×1DÑ1DÔEˆŒÜ 8×1DÑ1DÔEˆŒØ$ q�=¨F×,IÑ,IÕIÈQÑNˆŒä—G’GØØ×/Ñ/ð —?—?�?ð ×+Ò+à×1Ñ1ô
ˆŽ	r   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   r)   ÚmaskÚcacheÚreturn©r   Úarrayr   r   )r   r   s   "€r   r   r<   :   sH   ø€ ÷ !#ñ !#á�8‰8ð!#ñ ‘r—x‘xÕ ð!#ñ ™�}ð	!#ñ
 
�‰ñ!#r   c           	     ó:  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pVP	                  WEV P
                  R4      P                  ^ ^^^4      pVP	                  WEV P                  R4      P                  ^ ^^^4      pV	P	                  WEV P                  R4      P                  ^ ^^^4      p	V P                  V4      pV P                  V4      pVeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      pVei   \        V\        P                  4      '       dI   VP                   R,          VP                   R,          8w  d    VRVP                   R,          ) R13,          p\!        WxW“V P"                  VR7      p
V
P                  ^ ^^^4      P	                  WER4      p
V P%                  V
4      # )é   N)Úoffset.)rK   r`   éÿÿÿÿéþÿÿÿ)ÚshaperN   rO   rP   ÚreshaperE   Ú	transposerG   rS   rT   r[   rg   Úupdate_and_fetchÚ
isinstancer   rd   r   rK   rQ   )r"   r)   r`   ra   ÚBÚLÚ_ÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r   r*   ÚAttention.__call__:   s¹  € ð —'‘'‰ˆˆaØ $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆØ—/‘/ !¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆà�|‰|˜A $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆà—+‘+˜gÓ&ˆØ�{‰{˜4Ó ˆàÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDð Ò¤
¨4´·±× :Ò :Ø�z‰z˜"�~ §¡¨B¥Ô/Ø˜C $§*¡*¨R¥. Ñ!2Ð2Õ3�ä-Ø˜6°·
±
Àô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r   )rI   rV   rT   rO   r:   rE   rG   rQ   rS   rN   rH   r[   rK   rP   ©NNr,   r4   s   @@r   r6   r6      s   ù‡ € ÷
ó 
÷>!#÷ !#ò !#r   r6   c                   óD   a a€ ] tR t^^t oV 3R ltV3R lR ltRtVtV ;t# )ÚMLPc                óÒ   <€ \         SV `  4        \        P                  ! WR R7      V n        \        P                  ! W!R R7      V n        \        P                  ! WR R7      V n        R# ©Fr?   N)r   r   rL   rM   Ú	gate_projÚ	down_projÚup_proj)r"   r\   Ú
hidden_dimr#   s   &&&€r   r   ÚMLP.__init___   sD   ø€ Ü‰ÑÔÜŸš 3¸Ô?ˆŒÜŸš :¸Ô?ˆŒÜ—y’y °uÔ=ˆŽr   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   rb   ©r   rd   )r   r   s   "€r   r   ÚMLP.__annotate__e   s   ø€ ÷ Sñ S™RŸX™Xñ Sr   c                ó–   € V P                  \        P                  ! V P                  V4      4      V P	                  V4      ,          4      # r   )r}   rL   Úgelu_approxr|   r~   r(   s   &&r   r*   ÚMLP.__call__e   s0   € à�~‰~œbŸnšn¨T¯^©^¸AÓ->Ó?À$Ç,Á,ÈqÃ/ÕQÓRÐRr   )r}   r|   r~   r,   r4   s   @@r   ry   ry   ^   s   ù‡ € õ>÷S÷ Sð Sr   ry   T)Ú	shapelessc                 óŠ  € \         P                  ! \         P                  4      P                  pVfu   V P                  \         P                  8X  dT   \         P
                  ! V P                  \         P                  4      V) V4      P                  \         P                  4      # V # V P                  \         P                  8w  d	   W,           # \         P
                  ! V P                  \         P                  4      VP                  \         P                  4      ,           V) V4      P                  \         P                  4      # r   )r   ÚfinfoÚfloat16ÚmaxÚdtypeÚclipÚastypeÚfloat32)r)   ÚyÚbounds   && r   Úclip_residualr’   j   sÄ   € ä�HŠH”R—Z‘ZÓ ×$Ñ$€EØ‚yØ�7‰7”b—j‘jÔ Ü—7’7˜1Ÿ8™8¤B§J¡JÓ/°%°¸Ó?×FÑFÄrÇzÁzÓRÐRàˆHà‡w�w”"—*‘*ÔØ�uˆä�7Š7�1—8‘8œBŸJ™JÓ'¨!¯(©(´2·:±:Ó*>Õ>ÀÀÈÓN×UÑUÜ
�
‰
óð r   c                   óT   a a€ ] tR t^{t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚTransformerBlockc                ó&   <€ V ^8„  d   QhRS[ RS[/# r8   r;   )r   r   s   "€r   r   ÚTransformerBlock.__annotate__|   s   ø€ ÷ 
ñ 
™zð 
±cñ 
r   c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        \	        W4      V n        \        VP                  VP                  4      V n        \        VP                  VP                  R 7      V n        \        VP                  VP                  R 7      V n        \        VP                  VP                  R 7      V n        \        VP                  VP                  R 7      V n        R# )©r   N)r   r   rD   rC   r6   Ú	self_attnry   Úintermediate_sizeÚmlpr   rR   Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernorm©r"   r9   r:   r#   s   &&&€r   r   ÚTransformerBlock.__init__|   sÉ   ø€ Ü‰ÑÔØ#)×#=Ñ#=ˆÔ Ø!×-Ñ-ˆÔÜ" 6Ó5ˆŒÜ�v×)Ñ)¨6×+CÑ+CÓDˆŒÜ& v×'9Ñ'9¸v×?RÑ?RÔSˆÔÜ(/Ø×Ñ F×$7Ñ$7ô)
ˆÔ%ô *1Ø×Ñ F×$7Ñ$7ô*
ˆÔ&ô +2Ø×Ñ F×$7Ñ$7ô+
ˆÖ'r   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# r_   rc   )r   r   s   "€r   r   r–   �   sH   ø€ ÷ ñ á�8‰8ðñ ‘r—x‘xÕ ðñ ™�}ð	ñ
 
�‰ñr   c                ó&  € \        V4      pV P                  V P                  V4      W#4      pV P                  V4      p\        W,           4      pV P	                  V P                  V4      4      pV P                  V4      p\        WV,           4      pV# r   )r’   r™   rœ   r�   r›   rž   rŸ   )r"   r)   r`   ra   ÚrÚhÚouts   &&&&   r   r*   ÚTransformerBlock.__call__�   s‡   € ô ˜!Óˆð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ×)Ñ)¨!Ó,ˆô ˜!�%Ó ˆð �H‰H�T×3Ñ3°AÓ6Ó7ˆØ×-Ñ-¨aÓ0ˆô ˜A�GÓ$ˆàˆ
r   )rC   rœ   r›   rD   r�   rŸ   rž   r™   rw   r,   r4   s   @@r   r”   r”   {   s   ù‡ € ÷
ó 
÷"÷ ò r   r”   c                   óT   a a€ ] tR t^®t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma3Modelc                ó    <€ V ^8„  d   QhRS[ /# ©r   r9   r   )r   r   s   "€r   r   ÚGemma3Model.__annotate__¯   s   ø€ ÷ Iñ I™zñ Ir   c                óü  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        VP                  V n        VP                  V n        V P                  ^ 8”  g   Q h\        P                  ! VP                  VP                  4      V n        \        VP                  4       Uu. uF  p\        WR7      NK  	  upV n        \        VP                  VP                   R7      V n        R# u upi )r   )r9   r:   r˜   N)r   r   r9   Ú
vocab_sizeÚsliding_windowÚwindow_sizerU   Únum_hidden_layersrL   Ú	EmbeddingrC   Úembed_tokensÚranger”   Úlayersr   rR   Únormr    s   && €r   r   ÚGemma3Model.__init__¯   sÍ   ø€ Ü‰ÑÔØŒØ ×+Ñ+ˆŒØ!×0Ñ0ˆÔØ&,×&CÑ&CˆÔ#Ø!'×!9Ñ!9ˆÔØ�‰ Ô"Ð"Ð"ÜŸLšL¨×):Ñ):¸F×<NÑ<NÓOˆÔô # 6×#;Ñ#;Ô<ó
á<�	ô  F×@Ù<ñ
ˆŒô ˜F×.Ñ.°F×4GÑ4GÔHˆŽ	ùò	
s   Â4C9c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# ©r   ÚinputsÚinputs_embedsr`   r‚   )r   r   s   "€r   r   r¬   ¾   s5   ø€ ÷ ,ñ ,á—‘ð,ñ —x‘xð,ñ �h‰hñ	,r   c                óì  € Vf   V P                  V4      pMTpV\        P                  ! V P                  P                  R,          \        P
                  4      P                  VP                  4      ,          pVf   R .\        V P                  4      ,          pVfW   \        WTV P                  ^,
          ,          4      pV P                  ^8”  d!   \        VV^ ,          V P                  R7      pMR p\        \        V P                  V4      4       FN  w  pw  ršW€P                  ,          V P                  ^,
          8H  pTpVf   V'       d   XpMVf   XpV	! W\V
4      pKP  	  V P                  V4      # )Nr>   )r°   )r³   r   rd   r9   rC   Úbfloat16rŽ   rŒ   Úlenrµ   r   rU   r°   Ú	enumerateÚzipr¶   )r"   rº   r»   r`   ra   r¥   Úglobal_maskÚsliding_window_maskÚiÚlayerÚcÚ	is_globalÚ
local_masks   &&&&&        r   r*   ÚGemma3Model.__call__¾   sD  € ð Ò Ø×!Ñ! &Ó)‰AàˆAà	ŒR�XŠX�d—k‘k×-Ñ-¨sÕ2´B·K±KÓ@×GÑGÈÏÉÓPÕPˆàŠ=Ø�FœS §¡Ó-Õ-ˆEàŠ<Ü/Ø˜×4Ñ4°qÕ8Õ9óˆKð ×*Ñ*¨QÔ.Ü&;ØØ˜!•HØ $× 0Ñ 0ô'Ñ#ð '+Ð#ä&¤s¨4¯;©;¸Ó'>Ö?‰MˆA‰z�à×/Ñ/Õ/°4×3NÑ3NÐQRÕ3RÑRð ð ˆJØŠ|§	Ø(‘
Ø’Ø0�
á�a QÓ'ŠAñ @ð �y‰y˜‹|Ðr   )r9   r³   rµ   r¶   r±   rU   r®   r°   ©NNNr,   r4   s   @@r   r©   r©   ®   s    ù‡ € ÷Ió I÷,÷ ,ò ,r   r©   c                   ó�   a a€ ] tR t^ít oV3R lV 3R lltRV3R lR lltR t]R 4       t]R 4       t	]R 4       t
R	 tR
tVtV ;t# )ÚLanguageModelc                ó    <€ V ^8„  d   QhRS[ /# r«   r   )r   r   s   "€r   r   ÚLanguageModel.__annotate__î   s   ø€ ÷ Tñ T™zñ Tr   c                óÖ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        \        P                  ! VP                  VP                  R R7      V n
        R# r{   )r   r   r9   Ú
model_typer©   ÚmodelrL   rM   rC   r®   Úlm_head)r"   r9   r#   s   &&€r   r   ÚLanguageModel.__init__î   sL   ø€ Ü‰ÑÔØŒØ ×+Ñ+ˆŒÜ  Ó(ˆŒ
Ü—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŽr   c                óˆ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          /# r¹   )r   rd   r   )r   r   s   "€r   r   rÍ   õ   s?   ø€ ÷ 
/ñ 
/á—‘ð
/ñ  ¡§¡Õ)ð
/ñ ‘r—x‘xÕ ñ	
/r   c                ób   € V P                  WW4R 7      pV P                  V4      p\        VR7      # ))r»   r`   ra   )Úlogits)rÐ   rÑ   r   )r"   rº   r»   r`   ra   Úkwargsr¦   s   &&&&&, r   r*   ÚLanguageModel.__call__õ   s/   € ð �j‰j˜À4ˆjÓUˆØ�l‰l˜3ÓˆÜ"¨#Ô.Ð.r   c                óŽ   € R V9  d   VR,          VR&   VP                  4        UUu/ uF  w  r#RV9  g   K  W#bK  	  upp# u uppi )zlm_head.weightz(language_model.model.embed_tokens.weightzlanguage_model.lm_head.weightzself_attn.rotary_emb.inv_freq)Úitems)r"   ÚweightsÚkÚvs   &&  r   ÚsanitizeÚLanguageModel.sanitize  sU   € Ø 7Ô*Ø7>Ø:õ8ˆGÐ3Ñ4ð %Ÿ]™]œ_ô
Ù,‘T�QÐ0OÐWXÑ0XŒDˆAŠD™_ò
ð 	
ùó 
s
   §A·Ac                ó.   € V P                   P                  # r   )rÐ   rµ   ©r"   s   &r   rµ   ÚLanguageModel.layers
  s   € à�z‰z× Ñ Ð r   c                ó.   € V P                   P                  # r   )r9   rI   rà   s   &r   rI   ÚLanguageModel.head_dim  s   € à�{‰{×#Ñ#Ð#r   c                ó.   € V P                   P                  # r   )r9   rF   rà   s   &r   rG   ÚLanguageModel.n_kv_heads  s   € à�{‰{×.Ñ.Ð.r   c                ód  € . p\        V P                  P                  4       FŠ  pW P                  P                  ,          V P                  P                  ^,
          8X  d   VP	                  \        4       4       KZ  VP	                  \        V P                  P                  ^ R7      4       KŒ  	  V# )rf   )Úmax_sizeÚkeep)r´   r9   r±   rU   Úappendr	   r
   r¯   )r"   ÚcachesrÃ   s   &  r   Ú
make_cacheÚLanguageModel.make_cache  s‚   € ØˆÜ�t—{‘{×4Ñ4Ö5ˆAà—K‘K×6Ñ6Õ6Ø—;‘;×5Ñ5¸Õ9ô:ð —‘œg›iÖ(à—‘Ü#Ø!%§¡×!;Ñ!;Øôöñ 6ð ˆr   )r9   rÑ   rÐ   rÏ   rÉ   )r-   r.   r/   r0   r   r*   rÝ   Úpropertyrµ   rI   rG   rë   r1   r2   r3   r4   s   @@r   rË   rË   í   sf   ù‡ € ÷Tó T÷
/ò 
/ò
ð ñ!ó ð!ð ñ$ó ð$ð ñ/ó ð/÷ò r   rË   r   )Ú	functoolsr   Útypingr   r   Úmlx.coreÚcorer   Úmlx.nnrL   rB   r   r   r   ra   r	   r
   r9   r   ÚModuler   r6   ry   Úcompiler’   r”   r©   rË   © r   r   Ú<module>rö      s«   ðÝ ß  å Ý ÷ñ ÷
 -Ý ô@ˆb�i‰iô @ôA#�—	‘	ô A#ôH	Sˆ"�)‰)ô 	Sñ 	ˆ�‰˜tÔ$óó %ðô 0�r—y‘yô 0ôf<�"—)‘)ô <ô~8�B—I‘Iö 8r   