+
    TV-j×!  ã                   óÊ  € ^ RI Ht ^ RIHt ^ RIHtHtHt ^ RIH	t
 ^ RIHt ^RIHtHtHt ^RIHtHt ^RIHt ] ! R R	]4      4       t ! R
 R]P.                  4      t ! R R]P.                  4      t ! R R]P.                  4      t]! ]
P6                  RR7      R 4       t ! R R]P.                  4      t ! R R]P.                  4      t ! R R]P.                  4      tR# )é    )Ú	dataclass)Úpartial)ÚAnyÚDictÚOptionalN)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCacheÚRotatingKVCache)Úinitialize_ropec                   óh   a € ] tR t^t o Rt^tRt^tRtRt	Rt
^tRtRtRtRt^tR	tR
tV 3R ltRtV tR
# )Ú	ModelArgsi€  i   é   g�íµ ÷Æ°>i   g    €„.Ag     ˆÃ@i   i €  Nc                óÚ   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   # )é   Ú
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚhead_dimÚrms_norm_epsÚ
vocab_sizeÚnum_key_value_headsÚ
rope_thetaÚrope_local_base_freqÚquery_pre_attn_scalarÚsliding_windowÚsliding_window_patternÚmax_position_embeddingsÚrope_scaling)ÚstrÚintÚfloatr   )ÚformatÚ__classdict__s   "€Új/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/gemma3_text.pyÚ__annotate__ÚModelArgs.__annotate__   sÒ   ø‡ ‚ á�Oñ ñ Ññ ñ Ññ	 ñ
 Ñ!ñ ñ Ñ ñ ñ Ññ ñ Ñ ñ ñ Ññ ñ Ñ ñ ñ Ñ#ñ ñ  Ñ*ñ ñ !Ñ&ñ ñ Ññ ñ  Ñ#ñ ñ  !Ñ(ñ! ñ" Ñò# ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r'   s   @r(   r   r      sa   ø‡ € ð €KØÐØ!ÐØ ÐØ€HØ €LØ€JØ ÐØ#€JØ"*ÐØ#&ÐØ€NØ"#ÐØ#(ÐØ€L÷# ƒ r+   r   c                   óT   a a€ ] tR t^#t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r   ÚargsÚ	layer_idx©r   r$   )r&   r'   s   "€r(   r)   ÚAttention.__annotate__$   s   ø€ ÷ "ñ "™Yð "±3ñ "r+   c                óœ  <€ \         SV `  4        VP                  pVP                  ;V n        pVP
                  ;V n        pWE,          V n        VP                  ;V n        pW n	        VP                  R,          V n        \        P                  ! W4V,          RR7      V n        \        P                  ! W5V,          RR7      V n        \        P                  ! W5V,          RR7      V n        \        P                  ! WF,          VRR7      V n        \%        WaP&                  R7      V n        \%        WaP&                  R7      V n        V^,           VP,                  ,          ^ 8g  V n        V P.                  '       d    \1        VVP2                  RR7      V n        R# \1        VVP6                  RVP8                  VP:                  R7      V n        R# )ç      à?F©Úbias)ÚdimsÚeps)r@   ÚbaseÚtraditional)r@   rB   rC   r!   Úscaling_configNg      à¿)ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsÚrepeatsr   r9   r   ÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚo_projÚRMSNormr   Úq_normÚk_normr    Ú
is_slidingr   r   Úroper   r!   r"   )Úselfr8   r9   ÚdimrG   rH   r   Ú	__class__s   &&&    €r(   rF   ÚAttention.__init__$   sZ  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*ØÕ,ˆŒØ#'§=¡=Ð0ˆŒ˜Ø"Œà×/Ñ/°Õ5ˆŒ
ä—i’i ¨xÕ%7¸eÔDˆŒÜ—i’i °(Õ%:ÀÔGˆŒÜ—i’i °(Õ%:ÀÔGˆŒÜ—i’i Õ 2°C¸eÔDˆŒä 8×1BÑ1BÔCˆŒÜ 8×1BÑ1BÔCˆŒØ$ q�=¨D×,GÑ,GÕGÈ1ÑLˆŒà�?�?ˆ?Ü'ØØ×.Ñ.Ø!ôˆDŽIô (ØØ—_‘_Ø!Ø(,×(DÑ(DØ#×0Ñ0ôˆDŽIr+   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   ÚxÚmaskÚcacheÚreturn©ÚmxÚarrayr   r   )r&   r'   s   "€r(   r)   r;   H   sH   ø€ ÷ #ñ #á�8‰8ð#ñ ‘r—x‘xÕ ð#ñ ™�}ð	#ñ
 
�‰ñ#r+   c           	     ób  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pVP	                  WEV P
                  R4      P                  ^ ^^^4      pVP	                  WEV P                  R4      P                  ^ ^^^4      pV	P	                  WEV P                  R4      P                  ^ ^^^4      p	V P                  V4      pV P                  V4      pVeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      P	                  WER4      p
V P                  V
4      # )é   )Úoffset)r^   rJ   r]   éÿÿÿÿ)ÚshaperM   rN   rO   ÚreshaperG   Ú	transposerH   rR   rS   rU   re   Úupdate_and_fetchr
   rJ   rP   )rV   r\   r]   r^   ÚBÚLÚ_ÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r(   Ú__call__ÚAttention.__call__H   so  € ð —'‘'‰ˆˆaØ $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆØ—/‘/ !¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆà�|‰|˜A $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆà—+‘+˜gÓ&ˆØ�{‰{˜4Ó ˆàÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDô .Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r+   )r   rT   rS   rN   r9   rG   rH   rP   rR   rM   rI   rU   rJ   rO   ©NN©	r-   r.   r/   r0   rF   rr   r2   r3   Ú__classcell__©rX   r'   s   @@r(   r5   r5   #   s   ù‡ € ÷"ó "÷H#÷ #ò #r+   r5   c                   óH   a a€ ] tR t^ht oRV3R lV 3R llltR tRtVtV ;t# )rQ   c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   r@   rA   )r$   r%   )r&   r'   s   "€r(   r)   ÚRMSNorm.__annotate__i   s   ø€ ÷ ñ ™Sð ¡uñ r+   c                óh   <€ \         SV `  4        \        P                  ! V34      V n        W n        R # ©N)rE   rF   ra   ÚonesÚweightrA   )rV   r@   rA   rX   s   &&&€r(   rF   ÚRMSNorm.__init__i   s%   ø€ Ü‰ÑÔÜ—g’g˜t˜gÓ&ˆŒØŽr+   c                óz   € \         P                  P                  VR V P                  ,           V P                  4      # )g      ð?)ra   ÚfastÚrms_normr~   rA   ©rV   r\   s   &&r(   rr   ÚRMSNorm.__call__n   s(   € Ü�w‰w×Ñ  3¨¯©Õ#4°d·h±hÓ?Ð?r+   )rA   r~   )gñhãˆµøä>ru   rw   s   @@r(   rQ   rQ   h   s   ù‡ € ÷õ ÷
@ò @r+   rQ   c                   óD   a a€ ] tR t^rt oV 3R ltV3R lR ltRtVtV ;t# )ÚMLPc                óÒ   <€ \         SV `  4        \        P                  ! WR R7      V n        \        P                  ! W!R R7      V n        \        P                  ! WR R7      V n        R# ©Fr>   N)rE   rF   rK   rL   Ú	gate_projÚ	down_projÚup_proj)rV   rW   Ú
hidden_dimrX   s   &&&€r(   rF   ÚMLP.__init__s   sD   ø€ Ü‰ÑÔÜŸš 3¸Ô?ˆŒÜŸš :¸Ô?ˆŒÜ—y’y °uÔ=ˆŽr+   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   r_   )ra   rb   )r&   r'   s   "€r(   r)   ÚMLP.__annotate__y   s   ø€ ÷ Sñ S™RŸX™Xñ Sr+   c                ó–   € V P                  \        P                  ! V P                  V4      4      V P	                  V4      ,          4      # r|   )rŠ   rK   Úgelu_approxr‰   r‹   rƒ   s   &&r(   rr   ÚMLP.__call__y   s0   € Ø�~‰~œbŸnšn¨T¯^©^¸AÓ->Ó?À$Ç,Á,ÈqÃ/ÕQÓRÐRr+   )rŠ   r‰   r‹   ru   rw   s   @@r(   r†   r†   r   s   ù‡ € õ>÷S÷ Sð Sr+   r†   T)Ú	shapelessc                 óš  € V P                   \        P                  8w  d	   W,           # \        P                  ! \        P                  4      P                  p\        P
                  ! V P                  \        P                  4      VP                  \        P                  4      ,           V) V4      P                  \        P                  4      # r|   )Údtypera   Úfloat16ÚfinfoÚmaxÚclipÚastypeÚfloat32)r\   ÚyÚbounds   && r(   Úclip_residualrž   }   sy   € à‡w�w”"—*‘*ÔØ�uˆÜ�HŠH”R—Z‘ZÓ ×$Ñ$€EÜ�7Š7�1—8‘8œBŸJ™JÓ'¨!¯(©(´2·:±:Ó*>Õ>ÀÀÈÓN×UÑUÜ
�
‰
óð r+   c                   óT   a a€ ] tR t^‡t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚTransformerBlockc                ó&   <€ V ^8„  d   QhRS[ RS[/# r7   r:   )r&   r'   s   "€r(   r)   ÚTransformerBlock.__annotate__ˆ   s   ø€ ÷ 
ñ 
™Yð 
±3ñ 
r+   c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        \	        W4      V n        \        VP                  VP                  4      V n        \        VP                  VP                  R 7      V n        \        VP                  VP                  R 7      V n        \        VP                  VP                  R 7      V n        \        VP                  VP                  R 7      V n        R# )©rA   N)rE   rF   r   r   r5   Ú	self_attnr†   r   ÚmlprQ   r   Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernorm©rV   r8   r9   rX   s   &&&€r(   rF   ÚTransformerBlock.__init__ˆ   sÄ   ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø×+Ñ+ˆÔÜ" 4Ó3ˆŒÜ�t×'Ñ'¨×)?Ñ)?Ó@ˆŒÜ& t×'7Ñ'7¸T×=NÑ=NÔOˆÔÜ(/°×0@Ñ0@Àd×FWÑFWÔ(XˆÔ%Ü)0Ø×Ñ $×"3Ñ"3ô*
ˆÔ&ô +2Ø×Ñ $×"3Ñ"3ô+
ˆÖ'r+   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# r[   r`   )r&   r'   s   "€r(   r)   r¢   —   sH   ø€ ÷ 
ñ 
á�8‰8ð
ñ ‘r—x‘xÕ ð
ñ ™�}ð	
ñ
 
�‰ñ
r+   c                óð   € V P                  V P                  V4      W#4      p\        WP                  V4      4      pV P	                  V P                  V4      4      p\        WPP                  V4      4      pV# r|   )r¥   r§   rž   r¨   r¦   r©   rª   )rV   r\   r]   r^   ÚrÚhÚouts   &&&&   r(   rr   ÚTransformerBlock.__call__—   sg   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆÜ˜!×:Ñ:¸1Ó=Ó>ˆØ�H‰H�T×3Ñ3°AÓ6Ó7ˆÜ˜A×>Ñ>¸qÓAÓBˆØˆ
r+   )r   r§   r¦   r   r¨   rª   r©   r¥   rt   ru   rw   s   @@r(   r    r    ‡   s   ù‡ € ÷
ó 
÷
÷ 
ò 
r+   r    c                   óT   a a€ ] tR t^¤t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGemma3Modelc                ó    <€ V ^8„  d   QhRS[ /# ©r   r8   ©r   )r&   r'   s   "€r(   r)   ÚGemma3Model.__annotate__¥   s   ø€ ÷ Eñ E™Yñ Er+   c                óü  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        V P                  ^ 8”  g   Q h\        P                  ! VP                  VP                  4      V n        \        VP                  4       Uu. uF  p\        WR7      NK  	  upV n        \        VP                  VP                   R7      V n        R# u upi )r   )r8   r9   r¤   N)rE   rF   r8   r   Úwindow_sizer    r   r   rK   Ú	Embeddingr   Úembed_tokensÚranger    ÚlayersrQ   r   Únormr«   s   && €r(   rF   ÚGemma3Model.__init__¥   sÉ   ø€ Ü‰ÑÔØŒ	Ø×.Ñ.ˆÔØ&*×&AÑ&AˆÔ#ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ�‰ Ô"Ð"Ð"ÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔô # 4×#9Ñ#9Ô:ó
á:�	ô  $×<Ù:ñ
ˆŒô ˜D×,Ñ,°$×2CÑ2CÔDˆŽ	ùò	
s   Â4C9c                ó^   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          /# ©r   ÚinputsÚinput_embeddings©ra   rb   r   )r&   r'   s   "€r(   r)   r¸   ´   s,   ø€ ÷  ñ  á—‘ð ñ #¡2§8¡8Õ,ñ	 r+   c                óÎ  € Ve   TpMV P                  V4      pV\        P                  ! V P                  P                  R,          \        P
                  4      P                  VP                  4      ,          pVf   R .\        V P                  4      ,          p\        WBV P                  ^,
          ,          4      pV P                  ^8”  d!   \        VV^ ,          V P                  R7      pMR p\        \        V P                  V4      4       FC  w  pw  r‰WpP                  ,          V P                  ^,
          8H  p
V
'       d   TMTpV! WKV	4      pKE  	  V P                  V4      # )Nr=   )rº   )r¼   ra   rb   r8   r   Úbfloat16rš   r•   Úlenr¾   r	   r    rº   Ú	enumerateÚzipr¿   )rV   rÃ   r^   rÄ   r°   Úglobal_maskÚsliding_window_maskÚiÚlayerÚcÚ	is_globalr]   s   &&&&        r(   rr   ÚGemma3Model.__call__´   s*  € ð Ò'Ø ‰Aà×!Ñ! &Ó)ˆAØ	ŒR�XŠX�d—i‘i×+Ñ+¨SÕ0´"·+±+Ó>×EÑEÀaÇgÁgÓNÕNˆàŠ=Ø�FœS §¡Ó-Õ-ˆEä+¨A°T×5PÑ5PÐSTÕ5TÕ/UÓVˆà×&Ñ&¨Ô*Ü"7ØØ�a•Ø ×,Ñ,ô#Ñð #'ÐÜ&¤s¨4¯;©;¸Ó'>Ö?‰MˆA‰z�à×/Ñ/Õ/°4×3NÑ3NÐQRÕ3RÑRð ÷ #,‘;Ð1DˆDÙ�a˜qÓ!ŠAñ @ð �y‰y˜‹|Ðr+   )r8   r¼   r¾   r¿   r   r    r   rº   rt   ru   rw   s   @@r(   r´   r´   ¤   s    ù‡ € ÷Eó E÷ ÷  ò  r+   r´   c                   óp   a a€ ] tR t^×t oV3R lV 3R lltR	V3R lR lltR t]R 4       tR t	Rt
VtV ;t# )
ÚModelc                ó    <€ V ^8„  d   QhRS[ /# r¶   r·   )r&   r'   s   "€r(   r)   ÚModel.__annotate__Ø   s   ø€ ÷ )ñ )™Yñ )r+   c                óä   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        \        P                  ! VP                  VP                  R R7      V n
        R V n        R# rˆ   )rE   rF   r8   r   r´   ÚmodelrK   rL   r   r   Úlm_headÚtie_word_embeddings)rV   r8   rX   s   &&€r(   rF   ÚModel.__init__Ø   sP   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ  Ó&ˆŒ
Ü—y’y ×!1Ñ!1°4·?±?ÈÔOˆŒØ#(ˆÖ r+   c                ó^   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          /# rÂ   rÅ   )r&   r'   s   "€r(   r)   rÕ   à   s,   ø€ ÷ ñ á—‘ðñ #¡2§8¡8Õ,ñ	r+   c                ó¾   € V P                  WV4      pV P                  '       d(   V P                   P                  P                  V4      pV# V P	                  V4      pV# r|   )r×   rÙ   r¼   Ú	as_linearrØ   )rV   rÃ   r^   rÄ   r±   s   &&&& r(   rr   ÚModel.__call__à   sW   € ð �j‰j˜Ð(8Ó9ˆØ×#×#Ð#Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r+   c                óD   € R V9  d   RV n         V P                  R4       V# )zlm_head.weightTrØ   )rÙ   Úpop)rV   Úweightss   &&r(   ÚsanitizeÚModel.sanitizeí   s#   € Ø 7Ô*Ø'+ˆDÔ$Ø�H‰H�YÔØˆr+   c                ó.   € V P                   P                  # r|   )r×   r¾   )rV   s   &r(   r¾   ÚModel.layersó   s   € à�z‰z× Ñ Ð r+   c                ób  € . p\        V P                  P                  4       F‰  pW P                  P                  ,          V P                  P                  ^,
          8X  d   VP	                  \        4       4       KZ  VP	                  \        V P                  P                  R7      4       K‹  	  V# )rd   )Úmax_size)r½   r8   r   r    Úappendr   r   r   )rV   ÚcachesrÍ   s   &  r(   Ú
make_cacheÚModel.make_cache÷   sy   € ØˆÜ�t—y‘y×2Ñ2Ö3ˆAà—I‘I×4Ñ4Õ4Ø—9‘9×3Ñ3°aÕ7ô8ð —‘œg›iÖ(à—‘œo°t·y±y×7OÑ7OÔPÖQñ 4ð ˆr+   )r8   rØ   r×   r   rÙ   rt   )r-   r.   r/   r0   rF   rr   râ   Úpropertyr¾   rê   r2   r3   rv   rw   s   @@r(   rÓ   rÓ   ×   s<   ù‡ € ÷)ó )÷ò òð ñ!ó ð!÷
ò 
r+   rÓ   ) Údataclassesr   Ú	functoolsr   Útypingr   r   r   Úmlx.coreÚcorera   Úmlx.nnrK   rB   r   r	   r
   r^   r   r   Ú
rope_utilsr   r   ÚModuler5   rQ   r†   Úcompilerž   r    r´   rÓ   r,   r+   r(   Ú<module>rö      sÇ   ðõ "Ý ß &Ñ &å Ý ç TÑ Tß +Ý 'ð ô�ó ó ðô&B#�—	‘	ô B#ôJ@ˆb�i‰iô @ôSˆ"�)‰)ô Sñ 	ˆ�‰˜tÔ$ñó %ðô�r—y‘yô ô:0�"—)‘)ô 0ôf*ˆB�I‰Iö *r+   