+
    UV-jœ&  ã                   ó4  € ^ RI HtHt ^ RIHt ^ RIHt ^ RIH	t	 ^ RI
Ht ^RIHtHt R R lt ! R R	]P                   4      t ! R
 R]P                   4      t ! R R]P                   4      t ! R R]P                   4      t ! R R]P                   4      tR# )é    )ÚAnyÚOptionalN)Úinitialize_rope)Ú	SwitchGLU)Úcreate_attention_maskÚscaled_dot_product_attentionc                óH   € V ^8„  d   QhR\         R\         R\        R\         /# )é   ÚstartÚstopÚbetaÚmax_position_embeddings)ÚintÚfloat)Úformats   "Úq/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/mistral4/language.pyÚ__annotate__r      s,   € ÷ ñ ÜðÜðÜ!&ðÜADñó    c                 óÈ   € ^V\         P                  ! ^\         P                  ! \         P                  ! W4      V,          4      ,           4      ,          ,           pVR,          # )é   ):NNNN)ÚmxÚlogÚfloorÚarange)r   r   r   r   Úscalings   &&&& r   Ú_get_llama_4_attn_scaler      sM   € ð �$œŸšØ	ŒB�HŠH”R—Y’Y˜uÓ+Ð.EÕEÓFÕFóõ õ €Gð �7ÕÐr   c                   óL   a a€ ] tR t^t oRtV 3R ltRV3R lR lltRtVtV ;t	# )ÚMistral4Attentionz<Multi-Latent Attention (MLA) with compressed KV projections.c                ó  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  V n	        VP                  VP                  ,           V n
        V P                  R,          V n        V P
                  fE   \        P                  ! VP                  V P                  V P                  ,          RR7      V n        M¤\        P                  ! VP                  VP
                  VP                   R7      V n        \        P$                  ! VP
                  4      V n        \        P                  ! VP
                  V P                  V P                  ,          RR7      V n        \        P                  ! VP                  V P                  V P                  ,           VP                   R7      V n        \        P$                  ! V P                  4      V n        \        P                  ! V P                  V P                  V P                  V P                  ,           ,          RR7      V n        \        P                  ! V P                  V P                  ,          VP                  VP                   R7      V n        \3        V P                  VP4                  R,          VP6                  VP4                  VP8                  4      V n        R# )g      à?NF©ÚbiasÚ
rope_thetag      à¿)ÚsuperÚ__init__ÚconfigÚnum_attention_headsÚ	num_headsÚq_lora_rankÚqk_rope_head_dimÚkv_lora_rankÚ
v_head_dimÚqk_nope_head_dimÚqk_head_dimÚscaleÚnnÚLinearÚhidden_sizeÚq_projÚattention_biasÚq_a_projÚRMSNormÚq_a_layernormÚq_b_projÚkv_a_proj_with_mqaÚkv_a_layernormÚ	kv_b_projÚo_projr   Úrope_parametersÚrope_traditionalr   Úrope)Úselfr%   Ú	__class__s   &&€r   r$   ÚMistral4Attention.__init__   s=  ø€ Ü‰ÑÔØŒØ×3Ñ3ˆŒØ!×-Ñ-ˆÔØ &× 7Ñ 7ˆÔØ"×/Ñ/ˆÔØ ×+Ñ+ˆŒØ &× 7Ñ 7ˆÔØ!×2Ñ2°V×5LÑ5LÕLˆÔà×%Ñ% tÕ+ˆŒ
ð ×ÑÒ#ÜŸ)š)Ø×"Ñ"Ø—‘ ×!1Ñ!1Õ1ØôˆD�Kô ŸIšIØ×"Ñ"Ø×"Ñ"Ø×*Ñ*ôˆDŒMô
 "$§¢¨F×,>Ñ,>Ó!?ˆDÔÜŸIšIØ×"Ñ"Ø—‘ ×!1Ñ!1Õ1ØôˆDŒMô #%§)¢)Ø×ÑØ×Ñ × 5Ñ 5Õ5Ø×&Ñ&ô#
ˆÔô
 !Ÿjšj¨×):Ñ):Ó;ˆÔÜŸšØ×ÑØ�N‰N˜d×3Ñ3°d·o±oÕEÕFØô
ˆŒô —i’iØ�N‰N˜TŸ_™_Õ,Ø×ÑØ×&Ñ&ô
ˆŒô $Ø×!Ñ!Ø×"Ñ" <Õ0Ø×#Ñ#Ø×"Ñ"Ø×*Ñ*ó
ˆŽ	r   c          
      ó¨   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r
   ÚxÚ
attn_scaleÚmaskÚcacheÚreturn©r   Úarrayr   r   )r   Ú__classdict__s   "€r   r   ÚMistral4Attention.__annotate__T   sV   ø€ ÷ 4#ñ 4#á�8‰8ð4#ñ —H‘Hð4#ñ ‘r—x‘xÕ ð	4#ñ
 ™�}ð4#ñ 
�‰ñ4#r   c           	     ó.  € VP                   w  rVpV P                  f   V P                  V4      pM/V P                  V P	                  V P                  V4      4      4      pVP                  WVV P                  V P                  4      P                  ^ ^^^4      p\        P                  ! W€P                  .RR7      w  ršV P                  V4      p\        P                  ! W°P                  .RR7      w  r¼VP                  WV^V P                  4      P                  ^ ^^^4      pV P!                  V P#                  V4      4      pVP                  WVV P                  R4      P                  ^ ^^^4      p\        P                  ! WÐP                  .RR7      w  rïVe�   V P%                  W¤P&                  R7      p
V P%                  WÄP&                  R7      p\        P(                  ! WÎP                   R R V P                  3,           4      p\        P*                  ! Wì.RR7      pVP-                  VV4      w  ppMpV P%                  V
4      p
V P%                  V4      p\        P(                  ! WÎP                   R R V P                  3,           4      p\        P*                  ! Wì.RR7      p\        P*                  ! Wš.RR7      pVV,          p\/        VVWôV P0                  VR7      pVP                  ^ ^^^4      P                  WVR4      pV P3                  V4      # )N©Úaxis)Úoffset)rG   r.   rF   éÿÿÿÿ)Úshaper(   r2   r7   r6   r4   Úreshaper'   r-   Ú	transposer   Úsplitr,   r8   r*   r)   r:   r9   r>   rP   Úbroadcast_toÚconcatenateÚupdate_and_fetchr   r.   r;   )r?   rD   rE   rF   rG   ÚBÚLÚDÚqÚq_nopeÚq_peÚcompressed_kvÚk_peÚkvÚk_nopeÚvaluesÚkeysÚqueriesÚoutputs   &&&&&              r   Ú__call__ÚMistral4Attention.__call__T   s‹  € ð —'‘'‰ˆˆað ×ÑÒ#Ø—‘˜A“‰Aà—‘˜d×0Ñ0°·±¸qÓ1AÓBÓCˆAà�I‰I�a˜DŸN™N¨D×,<Ñ,<Ó=×GÑGÈÈ1ÈaÐQRÓSˆÜ—x’x ×$9Ñ$9Ð#:ÀÔD‰ˆð ×/Ñ/°Ó2ˆÜ Ÿhšh }×7HÑ7HÐ6IÐPRÔSÑˆð �|‰|˜A ! T×%:Ñ%:Ó;×EÑEÀaÈÈAÈqÓQˆð �^‰^˜D×/Ñ/°Ó>Ó?ˆØ�Z‰Z˜˜dŸn™n¨bÓ1×;Ñ;¸A¸qÀ!ÀQÓGˆÜŸš "×'<Ñ'<Ð&=ÀBÔG‰ˆð ÒØ—9‘9˜T¯,©,�9Ó7ˆDØ—9‘9˜T¯,©,�9Ó7ˆDÜ—?’? 4¯©°c°rÐ):¸d×>SÑ>SÐ=UÕ)UÓVˆDÜ—>’> 6 .°rÔ:ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—9‘9˜T“?ˆDØ—9‘9˜T“?ˆDÜ—?’? 4¯©°c°rÐ):¸d×>SÑ>SÐ=UÕ)UÓVˆDÜ—>’> 6 .°rÔ:ˆDä—.’. & °bÔ9ˆð ˜JÕ&ˆä-Ø�T˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r   )r%   r9   r8   r:   r*   r'   r;   r6   r4   r7   r(   r2   r-   r,   r)   r>   r.   r+   ©NN)
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r$   rg   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r@   rK   s   @@r   r   r      s   ù‡ € ÙFõ;
÷z4#÷ 4#ò 4#r   r   c                   óT   a a€ ] tR t^‹t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚMistral4MLPc                ó    <€ V ^8„  d   QhRS[ /# )r
   Úintermediate_size©r   )r   rK   s   "€r   r   ÚMistral4MLP.__annotate__Œ   s   ø€ ÷ @ñ @±#ñ @r   c                ó  <€ \         SV `  4        VP                  pT;'       g    VP                  p\        P
                  ! W4R R7      V n        \        P
                  ! W4R R7      V n        \        P
                  ! WCR R7      V n        R# )Fr    N)	r#   r$   r1   rv   r/   r0   Ú	gate_projÚup_projÚ	down_proj)r?   r%   rv   ÚdimÚ
hidden_dimr@   s   &&&  €r   r$   ÚMistral4MLP.__init__Œ   sb   ø€ Ü‰ÑÔØ× Ñ ˆØ&×BÐB¨&×*BÑ*Bˆ
ÜŸš 3¸Ô?ˆŒÜ—y’y °uÔ=ˆŒÜŸš :¸Ô?ˆŽr   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r
   rH   ©r   rJ   )r   rK   s   "€r   r   rx   ”   s   ø€ ÷ Lñ L™RŸX™Xñ Lr   c                ó–   € V P                  \        P                  ! V P                  V4      4      V P	                  V4      ,          4      # ©N)r|   r/   Úsilurz   r{   )r?   rD   s   &&r   rg   ÚMistral4MLP.__call__”   s0   € Ø�~‰~œbŸgšg d§n¡n°QÓ&7Ó8¸4¿<¹<È»?ÕJÓKÐKr   )r|   rz   r{   rƒ   ©	rj   rk   rl   rm   r$   rg   ro   rp   rq   rr   s   @@r   rt   rt   ‹   s#   ù‡ € ÷@õ @÷L÷ Lð Lr   rt   c                   óD   a a€ ] tR t^˜t oV 3R ltV3R lR ltRtVtV ;t# )ÚMistral4MoEc                ó  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        \        P                  ! VP                  VP
                  R R7      V n        \        VP                  VP                  VP
                  4      V n        VP                  eD   VP                  ^ 8”  d1   VP                  VP                  ,          p\!        WR7      V n        R# R# R# )Fr    N)rv   )r#   r$   r%   Únum_experts_per_tokÚtop_kÚn_routed_expertsÚnorm_topk_probÚrouted_scaling_factorr/   r0   r1   Úgater   Úmoe_intermediate_sizeÚ
switch_mlpÚn_shared_expertsrt   Úshared_experts)r?   r%   Úshared_intermediater@   s   && €r   r$   ÚMistral4MoE.__init__š   sß   ø€ Ü‰ÑÔØŒØ×/Ñ/ˆŒ
Ø &× 7Ñ 7ˆÔØ$×3Ñ3ˆÔØ%+×%AÑ%AˆÔ"ô —I’I˜f×0Ñ0°&×2IÑ2IÐPUÔVˆŒ	ô $Ø×Ñ × <Ñ <¸f×>UÑ>Uó
ˆŒð ×"Ñ"Ò.°6×3JÑ3JÈQÔ3NØ"(×">Ñ">À×AXÑAXÕ"XÐÜ"-Øô#ˆDÖñ 4OÑ.r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r
   rD   rH   r�   )r   rK   s   "€r   r   ÚMistral4MoE.__annotate__°   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                óF  € TpV P                  V4      p\        P                  ! VR
RR7      pV P                  p\        P                  ! V) V^,
          R
R7      RRV13,          p\        P
                  ! W5R
R7      pV P                  '       d(   V\        P                  ! VR
RR7      R,           ,          pW`P                  ,          pV P                  W4      pWvR,          ,          P                  RR7      p\        V R	4      '       d   WpP                  V4      ,           pV# )r   T)rO   Úprecise)ÚkthrO   .NrN   )rO   Úkeepdimsg#B’¡œÇ;r“   rQ   ).Néþÿÿÿ)r�   r   Úsoftmaxr‹   ÚargpartitionÚtake_along_axisr�   ÚsumrŽ   r‘   Úhasattrr“   )r?   rD   Ú	residualsÚgatesÚkÚindsÚscoresÚys   &&      r   rg   ÚMistral4MoE.__call__°   só   € Øˆ	ð —	‘	˜!“ˆÜ—
’
˜5 r°4Ô8ˆð �J‰JˆÜ�Š ˜v¨1¨q­5°rÔ:¸3ÀÀÀ¸7ÕCˆÜ×#Ò# E°bÔ9ˆà××ÐØœrŸvšv f°2ÀÔEÈÕMÕNˆFà×4Ñ4Õ4ˆð �O‰O˜AÓ$ˆØ˜	Õ"Õ"×'Ñ'¨RÐ'Ó0ˆô �4Ð)×*Ò*Ø×'Ñ'¨	Ó2Õ2ˆAàˆr   )r%   r�   rŒ   r�   rŽ   r“   r‘   r‹   r†   rr   s   @@r   rˆ   rˆ   ˜   s   ù‡ € õ÷,÷ ð r   rˆ   c                   óT   a a€ ] tR t^Ìt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMistral4TransformerBlockc                ó    <€ V ^8„  d   QhRS[ /# )r
   Ú	layer_idxrw   )r   rK   s   "€r   r   Ú%Mistral4TransformerBlock.__annotate__Í   s   ø€ ÷ 
ñ 
©#ñ 
r   c                óŽ  <€ \         SV `  4        \        V4      V n        W!P                  8¼  d$   VP
                  '       d   \        V4      V n        M\        V4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  VP                  R 7      V n        R# )©ÚepsN)r#   r$   r   Ú	self_attnÚfirst_k_dense_replacerŒ   rˆ   Úmlprt   r/   r5   r1   Úrms_norm_epsÚinput_layernormÚpost_attention_layernorm)r?   r%   r¬   r@   s   &&&€r   r$   Ú!Mistral4TransformerBlock.__init__Í   sŠ   ø€ Ü‰ÑÔÜ*¨6Ó2ˆŒà×4Ñ4Ô4¸×9P×9PÐ9PÜ" 6Ó*ˆD�Hä" 6Ó*ˆDŒHä!Ÿzšz¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(*¯
ª
Ø×Ñ F×$7Ñ$7ô)
ˆÖ%r   c          
      ó¨   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# rC   rI   )r   rK   s   "€r   r   r­   Û   sV   ø€ ÷ ñ á�8‰8ðñ —H‘Hðñ ‘r—x‘xÕ ð	ñ
 ™�}ðñ 
�‰ñr   c                óª   € V P                  V P                  V4      W#V4      pW,           pV P                  V P                  V4      4      pWe,           pV# rƒ   )r±   rµ   r³   r¶   )r?   rD   rE   rF   rG   ÚrÚhÚouts   &&&&&   r   rg   Ú!Mistral4TransformerBlock.__call__Û   sM   € ð �N‰N˜4×/Ñ/°Ó2°JÀeÓLˆØ�EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�eˆØˆ
r   )rµ   r³   r¶   r±   ri   r†   rr   s   @@r   rª   rª   Ì   s   ù‡ € ÷
ó 
÷÷ ò r   rª   c                   óH   a a€ ] tR t^ét oV 3R ltRV3R lR lltRtVtV ;t# )ÚMistral4Modelc                ó†  <€ \         SV `  4        Wn        VP                  V n        \        P
                  ! VP                  VP                  4      V n        \        VP                  4       Uu. uF  p\        WR 7      NK  	  upV n        \        P                  ! VP                  VP                  R7      V n        R# u upi ))r¬   r¯   N)r#   r$   r%   Ú
vocab_sizer/   Ú	Embeddingr1   Úembed_tokensÚrangeÚnum_hidden_layersrª   Úlayersr5   r´   Únorm)r?   r%   Úir@   s   && €r   r$   ÚMistral4Model.__init__ê   s•   ø€ Ü‰ÑÔØŒØ ×+Ñ+ˆŒÜŸLšL¨×):Ñ):¸F×<NÑ<NÓOˆÔô ˜6×3Ñ3Ô4ó
á4�ô % V×9Ù4ñ
ˆŒô —J’J˜v×1Ñ1°v×7JÑ7JÔKˆŽ	ùò	
s   Á.B>c                ó^   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          /# )r
   ÚinputsÚinputs_embeds)r   rJ   r   )r   rK   s   "€r   r   ÚMistral4Model.__annotate__õ   s,   ø€ ÷ &ñ &á—‘ð&ñ  ¡§¡Õ)ñ	&r   c                ó*  € Ve   TpMV P                  V4      pVf   R .\        V P                  4      ,          p^ pV^ ,          e�   V^ ,          P                  p\	        V\
        4      '       d   TpMb\	        V\        P                  4      '       d,   VP                  ^ 8X  d   TMV^ ,          P                  4       pM\        R\        V4       24      h\        WB^ ,          4      p\        VWTP                  ^,          ,           V P                  P                   R,          V P                  P                   R,          4      P#                  VP$                  4      p\'        V P                  V4       F  w  ršV	! WHWzR7      pK  	  V P)                  V4      # )NzUnexpected cache offset type: Úllama_4_scaling_betaÚ original_max_position_embeddings)rG   )rÃ   ÚlenrÆ   rP   Ú
isinstancer   r   rJ   ÚndimÚitemÚ
ValueErrorÚtyper   r   rR   r%   r<   ÚastypeÚdtypeÚziprÇ   )r?   rË   rG   rÌ   r»   Úcache_offsetrP   rF   rE   ÚlayerÚcs   &&&&       r   rg   ÚMistral4Model.__call__õ   sH  € ð Ò$Ø‰Aà×!Ñ! &Ó)ˆAàŠ=Ø�FœS §¡Ó-Õ-ˆEð ˆØ��8ÒØ˜1•X—_‘_ˆFÜ˜&¤#×&Ò&Ø%‘Ü˜F¤B§H¡H×-Ò-Ø*0¯+©+¸Ô*:¡ÀÀqÅ	×OÑOÓQ‘ä Ð#AÄ$ÀvÃ,ÀÐ!PÓQÐQä$ Q¨a­Ó1ˆô -ØØŸ7™7 1�:Õ%Ø�K‰K×'Ñ'Ð(>Õ?Ø�K‰K×'Ñ'Ð(JÕKó	
÷
 ‰&�—‘‹/ð 	ô ˜DŸK™K¨Ö/‰HˆEÙ�a TÔ3ŠAñ 0ð �y‰y˜‹|Ðr   )r%   rÃ   rÆ   rÇ   rÁ   ri   r†   rr   s   @@r   r¿   r¿   é   s   ù‡ € õ	L÷&÷ &ò &r   r¿   )Útypingr   r   Úmlx.coreÚcorer   Úmlx.nnr/   Úmlx_lm.models.rope_utilsr   Úmlx_lm.models.switch_layersr   Úbaser   r   r   ÚModuler   rt   rˆ   rª   r¿   © r   r   Ú<module>rç      sr   ðß  å Ý Ý 4Ý 1ç Fõôt#˜Ÿ	™	ô t#ôn
L�"—)‘)ô 
Lô1�"—)‘)ô 1ôh˜rŸy™yô ô:2�B—I‘Iö 2r   