+
    TV-j—   ã                   óŽ  € ^ RI Ht ^ RIHtHtHtHtHt ^ RIH	t
 ^ RIHt ^RIHt ^RIHtHtHt ^RIHt ] ! R R]4      4       t ! R	 R
]P,                  4      t ! R R]P,                  4      t ! R R]P,                  4      t ! R R]P,                  4      t ! R R]P,                  4      t ! R R]P,                  4      tR# )é    )Ú	dataclass)ÚAnyÚDictÚListÚOptionalÚUnionN)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Ú	SwitchGLUc                   ó0   a € ] tR t^t o RtV 3R ltRtV tR# )Ú	ModelArgsNc                óJ  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[,          ;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[S[ S[S[S[ 3,          3,          ,          ;R&   # )é   Ú
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚnum_expertsÚnum_experts_per_tokÚdecoder_sparse_stepÚmlp_only_layersÚmoe_intermediate_sizeÚrms_norm_epsÚ
vocab_sizeÚnum_key_value_headsÚhead_dimÚ
rope_thetaÚtie_word_embeddingsÚmax_position_embeddingsÚnorm_topk_probÚrope_scaling)ÚstrÚintr   ÚfloatÚboolr   r   r   )ÚformatÚ__classdict__s   "€Úh/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/qwen3_moe.pyÚ__annotate__ÚModelArgs.__annotate__   s  ø‡ ‚ á�Oñ ñ Ññ ñ Ññ	 ñ
 Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ ™#•YÑñ ñ Ññ ñ Ññ ñ �Oñ ñ Ññ ñ �Mñ ñ  Ññ! ñ" Ññ# ñ$ !Ñ ñ% ñ& Ññ' ñ( ™4¡¡U©5±#¨:Õ%6Ð 6Õ7Õ8Ñ?ò) ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r$   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r*   s   @r+   r   r      s   ø‡ € ð( <@€L÷) ƒ r.   r   c                   óT   a a€ ] tR t^%t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r   ÚargsÚ	layer_idx©r   r&   )r)   r*   s   "€r+   r,   ÚAttention.__annotate__&   s   ø€ ÷ 
ñ 
™Yð 
±3ñ 
r.   c                ó  <€ \         SV `  4        VP                  pVP                  ;V n        pVP
                  f   Q hVP
                  ;V n        p\        VRVP                  VP                  ,          4      pVR,          V n        \        P                  ! W4V,          RR7      V n        \        P                  ! W5V,          RR7      V n        \        P                  ! W5V,          RR7      V n        \        P                  ! WF,          VRR7      V n        \        P                  ! WaP                   R7      V n        \        P                  ! WaP                   R7      V n        \        P&                  ! VRVP(                  R7      V n        R # )Nr   F©Úbias©Úeps)ÚtraditionalÚbaseg      à¿)ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsÚgetattrÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚo_projÚRMSNormr   Úq_normÚk_normÚRoPEr    Úrope)Úselfr;   r<   ÚdimrH   rI   r   Ú	__class__s   &&&    €r+   rG   ÚAttention.__init__&   s#  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ×'Ñ'Ò3Ð3Ð3Ø'+×'?Ñ'?Ð?ˆŒ˜*äØ�*˜d×.Ñ.°$×2JÑ2JÕJó
ˆð ˜t•^ˆŒ
ä—i’i ¨xÕ%7¸eÔDˆŒÜ—i’i °(Õ%:ÀÔGˆŒÜ—i’i °(Õ%:ÀÔGˆŒÜ—i’i Õ 2°C¸eÔDˆŒä—j’j ×/@Ñ/@ÔAˆŒÜ—j’j ×/@Ñ/@ÔAˆŒä—G’GØØØ—‘ô
ˆŽ	r.   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   ÚxÚmaskÚcacheÚreturn©ÚmxÚarrayr   r   )r)   r*   s   "€r+   r,   r>   A   sH   ø€ ÷ #ñ #á�8‰8ð#ñ ‘r—x‘xÕ ð#ñ ™�}ð	#ñ
 
�‰ñ#r.   c           	     óZ  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pV P	                  VP                  WEV P                  R4      4      P                  ^ ^^^4      pV P                  VP                  WEV P                  R4      4      P                  ^ ^^^4      pV	P                  WEV P                  R4      P                  ^ ^^^4      p	VeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      P                  WER4      p
V P                  V
4      # )é   )Úoffset)r_   rK   r^   éÿÿÿÿ)ÚshaperN   rO   rP   rS   ÚreshaperH   Ú	transposerT   rI   rV   rf   Úupdate_and_fetchr   rK   rQ   )rW   r]   r^   r_   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r+   Ú__call__ÚAttention.__call__A   so  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆð —+‘+˜gŸo™o¨a°D·L±LÀ"ÓEÓF×PÑPØˆq�!�Qó
ˆð �{‰{˜4Ÿ<™<¨¨d¯o©o¸rÓBÓC×MÑMØˆq�!�Qó
ˆð —‘  d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r.   )
rT   rO   rH   rI   rQ   rS   rN   rV   rK   rP   ©NN©	r0   r1   r2   r3   rG   rs   r5   r6   Ú__classcell__©rY   r*   s   @@r+   r8   r8   %   s   ù‡ € ÷
ó 
÷6#÷ #ò #r.   r8   c                   óD   a a€ ] tR t^ct oV 3R ltV3R lR ltRtVtV ;t# )ÚMLPc                óÒ   <€ \         SV `  4        \        P                  ! WR R7      V n        \        P                  ! W!R R7      V n        \        P                  ! WR R7      V n        R# ©Fr@   N)rF   rG   rL   rM   Ú	gate_projÚ	down_projÚup_proj)rW   rX   Ú
hidden_dimrY   s   &&&€r+   rG   ÚMLP.__init__d   sD   ø€ Ü‰ÑÔÜŸš 3¸Ô?ˆŒÜŸš :¸Ô?ˆŒÜ—y’y °uÔ=ˆŽr.   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   r`   ©rb   rc   )r)   r*   s   "€r+   r,   ÚMLP.__annotate__j   s   ø€ ÷ Jñ J™RŸX™Xñ Jr.   c                ót   € V P                  \        V P                  V4      V P                  V4      4      4      # ©N)r~   r	   r}   r   )rW   r]   s   &&r+   rs   ÚMLP.__call__j   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr.   )r~   r}   r   rv   rx   s   @@r+   rz   rz   c   s   ù‡ € õ>÷J÷ Jð Jr.   rz   c                   óP   a a€ ] tR t^nt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚQwen3MoeSparseMoeBlockc                ó    <€ V ^8„  d   QhRS[ /# ©r   r;   ©r   )r)   r*   s   "€r+   r,   Ú#Qwen3MoeSparseMoeBlock.__annotate__o   s   ø€ ÷ 
Iñ 
I™Yñ 
Ir.   c                ó  <€ \         SV `  4        VP                  pVP                  pVP                  ;V n        pVP
                  V n        VP                  V n        \        P                  ! W$R R7      V n
        \        W#V4      V n        R# r|   )rF   rG   r   r   r   r   Útop_kr#   rL   rM   Úgater   Ú
switch_mlp)rW   r;   rX   r   r   rY   s   &&   €r+   rG   ÚQwen3MoeSparseMoeBlock.__init__o   ss   ø€ Ü‰ÑÔØ×ÑˆØ ×6Ñ6Ðà)-×)9Ñ)9Ð9ˆÔ˜;Ø×-Ñ-ˆŒ
Ø"×1Ñ1ˆÔä—I’I˜c°UÔ;ˆŒ	Ü# C¸KÓHˆŽr.   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   r]   r`   rƒ   )r)   r*   s   "€r+   r,   r�   {   s'   ø€ ÷ ñ á�8‰8ðñ 
�‰ñr.   c                ó°  € V P                  V4      p\        P                  ! VRRR7      pV P                  p\        P                  ! W#) RR7      RV) R13,          p\        P
                  ! W$RR7      pV P                  '       d!   V\        P                  ! VRRR7      ,          pV P                  W4      pWeR	,          ,          P                  R
R7      pV# )re   T)ÚaxisÚprecise)Úkthr•   .N)r•   )r•   Úkeepdimsrg   ).Néþÿÿÿ)	r�   rb   Úsoftmaxr�   ÚargpartitionÚtake_along_axisr#   Úsumr‘   )rW   r]   ÚgatesÚkÚindsÚscoresÚys   &&     r+   rs   ÚQwen3MoeSparseMoeBlock.__call__{   s®   € ð —	‘	˜!“ˆÜ—
’
˜5 r°4Ô8ˆà�J‰JˆÜ�Š˜u¨"°2Ô6°s¸Q¸B¹C°xÕ@ˆÜ×#Ò# E°bÔ9ˆØ××ÐØ”b—f’f˜V¨"°tÔ<Õ<ˆFà�O‰O˜AÓ$ˆØ˜	Õ"Õ"×'Ñ'¨RÐ'Ó0ˆàˆr.   )r�   r#   r   r‘   r�   rv   rx   s   @@r+   r‰   r‰   n   s    ù‡ € ÷
Ió 
I÷÷ ð r.   r‰   c                   óT   a a€ ] tR t^Žt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚQwen3MoeDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# r:   r=   )r)   r*   s   "€r+   r,   Ú!Qwen3MoeDecoderLayer.__annotate__�   s   ø€ ÷ Eñ E™Yð E±3ñ Er.   c                ó$  <€ \         SV `  4        VP                  V n        \        W4      V n        \
        P                  ! VP                  VP                  R 7      V n        \
        P                  ! VP                  VP                  R 7      V n	        Wn
        W!P                  9  dC   VP                  ^ 8”  d2   V^,           VP                  ,          ^ 8X  d   \        V4      V n        R# \!        VP                  VP"                  4      V n        R# )rB   N)rF   rG   r   r8   Ú	self_attnrL   rR   r   Úinput_layernormÚpost_attention_layernormr;   r   r   r   r‰   Úmlprz   r   )rW   r;   r<   rY   s   &&&€r+   rG   ÚQwen3MoeDecoderLayer.__init__�   sÇ   ø€ Ü‰ÑÔØ×+Ñ+ˆÔÜ" 4Ó3ˆŒä!Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
ª
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ð Œ	à×1Ñ1Ô1Ø×Ñ˜qÔ  i°!¥m°t×7OÑ7OÕ%OÐSTÔ%Tä-¨dÓ3ˆDŽHä˜4×+Ñ+¨T×-CÑ-CÓDˆDŽHr.   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# r\   ra   )r)   r*   s   "€r+   r,   r§   ¡   sH   ø€ ÷ 
ñ 
á�8‰8ð
ñ ‘r—x‘xÕ ð
ñ ™�}ð	
ñ
 
�‰ñ
r.   c                ó¨   € V P                  V P                  V4      W#4      pW,           pV P                  V P                  V4      4      pWT,           pV# r†   )r©   rª   r¬   r«   )rW   r]   r^   r_   ÚrÚhÚouts   &&&&   r+   rs   ÚQwen3MoeDecoderLayer.__call__¡   sK   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ�EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�eˆØˆ
r.   )r;   r   rª   r¬   r«   r©   ru   rv   rx   s   @@r+   r¥   r¥   Ž   s    ù‡ € ÷Eó E÷$
÷ 
ò 
r.   r¥   c                   óT   a a€ ] tR t^®t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚQwen3MoeModelc                ó    <€ V ^8„  d   QhRS[ /# r‹   rŒ   )r)   r*   s   "€r+   r,   ÚQwen3MoeModel.__annotate__¯   s   ø€ ÷ Hñ H™Yñ Hr.   c                óÎ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        V P                  ^ 8”  g   Q h\
        P                  ! VP                  VP                  4      V n        \        VP                  4       Uu. uF  p\        WR7      NK  	  upV n        \
        P                  ! VP                  VP                  R7      V n        R# u upi )r   )r;   r<   rB   N)rF   rG   r;   r   r   rL   Ú	Embeddingr   Úembed_tokensÚranger¥   ÚlayersrR   r   Únorm)rW   r;   ÚirY   s   && €r+   rG   ÚQwen3MoeModel.__init__¯   s±   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ�‰ Ô"Ð"Ð"ÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔô ˜4×1Ñ1Ô2ó
á2�ô ! d×8Ù2ñ
ˆŒô —J’J˜t×/Ñ/°T×5FÑ5FÔGˆŽ	ùò	
s   ÂC"c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[ P                  /# ©r   ÚinputsÚinput_embeddingsr`   ©rb   rc   r   )r)   r*   s   "€r+   r,   r·   ¼   s:   ø€ ÷ ñ á—‘ðñ #¡2§8¡8Õ,ð	ñ
 
�‰ñr.   c                ó
  € Ve   TpMV P                  V4      pVf   R .\        V P                  4      ,          p\        WB^ ,          4      p\	        V P                  V4       F  w  rgV! WEV4      pK  	  V P                  V4      # r†   )rº   Úlenr¼   r   Úzipr½   )rW   rÂ   r_   rÃ   r±   r^   ÚlayerÚcs   &&&&    r+   rs   ÚQwen3MoeModel.__call__¼   sz   € ð Ò'Ø ‰Aà×!Ñ! &Ó)ˆAàŠ=Ø�FœS §¡Ó-Õ-ˆEä$ Q¨a­Ó1ˆä˜DŸK™K¨Ö/‰HˆEÙ�a˜qÓ!ŠAñ 0ð �y‰y˜‹|Ðr.   )r;   rº   r¼   r½   r   r   ru   rv   rx   s   @@r+   rµ   rµ   ®   s    ù‡ € ÷Hó H÷÷ ò r.   rµ   c                   óz   a a€ ] tR t^Òt oV3R lV 3R lltR	V3R lR lltR t]R 4       t]R 4       t	Rt
VtV ;t# )
ÚModelc                ó    <€ V ^8„  d   QhRS[ /# r‹   rŒ   )r)   r*   s   "€r+   r,   ÚModel.__annotate__Ó   s   ø€ ÷ Tñ T™Yñ Tr.   c                óþ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        VP                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# R# r|   )rF   rG   r;   r   rµ   Úmodelr!   rL   rM   r   r   Úlm_head)rW   r;   rY   s   &&€r+   rG   ÚModel.__init__Ó   sY   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ" 4Ó(ˆŒ
Ø×'×'Ð'ÜŸ9š9 T×%5Ñ%5°t·±ÈUÔSˆDŽLñ (r.   c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[ P                  /# rÁ   rÄ   )r)   r*   s   "€r+   r,   rÎ   Û   s:   ø€ ÷ ñ á—‘ðñ #¡2§8¡8Õ,ð	ñ
 
�‰ñr.   c                óÒ   € V P                  WV4      pV P                  P                  '       d(   V P                   P                  P	                  V4      pV# V P                  V4      pV# r†   )rÐ   r;   r!   rº   Ú	as_linearrÑ   )rW   rÂ   r_   rÃ   r²   s   &&&& r+   rs   ÚModel.__call__Û   s[   € ð �j‰j˜Ð(8Ó9ˆØ�9‰9×(×(Ð(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r.   c                óÌ  € V P                   P                  '       d   VP                  R R4       RV9  d   V# \        V P                   P                  4       F„  pRV 2pR	 Fv  pV RV R2V9   g   K  \        V P                   P
                  4       Uu. uF  pVP                  V RV RV R24      NK  	  pp\        P                  ! V4      W RV R2&   Kx  	  K†  	  V# u upi )
zlm_head.weightNz+model.layers.0.mlp.experts.0.up_proj.weightzmodel.layers.z.mlp.experts.0.z.weightz.mlp.experts.Ú.z.mlp.switch_mlp.)r   r~   r}   )r;   r!   Úpopr»   r   r   rb   Ústack)rW   ÚweightsÚlÚprefixÚnÚeÚto_joins   &&     r+   ÚsanitizeÚModel.sanitizeè   sî   € Ø�9‰9×(×(Ð(Ø�K‰KÐ(¨$Ô/Ø8ÀÔGØˆNÜ�t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFÛ:�Ø�X˜_¨Q¨C¨wÐ7¸7ÖBô "' t§y¡y×'<Ñ'<Ô!=óá!=˜Að  Ÿ™ v h¨m¸A¸3¸aÀ¸sÀ'Ð$JÖKÙ!=ð ð ô FHÇXÂXÈgÓEV�G˜hÐ&6°q°c¸ÐAÓBó ;ñ 4ð ˆùòs   Â#C!c                ó   € R  pV# )c                 ó@   € V P                  R 4      '       d   R^@R^/# R# )zmlp.gateÚ
group_sizeÚbitsT)Úendswith)ÚpathÚ_s   &&r+   Ú	predicateÚ(Model.quant_predicate.<locals>.predicateú   s#   € Ø�}‰}˜Z×(Ò(Ø$ b¨&°!Ð4Ð4Ùr.   r/   )rW   rê   s   & r+   Úquant_predicateÚModel.quant_predicateø   s   € ò	ð
 Ðr.   c                ó.   € V P                   P                  # r†   )rÐ   r¼   )rW   s   &r+   r¼   ÚModel.layers  s   € à�z‰z× Ñ Ð r.   )r;   rÑ   rÐ   r   ru   )r0   r1   r2   r3   rG   rs   rá   Úpropertyrì   r¼   r5   r6   rw   rx   s   @@r+   rÌ   rÌ   Ò   sM   ù‡ € ÷Tó T÷ò òð  ñó ðð ñ!ó ÷!ð !r.   rÌ   )Údataclassesr   Útypingr   r   r   r   r   Úmlx.coreÚcorerb   Úmlx.nnrL   Úactivationsr	   rE   r
   r   r   Úswitch_layersr   r   ÚModuler8   rz   r‰   r¥   rµ   rÌ   r/   r.   r+   Ú<module>rù      s¦   ðõ "ß 3Õ 3å Ý å ß TÑ TÝ $ð ô@�ó @ó ð@ô,;#�—	‘	ô ;#ô|Jˆ"�)‰)ô Jô˜RŸY™Yô ô@˜2Ÿ9™9ô ô@!�B—I‘Iô !ôH1!ˆB�I‰Iö 1!r.   