+
    TV-jÀ?  ã                  ój  € ^ RI Ht ^ RIHt ^ RIHt ^ RIHtHtH	t	H
t
HtHt ^ RIHt ^ RIHt ^ RIHt ^RIHt ^RIHtHtHtHt ^R	IHtHt ^R
IHt ^RI H!t! ^RI"H#t# ] ! R R]4      4       t$]! ]PJ                  RR7      R 4       t& ! R R]PN                  4      t( ! R R]PN                  4      t) ! R R]PN                  4      t* ! R R]PN                  4      t+ ! R R]PN                  4      t, ! R R]PN                  4      t- ! R R]PN                  4      t. ! R  R!]PN                  4      t/R# )"é    )Úannotations)Ú	dataclass)Úpartial)ÚAnyÚDictÚListÚOptionalÚTupleÚUnionN)Úsum_gradients)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚcreate_ssm_maskÚscaled_dot_product_attention)ÚArraysCacheÚKVCache)Úgated_delta_update)Úinitialize_rope)Ú	SwitchGLUc                  óD  € ] tR t^t$ R]R&   R]R&   R]R&   R]R&   R]R&   R]R&   R]R	&   R]R
&   R]R&   R]R&   R]R&   R]R&   R]R&   R]R&   R]R&   R]R&   R]R&   R]R&   R]R&   R]R&   R]R&   R]R&   R]R&   RtR]R&   RtR]R&   RtR]R &   R!tR"]R#&   ^t	R]R$&   R%t
R!# )&Ú	ModelArgsÚstrÚ
model_typeÚintÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚlinear_num_value_headsÚlinear_num_key_headsÚlinear_key_head_dimÚlinear_value_head_dimÚlinear_conv_kernel_dimÚnum_expertsÚnum_experts_per_tokÚdecoder_sparse_stepÚshared_expert_intermediate_sizez	List[int]Úmlp_only_layersÚmoe_intermediate_sizeÚfloatÚrms_norm_epsÚ
vocab_sizeÚnum_key_value_headsÚ
rope_thetaÚpartial_rotary_factorÚmax_position_embeddingsÚhead_dimFÚboolÚnorm_topk_probÚtie_word_embeddingsÚattention_biasNz&Optional[Dict[str, Union[float, str]]]Úrope_scalingÚfull_attention_interval© )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__annotations__r4   r5   r6   r7   r8   Ú__static_attributes__r9   ó    Úi/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/qwen3_next.pyr   r      s¾   ‡ àƒOØÓØÓØÓØÓØÓØÓØÓØÓØÓØÓØÓØÓØ%(Ó(ØÓØÓØÓØƒOØÓØÓØ Ó Ø Ó ØƒMØ €N�DÓ Ø %Ð˜Ó%Ø €N�DÓ Ø;?€LÐ8Ó?Ø#$Ð˜S×$r@   r   T)Ú	shapelessc                óè   € \         P                  ! VP                  \        P                  4      4      pVP                  \        P                  4      pW,          P                  V P
                  4      # ©N)ÚnnÚsiluÚastypeÚmxÚfloat32Údtype)ÚhÚgateÚxs   &&&rA   Ú_precise_swiglurN   :   sF   € ä�7Š7�4—;‘;œrŸz™zÓ*Ó+€DØ	�‰”—‘Ó€AØ�H×Ñ˜QŸW™WÓ%Ð%r@   c                  óB   a € ] tR t^AtRR V 3R llltRR R lltRtV ;t# )ÚQwen3NextRMSNormGatedc               ó    € V ^8„  d   QhRRRR/# )é   r   r   Úepsr+   r9   )Úformats   "rA   Ú__annotate__Ú"Qwen3NextRMSNormGated.__annotate__B   s   € ÷ +ñ + Cð +¨eñ +r@   c                	óf   <€ \         SV `  4        W n        \        P                  ! V4      V n        R # rD   )ÚsuperÚ__init__rS   rH   ÚonesÚweight)Úselfr   rS   Ú	__class__s   &&&€rA   rY   ÚQwen3NextRMSNormGated.__init__B   s#   ø€ Ü‰ÑÔØŒÜ—g’g˜kÓ*ˆŽr@   c               ó$   € V ^8„  d   QhRRRRRR/# )rR   Úhidden_statesúmx.arrayrL   zmx.array | NoneÚreturnr9   )rT   s   "rA   rU   rV   G   s$   € ÷ 1ñ 1Ø%ð1Ø-<ð1à	ñ1r@   c                	óÀ   € \         P                  P                  WP                  V P                  4      pVe   \        WV4      # VP                  VP                  4      # rD   )rH   ÚfastÚrms_normr[   rS   rN   rG   rJ   )r\   r`   rL   rM   s   &&& rA   Ú__call__ÚQwen3NextRMSNormGated.__call__G   sJ   € ô �G‰G×Ñ˜]¯K©K¸¿¹ÓBˆØÒÜ" =¸Ó:Ð:à—8‘8˜M×/Ñ/Ó0Ð0r@   )rS   r[   )ç�íµ ÷Æ°>rD   ©r:   r;   r<   r=   rY   rf   r?   Ú__classcell__©r]   s   @rA   rP   rP   A   s   ø† ÷+ò +÷
1õ 1r@   rP   c                  ó>   a € ] tR t^QtR V 3R lltRR R lltRtV ;t# )ÚQwen3NextAttentionc               ó   € V ^8„  d   QhRR/# ©rR   Úargsr   r9   )rT   s   "rA   rU   ÚQwen3NextAttention.__annotate__R   s   € ÷ %
ñ %
˜Yñ %
r@   c                	ó¢  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        V P                  R,          V n        \        P                  ! VP                  V P                  V P                  ,          ^,          VP                  R7      V n
        \        P                  ! VP                  V P                  V P                  ,          VP                  R7      V n        \        P                  ! VP                  V P                  V P                  ,          VP                  R7      V n        \        P                  ! V P                  V P                  ,          VP                  VP                  R7      V n        \        P                  ! V P                  VP                  R7      V n        \        P                  ! V P                  VP                  R7      V n        \%        \'        V P                  VP(                  ,          4      VP*                  RVP,                  VP.                  R7      V n        R# )g      à?©Úbias©rS   F)ÚbaseÚtraditionalÚscaling_configr1   Nç      à¿)rX   rY   r.   r   r2   ÚscalerE   ÚLinearr   r6   Úq_projÚk_projÚv_projÚo_projÚRMSNormr,   Úq_normÚk_normr   r   r0   r/   r7   r1   Úrope©r\   rp   r]   s   &&€rA   rY   ÚQwen3NextAttention.__init__R   s™  ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø#'×#;Ñ#;ˆÔ ØŸ™ˆŒØ—]‘] DÕ(ˆŒ
ä—i’iØ×ÑØ×$Ñ$ t§}¡}Õ4°qÕ8Ø×$Ñ$ô
ˆŒô
 —i’iØ×ÑØ×$Ñ$ t§}¡}Õ4Ø×$Ñ$ô
ˆŒô
 —i’iØ×ÑØ×$Ñ$ t§}¡}Õ4Ø×$Ñ$ô
ˆŒô
 —i’iØ×$Ñ$ t§}¡}Õ4Ø×ÑØ×$Ñ$ô
ˆŒô —j’j §¡°D×4EÑ4EÔFˆŒÜ—j’j §¡°D×4EÑ4EÔFˆŒä#Ü�—‘ × :Ñ :Õ:Ó;Ø—‘ØØ×,Ñ,Ø$(×$@Ñ$@ô
ˆŽ	r@   c               ó(   € V ^8„  d   QhRRRRRRRR/# ©rR   rM   ra   ÚmaskúOptional[mx.array]ÚcacheúOptional[Any]rb   r9   )rT   s   "rA   rU   rq   y   s2   € ÷ %6ñ %6àð%6ð !ð%6ð ð	%6ð
 
ñ%6r@   c           	     	óê  € VP                   w  rEpV P                  V4      p\        P                  ! VP	                  WEV P
                  R4      ^RR7      w  r‰V	P	                  WER4      p	V P                  V4      V P                  V4      rºV P                  V4      P                  ^ ^^^4      pV P                  V
P	                  WEV P                  R4      4      P                  ^ ^^^4      p
VP	                  WEV P                  R4      P                  ^ ^^^4      pVeM   V P                  WƒP                  R7      pV P                  W£P                  R7      p
VP                  W«4      w  r«M"V P                  V4      pV P                  V
4      p
\        WŠW³V P                   VR7      pVP                  ^ ^^^4      P	                  WER4      pV P#                  V\        P$                  ! V	4      ,          4      # )é   ©Úaxis)Úoffset)rŠ   rz   rˆ   éÿÿÿÿ)Úshaper|   rH   ÚsplitÚreshaper   r}   r~   r�   Ú	transposer‚   r.   rƒ   r�   Úupdate_and_fetchr   rz   r   Úsigmoid)r\   rM   rˆ   rŠ   ÚBÚLÚDÚq_proj_outputÚqueriesrL   ÚkeysÚvaluesÚoutputs   &&&&         rA   rf   ÚQwen3NextAttention.__call__y   s­  € ð —'‘'‰ˆˆaàŸ™ A›ˆÜŸšØ×!Ñ! !¨×(@Ñ(@À"ÓEÀqÈrô
‰ˆð �|‰|˜A "Ó%ˆà—{‘{ 1“~ t§{¡{°1£~ˆfà—+‘+˜gÓ&×0Ñ0°°A°q¸!Ó<ˆØ�{‰{˜4Ÿ<™<¨¨d×.FÑ.FÈÓKÓL×VÑVØˆq�!�Qó
ˆð —‘  d×&>Ñ&>ÀÓC×MÑMØˆq�!�Qó
ˆð ÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆà�{‰{˜6¤B§J¢J¨tÓ$4Õ4Ó5Ð5r@   )r2   r‚   r}   r   r.   r   r�   r|   rƒ   rz   r~   ©NNri   rk   s   @rA   rm   rm   Q   s   ø† ÷%
ð %
÷N%6õ %6r@   rm   c                  ó4   a € ] tR t^¡tV 3R ltR R ltRtV ;t# )ÚQwen3NextMLPc                	óÒ   <€ \         SV `  4        \        P                  ! WR R7      V n        \        P                  ! W!R R7      V n        \        P                  ! WR R7      V n        R# ©Frs   N)rX   rY   rE   r{   Ú	gate_projÚ	down_projÚup_proj)r\   ÚdimÚ
hidden_dimr]   s   &&&€rA   rY   ÚQwen3NextMLP.__init__¢   sD   ø€ Ü‰ÑÔÜŸš 3¸Ô?ˆŒÜŸš :¸Ô?ˆŒÜ—y’y °uÔ=ˆŽr@   c               ó   € V ^8„  d   QhRR/# )rR   rb   ra   r9   )rT   s   "rA   rU   ÚQwen3NextMLP.__annotate__¨   s   € ÷ Jñ J˜Xñ Jr@   c                	ót   € V P                  \        V P                  V4      V P                  V4      4      4      # rD   )r§   r   r¦   r¨   )r\   rM   s   &&rA   rf   ÚQwen3NextMLP.__call__¨   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr@   )r§   r¦   r¨   ri   rk   s   @rA   r£   r£   ¡   s   ø† õ>÷Jó Jr@   r£   c                  óJ   a € ] tR t^¬tR V 3R lltR R ltRR R lltRtV ;t# )	ÚQwen3NextGatedDeltaNetc               ó   € V ^8„  d   QhRR/# )rR   Úconfigr   r9   )rT   s   "rA   rU   Ú#Qwen3NextGatedDeltaNet.__annotate__­   s   € ÷ 'Pñ 'P˜yñ 'Pr@   c           	     	ó¬  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        VP                  V n        VP                  V n
        V P                  V P                  ,          V n        V P                  V P                  ,          V n        V P                  V P                  ,          ^ 8w  d'   \        RV P                   RV P                   R24      hVP                  V n        VP                   V n        V P                  ^,          V P                  ,           V n        \&        P(                  ! V P$                  V P$                  RV P                  V P$                  ^ R7      V n        \&        P,                  ! V P                  V P                  ^,          V P                  ^,          ,           RR7      V n        \&        P,                  ! V P                  V P                  ^,          RR7      V n        \2        P4                  ! V P                  4      V n        \2        P8                  P;                  ^ ^V P                  3R7      p\2        P<                  ! V4      V n        \A        V P                  V P"                  R7      V n!        \&        P,                  ! V P                  V P                  RR7      V n"        R	# )
r   znum_v_heads (z$) must be divisible by num_k_heads (Ú)F)Úin_channelsÚout_channelsrt   Úkernel_sizeÚgroupsÚpaddingrs   )ÚlowÚhighr’   ru   N)#rX   rY   r   r    Únum_v_headsr!   Únum_k_headsr"   Ú
head_k_dimr#   Ú
head_v_dimÚkey_dimÚ	value_dimÚ
ValueErrorr$   Úconv_kernel_sizer,   Úlayer_norm_epsilonÚconv_dimrE   ÚConv1dÚconv1dr{   Úin_proj_qkvzÚ
in_proj_barH   rZ   Údt_biasÚrandomÚuniformÚlogÚA_logrP   ÚnormÚout_proj)r\   r³   ÚAr]   s   && €rA   rY   ÚQwen3NextGatedDeltaNet.__init__­   s  ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ!×8Ñ8ˆÔØ!×6Ñ6ˆÔØ ×4Ñ4ˆŒØ ×6Ñ6ˆŒØ—‘¨×)9Ñ)9Õ9ˆŒØŸ™¨4×+;Ñ+;Õ;ˆŒØ×Ñ˜d×.Ñ.Õ.°!Ô3ÜØ × 0Ñ 0Ð1Ð1UÐVZ×VfÑVfÐUgÐghÐióð ð !'× =Ñ =ˆÔØ"(×"5Ñ"5ˆÔàŸ™ qÕ(¨4¯>©>Õ9ˆŒÜ—i’iØŸ™ØŸ™ØØ×-Ñ-Ø—=‘=Øô
ˆŒô ŸIšIØ×Ñ˜dŸl™l¨QÕ.°·±À!Õ1CÕCÈ%ô
ˆÔô Ÿ)š) D×$4Ñ$4°d×6FÑ6FÈÕ6JÐQVÔWˆŒä—w’w˜t×/Ñ/Ó0ˆŒä�I‰I×Ñ !¨"°T×5EÑ5EÐ4GÐÓHˆÜ—V’V˜A“YˆŒ
ä)¨$¯/©/¸t×?VÑ?VÔWˆŒ	äŸ	š	 $§.¡.°$×2BÑ2BÈÔOˆŽr@   c               ó$   € V ^8„  d   QhRRRRRR/# )rR   Ú
mixed_qkvzra   Úmixed_barb   r9   )rT   s   "rA   rU   r´   Ö   s$   € ÷ 
ñ 
Ø"ð
Ø.6ð
à	ñ
r@   c           
     	óð  € V P                   V P                  V P                  V P                  3w  r4rVVP                  ! . VP
                  R R OVNRN5!  pVP                  ! . VP
                  R R OVNRN5!  p\        P                  ! W^V,          ^V,          WS,          V,          ,           .RR7      w  rxrš\        P                  ! W%V,          .RR7      w  r¼VVV	P                  ! . V	P
                  R,          ORNVN5!  V
P                  ! . V
P
                  R,          ORNVN5!  VP                  ! . VP
                  R,          OVN5!  VP                  ! . VP
                  R,          OVN5!  3# )NrŽ   :NrR   Nr‘   )r¿   rÀ   r¾   rÁ   r”   r’   rH   r“   )r\   rÖ   r×   ÚnkÚdnÚnvÚdvÚqÚkÚvÚzÚbÚas   &&&          rA   Úfix_query_key_value_orderingÚ3Qwen3NextGatedDeltaNet.fix_query_key_value_orderingÖ   sT  € ð ×ÑØ�O‰OØ×ÑØ�O‰Oð	
‰ˆ�ð  ×'Ò'ÐG¨×)9Ñ)9¸#¸2Ð)>ÐGÀÐGÀBÓGˆ
Ø×#Ò#ÐA X§^¡^°C°RÐ%8ÐA¸"ÐA¸bÓAˆÜ—X’X˜j¨q°2­v°q¸2µvÀÅÈ2ÅÕ7MÐ*NÐUWÔX‰
ˆˆaÜ�xŠx˜¨¥8 *°2Ô6‰ˆàØØ�IŠIÐ+�q—w‘w˜r•{Ð+ BÐ+¨Ó+Ø�IŠIÐ+�q—w‘w˜r•{Ð+ BÐ+¨Ó+Ø�IŠIÐ'�q—w‘w˜r•{Ð' BÓ'Ø�IŠIÐ'�q—w‘w˜r•{Ð' BÓ'ð
ð 	
r@   c               ó(   € V ^8„  d   QhRRRRRRRR/# )rR   Úinputsra   rˆ   r‰   rŠ   r‹   rb   r9   )rT   s   "rA   rU   r´   ì   s8   € ÷ E4ñ E4àðE4ð !ðE4ð ð	E4ð
 
ñE4r@   c                	ó|  € VP                   w  rEpV P                  V P                  V4      V P                  V4      4      w  rxršr¼Ve   V^ ,          e   V^ ,          pM?\        P
                  ! W@P                  ^,
          V P                  3VP                  R7      p\        P                  ! VP                  WER4      VP                  WER4      V	P                  WER4      .RR7      pVe    \        P                  ! VR,          V^ 4      p\        P                  ! WÞ.^R7      pVe²   V P                  ^,
          pVP                  ek   \        P                  ! VP                  ^ V4      pVR,          \        P                  ! V4      ,           R,          p\        P                  ! VV^R7      V^ &   M&\        P                   ! VRV) R 1R3,          4      V^ &   \"        P$                  ! V P'                  V4      4      p\)        \        P*                  ! VV P,                  ^V P,                  ,          .R4      V P.                  V P.                  V P0                  .V P2                  V P2                  V P4                  .4       UUUu. uF  w  pppVP                  WEVV4      NK  	  upppw  rxp	V'       d
   V^,          MR pVP                   R,          R	,          pV^,          \        P6                  P9                  VR R4      ,          pV\        P6                  P9                  VR R4      ,          p\;        VVV	VVV P<                  V P>                  VVV P@                  '       * R7
      w  ppVe   VV^&   VPC                  V4       V PE                  VV
4      pV PG                  VP                  WER4      4      # u upppi )
N)rJ   rŽ   ºNNNrh   )Ú
use_kernelr‘   ©.N)rè   Nry   )$r’   rã   rÊ   rË   rH   ÚzerosrÅ   rÇ   rJ   Úconcatenater”   ÚwhereÚlengthsÚclipÚarangeÚtake_along_axisÚ
contiguousrE   rF   rÉ   Úzipr“   rÂ   r¿   r¾   rÀ   rÁ   rd   re   r   rÐ   rÌ   ÚtrainingÚadvancerÑ   rÒ   )r\   ræ   rˆ   rŠ   r˜   ÚSÚ_rÝ   rÞ   rß   rà   rá   râ   Ú
conv_stateÚ	mixed_qkvÚ
conv_inputÚn_keepÚendsÚ	positionsÚconv_outÚtrK   ÚdÚstateÚ	inv_scaleÚouts   &&&&                      rA   rf   ÚQwen3NextGatedDeltaNet.__call__ì   s  € ð —,‘,‰ˆˆaØ×<Ñ<Ø×Ñ˜fÓ% t§¡°vÓ'>ó
Ñˆˆa�Að Ò  q¥Ò!5Ø˜q�‰JäŸšØ×)Ñ)¨AÕ-¨t¯}©}Ð=Ø—l‘lôˆJô
 —N’NØ�Y‰Y�q˜RÓ  !§)¡)¨A°"Ó"5°q·y±yÀÀrÓ7JÐKÐRTô
ˆ	ð ÒÜŸš  i¥°)¸QÓ?ˆIÜ—^’^ ZÐ$;À!ÔDˆ
àÒØ×*Ñ*¨QÕ.ˆFØ�}‰}Ò(Ü—w’w˜uŸ}™}¨a°Ó3�Ø! '�]¬R¯YªY°vÓ->Õ>À	ÕJ�	Ü×-Ò-¨j¸)È!ÔL��a’äŸ=š=¨°A¸°w±xÀ°NÕ)CÓD��a‘ä—7’7˜4Ÿ;™; zÓ2Ó3ˆô Ü—’˜ D§L¡L°!°d·l±lÕ2BÐ#CÀRÓHØ×!Ñ! 4×#3Ñ#3°T×5EÑ5EÐFØ—‘ $§/¡/°4·?±?ÐCôõ
ñ‘��1�að �I‰I�a˜A˜qÖ!ñó
‰ˆˆa÷ "��a– tˆØ—G‘G˜B•K 4Õ'ˆ	Ø˜�\œRŸW™W×-Ñ-¨a°°tÓ<Õ<ˆØœŸ™×(Ñ(¨¨D°$Ó7Õ7ˆä'ØØØØØØ�J‰JØ�L‰LØØØŸ=™=Ô(ô
‰
ˆˆUð ÒØˆE�!‰HØ�M‰M˜!Ôà�i‰i˜˜QÓˆØ�}‰}˜SŸ[™[¨¨rÓ2Ó3Ð3ùôA
s   Ê N7)rÐ   rÉ   rÇ   rÅ   rÌ   rÀ   rÁ   r   rË   rÊ   rÂ   rÆ   rÑ   r¿   r¾   rÒ   rÃ   r¡   )	r:   r;   r<   r=   rY   rã   rf   r?   rj   rk   s   @rA   r±   r±   ¬   s!   ø† ÷'Pð 'PõR
÷,E4õ E4r@   r±   c                  ó:   a € ] tR tRtR V 3R lltR R ltRtV ;t# )ÚQwen3NextSparseMoeBlocki4  c               ó   € V ^8„  d   QhRR/# ro   r9   )rT   s   "rA   rU   Ú$Qwen3NextSparseMoeBlock.__annotate__5  s   € ÷ #ñ #˜Yñ #r@   c                	óœ  <€ \         SV `  4        VP                  pVP                  pVP                  pVP
                  V n        VP                  ;V n        pVP                  V n        \        P                  ! W%R R7      V n        \        W#V4      V n        \        W$4      V n        \        P                  ! V^R R7      V n        RV n        R# r¥   )rX   rY   r   r*   r(   r4   r%   r&   Útop_krE   r{   rL   r   Ú
switch_mlpr£   Úshared_expertÚshared_expert_gateÚsharding_group)r\   rp   r©   r   r(   r%   r]   s   &&    €rA   rY   Ú Qwen3NextSparseMoeBlock.__init__5  s©   ø€ Ü‰ÑÔØ×ÑˆØ ×6Ñ6ÐØ*.×*NÑ*NÐ'à"×1Ñ1ˆÔØ)-×)9Ñ)9Ð9ˆÔ˜;Ø×-Ñ-ˆŒ
ä—I’I˜c°UÔ;ˆŒ	Ü# C¸KÓHˆŒä)¨#ÓOˆÔÜ"$§)¢)¨C°¸Ô"?ˆÔà"ˆÖr@   c               ó    € V ^8„  d   QhRRRR/# )rR   rM   ra   rb   r9   )rT   s   "rA   rU   r  G  s   € ÷ ñ àðð 
ñr@   c                	óî  € V P                   e   \        V P                   4      ! V4      pV P                  V4      p\        P                  ! VRRR7      pV P
                  p\        P                  ! W#) RR7      RV) R 13,          p\        P                  ! W$RR7      pV P                  '       d   WUP                  RRR7      ,          pV P                  W4      pWeR	,          ,          P                  R
R7      pV P                  V4      p\        P                  ! V P                  V4      4      V,          pWg,           pV P                   e+   \        P                  P                  W`P                   R7      pV# )NT)r�   Úprecise)Úkthr�   .rŽ   )r�   Úkeepdims)Úgroupr‘   rê   éþÿÿÿ)r  r   rL   rH   Úsoftmaxr
  Úargpartitionrñ   r4   Úsumr  r  r—   r  ÚdistributedÚall_sum)r\   rM   ÚgatesrÞ   ÚindsÚscoresÚyÚshared_ys   &&      rA   rf   Ú Qwen3NextSparseMoeBlock.__call__G  s+  € ð ×ÑÒ*Ü˜d×1Ñ1Ô2°1Ó5ˆAà—	‘	˜!“ˆÜ—
’
˜5 r°4Ô8ˆà�J‰JˆÜ�Š˜u¨"°2Ô6°s¸Q¸B¹C°xÕ@ˆÜ×#Ò# E°bÔ9ˆØ××ÐØŸj™j¨b¸4˜jÓ@Õ@ˆFà�O‰O˜AÓ$ˆØ˜	Õ"Õ"×'Ñ'¨RÐ'Ó0ˆà×%Ñ% aÓ(ˆÜ—:’:˜d×5Ñ5°aÓ8Ó9¸HÕDˆà�Lˆà×ÑÒ*Ü—‘×&Ñ& q×0CÑ0CÐ&ÓDˆAàˆr@   )rL   r4   r%   r  r  r  r  r
  ri   rk   s   @rA   r  r  4  s   ø† ÷#ð #÷$ó r@   r  c                  ó>   a € ] tR tRtR V 3R lltRR R lltRtV ;t# )ÚQwen3NextDecoderLayerie  c               ó    € V ^8„  d   QhRRRR/# )rR   rp   r   Ú	layer_idxr   r9   )rT   s   "rA   rU   Ú"Qwen3NextDecoderLayer.__annotate__f  s   € ÷ Nñ N˜Yð N°3ñ Nr@   c                	ó€  <€ \         SV `  4        V^,           VP                  ,          ^ 8g  V n        V P                  '       d   \	        V4      V n        M\        V4      V n        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  R7      V n        W!P                  9  dC   VP                  ^ 8”  d2   V^,           VP                   ,          ^ 8X  d   \#        V4      V n        R# \'        VP                  VP(                  4      V n        R# )r�   ru   N)rX   rY   r8   Ú	is_linearr±   Úlinear_attnrm   Ú	self_attnrE   r€   r   r,   Úinput_layernormÚpost_attention_layernormr)   r%   r'   r  Úmlpr£   r   )r\   rp   r%  r]   s   &&&€rA   rY   ÚQwen3NextDecoderLayer.__init__f  så   ø€ Ü‰ÑÔØ# a�-¨4×+GÑ+GÕGÈ1ÑLˆŒØ�>�>ˆ>Ü5°dÓ;ˆDÕä/°Ó5ˆDŒNä!Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
ª
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ð ×1Ñ1Ô1Ø×Ñ˜qÔ  i°!¥m°t×7OÑ7OÕ%OÐSTÔ%Tä.¨tÓ4ˆDŽHä# D×$4Ñ$4°d×6LÑ6LÓMˆDŽHr@   c               ó(   € V ^8„  d   QhRRRRRRRR/# r‡   r9   )rT   s   "rA   rU   r&  y  s2   € ÷ ñ àðð !ðð ð	ð
 
ñr@   c                	ó  € V P                   '       d#   V P                  V P                  V4      W#4      pM!V P                  V P                  V4      W#4      pW,           pWPP	                  V P                  V4      4      ,           pV# rD   )r(  r)  r+  r*  r-  r,  )r\   rM   rˆ   rŠ   ÚrrK   r  s   &&&&   rA   rf   ÚQwen3NextDecoderLayer.__call__y  sm   € ð �>�>ˆ>Ø× Ñ  ×!5Ñ!5°aÓ!8¸$ÓF‰Aà—‘˜t×3Ñ3°AÓ6¸ÓDˆAØ�EˆØ—(‘(˜4×8Ñ8¸Ó;Ó<Õ<ˆØˆ
r@   )r+  r(  r)  r-  r,  r*  r¡   ri   rk   s   @rA   r#  r#  e  s   ø† ÷Nð N÷&õ r@   r#  c                  ó>   a € ] tR tRtR V 3R lltRR R lltRtV ;t# )ÚQwen3NextModeliˆ  c               ó   € V ^8„  d   QhRR/# ro   r9   )rT   s   "rA   rU   ÚQwen3NextModel.__annotate__‰  s   € ÷ 	7ñ 	7˜Yñ 	7r@   c                	ó–  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  4       Uu. uF  p\        WR 7      NK  	  upV n
        \        P                  ! VP
                  VP                  R7      V n        ^ V n        VP                  ^,
          V n        R# u upi ))rp   r%  ru   N)rX   rY   rE   Ú	Embeddingr-   r   Úembed_tokensÚranger   r#  Úlayersr€   r,   rÑ   Ússm_idxr8   Úfa_idx)r\   rp   Úir]   s   && €rA   rY   ÚQwen3NextModel.__init__‰  sš   ø€ Ü‰ÑÔÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔô ˜4×1Ñ1Ô2ó
á2�ô " t×9Ù2ñ
ˆŒô —J’J˜t×/Ñ/°T×5FÑ5FÔGˆŒ	ØˆŒØ×2Ñ2°QÕ6ˆŽùò
s   ÁCc               ó$   € V ^8„  d   QhRRRRRR/# ©rR   ræ   ra   rŠ   r‹   rb   r9   )rT   s   "rA   rU   r6  ”  s(   € ÷ (ñ (àð(ð ð(ð 
ñ	(r@   c                	óv  € V P                  V4      pVf   R .\        V P                  4      ,          p\        W2V P                  ,          4      p\        W2V P                  ,          4      p\        V P                  V4       F%  w  rgVP                  '       d   TMTpV! W8VR7      pK'  	  V P                  V4      # )N)rˆ   rŠ   )
r9  Úlenr;  r   r=  r   r<  ró   r(  rÑ   )	r\   ræ   rŠ   r`   Úfa_maskÚssm_maskÚlayerÚcrˆ   s	   &&&      rA   rf   ÚQwen3NextModel.__call__”  s—   € ð
 ×)Ñ)¨&Ó1ˆàŠ=Ø�FœS §¡Ó-Õ-ˆEä'¨¸T¿[¹[Õ7IÓJˆÜ" =¸¿¹Õ2EÓFˆä˜DŸK™K¨Ö/‰HˆEØ$ŸŸ˜‘8°GˆDÙ! -À!ÔDŠMñ 0ð �y‰y˜Ó'Ð'r@   )r9  r=  r;  rÑ   r<  rD   ri   rk   s   @rA   r4  r4  ˆ  s   ø† ÷	7ð 	7÷(õ (r@   r4  c                  ój   a € ] tR tRtR V 3R lltRR R llt]R 4       tR tR t	]R	 4       t
R
tV ;t# )ÚModeli¨  c               ó   € V ^8„  d   QhRR/# ro   r9   )rT   s   "rA   rU   ÚModel.__annotate__©  s   € ÷ Tñ T˜Yñ Tr@   c                	óþ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        VP                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# R# r¥   )rX   rY   rp   r   r4  Úmodelr5   rE   r{   r   r-   Úlm_headr„   s   &&€rA   rY   ÚModel.__init__©  sY   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ# DÓ)ˆŒ
Ø×'×'Ð'ÜŸ9š9 T×%5Ñ%5°t·±ÈUÔSˆDŽLñ (r@   c               ó$   € V ^8„  d   QhRRRRRR/# rA  r9   )rT   s   "rA   rU   rL  ±  s(   € ÷ 
ñ 
àð
ð ð
ð 
ñ	
r@   c                	óÐ   € V P                  W4      pV P                  P                  '       d(   V P                   P                  P	                  V4      pV# V P                  V4      pV# rD   )rN  rp   r5   r9  Ú	as_linearrO  )r\   ræ   rŠ   r  s   &&& rA   rf   ÚModel.__call__±  sX   € ð
 �j‰j˜Ó'ˆØ�9‰9×(×(Ð(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r@   c                	ó.   € V P                   P                  # rD   )rN  r;  )r\   s   &rA   r;  ÚModel.layers½  s   € à�z‰z× Ñ Ð r@   c                	óŽ   € V P                    Uu. uF*  qP                  '       d   \        ^R7      M	\        4       NK,  	  up# u upi )rR   )Úsize)r;  r(  r   r   )r\   Úls   & rA   Ú
make_cacheÚModel.make_cacheÁ  s0   € ØKOÏ;Ê;ÓWÉ;Àa§{§{ {” Õ#¼»	ÒAÉ;ÑWÐWùÒWs   �0Ac                	óx  a€ R V9  d   V# VP                  4        UUu/ uF  w  r#RV9  g   K  W#bK  	  pppV P                  P                  '       d   VP                  RR4       \	        V P                  P
                  4       Fv  pRV R2pR Fg  p\	        V P                  P                  4       Uu. uF  pVP                  V RV RV R24      NK  	  pp\        P                  ! V4      W R	V R2&   Ki  	  Kx  	  Rp	VP                  4        F›  w  op
R
S9   d.   V
P                  R,          ^8w  d   V
P                  ^^4      VS&   \        ;QJ d    V3R lV	 4       F  '       g   K   RM	  RM! V3R lV	 4       4      '       g   K|  V
P                  ^8X  g   K�  V
R,           VS&   K�  	  V# u uppi u upi )z+model.layers.0.mlp.experts.0.up_proj.weightzmtp.zlm_head.weightNzmodel.layers.z.mlpz	.experts.Ú.z.weightz.switch_mlp.zconv1d.weightc              3  óF   <"  € T F  pSP                  V4      x € K  	  R # 5irD   ©Úendswith)Ú.0ÚsfxrÞ   s   & €rA   Ú	<genexpr>Ú!Model.sanitize.<locals>.<genexpr>ß  s   øé € Ð8©i s�1—:‘:˜c—?�?«iùs   ƒ!TFg      ð?)r¨   r§   r¦   )z.input_layernorm.weightz .post_attention_layernorm.weightzmodel.norm.weightz.q_norm.weightz.k_norm.weightr‘   )Úitemsrp   r5   Úpopr:  r   r%   rH   Ústackr’   ÚmoveaxisÚanyÚndim)r\   ÚweightsÚkeyÚvaluerY  ÚprefixÚnÚeÚto_joinÚ	norm_keysrß   rÞ   s   &&         @rA   ÚsanitizeÚModel.sanitizeÄ  s‰  ø€ Ø8ÀÔGØˆNØ07·±´ÔU±¡* #À6ÐQTÑCT”:�3’:±ˆÑUà�9‰9×(×(Ð(Ø�K‰KÐ(¨$Ô/ä�t—y‘y×2Ñ2Ö3ˆAØ$ Q C tÐ,ˆFÛ:�ô # 4§9¡9×#8Ñ#8Ô9óá9˜ð —K‘K 6 (¨)°A°3°a¸°s¸'Ð BÖCÙ9ð ð ô >@¿XºXÀgÓ=N�˜( ,¨q¨c°Ð9Ó:ó ;ñ 4ð
ˆ	ð —M‘M–O‰DˆAˆqØ !Ô#¨¯©°­°qÔ(8ØŸZ™Z¨¨1Ó-�˜‘
ß‹sÔ8©iÓ8�s�sŠsÔ8©iÓ8×8Ô8Ø—6‘6˜Q–;Ø!" S¥�G˜A“Jñ $ð ˆùó7 Vùòs   žF1®F1Â3#F7c                	ó   € R  pV# )c                ón   € V P                  R 4      '       g   V P                  R4      '       d   R^@R^/# R# )zmlp.gater  Ú
group_sizeÚbitsTr_  )Úpathr÷   s   &&rA   Ú	predicateÚ(Model.quant_predicate.<locals>.predicateæ  s2   € Ø�}‰}˜Z×(Ò(¨D¯M©MÐ:N×,OÒ,OØ$ b¨&°!Ð4Ð4Ùr@   r9   )r\   rz  s   & rA   Úquant_predicateÚModel.quant_predicateä  s   € ò	ð
 Ðr@   )rp   rO  rN  r   rD   )r:   r;   r<   r=   rY   rf   Úpropertyr;  rZ  rs  r|  r?   rj   rk   s   @rA   rJ  rJ  ¨  sH   ø† ÷Tð T÷
ð ñ!ó ð!òXòð@ ñó ör@   rJ  )0Ú
__future__r   Údataclassesr   Ú	functoolsr   Útypingr   r   r   r	   r
   r   Úmlx.coreÚcorerH   Úmlx.nnrE   Úmlx.nn.layers.distributedr   Úactivationsr   rv   r   r   r   r   rŠ   r   r   Úgated_deltar   Ú
rope_utilsr   Úswitch_layersr   r   ÚcompilerN   ÚModulerP   rm   r£   r±   r  r#  r4  rJ  r9   r@   rA   Ú<module>r�     s  ðõ #å !Ý ß :× :å Ý Ý 3å ÷ó ÷ (Ý +Ý 'Ý $ð ô%�ó %ó ð%ñ> 	ˆ�‰˜tÔ$ñ&ó %ð&ô1˜BŸI™Iô 1ô M6˜Ÿ™ô M6ô`J�2—9‘9ô JôE4˜RŸY™Yô E4ôP.˜bŸi™iô .ôb ˜BŸI™Iô  ôF(�R—Y‘Yô (ô@CˆB�I‰Iö Cr@   