Ë
    îÿæi±?  ã                  óh  — d dl mZ d dlmZ d dlmZ d dlmZmZm	Z	m
Z
mZmZ d dlmZ d dlmZ d dlmZ ddlmZ dd	lmZmZmZmZ dd
lmZmZ ddlmZ ddl m!Z! ddl"m#Z# e G d„ de«      «       Z$ eejJ                  d¬«      d„ «       Z& G d„ dejN                  «      Z( G d„ dejN                  «      Z) G d„ dejN                  «      Z* G d„ dejN                  «      Z+ G d„ dejN                  «      Z, G d„ dejN                  «      Z- G d„ d ejN                  «      Z. G d!„ d"ejN                  «      Z/y)#é    )Úannotations)Ú	dataclass)Úpartial)ÚAnyÚDictÚListÚOptionalÚTupleÚUnionN)Úsum_gradientsé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚcreate_ssm_maskÚscaled_dot_product_attention)ÚArraysCacheÚKVCache)Úgated_delta_update)Úinitialize_rope)Ú	SwitchGLUc                  ó:  — e Zd ZU ded<   ded<   ded<   ded<   ded<   ded<   ded	<   ded
<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   dZded<   dZded<   dZded <   d!Zd"ed#<   d$Zded%<   y!)&Ú	ModelArgsÚstrÚ
model_typeÚintÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚlinear_num_value_headsÚlinear_num_key_headsÚlinear_key_head_dimÚlinear_value_head_dimÚlinear_conv_kernel_dimÚnum_expertsÚnum_experts_per_tokÚdecoder_sparse_stepÚshared_expert_intermediate_sizez	List[int]Úmlp_only_layersÚmoe_intermediate_sizeÚfloatÚrms_norm_epsÚ
vocab_sizeÚnum_key_value_headsÚ
rope_thetaÚpartial_rotary_factorÚmax_position_embeddingsÚhead_dimFÚboolÚnorm_topk_probÚtie_word_embeddingsÚattention_biasNz&Optional[Dict[str, Union[float, str]]]Úrope_scalingé   Úfull_attention_interval)	Ú__name__Ú
__module__Ú__qualname__Ú__annotations__r5   r6   r7   r8   r:   © ó    úm/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/qwen3_next.pyr   r      s¾   … àƒOØÓØÓØÓØÓØÓØÓØÓØÓØÓØÓØÓØÓØ%(Ó(ØÓØÓØÓØƒOØÓØÓØ Ó Ø Ó ØƒMØ €N�DÓ Ø %Ð˜Ó%Ø €N�DÓ Ø;?€LÐ8Ó?Ø#$Ð˜SÔ$r@   r   T)Ú	shapelessc                óà   — t        j                  |j                  t        j                  «      «      }|j                  t        j                  «      }||z  j                  | j
                  «      S ©N)ÚnnÚsiluÚastypeÚmxÚfloat32Údtype)ÚhÚgateÚxs      rA   Ú_precise_swiglurN   :   sH   € ä�7‰7�4—;‘;œrŸz™zÓ*Ó+€DØ	�‰”—‘Ó€AØ�1‰H×Ñ˜QŸW™WÓ%Ð%r@   c                  ó8   ‡ — e Zd Zddˆ fd„Z	 d	 	 	 	 	 dd„Zˆ xZS )ÚQwen3NextRMSNormGatedc                ód   •— t         ‰| �  «        || _        t        j                  |«      | _        y rD   )ÚsuperÚ__init__ÚepsrH   ÚonesÚweight)Úselfr   rT   Ú	__class__s      €rA   rS   zQwen3NextRMSNormGated.__init__B   s%   ø€ Ü‰ÑÔØˆŒÜ—g‘g˜kÓ*ˆ�r@   c                óÀ   — t         j                  j                  || j                  | j                  «      }|�t        |||«      S |j                  |j                  «      S rD   )rH   ÚfastÚrms_normrV   rT   rN   rG   rJ   )rW   Úhidden_statesrL   rM   s       rA   Ú__call__zQwen3NextRMSNormGated.__call__G   sN   € ô �G‰G×Ñ˜]¨D¯K©K¸¿¹ÓBˆØÐÜ" =°$¸Ó:Ð:à—8‘8˜M×/Ñ/Ó0Ð0r@   )ç�íµ ÷Æ°>)r   r   rT   r,   rD   )r\   úmx.arrayrL   zmx.array | NoneÚreturnr_   ©r;   r<   r=   rS   r]   Ú__classcell__©rX   s   @rA   rP   rP   A   s,   ø„ ö+ð @Dð1Ø%ð1Ø-<ð1à	÷1r@   rP   c                  ó<   ‡ — e Zd Zdˆ fd„Z	 	 d	 	 	 	 	 	 	 dd„Zˆ xZS )ÚQwen3NextAttentionc                ó\  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        | j                  dz  | _        t        j                  |j                  | j                  | j                  z  dz  |j                  ¬«      | _
        t        j                  |j                  | j                  | j                  z  |j                  ¬«      | _        t        j                  |j                  | j                  | j                  z  |j                  ¬«      | _        t        j                  | j                  | j                  z  |j                  |j                  ¬«      | _        t        j                  | j                  |j                  ¬«      | _        t        j                  | j                  |j                  ¬«      | _        t%        t'        | j                  |j(                  z  «      |j*                  d|j,                  |j.                  ¬«      | _        y )Nç      à¿é   ©Úbias©rT   F)ÚbaseÚtraditionalÚscaling_configr2   )rR   rS   r/   r    r3   ÚscalerE   ÚLinearr   r7   Úq_projÚk_projÚv_projÚo_projÚRMSNormr-   Úq_normÚk_normr   r   r1   r0   r8   r2   Úrope©rW   ÚargsrX   s     €rA   rS   zQwen3NextAttention.__init__R   s™  ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø#'×#;Ñ#;ˆÔ ØŸ™ˆŒØ—]‘] DÑ(ˆŒ
ä—i‘iØ×ÑØ×$Ñ$ t§}¡}Ñ4°qÑ8Ø×$Ñ$ô
ˆŒô
 —i‘iØ×ÑØ×$Ñ$ t§}¡}Ñ4Ø×$Ñ$ô
ˆŒô
 —i‘iØ×ÑØ×$Ñ$ t§}¡}Ñ4Ø×$Ñ$ô
ˆŒô
 —i‘iØ×$Ñ$ t§}¡}Ñ4Ø×ÑØ×$Ñ$ô
ˆŒô —j‘j §¡°D×4EÑ4EÔFˆŒÜ—j‘j §¡°D×4EÑ4EÔFˆŒä#Ü�—‘ × :Ñ :Ñ:Ó;Ø—‘ØØ×,Ñ,Ø$(×$@Ñ$@ô
ˆ�	r@   c                óö  — |j                   \  }}}| j                  |«      }t        j                  |j	                  ||| j
                  d«      dd¬«      \  }}	|	j	                  ||d«      }	| j                  |«      | j                  |«      }}
| j                  |«      j                  dddd«      }| j                  |
j	                  ||| j                  d«      «      j                  dddd«      }
|j	                  ||| j                  d«      j                  dddd«      }|�P| j                  ||j                  ¬«      }| j                  |
|j                  ¬«      }
|j                  |
|«      \  }
}n"| j                  |«      }| j                  |
«      }
t        ||
||| j                   |¬«      }|j                  dddd«      j	                  ||d«      }| j#                  |t        j$                  |	«      z  «      S )	Néÿÿÿÿrh   ©Úaxisr   r   é   )Úoffset)Úcachero   Úmask)Úshaperq   rH   ÚsplitÚreshaper    rr   rs   rv   Ú	transposerw   r/   rx   r€   Úupdate_and_fetchr   ro   rt   Úsigmoid)rW   rM   r‚   r�   ÚBÚLÚDÚq_proj_outputÚqueriesrL   ÚkeysÚvaluesÚoutputs                rA   r]   zQwen3NextAttention.__call__y   sÇ  € ð —'‘'‰ˆˆ1ˆaàŸ™ A›ˆÜŸ™Ø×!Ñ! ! Q¨×(@Ñ(@À"ÓEÀqÈrô
‰ˆ�ð �|‰|˜A˜q "Ó%ˆà—{‘{ 1“~ t§{¡{°1£~ˆfˆà—+‘+˜gÓ&×0Ñ0°°A°q¸!Ó<ˆØ�{‰{˜4Ÿ<™<¨¨1¨d×.FÑ.FÈÓKÓL×VÑVØˆq�!�Qó
ˆð —‘  1 d×&>Ñ&>ÀÓC×MÑMØˆq�!�Qó
ˆð ÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆà�{‰{˜6¤B§J¡J¨tÓ$4Ñ4Ó5Ð5r@   ©rz   r   ©NN©rM   r_   r‚   úOptional[mx.array]r�   úOptional[Any]r`   r_   ra   rc   s   @rA   re   re   Q   s<   ø„ õ%
ðT $(Ø#ð	%6àð%6ð !ð%6ð ð	%6ð
 
÷%6r@   re   c                  ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚQwen3NextMLPc                óÐ   •— t         ‰| �  «        t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        y ©NFri   )rR   rS   rE   rp   Ú	gate_projÚ	down_projÚup_proj)rW   ÚdimÚ
hidden_dimrX   s      €rA   rS   zQwen3NextMLP.__init__¢   sJ   ø€ Ü‰ÑÔÜŸ™ 3¨
¸Ô?ˆŒÜŸ™ :¨s¸Ô?ˆŒÜ—y‘y  j°uÔ=ˆ�r@   c                ót   — | j                  t        | j                  |«      | j                  |«      «      «      S rD   )r›   r   rš   rœ   )rW   rM   s     rA   r]   zQwen3NextMLP.__call__¨   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr@   )r`   r_   ra   rc   s   @rA   r—   r—   ¡   s   ø„ ô>÷Jr@   r—   c                  óP   ‡ — e Zd Zdˆ fd„Z	 	 	 	 	 	 dd„Z	 	 d	 	 	 	 	 	 	 dd„Zˆ xZS )ÚQwen3NextGatedDeltaNetc                óR  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _
        | j                  | j                  z  | _        | j                  | j                  z  | _        | j                  | j                  z  dk7  r&t        d| j                  › d| j                  › d�«      ‚|j                  | _        |j                   | _        | j                  dz  | j                  z   | _        t'        j(                  | j$                  | j$                  d| j                  | j$                  d¬«      | _        t'        j,                  | j                  | j                  dz  | j                  dz  z   d¬«      | _        t'        j,                  | j                  | j                  dz  d¬«      | _        t3        j4                  | j                  «      | _        t2        j8                  j;                  dd	| j                  f¬
«      }t3        j<                  |«      | _        tA        | j                  | j"                  ¬«      | _!        t'        j,                  | j                  | j                  d¬«      | _"        y )Nr   znum_v_heads (z$) must be divisible by num_k_heads (Ú)rh   F)Úin_channelsÚout_channelsrj   Úkernel_sizeÚgroupsÚpaddingri   é   )ÚlowÚhighrƒ   rk   )#rR   rS   r   r!   Únum_v_headsr"   Únum_k_headsr#   Ú
head_k_dimr$   Ú
head_v_dimÚkey_dimÚ	value_dimÚ
ValueErrorr%   Úconv_kernel_sizer-   Úlayer_norm_epsilonÚconv_dimrE   ÚConv1dÚconv1drp   Úin_proj_qkvzÚ
in_proj_barH   rU   Údt_biasÚrandomÚuniformÚlogÚA_logrP   ÚnormÚout_proj)rW   ÚconfigÚArX   s      €rA   rS   zQwen3NextGatedDeltaNet.__init__­   s  ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ!×8Ñ8ˆÔØ!×6Ñ6ˆÔØ ×4Ñ4ˆŒØ ×6Ñ6ˆŒØ—‘¨×)9Ñ)9Ñ9ˆŒØŸ™¨4×+;Ñ+;Ñ;ˆŒØ×Ñ˜d×.Ñ.Ñ.°!Ò3ÜØ × 0Ñ 0Ð1Ð1UÐVZ×VfÑVfÐUgÐghÐióð ð !'× =Ñ =ˆÔØ"(×"5Ñ"5ˆÔàŸ™ qÑ(¨4¯>©>Ñ9ˆŒÜ—i‘iØŸ™ØŸ™ØØ×-Ñ-Ø—=‘=Øô
ˆŒô ŸI™IØ×Ñ˜dŸl™l¨QÑ.°·±À!Ñ1CÑCÈ%ô
ˆÔô Ÿ)™) D×$4Ñ$4°d×6FÑ6FÈÑ6JÐQVÔWˆŒä—w‘w˜t×/Ñ/Ó0ˆŒä�I‰I×Ñ !¨"°T×5EÑ5EÐ4GÐÓHˆÜ—V‘V˜A“YˆŒ
ä)¨$¯/©/¸t×?VÑ?VÔWˆŒ	äŸ	™	 $§.¡.°$×2BÑ2BÈÔOˆ�r@   c                ó   — | j                   | j                  | j                  | j                  f\  }}}} |j                  g |j
                  d d ¢|‘d‘­Ž } |j                  g |j
                  d d ¢|‘d‘­Ž }t        j                  ||d|z  d|z  ||z  |z  z   gd¬«      \  }}}	}
t        j                  |||z  gd¬«      \  }}|| |	j                  g |	j
                  d d ¢d‘|‘­Ž  |
j                  g |
j
                  d d ¢d‘|‘­Ž  |j                  g |j
                  d d ¢|‘­Ž  |j                  g |j
                  d d ¢|‘­Ž fS )Nr|   rh   r}   )r­   r®   r¬   r¯   r…   rƒ   rH   r„   )rW   Ú
mixed_qkvzÚmixed_baÚnkÚdnÚnvÚdvÚqÚkÚvÚzÚbÚas                rA   Úfix_query_key_value_orderingz3Qwen3NextGatedDeltaNet.fix_query_key_value_orderingÖ   sx  € ð ×ÑØ�O‰OØ×ÑØ�O‰Oð	
‰ˆˆB��Bð (�Z×'Ñ'ÐG¨×)9Ñ)9¸#¸2Ð)>ÐGÀÐGÀBÒGˆ
Ø#�8×#Ñ#ÐA X§^¡^°C°RÐ%8ÐA¸"ÐA¸bÒAˆÜ—X‘X˜j¨2¨q°2©v°q¸2±vÀÀbÁÈ2ÁÑ7MÐ*NÐUWÔX‰
ˆˆ1ˆa�Ü�x‰x˜ 2¨¡8 *°2Ô6‰ˆˆ1àØØˆA�I‰IÐ+�q—w‘w˜r �{Ð+ BÐ+¨Ò+ØˆA�I‰IÐ+�q—w‘w˜r �{Ð+ BÐ+¨Ò+ØˆA�I‰IÐ'�q—w‘w˜r �{Ð' BÒ'ØˆA�I‰IÐ'�q—w‘w˜r �{Ð' BÒ'ð
ð 	
r@   c                óº  — |j                   \  }}}| j                  | j                  |«      | j                  |«      «      \  }}}	}
}}|�|d   �|d   }n;t	        j
                  || j                  dz
  | j                  f|j                  ¬«      }t	        j                  |j                  ||d«      |j                  ||d«      |	j                  ||d«      gd¬«      }|�t	        j                  |d   |d«      }t	        j                  ||gd¬«      }|�Œ| j                  dz
  }|j                  �_t	        j                  |j                  d|«      }|d d …d f   t	        j                  |«      z   d   }t	        j                  ||d¬«      |d<   n|d d …| d …d d …f   |d<   t!        j"                  | j%                  |«      «      }t'        t	        j(                  || j*                  d| j*                  z  gd«      | j,                  | j,                  | j.                  g| j0                  | j0                  | j2                  g«      D ���cg c]  \  }}}|j                  ||||«      ‘Œ c}}}\  }}}	|r|d   nd }|j                   d   dz  }|dz  t        j4                  j7                  |d d	«      z  }|t        j4                  j7                  |d d	«      z  }t9        |||	||| j:                  | j<                  ||| j>                   ¬
«
      \  }}|�||d<   |jA                  |«       | jC                  ||
«      }| jE                  |j                  ||d«      «      S c c}}}w )Nr   r   )rJ   r|   r}   ©.Nrh   rg   r^   )Ú
use_kernel)#rƒ   rÐ   r¸   r¹   rH   Úzerosr³   rµ   rJ   Úconcatenater…   ÚwhereÚlengthsÚclipÚarangeÚtake_along_axisrE   rF   r·   Úzipr„   r°   r­   r¬   r®   r¯   rZ   r[   r   r¾   rº   ÚtrainingÚadvancer¿   rÀ   )rW   Úinputsr‚   r�   r‰   ÚSÚ_rÊ   rË   rÌ   rÍ   rÎ   rÏ   Ú
conv_stateÚ	mixed_qkvÚ
conv_inputÚn_keepÚendsÚ	positionsÚconv_outÚtrK   ÚdÚstateÚ	inv_scaleÚouts                             rA   r]   zQwen3NextGatedDeltaNet.__call__ì   s%  € ð —,‘,‰ˆˆ1ˆaØ×<Ñ<Ø×Ñ˜fÓ% t§¡°vÓ'>ó
Ñˆˆ1ˆa��A�qð Ð  q¡Ð!5Ø˜q™‰JäŸ™Ø�D×)Ñ)¨AÑ-¨t¯}©}Ð=Ø—l‘lôˆJô
 —N‘NØ�Y‰Y�q˜!˜RÓ  !§)¡)¨A¨q°"Ó"5°q·y±yÀÀAÀrÓ7JÐKÐRTô
ˆ	ð ÐÜŸ™  i¡°)¸QÓ?ˆIÜ—^‘^ Z°Ð$;À!ÔDˆ
àÐØ×*Ñ*¨QÑ.ˆFØ�}‰}Ð(Ü—w‘w˜uŸ}™}¨a°Ó3�Ø!¢! T '™]¬R¯Y©Y°vÓ->Ñ>À	ÑJ�	Ü×-Ñ-¨j¸)È!ÔL��a’à%¢a¨&¨©²1 nÑ5��a‘ä—7‘7˜4Ÿ;™; zÓ2Ó3ˆô Ü—‘˜ D§L¡L°!°d·l±lÑ2BÐ#CÀRÓHØ×!Ñ! 4×#3Ñ#3°T×5EÑ5EÐFØ—‘ $§/¡/°4·?±?ÐCôõ
ñ‘��1�að �I‰I�a˜˜A˜qÕ!ðó
‰ˆˆ1ˆañ "��a’ tˆØ—G‘G˜B‘K 4Ñ'ˆ	Ø˜‰\œRŸW™W×-Ñ-¨a°°tÓ<Ñ<ˆØœŸ™×(Ñ(¨¨D°$Ó7Ñ7ˆä'ØØØØØØ�J‰JØ�L‰LØØØŸ=™=Ð(ô
‰
ˆˆUð ÐØˆE�!‰HØ�M‰M˜!Ôà�i‰i˜˜QÓˆØ�}‰}˜SŸ[™[¨¨A¨rÓ2Ó3Ð3ùôA
s   ÉM)rÁ   r   )rÄ   r_   rÅ   r_   r`   r_   r’   )rÞ   r_   r‚   r”   r�   r•   r`   r_   )r;   r<   r=   rS   rÐ   r]   rb   rc   s   @rA   r¡   r¡   ¬   s_   ø„ õ'PðR
Ø"ð
Ø.6ð
à	ó
ð2 $(Ø#ð	E4àðE4ð !ðE4ð ð	E4ð
 
÷E4r@   r¡   c                  ó0   ‡ — e Zd Zdˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚQwen3NextSparseMoeBlockc                óœ  •— t         ‰| �  «        |j                  }|j                  }|j                  }|j
                  | _        |j                  x| _        }|j                  | _        t        j                  ||d¬«      | _        t        |||«      | _        t        ||«      | _        t        j                  |dd¬«      | _        d | _        y )NFri   r   )rR   rS   r   r+   r)   r5   r&   r'   Útop_krE   rp   rL   r   Ú
switch_mlpr—   Úshared_expertÚshared_expert_gateÚsharding_group)rW   rz   r�   r   r)   r&   rX   s         €rA   rS   z Qwen3NextSparseMoeBlock.__init__5  s±   ø€ Ü‰ÑÔØ×ÑˆØ ×6Ñ6ÐØ*.×*NÑ*NÐ'à"×1Ñ1ˆÔØ)-×)9Ñ)9Ð9ˆÔ˜;Ø×-Ñ-ˆŒ
ä—I‘I˜c ;°UÔ;ˆŒ	Ü# CÐ):¸KÓHˆŒä)¨#Ð/NÓOˆÔÜ"$§)¡)¨C°¸Ô"?ˆÔà"ˆÕr@   c                ó°  — | j                   � t        | j                   «      |«      }| j                  |«      }t        j                  |dd¬«      }| j
                  }t        j                  || d¬«      d| d …f   }t        j                  ||d¬«      }| j                  r||j                  dd¬«      z  }| j                  ||«      }||d   z  j                  d	¬«      }| j                  |«      }t        j                  | j                  |«      «      |z  }||z   }| j                   �+t        j                  j                  || j                   ¬
«      }|S )Nr|   T)r~   Úprecise)Úkthr~   .r}   )r~   ÚkeepdimsrÒ   éþÿÿÿ)Úgroup)rô   r   rL   rH   Úsoftmaxrð   ÚargpartitionrÚ   r5   Úsumrñ   rò   rˆ   ró   ÚdistributedÚall_sum)rW   rM   ÚgatesrË   ÚindsÚscoresÚyÚshared_ys           rA   r]   z Qwen3NextSparseMoeBlock.__call__G  s8  € ð ×ÑÐ*Ø2”˜d×1Ñ1Ó2°1Ó5ˆAà—	‘	˜!“ˆÜ—
‘
˜5 r°4Ô8ˆà�J‰JˆÜ�‰˜u¨1¨"°2Ô6°s¸Q¸B¹C°xÑ@ˆÜ×#Ñ# E¨4°bÔ9ˆØ×ÒØ˜fŸj™j¨b¸4˜jÓ@Ñ@ˆFà�O‰O˜A˜tÓ$ˆØ�˜	Ñ"Ñ"×'Ñ'¨RÐ'Ó0ˆà×%Ñ% aÓ(ˆÜ—:‘:˜d×5Ñ5°aÓ8Ó9¸HÑDˆà�‰Lˆà×ÑÐ*Ü—‘×&Ñ& q°×0CÑ0CÐ&ÓDˆAàˆr@   r‘   )rM   r_   r`   r_   ra   rc   s   @rA   rî   rî   4  s   ø„ õ#ð$àðð 
÷r@   rî   c                  ó<   ‡ — e Zd Zdˆ fd„Z	 	 d	 	 	 	 	 	 	 dd„Zˆ xZS )ÚQwen3NextDecoderLayerc                ó@  •— t         ‰| �  «        |dz   |j                  z  dk7  | _        | j                  rt	        |«      | _        nt        |«      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _        ||j                  vr5|j                  dkD  r&|dz   |j                   z  dk(  rt#        |«      | _        y t'        |j                  |j(                  «      | _        y )Nr   r   rk   )rR   rS   r:   Ú	is_linearr¡   Úlinear_attnre   Ú	self_attnrE   ru   r   r-   Úinput_layernormÚpost_attention_layernormr*   r&   r(   rî   Úmlpr—   r   )rW   rz   Ú	layer_idxrX   s      €rA   rS   zQwen3NextDecoderLayer.__init__f  så   ø€ Ü‰ÑÔØ# a™-¨4×+GÑ+GÑGÈ1ÑLˆŒØ�>Š>Ü5°dÓ;ˆDÕä/°Ó5ˆDŒNä!Ÿz™z¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
©
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ð ˜T×1Ñ1Ñ1Ø×Ñ˜qÒ  i°!¡m°t×7OÑ7OÑ%OÐSTÒ%Tä.¨tÓ4ˆD�Hä# D×$4Ñ$4°d×6LÑ6LÓMˆD�Hr@   c                óø   — | j                   r#| j                  | j                  |«      ||«      }n"| j                  | j                  |«      ||«      }||z   }|| j	                  | j                  |«      «      z   }|S rD   )r  r	  r  r
  r  r  )rW   rM   r‚   r�   ÚrrK   rì   s          rA   r]   zQwen3NextDecoderLayer.__call__y  ss   € ð �>Š>Ø× Ñ  ×!5Ñ!5°aÓ!8¸$ÀÓF‰Aà—‘˜t×3Ñ3°AÓ6¸¸eÓDˆAØ�‰EˆØ�$—(‘(˜4×8Ñ8¸Ó;Ó<Ñ<ˆØˆ
r@   )rz   r   r  r   r’   r“   ra   rc   s   @rA   r  r  e  s<   ø„ õNð, $(Ø#ð	àðð !ðð ð	ð
 
÷r@   r  c                  ó6   ‡ — e Zd Zdˆ fd„Z	 d	 	 	 	 	 dd„Zˆ xZS )ÚQwen3NextModelc                ó†  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}| _
        t        j                  |j
                  |j                  ¬«      | _        d| _        |j                  dz
  | _        y c c}w )N)rz   r  rk   r   r   )rR   rS   rE   Ú	Embeddingr.   r   Úembed_tokensÚranger   r  Úlayersru   r-   r¿   Ússm_idxr:   Úfa_idx)rW   rz   ÚirX   s      €rA   rS   zQwen3NextModel.__init__‰  sœ   ø€ Ü‰ÑÔÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔô ˜4×1Ñ1Ô2ó
á2�ô " t°qÖ9Ø2ñ
ˆŒô —J‘J˜t×/Ñ/°T×5FÑ5FÔGˆŒ	ØˆŒØ×2Ñ2°QÑ6ˆ�ùò
s   ÁB>c                óR  — | j                  |«      }|€d gt        | j                  «      z  }t        ||| j                     «      }t        ||| j                     «      }t        | j                  |«      D ]   \  }}|j                  r|n|} ||||¬«      }Œ" | j                  |«      S )N)r‚   r�   )
r  Úlenr  r   r  r   r  rÛ   r  r¿   )	rW   rÞ   r�   r\   Úfa_maskÚssm_maskÚlayerÚcr‚   s	            rA   r]   zQwen3NextModel.__call__”  s�   € ð
 ×)Ñ)¨&Ó1ˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEä'¨°u¸T¿[¹[Ñ7IÓJˆÜ" =°%¸¿¹Ñ2EÓFˆä˜DŸK™K¨Ö/‰HˆE�1Ø$Ÿš‘8°GˆDÙ! -°dÀ!ÔD‰Mð 0ð �y‰y˜Ó'Ð'r@   r‘   rD   ©rÞ   r_   r�   r•   r`   r_   ra   rc   s   @rA   r  r  ˆ  s.   ø„ õ	7ð  $ð(àð(ð ð(ð 
÷	(r@   r  c                  ób   ‡ — e Zd Zdˆ fd„Z	 d	 	 	 	 	 d	d„Zed„ «       Zd„ Zd„ Zed„ «       Z	ˆ xZ
S )
ÚModelc                óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y r™   )rR   rS   rz   r   r  Úmodelr6   rE   rp   r   r.   Úlm_headry   s     €rA   rS   zModel.__init__©  sX   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ# DÓ)ˆŒ
Ø×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (r@   c                óÆ   — | j                  ||«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S rD   )r%  rz   r6   r  Ú	as_linearr&  )rW   rÞ   r�   rì   s       rA   r]   zModel.__call__±  sW   € ð
 �j‰j˜ Ó'ˆØ�9‰9×(Ò(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r@   c                ó.   — | j                   j                  S rD   )r%  r  )rW   s    rA   r  zModel.layers½  s   € à�z‰z× Ñ Ð r@   c                ó€   — | j                   D �cg c]$  }|j                  rt        d¬«      n	t        «       ‘Œ& c}S c c}w )Nrh   )Úsize)r  r  r   r   )rW   Úls     rA   Ú
make_cachezModel.make_cacheÁ  s0   € ØKOÏ;Ê;ÓWÉ;Àa q§{¢{” Õ#¼»	ÑAÈ;ÑWÐWùÒWs   �);c                óÔ  ‡— d|vr|S |j                  «       D ��ci c]  \  }}d|vsŒ||“Œ }}}| j                  j                  r|j                  dd «       t	        | j                  j
                  «      D ]q  }d|› d�}dD ]d  }t	        | j                  j                  «      D �cg c]  }|j                  |› d|› d|› d	�«      ‘Œ }}t        j                  |«      ||› d
|› d	�<   Œf Œs d}	|j                  «       D ]]  \  Š}
d‰v r'|
j                  d   dk7  r|
j                  dd«      |‰<   t        ˆfd„|	D «       «      sŒF|
j                  dk(  sŒV|
dz   |‰<   Œ_ |S c c}}w c c}w )Nz+model.layers.0.mlp.experts.0.up_proj.weightzmtp.zlm_head.weightzmodel.layers.z.mlp)rœ   r›   rš   z	.experts.Ú.z.weightz.switch_mlp.)z.input_layernorm.weightz .post_attention_layernorm.weightzmodel.norm.weightz.q_norm.weightz.k_norm.weightzconv1d.weightr|   r   rh   c              3  ó@   •K  — | ]  }‰j                  |«      –— Œ y ­wrD   ©Úendswith)Ú.0ÚsfxrË   s     €rA   Ú	<genexpr>z!Model.sanitize.<locals>.<genexpr>ß  s   øè ø€ Ð8©i s�1—:‘:˜c—?©iùs   ƒg      ð?)Úitemsrz   r6   Úpopr  r   r&   rH   Ústackrƒ   ÚmoveaxisÚanyÚndim)rW   ÚweightsÚkeyÚvaluer,  ÚprefixÚnÚeÚto_joinÚ	norm_keysrÌ   rË   s              @rA   ÚsanitizezModel.sanitizeÄ  sw  ø€ Ø8ÀÑGØˆNØ07·±´ÔU±¡* # uÀ6ÐQTÒCT�3˜‘:°ˆÑUà�9‰9×(Ò(Ø�K‰KÐ(¨$Ô/ä�t—y‘y×2Ñ2Ö3ˆAØ$ Q C tÐ,ˆFÛ:�ô # 4§9¡9×#8Ñ#8Ô9óá9˜ð —K‘K 6 (¨)°A°3°a¸°s¸'Ð BÕCØ9ð ð ô >@¿X¹XÀgÓ=N�˜6˜( ,¨q¨c°Ð9Ò:ñ ;ð 4ð
ˆ	ð —M‘M–O‰DˆAˆqØ !Ñ#¨¯©°©°qÒ(8ØŸZ™Z¨¨1Ó-�˜‘
ÜÓ8©iÓ8Õ8Ø—6‘6˜Q“;Ø!" S¡�G˜A’Jð $ð ˆùó7 Vùòs   ›E¨EÂ&!E%c                ó   — d„ }|S )Nc                óR   — | j                  d«      s| j                  d«      rdddœS y)Nzmlp.gateró   é@   é   )Ú
group_sizeÚbitsTr1  )Úpathrà   s     rA   Ú	predicatez(Model.quant_predicate.<locals>.predicateæ  s(   € Ø�}‰}˜ZÔ(¨D¯M©MÐ:NÔ,OØ&(°!Ñ4Ð4Ør@   r?   )rW   rL  s     rA   Úquant_predicatezModel.quant_predicateä  s   € ò	ð
 Ðr@   r‘   rD   r!  )r;   r<   r=   rS   r]   Úpropertyr  r-  rD  rM  rb   rc   s   @rA   r#  r#  ¨  sc   ø„ õTð  $ð
àð
ð ð
ð 
ó	
ð ñ!ó ð!òXòð@ ñó ôr@   r#  )0Ú
__future__r   Údataclassesr   Ú	functoolsr   Útypingr   r   r   r	   r
   r   Úmlx.coreÚcorerH   Úmlx.nnrE   Úmlx.nn.layers.distributedr   Úactivationsr   rl   r   r   r   r   r�   r   r   Úgated_deltar   Ú
rope_utilsr   Úswitch_layersr   r   ÚcompilerN   ÚModulerP   re   r—   r¡   rî   r  r  r#  r?   r@   rA   Ú<module>r]     s  ðõ #å !Ý ß :× :å Ý Ý 3å ÷ó ÷ (Ý +Ý 'Ý $ð ô%�ó %ó ð%ñ> 	ˆ�‰˜tÔ$ñ&ó %ð&ô1˜BŸI™Iô 1ô M6˜Ÿ™ô M6ô`J�2—9‘9ô JôE4˜RŸY™Yô E4ôP.˜bŸi™iô .ôb ˜BŸI™Iô  ôF(�R—Y‘Yô (ô@CˆB�I‰Iõ Cr@   