Ë
    îÿæi—   ã                   óŒ  — d dl mZ d dlmZmZmZmZmZ d dlm	Z
 d dlmZ ddlmZ ddlmZmZmZ ddlmZ e G d„ d	e«      «       Z G d
„ dej,                  «      Z G d„ dej,                  «      Z G d„ dej,                  «      Z G d„ dej,                  «      Z G d„ dej,                  «      Z G d„ dej,                  «      Zy)é    )Ú	dataclass)ÚAnyÚDictÚListÚOptionalÚUnionNé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Ú	SwitchGLUc                   óð   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   ee   ed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZ	e
eeeeef   f      ed<   y)Ú	ModelArgsÚ
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚnum_expertsÚnum_experts_per_tokÚdecoder_sparse_stepÚmlp_only_layersÚmoe_intermediate_sizeÚrms_norm_epsÚ
vocab_sizeÚnum_key_value_headsÚhead_dimÚ
rope_thetaÚtie_word_embeddingsÚmax_position_embeddingsÚnorm_topk_probNÚrope_scaling)Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__Úintr   ÚfloatÚboolr#   r   r   r   © ó    úl/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/qwen3_moe.pyr   r      s‘   … àƒOØÓØÓØÓØÓØÓØÓØÓØ˜#‘YÓØÓØÓØƒOØÓØƒMØÓØÓØ Ó ØÓØ;?€L�(˜4  U¨5°#¨:Ñ%6Ð 6Ñ7Ñ8Ô?r-   r   c            	       óŽ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
Ú	AttentionÚargsÚ	layer_idxc                 óä  •— t         ‰| �  «        |j                  }|j                  x| _        }|j
                  €J ‚|j
                  x| _        }t        |d|j                  |j                  z  «      }|dz  | _        t        j                  |||z  d¬«      | _        t        j                  |||z  d¬«      | _        t        j                  |||z  d¬«      | _        t        j                  ||z  |d¬«      | _        t        j                  ||j                   ¬«      | _        t        j                  ||j                   ¬«      | _        t        j&                  |d|j(                  ¬«      | _        y )Nr   g      à¿F©Úbias©Úeps)ÚtraditionalÚbase)ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsÚgetattrÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚo_projÚRMSNormr   Úq_normÚk_normÚRoPEr   Úrope)Úselfr1   r2   Údimr<   r=   r   Ú	__class__s          €r.   r;   zAttention.__init__&   s/  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ×'Ñ'Ð3Ð3Ð3Ø'+×'?Ñ'?Ð?ˆŒ˜*äØ�*˜d×.Ñ.°$×2JÑ2JÑJó
ˆð ˜t‘^ˆŒ
ä—i‘i  W¨xÑ%7¸eÔDˆŒÜ—i‘i  Z°(Ñ%:ÀÔGˆŒÜ—i‘i  Z°(Ñ%:ÀÔGˆŒÜ—i‘i ¨(Ñ 2°C¸eÔDˆŒä—j‘j ¨t×/@Ñ/@ÔAˆŒÜ—j‘j ¨t×/@Ñ/@ÔAˆŒä—G‘GØØØ—‘ô
ˆ�	r-   ÚxÚmaskÚcacheÚreturnc                 ón  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}| j	                  |j                  ||| j                  d«      «      j                  dddd«      }| j                  |j                  ||| j                  d«      «      j                  dddd«      }|	j                  ||| j                  d«      j                  dddd«      }	|�P| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||	«      \  }}	n"| j                  |«      }| j                  |«      }t        |||	|| j                  |¬«      }
|
j                  dddd«      j                  ||d«      }
| j                  |
«      S )Néÿÿÿÿr   é   r	   é   )Úoffset)rP   r?   rO   )ÚshaperB   rC   rD   rG   Úreshaper<   Ú	transposerH   r=   rJ   rV   Úupdate_and_fetchr   r?   rE   )rK   rN   rO   rP   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs              r.   Ú__call__zAttention.__call__A   s…  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆð —+‘+˜gŸo™o¨a°°D·L±LÀ"ÓEÓF×PÑPØˆq�!�Qó
ˆð �{‰{˜4Ÿ<™<¨¨1¨d¯o©o¸rÓBÓC×MÑMØˆq�!�Qó
ˆð —‘  1 d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r-   ©NN©r$   r%   r&   r   r)   r;   ÚmxÚarrayr   r   rb   Ú__classcell__©rM   s   @r.   r0   r0   %   s^   ø„ ð
˜Yð 
°3õ 
ð< $(Ø#ñ	#à�8‰8ð#ð �r—x‘xÑ ð#ð ˜‰}ð	#ð
 
�‰÷#r-   r0   c                   ó>   ‡ — e Zd Zˆ fd„Zdej
                  fd„Zˆ xZS )ÚMLPc                 óÐ   •— t         ‰| �  «        t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        y ©NFr4   )r:   r;   r@   rA   Ú	gate_projÚ	down_projÚup_proj)rK   rL   Ú
hidden_dimrM   s      €r.   r;   zMLP.__init__d   sJ   ø€ Ü‰ÑÔÜŸ™ 3¨
¸Ô?ˆŒÜŸ™ :¨s¸Ô?ˆŒÜ—y‘y  j°uÔ=ˆ�r-   rQ   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S ©N)rn   r
   rm   ro   )rK   rN   s     r.   rb   zMLP.__call__j   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr-   )r$   r%   r&   r;   re   rf   rb   rg   rh   s   @r.   rj   rj   c   s   ø„ ô>ðJ˜RŸX™X÷ Jr-   rj   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚQwen3MoeSparseMoeBlockr1   c                 ó  •— t         ‰| �  «        |j                  }|j                  }|j                  x| _        }|j
                  | _        |j                  | _        t        j                  ||d¬«      | _
        t        |||«      | _        y rl   )r:   r;   r   r   r   r   Útop_kr"   r@   rA   Úgater   Ú
switch_mlp)rK   r1   rL   r   r   rM   s        €r.   r;   zQwen3MoeSparseMoeBlock.__init__o   sx   ø€ Ü‰ÑÔØ×ÑˆØ ×6Ñ6Ðà)-×)9Ñ)9Ð9ˆÔ˜;Ø×-Ñ-ˆŒ
Ø"×1Ñ1ˆÔä—I‘I˜c ;°UÔ;ˆŒ	Ü# CÐ):¸KÓHˆ�r-   rN   rQ   c                 ó„  — | j                  |«      }t        j                  |dd¬«      }| j                  }t        j                  || d¬«      d| d …f   }t        j
                  ||d¬«      }| j                  r|t        j                  |dd¬«      z  }| j                  ||«      }||d   z  j                  d	¬«      }|S )
NrS   T)ÚaxisÚprecise)Úkthrz   .)rz   )rz   Úkeepdims).Néþÿÿÿ)	rw   re   Úsoftmaxrv   ÚargpartitionÚtake_along_axisr"   Úsumrx   )rK   rN   ÚgatesÚkÚindsÚscoresÚys          r.   rb   zQwen3MoeSparseMoeBlock.__call__{   s³   € ð —	‘	˜!“ˆÜ—
‘
˜5 r°4Ô8ˆà�J‰JˆÜ�‰˜u¨1¨"°2Ô6°s¸Q¸B¹C°xÑ@ˆÜ×#Ñ# E¨4°bÔ9ˆØ×ÒØ”b—f‘f˜V¨"°tÔ<Ñ<ˆFà�O‰O˜A˜tÓ$ˆØ�˜	Ñ"Ñ"×'Ñ'¨RÐ'Ó0ˆàˆr-   )	r$   r%   r&   r   r;   re   rf   rb   rg   rh   s   @r.   rt   rt   n   s0   ø„ ð
I˜Yõ 
Iðà�8‰8ðð 
�‰÷r-   rt   c            	       óŽ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
ÚQwen3MoeDecoderLayerr1   r2   c                 ó  •— t         ‰| �  «        |j                  | _        t        ||«      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _	        || _
        ||j                  vr5|j                  dkD  r&|dz   |j                  z  dk(  rt        |«      | _        y t!        |j                  |j"                  «      | _        y )Nr6   r   r	   )r:   r;   r   r0   Ú	self_attnr@   rF   r   Úinput_layernormÚpost_attention_layernormr1   r   r   r   rt   Úmlprj   r   )rK   r1   r2   rM   s      €r.   r;   zQwen3MoeDecoderLayer.__init__�   sÍ   ø€ Ü‰ÑÔØ×+Ñ+ˆÔÜ" 4¨Ó3ˆŒä!Ÿz™z¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
©
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ð ˆŒ	à˜T×1Ñ1Ñ1Ø×Ñ˜qÒ  i°!¡m°t×7OÑ7OÑ%OÐSTÒ%Tä-¨dÓ3ˆD�Hä˜4×+Ñ+¨T×-CÑ-CÓDˆD�Hr-   rN   rO   rP   rQ   c                 óž   — | j                  | j                  |«      ||«      }||z   }| j                  | j                  |«      «      }||z   }|S rr   )r‹   rŒ   rŽ   r�   )rK   rN   rO   rP   ÚrÚhÚouts          r.   rb   zQwen3MoeDecoderLayer.__call__¡   sQ   € ð �N‰N˜4×/Ñ/°Ó2°D¸%Ó@ˆØ�‰EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�!‰eˆØˆ
r-   rc   rd   rh   s   @r.   r‰   r‰   Ž   sa   ø„ ðE˜Yð E°3õ Eð* $(Ø#ñ	
à�8‰8ð
ð �r—x‘xÑ ð
ð ˜‰}ð	
ð
 
�‰÷
r-   r‰   c                   ó€   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dej                  fd„Zˆ xZ	S )ÚQwen3MoeModelr1   c                 óÄ  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        | j                  dkD  sJ ‚t        j                  |j                  |j                  «      | _        t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}| _        t        j                  |j                  |j                  ¬«      | _        y c c}w )Nr   )r1   r2   r6   )r:   r;   r1   r   r   r@   Ú	Embeddingr   Úembed_tokensÚranger‰   ÚlayersrF   r   Únorm)rK   r1   ÚirM   s      €r.   r;   zQwen3MoeModel.__init__¯   sµ   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ�‰ Ò"Ð"Ð"ÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔô ˜4×1Ñ1Ô2ó
á2�ô ! d°aÖ8Ø2ñ
ˆŒô —J‘J˜t×/Ñ/°T×5FÑ5FÔGˆ�	ùò	
s   ÂCÚinputsÚinput_embeddingsrQ   c                 óô   — |�|}n| j                  |«      }|€d gt        | j                  «      z  }t        ||d   «      }t	        | j                  |«      D ]  \  }} ||||«      }Œ | j                  |«      S )Nr   )r—   Úlenr™   r   Úziprš   )rK   rœ   rP   r�   r‘   rO   ÚlayerÚcs           r.   rb   zQwen3MoeModel.__call__¼   s€   € ð Ð'Ø ‰Aà×!Ñ! &Ó)ˆAàˆ=Ø�FœS §¡Ó-Ñ-ˆEä$ Q¨¨a©Ó1ˆä˜DŸK™K¨Ö/‰HˆE�1Ù�a˜˜qÓ!‰Að 0ð �y‰y˜‹|Ðr-   rc   )
r$   r%   r&   r   r;   re   rf   r   rb   rg   rh   s   @r.   r”   r”   ®   sK   ø„ ðH˜Yõ Hð  Ø/3ñ	à—‘ðð # 2§8¡8Ñ,ð	ð
 
�‰÷r-   r”   c                   ó¦   ‡ — e Zd Zdefˆ fd„Z	 	 d
dej                  deej                     dej                  fd„Zd„ Z	e
d„ «       Ze
d	„ «       Zˆ xZS )ÚModelr1   c                 óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y rl   )r:   r;   r1   r   r”   Úmodelr    r@   rA   r   r   Úlm_head)rK   r1   rM   s     €r.   r;   zModel.__init__Ó   sX   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ" 4Ó(ˆŒ
Ø×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (r-   rœ   r�   rQ   c                 óÈ   — | j                  |||«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S rr   )r¦   r1   r    r—   Ú	as_linearr§   )rK   rœ   rP   r�   r’   s        r.   rb   zModel.__call__Û   sZ   € ð �j‰j˜ Ð(8Ó9ˆØ�9‰9×(Ò(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r-   c                 ó¦  — | j                   j                  r|j                  dd «       d|vr|S t        | j                   j                  «      D ]{  }d|› �}dD ]o  }|› d|› d�|v sŒt        | j                   j
                  «      D �cg c]  }|j                  |› d|› d|› d�«      ‘Œ }}t        j                  |«      ||› d	|› d�<   Œq Œ} |S c c}w )
Nzlm_head.weightz+model.layers.0.mlp.experts.0.up_proj.weightzmodel.layers.)ro   rn   rm   z.mlp.experts.0.z.weightz.mlp.experts.Ú.z.mlp.switch_mlp.)r1   r    Úpopr˜   r   r   re   Ústack)rK   ÚweightsÚlÚprefixÚnÚeÚto_joins          r.   ÚsanitizezModel.sanitizeè   sí   € Ø�9‰9×(Ò(Ø�K‰KÐ(¨$Ô/Ø8ÀÑGØˆNÜ�t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFÛ:�Ø�X˜_¨Q¨C¨wÐ7¸7ÒBô "' t§y¡y×'<Ñ'<Ô!=óá!=˜Að  Ÿ™ v h¨m¸A¸3¸aÀ¸sÀ'Ð$JÕKØ!=ð ð ô FHÇXÁXÈgÓEV�G˜v˜hÐ&6°q°c¸ÐAÒBñ ;ð 4ð ˆùòs   Â!Cc                 ó   — d„ }|S )Nc                 ó0   — | j                  d«      rdddœS y)Nzmlp.gateé@   é   )Ú
group_sizeÚbitsT)Úendswith)ÚpathÚ_s     r.   Ú	predicatez(Model.quant_predicate.<locals>.predicateú   s   € Ø�}‰}˜ZÔ(Ø&(°!Ñ4Ð4Ør-   r,   )rK   r¾   s     r.   Úquant_predicatezModel.quant_predicateø   s   € ò	ð
 Ðr-   c                 ó.   — | j                   j                  S rr   )r¦   r™   )rK   s    r.   r™   zModel.layers  s   € à�z‰z× Ñ Ð r-   rc   )r$   r%   r&   r   r;   re   rf   r   rb   r´   Úpropertyr¿   r™   rg   rh   s   @r.   r¤   r¤   Ò   sx   ø„ ðT˜Yõ Tð Ø/3ñ	à—‘ðð # 2§8¡8Ñ,ð	ð
 
�‰óòð  ñó ðð ñ!ó ô!r-   r¤   )Údataclassesr   Útypingr   r   r   r   r   Úmlx.coreÚcorere   Úmlx.nnr@   Úactivationsr
   r9   r   r   r   Úswitch_layersr   r   ÚModuler0   rj   rt   r‰   r”   r¤   r,   r-   r.   Ú<module>rÊ      s¦   ðõ "ß 3Õ 3å Ý å ß TÑ TÝ $ð ô@�ó @ó ð@ô,;#�—	‘	ô ;#ô|Jˆ"�)‰)ô Jô˜RŸY™Yô ô@˜2Ÿ9™9ô ô@!�B—I‘Iô !ôH1!ˆB�I‰Iõ 1!r-   