Ë
    îÿæiÎ  ã                   óˆ  — d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZ e G d„ d	e«      «       Z G d
„ dej*                  «      Z G d„ dej*                  «      Z G d„ dej*                  «      Z G d„ dej*                  «      Z G d„ dej*                  «      Z G d„ dej*                  «      Zy)é    )Ú	dataclass)ÚAnyÚDictÚOptionalÚUnionNé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Ú	SwitchGLUc                   óè   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   dZee   ed<   dZ	eed<   dZ
eed<   dZeeeeeef   f      ed<   dZeed<   d„ Zy)Ú	ModelArgsÚ
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚnum_experts_per_tokÚnum_expertsÚmoe_intermediate_sizeÚshared_expert_intermediate_sizeÚrms_norm_epsÚ
vocab_sizeNÚnum_key_value_headsi@B Ú
rope_thetaFÚrope_traditionalÚrope_scalingÚtie_word_embeddingsc                 óà   ‡ — ‰ j                   €‰ j                  ‰ _         ‰ j                  rDddh}t        ˆ fd„|D «       «      st	        d|› �«      ‚‰ j                  d   dk7  rt	        d«      ‚y y )NÚfactorÚtypec              3   ó:   •K  — | ]  }|‰j                   v –— Œ y ­w©N)r   )Ú.0ÚkeyÚselfs     €úl/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/qwen2_moe.pyÚ	<genexpr>z*ModelArgs.__post_init__.<locals>.<genexpr>'   s   øè ø€ ÐI¹=°C�s˜d×/Ñ/Ô/¹=ùs   ƒzrope_scaling must contain keys Úlinearz4rope_scaling 'type' currently only supports 'linear')r   r   r   ÚallÚ
ValueError)r'   Úrequired_keyss   ` r(   Ú__post_init__zModelArgs.__post_init__!   sz   ø€ Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$à×ÒØ% vÐ.ˆMÜÓI¹=ÓIÔIÜ Ð#BÀ=À/Ð!RÓSÐSà× Ñ  Ñ(¨HÒ4Ü Ð!WÓXÐXð 5ð ó    )Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatr   r   r   r   Úboolr   r   r   r   r.   © r/   r(   r   r      s™   … àƒOØÓØÓØÓØÓØÓØÓØÓØ%(Ó(ØÓØƒOØ)-Ð˜ #™Ó-Ø€J�ÓØ"Ð�dÓ"Ø;?€L�(˜4  U¨5°#¨:Ñ%6Ð 6Ñ7Ñ8Ó?Ø %Ð˜Ó%ó
Yr/   r   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	Ú	AttentionÚargsc                 ó6  •— t         ‰| �  «        |j                  }|j                  x| _        }|j
                  €J ‚|j
                  x| _        }|j                  |z  }|dz  | _        t        j                  |||z  d¬«      | _
        t        j                  |||z  d¬«      | _        t        j                  |||z  d¬«      | _        t        j                  ||z  |d¬«      | _        t        j                  ||j                  |j                   ¬«      | _        y )Ng      à¿T©ÚbiasF)ÚtraditionalÚbase)ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚo_projÚRoPEr   r   Úrope)r'   r;   ÚdimrC   rD   Úhead_dimÚ	__class__s         €r(   rB   zAttention.__init__/   sñ   ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ×'Ñ'Ð3Ð3Ð3Ø'+×'?Ñ'?Ð?ˆŒ˜*à×#Ñ# wÑ.ˆØ˜t‘^ˆŒ
ä—i‘i  W¨xÑ%7¸dÔCˆŒÜ—i‘i  Z°(Ñ%:ÀÔFˆŒÜ—i‘i  Z°(Ñ%:ÀÔFˆŒÜ—i‘i ¨(Ñ 2°C¸eÔDˆŒä—G‘GØØ×-Ñ-Ø—‘ô
ˆ�	r/   ÚxÚmaskÚcacheÚreturnc                 ó2  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}|j	                  ||| j
                  d«      j                  dddd«      }|j	                  ||| j                  d«      j                  dddd«      }|	j	                  ||| j                  d«      j                  dddd«      }	|�P| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||	«      \  }}	n"| j                  |«      }| j                  |«      }t        |||	|| j                  |¬«      }
|
j                  dddd«      j	                  ||d«      }
| j                  |
«      S )Néÿÿÿÿr   é   r   é   )Úoffset)rS   rE   rR   )ÚshaperH   rI   rJ   ÚreshaperC   Ú	transposerD   rM   rY   Úupdate_and_fetchr   rE   rK   )r'   rQ   rR   rS   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs              r(   Ú__call__zAttention.__call__E   si  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆð —/‘/ ! Q¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A˜q $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  1 d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r/   ©NN©r0   r1   r2   r   rB   ÚmxÚarrayr   r   re   Ú__classcell__©rP   s   @r(   r:   r:   .   sW   ø„ ð
˜Yõ 
ð2 $(Ø#ñ	#à�8‰8ð#ð �r—x‘xÑ ð#ð ˜‰}ð	#ð
 
�‰÷#r/   r:   c                   ó>   ‡ — e Zd Zˆ fd„Zdej
                  fd„Zˆ xZS )ÚMLPc                 óÐ   •— t         ‰| �  «        t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        y ©NFr=   )rA   rB   rF   rG   Ú	gate_projÚ	down_projÚup_proj)r'   rN   Ú
hidden_dimrP   s      €r(   rB   zMLP.__init__d   sJ   ø€ Ü‰ÑÔÜŸ™ 3¨
¸Ô?ˆŒÜŸ™ :¨s¸Ô?ˆŒÜ—y‘y  j°uÔ=ˆ�r/   rT   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S r$   )rq   r	   rp   rr   )r'   rQ   s     r(   re   zMLP.__call__j   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr/   )r0   r1   r2   rB   rh   ri   re   rj   rk   s   @r(   rm   rm   c   s   ø„ ô>ðJ˜RŸX™X÷ Jr/   rm   c                   óD   ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zˆ xZS )ÚQwen2MoeSparseMoeBlockr;   c                 ól  •— t         ‰| �  «        |j                  }|j                  }|j                  }|j
                  x| _        }|j                  | _        t        j                  ||d¬«      | _
        t        |||«      | _        t        ||«      | _        t        j                  |dd¬«      | _        y )NFr=   r   )rA   rB   r   r   r   r   r   Útop_krF   rG   Úgater   Ú
switch_mlprm   Úshared_expertÚshared_expert_gate)r'   r;   rN   r   r   r   rP   s         €r(   rB   zQwen2MoeSparseMoeBlock.__init__o   s›   ø€ Ü‰ÑÔØ×ÑˆØ ×6Ñ6ÐØ*.×*NÑ*NÐ'à)-×)9Ñ)9Ð9ˆÔ˜;Ø×-Ñ-ˆŒ
ä—I‘I˜c ;°UÔ;ˆŒ	Ü# CÐ):¸KÓHˆŒä  Ð&EÓFˆÔÜ"$§)¡)¨C°¸Ô"?ˆÕr/   rQ   c                 óÖ  — | j                  |«      }t        j                  |dd¬«      }| j                  }t        j                  t        j
                  | |dz
  d¬«      dd |…f   «      }t        j                  ||d¬«      }| j                  ||«      }||d   z  j                  d	¬«      }| j                  |«      }t        j                  | j                  |«      «      |z  }||z   S )
NrV   T)ÚaxisÚpreciser   )Úkthr~   .)r~   ).Néþÿÿÿ)ry   rh   Úsoftmaxrx   Ústop_gradientÚargpartitionÚtake_along_axisrz   Úsumr{   Úsigmoidr|   )r'   rQ   ÚgatesÚkÚindsÚscoresÚyÚshared_expert_outputs           r(   re   zQwen2MoeSparseMoeBlock.__call__~   sá   € ð —	‘	˜!“ˆÜ—
‘
˜5 r°4Ô8ˆà�J‰JˆÜ×Ñ¤§¡°°¸AÀ¹EÈÔ KÈCÐQSÐRSÐQSÈGÑ TÓUˆÜ×#Ñ# E¨4°bÔ9ˆà�O‰O˜A˜tÓ$ˆØ�˜	Ñ"Ñ"×'Ñ'¨RÐ'Ó0ˆà#×1Ñ1°!Ó4Ðä�J‰J�t×.Ñ.¨qÓ1Ó2Ð5IÑIð 	ð Ð'Ñ'Ð'r/   ©	r0   r1   r2   r   rB   rh   ri   re   rj   rk   s   @r(   rv   rv   n   s"   ø„ ð@˜Yõ @ð(à�8‰8÷(r/   rv   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚQwen2MoeDecoderLayerr;   c                 óR  •— t         ‰| �  «        |j                  | _        t        |«      | _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _
        t        j                  |j                  |j                  ¬«      | _        || _        y )N©Úeps)rA   rB   r   r:   Ú	self_attnrv   ÚmlprF   ÚRMSNormr   Úinput_layernormÚpost_attention_layernormr;   ©r'   r;   rP   s     €r(   rB   zQwen2MoeDecoderLayer.__init__•   s}   ø€ Ü‰ÑÔØ×+Ñ+ˆÔÜ" 4›ˆŒÜ)¨$Ó/ˆŒä!Ÿz™z¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
©
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ð ˆ�	r/   rQ   rR   rS   rT   c                 óž   — | j                  | j                  |«      ||«      }||z   }| j                  | j                  |«      «      }||z   }|S r$   )r”   r—   r•   r˜   )r'   rQ   rR   rS   ÚrÚhÚouts          r(   re   zQwen2MoeDecoderLayer.__call__¡   sQ   € ð �N‰N˜4×/Ñ/°Ó2°D¸%Ó@ˆØ�‰EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�!‰eˆØˆ
r/   rf   rg   rk   s   @r(   r�   r�   ”   sW   ø„ ð
˜Yõ 
ð $(Ø#ñ	
à�8‰8ð
ð �r—x‘xÑ ð
ð ˜‰}ð	
ð
 
�‰÷
r/   r�   c                   óH   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zˆ xZS )ÚQwen2MoeModelr;   c                 óÂ  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        | j                  dkD  sJ ‚t        j                  |j                  |j                  «      | _        t        |j                  «      D �cg c]  }t        |¬«      ‘Œ c}| _        t        j                  |j                  |j                  ¬«      | _        y c c}w )Nr   )r;   r’   )rA   rB   r;   r   r   rF   Ú	Embeddingr   Úembed_tokensÚranger�   Úlayersr–   r   Únorm)r'   r;   Ú_rP   s      €r(   rB   zQwen2MoeModel.__init__¯   s¯   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ�‰ Ò"Ð"Ð"ÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔä5:¸4×;QÑ;QÔ5Ró
Ù5R°Ô  dÖ+Ð5Rñ
ˆŒô —J‘J˜t×/Ñ/°T×5FÑ5FÔGˆ�	ùò
s   ÂCÚinputsc                 óê   — | j                  |«      }|€d gt        | j                  «      z  }t        ||d   «      }t	        | j                  |«      D ]  \  }} ||||«      }Œ | j                  |«      S )Nr   )r¢   Úlenr¤   r   Úzipr¥   )r'   r§   rS   rœ   rR   ÚlayerÚcs          r(   re   zQwen2MoeModel.__call__»   su   € ð
 ×Ñ˜fÓ%ˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEä$ Q¨¨a©Ó1ˆä˜DŸK™K¨Ö/‰HˆE�1Ù�a˜˜qÓ!‰Að 0ð �y‰y˜‹|Ðr/   r$   rŽ   rk   s   @r(   rŸ   rŸ   ®   s'   ø„ ð
H˜Yõ 
Hð ñà—‘÷r/   rŸ   c                   ó^   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zd„ Ze	d„ «       Z
ˆ xZS )ÚModelr;   c                 óÔ   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        t        j                  |j                  |j                  d¬«      | _
        y ro   )rA   rB   r;   r   rŸ   ÚmodelrF   rG   r   r   Úlm_headr™   s     €r(   rB   zModel.__init__Î   sJ   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ" 4Ó(ˆŒ
Ü—y‘y ×!1Ñ!1°4·?±?ÈÔOˆ�r/   r§   c                 óH   — | j                  ||«      }| j                  |«      S r$   )r°   r±   )r'   r§   rS   r�   s       r(   re   zModel.__call__Õ   s#   € ð
 �j‰j˜ Ó'ˆØ�|‰|˜CÓ Ð r/   c                 óp  — d|vr|S t        | j                  j                  «      D ]ˆ  }d|› �}dD ]|  }dD ]u  }|› d|› d|› �|v sŒt        | j                  j                  «      D �cg c]  }|j	                  |› d|› d|› d|› �«      ‘Œ  }}t        j                  |«      ||› d|› d|› �<   Œw Œ~ ŒŠ |S c c}w )	Nz+model.layers.0.mlp.experts.0.up_proj.weightzmodel.layers.)rr   rq   rp   )ÚweightÚscalesÚbiasesz.mlp.experts.0.Ú.z.mlp.experts.z.mlp.switch_mlp.)r£   r;   r   r   Úpoprh   Ústack)r'   ÚweightsÚlÚprefixÚnr‰   ÚeÚto_joins           r(   ÚsanitizezModel.sanitizeÝ   sç   € Ø8ÀÑGØˆNÜ�t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFÛ:�Û7�AØ ˜ °°°1°Q°CÐ8¸GÒCô &+¨4¯9©9×+@Ñ+@Ô%Aó#á%A ð $ŸK™K¨6¨(°-À¸sÀ!ÀAÀ3ÀaÈÀsÐ(KÕLØ%Að  ð #ô GIÇhÁhÈwÓFW˜ 6 (Ð*:¸1¸#¸Q¸q¸cÐ BÒCñ 8ñ ;ð 4ð ˆùò#s   Á&#B3
c                 ó.   — | j                   j                  S r$   )r°   r¤   )r'   s    r(   r¤   zModel.layersì   s   € à�z‰z× Ñ Ð r/   r$   )r0   r1   r2   r   rB   rh   ri   re   rÀ   Úpropertyr¤   rj   rk   s   @r(   r®   r®   Í   s@   ø„ ðP˜Yõ Pð ñ!à—‘ó!òð ñ!ó ô!r/   r®   )Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorerh   Úmlx.nnrF   Úactivationsr	   r@   r
   r   r   Úswitch_layersr   r   ÚModuler:   rm   rv   r�   rŸ   r®   r8   r/   r(   Ú<module>rË      s¤   ðõ "ß -Ó -å Ý å ß TÑ TÝ $ð ôY�ó Yó ðYô>2#�—	‘	ô 2#ôjJˆ"�)‰)ô Jô#(˜RŸY™Yô #(ôL˜2Ÿ9™9ô ô4�B—I‘Iô ô>!!ˆB�I‰Iõ !!r/   