Ë
    îÿæi  ã                   óf  — d dl Z d dlmZ d dlmZmZmZmZ d dlm	Z
 d dlmZ ddlmZmZmZ ddlmZ ddlmZ e G d„ d	e«      «       Z G d
„ dej,                  «      Z G d„ dej,                  «      Z G d„ dej,                  «      Z G d„ dej,                  «      Z G d„ dej,                  «      Zy)é    N)Ú	dataclass)ÚDictÚListÚOptionalÚUnioné   )ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚSuScaledRoPE)Ú	SwitchGLUc                   óì   — e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   d	Z
eed
<   d	Zeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeeeeee   f   f   ed<   dZeed<   dZeed<   dZeed<   y)Ú	ModelArgsÚphimoeÚ
model_typei@}  Ú
vocab_sizei   Úhidden_sizei   Úintermediate_sizeé    Únum_hidden_layersÚnum_attention_headsé   Únum_key_value_headsi   Úmax_position_embeddingsÚ original_max_position_embeddingsg�íµ ÷Æ°>Úrms_norm_epsNÚrope_scalingé   Únum_local_expertsé   Únum_experts_per_tokg     ˆÃ@Ú
rope_theta)Ú__name__Ú
__module__Ú__qualname__r   ÚstrÚ__annotations__r   Úintr   r   r   r   r   r   r   r   Úfloatr   r   r   r   r   r!   r"   © ó    úi/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/phimoe.pyr   r      s®   … à€J�ÓØ€J�ÓØ€K�ÓØ!Ð�sÓ!ØÐ�sÓØ!Ð˜Ó!Ø Ð˜Ó Ø#)Ð˜SÓ)Ø,0Ð$ cÓ0Ø€L�%ÓØ9=€L�$�s˜E %¨¨e©Ð"4Ñ5Ð5Ñ6Ó=ØÐ�sÓØ Ð˜Ó Ø€J�Ôr+   r   c                   ó€   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dej                  fd„Zˆ xZ	S )Ú	AttentionÚargsc                 óŒ  •— t         ‰| �  «        |j                  }|j                  x| _        }|j
                  x| _        }|j                  |z  }|dz  | _        t        j                  |||z  d¬«      | _
        t        j                  |||z  d¬«      | _        t        j                  |||z  d¬«      | _        t        j                  ||z  |d¬«      | _        t        ||j                  |j                   |j"                  |j$                  d   |j$                  d   |j$                  d   |j$                  d   ¬«      | _        y )	Ng      à¿T©ÚbiasÚshort_factorÚlong_factorÚshort_mscaleÚlong_mscale)Úbaser   r   r3   r4   r5   r6   )ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚo_projr   r"   r   r   r   Úrope)Úselfr/   Údimr:   r;   Úhead_dimÚ	__class__s         €r,   r9   zAttention.__init__!   s  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*à×#Ñ# wÑ.ˆØ˜t‘^ˆŒ
ä—i‘i  W¨xÑ%7¸dÔCˆŒÜ—i‘i  Z°(Ñ%:ÀÔFˆŒÜ—i‘i  Z°(Ñ%:ÀÔFˆŒÜ—i‘i ¨(Ñ 2°C¸dÔCˆŒä ØØ—‘Ø$(×$@Ñ$@Ø-1×-RÑ-RØ×*Ñ*¨>Ñ:Ø×)Ñ)¨-Ñ8Ø×*Ñ*¨>Ñ:Ø×)Ñ)¨-Ñ8ô	
ˆ�	r+   ÚxÚmaskÚreturnc                 ó2  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}|j	                  ||| j
                  d«      j                  dddd«      }|j	                  ||| j                  d«      j                  dddd«      }|	j	                  ||| j                  d«      j                  dddd«      }	|�P| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||	«      \  }}	n"| j                  |«      }| j                  |«      }t        |||	|| j                  |¬«      }
|
j                  dddd«      j	                  ||d«      }
| j                  |
«      S )Néÿÿÿÿr   r    r   é   )Úoffset)Úcacher<   rI   )Úshaper?   r@   rA   Úreshaper:   Ú	transposer;   rC   rN   Úupdate_and_fetchr   r<   rB   )rD   rH   rI   rO   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs              r,   Ú__call__zAttention.__call__;   si  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆð —/‘/ ! Q¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A˜q $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  1 d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r+   ©NN©
r#   r$   r%   r   r9   ÚmxÚarrayr   r[   Ú__classcell__©rG   s   @r,   r.   r.       sI   ø„ ð
˜Yõ 
ð: $(Øñ	#à�8‰8ð#ð �r—x‘xÑ ð#ð
 
�‰÷#r+   r.   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚPhiMoESparseMoeBlockr/   c                 ól  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  | j                  | j                  d¬«      | _        t        | j                  | j
                  | j                  «      | _        y )NFr1   )r8   r9   r   Ú
hidden_dimr   Úffn_dimr   Únum_expertsr!   Útop_kr=   r>   Úgater   Ú
switch_mlp©rD   r/   rG   s     €r,   r9   zPhiMoESparseMoeBlock.__init__Z   s   ø€ Ü‰ÑÔØ×*Ñ*ˆŒØ×-Ñ-ˆŒØ×1Ñ1ˆÔØ×-Ñ-ˆŒ
ä—I‘I˜dŸo™o¨t×/?Ñ/?ÀeÔLˆŒ	Ü# D§O¡O°T·\±\À4×CSÑCSÓTˆ�r+   rH   rJ   c                 ó`  — | j                  |«      }| j                  }t        j                  t        j                  | |dz
  d¬«      dd |…f   «      }t        j
                  ||d¬«      }t        j                  |dd¬«      }| j                  ||«      }||d   z  j                  d	¬«      }|S )
Nr   rL   )ÚkthÚaxis.)rn   T)rn   Úprecise).Néþÿÿÿ)	ri   rh   r^   Ústop_gradientÚargpartitionÚtake_along_axisÚsoftmaxrj   Úsum)rD   rH   ÚgatesÚkÚindsÚscoresÚys          r,   r[   zPhiMoESparseMoeBlock.__call__d   s£   € Ø—	‘	˜!“ˆà�J‰JˆÜ×Ñ¤§¡°°¸AÀ¹EÈÔ KÈCÐQSÐRSÐQSÈGÑ TÓUˆÜ×#Ñ# E¨4°bÔ9ˆÜ—‘˜F¨°TÔ:ˆà�O‰O˜A˜tÓ$ˆØ�˜	Ñ"Ñ"×'Ñ'¨RÐ'Ó0ˆàˆr+   ©	r#   r$   r%   r   r9   r^   r_   r[   r`   ra   s   @r,   rc   rc   Y   s,   ø„ ðU˜Yõ Uð˜"Ÿ(™(ð  r§x¡x÷ r+   rc   c                   ó€   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dej                  fd„Zˆ xZ	S )ÚPhiMoEDecoderLayerr/   c                 óD  •— t         ‰| �  «        |j                  | _        t        |«      | _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _
        t        j                  |j                  |j                  ¬«      | _        y ©N)Úeps)r8   r9   r   r.   Ú	self_attnrc   Úblock_sparse_moer=   Ú	LayerNormr   Úinput_layernormÚpost_attention_layernormrk   s     €r,   r9   zPhiMoEDecoderLayer.__init__s   su   ø€ Ü‰ÑÔØ×+Ñ+ˆÔä" 4›ˆŒÜ 4°TÓ :ˆÔÜ!Ÿ|™|¨D×,<Ñ,<À$×BSÑBSÔTˆÔÜ(*¯©Ø×Ñ $×"3Ñ"3ô)
ˆÕ%r+   rH   rI   rJ   c                 ó°   — |}| j                  |«      }| j                  |||¬«      }||z   }|}| j                  |«      }| j                  |«      }||z   }|S )N)rI   rO   )r„   r�   r…   r‚   )rD   rH   rI   rO   ÚresidualÚhidden_statess         r,   r[   zPhiMoEDecoderLayer.__call__~   sl   € ð ˆØ×,Ñ,¨QÓ/ˆØŸ™ }¸4Àu˜ÓMˆØ  =Ñ0ˆà ˆØ×5Ñ5°mÓDˆØ×-Ñ-¨mÓ<ˆØ  =Ñ0ˆàÐr+   r\   r]   ra   s   @r,   r}   r}   r   sI   ø„ ð	
˜Yõ 	
ð $(Øñ	à�8‰8ðð �r—x‘xÑ ðð
 
�‰÷r+   r}   c                   ó`   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dej                  fd„Zˆ xZS )ÚPhiMoEModelr/   c                 ó|  •— t         ‰| �  «        || _        |j                  | _        t	        j
                  |j                  |j                  «      | _        t        |j                  «      D �cg c]  }t        |«      ‘Œ c}| _        t	        j                  |j                  |j                  ¬«      | _        y c c}w r   )r8   r9   r/   r   r=   Ú	Embeddingr   Úembed_tokensÚranger   r}   Úlayersrƒ   r   Únorm)rD   r/   Ú_rG   s      €r,   r9   zPhiMoEModel.__init__’   sŠ   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒäŸL™L¨¯©¸$×:JÑ:JÓKˆÔÜ9>¸t×?UÑ?UÔ9VÓWÑ9V°AÔ)¨$Õ/Ð9VÑWˆŒÜ—L‘L ×!1Ñ!1°t×7HÑ7HÔIˆ�	ùò Xs   Á.B9ÚinputsrJ   c                 óê   — | j                  |«      }|€d gt        | j                  «      z  }t        ||d   «      }t	        | j                  |«      D ]  \  }} ||||«      }Œ | j                  |«      S )Nr   )r�   Úlenr�   r
   Úzipr�   )rD   r’   rO   ÚhrI   ÚlayerÚcs          r,   r[   zPhiMoEModel.__call__›   su   € ð
 ×Ñ˜fÓ%ˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEÜ$ Q¨¨a©Ó1ˆä˜DŸK™K¨Ö/‰HˆE�1Ù�a˜˜qÓ!‰Að 0ð �y‰y˜‹|Ðr+   ©Nr{   ra   s   @r,   rŠ   rŠ   ‘   s5   ø„ ðJ˜Yõ Jð ñà—‘ðð 
�‰÷	r+   rŠ   c                   ó^   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zd„ Ze	d„ «       Z
ˆ xZS )ÚModelr/   c                 óÔ   •— t         ‰| �  «        |j                  | _        || _        t	        |«      | _        t        j                  |j                  |j                  d¬«      | _
        y )NTr1   )r8   r9   r   r/   rŠ   Úmodelr=   r>   r   r   Úlm_headrk   s     €r,   r9   zModel.__init__­   sJ   ø€ Ü‰ÑÔØŸ/™/ˆŒØˆŒ	Ü  Ó&ˆŒ
Ü—y‘y ×!1Ñ!1°4·?±?ÈÔNˆ�r+   r’   c                 óH   — | j                  ||«      }| j                  |«      S r™   )r�   rž   )rD   r’   rO   Úouts       r,   r[   zModel.__call__´   s#   € ð
 �j‰j˜ Ó'ˆØ�|‰|˜CÓ Ð r+   c                 óv  — d|vr|S t        | j                  j                  «      D ]‹  }d|› �}dD ]  \  }}dD ]u  }|› d|› d|› �|v sŒt        | j                  j                  «      D �cg c]  }|j	                  |› d|› d|› d|› �«      ‘Œ  }}t        j                  |«      ||› d|› d|› �<   Œw Œ� Œ� |S c c}w )	Nz3model.layers.0.block_sparse_moe.experts.0.w1.weightzmodel.layers.))Úw1Ú	gate_proj)Úw2Ú	down_proj)Úw3Úup_proj)ÚweightÚscalesÚbiasesz.block_sparse_moe.experts.0.Ú.z.block_sparse_moe.experts.z.block_sparse_moe.switch_mlp.)rŽ   r/   r   r   Úpopr^   Ústack)	rD   ÚweightsÚlÚprefixÚnÚmrw   ÚeÚto_joins	            r,   ÚsanitizezModel.sanitize¼   sø   € Ø@ÈÑOØˆNÜ�t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFÛU‘��1Û7�AØ ˜Ð!=¸a¸SÀÀ!ÀÐEÈÒPô
 &+¨4¯9©9×+FÑ+FÔ%Gó	#ñ &H ð $ŸK™KØ#) (Ð*DÀQÀCÀqÈÈÈ1ÈQÈCÐ Põð &Hð	  ð #ô ŸH™H WÓ-ð   6 (Ð*GÈÀsÈ!ÈAÈ3Ð OÒPñ 8ñ Vð 4ð ˆùò#s   Á)#B6
c                 ó.   — | j                   j                  S r™   )r�   r�   )rD   s    r,   r�   zModel.layersÐ   s   € à�z‰z× Ñ Ð r+   r™   )r#   r$   r%   r   r9   r^   r_   r[   rµ   Úpropertyr�   r`   ra   s   @r,   r›   r›   ¬   s@   ø„ ðO˜Yõ Oð ñ!à—‘ó!òð( ñ!ó ô!r+   r›   )ÚmathÚdataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorer^   Úmlx.nnr=   r7   r	   r
   r   Ú
rope_utilsr   Úswitch_layersr   r   ÚModuler.   rc   r}   rŠ   r›   r*   r+   r,   Ú<module>rÁ      s�   ðã Ý !ß .Ó .å Ý ç TÑ TÝ $Ý $ð ô �ó  ó ð ô"6#�—	‘	ô 6#ôr˜2Ÿ9™9ô ô2˜Ÿ™ô ô>�"—)‘)ô ô6&!ˆB�I‰Iõ &!r+   