Ë
    îÿæi"  ã                   ó®  — d dl mZ d dlmZmZmZ d dlmZ d dl	m
Z
 ddlmZ ddlmZmZmZ ddlmZ e G d„ d	e«      «       Z G d
„ de
j(                  «      Z G d„ de
j(                  «      Z G d„ de
j(                  «      Z G d„ de
j(                  «      Z G d„ de
j(                  «      Z G d„ de
j(                  «      Z G d„ de
j(                  «      Zy)é    )Ú	dataclass)ÚAnyÚDictÚOptionalNé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Ú	SwitchGLUc                   ó"  — e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZee   ed<   dZee   ed<   dZee   ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZee   ed<   dZeed <   y)!Ú	ModelArgsÚdeepseekÚ
model_typei � Ú
vocab_sizei   Úhidden_sizei +  Úintermediate_sizei  Úmoe_intermediate_sizeé   Únum_hidden_layersé    Únum_attention_headsÚnum_key_value_headsNÚn_shared_expertsÚn_routed_expertsÚnum_experts_per_tokr   Úmoe_layer_freqr   Úfirst_k_dense_replacei   Úmax_position_embeddingsg�íµ ÷Æ°>Úrms_norm_epsg     ˆÃ@Ú
rope_thetaÚrope_scalingFÚattention_bias)Ú__name__Ú
__module__Ú__qualname__r   ÚstrÚ__annotations__r   Úintr   r   r   r   r   r   r   r   r   r   r   r   r   r    Úfloatr!   r"   r   r#   Úbool© ó    úk/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/deepseek.pyr   r      sÐ   … à €J�Ó Ø€J�ÓØ€K�ÓØ"Ð�sÓ"Ø!%Ð˜3Ó%ØÐ�sÓØ!Ð˜Ó!Ø!Ð˜Ó!Ø&*Ð�h˜s‘mÓ*Ø&*Ð�h˜s‘mÓ*Ø)-Ð˜ #™Ó-Ø€N�CÓØ!"Ð˜3Ó"Ø#'Ð˜SÓ'Ø€L�%ÓØ€J�ÓØ#'€L�(˜4‘.Ó'Ø €N�DÔ r-   r   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚDeepseekAttentionÚconfigc                 ó  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        |j
                  | _        |j                  |j                  z  | _        | j                  dz  | _        t        |dd«      }t        j                  | j                  |j                  | j                  z  |¬«      | _        t        j                  | j                  |j
                  | j                  z  |¬«      | _        t        j                  | j                  |j
                  | j                  z  |¬«      | _        t        j                  | j                  |j                  | j                  z  |¬«      | _        d}|j                   rC|j                   d   dk(  r1t#        |j                   d   t$        «      sJ ‚d	|j                   d   z  }t        j&                  | j                  |j(                  |¬
«      | _        y )Ng      à¿r#   F©Úbiasg      ð?ÚtypeÚlinearÚfactorr   )ÚbaseÚscale)ÚsuperÚ__init__r1   r   r   r   Únum_kv_headsÚhead_dimr9   ÚgetattrÚnnÚLinearÚq_projÚk_projÚv_projÚo_projr"   Ú
isinstancer*   ÚRoPEr!   Úrope)Úselfr1   r#   Ú
rope_scaleÚ	__class__s       €r.   r;   zDeepseekAttention.__init__#   s©  ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ Ø"×6Ñ6ˆÔØ×*Ñ*¨f×.HÑ.HÑHˆŒØ—]‘] DÑ(ˆŒ
ä  Ð)9¸5ÓAˆä—i‘iØ×ÑØ×&Ñ&¨¯©Ñ6Øô
ˆŒô
 —i‘iØ×ÑØ×&Ñ&¨¯©Ñ6Øô
ˆŒô
 —i‘iØ×ÑØ×&Ñ&¨¯©Ñ6Øô
ˆŒô
 —i‘iØ×ÑØ×&Ñ&¨¯©Ñ6Øô
ˆŒð ˆ
Ø×Ò 6×#6Ñ#6°vÑ#>À(Ò#JÜ˜f×1Ñ1°(Ñ;¼UÔCÐCÐCØ˜V×0Ñ0°Ñ:Ñ:ˆJÜ—G‘GØ�M‰MØ×"Ñ"Øô
ˆ�	r-   ÚxÚmaskÚcacheÚreturnc                 ó2  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}|j	                  ||| j
                  d«      j                  dddd«      }|j	                  ||| j                  d«      j                  dddd«      }|	j	                  ||| j                  d«      j                  dddd«      }	|�P| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||	«      \  }}	n"| j                  |«      }| j                  |«      }t        |||	|| j                  |¬«      }
|
j                  dddd«      j	                  ||d«      }
| j                  |
«      S )Néÿÿÿÿr   é   r   é   )Úoffset)rM   r9   rL   )ÚshaperA   rB   rC   Úreshaper   Ú	transposer<   rG   rS   Úupdate_and_fetchr   r9   rD   )rH   rK   rL   rM   ÚBÚLÚ_ÚqueriesÚkeysÚvaluesÚoutputs              r.   Ú__call__zDeepseekAttention.__call__M   ss  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆà—/‘/ ! Q¨×(@Ñ(@À"ÓE×OÑOØˆq�!�Qó
ˆð �|‰|˜A˜q $×"3Ñ"3°RÓ8×BÑBÀ1ÀaÈÈAÓNˆØ—‘  1 d×&7Ñ&7¸Ó<×FÑFÀqÈ!ÈQÐPQÓRˆàÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r-   ©NN©r$   r%   r&   r   r;   ÚmxÚarrayr   r   r_   Ú__classcell__©rJ   s   @r.   r0   r0   "   sX   ø„ ð(
˜yõ (
ðZ $(Ø#ñ	#à�8‰8ð#ð �r—x‘xÑ ð#ð ˜‰}ð	#ð
 
�‰÷#r-   r0   c                   óv   ‡ — e Zd Z	 	 ddedee   dee   fˆ fd„Zdej                  dej                  fd„Z	ˆ xZ
S )	ÚDeepseekMLPr1   r   r   c                 óª  •— t         ‰| �  «        || _        |xs |j                  | _        |xs |j                  | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _	        y ©NFr3   )
r:   r;   r1   r   r   r?   r@   Ú	gate_projÚup_projÚ	down_proj)rH   r1   r   r   rJ   s       €r.   r;   zDeepseekMLP.__init__m   s    ø€ ô 	‰ÑÔØˆŒØ&Ò<¨&×*<Ñ*<ˆÔØ!2Ò!N°f×6NÑ6NˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆ�r-   rK   rN   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S ©N)rl   r   rj   rk   )rH   rK   s     r.   r_   zDeepseekMLP.__call__{   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr-   r`   )r$   r%   r&   r   r   r)   r;   rb   rc   r_   rd   re   s   @r.   rg   rg   l   sV   ø„ ð &*Ø+/ñ	YàðYð ˜c‘]ðYð $ C™=õ	YðJ˜"Ÿ(™(ð J r§x¡x÷ Jr-   rg   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚMoEGater1   c                 óÔ   •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        t        j                  | j
                  |j                  f«      | _	        y rn   )
r:   r;   r1   r   Útop_kr   rb   Úzerosr   Úweight©rH   r1   rJ   s     €r.   r;   zMoEGate.__init__€   sQ   ø€ Ü‰ÑÔØˆŒØ×/Ñ/ˆŒ
Ø &× 7Ñ 7ˆÔÜ—h‘h × 5Ñ 5°v×7IÑ7IÐJÓKˆ�r-   c                 ó   — || j                   j                  z  }t        j                  |dd¬«      }| j                  }t        j
                  t        j                  | |dz
  d¬«      dd |…f   «      }t        j                  ||d¬«      }||fS )NrP   T)ÚaxisÚpreciser   )Úkthrw   .©rw   )rt   ÚTrb   Úsoftmaxrr   Ústop_gradientÚargpartitionÚtake_along_axis)rH   rK   ÚgatesÚscoresÚkÚindss         r.   r_   zMoEGate.__call__‡   s€   € Ø�D—K‘K—M‘MÑ!ˆÜ—‘˜E¨°DÔ9ˆØ�J‰JˆÜ×Ñ¤§¡°°¸QÀ¹UÈÔ LÈSÐRTÐSTÐRTÈWÑ UÓVˆÜ×#Ñ# F¨D°rÔ:ˆØ�Vˆ|Ðr-   ©r$   r%   r&   r   r;   r_   rd   re   s   @r.   rp   rp      s   ø„ ðL˜yõ Lör-   rp   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚDeepseekMoEr1   c                 ó   •— t         ‰| �  «        || _        t        |j                  |j
                  |j                  «      | _        t        |«      | _	        |j                  �,|j
                  |j                  z  }t        ||¬«      | _        y y )N)r1   r   )r:   r;   r1   r   r   r   r   Ú
switch_mlprp   Úgater   rg   Úshared_experts)rH   r1   r   rJ   s      €r.   r;   zDeepseekMoE.__init__‘   s‚   ø€ Ü‰ÑÔØˆŒÜ#Ø×Ñ × <Ñ <¸f×>UÑ>Uó
ˆŒô ˜F“OˆŒ	Ø×"Ñ"Ð.Ø &× <Ñ <¸v×?VÑ?VÑ VÐÜ"-ØÐ1Bô#ˆDÕð /r-   c                 óÖ   — | j                  |«      \  }}| j                  ||«      }||d   z  j                  d¬«      }| j                  j                  �|| j                  |«      z   }|S )N).Néþÿÿÿrz   )r‰   rˆ   Úsumr1   r   rŠ   )rH   rK   rƒ   r�   Úys        r.   r_   zDeepseekMoE.__call__Ÿ   sj   € Ø—y‘y “|‰ˆˆfØ�O‰O˜A˜tÓ$ˆØ�˜	Ñ"Ñ"×'Ñ'¨RÐ'Ó0ˆØ�;‰;×'Ñ'Ð3Ø�D×'Ñ'¨Ó*Ñ*ˆAàˆr-   r„   re   s   @r.   r†   r†   �   s   ø„ ð˜yõ ör-   r†   c            	       óŽ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
ÚDeepseekDecoderLayerr1   Ú	layer_idxc                 ó’  •— t         ‰| �  «        t        |«      | _        |j                  �,||j
                  k\  r||j                  z  dk(  rt        |«      n
t        |«      | _	        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _        y )Nr   ©Úeps)r:   r;   r0   Ú	self_attnr   r   r   r†   rg   Úmlpr?   ÚRMSNormr   r    Úinput_layernormÚpost_attention_layernorm)rH   r1   r‘   rJ   s      €r.   r;   zDeepseekDecoderLayer.__init__ª   s§   ø€ Ü‰ÑÔÜ*¨6Ó2ˆŒð ×'Ñ'Ð3Ø ×!=Ñ!=Ò=Ø × 5Ñ 5Ñ5¸Ò:ô	 ˜Ôô ˜VÓ$ð 	Œô  "Ÿz™z¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(*¯
©
Ø×Ñ F×$7Ñ$7ô)
ˆÕ%r-   rK   rL   rM   rN   c                 óž   — | j                  | j                  |«      ||«      }||z   }| j                  | j                  |«      «      }||z   }|S rn   )r•   r˜   r–   r™   )rH   rK   rL   rM   ÚrÚhÚouts          r.   r_   zDeepseekDecoderLayer.__call__»   sQ   € ð �N‰N˜4×/Ñ/°Ó2°D¸%Ó@ˆØ�‰EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�!‰eˆØˆ
r-   r`   )r$   r%   r&   r   r)   r;   rb   rc   r   r   r_   rd   re   s   @r.   r�   r�   ©   s^   ø„ ð
˜yð 
°Sõ 
ð( $(Ø#ñ	
à�8‰8ð
ð �r—x‘xÑ ð
ð ˜‰}ð	
ð
 
�‰÷
r-   r�   c                   ój   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dee   dej                  fd„Z	ˆ xZ
S )ÚDeepseekModelr1   c                 ó\  •— t         ‰| �  «        || _        t        j                  |j
                  |j                  «      | _        t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}| _        t        j                  |j                  |j                  ¬«      | _        y c c}w )Nr“   )r:   r;   r1   r?   Ú	Embeddingr   r   Úembed_tokensÚranger   r�   Úlayersr—   r    Únorm)rH   r1   ÚidxrJ   s      €r.   r;   zDeepseekModel.__init__É   sˆ   ø€ Ü‰ÑÔØˆŒÜŸL™L¨×):Ñ):¸F×<NÑ<NÓOˆÔä9>¸v×?WÑ?WÔ9Xó
Ù9X°#Ô  ¨Õ-Ð9Xñ
ˆŒô —J‘J˜v×1Ñ1°v×7JÑ7JÔKˆ�	ùò
s   ÁB)rK   rM   rN   c                 óê   — | j                  |«      }|€d gt        | j                  «      z  }t        ||d   «      }t	        | j                  |«      D ]  \  }} ||||«      }Œ | j                  |«      S )Nr   )r¢   Úlenr¤   r
   Úzipr¥   )rH   rK   rM   rœ   rL   ÚlayerÚcs          r.   r_   zDeepseekModel.__call__Ò   su   € ð
 ×Ñ˜aÓ ˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEä$ Q¨¨a©Ó1ˆä˜DŸK™K¨Ö/‰HˆE�1Ù�a˜˜qÓ!‰Að 0ð �y‰y˜‹|Ðr-   rn   ra   re   s   @r.   rŸ   rŸ   È   sC   ø„ ðL˜yõ Lð  $ñà�8‰8ðð ˜‰}ðð 
�‰÷	r-   rŸ   c                   óh   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dee   fd„Z	d„ Z
ed„ «       Zˆ xZS )	ÚModelr1   c                 óÔ   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        t        j                  |j                  |j                  d¬«      | _
        y ri   )r:   r;   Úargsr   rŸ   Úmodelr?   r@   r   r   Úlm_headru   s     €r.   r;   zModel.__init__å   sN   ø€ Ü‰ÑÔØˆŒ	Ø ×+Ñ+ˆŒÜ" 6Ó*ˆŒ
Ü—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆ�r-   ÚinputsrM   c                 óH   — | j                  ||«      }| j                  |«      S rn   )r°   r±   )rH   r²   rM   r�   s       r.   r_   zModel.__call__ì   s#   € ð
 �j‰j˜ Ó'ˆØ�|‰|˜CÓ Ð r-   c                 ód  — t        | j                  j                  «      D ]ˆ  }d|› �}dD ]|  }dD ]u  }|› d|› d|› �|v sŒt        | j                  j                  «      D �cg c]  }|j	                  |› d|› d|› d|› �«      ‘Œ  }}t        j                  |«      ||› d|› d|› �<   Œw Œ~ ŒŠ |S c c}w )Nzmodel.layers.)rj   rl   rk   )rt   ÚscalesÚbiasesz.mlp.experts.0.Ú.z.mlp.experts.z.mlp.switch_mlp.)r£   r¯   r   r   Úpoprb   Ústack)rH   ÚweightsÚlÚprefixÚmr‚   ÚeÚto_joins           r.   ÚsanitizezModel.sanitizeô   sÚ   € Ü�t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFÛ:�Û7�AØ ˜ °°°1°Q°CÐ8¸GÒCô &+¨4¯9©9×+EÑ+EÔ%Fó#á%F ð $ŸK™K¨6¨(°-À¸sÀ!ÀAÀ3ÀaÈÀsÐ(KÕLØ%Fð  ð #ô GIÇhÁhÈwÓFW˜ 6 (Ð*:¸1¸#¸Q¸q¸cÐ BÒCñ 8ñ ;ð 4ð ˆùò#s   Á #B-
c                 ó.   — | j                   j                  S rn   )r°   r¤   )rH   s    r.   r¤   zModel.layers  s   € à�z‰z× Ñ Ð r-   rn   )r$   r%   r&   r   r;   rb   rc   r   r   r_   rÀ   Úpropertyr¤   rd   re   s   @r.   r­   r­   ä   sN   ø„ ðT˜yõ Tð  $ñ!à—‘ð!ð ˜‰}ó!òð ñ!ó ô!r-   r­   )Údataclassesr   Útypingr   r   r   Úmlx.coreÚcorerb   Úmlx.nnr?   Úactivationsr   r8   r	   r
   r   Úswitch_layersr   r   ÚModuler0   rg   rp   r†   r�   rŸ   r­   r,   r-   r.   Ú<module>rË      s°   ðÝ !ß &Ñ &å Ý å ß TÑ TÝ $ð ô!�ó !ó ð!ô*G#˜Ÿ	™	ô G#ôTJ�"—)‘)ô Jô&ˆb�i‰iô ô"�"—)‘)ô ô2˜2Ÿ9™9ô ô>�B—I‘Iô ô8!ˆB�I‰Iõ !r-   