Ë
    îÿæiE  ã                   ó$  — d dl Z d dlmZ d dlmZmZmZ d dlmZ	 d dl
mZ d dlmZmZmZ ddlmZ ddlmZmZmZ ddlmZ dd	lmZ e G d
„ de«      «       Z	 d d„Z	 d d„Zd!d„Zd„ Z G d„ dej>                  «      Z  G d„ dej>                  «      Z! G d„ dej>                  «      Z" G d„ dej>                  «      Z# G d„ dej>                  «      Z$ G d„ dej>                  «      Z% G d„ deej>                  «      Z& G d„ dej>                  «      Z'y)"é    N)Ú	dataclass)ÚAnyÚDictÚOptional)Úshard_inplaceÚshard_linearÚsum_gradientsé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚPipelineMixin)Ú	SwitchGLUc                   ó¦  — e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZee   ed<   dZee   ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZee   ed <   dZee   ed!<   dZee   ed"<   d#Zeed$<   d%Zeed&<   d'Zeed(<   d)Zeed*<   d+Z eed,<   dZ!e"ed-<   d.Z#e$ed/<   y)0Ú	ModelArgsÚdeepseek_v2Ú
model_typei � Ú
vocab_sizeé   Úhidden_sizei +  Úintermediate_sizei  Úmoe_intermediate_sizeé   Únum_hidden_layersé    Únum_attention_headsÚnum_key_value_headsNÚn_shared_expertsÚn_routed_expertsç      ð?Úrouted_scaling_factori   Úkv_lora_ranki   Úq_lora_ranké@   Úqk_rope_head_dimé€   Ú
v_head_dimÚqk_nope_head_dimÚgreadyÚtopk_methodÚn_groupÚ
topk_groupÚnum_experts_per_tokr
   Úmoe_layer_freqr   Úfirst_k_dense_replaceé   Úmax_position_embeddingsç�íµ ÷Æ°>Úrms_norm_epsg     ˆÃ@Ú
rope_thetaÚrope_scalingFÚattention_bias)%Ú__name__Ú
__module__Ú__qualname__r   ÚstrÚ__annotations__r   Úintr   r   r   r   r   r   r   r   r    r"   Úfloatr#   r$   r&   r(   r)   r+   r,   r-   r.   r/   r0   r2   r4   r5   r6   r   r7   Úbool© ó    ún/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/deepseek_v2.pyr   r      s1  … à#€J�Ó#Ø€J�ÓØ€K�ÓØ"Ð�sÓ"Ø!%Ð˜3Ó%ØÐ�sÓØ!Ð˜Ó!Ø!Ð˜Ó!Ø&*Ð�h˜s‘mÓ*Ø&*Ð�h˜s‘mÓ*Ø#&Ð˜5Ó&Ø€L�#ÓØ€K�ÓØÐ�cÓØ€J�ÓØÐ�cÓØ€K�ÓØ!€GˆX�c‰]Ó!Ø $€J�˜‘Ó$Ø)-Ð˜ #™Ó-Ø€N�CÓØ!"Ð˜3Ó"Ø#'Ð˜SÓ'Ø€L�%ÓØ€J�ÓØ€L�$ÓØ €N�DÔ rA   r   c                 ó’   — |t        j                  || dz  t         j                  z  z  «      z  dt        j                  |«      z  z  S )Né   )ÚmathÚlogÚpi)Únum_rotationsÚdimÚbaser2   s       rB   Úyarn_find_correction_dimrK   0   sC   € ð ”$—(‘(Ð2°mÀaÑ6GÌ$Ï'É'Ñ6QÑRÓSÑSØ	ŒD�H‰H�T‹NÑñð rA   c                 ó¼   — t        j                  t        | |||«      «      }t        j                  t        ||||«      «      }t	        |d«      t        ||dz
  «      fS )Nr   r
   )rE   ÚfloorrK   ÚceilÚmaxÚmin)Úlow_rotÚhigh_rotrI   rJ   r2   ÚlowÚhighs          rB   Úyarn_find_correction_rangerU   8   s^   € ô �*‰*Ü  ¨#¨tÐ5LÓMó€Cô �9‰9Ü  ¨3°Ð6MÓNó€Dô ˆs�A‹;œ˜D #¨¡'Ó*Ð*Ð*rA   c                 óJ   — | dk  ryd|z  t        j                  | «      z  dz   S )Nr
   r!   gš™™™™™¹?)rE   rF   )ÚscaleÚmscales     rB   Úyarn_get_mscalerY   D   s(   € Ø�‚zØØ�‰<œ$Ÿ(™( 5›/Ñ)¨CÑ/Ð/rA   c                 ó    — | |k(  r|dz  }t        j                  |t         j                  ¬«      | z
  || z
  z  }t        j                  |dd«      S )Ngü©ñÒMbP?©Údtyper   r
   )ÚmxÚarangeÚfloat32Úclip)Úmin_valÚmax_valrI   Úlinear_funcs       rB   Úyarn_linear_ramp_maskrd   J   sJ   € Ø�'ÒØ�5Ñˆä—9‘9˜S¬¯
©
Ô3°gÑ=À'ÈGÑBSÑT€KÜ�7‰7�;  1Ó%Ð%rA   c                   ó8   ‡ — e Zd Z	 	 	 	 	 	 	 	 dˆ fd„	Zdd„Zˆ xZS )ÚDeepseekV2YarnRotaryEmbeddingc
                 ó”  •— t         ‰| �  «        t        ||«      t        ||	«      z  | _        |t	        j
                  d|dt        j                  ¬«      |z  z  }
||t	        j
                  d|dt        j                  ¬«      |z  z  z  }t        |||||«      \  }}dt        |||dz  «      z
  }||
z  ||z  |
d|z
  z  z   z  | _	        y )Nr   rD   r[   r!   r
   )
ÚsuperÚ__init__rY   rX   r]   r^   r_   rU   rd   Ú_freqs)ÚselfrI   r2   rJ   Úscaling_factorÚ original_max_position_embeddingsÚ	beta_fastÚ	beta_slowrX   Úmscale_all_dimÚ
freq_extraÚ
freq_interrS   rT   Ú	freq_maskÚ	__class__s                  €rB   ri   z&DeepseekV2YarnRotaryEmbedding.__init__S   sà   ø€ ô 	‰ÑÔÜ% n°fÓ=ÄØ˜NóA
ñ 
ˆŒð œbŸi™i¨¨3°¼¿¹ÔDÀsÑJÑKˆ
Ø# dÜ�I‰I�a˜˜a¤r§z¡zÔ2°SÑ8ñ'
ñ 
ˆ
ô /ØØØØØ,ó
‰	ˆˆTð Ô/°°T¸3À!¹8ÓDÑDˆ	Ø! JÑ.Ø˜Ñ" Z°1°y±=Ñ%AÑAñ
ˆ�rA   c           	      ó¸   — | j                   dk7  r| j                   |z  }t        j                  j                  ||j                  d   dd d|| j
                  ¬«      S )Nr!   éÿÿÿÿT)ÚtraditionalrJ   rW   ÚoffsetÚfreqs)rX   r]   ÚfastÚropeÚshaperj   )rk   Úxrx   s      rB   Ú__call__z&DeepseekV2YarnRotaryEmbedding.__call__s   sV   € Ø�;‰;˜#ÒØ—‘˜a‘ˆAÜ�w‰w�|‰|ØØ�G‰G�B‰KØØØØØ—+‘+ð ó 
ð 	
rA   )r1   é'  r!   r   r   r
   r
   r   )r   )r8   r9   r:   ri   r~   Ú__classcell__©rt   s   @rB   rf   rf   R   s(   ø„ ð !%ØØØ)-ØØØØõ
÷@
rA   rf   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚDeepseekV2AttentionÚconfigc                 ól  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        |j                  | _
        |j                  | _        |j                  | _        |j                  |j                  z   | _        | j                  dz  | _        | j                  €?t        j                   | j                  | j
                  | j                  z  d¬«      | _        nŸt        j                   | j                  | j                  |j$                  ¬«      | _        t        j(                  | j                  d¬«      | _        t        j                   | j                  | j
                  | j                  z  d¬«      | _        t        j                   | j                  | j                  | j                  z   |j$                  ¬«      | _        t        j(                  | j                  d¬«      | _        t        j                   | j                  | j
                  | j                  | j                  z
  | j                  z   z  d¬«      | _        t        j                   | j
                  | j                  z  | j                  |j$                  ¬«      | _        | j                  j6                  j9                  dd«      }| j                  j6                  d   }|r#t;        ||«      }| j                  |z  |z  | _        d	D �ci c]4  }|| j                  j6                  v r|| j                  j6                  |   “Œ6 }}t=        d| j                  | j                  || j                  d
œ|¤Ž| _        y c c}w )Ng      à¿F©Úbiasr3   ©Úepsrp   r   Úfactor)rm   rn   ro   rX   rp   )rI   r2   rl   rJ   r@   ) rh   ri   r„   r   r   Ú	num_headsr2   r5   r$   r&   r#   r(   r)   Ú
q_head_dimrW   ÚnnÚLinearÚq_projr7   Úq_a_projÚRMSNormÚq_a_layernormÚq_b_projÚkv_a_proj_with_mqaÚkv_a_layernormÚ	kv_b_projÚo_projr6   ÚgetrY   rf   r{   )rk   r„   rp   rl   rX   ÚkeyÚrope_kwargsrt   s          €rB   ri   zDeepseekV2Attention.__init__‚   s  ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ×3Ñ3ˆŒØ'-×'EÑ'EˆÔ$Ø ×+Ñ+ˆŒØ!×-Ñ-ˆÔØ &× 7Ñ 7ˆÔØ"×/Ñ/ˆÔØ ×+Ñ+ˆŒØ &× 7Ñ 7ˆÔØ ×1Ñ1°F×4KÑ4KÑKˆŒà—_‘_ dÑ*ˆŒ
à×ÑÐ#ÜŸ)™)Ø× Ñ  $§.¡.°4·?±?Ñ"BÈôˆD�Kô ŸI™IØ× Ñ  $×"2Ñ"2¸×9NÑ9NôˆDŒMô "$§¡¨D×,<Ñ,<À$Ô!GˆDÔÜŸI™IØ× Ñ  $§.¡.°4·?±?Ñ"BÈôˆDŒMô #%§)¡)Ø×ÑØ×Ñ × 5Ñ 5Ñ5Ø×&Ñ&ô#
ˆÔô
 !Ÿj™j¨×):Ñ):ÀÔEˆÔÜŸ™Ø×ÑØ�N‰NØ�‰ ×!6Ñ!6Ñ6¸¿¹ÑHñJàô	
ˆŒô —i‘iØ�N‰N˜TŸ_™_Ñ,Ø×ÑØ×&Ñ&ô
ˆŒð Ÿ™×1Ñ1×5Ñ5Ð6FÈÓJˆØŸ™×1Ñ1°(Ñ;ˆÙÜ$ ^°^ÓDˆFØŸ™ fÑ,¨vÑ5ˆDŒJñó

ñ�ð �d—k‘k×.Ñ.Ñ.ð �—‘×)Ñ)¨#Ñ.Ñ.ðð 	ð 

ô 2ð 
Ø×%Ñ%Ø$(×$@Ñ$@Ø)Ø—‘ñ	
ð
 ñ
ˆ�	ùò

s   Í9N1r}   ÚmaskÚcacheÚreturnc                 óÖ  — |j                   \  }}}| j                  €| j                  |«      }n/| j                  | j	                  | j                  |«      «      «      }|j                  ||| j                  | j                  «      j                  dddd«      }t        j                  || j                  gd¬«      \  }}	| j                  |«      }
t        j                  |
| j                  gd¬«      \  }
}|j                  ||d| j                  «      j                  dddd«      }| j!                  | j#                  |
«      «      }|j                  ||| j                  d«      j                  dddd«      }t        j                  || j                  gd¬«      \  }}|�‡| j%                  |	|j&                  «      }	| j%                  ||j&                  «      }t        j(                  || j                  d¬«      }|j+                  t        j,                  ||gd¬«      |«      \  }}n]| j%                  |	«      }	| j%                  |«      }t        j(                  || j                  d¬«      }t        j,                  ||gd¬«      }t        j,                  ||	gd¬«      }t/        ||||| j0                  |¬«      }|j                  dddd«      j                  ||d«      }| j3                  |«      S )Nr   rD   r
   é   rv   ©Úaxis)rœ   rW   r›   )r|   r$   r�   r“   r’   r�   Úreshaper‹   rŒ   Ú	transposer]   Úsplitr)   r”   r#   r&   r–   r•   r{   rx   ÚrepeatÚupdate_and_fetchÚconcatenater   rW   r—   )rk   r}   r›   rœ   ÚBÚLÚDÚqÚq_nopeÚq_peÚcompressed_kvÚk_peÚkvÚk_nopeÚvaluesÚkeysÚqueriesÚoutputs                     rB   r~   zDeepseekV2Attention.__call__Ë   so  € ð —'‘'‰ˆˆ1ˆaà×ÑÐ#Ø—‘˜A“‰Aà—‘˜d×0Ñ0°·±¸qÓ1AÓBÓCˆAà�I‰I�a˜˜DŸN™N¨D¯O©OÓ<×FÑFÀqÈ!ÈQÐPQÓRˆÜ—x‘x  D×$9Ñ$9Ð#:ÀÔD‰ˆ�Ø×/Ñ/°Ó2ˆÜ Ÿh™h }°t×7HÑ7HÐ6IÐPRÔSÑˆ�tØ�|‰|˜A˜q ! T×%:Ñ%:Ó;×EÑEÀaÈÈAÈqÓQˆØ�^‰^˜D×/Ñ/°Ó>Ó?ˆØ�Z‰Z˜˜1˜dŸn™n¨bÓ1×;Ñ;¸A¸qÀ!ÀQÓGˆäŸ™ " t×'<Ñ'<Ð&=ÀBÔG‰ˆ�àÐØ—9‘9˜T 5§<¡<Ó0ˆDØ—9‘9˜T 5§<¡<Ó0ˆDÜ—9‘9˜T 4§>¡>¸Ô:ˆDØ ×1Ñ1Ü—‘ ¨˜~°BÔ7¸ó‰LˆD‘&ð —9‘9˜T“?ˆDØ—9‘9˜T“?ˆDÜ—9‘9˜T 4§>¡>¸Ô:ˆDÜ—>‘> 6¨4 .°rÔ:ˆDä—.‘. &¨$ °bÔ9ˆä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"rA   ©NN©r8   r9   r:   r   ri   r]   Úarrayr   r   r~   r€   r�   s   @rB   rƒ   rƒ   �   sZ   ø„ ðG
˜yõ G
ðX $(Ø#ñ	*#à�8‰8ð*#ð �r—x‘xÑ ð*#ð ˜‰}ð	*#ð
 
�‰÷*#rA   rƒ   c                   ó6   ‡ — e Zd Z	 ddededefˆ fd„Zd„ Zˆ xZS )ÚDeepseekV2MLPr„   r   r   c                 óª  •— t         ‰| �  «        || _        |€|j                  n|| _        |€|j                  n|| _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _	        y ©NFr†   )
rh   ri   r„   r   r   r�   rŽ   Ú	gate_projÚup_projÚ	down_proj)rk   r„   r   r   rt   s       €rB   ri   zDeepseekV2MLP.__init__ù   sª   ø€ ô 	‰ÑÔØˆŒØ1<Ð1D˜6×-Ò-È+ˆÔà(9Ð(AˆF×$Ò$ÐGXð 	Ôô Ÿ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆ�rA   c                 óx   — | j                  t        | j                  |«      | j                  |«      «      «      }|S ©N)r¿   r   r½   r¾   )rk   r}   r¿   s      rB   r~   zDeepseekV2MLP.__call__  s.   € Ø—N‘N¤6¨$¯.©.¸Ó*;¸T¿\¹\È!»_Ó#MÓNˆ	ØÐrA   r¶   )r8   r9   r:   r   r=   ri   r~   r€   r�   s   @rB   rº   rº   ø   s,   ø„ àSWñYØðYØ.1ðYØMPõYörA   rº   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚMoEGater„   c                 ó\  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  | j
                  |j                  f«      | _        y rÁ   )rh   ri   r„   r.   Útop_kr    r"   r+   r,   r-   r]   Úzerosr   Úweight©rk   r„   rt   s     €rB   ri   zMoEGate.__init__  s…   ø€ Ü‰ÑÔØˆŒØ×/Ñ/ˆŒ
Ø &× 7Ñ 7ˆÔØ%+×%AÑ%AˆÔ"Ø!×-Ñ-ˆÔØ—~‘~ˆŒØ ×+Ñ+ˆŒÜ—h‘h × 5Ñ 5°v×7IÑ7IÐJÓKˆ�rA   c                 óÌ  — || j                   j                  z  }t        j                  |dd¬«      }| j                  dk(  rË|j
                  d d \  }}|j                  ||| j                  d«      }|j                  dd¬«      }| j                  | j                  z
  }t        j                  ||dz
  d¬	«      d
d |…d d …f   }t        j                  ||t        j                  d|j                  «      d¬«      }|j                  ||d«      }| j                  }t        j                  | |dz
  d¬	«      d
d |…f   }	t        j                  ||	d¬«      }|| j                   z  }|	|fS )Nrv   T)r¡   ÚpreciseÚgroup_limited_greedyrD   )r¡   Úkeepdimsr
   éþÿÿÿ)Úkthr¡   .g        r    )rÇ   ÚTr]   Úsoftmaxr+   r|   r¢   r,   rO   r-   ÚargpartitionÚput_along_axisr¸   r\   rÅ   Útake_along_axisr"   )
rk   r}   ÚgatesÚscoresÚbszÚseq_lenÚgroup_scoresÚkÚ	group_idxÚindss
             rB   r~   zMoEGate.__call__  sC  € Ø�D—K‘K—M‘MÑ!ˆä—‘˜E¨°DÔ9ˆà×ÑÐ5Ò5ØŸ7™7 2 A˜;‰LˆC�Ø—^‘^ C¨°$·,±,ÀÓCˆFØ!Ÿ:™:¨2¸˜:Ó=ˆLØ—‘˜tŸ™Ñ.ˆAÜŸ™¨¸!¸a¹%ÀbÔIÈ#ÈrÐPQÈrÒSTÈ*ÑUˆIÜ×&Ñ&Ø˜	¤2§8¡8¨C°·±Ó#>ÀRôˆFð —^‘^ C¨°"Ó5ˆFà�J‰JˆÜ�‰ ˜w¨A°©E¸Ô;¸CÀÀ!À¸GÑDˆÜ×#Ñ# F¨D°rÔ:ˆØ˜$×4Ñ4Ñ4ˆà�Vˆ|ÐrA   ©r8   r9   r:   r   ri   r~   r€   r�   s   @rB   rÃ   rÃ     s   ø„ ð	L˜yõ 	LörA   rÃ   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚDeepseekV2MoEr„   c                 óN  •— t         ‰| �  «        || _        |j                  | _        t	        |j
                  |j                  |j                  «      | _        t        |«      | _
        |j                  �+|j                  |j                  z  }t        ||¬«      | _        d | _        y )N)r„   r   )rh   ri   r„   r.   r   r   r   r    Ú
switch_mlprÃ   Úgater   rº   Úshared_expertsÚsharding_group)rk   r„   r   rt   s      €rB   ri   zDeepseekV2MoE.__init__1  s•   ø€ Ü‰ÑÔØˆŒØ#)×#=Ñ#=ˆÔ Ü#Ø×Ñ × <Ñ <¸f×>UÑ>Uó
ˆŒô ˜F“OˆŒ	Ø×"Ñ"Ð.Ø &× <Ñ <¸v×?VÑ?VÑ VÐÜ"/ØÐ1Bô#ˆDÔð #ˆÕrA   c                 ó’  — | j                   � t        | j                   «      |«      }| j                  |«      \  }}| j                  ||«      }||d   z  j	                  d¬«      }| j
                  j                  �|| j                  |«      z   }| j                   �+t        j                  j                  || j                   ¬«      }|S )N).NrÍ   r    ©Úgroup)rã   r	   rá   rà   Úsumr„   r   râ   r]   ÚdistributedÚall_sum)rk   r}   rÛ   rÕ   Úys        rB   r~   zDeepseekV2MoE.__call__B  s¸   € Ø×ÑÐ*Ø2”˜d×1Ñ1Ó2°1Ó5ˆAà—y‘y “|‰ˆˆfØ�O‰O˜A˜tÓ$ˆØ�˜	Ñ"Ñ"×'Ñ'¨RÐ'Ó0ˆØ�;‰;×'Ñ'Ð3Ø�D×'Ñ'¨Ó*Ñ*ˆAà×ÑÐ*Ü—‘×&Ñ& q°×0CÑ0CÐ&ÓDˆAàˆrA   rÜ   r�   s   @rB   rÞ   rÞ   0  s   ø„ ð#˜yõ #ö"rA   rÞ   c            	       óŽ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
ÚDeepseekV2DecoderLayerr„   Ú	layer_idxc                 ó’  •— t         ‰| �  «        t        |«      | _        |j                  �,||j
                  k\  r||j                  z  dk(  rt        |«      n
t        |«      | _	        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _        y )Nr   rˆ   )rh   ri   rƒ   Ú	self_attnr    r0   r/   rÞ   rº   Úmlpr�   r‘   r   r4   Úinput_layernormÚpost_attention_layernorm)rk   r„   rí   rt   s      €rB   ri   zDeepseekV2DecoderLayer.__init__S  s§   ø€ Ü‰ÑÔÜ,¨VÓ4ˆŒð ×'Ñ'Ð3Ø ×!=Ñ!=Ò=Ø × 5Ñ 5Ñ5¸Ò:ô	 ˜&Ô!ô ˜vÓ&ð 	Œô  "Ÿz™z¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(*¯
©
Ø×Ñ F×$7Ñ$7ô)
ˆÕ%rA   r}   r›   rœ   r�   c                 óž   — | j                  | j                  |«      ||«      }||z   }| j                  | j                  |«      «      }||z   }|S rÁ   )rï   rñ   rð   rò   )rk   r}   r›   rœ   ÚrÚhÚouts          rB   r~   zDeepseekV2DecoderLayer.__call__d  sQ   € ð �N‰N˜4×/Ñ/°Ó2°D¸%Ó@ˆØ�‰EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�!‰eˆØˆ
rA   r¶   )r8   r9   r:   r   r=   ri   r]   r¸   r   r   r~   r€   r�   s   @rB   rì   rì   R  s^   ø„ ð
˜yð 
°Sõ 
ð( $(Ø#ñ	
à�8‰8ð
ð �r—x‘xÑ ð
ð ˜‰}ð	
ð
 
�‰÷
rA   rì   c                   ój   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dee   dej                  fd„Z	ˆ xZ
S )ÚDeepseekV2Modelr„   c                 óp  •— t         ‰| �  «        |j                  | _        t        j                  |j                  |j
                  «      | _        t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}| _
        t        j                  |j
                  |j                  ¬«      | _        y c c}w )Nrˆ   )rh   ri   r   r�   Ú	Embeddingr   Úembed_tokensÚranger   rì   Úlayersr‘   r4   Únorm)rk   r„   Úidxrt   s      €rB   ri   zDeepseekV2Model.__init__r  s’   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜŸL™L¨×):Ñ):¸F×<NÑ<NÓOˆÔô ˜V×5Ñ5Ô6ó
á6�ô # 6¨3Õ/Ø6ñ
ˆŒô —J‘J˜v×1Ñ1°v×7JÑ7JÔKˆ�	ùò	
s   Á'B3r}   rœ   r�   c                 ó�  — | j                  |«      }| j                  }| j                  }|€d gt        | j                  «      z  }t        ||d   «      }||dz
  k  r#t        j                  j                  ||dz   «      }t        | j                  |«      D ]  \  }} ||||¬«      }Œ |dk7  rVt        j                  j                  ||dz
  |z  «      }|d   �+t        j                  |d   j                  |«      |d   _        |dkD  r/t        j                  j                  |«      d |j                  d    }| j                  |«      S )Nr   r
   )rœ   rv   )rû   Úpipeline_rankÚpipeline_sizeÚlenÚpipeline_layersr   r]   rè   Ú	recv_likeÚzipÚsendÚdependsr³   Ú
all_gatherr|   rþ   )	rk   r}   rœ   rõ   r  r  r›   ÚlÚcs	            rB   r~   zDeepseekV2Model.__call__|  s6  € ð
 ×Ñ˜aÓ ˆà×*Ñ*ˆØ×*Ñ*ˆàˆ=Ø�FœS ×!5Ñ!5Ó6Ñ6ˆEÜ$ Q¨¨a©Ó1ˆð ˜=¨1Ñ,Ò,Ü—‘×(Ñ(¨¨]¸QÑ->Ó@ˆAä˜×,Ñ,¨eÖ4‰DˆAˆqÙ�!�T Ô#‰Að 5ð ˜AÒÜ—‘×#Ñ# A¨¸Ñ(9¸]Ñ'JÓKˆAØ�R‰yÐ$Ü!#§¡¨E°"©I¯N©N¸AÓ!>��b‘	”ð ˜1ÒÜ—‘×)Ñ)¨!Ó,¨\¨q¯w©w°q©zÐ:ˆAà�y‰y˜‹|ÐrA   rÁ   r·   r�   s   @rB   rø   rø   q  sC   ø„ ðL˜yõ Lð  $ñà�8‰8ðð ˜‰}ðð 
�‰÷	rA   rø   c                   ó¤   ‡ — e Zd Zdefˆ fd„Z	 d
dej                  dee   fd„Z	d„ Z
d
deej                  j                     fd„Zed	„ «       Zˆ xZS )ÚModelr„   c                 óÔ   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        t        j                  |j                  |j                  d¬«      | _
        y r¼   )rh   ri   Úargsr   rø   Úmodelr�   rŽ   r   r   Úlm_headrÈ   s     €rB   ri   zModel.__init__Ÿ  sN   ø€ Ü‰ÑÔØˆŒ	Ø ×+Ñ+ˆŒÜ$ VÓ,ˆŒ
Ü—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆ�rA   Úinputsrœ   c                 óH   — | j                  ||«      }| j                  |«      S rÁ   )r  r  )rk   r  rœ   rö   s       rB   r~   zModel.__call__¦  s#   € ð
 �j‰j˜ Ó'ˆØ�|‰|˜CÓ Ð rA   c                 ój  — t        | j                  j                  «      D ]‹  }d|› �}dD ]  \  }}dD ]u  }|› d|› d|› �|v sŒt        | j                  j                  «      D �cg c]  }|j	                  |› d|› d|› d|› �«      ‘Œ  }}t        j                  |«      ||› d|› d|› �<   Œw Œ� Œ� |S c c}w )Nzmodel.layers.))Úw1r½   )Úw2r¿   )Úw3r¾   )rÇ   ÚscalesÚbiasesz.mlp.experts.0.Ú.z.mlp.experts.z.mlp.switch_mlp.)rü   r  r   r    Úpopr]   Ústack)	rk   Úweightsr
  ÚprefixÚnÚmrÙ   ÚeÚto_joins	            rB   ÚsanitizezModel.sanitize®  sß   € Ü�t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFÛU‘��1Û7�AØ ˜ °°°1°Q°CÐ8¸GÒCô &+¨4¯9©9×+EÑ+EÔ%Fó#á%F ð $ŸK™K¨6¨(°-À¸sÀ!ÀAÀ3ÀaÈÀsÐ(KÕLØ%Fð  ð #ô GIÇhÁhÈwÓFW˜ 6 (Ð*:¸1¸#¸Q¸q¸cÐ BÒCñ 8ñ Vð 4ð ˆùò#s   Á##B0
ræ   c                 ó&  — |xs t         j                  j                  «       }|j                  «       }| j                  j
                  D �]Å  }|j                  j                  €2t        |j                  j                  d|¬«      |j                  _	        n1t        |j                  j                  d|¬«      |j                  _
        t        |j                  j                  d|¬«      |j                  _        t        |j                  j                  d|¬«      |j                  _        |j                  xj                  |z  c_        t        |j                  t         «      r•t        |j                  j"                  d|¬«      |j                  _        t        |j                  j$                  d|¬«      |j                  _        t        |j                  j&                  d|¬«      |j                  _        �Œ­||j                  _        t+        |j                  j,                  j"                  d|¬«       t+        |j                  j,                  j$                  d|¬«       t+        |j                  j,                  j&                  d|¬«       t+        |j                  j.                  j"                  d|¬«       t+        |j                  j.                  j$                  d|¬«       t+        |j                  j.                  j&                  d|¬«       �ŒÈ y )Nzall-to-shardedrå   zsharded-to-all)r]   rè   ÚinitÚsizer  rý   rï   r$   r   r�   r“   r–   r—   r‹   Ú
isinstancerð   rº   r½   r¿   r¾   rã   r   râ   rà   )rk   ræ   ÚNÚlayers       rB   ÚshardzModel.shard»  sI  € ØÒ.œŸ™×,Ñ,Ó.ˆØ�J‰J‹LˆØ—Z‘Z×&Õ&ˆEà�‰×*Ñ*Ð2Ü)5Ø—O‘O×*Ñ*Ð,<ÀEô*�—‘Õ&ô ,8Ø—O‘O×,Ñ,Ð.>Àeô,�—‘Ô(ô )5Ø—‘×)Ñ)Ð+;À5ô)ˆE�O‰OÔ%ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ð �O‰O×%Ò%¨!Ñ+Õ%ô ˜%Ÿ)™)¤]Ô3Ü&2Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	Ô#ô '3Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	Ô#ô %1Ø—I‘I×%Ñ%Ð'7¸uô%�—	‘	Ö!ð ,1�—	‘	Ô(ÜØ—I‘I×,Ñ,×6Ñ6Ð8HÐPUõô Ø—I‘I×,Ñ,×6Ñ6Ð8HÐPUõô Ø—I‘I×,Ñ,×4Ñ4Ð6FÈeõô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈEõô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈEõô Ø—I‘I×(Ñ(×0Ñ0Ð2BÈ%÷ñc 'rA   c                 ó.   — | j                   j                  S rÁ   )r  r  )rk   s    rB   rý   zModel.layersó  s   € à�z‰z×)Ñ)Ð)rA   rÁ   )r8   r9   r:   r   ri   r]   r¸   r   r   r~   r#  rè   ÚGroupr*  Úpropertyrý   r€   r�   s   @rB   r  r  ž  sj   ø„ ðT˜yõ Tð  $ñ!à—‘ð!ð ˜‰}ó!òñ6˜8 B§N¡N×$8Ñ$8Ñ9ó 6ðp ñ*ó ô*rA   r  )r   r1   )r
   r
   )(rE   Údataclassesr   Útypingr   r   r   Úmlx.coreÚcorer]   Úmlx.nnr�   Úmlx.nn.layers.distributedr   r   r	   Úactivationsr   rJ   r   r   r   Úpipeliner   Úswitch_layersr   r   rK   rU   rY   rd   ÚModulerf   rƒ   rº   rÃ   rÞ   rì   rø   r  r@   rA   rB   Ú<module>r8     sõ   ðó Ý !ß &Ñ &å Ý ß PÑ På ß TÑ TÝ #Ý $ð ô!�ó !ó ð!ð> =Aóð AEó	+ó0ò&ô,
 B§I¡Iô ,
ô^t#˜"Ÿ)™)ô t#ôn�B—I‘Iô ô(!ˆb�i‰iô !ôH�B—I‘Iô ôD˜RŸY™Yô ô>*�m R§Y¡Yô *ôZW*ˆB�I‰Iõ W*rA   