Ë
    îÿæi9O  ã                   ó   — d dl Z d dlmZ d dlmZ d dlmZmZmZ d dl	m
Z d dlmZ d dlmZmZmZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ e G d„ de«      «       Z  G d„ dejB                  «      Z" G d„ dejB                  «      Z#ejH                  d„ «       Z% G d„ dejB                  «      Z& G d„ dejB                  «      Z' G d„ dejB                  «      Z( G d„ deejB                  «      Z) G d„ dejB                  «      Z*y)é    N)Ú	dataclass)Úpartial)ÚAnyÚDictÚOptional)Úshard_inplaceÚshard_linearÚsum_gradientsé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚMultiLinear)ÚPipelineMixin)Úinitialize_rope)Ú	SwitchGLUc                   ó°  — e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZee   ed<   dZee   ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   d Zeed!<   d"Zeed#<   d$Zeed%<   d$Zeed&<   d$Zeed'<   d$Zeed(<   d)Z eed*<   d+Z!eed,<   d-Z"eed.<   d/Z#eed0<   dZ$e%ed1<   d2Z&eed3<   y)4Ú	ModelArgsÚdeepseek_v3Ú
model_typei � Ú
vocab_sizei   Úhidden_sizei +  Úintermediate_sizei  Úmoe_intermediate_sizeé   Únum_hidden_layersé    Únum_attention_headsÚnum_key_value_headsNÚn_shared_expertsÚn_routed_expertsç      ð?Úrouted_scaling_factori   Úkv_lora_ranki   Úq_lora_ranké@   Úqk_rope_head_dimé€   Ú
v_head_dimÚqk_nope_head_dimÚnoaux_tcÚtopk_methodÚsigmoidÚscoring_funcTÚnorm_topk_probr   Ún_groupÚ
topk_groupÚnum_experts_per_tokÚmoe_layer_freqr   Úfirst_k_dense_replacei   Úmax_position_embeddingsç�íµ ÷Æ°>Úrms_norm_epsg     ˆÃ@Ú
rope_thetaÚrope_scalingFÚattention_bias)'Ú__name__Ú
__module__Ú__qualname__r   ÚstrÚ__annotations__r   Úintr   r   r   r   r   r    r!   r   r"   r$   Úfloatr%   r&   r(   r*   r+   r-   r/   r0   Úboolr1   r2   r3   r4   r5   r6   r8   r9   r:   r   r;   © ó    ún/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/deepseek_v3.pyr   r      s9  … à#€J�Ó#Ø€J�ÓØ€K�ÓØ"Ð�sÓ"Ø!%Ð˜3Ó%ØÐ�sÓØ!Ð˜Ó!Ø!Ð˜Ó!Ø&*Ð�h˜s‘mÓ*Ø&*Ð�h˜s‘mÓ*Ø#&Ð˜5Ó&Ø€L�#ÓØ€K�ÓØÐ�cÓØ€J�ÓØÐ�cÓØ!€K�Ó!Ø!€L�#Ó!Ø€N�DÓØ€GˆSÓØ€J�ÓØ Ð˜Ó Ø€N�CÓØ!"Ð˜3Ó"Ø#'Ð˜SÓ'Ø€L�%ÓØ€J�ÓØ€L�$ÓØ €N�DÔ rE   r   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚDeepseekV3AttentionÚconfigc                 óv  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        |j                  | _
        |j                  | _        |j                  | _        |j                  |j                  z   | _        | j                  dz  | _        | j                  €?t        j                   | j                  | j
                  | j                  z  d¬«      | _        nŸt        j                   | j                  | j                  |j$                  ¬«      | _        t        j(                  | j                  d¬«      | _        t        j                   | j                  | j
                  | j                  z  d¬«      | _        t        j                   | j                  | j                  | j                  z   |j$                  ¬«      | _        t        j(                  | j                  d¬«      | _        t3        | j                  | j                  | j
                  «      | _        t3        | j                  | j                  | j
                  «      | _        t        j                   | j
                  | j                  z  | j                  |j$                  ¬«      | _        | j                  j:                  �{| j                  j:                  j=                  dd«      }|rS| j                  j:                  d   }|d	kD  r5d
|z  t?        j@                  |«      z  dz   }| j                  |z  |z  | _        tC        | j                  | j                  d| j                  | j                  j:                  ¬«      | _"        y )Ng      à¿F©Úbiasr7   ©ÚepsÚmscale_all_dimr   Úfactorr   gš™™™™™¹?r#   T)ÚdimsÚbaseÚtraditionalr6   Úscaling_config)#ÚsuperÚ__init__rI   r   r   Ú	num_headsr6   r9   r&   r(   r%   r*   r+   Ú
q_head_dimÚscaleÚnnÚLinearÚq_projr;   Úq_a_projÚRMSNormÚq_a_layernormÚq_b_projÚkv_a_proj_with_mqaÚkv_a_layernormr   Úembed_qÚunembed_outÚo_projr:   ÚgetÚmathÚlogr   Úrope)ÚselfrI   rO   Úscaling_factorÚsÚ	__class__s        €rF   rV   zDeepseekV3Attention.__init__6   sá  ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ×3Ñ3ˆŒØ'-×'EÑ'EˆÔ$Ø ×+Ñ+ˆŒØ!×-Ñ-ˆÔØ &× 7Ñ 7ˆÔØ"×/Ñ/ˆÔØ ×+Ñ+ˆŒØ &× 7Ñ 7ˆÔØ ×1Ñ1°F×4KÑ4KÑKˆŒà—_‘_ dÑ*ˆŒ
à×ÑÐ#ÜŸ)™)Ø× Ñ  $§.¡.°4·?±?Ñ"BÈôˆD�Kô ŸI™IØ× Ñ  $×"2Ñ"2¸×9NÑ9NôˆDŒMô "$§¡¨D×,<Ñ,<À$Ô!GˆDÔÜŸI™IØ× Ñ  $§.¡.°4·?±?Ñ"BÈôˆDŒMô #%§)¡)Ø×ÑØ×Ñ × 5Ñ 5Ñ5Ø×&Ñ&ô#
ˆÔô
 !Ÿj™j¨×):Ñ):ÀÔEˆÔÜ"Ø×!Ñ! 4×#4Ñ#4°d·n±nó
ˆŒô 'Ø×Ñ˜tŸ™°·±ó
ˆÔô —i‘iØ�N‰N˜TŸ_™_Ñ,Ø×ÑØ×&Ñ&ô
ˆŒð �;‰;×#Ñ#Ð/Ø!Ÿ[™[×5Ñ5×9Ñ9Ð:JÈAÓNˆNÙØ!%§¡×!9Ñ!9¸(Ñ!C�Ø! AÒ%Ø˜nÑ,¬t¯x©x¸Ó/GÑGÈ#ÑM�AØ!%§¡¨a¡°!Ñ!3�D”Jä#Ø×&Ñ&Ø—‘ØØ$(×$@Ñ$@ØŸ;™;×3Ñ3ô
ˆ�	rE   ÚxÚmaskÚcacheÚreturnc           	      ó|  — |j                   \  }}}| j                  €| j                  |«      }n/| j                  | j	                  | j                  |«      «      «      }|j                  ||| j                  | j                  «      j                  dddd«      }t        j                  || j                  gd¬«      \  }}	| j                  |«      }
t        j                  |
| j                  gd¬«      \  }
}|j                  ||d| j                  «      j                  dddd«      }| j!                  |
«      }|�|j"                  nd}| j%                  |	|«      }	| j%                  ||«      }t        j&                  |d¬«      }|�|j)                  ||«      \  }}|	| j*                  z  |j-                  dd«      z  }|�\t        j.                  ||t        j0                  t        j2                  |j4                  «      j6                  |j4                  «      «      }|dk(  r| j9                  |«      }|x}}n$| j9                  |d¬	«      }| j;                  |«      }t=        ||||| j*                  |¬
«      }|dk(  r| j;                  |«      }|j                  dddd«      j                  ||d«      }| j?                  |«      S )Nr   é   r   é   éÿÿÿÿ©ÚaxiséþÿÿÿF)Ú	transpose)rp   rY   ro   ) Úshaper&   r\   r`   r_   r]   ÚreshaperW   rX   ry   ÚmxÚsplitr+   ra   r%   r(   rb   Úoffsetri   Úexpand_dimsÚupdate_and_fetchrY   ÚswapaxesÚwhereÚarrayÚfinfoÚdtypeÚminrc   rd   r   re   )rj   rn   ro   rp   ÚBÚLÚDÚqÚq_nopeÚq_peÚcompressed_kvÚk_peÚ	kv_latentr~   Ú	pe_scoresÚkÚvÚoutputs                     rF   Ú__call__zDeepseekV3Attention.__call__v   sk  € ð —'‘'‰ˆˆ1ˆaà×ÑÐ#Ø—‘˜A“‰Aà—‘˜d×0Ñ0°·±¸qÓ1AÓBÓCˆAà�I‰I�a˜˜DŸN™N¨D¯O©OÓ<×FÑFÀqÈ!ÈQÐPQÓRˆÜ—x‘x  D×$9Ñ$9Ð#:ÀÔD‰ˆ�Ø×/Ñ/°Ó2ˆÜ Ÿh™h }°t×7HÑ7HÐ6IÐPRÔSÑˆ�tØ�|‰|˜A˜q ! T×%:Ñ%:Ó;×EÑEÀaÈÈAÈqÓQˆØ×'Ñ'¨Ó6ˆ	à!&Ð!2�—’¸ˆØ�y‰y˜˜vÓ&ˆØ�y‰y˜˜vÓ&ˆä—N‘N 9°1Ô5ˆ	àÐØ#×4Ñ4°YÀÓE‰OˆI�tà˜DŸJ™JÑ&¨$¯-©-¸¸BÓ*?Ñ?ˆ	ØÐÜŸ™ØØÜ—‘œŸ™ )§/¡/Ó2×6Ñ6¸	¿¹ÓHóˆIð �Š6Ø—\‘\ &Ó)ˆFØÐˆA‘à—‘˜Y°%�Ó8ˆAØ× Ñ  Ó+ˆAä-Ø�A�q ¨T¯Z©Z¸iô
ˆð �Š6Ø×%Ñ% fÓ-ˆFà×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"rE   ©NN©r<   r=   r>   r   rV   r|   rƒ   r   r   r”   Ú__classcell__©rm   s   @rF   rH   rH   5   sX   ø„ ð>
˜yõ >
ðF $(Ø#ñ	3#à�8‰8ð3#ð �r—x‘xÑ ð3#ð ˜‰}ð	3#ð
 
�‰÷3#rE   rH   c                   ó6   ‡ — e Zd Z	 ddededefˆ fd„Zd„ Zˆ xZS )ÚDeepseekV3MLPrI   r   r   c                 óª  •— t         ‰| �  «        || _        |€|j                  n|| _        |€|j                  n|| _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _	        y ©NFrK   )
rU   rV   rI   r   r   rZ   r[   Ú	gate_projÚup_projÚ	down_proj)rj   rI   r   r   rm   s       €rF   rV   zDeepseekV3MLP.__init__­   sª   ø€ ô 	‰ÑÔØˆŒØ1<Ð1D˜6×-Ò-È+ˆÔà(9Ð(AˆF×$Ò$ÐGXð 	Ôô Ÿ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆ�rE   c                 óx   — | j                  t        | j                  |«      | j                  |«      «      «      }|S ©N)rŸ   r   r�   rž   )rj   rn   rŸ   s      rF   r”   zDeepseekV3MLP.__call__»   s.   € Ø—N‘N¤6¨$¯.©.¸Ó*;¸T¿\¹\È!»_Ó#MÓNˆ	ØÐrE   r•   )r<   r=   r>   r   rA   rV   r”   r—   r˜   s   @rF   rš   rš   ¬   s,   ø„ àSWñYØðYØ.1ðYØMPõYörE   rš   c                 óÎ  — t        j                  | j                  t         j                  «      «      }|}||z   }|dkD  rÃt        j                  |d|df¬«      }t        j
                  |dd¬«      j                  dd¬«      }	||z
  }
t        j                  |	|
dz
  d¬	«      d
d |
…d d …f   }t        j                  |t        j                  |«      t        j                  d«      d¬«      }t        j                  |dd«      }|}
t        j                  | |
dz
  d¬	«      d
d |
…f   }t        j                  ||d¬«      }|dkD  r|r|j                  dd¬«      }||z  }||z  }||fS )Nr   ru   )rw   rz   rs   rv   T)rw   Úkeepdimsrx   )Úkthrw   .g        )r|   r.   ÚastypeÚfloat32Ú	unflattenÚtopkÚsumÚargpartitionÚput_along_axisÚstop_gradientrƒ   ÚflattenÚtake_along_axis)ÚgatesÚe_score_correction_biasÚtop_kr1   r2   r$   r0   ÚscoresÚorig_scoresÚgroup_scoresr‘   Ú	group_idxÚindsÚdenominators                 rF   Úgroup_expert_selectr¸   À   sU  € ô �Z‰Z˜Ÿ™¤R§Z¡ZÓ0Ó1€FØ€KØÐ-Ñ-€FØ�‚{Ü—‘˜f¨2°g¸r°]ÔCˆÜ—w‘w˜v q¨rÔ2×6Ñ6¸BÈÐ6ÓNˆØ�jÑ ˆÜ—O‘O L°a¸!±eÀ"ÔEÀcÈ2ÈAÈ2ÊqÀjÑQˆ	Ü×"Ñ"Ø”B×$Ñ$ YÓ/´·±¸#³ÀRô
ˆô —‘˜F B¨Ó+ˆà€AÜ�?‰?˜F˜7¨¨A©°BÔ7¸¸R¸a¸R¸Ñ@€DÜ×Ñ ¨T¸Ô;€FØˆq‚y‘^Ø—j‘j b°4�jÓ8ˆØ˜+Ñ%ˆØÐ+Ñ+€Fà�ˆ<ÐrE   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚMoEGaterI   c                 óÒ  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  | j                  |j                  f«      | _        t        j                  | j                  f«      | _        |j                  dk(  sJ d«       ‚y )Nr,   zUnsupported topk method.)rU   rV   rI   r3   r±   r0   r"   r$   r1   r2   r|   Úzerosr   Úweightr°   r-   ©rj   rI   rm   s     €rF   rV   zMoEGate.__init__ä   s¹   ø€ Ü‰ÑÔØˆŒØ×/Ñ/ˆŒ
Ø$×3Ñ3ˆÔØ &× 7Ñ 7ˆÔØ%+×%AÑ%AˆÔ"Ø—~‘~ˆŒØ ×+Ñ+ˆŒÜ—h‘h × 5Ñ 5°v×7IÑ7IÐJÓKˆŒÜ')§x¡x°×1FÑ1FÐ0HÓ'IˆÔ$Ø×!Ñ! ZÒ/ÐKÐ1KÓKÑ/rE   c           	      óÊ   — t        || j                  j                  z  | j                  | j                  | j
                  | j                  | j                  | j                  «      S r¡   )	r¸   r½   ÚTr°   r±   r1   r2   r$   r0   )rj   rn   s     rF   r”   zMoEGate.__call__ñ   sO   € Ü"Ø�—‘—‘ÑØ×(Ñ(Ø�J‰JØ�L‰LØ�O‰OØ×&Ñ&Ø×Ñó
ð 	
rE   ©r<   r=   r>   r   rV   r”   r—   r˜   s   @rF   rº   rº   ã   s   ø„ ðL˜yõ Lö	
rE   rº   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚDeepseekV3MoErI   c                 óN  •— t         ‰| �  «        || _        |j                  | _        t	        |j
                  |j                  |j                  «      | _        t        |«      | _
        |j                  �+|j                  |j                  z  }t        ||¬«      | _        d | _        y )N)rI   r   )rU   rV   rI   r3   r   r   r   r"   Ú
switch_mlprº   Úgater!   rš   Úshared_expertsÚsharding_group)rj   rI   r   rm   s      €rF   rV   zDeepseekV3MoE.__init__þ   s—   ø€ Ü‰ÑÔØˆŒØ#)×#=Ñ#=ˆÔ Ü#Ø×ÑØ×(Ñ(Ø×#Ñ#ó
ˆŒô ˜F“OˆŒ	Ø×"Ñ"Ð.Ø &× <Ñ <¸v×?VÑ?VÑ VÐÜ"/ØÐ1Bô#ˆDÔð #ˆÕrE   c                 óÄ  — | j                   � t        | j                   «      |«      }| j                  |«      \  }}| j                  ||«      }||d   z  j	                  d¬«      j                  |j                  «      }| j                  j                  �|| j                  |«      z   }| j                   �+t        j                  j                  || j                   ¬«      }|S )N).Nrx   rv   ©Úgroup)rÈ   r
   rÆ   rÅ   r©   r¥   r…   rI   r!   rÇ   r|   ÚdistributedÚall_sum)rj   rn   r¶   r²   Úys        rF   r”   zDeepseekV3MoE.__call__  sÇ   € Ø×ÑÐ*Ø2”˜d×1Ñ1Ó2°1Ó5ˆAà—y‘y “|‰ˆˆfØ�O‰O˜A˜tÓ$ˆØ�˜	Ñ"Ñ"×'Ñ'¨RÐ'Ó0×7Ñ7¸¿¹Ó@ˆØ�;‰;×'Ñ'Ð3Ø�D×'Ñ'¨Ó*Ñ*ˆAà×ÑÐ*Ü—‘×&Ñ& q°×0CÑ0CÐ&ÓDˆAàˆrE   rÁ   r˜   s   @rF   rÃ   rÃ   ý   s   ø„ ð#˜yõ #ö&rE   rÃ   c            	       óŽ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
ÚDeepseekV3DecoderLayerrI   Ú	layer_idxc                 ó’  •— t         ‰| �  «        t        |«      | _        |j                  �,||j
                  k\  r||j                  z  dk(  rt        |«      n
t        |«      | _	        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _        y )Nr   rM   )rU   rV   rH   Ú	self_attnr"   r5   r4   rÃ   rš   ÚmlprZ   r^   r   r8   Úinput_layernormÚpost_attention_layernorm)rj   rI   rÑ   rm   s      €rF   rV   zDeepseekV3DecoderLayer.__init__"  s§   ø€ Ü‰ÑÔÜ,¨VÓ4ˆŒð ×'Ñ'Ð3Ø ×!=Ñ!=Ò=Ø × 5Ñ 5Ñ5¸Ò:ô	 ˜&Ô!ô ˜vÓ&ð 	Œô  "Ÿz™z¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(*¯
©
Ø×Ñ F×$7Ñ$7ô)
ˆÕ%rE   rn   ro   rp   rq   c                 óš   — | j                  | j                  |«      ||«      }||z   }| j                  | j                  |«      «      }||z   S r¡   )rÓ   rÕ   rÔ   rÖ   )rj   rn   ro   rp   ÚrÚhs         rF   r”   zDeepseekV3DecoderLayer.__call__3  sL   € ð �N‰N˜4×/Ñ/°Ó2°D¸%Ó@ˆØ�‰EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�1‰uˆrE   r•   )r<   r=   r>   r   rA   rV   r|   rƒ   r   r   r”   r—   r˜   s   @rF   rÐ   rÐ   !  s^   ø„ ð
˜yð 
°Sõ 
ð( $(Ø#ñ		à�8‰8ð	ð �r—x‘xÑ ð	ð ˜‰}ð		ð
 
�‰÷	rE   rÐ   c                   ój   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dee   dej                  fd„Z	ˆ xZ
S )ÚDeepseekV3ModelrI   c                 óp  •— t         ‰| �  «        |j                  | _        t        j                  |j                  |j
                  «      | _        t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}| _
        t        j                  |j
                  |j                  ¬«      | _        y c c}w )NrM   )rU   rV   r   rZ   Ú	Embeddingr   Úembed_tokensÚranger   rÐ   Úlayersr^   r8   Únorm)rj   rI   Úidxrm   s      €rF   rV   zDeepseekV3Model.__init__@  s’   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜŸL™L¨×):Ñ):¸F×<NÑ<NÓOˆÔô ˜V×5Ñ5Ô6ó
á6�ô # 6¨3Õ/Ø6ñ
ˆŒô —J‘J˜v×1Ñ1°v×7JÑ7JÔKˆ�	ùò	
s   Á'B3rn   rp   rq   c                 ó”  — | j                  |«      }| j                  }| j                  }|€d gt        | j                  «      z  }t        ||d   d¬«      }||dz
  k  r#t        j                  j                  ||dz   «      }t        | j                  |«      D ]  \  }} ||||¬«      }Œ |dk7  rVt        j                  j                  ||dz
  |z  «      }|d   �+t        j                  |d   j                  |«      |d   _        |dkD  r/t        j                  j                  |«      d |j                  d    }| j                  |«      S )Nr   T)Úreturn_arrayr   )rp   ru   )rÞ   Úpipeline_rankÚpipeline_sizeÚlenÚpipeline_layersr   r|   rÌ   Ú	recv_likeÚzipÚsendÚdependsÚkeysÚ
all_gatherrz   rá   )	rj   rn   rp   rÙ   rå   ræ   ro   ÚlÚcs	            rF   r”   zDeepseekV3Model.__call__J  s8  € ð
 ×Ñ˜aÓ ˆà×*Ñ*ˆØ×*Ñ*ˆàˆ=Ø�FœS ×!5Ñ!5Ó6Ñ6ˆEÜ$ Q¨¨a©¸tÔDˆð ˜=¨1Ñ,Ò,Ü—‘×(Ñ(¨¨]¸QÑ->Ó@ˆAä˜×,Ñ,¨eÖ4‰DˆAˆqÙ�!�T Ô#‰Að 5ð ˜AÒÜ—‘×#Ñ# A¨¸Ñ(9¸]Ñ'JÓKˆAØ�R‰yÐ$Ü!#§¡¨E°"©I¯N©N¸AÓ!>��b‘	”ð ˜1ÒÜ—‘×)Ñ)¨!Ó,¨\¨q¯w©w°q©zÐ:ˆAà�y‰y˜‹|ÐrE   r¡   r–   r˜   s   @rF   rÛ   rÛ   ?  sC   ø„ ðL˜yõ Lð  $ñà�8‰8ðð ˜‰}ðð 
�‰÷	rE   rÛ   c                   ó´   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dee   fd„Z	d„ Z
ddeej                  j                     fd„Zed	„ «       Zed
„ «       Zˆ xZS )ÚModelrI   c                 óÔ   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        t        j                  |j                  |j                  d¬«      | _
        y rœ   )rU   rV   Úargsr   rÛ   ÚmodelrZ   r[   r   r   Úlm_headr¾   s     €rF   rV   zModel.__init__m  sN   ø€ Ü‰ÑÔØˆŒ	Ø ×+Ñ+ˆŒÜ$ VÓ,ˆŒ
Ü—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆ�rE   Úinputsrp   c                 óH   — | j                  ||«      }| j                  |«      S r¡   )rõ   rö   )rj   r÷   rp   Úouts       rF   r”   zModel.__call__t  s#   € ð
 �j‰j˜ Ó'ˆØ�|‰|˜CÓ Ð rE   c                 ó  — d„ }i }|j                  «       D ]˜  \  }}|j                  d«      rY|j                  dd«      }||dz      j                  t        j
                  «      ||dz   <   ||dz      }|||dz   <   d|z  ||d	z   <   Œp|j                  d«      rŒ‚|j                  d«      rŒ”|||<   Œš |}i }|j                  «       D ];  \  }}d
|v r(|}|j                  dd«      }	||	   }
 ||
|«      }
|
||	<   Œ2||vsŒ7|||<   Œ= |}t        | j                  j                  «      D �]�  }d|› �}dD ]  \  }}dD ]u  }|› d|› d|› �|v sŒt        | j                  j                  «      D �cg c]  }|j                  |› d|› d|› d|› �«      ‘Œ  }}t	        j                  |«      ||› d|› d|› �<   Œw Œ� d|› d�}|› d�|v sŒ›| j                  j                  |   j                  j                  }|› d�|v }|j                  |› d�«      }| j                  j                   | j                  j"                  z   }|r| j                  j$                  }|j                  |› d�«      }|j                  |› d�«      }|j&                  d   dz  |z  }||j&                  d   z  }t	        j(                  |||||¬«      }| j                  j*                  }|j-                  ||d«      }t	        j.                  |d d …d | j                  j                   …d d …f   j1                  dd«      «      }	t	        j.                  |d d …| j                  j                   d …d d …f   «      }|rXt	        j2                  |	¬«      \  }	}}t	        j2                  |||¬«      \  }}}|||› d�<   |||› d�<   |||› d�<   |||› d�<   |	||› d�<   |||› d �<   �Œ  |j                  «       D ��ci c]  \  }}|j5                  d!«      sd"|vr||“Œ c}}S c c}w c c}}w )#Nc                 ó   — t         j                  }t        j                  | t         j                  ¬«      } d}| j                  \  }}| |z  }| |z  }t        j                  | d|fd|ff«      } | j                  ||z   |z  |||z   |z  |f«      } | |d d …d d d …d f   z  j                  ||z   ||z   «      } | d |…d |…f   j                  |«      S )N)r…   r)   r   )r|   Úbfloat16Úfrom_fp8rz   Úpadr{   r¥   )r½   Ú	scale_invr…   ÚbsÚmÚnÚ
pad_bottomÚpad_sides           rF   ÚdequantzModel.sanitize.<locals>.dequant}  så   € Ü—K‘KˆEÜ—[‘[ ¬r¯{©{Ô;ˆFØˆBØ—<‘<‰DˆAˆqØ˜" ™ˆJØ˜˜b‘yˆHÜ—V‘V˜F a¨ _°q¸(°mÐ$DÓEˆFØ—^‘^Ø�j‘. RÑ'¨¨a°(©l¸rÑ-AÀ2ÐFóˆFð ˜yª¨D²!°TÐ)9Ñ:Ñ:×CÑCØ�J‘  H¡óˆFð ˜"˜1˜"˜b˜q˜b˜&‘>×(Ñ(¨Ó/Ð/rE   Úweight_shapeÚ Úweight_packedr½   Úweight_scaleÚscalesiøÿÿÿÚbiasesÚweight_scale_invÚ
_scale_invzmodel.layers.))Úw1r�   )Úw2rŸ   )Úw3rž   )r½   r
  r  z.mlp.experts.0.Ú.z.mlp.experts.z.mlp.switch_mlp.z
.self_attnz.kv_b_proj.weightz.kv_b_proj.scalesz.kv_b_proj.biasesru   r   )ÚbitsÚ
group_sizerx   z.embed_q.scalesz.unembed_out.scalesz.embed_q.biasesz.unembed_out.biasesz.embed_q.weightz.unembed_out.weightzmodel.layers.61zrotary_emb.inv_freq)ÚitemsÚendswithÚreplaceÚviewr|   Úuint32rß   rô   r   r"   ÚpopÚstackrõ   rà   rÓ   rc   r+   r*   r%   rz   Ú
dequantizer   r{   Ú
contiguousr�   ÚquantizeÚ
startswith)rj   Úweightsr  Únew_weightsr‘   r’   rR   rl   rÿ   Úwkr½   rï   Úprefixr  r  ÚeÚto_joinÚlayerÚ	quantizedÚhead_dimrQ   r
  r  r  r  rW   ÚwvÚ	wk_scalesÚ	wk_biasesÚ	wv_scalesÚ	wv_biasess                                  rF   ÚsanitizezModel.sanitize|  s‹  € ò	0ð" ˆØ—M‘M–O‰DˆAˆqØ�z‰z˜.Ô)Ø—y‘y °Ó4�Ø/6°t¸oÑ7MÑ/N×/SÑ/SÜ—I‘Ió0�˜D 8™OÑ,ð ˜D >Ñ1Ñ2�Ø/0�˜D 8™OÑ,Ø/1°A©v�˜D 8™OÒ,Ø—j‘j Õ0°A·J±J¸Õ4OØ!"�˜A’ð $ð ˆð ˆØ—M‘M–O‰DˆAˆqØ! QÑ&Ø�	Ø—Y‘Y˜|¨RÓ0�Ø  ™�Ù  ¨Ó3�Ø"(�˜B’Ø˜+Ò%Ø!"�˜A’ð $ð ˆô �t—y‘y×2Ñ2×3ˆAØ$ Q CÐ(ˆFÛU‘��1Û7�AØ ˜ °°°1°Q°CÐ8¸GÒCô &+¨4¯9©9×+EÑ+EÔ%Fó#á%F ð $ŸK™K¨6¨(°-À¸sÀ!ÀAÀ3ÀaÈÀsÐ(KÕLØ%Fð  ð #ô GIÇhÁhÈwÓFW˜ 6 (Ð*:¸1¸#¸Q¸q¸cÐ BÒCñ 8ð Vð % Q C zÐ2ˆFØ�Ð*Ð+¨wÒ6ØŸ
™
×)Ñ)¨!Ñ,×6Ñ6×>Ñ>�Ø%˜hÐ&7Ð8¸GÐC�	Ø—K‘K 6 (Ð*;Ð <Ó=�ØŸ9™9×5Ñ5¸¿	¹	×8LÑ8LÑL�áØŸ9™9×1Ñ1�DØ$Ÿ[™[¨F¨8Ð3DÐ)EÓF�FØ$Ÿ[™[¨F¨8Ð3DÐ)EÓF�FàŸG™G B™K¨"Ñ,°Ñ5�DØ!%¨¯©°bÑ)9Ñ!9�JÜŸ™Ø˜6 6°Àô�Að !ŸI™I×9Ñ9�	Ø—I‘I˜i¨°2Ó6�Ü—]‘]Ø’aÐ5˜4Ÿ9™9×5Ñ5Ð5²qÐ8Ñ9×BÑBÀ2ÀrÓJó�ô —]‘] 1¢Q¨¯	©	×(BÑ(BÑ(DÂaÐ%GÑ#HÓI�ÙÜ/1¯{©{Ø °*ô0Ñ,�B˜	 9ô 02¯{©{Ø °*ô0Ñ,�B˜	 9ð ;D�G˜v˜h oÐ6Ñ7Ø>G�G˜v˜hÐ&9Ð:Ñ;Ø:C�G˜v˜h oÐ6Ñ7Ø>G�G˜v˜hÐ&9Ð:Ñ;Ø68�˜6˜( /Ð2Ñ3Ø:<�˜6˜(Ð"5Ð6Ó7ð[ 4ðd  Ÿ™œô
á'‘��1Ø—<‘<Ð 1Ô2Ð7LÐTUÑ7Uð ˆq‰DØ'ò
ð 	
ùòW#ùóV
s   Å(#O>
Ï"PrË   c                 óÈ  ‡‡— |xs t         j                  j                  «       }|j                  «       }|j	                  «       }| j
                  j                  D �]  }|j                  j                  €2t        |j                  j                  d|¬«      |j                  _
        n1t        |j                  j                  d|¬«      |j                  _        |j                  xj                  |z  c_        |j                  j                  }||z  Š‰|z   Šˆˆfd„}|j                  j                  j                  |«       |j                  j                  j                  |«       t        |j                  j                   d|¬«      |j                  _        t#        |j$                  t&        «      r•t        |j$                  j(                  d|¬«      |j$                  _        t        |j$                  j*                  d|¬«      |j$                  _        t        |j$                  j,                  d|¬«      |j$                  _        �Œì||j$                  _        t1        |j$                  j2                  j(                  d|¬«       t1        |j$                  j2                  j*                  d|¬«       t1        |j$                  j2                  j,                  d|¬«       t1        |j$                  j4                  j(                  d|¬«       t1        |j$                  j4                  j*                  d|¬«       t1        |j$                  j4                  j,                  d|¬«       �Œ y )Nzall-to-shardedrÊ   c                 ó   •— | ‰‰ S r¡   rD   )ÚwÚehÚshs    €€rF   Úshard_headsz Model.shard.<locals>.shard_headsó  s   ø€ Ø˜˜B�x�rE   zsharded-to-all)r|   rÌ   ÚinitÚsizeÚrankrõ   rà   rÓ   r&   r	   r\   r`   rW   rc   Úapplyrd   re   Ú
isinstancerÔ   rš   r�   rŸ   rž   rÈ   r   rÇ   rÅ   )	rj   rË   ÚNr6  r%  rW   r3  r1  r2  s	          @@rF   ÚshardzModel.shardà  s‡  ù€ ØÒ.œŸ™×,Ñ,Ó.ˆØ�J‰J‹LˆØ�z‰z‹|ˆØ—Z‘Z×&Õ&ˆEà�‰×*Ñ*Ð2Ü)5Ø—O‘O×*Ñ*Ð,<ÀEô*�—‘Õ&ô ,8Ø—O‘O×,Ñ,Ð.>Àeô,�—‘Ô(ð �O‰O×%Ò%¨!Ñ+Õ%ØŸ™×1Ñ1ˆIØ˜	Ñ!ˆBØ�i‘ˆBõ ð �O‰O×#Ñ#×)Ñ)¨+Ô6Ø�O‰O×'Ñ'×-Ñ-¨kÔ:ä%1Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô
 ˜%Ÿ)™)¤]Ô3Ü&2Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	Ô#ô '3Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	Ô#ô %1Ø—I‘I×%Ñ%Ð'7¸uô%�—	‘	Ö!ð ,1�—	‘	Ô(ÜØ—I‘I×,Ñ,×6Ñ6Ð8HÐPUõô Ø—I‘I×,Ñ,×6Ñ6Ð8HÐPUõô Ø—I‘I×,Ñ,×4Ñ4Ð6FÈeõô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈEõô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈEõô Ø—I‘I×(Ñ(×0Ñ0Ð2BÈ%÷ñq 'rE   c                 ó.   — | j                   j                  S r¡   )rõ   rè   )rj   s    rF   rà   zModel.layers   s   € à�z‰z×)Ñ)Ð)rE   c                 ó   — d„ }|S )Nc                 ó
   — d| vS )Nr°   rD   )r‘   s    rF   Ú	predicatez'Model.cast_predicate.<locals>.predicate&  s   € Ø,°AÐ5Ð5rE   rD   )rj   r>  s     rF   Úcast_predicatezModel.cast_predicate$  s   € ò	6ð ÐrE   r¡   )r<   r=   r>   r   rV   r|   rƒ   r   r   r”   r-  rÌ   ÚGroupr:  Úpropertyrà   r?  r—   r˜   s   @rF   rò   rò   l  s€   ø„ ðT˜yõ Tð  $ñ!à—‘ð!ð ˜‰}ó!òb
ñH>˜8 B§N¡N×$8Ñ$8Ñ9ó >ð@ ñ*ó ð*ð ñó ôrE   rò   )+rg   Údataclassesr   Ú	functoolsr   Útypingr   r   r   Úmlx.coreÚcorer|   Úmlx.nnrZ   Úmlx.nn.layers.distributedr   r	   r
   Úactivationsr   rR   r   r   r   Úmlar   Úpipeliner   Ú
rope_utilsr   Úswitch_layersr   r   ÚModulerH   rš   Úcompiler¸   rº   rÃ   rÐ   rÛ   rò   rD   rE   rF   Ú<module>rP     så   ðó Ý !Ý ß &Ñ &å Ý ß PÑ På ß TÑ TÝ Ý #Ý 'Ý $ð ô!�ó !ó ð!ô@t#˜"Ÿ)™)ô t#ôn�B—I‘Iô ð( ‡�ñó ðôD
ˆb�i‰iô 
ô4!�B—I‘Iô !ôH˜RŸY™Yô ô<*�m R§Y¡Yô *ôZ}ˆB�I‰Iõ }rE   