+
    TV-j9O  ã                   ó"  € ^ RI t ^ RIHt ^ RIHt ^ RIHtHtHt ^ RI	H
t ^ RIHt ^ RIHtHtHt ^RIHt ^RIHtHtHt ^RIHt ^R	IHt ^R
IHt ^RIHt ] ! R R]4      4       t  ! R R]PB                  4      t" ! R R]PB                  4      t#]PH                  R 4       t% ! R R]PB                  4      t& ! R R]PB                  4      t' ! R R]PB                  4      t( ! R R]]PB                  4      t) ! R R]PB                  4      t*R# )é    N)Ú	dataclass)Úpartial)ÚAnyÚDictÚOptional)Úshard_inplaceÚshard_linearÚsum_gradients)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚMultiLinear)ÚPipelineMixin)Úinitialize_rope)Ú	SwitchGLUc                   ó    a € ] tR t^t o RtRtRtRtRt^t	^ t
^ tRtRtRtRtR	t^@t^€t^€tR
tRtRt^t^t^t^t^ tRtRtRtRtRt V 3R lt!Rt"V t#R# )Ú	ModelArgsÚdeepseek_v3i � i   i +  i  Nç      ð?i   i   Únoaux_tcÚsigmoidTi   ç�íµ ÷Æ°>g     ˆÃ@Fc                ó–  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[,          ;R	&   S[S[,          ;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[ ;R&   S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   # )é   Ú
model_typeÚ
vocab_sizeÚhidden_sizeÚintermediate_sizeÚmoe_intermediate_sizeÚnum_hidden_layersÚnum_attention_headsÚnum_key_value_headsÚn_shared_expertsÚn_routed_expertsÚrouted_scaling_factorÚkv_lora_rankÚq_lora_rankÚqk_rope_head_dimÚ
v_head_dimÚqk_nope_head_dimÚtopk_methodÚscoring_funcÚnorm_topk_probÚn_groupÚ
topk_groupÚnum_experts_per_tokÚmoe_layer_freqÚfirst_k_dense_replaceÚmax_position_embeddingsÚrms_norm_epsÚ
rope_thetaÚrope_scalingÚattention_bias)ÚstrÚintr   ÚfloatÚboolr   )ÚformatÚ__classdict__s   "€Új/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/deepseek_v3.pyÚ__annotate__ÚModelArgs.__annotate__   s„  ø‡ ‚ áÑ#ñ ñ Ññ ñ Ññ	 ñ
 Ñ"ñ ñ Ñ%ñ ñ Ññ ñ Ñ!ñ ñ Ñ!ñ ñ ™s•mÑ*ñ ñ ™s•mÑ*ñ ñ !Ñ&ñ ñ Ññ ñ Ññ ñ Ññ ñ  Ññ! ñ" Ññ# ñ$ Ñ!ñ% ñ& Ñ!ñ' ñ( Ññ) ñ* Ññ+ ñ, Ññ- ñ. Ñ ñ/ ñ0 Ññ1 ñ2 Ñ"ñ3 ñ4 !Ñ'ñ5 ñ6 Ññ7 ñ8 Ññ9 ñ: Ññ; ñ< Ñ ò= ó    © )$Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r>   s   @r?   r   r      sª   ø‡ € à#€JØ€JØ€KØ"ÐØ!%ÐØÐØ!ÐØ!ÐØ&*ÐØ&*ÐØ#&ÐØ€LØ€KØÐØ€JØÐØ!€KØ!€LØ€NØ€GØ€JØ ÐØ€NØ!"ÐØ#'ÐØ€LØ€JØ€LØ €N÷= ƒ rB   r   c                   óT   a a€ ] tR t^5t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚDeepseekV3Attentionc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úconfig©r   )r=   r>   s   "€r?   r@   Ú DeepseekV3Attention.__annotate__6   s   ø€ ÷ >
ñ >
™yñ >
rB   c                óþ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  V n	        VP                  V n
        VP                  V n        VP                  V n        VP                  VP                  ,           V n        V P                  R,          V n        V P                  fE   \        P                   ! V P                  V P
                  V P                  ,          RR7      V n        M¦\        P                   ! V P                  V P                  VP$                  R7      V n        \        P(                  ! V P                  RR7      V n        \        P                   ! V P                  V P
                  V P                  ,          RR7      V n        \        P                   ! V P                  V P                  V P                  ,           VP$                  R7      V n        \        P(                  ! V P                  RR7      V n        \3        V P                  V P                  V P
                  4      V n        \3        V P                  V P                  V P
                  4      V n        \        P                   ! V P
                  V P                  ,          V P                  VP$                  R7      V n        V P                  P:                  e�   V P                  P:                  P=                  R^ 4      pV'       do   V P                  P:                  R,          pV^8”  dK   RV,          \>        P@                  ! V4      ,          R	,           pV P                  V,          V,          V n        \C        V P                  V P                  R
V P                  V P                  P:                  R7      V n"        R# )g      à?NF©Úbiasr   ©ÚepsÚmscale_all_dimÚfactorgš™™™™™¹?r   T)ÚdimsÚbaseÚtraditionalr4   Úscaling_configg      à¿)#ÚsuperÚ__init__rO   r   r"   Ú	num_headsr4   r6   r(   r)   r'   r*   r+   Ú
q_head_dimÚscaleÚnnÚLinearÚq_projr8   Úq_a_projÚRMSNormÚq_a_layernormÚq_b_projÚkv_a_proj_with_mqaÚkv_a_layernormr   Úembed_qÚunembed_outÚo_projr7   ÚgetÚmathÚlogr   Úrope)ÚselfrO   rW   Úscaling_factorÚsÚ	__class__s   &&   €r?   r^   ÚDeepseekV3Attention.__init__6   sß  ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ×3Ñ3ˆŒØ'-×'EÑ'EˆÔ$Ø ×+Ñ+ˆŒØ!×-Ñ-ˆÔØ &× 7Ñ 7ˆÔØ"×/Ñ/ˆÔØ ×+Ñ+ˆŒØ &× 7Ñ 7ˆÔØ ×1Ñ1°F×4KÑ4KÕKˆŒà—_‘_ dÕ*ˆŒ
à×ÑÒ#ÜŸ)š)Ø× Ñ  $§.¡.°4·?±?Õ"BÈôˆD�Kô ŸIšIØ× Ñ  $×"2Ñ"2¸×9NÑ9NôˆDŒMô "$§¢¨D×,<Ñ,<À$Ô!GˆDÔÜŸIšIØ× Ñ  $§.¡.°4·?±?Õ"BÈôˆDŒMô #%§)¢)Ø×ÑØ×Ñ × 5Ñ 5Õ5Ø×&Ñ&ô#
ˆÔô
 !Ÿjšj¨×):Ñ):ÀÔEˆÔÜ"Ø×!Ñ! 4×#4Ñ#4°d·n±nó
ˆŒô 'Ø×Ñ˜tŸ™°·±ó
ˆÔô —i’iØ�N‰N˜TŸ_™_Õ,Ø×ÑØ×&Ñ&ô
ˆŒð �;‰;×#Ñ#Ò/Ø!Ÿ[™[×5Ñ5×9Ñ9Ð:JÈAÓNˆNßØ!%§¡×!9Ñ!9¸(Õ!C�Ø! AÔ%Ø˜nÕ,¬t¯xªx¸Ó/GÕGÈ#ÕM�AØ!%§¡¨a¥°!Õ!3�D”Jä#Ø×&Ñ&Ø—‘ØØ$(×$@Ñ$@ØŸ;™;×3Ñ3ô
ˆŽ	rB   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   ÚxÚmaskÚcacheÚreturn©ÚmxÚarrayr   r   )r=   r>   s   "€r?   r@   rQ   v   sH   ø€ ÷ 3#ñ 3#á�8‰8ð3#ñ ‘r—x‘xÕ ð3#ñ ™�}ð	3#ñ
 
�‰ñ3#rB   c           	     ó’  € VP                   w  rEpV P                  f   V P                  V4      pM/V P                  V P	                  V P                  V4      4      4      pVP                  WEV P                  V P                  4      P                  ^ ^^^4      p\        P                  ! WpP                  .RR7      w  r‰V P                  V4      p
\        P                  ! W P                  .RR7      w  r«VP                  WE^V P                  4      P                  ^ ^^^4      pV P!                  V
4      pVe   VP"                  M^ pV P%                  W�4      p	V P%                  W½4      p\        P&                  ! V^R7      pVe   VP)                  WË4      w  rËW�P*                  ,          VP-                  RR4      ,          pVe`   \        P.                  ! VV\        P0                  ! \        P2                  ! VP4                  4      P6                  VP4                  4      4      pV^8X  d   V P9                  V4      pT;ppM$V P9                  VRR7      pV P;                  V4      p\=        W�VW0P*                  VR7      pV^8X  d   V P;                  V4      pVP                  ^ ^^^4      P                  WER4      pV P?                  V4      # )N©ÚaxisF)Ú	transpose)r{   ra   rz   éÿÿÿÿéþÿÿÿ) Úshaper(   rd   rh   rg   re   Úreshaper_   r`   rƒ   r~   Úsplitr+   ri   r'   r)   rj   Úoffsetrq   Úexpand_dimsÚupdate_and_fetchra   ÚswapaxesÚwherer   ÚfinfoÚdtypeÚminrk   rl   r   rm   )rr   ry   rz   r{   ÚBÚLÚDÚqÚq_nopeÚq_peÚcompressed_kvÚk_peÚ	kv_latentr‰   Ú	pe_scoresÚkÚvÚoutputs   &&&&              r?   Ú__call__ÚDeepseekV3Attention.__call__v   sM  € ð —'‘'‰ˆˆaà×ÑÒ#Ø—‘˜A“‰Aà—‘˜d×0Ñ0°·±¸qÓ1AÓBÓCˆAà�I‰I�a˜DŸN™N¨D¯O©OÓ<×FÑFÀqÈ!ÈQÐPQÓRˆÜ—x’x ×$9Ñ$9Ð#:ÀÔD‰ˆØ×/Ñ/°Ó2ˆÜ Ÿhšh }×7HÑ7HÐ6IÐPRÔSÑˆØ�|‰|˜A ! T×%:Ñ%:Ó;×EÑEÀaÈÈAÈqÓQˆØ×'Ñ'¨Ó6ˆ	à!&Ò!2�—’¸ˆØ�y‰y˜Ó&ˆØ�y‰y˜Ó&ˆä—N’N 9°1Ô5ˆ	àÒØ#×4Ñ4°YÓE‰OˆIàŸJ™JÕ&¨$¯-©-¸¸BÓ*?Õ?ˆ	ØÒÜŸšØØÜ—’œŸš )§/¡/Ó2×6Ñ6¸	¿¹ÓHóˆIð �Œ6Ø—\‘\ &Ó)ˆFØÐˆA‘à—‘˜Y°%�Ó8ˆAØ× Ñ  Ó+ˆAä-Ø�q ¯Z©Z¸iô
ˆð �Œ6Ø×%Ñ% fÓ-ˆFà×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"rB   )rO   rk   r   rj   ri   r'   r4   r_   rm   rg   re   rh   r`   r(   rd   r+   r)   rq   r6   ra   rl   r*   ©NN©	rD   rE   rF   rG   r^   rž   rI   rJ   Ú__classcell__©ru   r>   s   @@r?   rL   rL   5   s   ù‡ € ÷>
ó >
÷@3#÷ 3#ò 3#rB   rL   c                   óH   a a€ ] tR t^¬t oRV3R lV 3R llltR tRtVtV ;t# )ÚDeepseekV3MLPc                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r   rO   r   r   ©r   r:   )r=   r>   s   "€r?   r@   ÚDeepseekV3MLP.__annotate__­   s*   ø€ ÷ Yñ YÙðYÙ.1ðYÙMPñYrB   c                ó¸  <€ \         SV `  4        Wn        Vf   VP                  MTV n        Vf   VP                  MTV n        \
        P                  ! V P                  V P                  RR7      V n        \
        P                  ! V P                  V P                  RR7      V n        \
        P                  ! V P                  V P                  RR7      V n	        R # )NFrS   )
r]   r^   rO   r   r   rb   rc   Ú	gate_projÚup_projÚ	down_proj)rr   rO   r   r   ru   s   &&&&€r?   r^   ÚDeepseekV3MLP.__init__­   s¨   ø€ ô 	‰ÑÔØŒØ1<Ò1D˜6×-Ò-È+ˆÔà(9Ò(AˆF×$Ò$ÐGXð 	Ôô Ÿš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŽrB   c                óx   € V P                  \        V P                  V4      V P                  V4      4      4      pV# ©N)r¬   r   rª   r«   )rr   ry   r¬   s   && r?   rž   ÚDeepseekV3MLP.__call__»   s.   € Ø—N‘N¤6¨$¯.©.¸Ó*;¸T¿\¹\È!»_Ó#MÓNˆ	ØÐrB   )rO   r¬   rª   r   r   r«   r    r¡   r£   s   @@r?   r¥   r¥   ¬   s   ù‡ € ÷Yõ Y÷ò rB   r¥   c                 ó$  € \         P                  ! V P                  \         P                  4      4      pTpWq,           pV^8”  dÓ   \         P                  ! VR
VR
3R7      p\         P
                  ! V^R
R7      P                  R
RR7      p	W4,
          p
\         P                  ! Wš^,
          RR7      RRV
1R3,          p\         P                  ! V\         P                  ! V4      \         P                  ! R	4      RR7      p\         P                  ! VRR
4      pTp
\         P                  ! V) V
^,
          R
R7      RRV
13,          p\         P                  ! WŒR
R7      pV^8”  d$   V'       d   VP                  R
RR7      pW},          pWu,          pWÇ3# )é   )r‚   r†   r�   T)r‚   Úkeepdims)Úkthr‚   .NºNNNg        r„   r…   )r~   r   ÚastypeÚfloat32Ú	unflattenÚtopkÚsumÚargpartitionÚput_along_axisÚstop_gradientr   ÚflattenÚtake_along_axis)ÚgatesÚe_score_correction_biasÚtop_kr/   r0   r&   r.   ÚscoresÚorig_scoresÚgroup_scoresr›   Ú	group_idxÚindsÚdenominators   &&&&&&&       r?   Úgroup_expert_selectrÉ   À   sE  € ô �ZŠZ˜Ÿ™¤R§Z¡ZÓ0Ó1€FØ€KØÕ-€FØ�„{Ü—’˜f¨2°g¸r°]ÔCˆÜ—w’w˜v q¨rÔ2×6Ñ6¸BÈÐ6ÓNˆØÕ ˆÜ—O’O L¸!µeÀ"ÔEÀcÈ2ÈAÈ2ÈqÀjÕQˆ	Ü×"Ò"Ø”B×$Ò$ YÓ/´·²¸#³ÀRô
ˆô —’˜F B¨Ó+ˆà€AÜ�?Š?˜F˜7¨¨A­°BÔ7¸¸R¸a¸R¸Õ@€DÜ×Ò ¸Ô;€FØˆq„y—^Ø—j‘j b°4�jÓ8ˆØÕ%ˆØÕ+€Fàˆ<ÐrB   c                   óD   a a€ ] tR t^ãt oV3R lV 3R lltR tRtVtV ;t# )ÚMoEGatec                ó    <€ V ^8„  d   QhRS[ /# rN   rP   )r=   r>   s   "€r?   r@   ÚMoEGate.__annotate__ä   s   ø€ ÷ Lñ L™yñ LrB   c                óÚ  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  V n	        \        P                  ! V P                  VP                  34      V n        \        P                  ! V P                  34      V n        VP                  R 8X  g   Q R4       hR# )r   zUnsupported topk method.N)r]   r^   rO   r1   rÂ   r.   r%   r&   r/   r0   r~   Úzerosr   ÚweightrÁ   r,   ©rr   rO   ru   s   &&€r?   r^   ÚMoEGate.__init__ä   s·   ø€ Ü‰ÑÔØŒØ×/Ñ/ˆŒ
Ø$×3Ñ3ˆÔØ &× 7Ñ 7ˆÔØ%+×%AÑ%AˆÔ"Ø—~‘~ˆŒØ ×+Ñ+ˆŒÜ—h’h × 5Ñ 5°v×7IÑ7IÐJÓKˆŒÜ')§x¢x°×1FÑ1FÐ0HÓ'IˆÔ$Ø×!Ñ! ZÔ/ÐKÐ1KÓKÒ/rB   c           	     óÐ   € \        WP                  P                  ,          V P                  V P                  V P
                  V P                  V P                  V P                  4      # r¯   )	rÉ   rÐ   ÚTrÁ   rÂ   r/   r0   r&   r.   )rr   ry   s   &&r?   rž   ÚMoEGate.__call__ñ   sM   € Ü"Ø—‘—‘ÕØ×(Ñ(Ø�J‰JØ�L‰LØ�O‰OØ×&Ñ&Ø×Ñó
ð 	
rB   )	rO   rÁ   r/   r%   r.   r&   rÂ   r0   rÐ   r¡   r£   s   @@r?   rË   rË   ã   s   ù‡ € ÷Ló L÷	
ò 	
rB   rË   c                   óD   a a€ ] tR t^ýt oV3R lV 3R lltR tRtVtV ;t# )ÚDeepseekV3MoEc                ó    <€ V ^8„  d   QhRS[ /# rN   rP   )r=   r>   s   "€r?   r@   ÚDeepseekV3MoE.__annotate__þ   s   ø€ ÷ #ñ #™yñ #rB   c                óX  <€ \         SV `  4        Wn        VP                  V n        \	        VP
                  VP                  VP                  4      V n        \        V4      V n
        VP                  e/   VP                  VP                  ,          p\        WR7      V n        R V n        R # )N)rO   r   )r]   r^   rO   r1   r   r   r    r%   Ú
switch_mlprË   Úgater$   r¥   Úshared_expertsÚsharding_group)rr   rO   r   ru   s   && €r?   r^   ÚDeepseekV3MoE.__init__þ   s’   ø€ Ü‰ÑÔØŒØ#)×#=Ñ#=ˆÔ Ü#Ø×ÑØ×(Ñ(Ø×#Ñ#ó
ˆŒô ˜F“OˆŒ	Ø×"Ñ"Ò.Ø &× <Ñ <¸v×?VÑ?VÕ VÐÜ"/Øô#ˆDÔð #ˆÖrB   c                óÞ  € V P                   e   \        V P                   4      ! V4      pV P                  V4      w  r#V P                  W4      pWCR,          ,          P	                  RR7      P                  VP                  4      pV P                  P                  e   W@P                  V4      ,           pV P                   e+   \        P                  P                  W@P                   R7      pV# )Nr�   ©Úgroup).Nr…   )rÞ   r
   rÜ   rÛ   rº   r¶   r�   rO   r$   rÝ   r~   ÚdistributedÚall_sum)rr   ry   rÇ   rÃ   Úys   &&   r?   rž   ÚDeepseekV3MoE.__call__  s»   € Ø×ÑÒ*Ü˜d×1Ñ1Ô2°1Ó5ˆAà—y‘y “|‰ˆØ�O‰O˜AÓ$ˆØ˜	Õ"Õ"×'Ñ'¨RÐ'Ó0×7Ñ7¸¿¹Ó@ˆØ�;‰;×'Ñ'Ò3Ø×'Ñ'¨Ó*Õ*ˆAà×ÑÒ*Ü—‘×&Ñ& q×0CÑ0CÐ&ÓDˆAàˆrB   )rO   rÜ   r1   rÞ   rÝ   rÛ   r¡   r£   s   @@r?   r×   r×   ý   s   ù‡ € ÷#ó #÷&ò rB   r×   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚDeepseekV3DecoderLayeri!  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   rO   Ú	layer_idxr§   )r=   r>   s   "€r?   r@   Ú#DeepseekV3DecoderLayer.__annotate__"  s   ø€ ÷ 
ñ 
™yð 
±Sñ 
rB   c                ó¨  <€ \         SV `  4        \        V4      V n        VP                  e3   W!P
                  8¼  d#   W!P                  ,          ^ 8X  d   \        V4      M
\        V4      V n	        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  R7      V n        R # )NrU   )r]   r^   rL   Ú	self_attnr%   r3   r2   r×   r¥   Úmlprb   rf   r   r5   Úinput_layernormÚpost_attention_layernorm)rr   rO   rê   ru   s   &&&€r?   r^   ÚDeepseekV3DecoderLayer.__init__"  s£   ø€ Ü‰ÑÔÜ,¨VÓ4ˆŒð ×'Ñ'Ò3Ø×!=Ñ!=Ô=Ø× 5Ñ 5Õ5¸Ô:ô	 ˜&Ô!ô ˜vÓ&ð 	Œô  "Ÿzšz¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(*¯
ª
Ø×Ñ F×$7Ñ$7ô)
ˆÖ%rB   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# rx   r}   )r=   r>   s   "€r?   r@   rë   3  sH   ø€ ÷ 	ñ 	á�8‰8ð	ñ ‘r—x‘xÕ ð	ñ ™�}ð		ñ
 
�‰ñ	rB   c                ó¤   € V P                  V P                  V4      W#4      pW,           pV P                  V P                  V4      4      pWT,           # r¯   )rí   rï   rî   rð   )rr   ry   rz   r{   ÚrÚhs   &&&&  r?   rž   ÚDeepseekV3DecoderLayer.__call__3  sF   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ�EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�uˆrB   )rï   rî   rð   rí   r    r¡   r£   s   @@r?   rè   rè   !  s   ù‡ € ÷
ó 
÷"	÷ 	ò 	rB   rè   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚDeepseekV3Modeli?  c                ó    <€ V ^8„  d   QhRS[ /# rN   rP   )r=   r>   s   "€r?   r@   ÚDeepseekV3Model.__annotate__@  s   ø€ ÷ Lñ L™yñ LrB   c                óx  <€ \         SV `  4        VP                  V n        \        P                  ! VP                  VP
                  4      V n        \        VP                  4       Uu. uF  p\        W4      NK  	  upV n
        \        P                  ! VP
                  VP                  R 7      V n        R# u upi )rU   N)r]   r^   r   rb   Ú	Embeddingr   Úembed_tokensÚranger!   rè   Úlayersrf   r5   Únorm)rr   rO   Úidxru   s   && €r?   r^   ÚDeepseekV3Model.__init__@  s�   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜŸLšL¨×):Ñ):¸F×<NÑ<NÓOˆÔô ˜V×5Ñ5Ô6ó
á6�ô # 6Ö/Ù6ñ
ˆŒô —J’J˜v×1Ñ1°v×7JÑ7JÔKˆŽ	ùò	
s   Á(B7c                ód   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[ P                  /# )r   ry   r{   r|   r}   )r=   r>   s   "€r?   r@   rú   J  s5   ø€ ÷ ñ á�8‰8ðñ ™�}ðñ 
�‰ñ	rB   c                óò  € V P                  V4      pV P                  pV P                  pVf   R .\        V P                  4      ,          p\        W2^ ,          RR7      pWE^,
          8  d'   \        P                  P                  W4^,           4      p\        V P                  V4       F  w  rxV! W6VR7      pK  	  V^ 8w  dm   \        P                  P                  W4^,
          V,          4      pVR,          e5   \        P                  ! VR,          P                  V4      VR,          n        V^8”  d4   \        P                  P                  V4      R VP                  ^ ,           pV P                  V4      # )NT)Úreturn_array)r{   r„   )rý   Úpipeline_rankÚpipeline_sizeÚlenÚpipeline_layersr   r~   rã   Ú	recv_likeÚzipÚsendÚdependsÚkeysÚ
all_gatherr†   r   )	rr   ry   r{   rõ   r  r  rz   ÚlÚcs	   &&&      r?   rž   ÚDeepseekV3Model.__call__J  s,  € ð
 ×Ñ˜aÓ ˆà×*Ñ*ˆØ×*Ñ*ˆàŠ=Ø�FœS ×!5Ñ!5Ó6Õ6ˆEÜ$ Q¨a­¸tÔDˆð ¨1Õ,Ô,Ü—‘×(Ñ(¨¸QÕ->Ó@ˆAä˜×,Ñ,¨eÖ4‰DˆAÙ�! Ô#ŠAñ 5ð ˜AÔÜ—‘×#Ñ# A¸Õ(9¸]Õ'JÓKˆAØ�R�yÒ$Ü!#§¢¨E°"­I¯N©N¸AÓ!>��b•	”ð ˜1ÔÜ—‘×)Ñ)¨!Ó,¨\¨q¯w©w°q­zÐ:ˆAà�y‰y˜‹|ÐrB   )rý   rÿ   r   r   r¯   r¡   r£   s   @@r?   rø   rø   ?  s    ù‡ € ÷Ló L÷÷ ò rB   rø   c                   ó�   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltR tRV3R lR llt]R	 4       t	]R
 4       t
RtVtV ;t# )ÚModelil  c                ó    <€ V ^8„  d   QhRS[ /# rN   rP   )r=   r>   s   "€r?   r@   ÚModel.__annotate__m  s   ø€ ÷ Tñ T™yñ TrB   c                óÖ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        \        P                  ! VP                  VP                  R R7      V n
        R# )FrS   N)r]   r^   Úargsr   rø   Úmodelrb   rc   r   r   Úlm_headrÑ   s   &&€r?   r^   ÚModel.__init__m  sL   ø€ Ü‰ÑÔØŒ	Ø ×+Ñ+ˆŒÜ$ VÓ,ˆŒ
Ü—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŽrB   c                óJ   <€ V ^8„  d   QhRS[ P                  RS[S[,          /# )r   Úinputsr{   r}   )r=   r>   s   "€r?   r@   r  t  s'   ø€ ÷ !ñ !á—‘ð!ñ ™�}ñ!rB   c                óF   € V P                  W4      pV P                  V4      # r¯   )r  r  )rr   r  r{   Úouts   &&& r?   rž   ÚModel.__call__t  s!   € ð
 �j‰j˜Ó'ˆØ�|‰|˜CÓ Ð rB   c                ó
	  € R  p/ pVP                  4        FÇ  w  rEVP                  R4      '       dv   VP                  RR4      pWR,           ,          P                  \        P
                  4      W6R,           &   WR,           ,          pWsVR,           &   RV,          W6R,           &   K‘  VP                  R4      '       d   Kª  VP                  R4      '       d   KÃ  WSV&   KÉ  	  Tp/ pVP                  4        FB  w  rERV9   d+   TpVP                  R	R4      p	W,          p
V! W¨4      p
W£V	&   K6  WC9  g   K>  WSV&   KD  	  Tp\        V P                  P                  4       EFÓ  pR
V 2pR F‡  w  rÞR  F|  pV RV RV 2V9   g   K  \        V P                  P                  4       Uu. uF  pVP                  V RV RV RV 24      NK!  	  pp\        P                  ! V4      W RV RV 2&   K~  	  K‰  	  R
V R2pV R2V9   g   K¨  V P                  P                  V,          P                  P                  pV R2V9   pVP                  V R24      pV P                  P                   V P                  P"                  ,           pV'       d•   V P                  P$                  pVP                  V R24      pVP                  V R24      pVP&                  R!,          ^ ,          V,          pVVP&                  R!,          ,          p\        P(                  ! VVVVVR7      pV P                  P*                  pVP-                  VVR!4      p\        P.                  ! VRRV P                  P                   1R3,          P1                  R!R"4      4      p	\        P.                  ! VRV P                  P                   R1R3,          4      pV'       dW   \        P2                  ! V	XXR7      w  p	pp\        P2                  ! VVVR7      w  pppVW R2&   VW R2&   VW R2&   VW R2&   W‘V R2&   VW R2&   EKÖ  	  VP                  4        UUu/ uF)  w  rEVP5                  R4      '       d   K  RV9  g   K'  WEbK+  	  upp# u upi u uppi )#c                 óà  € \         P                  p\         P                  ! V \         P                  R 7      p ^€pV P                  w  rEV) V,          pV) V,          p\         P                  ! V ^ V3^ V334      p V P                  WF,           V,          W5V,           V,          V34      p WR,          ,          P                  WF,           WW,           4      p V RV1RV13,          P                  V4      # ))r�   N)rµ   Nrµ   N)r~   Úbfloat16Úfrom_fp8r†   Úpadr‡   r¶   )rÐ   Ú	scale_invr�   ÚbsÚmÚnÚ
pad_bottomÚpad_sides   &&      r?   ÚdequantÚModel.sanitize.<locals>.dequant}  sÑ   € Ü—K‘KˆEÜ—[’[ ¬r¯{©{Ô;ˆFØˆBØ—<‘<‰DˆAØ˜" �ˆJØ˜˜b•yˆHÜ—V’V˜F a¨ _°q¸(°mÐ$DÓEˆFØ—^‘^Ø•. RÕ'¨°(­l¸rÕ-AÀ2ÐFóˆFð Ð)9Õ:Õ:×CÑCØ• ¥óˆFð ˜"˜1˜"˜b˜q˜b˜&•>×(Ñ(¨Ó/Ð/rB   Úweight_shapeÚ Úweight_packedrÐ   Úweight_scaleÚscalesÚbiasesÚweight_scale_invÚ
_scale_invzmodel.layers.z.mlp.experts.0.Ú.z.mlp.experts.z.mlp.switch_mlp.z
.self_attnz.kv_b_proj.weightz.kv_b_proj.scalesz.kv_b_proj.biases)ÚbitsÚ
group_sizerµ   Nz.embed_q.scalesz.unembed_out.scalesz.embed_q.biasesz.unembed_out.biasesz.embed_q.weightz.unembed_out.weightzmodel.layers.61zrotary_emb.inv_freqiøÿÿÿ))Úw1rª   )Úw2r¬   )Úw3r«   )rÐ   r2  r3  r„   r…   )ÚitemsÚendswithÚreplaceÚviewr~   Úuint32rþ   r  r!   r%   ÚpopÚstackr  rÿ   rí   rk   r+   r*   r'   r†   Ú
dequantizer"   r‡   Ú
contiguousrŒ   ÚquantizeÚ
startswith)rr   Úweightsr,  Únew_weightsr›   rœ   rZ   rt   r&  ÚwkrÐ   r  Úprefixr)  r(  ÚeÚto_joinÚlayerÚ	quantizedÚhead_dimrY   r2  r3  r7  r8  r_   ÚwvÚ	wk_scalesÚ	wk_biasesÚ	wv_scalesÚ	wv_biasess   &&                             r?   ÚsanitizeÚModel.sanitize|  sm  € ò	0ð" ˆØ—M‘M–O‰DˆAØ�z‰z˜.×)Ò)Ø—y‘y °Ó4�Ø/6¸oÕ7MÕ/N×/SÑ/SÜ—I‘Ió0� 8�OÑ,ð  >Õ1Õ2�Ø/0˜D 8�OÑ,Ø/1°A­v� 8�OÓ,Ø—j‘j ×0Ô0°A·J±J¸×4OÔ4OØ!"˜A“ñ $ð ˆð ˆØ—M‘M–O‰DˆAØ! QÔ&Ø�	Ø—Y‘Y˜|¨RÓ0�Ø ��Ù  Ó3�Ø"(˜B“ØÖ%Ø!"˜A“ñ $ð ˆô �t—y‘y×2Ñ2×3ˆAØ$ Q CÐ(ˆFÛU‘�Û7�AØ ˜ °°°1°Q°CÐ8¸GÖCô &+¨4¯9©9×+EÑ+EÔ%Fó#á%F ð $ŸK™K¨6¨(°-À¸sÀ!ÀAÀ3ÀaÈÀsÐ(KÖLÙ%Fð  ð #ô GIÇhÂhÈwÓFW˜ (Ð*:¸1¸#¸Q¸q¸cÐ BÓCó 8ñ Vð % Q C zÐ2ˆFØ�Ð*Ð+¨wÖ6ØŸ
™
×)Ñ)¨!Õ,×6Ñ6×>Ñ>�Ø%˜hÐ&7Ð8¸GÑC�	Ø—K‘K 6 (Ð*;Ð <Ó=�ØŸ9™9×5Ñ5¸¿	¹	×8LÑ8LÕL�çØŸ9™9×1Ñ1�DØ$Ÿ[™[¨F¨8Ð3DÐ)EÓF�FØ$Ÿ[™[¨F¨8Ð3DÐ)EÓF�FàŸG™G B�K¨"Õ,°Õ5�DØ!%¨¯©°bÕ)9Õ!9�JÜŸšØ˜6 6°Àô�Að !ŸI™I×9Ñ9�	Ø—I‘I˜i¨°2Ó6�Ü—]’]Ø�aÐ5˜4Ÿ9™9×5Ñ5Ð5°qÐ8Õ9×BÑBÀ2ÀrÓJó�ô —]’] 1 Q¨¯	©	×(BÑ(BÑ(DÀaÐ%GÕ#HÓI�ßÜ/1¯{ª{Ø °*ô0Ñ,�B˜	 9ô 02¯{ª{Ø °*ô0Ñ,�B˜	 9ð ;D�G˜h oÐ6Ñ7Ø>G�G˜hÐ&9Ð:Ñ;Ø:C�G˜h oÐ6Ñ7Ø>G�G˜hÐ&9Ð:Ñ;Ø68˜6˜( /Ð2Ñ3Ø:<�˜(Ð"5Ð6Ô7ñ[ 4ðd  Ÿ™œô
á'‘�Ø—<‘<Ð 1×2ô à7LÐTUÑ7Uô ˆAŠDÙ'ò
ð 	
ùòW#ùóV
s   Æ#%Q:
ÑQ?Ñ'Q?Ñ0Q?c                óX   <€ V ^8„  d   QhRS[ S[P                  P                  ,          /# )r   râ   )r   r~   rã   ÚGroup)r=   r>   s   "€r?   r@   r  à  s#   ø€ ÷ >ñ >™8¡B§N¡N×$8Ñ$8Õ9ñ >rB   c                ó  aa€ T;'       g    \         P                  P                  4       pVP                  4       pVP	                  4       pV P
                  P                   EF  pVP                  P                  f3   \        VP                  P                  RVR7      VP                  n
        M1\        VP                  P                  RVR7      VP                  n        VP                  ;P                  V,          un        VP                  P                  pW5,          oSV,           oVV3R lpVP                  P                  P                  V4       VP                  P                  P                  V4       \        VP                  P                   RVR7      VP                  n        \#        VP$                  \&        4      '       d—   \        VP$                  P(                  RVR7      VP$                  n        \        VP$                  P*                  RVR7      VP$                  n        \        VP$                  P,                  RVR7      VP$                  n        EK  WP$                  n        \1        VP$                  P2                  P(                  RVR7       \1        VP$                  P2                  P*                  RVR7       \1        VP$                  P2                  P,                  RVR7       \1        VP$                  P4                  P(                  RVR7       \1        VP$                  P4                  P*                  RVR7       \1        VP$                  P4                  P,                  RVR7       EK  	  R # )Nzall-to-shardedrá   c                 ó   <€ V SS # r¯   rC   )ÚwÚehÚshs   &€€r?   Úshard_headsÚ Model.shard.<locals>.shard_headsó  s   ø€ Ø˜˜B�x�rB   zsharded-to-all)r~   rã   ÚinitÚsizeÚrankr  rÿ   rí   r(   r	   rd   rh   r_   rk   Úapplyrl   rm   Ú
isinstancerî   r¥   rª   r¬   r«   rÞ   r   rÝ   rÛ   )	rr   râ   ÚNrb  rM  r_   r^  r\  r]  s	   &&     @@r?   ÚshardÚModel.shardà  sŽ  ù€ Ø×.Ð.œŸ™×,Ñ,Ó.ˆØ�J‰J‹LˆØ�z‰z‹|ˆØ—Z‘Z×&Õ&ˆEà�‰×*Ñ*Ò2Ü)5Ø—O‘O×*Ñ*Ð,<ÀEô*�—‘Õ&ô ,8Ø—O‘O×,Ñ,Ð.>Àeô,�—‘Ô(ð �O‰O×%Ò%¨!Õ+Õ%ØŸ™×1Ñ1ˆIØÕ!ˆBØ�i•ˆBö ð �O‰O×#Ñ#×)Ñ)¨+Ô6Ø�O‰O×'Ñ'×-Ñ-¨kÔ:ä%1Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô
 ˜%Ÿ)™)¤]×3Ò3Ü&2Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	Ô#ô '3Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	Ô#ô %1Ø—I‘I×%Ñ%Ð'7¸uô%�—	‘	×!ð ,1—	‘	Ô(ÜØ—I‘I×,Ñ,×6Ñ6Ð8HÐPUõô Ø—I‘I×,Ñ,×6Ñ6Ð8HÐPUõô Ø—I‘I×,Ñ,×4Ñ4Ð6FÈeõô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈEõô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈEõô Ø—I‘I×(Ñ(×0Ñ0Ð2BÈ%÷ð óq 'rB   c                ó.   € V P                   P                  # r¯   )r  r	  )rr   s   &r?   rÿ   ÚModel.layers   s   € à�z‰z×)Ñ)Ð)rB   c                ó   € R  pV# )c                 ó   € R V 9  # )rÁ   rC   )r›   s   &r?   Ú	predicateÚ'Model.cast_predicate.<locals>.predicate&  s   € Ø,°AÑ5Ð5rB   rC   )rr   rl  s   & r?   Úcast_predicateÚModel.cast_predicate$  s   € ò	6ð ÐrB   )r  r  r  r   r¯   )rD   rE   rF   rG   r^   rž   rU  rf  Úpropertyrÿ   rn  rI   rJ   r¢   r£   s   @@r?   r  r  l  sZ   ù‡ € ÷Tó T÷!ò !òb
÷H>ò >ð@ ñ*ó ð*ð ñó ÷ð rB   r  )+ro   Údataclassesr   Ú	functoolsr   Útypingr   r   r   Úmlx.coreÚcorer~   Úmlx.nnrb   Úmlx.nn.layers.distributedr   r	   r
   Úactivationsr   rZ   r   r   r   Úmlar   Úpipeliner   Ú
rope_utilsr   Úswitch_layersr   r   ÚModulerL   r¥   ÚcompilerÉ   rË   r×   rè   rø   r  rC   rB   r?   Ú<module>r     så   ðó Ý !Ý ß &Ñ &å Ý ß PÑ På ß TÑ TÝ Ý #Ý 'Ý $ð ô!�ó !ó ð!ô@t#˜"Ÿ)™)ô t#ôn�B—I‘Iô ð( ‡�ñó ðôD
ˆb�i‰iô 
ô4!�B—I‘Iô !ôH˜RŸY™Yô ô<*�m R§Y¡Yô *ôZ}ˆB�I‰Iö }rB   