+
    UV-jËK  ã                   ó   € ^ RI t ^ RIHtHt ^ RIHt ^ RIHt ^ RI	H
t
 ^RIHtHtHt ^RIHt RR ltRR ltRR ltR	 t ! R
 R]P*                  4      t ! R R]P*                  4      t ! R R]P*                  4      t ! R R]P*                  4      t ! R R]P*                  4      t ! R R]P*                  4      t ! R R]P*                  4      t ! R R]P*                  4      t ! R R]P*                  4      tR# )é    N)ÚAnyÚOptional)Ú	SwitchGLU)ÚLanguageModelOutputÚcreate_attention_maskÚscaled_dot_product_attention©Ú
TextConfigc                 óÄ   € V\         P                  ! W0^,          \         P                  ,          ,          4      ,          ^\         P                  ! V4      ,          ,          # )é   )ÚmathÚlogÚpi)Únum_rotationsÚdimÚbaseÚmax_position_embeddingss   &&&&Út/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/deepseekocr/language.pyÚyarn_find_correction_dimr      sA   € ð ”$—(’(Ð2ÀaÕ6GÌ$Ï'É'Õ6QÕRÓSÕSØ	ŒD�HŠH�T‹NÕõð ó    c                 ó¾   € \         P                  ! \        WW44      4      p\         P                  ! \        WW44      4      p\	        V^ 4      \        Wb^,
          4      3# ©r   )r   Úfloorr   ÚceilÚmaxÚmin)Úlow_rotÚhigh_rotr   r   r   ÚlowÚhighs   &&&&&  r   Úyarn_find_correction_ranger!      sR   € ô �*Š*Ü  ¨tÓMó€Cô �9Š9Ü  °ÓNó€Dô ˆs�A‹;œ˜D¨¥'Ó*Ð*Ð*r   c                 ój   € V ^8:  d   R# RV,          \         P                  ! V 4      ,          R,           # )é   ç      ð?gš™™™™™¹?)r   r   )ÚscaleÚmscales   &&r   Úyarn_get_mscaler'   $   s(   € Ø�„zÙØ��<œ$Ÿ(š( 5›/Õ)¨CÕ/Ð/r   c                 óÄ   € W8X  d
   VR ,          p\         P                  ! V\         P                  R7      V ,
          W,
          ,          p\         P                  ! V^ ^4      # )gü©ñÒMbP?©Údtype)ÚmxÚarangeÚfloat32Úclip)Úmin_valÚmax_valr   Úlinear_funcs   &&& r   Úyarn_linear_ramp_maskr2   *   sF   € ØÔØ�5Õˆä—9’9˜S¬¯
©
Ô3°gÕ=À'ÕBSÕT€KÜ�7Š7�;  1Ó%Ð%r   c                   ó@   a a€ ] tR t^2t oRV 3R lltRR ltRtVtV ;t# )ÚDeepseekV2YarnRotaryEmbeddingc
           	     óü  <€ \         SV `  4        \        WH4      \        WI4      ,          V n        V\        P
                  ! ^ V^\        P                  R7      V,          ,          p
WC\        P
                  ! ^ V^\        P                  R7      V,          ,          ,          p\        VVVVV4      w  rÍR\        WÍV^,          4      ,
          pWº,          W¾,          V
^V,
          ,          ,           ,          V n	        R# )r   r)   r$   N)
ÚsuperÚ__init__r'   r&   r+   r,   r-   r!   r2   Ú_freqs)Úselfr   r   r   Úscaling_factorÚ original_max_position_embeddingsÚ	beta_fastÚ	beta_slowr&   Úmscale_all_dimÚ
freq_extraÚ
freq_interr   r    Ú	freq_maskÚ	__class__s   &&&&&&&&&&     €r   r7   Ú&DeepseekV2YarnRotaryEmbedding.__init__3   sÒ   ø€ ô 	‰ÑÔÜ% nÓ=ÄØóA
õ 
ˆŒð œbŸiši¨¨3°¼¿¹ÔDÀsÕJÕKˆ
Ø#Ü�IŠI�a˜˜a¤r§z¡zÔ2°SÕ8õ'
õ 
ˆ
ô /ØØØØØ,ó
‰	ˆð Ô/°¸3À!½8ÓDÕDˆ	Ø!Õ.ØÕ" Z°1°yµ=Õ%AÕAõ
ˆŽr   c           
     óÌ   € V P                   R 8w  d   V P                   V,          p\        P                  P                  VVP                  R,          RRR VV P
                  R7      # )r$   TN)Útraditionalr   r%   ÚoffsetÚfreqséÿÿÿÿ)r&   r+   ÚfastÚropeÚshaper8   )r9   ÚxrF   s   &&&r   Ú__call__Ú&DeepseekV2YarnRotaryEmbedding.__call__S   sV   € Ø�;‰;˜#ÔØ—‘˜a•ˆAÜ�w‰w�|‰|ØØ�G‰G�B�KØØØØØ—+‘+ð ó 
ð 	
r   )r8   r&   )é   é'  r$   i   é    r#   r#   r   r   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r7   rM   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©rB   Ú__classdict__s   @@r   r4   r4   2   s   ù‡ € ÷
÷@
ô 
r   r4   c                   óT   a a€ ] tR t^at oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚDeepseekV2Attentionc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úconfigr	   )Úformatr[   s   "€r   Ú__annotate__Ú DeepseekV2Attention.__annotate__b   s   ø€ ÷ Nñ N™zñ Nr   c                óÐ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  V n	        VP                  V n
        VP                  V n        VP                  V n        VP                  VP                  ,           V n        V P                  R,          V n        V P                  fE   \        P                   ! V P                  V P
                  V P                  ,          RR7      V n        M¤\        P                   ! V P                  V P                  VP$                  R7      V n        \        P(                  ! V P                  4      V n        \        P                   ! V P                  V P
                  V P                  ,          RR7      V n        \        P                   ! V P                  V P                  V P                  ,           VP$                  R7      V n        \        P(                  ! V P                  4      V n        \        P                   ! V P                  V P
                  V P                  V P                  ,
          V P                  ,           ,          RR7      V n        \        P                   ! V P
                  V P                  ,          V P                  VP$                  R7      V n        V P                  P6                  fI   \        P8                  ! V P                  V P                  P:                  V P                  R7      V n        R# V P                  P6                  P?                  R^ 4      pV P                  P6                  P?                  R^4      pV'       d«   \A        W24      pV P                  V,          V,          V n        R Uu/ uF<  pWPP                  P6                  9   g   K  WPP                  P6                  V,          bK>  	  pp\C        RRV P                  RV P                  R	VR
V P                  /VB V n        R# R# u upi )ç      à?NF©Úbias)rE   r   r>   Úfactorr   r   r:   r   ç      à¿)r;   r<   r=   r&   r>   © )"r6   r7   r`   Úhidden_sizeÚnum_attention_headsÚ	num_headsr   Ú
rope_thetaÚq_lora_rankÚqk_rope_head_dimÚkv_lora_rankÚ
v_head_dimÚqk_nope_head_dimÚ
q_head_dimr%   ÚnnÚLinearÚq_projÚattention_biasÚq_a_projÚRMSNormÚq_a_layernormÚq_b_projÚkv_a_proj_with_mqaÚkv_a_layernormÚ	kv_b_projÚo_projÚrope_scalingÚRoPEÚrope_traditionalrJ   Úgetr'   r4   )r9   r`   r>   r:   r&   ÚkeyÚrope_kwargsrB   s   &&     €r   r7   ÚDeepseekV2Attention.__init__b   sb  ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ×3Ñ3ˆŒØ'-×'EÑ'EˆÔ$Ø ×+Ñ+ˆŒØ!×-Ñ-ˆÔØ &× 7Ñ 7ˆÔØ"×/Ñ/ˆÔØ ×+Ñ+ˆŒØ &× 7Ñ 7ˆÔØ ×1Ñ1°F×4KÑ4KÕKˆŒà—_‘_ dÕ*ˆŒ
à×ÑÒ#ÜŸ)š)Ø× Ñ  $§.¡.°4·?±?Õ"BÈôˆD�Kô ŸIšIØ× Ñ  $×"2Ñ"2¸×9NÑ9NôˆDŒMô "$§¢¨D×,<Ñ,<Ó!=ˆDÔÜŸIšIØ× Ñ  $§.¡.°4·?±?Õ"BÈôˆDŒMô #%§)¢)Ø×ÑØ×Ñ × 5Ñ 5Õ5Ø×&Ñ&ô#
ˆÔô
 !Ÿjšj¨×):Ñ):Ó;ˆÔÜŸšØ×ÑØ�N‰NØ�‰ ×!6Ñ!6Õ6¸¿¹ÕHõJàô	
ˆŒô —i’iØ�N‰N˜TŸ_™_Õ,Ø×ÑØ×&Ñ&ô
ˆŒð �;‰;×#Ñ#Ò+ÜŸšØ×%Ñ%Ø ŸK™K×8Ñ8Ø—_‘_ôˆDŽIð "Ÿ[™[×5Ñ5×9Ñ9Ð:JÈAÓNˆNØ!Ÿ[™[×5Ñ5×9Ñ9¸(ÀAÓFˆNßÜ(¨ÓH�Ø!ŸZ™Z¨&Õ0°6Õ9�”
ñ ó
ñ ˜ð Ÿk™k×6Ñ6Ñ6ô 7�CŸ™×1Ñ1°#Õ6Ò6ñ ð ð 
ô :ñ Ø×-Ñ-ðà,0×,HÑ,Hðð $2ðð Ÿ™ð	ð
 "ñ�–	ñ ùò
s   Ï%Q#Ð!Q#c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   rL   ÚmaskÚcacheÚreturn©r+   Úarrayr   r   )ra   r[   s   "€r   rb   rc   ²   sH   ø€ ÷ *#ñ *#á�8‰8ð*#ñ ‘r—x‘xÕ ð*#ñ ™�}ð	*#ñ
 
�‰ñ*#r   c           	     óÄ  € VP                   w  rEpV P                  f   V P                  V4      pM/V P                  V P	                  V P                  V4      4      4      pVP                  WEV P                  V P                  4      P                  ^ ^^^4      p\        P                  ! WpP                  .RR7      w  r‰V P                  V4      p
\        P                  ! W P                  .RR7      w  r«VP                  WE^V P                  4      P                  ^ ^^^4      pV P!                  V P#                  V
4      4      pVP                  WEV P                  R4      P                  ^ ^^^4      p\        P                  ! WÀP                  .RR7      w  rÞVe…   V P%                  W“P&                  4      p	V P%                  W³P&                  4      p\        P(                  ! W°P                  ^R7      pVP+                  \        P,                  ! WÛ.RR7      V4      w  rþM]V P%                  V	4      p	V P%                  V4      p\        P(                  ! W°P                  ^R7      p\        P,                  ! WÛ.RR7      p\        P,                  ! W‰.RR7      p\/        VWþW0P0                  VR7      pVP                  ^ ^^^4      P                  WER4      pV P3                  V4      # )N©Úaxis©r%   rŠ   rH   )rK   ro   rw   r|   r{   ry   Úreshaperm   rt   Ú	transposer+   Úsplitrs   r}   rq   rp   r   r~   rJ   rF   ÚrepeatÚupdate_and_fetchÚconcatenater   r%   r€   )r9   rL   rŠ   r‹   ÚBÚLÚDÚqÚq_nopeÚq_peÚcompressed_kvÚk_peÚkvÚk_nopeÚvaluesÚkeysÚqueriesÚoutputs   &&&&              r   rM   ÚDeepseekV2Attention.__call__²   sG  € ð —'‘'‰ˆˆaà×ÑÒ#Ø—‘˜A“‰Aà—‘˜d×0Ñ0°·±¸qÓ1AÓBÓCˆAà�I‰I�a˜DŸN™N¨D¯O©OÓ<×FÑFÀqÈ!ÈQÐPQÓRˆÜ—x’x ×$9Ñ$9Ð#:ÀÔD‰ˆØ×/Ñ/°Ó2ˆÜ Ÿhšh }×7HÑ7HÐ6IÐPRÔSÑˆØ�|‰|˜A ! T×%:Ñ%:Ó;×EÑEÀaÈÈAÈqÓQˆØ�^‰^˜D×/Ñ/°Ó>Ó?ˆØ�Z‰Z˜˜dŸn™n¨bÓ1×;Ñ;¸A¸qÀ!ÀQÓGˆäŸš "×'<Ñ'<Ð&=ÀBÔG‰ˆàÒØ—9‘9˜T§<¡<Ó0ˆDØ—9‘9˜T§<¡<Ó0ˆDÜ—9’9˜T§>¡>¸Ô:ˆDØ ×1Ñ1Ü—’ ˜~°BÔ7¸ó‰LˆD�&ð —9‘9˜T“?ˆDØ—9‘9˜T“?ˆDÜ—9’9˜T§>¡>¸Ô:ˆDÜ—>’> 6 .°rÔ:ˆDä—.’. & °bÔ9ˆä-Ø�T 5·
±
Àô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r   )r`   rk   r~   r}   r   rq   r   rm   r€   r{   ry   r|   rt   ro   rw   rs   rp   rJ   rn   r%   rr   ©NNrR   rZ   s   @@r   r]   r]   a   s!   ù‡ € ÷Nó N÷`*#÷ *#ò *#r   r]   c                   óT   a a€ ] tR t^ßt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚLlamaAttentionc                ó    <€ V ^8„  d   QhRS[ /# r_   r	   )ra   r[   s   "€r   rb   ÚLlamaAttention.__annotate__à   s   ø€ ÷ 
ñ 
™zñ 
r   c                ó"  <€ \         SV `  4        VP                  pVP                  ;V n        pVP
                  ;V n        pVP                  V,          ;V n        pVR,          V n        VP                  '       d   VP                  pMRp\        P                  ! W#V,          VR7      V n        \        P                  ! W$V,          VR7      V n        \        P                  ! W$V,          VR7      V n        \        P                  ! W5,          W&R7      V n        VP                   e3   VP                   R,          R8X  d   ^VP                   R,          ,          M^p\        P"                  ! VVP$                  VP&                  VR7      V n        R# )	re   Frf   NÚtypeÚlinearrh   )rE   r   r%   ri   )r6   r7   rk   rl   Ún_headsÚnum_key_value_headsÚ
n_kv_headsÚhead_dimr%   rx   ru   rv   rw   Úk_projÚv_projr€   r�   r‚   rƒ   rn   rJ   )	r9   r`   r   r°   r²   r³   rx   Ú
rope_scalerB   s	   &&      €r   r7   ÚLlamaAttention.__init__à   s?  ø€ Ü‰ÑÔà× Ñ ˆØ!'×!;Ñ!;Ð;ˆŒ�wØ'-×'AÑ'AÐAˆŒ˜*à#)×#5Ñ#5¸Õ#@Ð@ˆŒ˜à˜t•^ˆŒ
Ø× × Ð Ø#×2Ñ2‰Nà"ˆNä—i’i ¨xÕ%7¸nÔMˆŒÜ—i’i °(Õ%:ÀÔPˆŒÜ—i’i °(Õ%:ÀÔPˆŒÜ—i’i Õ 2°CÔMˆŒð ×"Ñ"Ò.Ø×#Ñ# FÕ+¨xÔ7ð �×#Ñ# HÕ-Ö-ð ð	 	ô —G’GØØ×/Ñ/Ø×"Ñ"Øô	
ˆŽ	r   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# r‰   r�   )ra   r[   s   "€r   rb   r¬     sH   ø€ ÷ #ñ #á�8‰8ð#ñ ‘r—x‘xÕ ð#ñ ™�}ð	#ñ
 
�‰ñ#r   c           	     ó  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pVP	                  WEV P
                  R4      P                  ^ ^^^4      pVP	                  WEV P                  R4      P                  ^ ^^^4      pV	P	                  WEV P                  R4      P                  ^ ^^^4      p	VeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      P	                  WER4      p
V P                  V
4      # )r#   )rF   r’   rH   )rK   rw   r´   rµ   r“   r°   r”   r²   rJ   rF   r—   r   r%   r€   )r9   rL   rŠ   r‹   r™   rš   r›   r¥   r¤   r£   r¦   s   &&&&       r   rM   ÚLlamaAttention.__call__  sS  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆð —/‘/ !¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°·
±
Àô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r   )	r³   r´   r°   r²   r€   rw   rJ   r%   rµ   r¨   rR   rZ   s   @@r   rª   rª   ß   s   ù‡ € ÷
ó 
÷B#÷ #ò #r   rª   c                   óH   a a€ ] tR tRt oRV3R lV 3R llltR tRtVtV ;t# )ÚDeepseekV2MLPi   c                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r   r`   rk   Úintermediate_size©r
   Úint)ra   r[   s   "€r   rb   ÚDeepseekV2MLP.__annotate__!  s*   ø€ ÷ Yñ YÙ ðYÙ/2ðYÙNQñYr   c                ó¸  <€ \         SV `  4        Wn        Vf   VP                  MTV n        Vf   VP                  MTV n        \
        P                  ! V P                  V P                  RR7      V n        \
        P                  ! V P                  V P                  RR7      V n        \
        P                  ! V P                  V P                  RR7      V n	        R # )NFrf   )
r6   r7   r`   rk   r¾   ru   rv   Ú	gate_projÚup_projÚ	down_proj)r9   r`   rk   r¾   rB   s   &&&&€r   r7   ÚDeepseekV2MLP.__init__!  s¨   ø€ ô 	‰ÑÔØŒØ1<Ò1D˜6×-Ò-È+ˆÔà(9Ò(AˆF×$Ò$ÐGXð 	Ôô Ÿš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŽr   c                óš   € V P                  \        P                  ! V P                  V4      4      V P	                  V4      ,          4      pV# ©N)rÅ   ru   ÚsilurÃ   rÄ   )r9   rL   rÅ   s   && r   rM   ÚDeepseekV2MLP.__call__/  s5   € Ø—N‘N¤2§7¢7¨4¯>©>¸!Ó+<Ó#=ÀÇÁÈQÃÕ#OÓPˆ	ØÐr   )r`   rÅ   rÃ   rk   r¾   rÄ   r¨   rR   rZ   s   @@r   r¼   r¼      s   ù‡ € ÷Yõ Y÷ò r   r¼   c                   óD   a a€ ] tR tRt oV3R lV 3R lltR tRtVtV ;t# )ÚMoEGatei4  c                ó    <€ V ^8„  d   QhRS[ /# r_   r	   )ra   r[   s   "€r   rb   ÚMoEGate.__annotate__5  s   ø€ ÷ Lñ L™zñ Lr   c                óì  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  V n	        VP                  V n
        V P                  R 8X  d&   \        P                  ! V P                  4      V n        \        P                  ! V P                  VP                  34      V n        R# )Únoaux_tcN)r6   r7   r`   Úscoring_funcÚnum_experts_per_tokÚtop_kÚn_routed_expertsÚrouted_scaling_factorÚtopk_methodÚn_groupÚ
topk_groupr+   ÚzerosÚe_score_correction_biasrk   Úweight©r9   r`   rB   s   &&€r   r7   ÚMoEGate.__init__5  s¶   ø€ Ü‰ÑÔØŒØ"×/Ñ/ˆÔØ×/Ñ/ˆŒ
Ø &× 7Ñ 7ˆÔØ%+×%AÑ%AˆÔ"Ø!×-Ñ-ˆÔØ—~‘~ˆŒØ ×+Ñ+ˆŒØ×Ñ˜zÔ)Ü+-¯8ª8°T×5JÑ5JÓ+LˆDÔ(Ü—h’h × 5Ñ 5°v×7IÑ7IÐJÓKˆŽr   c                óX	  € VP                   R ,          w  r#\        V RR4      pWP                  P                  ,          pV P                  R8X  d   \
        P                  ! VRRR7      pM@V P                  R8X  d   \
        P                  ! V4      pM\        RV P                   24      hV P                  R8X  dN   TpV P                  p\
        P                  ! Wx) RR	7      R
V) R13,          p	\
        P                  ! WyRR7      p
EM�V P                  R8X  Edf   V P                  V P                  ,          pV P                  '       d   V P                  MV P                  pVP!                  W#V P                  4      pV P"                  P%                  \
        P&                  4      p\
        P                   ! V^^V P                  34      pWÞ,           pVP!                  W#V P                  V4      p\
        P(                  ! VRR7      pV^8¼  d   VR
RR13,          pMTp\
        P*                  ! VRR7      pWÀP                  8  Ed   V P                  V,
          p\
        P                  ! VVRR	7      R
V) R13,          p\
        P                   ! \
        P,                  ! V P                  \
        P.                  R7      ^^V P                  34      p\
        P0                  ! VP                   \
        P2                  R7      p\5        V4       FP  p\
        P6                  ! VR
V3,          RR7      p\
        P8                  ! V\
        P:                  ! VV4      4      pKR  	  M0\
        P<                  ! VP                   \
        P2                  R7      p\
        P6                  ! VRR7      p\
        P>                  ! VV\
        P@                  ! V4      4      pVP!                  W#R4      pTpVP                   R,          p\C        VV P                  ,
          ^ 4      p\
        P                  ! VVRR	7      R
V P                  ) R13,          P%                  \
        P.                  4      p	\
        P                  ! VV	RR7      p
M\        RV P                   24      hW PD                  ,          p
Wš3# ):Nr   NÚ_layer_indexNÚsoftmaxT)r‘   ÚpreciseÚsigmoidzUnknown scoring function: Úgreedy)Úkthr‘   .r�   rÐ   r)   zUnknown topk method: rH   éþÿÿÿ)#rK   ÚgetattrrÛ   ÚTrÑ   r+   rà   râ   Ú
ValueErrorrÖ   rÓ   ÚargpartitionÚtake_along_axisrÔ   r×   rØ   r“   rÚ   Úastyper-   ÚsortÚsumr,   Úint32rÙ   Úbool_ÚrangeÚexpand_dimsÚ
logical_orÚequalÚonesÚwhereÚ
zeros_liker   rÕ   ) r9   rL   ÚbszÚseq_lenÚ	layer_idxÚgatesÚscoresÚflat_scoresÚkÚindsÚscores_selectedÚexperts_per_grouprØ   Ú	scores_btrg   Ú	correctedÚcorrected_groupsÚsorted_groupsÚ
top_valuesÚgroup_scoresÚ	kth_groupÚ	group_idxÚ
group_axisrŠ   ÚslotÚidxÚmask_expandedÚcorrected_maskedÚcorrected_flatÚraw_flatÚtotal_expertsÚ	kth_values    &&                              r   rM   ÚMoEGate.__call__C  s˜  € Ø—w‘w˜r•{‰ˆä˜D .°$Ó7ˆ	à—K‘K—M‘MÕ!ˆà×Ñ 	Ô)Ü—Z’Z ¨B¸Ô=‰FØ×Ñ )Ô+Ü—Z’Z Ó&‰FäÐ9¸$×:KÑ:KÐ9LÐMÓNÐNà×Ñ˜xÔ'Ø ˆKØ—
‘
ˆAÜ—?’? ;°B¸RÔ@ÀÀqÀbÁcÀÕJˆDÜ ×0Ò0°ÈÔLŠOà×Ñ Õ+Ø $× 5Ñ 5¸¿¹Õ EÐØ,0¯O¯O¨O˜ŸšÀÇÁˆJàŸ™ s°T×5JÑ5JÓKˆIØ×/Ñ/×6Ñ6´r·z±zÓBˆDÜ—:’:˜d Q¨¨4×+@Ñ+@Ð$AÓBˆDØ!Õ(ˆIØ(×0Ñ0Ø˜dŸl™lÐ,=ó Ðô ŸGšGÐ$4¸2Ô>ˆMØ  AÔ%Ø*¨3°±¨8Õ4‘
à*�
ÜŸ6š6 *°2Ô6ˆLàŸL™LÕ(Ø ŸL™L¨:Õ5�	ÜŸOšO¨L¸iÈbÔQØ˜*˜™Ð%õ�	ô  ŸZšZÜ—I’I˜dŸl™l´"·(±(Ô;¸aÀÀDÇLÁLÐ=Qó�
ô —x’x × 2Ñ 2¼"¿(¹(ÔC�Ü! *Ö-�DÜŸ.š.¨°3¸°9Õ)=ÀBÔG�CÜŸ=š=¨¬r¯xªx¸¸ZÓ/HÓI’Dò .ô —w’w˜|×1Ñ1¼¿¹ÔB�äŸNšN¨4°bÔ9ˆMÜ!ŸxšxØØ Ü—’Ð.Ó/ó Ðð .×5Ñ5°cÀBÓGˆNØ ˆHà*×0Ñ0°Õ4ˆMÜ˜M¨D¯J©JÕ6¸Ó:ˆIÜ—?’? >°yÀrÔJØ�d—j‘j�[‘]Ð"õç‰f”R—X‘XÓð ô !×0Ò0°¸4ÀbÔI‰Oô Ð4°T×5EÑ5EÐ4FÐGÓHÐHà)×,FÑ,FÕFˆØÐ$Ð$r   )
r`   rÚ   r×   rÔ   rÕ   rÑ   rÓ   rØ   rÖ   rÛ   rR   rZ   s   @@r   rÌ   rÌ   4  s   ù‡ € ÷Ló L÷K%ò K%r   rÌ   c                   óD   a a€ ] tR tRt oV3R lV 3R lltR tRtVtV ;t# )ÚDeepseekV2MoEi‘  c                ó    <€ V ^8„  d   QhRS[ /# r_   r	   )ra   r[   s   "€r   rb   ÚDeepseekV2MoE.__annotate__’  s   ø€ ÷ ñ ™zñ r   c                óN  <€ \         SV `  4        Wn        VP                  V n        \	        VP
                  VP                  VP                  4      V n        \        V4      V n
        VP                  e1   VP                  VP                  ,          p\        WR7      V n        R # R # )N)r`   r¾   )r6   r7   r`   rÒ   r   rk   Úmoe_intermediate_sizerÔ   Ú
switch_mlprÌ   ÚgateÚn_shared_expertsr¼   Úshared_experts)r9   r`   r¾   rB   s   && €r   r7   ÚDeepseekV2MoE.__init__’  s‹   ø€ Ü‰ÑÔØŒØ#)×#=Ñ#=ˆÔ Ü#Ø×Ñ × <Ñ <¸f×>UÑ>Uó
ˆŒô ˜F“OˆŒ	Ø×"Ñ"Ò.Ø &× <Ñ <¸v×?VÑ?VÕ VÐÜ"/Øô#ˆDÖñ /r   c                óê   € V P                  V4      w  r#V P                  W4      pWCR,          ,          P                  RR7      pV P                  P                  e   W@P                  V4      ,           pV# ).r�   ).Nrå   )r  r  rí   r`   r  r  )r9   rL   rþ   rû   Úys   &&   r   rM   ÚDeepseekV2MoE.__call__¡  sb   € Ø—y‘y “|‰ˆØ�O‰O˜AÓ$ˆØ˜	Õ"Õ"×'Ñ'¨RÐ'Ó0ˆØ�;‰;×'Ñ'Ò3Ø×'Ñ'¨Ó*Õ*ˆAàˆr   )r`   r  rÒ   r  r  rR   rZ   s   @@r   r  r  ‘  s   ù‡ € ÷ó ÷ò r   r  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚDeepseekV2DecoderLayeri«  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   r`   rù   r¿   )ra   r[   s   "€r   rb   Ú#DeepseekV2DecoderLayer.__annotate__¬  s   ø€ ÷ 
ñ 
™zð 
±cñ 
r   c                ó  <€ \         SV `  4        VP                  V n        V P                  R 8X  d   \        V4      M
\	        V4      V n        VP                  e3   W!P                  8¼  d#   W!P                  ,          ^ 8X  d   \        V4      M
\        V4      V n        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  R7      V n        R# )r]   N©Úeps)r6   r7   Ú	attn_typer]   rª   Ú	self_attnrÔ   Úfirst_k_dense_replaceÚmoe_layer_freqr  r¼   Úmlpru   rz   rk   Úrms_norm_epsÚinput_layernormÚpost_attention_layernorm)r9   r`   rù   rB   s   &&&€r   r7   ÚDeepseekV2DecoderLayer.__init__¬  sÌ   ø€ Ü‰ÑÔØ×)Ñ)ˆŒð �~‰~Ð!6Ô6ô   Ô'ä Ó'ð 	Œð ×'Ñ'Ò3Ø×!=Ñ!=Ô=Ø× 5Ñ 5Õ5¸Ô:ô	 ˜&Ô!ô ˜vÓ&ð 	Œô  "Ÿzšz¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(*¯
ª
Ø×Ñ F×$7Ñ$7ô)
ˆÖ%r   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# r‰   r�   )ra   r[   s   "€r   rb   r$  Â  sH   ø€ ÷ 
ñ 
á�8‰8ð
ñ ‘r—x‘xÕ ð
ñ ™�}ð	
ñ
 
�‰ñ
r   c                ó¨   € V P                  V P                  V4      W#4      pW,           pV P                  V P                  V4      4      pWT,           pV# rÈ   )r)  r.  r,  r/  )r9   rL   rŠ   r‹   ÚrÚhÚouts   &&&&   r   rM   ÚDeepseekV2DecoderLayer.__call__Â  sK   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ�EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�eˆØˆ
r   )r(  r.  r,  r/  r)  r¨   rR   rZ   s   @@r   r"  r"  «  s   ù‡ € ÷
ó 
÷,
÷ 
ò 
r   r"  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚDeepseekV2ModeliÏ  c                ó    <€ V ^8„  d   QhRS[ /# r_   r	   )ra   r[   s   "€r   rb   ÚDeepseekV2Model.__annotate__Ð  s   ø€ ÷ Lñ L™zñ Lr   c                óx  <€ \         SV `  4        VP                  V n        \        P                  ! VP                  VP
                  4      V n        \        VP                  4       Uu. uF  p\        W4      NK  	  upV n
        \        P                  ! VP
                  VP                  R 7      V n        R# u upi )r&  N)r6   r7   Ú
vocab_sizeru   Ú	Embeddingrk   Úembed_tokensrð   Únum_hidden_layersr"  Úlayersrz   r-  Únorm)r9   r`   r  rB   s   && €r   r7   ÚDeepseekV2Model.__init__Ð  s�   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜŸLšL¨×):Ñ):¸F×<NÑ<NÓOˆÔô ˜V×5Ñ5Ô6ó
á6�ô # 6Ö/Ù6ñ
ˆŒô —J’J˜v×1Ñ1°v×7JÑ7JÔKˆŽ	ùò	
s   Á(B7c          
      ó¸   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[,          RS[ P                  /# )r   rL   rŠ   Úinputs_embedsr‹   rŒ   r�   )ra   r[   s   "€r   rb   r:  Ú  s[   ø€ ÷ ñ á�8‰8ðñ ‘r—x‘xÕ ðñ  ¡§¡Õ)ð	ñ
 ™�}ðñ 
�‰ñr   c                ó
  € Vf   V P                  V4      pMTpVf   R .\        V P                  4      ,          p\        WT^ ,          4      p\	        V P                  V4       F  w  rgV! WRV4      pK  	  V P                  V4      # rÈ   )r>  Úlenr@  r   ÚziprA  )r9   rL   rŠ   rD  r‹   r4  ÚlayerÚcs   &&&&&   r   rM   ÚDeepseekV2Model.__call__Ú  sz   € ð Ò Ø×!Ñ! !Ó$‰AàˆAàŠ=Ø�FœS §¡Ó-Õ-ˆEä$ Q¨a­Ó1ˆä˜DŸK™K¨Ö/‰HˆEÙ�a˜qÓ!ŠAñ 0ð �y‰y˜‹|Ðr   )r>  r@  rA  r<  ©NNNrR   rZ   s   @@r   r8  r8  Ï  s    ù‡ € ÷Ló L÷÷ ò r   r8  c                   óŠ   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltR t]R 4       t]R 4       t	]R	 4       t
R
tVtV ;t# )ÚLanguageModeliò  c                ó    <€ V ^8„  d   QhRS[ /# r_   r	   )ra   r[   s   "€r   rb   ÚLanguageModel.__annotate__ó  s   ø€ ÷ Tñ T™zñ Tr   c                óÖ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        \        P                  ! VP                  VP                  R R7      V n
        R# )Frf   N)r6   r7   r`   Ú
model_typer8  Úmodelru   rv   rk   r<  Úlm_headrÜ   s   &&€r   r7   ÚLanguageModel.__init__ó  sL   ø€ Ü‰ÑÔØŒØ ×+Ñ+ˆŒÜ$ VÓ,ˆŒ
Ü—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŽr   c          	      óž   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[S[,          /# )r   ÚinputsrD  rŠ   r‹   r�   )ra   r[   s   "€r   rb   rO  ú  sM   ø€ ÷ 
/ñ 
/á—‘ð
/ñ  ¡§¡Õ)ð
/ñ ‘r—x‘xÕ ð	
/ñ
 ™�}ñ
/r   c                ób   € V P                  WW$R 7      pV P                  V4      p\        VR7      # ))rŠ   rD  r‹   )Úlogits)rR  rS  r   )r9   rV  rD  rŠ   r‹   Úkwargsr5  s   &&&&&, r   rM   ÚLanguageModel.__call__ú  s/   € ð �j‰j˜¸-ˆjÓUˆØ�l‰l˜3ÓˆÜ"¨#Ô.Ð.r   c                ó€  € \        V P                  P                  4       F•  pR V 2pR F‡  w  rER F|  pV RV RV 2V9   g   K  \        V P                  P                  4       Uu. uF  pVP	                  V RV RV RV 24      NK!  	  pp\
        P                  ! V4      W RV RV 2&   K~  	  K‰  	  K—  	  V# u upi )zlanguage_model.model.layers.z.mlp.experts.0.Ú.z.mlp.experts.z.mlp.switch_mlp.))Úw1rÃ   )Úw2rÅ   )Úw3rÄ   )rÛ   ÚscalesÚbiases)rð   r`   r?  rÔ   Úpopr+   Ústack)	r9   ÚweightsÚlÚprefixÚnÚmrý   ÚeÚto_joins	   &&       r   ÚsanitizeÚLanguageModel.sanitize  sÛ   € Ü�t—{‘{×4Ñ4Ö5ˆAØ3°A°3Ð7ˆFÛU‘�Û7�AØ ˜ °°°1°Q°CÐ8¸GÖCô &+¨4¯;©;×+GÑ+GÔ%Hó#á%H ð $ŸK™K¨6¨(°-À¸sÀ!ÀAÀ3ÀaÈÀsÐ(KÖLÙ%Hð  ð #ô GIÇhÂhÈwÓFW˜ (Ð*:¸1¸#¸Q¸q¸cÐ BÓCó 8ó Vñ 6ð ˆùò#s   Á&%B;
c                ó.   € V P                   P                  # rÈ   )rR  r@  ©r9   s   &r   r@  ÚLanguageModel.layers  s   € à�z‰z× Ñ Ð r   c                ó(  € V P                   P                  R 8X  dH   V P                   P                  V P                   P                  ,           V P                   P                  3# V P                   P
                  V P                   P                  ,          # )r]   )r`   r(  rs   rp   rr   rk   r±   rn  s   &r   r³   ÚLanguageModel.head_dim  si   € à�;‰;× Ñ Ð$9Ô9à—‘×,Ñ,¨t¯{©{×/KÑ/KÕKØ—‘×&Ñ&ðð ð
 —;‘;×*Ñ*¨d¯k©k×.MÑ.MÕMÐMr   c                ó.   € V P                   P                  # rÈ   )r`   r±   rn  s   &r   r²   ÚLanguageModel.n_kv_heads!  s   € à�{‰{×.Ñ.Ð.r   )r`   rS  rR  rQ  rK  )rS   rT   rU   rV   r7   rM   rk  Úpropertyr@  r³   r²   rW   rX   rY   rZ   s   @@r   rM  rM  ò  sc   ù‡ € ÷Tó T÷
/ò 
/òð ñ!ó ð!ð ñNó ðNð ñ/ó ÷/ð /r   rM  )rP   rO   )r#   r#   )r   Útypingr   r   Úmlx.coreÚcorer+   Úmlx.nnru   Úmlx_lm.models.switch_layersr   r   r   r   r   r`   r
   r   r!   r'   r2   ÚModuler4   r]   rª   r¼   rÌ   r  r"  r8  rM  rj   r   r   Ú<module>r{     sÑ   ðÛ ß  å Ý Ý 1÷ñ õ
 ôô	+ô0ò&ô,
 B§I¡Iô ,
ô^{#˜"Ÿ)™)ô {#ô|>#�R—Y‘Yô >#ôB�B—I‘Iô ô(Z%ˆb�i‰iô Z%ôz�B—I‘Iô ô4!˜RŸY™Yô !ôH �b—i‘iô  ôF1/�B—I‘Iö 1/r   