+
    TV-jí+  ã                   óè  € ^ RI Ht ^ RIHtHtHtHtHt ^ RIH	t
 ^ RIHt ^RIHt ^RIHtHtHt ^RIHt ] ! R R]4      4       tR	 t ! R
 R]P.                  4      t ! R R]P.                  4      t ! R R]P.                  4      t ! R R]P.                  4      t ! R R]P.                  4      t ! R R]P.                  4      t ! R R]P.                  4      t ! R R]P.                  4      tR# )é    )Ú	dataclass)ÚAnyÚDictÚOptionalÚTupleÚUnionN)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Ú	SwitchGLUc                   ó>   a € ] tR t^t o RtRtRtR tV 3R ltRt	V t
R# )Ú	ModelArgsNFc                óÔ   a € S P                   '       dU   R R0p\        ;QJ d    V 3R lV 4       F  '       d   K   RM	  RM! V 3R lV 4       4      '       g   \        RV 24      hR# R# )ÚfactorÚtypec              3   ó@   <"  € T F  qSP                   9   x € K  	  R # 5i©N)Úrope_scaling)Ú.0ÚkeyÚselfs   & €Úf/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/hunyuan.pyÚ	<genexpr>Ú*ModelArgs.__post_init__.<locals>.<genexpr>)   s   øé € ÐI¹=°C˜d×/Ñ/Ö/»=ùs   ƒFTzrope_scaling must contain keys N)r   ÚallÚ
ValueError)r   Úrequired_keyss   f r   Ú__post_init__ÚModelArgs.__post_init__%   sX   ø€ à××ÐØ% vÐ.ˆMß“3ÔI¹=ÓI—3—3’3ÔI¹=ÓI×IÒIÜ Ð#BÀ=À/Ð!RÓSÐSñ Jñ ó    c                ój  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   ^;R&   S[S[S[S[3,          ,          ;R&   S[S[S[ S[S[S[ 3,          3,          ,          ;R&   S[;R&   # )é   Ú
model_typeÚ
vocab_sizeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚnum_key_value_headsÚattention_biasÚmoe_topkÚnum_expertsÚnum_shared_expertÚuse_mixed_mlp_moeÚuse_qk_normÚrms_norm_epsÚ
rope_thetaÚuse_claÚcla_share_factorÚmoe_intermediate_sizer   Útie_word_embeddings)ÚstrÚintÚboolÚfloatr   r   Úlistr   )ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚModelArgs.__annotate__   s)  ø‡ ‚ á�Oñ ñ �Oñ ñ Ññ	 ñ
 Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ �Mñ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ  Ññ! ñ" �Mñ# ð$ Ññ% ñ& $¡E©#©t¨)Õ$4Õ5Ñ<ñ' ñ( ™4¡¡U©5±#¨:Õ%6Ð 6Õ7Õ8Ñ?ñ) ñ* Ñ%ò+ r!   © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r5   r   r6   r   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r=   s   @r   r   r      s%   ø‡ € ð& 9=ÐØ;?€LØ %ÐòT÷/ ƒ r!   r   c                 óB   € \        V \        4      '       d	   W,          # V # r   )Ú
isinstancer;   )ÚargÚidxs   &&r   Ú_int_or_listrL   -   s   € Ü�#”t×ÒØ�xˆØ€Jr!   c                   óX   a a€ ] tR t^3t oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚDynamicNTKAlphaRoPEc                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r#   ÚdimsÚbaseÚscaling_alpha)r8   r:   )r<   r=   s   "€r   r>   Ú DynamicNTKAlphaRoPE.__annotate__4   s.   ø€ ÷ 	Gñ 	Gáð	Gñ ð	Gñ ñ		Gr!   c                óæ   <€ \         SV `  4        Wn        W#W^,
          ,          ,          ,          pV\        P                  ! ^ V P                  ^4      V P                  ,          ,          V n        R# )r#   N)ÚsuperÚ__init__rP   ÚmxÚarangeÚ_freqs)r   rP   rQ   rR   Ú	__class__s   &&&&€r   rV   ÚDynamicNTKAlphaRoPE.__init__4   sO   ø€ ô 	‰ÑÔØŒ	Ø¨°qµÕ(9Õ:Õ:ˆØœrŸyšy¨¨D¯I©I°qÓ9¸D¿I¹IÕEÕFˆŽr!   c                ó    <€ V ^8„  d   QhRS[ /# )r#   Úoffset)r8   )r<   r=   s   "€r   r>   rS   ?   s   ø€ ÷ 	
ñ 	
¡#ñ 	
r!   c           
     óv   € \         P                  P                  VV P                  R RRVV P                  R7      # )FNç      ð?)ÚtraditionalrQ   Úscaler]   Úfreqs)rW   ÚfastÚroperP   rY   )r   Úxr]   s   &&&r   Ú__call__ÚDynamicNTKAlphaRoPE.__call__?   s9   € Ü�w‰w�|‰|ØØ�I‰IØØØØØ—+‘+ð ó 
ð 	
r!   )rY   rP   )i'  r_   ©r   ©	rA   rB   rC   rD   rV   rf   rF   rG   Ú__classcell__©rZ   r=   s   @@r   rN   rN   3   s    ù‡ € ÷	Gõ 	G÷	
÷ 	
ò 	
r!   rN   c                   óT   a a€ ] tR t^Kt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r#   Úkv_projÚargs)r9   r   )r<   r=   s   "€r   r>   ÚAttention.__annotate__L   s   ø€ ÷ 
ñ 
¡ð 
©Iñ 
r!   c                óž  <€ \         SV `  4        VP                  pVP                  ;V n        pVP
                  f   Q hVP
                  ;V n        pVP                  V,          pVR,          V n        \        P                  ! W4V,          VP                  R7      V n        V'       d]   \        P                  ! W5V,          VP                  R7      V n        \        P                  ! W5V,          VP                  R7      V n        \        P                  ! WF,          W2P                  R7      V n        VP                  V n        V P                  '       dK   \        P                   ! WbP"                  4      V n        \        P                   ! WbP"                  4      V n        \)        VVP*                  VP,                  R,          R7      V n        R # )N©ÚbiasÚalpha)rQ   rR   g      à¿)rU   rV   r&   r)   Ún_headsr*   Ú
n_kv_headsra   ÚnnÚLinearr+   Úq_projÚk_projÚv_projÚo_projr0   ÚRMSNormr1   Úquery_layernormÚkey_layernormrN   r2   r   rd   )r   ro   rp   Údimrv   rw   Úhead_dimrZ   s   &&&    €r   rV   ÚAttention.__init__L   sU  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ×'Ñ'Ò3Ð3Ð3Ø'+×'?Ñ'?Ð?ˆŒ˜*à×#Ñ# wÕ.ˆØ˜t•^ˆŒ
Ü—i’i ¨xÕ%7¸d×>QÑ>QÔRˆŒßÜŸ)š)Ø (Õ*°×1DÑ1DôˆDŒKô Ÿ)š)Ø (Õ*°×1DÑ1DôˆDŒKô —i’i Õ 2°C×>QÑ>QÔRˆŒØ×+Ñ+ˆÔØ××ÐÜ#%§:¢:¨h×8IÑ8IÓ#JˆDÔ Ü!#§¢¨H×6GÑ6GÓ!HˆDÔä'ØØ—‘Ø×+Ñ+¨GÕ4ô
ˆŽ	r!   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# )r#   re   ÚmaskÚcacheÚreturn)rW   Úarrayr   r   )r<   r=   s   "€r   r>   rq   j   sH   ø€ ÷ #.ñ #.á�8‰8ð#.ñ ‘r—x‘xÕ ð#.ñ ™�}ð	#.ñ 
�‰ñ#.r!   c           	     ó`  € VP                   w  rVpV P                  V4      pVf&   V P                  V4      V P                  V4      r©Wš3pMVw  ršVP	                  WVV P
                  R4      P                  ^ ^^^4      pV	P	                  WVV P                  R4      P                  ^ ^^^4      p	V
P	                  WVV P                  R4      P                  ^ ^^^4      p
V'       d   VP                  M^ pV P                  W‹R7      pV P                  W›R7      p	V P                  '       d#   V P                  V4      pV P                  V	4      p	Ve   VP                  Wš4      w  rš\        W‰W£V P                  VR7      pVP                  ^ ^^^4      P	                  WVR4      pV P!                  V4      V3# )N)r]   )r†   ra   r…   éÿÿÿÿ)Úshaperz   r{   r|   Úreshaperv   Ú	transposerw   r]   rd   r0   r   r€   Úupdate_and_fetchr   ra   r}   )r   re   r…   r†   Ú	kv_statesÚBÚLÚDÚqueriesÚkeysÚvaluesr]   Úoutputs   &&&&&        r   rf   ÚAttention.__call__j   s�  € ð —'‘'‰ˆˆaà—+‘+˜a“.ˆØÒØŸ;™; q›>¨4¯;©;°q«>�&Ø˜‰Ià$‰LˆDð —/‘/ !¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆç!&�—’¨AˆØ—)‘)˜G�)Ó3ˆØ�y‰y˜ˆyÓ-ˆØ××ÐØ×*Ñ*¨7Ó3ˆGØ×%Ñ% dÓ+ˆDàÒØ ×1Ñ1°$Ó?‰LˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó" IÐ-Ð-r!   )r{   r€   rv   rw   r}   rz   r   rd   ra   r0   r|   ©NNNri   rk   s   @@r   rm   rm   K   s   ù‡ € ÷
ó 
÷<#.÷ #.ò #.r!   rm   c                   óD   a a€ ] tR t^�t oV 3R ltV3R lR ltRtVtV ;t# )ÚMLPc                óÒ   <€ \         SV `  4        \        P                  ! WR R7      V n        \        P                  ! W!R R7      V n        \        P                  ! WR R7      V n        R# ©Frs   N)rU   rV   rx   ry   Ú	gate_projÚ	down_projÚup_proj)r   r�   Ú
hidden_dimrZ   s   &&&€r   rV   ÚMLP.__init__‘   sD   ø€ Ü‰ÑÔÜŸš 3¸Ô?ˆŒÜŸš :¸Ô?ˆŒÜ—y’y °uÔ=ˆŽr!   c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r#   r‡   ©rW   rˆ   )r<   r=   s   "€r   r>   ÚMLP.__annotate__—   s   ø€ ÷ Jñ J™RŸX™Xñ Jr!   c                ót   € V P                  \        V P                  V4      V P                  V4      4      4      # r   )rž   r	   r�   rŸ   ©r   re   s   &&r   rf   ÚMLP.__call__—   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr!   )rž   r�   rŸ   ri   rk   s   @@r   rš   rš   �   s   ù‡ € õ>÷J÷ Jð Jr!   rš   c                   óD   a a€ ] tR t^›t oV 3R ltV3R lR ltRtVtV ;t# )ÚGatec                ó^   <€ \         SV `  4        \        P                  ! WR R7      V n        R# rœ   )rU   rV   rx   ry   Úwg)r   r�   r-   rZ   s   &&&€r   rV   ÚGate.__init__œ   s    ø€ Ü‰ÑÔÜ—)’)˜C°5Ô9ˆŽr!   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r£   r¤   )r<   r=   s   "€r   r>   ÚGate.__annotate__    s   ø€ ÷ ñ ™RŸX™Xñ r!   c                ó$   € V P                  V4      # r   ©r¬   r§   s   &&r   rf   ÚGate.__call__    s   € Ø�w‰w�q‹zÐr!   r±   ri   rk   s   @@r   rª   rª   ›   s   ù‡ € õ:÷÷ ð r!   rª   c                   óT   a a€ ] tR t^¤t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚMoeBlockc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r#   rp   Ú	layer_idx)r   r8   )r<   r=   s   "€r   r>   ÚMoeBlock.__annotate__¥   s   ø€ ÷ Pñ P™Yð P±3ñ Pr!   c                óð  <€ \         SV `  4        VP                  pVP                  pVP                  V n        VP                  '       d7   \        VP                  V4      p\        V\        WE,          4      4      V n
        VP                  ;V n        p\        VP                  V4      V n        \        W64      V n        TpVP                   e   \        VP                   V4      p\#        W7V4      V n        R # r   )rU   rV   r&   r(   r/   Úuse_shared_mlprL   r.   rš   r8   Ú
shared_mlpr-   r,   Útop_krª   Úgater5   r   Ú
switch_mlp)	r   rp   r¶   r�   r(   Ú
num_sharedr-   Úexpert_intermediate_sizerZ   s	   &&&     €r   rV   ÚMoeBlock.__init__¥   sÓ   ø€ Ü‰ÑÔØ×ÑˆØ ×2Ñ2ÐØ"×4Ñ4ˆÔà×!×!Ð!Ü% d×&<Ñ&<¸iÓHˆJÜ! #¤sÐ+<Õ+IÓ'JÓKˆDŒOà)-×)9Ñ)9Ð9ˆÔ˜;Ü! $§-¡-°Ó;ˆŒ
ä˜Ó*ˆŒ	ð $5Ð Ø×%Ñ%Ò1Ü'3Ø×*Ñ*¨Ió(Ð$ô $ CÀ;ÓOˆŽr!   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r#   re   r¤   )r<   r=   s   "€r   r>   r·   ½   s   ø€ ÷ ñ á�8‰8ñr!   c                óD  € V P                  V4      p\        P                  ! VRRR7      pV P                  p\        P                  ! \        P
                  ! V) V^,
          RR7      RRV13,          4      p\        P                  ! W$RR7      pV P                  W4      pWeR,          P                  \        P                  4      ,          P                  R	R7      P                  VP                  4      pV P                  '       d   V P                  V4      pWg,           pV# )
é   T)ÚaxisÚprecise)ÚkthrÄ   .N©rÄ   rŠ   ).Néþÿÿÿ)r¼   rW   Úsoftmaxr»   Ústop_gradientÚargpartitionÚtake_along_axisr½   ÚastypeÚfloat32ÚsumÚdtyper¹   rº   )r   re   ÚgatesÚkÚindsÚscoresÚyÚshared_expert_outputs   &&      r   rf   ÚMoeBlock.__call__½   sà   € ð —	‘	˜!“ˆÜ—
’
˜5 r°4Ô8ˆà�J‰JˆÜ×Ò¤§¢°°¸AÀ½EÈÔ KÈCÐQSÐRSÐQSÈGÕ TÓUˆÜ×#Ò# E°bÔ9ˆà�O‰O˜AÓ$ˆØ˜	Õ"×)Ñ)¬"¯*©*Ó5Õ5×:Ñ:ÀÐ:ÓC×JÑJÈ1Ï7É7ÓSˆà××ÐØ#'§?¡?°1Ó#5Ð ØÕ(ˆAàˆr!   )r¼   r-   rº   r½   r»   r¹   rh   ri   rk   s   @@r   r´   r´   ¤   s    ù‡ € ÷Põ P÷0÷ ð r!   r´   c                   óX   a a€ ] tR t^Òt oRV3R lV 3R llltRV3R lR lltRtVtV ;t# )ÚDecoderLayerc                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r#   rp   ro   r¶   )r   r9   r8   )r<   r=   s   "€r   r>   ÚDecoderLayer.__annotate__Ó   s"   ø€ ÷ ñ ™Yð ±ð Á#ñ r!   c                óÄ  <€ \         SV `  4        VP                  V n        \        W!4      V n        VP
                  ^8X  d'   \        VP                  VP                  4      V n        M\        W4      V n        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  R7      V n        Wn        R# )rÃ   ©ÚepsN)rU   rV   r&   rm   Ú	self_attnr-   rš   r(   Úmlpr´   rx   r~   r1   Úinput_layernormÚpost_attention_layernormrp   )r   rp   ro   r¶   rZ   s   &&&&€r   rV   ÚDecoderLayer.__init__Ó   s¤   ø€ Ü‰ÑÔØ×+Ñ+ˆÔÜ" 7Ó1ˆŒØ×Ñ˜qÔ Ü˜4×+Ñ+¨T×-CÑ-CÓDˆD�Hä Ó0ˆDŒHä!Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
ª
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ð Ž	r!   c                óÈ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[S[S[ P                  S[ P                  3,          ,          /# )r#   re   r…   r†   Úshared_kv_states)rW   rˆ   r   r   r   )r<   r=   s   "€r   r>   rÛ   â   s[   ø€ ÷ %ñ %á�8‰8ð%ñ ‘r—x‘xÕ ð%ñ ™�}ð	%ñ
 #¡5©¯©±2·8±8Ð);Õ#<Õ=ñ%r!   c                ó°   € V P                  V P                  V4      W#V4      w  rTW,           pV P                  V P                  V4      4      pWe,           pWt3# r   )rß   rá   rà   râ   )r   re   r…   r†   rå   ÚrÚhÚouts   &&&&&   r   rf   ÚDecoderLayer.__call__â   sZ   € ð #Ÿn™nØ× Ñ  Ó# TÐ2Bó
Ñˆð �EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�eˆØÐ$Ð$r!   )rp   r&   rá   rà   râ   rß   rh   r˜   ri   rk   s   @@r   rÙ   rÙ   Ò   s   ù‡ € ÷õ ÷%÷ %ò %r!   rÙ   c                   óT   a a€ ] tR t^òt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚHunYuanModelc                ó    <€ V ^8„  d   QhRS[ /# ©r#   rp   ©r   )r<   r=   s   "€r   r>   ÚHunYuanModel.__annotate__ó   s   ø€ ÷ Hñ H™Yñ Hr!   c           	     ó*  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        V P                  ^ 8”  g   Q h\
        P                  ! VP                  VP                  4      V n        \        VP                  4       Uu. uF=  p\        TVP                  '       * ;'       g    W!P                  ,          ^ 8H  VR7      NK?  	  upV n        \
        P                  ! VP                  VP                  R7      V n        R# u upi )r   )rp   ro   r¶   rÝ   N)rU   rV   rp   r%   r'   rx   Ú	Embeddingr&   Úembed_tokensÚrangerÙ   r3   r4   Úlayersr~   r1   Únorm)r   rp   ÚirZ   s   && €r   rV   ÚHunYuanModel.__init__ó   sÛ   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ�‰ Ô"Ð"Ð"ÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔô ˜4×1Ñ1Ô2ó
ñ 3�ô ØØ!Ÿ\™\Ô)×NÐN¨q×3HÑ3HÕ/HÈQÑ.NØ÷ñ
 3ñ
ˆŒô —J’J˜t×/Ñ/°T×5FÑ5FÔGˆŽ	ùò
s   Â"DÂ5 Dc                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r#   Úinputsr¤   )r<   r=   s   "€r   r>   rð     s   ø€ ÷ ñ á—‘ñr!   c                ó˜  € V P                  V4      pVf   R .\        V P                  4      ,          p\        W2^ ,          4      p\	        \        V P                  V4      4       FS  w  pw  rgV P                  P                  '       d"   WPP                  P                  ,          ^ 8X  d   R pV! W4VX4      w  r8KU  	  V P                  V4      # r   )
ró   Úlenrõ   r   Ú	enumerateÚziprp   r3   r4   rö   )	r   rû   r†   rè   r…   r÷   ÚlayerÚcrå   s	   &&&      r   rf   ÚHunYuanModel.__call__  s¬   € ð
 ×Ñ˜fÓ%ˆàŠ=Ø�FœS §¡Ó-Õ-ˆEä$ Q¨a­Ó1ˆÜ&¤s¨4¯;©;¸Ó'>Ö?‰MˆA‰z�Ø—I‘I×%×%Ð%¨!¯i©i×.HÑ.HÕ*HÈAÔ*MØ#'Ð Ù"'¨°Ð4DÓ"EÑˆAÑñ @ð
 �y‰y˜‹|Ðr!   )rp   ró   rõ   rö   r'   r%   r   ri   rk   s   @@r   rì   rì   ò   s    ù‡ € ÷Hó H÷"÷ ò r!   rì   c                   ój   a a€ ] tR tRt oV3R lV 3R lltR	V3R lR lltR t]R 4       tRt	Vt
V ;t# )
ÚModeli  c                ó    <€ V ^8„  d   QhRS[ /# rî   rï   )r<   r=   s   "€r   r>   ÚModel.__annotate__  s   ø€ ÷ (ñ (™Yñ (r!   c                ór   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        R # r   )rU   rV   rp   r$   rì   Úmodel)r   rp   rZ   s   &&€r   rV   ÚModel.__init__  s*   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ! $Ó'ˆŽ
r!   c                ó4   <€ V ^8„  d   QhRS[ P                  /# rú   r¤   )r<   r=   s   "€r   r>   r    s   ø€ ÷ 6ñ 6á—‘ñ6r!   c                ón   € V P                  W4      pV P                   P                  P                  V4      # r   )r  ró   Ú	as_linear)r   rû   r†   ré   s   &&& r   rf   ÚModel.__call__  s-   € ð
 �j‰j˜Ó'ˆØ�z‰z×&Ñ&×0Ñ0°Ó5Ð5r!   c                ó>  € R V9   EdQ   / pV P                   P                  pV P                   P                  pV P                   P                  V,          pW0P                   P                  ,          pVP	                  4        FÓ  w  rxRV9   d{   VP                  WE^,           VR4      pVP                  WU^,           .^R7      p	\        . ROV	4       F1  w  r«VP                  RV
4      p\        P                  ! V^ ^4      W,&   K3  	  K†  RV9   dC   VP                  ^^ R7      p	\        RR.V	4       F  w  r«VP                  RV
4      pW²V&   K  	  KÏ  W‚V&   KÕ  	  TpRV9  d   V# \        V P                   P                  4       F“  pRV 2pR F…  pR F|  pV RV R	V 2V9   g   K  \        V P                   P                  4       Uu. uF  pVP                  V R
V R	V R	V 24      NK!  	  pp\        P                  ! V4      W RV R	V 2&   K~  	  K‡  	  K•  	  V# u upi )z*model.layers.0.mlp.gate_and_up_proj.weightÚqkv_projrÇ   Úgate_and_up_projrŸ   r�   z+model.layers.0.mlp.experts.0.up_proj.weightzmodel.layers.z.mlp.experts.0.Ú.z.mlp.experts.z.mlp.switch_mlp.rŠ   )rz   r{   r|   )rŸ   rž   r�   )ÚweightÚscalesÚbiases)rp   r&   r*   r)   ÚitemsrŒ   Úsplitrÿ   ÚreplacerW   Úflattenrô   r'   r-   ÚpopÚstack)r   ÚweightsÚnew_weightsr’   rw   Ún_kv_groupsr‚   rÒ   ÚvÚsplitsÚk_upÚv_newÚk_newÚlÚprefixÚnÚeÚto_joins   &&                r   ÚsanitizeÚModel.sanitize&  s  € à7¸7ÕBØˆKØ—	‘	×%Ñ%ˆAØŸ™×6Ñ6ˆJØŸ)™)×7Ñ7¸:ÕEˆKØŸI™I×9Ñ9Õ9ˆHØŸ™ž‘�Ø ”?ØŸ	™	 *¸A­o¸xÈÓL�AØŸW™W kÀµ?Ð%CÈ!˜WÓL�FÜ'*Ò+IÈ6Ö'R™˜Ø !§	¡	¨*°dÓ ;˜Ü-/¯ZªZ¸¸qÀ!Ó-D˜Ó*ó (Sð (¨1Ô,ØŸW™W Q¨Q˜WÓ/�FÜ'*¨I°{Ð+CÀVÖ'L™˜Ø !§	¡	Ð*<¸dÓ C˜Ø-2 EÓ*ó (Mð &' “Nñ (ð "ˆGà8ÀÔGØˆNÜ�t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFÛ:�Û7�AØ ˜ °°°1°Q°CÐ8¸GÖCô &+¨4¯9©9×+@Ñ+@Ô%Aó#á%A ð $ŸK™K¨6¨(°-À¸sÀ!ÀAÀ3ÀaÈÀsÐ(KÖLÙ%Að  ð #ô GIÇhÂhÈwÓFW˜ (Ð*:¸1¸#¸Q¸q¸cÐ BÓCó 8ó ;ñ 4ð ˆùò#s   Ç%H
c                ó.   € V P                   P                  # r   )r  rõ   )r   s   &r   rõ   ÚModel.layersL  s   € à�z‰z× Ñ Ð r!   )rp   r  r$   r   )rA   rB   rC   rD   rV   rf   r(  Úpropertyrõ   rF   rG   rj   rk   s   @@r   r  r    s8   ù‡ € ÷(ó (÷6ò 6ò$ðL ñ!ó ÷!ð !r!   r  ) Údataclassesr   Útypingr   r   r   r   r   Úmlx.coreÚcorerW   Úmlx.nnrx   Úactivationsr	   rQ   r
   r   r   Úswitch_layersr   r   rL   ÚModulerN   rm   rš   rª   r´   rÙ   rì   r  r@   r!   r   Ú<module>r5     sÍ   ðõ "ß 4Õ 4å Ý å ß TÑ TÝ $ð ôT�ó Tó ðTò<ô
˜"Ÿ)™)ô 
ô0B.�—	‘	ô B.ôJJˆ"�)‰)ô Jôˆ2�9‰9ô ô+ˆr�y‰yô +ô\%�2—9‘9ô %ô@"�2—9‘9ô "ôJ7!ˆB�I‰Iö 7!r!   