+
    TV-jö2  ã                   óª  € ^ RI Ht ^ RIHt ^ RIHtHtHt ^ RIH	t
 ^ RIHt ^ RIHtHtHt ^RIHtHtHt ^RIHt ] ! R R	]4      4       t]R
 4       t ! R R]P2                  4      t ! R R]P2                  4      t ! R R]P2                  4      t ! R R]P2                  4      t ! R R]P2                  4      t ! R R]P2                  4      tR# )é    )Ú	dataclass)Ú	lru_cache)ÚAnyÚListÚOptionalN)Úshard_inplaceÚshard_linearÚsum_gradients)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Ú	SwitchGLUc                   ó<   a € ] tR t^t o RtRtRtRtV 3R ltRt	V t
R# )Ú	ModelArgsFÚsigmoidNTc                ó  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[ ;R&   S[S[,          ;R&   S[;R&   # )é   Ú
model_typeÚhidden_sizeÚintermediate_sizeÚnum_attention_headsÚnum_key_value_headsÚmax_position_embeddingsÚnum_experts_per_tokÚnum_local_expertsÚshared_intermediate_sizeÚnum_hidden_layersÚrms_norm_epsÚ
rope_thetaÚ
rotary_dimÚ
vocab_sizeÚtie_word_embeddingsÚscoring_funcÚhead_dimÚuse_qk_norm)ÚstrÚintÚfloatÚboolr   )ÚformatÚ__classdict__s   "€Úf/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/minimax.pyÚ__annotate__ÚModelArgs.__annotate__   sî   ø‡ ‚ á�Oñ ñ Ññ ñ Ññ	 ñ
 Ññ ñ Ññ ñ !Ñ ñ ñ Ññ ñ Ññ ñ "Ñ!ñ ñ Ññ ñ Ññ ñ Ññ ñ �Oñ ñ �Oñ ñ  Ñ%ñ! ñ" Ñ!ñ# ñ$ ‘s�mÑ"ñ% ñ& Ñò' ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r"   r#   r$   r%   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r+   s   @r,   r   r      s#   ø‡ € ð  !&ÐØ!€LØ"€HØ€K÷' ƒ r/   r   c                 óx   a aa€ \         P                  R  4       o\         P                  V 3R l4       oVV3R lpV# )c                 ó~   € V P                  \        P                  4      P                  4       P	                  RRR7      # )é   T)Úkeepdimséÿÿÿÿ)ÚastypeÚmxÚfloat32ÚsquareÚsum)Úxs   &r,   Ú_cast_square_sumÚ*sharded_rms_norm.<locals>._cast_square_sum'   s.   € à�x‰xœŸ
™
Ó#×*Ñ*Ó,×0Ñ0°¸dÐ0ÓCÐCr/   c                 ód  <€ \         P                  P                  VSR 7      p\         P                  ! WP                  R,          SP                  4       ,          ,          V,           4      pV P                  \         P                  4      V,          V,          P                  V P                  4      # )©Úgroupr<   )	r>   ÚdistributedÚall_sumÚrsqrtÚshapeÚsizer=   r?   Údtype)rB   Únorm2ÚwÚepsÚnormrG   s   &&&& €r,   Ú
_normalizeÚ$sharded_rms_norm.<locals>._normalize+   sq   ø€ ä—‘×&Ñ& u°EÐ&Ó:ˆÜ�xŠx˜§¡¨¥¨u¯z©z«|Õ!;Õ<¸sÕBÓCˆØ—‘œŸ™Ó$ tÕ+¨aÕ/×7Ñ7¸¿¹Ó@Ð@r/   c                 ó$   <€ S! V S! V 4      W4      # ©Nr0   )rB   rO   rP   rC   rR   s   &&&€€r,   Ú_inner_sharded_rms_normÚ1sharded_rms_norm.<locals>._inner_sharded_rms_norm4   s   ø€ Ù˜!Ñ-¨aÓ0°!Ó9Ð9r/   )r>   Úcompile)rG   rV   rC   rR   s   f @@r,   Úsharded_rms_normrY   %   sB   ú€ ä‡Z�ZñDó ðDô ‡Z�ZôAó ðAö:ð #Ð"r/   c                   ór   a a€ ] tR t^:t oR
V3R lV 3R llltR tR t]RR/V3R lR ll4       tR	t	Vt
V ;t# )ÚShardedRMSNormNc                ód   <€ V ^8„  d   QhRS[ RS[RS[S[P                  P
                  ,          /# )r   ÚdimsrP   rG   )r'   r(   r   r>   rH   ÚGroup)r*   r+   s   "€r,   r-   ÚShardedRMSNorm.__annotate__;   s4   ø€ ÷ ñ ÙðÙ#ðÙ3;¹B¿N¹N×<PÑ<PÕ3Qñr/   c                óì   <€ \         SV `  4        T;'       g    \        P                  P	                  4       p\        P
                  ! WP                  4       ,          34      V n        W0n        W n	        R # rU   )
ÚsuperÚ__init__r>   rH   ÚinitÚonesrL   ÚweightrG   rP   )Úselfr]   rP   rG   Ú	__class__s   &&&&€r,   rb   ÚShardedRMSNorm.__init__;   sP   ø€ ô 	‰ÑÔØ×.Ð.œŸ™×,Ñ,Ó.ˆÜ—g’g˜t§z¡z£|Õ3Ð5Ó6ˆŒØŒ
ØŽr/   c                ó˜   € V P                   P                  ^ ,          V P                  P                  4       ,           RV P                   2# )r   z, eps=)re   rK   rG   rL   rP   ©rf   s   &r,   Ú_extra_reprÚShardedRMSNorm._extra_reprD   s7   € Ø—+‘+×#Ñ# AÕ&¨¯©¯©Ó):Õ:Ð;¸6À$Ç(Á(ÀÐLÐLr/   c                ó\   € \        V P                  4      ! WR ,          V P                  4      # )re   )rY   rG   rP   )rf   rB   s   &&r,   Ú__call__ÚShardedRMSNorm.__call__G   s    € Ü §
¡
Ô+¨A°H­~¸t¿x¹xÓHÐHr/   rG   c                óX   <€ V ^8„  d   QhRS[ S[P                  P                  ,          /# ©r   rG   ©r   r>   rH   r^   )r*   r+   s   "€r,   r-   r_   K   s$   ø€ ÷ ñ Ù$,©R¯^©^×-AÑ-AÕ$Bñr/   c               ó$  € V ! VP                   P                  ^ ,          VP                  VR7      p\        P                  ! \        P
                  ! VP                   VP                  4       RR7      VP                  4       ,          4      Vn         V# )r   rF   ©Úaxisr<   )re   rK   rP   r>   Ú
contiguousÚsplitrL   Úrank)ÚclsÚnorm_modulerG   Úsns   &&$ r,   Úfrom_rms_normÚShardedRMSNorm.from_rms_normJ   se   € ñ �×#Ñ#×)Ñ)¨!Õ,¨k¯o©oÀUÔKˆÜ—M’MÜ�HŠH�[×'Ñ'¨¯©«¸BÔ?ÀÇ
Á
ÃÕMó
ˆŒ	ð ˆ	r/   )rP   rG   re   )gñhãˆµøä>N)r1   r2   r3   r4   rb   rk   rn   Úclassmethodr|   r6   r7   Ú__classcell__©rg   r+   s   @@r,   r[   r[   :   sA   ù‡ € ÷õ òMòIð ðØEI÷ñ ó ÷ð r/   r[   c                   óT   a a€ ] tR t^Vt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMiniMaxAttentionc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úargs©r   )r*   r+   s   "€r,   r-   ÚMiniMaxAttention.__annotate__W   s   ø€ ÷ "Vñ "V™Yñ "Vr/   c                óz  <€ \         SV `  4        VP                  ;V n        pVP                  V n        VP
                  V n        VP                  ;'       g    W!P                  ,          ;V n        pVR,          V n        \        P                  ! VP                  V P                  V,          RR7      V n
        \        P                  ! VP                  V P
                  V,          RR7      V n        \        P                  ! VP                  V P
                  V,          RR7      V n        \        P                  ! V P                  V,          VP                  RR7      V n        \        VR4      '       d   VP                  MRV n        V P                  '       do   \        P                   ! W0P                  ,          VP"                  R7      V n        \        P                   ! W0P
                  ,          VP"                  R7      V n        \        P(                  ! VP*                  RVP,                  R7      V n        R# )g      à?F©Úbiasr%   ©rP   )ÚtraditionalÚbaseNg      à¿)ra   rb   r   Ú
hidden_dimr   r   r$   ÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚo_projÚhasattrr%   ÚRMSNormr   Úq_normÚk_normÚRoPEr    r   Úrope)rf   r…   r   r$   rg   s   &&  €r,   rb   ÚMiniMaxAttention.__init__W   s¥  ø€ Ü‰ÑÔà(,×(8Ñ(8Ð8ˆŒ˜+à#'×#;Ñ#;ˆÔ Ø#'×#;Ñ#;ˆÔ à�M‰M×DÐD˜[×,DÑ,DÕDð	
ˆŒ˜ð ˜t•^ˆŒ
ä—i’iØ×Ñ˜d×6Ñ6¸ÕAÈô
ˆŒô —i’iØ×Ñ˜d×6Ñ6¸ÕAÈô
ˆŒô —i’iØ×Ñ˜d×6Ñ6¸ÕAÈô
ˆŒô —i’iØ×$Ñ$ xÕ/°×1AÑ1AÈô
ˆŒô 07°t¸]×/KÒ/K˜4×+Ò+ÐQVˆÔØ××ÐÜŸ*š*Ø×3Ñ3Õ3¸×9JÑ9JôˆDŒKô Ÿ*š*Ø×3Ñ3Õ3¸×9JÑ9JôˆDŒKô —G’G˜DŸO™O¸ÀTÇ_Á_ÔUˆŽ	r/   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   rB   ÚmaskÚcacheÚreturn©r>   Úarrayr   r   )r*   r+   s   "€r,   r-   r‡   {   sH   ø€ ÷ $#ñ $#á�8‰8ð$#ñ ‘r—x‘xÕ ð$#ñ ™�}ð	$#ñ
 
�‰ñ$#r/   c           	     ó†  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pV P                  '       d#   V P                  V4      pV P                  V4      pVP                  WEV P                  R4      P                  ^ ^^^4      pVP                  WEV P                  R4      P                  ^ ^^^4      pV	P                  WEV P                  R4      P                  ^ ^^^4      p	VeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      P                  WER4      p
V P!                  V
4      # )r:   )Úoffset)r    r�   rŸ   r<   )rK   r’   r“   r”   r%   r˜   r™   Úreshaper   Ú	transposer   r›   r¥   Úupdate_and_fetchr   r�   r•   )rf   rB   rŸ   r    ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r,   rn   ÚMiniMaxAttention.__call__{   s‹  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆà××ÐØ—k‘k 'Ó*ˆGØ—;‘;˜tÓ$ˆDà—/‘/ !¨×(@Ñ(@À"ÓE×OÑOØˆq�!�Qó
ˆð �|‰|˜A $×":Ñ":¸BÓ?×IÑIÈ!ÈQÐPQÐSTÓUˆØ—‘  d×&>Ñ&>ÀÓC×MÑMØˆq�!�Qó
ˆð ÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆà�{‰{˜6Ó"Ð"r/   )r$   rŽ   r™   r“   r   r   r•   r˜   r’   r›   r�   r%   r”   ©NN©	r1   r2   r3   r4   rb   rn   r6   r7   r   r€   s   @@r,   r‚   r‚   V   s!   ù‡ € ÷"Vó "V÷H$#÷ $#ò $#r/   r‚   c                   óP   a a€ ] tR t^¢t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚMiniMaxSparseMoeBlockc                ó    <€ V ^8„  d   QhRS[ /# r„   r†   )r*   r+   s   "€r,   r-   Ú"MiniMaxSparseMoeBlock.__annotate__£   s   ø€ ÷ 	#ñ 	#™Yñ 	#r/   c                ód  <€ \         SV `  4        VP                  V n        \        P                  ! VP
                  VP                  R R7      V n        \        VP
                  VP                  VP                  4      V n
        \        P                  ! VP                  34      V n        RV n        R# ©Fr‰   N)ra   rb   r   r�   r‘   r   r   Úgater   r   Ú
switch_mlpr>   ÚzerosÚe_score_correction_biasÚsharding_group©rf   r…   rg   s   &&€r,   rb   ÚMiniMaxSparseMoeBlock.__init__£   s…   ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ ä—I’I˜d×.Ñ.°×0FÑ0FÈUÔSˆŒ	Ü#Ø×Ñ˜d×4Ñ4°d×6LÑ6Ló
ˆŒô (*§x¢x°×1GÑ1GÐ0IÓ'JˆÔ$Ø"ˆÖr/   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   rB   r¡   )r>   r£   )r*   r+   s   "€r,   r-   r¶   ®   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r/   c                óü  € V P                   e   \        V P                   4      ! V4      pV P                  VP                  \        P
                  4      4      p\        P                  ! V4      pTpW0P                  ,           pV P                  p\        P                  ! V) V^,
          RR7      RR V13,          p\        P                  ! WFRR7      pV\        P                  ! VRRR7      R,           ,          pVP                  VP                  4      pV P                  W4      pWsR	,          ,          P                  R
R7      pV P                   e+   \        P                  P                  WpP                   R7      pV# )N)Úkthru   .rt   T)ru   r;   g#B’¡œÇ;rF   r<   ).Néþÿÿÿ)r½   r
   r¹   r=   r>   r?   r   r¼   r   ÚargpartitionÚtake_along_axisrA   rM   rº   rH   rI   )rf   rB   ÚgatesÚscoresÚorig_scoresÚkÚindsÚys   &&      r,   rn   ÚMiniMaxSparseMoeBlock.__call__®   s&  € Ø×ÑÒ*Ü˜d×1Ñ1Ô2°1Ó5ˆAà—	‘	˜!Ÿ(™(¤2§:¡:Ó.Ó/ˆä—’˜EÓ"ˆØˆØ×6Ñ6Õ6ˆà×$Ñ$ˆÜ�Š ˜w¨A°­E¸Ô;¸CÀÀ!À¸GÕDˆÜ×#Ò# K¸BÔ?ˆàœ2Ÿ6š6 &¨r¸DÔAÀEÕIÕJˆØ—‘˜qŸw™wÓ'ˆà�O‰O˜AÓ$ˆØ˜	Õ"Õ"×'Ñ'¨RÐ'Ó0ˆà×ÑÒ*Ü—‘×&Ñ& q×0CÑ0CÐ&ÓDˆAàˆr/   )r¼   r¹   r   r½   rº   r²   r€   s   @@r,   r´   r´   ¢   s   ù‡ € ÷	#ó 	#÷÷ ð r/   r´   c                   óT   a a€ ] tR t^Èt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMiniMaxDecoderLayerc                ó    <€ V ^8„  d   QhRS[ /# r„   r†   )r*   r+   s   "€r,   r-   Ú MiniMaxDecoderLayer.__annotate__É   s   ø€ ÷ 

ñ 

™Yñ 

r/   c                ó(  <€ \         SV `  4        \        V4      V n        \	        V4      V n        \        P                  ! VP                  VP                  R 7      V n
        \        P                  ! VP                  VP                  R 7      V n        R# )r‹   N)ra   rb   r‚   Ú	self_attnr´   Úblock_sparse_moer�   r—   r   r   Úinput_layernormÚpost_attention_layernormr¾   s   &&€r,   rb   ÚMiniMaxDecoderLayer.__init__É   sh   ø€ Ü‰ÑÔä)¨$Ó/ˆŒä 5°dÓ ;ˆÔä!Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
ª
Ø×Ñ $×"3Ñ"3ô)
ˆÖ%r/   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# rž   r¢   )r*   r+   s   "€r,   r-   rÐ   Õ   sH   ø€ ÷ ñ á�8‰8ðñ ‘r—x‘xÕ ðñ ™�}ð	ñ
 
�‰ñr/   c                ó    € WP                  V P                  V4      W#4      ,           pW@P                  V P                  V4      4      ,           pV# rU   )rÒ   rÔ   rÓ   rÕ   )rf   rB   rŸ   r    Úrs   &&&& r,   rn   ÚMiniMaxDecoderLayer.__call__Õ   sE   € ð —‘˜t×3Ñ3°AÓ6¸ÓDÕDˆØ×%Ñ% d×&CÑ&CÀAÓ&FÓGÕGˆØˆr/   )rÓ   rÔ   rÕ   rÒ   r±   r²   r€   s   @@r,   rÎ   rÎ   È   s   ù‡ € ÷

ó 

÷÷ ò r/   rÎ   c                   óT   a a€ ] tR t^àt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMiniMaxModelc                ó    <€ V ^8„  d   QhRS[ /# r„   r†   )r*   r+   s   "€r,   r-   ÚMiniMaxModel.__annotate__á   s   ø€ ÷ Hñ H™Yñ Hr/   c                óX  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  4       Uu. uF  p\        VR 7      NK  	  upV n
        \        P                  ! VP
                  VP                  R7      V n        R# u upi ))r…   r‹   N)ra   rb   r�   Ú	Embeddingr!   r   Úembed_tokensÚranger   rÎ   Úlayersr—   r   rQ   )rf   r…   Ú_rg   s   && €r,   rb   ÚMiniMaxModel.__init__á   s   ø€ Ü‰ÑÔÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔô 5:¸$×:PÑ:PÔ4Qó
Ù4Q¨qÔ T×*Ñ4Qñ
ˆŒô —J’J˜t×/Ñ/°T×5FÑ5FÔGˆŽ	ùò	
s   ÁB'c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# )r   ÚinputsrŸ   r    r¡   r¢   )r*   r+   s   "€r,   r-   rÞ   ë   sH   ø€ ÷ ñ á—‘ðñ ‘r—x‘xÕ ðñ ™�}ð	ñ
 
�‰ñr/   c                óü   € V P                  V4      pVf   R .\        V P                  4      ,          p\        WC^ ,          4      p\	        V P                  V4       F  w  rVV! WBV4      pK  	  V P                  V4      # rU   )rá   Úlenrã   r   ÚziprQ   )rf   rç   rŸ   r    ÚhÚlayerÚcs   &&&&   r,   rn   ÚMiniMaxModel.__call__ë   so   € ð ×Ñ˜fÓ%ˆàŠ=Ø�FœS §¡Ó-Õ-ˆEä$ Q¨a­Ó1ˆä˜DŸK™K¨Ö/‰HˆEÙ�a˜qÓ!ŠAñ 0ð �y‰y˜‹|Ðr/   )rá   rã   rQ   r±   r²   r€   s   @@r,   rÜ   rÜ   à   s    ù‡ € ÷Hó H÷÷ ò r/   rÜ   c                   ó    a a€ ] tR t^þt oV3R lV 3R lltRV3R lR lltR tRV3R lR llt]R 4       t	]R	 4       t
]R
 4       tRtVtV ;t# )ÚModelc                ó    <€ V ^8„  d   QhRS[ /# r„   r†   )r*   r+   s   "€r,   r-   ÚModel.__annotate__ÿ   s   ø€ ÷ Tñ T™Yñ Tr/   c                óþ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        VP                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# R# r¸   )ra   rb   r…   r   rÜ   Úmodelr"   r�   r‘   r   r!   Úlm_headr¾   s   &&€r,   rb   ÚModel.__init__ÿ   sY   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ! $Ó'ˆŒ
Ø×'×'Ð'ÜŸ9š9 T×%5Ñ%5°t·±ÈUÔSˆDŽLñ (r/   c                ót   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          /# )r   rç   rŸ   r    r¢   )r*   r+   s   "€r,   r-   rò     s:   ø€ ÷ ñ á—‘ðñ ‘r—x‘xÕ ðñ ™�}ñ	r/   c                óÔ   € V P                  WVR 7      pV P                  P                  '       d(   V P                   P                  P	                  V4      pV# V P                  V4      pV# ))rç   rŸ   r    )rô   r…   r"   rá   Ú	as_linearrõ   )rf   rç   rŸ   r    Úouts   &&&& r,   rn   ÚModel.__call__  s\   € ð �j‰j ¸ˆjÓ?ˆØ�9‰9×(×(Ð(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r/   c                ó\  € R p/ pVP                  4        FB  w  rERV9   d+   TpVP                  RR4      pW,          pV! W†4      pWƒV&   K6  WC9  g   K>  WSV&   KD  	  TpRV9  d   V# \        V P                  P                  4       F�  p	RV	 2p
RRR	R
RR/pVP                  4        Fy  w  rÍV
 RV R2V9   g   K  \        V P                  P
                  4       Uu. uF  pVP                  V
 RV RV R24      NK  	  pp\        P                  ! V4      VV
 RV R2&   K{  	  KŸ  	  V# u upi )z3Dequantize FP8 weights and restructure MoE experts.c                 óà  € \         P                  p\         P                  ! V \         P                  R 7      p ^€pV P                  w  rEV) V,          pV) V,          p\         P                  ! V ^ V3^ V334      p V P                  WF,           V,          W5V,           V,          V34      p WR,          ,          P                  WF,           WW,           4      p V RV1RV13,          P                  V4      # ))rM   N)ºNNNNrþ   N)r>   Úbfloat16Úfrom_fp8rK   Úpadr¦   r=   )re   Ú	scale_invrM   ÚbsÚmÚnÚ
pad_bottomÚpad_sides   &&      r,   ÚdequantÚModel.sanitize.<locals>.dequant  sÑ   € Ü—K‘KˆEÜ—[’[ ¬r¯{©{Ô;ˆFØˆBØ—<‘<‰DˆAØ˜" �ˆJØ˜˜b•yˆHÜ—V’V˜F a¨ _°q¸(°mÐ$DÓEˆFØ—^‘^Ø•. RÕ'¨°(­l¸rÕ-AÀ2ÐFóˆFð Ð)9Õ:Õ:×CÑCØ• ¥óˆFð ˜"˜1˜"˜b˜q˜b˜&•>×(Ñ(¨Ó/Ð/r/   Úweight_scale_invÚ
_scale_invÚ z3model.layers.0.block_sparse_moe.experts.0.w1.weightzmodel.layers.Úw1Ú	gate_projÚw2Ú	down_projÚw3Úup_projz.block_sparse_moe.experts.0.z.weightz.block_sparse_moe.experts.Ú.z.block_sparse_moe.switch_mlp.)	ÚitemsÚreplacerâ   r…   r   r   Úpopr>   Ústack)rf   Úweightsr  Únew_weightsrÉ   Úvr  Úwkre   ÚlÚprefixÚmappingÚ	orig_nameÚnew_nameÚeÚto_joins   &&              r,   ÚsanitizeÚModel.sanitize  sf  € ò	0ð" ˆØ—M‘M–O‰DˆAØ! QÔ&Ø�	Ø—Y‘Y˜|¨RÓ0�Ø ��Ù  Ó3�Ø"(˜B“ØÖ%Ø!"˜A“ñ $ð ˆð AÈÔOØˆNä�t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFØ˜[¨$°¸TÀ9ÐMˆGØ'.§}¡}¦Ñ#�	Ø�XÐ9¸)¸ÀGÐLÐPWÖWô
 "' t§y¡y×'BÑ'BÔ!Có	ñ "D˜Að  Ÿ™Ø%˜hÐ&@ÀÀÀ1ÀYÀKÈwÐWöñ "Dð	 ð ô Ÿš Ó)ð Ø!˜(Ð"?À¸zÈÐQóó (7ñ 4ð ˆùòs   Ã#D)c                óX   <€ V ^8„  d   QhRS[ S[P                  P                  ,          /# rq   rr   )r*   r+   s   "€r,   r-   rò   I  s#   ø€ ÷ -:ñ -:™8¡B§N¡N×$8Ñ$8Õ9ñ -:r/   c                ó0  € T;'       g    \         P                  P                  4       pVP                  4       pVP	                  4       pV P
                  P                   EF2  p\        VP                  P                  R VR7      VP                  n	        \        VP                  P                  R VR7      VP                  n
        \        VP                  P                  R VR7      VP                  n        \        VP                  P                  RVR7      VP                  n        VP                  P                  '       du   \        P                  VP                  P                   VR7      VP                  n        \        P                  VP                  P"                  VR7      VP                  n        VP                  ;P$                  V,          un        VP                  ;P&                  V,          un        \)        VP*                  P,                  P.                  R VR7       \)        VP*                  P,                  P0                  RVR7       \)        VP*                  P,                  P2                  R VR7       WP*                  n        EK5  	  R# )zall-to-shardedrF   zsharded-to-allN)r>   rH   rc   rL   rx   rô   rã   r	   rÒ   r’   r“   r”   r•   r%   r[   r|   r˜   r™   r   r   r   rÓ   rº   r  r  r  r½   )rf   rG   ÚNrx   rì   s   &&   r,   ÚshardÚModel.shardI  sî  € Ø×.Ð.œŸ™×,Ñ,Ó.ˆØ�J‰J‹LˆØ�z‰z‹|ˆØ—Z‘Z×&Õ&ˆEä%1Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ð �‰×*×*Ð*Ü)7×)EÑ)EØ—O‘O×*Ñ*°%ð *Fó *�—‘Ô&ô *8×)EÑ)EØ—O‘O×*Ñ*°%ð *Fó *�—‘Ô&ð �O‰O×/Ò/°AÕ5Õ/Ø�O‰O×/Ò/°AÕ5Õ/ô Ø×&Ñ&×1Ñ1×;Ñ;Ø Øõô
 Ø×&Ñ&×1Ñ1×;Ñ;Ø Øõô
 Ø×&Ñ&×1Ñ1×9Ñ9Ø Øõð
 5:×"Ñ"×1óS 'r/   c                ó.   € V P                   P                  # rU   )rô   rã   rj   s   &r,   rã   ÚModel.layersx  s   € à�z‰z× Ñ Ð r/   c                ó   € R  pV# )c                 ó   € R V 9  # )r¼   r0   )rÉ   s   &r,   Ú	predicateÚ'Model.cast_predicate.<locals>.predicate~  s   € Ø,°AÑ5Ð5r/   r0   ©rf   r.  s   & r,   Úcast_predicateÚModel.cast_predicate|  s   € ò	6ð Ðr/   c                ó   € R  pV# )c                 ó@   € V P                  R 4      '       d   R^@R^/# R# )zblock_sparse_moe.gateÚ
group_sizeÚbitsT)Úendswith)Úpathrä   s   &&r,   r.  Ú(Model.quant_predicate.<locals>.predicate…  s$   € Ø�}‰}Ð4×5Ò5Ø$ b¨&°!Ð4Ð4Ùr/   r0   r0  s   & r,   Úquant_predicateÚModel.quant_predicateƒ  s   € ò	ð
 Ðr/   )r…   rõ   rô   r   r±   rU   )r1   r2   r3   r4   rb   rn   r#  r(  Úpropertyrã   r1  r:  r6   r7   r   r€   s   @@r,   rð   rð   þ   sm   ù‡ € ÷Tó T÷ò ò3÷j-:ò -:ð^ ñ!ó ð!ð ñó ðð ñó ÷ð r/   rð   ) Údataclassesr   Ú	functoolsr   Útypingr   r   r   Úmlx.coreÚcorer>   Úmlx.nnr�   Úmlx.nn.layers.distributedr   r	   r
   r�   r   r   r   Úswitch_layersr   r   rY   ÚModuler[   r‚   r´   rÎ   rÜ   rð   r0   r/   r,   Ú<module>rF     s½   ðõ "Ý ß &Ñ &å Ý ß PÑ Pç TÑ TÝ $ð ô�ó ó ðð* ñ#ó ð#ô(�R—Y‘Yô ô8I#�r—y‘yô I#ôX#˜BŸI™Iô #ôL˜"Ÿ)™)ô ô0�2—9‘9ô ô<LˆB�I‰Iö Lr/   