+
    TV-jâ2  ã                   ó$  € ^ RI Ht ^ RIHt ^ RIHtHtHtHt ^ RI	H
t ^ RIHt ^RIHt ^RIHtHtHt ^RIHt ^RIHt ] ! R	 R
]4      4       t]! ]P2                  RR7      R 4       t ! R R]P6                  4      t ! R R]P6                  4      t]P2                  R 4       t ! R R]P6                  4      t ! R R]P6                  4      t  ! R R]P6                  4      t! ! R R]P6                  4      t" ! R R]P6                  4      t#R# )é    )Ú	dataclass)Úpartial)ÚAnyÚDictÚOptionalÚUnionN)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Úinitialize_rope)Ú	SwitchGLUc                   óp   a € ] tR t^t o RtRtRtRtRtRt	Rt
RtRtRtRtRtRt^t^tRtRtV 3R ltRtV tR# )Ú	ModelArgsNFg      ð?TÚsoftmaxc                ó  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[S[ S[S[S[ 3,          3,          ,          ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[,          ;R&   S[;R&   S[;R&   S[;R&   S[ ;R&   S[;R&   S[;R&   S[S[,          ;R &   S[;R!&   # )"é   Ú
model_typeÚhidden_sizeÚintermediate_sizeÚmax_position_embeddingsÚmoe_intermediate_sizeÚnum_expertsÚnum_shared_expertsÚnorm_topk_probÚnum_attention_headsÚnum_experts_per_tokÚnum_hidden_layersÚnum_key_value_headsÚrms_norm_epsÚ
rope_thetaÚ
vocab_sizeÚfirst_k_dense_replaceÚrope_scalingÚuse_biasÚuse_qkv_biasÚ	norm_headÚnorm_softmaxÚuse_qk_normÚtie_word_embeddingsÚpartial_rotary_factorÚ
rotary_dimÚmoe_router_enable_expert_biasÚ moe_router_enable_routed_scalingÚrouted_scaling_factorÚscore_functionÚn_groupÚ
topk_groupÚ#moe_shared_expert_intermediate_sizeÚmoe_router_enable_shared_expert)ÚstrÚintÚboolÚfloatr   r   r   )ÚformatÚ__classdict__s   "€Új/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/bailing_moe.pyÚ__annotate__ÚModelArgs.__annotate__   sÔ  ø‡ ‚ á�Oñ ñ Ññ ñ Ññ	 ñ
 !Ñ ñ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ  �Oñ! ñ" Ññ# ñ$ ™4¡¡U©5±#¨:Õ%6Ð 6Õ7Õ8Ñ?ñ% ñ& Ññ' ñ( Ññ) ñ* Ññ+ ñ, Ññ- ñ. Ññ/ ñ0 Ñ%ñ1 ñ2 !Ñ&ñ3 ñ4 ™•Ñ$ñ5 ñ6 $(Ñ/ñ7 ñ8 '+Ñ1ñ9 ñ: !Ñ&ñ; ñ< Ñ#ñ= ñ> Ññ? ñ@ ÑñA ñB *2±#­Ñ=ñC ñD &*Ñ0òE ó    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r:   s   @r;   r   r      sl   ø‡ € ð$ <@€LØ€HØ€LØ€IØ€LØ€KØ %ÐØ#&ÐØ $€JØ*/Ð!Ø-1Ð$Ø#&ÐØ#€NØ€GØ€JØ9=Ð'Ø,0Ð#÷E ƒ r>   r   T)Ú	shapelessc                 ór   € WR,          ,          P                  RR7      P                  V P                  4      # ).©Úaxis).Néþÿÿÿ)ÚsumÚastypeÚdtype)Úexpert_outputsÚscoress   &&r;   Úaggregate_expert_outputsrQ   5   s3   € ð 
 Õ+Õ	+×0Ñ0°bÐ0Ó9×@Ñ@À×AUÑAUÓVðr>   c                   óT   a a€ ] tR t^<t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚBailingMoeMLPc                ó6   <€ V ^8„  d   QhRS[ RS[S[,          /# )r   Úargsr   )r   r   r6   )r9   r:   s   "€r;   r<   ÚBailingMoeMLP.__annotate__=   s   ø€ ÷ 
ñ 
™Yð 
¹8ÁC½=ñ 
r>   c                óº  <€ \         SV `  4        Ve   TMVP                  V n        \        P                  ! VP
                  V P                  VP                  R7      V n        \        P                  ! V P                  VP
                  VP                  R7      V n        \        P                  ! VP
                  V P                  VP                  R7      V n	        R # )N©Úbias)
ÚsuperÚ__init__r   ÚnnÚLinearr   r%   Ú	gate_projÚ	down_projÚup_proj)ÚselfrU   r   Ú	__class__s   &&&€r;   r[   ÚBailingMoeMLP.__init__=   s§   ø€ Ü‰ÑÔð !Ò,ñ à×'Ñ'ð 	Ôô ŸšØ×Ñ˜d×4Ñ4¸4¿=¹=ô
ˆŒô ŸšØ×"Ñ" D×$4Ñ$4¸4¿=¹=ô
ˆŒô —y’yØ×Ñ˜d×4Ñ4¸4¿=¹=ô
ˆŽr>   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   Úreturn©ÚmxÚarray)r9   r:   s   "€r;   r<   rV   O   s   ø€ ÷ Jñ J™RŸX™Xñ Jr>   c                ót   € V P                  \        V P                  V4      V P                  V4      4      4      # ©N)r_   r	   r^   r`   ©ra   Úxs   &&r;   Ú__call__ÚBailingMoeMLP.__call__O   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr>   )r_   r^   r   r`   rj   ©	r@   rA   rB   rC   r[   rm   rE   rF   Ú__classcell__©rb   r:   s   @@r;   rS   rS   <   s!   ù‡ € ÷
õ 
÷$J÷ Jð Jr>   rS   c                   óT   a a€ ] tR t^St oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚBailingMoeAttentionc                ó    <€ V ^8„  d   QhRS[ /# ©r   rU   ©r   )r9   r:   s   "€r;   r<   Ú BailingMoeAttention.__annotate__T   s   ø€ ÷ 
ñ 
™Yñ 
r>   c                ó  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP
                  V P                  ,          V n        V P                  R,          V n        \        P                  ! VP
                  V P                  ^V P                  ,          ,           V P                  ,          VP                  R7      V n        \        P                  ! V P                  V P                  ,          VP
                  VP                  R7      V n        VP                  '       dc   \        P                  ! V P                  VP                  R7      V n        \        P                  ! V P                  VP                  R7      V n        VP$                  ;pf'   \'        V P                  VP(                  ,          4      p\+        VVP,                  RVP.                  VP0                  R7      V n        R# )g      à?rX   ©ÚepsNF)ÚtraditionalÚscaling_configr   g      à¿)rZ   r[   r)   r   r   r   Úhead_dimÚscaler\   r]   r&   Úquery_key_valuer%   ÚdenseÚRMSNormr    Úkey_layernormÚquery_layernormr,   r6   r+   r   r!   r$   r   Úrope)ra   rU   Úrope_dimrb   s   && €r;   r[   ÚBailingMoeAttention.__init__T   st  ø€ Ü‰ÑÔØ×+Ñ+ˆÔØ#'×#;Ñ#;ˆÔ Ø#'×#;Ñ#;ˆÔ Ø×(Ñ(¨D×,DÑ,DÕDˆŒØ—]‘] DÕ(ˆŒ
ä!ŸyšyØ×ÑØ×%Ñ%¨¨D×,DÑ,DÕ(DÕDÈÏÉÕUØ×"Ñ"ô 
ˆÔô
 —Y’YØ×$Ñ$ t§}¡}Õ4Ø×ÑØ—‘ô
ˆŒ
ð ××ÐÜ!#§¢¨D¯M©M¸t×?PÑ?PÔ!QˆDÔÜ#%§:¢:¨d¯m©mÀ×ARÑARÔ#SˆDÔ àŸ™Ð'ˆHÒ0Ü˜4Ÿ=™=¨4×+EÑ+EÕEÓFˆHÜ#ØØ�O‰OØØ×,Ñ,Ø$(×$@Ñ$@ô
ˆŽ	r>   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   rl   ÚmaskÚcachere   ©rg   rh   r   r   )r9   r:   s   "€r;   r<   rw   u   sH   ø€ ÷ )"ñ )"á�8‰8ð)"ñ ‘r—x‘xÕ ð)"ñ ™�}ð	)"ñ
 
�‰ñ)"r>   c           	     ó:  € VP                   w  rEpV P                  V4      pV P                  V P                  ,          pV P                  V P                  ,          p	\
        P                  ! WxW‰,           .RR7      w  r«pV
P                  WEV P                  V P                  4      P                  ^ ^^^4      pVP                  WEV P                  V P                  4      P                  ^ ^^^4      pVP                  WEV P                  V P                  4      P                  ^ ^^^4      pV P                  '       d#   V P                  V4      pV P                  V4      pVeM   V P                  WÓP                  R7      pV P                  WãP                  R7      pVP                  Wï4      w  rïM"V P                  V4      pV P                  V4      p\        WÞWóV P                   VR7      pVP                  ^ ^^^4      P                  WER4      pV P#                  V4      # )é   rI   )Úoffset)rŠ   r~   r‰   éÿÿÿÿ)Úshaper   r   r}   r   rg   ÚsplitÚreshapeÚ	transposer)   rƒ   r‚   r„   rŽ   Úupdate_and_fetchr   r~   r€   )ra   rl   r‰   rŠ   ÚBÚLÚDÚqkvÚq_sizeÚkv_sizeÚqÚkÚvÚqueriesÚkeysÚvaluesÚoutputs   &&&&             r;   rm   ÚBailingMoeAttention.__call__u   sÏ  € ð —'‘'‰ˆˆaà×"Ñ" 1Ó%ˆà×)Ñ)¨D¯M©MÕ9ˆØ×*Ñ*¨T¯]©]Õ:ˆÜ—(’(˜3¨Õ)9Ð :ÀÔD‰ˆˆaà—)‘)˜A $×":Ñ":¸D¿M¹MÓJ×TÑTØˆq�!�Qó
ˆð �y‰y˜˜t×7Ñ7¸¿¹ÓG×QÑQØˆq�!�Qó
ˆð —‘˜1 ×!9Ñ!9¸4¿=¹=ÓI×SÑSØˆq�!�Qó
ˆð ××ÐØ×*Ñ*¨7Ó3ˆGØ×%Ñ% dÓ+ˆDàÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�z‰z˜&Ó!Ð!r>   )
r€   r}   r‚   r   r   r   rƒ   r„   r~   r)   ©NNro   rq   s   @@r;   rs   rs   S   s   ù‡ € ÷
ó 
÷B)"÷ )"ò )"r>   rs   c                 óò  € V P                   pVR 8X  d5   \        P                  ! V P                  \        P                  4      4      p	M5\        P
                  ! V P                  \        P                  4      RR7      p	T	p
Ve	   W‘,           p	V^8”  dÞ   \        P                  ! V	RVR3R7      p	\        P                  ! V	^RR7      P                  RRR7      pW4,
          p\        P                  ! W¼^,
          RR7      RRV1R3,          p\        P                  ! V	\        P                  ! V4      \        P                  ! R	V	P                   4      RR7      p	\        P                  ! V	RR4      p	Tp\        P                  ! Wœ) RR7      RV) R13,          p\        P                  ! W®RR7      p	V^8”  d+   V'       d#   V	P                  RRR7      R
,           pWŸ,          p	W•,          p	WéP                  V4      3# )ÚsigmoidrI   N)rJ   r�   T©rJ   Úkeepdims)ÚkthrJ   .:NNNg        g#B’¡œÇ;r�   rK   )rN   rg   r¥   rM   Úfloat32r   Ú	unflattenÚtopkrL   ÚargpartitionÚput_along_axisÚstop_gradientrh   ÚflattenÚtake_along_axis)ÚgatesÚe_score_correction_biasÚtop_kr1   r2   r/   r   r0   Úin_typerP   Úorig_scoresÚgroup_scoresrœ   Ú	group_idxÚindsÚdenominators   &&&&&&&&        r;   Úgroup_expert_selectrº   ¡   sŠ  € ð �k‰k€GØ˜Ô"Ü—’˜EŸL™L¬¯©Ó4Ó5‰ä—’˜EŸL™L¬¯©Ó4¸2Ô>ˆØ€KØÒ*ØÕ1ˆØ�„{Ü—’˜f¨2°g¸r°]ÔCˆÜ—w’w˜v q¨rÔ2×6Ñ6¸BÈÐ6ÓNˆØÕ ˆÜ—O’O L¸!µeÀ"ÔEÀcÈ2ÈAÈ2ÈqÀjÕQˆ	Ü×"Ò"Ø”B×$Ò$ YÓ/´·²¸#¸v¿|¹|Ó1LÐSUô
ˆô —’˜F B¨Ó+ˆà€AÜ�?Š?˜6 r°Ô3°C¸!¸¹°HÕ=€DÜ×Ò ¸Ô;€FØˆq„y—^Ø—j‘j b°4�jÓ8¸5Õ@ˆØÕ%ˆØÕ+€Fà—‘˜wÓ'Ð'Ð'r>   c                   óD   a a€ ] tR t^Êt oV3R lV 3R lltR tRtVtV ;t# )ÚBailingMoeGatec                ó    <€ V ^8„  d   QhRS[ /# ru   rv   )r9   r:   s   "€r;   r<   ÚBailingMoeGate.__annotate__Ë   s   ø€ ÷ 2ñ 2™Yñ 2r>   c                óê  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        VP                  V n	        \        P                  ! VP                  VP                  R R7      V n        VP                  '       d"   \         P"                  ! VP                  34      MRV n        VP&                  V n        R# ©FrX   N)rZ   r[   r   r   r³   r1   r2   r/   r.   Úenable_routed_scalingr\   r]   r   r   r^   r-   rg   ÚzerosÚexpert_biasr0   ©ra   rU   rb   s   &&€r;   r[   ÚBailingMoeGate.__init__Ë   s»   ø€ Ü‰ÑÔØ"×1Ñ1ˆÔà×-Ñ-ˆŒ
Ø—|‘|ˆŒØŸ/™/ˆŒØ%)×%?Ñ%?ˆÔ"Ø%)×%JÑ%JˆÔ"äŸš 4×#3Ñ#3°T×5EÑ5EÈEÔRˆŒð ×1×1Ð1ô �HŠH�d×&Ñ&Ð(Ô)àð 	Ôð
 #×1Ñ1ˆÖr>   c           
     óÐ   € \        V P                  V4      V P                  V P                  V P                  V P
                  V P                  V P                  V P                  4      # rj   )	rº   r^   rÃ   r³   r1   r2   r/   r   r0   rk   s   &&r;   rm   ÚBailingMoeGate.__call__Ý   sT   € Ü"Ø�N‰N˜1ÓØ×ÑØ�J‰JØ�L‰LØ�O‰OØ×&Ñ&Ø×ÑØ×Ñó	
ð 		
r>   )	rÁ   rÃ   r^   r1   r   r/   r0   r³   r2   ro   rq   s   @@r;   r¼   r¼   Ê   s   ù‡ € ÷2ó 2÷$

ò 

r>   r¼   c                   óD   a a€ ] tR t^êt oV3R lV 3R lltR tRtVtV ;t# )ÚBailingMoeSparseMoeBlockc                ó    <€ V ^8„  d   QhRS[ /# ru   rv   )r9   r:   s   "€r;   r<   Ú%BailingMoeSparseMoeBlock.__annotate__ë   s   ø€ ÷ 
ñ 
™Yñ 
r>   c                óÆ  <€ \         SV `  4        Wn        VP                  V n        \	        VP
                  VP                  VP                  VP                  R 7      V n	        \        V4      V n        VP                  ;'       g    VP                  pVP                  ^ 8”  d7   VP                  '       d%   \        VW!P                  ,          R7      V n        R# RV n        R# )rX   )rU   r   N)rZ   r[   rU   r   r   r   r   r   r%   Ú
switch_mlpr¼   Úgater3   r   r4   rS   Úshared_experts)ra   rU   Ú
shared_dimrb   s   && €r;   r[   Ú!BailingMoeSparseMoeBlock.__init__ë   sÂ   ø€ Ü‰ÑÔØŒ	Ø#'×#;Ñ#;ˆÔ Ü#Ø×ÑØ×&Ñ&Ø×ÑØ—‘ô	
ˆŒô # 4Ó(ˆŒ	à×4Ñ4×RÐR¸×8RÑ8Rð 	ð ×&Ñ&¨Ô*¨t×/S×/SÐ/Sô	 ØØ",×/FÑ/FÕ"Fôð 	Öð ð 	Ör>   c                ó®   € V P                  V4      w  r#V P                  W4      p\        WC4      pV P                  e   W@P                  V4      ,           pV# rj   )rÎ   rÍ   rQ   rÏ   )ra   rl   Útopk_idxÚtopk_weightÚouts   &&   r;   rm   Ú!BailingMoeSparseMoeBlock.__call__  sN   € Ø $§	¡	¨!£ÑˆØ�o‰o˜aÓ*ˆÜ& sÓ8ˆØ×ÑÒ*Ø×+Ñ+¨AÓ.Õ.ˆCØˆ
r>   )rU   rÎ   r   rÏ   rÍ   ro   rq   s   @@r;   rÉ   rÉ   ê   s   ù‡ € ÷
ó 
÷.ò r>   rÉ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚBailingMoeDecoderLayeri  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   rU   Ú	layer_idx)r   r6   )r9   r:   s   "€r;   r<   Ú#BailingMoeDecoderLayer.__annotate__  s   ø€ ÷ 
ñ 
™Yð 
±3ñ 
r>   c                óz  <€ \         SV `  4        \        V4      V n        VP                  e   W!P
                  8¼  d   \        V4      M
\        V4      V n        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  R7      V n        R # )Nry   )rZ   r[   rs   Ú	attentionr   r#   rÉ   rS   Úmlpr\   r�   r   r    Úinput_layernormÚpost_attention_layernorm)ra   rU   rÚ   rb   s   &&&€r;   r[   ÚBailingMoeDecoderLayer.__init__  s‘   ø€ Ü‰ÑÔÜ,¨TÓ2ˆŒð
 × Ñ Ò,°×>XÑ>XÔ1Xô % TÔ*ô ˜tÓ$ð 	Œô  "Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
ª
Ø×Ñ $×"3Ñ"3ô)
ˆÖ%r>   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# rˆ   r‹   )r9   r:   s   "€r;   r<   rÛ     sH   ø€ ÷ 	ñ 	á�8‰8ð	ñ ‘r—x‘xÕ ð	ñ ™�}ð		ñ
 
�‰ñ	r>   c                ó¤   € V P                  V P                  V4      W#4      pW,           pV P                  V P                  V4      4      pWT,           # rj   ©rÝ   rß   rÞ   rà   )ra   rl   r‰   rŠ   ÚrÚhs   &&&&  r;   rm   ÚBailingMoeDecoderLayer.__call__  sF   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ�EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�uˆr>   rä   r£   ro   rq   s   @@r;   rØ   rØ     s   ù‡ € ÷
ó 
÷ 	÷ 	ò 	r>   rØ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚBailingMoeModeli(  c                ó    <€ V ^8„  d   QhRS[ /# ru   rv   )r9   r:   s   "€r;   r<   ÚBailingMoeModel.__annotate__)  s   ø€ ÷ Hñ H™Yñ Hr>   c                óX  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  4       Uu. uF  p\        WR 7      NK  	  upV n
        \        P                  ! VP
                  VP                  R7      V n        R# u upi ))rÚ   ry   N)rZ   r[   r\   Ú	Embeddingr"   r   Úword_embeddingsÚranger   rØ   Úlayersr�   r    Únorm)ra   rU   Úirb   s   && €r;   r[   ÚBailingMoeModel.__init__)  s�   ø€ Ü‰ÑÔÜ!Ÿ|š|¨D¯O©O¸T×=MÑ=MÓNˆÔô ˜4×1Ñ1Ô2ó
á2�ô # 4×5Ù2ñ
ˆŒô —J’J˜t×/Ñ/°T×5FÑ5FÔGˆŽ	ùò	
s   ÁB'c                óJ   <€ V ^8„  d   QhRS[ P                  RS[S[,          /# )r   ÚinputsrŠ   r‹   )r9   r:   s   "€r;   r<   rë   2  s'   ø€ ÷ ñ á—‘ðñ ™�}ñr>   c                óü   € V P                  V4      pVf   R .\        V P                  4      ,          p\        W2^ ,          4      p\	        V P                  V4       F  w  rVV! W4V4      pK  	  V P                  V4      # rj   )rî   Úlenrð   r   Úziprñ   )ra   rõ   rŠ   ræ   r‰   ÚlayerÚcs   &&&    r;   rm   ÚBailingMoeModel.__call__2  so   € ð
 × Ñ  Ó(ˆàŠ=Ø�FœS §¡Ó-Õ-ˆEä$ Q¨a­Ó1ˆä˜DŸK™K¨Ö/‰HˆEÙ�a˜qÓ!ŠAñ 0ð �y‰y˜‹|Ðr>   )rð   rñ   rî   rj   ro   rq   s   @@r;   ré   ré   (  s    ù‡ € ÷Hó H÷÷ ò r>   ré   c                   óŠ   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltR t]R 4       t]R 4       t	]R	 4       t
R
tVtV ;t# )ÚModeliD  c                ó    <€ V ^8„  d   QhRS[ /# ru   rv   )r9   r:   s   "€r;   r<   ÚModel.__annotate__E  s   ø€ ÷ Tñ T™Yñ Tr>   c                ó   <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \        V4      V n        VP                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# R# rÀ   )rZ   r[   rU   r'   r   ré   Úmodelr*   r\   r]   r   r"   Úlm_headrÄ   s   &&€r;   r[   ÚModel.__init__E  sd   ø€ Ü‰ÑÔØŒ	ØŸ™ˆŒØŸ/™/ˆŒÜ$ TÓ*ˆŒ
Ø×'×'Ð'ÜŸ9š9 T×%5Ñ%5°t·±ÈUÔSˆDŽLñ (r>   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   rõ   rf   )r9   r:   s   "€r;   r<   rÿ   N  s   ø€ ÷ 
ñ 
á—‘ñ
r>   c                óÐ   € V P                  W4      pV P                  P                  '       d(   V P                   P                  P	                  V4      pV# V P                  V4      pV# rj   )r  rU   r*   rî   Ú	as_linearr  )ra   rõ   rŠ   rÕ   s   &&& r;   rm   ÚModel.__call__N  sX   € ð
 �j‰j˜Ó'ˆØ�9‰9×(×(Ð(Ø—*‘*×,Ñ,×6Ñ6°sÓ;ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r>   c                ó¶  € V P                   P                  '       d   VP                  R R4       V P                  '       dv   VR ,          pVP                  p\
        P                  P                  VP                  \
        P                  4      ^ RR7      R,           pW$,          P                  V4      VR &   \        V P                   P                  4       Fû  pRV 2pWPP                   P                  8¼  g   K$  R F…  pR F|  pV RV RV 2V9   g   K  \        V P                   P                  4       U	u. uF  p	VP                  V RV	 RV RV 24      NK!  	  p
p	\
        P                  ! V
4      W R	V RV 2&   K~  	  K‡  	  V R
2V9   d   VP                  V R
24      pW±V R2&   V R2V9   g   Kà  VP                  V R24      pWÁV R2&   Ký  	  V# u up	i )zlm_head.weightNTr¦   gH¯¼šò×z>zmodel.layers.z.mlp.experts.0.Ú.z.mlp.experts.z.mlp.switch_mlp.z.mlp.gate.weightz.mlp.gate.gate_proj.weightz.mlp.gate.biasz.mlp.gate.gate_proj.bias)r^   r_   r`   )ÚweightÚscalesÚbiases)rU   r*   Úpopr'   rN   rg   Úlinalgrñ   rM   r©   rï   r   r#   r   Ústack)ra   ÚweightsÚwrN   Úweight_normÚlÚprefixÚmrœ   ÚeÚto_joinÚgate_weightÚ	gate_biass   &&           r;   ÚsanitizeÚModel.sanitizeZ  sÛ  € Ø�9‰9×(×(Ð(Ø�K‰KÐ(¨$Ô/à�>�>ˆ>ØÐ(Õ)ˆAØ—G‘GˆEä—	‘	—‘˜qŸx™x¬¯
©
Ó3¸!Àd�ÓKÈdÕRð ð *+­×(@Ñ(@ÀÓ(GˆGÐ$Ñ%ä�t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFà—I‘I×3Ñ3Ö3Û>�AÛ;˜Ø$˜X _°Q°C°q¸¸Ð<ÀÖGô */¨t¯y©y×/DÑ/DÔ)Eó'á)E Að !(§¡¨v¨h°mÀAÀ3ÀaÈÀsÈ!ÈAÈ3Ð,OÖ PÙ)Eð $ð 'ô KMÏ(Ê(Ø 'óK˜G hÐ.>¸q¸cÀÀ1À#Ð$FÓGó <ñ ?ð �XÐ-Ð.°'Ô9Ø")§+¡+°°Ð8HÐ.IÓ"J�KØEP˜v˜hÐ&@ÐAÑBà�X˜^Ð,°Ö7Ø '§¡¨v¨h°nÐ,EÓ F�IØCL˜v˜hÐ&>Ð?Ó@ñ+ 4ð. ˆùò!'s   Ä5%G
c                ó   € R  pV# )c                 ó@   € V P                  R 4      '       d   R^@R^/# R# )zmlp.gate.gate_projÚ
group_sizeÚbitsT)Úendswith)ÚpathÚ_s   &&r;   Ú	predicateÚ(Model.quant_predicate.<locals>.predicate�  s$   € Ø�}‰}Ð1×2Ò2Ø$ b¨&°!Ð4Ð4Ùr>   r?   ©ra   r#  s   & r;   Úquant_predicateÚModel.quant_predicate  s   € ò	ð
 Ðr>   c                ó   € R  pV# )c                 ó   € R V 9  # )rÃ   r?   )rœ   s   &r;   r#  Ú'Model.cast_predicate.<locals>.predicateŠ  s   € Ø ¨Ñ)Ð)r>   r?   r%  s   & r;   Úcast_predicateÚModel.cast_predicateˆ  s   € ò	*ð Ðr>   c                ó.   € V P                   P                  # rj   )r  rð   )ra   s   &r;   rð   ÚModel.layers�  s   € à�z‰z× Ñ Ð r>   )rU   r  r  r   r'   rj   )r@   rA   rB   rC   r[   rm   r  Úpropertyr&  r+  rð   rE   rF   rp   rq   s   @@r;   rý   rý   D  sb   ù‡ € ÷Tó T÷
ò 
ò#ðJ ñó ðð ñó ðð ñ!ó ÷!ð !r>   rý   )$Údataclassesr   Ú	functoolsr   Útypingr   r   r   r   Úmlx.coreÚcorerg   Úmlx.nnr\   Úactivationsr	   Úbaser
   r   r   Ú
rope_utilsr   Úswitch_layersr   r   ÚcompilerQ   ÚModulerS   rs   rº   r¼   rÉ   rØ   ré   rý   r?   r>   r;   Ú<module>r<     sõ   ðõ "Ý ß -Ó -å Ý å ß TÑ TÝ 'Ý $ð ô!1�ó !1ó ð!1ñH 	ˆ�‰˜tÔ$ñó %ðôJ�B—I‘Iô Jô.K"˜"Ÿ)™)ô K"ð\ ‡�ñ%(ó ð%(ôP
�R—Y‘Yô 
ô@˜rŸy™yô ôB˜RŸY™Yô ô:�b—i‘iô ô8M!ˆB�I‰Iö M!r>   