+
    TV-jÅ5  ã                   ó
  € ^ RI t ^ RIHt ^ RIHt ^ RIHtHtHt ^ RI	H
t ^ RIHt ^ RIHtHtHt ^RIHt ^RIHtHtHt ^RIHt ^R	IHt ] ! R
 R]4      4       t ! R R]P:                  4      t ! R R]P:                  4      t]P@                  R 4       t! ! R R]P:                  4      t" ! R R]P:                  4      t# ! R R]P:                  4      t$ ! R R]]P:                  4      t% ! R R]P:                  4      t&R# )é    N)Ú	dataclass)Úpartial)ÚAnyÚDictÚOptional)Úshard_inplaceÚshard_linearÚsum_gradients)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚPipelineMixin)Ú	SwitchGLUc                   ó4   a € ] tR t^t o RtRtV 3R ltRtV tR# )Ú	ModelArgsÚsigmoidÚnoaux_tcc                ón  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[,          ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[ ;R&   S[ ;R&   # )é   Ú
model_typeÚ
vocab_sizeÚhidden_sizeÚintermediate_sizeÚmax_position_embeddingsÚmoe_intermediate_sizeÚnorm_topk_probÚnum_attention_headsÚn_groupÚhead_dimÚ
topk_groupÚn_shared_expertsÚn_routed_expertsÚrouted_scaling_factorÚnum_experts_per_tokÚfirst_k_dense_replaceÚnum_hidden_layersÚnum_key_value_headsÚrms_norm_epsÚ
rope_thetaÚrope_scalingÚuse_qk_normÚtie_word_embeddingsÚattention_biasÚpartial_rotary_factorÚscoring_funcÚtopk_method)ÚstrÚintÚboolÚfloatr   r   )ÚformatÚ__classdict__s   "€Úg/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/glm4_moe.pyÚ__annotate__ÚModelArgs.__annotate__   sa  ø‡ ‚ á�Oñ ñ �Oñ ñ Ññ	 ñ
 Ññ ñ !Ñ ñ ñ Ññ ñ Ññ ñ Ññ ñ �Lñ ñ �Mñ ñ �Oñ ñ Ññ ñ Ññ ñ !Ñ ñ ñ  Ññ! ñ" Ññ# ñ$ Ññ% ñ& Ññ' ñ( Ññ) ñ* Ññ+ ñ, ™4•.Ñ ñ- ñ. Ññ/ ñ0 Ññ1 ñ2 Ññ3 ñ4 !Ñ ñ5 ñ6 Ñ!ñ7 ñ8 Ñ!ò9 ó    © N)	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r0   r1   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r7   s   @r8   r   r      s   ø‡ € ð6 "€LØ!€K÷9 ƒ r;   r   c                   óT   a a€ ] tR t^1t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó    <€ V ^8„  d   QhRS[ /# )r   Úargs©r   )r6   r7   s   "€r8   r9   ÚAttention.__annotate__2   s   ø€ ÷ 
ñ 
™Yñ 
r;   c                óx  <€ \         SV `  4        VP                  pVP                  ;V n        pVP
                  ;V n        pVP                  pVR,          V n        \        P                  ! W#V,          VP                  R7      V n        \        P                  ! W$V,          VP                  R7      V n        \        P                  ! W$V,          VP                  R7      V n        \        P                  ! W5,          VRR7      V n        VP                   V n        V P                   '       dM   \        P"                  ! WQP$                  R7      V n        \        P"                  ! WQP$                  R7      V n        \        P*                  ! \-        WQP.                  ,          4      RVP0                  R7      V n        R# )g      à?©ÚbiasF©Úeps)ÚtraditionalÚbaseNg      à¿)ÚsuperÚ__init__r   r   Ún_headsr(   Ú
n_kv_headsr    ÚscaleÚnnÚLinearr.   Úq_projÚk_projÚv_projÚo_projr,   ÚRMSNormr)   Úq_normÚk_normÚRoPEr3   r/   r*   Úrope)ÚselfrG   ÚdimrS   rT   r    Ú	__class__s   &&    €r8   rR   ÚAttention.__init__2   s0  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*à—=‘=ˆØ˜t•^ˆŒ
ä—i’i ¨xÕ%7¸d×>QÑ>QÔRˆŒÜ—i’i °(Õ%:À×ATÑATÔUˆŒÜ—i’i °(Õ%:À×ATÑATÔUˆŒÜ—i’i Õ 2°C¸eÔDˆŒà×+Ñ+ˆÔØ××ÐÜŸ*š* X×3DÑ3DÔEˆDŒKÜŸ*š* X×3DÑ3DÔEˆDŒKä—G’GÜ�×5Ñ5Õ5Ó6ØØ—‘ô
ˆŽ	r;   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   ÚxÚmaskÚcacheÚreturn©ÚmxÚarrayr   r   )r6   r7   s   "€r8   r9   rI   L   sH   ø€ ÷  #ñ  #á�8‰8ð #ñ ‘r—x‘xÕ ð #ñ ™�}ð	 #ñ
 
�‰ñ #r;   c           	     óŽ  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pVP	                  WEV P
                  R4      pVP	                  WEV P                  R4      pV P                  '       d#   V P                  V4      pV P                  V4      pVP                  ^ ^^^4      pVP                  ^ ^^^4      pV	P	                  WEV P                  R4      P                  ^ ^^^4      p	VeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      P	                  WER4      p
V P!                  V
4      # )é   )Úoffset)ri   rU   rh   éÿÿÿÿ)ÚshaperX   rY   rZ   ÚreshaperS   rT   r,   r]   r^   Ú	transposer`   rp   Úupdate_and_fetchr   rU   r[   )ra   rg   rh   ri   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r8   Ú__call__ÚAttention.__call__L   s�  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆà—/‘/ !¨¯©°bÓ9ˆØ�|‰|˜A $§/¡/°2Ó6ˆà××ÐØ—k‘k 'Ó*ˆGØ—;‘;˜tÓ$ˆDà×#Ñ# A q¨!¨QÓ/ˆØ�~‰~˜a  A qÓ)ˆØ—‘  d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆØÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r;   )r^   rY   rS   rT   r[   r]   rX   r`   rU   r,   rZ   ©NN©	r=   r>   r?   r@   rR   r}   rB   rC   Ú__classcell__©rc   r7   s   @@r8   rE   rE   1   s   ù‡ € ÷
ó 
÷4 #÷  #ò  #r;   rE   c                   óH   a a€ ] tR t^ot oRV3R lV 3R llltR tRtVtV ;t# )ÚMLPc                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r   Úconfigr   r   ©r   r3   )r6   r7   s   "€r8   r9   ÚMLP.__annotate__p   s*   ø€ ÷ Yñ YÙðYÙ.1ðYÙMPñYr;   c                ó¸  <€ \         SV `  4        Wn        Vf   VP                  MTV n        Vf   VP                  MTV n        \
        P                  ! V P                  V P                  RR7      V n        \
        P                  ! V P                  V P                  RR7      V n        \
        P                  ! V P                  V P                  RR7      V n	        R # )NFrK   )
rQ   rR   r†   r   r   rV   rW   Ú	gate_projÚup_projÚ	down_proj)ra   r†   r   r   rc   s   &&&&€r8   rR   ÚMLP.__init__p   s¨   ø€ ô 	‰ÑÔØŒØ1<Ò1D˜6×-Ò-È+ˆÔà(9Ò(AˆF×$Ò$ÐGXð 	Ôô Ÿš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŽr;   c                óx   € V P                  \        V P                  V4      V P                  V4      4      4      pV# ©N)rŒ   r   rŠ   r‹   )ra   rg   rŒ   s   && r8   r}   ÚMLP.__call__~   s.   € Ø—N‘N¤6¨$¯.©.¸Ó*;¸T¿\¹\È!»_Ó#MÓNˆ	ØÐr;   )r†   rŒ   rŠ   r   r   r‹   r   r€   r‚   s   @@r8   r„   r„   o   s   ù‡ € ÷Yõ Y÷ò r;   r„   c                 ó$  € \         P                  ! V P                  \         P                  4      4      pTpWq,           pV^8”  dÓ   \         P                  ! VR
VR
3R7      p\         P
                  ! V^R
R7      P                  R
RR7      p	W4,
          p
\         P                  ! Wš^,
          RR7      RRV
1R3,          p\         P                  ! V\         P                  ! V4      \         P                  ! R	4      RR7      p\         P                  ! VRR
4      pTp
\         P                  ! V) V
^,
          R
R7      RRV
13,          p\         P                  ! WŒR
R7      pV^8”  d$   V'       d   VP                  R
RR7      pW},          pWu,          pWÇ3# )ro   )Úaxisrr   ©r’   T)r’   Úkeepdims)Úkthr’   .N:NNNg        rq   éþÿÿÿ)rl   r   ÚastypeÚfloat32Ú	unflattenÚtopkÚsumÚargpartitionÚput_along_axisÚstop_gradientrm   ÚflattenÚtake_along_axis)ÚgatesÚe_score_correction_biasÚtop_kr   r!   r$   r   ÚscoresÚorig_scoresÚgroup_scoresÚkÚ	group_idxÚindsÚdenominators   &&&&&&&       r8   Úgroup_expert_selectr«   ƒ   sE  € ô �ZŠZ˜Ÿ™¤R§Z¡ZÓ0Ó1€FØ€KØÕ-€FØ�„{Ü—’˜f¨2°g¸r°]ÔCˆÜ—w’w˜v q¨rÔ2×6Ñ6¸BÈÐ6ÓNˆØÕ ˆÜ—O’O L¸!µeÀ"ÔEÀcÈ2ÈAÈ2ÈqÀjÕQˆ	Ü×"Ò"Ø”B×$Ò$ YÓ/´·²¸#³ÀRô
ˆô —’˜F B¨Ó+ˆà€AÜ�?Š?˜F˜7¨¨A­°BÔ7¸¸R¸a¸R¸Õ@€DÜ×Ò ¸Ô;€FØˆq„y—^Ø—j‘j b°4�jÓ8ˆØÕ%ˆØÕ+€Fàˆ<Ðr;   c                   óD   a a€ ] tR t^¦t oV3R lV 3R lltR tRtVtV ;t# )ÚMoEGatec                ó    <€ V ^8„  d   QhRS[ /# ©r   r†   rH   )r6   r7   s   "€r8   r9   ÚMoEGate.__annotate__§   s   ø€ ÷ Lñ L™yñ Lr;   c                óÚ  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  V n	        \        P                  ! V P                  VP                  34      V n        \        P                  ! V P                  34      V n        VP                  R 8X  g   Q R4       hR# )r   zUnsupported topk method.N)rQ   rR   r†   r%   r£   r   r#   r$   r   r!   rl   Úzerosr   Úweightr¢   r1   ©ra   r†   rc   s   &&€r8   rR   ÚMoEGate.__init__§   s·   ø€ Ü‰ÑÔØŒØ×/Ñ/ˆŒ
Ø$×3Ñ3ˆÔØ &× 7Ñ 7ˆÔØ%+×%AÑ%AˆÔ"Ø—~‘~ˆŒØ ×+Ñ+ˆŒÜ—h’h × 5Ñ 5°v×7IÑ7IÐJÓKˆŒÜ')§x¢x°×1FÑ1FÐ0HÓ'IˆÔ$Ø×!Ñ! ZÔ/ÐKÐ1KÓKÒ/r;   c           	     óÐ   € \        WP                  P                  ,          V P                  V P                  V P
                  V P                  V P                  V P                  4      # r�   )	r«   r³   ÚTr¢   r£   r   r!   r$   r   )ra   rg   s   &&r8   r}   ÚMoEGate.__call__´   sM   € Ü"Ø—‘—‘ÕØ×(Ñ(Ø�J‰JØ�L‰LØ�O‰OØ×&Ñ&Ø×Ñó
ð 	
r;   )	r†   r¢   r   r#   r   r$   r£   r!   r³   r€   r‚   s   @@r8   r­   r­   ¦   s   ù‡ € ÷Ló L÷	
ò 	
r;   r­   c                   óD   a a€ ] tR t^Àt oV3R lV 3R lltR tRtVtV ;t# )ÚMoEc                ó    <€ V ^8„  d   QhRS[ /# r¯   rH   )r6   r7   s   "€r8   r9   ÚMoE.__annotate__Á   s   ø€ ÷ #ñ #™yñ #r;   c                óX  <€ \         SV `  4        Wn        VP                  V n        \	        VP
                  VP                  VP                  4      V n        \        V4      V n
        VP                  e/   VP                  VP                  ,          p\        WR7      V n        R V n        R # )N)r†   r   )rQ   rR   r†   r%   r   r   r   r#   Ú
switch_mlpr­   Úgater"   r„   Úshared_expertsÚsharding_group)ra   r†   r   rc   s   && €r8   rR   ÚMoE.__init__Á   s’   ø€ Ü‰ÑÔØŒØ#)×#=Ñ#=ˆÔ Ü#Ø×ÑØ×(Ñ(Ø×#Ñ#ó
ˆŒô ˜F“OˆŒ	Ø×"Ñ"Ò.Ø &× <Ñ <¸v×?VÑ?VÕ VÐÜ"%Øô#ˆDÔð #ˆÖr;   c                óÞ  € V P                   e   \        V P                   4      ! V4      pV P                  V4      w  r#V P                  W4      pWCR,          ,          P	                  RR7      P                  VP                  4      pV P                  P                  e   W@P                  V4      ,           pV P                   e+   \        P                  P                  W@P                   R7      pV# )Nr“   ©Úgroup).Nr–   )rÁ   r
   r¿   r¾   r›   r—   Údtyper†   r"   rÀ   rl   ÚdistributedÚall_sum)ra   rg   r©   r¤   Úys   &&   r8   r}   ÚMoE.__call__Ô   s»   € Ø×ÑÒ*Ü˜d×1Ñ1Ô2°1Ó5ˆAà—y‘y “|‰ˆØ�O‰O˜AÓ$ˆØ˜	Õ"Õ"×'Ñ'¨RÐ'Ó0×7Ñ7¸¿¹Ó@ˆØ�;‰;×'Ñ'Ò3Ø×'Ñ'¨Ó*Õ*ˆAà×ÑÒ*Ü—‘×&Ñ& q×0CÑ0CÐ&ÓDˆAàˆr;   )r†   r¿   r%   rÁ   rÀ   r¾   r€   r‚   s   @@r8   rº   rº   À   s   ù‡ € ÷#ó #÷&ò r;   rº   c                   óT   a a€ ] tR t^ät oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   r†   Ú	layer_idxr‡   )r6   r7   s   "€r8   r9   ÚDecoderLayer.__annotate__å   s   ø€ ÷ 
ñ 
™yð 
±Sñ 
r;   c                óz  <€ \         SV `  4        \        V4      V n        VP                  e   W!P
                  8¼  d   \        V4      M
\        V4      V n        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  R7      V n        R # )NrM   )rQ   rR   rE   Ú	self_attnr#   r&   rº   r„   ÚmlprV   r\   r   r)   Úinput_layernormÚpost_attention_layernorm)ra   r†   rÎ   rc   s   &&&€r8   rR   ÚDecoderLayer.__init__å   s�   ø€ Ü‰ÑÔÜ" 6Ó*ˆŒð ×'Ñ'Ò3Ø×!=Ñ!=Ô=ô �ŒKô
 �V“ð 	Œô  "Ÿzšz¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(*¯
ª
Ø×Ñ F×$7Ñ$7ô)
ˆÖ%r;   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# rf   rk   )r6   r7   s   "€r8   r9   rÏ   õ   sH   ø€ ÷ 	ñ 	á�8‰8ð	ñ ‘r—x‘xÕ ð	ñ ™�}ð		ñ
 
�‰ñ	r;   c                ó¤   € V P                  V P                  V4      W#4      pW,           pV P                  V P                  V4      4      pWT,           # r�   )rÑ   rÓ   rÒ   rÔ   )ra   rg   rh   ri   ÚrÚhs   &&&&  r8   r}   ÚDecoderLayer.__call__õ   sF   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ�EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�uˆr;   )rÓ   rÒ   rÔ   rÑ   r   r€   r‚   s   @@r8   rÌ   rÌ   ä   s   ù‡ € ÷
ó 
÷ 	÷ 	ò 	r;   rÌ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚLanguageModeli  c                ó    <€ V ^8„  d   QhRS[ /# r¯   rH   )r6   r7   s   "€r8   r9   ÚLanguageModel.__annotate__  s   ø€ ÷ Lñ L™yñ Lr;   c                óx  <€ \         SV `  4        VP                  V n        \        P                  ! VP                  VP
                  4      V n        \        VP                  4       Uu. uF  p\        W4      NK  	  upV n
        \        P                  ! VP
                  VP                  R 7      V n        R# u upi )rM   N)rQ   rR   r   rV   Ú	Embeddingr   Úembed_tokensÚranger'   rÌ   Úlayersr\   r)   Únorm)ra   r†   Úidxrc   s   && €r8   rR   ÚLanguageModel.__init__  s‹   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜŸLšL¨×):Ñ):¸F×<NÑ<NÓOˆÔä16°v×7OÑ7OÔ1Pó
Ù1P¨#ŒL˜Ö%Ñ1Pñ
ˆŒô —J’J˜v×1Ñ1°v×7JÑ7JÔKˆŽ	ùò
s   Á(B7c                ód   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[ P                  /# )r   rg   ri   rj   rk   )r6   r7   s   "€r8   r9   rÞ     s5   ø€ ÷ ñ á�8‰8ðñ ™�}ðñ 
�‰ñ	r;   c                óî  € V P                  V4      pV P                  pV P                  pVf   R .\        V P                  4      ,          p\        W2^ ,          4      pWE^,
          8  d'   \        P                  P                  W4^,           4      p\        V P                  V4       F  w  rxV! W6VR7      pK  	  V^ 8w  dm   \        P                  P                  W4^,
          V,          4      pVR,          e5   \        P                  ! VR,          P                  V4      VR,          n        V^8”  d4   \        P                  P                  V4      R VP                  ^ ,           pV P                  V4      # )N)ri   rq   )rá   Úpipeline_rankÚpipeline_sizeÚlenÚpipeline_layersr   rl   rÇ   Ú	recv_likeÚzipÚsendÚdependsrz   Ú
all_gatherrr   rä   )	ra   rg   ri   rÙ   ré   rê   rh   ÚlÚcs	   &&&      r8   r}   ÚLanguageModel.__call__  s*  € ð
 ×Ñ˜aÓ ˆà×*Ñ*ˆØ×*Ñ*ˆàŠ=Ø�FœS ×!5Ñ!5Ó6Õ6ˆEÜ$ Q¨a­Ó1ˆð ¨1Õ,Ô,Ü—‘×(Ñ(¨¸QÕ->Ó@ˆAä˜×,Ñ,¨eÖ4‰DˆAÙ�! Ô#ŠAñ 5ð ˜AÔÜ—‘×#Ñ# A¸Õ(9¸]Õ'JÓKˆAØ�R�yÒ$Ü!#§¢¨E°"­I¯N©N¸AÓ!>��b•	”ð ˜1ÔÜ—‘×)Ñ)¨!Ó,¨\¨q¯w©w°q­zÐ:ˆAà�y‰y˜‹|Ðr;   )rá   rã   rä   r   r�   r€   r‚   s   @@r8   rÜ   rÜ     s    ù‡ € ÷Ló L÷÷ ò r;   rÜ   c                   ó�   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltR tRV3R lR llt]R	 4       t	]R
 4       t
RtVtV ;t# )ÚModeli-  c                ó    <€ V ^8„  d   QhRS[ /# r¯   rH   )r6   r7   s   "€r8   r9   ÚModel.__annotate__.  s   ø€ ÷ Tñ T™yñ Tr;   c                óÖ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        \        P                  ! VP                  VP                  R R7      V n
        R# )FrK   N)rQ   rR   rG   r   rÜ   ÚmodelrV   rW   r   r   Úlm_headr´   s   &&€r8   rR   ÚModel.__init__.  sL   ø€ Ü‰ÑÔØŒ	Ø ×+Ñ+ˆŒÜ" 6Ó*ˆŒ
Ü—y’y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŽr;   c                óJ   <€ V ^8„  d   QhRS[ P                  RS[S[,          /# )r   Úinputsri   rk   )r6   r7   s   "€r8   r9   rø   5  s'   ø€ ÷ !ñ !á—‘ð!ñ ™�}ñ!r;   c                óF   € V P                  W4      pV P                  V4      # r�   )rú   rû   )ra   rþ   ri   Úouts   &&& r8   r}   ÚModel.__call__5  s!   € ð
 �j‰j˜Ó'ˆØ�|‰|˜CÓ Ð r;   c                ó4  € V P                   P                  p\        V P                   P                  4       F•  pR V 2pR F‡  w  rVR F|  pV RV RV 2V9   g   K  \        V P                   P                  4       Uu. uF  pVP	                  V RV RV RV 24      NK!  	  p	p\
        P                  ! V	4      W RV RV 2&   K~  	  K‰  	  K—  	  VP                  4        UU
u/ uF#  w  rzVP                  R V 24      '       d   K!  WzbK%  	  up
p# u upi u up
pi )zmodel.layers.z.mlp.experts.0.Ú.z.mlp.experts.z.mlp.switch_mlp.))Úw1rŠ   )Úw2rŒ   )Úw3r‹   )r³   ÚscalesÚbiases)	rG   r'   râ   r#   Úpoprl   ÚstackÚitemsÚ
startswith)ra   ÚweightsÚ	mpt_layerrò   ÚprefixÚnÚmr§   ÚeÚto_joinÚvs   &&         r8   ÚsanitizeÚModel.sanitize=  s+  € Ø—I‘I×/Ñ/ˆ	ô �t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFÛU‘�Û7�AØ ˜ °°°1°Q°CÐ8¸GÖCô &+¨4¯9©9×+EÑ+EÔ%Fó#á%F ð $ŸK™K¨6¨(°-À¸sÀ!ÀAÀ3ÀaÈÀsÐ(KÖLÙ%Fð  ð #ô GIÇhÂhÈwÓFW˜ (Ð*:¸1¸#¸Q¸q¸cÐ BÓCó 8ó Vñ 4ð  Ÿ™œô
á'‘�Ø—<‘< -°	¨{Ð ;×<ô ˆAŠDÙ'ò
ð 	
ùò#ùó
s   Á<%D
Ã"DÄDc                óX   <€ V ^8„  d   QhRS[ S[P                  P                  ,          /# )r   rÅ   )r   rl   rÇ   ÚGroup)r6   r7   s   "€r8   r9   rø   S  s#   ø€ ÷ 5ñ 5™8¡B§N¡N×$8Ñ$8Õ9ñ 5r;   c                ód  € T;'       g    \         P                  P                  4       pVP                  4       pV P                  P
                   EFÜ  p\        VP                  P                  R VR7      VP                  n        \        VP                  P                  R VR7      VP                  n	        \        VP                  P                  R VR7      VP                  n
        \        VP                  P                  RVR7      VP                  n        VP                  ;P                  V,          un        VP                  ;P                  V,          un        \        VP                  \         4      '       d—   \        VP                  P"                  R VR7      VP                  n        \        VP                  P$                  RVR7      VP                  n        \        VP                  P&                  R VR7      VP                  n        EKÄ  WP                  n        \+        VP                  P,                  P"                  R VR7       \+        VP                  P,                  P$                  RVR7       \+        VP                  P,                  P&                  R VR7       \+        VP                  P.                  P"                  R VR7       \+        VP                  P.                  P$                  RVR7       \+        VP                  P.                  P&                  R VR7       EKß  	  R# )zall-to-shardedrÄ   zsharded-to-allN)rl   rÇ   ÚinitÚsizerú   rã   r	   rÑ   rX   rY   rZ   r[   rS   rT   Ú
isinstancerÒ   r„   rŠ   rŒ   r‹   rÁ   r   rÀ   r¾   )ra   rÅ   ÚNÚlayers   &&  r8   ÚshardÚModel.shardS  sW  € Ø×.Ð.œŸ™×,Ñ,Ó.ˆØ�J‰J‹LˆØ—Z‘Z×&Õ&ˆEä%1Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ð �O‰O×#Ò#¨Õ)Õ#Ø�O‰O×&Ò&¨1Õ,Õ&ô ˜%Ÿ)™)¤S×)Ò)Ü&2Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	Ô#ô '3Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	Ô#ô %1Ø—I‘I×%Ñ%Ð'7¸uô%�—	‘	×!ð ,1—	‘	Ô(ÜØ—I‘I×,Ñ,×6Ñ6Ð8HÐPUõô Ø—I‘I×,Ñ,×6Ñ6Ð8HÐPUõô Ø—I‘I×,Ñ,×4Ñ4Ð6FÈeõô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈEõô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈEõô Ø—I‘I×(Ñ(×0Ñ0Ð2BÈ%÷ð óa 'r;   c                ó.   € V P                   P                  # r�   )rú   rì   )ra   s   &r8   rã   ÚModel.layersŠ  s   € à�z‰z×)Ñ)Ð)r;   c                ó   € R  pV# )c                 ó   € R V 9  # )r¢   r<   )r§   s   &r8   Ú	predicateÚ'Model.cast_predicate.<locals>.predicate�  s   € Ø,°AÑ5Ð5r;   r<   )ra   r%  s   & r8   Úcast_predicateÚModel.cast_predicateŽ  s   € ò	6ð Ðr;   )rG   rû   rú   r   r�   )r=   r>   r?   r@   rR   r}   r  r  Úpropertyrã   r'  rB   rC   r�   r‚   s   @@r8   rö   rö   -  sX   ù‡ € ÷Tó T÷!ò !ò
÷,5ò 5ðn ñ*ó ð*ð ñó ÷ð r;   rö   )'ÚmathÚdataclassesr   Ú	functoolsr   Útypingr   r   r   Úmlx.coreÚcorerl   Úmlx.nnrV   Úmlx.nn.layers.distributedr   r	   r
   Úactivationsr   rP   r   r   r   Úpipeliner   Úswitch_layersr   r   ÚModulerE   r„   Úcompiler«   r­   rº   rÌ   rÜ   rö   r<   r;   r8   Ú<module>r7     sÜ   ðó Ý !Ý ß &Ñ &å Ý ß PÑ På ß TÑ TÝ #Ý $ð ô"�ó "ó ð"ô<;#�—	‘	ô ;#ô|ˆ"�)‰)ô ð( ‡�ñó ðôD
ˆb�i‰iô 
ô4!ˆ"�)‰)ô !ôH�2—9‘9ô ô:)�M 2§9¡9ô )ôXfˆB�I‰Iö fr;   