+
    TV-j/0  ã                   óÄ  € ^ RI Ht ^ RIHtHtHt ^ RIHt ^ RI	H
t
 ^RIHt ^RIHtHtHtHt ^RIHtHt ^RIHt ] ! R R	]4      4       t ! R
 R]
P0                  4      t ! R R]
P0                  4      t ! R R]
P0                  4      t ! R R]
P0                  4      t ! R R]
P0                  4      t ! R R]
P0                  4      t ! R R]
P0                  4      tR# )é    )Ú	dataclass)ÚAnyÚListÚOptionalN)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚcreate_ssm_maskÚscaled_dot_product_attention)ÚArraysCacheÚKVCache)Ú	SwitchGLUc                   óB   a € ] tR t^t o RtRtRtRtR tV 3R lt	Rt
V tR# )Ú	ModelArgsg    €„.ANc                ó  € V P                   e*   RV P                   9   d   V P                   R,          V n        V P                  f:   \        V P                  4       UUu. uF  w  rVR8X  g   K  VNK  	  uppV n        R # R # u uppi )NÚ
rope_thetaÚfull_attention)Úrope_parametersr   Úfull_attn_idxsÚ	enumerateÚlayer_types)ÚselfÚiÚ
layer_types   &  Úg/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/lfm2_moe.pyÚ__post_init__ÚModelArgs.__post_init__+   s}   € Ø×ÑÒ+°À×@TÑ@TÔ0TØ"×2Ñ2°<Õ@ˆDŒOØ×ÑÒ&ô &/¨t×/?Ñ/?Ô%@ô#á%@‘M�AØÐ!1Ñ1÷ �Ù%@ò#ˆDÖñ 'ùó#s   ÁBÁ.Bc                óf  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[,          ;R&   S[S[S[,          ,          ;R&   S[S[S[ ,          ,          ;R&   # )é   Ú
model_typeÚ
vocab_sizeÚhidden_sizeÚintermediate_sizeÚmoe_intermediate_sizeÚnum_hidden_layersÚnum_expertsÚnum_experts_per_tokÚnorm_topk_probÚnum_attention_headsÚnum_key_value_headsÚmax_position_embeddingsÚuse_expert_biasÚnum_dense_layersÚnorm_epsÚ	conv_biasÚconv_L_cacher   r   r   r   )ÚstrÚintÚboolÚfloatr   Údictr   )ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚModelArgs.__annotate__   s&  ø‡ ‚ á�Oñ ñ �Oñ ñ Ññ	 ñ
 Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ !Ñ ñ ñ Ññ ñ Ññ ñ  �Oñ! ñ" �Oñ# ñ$ Ññ% ñ& Ñ!ñ' ñ( ™d•^Ñ*ñ) ñ* ™T¡#�YÕ'Ñ.ñ+ ñ, ™$™s�)Õ$Ñ+ò- ó    )r   r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r   r   r   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r7   s   @r   r   r      s'   ø‡ € ð& "€JØ&*€OØ*.€NØ'+€Kò÷1 ƒ r:   r   c                   óT   a a€ ] tR t^6t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úargs©r   )r6   r7   s   "€r   r8   ÚAttention.__annotate__7   s   ø€ ÷ 
ñ 
™Yñ 
r:   c                óô  <€ \         SV `  4        VP                  pVP                  ;V n        pVP
                  ;V n        pVP                  V,          ;V n        pVR,          V n        \        P                  ! WQP                  R7      V n        \        P                  ! WQP                  R7      V n        \        P                  ! W#V,          RR7      V n        \        P                  ! W$V,          RR7      V n        \        P                  ! W$V,          RR7      V n        \        P                  ! W5,          VRR7      V n        \        P&                  ! V P                  VP(                  RR7      V n        R# )g      à?©ÚepsF©Úbias)ÚbaseÚtraditionalNg      à¿)ÚsuperÚ__init__r"   r)   Ún_headsr*   Ú
n_kv_headsÚhead_dimÚscaleÚnnÚRMSNormr.   Úq_layernormÚk_layernormÚLinearÚq_projÚk_projÚv_projÚout_projÚRoPEr   Úrope)r   rF   ÚdimrR   rS   rT   Ú	__class__s   &&    €r   rQ   ÚAttention.__init__7   s  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*à#'×#3Ñ#3°wÕ#>Ð>ˆŒ˜à˜t•^ˆŒ
äŸ:š: h·M±MÔBˆÔÜŸ:š: h·M±MÔBˆÔä—i’i ¨xÕ%7¸eÔDˆŒÜ—i’i °(Õ%:ÀÔGˆŒÜ—i’i °(Õ%:ÀÔGˆŒÜŸ	š	 'Õ"4°cÀÔFˆŒä—G’GØ�M‰MØ—‘Øô
ˆŽ	r:   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   ÚxÚmaskÚcacheÚreturn©ÚmxÚarrayr   r   )r6   r7   s   "€r   r8   rH   P   sH   ø€ ÷ %ñ %á�8‰8ð%ñ ‘r—x‘xÕ ð%ñ ™�}ð	%ñ
 
�‰ñ%r:   c           	     óX  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pV P	                  VP                  WEV P                  R4      4      P                  ^ ^^^4      pV P                  VP                  WEV P                  R4      4      P                  ^ ^^^4      pV	P                  WEV P                  R4      P                  ^ ^^^4      p	VeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW“W P                  R7      p
V
P                  ^ ^^^4      P                  WER4      p
V P                  V
4      # )é   )Úoffset)rh   rg   rU   éÿÿÿÿ)Úshaper[   r\   r]   rX   ÚreshaperR   Ú	transposerY   rS   r`   ro   Úupdate_and_fetchr   rU   r^   )r   rf   rg   rh   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r   Ú__call__ÚAttention.__call__P   so  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆà×"Ñ" 7§?¡?°1¸¿¹ÀrÓ#JÓK×UÑUØˆq�!�Qó
ˆð ×Ñ §¡¨Q°4·?±?ÀBÓ GÓH×RÑRØˆq�!�Qó
ˆð —‘  d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°TÇÁô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�}‰}˜VÓ$Ð$r:   )rT   rY   r\   rR   rS   r^   rX   r[   r`   rU   r]   ©NN©	r;   r<   r=   r>   rQ   r|   r@   rA   Ú__classcell__©rb   r7   s   @@r   rC   rC   6   s   ù‡ € ÷
ó 
÷2%÷ %ò %r:   rC   c                   óT   a a€ ] tR t^qt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	ShortConvc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r   rF   Ú	layer_idx©r   r2   )r6   r7   s   "€r   r8   ÚShortConv.__annotate__r   s#   ø€ ÷ Vñ VáðVñ ñVr:   c                ó"  <€ \         SV `  4        Wn        W n        VP                  V n        VP                  V n        \        P                  ! VP                  VP                  V P
                  VP                  V P                  R 7      V n        \        P                  ! VP                  ^VP                  ,          V P                  R7      V n        \        P                  ! VP                  VP                  V P                  R7      V n        R# ))Úin_channelsÚout_channelsÚkernel_sizeÚgroupsrM   rL   N)rP   rQ   rF   r†   r0   ÚL_cacher/   rM   rV   ÚConv1dr"   ÚconvrZ   Úin_projr^   ©r   rF   r†   rb   s   &&&€r   rQ   ÚShortConv.__init__r   sº   ø€ ô
 	‰ÑÔØŒ	Ø"ŒØ×(Ñ(ˆŒØ—N‘NˆŒ	ä—I’IØ×(Ñ(Ø×)Ñ)ØŸ™Ø×#Ñ#Ø—‘ô
ˆŒ	ô —y’y ×!1Ñ!1°1°t×7GÑ7GÕ3GÈdÏiÉiÔXˆŒÜŸ	š	 $×"2Ñ"2°D×4DÑ4DÈ4Ï9É9ÔUˆŽr:   c                ót   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          /# )r   rf   rg   rh   rj   )r6   r7   s   "€r   r8   rˆ   ‡   s:   ø€ ÷ $ ñ $ á�8‰8ð$ ñ ‘r—x‘xÕ ð$ ñ ™�}ñ	$ r:   c                óþ  € V P                  V4      p\        P                  ! V^RR7      w  rVpWQ,          pVe    \        P                  ! VR,          V^ 4      pVEeK   V^ ,          f]   \        P                  ! VP
                  ^ ,          V P                  ^,
          V P                  P                  3VP                  R7      pM	V^ ,          p\        P                  ! W‡.^R7      pV P                  ^,
          p	VP
                  ^,          p
VP                  ej   \        P                  ! VP                  ^ V
4      pVR,          \        P                  ! V	4      ,           R,          p\        P                  ! W|^R7      V^ &   MVRV	) R1R3,          V^ &   VP                  V
4       M-\        P                   ! VRV P                  ^,
          ^ 3R.4      pV P#                  V4      pWm,          pV P%                  V4      # )	é   ©ÚaxisN)ÚdtypeºNNNrp   ©.N)rš   N)r   r   )r‘   rk   ÚsplitÚwhereÚzerosrq   rŽ   rF   r"   r™   ÚconcatenateÚlengthsÚclipÚarangeÚtake_along_axisÚadvanceÚpadr�   r^   )r   rf   rg   rh   ÚBCxru   ÚCÚBxÚstateÚn_keepÚtÚendsÚ	positionsÚconv_outÚys   &&&&           r   r|   ÚShortConv.__call__‡   sŠ  € ð �l‰l˜1‹oˆÜ—(’(˜3 ¨Ô+‰ˆˆaØ�UˆØÒÜ—’˜$˜y�/¨2¨qÓ1ˆBàÓØ�Q�xÒÜŸšØ—X‘X˜a•[ $§,¡,°Õ"2°D·I±I×4IÑ4IÐJØŸ(™(ô‘ð
 ˜a��Ü—’  °!Ô4ˆBØ—\‘\ AÕ%ˆFØ—‘˜•
ˆAØ�}‰}Ò(Ü—w’w˜uŸ}™}¨a°Ó3�Ø! '�]¬R¯YªY°vÓ->Õ>À	ÕJ�	Ü×-Ò-¨bÀ!ÔD��a’à˜a & ¡¨1˜nÕ-��a‘Ø�M‰M˜!Õä—’˜˜V d§l¡l°QÕ&6¸Ð%:¸FÐCÓDˆBà—9‘9˜R“=ˆà�LˆØ�}‰}˜QÓÐr:   )rŽ   rF   rM   r�   r‘   r†   r^   r~   r   r�   s   @@r   rƒ   rƒ   q   s    ù‡ € ÷Vó V÷*$ ÷ $ ò $ r:   rƒ   c                   óT   a a€ ] tR t^®t oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚMLPc                ó6   <€ V ^8„  d   QhRS[ RS[S[,          /# )r   Úconfigr#   )r   r   r2   )r6   r7   s   "€r   r8   ÚMLP.__annotate__¯   s#   ø€ ÷ Yñ Y™yð Y¹XÁc½]ñ Yr:   c                ó   <€ \         SV `  4        VP                  V n        Vf   VP                  MTV n        \        P
                  ! V P                  V P                  RR7      V n        \        P
                  ! V P                  V P                  RR7      V n        \        P
                  ! V P                  V P                  RR7      V n        R # )NFrL   )	rP   rQ   r"   r#   rV   rZ   Ú	gate_projÚup_projÚ	down_proj)r   r´   r#   rb   s   &&&€r   rQ   ÚMLP.__init__¯   sš   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔà(9Ò(AˆF×$Ò$ÐGXð 	Ôô Ÿš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŽr:   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   ri   ©rk   rl   )r6   r7   s   "€r   r8   rµ   ¹   s   ø€ ÷ Jñ J™RŸX™Xñ Jr:   c                ót   € V P                  \        V P                  V4      V P                  V4      4      4      # ©N)r¹   r   r·   r¸   )r   rf   s   &&r   r|   ÚMLP.__call__¹   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr:   )r¹   r·   r"   r#   r¸   r¾   r   r�   s   @@r   r²   r²   ®   s#   ù‡ € ÷Yõ Y÷J÷ Jð Jr:   r²   c                   óP   a a€ ] tR t^½t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚLfm2MoeSparseMoeBlockc                ó    <€ V ^8„  d   QhRS[ /# rE   rG   )r6   r7   s   "€r   r8   Ú"Lfm2MoeSparseMoeBlock.__annotate__¾   s   ø€ ÷ =ñ =™Yñ =r:   c                ó°  <€ \         SV `  4        VP                  pVP                  pVP                  ;V n        pVP
                  V n        VP                  V n        VP                  V n        \        P                  ! W$R R7      V n        \        W#V4      V n        V P                  '       d)   \        P                  ! V P                  34      V n        R# R# )FrL   N)rP   rQ   r"   r$   r&   r'   Útop_kr(   r,   rV   rZ   Úgater   Ú
switch_mlprk   rž   Úexpert_bias)r   rF   ra   r#   r&   rb   s   &&   €r   rQ   ÚLfm2MoeSparseMoeBlock.__init__¾   s¬   ø€ Ü‰ÑÔØ×ÑˆØ ×6Ñ6Ðà)-×)9Ñ)9Ð9ˆÔ˜;Ø×-Ñ-ˆŒ
Ø"×1Ñ1ˆÔØ#×3Ñ3ˆÔä—I’I˜c°UÔ;ˆŒ	Ü# C¸KÓHˆŒØ××ÐÜ!Ÿxšx¨×)9Ñ)9Ð(;Ó<ˆDÖñ  r:   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   rf   r¼   )r6   r7   s   "€r   r8   rÃ   Í   s   ø€ ÷ ñ á�8‰8ñr:   c                ót  € V P                  V4      P                  \        P                  4      p\        P                  ! VRR7      pV P
                  '       d   W P                  ,          pV P                  p\        P                  ! W#) RR7      RV) R13,          p\        P                  ! W$RR7      pV P                  '       d(   V\        P                  ! VRRR7      R,           ,          pVP                  VP                  4      pV P                  W4      pWeR	,          ,          P                  R
R7      pV# )rn   r—   )Úkthr˜   .NT)r˜   Úkeepdimsg#B’¡œÇ;rp   r›   éþÿÿÿ)rÆ   Úastyperk   Úfloat32Úsoftmaxr,   rÈ   rÅ   Úargpartitionr£   r(   Úsumr™   rÇ   )r   rf   ÚgatesÚkÚindsÚscoresr¯   s   &&     r   r|   ÚLfm2MoeSparseMoeBlock.__call__Í   sì   € ð —	‘	˜!“×#Ñ#¤B§J¡JÓ/ˆÜ—
’
˜5 rÔ*ˆà××ÐØ×%Ñ%Õ%ˆEà�J‰JˆÜ�Š˜u¨"°2Ô6°s¸Q¸B¹C°xÕ@ˆä×#Ò# E°bÔ9ˆØ××ÐØ”b—f’f˜V¨"°tÔ<¸uÕDÕDˆFØ—‘˜qŸw™wÓ'ˆà�O‰O˜AÓ$ˆØ˜	Õ"Õ"×'Ñ'¨RÐ'Ó0ˆàˆr:   )rÈ   rÆ   r(   r&   rÇ   rÅ   r,   r   r�   s   @@r   rÁ   rÁ   ½   s   ù‡ € ÷=ó =÷÷ ð r:   rÁ   c                   óT   a a€ ] tR t^åt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚLfm2DecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# r…   r‡   )r6   r7   s   "€r   r8   ÚLfm2DecoderLayer.__annotate__æ   s   ø€ ÷ Hñ H™Yð H±3ñ Hr:   c                óâ  <€ \         SV `  4        W!P                  9   V n        V P                  '       d   \	        V4      V n        M\        W4      V n        W!P                  8  d   \        VVP                  R 7      M
\        V4      V n        \        P                  ! VP                  VP                   R7      V n        \        P                  ! VP                  VP                   R7      V n        R# ))r´   r#   rJ   N)rP   rQ   r   Úis_attention_layerrC   Ú	self_attnrƒ   r�   r-   r²   r#   rÁ   Úfeed_forwardrV   rW   r"   r.   Úoperator_normÚffn_normr’   s   &&&€r   rQ   ÚLfm2DecoderLayer.__init__æ   s³   ø€ Ü‰ÑÔØ"+×/BÑ/BÑ"BˆÔà×"×"Ð"Ü& t›_ˆD�Nä! $Ó2ˆDŒIð ×0Ñ0Ô0ô	 ØØ"&×"8Ñ"8õô
 ' tÓ,ð 	Ôô  ŸZšZ¨×(8Ñ(8¸d¿m¹mÔLˆÔÜŸ
š
 4×#3Ñ#3¸¿¹ÔGˆŽr:   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# re   rj   )r6   r7   s   "€r   r8   rÜ   ú   sH   ø€ ÷ ñ á�8‰8ðñ ‘r—x‘xÕ ðñ ™�}ð	ñ
 
�‰ñr:   c                ó  € V P                   '       d$   V P                  V P                  V4      W#R 7      pM#V P                  V P                  V4      VVR 7      pW,           pWPP	                  V P                  V4      4      ,           pV# ))rg   rh   )rÞ   rß   rá   r�   rà   râ   )r   rf   rg   rh   ÚrÚhÚouts   &&&&   r   r|   ÚLfm2DecoderLayer.__call__ú   s~   € ð ×"×"Ð"Ø—‘˜t×1Ñ1°!Ó4¸4�ÓM‰Aà—	‘	Ø×"Ñ" 1Ó%ØØð ó ˆAð
 �EˆØ×#Ñ# D§M¡M°!Ó$4Ó5Õ5ˆØˆ
r:   )r�   rà   râ   rÞ   rá   rß   r~   r   r�   s   @@r   rÚ   rÚ   å   s    ù‡ € ÷Hó H÷(÷ ò r:   rÚ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Lfm2Modeli  c                ó    <€ V ^8„  d   QhRS[ /# rE   rG   )r6   r7   s   "€r   r8   ÚLfm2Model.__annotate__  s   ø€ ÷ ñ ™Yñ r:   c                óv  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  4      V n        \        VP                  4       Uu. uF  p\        WR 7      NK  	  upV n        \
        P                  ! VP                  VP                  R7      V n        VP                  ^ ,          V n        ^ V n        \        VP                  4       F/  pW!P                  9   d   V ;P"                  ^,          un        K.   R# 	  R# u upi ))r†   rJ   N)rP   rQ   rF   r!   r%   rV   Ú	Embeddingr"   Úembed_tokensÚrangerÚ   ÚlayersrW   r.   Úembedding_normr   Úfa_idxÚconv_idx)r   rF   r   rb   s   && €r   rQ   ÚLfm2Model.__init__  så   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔä9>¸t×?UÑ?UÔ9Vó
Ù9V°AÔ˜T×/Ñ9Vñ
ˆŒô !Ÿjšj¨×)9Ñ)9¸t¿}¹}ÔMˆÔà×)Ñ)¨!Õ,ˆŒØˆŒÜ�t×-Ñ-Ö.ˆAØ×'Ñ'Ô'Ø—’ Õ"—âó	 /ùò
s   Á?D6c                ó^   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          /# ©r   ÚinputsÚinput_embeddings©rk   rl   r   )r6   r7   s   "€r   r8   rí   #  s,   ø€ ÷ &ñ &á—‘ð&ñ #¡2§8¡8Õ,ñ	&r:   c                ó„  € Ve   TpMV P                  V4      pVf   R .\        V P                  4      ,          p\        WBV P                  ,          4      p\        WBV P                  ,          4      p\        V P                  V4       F%  w  rxVP                  '       d   TMTp	V! WIVR7      pK'  	  V P                  V4      # )N)rh   )
rð   Úlenrò   r	   rô   r
   rõ   ÚziprÞ   ró   )
r   rù   rh   rú   rç   Ú	attn_maskÚ	conv_maskÚlayerÚcrg   s
   &&&&      r   r|   ÚLfm2Model.__call__#  s§   € ð Ò'Ø ‰Aà×!Ñ! &Ó)ˆAàŠ=Ø�FœS §¡Ó-Õ-ˆEä)¨!°4·;±;Õ-?Ó@ˆ	Ü# A¨T¯]©]Õ';Ó<ˆ	ä˜DŸK™K¨Ö/‰HˆEØ %× 8× 8Ð 8‘9¸iˆDÙ�a QÔ'ŠAñ 0ð ×"Ñ" 1Ó%Ð%r:   )rF   rõ   rð   ró   rô   rò   r%   r!   r~   r   r�   s   @@r   rë   rë     s   ù‡ € ÷ó ÷(&÷ &ò &r:   rë   c                   ó�   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltR t]R 4       tR t	]R	 4       t
]R
 4       tRtVtV ;t# )ÚModeli;  c                ó    <€ V ^8„  d   QhRS[ /# rE   rG   )r6   r7   s   "€r   r8   ÚModel.__annotate__<  s   ø€ ÷ %ñ %™Yñ %r:   c                ór   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        R # r¾   )rP   rQ   rF   r    rë   Úmodel)r   rF   rb   s   &&€r   rQ   ÚModel.__init__<  s)   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ˜t“_ˆŽ
r:   c                ó^   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          /# rø   rû   )r6   r7   s   "€r   r8   r  B  s,   ø€ ÷ 6ñ 6á—‘ð6ñ #¡2§8¡8Õ,ñ	6r:   c                óp   € V P                  WV4      pV P                   P                  P                  V4      # r¾   )r	  rð   Ú	as_linear)r   rù   rh   rú   rè   s   &&&& r   r|   ÚModel.__call__B  s0   € ð �j‰j˜Ð(8Ó9ˆØ�z‰z×&Ñ&×0Ñ0°Ó5Ð5r:   c                ó˜  € / pVP                  4        F†  w  r4R V9   d=   VP                  R,          VP                  ^,          8”  d   VP                  ^ ^^4      pRRRRRR/pVP                  4        F  w  rgWc9   g   K  VP                  Wg4      pK   	  WBV&   Kˆ  	  \	        V P
                  P                  4       F…  pRV 2p	R Fw  p
V	 RV
 R	2V9   g   K  \	        V P
                  P                  4       Uu. uF  pVP                  V	 R
V RV
 R	24      NK  	  pp\        P                  ! V4      VV	 RV
 R	2&   Ky  	  K‡  	  V# u upi )zconv.weightz	w1.weightzgate_proj.weightz	w2.weightzdown_proj.weightz	w3.weightzup_proj.weightzmodel.layers.z.feed_forward.experts.0.z.weightz.feed_forward.experts.Ú.z.feed_forward.switch_mlp.rp   )r·   r¹   r¸   )Úitemsrq   rs   Úreplacerñ   rF   r%   r&   Úpoprk   Ústack)r   ÚweightsÚsanitized_weightsÚnameÚparamÚreplacementsÚoldÚnewÚlÚprefixÚnÚeÚto_joins   &&           r   ÚsanitizeÚModel.sanitizeK  sn  € ØÐØ"Ÿ=™=ž?‰KˆDØ Ô$Ø—;‘;˜r•? U§[¡[°¥^Ô3Ø!ŸO™O¨A¨q°!Ó4�EàÐ/ØÐ/ØÐ-ðˆLð
 )×.Ñ.Ö0‘�Ø–;ØŸ<™<¨Ó1’Dñ 1ð ',˜dÓ#ñ +ô �t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFã:�Ø�XÐ5°a°S¸Ð@ÐDUÖUô
 "' t§y¡y×'<Ñ'<Ô!=ó	ñ ">˜Að *×-Ñ-Ø%˜hÐ&<¸Q¸C¸qÀÀÀ7ÐKöñ ">ð	 ð ô Ÿš Ó)ð &Ø!˜(Ð";¸A¸3¸gÐFóó ;ñ 4ð !Ð ùòs   Ã9#Ec                ó.   € V P                   P                  # r¾   )r	  rò   )r   s   &r   rò   ÚModel.layersk  s   € à�z‰z× Ñ Ð r:   c                ó�   € V P                    Uu. uF+  pVP                  '       d   \        4       M\        ^R7      NK-  	  up# u upi )rn   )Úsize)rò   rÞ   r   r   )r   r  s   & r   Ú
make_cacheÚModel.make_cacheo  sD   € ð —[’[ó
á �ð ×-×-Ð-ŒGŒI´;ÀAÔ3FÒFÙ ñ
ð 	
ùò 
s
   �A¦Ac                ó   € R  pV# )c                 ó@   € V P                  R 4      '       d   R^@R^/# R# )zfeed_forward.gateÚ
group_sizeÚbitsT)Úendswith)ÚpathÚ_s   &&r   Ú	predicateÚ(Model.quant_predicate.<locals>.predicatew  s$   € Ø�}‰}Ð0×1Ò1Ø$ b¨&°!Ð4Ð4Ùr:   © ©r   r0  s   & r   Úquant_predicateÚModel.quant_predicateu  s   € ò	ð
 Ðr:   c                ó   € R  pV# )c                 ó   € R V 9  # )rÈ   r2  )rÕ   s   &r   r0  Ú'Model.cast_predicate.<locals>.predicate€  s   € Ø ¨Ñ)Ð)r:   r2  r3  s   & r   Úcast_predicateÚModel.cast_predicate~  s   € ò	*ð Ðr:   )rF   r	  r    r~   )r;   r<   r=   r>   rQ   r|   r!  Úpropertyrò   r'  r4  r9  r@   rA   r€   r�   s   @@r   r  r  ;  se   ù‡ € ÷%ó %÷6ò 6ò!ð@ ñ!ó ð!ò
ð ñó ðð ñó ÷ð r:   r  ) Údataclassesr   Útypingr   r   r   Úmlx.coreÚcorerk   Úmlx.nnrV   Úactivationsr   rN   r   r	   r
   r   rh   r   r   Úswitch_layersr   r   ÚModulerC   rƒ   r²   rÁ   rÚ   rë   r  r2  r:   r   Ú<module>rD     s¾   ðå !ß &Ñ &å Ý å ÷ó ÷ (Ý $ð ô�ó ó ðôD8%�—	‘	ô 8%ôv: �—	‘	ô : ôzJˆ"�)‰)ô Jô%˜BŸI™Iô %ôP&�r—y‘yô &ôR*&�—	‘	ô *&ôZHˆB�I‰Iö Hr:   