+
    TV-jÉC  ã                   ó   € ^ RI Ht ^ RIHt ^ RIHtHtHtHt ^ RI	H
t ^ RIHt ^ RIHtHtHt ^RIHt ^RIHtHtHt ^RIHtHt ^R	IHt ^R
IHtHt ]! ]P@                  RR7      R 4       t! ! R R]PD                  4      t#] ! R R]4      4       t$ ! R R]PD                  4      t% ! R R]PD                  4      t&]P@                  R 4       t' ! R R]PD                  4      t( ! R R]PD                  4      t) ! R R]PD                  4      t* ! R R]PD                  4      t+ ! R R ]PD                  4      t, ! R! R"]PD                  4      t-R# )#é    )Ú	dataclass)Úpartial)ÚAnyÚDictÚListÚOptionalN)Úshard_inplaceÚshard_linearÚsum_gradients)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCacheÚRotatingKVCache)Úinitialize_rope)ÚSwiGLUÚ	SwitchGLUT)Ú	shapelessc                 óž   € \         P                  ! \        P                  ! V4      R VR7      p\         P                  ! W) VR7      p W,          # )N)Úa_minÚa_max)ÚmxÚclipÚnnÚsilu)ÚxÚgateÚlimits   &&&Úf/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/step3p5.pyÚclamped_swiglur!      s5   € ä�7Š7”2—7’7˜4“=¨°EÔ:€DÜ
�Š�˜ uÔ-€AØ�8€Oó    c                   óP   a a€ ] tR t^t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚClampedSwiGLUc                ó    <€ V ^8„  d   QhRS[ /# )é   r   )Úfloat)ÚformatÚ__classdict__s   "€r    Ú__annotate__ÚClampedSwiGLU.__annotate__   s   ø€ ÷ ñ ™eñ r"   c                ó0   <€ \         SV `  4        Wn        R # ©N)ÚsuperÚ__init__r   )Úselfr   Ú	__class__s   &&€r    r/   ÚClampedSwiGLU.__init__   s   ø€ Ü‰ÑÔØŽ
r"   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r&   r   r   Úreturn©r   Úarray)r(   r)   s   "€r    r*   r+      s.   ø€ ÷ 3ñ 3™"Ÿ(™(ð 3©"¯(©(ð 3±r·x±xñ 3r"   c                ó.   € \        WV P                  4      # r-   )r!   r   )r0   r   r   s   &&&r    Ú__call__ÚClampedSwiGLU.__call__   s   € Ü˜a t§z¡zÓ2Ð2r"   )r   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r/   r8   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r1   r)   s   @@r    r$   r$      s   ù‡ € ÷ó ÷3÷ 3ð 3r"   r$   c                   ó|   a € ] tR t^"t o RtRtRtRtRtRt	Rt
RtRtRtRt^tRtRtRtR	tRtRtRtR
tV 3R ltRtV tR# )Ú	ModelArgsçñhãˆµøä>g     ˆÃ@Ni   i   Ti   i   g      @Fc                ó:  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[S[,          ;R&   S[;R&   S[;R&   S[S[S[ ,          ,          ;R&   S[S[S[ ,          ,          ;R&   S[S[S[,          ,          ;R&   S[S[,          ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[ ,          ;R&   S[;R&   S[;R&   S[S[S[,          ,          ;R&   S[S[S[,          ,          ;R&   S[;R&   # )r&   Ú
model_typeÚhidden_sizeÚnum_hidden_layersÚ
vocab_sizeÚnum_attention_headsÚnum_attention_groupsÚhead_dimÚintermediate_sizeÚrms_norm_epsÚ
rope_thetaÚrope_scalingÚmax_position_embeddingsÚsliding_windowÚlayer_typesÚyarn_only_typesÚpartial_rotary_factorsÚattention_other_settingÚuse_head_wise_attn_gateÚmoe_num_expertsÚ	moe_top_kÚmoe_intermediate_sizeÚshare_expert_dimÚmoe_layers_enumÚmoe_router_scaling_factorÚnorm_expert_weightÚswiglu_limitsÚswiglu_limits_sharedÚtie_word_embeddings)ÚstrÚintr'   r   r   r   Úbool)r(   r)   s   "€r    r*   ÚModelArgs.__annotate__"   s¥  ø‡ ‚ á�Oñ ñ Ññ ñ Ññ	 ñ
 �Oñ ñ Ññ ñ Ññ ñ �Mñ ñ Ññ ñ Ññ ñ Ññ ñ ™4•.Ñ'ñ ñ !Ñ)ñ ñ Ññ ñ ™$™s�)Õ$Ñ+ñ ñ  ™d¡3�iÕ(Ñ/ñ! ñ" %¡T©%¥[Õ1Ñ8ñ# ñ$ &¡d�^Ñ2ñ% ñ& "Ñ(ñ' ñ( Ññ) ñ* Ññ+ ñ, Ñ%ñ- ñ. Ñ ñ/ ñ0 ™c•]Ñ)ñ1 ñ2  %Ñ*ñ3 ñ4 Ñ#ñ5 ñ6 ™D¡�KÕ(Ñ/ñ7 ñ8 #¡4©¥;Õ/Ñ6ñ9 ñ: Ñ%ò; r"   © )r;   r<   r=   r>   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   Ú__annotate_func__r?   r@   )r)   s   @r    rD   rD   "   s|   ø‡ € ð €LØ€JØ#'€LØ#)ÐØ€NØ'+€KØ+/€OØ48ÐØ.2ÐØ$(ÐØ€OØ€IØ!%ÐØ ÐØ%)€OØ'*ÐØ#ÐØ+/€MØ26ÐØ %Ð÷; ƒ r"   rD   c                   óT   a a€ ] tR t^Bt oRV3R lV 3R llltV3R lR ltRtVtV ;t# )ÚZeroCenteredRMSNormc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r&   ÚdimsÚeps)rd   r'   )r(   r)   s   "€r    r*   Ú ZeroCenteredRMSNorm.__annotate__C   s   ø€ ÷ ñ ™Sð ¡uñ r"   c                óh   <€ \         SV `  4        \        P                  ! V34      V n        W n        R # r-   )r.   r/   r   ÚonesÚweightrm   )r0   rl   rm   r1   s   &&&€r    r/   ÚZeroCenteredRMSNorm.__init__C   s%   ø€ Ü‰ÑÔÜ—g’g˜t˜gÓ&ˆŒØŽr"   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r&   r   r4   r5   )r(   r)   s   "€r    r*   rn   H   s#   ø€ ÷ :ñ :™"Ÿ(™(ð :¡r§x¡xñ :r"   c                ój   € \         P                  P                  WP                  V P                  4      # r-   )r   ÚfastÚrms_normrq   rm   ©r0   r   s   &&r    r8   ÚZeroCenteredRMSNorm.__call__H   s!   € Ü�w‰w×Ñ §;¡;°·±Ó9Ð9r"   )rm   rq   )rE   r:   rB   s   @@r    rj   rj   B   s   ù‡ € ÷õ ÷
:÷ :ð :r"   rj   c                   óT   a a€ ] tR t^Lt oRV3R lV 3R llltV3R lR ltRtVtV ;t# )Ú
Step3p5MLPc                ó,   <€ V ^8„  d   QhRS[ RS[RS[/# )r&   ÚargsrN   Úswiglu_limit)rD   rd   r'   )r(   r)   s   "€r    r*   ÚStep3p5MLP.__annotate__M   s*   ø€ ÷ Qñ QÙðQÙ25ðQÙEJñQr"   c                óº  <€ \         SV `  4        VP                  V n        W n        \        P
                  ! V P                  V P                  R R7      V n        \        P
                  ! V P                  V P                  R R7      V n        \        P
                  ! V P                  V P                  R R7      V n        V'       d   V^ 8”  d	   W0n	        R# RV n	        R# ©F©ÚbiasN)
r.   r/   rH   rN   r   ÚLinearÚ	gate_projÚup_projÚ	down_projr   )r0   r}   rN   r~   r1   s   &&&&€r    r/   ÚStep3p5MLP.__init__M   s›   ø€ ô 	‰ÑÔØ×+Ñ+ˆÔØ!2ÔäŸš 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒç%1°lÀQÔ6F�\Ž
ÈDˆŽ
r"   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rt   r5   )r(   r)   s   "€r    r*   r   Z   s'   ø€ ÷ Jñ J™"Ÿ(™(ð J¡r§x¡xñ Jr"   c                ó  € V P                   eE   V P                  \        V P                  V4      V P	                  V4      V P                   4      4      # V P                  \        V P	                  V4      V P                  V4      4      4      # r-   )r   r‡   r!   r†   r…   r   rx   s   &&r    r8   ÚStep3p5MLP.__call__Z   sg   € Ø�:‰:Ò!Ø—>‘>Ü˜tŸ|™|¨A›°·±¸qÓ0AÀ4Ç:Á:ÓNóð ð �~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr"   )r‡   r…   rH   rN   r   r†   )r   r:   rB   s   @@r    r{   r{   L   s#   ù‡ € ÷Qõ Q÷J÷ Jð Jr"   r{   c                 ór  € \         P                  ! V P                  \         P                  4      4      pWQ,           p\         P                  ! V) V^,
          RR7      RRV13,          p\         P
                  ! WWRR7      pV'       d(   V\         P                  ! VRRR7      R,           ,          pWxV,          3# )	é   )ÚkthÚaxis.N©r�   T)r�   Úkeepdimsg#B’¡œÇ;éÿÿÿÿ)r   ÚsigmoidÚastypeÚfloat32ÚargpartitionÚtake_along_axisÚsum)	ÚgatesÚrouter_biasÚtop_kÚrouted_scaling_factorÚnorm_topk_probÚscoresÚcorrected_scoresÚtopk_indicesÚtopk_weightss	   &&&&&    r    Úmoe_gate_selectr¢   b   s—   € ä�ZŠZ˜Ÿ™¤R§Z¡ZÓ0Ó1€FØÕ+Ðä—?’?Ð$4Ð#4¸%À!½)È"ÔMØˆVˆeˆVˆõ€Lô ×%Ò% fÀÔD€LçØ#Ü�FŠF�< b°4Ô8¸5Õ@õ
ˆð Ð(=Õ=Ð=Ð=r"   c                   óP   a a€ ] tR t^tt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚStep3p5MoEGatec                ó    <€ V ^8„  d   QhRS[ /# ©r&   r}   ©rD   )r(   r)   s   "€r    r*   ÚStep3p5MoEGate.__annotate__u   s   ø€ ÷ >ñ >™Yñ >r"   c                ó\  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP                  V n	        \        P                  ! VP                  V P
                  R R7      V n        \        P                  ! V P
                  34      V n        R# r�   )r.   r/   rZ   r›   rY   Ún_routed_expertsr^   rœ   r_   r�   r   r„   rH   r   r   Úzerosrš   ©r0   r}   r1   s   &&€r    r/   ÚStep3p5MoEGate.__init__u   s}   ø€ Ü‰ÑÔØ—^‘^ˆŒ
Ø $× 4Ñ 4ˆÔØ%)×%CÑ%CˆÔ"Ø"×5Ñ5ˆÔä—I’I˜d×.Ñ.°×0EÑ0EÈEÔRˆŒ	ÜŸ8š8 T×%:Ñ%:Ð$<Ó=ˆÖr"   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r&   r   r5   )r(   r)   s   "€r    r*   r¨      s   ø€ ÷ 
ñ 
™"Ÿ(™(ñ 
r"   c                óŽ   € \        V P                  V4      V P                  V P                  V P                  V P
                  4      # r-   )r¢   r   rš   r›   rœ   r�   rx   s   &&r    r8   ÚStep3p5MoEGate.__call__   s<   € ÜØ�I‰I�a‹LØ×ÑØ�J‰JØ×&Ñ&Ø×Ñó
ð 	
r"   )r   rª   r�   rœ   rš   r›   r:   rB   s   @@r    r¤   r¤   t   s   ù‡ € ÷>ó >÷
÷ 
ð 
r"   r¤   c                   óP   a a€ ] tR t^‰t oV3R lV 3R lltV3R lR ltRtVtV ;t# )Ú
Step3p5MoEc                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r&   r}   Ú	layer_idx©rD   rd   )r(   r)   s   "€r    r*   ÚStep3p5MoE.__annotate__Š   s   ø€ ÷ #ñ #™Yð #±3ñ #r"   c                óT  <€ \         SV `  4        ^ pVP                  '       d8   V\        VP                  4      8  d   VP                  V,          ;'       g    ^ p^ pVP                  '       d8   V\        VP                  4      8  d   VP                  V,          ;'       g    ^ p\        V4      V n        V^ 8”  d   \        V4      M	\        4       p\        VP                  VP                  VP                  VR7      V n        \        VVP                  VR7      V n        RV n        R# )r   )Ú
activation©rN   r~   N)r.   r/   r`   Úlenra   r¤   r   r$   r   r   rH   r[   rY   Ú
switch_mlpr{   r\   Úshare_expertÚsharding_group)r0   r}   rµ   r~   Úswiglu_limit_sharedr¹   r1   s   &&&   €r    r/   ÚStep3p5MoE.__init__Š   sù   ø€ Ü‰ÑÔàˆØ××Ð )¬c°$×2DÑ2DÓ.EÔ"EØ×-Ñ-¨iÕ8×=Ð=¸AˆLàÐØ×$×$Ð$¨´S¸×9RÑ9RÓ5SÔ)SØ"&×";Ñ";¸IÕ"F×"KÐ"KÈ!Ðä" 4Ó(ˆŒ	à4@À1Ô4D”] <Ô0Ì&Ë(ˆ
Ü#Ø×ÑØ×&Ñ&Ø× Ñ Ø!ô	
ˆŒô 'ØØ"×3Ñ3Ø,ô
ˆÔð #ˆÖr"   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rt   r5   )r(   r)   s   "€r    r*   r·   §   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r"   c                ó®  € V P                   e   \        V P                   4      ! V4      pV P                  V4      w  r#V P                  W4      pWCR,          ,          P	                  RR7      P                  VP                  4      pW@P                  V4      ,           pV P                   e+   \        P                  P                  WPP                   R7      pV# )Nr�   ©Úgroup©.Néþÿÿÿ)r¾   r   r   r¼   r˜   r”   Údtyper½   r   ÚdistributedÚall_sum)r0   r   r    r¡   Úrouted_outputÚys   &&    r    r8   ÚStep3p5MoE.__call__§   s¯   € Ø×ÑÒ*Ü˜d×1Ñ1Ô2°1Ó5ˆAà%)§Y¡Y¨q£\Ñ"ˆØŸ™¨Ó8ˆà¨)Õ4Õ4ß‰S�bˆS‹\ß‰V�M×'Ñ'Ó(ð 	ð
 ×-Ñ-¨aÓ0Õ0ˆà×ÑÒ*Ü—‘×&Ñ& q×0CÑ0CÐ&ÓDˆAàˆr"   )r   r¾   r½   r¼   r:   rB   s   @@r    r²   r²   ‰   s   ù‡ € ÷#ó #÷:÷ ð r"   r²   c                   óT   a a€ ] tR t^ºt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚStep3p5Attentionc                ó&   <€ V ^8„  d   QhRS[ RS[/# r´   r¶   )r(   r)   s   "€r    r*   ÚStep3p5Attention.__annotate__»   s   ø€ ÷ 7
ñ 7
™Yð 7
±3ñ 7
r"   c                óÀ  <€ \         SV `  4        VP                  pVP                  ;'       g    . pV'       d   WB,          R 8H  V n        MV^,          ^ 8H  V n        V P                  '       dD   VP
                  '       d2   VP
                  R,          V n        VP
                  R,          V n        M"VP                  V n        VP                  V n        VP                  V n
        V P                  R
,          V n        \        P                  ! W0P                  V P                  ,          RR7      V n        \        P                  ! W0P                  V P                  ,          RR7      V n        \        P                  ! W0P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  ,          VRR7      V n        \%        V P                  VP&                  R7      V n        \%        V P                  VP&                  R7      V n        VP,                  V n        V P,                  '       d(   \        P                  ! W0P                  RR7      V n        VP0                  p\3        V\4        4      '       d	   WR,          pRpVP6                  '       d.   V\9        VP6                  4      8  d   VP6                  V,          p\;        V P                  V,          4      pVP<                  ;'       g    . pV'       d	   WB,          MRp	V'       d
   W˜9  d   Rp
MVP>                  p
\A        VVRV
VPB                  R	7      V n"        R# )Úsliding_attentionrK   rL   Fr‚   ©rm   g      ð?Úfull_attentionN)rl   ÚbaseÚtraditionalÚscaling_configrR   g      à¿)#r.   r/   rH   rT   Ú
is_slidingrW   Ú	num_headsÚnum_kv_headsrK   rL   rM   Úscaler   r„   Úq_projÚk_projÚv_projÚo_projrj   rO   Úq_normÚk_normrX   Úg_projrP   Ú
isinstanceÚlistrV   r»   rd   rU   rQ   r   rR   Úrope)r0   r}   rµ   ÚdimrT   rP   Úpartial_rotary_factorÚ	rope_dimsrU   Ú
layer_typerQ   r1   s   &&&        €r    r/   ÚStep3p5Attention.__init__»   sb  ø€ Ü‰ÑÔØ×Ñˆà×&Ñ&×,Ð,¨"ˆßØ)Õ4Ð8KÑKˆD�Oà'¨!�m¨qÑ0ˆDŒOà�?�?ˆ?˜t×;×;Ð;Ø!×9Ñ9Ð:OÕPˆDŒNØ $× <Ñ <Ð=SÕ TˆDÕà!×5Ñ5ˆDŒNØ $× 9Ñ 9ˆDÔàŸ™ˆŒØ—]‘] DÕ(ˆŒ
ä—i’i §^¡^°d·m±mÕ%CÈ%ÔPˆŒÜ—i’i ×%6Ñ%6¸¿¹Õ%FÈUÔSˆŒÜ—i’i ×%6Ñ%6¸¿¹Õ%FÈUÔSˆŒÜ—i’i §¡°·±Õ >ÀÈ%ÔPˆŒä)¨$¯-©-¸T×=NÑ=NÔOˆŒÜ)¨$¯-©-¸T×=NÑ=NÔOˆŒà'+×'CÑ'CˆÔ$Ø×'×'Ð'ÜŸ)š) C¯©¸eÔDˆDŒKà—_‘_ˆ
Ü�j¤$×'Ò'Ø#Õ.ˆJà #ÐØ×&×&Ð&¨9´s¸4×;VÑ;VÓ7WÔ+WØ$(×$?Ñ$?À	Õ$JÐ!ä˜Ÿ™Ð(=Õ=Ó>ˆ	à×.Ñ.×4Ð4°"ˆß/:�[Ö+Ð@Pˆ
ß˜zÔ@Ø‰Là×,Ñ,ˆLä#ØØØØ'Ø$(×$@Ñ$@ô
ˆŽ	r"   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r&   r   ÚmaskÚcacher4   ©r   r6   r   r   )r(   r)   s   "€r    r*   rÐ   ô   sH   ø€ ÷ "5ñ "5á�8‰8ð"5ñ ‘r—x‘xÕ ð"5ñ ™�}ð	"5ñ
 
�‰ñ"5r"   c           	     óä  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pV P	                  VP                  WEV P                  R4      4      P                  ^ ^^^4      pV P                  VP                  WEV P                  R4      4      P                  ^ ^^^4      pV	P                  WEV P                  R4      P                  ^ ^^^4      p	VeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      p
V P                  '       d4   V
\         P"                  ! V P%                  V4      4      R,          ,          p
V P'                  V
P                  WER4      4      # )r�   )Úoffset)rî   rÛ   rí   r’   rÅ   )ÚshaperÜ   rÝ   rÞ   rà   ÚreshaperÙ   Ú	transposerá   rÚ   rå   rñ   Úupdate_and_fetchr   rÛ   rX   r   r“   râ   rß   )r0   r   rí   rî   ÚBÚLÚ_ÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r    r8   ÚStep3p5Attention.__call__ô   sœ  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆà—+‘+˜gŸo™o¨a°D·N±NÀBÓGÓH×RÑRØˆq�!�Qó
ˆð �{‰{˜4Ÿ<™<¨¨d×.?Ñ.?ÀÓDÓE×OÑOØˆq�!�Qó
ˆð —‘  d×&7Ñ&7¸Ó<×FÑFÀqÈ!ÈQÐPQÓRˆàÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-ˆà×'×'Ð'ØœbŸjšj¨¯©°Q«Ó8¸ÕCÕCˆFà�{‰{˜6Ÿ>™>¨!°Ó3Ó4Ð4r"   )râ   rM   rØ   rá   rÝ   rÙ   rÚ   rß   rà   rÜ   rå   rÛ   rX   rÞ   ©NNr:   rB   s   @@r    rÎ   rÎ   º   s   ù‡ € ÷7
ó 7
÷r"5÷ "5ò "5r"   rÎ   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚStep3p5DecoderLayeri  c                ó&   <€ V ^8„  d   QhRS[ RS[/# r´   r¶   )r(   r)   s   "€r    r*   Ú Step3p5DecoderLayer.__annotate__  s   ø€ ÷ 
ñ 
™Yð 
±3ñ 
r"   c                ó>  <€ \         SV `  4        \        W4      V n        V P                  P                  V n        \        4       pVP                  '       dC   VP                  P                  4       P                  R 4       Uu0 uF  p\        V4      kK  	  ppM\        \        ^VP                  4      4      pW#9   V n        V P                  '       d   \        W4      V n        Mh^ pVP                  '       d8   V\!        VP                  4      8  d   VP                  V,          ;'       g    ^ p\#        VVP$                  VR7      V n        \'        VP(                  VP*                  R7      V n        \'        VP(                  VP*                  R7      V n        R# u upi )Ú,rº   rÓ   N)r.   r/   rÎ   Ú	self_attnrØ   Úsetr]   ÚstripÚsplitrd   ÚrangerI   Úis_moe_layerr²   Úmlpra   r»   r{   rN   rj   rH   rO   Úinput_layernormÚpost_attention_layernorm)r0   r}   rµ   Úmoe_layers_idxÚir~   r1   s   &&&   €r    r/   ÚStep3p5DecoderLayer.__init__  sK  ø€ Ü‰ÑÔä)¨$Ó:ˆŒØŸ.™.×3Ñ3ˆŒä›ˆØ××ÐØ.2×.BÑ.B×.HÑ.HÓ.J×.PÑ.PÐQTÔ.UÓVÑ.U¨œc !žfÑ.UˆNÐVˆNä ¤ q¨$×*@Ñ*@Ó!AÓBˆNà%Ñ7ˆÔà××ÐÜ! $Ó2ˆD�HàˆLØ×(×(Ð(¨Y¼¸T×=VÑ=VÓ9WÔ-WØ#×8Ñ8¸ÕC×HÐHÀq�Ü!ØØ"&×"8Ñ"8Ø)ôˆDŒHô  3Ø×Ñ $×"3Ñ"3ô 
ˆÔô )<Ø×Ñ $×"3Ñ"3ô)
ˆÖ%ùò+ Ws   ÂFc                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# rì   rï   )r(   r)   s   "€r    r*   r  ;  sH   ø€ ÷ 	ñ 	á�8‰8ð	ñ ‘r—x‘xÕ ð	ñ ™�}ð		ñ
 
�‰ñ	r"   c                ó¦   € V P                  V P                  V4      W#R 7      pW,           pV P                  V P                  V4      4      pWT,           # )©rí   rî   )r  r  r  r  )r0   r   rí   rî   ÚrÚhs   &&&&  r    r8   ÚStep3p5DecoderLayer.__call__;  sH   € ð �N‰N˜4×/Ñ/°Ó2¸ˆNÓKˆØ�EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�uˆr"   )r  r
  rØ   r  r  r  rþ   r:   rB   s   @@r    r   r     s   ù‡ € ÷
ó 
÷B	÷ 	ò 	r"   r   c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚStep3p5ModeliG  c                ó    <€ V ^8„  d   QhRS[ /# r¦   r§   )r(   r)   s   "€r    r*   ÚStep3p5Model.__annotate__H  s   ø€ ÷ 
ñ 
™Yñ 
r"   c                ó<  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \        P                  ! VP                  VP                  4      V n	        \        VP                  4       Uu. uF  p\        W4      NK  	  upV n        \        VP                  VP                  R 7      V n        \!        R \#        V P                  4       4       R4      V n        \!        R \#        V P                  4       4       R4      V n        R# u upi )rÓ   c              3   óR   "  € T F  w  rVP                   '       g   K  Vx € K  	  R # 5ir-   ©rØ   ©Ú.0r  Úls   &  r    Ú	<genexpr>Ú(Step3p5Model.__init__.<locals>.<genexpr>V  s   é € ÐBÑ1‘4�1°Q·\µ\�QŠQÓ1ùó   ‚'�
'Nc              3   óR   "  € T F  w  rVP                   '       d   K  Vx € K  	  R # 5ir-   r  r  s   &  r    r!  r"  Y  s   é € ÐFÑ1‘4�1¸¿½�QŠQÓ1ùr#  )r.   r/   r}   rJ   rI   Ú
num_layersr   Ú	EmbeddingrH   Úembed_tokensr	  r   Úlayersrj   rO   ÚnormÚnextÚ	enumerateÚ_swa_idxÚ	_full_idx)r0   r}   rµ   r1   s   && €r    r/   ÚStep3p5Model.__init__H  sÚ   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒØ×0Ñ0ˆŒäŸLšL¨¯©¸$×:JÑ:JÓKˆÔô # 4×#9Ñ#9Ô:ó
á:�	ô   Ö0Ù:ñ
ˆŒô (¨×(8Ñ(8¸d×>OÑ>OÔPˆŒ	äÙBœ9 T§[¡[Ô1ÓBÀDó
ˆŒô ÙFœ9 T§[¡[Ô1ÓFÈó
ˆŽùò
s   Á?Dc                ót   <€ V ^8„  d   QhRS[ P                  RS[S[S[,          ,          RS[ P                  /# )r&   r   rî   r4   ©r   r6   r   r   r   )r(   r)   s   "€r    r*   r  \  s:   ø€ ÷ ñ á�8‰8ðñ ™™S�	Õ"ðñ 
�‰ñ	r"   c                óÐ  € V P                  V4      pVf   R .V P                  ,          pR pR pV P                  e   \        W2V P                  ,          4      pV P                  e3   \        W2V P                  ,          V P
                  P                  R7      p\        V P                  V4       F%  w  rgVP                  '       d   TMTpV! W8VR7      pK'  	  V P                  V4      # )N)Úwindow_sizer  )r'  r%  r-  r   r,  r}   rS   Úzipr(  rØ   r)  )	r0   r   rî   r  Ú	full_maskÚswa_maskÚlayerÚcrí   s	   &&&      r    r8   ÚStep3p5Model.__call__\  sÃ   € ð
 ×Ñ˜aÓ ˆàŠ=Ø�F˜TŸ_™_Õ,ˆEàˆ	Øˆà�>‰>Ò%Ü-¨a°t·~±~Õ1FÓGˆIà�=‰=Ò$Ü,Ø˜Ÿ™Õ'°T·Y±Y×5MÑ5MôˆHô ˜DŸK™K¨Ö/‰HˆEØ$×/×/Ð/‘8°YˆDÙ�a¨!Ô,ŠAñ 0ð �y‰y˜‹|Ðr"   )r-  r,  r}   r'  r(  r)  r%  rJ   r-   r:   rB   s   @@r    r  r  G  s   ù‡ € ÷
ó 
÷(÷ ò r"   r  c                   ó¦   a a€ ] tR tRt oV3R lV 3R lltRV3R lR llt]R 4       tR tR t	]R	 4       t
]R
 4       tRV3R lR lltRtVtV ;t# )ÚModelix  c                ó    <€ V ^8„  d   QhRS[ /# r¦   r§   )r(   r)   s   "€r    r*   ÚModel.__annotate__y  s   ø€ ÷ Pñ P™Yñ Pr"   c                óÖ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        \        P                  ! VP                  VP                  R R7      V n
        R# r�   )r.   r/   r}   rG   r  Úmodelr   r„   rH   rJ   Úlm_headr¬   s   &&€r    r/   ÚModel.__init__y  sH   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ! $Ó'ˆŒ
Ü—y’y ×!1Ñ!1°4·?±?ÈÔOˆŽr"   c                óZ   <€ V ^8„  d   QhRS[ P                  RS[S[S[,          ,          /# )r&   Úinputsrî   r0  )r(   r)   s   "€r    r*   r<  €  s,   ø€ ÷ !ñ !á—‘ð!ñ ™™S�	Õ"ñ!r"   c                óF   € V P                  W4      pV P                  V4      # r-   )r>  r?  )r0   rB  rî   Úouts   &&& r    r8   ÚModel.__call__€  s!   € ð
 �j‰j˜Ó'ˆØ�|‰|˜CÓ Ð r"   c                ó.   € V P                   P                  # r-   )r>  r(  )r0   s   &r    r(  ÚModel.layersˆ  s   € à�z‰z× Ñ Ð r"   c                ó¸   € V P                    Uu. uF?  pVP                  '       d!   \        V P                  P                  R 7      M	\        4       NKA  	  up# u upi ))Úmax_size)r(  rØ   r   r}   rS   r   )r0   r6  s   & r    Ú
make_cacheÚModel.make_cacheŒ  sX   € ð Ÿšó
ñ %�ð ×#×#Ð#ô  ¨¯©×)AÑ)AÕBä“Yòñ %ñ
ð 	
ùò 
s
   �A¦.Ac                ón  a	€ . R	Oo	\         ;QJ d    V	3R lV 4       F  '       g   K   RM	  RM! V	3R lV 4       4      p/ pVP                  4        Fß  w  rERV9   d   K  RV9   dl   VP                  R4      p\        V4      ^8”  dK   V^,          P	                  4       '       d.   \        V^,          4      V P                  P                  8¼  d   K€  S	 F&  w  rxWt9   g   K  W„9  g   K  VP                  Wx4      p M	  V'       d(   VP                  R4      '       d   RV9   d
   V^,           pWSV&   Ká  	  V# )
ú.moe.gate_proj.c              3   óZ   <"  € T F   pS F  w  r#W!9   ;'       d    W19  x € K  	  K"  	  R # 5ir-   rg   )r  ÚkÚsrcÚdstÚ
remappingss   &   €r    r!  Ú!Model.sanitize.<locals>.<genexpr>   s.   øé € ð 
Ù/6¨!Ãz¹8¸3ˆC‰H×%Ð%˜™Ô%ÁzÑ%«wùs   ƒ+š+TFz.mtpzmodel.layers.Ú.z.weightr)  ))rM  z.mlp.switch_mlp.gate_proj.)z.moe.up_proj.z.mlp.switch_mlp.up_proj.)z.moe.down_proj.z.mlp.switch_mlp.down_proj.)z
.moe.gate.z.mlp.gate.gate.)z.moe.router_biasz.mlp.gate.router_bias)z.share_expert.z.mlp.share_expert.)
ÚanyÚitemsr  r»   Úisdigitrd   r}   rI   ÚreplaceÚendswith)
r0   ÚweightsÚ
is_vanillaÚnew_weightsrO  ÚvÚpartsrP  rQ  rR  s
   &&       @r    ÚsanitizeÚModel.sanitize–  s  ø€ ò
ˆ
÷ “Sô 
Ù/6ó
—S—S’Sô 
Ù/6ó
ó 
ˆ
ð ˆØ—M‘M–O‰DˆAØ˜Œ{ÙØ !Ô#ØŸ™ ›�Ü�u“: ”> e¨A¥h×&6Ñ&6×&8Ò&8Ü˜5 �8“}¨¯	©	×(CÑ(CÔCÙ ã&‘�Ø–8 ¦ØŸ	™	 #Ó+�AÙñ '÷
 ˜aŸj™j¨×3Ò3¸À!¼Ø˜•E�à˜‹Nñ# $ð& Ðr"   c                ó   € R  pV# )c                 ó   € R V 9  # )rš   rg   )rO  s   &r    Ú	predicateÚ'Model.cast_predicate.<locals>.predicate¼  s   € Ø ¨Ñ)Ð)r"   rg   ©r0   rc  s   & r    Úcast_predicateÚModel.cast_predicateº  s   € ò	*ð Ðr"   c                ó   € R  pV# )c                 ó    € R V 9   d   R^@R^/# R# )zmlp.gate.gateÚ
group_sizeÚbitsTrg   )Úpathrø   s   &&r    rc  Ú(Model.quant_predicate.<locals>.predicateÃ  s   € Ø $Ô&Ø$ b¨&°!Ð4Ð4Ùr"   rg   re  s   & r    Úquant_predicateÚModel.quant_predicateÁ  s   € ò	ð
 Ðr"   c                óX   <€ V ^8„  d   QhRS[ S[P                  P                  ,          /# )r&   rÄ   )r   r   rÈ   ÚGroup)r(   r)   s   "€r    r*   r<  Ê  s#   ø€ ÷ 6ñ 6™8¡B§N¡N×$8Ñ$8Õ9ñ 6r"   c                óþ  € T;'       g    \         P                  P                  4       pVP                  4       pV P                  P
                   EF)  p\        VP                  P                  R VR7      VP                  n        \        VP                  P                  R VR7      VP                  n	        \        VP                  P                  R VR7      VP                  n
        \        VP                  P                  RVR7      VP                  n        VP                  ;P                  V,          un        VP                  ;P                  V,          un        VP                  P                  '       d2   \        VP                  P                  R VR7      VP                  n        \!        VP"                  \$        4      '       d—   \        VP"                  P&                  R VR7      VP"                  n        \        VP"                  P(                  R VR7      VP"                  n        \        VP"                  P*                  RVR7      VP"                  n        EK  WP"                  n        \/        VP"                  P0                  P&                  R VR7       \/        VP"                  P0                  P(                  R VR7       \/        VP"                  P0                  P*                  RVR7       \/        VP"                  P2                  P&                  R VR7       \/        VP"                  P2                  P(                  R VR7       \/        VP"                  P2                  P*                  RVR7       EK,  	  R# )zall-to-shardedrÃ   zsharded-to-allN)r   rÈ   ÚinitÚsizer>  r(  r
   r  rÜ   rÝ   rÞ   rß   rÙ   rÚ   rX   râ   rã   r  r{   r…   r†   r‡   r¾   r	   r½   r¼   )r0   rÄ   ÚNr6  s   &&  r    ÚshardÚModel.shardÊ  sˆ  € Ø×.Ð.œŸ™×,Ñ,Ó.ˆØ�J‰J‹Lˆà—Z‘Z×&Õ&ˆEÜ%1Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ð �O‰O×%Ò%¨!Õ+Õ%Ø�O‰O×(Ò(¨QÕ.Õ(à�‰×6×6Ð6Ü)5Ø—O‘O×*Ñ*Ð,<ÀEô*�—‘Ô&ô ˜%Ÿ)™)¤Z×0Ò0Ü&2Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	Ô#ô %1Ø—I‘I×%Ñ%Ð'7¸uô%�—	‘	Ô!ô '3Ø—I‘I×'Ñ'Ð)9Àô'�—	‘	×#ð ,1—	‘	Ô(ÜØ—I‘I×*Ñ*×4Ñ4Ð6FÈeõô Ø—I‘I×*Ñ*×2Ñ2Ð4DÈEõô Ø—I‘I×*Ñ*×4Ñ4Ð6FÈeõô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈEõô Ø—I‘I×(Ñ(×0Ñ0Ð2BÈ%õô Ø—I‘I×(Ñ(×2Ñ2Ð4DÈE÷ð óa 'r"   )r}   r?  r>  rG   r-   )r;   r<   r=   r>   r/   r8   Úpropertyr(  rJ  r_  rf  rn  rv  r?   r@   rA   rB   s   @@r    r:  r:  x  sq   ù‡ € ÷Pó P÷!ò !ð ñ!ó ð!ò
ò"ðH ñó ðð ñó ð÷6÷ 6ò 6r"   r:  ).Údataclassesr   Ú	functoolsr   Útypingr   r   r   r   Úmlx.coreÚcorer   Úmlx.nnr   Úmlx.nn.layers.distributedr	   r
   r   Úactivationsr   rÕ   r   r   r   rî   r   r   Ú
rope_utilsr   Úswitch_layersr   r   Úcompiler!   ÚModuler$   rD   rj   r{   r¢   r¤   r²   rÎ   r   r  r:  rg   r"   r    Ú<module>r…     s  ðõ "Ý ß ,Ó ,å Ý ß PÑ På ß TÑ Tß +Ý 'ß ,ñ 	ˆ�‰˜tÔ$ñó %ðô3�B—I‘Iô 3ð ô&�ó &ó ð&ô>:˜"Ÿ)™)ô :ôJ�—‘ô Jð, ‡�ñ>ó ð>ô"
�R—Y‘Yô 
ô*.�—‘ô .ôb\5�r—y‘yô \5ô~+˜"Ÿ)™)ô +ô\.�2—9‘9ô .ôbHˆB�I‰Iö Hr"   