+
    TV-j	H  ã                   ób  € ^ RI Ht ^ RIHt ^ RIHtHtHtHt ^ RI	H
t ^ RIHt ^RIHt ^RIHtHtHtHt ^RIHtHt ^RIHt ^R	IHt ]! 4        ! R
 R]4      4       t ! R R]P:                  4      t ! R R]P:                  4      t ! R R]P:                  4      t  ! R R]P:                  4      t!]PD                  R 4       t# ! R R]P:                  4      t$ ! R R]P:                  4      t% ! R R]P:                  4      t& ! R R]P:                  4      t' ! R R]P:                  4      t(R# )é    )Ú	dataclass)Úpartial)ÚAnyÚListÚOptionalÚTupleN)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚcreate_ssm_maskÚscaled_dot_product_attention)ÚArraysCacheÚKVCache)Ú
ssm_update)Ú	SwitchMLPc                   ó†   a € ] tR t^t o RtRtRtRtRtRt	Rt
RtRtRtRtRtRtRtRtRtRRRRRRRR	/tR
 tV 3R ltRtV tR# )Ú	ModelArgsNÚmambaÚMÚ	attentionÚ*ÚmoeÚEÚmlpÚ-c                ó>  € V P                   f   R\        R4      3V n         V P                  f@   V P                  e2   V P                   Uu. uF  qP                  V,          NK  	  upV n        V P                  e   \        V P                  4      V n        R # R # u upi )Nç        Úinf)Útime_step_limitÚfloatÚhybrid_override_patternÚlayers_block_typeÚ_block_type_to_charÚlenÚnum_hidden_layers)ÚselfÚts   & Úi/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/nemotron_h.pyÚ__post_init__ÚModelArgs.__post_init__?   s’   € Ø×ÑÒ'Ø$'¬¨u«Ð#6ˆDÔ ð ×'Ñ'Ò/°D×4JÑ4JÒ4Và59×5KÒ5Kó,Ù5K°×(Ñ(¨×+Ð+Ñ5Kñ,ˆDÔ(ð ×'Ñ'Ò3Ü%(¨×)EÑ)EÓ%FˆDÖ"ñ 4ùò,s   ÁBc                óô  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[S[ ,          ,          ;R&   S[S[S[ ,          ,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R &   S[S[S[S[3,          ,          ;R!&   S[S[,          ;R"&   S[S[,          ;R#&   # )$é   Ú
model_typeÚ
vocab_sizeÚhidden_sizeÚintermediate_sizer%   Úmax_position_embeddingsÚnum_attention_headsÚnum_key_value_headsÚattention_biasÚmamba_num_headsÚmamba_head_dimÚmamba_proj_biasÚssm_state_sizeÚconv_kernelÚn_groupsÚmlp_biasÚlayer_norm_epsilonÚuse_biasÚuse_conv_biasr!   r"   Úhead_dimÚmoe_intermediate_sizeÚ#moe_shared_expert_intermediate_sizeÚmoe_latent_sizeÚn_groupÚn_routed_expertsÚn_shared_expertsÚ
topk_groupÚnum_experts_per_tokÚnorm_topk_probÚrouted_scaling_factorr   Útime_step_minÚtime_step_max)ÚstrÚintÚboolr    r   r   r   )ÚformatÚ__classdict__s   "€r(   Ú__annotate__ÚModelArgs.__annotate__   s#  ø‡ ‚ á�Oñ ñ �Oñ ñ Ññ	 ñ
 Ññ ñ Ññ ñ !Ñ ñ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ  �Mñ! ñ" �Nñ# ñ$ Ññ% ñ& �Nñ' ñ( Ññ) ñ* &¡d©3¥iÕ0Ñ7ñ+ ñ,  ¡¡S¥	Õ*Ñ1ñ- ñ. ‘s�mÑ"ñ/ ñ0 $¡C�=Ñ/ñ1 ñ2 *2±#­Ñ=ñ3 ñ4 ™c•]Ñ)ñ5 ñ6 ‘c�]Ñ!ñ7 ñ8 ™s•mÑ*ñ9 ñ: ™s•mÑ*ñ; ñ< ™•Ñ$ñ= ñ> "¡#�Ñ-ñ? ñ@ ™T•NÑ)ñA ñB $¡E�?Ñ1ñC ñD ™e¡E©5 LÕ1Õ2Ñ9ñE ñF ™E•?Ñ)ñG ñH ™E•?Ñ)òI ó    )r!   r%   r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r!   r"   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   r   rJ   rK   r#   r)   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)rP   s   @r(   r   r      s†   ø‡ € ð* 48ÐØ-1ÐØ"€HØ+/ÐØ9=Ð'Ø%)€OØ!€GØ&*ÐØ&*ÐØ $€JØ)-ÐØ%)€NØ-1ÐØ59€OØ%)€MØ%)€Mð # C¨°c¸5À#ÀuÈcÐRÐò
G÷S ƒ rS   r   c                   óT   a a€ ] tR t^Lt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMambaRMSNormGatedc                ó,   <€ V ^8„  d   QhRS[ RS[RS[ /# )r,   r/   ÚepsÚ
group_size)rM   r    )rO   rP   s   "€r(   rQ   ÚMambaRMSNormGated.__annotate__M   s"   ø€ ÷ %ñ %¡Cð %©eð %Áñ %rS   c                ór   <€ \         SV `  4        W n        \        P                  ! V4      V n        W0n        R # ©N)ÚsuperÚ__init__r^   ÚmxÚonesÚweightr_   )r&   r/   r^   r_   Ú	__class__s   &&&&€r(   rd   ÚMambaRMSNormGated.__init__M   s(   ø€ Ü‰ÑÔØŒÜ—g’g˜kÓ*ˆŒØ$ŽrS   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r,   ÚxÚgateÚreturn)re   Úarray)rO   rP   s   "€r(   rQ   r`   S   s.   ø€ ÷ +ñ +™"Ÿ(™(ð +©"¯(©(ð +¹b¿h¹hñ +rS   c                ó  € Ve   \        W!4      p\        P                  ! VRRV P                  3R7      p\        P                  P                  VR V P                  R7      pV P                  VP                  R4      ,          # )N©ÚaxisÚshape)rg   r^   éÿÿÿÿéþÿÿÿ)	r	   re   Ú	unflattenr_   ÚfastÚrms_normr^   rg   Úflatten)r&   rk   rl   s   &&&r(   Ú__call__ÚMambaRMSNormGated.__call__S   sb   € ØÒÜ�t“ˆAÜ�LŠL˜ ¨B°·±Ð+@ÔAˆÜ�G‰G×Ñ˜Q t°·±ÐÓ:ˆØ�{‰{˜QŸY™Y r›]Õ*Ð*rS   )r^   r_   rg   rb   ©	rT   rU   rV   rW   rd   ry   rY   rZ   Ú__classcell__©rh   rP   s   @@r(   r\   r\   L   s   ù‡ € ÷%ó %÷+÷ +ò +rS   r\   c                   óx   a a€ ] tR t^[t oV3R lV 3R lltV3R lR ltV3R lR ltR
V3R lR lltR	tVt	V ;t
# )ÚNemotronHMamba2Mixerc                ó    <€ V ^8„  d   QhRS[ /# ©r,   Úargs©r   )rO   rP   s   "€r(   rQ   Ú!NemotronHMamba2Mixer.__annotate__\   s   ø€ ÷ (
ñ (
™Yñ (
rS   c           	     ó¨  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  V n        VP                  V n        VP                  VP                  ,          V n	        VP                  V n
        VP                  V n        VP                  V n        V P                  V P                  ,          V n        V P                  ^V P                  ,          V P
                  ,          ,           V n        \        P                   ! V P                  V P                  VP                  ^ V P                  VP"                  R7      V n        V P                  V P                  ,           V P                  ,           p\        P&                  ! V P                  W!P(                  R7      V n        \,        P.                  ! V P                  4      V n        \,        P2                  ! \,        P4                  ! ^V P                  ^,           \,        P6                  R7      4      V n        \,        P.                  ! V P                  4      V n        V P                  V P                  ,          p\=        V P                  VP>                  VR7      V n         \        P&                  ! V P                  V P                  VP(                  R7      V n!        R# )r,   )Úin_channelsÚout_channelsÚkernel_sizeÚpaddingÚgroupsÚbias©r‹   ©Údtype)r^   r_   N)"rc   rd   r5   Ú	num_headsr/   r8   r9   Úconv_kernel_sizer6   r0   r:   r?   r   Úheads_per_groupÚconv_dimÚnnÚConv1dr>   Úconv1dÚLinearr7   Úin_projre   rf   Údt_biasÚlogÚarangeÚfloat32ÚA_logÚDr\   r<   ÚnormÚout_proj)r&   r‚   Úprojection_sizer_   rh   s   &&  €r(   rd   ÚNemotronHMamba2Mixer.__init__\   sí  ø€ Ü‰ÑÔØ×-Ñ-ˆŒØ×+Ñ+ˆÔØ"×1Ñ1ˆÔØ $× 0Ñ 0ˆÔØ!%×!5Ñ!5¸×8KÑ8KÕ!KˆÔØŸ™ˆŒØ×+Ñ+ˆŒØ#×3Ñ3ˆÔØ#Ÿ~™~°·±Õ>ˆÔà×.Ñ.°°T·]±]Õ1BÀT×EXÑEXÕ1XÕXˆŒä—i’iØŸ™ØŸ™Ø×(Ñ(ØØ—=‘=Ø×#Ñ#ô
ˆŒð ×0Ñ0°4·=±=Õ@À4Ç>Á>ÕQˆÜ—y’yØ×Ñ˜o×4HÑ4Hô
ˆŒô —w’w˜tŸ~™~Ó.ˆŒÜ—V’VœBŸIšI a¨¯©¸!Õ);Ä2Ç:Á:ÔNÓOˆŒ
Ü—’˜Ÿ™Ó(ˆŒà×+Ñ+¨t¯}©}Õ<ˆ
Ü%Ø×"Ñ"Ø×'Ñ'Ø!ô
ˆŒ	ô
 Ÿ	š	Ø×"Ñ" D×$4Ñ$4¸4×;OÑ;Oô
ˆŽrS   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[S[ P                  ,          RS[ P                  /# )r,   Ú
conv_inputÚcacheÚmaskrm   ©re   rn   r   r   )rO   rP   s   "€r(   rQ   r„   †   sI   ø€ ÷  $ñ  $á—H‘Hð $ñ ™Õ$ð $ñ ‘r—x‘xÕ ð	 $ñ
 
�‰ñ $rS   c                ód  € Ve    \         P                  ! VR,          V^ 4      pVEe6   V^ ,          fS   \         P                  ! VP                  ^ ,          V P                  ^,
          V P
                  3VP                  R7      pM	V^ ,          p\         P                  ! WA.^R7      pV P                  ^,
          pVP                  eƒ   VP                  ^,          p\         P                  ! VP                  ^ Wv,
          4      pVR,          \         P                  ! V4      ,           R,          p	\         P                  ! WY^R7      V^ &   M@VRV) R 1R3,          V^ &   M-\         P                  ! VRV P                  ^,
          ^ 3R.4      pV P                  V4      p
\        P                  ! V
4      # )Nr�   ©rq   ºNNN©.N)r©   N)r   r   )re   ÚwhereÚzerosrr   r�   r’   rŽ   ÚconcatenateÚlengthsÚcliprš   Útake_along_axisÚpadr•   r“   Úsilu)r&   r£   r¤   r¥   Ú
conv_stateÚpadded_inputÚn_keepr'   ÚendsÚ	positionsÚconv_outputs   &&&&       r(   Ú_convÚNemotronHMamba2Mixer._conv†   s^  € ð ÒÜŸš $ y¥/°:¸qÓAˆJàÓØ�Q�xÒÜŸXšXØ×%Ñ% aÕ(¨$×*?Ñ*?À!Õ*CÀTÇ]Á]ÐSØ$×*Ñ*ô‘
ð
 # 1�X�
ÜŸ>š>¨:Ð*BÈÔKˆLØ×*Ñ*¨QÕ.ˆFØ�}‰}Ò(Ø ×&Ñ& qÕ)�Ü—w’w˜uŸ}™}¨a°µÓ<�Ø! '�]¬R¯YªY°vÓ->Õ>À	ÕJ�	Ü×-Ò-¨lÈAÔN��a’à'¨¨F¨7©8°Q¨Õ7��a’äŸ6š6Ø˜V d×&;Ñ&;¸aÕ&?ÀÐ%CÀVÐLóˆLð —k‘k ,Ó/ˆÜ�wŠw�{Ó#Ð#rS   c                óÜ   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  RS[S[,          RS[S[ P                  ,          RS[ P                  /# )r,   Úhidden_statesÚBÚCÚdtr¤   r¥   rm   r¦   )rO   rP   s   "€r(   rQ   r„   ¨   s|   ø€ ÷ %Fñ %Fá—x‘xð%Fñ �8‰8ð%Fñ �8‰8ð	%Fñ
 �H‰Hð%Fñ ™Õ$ð%Fñ ‘r—x‘xÕ ð%Fñ 
�‰ñ%FrS   c                óJ  € VP                   w  rxp	VP                  WxV P                  V P                  4      pVP                  WxV P                  V P
                  4      pVP                  WxV P                  V P
                  4      pV'       d   V^,          p
VP                  pMRRrº\        VV P                  VVV P                  P                  VP                  4      VV P                  V
V P                  V4
      w  rÊV'       d   W¥^&   VP                  WxV P                  4      # )é   N)rr   Úreshaper�   r?   r:   r8   r®   r   rœ   r�   ÚastyperŽ   r˜   r   r0   )r&   r¼   r½   r¾   r¿   r¤   r¥   Ú
batch_sizeÚseq_lenÚ_Ústater®   Úys   &&&&&&&      r(   Ú_ssmÚNemotronHMamba2Mixer._ssm¨   s÷   € ð "/×!4Ñ!4Ñˆ
˜Qà%×-Ñ-Ø §¡°·±ó
ˆð �I‰I�j¨4¯=©=¸$×:MÑ:MÓNˆØ�I‰I�j¨4¯=©=¸$×:MÑ:MÓNˆßØ˜!•HˆEØ—m‘m‰Gà! 4�7äØØ�J‰JØØØ�F‰F�M‰M˜-×-Ñ-Ó.ØØ�L‰LØØ× Ñ Øó
‰ˆ÷ Ø�!‰Hà�y‰y˜¨d×.DÑ.DÓEÐErS   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# )r,   r¼   r¥   r¤   rm   r¦   )rO   rP   s   "€r(   rQ   r„   Ï   sI   ø€ ÷  ñ  á—x‘xð ñ ‘r—x‘xÕ ð ñ ™Õ$ð	 ñ
 
�‰ñ rS   c                ó4  € V P                  V4      p\        P                  ! VV P                  V P                  V P                  ,           .RR7      w  rVpV P                  WcV4      p\        P                  ! VV P                  V P                  V P                  V P                  ,          ,           .RR7      w  ršpV P                  WšW·W24      pV'       d#   VP                  VP                  ^,          4       V P                  WÅ4      pV P                  V4      # )rÁ   r¨   rs   )r—   re   Úsplitr0   r’   r¹   r:   r8   rÉ   Úadvancerr   rž   rŸ   )r&   r¼   r¥   r¤   Ú	projectedrl   r£   r¿   r¸   Úhidden_states_ssmr½   r¾   rÈ   s   &&&&         r(   ry   ÚNemotronHMamba2Mixer.__call__Ï   sê   € ð —L‘L Ó/ˆ	ä!ŸxšxØØ×#Ñ# T×%;Ñ%;¸d¿m¹mÕ%KÐLØô 
Ñˆ˜"ð
 —j‘j °DÓ9ˆÜ"$§(¢(Øà×&Ñ&Ø×&Ñ&¨¯©¸×9LÑ9LÕ)LÕLðð ô#
ÑÐ˜að �I‰IÐ'¨A°5Ó?ˆßØ�M‰M˜!Ÿ'™' !�*Ô%Ø�I‰I�aÓˆØ�}‰}˜QÓÐrS   )rœ   r�   r•   r’   r�   r˜   r?   r‘   r/   r—   r0   r:   rž   r�   rŸ   r8   r   rb   )rT   rU   rV   rW   rd   r¹   rÉ   ry   rY   rZ   r|   r}   s   @@r(   r   r   [   s7   ù‡ € ÷(
ó (
÷T $ð  $÷D%Fð %F÷N ÷  ò  rS   r   c                   óT   a a€ ] tR t^ít oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚNemotronHAttentionc                ó    <€ V ^8„  d   QhRS[ /# r�   rƒ   )rO   rP   s   "€r(   rQ   ÚNemotronHAttention.__annotate__î   s   ø€ ÷ 
ñ 
™Yñ 
rS   c                óš  <€ \         SV `  4        VP                  V n        VP                  V n        VP
                  e   VP
                  MVP                  VP                  ,          V n        VP                  V n        V P
                  R,          V n        \        P                  ! V P                  V P                  V P
                  ,          VP                  R7      V n        \        P                  ! V P                  V P                  V P
                  ,          VP                  R7      V n        \        P                  ! V P                  V P                  V P
                  ,          VP                  R7      V n        \        P                  ! V P                  V P
                  ,          V P                  VP                  R7      V n        R # )NrŒ   g      à¿)rc   rd   r/   r2   r�   r?   r3   Úscaler“   r–   r4   Úq_projÚk_projÚv_projÚo_proj©r&   r‚   rh   s   &&€r(   rd   ÚNemotronHAttention.__init__î   sG  ø€ Ü‰ÑÔØ×+Ñ+ˆÔØ×1Ñ1ˆŒð �}‰}Ò(ð �MŠMà×"Ñ" d×&>Ñ&>Õ>ð 	Œð
 $(×#;Ñ#;ˆÔ Ø—]‘] DÕ(ˆŒ
ä—i’iØ×Ñ˜dŸn™n¨t¯}©}Õ<À4×CVÑCVô
ˆŒô —i’iØ×ÑØ×$Ñ$ t§}¡}Õ4Ø×$Ñ$ô
ˆŒô
 —i’iØ×ÑØ×$Ñ$ t§}¡}Õ4Ø×$Ñ$ô
ˆŒô
 —i’iØ�N‰N˜TŸ]™]Õ*¨D×,<Ñ,<À4×CVÑCVô
ˆŽrS   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# )r,   rk   r¥   r¤   rm   )re   rn   r   r   )rO   rP   s   "€r(   rQ   rÕ     sI   ø€ ÷ #ñ #á�8‰8ð#ñ ‘r—x‘xÕ ð#ñ ™Õ ð	#ñ
 
�‰ñ#rS   c           	     ó^  € VP                   w  rEpV P                  V4      P                  WEV P                  R4      P	                  ^ ^^^4      pV P                  V4      P                  WEV P                  R4      P	                  ^ ^^^4      pV P                  V4      P                  WEV P                  R4      P	                  ^ ^^^4      p	Ve   VP                  W‰4      w  r‰\        WxW“V P                  VR7      p
V
P	                  ^ ^^^4      P                  WER4      p
V P                  V
4      # )rÁ   )r¤   r×   r¥   rs   )rr   rØ   rÂ   r�   Ú	transposerÙ   r3   rÚ   Úupdate_and_fetchr   r×   rÛ   )r&   rk   r¥   r¤   r½   ÚLr�   ÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r(   ry   ÚNemotronHAttention.__call__  s  € ð —'‘'‰ˆˆaà—+‘+˜a“.×(Ñ(¨¨t¯~©~¸rÓB×LÑLÈQÐPQÐSTÐVWÓXˆà�K‰K˜‹Nß‰W�Q˜4×3Ñ3°RÓ8ß‰Y�q˜!˜Q Ó"ð 	ð �K‰K˜‹Nß‰W�Q˜4×3Ñ3°RÓ8ß‰Y�q˜!˜Q Ó"ð 	ð ÒØ ×1Ñ1°$Ó?‰LˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"rS   )	r?   r/   rÙ   r�   r3   rÛ   rØ   r×   rÚ   ©NNr{   r}   s   @@r(   rÓ   rÓ   í   s   ù‡ € ÷
ó 
÷:#÷ #ò #rS   rÓ   c                   óH   a a€ ] tR tRt oRV3R lV 3R llltR tRtVtV ;t# )ÚNemotronHMLPi)  c                ó    <€ V ^8„  d   QhRS[ /# r�   rƒ   )rO   rP   s   "€r(   rQ   ÚNemotronHMLP.__annotate__*  s   ø€ ÷ 	
ñ 	
™Yñ 	
rS   c                ó  <€ \         SV `  4        T;'       g    VP                  p\        P                  ! VP
                  W!P                  R 7      V n        \        P                  ! W!P
                  VP                  R 7      V n        R# )rŒ   N)	rc   rd   r0   r“   r–   r/   r;   Úup_projÚ	down_proj)r&   r‚   r0   rh   s   &&&€r(   rd   ÚNemotronHMLP.__init__*  sc   ø€ Ü‰ÑÔØ-×GÐG°×1GÑ1GÐä—y’yØ×ÑÐ/·m±mô
ˆŒô ŸšØ×/Ñ/°d·m±mô
ˆŽrS   c                ój   € V P                  \        P                  ! V P                  V4      4      4      # rb   )rï   r“   Úrelu2rî   ©r&   rk   s   &&r(   ry   ÚNemotronHMLP.__call__5  s"   € Ø�~‰~œbŸhšh t§|¡|°A£Ó7Ó8Ð8rS   )rï   rî   rb   r{   r}   s   @@r(   rê   rê   )  s   ù‡ € ÷	
õ 	
÷9ò 9rS   rê   c                 ó0  € \         P                  ! V P                  \         P                  4      4      ;rxW�,           pV^8”  dÓ   \         P                  ! VRVR3R7      p\         P
                  ! V^RR7      P                  RRR7      p	W4,
          p
\         P                  ! Wš^,
          RR7      RRV
1R3,          p\         P                  ! V\         P                  ! V4      \         P                  ! R	4      RR7      p\         P                  ! VRR4      pTp
\         P                  ! V) V
^,
          RR7      RRV
13,          p\         P                  ! W|RR7      pV^8”  d+   V'       d#   VP                  RRR7      pW�R
,           ,          pW…,          pWÈ3# )rÁ   rp   r¨   T)rq   Úkeepdims)Úkthrq   .Nr©   r   g#B’¡œÇ;rs   rt   )re   ÚsigmoidrÃ   r›   ru   ÚtopkÚsumÚargpartitionÚput_along_axisÚstop_gradientrn   rx   r°   )ÚgatesÚe_score_correction_biasÚtop_krC   rF   rI   rH   Úorig_scoresÚscoresÚgroup_scoresÚkÚ	group_idxÚindsÚdenominators   &&&&&&&       r(   Úgroup_expert_selectr  9  sH  € ô Ÿ:š: e§l¡l´2·:±:Ó&>Ó?Ð?€KØÕ-€FØ�„{Ü—’˜f¨2°g¸r°]ÔCˆÜ—w’w˜v q¨rÔ2×6Ñ6¸BÈÐ6ÓNˆØÕ ˆÜ—O’O L¸!µeÀ"ÔEÀcÈ2ÈAÈ2ÈqÀjÕQˆ	Ü×"Ò"Ø”B×$Ò$ YÓ/´·²¸#³ÀRô
ˆô —’˜F B¨Ó+ˆà€AÜ�?Š?˜F˜7¨¨A­°BÔ7¸¸R¸a¸R¸Õ@€DÜ×Ò ¸Ô;€FØˆq„y—^Ø—j‘j b°4�jÓ8ˆØ¨Õ.Õ/ˆØÕ+€Fàˆ<ÐrS   c                   óD   a a€ ] tR tRt oV3R lV 3R lltR tRtVtV ;t# )ÚMoEGatei[  c                ó    <€ V ^8„  d   QhRS[ /# ©r,   Úconfigrƒ   )rO   rP   s   "€r(   rQ   ÚMoEGate.__annotate__\  s   ø€ ÷ 
Jñ 
J™yñ 
JrS   c                óª  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  V n	        \        P                  ! V P                  VP                  34      V n        \        P                  ! V P                  34      V n        R # rb   )rc   rd   r  rG   r   rH   rD   rI   rC   rF   re   r¬   r/   rg   rÿ   )r&   r  rh   s   &&€r(   rd   ÚMoEGate.__init__\  s�   ø€ Ü‰ÑÔØŒØ×/Ñ/ˆŒ
Ø$×3Ñ3ˆÔØ &× 7Ñ 7ˆÔØ%+×%AÑ%AˆÔ"Ø—~‘~ˆŒØ ×+Ñ+ˆŒÜ—h’h × 5Ñ 5°v×7IÑ7IÐJÓKˆŒÜ')§x¢x°×1FÑ1FÐ0HÓ'IˆÖ$rS   c           	     óÐ   € \        WP                  P                  ,          V P                  V P                  V P
                  V P                  V P                  V P                  4      # rb   )	r  rg   ÚTrÿ   r   rC   rF   rI   rH   ró   s   &&r(   ry   ÚMoEGate.__call__h  sM   € Ü"Ø—‘—‘ÕØ×(Ñ(Ø�J‰JØ�L‰LØ�O‰OØ×&Ñ&Ø×Ñó
ð 	
rS   )	r  rÿ   rC   rD   rH   rI   r   rF   rg   r{   r}   s   @@r(   r
  r
  [  s   ù‡ € ÷
Jó 
J÷	
ò 	
rS   r
  c                   óD   a a€ ] tR tRt oV3R lV 3R lltR tRtVtV ;t# )ÚNemotronHMoEit  c                ó    <€ V ^8„  d   QhRS[ /# r  rƒ   )rO   rP   s   "€r(   rQ   ÚNemotronHMoE.__annotate__u  s   ø€ ÷ !ñ !™yñ !rS   c                ó¼  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        VP                  e   VP                  MVP
                  p\        VVP                  VP                  \        P                  ! 4       R7      V n        \        V4      V n        VP                  e   VP                  p\!        WR7      V n        VP                  e{   \        P$                  ! VP
                  VP                  VP&                  R7      V n        \        P$                  ! VP                  VP
                  VP&                  R7      V n        R # R # )N)Ú
activation)r0   rŒ   )rc   rd   r  rG   rB   r/   r   r@   rD   r“   ÚReLU2Ú
switch_mlpr
  rl   rE   rA   rê   Úshared_expertsr–   r;   Úfc1_latent_projÚfc2_latent_proj)r&   r  Úexpert_input_dimr0   rh   s   &&  €r(   rd   ÚNemotronHMoE.__init__u  s  ø€ Ü‰ÑÔØŒØ#)×#=Ñ#=ˆÔ Ø%×5Ñ5ˆÔð
 ×%Ñ%Ò1ð ×"Ò"à×#Ñ#ð 	ô
 $ØØ×(Ñ(Ø×#Ñ#Ü—x’x“zô	
ˆŒô ˜F“OˆŒ	Ø×"Ñ"Ò.Ø &× JÑ JÐÜ".Øô#ˆDÔð
 ×!Ñ!Ò-Ü#%§9¢9Ø×"Ñ" F×$:Ñ$:ÀÇÁô$ˆDÔ ô $&§9¢9Ø×&Ñ&¨×(:Ñ(:ÀÇÁô$ˆDÖ ñ	 .rS   c                óœ  € TpV P                  V4      w  r4V P                  e   V P                  V4      pV P                  W4      pWTR,          ,          P	                  RR7      P                  VP                  4      pV P                  e   V P                  V4      pV P                  P                  e   WPP                  V4      ,           pV# )Nr¨   rª   rt   )rl   rB   r  r  rú   rÃ   rŽ   r  r  rE   r  )r&   rk   Ú	residualsr  r  rÈ   s   &&    r(   ry   ÚNemotronHMoE.__call__˜  s®   € Øˆ	Ø—y‘y “|‰ˆà×ÑÒ+Ø×$Ñ$ QÓ'ˆAà�O‰O˜AÓ$ˆØ˜	Õ"Õ"×'Ñ'¨RÐ'Ó0×7Ñ7¸¿¹Ó@ˆà×ÑÒ+Ø×$Ñ$ QÓ'ˆAà�;‰;×'Ñ'Ò3Ø×'Ñ'¨	Ó2Õ2ˆAàˆrS   )r  r  r  rl   rB   rG   r  r  r{   r}   s   @@r(   r  r  t  s   ù‡ € ÷!ó !÷Fò rS   r  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚNemotronHBlocki«  c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r,   r‚   Ú
block_type)r   rL   )rO   rP   s   "€r(   rQ   ÚNemotronHBlock.__annotate__¬  s   ø€ ÷ ,ñ ,™Yð ,±Cñ ,rS   c                óª  <€ \         SV `  4        \        P                  ! VP                  VP
                  R 7      V n        W n        V P                  R8X  d   \        V4      V n	        R# V P                  R8X  d   \        V4      V n	        R# V P                  R8X  d   \        V4      V n	        R# V P                  R8X  d   \        V4      V n	        R# R# )©r^   r   r   r   r   N)rc   rd   r“   ÚRMSNormr/   r<   rž   r'  r   ÚmixerrÓ   rê   r  )r&   r‚   r'  rh   s   &&&€r(   rd   ÚNemotronHBlock.__init__¬  s–   ø€ Ü‰ÑÔÜ—J’J˜t×/Ñ/°T×5LÑ5LÔMˆŒ	à$Œà�?‰?˜cÔ!Ü-¨dÓ3ˆDŽJØ�_‰_ Ô#Ü+¨DÓ1ˆDŽJØ�_‰_ Ô#Ü% dÓ+ˆDŽJØ�_‰_ Ô#Ü% dÓ+ˆDŽJñ $rS   c                óZ   <€ V ^8„  d   QhRS[ S[P                  ,          RS[ S[,          /# )r,   r¥   r¤   )r   re   rn   r   )rO   rP   s   "€r(   rQ   r(  »  s.   ø€ ÷ !ñ !ñ ‘r—x‘xÕ ð!ñ ™�}ñ	!rS   c                óÐ   € V P                  V4      pV P                  R 8X  g   V P                  R8X  d   V P                  WBVR7      pW,           # V P                  V4      pW,           # )r   r   ©r¥   r¤   )rž   r'  r,  )r&   rk   r¥   r¤   r¼   s   &&&& r(   ry   ÚNemotronHBlock.__call__»  s^   € ð Ÿ	™	 !›ˆØ�?‰?˜cÔ! T§_¡_¸Ô%;Ø ŸJ™J }Àu˜JÓMˆMð Õ Ð ð !ŸJ™J }Ó5ˆMàÕ Ð rS   )r'  r,  rž   rè   r{   r}   s   @@r(   r%  r%  «  s   ù‡ € ÷,ó ,÷!÷ !ò !rS   r%  c                   óT   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚNemotronHModeliÊ  c                ó    <€ V ^8„  d   QhRS[ /# r�   rƒ   )rO   rP   s   "€r(   rQ   ÚNemotronHModel.__annotate__Ë  s   ø€ ÷ ñ ™Yñ rS   c                óZ  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        VP                   Uu. uF  p\        W4      NK  	  upV n	        \        P                  ! VP
                  VP                  R 7      V n        ^ V n        ^ V n        VP                   F.  pVR8X  d    M&VR8X  g   K  V ;P                  ^,          un        K0  	  VP                   F/  pVR8X  d   V ;P                  ^,          un        K%  VR8X  g   K.   R# 	  R# u upi )r*  r   r   N)rc   rd   r“   Ú	Embeddingr.   r/   Ú
embeddingsr!   r%  Úlayersr+  r<   Únorm_fÚfa_idxÚssm_idx)r&   r‚   r'  Úbrh   s   &&  €r(   rd   ÚNemotronHModel.__init__Ë  sé   ø€ Ü‰ÑÔÜŸ,š, t§¡¸×8HÑ8HÓIˆŒð #×:Ò:ó
á:�
ô ˜4Ö,Ù:ñ
ˆŒô —j’j ×!1Ñ!1°t×7NÑ7NÔOˆŒØˆŒØˆŒØ×-Ô-ˆAØ�CŒxÙØ�c–Ø—’˜qÕ —ñ	 .ð
 ×-Ô-ˆAØ�CŒxØ—’ Õ!—Ø�c–Úó	 .ùò
s   ÁD(c                ó0   <€ V ^8„  d   QhRS[ S[,          /# )r,   r¤   )r   r   )rO   rP   s   "€r(   rQ   r5  à  s   ø€ ÷ *ñ *ñ ™�}ñ*rS   c                óÎ  € V P                  V4      pVf   R .\        V P                  4      ,          p\        W2V P                  ,          4      p\        W2V P                  ,          4      p^ pV P                   FY  pVP                  R8X  g   VP                  R8X  d   W&,          pV^,          pMR pVP                  R8X  d   Tp	MTp	V! W9VR7      pK[  	  V P                  V4      # )Nr   r   r0  )	r8  r$   r9  r   r;  r   r<  r'  r:  )
r&   Úinputsr¤   r¼   Ú	attn_maskÚssm_maskÚcache_counterÚlayerÚcr¥   s
   &&&       r(   ry   ÚNemotronHModel.__call__à  sÌ   € ð
 Ÿ™¨Ó/ˆàŠ=Ø�FœS §¡Ó-Õ-ˆEÜ)¨-¸t¿{¹{Õ9KÓLˆ	Ü" =¸¿¹Õ2EÓFˆàˆØ—[”[ˆEØ×Ñ 3Ô&¨%×*:Ñ*:¸cÔ*AØÕ(�Ø Õ"‘à�à×Ñ 3Ô&Ø ‘à�Ù! -À!ÔDŠMñ !ð �{‰{˜=Ó)Ð)rS   )r8  r;  r9  r:  r<  rb   r{   r}   s   @@r(   r3  r3  Ê  s   ù‡ € ÷ó ÷**÷ *ò *rS   r3  c                   ó€   a a€ ] tR tRt oV3R lV 3R lltRV3R lR llt]R 4       tR tR t	]R	 4       t
R
tVtV ;t# )ÚModeliý  c                ó    <€ V ^8„  d   QhRS[ /# r�   rƒ   )rO   rP   s   "€r(   rQ   ÚModel.__annotate__þ  s   ø€ ÷ *ñ *™Yñ *rS   c                óÖ   <€ \         SV `  4        Wn        \        V4      V n        \
        P                  ! VP                  VP                  R R7      V n	        VP                  V n
        R# )FrŒ   N)rc   rd   r‚   r3  Úbackboner“   r–   r/   r.   Úlm_headr-   rÜ   s   &&€r(   rd   ÚModel.__init__þ  sH   ø€ Ü‰ÑÔØŒ	Ü& tÓ,ˆŒÜ—y’y ×!1Ñ!1°4·?±?ÈÔOˆŒØŸ/™/ˆŽrS   c                óJ   <€ V ^8„  d   QhRS[ P                  RS[S[,          /# )r,   rA  r¤   )re   rn   r   r   )rO   rP   s   "€r(   rQ   rK    s'   ø€ ÷ !ñ !á—‘ð!ñ ™�}ñ!rS   c                óH   € V P                  WR 7      pV P                  V4      # ))r¤   )rM  rN  )r&   rA  r¤   Úouts   &&& r(   ry   ÚModel.__call__  s#   € ð
 �m‰m˜FˆmÓ0ˆØ�|‰|˜CÓ Ð rS   c                ó.   € V P                   P                  # rb   )rM  r9  )r&   s   &r(   r9  ÚModel.layers  s   € à�}‰}×#Ñ#Ð#rS   c                óä   € . pV P                    F]  pVP                  R 8X  d   VP                  \        ^R7      4       K1  VP                  R8X  g   KD  VP                  \	        4       4       K_  	  V# )r   )Úsizer   )r9  r'  Úappendr   r   )r&   ÚcachesÚls   &  r(   Ú
make_cacheÚModel.make_cache  sS   € ØˆØ—”ˆAØ�|‰|˜sÔ"Ø—‘œk¨qÔ1Ö2Ø—‘ Ö$Ø—‘œg›iÖ(ñ	 ð
 ˆrS   c                óˆ  € VP                  4        UUu/ uF   w  r#VP                  R 4      '       d   K  W#bK"  	  pppVP                  4        F<  w  r#RV9   g   K  VP                  R	,          ^8w  g   K(  VP                  ^^4      W&   K>  	  \	        V P
                  P                  4       F‡  pRV R2pR
 Fx  w  rgV RV R2V9   g   K  \	        V P
                  P                  4       Uu. uF  pVP                  V RV RV R24      NK  	  p	p\        P                  ! V	4      W RV R2&   Kz  	  K‰  	  V# u uppi u upi )zmtp.zconv1d.weightzbackbone.layers.z.mixerz.experts.0.z.weightz	.experts.Ú.z.switch_mlp.rs   ))rï   Úfc2)rî   Úfc1)ÚitemsÚ
startswithrr   ÚmoveaxisÚranger‚   r%   rD   Úpopre   Ústack)
r&   Úweightsr  ÚvrZ  ÚprefixÚmÚnÚeÚto_joins
   &&        r(   ÚsanitizeÚModel.sanitize  s4  € Ø&-§m¡m¤oÔR¡o™F˜Q¸Q¿\¹\È&×=Q”4�1’4¡oˆÑRØ—M‘M–O‰DˆAØ !Ö#¨¯©°­°qÖ(8ØŸZ™Z¨¨1Ó-�“
ñ $ô
 �t—y‘y×2Ñ2Ö3ˆAØ'¨ s¨&Ð1ˆFÛB‘�Ø�X˜[¨¨¨7Ð3°wÖ>ô "' t§y¡y×'AÑ'AÔ!Bóá!B˜Að  Ÿ™ v h¨i¸°s¸!¸A¸3¸gÐ$FÖGÙ!Bð ð ô BDÇÂÈ'ÓAR�G˜h l°1°#°WÐ=Ó>ó Cñ 4ð ˆùó! Sùòs   ”D9´D9Ã,#D?c                ó   € R  pV# )c                 ó&   € R V 9  ;'       d    RV 9  # )rÿ   rœ   © )r  s   &r(   Ú	predicateÚ'Model.cast_predicate.<locals>.predicate/  s   € Ø,°AÑ5×JÐJ¸'ÈÑ:JÐJrS   rr  )r&   rs  s   & r(   Úcast_predicateÚModel.cast_predicate-  s   € ò	Kð ÐrS   )r‚   rM  rN  r-   rb   )rT   rU   rV   rW   rd   ry   Úpropertyr9  r[  rn  ru  rY   rZ   r|   r}   s   @@r(   rI  rI  ý  sP   ù‡ € ÷*ó *÷!ò !ð ñ$ó ð$òòð& ñó ÷ð rS   rI  ))Údataclassesr   Ú	functoolsr   Útypingr   r   r   r   Úmlx.coreÚcorere   Úmlx.nnr“   Úactivationsr	   Úbaser
   r   r   r   r¤   r   r   Ússmr   Úswitch_layersr   r   ÚModuler\   r   rÓ   rê   Úcompiler  r
  r  r%  r3  rI  rr  rS   r(   Ú<module>r„     s  ðõ "Ý ß -Ó -å Ý å ÷ó ÷ (Ý Ý $ñ ƒô2G�ó 2Gó ð2Gôj+˜Ÿ	™	ô +ôO ˜2Ÿ9™9ô O ôd9#˜Ÿ™ô 9#ôx9�2—9‘9ô 9ð  ‡�ñó ðôB
ˆb�i‰iô 
ô24�2—9‘9ô 4ôn!�R—Y‘Yô !ô>0*�R—Y‘Yô 0*ôf5ˆB�I‰Iö 5rS   