+
    TV-j07  ã            
       óª  € ^ RI HtHt ^ RIHtHtHtHtHt ^ RI	H
t ^ RIHt ^RIHtHtHt ^RIHt ^RIHt ]! RR7       ! R	 R
4      4       t]! RR7       ! R R4      4       t]! RR7       ! R R4      4       tR R ltR R ltR]P6                  R]P8                  R]P:                  R]P<                  R]P<                  /t] ! R R]4      4       t  ! R R]PB                  4      t" ! R R]PB                  4      t# ! R R]PB                  4      t$ ! R  R!]PB                  4      t% ! R" R#]PB                  4      t& ! R$ R%]PB                  4      t'R# )&é    )Ú	dataclassÚfield)ÚAnyÚDictÚListÚOptionalÚUnionN)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCache)Úinitialize_ropeT)Úfrozenc                   óR   a € ] tR t^t o RtRtRtRtRtRt	Rt
RtR tV 3R ltRtV tR# )ÚAttentionConfigFNc                óˆ  € V P                   '       g   V P                  '       dH   \        P                  V R R4       \        P                  V RR4       \        P                  V RR4       R# V P                   '       gE   V P                  f   \        R4      hV P                  ^ 8:  d   \        RV P                   24      hR# R# )Ún_heads_in_groupNÚwindow_lengthÚnum_sink_tokensz>n_heads_in_group must be specified for active attention blocksz'n_heads_in_group must be positive, got )Úno_opÚreplace_with_linearÚobjectÚ__setattr__r   Ú
ValueError©Úselfs   &Úk/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/nemotron-nas.pyÚ__post_init__ÚAttentionConfig.__post_init__   s«   € à�:�:ˆ:˜×1×1Ð1ä×Ñ˜tÐ%7¸Ô>Ü×Ñ˜t _°dÔ;Ü×Ñ˜tÐ%6¸Ö=à——�Ø×$Ñ$Ò,Ü ØTóð ð ×$Ñ$¨Ô)Ü Ø=¸d×>SÑ>SÐ=TÐUóð ñ *ñ ó    c                óÊ   <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[S[S[,          ,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[S[,          ;R&   S[ ;R&   S[ ;R&   # )	é   r   r   Úsparsifyr   r   r   Ú$use_prefill_window_in_sink_attentionÚunshifted_sink)Úboolr   ÚlistÚstrÚint)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚAttentionConfig.__annotate__   s‚   ø‡ ‚ áÑñ ñ Ñ%ñ ñ ‘t™C•yÕ!Ñ(ñ	 ñ
 ™s•mÑ*ñ ñ ™C•=Ñ'ñ ñ ™c•]Ñ)ñ ñ +/ñ ñ ñ Ñ ò r    © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r#   r   r   r   r$   r%   r   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__©r+   s   @r   r   r      s@   ø‡ € à€EØ %ÐØ$(€HØ&*ÐØ#'€MØ%)€Oàð )ð !€Nò÷ ƒ r    r   c                   óB   a € ] tR t^.t o RtRtRtRtR tV 3R lt	Rt
V tR# )Ú	FFNConfigFNc                ó0  € V P                   '       g   V P                  '       d   \        P                  V R R4       R# V P                   '       gG   V P                  f   \        R4      h\        P                  V R \        V P                  ^4      4       R# R# )Úffn_multNz0ffn_mult must be specified for active FFN blocks)r   r   r   r   r:   r   Úroundr   s   &r   r   ÚFFNConfig.__post_init__5   si   € à�:�:ˆ:˜×1×1Ð1Ü×Ñ˜t Z°Ö6à——�Ø�}‰}Ò$Ü Ð!SÓTÐTä×Ñ˜t Z´°t·}±}ÀaÓ1HÖIñ	 r    c                óz   <€ V ^8„  d   Qh/ S[ ;R&   S[ ;R&   S[S[S[,          ,          ;R&   S[S[,          ;R&   # )r"   r   r   r#   r:   )r&   r   r'   r(   Úfloat)r*   r+   s   "€r   r,   ÚFFNConfig.__annotate__.   sD   ø‡ ‚ áÑñ ñ Ñ%ñ ñ ‘t™C•yÕ!Ñ(ñ	 ñ
 ‘u�oÑ$ò r    r.   )r/   r0   r1   r2   r   r   r#   r:   r   r3   r4   r5   r6   s   @r   r8   r8   .   s'   ø‡ € à€EØ %ÐØ$(€HØ $€Hò	J÷ ƒ r    r8   c                   óH   a € ] tR t^At o ]V 3R lR l4       tV 3R ltRtV tR# )ÚBlockConfigc                ó    <€ V ^8„  d   QhRS[ /# )r"   Údata)Údict)r*   r+   s   "€r   r,   ÚBlockConfig.__annotate__G   s   ø€ ÷ 6ñ 6™Tñ 6r    c                ó€   € \        R/ VP                  R / 4      B p\        R/ VP                  R/ 4      B pV ! W#R7      # )Ú	attentionÚffn)rG   rH   r.   )r   Úgetr8   )ÚclsrC   Ú	attn_confÚffn_confs   &&  r   Ú	from_dictÚBlockConfig.from_dictF   s?   € ô $Ñ@ d§h¡h¨{¸BÓ&?Ñ@ˆ	ÜÑ3˜tŸx™x¨¨rÓ2Ñ3ˆÙ˜YÔ5Ð5r    c                ó2   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   # )r"   rG   rH   )r   r8   )r*   r+   s   "€r   r,   rE   A   s   ø‡ ‚ áÑñ ñ 
�Nò r    r.   N)	r/   r0   r1   r2   ÚclassmethodrM   r3   r4   r5   r6   s   @r   rA   rA   A   s    ø‡ € ð
 ÷6ó ð6÷ ƒ r    rA   c                ó<   € V ^8„  d   QhR\         R\         R\         /# )r"   ÚnÚkÚreturn)r)   )r*   s   "r   r,   r,   N   s!   € ÷ ñ ”cð œcð ¤cñ r    c                óJ   € W,          ^ 8X  d   V # W,           W,          ,
          # )z<Finds the smallest multiple of k greater than or equal to n.r.   )rR   rS   s   &&r   Ú_find_multiplerV   N   s   € à…u�„zØˆØ�5�A•E�?Ðr    c                ó<   € V ^8„  d   QhR\         R\        R\        /# )r"   r:   Ún_embdrT   )r>   r)   )r*   s   "r   r,   r,   U   s!   € ÷ 2ñ 2¬Uð 2¼Cð 2ÄCñ 2r    c                óZ   € \        ^V ,          V,          ^,          4      p\        VR4      # )zQCalculates intermediate size based on multiplier, rounding up to multiple of 256.é   )r)   rV   )r:   rX   Úintermediate_sizes   && r   Ú_ffn_mult_to_intermediate_sizer\   U   s)   € ä˜A �L¨6Õ1°AÕ5Ó6ÐÜÐ+¨SÓ1Ð1r    ÚsiluÚreluÚgeluÚgelu_newÚ	gelu_fastc                   óx   a € ] tR t^et o RtRt^Pt^@tRtRt	]
! ]R7      tRtRtRtRtR	tR
tRtR tV 3R ltRtV tR	# )Ú	ModelArgsznemotron-nasi    gñhãˆµøä>i õ )Údefault_factoryr]   Fg    €„ANi   c           
     óz  € V P                   '       d\   \        V P                   ^ ,          \        4      '       d5   V P                    Uu. uF  p\        P	                  V4      NK  	  upV n         \        V P                   4      V P                  8w  d0   \        R\        V P                   4       RV P                   R24      hV P                  '       dG   RV P                  9  d   \        R4      hV P                  P                  R4      pVf   \        R4      h\        V P                   4       F„  w  r4VP                  pVP                  '       d   K%  VP                  '       d   K9  V P                  VP                  ,          ^ 8w  g   K]  \        R	V R
V P                   RVP                   R24      h	  R# u upi )r   zNumber of block_configs (z ) must match num_hidden_layers (Ú)Úfactorz"rope_scaling must contain 'factor'Ú	rope_typeNz%rope_scaling must contain 'rope_type'zLayer z: num_attention_heads (z)) must be divisible by n_heads_in_group ()Úblock_configsÚ
isinstancerD   rA   rM   ÚlenÚnum_hidden_layersr   Úrope_scalingrI   Ú	enumeraterG   r   r   Únum_attention_headsr   )r   Úconfrh   ÚiÚ
block_confrK   s   &     r   r   ÚModelArgs.__post_init__v   sŒ  € à××Ð¤*¨T×-?Ñ-?ÀÕ-BÄD×"IÒ"Ià8<×8JÒ8Jó"Ù8J°”×%Ñ% dÖ+Ñ8Jñ"ˆDÔô ˆt×!Ñ!Ó" d×&<Ñ&<Ô<ÜØ+¬C°×0BÑ0BÓ,CÐ+Dð E&Ø&*×&<Ñ&<Ð%=¸Qð@óð ð ××ÐØ˜t×0Ñ0Ô0Ü Ð!EÓFÐFØ×)Ñ)×-Ñ-¨kÓ:ˆIØÒ Ü Ð!HÓIÐIô ' t×'9Ñ'9Ö:‰MˆAØ"×,Ñ,ˆIØ—?—?’?¨9×+H×+HÒ+HØ×+Ñ+¨i×.HÑ.HÕHÈAÖMÜ$Ø   Ð#:¸4×;SÑ;SÐ:Tð UBØBK×B\ÑB\ÐA]Ð]^ð`óð ó	 ;ùò'"s   ÁF8c                óþ   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[ ;R&   S[;R	&   S[;R
&   S[;R&   S[S[S[ S[S[S[ 3,          3,          ,          ;R&   S[;R&   S[;R&   # )r"   Ú
model_typeÚhidden_sizerl   ro   Úrms_norm_epsÚ
vocab_sizeri   Ú
hidden_actÚattention_biasÚmlp_biasÚ
rope_thetarm   Úmax_position_embeddingsÚtie_word_embeddings)r(   r)   r>   r'   r&   r   r   r	   )r*   r+   s   "€r   r,   ÚModelArgs.__annotate__e   sÑ   ø‡ ‚ áÑ$ñ ñ Ññ ñ Ññ	 ñ
 Ñ!ñ ñ Ññ ñ Ññ ñ Ñ5ñ ñ Ññ ñ Ñ ñ ñ Ññ ñ Ñ ñ ñ ™4¡¡U©5±#¨:Õ%6Ð 6Õ7Õ8Ñ?ñ ñ !Ñ)ñ ñ Ñ%ò r    )ri   )r/   r0   r1   r2   ru   rv   rl   ro   rw   rx   r   r'   ri   ry   rz   r{   r|   rm   r}   r~   r   r3   r4   r5   r6   s   @r   rc   rc   e   s`   ø‡ € à$€JØ€KØÐØ!ÐØ€LØ€JÙ°Ô5€MØ€JØ €NØ€HØ €JØ;?€LØ#)ÐØ %Ðò÷# ƒ r    rc   c                   óX   a a€ ] tR t^–t oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )Ú	Attentionz8Standard GQA Attention mechanism for layers that use it.c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r"   ÚargsÚattention_config)rc   r   )r*   r+   s   "€r   r,   ÚAttention.__annotate__™   s   ø€ ÷ 
ñ 
™Yð 
¹/ñ 
r    c                ó  <€ \         SV `  4        VP                  pVP                  ;V n        pWBP
                  ,          ;V n        pVP                  V,          ;V n        pV P                  V,          V8w  d   \        R V RV R24      hVR,          V n	        \        P                  ! W4V,          VP                  R7      V n        \        P                  ! W5V,          VP                  R7      V n        \        P                  ! W5V,          VP                  R7      V n        \        P                  ! WF,          W1P                  R7      V n        \#        V P                  VP$                  RVP&                  VP(                  4      V n        R# )zhidden_size (z,) must be divisible by num_attention_heads (rf   ©ÚbiasFNg      à¿)ÚsuperÚ__init__rv   ro   Ún_headsr   Ú
n_kv_headsÚhead_dimr   ÚscaleÚnnÚLinearrz   Úq_projÚk_projÚv_projÚo_projr   r|   rm   r}   Úrope)r   rƒ   r„   Údimr‹   rŒ   r�   Ú	__class__s   &&&    €r   rŠ   ÚAttention.__init__™   s;  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'.×2SÑ2SÕ'SÐSˆŒ˜*à#'×#3Ñ#3°wÕ#>Ð>ˆŒ˜Ø�M‰M˜GÕ#¨Ô+ÜØ ˜uÐ$PÐQXÐPYÐYZÐ[óð ð ˜t•^ˆŒ
ä—i’i ¨xÕ%7¸d×>QÑ>QÔRˆŒÜ—i’i °(Õ%:À×ATÑATÔUˆŒÜ—i’i °(Õ%:À×ATÑATÔUˆŒÜ—i’i Õ 2°C×>QÑ>QÔRˆŒô $Ø�M‰MØ�O‰OØØ×ÑØ×(Ñ(ó
ˆŽ	r    c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r"   ÚxÚmaskÚcacherT   ©ÚmxÚarrayr   r   )r*   r+   s   "€r   r,   r…   ¶   sH   ø€ ÷ #ñ #á�8‰8ð#ñ ‘r—x‘xÕ ð#ñ ™�}ð	#ñ
 
�‰ñ#r    c           	     óZ  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pVP	                  WEV P
                  V P                  4      P                  ^ ^^^4      pVP	                  WEV P                  V P                  4      P                  ^ ^^^4      pV	P	                  WEV P                  V P                  4      P                  ^ ^^^4      p	VeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      P	                  WER4      p
V P                  V
4      # )r   )Úoffset)r�   rŽ   rœ   éÿÿÿÿ)Úshaper‘   r’   r“   Úreshaper‹   r�   Ú	transposerŒ   r•   r¢   Úupdate_and_fetchr   rŽ   r”   )r   r›   rœ   r�   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r   Ú__call__ÚAttention.__call__¶   si  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆà—/‘/ !¨¯©°d·m±mÓD×NÑNØˆq�!�Qó
ˆð �|‰|˜A $§/¡/°4·=±=ÓA×KÑKÈAÈqÐRSÐUVÓWˆØ—‘  d§o¡o°t·}±}ÓE×OÑOØˆq�!�Qó
ˆð ÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r    )	r�   r’   r‹   rŒ   r”   r‘   r•   rŽ   r“   ©NN©
r/   r0   r1   r2   Ú__doc__rŠ   r¯   r4   r5   Ú__classcell__©r—   r+   s   @@r   r�   r�   –   s!   ù‡ € ÙB÷
ó 
÷:#÷ #ò #r    r�   c                   óT   a a€ ] tR t^×t oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚMLPz5Standard Feed-Forward Network for layers that use it.c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r"   rƒ   Ú
ffn_config)rc   r8   )r*   r+   s   "€r   r,   ÚMLP.__annotate__Ú   s   ø€ ÷ Pñ P™Yð P±Iñ Pr    c                óÎ  <€ \         SV `  4        VP                  p\        VP                  V4      p\
        P                  ! W4VP                  R 7      V n        \
        P                  ! WCVP                  R 7      V n	        \
        P                  ! W4VP                  R 7      V n
        VP                  V n        V P                  \        9  d   \        RVP                   24      hR# )r‡   zUnknown activation function: N)r‰   rŠ   rv   r\   r:   r�   r�   r{   Ú	gate_projÚ	down_projÚup_projry   Úact_fnÚ_ACT2FNr   )r   rƒ   r¹   r–   Ú
hidden_dimr—   s   &&&  €r   rŠ   ÚMLP.__init__Ú   s¡   ø€ Ü‰ÑÔà×Ñˆä3°J×4GÑ4GÈÓMˆ
äŸš 3¸¿¹ÔGˆŒÜŸš :¸¿¹ÔGˆŒÜ—y’y °t·}±}ÔEˆŒà—o‘oˆŒØ�;‰;œgÔ%ÜÐ<¸T¿_¹_Ð<MÐNÓOÐOñ &r    c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r"   rT   ©rŸ   r    )r*   r+   s   "€r   r,   rº   é   s   ø€ ÷ Kñ K™RŸX™Xñ Kr    c                ó¨   € \         V P                  ,          pV P                  V! V P                  V4      4      V P	                  V4      ,          4      # ©N)rÀ   r¿   r½   r¼   r¾   )r   r›   r¿   s   && r   r¯   ÚMLP.__call__é   s:   € Ü˜Ÿ™Õ%ˆØ�~‰~™f T§^¡^°AÓ%6Ó7¸$¿,¹,Àq»/ÕIÓJÐJr    )r¿   r½   r¼   r¾   r²   rµ   s   @@r   r·   r·   ×   s&   ù‡ € Ù?÷Pó P÷K÷ Kð Kr    r·   c                   óT   a a€ ] tR t^ît oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚLinearSubblockReplacementz>A simple linear layer used to replace Attention or MLP blocks.c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r"   rv   rˆ   )r)   r&   )r*   r+   s   "€r   r,   Ú&LinearSubblockReplacement.__annotate__ñ   s   ø€ ÷ Eñ E¡Cð E©tñ Er    c                ó^   <€ \         SV `  4        \        P                  ! WVR 7      V n        R# )r‡   N)r‰   rŠ   r�   r�   Úlinear)r   rv   rˆ   r—   s   &&&€r   rŠ   Ú"LinearSubblockReplacement.__init__ñ   s    ø€ Ü‰ÑÔÜ—i’i ¸tÔDˆŽr    c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r"   r›   rT   rÄ   )r*   r+   s   "€r   r,   rË   õ   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¹¿¹ñ r    c                ó$   € V P                  V4      # rÆ   ©rÍ   )r   r›   rƒ   Úkwargss   &&*,r   r¯   Ú"LinearSubblockReplacement.__call__õ   s   € à�{‰{˜1‹~Ðr    rÑ   r²   rµ   s   @@r   rÉ   rÉ   î   s#   ù‡ € ÙH÷Eó E÷÷ ð r    rÉ   c                   óX   a a€ ] tR t^út oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚTransformerBlockzFA single transformer block, potentially heterogeneous based on config.c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r"   rƒ   Ú	layer_idx)rc   r)   )r*   r+   s   "€r   r,   ÚTransformerBlock.__annotate__ý   s   ø€ ÷ (2ñ (2™Yð (2±3ñ (2r    c                ó  <€ \         SV `  4        VP                  V n        VP                  V,          pVP                  V n        VP                  V n        V P
                  P                  '       g3   \        P                  ! VP                  VP                  R 7      V n        MRV n        V P
                  P                  '       d	   RV n        M\V P
                  P                  '       d'   \        VP                  VP                   4      V n        M\#        WP
                  4      V n        V P                  P                  '       g3   \        P                  ! VP                  VP                  R 7      V n        MRV n        V P                  P                  '       d
   RV n        R# V P                  P                  '       d(   \        VP                  VP(                  4      V n        R# \+        WP                  4      V n        R# )©ÚepsN)r‰   rŠ   rv   ri   rG   r„   rH   r¹   r   r�   ÚRMSNormrw   Úinput_layernormÚ	self_attnr   rÉ   rz   r�   Úpost_attention_layernormÚmlpr{   r·   )r   rƒ   r×   Úblock_configr—   s   &&& €r   rŠ   ÚTransformerBlock.__init__ý   sq  ø€ Ü‰ÑÔØ×+Ñ+ˆÔà×)Ñ)¨)Õ4ˆØ ,× 6Ñ 6ˆÔØ&×*Ñ*ˆŒð ×$Ñ$×*×*Ð*Ü#%§:¢:¨d×.>Ñ.>ÀD×DUÑDUÔ#VˆDÕ à#'ˆDÔ ð × Ñ ×&×&Ð&Ø!ˆD�NØ×"Ñ"×6×6Ð6Ü6Ø× Ñ  $×"5Ñ"5óˆD�Nô
 ' t×-BÑ-BÓCˆDŒNð �‰×$×$Ð$Ü,.¯JªJØ× Ñ  d×&7Ñ&7ô-ˆDÕ)ð -1ˆDÔ)ð �?‰?× × Ð ØˆDŽHØ�_‰_×0×0Ð0Ü0°×1AÑ1AÀ4Ç=Á=ÓQˆDŽHô ˜4§¡Ó1ˆDŽHr    c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# rš   rž   )r*   r+   s   "€r   r,   rØ   '  sH   ø€ ÷ ñ á�8‰8ðñ ‘r—x‘xÕ ðñ ™�}ð	ñ
 
�‰ñr    c                óò   € V P                   e/   TpV P                  V4      pV P                  WRVR7      pWF,           pV P                  e-   TpV P                  V4      pV P                  V4      pWG,           pV# )N)rœ   r�   )rÞ   rÝ   rà   rß   )r   r›   rœ   r�   ÚresidualÚhÚattn_outÚmlp_outs   &&&&    r   r¯   ÚTransformerBlock.__call__'  sx   € ð �>‰>Ò%ØˆHØ×$Ñ$ QÓ'ˆAØ—~‘~ a¸%�~Ó@ˆHØÕ#ˆAð �8‰8ÒØˆHØ×-Ñ-¨aÓ0ˆAØ—h‘h˜q“kˆGØÕ"ˆAàˆr    )r„   r¹   rv   rÝ   rà   rß   rÞ   r±   r²   rµ   s   @@r   rÕ   rÕ   ú   s"   ù‡ € ÙP÷(2ó (2÷T÷ ò r    rÕ   c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚNemotronNASModeli?  z.The core Nemotron-NAS style transformer model.c                ó    <€ V ^8„  d   QhRS[ /# ©r"   rƒ   ©rc   )r*   r+   s   "€r   r,   ÚNemotronNASModel.__annotate__B  s   ø€ ÷ 
ñ 
™Yñ 
r    c                óê  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        \
        P                  ! VP                  VP                  4      V n        \        VP                  4       Uu. uF  p\        WR 7      NK  	  upV n        \
        P                  ! VP                  VP                  R7      V n        \        R V P                   4       4      V n        R# u upi ))rƒ   r×   rÚ   c              3   óD   "  € T F  qP                   f   K  ^x € K  	  R # 5irÆ   )rÞ   )Ú.0Úlayers   & r   Ú	<genexpr>Ú,NemotronNASModel.__init__.<locals>.<genexpr>M  s   é € ð #
Ù&�%¯/©/�AŠA›;ùs   ‚ –
 N)r‰   rŠ   rƒ   rx   rl   r�   Ú	Embeddingrv   Úembed_tokensÚrangerÕ   ÚlayersrÜ   rw   ÚnormÚsumÚnum_attn_layers)r   rƒ   rq   r—   s   && €r   rŠ   ÚNemotronNASModel.__init__B  s½   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔô ˜4×1Ñ1Ô2ó
á2�ô  $×4Ù2ñ
ˆŒô —J’J˜t×/Ñ/°T×5FÑ5FÔGˆŒ	Ü"ñ #
ØŸ;š;ó#
ó  
ˆÖùò
s   Á?C0c                óZ   <€ V ^8„  d   QhRS[ P                  RS[S[S[,          ,          /# )r"   Úinputsr�   )rŸ   r    r   r   r   )r*   r+   s   "€r   r,   rï   Q  s,   ø€ ÷ ñ á—‘ðñ ™™S�	Õ"ñr    c                ó  € V P                  V4      pVf   R .V P                  ,          p\        W2^ ,          4      p^ pV P                   F/  pVP                  e   W%,          pV^,          pMR pV! W4VR7      pK1  	  V P                  V4      # )N©r�   )r÷   rü   r   rù   rÞ   rú   )r   rÿ   r�   ræ   rœ   Ú	cache_idxró   Úcs   &&&     r   r¯   ÚNemotronNASModel.__call__Q  sˆ   € ð
 ×Ñ˜fÓ%ˆàŠ=Ø�F˜T×1Ñ1Õ1ˆEä$ Q¨a­Ó1ˆàˆ	Ø—[”[ˆEØ�‰Ò*ØÕ$�Ø˜Q•‘	à�Ù�a QÔ'ŠAñ !ð �y‰y˜‹|Ðr    )rƒ   r÷   rù   rú   rü   rl   rx   rÆ   r²   rµ   s   @@r   rë   rë   ?  s!   ù‡ € Ù8÷
ó 
÷÷ ò r    rë   c                   óp   a a€ ] tR tRt oV3R lV 3R lltR
V3R lR lltR t]R 4       tR t	R	t
VtV ;t# )ÚModelii  c                ó    <€ V ^8„  d   QhRS[ /# rí   rî   )r*   r+   s   "€r   r,   ÚModel.__annotate__k  s   ø€ ÷  ñ  ™Yñ  r    c                ó  <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        VP                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# RV n        R# )Fr‡   N)r‰   rŠ   rƒ   ru   rë   Úmodelr~   r�   r�   rv   rx   Úlm_head)r   rƒ   r—   s   &&€r   rŠ   ÚModel.__init__k  s[   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ% dÓ+ˆŒ
Ø×'×'Ð'ÜŸ9š9 T×%5Ñ%5°t·±ÈUÔSˆDŽLàˆDŽLr    c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r"   rÿ   rÄ   )r*   r+   s   "€r   r,   r  u  s   ø€ ÷ 
ñ 
á—‘ñ
r    c                óÒ   € V P                  WR 7      pV P                  P                  '       d(   V P                   P                  P	                  V4      pV# V P                  V4      pV# )r  )r
  rƒ   r~   r÷   Ú	as_linearr  )r   rÿ   r�   Úouts   &&& r   r¯   ÚModel.__call__u  sZ   € ð
 �j‰j˜ˆjÓ-ˆØ�9‰9×(×(Ð(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r    c                ób   € V P                   P                  '       d   VP                  R R4       V# )zlm_head.weightN)rƒ   r~   Úpop)r   Úweightss   &&r   ÚsanitizeÚModel.sanitize�  s&   € Ø�9‰9×(×(Ð(Ø�K‰KÐ(¨$Ô/Øˆr    c                ó.   € V P                   P                  # rÆ   )r
  rù   r   s   &r   rù   ÚModel.layers†  s   € à�z‰z× Ñ Ð r    c                ór   € V P                    Uu. uF  qP                  f   K  \        4       NK  	  up# u upi rÆ   )rù   rÞ   r   )r   ró   s   & r   Ú
make_cacheÚModel.make_cacheŠ  s&   € Ø'+§{¢{ÓR¡{˜e·o±o”	”–	¡{ÑRÐRùÒRs   �4£4)rƒ   r  r
  ru   rÆ   )r/   r0   r1   r2   rŠ   r¯   r  Úpropertyrù   r  r4   r5   r´   rµ   s   @@r   r  r  i  s>   ù‡ € ÷ ó  ÷
ò 
òð
 ñ!ó ð!÷Sò Sr    r  )(Údataclassesr   r   Útypingr   r   r   r   r	   Úmlx.coreÚcorerŸ   Úmlx.nnr�   Úbaser
   r   r   r�   r   Ú
rope_utilsr   r   r8   rA   rV   r\   r]   r^   r_   Úgelu_approxrÀ   rc   ÚModuler�   r·   rÉ   rÕ   rë   r  r.   r    r   Ú<module>r&     sD  ð÷ )ß 3Õ 3å Ý ç TÑ TÝ Ý 'ñ �$Ô÷ð ó ðñ> �$Ô÷Jð Jó ðJñ$ �$Ô÷	6ð 	6ó ð	6õõ2ð ˆB�G‰GØ
ˆB�G‰GØ
ˆB�G‰GØ�—‘Ø�—‘ð€ð ô-�ó -ó ð-ô`>#�—	‘	ô >#ôBKˆ"�)‰)ô Kô.	 §	¡	ô 	ôB�r—y‘yô BôJ'�r—y‘yô 'ôT"SˆB�I‰Iö "Sr    