Ë
    îÿæiB  ã                   ó`  — d dl mZ d dlmZ d dlmZmZmZmZ d dl	m
Z d dlmZ ddlmZ ddlmZmZmZmZ ddlmZmZ dd	lmZ dd
lmZ  e«        G d„ de«      «       Z G d„ dej:                  «      Z G d„ dej:                  «      Z G d„ dej:                  «      Z  G d„ dej:                  «      Z!ejD                  d„ «       Z# G d„ dej:                  «      Z$ G d„ dej:                  «      Z% G d„ dej:                  «      Z& G d„ dej:                  «      Z' G d„ dej:                  «      Z(y) é    )Ú	dataclass)Úpartial)ÚAnyÚListÚOptionalÚTupleNé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚcreate_ssm_maskÚscaled_dot_product_attention)ÚArraysCacheÚKVCache)Ú
ssm_update)Ú	SwitchMLPc                   óð  — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   ee   ed<   dZ	e
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
e   ed<   dZe
eeef      ed <   dZe
e   ed!<   dZe
e   ed"<   d#„ Zy)$Ú	ModelArgsÚ
model_typeÚ
vocab_sizeÚhidden_sizeÚintermediate_sizeÚnum_hidden_layersÚmax_position_embeddingsÚnum_attention_headsÚnum_key_value_headsÚattention_biasÚmamba_num_headsÚmamba_head_dimÚmamba_proj_biasÚssm_state_sizeÚconv_kernelÚn_groupsÚmlp_biasÚlayer_norm_epsilonÚuse_biasÚuse_conv_biasÚhybrid_override_patternNÚhead_dimÚmoe_intermediate_sizeÚ#moe_shared_expert_intermediate_sizeÚn_groupÚn_routed_expertsÚn_shared_expertsÚ
topk_groupÚnum_experts_per_tokÚnorm_topk_probÚrouted_scaling_factorÚtime_step_limitÚtime_step_minÚtime_step_maxc                 óŒ   — | j                   €8| j                  �+| j                  �| j                  | j                  f| _         y y y y ©N)r3   r4   r5   ©Úselfs    úm/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/nemotron_h.pyÚ__post_init__zModelArgs.__post_init__:   sN   € à× Ñ Ð(Ø×"Ñ"Ð.Ø×"Ñ"Ð.à$(×$6Ñ$6¸×8JÑ8JÐ#KˆDÕ ð /ð /ð )ó    )Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚboolÚfloatr   r)   r   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r   r4   r5   r;   © r<   r:   r   r      sE  … àƒOØƒOØÓØÓØÓØ Ó ØÓØÓØÓØÓØÓØÓØÓØÓØƒMØƒNØÓØƒNØÓØ! #™YÓ&Ø"€Hˆh�s‰mÓ"Ø+/Ð˜8 C™=Ó/Ø9=Ð'¨°#©Ó=Ø!€GˆX�c‰]Ó!Ø&*Ð�h˜s‘mÓ*Ø&*Ð�h˜s‘mÓ*Ø $€J�˜‘Ó$Ø)-Ð˜ #™Ó-Ø%)€N�H˜T‘NÓ)Ø-1Ð˜8 E™?Ó1Ø59€O�X˜e E¨5 LÑ1Ñ2Ó9Ø%)€M�8˜E‘?Ó)Ø%)€M�8˜E‘?Ó)óLr<   r   c                   ó~   ‡ — e Zd Zdededefˆ fd„Zd	dej                  dej                  dej                  fd„Zˆ xZ	S )
ÚMambaRMSNormGatedr   ÚepsÚ
group_sizec                 ór   •— t         ‰| �  «        || _        t        j                  |«      | _        || _        y r7   )ÚsuperÚ__init__rH   ÚmxÚonesÚweightrI   )r9   r   rH   rI   Ú	__class__s       €r:   rL   zMambaRMSNormGated.__init__D   s,   ø€ Ü‰ÑÔØˆŒÜ—g‘g˜kÓ*ˆŒØ$ˆ�r<   ÚxÚgateÚreturnc                 óú   — |�t        ||«      }t        j                  |dd| j                  f¬«      }t        j                  j                  |d | j                  ¬«      }| j                  |j                  d«      z  S )Néÿÿÿÿ©ÚaxisÚshape)rO   rH   éþÿÿÿ)	r
   rM   Ú	unflattenrI   ÚfastÚrms_normrH   rO   Úflatten)r9   rQ   rR   s      r:   Ú__call__zMambaRMSNormGated.__call__J   sd   € ØÐÜ�t˜Q“ˆAÜ�L‰L˜ ¨B°·±Ð+@ÔAˆÜ�G‰G×Ñ˜Q t°·±ÐÓ:ˆØ�{‰{˜QŸY™Y r›]Ñ*Ð*r<   r7   )
r=   r>   r?   rB   rD   rL   rM   Úarrayr^   Ú__classcell__©rP   s   @r:   rG   rG   C   sC   ø„ ð% Cð %¨eð %Àõ %ñ+˜"Ÿ(™(ð +¨"¯(©(ð +¸b¿h¹h÷ +r<   rG   c                   ó�  ‡ — e Zd Zdefˆ fd„Zdej                  dee   deej                     dej                  fd„Z	dej                  d	ej                  d
ej                  dej                  dee   deej                     dej                  fd„Z
	 ddej                  deej                     dee   dej                  fd„Zˆ xZS )ÚNemotronHMamba2MixerÚargsc                 óR  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  |j                  z  | _	        |j                  | _
        |j                  | _        |j                  | _        | j                  | j                  z  | _        | j                  d| j                  z  | j
                  z  z   | _        t        j                   | j                  | j                  |j                  d| j                  |j"                  ¬«      | _        | j                  | j                  z   | j                  z   }t        j&                  | j                  ||j(                  ¬«      | _        t-        j.                  | j                  «      | _        t-        j2                  t-        j4                  d| j                  dz   t,        j6                  ¬«      «      | _        t-        j.                  | j                  «      | _        | j                  | j                  z  }t=        | j                  |j>                  |¬«      | _         t        j&                  | j                  | j                  |j(                  ¬«      | _!        y )Né   r   )Úin_channelsÚout_channelsÚkernel_sizeÚpaddingÚgroupsÚbias©rl   r	   ©Údtype)rH   rI   )"rK   rL   r   Ú	num_headsr   r!   r"   Úconv_kernel_sizer   r   r#   r)   r3   Úheads_per_groupÚconv_dimÚnnÚConv1dr'   Úconv1dÚLinearr    Úin_projrM   rN   Údt_biasÚlogÚarangeÚfloat32ÚA_logÚDrG   r%   ÚnormÚout_proj)r9   rd   Úprojection_sizerI   rP   s       €r:   rL   zNemotronHMamba2Mixer.__init__S   sï  ø€ Ü‰ÑÔØ×-Ñ-ˆŒØ×+Ñ+ˆÔØ"×1Ñ1ˆÔØ $× 0Ñ 0ˆÔØ!%×!5Ñ!5¸×8KÑ8KÑ!KˆÔØŸ™ˆŒØ×+Ñ+ˆŒØ#×3Ñ3ˆÔØ#Ÿ~™~°·±Ñ>ˆÔà×.Ñ.°°T·]±]Ñ1BÀT×EXÑEXÑ1XÑXˆŒä—i‘iØŸ™ØŸ™Ø×(Ñ(ØØ—=‘=Ø×#Ñ#ô
ˆŒð ×0Ñ0°4·=±=Ñ@À4Ç>Á>ÑQˆÜ—y‘yØ×Ñ˜o°D×4HÑ4Hô
ˆŒô —w‘w˜tŸ~™~Ó.ˆŒÜ—V‘VœBŸI™I a¨¯©¸!Ñ);Ä2Ç:Á:ÔNÓOˆŒ
Ü—‘˜Ÿ™Ó(ˆŒà×+Ñ+¨t¯}©}Ñ<ˆ
Ü%Ø×"Ñ"Ø×'Ñ'Ø!ô
ˆŒ	ô
 Ÿ	™	Ø×"Ñ" D×$4Ñ$4¸4×;OÑ;Oô
ˆ�r<   Ú
conv_inputÚcacheÚmaskrS   c                 óò  — |�t        j                  |d   |d«      }|��|d   €It        j                  |j                  d   | j                  dz
  | j
                  f|j                  ¬«      }n|d   }t        j                  ||gd¬«      }| j                  dz
  }|j                  �q|j                  d   }t        j                  |j                  d||z
  «      }|d d …d f   t        j                  |«      z   d   }	t        j                  ||	d¬«      |d<   n;|d d …| d …d d …f   |d<   n(t        j                  |d| j                  dz
  dfdg«      }| j                  |«      }
t        j                  |
«      S )N©.Nr   r	   rn   ©rW   )r   r   )rM   ÚwhereÚzerosrX   rq   rs   ro   ÚconcatenateÚlengthsÚclipr{   Útake_along_axisÚpadrv   rt   Úsilu)r9   r‚   rƒ   r„   Ú
conv_stateÚpadded_inputÚn_keepÚtÚendsÚ	positionsÚconv_outputs              r:   Ú_convzNemotronHMamba2Mixer._conv}   sh  € ð ÐÜŸ™ $ y¡/°:¸qÓAˆJàÑØ�Q‰xÐÜŸX™XØ×%Ñ% aÑ(¨$×*?Ñ*?À!Ñ*CÀTÇ]Á]ÐSØ$×*Ñ*ô‘
ð
 # 1™X�
ÜŸ>™>¨:°zÐ*BÈÔKˆLØ×*Ñ*¨QÑ.ˆFØ�}‰}Ð(Ø ×&Ñ& qÑ)�Ü—w‘w˜uŸ}™}¨a°°V±Ó<�Ø!¢! T '™]¬R¯Y©Y°vÓ->Ñ>À	ÑJ�	Ü×-Ñ-¨l¸IÈAÔN��a’à'ª¨F¨7©8²Q¨Ñ7��a’äŸ6™6Ø˜V d×&;Ñ&;¸aÑ&?ÀÐ%CÀVÐLóˆLð —k‘k ,Ó/ˆÜ�w‰w�{Ó#Ð#r<   Úhidden_statesÚBÚCÚdtc                 ó<  — |j                   \  }}}	|j                  ||| j                  | j                  «      }|j                  ||| j                  | j
                  «      }|j                  ||| j                  | j
                  «      }|r|d   }
|j                  }nd\  }
}t        || j                  ||| j                  j                  |j                  «      || j                  |
| j                  |«
      \  }}
|r|
|d<   |j                  ||| j                  «      S )Nr	   ©NN)rX   Úreshaperp   r)   r#   r!   r‹   r   r}   r~   Úastypero   ry   r3   r   )r9   r˜   r™   rš   r›   rƒ   r„   Ú
batch_sizeÚseq_lenÚ_Ústater‹   Úys                r:   Ú_ssmzNemotronHMamba2Mixer._ssmŸ   s  € ð "/×!4Ñ!4Ñˆ
�G˜Qà%×-Ñ-Ø˜ §¡°·±ó
ˆð �I‰I�j '¨4¯=©=¸$×:MÑ:MÓNˆØ�I‰I�j '¨4¯=©=¸$×:MÑ:MÓNˆÙØ˜!‘HˆEØ—m‘m‰Gà'‰NˆE�7äØØ�J‰JØØØ�F‰F�M‰M˜-×-Ñ-Ó.ØØ�L‰LØØ× Ñ Øó
‰ˆˆ5ñ ØˆE�!‰Hà�y‰y˜ W¨d×.DÑ.DÓEÐEr<   c                 ó  — | j                  |«      }t        j                  || j                  | j                  | j                  z   gd¬«      \  }}}| j                  |||«      }t        j                  || j                  | j                  | j                  | j                  z  z   gd¬«      \  }	}
}| j                  |	|
||||«      }|r|j                  |j                  d   «       | j                  ||«      }| j                  |«      S )NrU   r‡   r	   )rx   rM   Úsplitr   rs   r—   r#   r!   r¥   ÚadvancerX   r   r€   )r9   r˜   r„   rƒ   Ú	projectedrR   r‚   r›   r–   Úhidden_states_ssmr™   rš   r¤   s                r:   r^   zNemotronHMamba2Mixer.__call__Æ   sø   € ð —L‘L Ó/ˆ	ä!Ÿx™xØØ×#Ñ# T×%;Ñ%;¸d¿m¹mÑ%KÐLØô 
Ñˆˆj˜"ð
 —j‘j ¨U°DÓ9ˆÜ"$§(¡(Øà×&Ñ&Ø×&Ñ&¨¯©¸×9LÑ9LÑ)LÑLðð ô#
ÑÐ˜1˜að �I‰IÐ'¨¨A¨r°5¸$Ó?ˆÙØ�M‰M˜!Ÿ'™' !™*Ô%Ø�I‰I�a˜ÓˆØ�}‰}˜QÓÐr<   r7   )r=   r>   r?   r   rL   rM   r_   r   r   r—   r¥   r^   r`   ra   s   @r:   rc   rc   R   s  ø„ ð(
˜Yõ (
ðT $à—H‘Hð $ð ˜Ñ$ð $ð �r—x‘xÑ ð	 $ð
 
�‰ó $ðD%Fà—x‘xð%Fð �8‰8ð%Fð �8‰8ð	%Fð
 �H‰Hð%Fð ˜Ñ$ð%Fð �r—x‘xÑ ð%Fð 
�‰ó%FðV (,ñ	 à—x‘xð ð �r—x‘xÑ ð ð ˜Ñ$ð	 ð
 
�‰÷ r<   rc   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚNemotronHAttentionrd   c                 ó\  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  �|j
                  n|j                  |j                  z  | _        |j                  | _        | j
                  dz  | _        t        j                  | j                  | j                  | j
                  z  |j                  ¬«      | _        t        j                  | j                  | j                  | j
                  z  |j                  ¬«      | _        t        j                  | j                  | j                  | j
                  z  |j                  ¬«      | _        t        j                  | j                  | j
                  z  | j                  |j                  ¬«      | _        y )Ng      à¿rm   )rK   rL   r   r   rp   r)   r   Úscalert   rw   r   Úq_projÚk_projÚv_projÚo_proj©r9   rd   rP   s     €r:   rL   zNemotronHAttention.__init__å   sG  ø€ Ü‰ÑÔØ×+Ñ+ˆÔØ×1Ñ1ˆŒð �}‰}Ð(ð �MŠMà×"Ñ" d×&>Ñ&>Ñ>ð 	Œð
 $(×#;Ñ#;ˆÔ Ø—]‘] DÑ(ˆŒ
ä—i‘iØ×Ñ˜dŸn™n¨t¯}©}Ñ<À4×CVÑCVô
ˆŒô —i‘iØ×ÑØ×$Ñ$ t§}¡}Ñ4Ø×$Ñ$ô
ˆŒô
 —i‘iØ×ÑØ×$Ñ$ t§}¡}Ñ4Ø×$Ñ$ô
ˆŒô
 —i‘iØ�N‰N˜TŸ]™]Ñ*¨D×,<Ñ,<À4×CVÑCVô
ˆ�r<   rQ   r„   rƒ   rS   c                 ól  — |j                   \  }}}| j                  |«      j                  ||| j                  d«      j	                  dddd«      }| j                  |«      j                  ||| j                  d«      j	                  dddd«      }| j                  |«      j                  ||| j                  d«      j	                  dddd«      }	|�|j                  ||	«      \  }}	t        |||	|| j                  |¬«      }
|
j	                  dddd«      j                  ||d«      }
| j                  |
«      S )NrU   r   rf   r	   é   )rƒ   r®   r„   )rX   r¯   rž   rp   Ú	transposer°   r   r±   Úupdate_and_fetchr   r®   r²   )r9   rQ   r„   rƒ   r™   ÚLr~   ÚqueriesÚkeysÚvaluesÚoutputs              r:   r^   zNemotronHAttention.__call__  s&  € ð —'‘'‰ˆˆ1ˆaà—+‘+˜a“.×(Ñ(¨¨A¨t¯~©~¸rÓB×LÑLÈQÐPQÐSTÐVWÓXˆà�K‰K˜‹Nß‰W�Q˜˜4×3Ñ3°RÓ8ß‰Y�q˜!˜Q Ó"ð 	ð �K‰K˜‹Nß‰W�Q˜˜4×3Ñ3°RÓ8ß‰Y�q˜!˜Q Ó"ð 	ð ÐØ ×1Ñ1°$¸Ó?‰LˆD�&ä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r<   r�   )r=   r>   r?   r   rL   rM   r_   r   r   r^   r`   ra   s   @r:   r¬   r¬   ä   sY   ø„ ð
˜Yõ 
ð@ $(Ø#'ñ	#à�8‰8ð#ð �r—x‘xÑ ð#ð ˜Ñ ð	#ð
 
�‰÷#r<   r¬   c                   ó,   ‡ — e Zd Zddefˆ fd„Zd„ Zˆ xZS )ÚNemotronHMLPrd   c                 ó  •— t         ‰| �  «        |xs |j                  }t        j                  |j
                  ||j                  ¬«      | _        t        j                  ||j
                  |j                  ¬«      | _        y )Nrm   )	rK   rL   r   rt   rw   r   r$   Úup_projÚ	down_proj)r9   rd   r   rP   s      €r:   rL   zNemotronHMLP.__init__!  sd   ø€ Ü‰ÑÔØ-ÒG°×1GÑ1GÐä—y‘yØ×ÑÐ/°d·m±mô
ˆŒô Ÿ™Ø˜t×/Ñ/°d·m±mô
ˆ�r<   c                 óh   — | j                  t        j                  | j                  |«      «      «      S r7   )rÁ   rt   Úrelu2rÀ   ©r9   rQ   s     r:   r^   zNemotronHMLP.__call__,  s"   € Ø�~‰~œbŸh™h t§|¡|°A£Ó7Ó8Ð8r<   r7   ©r=   r>   r?   r   rL   r^   r`   ra   s   @r:   r¾   r¾      s   ø„ ñ	
˜Yõ 	
ö9r<   r¾   c                 óÔ  — t        j                  | j                  t         j                  «      «      x}}||z   }|dkD  rÃt        j                  |d|df¬«      }t        j
                  |dd¬«      j                  dd¬«      }	||z
  }
t        j                  |	|
dz
  d¬	«      d
d |
…d d …f   }t        j                  |t        j                  |«      t        j                  d«      d¬«      }t        j                  |dd«      }|}
t        j                  | |
dz
  d¬	«      d
d |
…f   }t        j                  ||d¬«      }|dkD  r|r|j                  dd¬«      }||dz   z  }||z  }||fS )Nr	   rU   rV   rf   r‡   T)rW   ÚkeepdimsrY   )ÚkthrW   .g        g#B’¡œÇ;)rM   ÚsigmoidrŸ   r|   rZ   ÚtopkÚsumÚargpartitionÚput_along_axisÚstop_gradientr_   r]   r�   )ÚgatesÚe_score_correction_biasÚtop_kr,   r/   r2   r1   Úorig_scoresÚscoresÚgroup_scoresÚkÚ	group_idxÚindsÚdenominators                 r:   Úgroup_expert_selectrÙ   0  sZ  € ô Ÿ:™: e§l¡l´2·:±:Ó&>Ó?Ð?€K�&ØÐ-Ñ-€FØ�‚{Ü—‘˜f¨2°g¸r°]ÔCˆÜ—w‘w˜v q¨rÔ2×6Ñ6¸BÈÐ6ÓNˆØ�jÑ ˆÜ—O‘O L°a¸!±eÀ"ÔEÀcÈ2ÈAÈ2ÊqÀjÑQˆ	Ü×"Ñ"Ø”B×$Ñ$ YÓ/´·±¸#³ÀRô
ˆô —‘˜F B¨Ó+ˆà€AÜ�?‰?˜F˜7¨¨A©°BÔ7¸¸R¸a¸R¸Ñ@€DÜ×Ñ ¨T¸Ô;€FØˆq‚y‘^Ø—j‘j b°4�jÓ8ˆØ˜;¨Ñ.Ñ/ˆØÐ+Ñ+€Fà�ˆ<Ðr<   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚMoEGateÚconfigc                 ó¦  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  | j                  |j                  f«      | _        t        j                  | j                  f«      | _        y r7   )rK   rL   rÜ   r0   rÑ   r1   r-   r2   r,   r/   rM   r‰   r   rO   rÐ   )r9   rÜ   rP   s     €r:   rL   zMoEGate.__init__S  sŸ   ø€ Ü‰ÑÔØˆŒØ×/Ñ/ˆŒ
Ø$×3Ñ3ˆÔØ &× 7Ñ 7ˆÔØ%+×%AÑ%AˆÔ"Ø—~‘~ˆŒØ ×+Ñ+ˆŒÜ—h‘h × 5Ñ 5°v×7IÑ7IÐJÓKˆŒÜ')§x¡x°×1FÑ1FÐ0HÓ'IˆÕ$r<   c           	      óÊ   — t        || j                  j                  z  | j                  | j                  | j
                  | j                  | j                  | j                  «      S r7   )	rÙ   rO   ÚTrÐ   rÑ   r,   r/   r2   r1   rÄ   s     r:   r^   zMoEGate.__call___  sO   € Ü"Ø�—‘—‘ÑØ×(Ñ(Ø�J‰JØ�L‰LØ�O‰OØ×&Ñ&Ø×Ñó
ð 	
r<   rÅ   ra   s   @r:   rÛ   rÛ   R  s   ø„ ð
J˜yõ 
Jö	
r<   rÛ   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚNemotronHMoErÜ   c                 óP  •— t         ‰| �  «        || _        |j                  | _        t	        |j
                  |j                  |j                  t        j                  «       ¬«      | _
        t        |«      | _        |j                  �|j                  }t        ||¬«      | _        y y )N)Ú
activation)r   )rK   rL   rÜ   r0   r   r   r*   r-   rt   ÚReLU2Ú
switch_mlprÛ   rR   r.   r+   r¾   Úshared_experts)r9   rÜ   r   rP   s      €r:   rL   zNemotronHMoE.__init__l  s�   ø€ Ü‰ÑÔØˆŒØ#)×#=Ñ#=ˆÔ Ü#Ø×ÑØ×(Ñ(Ø×#Ñ#Ü—x‘x“zô	
ˆŒô ˜F“OˆŒ	Ø×"Ñ"Ð.Ø &× JÑ JÐÜ".ØÐ*;ô#ˆDÕð /r<   c                 ó  — | j                  |«      \  }}| j                  ||«      }||d   z  j                  d¬«      j                  |j                  «      }| j
                  j                  �|| j                  |«      z   }|S )Nr†   rY   r‡   )rR   rå   rË   rŸ   ro   rÜ   r.   ræ   )r9   rQ   r×   rÓ   r¤   s        r:   r^   zNemotronHMoE.__call__~  sy   € Ø—y‘y “|‰ˆˆfØ�O‰O˜A˜tÓ$ˆØ�˜	Ñ"Ñ"×'Ñ'¨RÐ'Ó0×7Ñ7¸¿¹Ó@ˆØ�;‰;×'Ñ'Ð3Ø�D×'Ñ'¨Ó*Ñ*ˆAàˆr<   rÅ   ra   s   @r:   rá   rá   k  s   ø„ ð˜yõ ö$r<   rá   c                   ó^   ‡ — e Zd Zdedefˆ fd„Z	 	 ddeej                     dee	   fd„Z
ˆ xZS )ÚNemotronHBlockrd   Ú
block_typec                 ó�  •— t         ‰| �  «        t        j                  |j                  |j
                  ¬«      | _        || _        | j                  dk(  rt        |«      | _	        y | j                  dk(  rt        |«      | _	        y | j                  dk(  rt        |«      | _	        y | j                  dk(  rt        |«      | _	        y y )N©rH   ÚMÚ*Ú-ÚE)rK   rL   rt   ÚRMSNormr   r%   r   rê   rc   Úmixerr¬   r¾   rá   )r9   rd   rê   rP   s      €r:   rL   zNemotronHBlock.__init__‰  s˜   ø€ Ü‰ÑÔÜ—J‘J˜t×/Ñ/°T×5LÑ5LÔMˆŒ	à$ˆŒà�?‰?˜cÒ!Ü-¨dÓ3ˆD�JØ�_‰_ Ò#Ü+¨DÓ1ˆD�JØ�_‰_ Ò#Ü% dÓ+ˆD�JØ�_‰_ Ò#Ü% dÓ+ˆD�Jð $r<   r„   rƒ   c                 ó¾   — | j                  |«      }| j                  dk(  s| j                  dk(  r| j                  |||¬«      }||z   S | j                  |«      }||z   S )Nrí   rî   ©r„   rƒ   )r   rê   rò   )r9   rQ   r„   rƒ   r˜   s        r:   r^   zNemotronHBlock.__call__˜  sd   € ð Ÿ	™	 !›ˆØ�?‰?˜cÒ! T§_¡_¸Ò%;Ø ŸJ™J }¸4Àu˜JÓMˆMð �=Ñ Ð ð !ŸJ™J }Ó5ˆMà�=Ñ Ð r<   r�   )r=   r>   r?   r   r@   rL   r   rM   r_   r   r^   r`   ra   s   @r:   ré   ré   ˆ  sD   ø„ ð,˜Yð ,°Cõ ,ð$ $(Ø#ñ	!ð �r—x‘xÑ ð!ð ˜‰}÷	!r<   ré   c                   ó:   ‡ — e Zd Zdefˆ fd„Z	 ddee   fd„Zˆ xZS )ÚNemotronHModelrd   c                 ó$  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        |j                  D �cg c]  }t        ||«      ‘Œ c}| _	        t        j                  |j
                  |j                  ¬«      | _        d| _        d| _        |j                  D ]$  }|dk(  r n|dk(  sŒ| xj                  dz  c_        Œ& |j                  D ]$  }|dk(  r| xj                  dz  c_        Œ|dk(  sŒ$ y  y c c}w )Nrì   r   rî   rí   r	   )rK   rL   rt   Ú	Embeddingr   r   Ú
embeddingsr(   ré   Úlayersrñ   r%   Únorm_fÚfa_idxÚssm_idx)r9   rd   rê   ÚbrP   s       €r:   rL   zNemotronHModel.__init__¨  së   ø€ Ü‰ÑÔÜŸ,™, t§¡¸×8HÑ8HÓIˆŒð #×:Ò:ó
á:�
ô ˜4 Õ,Ø:ñ
ˆŒô —j‘j ×!1Ñ!1°t×7NÑ7NÔOˆŒØˆŒØˆŒØ×-Ô-ˆAØ�CŠxÙØ�c“Ø—’˜qÑ –ð	 .ð
 ×-Ô-ˆAØ�CŠxØ—’ Ñ!–Ø�c“Ùñ	 .ùò
s   ÁDrƒ   c                 óš  — | j                  |«      }|€d gt        | j                  «      z  }t        ||| j                     «      }t        ||| j                     «      }d}| j                  D ]L  }|j                  dk(  s|j                  dk(  r||   }|dz  }nd }|j                  dk(  r|}	n|}	 |||	|¬«      }ŒN | j                  |«      S )Nr   rí   rî   r	   rô   )	rù   Úlenrú   r   rü   r   rý   rê   rû   )
r9   Úinputsrƒ   r˜   Ú	attn_maskÚssm_maskÚcache_counterÚlayerÚcr„   s
             r:   r^   zNemotronHModel.__call__½  sÔ   € ð
 Ÿ™¨Ó/ˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEÜ)¨-¸¸t¿{¹{Ñ9KÓLˆ	Ü" =°%¸¿¹Ñ2EÓFˆàˆØ—[”[ˆEØ×Ñ 3Ò&¨%×*:Ñ*:¸cÒ*AØ˜-Ñ(�Ø Ñ"‘à�à×Ñ 3Ò&Ø ‘à�Ù! -°dÀ!ÔD‰Mð !ð �{‰{˜=Ó)Ð)r<   r7   )	r=   r>   r?   r   rL   r   r   r^   r`   ra   s   @r:   rö   rö   §  s'   ø„ ð˜Yõ ð0  $ñ*ð ˜‰}÷*r<   rö   c                   ó~   ‡ — e Zd Zdefˆ fd„Z	 d
dej                  dee   fd„Z	e
d„ «       Zd„ Zd„ Ze
d	„ «       Zˆ xZS )ÚModelrd   c                 óÔ   •— t         ‰| �  «        || _        t        |«      | _        t        j                  |j                  |j                  d¬«      | _	        |j                  | _
        y )NFrm   )rK   rL   rd   rö   Úbackbonert   rw   r   r   Úlm_headr   r³   s     €r:   rL   zModel.__init__Û  sJ   ø€ Ü‰ÑÔØˆŒ	Ü& tÓ,ˆŒÜ—y‘y ×!1Ñ!1°4·?±?ÈÔOˆŒØŸ/™/ˆ�r<   r  rƒ   c                 óJ   — | j                  ||¬«      }| j                  |«      S )N)rƒ   )r
  r  )r9   r  rƒ   Úouts       r:   r^   zModel.__call__â  s%   € ð
 �m‰m˜F¨%ˆmÓ0ˆØ�|‰|˜CÓ Ð r<   c                 ó.   — | j                   j                  S r7   )r
  rú   r8   s    r:   rú   zModel.layersê  s   € à�}‰}×#Ñ#Ð#r<   c                 óÔ   — g }| j                   D ]V  }|j                  dk(  r|j                  t        d¬«      «       Œ.|j                  dk(  sŒ>|j                  t	        «       «       ŒX |S )Nrí   rf   )Úsizerî   )rú   rê   Úappendr   r   )r9   ÚcachesÚls      r:   Ú
make_cachezModel.make_cacheî  sS   € ØˆØ—”ˆAØ�|‰|˜sÒ"Ø—‘œk¨qÔ1Õ2Ø—‘ Ó$Ø—‘œg›iÕ(ð	 ð
 ˆr<   c                 óÜ  — |j                  «       D ]2  \  }}d|v sŒ|j                  d   dk7  sŒ|j                  dd«      ||<   Œ4 t        | j                  j
                  «      D ]  }d|› d�}dD ]r  \  }}|› d|› d	�|v sŒt        | j                  j                  «      D �cg c]  }|j                  |› d
|› d|› d	�«      ‘Œ }	}t        j                  |	«      ||› d|› d	�<   Œt Œ� |S c c}w )Nzconv1d.weightrU   r	   rf   zbackbone.layers.z.mixer))rÁ   Úfc2)rÀ   Úfc1z.experts.0.z.weightz	.experts.Ú.z.switch_mlp.)
ÚitemsrX   ÚmoveaxisÚrangerd   r   r-   ÚpoprM   Ústack)
r9   ÚweightsrÕ   Úvr  ÚprefixÚmÚnÚeÚto_joins
             r:   ÚsanitizezModel.sanitize÷  s  € Ø—M‘M–O‰DˆAˆqØ !Ò#¨¯©°©°qÓ(8ØŸZ™Z¨¨1Ó-�˜’
ð $ô
 �t—y‘y×2Ñ2Ö3ˆAØ'¨ s¨&Ð1ˆFÛB‘��1Ø�X˜[¨¨¨7Ð3°wÒ>ô "' t§y¡y×'AÑ'AÔ!Bóá!B˜Að  Ÿ™ v h¨i¸°s¸!¸A¸3¸gÐ$FÕGØ!Bð ð ô BDÇÁÈ'ÓAR�G˜v˜h l°1°#°WÐ=Ò>ñ Cð 4ð ˆùòs   Â"!C)c                 ó   — d„ }|S )Nc                 ó   — d| vxr d| vS )NrÐ   r}   rE   )rÕ   s    r:   Ú	predicatez'Model.cast_predicate.<locals>.predicate  s   € Ø,°AÐ5ÒJ¸'ÈÐ:JÐJr<   rE   )r9   r(  s     r:   Úcast_predicatezModel.cast_predicate	  s   € ò	Kð Ðr<   r7   )r=   r>   r?   r   rL   rM   r_   r   r   r^   Úpropertyrú   r  r%  r)  r`   ra   s   @r:   r  r  Ú  se   ø„ ð*˜Yõ *ð  $ñ!à—‘ð!ð ˜‰}ó!ð ñ$ó ð$òòð$ ñó ôr<   r  ))Údataclassesr   Ú	functoolsr   Útypingr   r   r   r   Úmlx.coreÚcorerM   Úmlx.nnrt   Úactivationsr
   Úbaser   r   r   r   rƒ   r   r   Ússmr   Úswitch_layersr   r   ÚModulerG   rc   r¬   r¾   ÚcompilerÙ   rÛ   rá   ré   rö   r  rE   r<   r:   Ú<module>r7     s   ðõ "Ý ß -Ó -å Ý å ÷ó ÷ (Ý Ý $ñ ƒô)L�ó )Ló ð)LôX+˜Ÿ	™	ô +ôO ˜2Ÿ9™9ô O ôd9#˜Ÿ™ô 9#ôx9�2—9‘9ô 9ð  ‡�ñó ðôB
ˆb�i‰iô 
ô2�2—9‘9ô ô:!�R—Y‘Yô !ô>0*�R—Y‘Yô 0*ôf4ˆB�I‰Iõ 4r<   