+
    TV-jD  ã                   ó~  € ^ RI Ht ^ RIHtHtHtHt ^ RIHt	 ^ RI
Ht ^RIHt ^RIHtHtHt ] ! R R]4      4       t ! R R	]P&                  4      t ! R
 R]P&                  4      t ! R R]P&                  4      t ! R R]P&                  4      t ! R R]P&                  4      t ! R R]P&                  4      tR# )é    )Ú	dataclass)ÚAnyÚDictÚOptionalÚUnionN)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attentionc                   óR   a € ] tR t^t o RtRtRtRtRtRt	Rt
RtR tV 3R ltRtV tR# )Ú	ModelArgsFi €  Né'  c                óX  a € S P                   f   S P                  S n         S P                  '       dx   RR0p\        ;QJ d    V 3R lV 4       F  '       d   K   RM	  RM! V 3R lV 4       4      '       g   \	        RV 24      hS P                  R,          R9  d   \	        R4      hR # R # )	NÚfactorÚ	rope_typec              3   ó@   <"  € T F  qSP                   9   x € K  	  R # 5i©N)Úrope_scaling)Ú.0ÚkeyÚselfs   & €Úh/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/internlm3.pyÚ	<genexpr>Ú*ModelArgs.__post_init__.<locals>.<genexpr>%   s   øé € ÐI¹=°C˜d×/Ñ/Ö/»=ùs   ƒFTzrope_scaling must contain keys zErope_scaling 'rope_type' currently only supports 'linear' or 'dynamic)ÚlinearÚdynamic)Únum_key_value_headsÚnum_attention_headsr   ÚallÚ
ValueError)r   Úrequired_keyss   f r   Ú__post_init__ÚModelArgs.__post_init__   s–   ø€ Ø×#Ñ#Ò+Ø'+×'?Ñ'?ˆDÔ$à××ÐØ% {Ð3ˆMß“3ÔI¹=ÓI—3—3’3ÔI¹=ÓI×IÒIÜ Ð#BÀ=À/Ð!RÓSÐSà× Ñ  Õ-Ð5JÔJÜ Ø[óð ñ Kñ ó    c                ó
  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[S[S[ S[S[S[ 3,          3,          ,          ;R&   S[;R&   # )é   Ú
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizer   Úrms_norm_epsÚ
vocab_sizeÚbiasÚqkv_biasÚmax_position_embeddingsr   Ú
rope_thetaÚrope_traditionalr   Útie_word_embeddings)ÚstrÚintÚfloatÚboolr   r   r   )ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚModelArgs.__annotate__   sÜ   ø‡ ‚ á�Oñ ñ Ññ ñ Ññ	 ñ
 Ññ ñ Ññ ñ Ññ ñ �Oñ ñ Ññ ñ Ññ ñ !Ñ(ñ ñ Ñ#ñ ñ Ññ ñ Ñ"ñ ñ ™4¡¡U©5±#¨:Õ%6Ð 6Õ7Õ8Ñ?ñ ñ  Ñ%ò! r$   )r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r-   r.   r/   r   r0   r1   r   r2   r"   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r8   s   @r   r   r      s?   ø‡ € ð €DØ€HØ#(ÐØ#ÐØ€JØ"ÐØ;?€LØ %Ðò÷% ƒ r$   r   c                   ób   a a€ ] tR t^.t oRtRV3R lV 3R llltR tR	V3R lR lltRtVt	V ;t
# )
ÚDynamicNTKScalingRoPEzCImplements the rotary positional encoding with Dynamic NTK scaling.c          
      ó8   <€ V ^8„  d   QhRS[ RS[ RS[RS[RS[/# )r&   Údimsr/   ÚtraditionalÚbaseÚscale)r4   r6   r5   )r7   r8   s   "€r   r9   Ú"DynamicNTKScalingRoPE.__annotate__1   s=   ø€ ÷ ñ áðñ "%ðñ ð	ñ
 ðñ ñr$   c                ó`   <€ \         SV `  4        W n        W@n        Wn        W0n        WPn        R # r   )ÚsuperÚ__init__r/   Úoriginal_baserE   rF   rH   )r   rE   r/   rF   rG   rH   Ú	__class__s   &&&&&&€r   rL   ÚDynamicNTKScalingRoPE.__init__1   s,   ø€ ô 	‰ÑÔØ'>Ô$Ø!ÔØŒ	Ø&ÔØŽ
r$   c                ól   € V P                    R V P                   RV P                   RV P                   2# )z, traditional=z, max_position_embeddings=z, scaling_factor=)rE   rF   r/   Úscaling_factor©r   s   &r   Ú
extra_reprÚ DynamicNTKScalingRoPE.extra_repr@   sV   € Ø—)‘)�˜N¨4×+;Ñ+;Ð*<Ð<VÐW[×WsÑWsÐVtð  uFð  GK÷  GZñ  GZð  F[ð  \ð  	\r$   c                ó    <€ V ^8„  d   QhRS[ /# )r&   Úoffset)r4   )r7   r8   s   "€r   r9   rI   C   s   ø€ ÷ 
ñ 
¡#ñ 
r$   c           	     óâ  € VP                   ^,          V,           pW0P                  8”  dx   V P                  V P                  V,          V P                  ,          V P                  ^,
          ,
          V P                  V P                  ^,
          ,          ,          ,          pMV P                  p\
        P                  P                  VV P                  V P                  VV P                  VR7      # )é   )rF   rG   rH   rV   )	Úshaper/   rM   rH   rE   ÚmxÚfastÚroperF   )r   ÚxrV   Úseq_lenrG   s   &&&  r   Ú__call__ÚDynamicNTKScalingRoPE.__call__C   sµ   € Ø—'‘'˜!•*˜vÕ%ˆØ×1Ñ1Ô1Ø×%Ñ%Ø—‘˜gÕ%¨×(DÑ(DÕDÈÏÉÐVWÍÕXØ—)‘)˜tŸy™y¨1�}Õ-õ)/õ /‰Dð ×%Ñ%ˆDä�w‰w�|‰|ØØ�I‰IØ×(Ñ(ØØ—*‘*Øð ó 
ð 	
r$   )rE   r/   rM   rH   rF   )i   Fr   g      ð?)r   )r;   r<   r=   r>   Ú__doc__rL   rS   r_   r@   rA   Ú__classcell__©rN   r8   s   @@r   rC   rC   .   s'   ù‡ € ÙM÷õ ò\÷
÷ 
ò 
r$   rC   c                   óT   a a€ ] tR t^Vt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó    <€ V ^8„  d   QhRS[ /# ©r&   Úargs©r   )r7   r8   s   "€r   r9   ÚAttention.__annotate__W   s   ø€ ÷ 
ñ 
™Yñ 
r$   c                ó&  <€ \         SV `  4        VP                  pVP                  pVP                  ;V n        pVP                  ;V n        pWAP                  ,          V n        VP                  V,          ;V n	        pVR,          V n
        \        P                  ! W$V,          VR7      V n        \        P                  ! W%V,          VR7      V n        \        P                  ! W%V,          VR7      V n        \        P                  ! WF,          W#R7      V n        VP"                  e3   VP"                  R,          R8X  d   ^VP"                  R,          ,          MRp\%        VVP&                  VP(                  VP*                  VR7      V n        R# )	g      à?©r-   Nr   r   r   g       @)r/   rF   rG   rH   g      à¿)rK   rL   r(   r.   r   Ún_headsr   Ú
n_kv_headsÚn_kv_groupsÚhead_dimrH   ÚnnÚLinearÚq_projÚk_projÚv_projÚo_projr   rC   r/   r1   r0   r\   )	r   rh   Údimr.   rm   rn   rp   Ú
rope_scalerN   s	   &&      €r   rL   ÚAttention.__init__W   s@  ø€ Ü‰ÑÔà×ÑˆØ—=‘=ˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*Ø"×&>Ñ&>Õ>ˆÔà#'×#3Ñ#3°wÕ#>Ð>ˆŒ˜Ø˜t•^ˆŒ
ä—i’i ¨xÕ%7¸hÔGˆŒÜ—i’i °(Õ%:ÀÔJˆŒÜ—i’i °(Õ%:ÀÔJˆŒÜ—i’i Õ 2°CÔGˆŒð × Ñ Ò,Ø×!Ñ! +Õ.°(Ô:ð �×!Ñ! (Õ+Ö+ð ð	 	ô *ØØ$(×$@Ñ$@Ø×-Ñ-Ø—‘Øô
ˆŽ	r$   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r&   r]   ÚmaskÚcacheÚreturn©rZ   Úarrayr   r   )r7   r8   s   "€r   r9   rj   w   sH   ø€ ÷ #ñ #á�8‰8ð#ñ ‘r—x‘xÕ ð#ñ ™�}ð	#ñ
 
�‰ñ#r$   c           	     ó  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pVP	                  WEV P
                  R4      P                  ^ ^^^4      pVP	                  WEV P                  R4      P                  ^ ^^^4      pV	P	                  WEV P                  R4      P                  ^ ^^^4      p	VeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      P	                  WER4      p
V P                  V
4      # )rX   )rV   )r}   rH   r|   éÿÿÿÿ)rY   rs   rt   ru   Úreshaperm   Ú	transposern   r\   rV   Úupdate_and_fetchr   rH   rv   )r   r]   r|   r}   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r   r_   ÚAttention.__call__w   sS  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆð —/‘/ !¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r$   )
rp   rt   rm   ro   rn   rv   rs   r\   rH   ru   ©NN©	r;   r<   r=   r>   rL   r_   r@   rA   rb   rc   s   @@r   re   re   V   s   ù‡ € ÷
ó 
÷@#÷ #ò #r$   re   c                   óD   a a€ ] tR t^–t oV 3R ltV3R lR ltRtVtV ;t# )ÚMLPc                óÒ   <€ \         SV `  4        \        P                  ! WVR 7      V n        \        P                  ! W!VR 7      V n        \        P                  ! WVR 7      V n        R# )rl   N)rK   rL   rq   rr   Ú	gate_projÚ	down_projÚup_proj)r   rw   Ú
hidden_dimr-   rN   s   &&&&€r   rL   ÚMLP.__init__—   sD   ø€ Ü‰ÑÔÜŸš 3¸Ô>ˆŒÜŸš :¸Ô>ˆŒÜ—y’y °tÔ<ˆŽr$   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r&   r~   ©rZ   r€   )r7   r8   s   "€r   r9   ÚMLP.__annotate__�   s   ø€ ÷ Jñ J™RŸX™Xñ Jr$   c                ót   € V P                  \        V P                  V4      V P                  V4      4      4      # r   )r”   r   r“   r•   )r   r]   s   &&r   r_   ÚMLP.__call__�   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr$   )r”   r“   r•   r�   rc   s   @@r   r‘   r‘   –   s   ù‡ € õ=÷J÷ Jð Jr$   r‘   c                   óT   a a€ ] tR t^¡t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚTransformerBlockc                ó    <€ V ^8„  d   QhRS[ /# rg   ri   )r7   r8   s   "€r   r9   ÚTransformerBlock.__annotate__¢   s   ø€ ÷ 
ñ 
™Yñ 
r$   c                óh  <€ \         SV `  4        \        V4      V n        \	        VP
                  VP                  VP                  4      V n        \        P                  ! VP
                  VP                  R 7      V n        \        P                  ! VP
                  VP                  R 7      V n        R# )©ÚepsN)rK   rL   re   Ú	self_attnr‘   r(   r*   r-   Úmlprq   ÚRMSNormr+   Úinput_layernormÚpost_attention_layernorm©r   rh   rN   s   &&€r   rL   ÚTransformerBlock.__init__¢   sz   ø€ Ü‰ÑÔÜ" 4›ˆŒÜ�t×'Ñ'¨×)?Ñ)?ÀÇÁÓKˆŒÜ!Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
ª
Ø×Ñ $×"3Ñ"3ô)
ˆÖ%r$   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# r{   r   )r7   r8   s   "€r   r9   r    «   sH   ø€ ÷ 
ñ 
á�8‰8ð
ñ ‘r—x‘xÕ ð
ñ ™�}ð	
ñ
 
�‰ñ
r$   c                ó¨   € V P                  V P                  V4      W#4      pW,           pV P                  V P                  V4      4      pWT,           pV# r   )r¤   r§   r¥   r¨   )r   r]   r|   r}   ÚrÚhÚouts   &&&&   r   r_   ÚTransformerBlock.__call__«   sK   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ�EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�eˆØˆ
r$   )r§   r¥   r¨   r¤   rŽ   r�   rc   s   @@r   rž   rž   ¡   s   ù‡ € ÷
ó 
÷
÷ 
ò 
r$   rž   c                   óT   a a€ ] tR t^¸t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚInternLM2Modelc                ó    <€ V ^8„  d   QhRS[ /# rg   ri   )r7   r8   s   "€r   r9   ÚInternLM2Model.__annotate__¹   s   ø€ ÷ Hñ H™Yñ Hr$   c                ó~  <€ \         SV `  4        VP                  ^ 8”  g   Q h\        P                  ! VP                  VP
                  4      V n        \        VP                  4       Uu. uF  p\        VR7      NK  	  upV n
        \        P                  ! VP
                  VP                  R7      V n        R# u upi )r   )rh   r¢   N)rK   rL   r,   rq   Ú	Embeddingr(   Úembed_tokensÚranger)   rž   Úlayersr¦   r+   Únorm)r   rh   Ú_rN   s   && €r   rL   ÚInternLM2Model.__init__¹   s�   ø€ Ü‰ÑÔØ�‰ Ô"Ð"Ð"ÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔä16°t×7MÑ7MÔ1Nó
Ù1N¨AÔ $×'Ñ1Nñ
ˆŒô —J’J˜t×/Ñ/°T×5FÑ5FÔGˆŽ	ùò
s   Á*B:c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r&   Úinputsr™   )r7   r8   s   "€r   r9   r´   Â   s   ø€ ÷ ñ á—‘ñr$   c                óþ   € V P                  V4      pVf   R .\        V P                  4      ,          p\        W2^ ,          4      p\	        V P                  V4       F  w  rVV! W4VR7      pK  	  V P                  V4      # )N)r}   )r·   Úlenr¹   r
   Úziprº   )r   r¿   r}   r®   r|   ÚlayerÚcs   &&&    r   r_   ÚInternLM2Model.__call__Â   so   € ð
 ×Ñ˜fÓ%ˆàŠ=Ø�FœS §¡Ó-Õ-ˆEä$ Q¨a­Ó1ˆÜ˜DŸK™K¨Ö/‰HˆEÙ�a QÔ'ŠAñ 0ð �y‰y˜‹|Ðr$   )r·   r¹   rº   r   r�   rc   s   @@r   r²   r²   ¸   s    ù‡ € ÷Hó H÷÷ ò r$   r²   c                   ój   a a€ ] tR t^Ót oV3R lV 3R lltRV3R lR lltR t]R 4       tRt	Vt
V ;t# )	ÚModelc                ó    <€ V ^8„  d   QhRS[ /# rg   ri   )r7   r8   s   "€r   r9   ÚModel.__annotate__Ô   s   ø€ ÷ Tñ T™Yñ Tr$   c                óþ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        VP                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# R# )Frl   N)rK   rL   rh   r'   r²   Úmodelr2   rq   rr   r(   r,   Úlm_headr©   s   &&€r   rL   ÚModel.__init__Ô   sY   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ# DÓ)ˆŒ
Ø×'×'Ð'ÜŸ9š9 T×%5Ñ%5°t·±ÈUÔSˆDŽLñ (r$   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r¾   r™   )r7   r8   s   "€r   r9   rÉ   Ü   s   ø€ ÷ 
ñ 
á—‘ñ
r$   c                óÐ   € V P                  W4      pV P                  P                  '       d(   V P                   P                  P	                  V4      pV# V P                  V4      pV# r   )rË   rh   r2   r·   Ú	as_linearrÌ   )r   r¿   r}   r¯   s   &&& r   r_   ÚModel.__call__Ü   sX   € ð
 �j‰j˜Ó'ˆØ�9‰9×(×(Ð(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r$   c                óh   € VP                  4        UUu/ uF  w  r#R V9  g   K  W#bK  	  upp# u uppi )zattention.rope.inv_freq)Úitems)r   ÚweightsÚkÚvs   &&  r   ÚsanitizeÚModel.sanitizeè   s/   € à!(§¡¤ÔW¡™˜Ð4MÐUVÑ4V”�’¡ÒWÐWùÓWs   ”.¤.c                ó.   € V P                   P                  # r   )rË   r¹   rR   s   &r   r¹   ÚModel.layersì   s   € à�z‰z× Ñ Ð r$   )rh   rÌ   rË   r'   r   )r;   r<   r=   r>   rL   r_   r×   Úpropertyr¹   r@   rA   rb   rc   s   @@r   rÇ   rÇ   Ó   s:   ù‡ € ÷Tó T÷
ò 
òXð ñ!ó ÷!ð !r$   rÇ   )Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorerZ   Úmlx.nnrq   Úactivationsr   rG   r	   r
   r   r   ÚModulerC   re   r‘   rž   r²   rÇ   © r$   r   Ú<module>rä      sŸ   ðõ "ß -Ó -å Ý å ß TÑ Tð ô�ó ó ðô@%
˜BŸI™Iô %
ôP=#�—	‘	ô =#ô@Jˆ"�)‰)ô Jô�r—y‘yô ô.�R—Y‘Yô ô6!ˆB�I‰Iö !r$   