Ë
    îÿæiD  ã                   ó|  — d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ e G d„ de«      «       Z G d	„ d
ej&                  «      Z G d„ dej&                  «      Z G d„ dej&                  «      Z G d„ dej&                  «      Z G d„ dej&                  «      Z G d„ dej&                  «      Zy)é    )Ú	dataclass)ÚAnyÚDictÚOptionalÚUnionNé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attentionc                   óä   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZeed	<   dZ	eed
<   dZ
eed<   dZeed<   dZeed<   dZeed<   dZeeeeeef   f      ed<   dZeed<   d„ Zy)Ú	ModelArgsÚ
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚrms_norm_epsÚ
vocab_sizeFÚbiasÚqkv_biasi €  Úmax_position_embeddingsNÚnum_key_value_headsé'  Ú
rope_thetaÚrope_traditionalÚrope_scalingÚtie_word_embeddingsc                 óÞ   ‡ — ‰ j                   €‰ j                  ‰ _         ‰ j                  rCddh}t        ˆ fd„|D «       «      st	        d|› �«      ‚‰ j                  d   dvrt	        d«      ‚y y )NÚfactorÚ	rope_typec              3   ó:   •K  — | ]  }|‰j                   v –— Œ y ­w©N)r   )Ú.0ÚkeyÚselfs     €úl/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/internlm3.pyÚ	<genexpr>z*ModelArgs.__post_init__.<locals>.<genexpr>%   s   øè ø€ ÐI¹=°C�s˜d×/Ñ/Ô/¹=ùs   ƒzrope_scaling must contain keys )ÚlinearÚdynamiczErope_scaling 'rope_type' currently only supports 'linear' or 'dynamic)r   r   r   ÚallÚ
ValueError)r&   Úrequired_keyss   ` r'   Ú__post_init__zModelArgs.__post_init__   s€   ø€ Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$à×ÒØ% {Ð3ˆMÜÓI¹=ÓIÔIÜ Ð#BÀ=À/Ð!RÓSÐSà× Ñ  Ñ-Ð5JÑJÜ Ø[óð ð Kð ó    )Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatr   Úboolr   r   r   r   r   r   r   r   r   r   r.   © r/   r'   r   r      s›   … àƒOØÓØÓØÓØÓØÓØƒOØ€Dˆ$ÓØ€HˆdÓØ#(Ð˜SÓ(Ø#Ð˜Ó#Ø€J�ÓØ"Ð�dÓ"Ø;?€L�(˜4  U¨5°#¨:Ñ%6Ð 6Ñ7Ñ8Ó?Ø %Ð˜Ó%ór/   r   c                   óV   ‡ — e Zd ZdZ	 	 	 	 ddededededef
ˆ fd„Zd„ Zdd	efd
„Z	ˆ xZ
S )ÚDynamicNTKScalingRoPEzCImplements the rotary positional encoding with Dynamic NTK scaling.Údimsr   ÚtraditionalÚbaseÚscalec                 óh   •— t         ‰| �  «        || _        || _        || _        || _        || _        y r#   )ÚsuperÚ__init__r   Úoriginal_baser;   r<   r>   )r&   r;   r   r<   r=   r>   Ú	__class__s         €r'   rA   zDynamicNTKScalingRoPE.__init__1   s6   ø€ ô 	‰ÑÔØ'>ˆÔ$Ø!ˆÔØˆŒ	Ø&ˆÔØˆ�
r/   c                 ól   — | j                   › d| j                  › d| j                  › d| j                  › �S )Nz, traditional=z, max_position_embeddings=z, scaling_factor=)r;   r<   r   Úscaling_factor©r&   s    r'   Ú
extra_reprz DynamicNTKScalingRoPE.extra_repr@   sV   € Ø—)‘)�˜N¨4×+;Ñ+;Ð*<Ð<VÐW[×WsÑWsÐVtð  uFð  GK÷  GZñ  GZð  F[ð  \ð  	\r/   Úoffsetc                 ó�  — |j                   d   |z   }|| j                  kD  rW| j                  | j                  |z  | j                  z  | j                  dz
  z
  | j                  | j                  dz
  z  z  z  }n| j                  }t
        j                  j                  || j                  | j                  || j                  |¬«      S )Nr   é   )r<   r=   r>   rH   )	Úshaper   rB   r>   r;   ÚmxÚfastÚroper<   )r&   ÚxrH   Úseq_lenr=   s        r'   Ú__call__zDynamicNTKScalingRoPE.__call__C   s·   € Ø—'‘'˜!‘*˜vÑ%ˆØ�T×1Ñ1Ò1Ø×%Ñ%Ø—‘˜gÑ%¨×(DÑ(DÑDÈÏÉÐVWÉÑXØ—)‘)˜tŸy™y¨1™}Ñ-ñ)/ñ /‰Dð ×%Ñ%ˆDä�w‰w�|‰|ØØ�I‰IØ×(Ñ(ØØ—*‘*Øð ó 
ð 	
r/   )i   Fr   g      ð?)r   )r0   r1   r2   Ú__doc__r5   r7   r6   rA   rG   rQ   Ú__classcell__©rC   s   @r'   r:   r:   .   s[   ø„ ÙMð
 (,Ø!ØØñàðð "%ðð ð	ð
 ðð õò\ñ
 #÷ 
r/   r:   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	Ú	AttentionÚargsc                 óÐ  •— t         ‰| �  «        |j                  }|j                  }|j                  x| _        }|j                  x| _        }||j                  z  | _        |j                  |z  x| _	        }|dz  | _
        t        j                  |||z  |¬«      | _        t        j                  |||z  |¬«      | _        t        j                  |||z  |¬«      | _        t        j                  ||z  ||¬«      | _        |j"                  �$|j"                  d   dk(  rd|j"                  d   z  nd}t%        ||j&                  |j(                  |j*                  |¬«      | _        y )	Ng      à¿©r   r!   r)   r   r    g       @)r   r<   r=   r>   )r@   rA   r   r   r   Ún_headsr   Ú
n_kv_headsÚn_kv_groupsÚhead_dimr>   ÚnnÚLinearÚq_projÚk_projÚv_projÚo_projr   r:   r   r   r   rN   )	r&   rW   Údimr   rZ   r[   r]   Ú
rope_scalerC   s	           €r'   rA   zAttention.__init__W   sL  ø€ Ü‰ÑÔà×ÑˆØ—=‘=ˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*Ø" d×&>Ñ&>Ñ>ˆÔà#'×#3Ñ#3°wÑ#>Ð>ˆŒ˜Ø˜t‘^ˆŒ
ä—i‘i  W¨xÑ%7¸hÔGˆŒÜ—i‘i  Z°(Ñ%:ÀÔJˆŒÜ—i‘i  Z°(Ñ%:ÀÔJˆŒÜ—i‘i ¨(Ñ 2°C¸hÔGˆŒð × Ñ Ð,Ø×!Ñ! +Ñ.°(Ò:ð �×!Ñ! (Ñ+Ò+ð ð	 	ô *ØØ$(×$@Ñ$@Ø×-Ñ-Ø—‘Øô
ˆ�	r/   rO   ÚmaskÚcacheÚreturnc                 ó2  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}|j	                  ||| j
                  d«      j                  dddd«      }|j	                  ||| j                  d«      j                  dddd«      }|	j	                  ||| j                  d«      j                  dddd«      }	|�P| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||	«      \  }}	n"| j                  |«      }| j                  |«      }t        |||	|| j                  |¬«      }
|
j                  dddd«      j	                  ||d«      }
| j                  |
«      S )Néÿÿÿÿr   rJ   r   é   )rH   )rg   r>   rf   )rK   r`   ra   rb   ÚreshaperZ   Ú	transposer[   rN   rH   Úupdate_and_fetchr   r>   rc   )r&   rO   rf   rg   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs              r'   rQ   zAttention.__call__w   si  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆð —/‘/ ! Q¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A˜q $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  1 d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r/   ©NN©r0   r1   r2   r   rA   rL   Úarrayr   r   rQ   rS   rT   s   @r'   rV   rV   V   sX   ø„ ð
˜Yõ 
ðF $(Ø#ñ	#à�8‰8ð#ð �r—x‘xÑ ð#ð ˜‰}ð	#ð
 
�‰÷#r/   rV   c                   ó>   ‡ — e Zd Zˆ fd„Zdej
                  fd„Zˆ xZS )ÚMLPc                 óÐ   •— t         ‰| �  «        t        j                  |||¬«      | _        t        j                  |||¬«      | _        t        j                  |||¬«      | _        y )NrY   )r@   rA   r^   r_   Ú	gate_projÚ	down_projÚup_proj)r&   rd   Ú
hidden_dimr   rC   s       €r'   rA   zMLP.__init__—   sJ   ø€ Ü‰ÑÔÜŸ™ 3¨
¸Ô>ˆŒÜŸ™ :¨s¸Ô>ˆŒÜ—y‘y  j°tÔ<ˆ�r/   rh   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S r#   )r}   r	   r|   r~   )r&   rO   s     r'   rQ   zMLP.__call__�   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr/   )r0   r1   r2   rA   rL   rx   rQ   rS   rT   s   @r'   rz   rz   –   s   ø„ ô=ðJ˜RŸX™X÷ Jr/   rz   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚTransformerBlockrW   c                 ób  •— t         ‰| �  «        t        |«      | _        t	        |j
                  |j                  |j                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j
                  |j                  ¬«      | _        y )N©Úeps)r@   rA   rV   Ú	self_attnrz   r   r   r   Úmlpr^   ÚRMSNormr   Úinput_layernormÚpost_attention_layernorm©r&   rW   rC   s     €r'   rA   zTransformerBlock.__init__¢   sz   ø€ Ü‰ÑÔÜ" 4›ˆŒÜ�t×'Ñ'¨×)?Ñ)?ÀÇÁÓKˆŒÜ!Ÿz™z¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
©
Ø×Ñ $×"3Ñ"3ô)
ˆÕ%r/   rO   rf   rg   rh   c                 óž   — | j                  | j                  |«      ||«      }||z   }| j                  | j                  |«      «      }||z   }|S r#   )r†   r‰   r‡   rŠ   )r&   rO   rf   rg   ÚrÚhÚouts          r'   rQ   zTransformerBlock.__call__«   sQ   € ð �N‰N˜4×/Ñ/°Ó2°D¸%Ó@ˆØ�‰EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�!‰eˆØˆ
r/   rv   rw   rT   s   @r'   r‚   r‚   ¡   sW   ø„ ð
˜Yõ 
ð $(Ø#ñ	
à�8‰8ð
ð �r—x‘xÑ ð
ð ˜‰}ð	
ð
 
�‰÷
r/   r‚   c                   óH   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zˆ xZS )ÚInternLM2ModelrW   c                 óp  •— t         ‰| �  «        |j                  dkD  sJ ‚t        j                  |j                  |j
                  «      | _        t        |j                  «      D �cg c]  }t        |¬«      ‘Œ c}| _
        t        j                  |j
                  |j                  ¬«      | _        y c c}w )Nr   )rW   r„   )r@   rA   r   r^   Ú	Embeddingr   Úembed_tokensÚranger   r‚   Úlayersrˆ   r   Únorm)r&   rW   Ú_rC   s      €r'   rA   zInternLM2Model.__init__¹   s�   ø€ Ü‰ÑÔØ�‰ Ò"Ð"Ð"ÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔä16°t×7MÑ7MÔ1Nó
Ù1N¨AÔ $Ö'Ð1Nñ
ˆŒô —J‘J˜t×/Ñ/°T×5FÑ5FÔGˆ�	ùò
s   Á'B3Úinputsc                 óì   — | j                  |«      }|€d gt        | j                  «      z  }t        ||d   «      }t	        | j                  |«      D ]  \  }} ||||¬«      }Œ | j                  |«      S )Nr   )rg   )r”   Úlenr–   r   Úzipr—   )r&   r™   rg   rŽ   rf   ÚlayerÚcs          r'   rQ   zInternLM2Model.__call__Â   su   € ð
 ×Ñ˜fÓ%ˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEä$ Q¨¨a©Ó1ˆÜ˜DŸK™K¨Ö/‰HˆE�1Ù�a˜ QÔ'‰Að 0ð �y‰y˜‹|Ðr/   r#   )	r0   r1   r2   r   rA   rL   rx   rQ   rS   rT   s   @r'   r‘   r‘   ¸   s'   ø„ ðH˜Yõ Hð ñà—‘÷r/   r‘   c                   ó^   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zd„ Ze	d„ «       Z
ˆ xZS )ÚModelrW   c                 óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y )NFrY   )r@   rA   rW   r   r‘   Úmodelr   r^   r_   r   r   Úlm_headr‹   s     €r'   rA   zModel.__init__Ô   sX   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ# DÓ)ˆŒ
Ø×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (r/   r™   c                 óÆ   — | j                  ||«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S r#   )r¢   rW   r   r”   Ú	as_linearr£   )r&   r™   rg   r�   s       r'   rQ   zModel.__call__Ü   sW   € ð
 �j‰j˜ Ó'ˆØ�9‰9×(Ò(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r/   c                 ó`   — |j                  «       D ��ci c]  \  }}d|vsŒ||“Œ c}}S c c}}w )Nzattention.rope.inv_freq)Úitems)r&   ÚweightsÚkÚvs       r'   ÚsanitizezModel.sanitizeè   s1   € à!(§¡¤ÔW¡™˜˜AÐ4MÐUVÒ4V��1‘ ÒWÐWùÓWs   ”*¡*c                 ó.   — | j                   j                  S r#   )r¢   r–   rF   s    r'   r–   zModel.layersì   s   € à�z‰z× Ñ Ð r/   r#   )r0   r1   r2   r   rA   rL   rx   rQ   r«   Úpropertyr–   rS   rT   s   @r'   r    r    Ó   sA   ø„ ðT˜Yõ Tð ñ
à—‘ó
òXð ñ!ó ô!r/   r    )Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorerL   Úmlx.nnr^   Úactivationsr	   r=   r
   r   r   r   ÚModuler:   rV   rz   r‚   r‘   r    r8   r/   r'   Ú<module>rµ      sŸ   ðõ "ß -Ó -å Ý å ß TÑ Tð ô�ó ó ðô@%
˜BŸI™Iô %
ôP=#�—	‘	ô =#ô@Jˆ"�)‰)ô Jô�r—y‘yô ô.�R—Y‘Yô ô6!ˆB�I‰Iõ !r/   