Ë
    îÿæi  ã                   ó|  — d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ e G d„ de«      «       Z G d	„ d
ej&                  «      Z G d„ dej&                  «      Z G d„ dej&                  «      Z G d„ dej&                  «      Z G d„ dej&                  «      Z G d„ dej&                  «      Zy)é    )Ú	dataclass)ÚAnyÚDictÚOptionalÚUnionNé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attentionc                   óÖ   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZeed	<   d
Z	eed<   dZ
eed<   dZeed<   dZeed<   dZeeeeeef   f      ed<   dZeed<   d„ Zy)Ú	ModelArgsÚ
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚrms_norm_epsÚ
vocab_sizeTÚbiasi €  Úmax_position_embeddingsNÚnum_key_value_headsé'  Ú
rope_thetaFÚrope_traditionalÚrope_scalingÚtie_word_embeddingsc                 óÞ   ‡ — ‰ j                   €‰ j                  ‰ _         ‰ j                  rCddh}t        ˆ fd„|D «       «      st	        d|› �«      ‚‰ j                  d   dvrt	        d«      ‚y y )NÚfactorÚtypec              3   ó:   •K  — | ]  }|‰j                   v –— Œ y ­w©N)r   )Ú.0ÚkeyÚselfs     €úl/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/internlm2.pyÚ	<genexpr>z*ModelArgs.__post_init__.<locals>.<genexpr>$   s   øè ø€ ÐI¹=°C�s˜d×/Ñ/Ô/¹=ùs   ƒzrope_scaling must contain keys )ÚlinearÚdynamicz@rope_scaling 'type' currently only supports 'linear' or 'dynamic)r   r   r   ÚallÚ
ValueError)r%   Úrequired_keyss   ` r&   Ú__post_init__zModelArgs.__post_init__   s€   ø€ Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$à×ÒØ% vÐ.ˆMÜÓI¹=ÓIÔIÜ Ð#BÀ=À/Ð!RÓSÐSà× Ñ  Ñ(Ð0EÑEÜ ØVóð ð Fð ó    )Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatr   Úboolr   r   r   r   r   r   r   r   r   r-   © r.   r&   r   r      s‘   … àƒOØÓØÓØÓØÓØÓØƒOØ€Dˆ$ÓØ#(Ð˜SÓ(Ø#Ð˜Ó#Ø€J�ÓØ"Ð�dÓ"Ø;?€L�(˜4  U¨5°#¨:Ñ%6Ð 6Ñ7Ñ8Ó?Ø %Ð˜Ó%ór.   r   c                   óV   ‡ — e Zd ZdZ	 	 	 	 ddededededef
ˆ fd„Zd„ Zdd	efd
„Z	ˆ xZ
S )ÚDynamicNTKScalingRoPEzCImplements the rotary positional encoding with Dynamic NTK scaling.Údimsr   ÚtraditionalÚbaseÚscalec                 óh   •— t         ‰| �  «        || _        || _        || _        || _        || _        y r"   )ÚsuperÚ__init__r   Úoriginal_baser:   r;   r=   )r%   r:   r   r;   r<   r=   Ú	__class__s         €r&   r@   zDynamicNTKScalingRoPE.__init__0   s6   ø€ ô 	‰ÑÔØ'>ˆÔ$Ø!ˆÔØˆŒ	Ø&ˆÔØˆ�
r.   c                 ól   — | j                   › d| j                  › d| j                  › d| j                  › �S )Nz, traditional=z, max_position_embeddings=z, scaling_factor=)r:   r;   r   Úscaling_factor©r%   s    r&   Ú
extra_reprz DynamicNTKScalingRoPE.extra_repr?   sV   € Ø—)‘)�˜N¨4×+;Ñ+;Ð*<Ð<VÐW[×WsÑWsÐVtð  uFð  GK÷  GZñ  GZð  F[ð  \ð  	\r.   Úoffsetc                 ó�  — |j                   d   |z   }|| j                  kD  rW| j                  | j                  |z  | j                  z  | j                  dz
  z
  | j                  | j                  dz
  z  z  z  }n| j                  }t
        j                  j                  || j                  | j                  || j                  |¬«      S )Nr   é   )r;   r<   r=   rG   )	Úshaper   rA   r=   r:   ÚmxÚfastÚroper;   )r%   ÚxrG   Úseq_lenr<   s        r&   Ú__call__zDynamicNTKScalingRoPE.__call__B   s·   € Ø—'‘'˜!‘*˜vÑ%ˆØ�T×1Ñ1Ò1Ø×%Ñ%Ø—‘˜gÑ%¨×(DÑ(DÑDÈÏÉÐVWÉÑXØ—)‘)˜tŸy™y¨1™}Ñ-ñ)/ñ /‰Dð ×%Ñ%ˆDä�w‰w�|‰|ØØ�I‰IØ×(Ñ(ØØ—*‘*Øð ó 
ð 	
r.   )i   Fr   g      ð?)r   )r/   r0   r1   Ú__doc__r4   r6   r5   r@   rF   rP   Ú__classcell__©rB   s   @r&   r9   r9   -   s[   ø„ ÙMð
 (,Ø!ØØñàðð "%ðð ð	ð
 ðð õò\ñ
 #÷ 
r.   r9   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	Ú	AttentionÚargsc                 ól  •— t         ‰| �  «        |j                  }|j                  x| _        }|j
                  x| _        }||j
                  z  | _        |j                  |z  x| _        }|dz  | _	        t        j                  ||d|z  z   |z  |j                  ¬«      | _        t        j                  ||z  ||j                  ¬«      | _        |j                  �$|j                  d   dk(  rd|j                  d   z  nd}t!        ||j"                  |j$                  |j&                  |¬	«      | _        y )
Ng      à¿rI   ©r   r    r(   r   r   g       @)r   r;   r<   r=   )r?   r@   r   r   Ún_headsr   Ú
n_kv_headsÚn_kv_groupsÚhead_dimr=   ÚnnÚLinearr   ÚwqkvÚwor   r9   r   r   r   rM   )r%   rV   ÚdimrY   rZ   r\   Ú
rope_scalerB   s          €r&   r@   zAttention.__init__V   s$  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*Ø" d×&>Ñ&>Ñ>ˆÔà#'×#3Ñ#3°wÑ#>Ð>ˆŒ˜Ø˜t‘^ˆŒ
ä—I‘IØ�'˜A 
™NÑ*¨hÑ6¸T¿Y¹Yô
ˆŒ	ô —)‘)˜G hÑ.°¸$¿)¹)ÔDˆŒð × Ñ Ð,°×1BÑ1BÀ6Ñ1JÈhÒ1Vð �×!Ñ! (Ñ+Ò+àð 	ô *ØØ$(×$@Ñ$@Ø×-Ñ-Ø—‘Øô
ˆ�	r.   rN   ÚmaskÚcacheÚreturnc                 óÖ  — |j                   \  }}}| j                  |«      }|j                  ||dd| j                  z   | j                  «      }|dd | j                  …d d …f   }|j                  ||d| j                  «      }|ddd d …f   }	|ddd d …f   }
|j                  ||| j
                  d«      j                  dddd«      }|	j                  ||| j                  d«      j                  dddd«      }	|
j                  ||| j                  d«      j                  dddd«      }
|�P| j                  ||j                  ¬«      }| j                  |	|j                  ¬«      }	|j                  |	|
«      \  }	}
n"| j                  |«      }| j                  |	«      }	t        ||	|
|| j                  |¬	«      }|j                  dddd«      j                  ||d«      }| j                  |«      S )
NéÿÿÿÿrI   .éþÿÿÿr   r   é   )rG   )rd   r=   rc   )rJ   r_   Úreshaper[   r\   rY   Ú	transposerZ   rM   rG   Úupdate_and_fetchr   r=   r`   )r%   rN   rc   rd   ÚBÚLÚDÚ
qkv_statesÚqueriesÚkeysÚvaluesÚoutputs               r&   rP   zAttention.__call__t   sÌ  € ð —'‘'‰ˆˆ1ˆaà—Y‘Y˜q“\ˆ
Ø×'Ñ'¨¨1¨b°!°d×6FÑ6FÑ2FÈÏÉÓVˆ
à˜SÐ"4 D×$4Ñ$4Ð"4²aÐ7Ñ8ˆØ—/‘/ ! Q¨¨D¯M©MÓ:ˆØ˜#˜r¢1˜*Ñ%ˆØ˜C ¢Q˜JÑ'ˆð —/‘/ ! Q¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A˜q $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  1 d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�w‰w�v‹Ðr.   ©NN©r/   r0   r1   r   r@   rK   Úarrayr   r   rP   rR   rS   s   @r&   rU   rU   U   sX   ø„ ð
˜Yõ 
ðB $(Ø#ñ	!à�8‰8ð!ð �r—x‘xÑ ð!ð ˜‰}ð	!ð
 
�‰÷!r.   rU   c                   ó>   ‡ — e Zd Zˆ fd„Zdej
                  fd„Zˆ xZS )ÚMLPc                 óÐ   •— t         ‰| �  «        t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        y ©NFrX   )r?   r@   r]   r^   Úw1Úw2Úw3)r%   ra   Ú
hidden_dimrB   s      €r&   r@   zMLP.__init__™   sJ   ø€ Ü‰ÑÔÜ—)‘)˜C °%Ô8ˆŒÜ—)‘)˜J¨°%Ô8ˆŒÜ—)‘)˜C °%Ô8ˆ�r.   re   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S r"   )r}   r	   r|   r~   )r%   rN   s     r&   rP   zMLP.__call__Ÿ   s(   € Ø�w‰w”v˜dŸg™g a›j¨$¯'©'°!«*Ó5Ó6Ð6r.   )r/   r0   r1   r@   rK   rw   rP   rR   rS   s   @r&   ry   ry   ˜   s   ø„ ô9ð7˜RŸX™X÷ 7r.   ry   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚTransformerBlockrV   c                 óL  •— t         ‰| �  «        t        |«      | _        t	        |j
                  |j                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j
                  |j                  ¬«      | _        y )N©Úeps)r?   r@   rU   Ú	attentionry   r   r   Úfeed_forwardr]   ÚRMSNormr   Úattention_normÚffn_norm©r%   rV   rB   s     €r&   r@   zTransformerBlock.__init__¤   sq   ø€ Ü‰ÑÔÜ" 4›ˆŒÜ × 0Ñ 0°$×2HÑ2HÓIˆÔÜ Ÿj™j¨×)9Ñ)9¸t×?PÑ?PÔQˆÔÜŸ
™
 4×#3Ñ#3¸×9JÑ9JÔKˆ�r.   rN   rc   rd   re   c                 óž   — | j                  | j                  |«      ||«      }||z   }| j                  | j                  |«      «      }||z   }|S r"   )r†   r‰   r‡   rŠ   )r%   rN   rc   rd   ÚrÚhÚouts          r&   rP   zTransformerBlock.__call__«   sQ   € ð �N‰N˜4×.Ñ.¨qÓ1°4¸Ó?ˆØ�‰EˆØ×Ñ˜dŸm™m¨AÓ.Ó/ˆØ�!‰eˆØˆ
r.   ru   rv   rS   s   @r&   r‚   r‚   £   sY   ø„ ðL˜Yõ Lð $(Ø#ñ	
à�8‰8ð
ð �r—x‘xÑ ð
ð ˜‰}ð	
ð
 
�‰÷
r.   r‚   c                   óH   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zˆ xZS )ÚInternLM2ModelrV   c                 óp  •— t         ‰| �  «        |j                  dkD  sJ ‚t        j                  |j                  |j
                  «      | _        t        |j                  «      D �cg c]  }t        |¬«      ‘Œ c}| _
        t        j                  |j
                  |j                  ¬«      | _        y c c}w )Nr   )rV   r„   )r?   r@   r   r]   Ú	Embeddingr   Útok_embeddingsÚranger   r‚   Úlayersrˆ   r   Únorm)r%   rV   Ú_rB   s      €r&   r@   zInternLM2Model.__init__¹   s�   ø€ Ü‰ÑÔØ�‰ Ò"Ð"Ð"Ü Ÿl™l¨4¯?©?¸D×<LÑ<LÓMˆÔä16°t×7MÑ7MÔ1Nó
Ù1N¨AÔ $Ö'Ð1Nñ
ˆŒô —J‘J˜t×/Ñ/°T×5FÑ5FÔGˆ�	ùò
s   Á'B3Úinputsc                 óì   — | j                  |«      }|€d gt        | j                  «      z  }t        ||d   «      }t	        | j                  |«      D ]  \  }} ||||¬«      }Œ | j                  |«      S )Nr   )rd   )r”   Úlenr–   r   Úzipr—   )r%   r™   rd   rŽ   rc   ÚlayerÚcs          r&   rP   zInternLM2Model.__call__Â   su   € ð
 ×Ñ Ó'ˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEä$ Q¨¨a©Ó1ˆÜ˜DŸK™K¨Ö/‰HˆE�1Ù�a˜ QÔ'‰Að 0ð �y‰y˜‹|Ðr.   r"   )	r/   r0   r1   r   r@   rK   rw   rP   rR   rS   s   @r&   r‘   r‘   ¸   s'   ø„ ðH˜Yõ Hð ñà—‘÷r.   r‘   c                   ó^   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zd„ Ze	d„ «       Z
ˆ xZS )ÚModelrV   c                 óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y r{   )r?   r@   rV   r   r‘   Úmodelr   r]   r^   r   r   rt   r‹   s     €r&   r@   zModel.__init__Ô   sX   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ# DÓ)ˆŒ
Ø×'Ò'ÜŸ)™) D×$4Ñ$4°d·o±oÈEÔRˆD�Kð (r.   r™   c                 óÆ   — | j                  ||«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S r"   )r¢   rV   r   r”   Ú	as_linearrt   )r%   r™   rd   r�   s       r&   rP   zModel.__call__Ü   sW   € ð
 �j‰j˜ Ó'ˆØ�9‰9×(Ò(Ø—*‘*×+Ñ+×5Ñ5°cÓ:ˆCð ˆ
ð —+‘+˜cÓ"ˆCØˆ
r.   c                 ó`   — |j                  «       D ��ci c]  \  }}d|vsŒ||“Œ c}}S c c}}w )Nzattention.rope.inv_freq)Úitems)r%   ÚweightsÚkÚvs       r&   ÚsanitizezModel.sanitizeè   s1   € à!(§¡¤ÔW¡™˜˜AÐ4MÐUVÒ4V��1‘ ÒWÐWùÓWs   ”*¡*c                 ó.   — | j                   j                  S r"   )r¢   r–   rE   s    r&   r–   zModel.layersì   s   € à�z‰z× Ñ Ð r.   r"   )r/   r0   r1   r   r@   rK   rw   rP   rª   Úpropertyr–   rR   rS   s   @r&   r    r    Ó   sA   ø„ ðS˜Yõ Sð ñ
à—‘ó
òXð ñ!ó ô!r.   r    )Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorerK   Úmlx.nnr]   Úactivationsr	   r<   r
   r   r   r   ÚModuler9   rU   ry   r‚   r‘   r    r7   r.   r&   Ú<module>r´      sž   ðõ "ß -Ó -å Ý å ß TÑ Tð ô�ó ó ðô>%
˜BŸI™Iô %
ôP@�—	‘	ô @ôF7ˆ"�)‰)ô 7ô�r—y‘yô ô*�R—Y‘Yô ô6!ˆB�I‰Iõ !r.   