Ë
    îÿæir  ã                   óˆ  — d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZ e G d„ d	e«      «       Z G d
„ dej*                  «      Z G d„ dej*                  «      Z G d„ dej*                  «      Z G d„ dej*                  «      Z G d„ dej*                  «      Z G d„ dej*                  «      Zy)é    )Ú	dataclass)ÚAnyÚDictÚOptionalÚUnionNé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Úinitialize_ropec                   óð   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zee   ed<   d
Z	ee   ed<   dZ
eed<   d
Zeeeeeef   f      ed<   dZeed<   dZeed<   dZeed<   y
)Ú	ModelArgsÚ
model_typeÚhidden_sizeÚ
num_layersÚintermediate_sizeÚnum_attention_headsÚ
vocab_sizeÚ
rope_thetaÚlayer_norm_epsilonÚnum_key_value_headsNÚhead_dimÚmax_position_embeddingsFÚrope_traditionalÚrope_scalingTÚtie_word_embeddingsÚattention_biasÚmlp_bias)Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatr   r   r   r   Úboolr   r   r   r   r   r   © ó    úi/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/exaone.pyr   r      sž   … àƒOØÓØƒOØÓØÓØƒOØÓØÓØÓØ"€Hˆh�s‰mÓ"Ø-1Ð˜X c™]Ó1Ø"Ð�dÓ"Ø;?€L�(˜4  U¨5°#¨:Ñ%6Ð 6Ñ7Ñ8Ó?Ø $Ð˜Ó$Ø €N�DÓ Ø€HˆdÔr)   r   c            	       óˆ   ‡ — e Zd Zdefˆ fd„Z	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚAttentionModuleÚargsc                 óª  •— t         ‰| �  «        |j                  }|j                  x| _        }|j
                  x| _        }|j                  xs ||z  x| _        }|dz  | _        t        j                  |||z  |j                  ¬«      | _        t        j                  |||z  |j                  ¬«      | _        t        j                  |||z  |j                  ¬«      | _        t        j                  ||z  ||j                  ¬«      | _        t!        | j                  |j"                  |j$                  |j&                  |j(                  «      | _        y )Ng      à¿©Úbias)ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsr   ÚscaleÚnnÚLinearr   Úq_projÚk_projÚv_projÚout_projr   r   r   r   r   Úrope)Úselfr-   Údimr3   r4   r   Ú	__class__s         €r*   r2   zAttentionModule.__init__#   s  ø€ Ü‰ÑÔØ×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*Ø#'§=¡=Ò#D°S¸G±^ÐDˆŒ˜Ø˜t‘^ˆŒ
ä—i‘i  W¨xÑ%7¸d×>QÑ>QÔRˆŒÜ—i‘i  Z°(Ñ%:À×ATÑATÔUˆŒÜ—i‘i  Z°(Ñ%:À×ATÑATÔUˆŒÜŸ	™	 '¨HÑ"4°cÀ×@SÑ@SÔTˆŒä#Ø�M‰MØ�O‰OØ×!Ñ!Ø×ÑØ×(Ñ(ó
ˆ�	r)   ÚxÚmaskÚcacheÚreturnc                 ó&  — |j                   \  }}}| j                  |«      j                  ||| j                  d«      j	                  dddd«      }| j                  |«      j                  ||| j                  d«      j	                  dddd«      }| j                  |«      j                  ||| j                  d«      j	                  dddd«      }	|�P| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||	«      \  }}	n"| j                  |«      }| j                  |«      }t        |||	|| j                  |¬«      }
|
j	                  dddd«      j                  |||«      }
| j                  |
«      S )Néÿÿÿÿr   é   r   é   )Úoffset)rB   r5   rA   )Úshaper8   Úreshaper3   Ú	transposer9   r4   r:   r<   rH   Úupdate_and_fetchr   r5   r;   )r=   r@   rA   rB   ÚBÚLÚDÚqÚkÚvÚouts              r*   Ú__call__zAttentionModule.__call__8   s_  € ð —'‘'‰ˆˆ1ˆaØ�K‰K˜‹N×"Ñ" 1 a¨¯©°rÓ:×DÑDÀQÈÈ1ÈaÓPˆØ�K‰K˜‹N×"Ñ" 1 a¨¯©¸"Ó=×GÑGÈÈ1ÈaÐQRÓSˆØ�K‰K˜‹N×"Ñ" 1 a¨¯©¸"Ó=×GÑGÈÈ1ÈaÐQRÓSˆàÐØ—	‘	˜! E§L¡L�	Ó1ˆAØ—	‘	˜! E§L¡L�	Ó1ˆAØ×)Ñ)¨!¨QÓ/‰DˆA‰qà—	‘	˜!“ˆAØ—	‘	˜!“ˆAä*Øˆq�!˜5¨¯
©
¸ô
ˆð �m‰m˜A˜q ! QÓ'×/Ñ/°°1°aÓ8ˆØ�}‰}˜SÓ!Ð!r)   ©NN©r    r!   r"   r   r2   ÚmxÚarrayr   r   rT   Ú__classcell__©r?   s   @r*   r,   r,   "   sP   ø„ ð
˜Yõ 
ð, TXñ"Ø—‘ð"Ø!)¨"¯(©(Ñ!3ð"ØCKÈCÁ=ð"à	�‰÷"r)   r,   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )Ú	Attentionr-   c                 óB   •— t         ‰| �  «        t        |«      | _        y ©N)r1   r2   r,   Ú	attention©r=   r-   r?   s     €r*   r2   zAttention.__init__P   s   ø€ Ü‰ÑÔÜ(¨Ó.ˆ�r)   )r    r!   r"   r   r2   rY   rZ   s   @r*   r\   r\   O   s   ø„ ð/˜Y÷ /ñ /r)   r\   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚMLPr-   c                 ó<  •— t         ‰| �  «        |j                  }|j                  }t	        j
                  |||j                  ¬«      | _        t	        j
                  |||j                  ¬«      | _        t	        j
                  |||j                  ¬«      | _	        y )Nr/   )
r1   r2   r   r   r6   r7   r   Úc_fc_0Úc_fc_1Úc_proj)r=   r-   r>   Ú
hidden_dimr?   s       €r*   r2   zMLP.__init__V   sl   ø€ Ü‰ÑÔØ×ÑˆØ×+Ñ+ˆ
Ü—i‘i  Z°d·m±mÔDˆŒÜ—i‘i  Z°d·m±mÔDˆŒÜ—i‘i 
¨C°d·m±mÔDˆ�r)   r@   rC   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S r^   )rf   r	   rd   re   )r=   r@   s     r*   rT   zMLP.__call__^   s(   € Ø�{‰{œ6 $§+¡+¨a£.°$·+±+¸a³.ÓAÓBÐBr)   ©	r    r!   r"   r   r2   rW   rX   rT   rY   rZ   s   @r*   rb   rb   U   s/   ø„ ðE˜Yõ EðC˜"Ÿ(™(ð C r§x¡x÷ Cr)   rb   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚTransformerBlockr-   c                 ó"  •— t         ‰| �  «        t        j                  |j                  |j
                  ¬«      | _        t        |«      | _        t        j                  |j                  |j
                  ¬«      | _	        t        |«      | _        y ©N)Úeps)r1   r2   r6   ÚRMSNormr   r   Úln_1r\   ÚattnÚln_2rb   Úmlpr`   s     €r*   r2   zTransformerBlock.__init__c   s`   ø€ Ü‰ÑÔÜ—J‘J˜t×/Ñ/°T×5LÑ5LÔMˆŒ	Ü˜d“OˆŒ	Ü—J‘J˜t×/Ñ/°T×5LÑ5LÔMˆŒ	Ü�t“9ˆ�r)   r@   rA   rB   rC   c                 óª   — || j                   j                  | j                  |«      ||«      z   }|| j                  | j	                  |«      «      z   }|S r^   )rq   r_   rp   rs   rr   )r=   r@   rA   rB   ÚhrS   s         r*   rT   zTransformerBlock.__call__j   sI   € ð �—	‘	×#Ñ# D§I¡I¨a£L°$¸Ó>Ñ>ˆØ�$—(‘(˜4Ÿ9™9 Q›<Ó(Ñ(ˆØˆ
r)   rU   rV   rZ   s   @r*   rk   rk   b   sW   ø„ ð˜Yõ ð $(Ø#ñ	à�8‰8ðð �r—x‘xÑ ðð ˜‰}ð	ð
 
�‰÷r)   rk   c                   óH   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zˆ xZS )ÚExaoneModelr-   c                 óL  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  «      D �cg c]  }t        |«      ‘Œ c}| _
        t        j                  |j
                  |j                  ¬«      | _        y c c}w rm   )r1   r2   r6   Ú	Embeddingr   r   ÚwteÚranger   rk   ru   ro   r   Úln_f)r=   r-   Ú_r?   s      €r*   r2   zExaoneModel.__init__v   su   ø€ Ü‰ÑÔÜ—<‘< §¡°×1AÑ1AÓBˆŒÜ27¸¿¹Ô2HÓIÑ2H¨QÔ" 4Õ(Ð2HÑIˆŒÜ—J‘J˜t×/Ñ/°T×5LÑ5LÔMˆ�	ùò Js   ÁB!Úinputsc                 óì   — | j                  |«      }|€d gt        | j                  «      z  }t        ||d   «      }t	        | j                  |«      D ]  \  }} ||||¬«      }Œ | j                  |«      S )Nr   )rB   )rz   Úlenru   r   Úzipr|   )r=   r~   rB   ru   rA   ÚlayerÚcs          r*   rT   zExaoneModel.__call__|   sr   € ð
 �H‰H�VÓˆàˆ=Ø�FœS §¡›[Ñ(ˆEä$ Q¨¨a©Ó1ˆä˜DŸF™F EÖ*‰HˆE�1Ù�a˜ QÔ'‰Að +ð �y‰y˜‹|Ðr)   r^   ri   rZ   s   @r*   rw   rw   u   s'   ø„ ðN˜Yõ Nð ñà—‘÷r)   rw   c                   óX   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zed„ «       Z	ˆ xZ
S )ÚModelr-   c                 óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y )NFr/   )r1   r2   r-   r   rw   Útransformerr   r6   r7   r   r   Úlm_headr`   s     €r*   r2   zModel.__init__�   sY   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ& tÓ,ˆÔØ×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (r)   r~   c                 óÆ   — | j                  ||«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S r^   )r‡   r-   r   rz   Ú	as_linearrˆ   )r=   r~   rB   rS   s       r*   rT   zModel.__call__—   s[   € ð
 ×Ñ˜v uÓ-ˆØ�9‰9×(Ò(Ø×"Ñ"×&Ñ&×0Ñ0°Ó5ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r)   c                 ó.   — | j                   j                  S r^   )r‡   ru   )r=   s    r*   ÚlayerszModel.layers£   s   € à×Ñ×!Ñ!Ð!r)   r^   )r    r!   r"   r   r2   rW   rX   rT   ÚpropertyrŒ   rY   rZ   s   @r*   r…   r…   Ž   s;   ø„ ðT˜Yõ Tð ñ
à—‘ó
ð ñ"ó ô"r)   r…   )Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorerW   Úmlx.nnr6   Úactivationsr	   Úbaser
   r   r   Ú
rope_utilsr   r   ÚModuler,   r\   rb   rk   rw   r…   r(   r)   r*   Ú<module>r—      s    ðõ "ß -Ó -å Ý å ß TÑ TÝ 'ð ô�ó ó ðô&*"�b—i‘iô *"ôZ/�—	‘	ô /ô
Cˆ"�)‰)ô 
Cô�r—y‘yô ô&�"—)‘)ô ô2"ˆB�I‰Iõ "r)   