Ë
    îÿæi•  ã                   ón  — d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZmZ ddlmZ e G d	„ d
e«      «       Z G d„ dej0                  «      Z G d„ dej0                  «      Z G d„ dej0                  «      Z G d„ dej0                  «      Z G d„ dej0                  «      Zy)é    )Ú	dataclass)ÚAnyÚDictÚOptionalÚUnionNé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCacheÚRotatingKVCache)Úinitialize_ropec                   óÄ   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   eed<   eee	eef   f   ed<   e
e   ed<   e
e   ed<   y)Ú	ModelArgsÚ
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚrms_norm_epsÚ
vocab_sizeÚnum_key_value_headsÚmax_position_embeddingsÚ
rope_thetaÚhead_dimÚtie_word_embeddingsÚrope_scalingÚsliding_windowÚsliding_window_patternN)Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatÚboolr   r   r   © ó    új/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/exaone4.pyr   r      st   … àƒOØÓØÓØÓØÓØÓØƒOØÓØ Ó ØÓØƒMØÓØ�s˜E %¨ *Ñ-Ð-Ñ.Ó.Ø˜S‘MÓ!Ø$ S™MÔ)r*   r   c            	       ó”   ‡ — e Zd Zdedee   fˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
Ú	AttentionÚargsÚis_localc                 ó  •— t         ‰| �  «        |j                  }|j                  x| _        }|j
                  €J ‚|j
                  x| _        }|j                  }|dz  | _        t        j                  |||z  d¬«      | _        t        j                  |||z  d¬«      | _        t        j                  |||z  d¬«      | _        t        j                  ||z  |d¬«      | _        t        j                  ||j                   ¬«      | _        t        j                  ||j                   ¬«      | _        |xs d| _        |d u xs || _        | j(                  r4t+        ||j,                  d|j.                  |j0                  ¬«      | _        y y )Ng      à¿F©Úbias©Úeps)ÚbaseÚtraditionalÚscaling_configr   )ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsr   ÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚo_projÚRMSNormr   Úq_normÚk_normr/   Úuse_roper   r   r   r   Úrope)Úselfr.   r/   Údimr:   r;   r   Ú	__class__s          €r+   r9   zAttention.__init__#   sK  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ×'Ñ'Ð3Ð3Ð3Ø'+×'?Ñ'?Ð?ˆŒ˜*à—=‘=ˆØ˜t‘^ˆŒ
ä—i‘i  W¨xÑ%7¸eÔDˆŒÜ—i‘i  Z°(Ñ%:ÀÔGˆŒÜ—i‘i  Z°(Ñ%:ÀÔGˆŒÜ—i‘i ¨(Ñ 2°C¸eÔDˆŒä—j‘j ¨t×/@Ñ/@ÔAˆŒÜ—j‘j ¨t×/@Ñ/@ÔAˆŒØ Ò) EˆŒØ  DÐ(Ò4¨HˆŒØ�=Š=Ü'ØØ—_‘_Ø!Ø#×0Ñ0Ø(,×(DÑ(DôˆD�Ið r*   ÚxÚmaskÚcacheÚreturnc                 óž  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}| j	                  |j                  ||| j                  d«      «      j                  dddd«      }| j                  |j                  ||| j                  d«      «      j                  dddd«      }|	j                  ||| j                  d«      j                  dddd«      }	|�\| j                  r:| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||	«      \  }}	n.| j                  r"| j                  |«      }| j                  |«      }t        |||	|| j                  |¬«      }
|
j                  dddd«      j                  ||d«      }
| j!                  |
«      S )Néÿÿÿÿr   é   r   é   )Úoffset)rM   r<   rL   )Úshaper?   r@   rA   rD   Úreshaper:   Ú	transposerE   r;   rF   rG   rS   Úupdate_and_fetchr   r<   rB   )rH   rK   rL   rM   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs              r+   Ú__call__zAttention.__call__@   s‘  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆà—+‘+˜gŸo™o¨a°°D·L±LÀ"ÓEÓF×PÑPØˆq�!�Qó
ˆð �{‰{˜4Ÿ<™<¨¨1¨d¯o©o¸rÓBÓC×MÑMØˆq�!�Qó
ˆð —‘  1 d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÐØ�}Š}ØŸ)™) G°E·L±L˜)ÓA�Ø—y‘y ¨e¯l©l�yÓ;�Ø ×1Ñ1°$¸Ó?‰LˆD‘&Ø�]Š]Ø—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r*   ©NN)r!   r"   r#   r   r   r(   r9   ÚmxÚarrayr   r_   Ú__classcell__©rJ   s   @r+   r-   r-   "   sc   ø„ ð˜Yð °(¸4±.õ ð@ $(Ø#ñ	#à�8‰8ð#ð �r—x‘xÑ ð#ð ˜‰}ð	#ð
 
�‰÷#r*   r-   c                   ó>   ‡ — e Zd Zˆ fd„Zdej
                  fd„Zˆ xZS )ÚMLPc                 óÐ   •— t         ‰| �  «        t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        t        j                  ||d¬«      | _        y ©NFr1   )r8   r9   r=   r>   Ú	gate_projÚ	down_projÚup_proj)rH   rI   Ú
hidden_dimrJ   s      €r+   r9   zMLP.__init__c   sJ   ø€ Ü‰ÑÔÜŸ™ 3¨
¸Ô?ˆŒÜŸ™ :¨s¸Ô?ˆŒÜ—y‘y  j°uÔ=ˆ�r*   rN   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S ©N)rj   r	   ri   rk   )rH   rK   s     r+   r_   zMLP.__call__i   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr*   )r!   r"   r#   r9   ra   rb   r_   rc   rd   s   @r+   rf   rf   b   s   ø„ ô>ðJ˜RŸX™X÷ Jr*   rf   c            	       óŽ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
ÚTransformerBlockr.   r/   c                 ó   •— t         ‰| �  «        |j                  | _        |j                  | _        t	        ||«      | _        t        |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _        || _        y )Nr3   )r8   r9   r   r   r-   Ú	self_attnrf   r   Úmlpr=   rC   r   Úpost_attention_layernormÚpost_feedforward_layernormr.   )rH   r.   r/   rJ   s      €r+   r9   zTransformerBlock.__init__n   s¡   ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø×+Ñ+ˆÔÜ" 4¨Ó2ˆŒÜ�t×'Ñ'¨×)?Ñ)?Ó@ˆŒÜ(*¯
©
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ô +-¯*©*Ø×Ñ $×"3Ñ"3ô+
ˆÔ'ð ˆ�	r*   rK   rL   rM   rN   c                 óž   — | j                  |||«      }|| j                  |«      z   }| j                  |«      }|| j                  |«      z   }|S rn   )rr   rt   rs   ru   )rH   rK   rL   rM   ÚrÚhÚouts          r+   r_   zTransformerBlock.__call__|   sR   € ð �N‰N˜1˜d EÓ*ˆØ�×-Ñ-¨aÓ0Ñ0ˆØ�H‰H�Q‹KˆØ�$×1Ñ1°!Ó4Ñ4ˆØˆ
r*   r`   )r!   r"   r#   r   r(   r9   ra   rb   r   r   r_   rc   rd   s   @r+   rp   rp   m   s^   ø„ ð˜Yð °$õ ð" $(Ø#ñ	
à�8‰8ð
ð �r—x‘xÑ ð
ð ˜‰}ð	
ð
 
�‰÷
r*   rp   c                   óH   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zˆ xZS )ÚExaoneModelr.   c                 ó¤  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        | j                  dkD  sJ ‚t        j                  |j                  |j                  «      | _        |j                  }t        |j                  «      D �cg c]%  }t        ||r||t        |«      z     dk(  nd ¬«      ‘Œ' c}| _        |r-|j                  d«      | _        |j                  d«      | _        nd | _        d| _        |j"                  | _        t        j&                  |j                  |j(                  ¬«      | _        y c c}w )Nr   rY   )r.   r/   ÚGr3   )r8   r9   r.   r   r   r=   Ú	Embeddingr   Úembed_tokensr    Úrangerp   ÚlenÚlayersÚindexÚswa_idxÚfull_idxr   Úwindow_sizerC   r   Únorm)rH   r.   ÚpatternÚirJ   s       €r+   r9   zExaoneModel.__init__Š   s  ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ�‰ Ò"Ð"Ð"ÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔØ×-Ñ-ˆô ˜4×1Ñ1Ô2ó
ñ
 3�ô	 ØÙ=D˜ ¤S¨£\Ñ!1Ñ2°cÒ9È$öð 3ñ
ˆŒñ Ø"Ÿ=™=¨Ó-ˆDŒLØ#ŸM™M¨#Ó.ˆD�MàˆDŒLØˆDŒMà×.Ñ.ˆÔÜ—J‘J˜t×/Ñ/°T×5FÑ5FÔGˆ�	ùò
s   Â*EÚinputsc                 óš  — | j                  |«      }|€d gt        | j                  «      z  }t        ||| j                     «      }| j
                  �&t        ||| j
                     | j                  ¬«      }nd }t        | j                  |«      D ])  \  }}|j                  j                  r|n|} ||||«      }Œ+ | j                  |«      S )N)r†   )r   r�   r‚   r   r…   r„   r†   Úziprr   r/   r‡   )	rH   rŠ   rM   rx   Úglobal_maskÚswa_maskÚlayerÚcrL   s	            r+   r_   zExaoneModel.__call__£   s¾   € ð
 ×Ñ˜fÓ%ˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEÜ+¨A¨u°T·]±]Ñ/CÓDˆØ�<‰<Ð#Ü,Ø�5˜Ÿ™Ñ&°D×4DÑ4Dô‰Hð ˆHä˜DŸK™K¨Ö/‰HˆE�1Ø$Ÿ™×7Ò7‘8¸[ˆDÙ�a˜˜qÓ!‰Að 0ð �y‰y˜‹|Ðr*   rn   )	r!   r"   r#   r   r9   ra   rb   r_   rc   rd   s   @r+   r{   r{   ‰   s'   ø„ ðH˜Yõ Hð8 ñà—‘÷r*   r{   c                   ó^   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zd„ Ze	d„ «       Z
ˆ xZS )ÚModelr.   c                 óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y rh   )r8   r9   r.   r   r{   Úmodelr   r=   r>   r   r   Úlm_head)rH   r.   rJ   s     €r+   r9   zModel.__init__¼   sX   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ  Ó&ˆŒ
Ø×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (r*   rŠ   c                 óÆ   — | j                  ||«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S rn   )r”   r.   r   r   Ú	as_linearr•   )rH   rŠ   rM   ry   s       r+   r_   zModel.__call__Ä   sW   € ð
 �j‰j˜ Ó'ˆØ�9‰9×(Ò(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r*   c                 ó¾   — | j                   D �cg c]C  }|j                  j                  r!t        | j                  j
                  d¬«      n	t        «       ‘ŒE c}S c c}w )Nr   )Úmax_sizeÚkeep)r‚   rr   r/   r   r.   r   r   )rH   Úls     r+   Ú
make_cachezModel.make_cacheÐ   s[   € ð —[’[ó
ñ !�ð —;‘;×'Ò'ô  ¨¯©×)AÑ)AÈÕJä“Yñð !ñ
ð 	
ùò 
s   �AAc                 ó.   — | j                   j                  S rn   )r”   r‚   )rH   s    r+   r‚   zModel.layersÚ   s   € à�z‰z× Ñ Ð r*   rn   )r!   r"   r#   r   r9   ra   rb   r_   rœ   Úpropertyr‚   rc   rd   s   @r+   r’   r’   »   s@   ø„ ðT˜Yõ Tð ñ
à—‘ó
ò
ð ñ!ó ô!r*   r’   )Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorera   Úmlx.nnr=   Úactivationsr	   r5   r
   r   r   rM   r   r   Ú
rope_utilsr   r   ÚModuler-   rf   rp   r{   r’   r)   r*   r+   Ú<module>r§      s”   ðõ "ß -Ó -å Ý å ß TÑ Tß +Ý 'ð ô*�ó *ó ð*ô$=#�—	‘	ô =#ô@Jˆ"�)‰)ô Jô�r—y‘yô ô8/�"—)‘)ô /ôd!!ˆB�I‰Iõ !!r*   