Ë
    îÿæi«  ã                   ór  — d dl mZ d dlmZmZmZmZmZ d dlm	Z
 d dlmZ ddlmZ ddlmZmZmZ ddlmZmZ ddlmZ e G d	„ d
e«      «       Z G d„ dej2                  «      Z G d„ dej2                  «      Z G d„ dej2                  «      Z G d„ dej2                  «      Z G d„ dej2                  «      Zy)é    )Ú	dataclass)ÚAnyÚDictÚListÚOptionalÚUnionNé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCacheÚRotatingKVCache)Úinitialize_ropec                   óþ   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   dZeed<   dZ	e
ee      ed<   dZe
e   ed<   dZe
e   ed<   dZe
eeeeef   f      ed<   dZeed<   d„ Zy)Ú	ModelArgsÚ
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚrms_norm_epsÚ
vocab_sizeÚmax_position_embeddingsÚsliding_windowÚ
rope_thetaFÚattention_biasNÚlayer_typesÚnum_key_value_headsÚhead_dimÚrope_scalingÚtie_word_embeddingsc                 óÌ   — | j                   €| j                  | _         | j                  €6t        | j                  «      D �cg c]  }|dz   dz  dk(  rdnd‘Œ c}| _        y y c c}w )Nr	   é   r   Úfull_attentionÚsliding_attention)r   r   r   Úranger   )ÚselfÚis     úh/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/olmo3.pyÚ__post_init__zModelArgs.__post_init__"   su   € Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$Ø×ÑÐ#ô ˜t×5Ñ5Ô6ó á6�Að &'¨¡U¨a¡K°1Ò$4Ñ Ð:MÑMØ6ñ ˆDÕð $ùò s   ÁA!)Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatr   Úboolr   r   r   r   r    r!   r   r   r"   r+   © ó    r*   r   r      s¨   … àƒOØÓØÓØÓØÓØÓØƒOØ Ó ØÓØÓØ €N�DÓ Ø'+€K�˜$˜s™)Ñ$Ó+Ø)-Ð˜ #™Ó-Ø"€Hˆh�s‰mÓ"Ø;?€L�(˜4  U¨5°#¨:Ñ%6Ð 6Ñ7Ñ8Ó?Ø %Ð˜Ó%ór5   r   c            	       óŽ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
ÚOlmo3AttentionÚargsÚ	layer_idxc                 ó*  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        |j
                  xs |j                  |j                  z  | _        | j
                  dz  | _        t        j                  |j                  |j                  | j
                  z  |j                  ¬«      | _        t        j                  |j                  |j                  | j
                  z  |j                  ¬«      | _        t        j                  |j                  |j                  | j
                  z  |j                  ¬«      | _        t        j                  |j                  | j
                  z  |j                  |j                  ¬«      | _        t        j                  |j                  | j
                  z  |j                   ¬«      | _        t        j                  |j                  | j
                  z  |j                   ¬«      | _        |j&                  |   dk7  r2t        j(                  | j
                  d|j*                  ¬«      | _        y t/        | j
                  d|j*                  |j0                  |j2                  ¬«      | _        y )Ng      à¿©Úbias©Úepsr%   F)ÚtraditionalÚbase)r?   r@   Úscaling_configr   )ÚsuperÚ__init__r   r   r9   r    r   ÚscaleÚnnÚLinearr   Úq_projÚk_projÚv_projÚo_projÚRMSNormr   Úq_normÚk_normr   ÚRoPEr   Úroper   r!   r   ©r(   r8   r9   Ú	__class__s      €r*   rC   zOlmo3Attention.__init__-   sñ  ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø#'×#;Ñ#;ˆÔ Ø"ˆŒàŸ™ÒU¨×)9Ñ)9¸T×=UÑ=UÑ)UˆŒØ—]‘] DÑ(ˆŒ
ä—i‘iØ×ÑØ×$Ñ$ t§}¡}Ñ4Ø×$Ñ$ô
ˆŒô
 —i‘iØ×ÑØ×$Ñ$ t§}¡}Ñ4Ø×$Ñ$ô
ˆŒô
 —i‘iØ×ÑØ×$Ñ$ t§}¡}Ñ4Ø×$Ñ$ô
ˆŒô
 —i‘iØ×$Ñ$ t§}¡}Ñ4Ø×ÑØ×$Ñ$ô
ˆŒô —j‘jØ×$Ñ$ t§}¡}Ñ4¸$×:KÑ:Kô
ˆŒô —j‘jØ×$Ñ$ t§}¡}Ñ4¸$×:KÑ:Kô
ˆŒð ×Ñ˜IÑ&Ð*:Ò:ÜŸ™ §¡¸5ÀtÇÁÔWˆD�Iä'Ø—‘Ø!Ø—_‘_Ø#×0Ñ0Ø(,×(DÑ(DôˆD�Ir5   ÚxÚmaskÚcacheÚreturnc                 ón  — |j                   \  }}}| j                  | j                  |«      «      }| j                  | j	                  |«      «      }| j                  |«      }	|j                  ||| j                  d«      j                  dddd«      }|j                  ||| j                  d«      j                  dddd«      }|	j                  ||| j                  d«      j                  dddd«      }	|�P| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||	«      \  }}	n"| j                  |«      }| j                  |«      }t        |||	|| j                  |¬«      }
|
j                  dddd«      j                  ||d«      }
| j                  |
«      S )Néÿÿÿÿr   é   r	   é   )Úoffset)rT   rD   rS   )ÚshaperL   rG   rM   rH   rI   Úreshaper   Ú	transposer   rO   rZ   Úupdate_and_fetchr   rD   rJ   )r(   rR   rS   rT   ÚBÚLÚ_ÚqueriesÚkeysÚvaluesÚoutputs              r*   Ú__call__zOlmo3Attention.__call__]   s�  € ð —'‘'‰ˆˆ1ˆaØ—+‘+˜dŸk™k¨!›nÓ-ˆØ�{‰{˜4Ÿ;™; q›>Ó*ˆØ—‘˜Q“ˆà—/‘/ ! Q¨×(@Ñ(@À"ÓE×OÑOØˆq�!�Qó
ˆð �|‰|˜A˜q $×":Ñ":¸BÓ?×IÑIÈ!ÈQÐPQÐSTÓUˆØ—‘  1 d×&>Ñ&>ÀÓC×MÑMØˆq�!�Qó
ˆð ÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r5   ©NN©r,   r-   r.   r   r1   rC   ÚmxÚarrayr   r   rf   Ú__classcell__©rQ   s   @r*   r7   r7   ,   s_   ø„ ð.˜Yð .°3õ .ðf $(Ø#ñ	#à�8‰8ð#ð �r—x‘xÑ ð#ð ˜‰}ð	#ð
 
�‰÷#r5   r7   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚOlmo3MLPr8   c                 óH  •— t         ‰| �  «        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j
                  |j                  d¬«      | _        t        j                  |j                  |j
                  d¬«      | _        y ©NFr;   )	rB   rC   rE   rF   r   r   Ú	gate_projÚ	down_projÚup_proj©r(   r8   rQ   s     €r*   rC   zOlmo3MLP.__init__€   sq   ø€ Ü‰ÑÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆ�r5   rR   rU   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S ©N)rr   r
   rq   rs   )r(   rR   s     r*   rf   zOlmo3MLP.__call__†   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr5   )	r,   r-   r.   r   rC   ri   rj   rf   rk   rl   s   @r*   rn   rn      s/   ø„ ðW˜Yõ WðJ˜"Ÿ(™(ð J r§x¡x÷ Jr5   rn   c            	       óŽ   ‡ — e Zd Zdedefˆ fd„Z	 	 d	dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )
ÚOlmo3DecoderLayerr8   r9   c                 óx  •— t         ‰| �  «        |j                  | _        |j                  | _        t	        ||¬«      | _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _        || _        y )N)r9   r=   )rB   rC   r   r   r7   Ú	self_attnrn   ÚmlprE   rK   r   Úpost_attention_layernormÚpost_feedforward_layernormr8   rP   s      €r*   rC   zOlmo3DecoderLayer.__init__‹   s’   ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø×+Ñ+ˆÔÜ'¨¸	ÔBˆŒÜ˜D“>ˆŒÜ(*¯
©
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ô +-¯*©*Ø×Ñ $×"3Ñ"3ô+
ˆÔ'ð ˆ�	r5   rR   rS   rT   rU   c                 óž   — | j                  | j                  |||«      «      }||z   }| j                  | j                  |«      «      }||z   }|S rv   )r|   rz   r}   r{   )r(   rR   rS   rT   ÚrÚhÚouts          r*   rf   zOlmo3DecoderLayer.__call__™   sP   € ð ×)Ñ)¨$¯.©.¸¸DÀ%Ó*HÓIˆØ�‰EˆØ×+Ñ+¨D¯H©H°Q«KÓ8ˆØ�!‰eˆØˆ
r5   rg   rh   rl   s   @r*   rx   rx   Š   s^   ø„ ð˜Yð °3õ ð" $(Ø#ñ	
à�8‰8ð
ð �r—x‘xÑ ð
ð ˜‰}ð	
ð
 
�‰÷
r5   rx   c                   ój   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dee   dej                  fd„Z	ˆ xZ
S )Ú
Olmo3Modelr8   c                 ó  •— t         ‰| �  «        |j                  | _        t        j                  |j
                  |j                  «      | _        t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}| _        t        j                  |j                  |j                  ¬«      | _        |j                  j!                  d«      | _        |j                  j!                  d«      | _        |j                  | _        y c c}w )N)r8   r9   r=   r&   r%   )rB   rC   r   rE   Ú	Embeddingr   r   Úembed_tokensr'   r   rx   ÚlayersrK   r   Únormr   ÚindexÚswa_idxÚga_idx)r(   r8   r)   rQ   s      €r*   rC   zOlmo3Model.__init__§   sÑ   ø€ Ü‰ÑÔØ"×1Ñ1ˆÔäŸL™L¨¯©¸$×:JÑ:JÓKˆÔô ˜4×1Ñ1Ô2ó
á2�ô  4°1Ö5Ø2ñ
ˆŒô —J‘J˜t×/Ñ/°T×5FÑ5FÔGˆŒ	à×'Ñ'×-Ñ-Ð.AÓBˆŒØ×&Ñ&×,Ñ,Ð-=Ó>ˆŒØ×+Ñ+ˆÕùò
s   Á'DÚinputsrT   rU   c                 ót  — | j                  |«      }|€d gt        | j                  «      z  }t        ||| j                     «      }t        ||| j
                     | j                  ¬«      }t        | j                  || j                  «      D ]  \  }}}|dk(  r|n|}	 |||	|¬«      }Œ | j                  |«      S )N)Úwindow_sizer%   )rT   )
r†   Úlenr‡   r   r‹   rŠ   r   Úzipr   rˆ   )
r(   rŒ   rT   r€   Ú	full_maskÚsliding_window_maskÚlayerÚcÚ
layer_typerS   s
             r*   rf   zOlmo3Model.__call__¶   s¹   € ð
 ×Ñ˜fÓ%ˆàˆ=Ø�FœS §¡Ó-Ñ-ˆEä)¨!¨U°4·;±;Ñ-?Ó@ˆ	Ü3Øˆu�T—\‘\Ñ"°×0CÑ0Cô
Ðô %(¨¯©°U¸D×<LÑ<LÖ$MÑ ˆE�1�jØ *Ð.>Ò >‘9ÐDWˆDÙ�a˜ QÔ'‰Að %Nð �y‰y˜‹|Ðr5   rv   )r,   r-   r.   r   rC   ri   rj   r   r   rf   rk   rl   s   @r*   rƒ   rƒ   ¦   sA   ø„ ð,˜Yõ ,ð$  $ñà—‘ðð ˜‰}ðð 
�‰÷	r5   rƒ   c                   ó€   ‡ — e Zd Zdefˆ fd„Z	 d	dej                  dee   dej                  fd„Z	e
d„ «       Zd„ Zˆ xZS )
ÚModelr8   c                 óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y rp   )rB   rC   r8   r   rƒ   Úmodelr"   rE   rF   r   r   Úlm_headrt   s     €r*   rC   zModel.__init__Í   sX   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ Ó%ˆŒ
Ø×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (r5   rŒ   rT   rU   c                 óÆ   — | j                  ||«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S rv   )r™   r8   r"   r†   Ú	as_linearrš   )r(   rŒ   rT   r�   s       r*   rf   zModel.__call__Õ   sW   € ð
 �j‰j˜ Ó'ˆØ�9‰9×(Ò(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r5   c                 ó.   — | j                   j                  S rv   )r™   r‡   )r(   s    r*   r‡   zModel.layersá   s   € à�z‰z× Ñ Ð r5   c                 óÜ   — g }| j                   j                  D ]P  }|dk(  r|j                  t        «       «       Œ"|j                  t	        | j
                  j                  ¬«      «       ŒR |S )Nr%   )Úmax_size)r™   r   Úappendr   r   r8   r   )r(   ÚcachesÚlts      r*   Ú
make_cachezModel.make_cacheå   sT   € ØˆØ—*‘*×(Ô(ˆBØÐ%Ò%Ø—‘œg›iÕ(à—‘œo°t·y±y×7OÑ7OÔPÕQð	 )ð
 ˆr5   rv   )r,   r-   r.   r   rC   ri   rj   r   r   rf   Úpropertyr‡   r£   rk   rl   s   @r*   r—   r—   Ì   s\   ø„ ðT˜Yõ Tð  $ñ
à—‘ð
ð ˜‰}ð
ð 
�‰ó	
ð ñ!ó ð!ör5   r—   )Údataclassesr   Útypingr   r   r   r   r   Úmlx.coreÚcoreri   Úmlx.nnrE   Úactivationsr
   r@   r   r   r   rT   r   r   Ú
rope_utilsr   r   ÚModuler7   rn   rx   rƒ   r—   r4   r5   r*   Ú<module>r­      s–   ðõ "ß 3Õ 3å Ý å ß TÑ Tß +Ý 'ð ô�ó ó ðô8P#�R—Y‘Yô P#ôfJˆr�y‰yô Jô˜Ÿ	™	ô ô8#�—‘ô #ôL ˆB�I‰Iõ  r5   