Ë
    îÿæiü#  ã                   ó~  — d dl mZ d dlmZmZmZmZmZ d dlm	Z
 d dlmZ d dlmZ ddlmZ ddlmZmZmZ ddlmZmZ dd	lmZ e G d
„ de«      «       Z G d„ dej6                  «      Z G d„ dej6                  «      Z G d„ dej6                  «      Z G d„ dej6                  «      Z G d„ dej6                  «      Z y)é    )Ú	dataclass)ÚAnyÚDictÚListÚOptionalÚUnionN)Úshard_linearé   )Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCacheÚRotatingKVCache)Úinitialize_ropec                   ó2  — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZee   ed	<   dZ	ee   ed
<   dZ
ee   ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeeeeeef   f      ed<   dZeed<   dZeee      ed<   dZee   ed<   d„ Zy)Ú	ModelArgsÚ
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚrms_norm_epsÚ
vocab_sizeNÚhead_dimÚmax_position_embeddingsÚnum_key_value_headsFÚattention_biasÚmlp_biasi'  Ú
rope_thetaÚrope_traditionalÚrope_scalingTÚtie_word_embeddingsÚlayer_typesÚsliding_windowc                 ó‚   — | j                   €| j                  | _         | j                  €dg| j                  z  | _        y y )NÚfull_attention)r   r   r$   r   ©Úselfs    úh/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/llama.pyÚ__post_init__zModelArgs.__post_init__%   sC   € Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$à×ÑÐ#Ø 0Ð1°D×4JÑ4JÑJˆDÕð $ó    )Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatr   r   r   r   r   Úboolr   r    r!   r"   r   r   r#   r$   r   r%   r+   © r,   r*   r   r      sÓ   … àƒOØÓØÓØÓØÓØÓØƒOØ"€Hˆh�s‰mÓ"Ø-1Ð˜X c™]Ó1Ø)-Ð˜ #™Ó-Ø €N�DÓ Ø€HˆdÓØ€J�ÓØ"Ð�dÓ"Ø;?€L�(˜4  U¨5°#¨:Ñ%6Ð 6Ñ7Ñ8Ó?Ø $Ð˜Ó$Ø'+€K�˜$˜s™)Ñ$Ó+Ø$(€N�H˜S‘MÓ(óKr,   r   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	Ú	AttentionÚargsc                 ó¤  •— t         ‰| �  «        |j                  }|j                  x| _        }|j
                  x| _        }|j                  xs |j                  |z  x| _        }|dz  | _        t        |d«      r|j                  }nd}t        j                  |||z  |¬«      | _        t        j                  |||z  |¬«      | _        t        j                  |||z  |¬«      | _        t        j                  ||z  ||¬«      | _        t#        | j                  |j$                  |j&                  |j(                  |j*                  «      | _        y )Ng      à¿r   F©Úbias)ÚsuperÚ__init__r   r   Ún_headsr   Ú
n_kv_headsr   ÚscaleÚhasattrr   ÚnnÚLinearÚq_projÚk_projÚv_projÚo_projr   r    r!   r"   r   Úrope)r)   r8   Údimr>   r?   r   r   Ú	__class__s          €r*   r=   zAttention.__init__.   s  ø€ Ü‰ÑÔà×ÑˆØ!%×!9Ñ!9Ð9ˆŒ�wØ'+×'?Ñ'?Ð?ˆŒ˜*à#'§=¡=Ò#O°D×4DÑ4DÈÑ4OÐOˆŒ˜à˜t‘^ˆŒ
Ü�4Ð)Ô*Ø!×0Ñ0‰Nà"ˆNä—i‘i  W¨xÑ%7¸nÔMˆŒÜ—i‘i  Z°(Ñ%:ÀÔPˆŒÜ—i‘i  Z°(Ñ%:ÀÔPˆŒÜ—i‘i ¨(Ñ 2°C¸nÔMˆŒä#Ø�M‰MØ�O‰OØ×!Ñ!Ø×ÑØ×(Ñ(ó
ˆ�	r,   ÚxÚmaskÚcacheÚreturnc                 ó2  — |j                   \  }}}| j                  |«      | j                  |«      | j                  |«      }	}}|j	                  ||| j
                  d«      j                  dddd«      }|j	                  ||| j                  d«      j                  dddd«      }|	j	                  ||| j                  d«      j                  dddd«      }	|�P| j                  ||j                  ¬«      }| j                  ||j                  ¬«      }|j                  ||	«      \  }}	n"| j                  |«      }| j                  |«      }t        |||	|| j                  |¬«      }
|
j                  dddd«      j	                  ||d«      }
| j                  |
«      S )Néÿÿÿÿr   é   r
   é   )Úoffset)rM   r@   rL   )ÚshaperD   rE   rF   Úreshaper>   Ú	transposer?   rH   rS   Úupdate_and_fetchr   r@   rG   )r)   rK   rL   rM   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs              r*   Ú__call__zAttention.__call__J   si  € ð —'‘'‰ˆˆ1ˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�v�ˆð —/‘/ ! Q¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A˜q $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  1 d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�{‰{˜6Ó"Ð"r,   ©NN)r-   r.   r/   r   r=   ÚmxÚarrayr   r   r_   Ú__classcell__©rJ   s   @r*   r7   r7   -   sW   ø„ ð
˜Yõ 
ð> $(Ø#ñ	#à�8‰8ð#ð �r—x‘xÑ ð#ð ˜‰}ð	#ð
 
�‰÷#r,   r7   c                   óD   ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zˆ xZS )ÚMLPr8   c                 ó6  •— t         ‰| �  «        |j                  }|j                  }t	        |d«      r|j
                  }nd}t        j                  |||¬«      | _        t        j                  |||¬«      | _	        t        j                  |||¬«      | _
        y )Nr   Fr:   )r<   r=   r   r   rA   r   rB   rC   Ú	gate_projÚ	down_projÚup_proj)r)   r8   rI   Ú
hidden_dimr   rJ   s        €r*   r=   zMLP.__init__j   sx   ø€ Ü‰ÑÔà×ÑˆØ×+Ñ+ˆ
Ü�4˜Ô$Ø—}‘}‰HàˆHäŸ™ 3¨
¸ÔBˆŒÜŸ™ :¨s¸ÔBˆŒÜ—y‘y  j°xÔ@ˆ�r,   rN   c                 ót   — | j                  t        | j                  |«      | j                  |«      «      «      S ©N)ri   r   rh   rj   )r)   rK   s     r*   r_   zMLP.__call__x   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr,   )	r-   r.   r/   r   r=   ra   rb   r_   rc   rd   s   @r*   rf   rf   i   s#   ø„ ðA˜Yõ AðJ˜RŸX™X÷ Jr,   rf   c            	       ó�   ‡ — e Zd Zd	dedefˆ fd„Z	 	 d
dej                  deej                     dee	   dej                  fd„Z
ˆ xZS )ÚTransformerBlockr8   Úuse_slidingc                 ó‚  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        t        |«      | _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _        || _        y )N©Úeps)r<   r=   r   r   rp   r7   Ú	self_attnrf   ÚmlprB   ÚRMSNormr   Úinput_layernormÚpost_attention_layernormr8   )r)   r8   rp   rJ   s      €r*   r=   zTransformerBlock.__init__}   s’   ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø×+Ñ+ˆÔØ&ˆÔÜ" 4›ˆŒÜ�t“9ˆŒÜ!Ÿz™z¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
©
Ø×Ñ $×"3Ñ"3ô)
ˆÔ%ð ˆ�	r,   rK   rL   rM   rN   c                 óž   — | j                  | j                  |«      ||«      }||z   }| j                  | j                  |«      «      }||z   }|S rm   )rt   rw   ru   rx   )r)   rK   rL   rM   ÚrÚhÚouts          r*   r_   zTransformerBlock.__call__Š   sQ   € ð �N‰N˜4×/Ñ/°Ó2°D¸%Ó@ˆØ�‰EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�!‰eˆØˆ
r,   )Fr`   )r-   r.   r/   r   r4   r=   ra   rb   r   r   r_   rc   rd   s   @r*   ro   ro   |   s^   ø„ ñ˜Yð °Tõ ð  $(Ø#ñ	
à�8‰8ð
ð �r—x‘xÑ ð
ð ˜‰}ð	
ð
 
�‰÷
r,   ro   c                   óh   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     fd„Zˆ xZ	S )Ú
LlamaModelr8   c                 ó®  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        | j                  dkD  sJ ‚t        j                  |j                  |j                  «      | _
        | j
                  D �cg c]  }t        ||dk(  ¬«      ‘Œ c}| _        t        j                  |j                  |j                  ¬«      | _        | j
                  j!                  d«      | _        d | _        t'        | j                  «      D ]  \  }}|j(                  sŒ|| _         y  y c c}w )Nr   Úsliding_attention)r8   rp   rr   r'   )r<   r=   r8   r   r   r$   r%   rB   Ú	Embeddingr   Úembed_tokensro   Úlayersrv   r   ÚnormÚindexÚfa_idxÚswa_idxÚ	enumeraterp   )r)   r8   Ú
layer_typeÚeÚlrJ   s        €r*   r=   zLlamaModel.__init__˜   s  ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ×+Ñ+ˆÔØ"×1Ñ1ˆÔØ�‰ Ò"Ð"Ð"ÜŸL™L¨¯©¸$×:JÑ:JÓKˆÔð #×.Ò.ó
á.�
ô  $°JÐBUÑ4UÖVØ.ñ
ˆŒô —J‘J˜t×/Ñ/°T×5FÑ5FÔGˆŒ	Ø×&Ñ&×,Ñ,Ð-=Ó>ˆŒØˆŒÜ˜dŸk™kÖ*‰DˆAˆqØ�}‹}Ø �”Ùñ +ùò
s   Â)EÚinputsÚinput_embeddingsc                 óŒ  — |�|}n| j                  |«      }|€d gt        | j                  «      z  }t        ||| j                     «      }| j
                  �%t        ||| j
                     | j                  ¬«      }t        | j                  |«      D ]   \  }}|j                  rn|} ||||¬«      }Œ" | j                  |«      S )N)Úwindow_size)rM   )
r‚   Úlenrƒ   r   r†   r‡   r%   Úziprp   r„   )	r)   rŒ   rM   r�   r{   Úfa_maskÚswa_maskÚlayerrL   s	            r*   r_   zLlamaModel.__call__­   sÀ   € ð Ð'Ø ‰Aà×!Ñ! &Ó)ˆAàˆ=Ø�FœS §¡Ó-Ñ-ˆEä'¨¨5°·±Ñ+=Ó>ˆØ�<‰<Ð#Ü,Ø�5˜Ÿ™Ñ&°D×4GÑ4GôˆHô   §¡¨UÖ3‰LˆE�5Ø$×0Ò0‘8°gˆDÙ�a˜ UÔ+‰Að 4ð �y‰y˜‹|Ðr,   r`   )
r-   r.   r/   r   r=   ra   rb   r   r_   rc   rd   s   @r*   r~   r~   —   s;   ø„ ð˜Yõ ð0 Ø/3ñ	à—‘ðð # 2§8¡8Ñ,÷	r,   r~   c                   óÀ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     fd„Zd„ Z	ddeej                  j                     fd„Zed	„ «       Zd
„ Zˆ xZS )ÚModelr8   c                 óî   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        |j                  s2t        j                  |j                  |j                  d¬«      | _        y y )NFr:   )r<   r=   r8   r   r~   Úmodelr#   rB   rC   r   r   Úlm_head)r)   r8   rJ   s     €r*   r=   zModel.__init__É   sX   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ Ó%ˆŒ
Ø×'Ò'ÜŸ9™9 T×%5Ñ%5°t·±ÈUÔSˆD�Lð (r,   rŒ   r�   c                 óÈ   — | j                  |||«      }| j                  j                  r'| j                   j                  j	                  |«      }|S | j                  |«      }|S rm   )r˜   r8   r#   r‚   Ú	as_linearr™   )r)   rŒ   rM   r�   r|   s        r*   r_   zModel.__call__Ñ   sZ   € ð �j‰j˜ Ð(8Ó9ˆØ�9‰9×(Ò(Ø—*‘*×)Ñ)×3Ñ3°CÓ8ˆCð ˆ
ð —,‘,˜sÓ#ˆCØˆ
r,   c                 ó²   — |j                  «       D ��ci c]  \  }}d|vsŒ||“Œ }}}| j                  j                  r|j                  dd «       |S c c}}w )Nzself_attn.rotary_emb.inv_freqzlm_head.weight)Úitemsr8   r#   Úpop)r)   ÚweightsÚkÚvs       r*   ÚsanitizezModel.sanitizeÞ   s_   € ð %Ÿ]™]œ_ô
Ù,‘T�Q˜Ð0OÐWXÒ0XˆAˆq‰D˜_ð 	ñ 
ð �9‰9×(Ò(Ø�K‰KÐ(¨$Ô/Øˆùó
s
   ”A¡AÚgroupc                 óÌ  — |xs t         j                  j                  «       }|j                  «       }| j                  j
                  D �]˜  }t        |j                  j                  d|¬«      |j                  _        t        |j                  j                  d|¬«      |j                  _	        t        |j                  j                  d|¬«      |j                  _
        t        |j                  j                  d|¬«      |j                  _        |j                  xj                  |z  c_        |j                  xj                  |z  c_        t        |j                  j                  d|¬«      |j                  _        t        |j                  j                   d|¬«      |j                  _        t        |j                  j"                  d|¬«      |j                  _        �Œ› y )Nzall-to-sharded)r£   zsharded-to-all)ra   ÚdistributedÚinitÚsizer˜   rƒ   r	   rt   rD   rE   rF   rG   r>   r?   ru   rh   ri   rj   )r)   r£   ÚNr”   s       r*   ÚshardzModel.shardç   se  € ØÒ.œŸ™×,Ñ,Ó.ˆØ�J‰J‹LˆØ—Z‘Z×&Õ&ˆEä%1Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ô &2Ø—‘×&Ñ&Ð(8Àô&ˆE�O‰OÔ"ð �O‰O×#Ò#¨Ñ)Õ#Ø�O‰O×&Ò&¨1Ñ,Õ&ô #/Ø—	‘	×#Ñ#Ð%5¸Uô#ˆE�I‰IÔô #/Ø—	‘	×#Ñ#Ð%5¸Uô#ˆE�I‰IÔô !-Ø—	‘	×!Ñ!Ð#3¸5ô!ˆE�I‰IÖñ1 'r,   c                 ó.   — | j                   j                  S rm   )r˜   rƒ   r(   s    r*   rƒ   zModel.layers  s   € à�z‰z× Ñ Ð r,   c                 ó¨   — | j                   D �cg c]8  }|j                  r t        | j                  j                  ¬«      n	t        «       ‘Œ: c}S c c}w )N)Úmax_size)rƒ   rp   r   r˜   r%   r   )r)   r”   s     r*   Ú
make_cachezModel.make_cache
  sU   € ð Ÿšó
ñ %�ð ×$Ò$ô  ¨¯©×)BÑ)BÕCä“Yñð %ñ
ð 	
ùò 
s   �=Ar`   rm   )r-   r.   r/   r   r=   ra   rb   r   r_   r¢   r¥   ÚGroupr©   Úpropertyrƒ   r­   rc   rd   s   @r*   r–   r–   È   sv   ø„ ðT˜Yõ Tð Ø/3ñ	à—‘ðð # 2§8¡8Ñ,ó	òñ˜8 B§N¡N×$8Ñ$8Ñ9ó ð> ñ!ó ð!ö
r,   r–   )!Údataclassesr   Útypingr   r   r   r   r   Úmlx.coreÚcorera   Úmlx.nnrB   Úmlx.nn.layers.distributedr	   Úactivationsr   Úbaser   r   r   rM   r   r   Ú
rope_utilsr   r   ÚModuler7   rf   ro   r~   r–   r5   r,   r*   Ú<module>rº      sœ   ðõ "ß 3Õ 3å Ý Ý 2å ß TÑ Tß +Ý 'ð ôK�ó Kó ðKô89#�—	‘	ô 9#ôxJˆ"�)‰)ô Jô&�r—y‘yô ô6.�—‘ô .ôbJ
ˆB�I‰Iõ J
r,   