Ë
    îÿæi-  ã                   ó>  — d dl mZ d dlmZmZ d dlmZ d dlm	Z	 ddl
mZmZmZ e G d„ de«      «       Z G d„ d	e	j                  «      Z G d
„ de	j                  «      Z G d„ de	j                  «      Z G d„ de	j                  «      Z G d„ de	j                  «      Zy)é    )Ú	dataclass)ÚAnyÚOptionalNé   )ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attentionc                   óŠ   — e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
Zeed<   dZ	eed<   d„ Z
y)Ú	ModelArgsÚ
model_typeÚmax_position_embeddingsÚhidden_sizeÚnum_attention_headsÚnum_hidden_layersÚlayer_norm_epsÚ
vocab_sizeÚrotary_emb_baseÚ
rotary_pctTÚuse_parallel_residualNÚnum_key_value_headsc                 ó@   — | j                   €| j                  | _         y y ©N)r   r   ©Úselfs    úk/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/gpt_neox.pyÚ__post_init__zModelArgs.__post_init__   s!   € Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÕ$ð ,ó    )Ú__name__Ú
__module__Ú__qualname__ÚstrÚ__annotations__ÚintÚfloatr   Úboolr   r   © r   r   r   r      sR   … àƒOØ Ó ØÓØÓØÓØÓØƒOØÓØÓØ"&Ð˜4Ó&Ø#Ð˜Ó#ó@r   r   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	Ú	AttentionÚargsc                 óh  •— t         ‰| �  «        |j                  |j                  z  dk(  sJ d«       ‚|j                  | _        |j                  | _        | j                  | j                  z  | _        t        j                  t        | j                  |j                  z  «      d|j                  ¬«      | _
        | j                  dz  | _        t        j                  | j                  d| j                  z  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        y )	Nr   z4hidden_size must be divisible by num_attention_headsF)ÚdimsÚtraditionalÚbaseg      à¿é   T©Úbias)ÚsuperÚ__init__r   r   Úhead_dimÚnnÚRoPEr#   r   r   ÚropeÚscaleÚLinearÚquery_key_valueÚdense©r   r)   Ú	__class__s     €r   r2   zAttention.__init__#   sÿ   ø€ Ü‰ÑÔð ×Ñ˜t×7Ñ7Ñ7¸1Ò<ð	BàAó	BØ<ð  ×+Ñ+ˆÔØ#'×#;Ñ#;ˆÔ Ø×(Ñ(¨D×,DÑ,DÑDˆŒä—G‘GÜ�T—]‘] T§_¡_Ñ4Ó5ØØ×%Ñ%ô
ˆŒ	ð —]‘] DÑ(ˆŒ
ä!Ÿy™yØ×Ñ˜a $×"2Ñ"2Ñ2¸ô 
ˆÔô —Y‘Y˜t×/Ñ/°×1AÑ1AÈÔMˆ�
r   ÚxÚmaskÚcacheÚreturnc           	      ó¸  — |j                   \  }}}| j                  |«      }|j                   d d | j                  d| j                  z  fz   } |j                  |Ž }|j                  dd«      D �cg c]  }|j                  dddd«      ‘Œ c}\  }	}
}|�P| j                  |	|j                  ¬«      }	| j                  |
|j                  ¬«      }
|j                  |
|«      \  }
}n"| j                  |	«      }	| j                  |
«      }
t        |	|
||| j                  |¬«      }|j                  dddd«      j	                  ||d«      }| j                  |«      S c c}w )Néÿÿÿÿr.   r   é   r   )Úoffset)r?   r7   r>   )Úshaper9   r   r3   ÚreshapeÚsplitÚ	transposer6   rD   Úupdate_and_fetchr	   r7   r:   )r   r=   r>   r?   ÚBÚLÚDÚqkvÚnew_qkv_shapeÚqueriesÚkeysÚvaluesÚoutputs                r   Ú__call__zAttention.__call__;   sL  € ð —'‘'‰ˆˆ1ˆaà×"Ñ" 1Ó%ˆàŸ	™	 # 2˜¨$×*BÑ*BÀAÈÏÉÑDUÐ)VÑVˆØˆc�k‰k˜=Ð)ˆàBEÇ)Á)ÈAÈrÔBRÓ SÑBR¸Q §¡¨Q°°1°aÕ!8ÐBRÑ SÑˆ��vàÐØ—i‘i °·±�iÓ=ˆGØ—9‘9˜T¨%¯,©,�9Ó7ˆDØ ×1Ñ1°$¸Ó?‰LˆD‘&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø�T˜6¨°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸¸BÓ?ˆØ�z‰z˜&Ó!Ð!ùò !Ts   Á0E©NN©r   r   r    r   r2   ÚmxÚarrayr   r   rS   Ú__classcell__©r<   s   @r   r(   r(   "   sY   ø„ ðN˜Yõ Nð6 $(Ø#ñ	"à�8‰8ð"ð �r—x‘xÑ ð"ð ˜‰}ð	"ð
 
�‰÷"r   r(   c                   óD   ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zˆ xZS )ÚMLPr)   c                 ó  •— t         ‰| �  «        |j                  | _        t        j                  | j                  d| j                  z  «      | _        t        j                  d| j                  z  | j                  «      | _        y )Né   )r1   r2   r   r4   r8   Údense_h_to_4hÚdense_4h_to_hr;   s     €r   r2   zMLP.__init__[   sd   ø€ Ü‰ÑÔà×+Ñ+ˆÔÜŸY™Y t×'7Ñ'7¸¸T×=MÑ=MÑ9MÓNˆÔÜŸY™Y q¨4×+;Ñ+;Ñ';¸T×=MÑ=MÓNˆÕr   r@   c                 óh   — | j                  t        j                  | j                  |«      «      «      S r   )r_   r4   Úgelu_approxr^   )r   r=   s     r   rS   zMLP.__call__b   s'   € à×!Ñ!¤"§.¡.°×1CÑ1CÀAÓ1FÓ"GÓHÐHr   ©	r   r   r    r   r2   rV   rW   rS   rX   rY   s   @r   r[   r[   Z   s#   ø„ ðO˜Yõ OðI˜RŸX™X÷ Ir   r[   c            	       óŠ   ‡ — e Zd Zdefˆ fd„Z	 	 ddej                  deej                     dee   dej                  fd„Z	ˆ xZ
S )	ÚTransformerBlockr)   c                 óˆ  •— t         ‰| �  «        |j                  | _        |j                  | _        t	        |«      | _        t        |«      | _        |j                  | _        t        j                  | j                  | j                  ¬«      | _        t        j                  | j                  | j                  ¬«      | _        y )N©Úeps)r1   r2   r   r   r(   Ú	attentionr[   Úmlpr   r4   Ú	LayerNormÚinput_layernormÚpost_attention_layernormr;   s     €r   r2   zTransformerBlock.__init__h   s•   ø€ Ü‰ÑÔà×+Ñ+ˆÔØ"×1Ñ1ˆÔÜ" 4›ˆŒÜ�t“9ˆŒØ%)×%?Ñ%?ˆÔ"Ü!Ÿ|™|Ø×ÑØ×#Ñ#ô 
ˆÔô )+¯©Ø×Ñ $×"5Ñ"5ô)
ˆÕ%r   r=   r>   r?   r@   c                 óR  — | j                   rN|}| j                  | j                  |«      ||«      }| j                  | j	                  |«      «      }||z   |z   }|S | j                  | j                  |«      ||«      }||z   }| j                  | j	                  |«      «      }	||	z   }|S r   )r   rh   rk   ri   rl   )
r   r=   r>   r?   ÚresidualÚattnÚffnÚoutÚattn_outputÚ
ffn_outputs
             r   rS   zTransformerBlock.__call__x   sª   € ð ×%Ò%ØˆHà—>‘> $×"6Ñ"6°qÓ"9¸4ÀÓGˆDØ—(‘(˜4×8Ñ8¸Ó;Ó<ˆCØ˜‘*˜xÑ'ˆCØˆJð Ÿ.™.¨×)=Ñ)=¸aÓ)@À$ÈÓNˆKØ�K‘ˆAØŸ™ $×"?Ñ"?ÀÓ"BÓCˆJØ�J‘ˆAØˆHr   rT   rU   rY   s   @r   rd   rd   g   sW   ø„ ð
˜Yõ 
ð& $(Ø#ñ	à�8‰8ðð �r—x‘xÑ ðð ˜‰}ð	ð
 
�‰÷r   rd   c                   óH   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zˆ xZS )ÚGPTNeoXModelr)   c                 óZ  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        |j
                  | _        | j                  dkD  sJ ‚t        j                  | j                  | j                  «      | _        t        j                  |j                  |j                  d¬«      | _
        t        | j                  «      D �cg c]  }t        |¬«      ‘Œ c}| _        t        j                  | j                  | j
                  ¬«      | _        y c c}w )Nr   Fr/   )r)   rf   )r1   r2   r   r   r   r   r4   Ú	EmbeddingÚembed_inr8   Ú	embed_outÚrangerd   Úhrj   Úfinal_layer_norm)r   r)   Ú_r<   s      €r   r2   zGPTNeoXModel.__init__�   sÝ   ø€ Ü‰ÑÔØ×+Ñ+ˆÔØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ"×1Ñ1ˆÔØ�‰ Ò"Ð"Ð"ÜŸ™ T§_¡_°d×6FÑ6FÓGˆŒÜŸ™ 4×#3Ñ#3°T·_±_È5ÔQˆŒÜ7<¸T×=SÑ=SÔ7TÓUÑ7T°!Ô"¨Ö-Ð7TÑUˆŒÜ "§¡¨T×-=Ñ-=À4×CVÑCVÔ WˆÕùò Vs   ÃD(Úinputsc                 ó0  — |j                   \  }}| j                  |«      }|€d gt        | j                  «      z  }t	        ||d   «      }t        | j                  |«      D ]  \  }} ||||¬«      }Œ | j                  |«      }	| j                  |	«      }	|	S )Nr   )r?   )rE   rx   Úlenr{   r   Úzipr|   ry   )
r   r~   r?   r}   rK   Úhidden_statesr>   ÚlayerÚcrq   s
             r   rS   zGPTNeoXModel.__call__›   s”   € ð
 �|‰|‰ˆˆ1àŸ™ fÓ-ˆàˆ=Ø�FœS §¡›[Ñ(ˆEä$ ]°E¸!±HÓ=ˆä˜DŸF™F EÖ*‰HˆE�1Ù! -°¸QÔ?‰Mð +ð ×#Ñ# MÓ2ˆØ�n‰n˜SÓ!ˆàˆ
r   r   rb   rY   s   @r   ru   ru   Ž   s'   ø„ ð
X˜Yõ 
Xð ñà—‘÷r   ru   c                   ó^   ‡ — e Zd Zdefˆ fd„Z	 ddej                  fd„Zd„ Ze	d„ «       Z
ˆ xZS )ÚModelr)   c                 ór   •— t         ‰| �  «        || _        |j                  | _        t	        |«      | _        y r   )r1   r2   r)   r   ru   Úmodelr;   s     €r   r2   zModel.__init__³   s,   ø€ Ü‰ÑÔØˆŒ	ØŸ/™/ˆŒÜ! $Ó'ˆ�
r   r~   c                 ó*   — | j                  ||«      }|S r   )rˆ   )r   r~   r?   rq   s       r   rS   zModel.__call__¹   s   € ð
 �j‰j˜ Ó'ˆØˆ
r   c                 ó  — i }|j                  «       D ]i  \  }}g d¢}d}|D ]  }|j                  |«      sŒd} n |rŒ+|j                  d«      sd|› �}|j                  dd«      }|j                  dd«      }|||<   Œk |S )	N)z.attention.biasz.attention.masked_biasz.attention.rotary_emb.inv_freqFTzmodel.z.gpt_neox.layers.z.h.z
.gpt_neox.Ú.)ÚitemsÚendswithÚ
startswithÚreplace)r   ÚweightsÚnew_weightsÚw_keyÚw_valueÚignore_suffixesÚskip_weightÚignored_suffixs           r   ÚsanitizezModel.sanitizeÁ   sœ   € Øˆà%Ÿm™mžo‰NˆE�7òˆOð  ˆKÛ"1�Ø—>‘> .Õ1Ø"&�KÙð #2ñ
 Øà×#Ñ# HÔ-Ø   Ð(�à—M‘MÐ"5°uÓ=ˆEØ—M‘M ,°Ó4ˆEà!(ˆK˜Òð/ .ð2 Ðr   c                 ó.   — | j                   j                  S r   )rˆ   r{   r   s    r   ÚlayerszModel.layersß   s   € à�z‰z�|‰|Ðr   r   )r   r   r    r   r2   rV   rW   rS   r—   Úpropertyr™   rX   rY   s   @r   r†   r†   ²   s>   ø„ ð(˜Yõ (ð ñà—‘óòð< ñó ôr   r†   )Údataclassesr   Útypingr   r   Úmlx.coreÚcorerV   Úmlx.nnr4   r-   r   r   r	   r   ÚModuler(   r[   rd   ru   r†   r&   r   r   Ú<module>r¡      sŒ   ðõ "ß  å Ý ç TÑ Tð ô@�ó @ó ð@ô$5"�—	‘	ô 5"ôp
Iˆ"�)‰)ô 
Iô$�r—y‘yô $ôN!�2—9‘9ô !ôH/ˆB�I‰Iõ /r   