+
    TV-j-  ã                   ó@  € ^ RI Ht ^ RIHtHt ^ RIHt ^ RIH	t	 ^RI
HtHtHt ] ! R R]4      4       t ! R R]	P                  4      t ! R	 R
]	P                  4      t ! R R]	P                  4      t ! R R]	P                  4      t ! R R]	P                  4      tR# )é    )Ú	dataclass)ÚAnyÚOptionalN)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attentionc                   ó:   a € ] tR t^t o RtRtR tV 3R ltRtV t	R# )Ú	ModelArgsTNc                óH   € V P                   f   V P                  V n         R # R # ©N)Únum_key_value_headsÚnum_attention_heads©Úselfs   &Úg/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/gpt_neox.pyÚ__post_init__ÚModelArgs.__post_init__   s!   € Ø×#Ñ#Ò+Ø'+×'?Ñ'?ˆDÖ$ñ ,ó    c                óž   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   # )é   Ú
model_typeÚmax_position_embeddingsÚhidden_sizer   Únum_hidden_layersÚlayer_norm_epsÚ
vocab_sizeÚrotary_emb_baseÚ
rotary_pctÚuse_parallel_residualr   )ÚstrÚintÚfloatÚbool)ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚModelArgs.__annotate__   s�   ø‡ ‚ á�Oñ ñ !Ñ ñ ñ Ññ	 ñ
 Ññ ñ Ññ ñ Ññ ñ �Oñ ñ Ññ ñ Ññ ñ  Ñ&ñ ñ Ñ#ò r   )r   )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r%   s   @r   r
   r
      s    ø‡ € ð #'ÐØ#Ðò@÷ ƒ r   r
   c                   óT   a a€ ] tR t^"t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úargs©r
   )r$   r%   s   "€r   r&   ÚAttention.__annotate__#   s   ø€ ÷ Nñ N™Yñ Nr   c                óœ  <€ \         SV `  4        VP                  VP                  ,          ^ 8X  g   Q R4       hVP                  V n        VP                  V n        V P                  V P                  ,          V n        \
        P                  ! \        V P                  VP                  ,          4      RVP                  R7      V n
        V P                  R,          V n        \
        P                  ! V P                  ^V P                  ,          RR7      V n        \
        P                  ! V P                  V P                  RR7      V n        R# )r   z4hidden_size must be divisible by num_attention_headsF)ÚdimsÚtraditionalÚbaseT©ÚbiasNg      à¿)ÚsuperÚ__init__r   r   Úhead_dimÚnnÚRoPEr!   r   r   ÚropeÚscaleÚLinearÚquery_key_valueÚdense©r   r3   Ú	__class__s   &&€r   r=   ÚAttention.__init__#   sÿ   ø€ Ü‰ÑÔð ×Ñ˜t×7Ñ7Õ7¸1Ô<ð	BàAó	BØ<ð  ×+Ñ+ˆÔØ#'×#;Ñ#;ˆÔ Ø×(Ñ(¨D×,DÑ,DÕDˆŒä—G’GÜ�T—]‘] T§_¡_Õ4Ó5ØØ×%Ñ%ô
ˆŒ	ð —]‘] DÕ(ˆŒ
ä!ŸyšyØ×Ñ˜a $×"2Ñ"2Õ2¸ô 
ˆÔô —Y’Y˜t×/Ñ/°×1AÑ1AÈÔMˆŽ
r   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   ÚxÚmaskÚcacheÚreturn©ÚmxÚarrayr   r   )r$   r%   s   "€r   r&   r5   ;   sH   ø€ ÷ "ñ "á�8‰8ð"ñ ‘r—x‘xÕ ð"ñ ™�}ð	"ñ
 
�‰ñ"r   c           	     ó¾  € VP                   w  rEpV P                  V4      pVP                   R R V P                  ^V P                  ,          3,           pVP                  ! V!  pVP                  ^R4       Uu. uF  qP                  ^ ^^^4      NK  	  upw  ršpVeM   V P                  W“P                  R7      p	V P                  W£P                  R7      p
VP                  W«4      w  r«M"V P                  V	4      p	V P                  V
4      p
\        WšW³V P                  VR7      pVP                  ^ ^^^4      P	                  WER4      pV P                  V4      # u upi )N)Úoffset)rM   rB   rL   éÿÿÿÿ)ÚshaperD   r   r>   ÚreshapeÚsplitÚ	transposerA   rS   Úupdate_and_fetchr   rB   rE   )r   rK   rL   rM   ÚBÚLÚDÚqkvÚnew_qkv_shapeÚqueriesÚkeysÚvaluesÚoutputs   &&&&         r   Ú__call__ÚAttention.__call__;   s8  € ð —'‘'‰ˆˆaà×"Ñ" 1Ó%ˆàŸ	™	 # 2˜¨$×*BÑ*BÀAÈÏÉÕDUÐ)VÕVˆØ�kŠk˜=Ñ)ˆàBEÇ)Á)ÈAÈrÔBRÓ SÑBR¸Q§¡¨Q°°1°aÖ!8ÑBRÑ SÑˆ�vàÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�z‰z˜&Ó!Ð!ùò !Ts   Á8E)rE   r>   r   r   rD   rA   rB   ©NN©	r(   r)   r*   r+   r=   rc   r-   r.   Ú__classcell__©rG   r%   s   @@r   r0   r0   "   s    ù‡ € ÷Nó N÷0"÷ "ò "r   r0   c                   óP   a a€ ] tR t^Zt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚMLPc                ó    <€ V ^8„  d   QhRS[ /# r2   r4   )r$   r%   s   "€r   r&   ÚMLP.__annotate__[   s   ø€ ÷ Oñ O™Yñ Or   c                ó"  <€ \         SV `  4        VP                  V n        \        P                  ! V P                  ^V P                  ,          4      V n        \        P                  ! ^V P                  ,          V P                  4      V n        R# )é   N)r<   r=   r   r?   rC   Údense_h_to_4hÚdense_4h_to_hrF   s   &&€r   r=   ÚMLP.__init__[   sd   ø€ Ü‰ÑÔà×+Ñ+ˆÔÜŸYšY t×'7Ñ'7¸¸T×=MÑ=MÕ9MÓNˆÔÜŸYšY q¨4×+;Ñ+;Õ';¸T×=MÑ=MÓNˆÖr   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   rN   ©rP   rQ   )r$   r%   s   "€r   r&   rl   b   s   ø€ ÷ Iñ I™RŸX™Xñ Ir   c                ój   € V P                  \        P                  ! V P                  V4      4      4      # r   )rp   r?   Úgelu_approxro   )r   rK   s   &&r   rc   ÚMLP.__call__b   s'   € à×!Ñ!¤"§.¢.°×1CÑ1CÀAÓ1FÓ"GÓHÐHr   )rp   ro   r   rf   rh   s   @@r   rj   rj   Z   s#   ù‡ € ÷Oó O÷I÷ Ið Ir   rj   c                   óT   a a€ ] tR t^gt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚTransformerBlockc                ó    <€ V ^8„  d   QhRS[ /# r2   r4   )r$   r%   s   "€r   r&   ÚTransformerBlock.__annotate__h   s   ø€ ÷ 
ñ 
™Yñ 
r   c                óŽ  <€ \         SV `  4        VP                  V n        VP                  V n        \	        V4      V n        \        V4      V n        VP                  V n        \        P                  ! V P                  V P                  R 7      V n        \        P                  ! V P                  V P                  R 7      V n        R# )©ÚepsN)r<   r=   r   r   r0   Ú	attentionrj   Úmlpr   r?   Ú	LayerNormÚinput_layernormÚpost_attention_layernormrF   s   &&€r   r=   ÚTransformerBlock.__init__h   s•   ø€ Ü‰ÑÔà×+Ñ+ˆÔØ"×1Ñ1ˆÔÜ" 4›ˆŒÜ�t“9ˆŒØ%)×%?Ñ%?ˆÔ"Ü!Ÿ|š|Ø×ÑØ×#Ñ#ô 
ˆÔô )+¯ªØ×Ñ $×"5Ñ"5ô)
ˆÖ%r   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# rJ   rO   )r$   r%   s   "€r   r&   rz   x   sH   ø€ ÷ ñ á�8‰8ðñ ‘r—x‘xÕ ðñ ™�}ð	ñ
 
�‰ñr   c                ót  € V P                   '       dU   TpV P                  V P                  V4      W#4      pV P                  V P	                  V4      4      pWV,           V,           pV# V P                  V P                  V4      W#4      pW,           pV P                  V P	                  V4      4      p	W,           pV# r   )r   r~   r�   r   r‚   )
r   rK   rL   rM   ÚresidualÚattnÚffnÚoutÚattn_outputÚ
ffn_outputs
   &&&&      r   rc   ÚTransformerBlock.__call__x   s£   € ð ×%×%Ð%ØˆHà—>‘> $×"6Ñ"6°qÓ"9¸4ÓGˆDØ—(‘(˜4×8Ñ8¸Ó;Ó<ˆCØ•*˜xÕ'ˆCØˆJð Ÿ.™.¨×)=Ñ)=¸aÓ)@À$ÓNˆKØ•ˆAØŸ™ $×"?Ñ"?ÀÓ"BÓCˆJØ•ˆAØˆHr   )r~   r   r�   r   r   r‚   r   re   rf   rh   s   @@r   rx   rx   g   s   ù‡ € ÷
ó 
÷ ÷ ò r   rx   c                   óT   a a€ ] tR t^Žt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚGPTNeoXModelc                ó    <€ V ^8„  d   QhRS[ /# r2   r4   )r$   r%   s   "€r   r&   ÚGPTNeoXModel.__annotate__�   s   ø€ ÷ 
Xñ 
X™Yñ 
Xr   c                ój  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP
                  V n        V P                  ^ 8”  g   Q h\        P                  ! V P                  V P                  4      V n        \        P                  ! VP                  VP                  RR7      V n
        \        V P                  4       Uu. uF  p\        VR7      NK  	  upV n        \        P                  ! V P                  V P
                  R7      V n        R# u upi )r   Fr:   )r3   r|   N)r<   r=   r   r   r   r   r?   Ú	EmbeddingÚembed_inrC   Ú	embed_outÚrangerx   Úhr€   Úfinal_layer_norm)r   r3   Ú_rG   s   && €r   r=   ÚGPTNeoXModel.__init__�   sÝ   ø€ Ü‰ÑÔØ×+Ñ+ˆÔØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔØ"×1Ñ1ˆÔØ�‰ Ô"Ð"Ð"ÜŸš T§_¡_°d×6FÑ6FÓGˆŒÜŸš 4×#3Ñ#3°T·_±_È5ÔQˆŒÜ7<¸T×=SÑ=SÔ7TÓUÑ7T°!Ô"¨×-Ñ7TÑUˆŒÜ "§¢¨T×-=Ñ-=À4×CVÑCVÔ WˆÖùò Vs   Ã D0c                ó4   <€ V ^8„  d   QhRS[ P                  /# ©r   Úinputsrs   )r$   r%   s   "€r   r&   r�   ›   s   ø€ ÷ ñ á—‘ñr   c                ó@  € VP                   w  r4V P                  V4      pVf   R .\        V P                  4      ,          p\	        WR^ ,          4      p\        V P                  V4       F  w  rxV! WVVR7      pK  	  V P                  V4      p	V P                  V	4      p	V	# )N)rM   )rU   r“   Úlenr–   r   Úzipr—   r”   )
r   rœ   rM   r˜   r[   Úhidden_statesrL   ÚlayerÚcr‰   s
   &&&       r   rc   ÚGPTNeoXModel.__call__›   sŒ   € ð
 �|‰|‰ˆàŸ™ fÓ-ˆàŠ=Ø�FœS §¡›[Õ(ˆEä$ ]¸!µHÓ=ˆä˜DŸF™F EÖ*‰HˆEÙ! -¸QÔ?ŠMñ +ð ×#Ñ# MÓ2ˆØ�n‰n˜SÓ!ˆàˆ
r   )r“   r”   r—   r–   r   r   r   r   r   rf   rh   s   @@r   rŽ   rŽ   Ž   s    ù‡ € ÷
Xó 
X÷÷ ò r   rŽ   c                   ój   a a€ ] tR t^²t oV3R lV 3R lltRV3R lR lltR t]R 4       tRt	Vt
V ;t# )	ÚModelc                ó    <€ V ^8„  d   QhRS[ /# r2   r4   )r$   r%   s   "€r   r&   ÚModel.__annotate__³   s   ø€ ÷ (ñ (™Yñ (r   c                ór   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        R # r   )r<   r=   r3   r   rŽ   ÚmodelrF   s   &&€r   r=   ÚModel.__init__³   s*   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ! $Ó'ˆŽ
r   c                ó4   <€ V ^8„  d   QhRS[ P                  /# r›   rs   )r$   r%   s   "€r   r&   r§   ¹   s   ø€ ÷ ñ á—‘ñr   c                ó(   € V P                  W4      pV# r   )r©   )r   rœ   rM   r‰   s   &&& r   rc   ÚModel.__call__¹   s   € ð
 �j‰j˜Ó'ˆØˆ
r   c                ó,  € / pVP                  4        F}  w  r4. ROpRpV F  pVP                  V4      '       g   K  Rp M	  V'       d   K9  VP                  R4      '       g   RV 2pVP                  RR4      pVP                  RR4      pWBV&   K  	  V# )	ú.attention.biasFTzmodel.z.gpt_neox.layers.z.h.z
.gpt_neox.Ú.)r¯   z.attention.masked_biasz.attention.rotary_emb.inv_freq)ÚitemsÚendswithÚ
startswithÚreplace)r   ÚweightsÚnew_weightsÚw_keyÚw_valueÚignore_suffixesÚskip_weightÚignored_suffixs   &&      r   ÚsanitizeÚModel.sanitizeÁ   sž   € Øˆà%Ÿm™mžo‰NˆEòˆOð  ˆKÛ"1�Ø—>‘> .×1Ô1Ø"&�KÙñ #2÷
 Ùà×#Ñ# H×-Ò-Ø   Ð(�à—M‘MÐ"5°uÓ=ˆEØ—M‘M ,°Ó4ˆEà!(˜Óñ/ .ð2 Ðr   c                ó.   € V P                   P                  # r   )r©   r–   r   s   &r   ÚlayersÚModel.layersß   s   € à�z‰z�|‰|Ðr   )r3   r©   r   r   )r(   r)   r*   r+   r=   rc   r¼   Úpropertyr¿   r-   r.   rg   rh   s   @@r   r¥   r¥   ²   s7   ù‡ € ÷(ó (÷ò òð< ñó ÷ð r   r¥   )Údataclassesr   Útypingr   r   Úmlx.coreÚcorerP   Úmlx.nnr?   r9   r   r   r   r
   ÚModuler0   rj   rx   rŽ   r¥   © r   r   Ú<module>rÉ      sŒ   ðõ "ß  å Ý ç TÑ Tð ô@�ó @ó ð@ô$5"�—	‘	ô 5"ôp
Iˆ"�)‰)ô 
Iô$�r—y‘yô $ôN!�2—9‘9ô !ôH/ˆB�I‰Iö /r   