+
    TV-j¥  ã                   óT  € ^ RI Ht ^ RIHtHtHtHt ^ RIHt	 ^ RI
Ht ^RIHtHtHt ^RIHt ] ! R R]4      4       t ! R R	]P&                  4      t ! R
 R]P&                  4      t ! R R]P&                  4      t ! R R]P&                  4      t ! R R]P&                  4      tR# )é    )Ú	dataclass)ÚAnyÚDictÚOptionalÚUnionN)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Ú	SwitchGLUc                   óf   a € ] tR t^t o RtRtRt^ t^ t^t	^t
^tRtRtRtRtRtR tV 3R	 ltR
tV tR# )Ú	ModelArgsi }  i   i 8  gñhãˆµøä>g    €„.AFNc                óH   € V P                   f   V P                  V n         R # R # ©N)Únum_key_value_headsÚnum_attention_heads©Úselfs   &Úf/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/mixtral.pyÚ__post_init__ÚModelArgs.__post_init__   s!   € Ø×#Ñ#Ò+Ø'+×'?Ñ'?ˆDÖ$ñ ,ó    c                óþ   <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[S[S[ S[S[S[ 3,          3,          ,          ;R&   S[;R&   # )é   Ú
model_typeÚ
vocab_sizeÚhidden_sizeÚintermediate_sizeÚnum_hidden_layersr   Únum_experts_per_tokr   Únum_local_expertsÚrms_norm_epsÚ
rope_thetaÚrope_traditionalÚrope_scalingÚtie_word_embeddings)ÚstrÚintÚfloatÚboolr   r   r   )ÚformatÚ__classdict__s   "€r   Ú__annotate__ÚModelArgs.__annotate__   sÐ   ø‡ ‚ á�Oñ ñ Ññ ñ Ññ	 ñ
 Ñ"ñ ñ Ññ ñ Ñ!ñ ñ Ñ ñ ñ Ñ ñ ñ Ññ ñ Ññ ñ Ññ ñ Ñ"ñ ñ ™4¡¡U©5±#¨:Õ%6Ð 6Õ7Õ8Ñ?ñ ñ Ñ%ò r   )r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r   Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r+   s   @r   r   r      s]   ø‡ € ð €JØ€KØ"ÐØÐØ!ÐØ ÐØ ÐØÐØ€LØ€JØ"ÐØ;?€LØ %Ðò@÷# ƒ r   r   c                   óT   a a€ ] tR t^#t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMixtralAttentionc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úargs©r   )r*   r+   s   "€r   r,   ÚMixtralAttention.__annotate__$   s   ø€ ÷ 
ñ 
™Yñ 
r   c                ó°  <€ \         SV `  4        VP                  V n        VP                  V n        V P                  V P                  ,          V n        VP                  V n        VP                  V n        V P
                  R,          V n        \        P                  ! V P                  V P                  V P
                  ,          RR7      V n        \        P                  ! V P                  V P                  V P
                  ,          RR7      V n        \        P                  ! V P                  V P                  V P
                  ,          RR7      V n        \        P                  ! V P                  V P
                  ,          V P                  RR7      V n        \        P                  ! V P
                  VP                   VP                  R7      V n        R# )g      à?F©Úbias)ÚtraditionalÚbaseNg      à¿)ÚsuperÚ__init__r   r   Ú	num_headsÚhead_dimr   r"   ÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚo_projÚRoPEr#   Úrope©r   r9   Ú	__class__s   &&€r   rB   ÚMixtralAttention.__init__$   sC  ø€ Ü‰ÑÔØ×+Ñ+ˆÔØ×1Ñ1ˆŒØ×(Ñ(¨D¯N©NÕ:ˆŒØ#'×#;Ñ#;ˆÔ ØŸ/™/ˆŒà—]‘] DÕ(ˆŒ
ä—i’iØ×Ñ˜dŸn™n¨t¯}©}Õ<À5ô
ˆŒô —i’iØ×Ñ˜d×6Ñ6¸¿¹ÕFÈUô
ˆŒô —i’iØ×Ñ˜d×6Ñ6¸¿¹ÕFÈUô
ˆŒô —i’iØ�N‰N˜TŸ]™]Õ*¨D×,<Ñ,<À5ô
ˆŒô —G’GØ�M‰MØ×-Ñ-Ø—‘ô
ˆŽ	r   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   ÚxÚmaskÚcacheÚreturn©ÚmxÚarrayr   r   )r*   r+   s   "€r   r,   r;   A   sH   ø€ ÷ #ñ #á�8‰8ð#ñ ‘r—x‘xÕ ð#ñ ™�}ð	#ñ
 
�‰ñ#r   c           	     ó  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pVP	                  WEV P
                  R4      P                  ^ ^^^4      pVP	                  WEV P                  R4      P                  ^ ^^^4      pV	P	                  WEV P                  R4      P                  ^ ^^^4      p	VeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      P	                  WER4      p
V P                  V
4      # )é   )Úoffset)rU   rE   rT   éÿÿÿÿ)ÚshaperH   rI   rJ   ÚreshaperC   Ú	transposer   rM   r\   Úupdate_and_fetchr
   rE   rK   )r   rS   rT   rU   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r   Ú__call__ÚMixtralAttention.__call__A   s^  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆð —/‘/ !¨¯©¸Ó;×EÑEÀaÈÈAÈqÓQˆØ�|‰|˜A $×":Ñ":¸BÓ?×IÑIÈ!ÈQÐPQÐSTÓUˆØ—‘  d×&>Ñ&>ÀÓC×MÑMØˆq�!�Qó
ˆð ÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r   )rD   r   rI   rC   r   rK   rH   rM   r"   rE   rJ   ©NN©	r.   r/   r0   r1   rB   ri   r3   r4   Ú__classcell__©rO   r+   s   @@r   r6   r6   #   s   ù‡ € ÷
ó 
÷:#÷ #ò #r   r6   c                   óP   a a€ ] tR t^at oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚMixtralSparseMoeBlockc                ó    <€ V ^8„  d   QhRS[ /# r8   r:   )r*   r+   s   "€r   r,   Ú"MixtralSparseMoeBlock.__annotate__b   s   ø€ ÷ 
Uñ 
U™Yñ 
Ur   c                óp  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP                  V n        \        P                  ! V P                  V P                  R R7      V n        \        V P                  V P
                  V P                  4      V n        R# ©Fr=   N)rA   rB   r   Ú
hidden_dimr   Úffn_dimr    Únum_expertsr   rF   rG   Úgater   Ú
switch_mlprN   s   &&€r   rB   ÚMixtralSparseMoeBlock.__init__b   s‚   ø€ Ü‰ÑÔØ×*Ñ*ˆŒØ×-Ñ-ˆŒØ×1Ñ1ˆÔØ#'×#;Ñ#;ˆÔ ô —I’I˜dŸo™o¨t×/?Ñ/?ÀeÔLˆŒ	ä# D§O¡O°T·\±\À4×CSÑCSÓTˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   rS   rV   )rX   rY   )r*   r+   s   "€r   r,   rr   n   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                ó‚  € V P                  V4      pV P                  p\        P                  ! \        P                  ! V) V^,
          RR7      RRV13,          4      p\        P
                  ! W$RR7      p\        P                  ! VRRR7      pV P                  W4      pWeR,          ,          P                  R	R7      pV# )
r[   )ÚkthÚaxis.N)r~   T)r~   Úpreciser]   ).Néþÿÿÿ)	rx   r   rX   Ústop_gradientÚargpartitionÚtake_along_axisÚsoftmaxry   Úsum)r   rS   ÚgatesÚkÚindsÚscoresÚys   &&     r   ri   ÚMixtralSparseMoeBlock.__call__n   sŸ   € Ø—	‘	˜!“ˆà×$Ñ$ˆÜ×Ò¤§¢°°¸AÀ½EÈÔ KÈCÐQSÐRSÐQSÈGÕ TÓUˆÜ×#Ò# E°bÔ9ˆÜ—’˜F¨°TÔ:ˆà�O‰O˜AÓ$ˆØ˜	Õ"Õ"×'Ñ'¨RÐ'Ó0ˆàˆr   )rv   rx   ru   rw   r   ry   rl   rn   s   @@r   rp   rp   a   s    ù‡ € ÷
Uó 
U÷÷ ð r   rp   c                   óT   a a€ ] tR t^|t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMixtralDecoderLayerc                ó    <€ V ^8„  d   QhRS[ /# r8   r:   )r*   r+   s   "€r   r,   Ú MixtralDecoderLayer.__annotate__}   s   ø€ ÷ 

ñ 

™Yñ 

r   c                óJ  <€ \         SV `  4        VP                  V n        \        V4      V n        \        V4      V n        \        P                  ! VP                  VP                  R 7      V n
        \        P                  ! VP                  VP                  R 7      V n        R# )©ÚepsN)rA   rB   r   r6   Ú	self_attnrp   Úblock_sparse_moerF   ÚRMSNormr!   Úinput_layernormÚpost_attention_layernormrN   s   &&€r   rB   ÚMixtralDecoderLayer.__init__}   sv   ø€ Ü‰ÑÔØ×+Ñ+ˆÔä)¨$Ó/ˆŒä 5°dÓ ;ˆÔÜ!Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
ª
Ø×Ñ $×"3Ñ"3ô)
ˆÖ%r   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# rR   rW   )r*   r+   s   "€r   r,   r�   ‰   sH   ø€ ÷ 
ñ 
á�8‰8ð
ñ ‘r—x‘xÕ ð
ñ ™�}ð	
ñ
 
�‰ñ
r   c                ó¨   € V P                  V P                  V4      W#4      pW,           pV P                  V P                  V4      4      pWT,           pV# r   )r“   r–   r”   r—   )r   rS   rT   rU   ÚrÚhÚouts   &&&&   r   ri   ÚMixtralDecoderLayer.__call__‰   sM   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ�EˆØ×!Ñ! $×"?Ñ"?ÀÓ"BÓCˆØ�eˆØˆ
r   )r”   r   r–   r—   r“   rk   rl   rn   s   @@r   r�   r�   |   s   ù‡ € ÷

ó 

÷
÷ 
ò 
r   r�   c                   óT   a a€ ] tR t^–t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚMixtralModelc                ó    <€ V ^8„  d   QhRS[ /# r8   r:   )r*   r+   s   "€r   r,   ÚMixtralModel.__annotate__—   s   ø€ ÷ 	Hñ 	H™Yñ 	Hr   c                óœ  <€ \         SV `  4        VP                  V n        VP                  V n        \        P
                  ! VP                  VP                  4      V n        \        VP                  4       Uu. uF  p\        VR 7      NK  	  upV n
        \        P                  ! VP                  VP                  R7      V n        R# u upi ))r9   r‘   N)rA   rB   r   r   rF   Ú	Embeddingr   Úembed_tokensÚranger�   Úlayersr•   r!   Únorm)r   r9   Ú_rO   s   && €r   rB   ÚMixtralModel.__init__—   s–   ø€ Ü‰ÑÔØŸ/™/ˆŒØ!%×!7Ñ!7ˆÔäŸLšL¨¯©¸$×:JÑ:JÓKˆÔä49¸$×:PÑ:PÔ4Qó
Ù4Q¨qÔ T×*Ñ4Qñ
ˆŒô —J’J˜t×/Ñ/°T×5FÑ5FÔGˆŽ	ùò
s   Á9C	c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[ P                  /# ©r   ÚinputsÚinput_embeddingsrV   ©rX   rY   r   )r*   r+   s   "€r   r,   r¢   ¢   s:   ø€ ÷ ñ á—‘ðñ #¡2§8¡8Õ,ð	ñ
 
�‰ñr   c                ó
  € Ve   TpMV P                  V4      pVf   R .\        V P                  4      ,          p\        WB^ ,          4      p\	        V P                  V4       F  w  rgV! WEV4      pK  	  V P                  V4      # r   )r¥   Úlenr§   r	   Úzipr¨   )r   r­   rU   r®   rœ   rT   ÚlayerÚcs   &&&&    r   ri   ÚMixtralModel.__call__¢   sz   € ð Ò'Ø ‰Aà×!Ñ! &Ó)ˆAàŠ=Ø�FœS §¡Ó-Õ-ˆEä$ Q¨a­Ó1ˆä˜DŸK™K¨Ö/‰HˆEÙ�a˜qÓ!ŠAñ 0ð �y‰y˜‹|Ðr   )r¥   r§   r¨   r   r   rk   rl   rn   s   @@r   r    r    –   s    ù‡ € ÷	Hó 	H÷÷ ò r   r    c                   ój   a a€ ] tR t^¸t oV3R lV 3R lltRV3R lR lltR t]R 4       tRt	Vt
V ;t# )	ÚModelc                ó    <€ V ^8„  d   QhRS[ /# r8   r:   )r*   r+   s   "€r   r,   ÚModel.__annotate__¹   s   ø€ ÷ Tñ T™Yñ Tr   c                óþ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        VP                  '       g5   \        P                  ! VP                  VP                  R R7      V n        R# R# rt   )rA   rB   r9   r   r    Úmodelr%   rF   rG   r   r   Úlm_headrN   s   &&€r   rB   ÚModel.__init__¹   sY   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ! $Ó'ˆŒ
Ø×'×'Ð'ÜŸ9š9 T×%5Ñ%5°t·±ÈUÔSˆDŽLñ (r   c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[ P                  /# r¬   r¯   )r*   r+   s   "€r   r,   r¹   Á   s:   ø€ ÷ 
%ñ 
%á—‘ð
%ñ #¡2§8¡8Õ,ð	
%ñ
 
�‰ñ
%r   c                óÊ   € V P                  WV4      pV P                  P                  '       d&   V P                   P                  P	                  V4      # V P                  V4      # r   )r»   r9   r%   r¥   Ú	as_linearr¼   )r   r­   rU   r®   r�   s   &&&& r   ri   ÚModel.__call__Á   sO   € ð �j‰j˜Ð(8Ó9ˆØ�9‰9×(×(Ð(Ø—:‘:×*Ñ*×4Ñ4°SÓ9Ð9à—<‘< Ó$Ð$r   c                óî  € V P                   P                  '       d   VP                  R R4       RV9  d   V# \        V P                   P                  4       F•  pRV 2pR F‡  w  rER	 F|  pV RV RV 2V9   g   K  \        V P                   P
                  4       Uu. uF  pVP                  V RV RV RV 24      NK!  	  pp\        P                  ! V4      W RV RV 2&   K~  	  K‰  	  K—  	  V# u upi )
zlm_head.weightNz3model.layers.0.block_sparse_moe.experts.0.w1.weightzmodel.layers.z.block_sparse_moe.experts.0.Ú.z.block_sparse_moe.experts.z.block_sparse_moe.switch_mlp.))Úw1Ú	gate_proj)Úw2Ú	down_proj)Úw3Úup_proj)ÚweightÚscalesÚbiases)r9   r%   Úpopr¦   r   r    rX   Ústack)	r   ÚweightsÚlÚprefixÚnÚmr‡   ÚeÚto_joins	   &&       r   ÚsanitizeÚModel.sanitizeÍ   s  € Ø�9‰9×(×(Ð(Ø�K‰KÐ(¨$Ô/Ø@ÈÔOØˆNÜ�t—y‘y×2Ñ2Ö3ˆAØ$ Q CÐ(ˆFÛU‘�Û7�AØ ˜Ð!=¸a¸SÀÀ!ÀÐEÈÖPô
 &+¨4¯9©9×+FÑ+FÔ%Gó	#ñ &H ð $ŸK™KØ#) (Ð*DÀQÀCÀqÈÈÈ1ÈQÈCÐ Pöñ &Hð	  ð #ô ŸHšH WÓ-ð   (Ð*GÈÀsÈ!ÈAÈ3Ð OÓPó 8ó Vñ 4ð ˆùò#s   Â%C2
c                ó.   € V P                   P                  # r   )r»   r§   r   s   &r   r§   ÚModel.layersâ   s   € à�z‰z× Ñ Ð r   )r9   r¼   r»   r   rk   )r.   r/   r0   r1   rB   ri   rÖ   Úpropertyr§   r3   r4   rm   rn   s   @@r   r·   r·   ¸   s9   ù‡ € ÷Tó T÷
%ò 
%òð* ñ!ó ÷!ð !r   r·   )Údataclassesr   Útypingr   r   r   r   Úmlx.coreÚcorerX   Úmlx.nnrF   r@   r   r	   r
   Úswitch_layersr   r   ÚModuler6   rp   r�   r    r·   © r   r   Ú<module>rã      s�   ðõ "ß -Ó -å Ý ç TÑ TÝ $ð ô@�ó @ó ð@ô*;#�r—y‘yô ;#ô|˜BŸI™Iô ô6˜"Ÿ)™)ô ô4�2—9‘9ô ôD,!ˆB�I‰Iö ,!r   