+
    TV-jÐ   ã                   ó†  € ^ RI Ht ^ RIHtHtHt ^ RIHt ^ RI	H
t
 ^RIHt ^RIHtHtHt ^RIHt ] ! R R]4      4       t ! R	 R
]
P(                  4      t ! R R]
P(                  4      t ! R R]
P(                  4      t ! R R]
P(                  4      t ! R R]
P(                  4      t ! R R]
P(                  4      tR# )é    )Ú	dataclass)ÚAnyÚListÚOptionalN)Úswiglu)ÚBaseModelArgsÚcreate_attention_maskÚscaled_dot_product_attention)Ú	SwitchGLUc                   ó,   a € ] tR t^t o V 3R ltRtV tR# )Ú	ModelArgsc                ó  <€ V ^8„  d   Qh/ S[ ;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[S[,          ;R&   S[;R&   S[;R	&   S[;R
&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   S[;R&   # )é   Ú
model_typeÚhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚattention_biasÚmlp_only_layersÚnum_expertsÚnum_experts_per_tokÚdecoder_sparse_stepÚn_shared_expertsÚmoe_intermediate_sizeÚrms_norm_epsÚ
vocab_sizeÚnum_key_value_headsÚ
rope_thetaÚmax_position_embeddingsÚnorm_topk_prob)ÚstrÚintÚboolr   Úfloat)ÚformatÚ__classdict__s   "€Úd/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_lm/models/Klear.pyÚ__annotate__ÚModelArgs.__annotate__   sï   ø‡ ‚ á�Oñ ñ Ññ ñ Ññ	 ñ
 Ññ ñ Ññ ñ Ññ ñ ™#•YÑñ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ Ññ ñ �Oñ ñ  Ññ! ñ" Ññ# ñ$ !Ñ ñ% ñ& Ñò' ó    © N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__annotate_func__Ú__static_attributes__Ú__classdictcell__)r'   s   @r(   r   r      s   ø‡ ‡ „ r+   r   c                   óT   a a€ ] tR t^$t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚKlearAttentionc                ó    <€ V ^8„  d   QhRS[ /# ©r   Úargs©r   )r&   r'   s   "€r(   r)   ÚKlearAttention.__annotate__%   s   ø€ ÷ $
ñ $
™Yñ $
r+   c                ól  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  VP                  ,          V n        V P
                  R,          V n        \        P                  ! VP                  V P                  V P
                  ,          VP                  R7      V n
        \        P                  ! VP                  V P                  V P
                  ,          VP                  R7      V n        \        P                  ! VP                  V P                  V P
                  ,          VP                  R7      V n        \        P                  ! V P                  V P
                  ,          VP                  VP                  R7      V n        \        P                  ! V P
                  VP                  R7      V n        \        P                  ! V P
                  VP                  R7      V n        \        P$                  ! V P
                  RVP&                  R7      V n        R# )g      à?©Úbias©ÚepsF)ÚtraditionalÚbaseNg      à¿)ÚsuperÚ__init__r   r   r   Úhead_dimÚscaleÚnnÚLinearr   Úq_projÚk_projÚv_projÚo_projÚRMSNormr   Úq_normÚk_normÚRoPEr   Úrope©Úselfr8   Ú	__class__s   &&€r(   rC   ÚKlearAttention.__init__%   sƒ  ø€ Ü‰ÑÔØ#'×#;Ñ#;ˆÔ Ø#'×#;Ñ#;ˆÔ à×(Ñ(¨D×,DÑ,DÕDˆŒØ—]‘] DÕ(ˆŒ
ä—i’iØ×ÑØ×$Ñ$ t§}¡}Õ4Ø×$Ñ$ô
ˆŒô
 —i’iØ×ÑØ×$Ñ$ t§}¡}Õ4Ø×$Ñ$ô
ˆŒô
 —i’iØ×ÑØ×$Ñ$ t§}¡}Õ4Ø×$Ñ$ô
ˆŒô
 —i’iØ×$Ñ$ t§}¡}Õ4Ø×ÑØ×$Ñ$ô
ˆŒô —j’j §¡°D×4EÑ4EÔFˆŒÜ—j’j §¡°D×4EÑ4EÔFˆŒä—G’GØ�M‰MØØ—‘ô
ˆŽ	r+   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   ÚxÚmaskÚcacheÚreturn©ÚmxÚarrayr   r   )r&   r'   s   "€r(   r)   r:   K   sH   ø€ ÷  #ñ  #á�8‰8ð #ñ ‘r—x‘xÕ ð #ñ ™�}ð	 #ñ
 
�‰ñ #r+   c           	     óZ  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pV P	                  VP                  WEV P                  R4      4      P                  ^ ^^^4      pV P                  VP                  WEV P                  R4      4      P                  ^ ^^^4      pV	P                  WEV P                  R4      P                  ^ ^^^4      p	VeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      P                  WER4      p
V P                  V
4      # )é   )Úoffset)rY   rE   rX   éÿÿÿÿ)ÚshaperH   rI   rJ   rM   Úreshaper   Ú	transposerN   r   rP   r`   Úupdate_and_fetchr
   rE   rK   )rR   rW   rX   rY   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r(   Ú__call__ÚKlearAttention.__call__K   sz  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆà—+‘+Ø�O‰O˜A $×":Ñ":¸BÓ?ó
ç
‰)�A�q˜!˜QÓ
ð 	ð �{‰{˜4Ÿ<™<¨¨d×.FÑ.FÈÓKÓL×VÑVØˆq�!�Qó
ˆð —‘  d×&>Ñ&>ÀÓC×MÑMØˆq�!�Qó
ˆð ÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°d·j±jÀtô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r+   )rD   rN   rI   r   r   rK   rM   rH   rP   rE   rJ   ©NN©	r-   r.   r/   r0   rC   rm   r2   r3   Ú__classcell__©rS   r'   s   @@r(   r5   r5   $   s   ù‡ € ÷$
ó $
÷L #÷  #ò  #r+   r5   c                   óD   a a€ ] tR t^nt oV 3R ltV3R lR ltRtVtV ;t# )ÚKlearMLPc                óÒ   <€ \         SV `  4        \        P                  ! WR R7      V n        \        P                  ! W!R R7      V n        \        P                  ! WR R7      V n        R# ©Fr<   N)rB   rC   rF   rG   Ú	gate_projÚ	down_projÚup_proj)rR   ÚdimÚ
hidden_dimrS   s   &&&€r(   rC   ÚKlearMLP.__init__o   sD   ø€ Ü‰ÑÔÜŸš 3¸Ô?ˆŒÜŸš :¸Ô?ˆŒÜ—y’y °uÔ=ˆŽr+   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   rZ   ©r\   r]   )r&   r'   s   "€r(   r)   ÚKlearMLP.__annotate__u   s   ø€ ÷ Jñ J™RŸX™Xñ Jr+   c                ót   € V P                  \        V P                  V4      V P                  V4      4      4      # ©N)rx   r   rw   ry   )rR   rW   s   &&r(   rm   ÚKlearMLP.__call__u   s)   € Ø�~‰~œf T§^¡^°AÓ%6¸¿¹ÀQ»ÓHÓIÐIr+   )rx   rw   ry   rp   rr   s   @@r(   rt   rt   n   s   ù‡ € õ>÷J÷ Jð Jr+   rt   c                   óP   a a€ ] tR t^yt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚKlearSparseMoeBlockc                ó    <€ V ^8„  d   QhRS[ /# r7   r9   )r&   r'   s   "€r(   r)   Ú KlearSparseMoeBlock.__annotate__z   s   ø€ ÷ Kñ K™Yñ Kr+   c                ót  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \        P                  ! VP                  VP                  R R7      V n	        \        VP                  VP                  VP                  4      V n        \        VP                  VP                  VP                  ,          R7      V n        \        P                  ! VP                  ^4      V n        \"        P$                  ! V P                  3\"        P&                  R7      V n        R# )Fr<   )r{   )ÚdtypeN)rB   rC   r!   r   r   Útop_krF   rG   r   Úgater   r   Úexpertsrt   r   Úshared_expertsÚcoefficientr\   ÚzerosÚfloat32Úexpert_biasrQ   s   &&€r(   rC   ÚKlearSparseMoeBlock.__init__z   sã   ø€ Ü‰ÑÔØ"×1Ñ1ˆÔØ×+Ñ+ˆÔØ×-Ñ-ˆŒ
ä—I’I˜d×.Ñ.°×0@Ñ0@ÀuÔMˆŒ	Ü Ø×Ñ˜d×8Ñ8¸$×:JÑ:Jó
ˆŒô 'Ø×ÑØ×1Ñ1°D×4IÑ4IÕIô
ˆÔô Ÿ9š9 T×%5Ñ%5°qÓ9ˆÔÜŸ8š8 T×%5Ñ%5Ð$7¼r¿z¹zÔJˆÖr+   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   rW   rZ   r~   )r&   r'   s   "€r(   r)   r†   ‹   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r+   c                ó  € \         P                  ! V P                  V4      P                  \         P                  4      4      pW P
                  P                  R	4      ,           pV P                  p\         P                  ! V) V^,
          RR7      RRV13,          p\         P                  ! W%RR7      pV P                  '       d!   V\         P                  ! VRRR7      ,          pVP                  VP                  4      pV P                  W4      pWvR
,          ,          P                  RR7      p\         P                  ! V P                  V4      RRR7      p	V P!                  V4      p
W‰R,          ,          W©R,          ,          ,           pV# )r_   )ÚkthÚaxis.N)r•   T)r•   Úkeepdims)r•   Úprecisera   )r_   r_   ra   ).Néþÿÿÿ).:Nr_   N).:r_   NN)r\   ÚsigmoidrŠ   Úastyper�   r�   rc   r‰   ÚargpartitionÚtake_along_axisr!   Úsumrˆ   r‹   Úsoftmaxr�   rŒ   )rR   rW   Úrouting_weightsÚbiased_weightsÚkÚindsÚscoresÚ
expert_outÚ	y_expertsÚcoefÚsharedÚys   &&          r(   rm   ÚKlearSparseMoeBlock.__call__‹   s(  € ÜŸ*š* T§Y¡Y¨q£\×%8Ñ%8¼¿¹Ó%DÓEˆØ(×+;Ñ+;×+CÑ+CÀJÓ+OÕOˆØ�J‰JˆÜ�Š ˜°A¸µEÀÔCÀCÈÈ!ÈÀGÕLˆÜ×#Ò# OÀÔCˆØ××ÐØœbŸfšf V°"¸tÔDÕDˆFØ—‘˜qŸw™wÓ'ˆØ—\‘\ !Ó*ˆ
Ø¨Õ"3Õ3×8Ñ8¸bÐ8ÓAˆ	Ü�zŠz˜$×*Ñ*¨1Ó-°BÀÔEˆØ×$Ñ$ QÓ'ˆØ˜W�Õ%¨°gµÕ(>Õ>ˆØˆr+   )r�   r�   r‹   rŠ   r!   r   rŒ   r‰   rp   rr   s   @@r(   r„   r„   y   s    ù‡ € ÷Kó K÷"÷ ð r+   r„   c                   óT   a a€ ] tR t^œt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚKlearDecoderLayerc                ó&   <€ V ^8„  d   QhRS[ RS[/# )r   r8   Ú	layer_idx)r   r#   )r&   r'   s   "€r(   r)   ÚKlearDecoderLayer.__annotate__�   s   ø€ ÷ 
ñ 
™Yð 
±3ñ 
r+   c                óô  <€ \         SV `  4        \        V4      V n        W!P                  9  dB   VP
                  ^ 8”  d1   V^,           VP                  ,          ^ 8X  d   \        V4      V n        M%\        VP                  VP                  4      V n        \        P                  ! VP                  VP                  R7      V n        \        P                  ! VP                  VP                  R7      V n        R# )r   r>   N)rB   rC   r5   Ú	self_attnr   r   r   r„   Úmlprt   r   r   rF   rL   r   Úinput_layernormÚpost_attention_layernorm)rR   r8   r­   rS   s   &&&€r(   rC   ÚKlearDecoderLayer.__init__�   s²   ø€ Ü‰ÑÔÜ'¨Ó-ˆŒà×1Ñ1Ô1Ø×Ñ˜qÔ  i°!¥m°t×7OÑ7OÕ%OÐSTÔ%Tä*¨4Ó0ˆD�Hä × 0Ñ 0°$×2HÑ2HÓIˆDŒHä!Ÿzšz¨$×*:Ñ*:À×@QÑ@QÔRˆÔÜ(*¯
ª
Ø×Ñ $×"3Ñ"3ô)
ˆÖ%r+   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# rV   r[   )r&   r'   s   "€r(   r)   r®   ­   sH   ø€ ÷ 
ñ 
á�8‰8ð
ñ ‘r—x‘xÕ ð
ñ ™�}ð	
ñ
 
�‰ñ
r+   c                ó¨   € V P                  V P                  V4      W#4      pW,           pV P                  V P                  V4      4      pWT,           pV# r�   )r°   r²   r±   r³   )rR   rW   rX   rY   ÚrÚhÚouts   &&&&   r(   rm   ÚKlearDecoderLayer.__call__­   sK   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ�EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�eˆØˆ
r+   )r²   r±   r³   r°   ro   rp   rr   s   @@r(   r«   r«   œ   s   ù‡ € ÷
ó 
÷ 
÷ 
ò 
r+   r«   c                   óT   a a€ ] tR t^ºt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú
KlearModelc                ó    <€ V ^8„  d   QhRS[ /# r7   r9   )r&   r'   s   "€r(   r)   ÚKlearModel.__annotate__»   s   ø€ ÷ Hñ H™Yñ Hr+   c                óX  <€ \         SV `  4        \        P                  ! VP                  VP
                  4      V n        \        VP                  4       Uu. uF  p\        WR 7      NK  	  upV n
        \        P                  ! VP
                  VP                  R7      V n        R# u upi ))r8   r­   r>   N)rB   rC   rF   Ú	Embeddingr   r   Úembed_tokensÚranger   r«   ÚlayersrL   r   Únorm)rR   r8   ÚirS   s   && €r(   rC   ÚKlearModel.__init__»   s�   ø€ Ü‰ÑÔÜŸLšL¨¯©¸$×:JÑ:JÓKˆÔô ˜4×1Ñ1Ô2ó
á2�ô  4×5Ù2ñ
ˆŒô —J’J˜t×/Ñ/°T×5FÑ5FÔGˆŽ	ùò	
s   ÁB'c                ód   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[ P                  /# )r   ÚinputsrY   rZ   r[   )r&   r'   s   "€r(   r)   r¾   Ä   s5   ø€ ÷ ñ á—‘ðñ ™�}ðñ 
�‰ñ	r+   c                óü   € V P                  V4      pVf   R .\        V P                  4      ,          p\        W2^ ,          4      p\	        V P                  V4       F  w  rVV! W4V4      pK  	  V P                  V4      # r�   )rÁ   ÚlenrÃ   r	   ÚziprÄ   )rR   rÈ   rY   r¸   rX   ÚlayerÚcs   &&&    r(   rm   ÚKlearModel.__call__Ä   so   € ð
 ×Ñ˜fÓ%ˆàŠ=Ø�FœS §¡Ó-Õ-ˆEä$ Q¨a­Ó1ˆä˜DŸK™K¨Ö/‰HˆEÙ�a˜qÓ!ŠAñ 0ð �y‰y˜‹|Ðr+   )rÁ   rÃ   rÄ   r�   rp   rr   s   @@r(   r¼   r¼   º   s    ù‡ € ÷Hó H÷÷ ò r+   r¼   c                   óŠ   a a€ ] tR t^Öt oV3R lV 3R lltR
V3R lR lltR t]R 4       t]R 4       t	]R 4       t
R	tVtV ;t# )ÚModelc                ó    <€ V ^8„  d   QhRS[ /# r7   r9   )r&   r'   s   "€r(   r)   ÚModel.__annotate__×   s   ø€ ÷ Pñ P™Yñ Pr+   c                óÖ   <€ \         SV `  4        Wn        VP                  V n        \	        V4      V n        \        P                  ! VP                  VP                  R R7      V n
        R# rv   )rB   rC   r8   r   r¼   ÚmodelrF   rG   r   r   Úlm_headrQ   s   &&€r(   rC   ÚModel.__init__×   sH   ø€ Ü‰ÑÔØŒ	ØŸ/™/ˆŒÜ Ó%ˆŒ
Ü—y’y ×!1Ñ!1°4·?±?ÈÔOˆŽr+   c                óJ   <€ V ^8„  d   QhRS[ P                  RS[S[,          /# )r   rÈ   rY   r[   )r&   r'   s   "€r(   r)   rÒ   Þ   s'   ø€ ÷ !ñ !á—‘ð!ñ ™�}ñ!r+   c                óF   € V P                  W4      pV P                  V4      # r�   )rÔ   rÕ   )rR   rÈ   rY   r¹   s   &&& r(   rm   ÚModel.__call__Þ   s!   € ð
 �j‰j˜Ó'ˆØ�|‰|˜CÓ Ð r+   c                óT  € R V9  d   V# \        V P                  P                  4       Fv  pRV R2pR Fg  p\        V P                  P                  4       Uu. uF  pVP	                  V RV RV R24      NK  	  pp\
        P                  ! V4      W RV R2&   Ki  	  Kx  	  V# u upi )z-model.layers.0.mlp.experts.0.gate_proj.weightzmodel.layers.z.mlp.expertsÚ.z.weight)rw   ry   rx   )rÂ   r8   r   r   Úpopr\   Ústack)rR   ÚweightsÚlÚprefixÚnameÚeÚstackeds   &&     r(   ÚsanitizeÚModel.sanitizeæ   s¹   € Ø:À'ÔIØˆNä�t—y‘y×2Ñ2Ö3ˆAØ$ Q C |Ð4ˆFÛ=�ô # 4§9¡9×#8Ñ#8Ô9óá9˜ð —K‘K 6 (¨!¨A¨3¨a°¨v°WÐ =Ö>Ù9ð ð ô 68·X²X¸gÓ5F�˜( ! D 6¨Ð1Ó2ó >ñ 4ð ˆùòs   Á#B%c                ó.   € V P                   P                  # r�   )rÔ   rÃ   )rR   s   &r(   rÃ   ÚModel.layersõ   s   € à�z‰z× Ñ Ð r+   c                ó   € R  pV# )c                 ó@   € V P                  R 4      '       d   R^@R^/# R# )zmlp.gateÚ
group_sizeÚbitsT)Úendswith)ÚpathÚ_s   &&r(   Ú	predicateÚ(Model.quant_predicate.<locals>.predicateû   s#   € Ø�}‰}˜Z×(Ò(Ø$ b¨&°!Ð4Ð4Ùr+   r,   ©rR   rï   s   & r(   Úquant_predicateÚModel.quant_predicateù   s   € ò	ð
 Ðr+   c                ó   € R  pV# )c                 ó   € R V 9  # )r�   r,   )r¡   s   &r(   rï   Ú'Model.cast_predicate.<locals>.predicate  s   € Ø ¨Ñ)Ð)r+   r,   rñ   s   & r(   Úcast_predicateÚModel.cast_predicate  s   € ò	*ð Ðr+   )r8   rÕ   rÔ   r   r�   )r-   r.   r/   r0   rC   rm   rä   ÚpropertyrÃ   rò   r÷   r2   r3   rq   rr   s   @@r(   rÐ   rÐ   Ö   sa   ù‡ € ÷Pó P÷!ò !òð ñ!ó ð!ð ñó ðð ñó ÷ð r+   rÐ   )Údataclassesr   Útypingr   r   r   Úmlx.coreÚcorer\   Úmlx.nnrF   Úactivationsr   rA   r   r	   r
   Úswitch_layersr   r   ÚModuler5   rt   r„   r«   r¼   rÐ   r,   r+   r(   Ú<module>r     s£   ðõ "ß &Ñ &å Ý å ß TÑ TÝ $ð ô�ó ó ðô*G#�R—Y‘Yô G#ôTJˆr�y‰yô Jô ˜"Ÿ)™)ô  ôF˜Ÿ	™	ô ô<�—‘ô ô81ˆB�I‰Iö 1r+   