+
    UV-jW  ã                   ó(  € ^ RI Ht ^ RIHt ^ RIHt ^RIHtH	t	H
t
 ^RIHt ^RIHt  ! R R]P                  4      t ! R R	]P                  4      t ! R
 R]P                  4      t ! R R]P                  4      t ! R R]P                  4      tR# )é    )ÚOptionalN)ÚLanguageModelOutputÚcreate_attention_maskÚscaled_dot_product_attention)ÚKVCache©Ú
TextConfigc                   óT   a a€ ] tR t^t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )Ú	Attentionc                ó    <€ V ^8„  d   QhRS[ /# ©é   Úconfigr   )ÚformatÚ__classdict__s   "€Ún/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/llava/language.pyÚ__annotate__ÚAttention.__annotate__   s   ø€ ÷ !
ñ !
™zñ !
ó    c                ó  <€ \         SV `  4        VP                  pVP                  ;V n        pVP
                  ;V n        pW4,          V n        VP                  V,          pVR
,          V n        VP                  R8X  d   RpMRp\        P                  ! W#V,          VR7      V n        \        P                  ! W$V,          VR7      V n        \        P                  ! W$V,          VR7      V n        \        P                  ! W5,          VRR7      V n        VP                   e3   VP                   R,          R8X  d   ^VP                   R,          ,          M^p\        P"                  ! VVP$                  VP&                  VR	7      V n        R# )g      à?Úqwen2TF©ÚbiasNÚtypeÚlinearÚfactor)ÚtraditionalÚbaseÚscaleg      à¿)ÚsuperÚ__init__Úhidden_sizeÚnum_attention_headsÚn_headsÚnum_key_value_headsÚ
n_kv_headsÚrepeatsr   Ú
model_typeÚnnÚLinearÚq_projÚk_projÚv_projÚo_projÚrope_scalingÚRoPEÚrope_traditionalÚ
rope_thetaÚrope)	Úselfr   Údimr$   r&   Úhead_dimÚattention_biasÚ
rope_scaleÚ	__class__s	   &&      €r   r!   ÚAttention.__init__   s@  ø€ Ü‰ÑÔà× Ñ ˆØ!'×!;Ñ!;Ð;ˆŒ�wØ'-×'AÑ'AÐAˆŒ˜*àÕ,ˆŒà×%Ñ%¨Õ0ˆØ˜t•^ˆŒ
à×Ñ Ô'Ø!‰Nà"ˆNä—i’i ¨xÕ%7¸nÔMˆŒÜ—i’i °(Õ%:ÀÔPˆŒÜ—i’i °(Õ%:ÀÔPˆŒÜ—i’i Õ 2°C¸eÔDˆŒð ×"Ñ"Ò.Ø×#Ñ# FÕ+¨xÔ7ð �×#Ñ# HÕ-Ö-ð ð	 	ô —G’GØØ×/Ñ/Ø×"Ñ"Øô	
ˆŽ	r   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# ©r   ÚxÚmaskÚcacheÚreturn©ÚmxÚarrayr   r   )r   r   s   "€r   r   r   3   sI   ø€ ÷ #ñ #á�8‰8ð#ñ ‘r—x‘xÕ ð#ñ ™Õ ð	#ñ
 
�‰ñ#r   c           	     ó  € VP                   w  rEpV P                  V4      V P                  V4      V P                  V4      r˜pVP	                  WEV P
                  R4      P                  ^ ^^^4      pVP	                  WEV P                  R4      P                  ^ ^^^4      pV	P	                  WEV P                  R4      P                  ^ ^^^4      p	VeM   V P                  WsP                  R7      pV P                  WƒP                  R7      pVP                  W‰4      w  r‰M"V P                  V4      pV P                  V4      p\        WxW“V P                  VR7      p
V
P                  ^ ^^^4      P	                  WER4      p
V P                  V
4      # )é   )Úoffset)r   r>   éÿÿÿÿ)Úshaper+   r,   r-   Úreshaper$   Ú	transposer&   r3   rF   Úupdate_and_fetchr   r   r.   )r4   r=   r>   r?   ÚBÚLÚDÚqueriesÚkeysÚvaluesÚoutputs   &&&&       r   Ú__call__ÚAttention.__call__3   sS  € ð —'‘'‰ˆˆaà $§¡¨A£°·±¸A³ÀÇÁÈAÃ�vˆð —/‘/ !¨¯©°bÓ9×CÑCÀAÀqÈ!ÈQÓOˆØ�|‰|˜A $§/¡/°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ—‘  d§o¡o°rÓ:×DÑDÀQÈÈ1ÈaÓPˆàÒØ—i‘i ·±�iÓ=ˆGØ—9‘9˜T¯,©,�9Ó7ˆDØ ×1Ñ1°$Ó?‰LˆD�&à—i‘i Ó(ˆGØ—9‘9˜T“?ˆDä-Ø˜6°·
±
Àô
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�{‰{˜6Ó"Ð"r   )	r,   r$   r&   r.   r+   r'   r3   r   r-   ©NN©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r!   rS   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r9   r   s   @@r   r   r      s   ù‡ € ÷!
ó !
÷F#÷ #ò #r   r   c                   óD   a a€ ] tR t^Qt oV 3R ltV3R lR ltRtVtV ;t# )ÚMLPc                óÒ   <€ \         SV `  4        \        P                  ! WR R7      V n        \        P                  ! W!R R7      V n        \        P                  ! WR R7      V n        R# )Fr   N)r    r!   r)   r*   Ú	gate_projÚ	down_projÚup_proj)r4   r5   Ú
hidden_dimr9   s   &&&€r   r!   ÚMLP.__init__R   sD   ø€ Ü‰ÑÔÜŸš 3¸Ô?ˆŒÜŸš :¸Ô?ˆŒÜ—y’y °uÔ=ˆŽr   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r   r@   )rB   rC   )r   r   s   "€r   r   ÚMLP.__annotate__X   s   ø€ ÷ Lñ L™RŸX™Xñ Lr   c                ó–   € V P                  \        P                  ! V P                  V4      4      V P	                  V4      ,          4      # ©N)rc   r)   Úsilurb   rd   )r4   r=   s   &&r   rS   ÚMLP.__call__X   s0   € Ø�~‰~œbŸgšg d§n¡n°QÓ&7Ó8¸4¿<¹<È»?ÕJÓKÐKr   )rc   rb   rd   rV   r^   s   @@r   r`   r`   Q   s   ù‡ € õ>÷L÷ Lð Lr   r`   c                   óT   a a€ ] tR t^\t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚTransformerBlockc                ó    <€ V ^8„  d   QhRS[ /# r   r   )r   r   s   "€r   r   ÚTransformerBlock.__annotate__]   s   ø€ ÷ 
ñ 
™zñ 
r   c                ó¢  <€ \         SV `  4        VP                  V n        VP                  V n        \	        V4      V n        \        VP                  VP                  4      V n        \        P                  ! VP                  VP                  R 7      V n        \        P                  ! VP                  VP                  R 7      V n        Wn        R# )©ÚepsN)r    r!   r#   r"   r   Ú	self_attnr`   Úintermediate_sizeÚmlpr)   ÚRMSNormÚrms_norm_epsÚinput_layernormÚpost_attention_layernormr   ©r4   r   r9   s   &&€r   r!   ÚTransformerBlock.__init__]   s˜   ø€ Ü‰ÑÔØ#)×#=Ñ#=ˆÔ Ø!×-Ñ-ˆÔÜ" 6Ó*ˆŒÜ�v×)Ñ)¨6×+CÑ+CÓDˆŒÜ!Ÿzšz¨&×*<Ñ*<À&×BUÑBUÔVˆÔÜ(*¯
ª
Ø×Ñ F×$7Ñ$7ô)
ˆÔ%ð Žr   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[,          RS[ P                  /# r<   rA   )r   r   s   "€r   r   rp   i   sI   ø€ ÷ 
ñ 
á�8‰8ð
ñ ‘r—x‘xÕ ð
ñ ™Õ ð	
ñ
 
�‰ñ
r   c                ó¨   € V P                  V P                  V4      W#4      pW,           pV P                  V P                  V4      4      pWT,           pV# rj   )rt   ry   rv   rz   )r4   r=   r>   r?   ÚrÚhÚouts   &&&&   r   rS   ÚTransformerBlock.__call__i   sK   € ð �N‰N˜4×/Ñ/°Ó2°DÓ@ˆØ�EˆØ�H‰H�T×2Ñ2°1Ó5Ó6ˆØ�eˆØˆ
r   )r   r"   ry   rv   r#   rz   rt   rU   rV   r^   s   @@r   rn   rn   \   s   ù‡ € ÷
ó 
÷
÷ 
ò 
r   rn   c                   óT   a a€ ] tR t^vt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚLlamac                ó    <€ V ^8„  d   QhRS[ /# r   r   )r   r   s   "€r   r   ÚLlama.__annotate__w   s   ø€ ÷ 
Lñ 
L™zñ 
Lr   c                óÎ  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        V P                  ^ 8”  g   Q h\
        P                  ! VP                  VP                  4      V n        \        VP                  4       Uu. uF  p\        VR7      NK  	  upV n        \
        P                  ! VP                  VP                  R7      V n        R# u upi )r   )r   rr   N)r    r!   r   Ú
vocab_sizeÚnum_hidden_layersr)   Ú	Embeddingr"   Úembed_tokensÚrangern   Úlayersrw   rx   Únorm)r4   r   Ú_r9   s   && €r   r!   ÚLlama.__init__w   s±   ø€ Ü‰ÑÔØŒØ ×+Ñ+ˆŒØ!'×!9Ñ!9ˆÔØ�‰ Ô"Ð"Ð"ÜŸLšL¨×):Ñ):¸F×<NÑ<NÓOˆÔä5:¸6×;SÑ;SÔ5Tó
Ù5T°Ô F×+Ñ5Tñ
ˆŒô —J’J˜v×1Ñ1°v×7JÑ7JÔKˆŽ	ùò
s   ÂC"c                óˆ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          /# ©r   ÚinputsÚinputs_embedsr>   ©rB   rC   r   )r   r   s   "€r   r   r†   ƒ   s?   ø€ ÷ ñ á—‘ðñ  ¡§¡Õ)ðñ ‘r—x‘xÕ ñ	r   c                óü   € Vf   V P                  V4      pMTpVf   R .\        V P                  4      ,          p\        WT4      p\	        V P                  V4       F  w  rgV! WSV4      pK  	  V P                  V4      # rj   )r‹   Úlenr�   r   ÚziprŽ   )r4   r“   r”   r>   r?   r€   ÚlayerÚcs   &&&&&   r   rS   ÚLlama.__call__ƒ   sx   € ð Ò Ø×!Ñ! &Ó)‰AàˆAàŠ=Ø�FœS §¡Ó-Õ-ˆEô % QÓ.ˆä˜DŸK™K¨Ö/‰HˆEÙ�a˜qÓ!ŠAñ 0ð �y‰y˜‹|Ðr   )r   r‹   r�   rŽ   r‰   rˆ   ©NNNrV   r^   s   @@r   r„   r„   v   s    ù‡ € ÷
Ló 
L÷÷ ò r   r„   c                   ó”   a a€ ] tR t^œt oV3R lV 3R lltR
V3R lR llt]R 4       t]R 4       t	]R 4       t
]R 4       tR	tVtV ;t# )ÚLanguageModelc                ó    <€ V ^8„  d   QhRS[ /# r   r   )r   r   s   "€r   r   ÚLanguageModel.__annotate__�   s   ø€ ÷ 
Xñ 
X™zñ 
Xr   c                óR  <€ \         SV `  4        Wn        VP                  V n        V P                  R9  d   \	        RV P                   R24      h\        V4      V n        VP                  '       g5   \        P                  ! VP                  VP                  RR7      V n        R# R# )ÚllamazModel type z1 not supported. Supported types: 'llama', 'qwen2'Fr   N)r¢   r   )r    r!   r   r(   Ú
ValueErrorr„   ÚmodelÚtie_word_embeddingsr)   r*   r"   rˆ   Úlm_headr{   s   &&€r   r!   ÚLanguageModel.__init__�   s‹   ø€ Ü‰ÑÔØŒØ ×+Ñ+ˆŒØ�?‰?Ð"4Ô4ÜØ˜dŸo™oÐ.Ð._Ð`óð ô ˜6“]ˆŒ
Ø×)×)Ð)ÜŸ9š9 V×%7Ñ%7¸×9JÑ9JÐQVÔWˆDŽLñ *r   c                óˆ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          /# r’   r•   )r   r   s   "€r   r   r    ©   s?   ø€ ÷ /ñ /á—‘ð/ñ  ¡§¡Õ)ð/ñ ‘r—x‘xÕ ñ	/r   c                óæ   € V P                  WWBR 7      pV P                  P                  '       d'   V P                   P                  P	                  V4      pMV P                  V4      p\        VR7      # ))r>   r?   r”   )Úlogits)r¤   r   r¥   r‹   Ú	as_linearr¦   r   )r4   r“   r”   r>   r?   Úkwargsr�   s   &&&&&, r   rS   ÚLanguageModel.__call__©   sY   € ð �j‰j˜°%ˆjÓUˆØ�;‰;×*×*Ð*Ø—*‘*×)Ñ)×3Ñ3°CÓ8‰Cà—,‘,˜sÓ#ˆCÜ"¨#Ô.Ð.r   c                óh   € V P                  4        UUu/ uF  w  rR V9  g   K  WbK  	  upp# u uppi )zself_attn.rotary_emb.inv_freq)Úitems)ÚweightsÚkÚvs   &  r   ÚsanitizeÚLanguageModel.sanitize¸   s:   € ð %Ÿ]™]œ_ô
Ù,‘T�QÐ0OÐWXÑ0XŒDˆAŠD™_ò
ð 	
ùó 
s   ”.¤.c                ó.   € V P                   P                  # rj   )r¤   r�   ©r4   s   &r   r�   ÚLanguageModel.layers¿   s   € à�z‰z× Ñ Ð r   c                ód   € V P                   P                  V P                   P                  ,          # rj   )r   r"   r#   r¶   s   &r   r6   ÚLanguageModel.head_dimÃ   s!   € à�{‰{×&Ñ&¨$¯+©+×*IÑ*IÕIÐIr   c                ó.   € V P                   P                  # rj   )r   r%   r¶   s   &r   r&   ÚLanguageModel.n_kv_headsÇ   s   € à�{‰{×.Ñ.Ð.r   )r   r¦   r¤   r(   rœ   )rW   rX   rY   rZ   r!   rS   Ústaticmethodr³   Úpropertyr�   r6   r&   r[   r\   r]   r^   s   @@r   rž   rž   œ   sr   ù‡ € ÷
Xó 
X÷/ò /ð ñ
ó ð
ð ñ!ó ð!ð ñJó ðJð ñ/ó ÷/ð /r   rž   )Útypingr   Úmlx.coreÚcorerB   Úmlx.nnr)   r   r   r   r   r?   r   r   r	   ÚModuler   r`   rn   r„   rž   © r   r   Ú<module>rÄ      st   ðÝ å Ý ÷ñ õ
 Ý ô?#�—	‘	ô ?#ôDLˆ"�)‰)ô Lô�r—y‘yô ô4#ˆB�I‰Iô #ôL-/�B—I‘Iö -/r   