+
    UV-jµ'  ã                   óp  € ^ RI Ht ^ RIHt ^ RIHt ^ RIt^RI	H
t
 ^RIHt  ! R R]P                  4      t ! R R]P                  4      t ! R	 R
]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      tR# )é    )ÚOptionalN)Úresize_bilinear©ÚVisionConfigc                   óL   a a€ ] tR t^t oRV3R lV 3R llltRR ltRtVtV ;t# )Ú	Attentionc                ó,   <€ V ^8„  d   QhRS[ RS[ RS[/# )é   ÚdimsÚ	num_headsÚbias)ÚintÚbool)ÚformatÚ__classdict__s   "€Úm/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/phi4mm/vision.pyÚ__annotate__ÚAttention.__annotate__   s)   ø€ ÷ 9ñ 9áð9ñ ð9ñ ñ	9ó    c                ó‚  <€ \         SV `  4        W,          ^ 8w  d   \        RV RV R24      hW n        W,          pVR,          V n        \
        P                  ! WVR7      V n        \
        P                  ! WVR7      V n        \
        P                  ! WVR7      V n	        \
        P                  ! WVR7      V n
        R# )r   zIThe input feature dimensions should be divisible by the number of heads (z % z) != 0©r   Ng      à¿)ÚsuperÚ__init__Ú
ValueErrorr   ÚscaleÚnnÚLinearÚq_projÚk_projÚv_projÚout_proj)Úselfr   r   r   Úhead_dimÚ	__class__s   &&&& €r   r   ÚAttention.__init__   s¡   ø€ ô 	‰ÑÔàÕ Ô"Üð$Ø$( 6¨¨Y¨K°vð?óð ð
 #ŒØÕ$ˆØ˜t•^ˆŒ
ä—i’i °Ô6ˆŒÜ—i’i °Ô6ˆŒÜ—i’i °Ô6ˆŒÜŸ	š	 $°4Ô8ˆŽr   c                ó\  € V P                  V4      pV P                  V4      pV P                  V4      pV P                  pVP                  w  rxp	VP                  w  r«p
VP                  WxVR4      P                  ^ ^^^4      pVP                  W{VR4      P                  ^ ^^^4      pVP                  W{VR4      P                  ^ ^^^4      p\        P                  P                  W4WPP                  VR7      pVP                  ^ ^^^4      P                  WxR4      pV P                  V4      # )é   )r   Úmaskéÿÿÿÿ)r   r   r    r   ÚshapeÚreshapeÚ	transposeÚmxÚfastÚscaled_dot_product_attentionr   r!   )r"   Úxr(   ÚqueriesÚkeysÚvaluesr   ÚBÚLÚDÚ_ÚSÚoutputs   &&&          r   Ú__call__ÚAttention.__call__#   s  € Ø—+‘+˜a“.ˆØ�{‰{˜1‹~ˆØ—‘˜Q“ˆà—N‘Nˆ	Ø—-‘-‰ˆˆaØ—*‘*‰ˆˆaØ—/‘/ !¨	°2Ó6×@Ñ@ÀÀAÀqÈ!ÓLˆØ�|‰|˜A )¨RÓ0×:Ñ:¸1¸aÀÀAÓFˆØ—‘  i°Ó4×>Ñ>¸qÀ!ÀQÈÓJˆä—‘×5Ñ5Ø˜6¯©¸$ð 6ó 
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°a¸BÓ?ˆØ�}‰}˜VÓ$Ð$r   )r   r   r!   r   r   r    )T©N©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r:   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r$   r   s   @@r   r   r      s   ù‡ € ÷9õ 9÷.%ô %r   r   c                   óP   a a€ ] tR t^6t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚMLPc                ó    <€ V ^8„  d   QhRS[ /# ©r
   Úconfigr   )r   r   s   "€r   r   ÚMLP.__annotate__7   s   ø€ ÷ Vñ V™|ñ Vr   c                ó$  <€ \         SV `  4        \        P                  ! R R7      V n        \        P
                  ! VP                  VP                  RR7      V n        \        P
                  ! VP                  VP                  RR7      V n	        R# )Úprecise)ÚapproxTr   N)
r   r   r   ÚGELUÚactivation_fnr   Úhidden_sizeÚintermediate_sizeÚfc1Úfc2©r"   rJ   r$   s   &&€r   r   ÚMLP.__init__7   sa   ø€ Ü‰ÑÔÜŸWšW¨IÔ6ˆÔÜ—9’9˜V×/Ñ/°×1IÑ1IÐPTÔUˆŒÜ—9’9˜V×5Ñ5°v×7IÑ7IÐPTÔUˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r
   r0   Úreturn)r-   Úarray)r   r   s   "€r   r   rK   =   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                ól   € V P                  V4      pV P                  V4      pV P                  V4      pV# r<   )rS   rP   rT   )r"   r0   s   &&r   r:   ÚMLP.__call__=   s1   € Ø�H‰H�Q‹KˆØ×Ñ˜qÓ!ˆØ�H‰H�Q‹KˆØˆr   )rP   rS   rT   r=   rE   s   @@r   rG   rG   6   s    ù‡ € ÷Vó V÷÷ ð r   rG   c                   óT   a a€ ] tR t^Dt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚEncoderLayerc                ó    <€ V ^8„  d   QhRS[ /# rI   r   )r   r   s   "€r   r   ÚEncoderLayer.__annotate__E   s   ø€ ÷ Sñ S™|ñ Sr   c                óx  <€ \         SV `  4        VP                  V n        \	        VP                  VP
                  R R7      V n        \        P                  ! V P                  VP                  R7      V n
        \        V4      V n        \        P                  ! V P                  VP                  R7      V n        R# )Tr   ©ÚepsN)r   r   rQ   Ú	embed_dimr   Únum_attention_headsÚ	self_attnr   Ú	LayerNormÚlayer_norm_epsÚlayer_norm1rG   ÚmlpÚlayer_norm2rU   s   &&€r   r   ÚEncoderLayer.__init__E   s�   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ"Ø×Ñ × :Ñ :Àô
ˆŒô Ÿ<š<¨¯©¸F×<QÑ<QÔRˆÔÜ�v“;ˆŒÜŸ<š<¨¯©¸F×<QÑ<QÔRˆÖr   c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[ P                  /# )r
   r0   r(   rX   ©r-   rY   r   )r   r   s   "€r   r   r_   O   s3   ø€ ÷ ñ ™"Ÿ(™(ð ©(±2·8±8Õ*<ð ÉÏÉñ r   c                ó¤   € V P                  V P                  V4      V4      pW,           pV P                  V P                  V4      4      pWC,           # r<   )re   rh   ri   rj   )r"   r0   r(   ÚrÚhs   &&&  r   r:   ÚEncoderLayer.__call__O   sD   € Ø�N‰N˜4×+Ñ+¨AÓ.°Ó5ˆØ�EˆØ�H‰H�T×%Ñ% aÓ(Ó)ˆØ�uˆr   )rc   rh   rj   ri   re   r<   r=   rE   s   @@r   r]   r]   D   s    ù‡ € ÷Só S÷÷ ò r   r]   c                   óT   a a€ ] tR t^Vt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚEncoderc                ó    <€ V ^8„  d   QhRS[ /# rI   r   )r   r   s   "€r   r   ÚEncoder.__annotate__W   s   ø€ ÷ Vñ V™|ñ Vr   c                ó”   <€ \         SV `  4        \        VP                  4       Uu. uF  p\	        V4      NK  	  upV n        R # u upi r<   )r   r   ÚrangeÚnum_hidden_layersr]   Úlayers)r"   rJ   r7   r$   s   && €r   r   ÚEncoder.__init__W   s9   ø€ Ü‰ÑÔÜ5:¸6×;SÑ;SÔ5TÓUÑ5T°”| FÖ+Ñ5TÑUˆŽùÒUs   §Ac                ód   <€ V ^8„  d   QhRS[ P                  RS[RS[S[ P                  ,          /# )r
   r0   Úoutput_hidden_statesr(   )r-   rY   r   r   )r   r   s   "€r   r   ru   [   s6   ø€ ÷ !ñ !á�8‰8ð!ñ #ð!ñ ‘r—x‘xÕ ñ	!r   c                ó„   € V'       d   V3MR pTpV P                    F  pV! WSR7      pV'       g   K  WE3,           pK!  	  WT3# )N)r(   ©ry   )r"   r0   r|   r(   Úencoder_statesrp   Úlayers   &&&&   r   r:   ÚEncoder.__call__[   sG   € ÷ "6˜!™¸4ˆØˆØ—[”[ˆEÙ�aÔ#ˆAß#Ñ#Ø!/°$Õ!6’ñ !ð Ð Ð r   r~   )FNr=   rE   s   @@r   rs   rs   V   s    ù‡ € ÷Vó V÷!÷ !ò !r   rs   c                   óp   a a€ ] tR t^jt oV3R lV 3R llt]V3R lR l4       tRV3R lR lltRtVt	V ;t
# )	ÚVisionEmbeddingsc                ó    <€ V ^8„  d   QhRS[ /# rI   r   )r   r   s   "€r   r   ÚVisionEmbeddings.__annotate__k   s   ø€ ÷ Sñ S™|ñ Sr   c                óð  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        \        P                  ! VP
                  VP
                  ,          VP                  ,          V P                  R R7      V n	        VP                  VP
                  ,          V n        V P                  ^,          V n        \        P                  ! V P                  V P                  4      V n        R# )Tr   N)r   r   rJ   rQ   rc   Ú
patch_sizer   r   Únum_channelsÚpatch_embeddingÚ
image_sizeÚposition_embedding_sizeÚnum_positionsÚ	EmbeddingÚposition_embeddingrU   s   &&€r   r   ÚVisionEmbeddings.__init__k   s´   ø€ Ü‰ÑÔØŒØ×+Ñ+ˆŒØ ×+Ñ+ˆŒô  "ŸyšyØ×Ñ × 1Ñ 1Õ1°F×4GÑ4GÕGØ�N‰NØô 
ˆÔð (.×'8Ñ'8¸F×<MÑ<MÕ'MˆÔ$Ø!×9Ñ9¸1Õ<ˆÔÜ"$§,¢,¨t×/AÑ/AÀ4Ç>Á>Ó"RˆÖr   c                ón   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[RS[ P                  /# )r
   Úpositional_embeddingsÚspatial_shapesÚ
max_lengthrX   )r-   rY   r   )r   r   s   "€r   r   r…   ~   s=   ø€ ÷ 0'ñ 0'Ù!Ÿx™xð0'áŸ™ð0'ñ ð0'ñ 
�‰ñ	0'r   c                óö  € VP                   ^ ,          pV P                   R,          pV P                  pV P                   ^ ,          pV P                  ^^ ^4      P                  ^WFV4      p. p\	        V4       Fê  p	\        W^ 3,          P                  4       4      p
\        W^3,          P                  4       4      p\        VW«3RRR7      pVP                  WJV,          4      P                  pVP                  V4      pW«,          pWÒ8  dA   W-,
          p\        P                  ! VR,          Wä34      p\        P                  ! WÏ.^ R7      pVP                  VRV 4       Kì  	  \        P                  ! V4      # )zû
Resize 2D positional embeddings per image and pad to fixed size.

Args:
    positional_embeddings: (pos_size, pos_size, embed_dim)
    spatial_shapes: (batch_size, 2) - (h_patches, w_patches) per image
    max_length: max number of patches to pad to
FT)Úalign_cornersÚ	antialias:r   r'   N)ÚaxisNr)   )r*   Údtyper,   r+   rw   r   Úitemr   ÚTÚastyper-   Úbroadcast_toÚconcatenateÚappendÚstack)r‘   r’   r“   Ú
batch_sizerc   Úsource_dtypeÚpos_sizeÚpos_embÚbatch_resultsÚiÚheightÚwidthÚresizedÚnum_patchesÚ	pad_countÚpaddings   &&&             r   Úresize_positional_embeddingsÚ-VisionEmbeddings.resize_positional_embeddings}   sT  € ð $×)Ñ)¨!Õ,ˆ
Ø)×/Ñ/°Õ3ˆ	Ø,×2Ñ2ˆà(×.Ñ.¨qÕ1ˆà'×1Ñ1°!°Q¸Ó:×BÑBØˆy Hó
ˆð ˆÜ�zÖ"ˆAÜ˜¨1¨Õ-×2Ñ2Ó4Ó5ˆFÜ˜¨! tÕ,×1Ñ1Ó3Ó4ˆEä%ØØ�Ø#Øô	ˆGð —o‘o i¸%µÓ@×BÑBˆGØ—n‘n \Ó2ˆGà �.ˆKØÔ'à&Õ4�	ÜŸ/š/¨'°#­,¸Ð8NÓO�ÜŸ.š.¨'Ð);À!ÔD�à× Ñ  ¨¨*Ð!5Ö6ñ- #ô0 �xŠx˜Ó&Ð&r   c                óx   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[ P                  /# )r
   r0   r’   rX   rm   )r   r   s   "€r   r   r…   °   s6   ø€ ÷ ñ Ù—‘ðÙ+3±B·H±HÕ+=ðá	�‰ñr   c                ó¦  € V P                  V4      pVej   V P                  P                  P                  V P                  V P                  R4      pV P                  WBVP                  ^,          R7      pW5,           pV# \        P                  ! \        P                  ! V P                  4      R,          4      pW0P                  V4      ,           pV# )N)r“   r)   )NºNNN)r‰   rŽ   Úweightr+   r‹   r¬   r*   r-   rY   ÚnpÚarangerŒ   )r"   r0   r’   Úpatch_embeddingsr‘   Úresized_pos_embeddingsÚ
embeddingsÚposition_idss   &&&     r   r:   ÚVisionEmbeddings.__call__°   sÀ   € ð  ×/Ñ/°Ó2ÐàÒ%à$(×$;Ñ$;×$BÑ$B×$JÑ$JØ×,Ñ,¨d×.JÑ.JÈBó%Ð!ð &*×%FÑ%FØ%À!Ç'Á'È!Å*ð &Gó &Ð"ð *ÕBˆJð Ðô Ÿ8š8¤B§I¢I¨d×.@Ñ.@Ó$AÀ'Õ$JÓKˆLØ)×,CÑ,CÀLÓ,QÕQˆJàÐr   )rJ   rc   rŒ   r‰   r‡   rŽ   r‹   r<   )r>   r?   r@   rA   r   Ústaticmethodr¬   r:   rB   rC   rD   rE   s   @@r   rƒ   rƒ   j   s5   ù‡ € ÷Só Sð$ ÷0'ó ð0'÷d÷ ò r   rƒ   c                   óX   a a€ ] tR t^Çt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚSigLip2VisionModelz)SigLIP2 vision model with NaFlex support.c                ó    <€ V ^8„  d   QhRS[ /# rI   r   )r   r   s   "€r   r   ÚSigLip2VisionModel.__annotate__Ê   s   ø€ ÷ 
ñ 
™|ñ 
r   c                óÆ   <€ \         SV `  4        \        V4      V n        \	        V4      V n        \        P                  ! VP                  VP                  R 7      V n
        R# )ra   N)r   r   rƒ   r¶   rs   Úencoderr   rf   rQ   rg   Úpost_layernormrU   s   &&€r   r   ÚSigLip2VisionModel.__init__Ê   sG   ø€ Ü‰ÑÔÜ*¨6Ó2ˆŒÜ˜v“ˆŒÜ ŸlšlØ×Ñ F×$9Ñ$9ô
ˆÖr   c                óŽ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          RS[/# )r
   r0   r’   Úpixel_attention_maskr|   )r-   rY   r   r   )r   r   s   "€r   r   r½   Ò   sI   ø€ ÷ /ñ /á�8‰8ð/ñ !¡§¡Õ*ð/ñ '¡r§x¡xÕ0ð	/ñ
 #ñ/r   c           	     ó  € V P                  WR 7      pVP                  V P                   P                  P                  P                  4      pRpVe‡   \
        P                  ! VR,          P                  \
        P                  4      \
        P                  ! R4      \
        P                  ! \        R4      4      4      P                  VP                  4      pV P                  WVR7      w  rgV P                  V4      pW�V3# ))r’   Ng        z-inf)r|   r(   )r°   NNr°   )r¶   r›   r‰   r±   r˜   r-   ÚwhereÚbool_rY   Úfloatr¿   rÀ   )	r"   r0   r’   rÃ   r|   r(   Úfinal_outputÚhidden_statesÚpooler_outputs	   &&&&&    r   r:   ÚSigLip2VisionModel.__call__Ò   sÔ   € ð �O‰O˜AˆOÓ=ˆØ�H‰H�T—_‘_×4Ñ4×;Ñ;×AÑAÓBˆð ˆØÒ+ô —8’8Ø$Ð%5Õ6×=Ñ=¼b¿h¹hÓGÜ—’˜“Ü—’œ˜v›Ó'ó÷ ‰f�Q—W‘W‹oð	 ð '+§l¡lØ¸tð '3ó '
Ñ#ˆð ×+Ñ+¨LÓ9ˆØ Ð.Ð.r   )r¶   r¿   rÀ   )NNF)
r>   r?   r@   rA   Ú__doc__r   r:   rB   rC   rD   rE   s   @@r   r»   r»   Ç   s!   ù‡ € Ù3÷
ó 
÷/÷ /ò /r   r»   c                   ó^   a a€ ] tR t^ït oRtV3R lV 3R lltRV3R lR lltR tRtVt	V ;t
# )	ÚVisionTowerzKWrapper matching the HF weight structure: model.vision_tower.vision_tower.*c                ó    <€ V ^8„  d   QhRS[ /# rI   r   )r   r   s   "€r   r   ÚVisionTower.__annotate__ò   s   ø€ ÷ ñ ™|ñ r   c                óR   <€ \         SV `  4        \        V4      V n        RV n        R# )r
   Néþÿÿÿ)r   r   r»   Úvision_towerÚselect_layerrU   s   &&€r   r   ÚVisionTower.__init__ò   s#   ø€ Ü‰ÑÔÜ.¨vÓ6ˆÔØˆÖr   c                óˆ   <€ V ^8„  d   QhRS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          /# )r
   Úpixel_valuesrÃ   r’   rm   )r   r   s   "€r   r   rÐ   ÷   s?   ø€ ÷ ñ á—h‘hðñ '¡r§x¡xÕ0ðñ !¡§¡Õ*ñ	r   c                ó~  € V P                  VVVRR7      w   rEWPP                  ,          pVe�   . p\        VP                  ^ ,          4       Fj  p\	        W8^ 3,          P                  4       4      p	\	        W8^3,          P                  4       4      p
Wš,          pWhRV13,          pVP                  V4       Kl  	  V# V# )zM
Returns list of image features (one per image, variable length for NaFlex).
T)r’   rÃ   r|   N)rÓ   rÔ   rw   r*   r   r™   rž   )r"   r×   rÃ   r’   r7   rÉ   ÚselectedÚfeaturesr¥   rp   ÚwÚ	num_validÚfeats   &&&&         r   r:   ÚVisionTower.__call__÷   sÂ   € ð #×/Ñ/ØØ)Ø!5Ø!%ð	 0ó 
Ñˆˆ1ð !×!2Ñ!2Õ3ˆð Ò%ØˆHÜ˜8Ÿ>™>¨!Õ,Ö-�Ü˜¨! tÕ,×1Ñ1Ó3Ó4�Ü˜¨! tÕ,×1Ñ1Ó3Ó4�Ø�E�	Ø : I : Õ.�Ø—‘ Ö%ñ .ð ˆOàˆr   c                óð   € / pVP                  4        F_  w  r4R V9   d   K  RV9   dG   VP                  ^8X  d6   VP                  ^ ^^^4      P                  VP                  ^ ,          R4      pWBV&   Ka  	  V# )r·   zpatch_embedding.weightr)   )ÚitemsÚndimr,   r+   r*   )r"   ÚweightsÚsanitized_weightsÚkÚvs   &&   r   ÚsanitizeÚVisionTower.sanitize  sr   € ØÐØ—M‘M–O‰DˆAØ Ô"Ùð (¨1Ô,°·±¸1´à—K‘K  1 a¨Ó+×3Ñ3°A·G±G¸AµJÀÓC�Ø#$˜aÓ ñ $ð !Ð r   )rÔ   rÓ   )NN)r>   r?   r@   rA   rÌ   r   r:   ræ   rB   rC   rD   rE   s   @@r   rÎ   rÎ   ï   s'   ù‡ € ÙU÷ó ÷
ò ÷@!ò !r   rÎ   )Útypingr   Úmlx.coreÚcorer-   Úmlx.nnr   Únumpyr²   Úinterpolater   rJ   r   ÚModuler   rG   r]   rs   rƒ   r»   rÎ   © r   r   Ú<module>rð      sŒ   ðÝ å Ý Û å )Ý  ô(%�—	‘	ô (%ôVˆ"�)‰)ô ô�2—9‘9ô ô$!ˆb�i‰iô !ô(Z�r—y‘yô Zôz%/˜Ÿ™ô %/ôP3!�"—)‘)ö 3!r   