+
    UV-j~Q  ã                   ó@  € ^ RI HtHtHt ^ RIHt ^ RIHt ^RI	H
t
  ! R R]P                  4      t ! R R]4      t ! R R	]P                  4      t ! R
 R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t^R]P0                  ]P2                  3R R lltR R lt ! R R ]P                  4      t ! R! R"]P                  4      t ! R# R$]P                  4      t ! R% R&]P                  4      t ! R' R(]P                  4      t  ! R) R*]P                  4      t!R# )+é    )ÚListÚOptionalÚTupleN©ÚVisionConfigc                   ó>   a a€ ] tR t^	t oV 3R ltR tR tRtVtV ;t	# )ÚNamedSequentialc                ó2   <€ \         SV `  4        . V n        R # ©N)ÚsuperÚ__init__Ú_order)ÚselfÚ	__class__s   &€Ún/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/fastvlm/vision.pyr   ÚNamedSequential.__init__
   s   ø€ Ü‰ÑÔØˆŽó    c                óT   € \        WV4       V P                  P                  V4       R # r   )Úsetattrr   Úappend)r   ÚnameÚmodules   &&&r   Ú
add_moduleÚNamedSequential.add_module   s   € Ü�˜FÔ#Ø�‰×Ñ˜4Ö r   c                óN   € V P                    F  p\        W4      ! V4      pK  	  V# r   )r   Úgetattr)r   Úxr   s   && r   Ú__call__ÚNamedSequential.__call__   s$   € Ø—K”KˆDÜ˜Ô# AÓ&ŠAñ  àˆr   )r   )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r   Ú__classdict__s   @@r   r	   r	   	   s   ù‡ € õò!÷ò r   r	   c                   ó2   a € ] tR t^t o V 3R lR ltRtV tR# )ÚCallableModuleListc                ó4   <€ V ^8„  d   QhRS[ P                  /# )é   r   ©ÚmxÚarray)Úformatr(   s   "€r   Ú__annotate__ÚCallableModuleList.__annotate__   s   ø€ ÷ ñ ™"Ÿ(™(ñ r   c                ó(   € V  F  pV! V4      pK  	  V# r   © )r   r   Úitems   && r   r   ÚCallableModuleList.__call__   s   € ÛˆDÙ�Q“ŠAñ àˆr   r4   N)r    r!   r"   r#   r   r$   r%   )r(   s   @r   r*   r*      s   ø‡ € ÷ö r   r*   c                   óX   a a€ ] tR t^t oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )ÚMHSAz›Multi-headed Self Attention module.

Source modified from:
https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py
c                ó<   <€ V ^8„  d   QhRS[ RS[ RS[RS[RS[RR/# )r,   ÚdimÚhead_dimÚqkv_biasÚ	attn_dropÚ	proj_dropÚreturnN)ÚintÚboolÚfloat)r0   r(   s   "€r   r1   ÚMHSA.__annotate__&   sG   ø€ ÷ /ñ /áð/ñ ð/ñ ð	/ñ
 ð/ñ ð/ð 
ñ/r   c                óx  <€ \         SV `  4        W,          ^ 8X  g   Q R4       hW n        W,          V n        VR,          V n        \
        P                  ! W^,          VR7      V n        \
        P                  ! V4      V n	        \
        P                  ! W4      V n
        \
        P                  ! V4      V n        R# )r   z#dim should be divisible by head_dim)ÚbiasNg      à¿)r   r   r;   Ú	num_headsÚscaleÚnnÚLinearÚqkvÚDropoutr=   Úprojr>   )r   r:   r;   r<   r=   r>   r   s   &&&&&&€r   r   ÚMHSA.__init__&   s…   ø€ ô 	‰ÑÔØ�~ Ô"ÐIÐ$IÓIÐ"Ø ŒØ�ˆŒØ˜t•^ˆŒ
ä—9’9˜S¨¥'°Ô9ˆŒÜŸš IÓ.ˆŒÜ—I’I˜cÓ'ˆŒ	ÜŸš IÓ.ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r,   r   r?   r-   )r0   r(   s   "€r   r1   rC   9   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                óH  € VP                  ^ ^^^4      pVP                  w  r#rEWE,          pVP                  ^R7      P                  ^ ^^4      pV P                  V4      P	                  W&^V P
                  V P                  4      P                  ^^ ^^^4      pVw  r‰p
\        P                  P                  W‰W P                  RR7      pVP                  ^ ^^^4      P	                  W&V4      pV P                  V4      pV P                  V4      pVP	                  W$WS4      pV# )r   )Ú
start_axisN)rG   Úmask)Ú	transposeÚshapeÚflattenrJ   ÚreshaperF   r;   r.   ÚfastÚscaled_dot_product_attentionrG   rL   r>   )r   r   ÚBÚCÚHÚWÚNrJ   ÚqÚkÚvs   &&         r   r   ÚMHSA.__call__9   sÿ   € à�K‰K˜˜1˜a Ó#ˆØ—W‘W‰
ˆˆaØ�EˆØ�I‰I ˆIÓ#×-Ñ-¨a°°AÓ6ˆà�H‰H�Q‹Kß‰W�Q˜1˜dŸn™n¨d¯m©mÓ<ß‰Y�q˜!˜Q  1Ó%ð 	ð
 ‰ˆˆaä�G‰G×0Ñ0°°qÇ
Á
ÐQUÐ0ÓVˆØ�K‰K˜˜1˜a Ó#×+Ñ+¨A°!Ó4ˆØ�I‰I�a‹LˆØ�N‰N˜1Óˆà�I‰I�a˜AÓ!ˆØˆr   )r=   r;   rF   rL   r>   rJ   rG   )é    Fç        rc   ©
r    r!   r"   r#   Ú__doc__r   r   r$   r%   r&   r'   s   @@r   r8   r8      s#   ù‡ € ñ÷/õ /÷&÷ ð r   r8   c                   ór   a a€ ] tR t^Ot oRtRR]P                  3V3R lV 3R llltV3R lR ltRt	Vt
V ;t# )ÚConvFFNzConvolutional FFN Module.Nc          
      ój   <€ V ^8„  d   QhRS[ RS[S[ ,          RS[S[ ,          RS[P                  RR/# )r,   Úin_channelsÚhidden_channelsÚout_channelsÚ	act_layerr?   N)r@   r   rH   ÚModule)r0   r(   s   "€r   r1   ÚConvFFN.__annotate__R   sP   ø€ ÷ Kñ KáðKñ "¡#�ðKñ ™s•mð	Kñ
 —9‘9ðKð 
ñKr   c                óÌ  <€ \         SV `  4        T;'       g    TpT;'       g    Tp\        4       V n        V P                  P	                  R \
        P                  ! VV^^VRR7      4       V P                  P	                  R\
        P                  ! VR7      4       \
        P                  ! W^R7      V n        V! 4       V n	        \
        P                  ! W#^R7      V n
        R# )ÚconvF)ri   rk   Úkernel_sizeÚpaddingÚgroupsrE   Úbn©Únum_features©rq   N)r   r   r	   rp   r   rH   ÚConv2dÚ	BatchNormÚfc1ÚactÚfc2)r   ri   rj   rk   rl   r   s   &&&&&€r   r   ÚConvFFN.__init__R   s¹   ø€ ô 	‰ÑÔØ#×2Ð2 {ˆØ)×8Ð8¨[ˆÜ#Ó%ˆŒ	Ø�	‰	×ÑØÜ�IŠIØ'Ø)ØØØ"Øôô
	
ð 	�	‰	×ÑØÜ�LŠL lÔ3ô	
ô —9’9˜[ÀqÔIˆŒÙ“;ˆŒÜ—9’9˜_ÈÔJˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rO   r-   )r0   r(   s   "€r   r1   rn   p   ó#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                óŽ   € V P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV# r   )rp   rz   r{   r|   ©r   r   s   &&r   r   ÚConvFFN.__call__p   s;   € Ø�I‰I�a‹LˆØ�H‰H�Q‹KˆØ�H‰H�Q‹KˆØ�H‰H�Q‹KˆØˆr   )r{   rp   rz   r|   ©r    r!   r"   r#   re   rH   ÚGELUr   r   r$   r%   r&   r'   s   @@r   rg   rg   O   s2   ù‡ € Ù#ð
 *.Ø&*Ø!Ÿw™w÷Kõ K÷<÷ ð r   rg   c                   óX   a a€ ] tR t^xt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )ÚLayerNormChannelzK
LayerNorm only for Channel Dimension.
Input: tensor in shape [B, H, W, C]
c                ó   <€ V ^8„  d   QhRR/# )r,   r?   Nr4   )r0   r(   s   "€r   r1   ÚLayerNormChannel.__annotate__~   s   ø€ ÷ ñ °4ñ r   c                óœ   <€ \         SV `  4        \        P                  ! V4      V n        \        P
                  ! V4      V n        W n        R # r   )r   r   r.   ÚonesÚweightÚzerosrE   Úeps)r   rv   r�   r   s   &&&€r   r   ÚLayerNormChannel.__init__~   s3   ø€ Ü‰ÑÔÜ—g’g˜lÓ+ˆŒÜ—H’H˜\Ó*ˆŒ	ØŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rO   r-   )r0   r(   s   "€r   r1   rˆ   „   r   r   c                ó6  € VP                  RRR7      p\        P                  ! W,
          ^4      P                  RRR7      pW,
          \        P                  ! W0P                  ,           4      ,          pV P
                  V,          V P                  ,           pV# )é   T)Úkeepdimséÿÿÿÿ)Úmeanr.   ÚpowerÚsqrtr�   r‹   rE   )r   r   ÚuÚss   &&  r   r   ÚLayerNormChannel.__call__„   sl   € Ø�F‰F�2 ˆFÓ%ˆÜ�HŠH�Q•U˜AÓ×#Ñ# B°Ð#Ó6ˆØ�U”b—g’g˜a§(¡(�lÓ+Õ+ˆØ�K‰K˜!�O˜dŸi™iÕ'ˆØˆr   )rE   r�   r‹   )gñhãˆµøä>rd   r'   s   @@r   r†   r†   x   s#   ù‡ € ñ÷
õ ÷÷ ð r   r†   c                   ó†   a a€ ] tR t^Œt oRtR]P                  ]P                  3V3R lV 3R llltV3R lR lt	Rt
VtV ;t# )ÚAttentionBlockzÓImplementation of metaformer block with MHSA as token mixer.

For more details on Metaformer structure, please refer to:
`MetaFormer Is Actually What You Need for Vision <https://arxiv.org/pdf/2111.11418.pdf>`_
ç      @c                óZ   <€ V ^8„  d   QhRS[ RS[RS[P                  RS[P                  /# )r,   r:   Ú	mlp_ratiorl   Ú
norm_layer©r@   rB   rH   rm   )r0   r(   s   "€r   r1   ÚAttentionBlock.__annotate__“   s;   ø€ ÷ 2ñ 2áð2ñ ð2ñ —9‘9ð	2ñ
 —I‘Iñ2r   c                ó\  <€ \         SV `  4        V! VR 7      V n        \        VR7      V n        V^ 8”  g   Q RP                  V4      4       h\        W,          4      p\        VVVR7      V n        \        P                  ! ^^V34      V n        \        P                  ! ^^V34      V n        R# )ru   )r:   ú-MLP ratio should be greater than 0, found: {}©ri   rj   rl   N)r   r   Únormr8   Útoken_mixerr0   r@   rg   Úconvffnr.   rŠ   Úlayer_scale_1Úlayer_scale_2)r   r:   rž   rl   rŸ   Úmlp_hidden_dimr   s   &&&&& €r   r   ÚAttentionBlock.__init__“   sŸ   ø€ ô 	‰ÑÔá¨CÔ0ˆŒ	Ü Cœ=ˆÔà˜1Œ}ð 	
ÐM×TÑTØó
ó 	
ˆ}ô ˜S�_Ó-ˆÜØØ*Øô
ˆŒô  ŸWšW a¨¨C [Ó1ˆÔÜŸWšW a¨¨C [Ó1ˆÖr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rO   r-   )r0   r(   s   "€r   r1   r¡   ¬   ó#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                óÄ   € WP                   V P                  V P                  V4      4      ,          ,           pWP                  V P	                  V4      ,          ,           pV# r   )r¨   r¦   r¥   r©   r§   r�   s   &&r   r   ÚAttentionBlock.__call__¬   sH   € Ø×"Ñ" T×%5Ñ%5°d·i±iÀ³lÓ%CÕCÕCˆØ×"Ñ" T§\¡\°!£_Õ4Õ4ˆØˆr   )r§   r¨   r©   r¥   r¦   )r    r!   r"   r#   re   rH   r„   ry   r   r   r$   r%   r&   r'   s   @@r   r›   r›   Œ   s6   ù‡ € ñð Ø!Ÿw™wØ "§¡÷2õ 2÷2÷ ð r   r›   c                   óX   a a€ ] tR t^²t oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )ÚRepCPEz¸Implementation of conditional positional encoding.

For more details refer to paper:
`Conditional Positional Encodings for Vision Transformers <https://arxiv.org/pdf/2102.10882.pdf>`_
c                ó*   <€ V ^8„  d   QhRS[ RS[ RR/# )r,   ri   Ú	embed_dimr?   N©r@   )r0   r(   s   "€r   r1   ÚRepCPE.__annotate__¹   s)   ø€ ÷ 
ñ 
áð
ñ ð
ð
 
ñ
r   c           
     ó„  <€ \         SV `  4        \        V\        4      '       d   \	        V.^,          4      p\        V\
        4      '       g   Q R\        V4       R24       h\        V4      ^8X  g   Q R\        V4       R24       h\        P                  ! VVV^\        V^ ,          ^,          4      VRR7      V n
        R# )r,   z/"spatial_shape" must by a sequence or int, get z	 instead.z+Length of "spatial_shape" should be 2, got T©ri   rk   rq   Ústriderr   rs   rE   N)r   r   Ú
isinstancer@   Útupler   ÚtypeÚlenrH   rx   Úreparam_conv)r   ri   r³   Úspatial_shaper   s   &&&&€r   r   ÚRepCPE.__init__¹   sÉ   ø€ ô 	‰ÑÔÜ�m¤S×)Ò)Ü! = /°AÕ"5Ó6ˆMÜ˜-¬×/Ò/ð 	
ðÜ˜Ó&Ð' yð2ó	
Ð/ô �=Ó! QÔ&ð 	
ðÜ�}Ó%Ð& ið1ó	
Ð&ô
 ŸIšIØ#Ø"Ø%ØÜ˜ aÕ(¨AÕ-Ó.ØØô
ˆÖr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rO   r-   )r0   r(   s   "€r   r1   rµ   Õ   ó#   ø€ ÷ $ñ $™"Ÿ(™(ð $¡r§x¡xñ $r   c                ó$   € V P                  V4      # r   ©r½   r�   s   &&r   r   ÚRepCPE.__call__Õ   ó   € Ø× Ñ  Ó#Ð#r   rÃ   )i   )é   rÆ   rd   r'   s   @@r   r±   r±   ²   s#   ù‡ € ñ÷
õ 
÷8$÷ $ð $r   r±   c                   óx   a a€ ] tR t^Ùt oRt]P                  ! 4       3V3R lV 3R llltV3R lR ltRt	Vt
V ;t# )ÚReparamLargeKernelConvzÐBuilding Block of RepLKNet

This class defines overparameterized large kernel conv block
introduced in `RepLKNet <https://arxiv.org/abs/2203.06717>`_

Reference: https://github.com/DingXiaoH/RepLKNet-pytorch
c                óV   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[ RS[P                  RR/# )	r,   ri   rk   rq   r¸   rs   Ú
activationr?   N)r@   rH   rm   )r0   r(   s   "€r   r1   Ú#ReparamLargeKernelConv.__annotate__â   sU   ø€ ÷ 
ñ 
áð
ñ ð
ñ ð	
ñ
 ð
ñ ð
ñ —I‘Ið
ð 
ñ
r   c                óŒ   <€ \         \        V `  4        W`n        \        P
                  ! VVVVV^,          ^VRR7      V n        R# )r,   T©ri   rk   rq   r¸   rr   Údilationrs   rE   N)r   rÈ   r   rÊ   rH   rx   Úlkb_reparam)r   ri   rk   rq   r¸   rs   rÊ   r   s   &&&&&&&€r   r   ÚReparamLargeKernelConv.__init__â   sF   ø€ ô 	Ô$ dÑ4Ô6Ø$ŒÜŸ9š9Ø#Ø%Ø#ØØ 1Õ$ØØØô	
ˆÖr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rO   r-   )r0   r(   s   "€r   r1   rË   ø   s#   ø€ ÷ 4ñ 4™"Ÿ(™(ð 4¡r§x¡xñ 4r   c                óB   € V P                  V P                  V4      4      # r   ©rÊ   rÏ   r�   s   &&r   r   ÚReparamLargeKernelConv.__call__ø   s   € Ø�‰˜t×/Ñ/°Ó2Ó3Ð3r   rÓ   rƒ   r'   s   @@r   rÈ   rÈ   Ù   s.   ù‡ € ñð !#§¢£	÷
õ 
÷,4÷ 4ð 4r   rÈ   c                   óT   a a€ ] tR t^üt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )Ú
PatchEmbedz$Convolutional patch embedding layer.c          
      ó6   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RR/# )r,   Ú
patch_sizer¸   ri   r³   r?   Nr´   )r0   r(   s   "€r   r1   ÚPatchEmbed.__annotate__ÿ   s=   ø€ ÷ 
ñ 
áð
ñ ð
ñ ð	
ñ
 ð
ð 
ñ
r   c                óê   <€ \         SV `  4        \        4       V n        V P                  P	                  \        VVVVVR 7      4       V P                  P	                  \        VV^^^ ^RR7      4       R# ))ri   rk   rq   r¸   rs   F©ri   rk   rq   r¸   rr   rs   Úuse_seN)r   r   r*   rL   r   rÈ   ÚMobileOneBlock)r   rØ   r¸   ri   r³   r   s   &&&&&€r   r   ÚPatchEmbed.__init__ÿ   st   ø€ ô 	‰ÑÔÜ&Ó(ˆŒ	Ø�	‰	×ÑÜ"Ø'Ø&Ø&ØØ"ôô	
ð 	�	‰	×ÑÜØ%Ø&ØØØØØôö
	
r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rO   r-   )r0   r(   s   "€r   r1   rÙ     s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                ó$   € V P                  V4      # r   ©rL   r�   s   &&r   r   ÚPatchEmbed.__call__  s   € Ø�y‰y˜‹|Ðr   rá   rd   r'   s   @@r   rÖ   rÖ   ü   s!   ù‡ € Ù.÷
ó 
÷<÷ ð r   rÖ   c                   óL   a a€ ] tR tRt oRtRV 3R lltV3R lR ltRtVtV ;t	# )ÚRepMixeri!  zÉReparameterizable token mixer.

For more details, please refer to Apple's paper:
`FastViT: A Fast Hybrid Vision Transformer using Structural Reparameterization <https://arxiv.org/pdf/2303.14189.pdf>`_
c           
     óò   <€ \         SV `  4        Wn        W n        \        P
                  ! V P                  V P                  V P                  ^V P                  ^,          V P                  RR7      V n        R# )r‘   Tr·   N)r   r   r:   rq   rH   rx   r½   )r   r:   rq   r   s   &&&€r   r   ÚRepMixer.__init__(  s^   ø€ ô
 	‰ÑÔØŒØ&ÔäŸIšIØŸ™ØŸ™Ø×(Ñ(ØØ×$Ñ$¨Õ)Ø—8‘8Øô
ˆÖr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rO   r-   )r0   r(   s   "€r   r1   ÚRepMixer.__annotate__;  rÁ   r   c                ó$   € V P                  V4      # r   rÃ   r�   s   &&r   r   ÚRepMixer.__call__;  rÅ   r   )r:   rq   r½   )é   rd   r'   s   @@r   rä   rä   !  s   ù‡ € ñ÷
÷&$÷ $ð $r   rä   c                   ór   a a€ ] tR tRt oRt^R]P                  3V3R lV 3R llltV3R lR ltRt	Vt
V ;t# )	ÚRepMixerBlocki?  z×Implementation of Metaformer block with RepMixer as token mixer.

For more details on Metaformer structure, please refer to:
`MetaFormer Is Actually What You Need for Vision <https://arxiv.org/pdf/2111.11418.pdf>`_
rœ   c                óF   <€ V ^8„  d   QhRS[ RS[ RS[RS[P                  /# )r,   r:   rq   rž   rl   r    )r0   r(   s   "€r   r1   ÚRepMixerBlock.__annotate__F  s7   ø€ ÷ 0ñ 0áð0ñ ð0ñ ð	0ñ
 —9‘9ñ0r   c                ó  <€ \         SV `  4        \        WR 7      V n        V^ 8”  g   Q RP	                  V4      4       h\        W,          4      p\        VVVR7      V n        \        P                  ! ^^V34      V n
        R# )rw   r£   r¤   N)r   r   rä   r¦   r0   r@   rg   r§   r.   rŠ   Úlayer_scale)r   r:   rq   rž   rl   rª   r   s   &&&&& €r   r   ÚRepMixerBlock.__init__F  s}   ø€ ô 	‰ÑÔä# CÔAˆÔà˜1Œ}ð 	
ÐM×TÑTØó
ó 	
ˆ}ô ˜S�_Ó-ˆÜØØ*Øô
ˆŒô
 Ÿ7š7 A q¨# ;Ó/ˆÖr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rO   r-   )r0   r(   s   "€r   r1   rï   \  r­   r   c                óx   € V P                  V4      pWP                  V P                  V4      ,          ,           pV# r   )r¦   rñ   r§   r�   s   &&r   r   ÚRepMixerBlock.__call__\  s2   € Ø×Ñ˜QÓˆØ× Ñ  4§<¡<°£?Õ2Õ2ˆØˆr   )r§   rñ   r¦   rƒ   r'   s   @@r   rí   rí   ?  s2   ù‡ € ñð ØØ!Ÿw™w÷0õ 0÷,÷ ð r   rí   rœ   c                ó¶   € V ^8„  d   QhR\         R\         R\        \         ,          R\        R\         R\        R\        P
                  R\        P
                  /# )	r,   r:   Úblock_indexÚ
num_blocksÚtoken_mixer_typerq   rž   rl   rŸ   )r@   r   ÚstrrB   rH   rm   )r0   s   "r   r1   r1   b  sd   € ÷ "ñ "Ü	ð"äð"ô ”S•	ð"ô ð	"ô
 ð"ô ð"ô �y‰yð"ô —	‘	ñ"r   c           
      ó  € \        4       p\        W!,          4       Fi  p	VR 8X  d!   VP                  \        V VVVR7      4       K*  VR8X  d!   VP                  \	        V VVVR7      4       KQ  \        RP                  V4      4      h	  V# )Úrepmixer)rq   rž   rl   Ú	attention)rž   rl   rŸ   z"Token mixer type: {} not supported)r*   Úranger   rí   r›   Ú
ValueErrorr0   )
r:   r÷   rø   rù   rq   rž   rl   rŸ   ÚblocksÚ_s
   &&&&&&&&  r   Úbasic_blocksr  b  s•   € ô  Ó!€FÜ�:Õ*Ö+ˆØ˜zÔ)Ø�M‰MÜØØ +Ø'Ø'ô	öð  Ô,Ø�M‰MÜØØ'Ø'Ø)ô	öô Ø4×;Ñ;Ð<LÓMóð ñ) ,ð. €Mr   c                ó$   € V ^8„  d   QhR\         /# ©r,   Úconfigr   )r0   s   "r   r1   r1   ‡  s   € ÷ $ñ $¤<ñ $r   c                 óz  € . p\        \        V P                  4      4       EF•  pV P                  V,          pVeB   \	        V P
                  V,          V P
                  V,          VR7      pVP                  V4       \        V P
                  V,          VV P                  V P                  V,          V P                  V P                  V,          \        R7      pVP                  V4       V\        V P                  4      ^,
          8¼  d    V# V P                  V,          '       g4   V P
                  V,          V P
                  V^,           ,          8w  g   EK:  VP                  \        V P                  V P                  V P
                  V,          V P
                  V^,           ,          R7      4       EK˜  	  V# )N)ri   r³   r¾   )rù   rq   rž   rŸ   )rØ   r¸   ri   r³   )rþ   r¼   ÚlayersÚpos_embs_shapesr±   Ú
embed_dimsr   r  Útoken_mixersÚrepmixer_kernel_sizeÚ
mlp_ratiosr†   ÚdownsamplesrÖ   Údown_patch_sizeÚdown_stride)r  ÚnetworkÚir¾   Úposition_embeddingsÚstages   &     r   Úbuild_fast_vit_networkr  ‡  si  € Ø€GÜ”3�v—}‘}Ó%×&ˆØ×.Ñ.¨qÕ1ˆØÒ$Ü"(Ø"×-Ñ-¨aÕ0Ø ×+Ñ+¨AÕ.Ø+ô#Ðð
 �N‰NÐ.Ô/äØ×Ñ˜aÕ ØØ�M‰MØ#×0Ñ0°Õ3Ø×3Ñ3Ø×'Ñ'¨Õ*Ü'ô
ˆð 	�‰�uÔà”�F—M‘MÓ" QÕ&Ô&Øð €Nð ×Ñ˜a× Ô  F×$5Ñ$5°aÕ$8¸F×<MÑ<MÈaÐRSÍeÕ<T×$TØ�N‰NÜØ%×5Ñ5Ø!×-Ñ-Ø &× 1Ñ 1°!Õ 4Ø$×/Ñ/°°AµÕ6ô	÷ñ5 'ðD €Nr   c                   óX   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )	ÚSEBlocki®  z{Squeeze and Excite module.

MLX implementation of `Squeeze-and-Excitation Networks` -
https://arxiv.org/pdf/1709.01507.pdf
c                ó&   <€ V ^8„  d   QhRS[ RS[/# )r,   ri   Úrd_ratio)r@   rB   )r0   r(   s   "€r   r1   ÚSEBlock.__annotate__µ  s   ø€ ÷ 
ñ 
¡Cð 
±5ñ 
r   c                óà   <€ \         SV `  4        \        P                  ! V\	        W,          4      ^^RR7      V n        \        P                  ! \	        W,          4      V^^RR7      V n        R# )z†Construct a Squeeze and Excite Module.

Args:
    in_channels: Number of input channels.
    rd_ratio: Input channel reduction ratio.
T)ri   rk   rq   r¸   rE   N)r   r   rH   rx   r@   ÚreduceÚexpand)r   ri   r  r   s   &&&€r   r   ÚSEBlock.__init__µ  s`   ø€ ô 	‰ÑÔÜ—i’iØ#Ü˜[Õ3Ó4ØØØô
ˆŒô —i’iÜ˜KÕ2Ó3Ø$ØØØô
ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r,   Úinputsr?   r-   )r0   r(   s   "€r   r1   r  Ì  s#   ø€ ÷ ñ ™rŸx™xð ©B¯H©Hñ r   c                óB  € VP                   w  r#rE\        P                  ! W4.R 7      ! V4      pV P                  V4      p\        P                  P                  V4      pV P                  V4      p\        P                  ! V4      pVP                  R^^V4      pW,          # )rw   r“   )
rT   rH   Ú	AvgPool2dr  r  Úrelur  r.   ÚsigmoidrV   )r   r  r  ÚhÚwÚcr   s   &&     r   r   ÚSEBlock.__call__Ì  sy   € Ø—\‘\‰
ˆˆaÜ�LŠL a VÕ,¨VÓ4ˆØ�K‰K˜‹NˆÜ�I‰I�N‰N˜1ÓˆØ�K‰K˜‹NˆÜ�JŠJ�q‹MˆØ�I‰I�b˜!˜Q Ó"ˆØ�zÐr   )r  r  )g      °?rd   r'   s   @@r   r  r  ®  s#   ù‡ € ñ÷
õ 
÷.÷ ð r   r  c                   óX   a a€ ] tR tRt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )	rÝ   i×  z}MobileOne building block.

This implementation only uses the inference time CNN architecture and uses FastViTHD conventions.
c                óJ   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[ RS[ RS[ RS[/# )	r,   ri   rk   rq   r¸   rr   rÎ   rs   rÜ   )r@   rA   )r0   r(   s   "€r   r1   ÚMobileOneBlock.__annotate__Ý  s[   ø€ ÷ $
ñ $
áð$
ñ ð$
ñ ð	$
ñ
 ð$
ñ ð$
ñ ð$
ñ ð$
ñ ñ$
r   c	                óX  <€ \         S	V `  4        Wpn        W@n        WPn        W`n        W0n        Wn        W n        V'       d   \        V4      V n
        M\        P                  ! 4       V n
        \        P                  ! 4       V n        \        P                  ! VVVVVVVR R7      V n        R# )TrÍ   N)r   r   rs   r¸   rr   rÎ   rq   ri   rk   r  ÚserH   ÚIdentityr„   rÊ   rx   r½   )
r   ri   rk   rq   r¸   rr   rÎ   rs   rÜ   r   s
   &&&&&&&&&€r   r   ÚMobileOneBlock.__init__Ý  sŠ   ø€ ô 	‰ÑÔØŒØŒØŒØ ŒØ&ÔØ&ÔØ(Ô÷ Ü˜lÓ+ˆD�Gä—k’k“mˆDŒGäŸ'š'›)ˆŒÜŸIšIØ#Ø%Ø#ØØØØØô	
ˆÖr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rO   r-   )r0   r(   s   "€r   r1   r*    s#   ø€ ÷ >ñ >™"Ÿ(™(ð >¡r§x¡xñ >r   c                ó`   € V P                  V P                  V P                  V4      4      4      # r   )rÊ   r,  r½   r�   s   &&r   r   ÚMobileOneBlock.__call__  s%   € Ø�‰˜tŸw™w t×'8Ñ'8¸Ó';Ó<Ó=Ð=r   )
rÊ   rÎ   rs   ri   rq   rk   rr   r½   r,  r¸   )r‘   r   r‘   r‘   Frd   r'   s   @@r   rÝ   rÝ   ×  s$   ù‡ € ñ÷
$
õ $
÷L>÷ >ð >r   rÝ   c                   óP   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚConvolutionalStemi  c                ó    <€ V ^8„  d   QhRS[ /# r  r   )r0   r(   s   "€r   r1   ÚConvolutionalStem.__annotate__  s   ø€ ÷ 
ñ 
™|ñ 
r   c                óÎ   <€ \         SV `  4        ^pVP                  ^ ,          p\        \	        VV^^^^R7      \	        VV^^^VR7      \	        VV^^^ ^R7      .4      V n        R# )rë   )ri   rk   rq   r¸   rr   rs   N)r   r   r	  r*   rÝ   r   )r   r  ri   rk   r   s   &&  €r   r   ÚConvolutionalStem.__init__  s†   ø€ Ü‰ÑÔØˆØ×(Ñ(¨Õ+ˆÜ(äØ +Ø!-Ø !ØØØôô Ø ,Ø!-Ø !ØØØ'ôô Ø ,Ø!-Ø !ØØØôð#ó
ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rO   r-   )r0   r(   s   "€r   r1   r5  )  s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                ó$   € V P                  V4      # r   ©r   r�   s   &&r   r   ÚConvolutionalStem.__call__)  s   € Ø�{‰{˜1‹~Ðr   r:  )	r    r!   r"   r#   r   r   r$   r%   r&   r'   s   @@r   r3  r3    s   ù‡ € ÷
ó 
÷B÷ ð r   r3  c                   óX   a a€ ] tR tRt oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )	ÚFastViTHDModeli-  a&  
Based on https://github.com/apple/ml-fastvlm/blob/592b4add3c1c8a518e77d95dc6248e76c1dd591f/llava/model/multimodal_encoder/mobileclip/mci.py
Hardcoded, for now, for:
- FastViTHD variant
- Use inference_mode (i.e., modules contain the convolutional reparameterized versions of the architecture)
c                ó    <€ V ^8„  d   QhRS[ /# r  r   )r0   r(   s   "€r   r1   ÚFastViTHDModel.__annotate__5  s   ø€ ÷ 
ñ 
™|ñ 
r   c           
     ó:  <€ \         SV `  4        VP                  f#   R .\        VP                  4      ,          Vn        Wn        \        V4      V n        \        V4      V n	        \        VP                  R,          \        VP                  R,          VP                  ,          4      ^^^VP                  R,          RR7      V n        \        P                   ! \        VP                  R,          VP                  ,          4      VP"                  4      V n        R # )NTrÛ   r“   )r   r   r  r¼   r  r  r3  Úpatch_embedr  r  rÝ   r	  r@   Ú	cls_ratioÚconv_exprH   rI   Únum_classesÚhead)r   r  r   s   &&€r   r   ÚFastViTHDModel.__init__5  sÜ   ø€ Ü‰ÑÔØ×!Ñ!Ò)Ø&* V¬c°&·-±-Ó.@Õ%@ˆFÔ"ØŒô -¨VÓ4ˆÔÜ-¨fÓ5ˆŒÜ&Ø×)Ñ)¨"Õ-Ü˜V×.Ñ.¨rÕ2°V×5EÑ5EÕEÓFØØØØ×$Ñ$ RÕ(Øô
ˆŒô —I’IÜ�×!Ñ! "Õ%¨×(8Ñ(8Õ8Ó9¸6×;MÑ;Mó
ˆŽ	r   c                óJ   <€ V ^8„  d   QhRS[ P                  RS[S[,          /# )r,   r   Úoutput_hidden_states©r.   r/   r   rA   )r0   r(   s   "€r   r1   r?  K  s'   ø€ ÷ *ñ *á�8‰8ð*ñ '¡t�nñ*r   c                óæ   € V P                  V4      pV'       d   V3MR pV P                   F  pV! V4      pV'       g   K  W13,           pK   	  V P                  V4      pV P                  V4      pWQV3# r   )rA  r  rC  rE  )r   r   rH  Úencoder_statesÚlayerÚcls_outs   &&&   r   r   ÚFastViTHDModel.__call__K  sn   € ð
 ×Ñ˜QÓˆç!5˜!™¸4ˆØ—\”\ˆEÙ�a“ˆAß#Ñ#Ø!/°$Õ!6’ñ "ð
 �M‰M˜!ÓˆØ—)‘)˜A“,ˆà˜>Ð)Ð)r   )r  rC  rE  r  rA  r   rd   r'   s   @@r   r=  r=  -  s#   ù‡ € ñ÷
ó 
÷,*÷ *ò *r   r=  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚGlobalPool2Di^  z<This class implements global pooling with linear projection.c                ó*   <€ V ^8„  d   QhRS[ RS[ RR/# )r,   Úin_dimÚout_dimr?   Nr´   )r0   r(   s   "€r   r1   ÚGlobalPool2D.__annotate__a  s"   ø€ ÷ 0ñ 0™sð 0©Sð 0°Tñ 0r   c                ó\   <€ \         SV `  4        \        P                  ! W34      V n        R # r   )r   r   r.   rŒ   rL   )r   rR  rS  r   s   &&&€r   r   ÚGlobalPool2D.__init__a  s!   ø€ Ü‰ÑÔÜ—H’H˜fÐ.Ó/ˆŽ	r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# rO   r-   )r0   r(   s   "€r   r1   rT  e  s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                ó´   € VP                   ^8X  g!   Q RP                  VP                  4      4       hVP                  ^^.R7      pWP                  ,          pV# )é   zNInput should be 4-dimensional (Batch x in_dim x in_height x in_width). Got: {})Úaxis)Úndimr0   rT   r”   rL   r�   s   &&r   r   ÚGlobalPool2D.__call__e  sU   € à�F‰F�aŒKð	
à[×bÑbØ�G‰Gó
ó	
Øð �F‰F˜˜A˜ˆFÓˆà—	‘	�MˆØˆr   rá   rd   r'   s   @@r   rP  rP  ^  s!   ù‡ € ÙF÷0ó 0÷÷ ð r   rP  c                   óZ   a a€ ] tR tRt oV3R lV 3R lltRV3R lR lltR tRtVtV ;t	# )	ÚVisionModelis  c                ó    <€ V ^8„  d   QhRS[ /# r  r   )r0   r(   s   "€r   r1   ÚVisionModel.__annotate__t  s   ø€ ÷ Qñ Q™|ñ Qr   c                óz  <€ \         SV `  4        VP                  V n        V P                  R9  d   \        RV P                   24      h\	        V4      V n        VP                  eT   \        VP                  R,          VP                  ,          4      p\        W!P                  4      V P
                  n        R# R# )Úllava_qwen2zUnsupported model type: N)rb  Úfastvlmr“   )r   r   Ú
model_typerÿ   r=  Úvision_modelÚprojection_dimr@   r	  rB  rP  rE  )r   r  rR  r   s   && €r   r   ÚVisionModel.__init__t  s™   ø€ Ü‰ÑÔà ×+Ñ+ˆŒØ�?‰?Ð"<Ô<ÜÐ7¸¿¹Ð7HÐIÓJÐJä*¨6Ó2ˆÔð × Ñ Ò,Ü˜×*Ñ*¨2Õ.°×1AÑ1AÕAÓBˆFÜ%1°&×:OÑ:OÓ%PˆD×ÑÖ"ñ -r   c                ód   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[ P                  /# )r,   r   rH  r?   rI  )r0   r(   s   "€r   r1   r`  ƒ  s1   ø€ ÷ :ñ :Ù—‘ð:Ù19¹$µð:á	�‰ñ:r   c                ó$   € V P                  W4      # r   )re  )r   r   rH  s   &&&r   r   ÚVisionModel.__call__ƒ  s   € ð × Ñ  Ó9Ð9r   c                ób  a€ VR ,          P                   RR w  r#W#8„  oV3R lp/ pVP                  4        Fu  w  rgV! V4      '       d0   VP                  ^8X  d   VP                  ^ ^^^4      WV&   K<  WuV&   KB  RV9   d    S'       g   VP                  ^^^ 4      WV&   Kh  RV9   d   Kq  WuV&   Kw  	  V# )zBvision_tower.vision_model.patch_embed.blocks.1.reparam_conv.weightNc                 óš   <€ S'       d   R # RV 9   d   R# RV 9   d   R# RV 9   d   R# RV 9   d   R# RV 9   d   R# RV 9   d   R# RV 9   d   R# R # )	Fz.reparam_conv.weightTz.conv.weightz.fc1.weightz.fc2.weightz.lkb_reparam.weightz.reduce.weightz.expand.weightr4   )r_   Úskip_transposes   &€r   Úis_convÚ%VisionModel.sanitize.<locals>.is_conv�  sY   ø€ ßÙØ%¨Ô*ÙØ Ô"ÙØ Ô!ÙØ Ô!ÙØ$¨Ô)ÙØ 1Ô$ÙØ 1Ô$ÙÙr   rñ   Únum_batches_trackedéþÿÿÿ)rT   Úitemsr[  rS   )	r   Úweightsr\   rZ   rn  Úsanitized_weightsr_   r`   rm  s	   &&      @r   ÚsanitizeÚVisionModel.sanitizeˆ  s¹   ø€ àØPõ
ç
‰%��ð‰ˆð ™ˆõ	ð& ÐØ—M‘M–O‰DˆAÙ�q�zŠzð
 —6‘6˜Q”;Ø+,¯;©;°q¸!¸QÀÓ+BÐ%Ó(à+, aÓ(Ø !Ô#¯NØ'(§{¡{°1°a¸Ó';Ð!Ó$Ø&¨!Ô+áà'( !Ó$ñ! $ð" !Ð r   )rd  re  r   )
r    r!   r"   r#   r   r   ru  r$   r%   r&   r'   s   @@r   r^  r^  s  s%   ù‡ € ÷Qó Q÷:ò :÷
,!ò ,!r   r^  )"Útypingr   r   r   Úmlx.coreÚcorer.   Úmlx.nnrH   r  r   rm   r	   Úlistr*   r8   rg   r†   r›   r±   rÈ   rÖ   rä   rí   r„   ry   r  r  r  rÝ   r3  r=  rP  r^  r4   r   r   Ú<module>r|     sO  ðß (Ñ (å Ý å  ô�b—i‘iô ô˜ô ô-ˆ2�9‰9ô -ô`&ˆb�i‰iô &ôR�r—y‘yô ô(#�R—Y‘Yô #ôL$$ˆR�Y‰Yô $$ôN 4˜RŸY™Yô  4ôF"�—‘ô "ôJ$ˆr�y‰yô $ô< �B—I‘Iô  ðP ØØŸ7™7ØŸL™L÷"õJ$ôN&ˆb�i‰iô &ôR->�R—Y‘Yô ->ô`#˜Ÿ	™	ô #ôL.*�R—Y‘Yô .*ôb�2—9‘9ô ô*A!�"—)‘)ö A!r   