+
    UV-j«P  ã                   ó(  € ^ RI Ht ^ RIHt ^ RIHt ^ RIt^RI	H
t
  ! R R]P                  4      tR R lt ! R R	]P                  4      t ! R
 R]P                  4      t ! R R]P                  4      tR tRR lt ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      tR# )é    )ÚOptionalN©ÚVisionConfigc                   óX   a a€ ] tR t^
t oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )ÚClippableLinearu]  Linear layer with optional input/output clamping.

Matches PyTorch's Gemma4ClippableLinear: wraps nn.Linear, clamps input/output.
Clip bounds are stored as buffers in the checkpoint (scalar tensors).
Initialized to Â±inf so clamping is a no-op until real values are loaded.
When use_clipping=False, behaves as a standard nn.Linear (no clip params).
c                ó2   <€ V ^8„  d   QhRS[ RS[ RS[RS[/# )é   Úin_featuresÚout_featuresÚbiasÚuse_clipping)ÚintÚbool)ÚformatÚ__classdict__s   "€Úm/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/gemma4/vision.pyÚ__annotate__ÚClippableLinear.__annotate__   s3   ø€ ÷ 5ñ 5áð5ñ ð5ñ ð	5ñ
 ñ5ó    c                óž  <€ \         SV `  4        \        P                  ! WVR 7      V n        W@n        V'       d“   \        P                  ! \        R4      4      V n	        \        P                  ! \        R4      4      V n
        \        P                  ! \        R4      4      V n        \        P                  ! \        R4      4      V n        R# R# )©r   z-infÚinfN)ÚsuperÚ__init__ÚnnÚLinearÚlinearr   ÚmxÚarrayÚfloatÚ	input_minÚ	input_maxÚ
output_minÚ
output_max)Úselfr
   r   r   r   Ú	__class__s   &&&&&€r   r   ÚClippableLinear.__init__   s€   ø€ ô 	‰ÑÔÜ—i’i ÀÔEˆŒØ(ÔßÜŸXšX¤e¨F£mÓ4ˆDŒNÜŸXšX¤e¨E£lÓ3ˆDŒNÜ Ÿhšh¤u¨V£}Ó5ˆDŒOÜ Ÿhšh¤u¨U£|Ó4ˆDŽOñ	 r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r	   ÚxÚreturn©r   r   )r   r   s   "€r   r   r   #   s#   ø€ ÷ ñ ™"Ÿ(™(ð ¡r§x¡xñ r   c                ó  € V P                   '       d,   \        P                  ! WP                  V P                  4      pV P                  V4      pV P                   '       d,   \        P                  ! WP                  V P                  4      pV# ©N)r   r   Úclipr!   r"   r   r#   r$   ©r%   r*   s   &&r   Ú__call__ÚClippableLinear.__call__#   s\   € Ø××ÐÜ—’˜Ÿ>™>¨4¯>©>Ó:ˆAØ�K‰K˜‹NˆØ××ÐÜ—’˜Ÿ?™?¨D¯O©OÓ<ˆAØˆr   )r"   r!   r   r$   r#   r   )FT©
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r1   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r&   r   s   @@r   r   r   
   s#   ù‡ € ñ÷5õ 5÷ ÷ ð r   r   c                ód   € V ^8„  d   QhR\         P                  R\        R\         P                  /# )r	   ÚindicesÚnum_classesr+   )r   r   r   )r   s   "r   r   r   ,   s.   € ÷ Vñ V”R—X‘Xð V¬Cð V´B·H±Hñ Vr   c                ó˜   € \         P                  ! V R4      \         P                  ! V4      8H  P                  \         P                  4      # )zOne-hot encoding.éÿÿÿÿ)r   Úexpand_dimsÚarangeÚastypeÚfloat32)r>   r?   s   &&r   Úone_hotrF   ,   s0   € ä�NŠN˜7 BÓ'¬2¯9ª9°[Ó+AÑA×IÑIÌ"Ï*É*ÓUÐUr   c                   óX   a a€ ] tR t^1t oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )ÚVisionRMSNormz‚RMS normalization with learned scale: normed * weight.

Matches PyTorch Gemma4RMSNorm(with_scale=True): full float32 computation.
c                ó&   <€ V ^8„  d   QhRS[ RS[/# ©r	   ÚdimÚeps©r   r    )r   r   s   "€r   r   ÚVisionRMSNorm.__annotate__7   s   ø€ ÷ &ñ &™Cð &¡eñ &r   c                óh   <€ \         SV `  4        W n        \        P                  ! V34      V n        R # r.   )r   r   rL   r   ÚonesÚweight©r%   rK   rL   r&   s   &&&€r   r   ÚVisionRMSNorm.__init__7   s$   ø€ Ü‰ÑÔØŒÜ—g’g˜s˜f“oˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r)   r,   )r   r   s   "€r   r   rN   <   s#   ø€ ÷ &ñ &™"Ÿ(™(ð &¡r§x¡xñ &r   c                ón  € VP                  \        P                  4      p\        P                  ! V^,          RRR7      pV\        P                  ! W0P
                  ,           4      ,          pW@P                  P                  \        P                  4      ,          pVP                  VP                  4      # ©r	   T©ÚaxisÚkeepdimsrA   )rD   r   rE   ÚmeanÚrsqrtrL   rQ   Údtype)r%   r*   Úx_floatÚvarÚnormedÚresults   &&    r   r1   ÚVisionRMSNorm.__call__<   sq   € Ø—(‘(œ2Ÿ:™:Ó&ˆÜ�gŠg�g˜q•j r°DÔ9ˆØœ2Ÿ8š8 C¯(©(¥NÓ3Õ3ˆØŸ+™+×,Ñ,¬R¯Z©ZÓ8Õ8ˆØ�}‰}˜QŸW™WÓ%Ð%r   ©rL   rQ   ©g�íµ ÷Æ°>r3   r<   s   @@r   rH   rH   1   s#   ù‡ € ñ÷
&õ &÷
&÷ &ð &r   rH   c                   óX   a a€ ] tR t^Dt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )ÚVisionRMSNormNoScalezˆRMS normalization without learnable scale (parameter-free).

Matches PyTorch Gemma4RMSNorm(with_scale=False): full float32 computation.
c                ó    <€ V ^8„  d   QhRS[ /# )r	   rL   )r    )r   r   s   "€r   r   Ú!VisionRMSNormNoScale.__annotate__J   s   ø€ ÷ ñ ™Eñ r   c                ó0   <€ \         SV `  4        Wn        R # r.   )r   r   rL   )r%   rL   r&   s   &&€r   r   ÚVisionRMSNormNoScale.__init__J   s   ø€ Ü‰ÑÔØŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r)   r,   )r   r   s   "€r   r   rg   N   s'   ø€ ÷ Dñ D™"Ÿ(™(ð D¡r§x¡xñ Dr   c                ó  € VP                  \        P                  4      p\        P                  ! V^,          RRR7      pV\        P                  ! W0P
                  ,           4      ,          P                  VP                  4      # rV   )rD   r   rE   rZ   r[   rL   r\   )r%   r*   r]   r^   s   &&  r   r1   ÚVisionRMSNormNoScale.__call__N   sS   € Ø—(‘(œ2Ÿ:™:Ó&ˆÜ�gŠg�g˜q•j r°DÔ9ˆØœ"Ÿ(š( 3¯©¥>Ó2Õ2×:Ñ:¸1¿7¹7ÓCÐCr   ©rL   rc   r3   r<   s   @@r   re   re   D   s&   ù‡ € ñ÷
õ ÷D÷ Dð Dr   re   c                   óX   a a€ ] tR t^Tt oRtRV3R lV 3R llltV3R lR ltRtVtV ;t	# )ÚRMSNormz1Standard Gemma4 RMSNorm: weight applied directly.c                ó&   <€ V ^8„  d   QhRS[ RS[/# rJ   rM   )r   r   s   "€r   r   ÚRMSNorm.__annotate__W   s   ø€ ÷ ñ ™Cð ¡eñ r   c                óh   <€ \         SV `  4        \        P                  ! V34      V n        W n        R # r.   )r   r   r   rP   rQ   rL   rR   s   &&&€r   r   ÚRMSNorm.__init__W   s$   ø€ Ü‰ÑÔÜ—g’g˜s˜f“oˆŒØŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r)   r,   )r   r   s   "€r   r   rq   \   s#   ø€ ÷ :ñ :™"Ÿ(™(ð :¡r§x¡xñ :r   c                ój   € \         P                  P                  WP                  V P                  4      # r.   )r   ÚfastÚrms_normrQ   rL   r0   s   &&r   r1   ÚRMSNorm.__call__\   s!   € Ü�w‰w×Ñ §;¡;°·±Ó9Ð9r   rb   rc   r3   r<   s   @@r   ro   ro   T   s!   ù‡ € Ù;÷õ ÷
:÷ :ð :r   ro   c                óÌ   € V RRV P                   R,          ^,          13,          pV RV P                   R,          ^,          R13,          p\        P                  ! V) V.RR7      # )z6Rotate half: [-x2, x1] matching PyTorch's rotate_half..N©rX   rA   )Úshaper   Úconcatenate)r*   Úx1Úx2s   &  r   Ú_rotate_halfr   `   sZ   € à	
ˆ3Ð"�!—'‘'˜"•+ Õ"Ð"Ð"Õ	#€BØ	
ˆ3�—‘˜•˜qÕ Ñ"Ð"Õ	#€BÜ�>Š>˜B˜3 ˜)¨"Ô-Ð-r   c                ó.  € V P                   R,          pVP                  ^8X  Edn   V^,          pRV,          \        P                  ! ^ V4      P	                  \        P
                  4      ,          p\        P                  ! W%4      pVR,          P	                  \        P
                  4      V,          p\        P                  ! V4      p\        P                  ! V4      p	\        P                  ! Wˆ.RR7      P	                  V P                  4      p\        P                  ! W™.RR7      P	                  V P                  4      p	\        P                  ! V^R7      p\        P                  ! V	^R7      p	W,          \        V 4      V	,          ,           # VP                   R,          p
^V^V
,          ,          ,          pV^,          p. p\        V
4       EF§  pV RWë,          V^,           V,          13,          pRV,          \        P                  ! ^ V4      P	                  \        P
                  4      ,          p\        P                  ! W%4      pVRWî^,           13,          P	                  \        P
                  4      V,          p\        P                  ! V4      p\        P                  ! V4      p\        P                  ! VV.RR7      P	                  V P                  4      p\        P                  ! VV.RR7      P	                  V P                  4      p\        P                  ! V^R7      p\        P                  ! V^R7      pVV,          \        V4      V,          ,           pVP                  V4       EKª  	  \        P                  ! VRR7      # )aH  Apply multidimensional RoPE matching PT's apply_multidimensional_rope.

Splits the head dimension into ndim parts and applies rotate_half
independently to each part (one per spatial dimension). This is critical:
rotate_half must NOT mix features across spatial dimensions.

inputs: [B, L, N, H], positions: [B, L, 2] or [B, L].
g       @.rz   rA   ).N)r{   Úndimr   rC   rD   rE   ÚpowerÚcosÚsinr|   r\   rB   r   ÚrangeÚappend)ÚinputsÚ	positionsÚbase_frequencyÚhead_dimÚhalfÚfreq_exponentsÚ	timescaleÚsinusoid_inpÚcos_valÚsin_valr�   Úchannels_per_dimÚhalf_per_dimÚresult_partsÚdÚx_partÚcos_dÚsin_dÚy_parts   &&&                r   Úapply_multidimensional_roper™   g   s   € ð �|‰|˜BÕ€Hà‡~�~˜Õà˜1�}ˆØ �.¬B¯IªI°a¸Ó,>×,EÑ,EÄbÇjÁjÓ,QÕQˆÜ—H’H˜^Ó<ˆ	Ø  Õ+×2Ñ2´2·:±:Ó>ÀÕJˆÜ—&’&˜Ó&ˆÜ—&’&˜Ó&ˆÜ—.’. 'Ð!3¸"Ô=×DÑDÀVÇ\Á\ÓRˆÜ—.’. 'Ð!3¸"Ô=×DÑDÀVÇ\Á\ÓRˆÜ—.’. ¨qÔ1ˆÜ—.’. ¨qÔ1ˆØÕ¤,¨vÓ"6¸Õ"@Õ@Ð@à�?‰?˜2Õ€DØ˜H¨¨T­Õ2Õ3ÐØ# qÕ(€Lð €LÜ�4�[ˆà˜˜QÕ1°Q¸µUÐ>NÕ4NÐNÐNÕOˆð Ð 0Õ0´B·I²I¸aÀÓ4N×4UÑ4UÜ�J‰Jó5
õ 
ˆô —H’H˜^Ó<ˆ	à�c˜1 1�u˜9�nÕ%×,Ñ,¬R¯Z©ZÓ8¸9ÕDð 	ô —’�|Ó$ˆÜ—’�|Ó$ˆä—’  u˜~°BÔ7×>Ñ>¸v¿|¹|ÓLˆÜ—’  u˜~°BÔ7×>Ñ>¸v¿|¹|ÓLˆÜ—’˜u¨1Ô-ˆÜ—’˜u¨1Ô-ˆð ˜%•¤,¨vÓ"6¸Õ">Õ>ˆØ×Ñ˜F×#ñ- ô0 �>Š>˜,¨RÔ0Ð0r   c                   óT   a a€ ] tR t^¡t oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚVisionAttentionc                ó    <€ V ^8„  d   QhRS[ /# ©r	   Úconfigr   )r   r   s   "€r   r   ÚVisionAttention.__annotate__¢   s   ø€ ÷ $.ñ $.™|ñ $.r   c                óD  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP                  V n        VP                  R ,          V n	        \        VRR4      p\        V P                  V P                  V P                  ,          RVR7      V n        \        V P                  V P
                  V P                  ,          RVR7      V n        \        V P                  V P
                  V P                  ,          RVR7      V n        \        V P                  V P                  ,          V P                  RVR7      V n        \!        V P                  4      V n        \!        V P                  4      V n        \'        4       V n        R# )Ú
rope_thetaÚuse_clipped_linearsF©r   r   N)r   r   Únum_attention_headsÚ	num_headsÚnum_key_value_headsÚnum_kv_headsrŠ   Úhidden_sizeÚrope_parametersÚrope_base_frequencyÚgetattrr   Úq_projÚk_projÚv_projÚo_projrH   Úq_normÚk_normre   Ú_v_norm©r%   rž   r/   r&   s   && €r   r   ÚVisionAttention.__init__¢   sF  ø€ Ü‰ÑÔØ×3Ñ3ˆŒØ"×6Ñ6ˆÔØŸ™ˆŒØ!×-Ñ-ˆÔØ#)×#9Ñ#9¸,Õ#GˆÔ ä�vÐ4°eÓ<ˆÜ%Ø×ÑØ�N‰N˜TŸ]™]Õ*ØØô	
ˆŒô &Ø×ÑØ×Ñ §¡Õ-ØØô	
ˆŒô &Ø×ÑØ×Ñ §¡Õ-ØØô	
ˆŒô &Ø�N‰N˜TŸ]™]Õ*Ø×ÑØØô	
ˆŒô $ D§M¡MÓ2ˆŒÜ# D§M¡MÓ2ˆŒÜ+Ó-ˆŽr   c                ó’   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          RS[ P                  /# ©r	   r*   rˆ   Úmaskr+   ©r   r   r   )r   r   s   "€r   r   rŸ   È   sB   ø€ ÷ (ñ (Ù—‘ð(Ù&(§h¡hð(Ù6>¹r¿x¹xÕ6Hð(á	�‰ñ(r   c                ó(  € VP                   w  rEpV P                  V4      P                  WEV P                  V P                  4      pV P                  V4      P                  WEV P                  V P                  4      pV P                  V4      P                  WEV P                  V P                  4      p	V P                  V4      pV P                  V4      pV P                  V	4      p	\        WrV P                  4      p\        W‚V P                  4      pVP                  ^ ^^^4      pVP                  ^ ^^^4      pV	P                  ^ ^^^4      p	^RIHp
 V
! WxV	RVR7      pVP                  ^ ^^^4      P                  WER4      pV P!                  V4      # )r   )Úensure_fused_sdpag      ð?)Úscaler·   rA   )r{   r¬   Úreshaper¥   rŠ   r­   r§   r®   r°   r±   r²   r™   rª   Ú	transposeÚbaserº   r¯   )r%   r*   rˆ   r·   ÚBÚLÚ_ÚqÚkÚvrº   Úattn_outputs   &&&&        r   r1   ÚVisionAttention.__call__È   sU  € ð —'‘'‰ˆˆaà�K‰K˜‹N×"Ñ" 1¨¯©¸¿¹ÓGˆØ�K‰K˜‹N×"Ñ" 1¨×):Ñ):¸D¿M¹MÓJˆØ�K‰K˜‹N×"Ñ" 1¨×):Ñ):¸D¿M¹MÓJˆà�K‰K˜‹NˆØ�K‰K˜‹NˆØ�L‰L˜‹Oˆô (¨°d×6NÑ6NÓOˆÜ'¨°d×6NÑ6NÓOˆð �K‰K˜˜1˜a Ó#ˆØ�K‰K˜˜1˜a Ó#ˆØ�K‰K˜˜1˜a Ó#ˆõ 	-á'¨¨a°sÀÔFˆð "×+Ñ+¨A¨q°!°QÓ7×?Ñ?ÀÀbÓIˆà�{‰{˜;Ó'Ð'r   )r²   rŠ   r¨   r±   r­   r¥   r§   r¯   r°   r¬   rª   r®   r.   ©	r4   r5   r6   r7   r   r1   r9   r:   r;   r<   s   @@r   r›   r›   ¡   s   ù‡ € ÷$.ó $.÷L(÷ (ò (r   r›   c                   óP   a a€ ] tR t^êt oV3R lV 3R lltV3R lR ltRtVtV ;t# )Ú	VisionMLPc                ó    <€ V ^8„  d   QhRS[ /# r�   r   )r   r   s   "€r   r   ÚVisionMLP.__annotate__ë   s   ø€ ÷ 
ñ 
™|ñ 
r   c                ó.  <€ \         SV `  4        \        VR R4      p\        VP                  VP
                  RVR7      V n        \        VP                  VP
                  RVR7      V n        \        VP
                  VP                  RVR7      V n        R# )r¢   Fr£   N)	r   r   r«   r   r¨   Úintermediate_sizeÚ	gate_projÚup_projÚ	down_projr³   s   && €r   r   ÚVisionMLP.__init__ë   s‡   ø€ Ü‰ÑÔÜ�vÐ4°eÓ<ˆÜ(Ø×Ñ × 8Ñ 8¸uÐSWô
ˆŒô 'Ø×Ñ × 8Ñ 8¸uÐSWô
ˆŒô )Ø×$Ñ$ f×&8Ñ&8¸uÐSWô
ˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r)   r,   )r   r   s   "€r   r   rË   ø   s'   ø€ ÷ Sñ S™"Ÿ(™(ð S¡r§x¡xñ Sr   c                ó–   € V P                  \        P                  ! V P                  V4      4      V P	                  V4      ,          4      # r.   )rÐ   r   Úgelu_approxrÎ   rÏ   r0   s   &&r   r1   ÚVisionMLP.__call__ø   s0   € Ø�~‰~œbŸnšn¨T¯^©^¸AÓ->Ó?À$Ç,Á,ÈqÃ/ÕQÓRÐRr   )rÐ   rÎ   rÏ   rÇ   r<   s   @@r   rÉ   rÉ   ê   s!   ù‡ € ÷
ó 
÷S÷ Sð Sr   rÉ   c                   óT   a a€ ] tR t^üt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚVisionTransformerBlockc                ó    <€ V ^8„  d   QhRS[ /# r�   r   )r   r   s   "€r   r   Ú#VisionTransformerBlock.__annotate__ý   s   ø€ ÷ 
ñ 
™|ñ 
r   c                ó”  <€ \         SV `  4        \        V4      V n        \	        V4      V n        \        VP                  VP                  R 7      V n	        \        VP                  VP                  R 7      V n
        \        VP                  VP                  R 7      V n        \        VP                  VP                  R 7      V n        R# )rm   N)r   r   r›   Ú	self_attnrÉ   Úmlpro   r¨   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernorm©r%   rž   r&   s   &&€r   r   ÚVisionTransformerBlock.__init__ý   sŸ   ø€ Ü‰ÑÔÜ(¨Ó0ˆŒÜ˜VÓ$ˆŒÜ& v×'9Ñ'9¸v×?RÑ?RÔSˆÔÜ(/Ø×Ñ F×$7Ñ$7ô)
ˆÔ%ô *1Ø×Ñ F×$7Ñ$7ô*
ˆÔ&ô +2Ø×Ñ F×$7Ñ$7ô+
ˆÖ'r   c                ó’   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          RS[ P                  /# r¶   r¸   )r   r   s   "€r   r   rÙ     sB   ø€ ÷ ñ Ù—‘ðÙ&(§h¡hðÙ6>¹r¿x¹xÕ6Hðá	�‰ñr   c                óð   € V P                  V4      pV P                  WBV4      pV P                  V4      pW,           pV P                  V4      pV P	                  V4      pV P                  V4      pWh,           # r.   )rÞ   rÛ   rß   rà   rÜ   rá   )	r%   r*   rˆ   r·   r_   Úattn_outÚhÚnormed_hÚffw_outs	   &&&&     r   r1   ÚVisionTransformerBlock.__call__  sq   € ð ×%Ñ% aÓ(ˆØ—>‘> &°TÓ:ˆØ×0Ñ0°Ó:ˆØ�Lˆà×1Ñ1°!Ó4ˆØ—(‘(˜8Ó$ˆØ×1Ñ1°'Ó:ˆØ�{Ðr   )rÞ   rÜ   rß   rá   rà   rÛ   r.   rÇ   r<   s   @@r   r×   r×   ü   s   ù‡ € ÷
ó 
÷÷ ò r   r×   c                   ót   a a€ ] tR tRt oV3R lV 3R lltV3R lR ltV3R lR ltV3R lR	 ltR
tVt	V ;t
# )ÚVisionPatchEmbedderi  c                ó    <€ V ^8„  d   QhRS[ /# r�   r   )r   r   s   "€r   r   Ú VisionPatchEmbedder.__annotate__  s   ø€ ÷ 

ñ 

™|ñ 

r   c                ón  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        \
        P                  ! ^V P                  ^,          ,          V P                  RR7      V n        \        P                  ! ^V P                  V P                  34      V n
        R# )é   Fr   N)r   r   r¨   Ú
patch_sizeÚposition_embedding_sizer   r   Ú
input_projr   rP   Úposition_embedding_tablerâ   s   &&€r   r   ÚVisionPatchEmbedder.__init__  s‹   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ ×+Ñ+ˆŒØ'-×'EÑ'EˆÔ$ÜŸ)š)Ø�—‘ Õ"Õ" D×$4Ñ$4¸5ô
ˆŒô )+¯ªØ�×,Ñ,¨d×.>Ñ.>Ð?ó)
ˆÖ%r   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r	   Úpatch_positionsÚpadding_positionsr+   r,   )r   r   s   "€r   r   rî   '  s1   ø€ ÷ #ñ #Ù!Ÿx™xð#Ù<>¿H¹Hð#á	�‰ñ#r   c                ó@  € \        WP                  4      pVP                  ^ ^^^4      P                  V P                  P
                  4      pW0P                  ,          pVP                  ^R7      p\        P                  ! \        P                  ! VR4      RV4      pV# )r   rz   ç        rA   )
rF   rò   r½   rD   rô   r\   Úsumr   ÚwhererB   )r%   r÷   rø   ÚohÚposition_embeddingss   &&&  r   Ú_position_embeddingsÚ(VisionPatchEmbedder._position_embeddings'  s�   € ô �_×&BÑ&BÓCˆà�\‰\˜!˜Q  1Ó%×,Ñ,¨T×-JÑ-J×-PÑ-PÓQˆØ ×#@Ñ#@Õ@ÐØ1×5Ñ5¸1Ð5Ó=ÐÜ ŸhšhÜ�NŠNÐ,¨bÓ1°3Ð8Kó
Ðð #Ð"r   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r	   Úpixel_valuesr+   r,   )r   r   s   "€r   r   rî   4  s'   ø€ ÷ Mñ M¡b§h¡hð M±2·8±8ñ Mr   c                ó’  € VP                   w  r#rEV P                  pWF,          pWV,          pVP                  W#WvW†4      p	V	P                  ^ ^^^^^4      p	V	P                  W'V,          W6,          V,          4      p	^V	R,
          ,          p	V P	                  V	P                  V P                  P                  P                  4      4      # )r   ç      à?)r{   rñ   r¼   r½   ró   rD   rQ   r\   )
r%   r  r¿   ÚCÚHÚWÚpÚpHÚpWÚpatchess
   &&        r   Ú	_patchifyÚVisionPatchEmbedder._patchify4  s¨   € à!×'Ñ'‰
ˆˆaØ�O‰OˆØ�VˆØ�Vˆð ×&Ñ& q¨R°BÓ:ˆØ×#Ñ# A q¨!¨Q°°1Ó5ˆØ—/‘/ !¨"¥W¨a­e°a­iÓ8ˆØ�w •}Õ%ˆØ�‰˜wŸ~™~¨d¯o©o×.DÑ.D×.JÑ.JÓKÓLÐLr   c                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r	   r  r÷   rø   r+   r,   )r   r   s   "€r   r   rî   B  sC   ø€ ÷ 
3ñ 
3á—h‘hð
3ñ Ÿ™ð
3ñ Ÿ8™8ð	
3ñ
 
�‰ñ
3r   c                óV   € V P                  V4      pV P                  W#4      pWE,           # r.   )r  rÿ   )r%   r  r÷   rø   Úhidden_statesrþ   s   &&&&  r   r1   ÚVisionPatchEmbedder.__call__B  s1   € ð Ÿ™ |Ó4ˆØ"×7Ñ7Øó
Ðð Õ2Ð2r   )r¨   ró   rñ   rò   rô   )r4   r5   r6   r7   r   rÿ   r  r1   r9   r:   r;   r<   s   @@r   rì   rì     s4   ù‡ € ÷

ó 

÷#ð #÷Mð M÷
3÷ 
3ð 
3r   rì   c                   óN   a a€ ] tR tRt oV3R lV 3R lltR tRR ltRtVtV ;t	# )ÚVisionPooleriO  c                ó    <€ V ^8„  d   QhRS[ /# r�   r   )r   r   s   "€r   r   ÚVisionPooler.__annotate__P  s   ø€ ÷ 6ñ 6™|ñ 6r   c                ó˜   <€ \         SV `  4        VP                  V n        VP                  V n        V P                  R ,          V n        R# )r  N)r   r   r¨   Údefault_output_lengthÚroot_hidden_sizerâ   s   &&€r   r   ÚVisionPooler.__init__P  s=   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ%+×%AÑ%AˆÔ"Ø $× 0Ñ 0°#Õ 5ˆÖr   c                ó   € VP                   ^,          p\        WC,          R,          4      pV^,          p\        P                  ! V^ R4      p\        P                  ! VR,          RRR7      ^,           p\        P
                  ! VP                  \        P                  4      V,          4      P                  \        P                  4      p	V	R,          W…,          V	R	,          ,          ,           p	\        W“4      P                  \        P                  4      V,          p
\        P                  ! RW¡4      P                  VP                  4      p\        P                  ! \        P                  ! V
^ 8H  ^R7      4      pW¼3# )
é   r  NTrW   zbLl,bLd->bldrz   ).r   rA   ).r  )r{   r   r   r/   ÚmaxÚfloorrD   rE   Úint32rF   Úeinsumr\   Úlogical_notÚall)r%   r*   r÷   ÚlengthÚinput_seq_lenrÃ   Ú	k_squaredÚclampedÚmax_xÚkernel_idxsÚweightsÚoutputr·   s   &&&&         r   Ú_avg_pool_by_positionsÚ#VisionPooler._avg_pool_by_positionsV  s  € ØŸ™ �
ˆÜ�Õ(¨SÕ0Ó1ˆØ�q•Dˆ	ä—'’'˜/¨1¨dÓ3ˆÜ—’�w˜v•¨R¸$Ô?À!ÕCˆÜ—h’h˜wŸ~™~¬b¯j©jÓ9¸AÕ=Ó>×EÑEÄbÇhÁhÓOˆØ! &Õ)¨U­Z¸;ÀvÕ;NÕ,NÕNˆÜ˜+Ó.×5Ñ5´b·j±jÓAÀIÕMˆÜ—’˜>¨7Ó6×=Ñ=¸a¿g¹gÓFˆÜ�~Š~œbŸfšf W°¡\¸Ô:Ó;ˆØˆ|Ðr   c                ó  € \         P                  ! \         P                  ! VR4      RV4      pT;'       g    V P                  pVP                  ^,          V8X  d   TpMV P                  WV4      w  rWP                  ,          pW3# )r  rú   rA   )r   rü   rB   r  r{   r*  r  )r%   r  r÷   rø   Úoutput_lengthr"  r·   s   &&&&&  r   r1   ÚVisionPooler.__call__d  sƒ   € ô ŸšÜ�NŠNÐ,¨bÓ1°3¸ó
ˆð ×<Ð< $×"<Ñ"<ˆØ×Ñ˜qÕ! VÔ+Ø$‰Dà"&×"=Ñ"=Ø°ó#ÑˆMð &×(=Ñ(=Õ=ˆØÐ"Ð"r   )r  r¨   r  r.   )
r4   r5   r6   r7   r   r*  r1   r9   r:   r;   r<   s   @@r   r  r  O  s   ù‡ € ÷6ó 6ò÷#ô #r   r  c                   óT   a a€ ] tR tRt oRtV3R lV 3R lltV3R lR ltRtVtV ;t	# )ÚVisionTransformerModeliw  zLHolds just the transformer layers (maps to vision_tower.encoder in weights).c                ó    <€ V ^8„  d   QhRS[ /# r�   r   )r   r   s   "€r   r   Ú#VisionTransformerModel.__annotate__z  s   ø€ ÷ 
ñ 
™|ñ 
r   c                ó”   <€ \         SV `  4        \        VP                  4       Uu. uF  p\	        V4      NK  	  upV n        R # u upi r.   )r   r   r…   Únum_hidden_layersr×   Úlayers)r%   rž   rÁ   r&   s   && €r   r   ÚVisionTransformerModel.__init__z  s@   ø€ Ü‰ÑÔä49¸&×:RÑ:RÔ4Só
Ù4S¨qÔ" 6Ö*Ñ4Sñ
ˆŽùò 
s   §Ac                ó‚   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  RS[ P                  /# )r	   r  rˆ   r·   r+   r,   )r   r   s   "€r   r   r2  €  s=   ø€ ÷ ñ ÙŸX™XðÙ24·(±(ðÙBDÇ(Á(ðá	�‰ñr   c                ó>   € V P                    F  pV! WV4      pK  	  V# r.   ©r5  )r%   r  rˆ   r·   Úlayers   &&&& r   r1   ÚVisionTransformerModel.__call__€  s$   € ð —[”[ˆEÙ! -¸DÓAŠMñ !àÐr   r9  r3   r<   s   @@r   r0  r0  w  s!   ù‡ € ÙV÷
ó 
÷÷ ð r   r0  c                   ój   a a€ ] tR tRt oRtV3R lV 3R lltR tV3R lR lt]R 4       t	R	t
VtV ;t# )
ÚVisionModeliˆ  z˜Top-level vision encoder matching PyTorch's Gemma4VisionEncoder.

Weight key structure:
  vision_tower.patch_embedder.*
  vision_tower.encoder.layers.*
c                ó    <€ V ^8„  d   QhRS[ /# r�   r   )r   r   s   "€r   r   ÚVisionModel.__annotate__�  s   ø€ ÷ <ñ <™|ñ <r   c                ó*  <€ \         SV `  4        Wn        VP                  V n        VP                  V n        VP
                  V n        VP                  V n        V P                  V P
                  ^,          ,          V n        \        V4      V n	        \        V4      V n        \        V4      V n        VP                  '       dO   \        P                   ! VP"                  34      V n        \        P&                  ! VP"                  34      V n        R# R# )r	   N)r   r   rž   Ú
model_typerñ   Úpooling_kernel_sizer  Úmax_patchesrì   Úpatch_embedderr0  Úencoderr  ÚpoolerÚstandardizer   Úzerosr¨   Ústd_biasrP   Ú	std_scalerâ   s   &&€r   r   ÚVisionModel.__init__�  sÐ   ø€ Ü‰ÑÔØŒØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ø%+×%AÑ%AˆÔ"Ø×5Ñ5¸×8PÑ8PÐRSÕ8SÕSˆÔä1°&Ó9ˆÔÜ-¨fÓ5ˆŒÜ" 6Ó*ˆŒà××ÐÜŸHšH f×&8Ñ&8Ð%:Ó;ˆDŒMÜŸWšW f×&8Ñ&8Ð%:Ó;ˆDŽNñ r   c                óÎ  € WP                   ,          pW P                   ,          pW4,          p\        P                  ! V4      p\        P                  ! V4      p\        P                  ! WgRR7      w  r‰\        P                  ! VP                  4       V	P                  4       .RR7      p
V P                  V,
          pV^ 8”  dD   \        P                  ! V^3R\        P                  R7      p\        P                  ! W¬.^ R7      pMV
RV P                   p\        P                  ! V P                  \        R7      pV^ 8”  d   RWåR% VP                  \        P                  4      Wå3# )z<Compute patch positions and padding mask for a single image.Úxy)Úindexingrz   ©r\   NTrA   )rñ   ÚnprC   ÚmeshgridÚstackÚflattenrC  ÚfullÚint64r|   rH  r   rD   r  )r%   r  r  r	  r
  Únum_patchesÚgrid_xÚgrid_yÚgxÚgyÚreal_positionsÚnum_paddingÚpad_positionsr÷   Úpadding_masks   &&&            r   Ú_patch_positions_singleÚ#VisionModel._patch_positions_single¡  s  € à—/‘/Õ!ˆØ—/‘/Õ!ˆØ•gˆä—’˜2“ˆÜ—’˜2“ˆÜ—’˜V°dÔ;‰ˆÜŸš 2§:¡:£<°·±³Ð">ÀRÔHˆà×&Ñ&¨Õ4ˆØ˜Œ?ÜŸGšG [°!Ð$4°bÄÇÁÔIˆMÜ Ÿnšn¨nÐ-LÐSTÔU‰Oà,Ð-?¨t×/?Ñ/?Ð@ˆOä—x’x × 0Ñ 0¼Ô=ˆØ˜Œ?Ø)-ˆL˜Ð&à×%Ñ%¤b§h¡hÓ/°ÐJÐJr   c                ó4   <€ V ^8„  d   QhRS[ P                  /# )r	   r+   r,   )r   r   s   "€r   r   r?  ¹  s   ø€ ÷ añ a©¯©ñ ar   c           	     óž  € \        V\        4      '       d›   . pV Fs  p\        V\        P                  4      '       g   \        P                  ! V4      pVP                  ^8X  d
   VR,          pV ! V4      pVP                  V^ ,          4       Ku  	  \        P                  ! V^ R7      R,          # \        V\        P                  4      '       g   \        P                  ! V4      pVP                  w  rVrxRp	V	'       EdQ   WpP                  ,          W€P                  ,          ,          p
\        W P                  4      p
V P                  Wx4      w  r¼p\        P                  ! \        P                  ! VR,          V^^34      4      p\        P                  ! \        P                  ! VR,          V^34      4      pV P                  VVRRV
13,          VRRV
13,          4      pV P                  V
,
          pV^ 8”  dQ   \        P                  ! VVVP                  R,          3VP                   R7      p\        P                  ! VV.^R7      pEM«. p. p. p\#        V4       EFM  pVVV^,            pVP                  w   pppV P                  VV4      w  ppp\        P                  ! VR,          4      p\        P                  ! VR,          4      p\        VV P                  4      pV P                  VVRRV13,          VRRV13,          4      pV P                  V,
          pV^ 8”  dQ   \        P                  ! ^VVP                  R,          3VP                   R7      p \        P                  ! VV .^R7      pVP                  V4       VP                  V4       VP                  V4       EKP  	  \        P                  ! V^ R7      p\        P                  ! V^ R7      p\        P                  ! V^ R7      pV( p!\        P$                  ! V!^4      \        P$                  ! V!^4      ,          p"\        P                  ! RVP                   R7      p#\        P&                  ! V"\        P                  ! RVP                   R7      V#4      p"\        P$                  ! V"^4      p"V P)                  VVV"4      p$V P+                  V$Wï4      w  p%p&V&P                  ^,          V P,                  8X  d   T&p!MV&( p!. p\#        V4       Fj  p\/        V!V,          P1                  \        P2                  4      P5                  4       P7                  4       4      p'VP                  V%VRV'13,          4       Kl  	  \        P                  ! V^ R7      R,          p$V P8                  P:                  '       d%   V$V P<                  ,
          V P>                  ,          p$V$# )	rð   Nrz   T:NNNrO  rú   rA   g     ˆÃÀ) Ú
isinstanceÚlistr   r   r�   r†   r|   r{   rñ   ÚminrC  r_  rP  ÚtilerD  rH  r\   r…   rB   rü   rE  rF  r  r   rD   r  rû   Úitemrž   rG  rI  rJ  )(r%   r  Úall_realÚimgr`   r¿   r  r  r  Úall_same_sizeÚnum_realrˆ   r^  rÁ   r÷   rø   Úinputs_embedsr\  Ú
pad_embedsÚ
all_embedsÚall_positionsÚall_paddingÚirç   ÚwÚposÚpad_maskÚn_realÚpos_mxÚpad_mxÚembÚn_padÚpad_embÚ
valid_maskÚ	attn_maskÚ	mask_fillr  ÚpooledÚ	pool_maskÚn_valids(   &&                                      r   r1   ÚVisionModel.__call__¹  sš  € ä�l¤D×)Ò)ØˆHÛ#�Ü! #¤r§x¡x×0Ò0ÜŸ(š( 3›-�CØ—8‘8˜q”=Ø˜d�)�CÙ˜c›�Ø—‘  q¥	Ö*ñ $ô —>’> (°Ô3°DÕ9Ð9ä˜,¬¯©×1Ò1ÜŸ8š8 LÓ1ˆLà!×'Ñ'‰
ˆˆaØˆçˆ=ØŸ_™_Õ,°·o±oÕ1EÕFˆHÜ˜8×%5Ñ%5Ó6ˆHØ)-×)EÑ)EÀaÓ)KÑ&ˆI Qä Ÿhšh¤r§w¢w¨y¸­ÀÀAÀqÀ	Ó'JÓKˆOÜ "§¢¬¯ª°¸dÕ1CÀaÈÀVÓ)LÓ MÐà ×/Ñ/ØØ  9 H 9 Õ-Ø! ! Y h Y ,Õ/óˆMð ×*Ñ*¨XÕ5ˆKØ˜QŒÜŸXšXØ˜ ]×%8Ñ%8¸Õ%<Ð=À]×EXÑEXô�
ô !#§¢°¸zÐ/JÐQRÔ S�ùð ˆJØˆMØˆKÜ˜1—X�Ø" 1 q¨1¥uÐ-�Ø ŸY™Y‘
��1�a˜Ø(,×(DÑ(DÀQÈÓ(JÑ%��X˜vÜŸš # d¥)Ó,�ÜŸš (¨4¥.Ó1�Ü˜V T×%5Ñ%5Ó6�à×)Ñ)¨#¨v°a¸¸&¸°jÕ/AÀ6È!ÈWÈfÈWÈ*ÕCUÓV�Ø×(Ñ(¨6Õ1�Ø˜1”9Ü Ÿhšh¨¨5°#·)±)¸Bµ-Ð'@ÈÏ	É	ÔR�GÜŸ.š.¨#¨w¨¸aÔ@�CØ×!Ñ! #Ô&Ø×$Ñ$ VÔ,Ø×"Ñ" 6×*ñ ô" ŸNšN¨:¸AÔ>ˆMÜ Ÿnšn¨]ÀÔCˆOÜ "§¢¨{ÀÔ CÐð
 (Ð'ˆ
Ü—N’N :¨qÓ1´B·N²NÀ:ÈqÓ4QÕQˆ	Ü—H’H˜T¨×)<Ñ)<Ô=ˆ	Ü—H’HØ”r—x’x ¨=×+>Ñ+>Ô?Àó
ˆ	ô —N’N 9¨aÓ0ˆ	àŸ™ ]°OÀYÓOˆà ŸK™KØ˜?ó
Ñˆ�	ð �?‰?˜1Õ ×!;Ñ!;Ô;Ø"‰Jà#˜ˆJàˆÜ�q–ˆAÜ˜* Q�-×.Ñ.¬r¯x©xÓ8×<Ñ<Ó>×CÑCÓEÓFˆGØ�O‰O˜F 1 h w h ;Õ/Ö0ñ ô Ÿš x°aÔ8¸Õ>ˆà�;‰;×"×"Ð"Ø*¨T¯]©]Õ:¸d¿n¹nÕLˆMàÐr   c                óD   € / pV P                  4        F	  w  r#W1V&   K  	  V# r.   )Úitems)r(  Ú	sanitizedrÃ   rÄ   s   &   r   ÚsanitizeÚVisionModel.sanitize  s&   € àˆ	Ø—M‘M–O‰DˆAØ�a‹Lñ $àÐr   )rž   r  rE  rC  rA  rD  rñ   rF  rB  rI  rJ  )r4   r5   r6   r7   r8   r   r_  r1   Ústaticmethodr…  r9   r:   r;   r<   s   @@r   r=  r=  ˆ  s@   ù‡ € ñ÷<ó <ò"K÷0að aðF ñó ÷ð r   r=  )g      Y@)Útypingr   Úmlx.coreÚcorer   Úmlx.nnr   ÚnumpyrP  rž   r   ÚModuler   rF   rH   re   ro   r   r™   r›   rÉ   r×   rì   r  r0  r=  © r   r   Ú<module>r�     sá   ðÝ å Ý Û å  ô�b—i‘iô õDVô
&�B—I‘Iô &ô&D˜2Ÿ9™9ô Dô 	:ˆb�i‰iô 	:ò.ô71ôtF(�b—i‘iô F(ôRS�—	‘	ô Sô$˜RŸY™Yô ô<23˜"Ÿ)™)ô 23ôj%#�2—9‘9ô %#ôP˜RŸY™Yô ô"Y�"—)‘)ö Yr   