+
    UV-jE  ã                   óÔ  € ^ RI HtHt ^ RIHt ^ RIHt ^RIH	t	 ^RI
Ht R t ! R R]P                  4      t ! R R	]P                  4      tR
 R ltR tR tR R lt ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t ! R R4      t ! R R]P                  4      tR# )é    )ÚOptionalÚTupleN)Úpixel_shuffle©ÚVisionConfigc                 ót   € V P                   p\        V4      ^8w  d   R# Vw  r#rEW#8¼  d   W$8¼  d	   W48X  d   R# R# )é   FT)ÚshapeÚlen)Úarrr
   Úout_channelsÚkHÚKWÚ_s   &     Úm/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx_vlm/models/llama4/vision.pyÚcheck_array_shaper   
   s<   € Ø�I‰I€Eô ˆ5ƒz�Q„Ùà#Ñ€L�bð 	Ô Ô!3¸"¼(Ùáó    c                   ó8   a a€ ] tR t^t oV 3R ltR tRtVtV ;t# )ÚLlama4MultiModalProjectorc                ó°   <€ \         SV `  4        \        P                  ! VP                  P
                  VP                  P                  R R7      V n        R# )F©ÚbiasN)	ÚsuperÚ__init__ÚnnÚLinearÚvision_configÚvision_output_dimÚtext_configÚhidden_sizeÚlinear_1©ÚselfÚconfigÚ	__class__s   &&€r   r   Ú"Llama4MultiModalProjector.__init__   s?   ø€ Ü‰ÑÔÜŸ	š	Ø× Ñ ×2Ñ2Ø×Ñ×*Ñ*Øô
ˆŽr   c                ó(   € V P                  V4      pV# ©N©r!   )r#   Úimage_featuresÚhidden_statess   && r   Ú__call__Ú"Llama4MultiModalProjector.__call__#   s   € ØŸ™ nÓ5ˆØÐr   r)   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r,   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r%   Ú__classdict__s   @@r   r   r      s   ù‡ € õ
÷ò r   r   c                   óD   a a€ ] tR t^(t oV 3R ltV3R lR ltRtVtV ;t# )ÚLlama4VisionPixelShuffleMLPc                óò   <€ \         SV `  4        VP                  V n        \        VP                  V P                  ^,          ,          4      V n        VP                  V n        \        VRRR7      V n	        R# )é   FT)r   Úis_projectorN)
r   r   Úpixel_shuffle_ratioÚintÚprojector_input_dimÚ	inner_dimÚprojector_output_dimÚ
output_dimÚLlama4VisionMLPÚmlpr"   s   &&€r   r   Ú$Llama4VisionPixelShuffleMLP.__init__)   s`   ø€ Ü‰ÑÔØ#)×#=Ñ#=ˆÔ ÜØ×&Ñ&¨4×+CÑ+CÀQÕ+FÕGó
ˆŒð !×5Ñ5ˆŒÜ" 6°ÀDÔIˆŽr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r;   Úencoded_patchesÚreturn©ÚmxÚarray)Úformatr7   s   "€r   Ú__annotate__Ú(Llama4VisionPixelShuffleMLP.__annotate__2   s#   ø€ ÷ )ñ )©¯©ð )±R·X±Xñ )r   c                óN   € \        WP                  4      pV P                  V4      # r(   )r   r=   rD   )r#   rG   s   &&r   r,   Ú$Llama4VisionPixelShuffleMLP.__call__2   s!   € Ü'¨×9QÑ9QÓRˆØ�x‰x˜Ó(Ð(r   )r@   rD   rB   r=   r.   r6   s   @@r   r9   r9   (   s   ù‡ € õJ÷)÷ )ð )r   r9   c                óX   € V ^8„  d   QhR\         P                  R\         P                  /# )r;   Úfreqs_ciÚqueryrI   )rL   s   "r   rM   rM   8   s"   € ÷ $ñ $¤B§H¡Hð $´R·X±Xñ $r   c                 óÂ   € VP                   p\        VP                  4       UUu. uF  w  r4V^8X  g   W2^,
          8X  d   TM^NK  	  pppV P                  ! V!  # u uppi ©é   )ÚndimÚ	enumerater
   Úreshape)rR   rS   rW   ÚiÚdr
   s   &&    r   Úreshape_for_broadcastr\   8   sV   € Ø�:‰:€DÜ=FÀuÇ{Á{Ô=SÔTÑ=S±T°Q�!�q”&˜A¨¥œM‰Q¨qÒ0Ñ=S€EÑTØ×Ò˜UÑ#Ð#ùó Us   ¥#Ac                óª   € V P                   R,          ^8X  g   Q RV P                   R,           24       hV R,          V R,          r!VRV,          ,           # )zÉ
Convert a tensor with shape (..., 2) to a complex tensor with shape (...).

Args:
    x: A real tensor with last dimension of size 2.

Returns:
    A complex tensor with size one less than the input.
zLast dimension must be 2, got y              ð?éÿÿÿÿ).r   ).rV   )r
   ©ÚxÚrealÚimags   &  r   Úview_as_complexrc   >   sQ   € ð �7‰7�2�;˜!ÔÐKÐ=¸a¿g¹gÀb½k¸]ÐKÓKÐð �6•˜A˜f�Iˆ$ð �"�t•)ÕÐr   c                óŒ   € \         P                  ! V 4      p\         P                  ! V 4      p\         P                  ! W.RR7      # )z°
Convert a complex tensor with shape (...) to a real tensor with shape (..., 2).

Args:
    x: A complex tensor.

Returns:
    A real tensor with an extra dimension of size 2.
©Úaxisr^   )rJ   ra   rb   Ústackr_   s   &  r   Úview_as_realrh   R   s3   € ô �7Š7�1‹:€DÜ�7Š7�1‹:€Dô �8Š8�T�L rÔ*Ð*r   c          
      óÎ   € V ^8„  d   QhR\         P                  R\         P                  R\         P                  R\        \         P                  \         P                  3,          /# )r;   rS   ÚkeyrR   rH   )rJ   rK   r   )rL   s   "r   rM   rM   d   sT   € ÷ Dñ DÜ�8‰8ðDä	�‰ðDô �h‰hðDô Œ2�8‰8”R—X‘XÐÕñ	Dr   c                 ó.  € \        V P                  \        P                  4      P                  ! . V P
                  R R ORN^N5!  4      p\        VP                  \        P                  4      P                  ! . VP
                  R R ORN^N5!  4      p\        W#R7      p\        W2,          4      P                  ^4      p\        WB,          4      P                  ^4      pVP                  V P                  4      VP                  VP                  4      3# )N)rR   rS   r^   )
rc   ÚastyperJ   Úfloat32rY   r
   r\   rh   ÚflattenÚdtype)rS   rj   rR   Úquery_Úkey_Ú	query_outÚkey_outs   &&&    r   Úvision_apply_rotary_embrt   d   sÙ   € ô ˜UŸ\™\¬"¯*©*Ó5×=Ò=ÐW¸u¿{¹{È3ÈBÐ?OÐWÐQSÐWÐUVÓWÓX€FÜ˜3Ÿ:™:¤b§j¡jÓ1×9Ò9ÐQ¸3¿9¹9ÀSÀb¸>ÐQÈ2ÐQÈqÓQÓR€DÜ$¨hÔE€HÜ˜VÕ.Ó/×7Ñ7¸Ó:€IÜ˜4�?Ó+×3Ñ3°AÓ6€GØ×Ñ˜EŸK™KÓ(¨'¯.©.¸¿¹Ó*CÐCÐCr   c                   óT   a a€ ] tR t^rt oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚLlama4VisionAttentionc                ó    <€ V ^8„  d   QhRS[ /# ©r;   r$   r   )rL   r7   s   "€r   rM   Ú"Llama4VisionAttention.__annotate__s   s   ø€ ÷ 
ñ 
™|ñ 
r   c                ó  <€ \         SV `  4        Wn        VP                  V n        VP
                  V n        VP                  VP
                  ,          V n        ^V n        V P                  R,          V n	        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  ,          V P                  RR7      V n        R# )rV   Tr   Nç      à¿)r   r   r$   r    Ú	embed_dimÚnum_attention_headsÚ	num_headsÚhead_dimÚnum_key_value_groupsÚscaler   r   Úq_projÚk_projÚv_projÚo_projr"   s   &&€r   r   ÚLlama4VisionAttention.__init__s   s  ø€ Ü‰ÑÔØŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØ×*Ñ*¨f×.HÑ.HÕHˆŒØ$%ˆÔ!Ø—]‘] DÕ(ˆŒ
ä—i’iØ�N‰N˜DŸN™N¨T¯]©]Õ:Àô
ˆŒô —i’iØ�N‰N˜DŸN™N¨T¯]©]Õ:Àô
ˆŒô —i’iØ�N‰N˜DŸN™N¨T¯]©]Õ:Àô
ˆŒô —i’iØ�N‰N˜TŸ]™]Õ*¨D¯N©NÀô
ˆŽr   c          	      ó¢   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          RS[S[ P                  ,          /# )r;   r+   rR   ÚmaskÚcache©rJ   rK   r   )rL   r7   s   "€r   rM   ry   ‰   sM   ø€ ÷ ñ á—x‘xðñ —(‘(ðñ ‘r—x‘xÕ ð	ñ
 ™Ÿ™Õ!ñr   c                ó‚  € VP                   w  rVpV P                  V4      P                  WVV P                  R4      pV P	                  V4      P                  WVV P                  R4      p	V P                  V4      P                  WVV P                  R4      p
\        W‰VR7      w  r‰VP                  ^ ^^^4      pV	P                  ^ ^^^4      p	V
P                  ^ ^^^4      p
\        P                  P                  W‰W P                  R7      pVP                  ^ ^^^4      P                  WVR4      pV P                  V4      pV# )rV   ©rR   )r�   r^   )r
   r‚   rY   r~   rƒ   r„   rt   Ú	transposerJ   ÚfastÚscaled_dot_product_attentionr�   r…   )r#   r+   rR   rˆ   r‰   ÚBÚLÚDÚquery_statesÚ
key_statesÚvalue_statesÚattn_outputs   &&&&&       r   r,   ÚLlama4VisionAttention.__call__‰   s)  € ð  ×%Ñ%‰ˆˆaà—{‘{ =Ó1×9Ñ9¸!ÀÇÁÐPRÓSˆØ—[‘[ Ó/×7Ñ7¸¸d¿n¹nÈbÓQˆ
Ø—{‘{ =Ó1×9Ñ9¸!ÀÇÁÐPRÓSˆä#:Ø¨xô$
Ñ ˆð $×-Ñ-¨a°°A°qÓ9ˆØ×)Ñ)¨!¨Q°°1Ó5ˆ
Ø#×-Ñ-¨a°°A°qÓ9ˆä—g‘g×:Ñ:Ø l¿*¹*ð ;ó 
ˆð "×+Ñ+¨A¨q°!°QÓ7×?Ñ?ÀÀbÓIˆØ—k‘k +Ó.ˆØÐr   )
r$   r|   r   rƒ   r~   r€   r…   r‚   r�   r„   )NNr.   r6   s   @@r   rv   rv   r   s   ù‡ € ÷
ó 
÷,÷ ò r   rv   c                   óH   a a€ ] tR t^§t oRV 3R lltV3R lR ltRtVtV ;t# )rC   c                ó8  <€ \         SV `  4        Wn        \        P                  ! R R7      V n        W0n        VP                  V n        VP                  V n        V'       d   V P                  MV P                  pV'       d   VP                  MV P                  p\        P                  ! WEVR7      V n        V'       d   VP                  MV P                  pV'       d   VP                  MV P                  p\        P                  ! WgVR7      V n        W0n        R# )rŽ   )Úapproxr   N)r   r   r$   r   ÚGELUÚactivation_fnr<   r    Úintermediate_sizer?   r   Úfc1rA   Úfc2)	r#   r$   r   r<   Úfc1_input_dimÚfc1_output_dimÚfc2_input_dimÚfc2_output_dimr%   s	   &&&&    €r   r   ÚLlama4VisionMLP.__init__¨   s×   ø€ Ü‰ÑÔØŒÜŸWšW¨FÔ3ˆÔØ(ÔØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔ÷ 3?˜×.Ò.ÀD×DTÑDTˆç*6ˆF×&Ò&¸D×<RÑ<Rð 	ô —9’9˜]ÀÔFˆŒ÷ ,8ˆF×'Ò'¸T×=SÑ=Sð 	÷ ,8ˆF×'Ò'¸T×=MÑ=Mð 	ô —9’9˜]ÀÔFˆŒà(Ör   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# ©r;   r+   rH   rI   )rL   r7   s   "€r   rM   ÚLlama4VisionMLP.__annotate__Ä   s#   ø€ ÷ 'ñ '¡b§h¡hð '±2·8±8ñ 'r   c                óÌ   € V P                  V4      pV P                  V4      pV P                  '       d!   V P                  V P                  V4      4      # V P                  V4      # r(   )rž   rœ   r<   rŸ   ©r#   r+   s   &&r   r,   ÚLlama4VisionMLP.__call__Ä   sU   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆà××ÐØ×%Ñ% d§h¡h¨}Ó&=Ó>Ð>à�x‰x˜Ó&Ð&r   )rœ   r$   rž   rŸ   r    r�   r<   )TFr.   r6   s   @@r   rC   rC   §   s   ù‡ € ÷)÷8'÷ 'ð 'r   rC   c                   óT   a a€ ] tR t^Ît oV3R lV 3R lltRV3R lR lltRtVtV ;t# )ÚLlama4VisionEncoderLayerc                ó    <€ V ^8„  d   QhRS[ /# rx   r   )rL   r7   s   "€r   rM   Ú%Llama4VisionEncoderLayer.__annotate__Ï   s   ø€ ÷ Iñ I™|ñ Ir   c                ó  <€ \         SV `  4        VP                  V n        \        V4      V n        \        V4      V n        \        P                  ! VP                  4      V n	        \        P                  ! VP                  4      V n
        R # r(   )r   r   r    rv   Ú	self_attnrC   rD   r   Ú	LayerNormÚinput_layernormÚpost_attention_layernormr"   s   &&€r   r   Ú!Llama4VisionEncoderLayer.__init__Ï   sb   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä.¨vÓ6ˆŒÜ" 6Ó*ˆŒä!Ÿ|š|¨F×,>Ñ,>Ó?ˆÔÜ(*¯ª°V×5GÑ5GÓ(HˆÖ%r   c                óx   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          /# )r;   Úhidden_staterR   rˆ   rŠ   )rL   r7   s   "€r   rM   r®   Ù   s:   ø€ ÷ ñ á—h‘hðñ —(‘(ðñ ‘r—x‘xÕ ñ	r   c                ó¼   € TpV P                  V4      pV P                  VVVR 7      pWA,           pTpV P                  V4      pV P                  V4      pWA,           pV# ))rR   rˆ   )r²   r°   r³   rD   )r#   r¶   rR   rˆ   Úresiduals   &&&& r   r,   Ú!Llama4VisionEncoderLayer.__call__Ù   sr   € ð  ˆà×+Ñ+¨LÓ9ˆà—~‘~ØØØð &ó 
ˆð
  Õ.ˆð  ˆØ×4Ñ4°\ÓBˆØ—x‘x Ó-ˆØÕ.ˆØÐr   )r    r²   rD   r³   r°   r(   r.   r6   s   @@r   r¬   r¬   Î   s    ù‡ € ÷Ió I÷÷ ò r   r¬   c                   óX   a a€ ] tR t^ót oRtV3R lV 3R lltRV3R lR lltRtVtV ;t	# )ÚLlama4VisionEncoderz£
Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
[`Llama4VisionEncoderLayer`].

Args:
    config: VisionConfig
c                ó    <€ V ^8„  d   QhRS[ /# rx   r   )rL   r7   s   "€r   rM   Ú Llama4VisionEncoder.__annotate__ü   s   ø€ ÷ ñ ™|ñ r   c                ó¬   <€ \         SV `  4        Wn        \        VP                  4       Uu. uF  p\        V4      NK  	  upV n        Wn        R # u upi r(   )r   r   r$   ÚrangeÚnum_hidden_layersr¬   Úlayers)r#   r$   r   r%   s   && €r   r   ÚLlama4VisionEncoder.__init__ü   sL   ø€ Ü‰ÑÔØŒä6;¸F×<TÑ<TÔ6Uó
Ù6U°Ô$ VÖ,Ñ6Uñ
ˆŒð Žùò
s   ­Ac                óx   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[S[ P                  ,          /# )r;   r+   rR   rˆ   rŠ   )rL   r7   s   "€r   rM   r½     s:   ø€ ÷ ñ á—x‘xðñ —(‘(ðñ ‘r—x‘xÕ ñ	r   c                óX   € \        V P                  4       F  w  rEV! VVVR 7      pK  	  V# ))r¶   rˆ   rR   )rX   rÁ   )r#   r+   rR   rˆ   rZ   Úencoder_layers   &&&&  r   r,   ÚLlama4VisionEncoder.__call__  s5   € ô !*¨$¯+©+Ö 6ÑˆAÙ)Ø*ØØ!ôŠMñ !7ð Ðr   )r$   rÁ   r(   )
r/   r0   r1   r2   Ú__doc__r   r,   r3   r4   r5   r6   s   @@r   r»   r»   ó   s#   ù‡ € ñ÷ó ÷÷ ò r   r»   c                   óP   a a€ ] tR tRt oV 3R ltR tR tV3R lR ltRtVt	V ;t
# )ÚLlama4UnfoldConvolutioni  c                ó8  <€ \         SV `  4        VP                  p\        V\        4      '       d   W"3pW n        VP                  V n        \        P                  ! VP                  V^ ,          ,          V^,          ,          VP                  RR7      V n        R# )r   Fr   N)r   r   Ú
patch_sizeÚ
isinstancer>   Úkernel_sizeÚstrider   r   Únum_channelsr    Úlinear)r#   r$   rÍ   r%   s   && €r   r   Ú Llama4UnfoldConvolution.__init__  sx   ø€ Ü‰ÑÔØ×'Ñ'ˆÜ�k¤3×'Ò'Ø&Ð4ˆKØ&ÔØ×'Ñ'ˆŒÜ—i’iØ×Ñ +¨a¥.Õ0°;¸qµ>ÕAØ×ÑØô
ˆŽr   c                óV   € \        V\        \        34      '       d   \        V4      # W3# )z"Convert input to a pair of values.)rÌ   ÚlistÚtuple)r#   r`   s   &&r   Ú_pairÚLlama4UnfoldConvolution._pair#  s#   € ä�aœ$¤˜×'Ò'Ü˜“8ˆOØˆvˆr   c                ó  € V P                  V P                  4      pV P                  V P                  4      pRpRpVP                  w  rgr‰V^V^ ,          ,          ,           V^ ,          V^ ,          ^,
          ,          ,
          ^,
          V^ ,          ,          ^,           p
V	^V^,          ,          ,           V^,          V^,          ^,
          ,          ,
          ^,
          V^,          ,          ^,           p. p\	        ^ W‚^ ,          V^ ,          ,          ,
          ^,           V^ ,          4       EF  p\	        ^ W’^,          V^,          ,          ,
          ^,           V^,          4       FÃ  p. p\	        V^ ,          4       Ff  p\	        V^,          4       FM  pVVV^ ,          ,          ,           pVVV^,          ,          ,           pVP                  VRRVV3,          4       KO  	  Kh  	  \        P                  ! V^R7      p\        P                  ! V. RO4      pVP                  V4       KÅ  	  EK  	  \        P                  ! VRR7      p\        P                  ! VVWr^ ,          ,          V^,          ,          W«,          34      pV# )aG  
Extract sliding local blocks from a batched input tensor (MLX implementation).

This is equivalent to PyTorch's nn.functional.unfold or im2col operation.

Args:
    input_tensor: Input tensor of shape (B, C, H, W)

Returns:
    Unfolded tensor of shape (B, C*kernel_height*kernel_width, L)
    where L is the number of blocks
ºNNNre   )r   r   )rV   rV   )r   r;   rV   r^   )
rÕ   rÍ   rÎ   r
   r¿   ÚappendrJ   rg   r�   rY   )r#   Úinput_tensorrÍ   rÎ   ÚpaddingÚdilationÚ
batch_sizeÚchannelsÚheightÚwidthÚ
height_outÚ	width_outÚblocksrZ   ÚjÚblockÚdiÚdjÚh_idxÚw_idxÚresults   &&                   r   ÚunfoldÚLlama4UnfoldConvolution.unfold)  s  € ð —j‘j ×!1Ñ!1Ó2ˆØ—‘˜DŸK™KÓ(ˆØˆØˆð /;×.@Ñ.@Ñ+ˆ
˜fð �Q˜ �•^Õ# h¨q¥k°[Àµ^ÀaÕ5GÕ&HÕHÈ1ÕLØ�A�Yõàõˆ
ð �A˜ �
•NÕ" X¨a¥[°KÀµNÀQÕ4FÕ%GÕGÈ!ÕKØ�A�Yõàõˆ	ð
 ˆô �q˜&¨q¥>°H¸QµKÕ#?Õ?À!ÕCÀVÈAÅY×OˆAÜ˜1˜e°!¥n°xÀµ{Õ&BÕBÀQÕFÈÈqÍ	ÖR�à�Ü ¨A¥Ö/�BÜ# K°¥NÖ3˜Ø ! B¨°!­Õ$4Õ 4˜Ø ! B¨°!­Õ$4Õ 4˜àŸ™ \°!°Q¸¸uÐ2DÕ%EÖFó	 4ñ 0ô Ÿš ¨QÔ/�ÜŸš UªIÓ6�Ø—‘˜eÖ$ô Sñ Pô" —’˜& rÔ*ˆô —’ØàØ q�>Õ)¨K¸­NÕ:ØÕ&ðó
ˆð ˆr   c                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# r¦   rI   )rL   r7   s   "€r   rM   Ú$Llama4UnfoldConvolution.__annotate__j  s#   ø€ ÷ ñ ¡b§h¡hð ±2·8±8ñ r   c                ón   € V P                  V4      pVP                  ^^4      pV P                  V4      pV# rU   )rë   ÚswapaxesrÐ   r©   s   &&r   r,   Ú Llama4UnfoldConvolution.__call__j  s6   € ØŸ™ MÓ2ˆØ%×.Ñ.¨q°!Ó4ˆØŸ™ MÓ2ˆØÐr   )rÍ   rÐ   rÎ   )r/   r0   r1   r2   r   rÕ   rë   r,   r3   r4   r5   r6   s   @@r   rÉ   rÉ     s$   ù‡ € õ
òò?÷B÷ ð r   rÉ   c                   ó8   a a€ ] tR tRt oV 3R ltR tRtVtV ;t# )ÚLlama4VisionRotaryEmbeddingiq  c           	     ó„  <€ \         SV `  4        VP                  VP                  ,          p\        P
                  ! V^,          \        P                  R7      P                  V^,          ^4      p\        P                  ! W3R,          .^ R7      pRVR	&   W2,          pW2,          pVP                  VP                  ,          ^,          pRVP                  \        P
                  ! ^ V^\        P                  R7      RV^,           V,          ,          ,          pV^,           R
,          VR,          ,          pV^,           R
,          VR,          ,          p	RR lp
V
! V^4      pV
! V	^4      p\        P                  ! W¼.RR7      P                  \        P                  4      R,          pVP                  R^^4      ^ 8  p\        P                  ! V\        P                  ! V4      V4      p\        P                   ! \        P"                  ! V4      \        P$                  ! V4      .RR7      p\'        V4      pWðn        R# )r;   )ro   :NrV   Nre   g      ð?Nc                 ó  € \        V P                  4      p\        P                  ! WR R VR,          ^.,           4      p \        P                  ! WRR7      p \        P                  ! WR R VR,          V,          .,           4      # )Nre   r^   )rÓ   r
   rJ   rY   Úrepeat)ÚtensorÚrepeatsÚdimr
   s   &&& r   Úrepeat_interleaveÚ?Llama4VisionRotaryEmbedding.__init__.<locals>.repeat_interleave‡  sm   € ä˜Ÿ™Ó&ˆEô —Z’Z ¨c¨r¨
°e¸BµiÀ°^Õ(CÓDˆFô —Y’Y˜v°RÔ8ˆFô —:’:˜f¨C¨R j°E¸"µIÀÕ4GÐ3HÕ&HÓIÐIr   éþÿÿÿr^   )r^   r^   ).N)NNrØ   )r^   ).:NNr;   )r   r   Ú
image_sizerË   rJ   ÚarangeÚint32rY   Úconcatenater    r}   Ú
rope_thetarm   rl   ÚwhereÚ
zeros_likerg   ÚcosÚsinrc   rR   )r#   r$   ÚidxÚimg_idxÚfrequencies_xÚfrequencies_yÚfreq_dimÚ	rope_freqÚfreqs_x_expandedÚfreqs_y_expandedrú   Úfreqs_xÚfreqs_yÚfreqsrˆ   Úfreq_cisr%   s   &&              €r   r   Ú$Llama4VisionRotaryEmbedding.__init__r  s½  ø€ Ü‰ÑÔØ×Ñ 6×#4Ñ#4Õ4ˆÜ—)’)˜C �F¬"¯(©(Ô3×;Ñ;¸CÀ½FÀAÓFˆÜ—.’. '°2­;Ð!7¸aÔ@ˆØˆ�‰Ø�ˆØ�ˆØ×%Ñ%¨×)CÑ)CÕCÀqÕHˆØØ×Ñä—	’	˜!˜X q´·
±
Ô;Ð<M¸xÈ1½}ÐNØõõõ
ˆ	ð *¨AÕ-¨yÕ9¸IÀmÕ<TÕTÐØ)¨AÕ-¨yÕ9¸IÀmÕ<TÕTÐô	Jñ $Ð$4°aÓ8ˆÙ#Ð$4°aÓ8ˆÜ—’ Ð1¸Ô;×BÑBÄ2Ç:Á:ÓNÈxÕXˆà�‰˜r 1 aÓ(¨1Ñ,ˆÜ—’˜œrŸ}š}¨UÓ3°UÓ;ˆÜ—8’8œRŸVšV E›]¬B¯FªF°5«MÐ:ÀÔDˆÜ" 8Ó,ˆØ Žr   c                ó   € V P                   # r(   rŒ   r©   s   &&r   r,   Ú$Llama4VisionRotaryEmbedding.__call__Ÿ  s   € Ø�}‰}Ðr   rŒ   r.   r6   s   @@r   ró   ró   q  s   ù‡ € õ+!÷Zò r   ró   c                   ó`   a a€ ] tR tRt oV3R lV 3R lltR tR	V3R lR lltR tRtVt	V ;t
# )
ÚVisionModeli£  c                ó    <€ V ^8„  d   QhRS[ /# rx   r   )rL   r7   s   "€r   rM   ÚVisionModel.__annotate__¤  s   ø€ ÷ Bñ B™|ñ Br   c                óÜ  <€ \         SV `  4        VP                  V n        VP                  V n        VP                  V n        VP
                  V n        VP                  V n        V P                  R9  d   \        RV P                   R24      hV P                  V P                  ,          ^,          ^,           V n        VP                  R,          V n	        V P                  \        P                  P                  V P                  34      ,          V n        V P                  \        P                  P                  V P                  V P                  34      ,          V n        \        V4      V n        \#        V4      V n        \&        P(                  ! V P                  4      V n        \&        P(                  ! V P                  4      V n        \/        V4      V n        \3        V4      V n        R# )Úllama4zModel type z not supportedN)r  Úllama4_vision_modelr{   )r   r   rý   rË   r    rÏ   Ú
model_typeÚ
ValueErrorÚnum_patchesr�   rJ   ÚrandomÚnormalÚclass_embeddingÚpositional_embedding_vlmrÉ   Úpatch_embeddingró   Úrotary_embeddingr   r±   Úlayernorm_preÚlayernorm_postr»   Úmodelr9   Úvision_adapterr"   s   &&€r   r   ÚVisionModel.__init__¤  sn  ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØ ×+Ñ+ˆŒØ!×-Ñ-ˆÔØ"×/Ñ/ˆÔØ ×+Ñ+ˆŒØ�?‰?Ð"CÔCÜ˜{¨4¯?©?Ð*;¸>ÐJÓKÐKà ŸO™O¨t¯©Õ>À1ÕDÀqÕHˆÔØ×'Ñ'¨Õ-ˆŒ
à#Ÿz™z¬B¯I©I×,<Ñ,<¸d×>NÑ>NÐ=PÓ,QÕQˆÔØ(,¯
©
´R·Y±Y×5EÑ5EØ×Ñ˜t×/Ñ/Ð0ó6
õ )
ˆÔ%ô  7°vÓ>ˆÔä ;¸FÓ CˆÔô  Ÿ\š\¨$×*:Ñ*:Ó;ˆÔÜ Ÿlšl¨4×+;Ñ+;Ó<ˆÔô )¨Ó0ˆŒ
Ü9¸&ÓAˆÖr   c                ó   € V P                   # )zW
This function is used to fetch the first embedding layer to activate grads on inputs.
)r#  )r#   s   &r   Úget_input_embeddingsÚ VisionModel.get_input_embeddingsÂ  s   € ð ×#Ñ#Ð#r   c          	      óv   <€ V ^8„  d   QhRS[ P                  RS[S[,          RS[S[,          RS[S[,          /# )r;   Úpixel_valuesÚoutput_attentionsÚoutput_hidden_statesÚcapture_activations)rJ   rK   r   Úbool)rL   r7   s   "€r   rM   r  È  sC   ø€ ÷ ;"ñ ;"á—h‘hð;"ñ $¡D�>ð;"ñ '¡t�nð	;"ñ
 &¡d�^ñ;"r   c                óÌ  € VP                   w  rVrx^p	^p
V P                  V4      pVP                   w  rÍpVP                  WY,          V
,          VV4      p\        P                  ! V P
                  VP                   ^ ,          ^VP                   R,          34      p\        P                  ! W¿.^R7      pV^,          pVP                  WY,          V
VV4      pV P                  pVV,           pV P                  V4      pVP                  VRV4      pV P                  V4      pV P                  VRVR7      pV P                  V4      pVRRR1R3,          pV P                  V4      pV# )rV   re   N)rˆ   rR   rØ   r^   )r
   r#  rY   rJ   Úbroadcast_tor!  r   r"  r%  r$  r'  r&  r(  )r#   r.  r/  r0  r1  Úbatch_size_times_num_tilesrÏ   rß   rà   Únum_concurrent_mediaÚ
num_chunksr¶   r   r  Ú
hidden_dimr!  Úpositional_embeddingrR   Úfinal_hidden_states   &&&&&              r   r,   ÚVisionModel.__call__È  s‰  € ð CO×BTÑBTÑ?Ð"°&Ø ÐØˆ
à×+Ñ+¨LÓ9ˆà%1×%7Ñ%7Ñ"ˆ˜
ð $×+Ñ+Ø&Õ=À
ÕJØØó
ˆô Ÿ/š/Ø× Ñ  <×#5Ñ#5°aÕ#8¸!¸\×=OÑ=OÐPRÕ=SÐ"Tó
ˆô —~’~ |Ð&EÈAÔNˆØ�qÕˆð $×+Ñ+Ø&Õ=ØØØó	
ˆð  $×<Ñ<ÐØ#Ð&:Õ:ˆà×)Ñ)¨,Ó7ˆà#×+Ñ+Ð,FÈÈJÓWˆØ×(Ñ(¨Ó6ˆà—z‘zØØØð "ó 
ˆð ×*Ñ*¨<Ó8ˆà# A s¨ s¨A IÕ.ˆð "×0Ñ0°Ó>Ðð "Ð!r   c                óV   € / pVP                  4        F  w  r4R V9   d   K  WBV&   K  	  V# )Úposition_ids)Úitems)r#   ÚweightsÚsanitized_weightsÚkÚvs   &&   r   ÚsanitizeÚVisionModel.sanitize  s5   € ØÐØ—M‘M–O‰DˆAØ Ô"áà'( !Ó$ñ $ð !Ð r   )r!  r    rý   r&  r%  r'  r  rÏ   r  r#  rË   r"  r$  r�   r(  )NNT)r/   r0   r1   r2   r   r+  r,   rC  r3   r4   r5   r6   s   @@r   r  r  £  s+   ù‡ € ÷Bó Bò<$÷;"ò ;"÷z	!ò 	!r   r  )Útypingr   r   Úmlx.coreÚcorerJ   Úmlx.nnr   Úbaser   r$   r   r   ÚModuler   r9   r\   rc   rh   rt   rv   rC   r¬   r»   rÉ   ró   r  © r   r   Ú<module>rL     sÂ   ðß "å Ý å  Ý  òô  §	¡	ô ô) "§)¡)ô )õ $òò(+õ$Dô2˜BŸI™Iô 2ôj$'�b—i‘iô $'ôN"˜rŸy™yô "ôJ˜"Ÿ)™)ô ôDY˜bŸi™iô Y÷x/ñ /ôdk!�"—)‘)ö k!r   