+
    QV-j²$  ã                   ó¨   € R t ^ RIt^ RIHt ^RIHt  ! R R]P                  4      t ! R R]P                  4      t ! R R]P                  4      t	R# )	a  

Generic interface to various configurations of the Perceiver Resampler, that simply takes in a series of (potentially
time-indexed) contextual embeddings, and "resamples" (compresses) them down to a pre-specified number of latents! Note
that the Perceiver in general resamples based solely off the *long-range* context; there's a nice opportunity here to
prime the Perceiver Resampler with say a single layer's worth of language embeddings (the target domain), and use that
to softly "retrieve & compress" what we need --> this would be a novel contribution we should explore.

References:
    - DeepMind's Flamingo: https://www.deepmind.com/blog/tackling-multiple-tasks-with-a-single-visual-language-model
    - Code borrowed w/ love from: https://github.com/lucidrains/flamingo-pytorch

N©ÚIdeficsConfigc                   óP   a a€ ] tR t^.t oV3R lV 3R lltV3R lR ltRtVtV ;t# )ÚIdeficsPerceiverResamplerc                óB   <€ V ^8„  d   QhRS[ RS[RS[RS[RS[RS[RR/# )	é   ÚconfigÚ	embed_dimÚdepthÚn_headsÚhead_dimÚ	n_latentsÚreturnN)r   Úint)ÚformatÚ__classdict__s   "€Úv/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/transformers/models/idefics/perceiver.pyÚ__annotate__Ú&IdeficsPerceiverResampler.__annotate__/   sE   ø€ ÷ ,7ñ ,7Ù#ð,7Ù03ð,7Ù<?ð,7ÙJMð,7ÙY\ð,7Ùilð,7à	ñ,7ó    c                ó:  <€ \         SV `  4        W$WV3w  V n        V n        V n        V n        VP                  P                  V n        \        P                  ! \        P                  ! V P
                  V P                  4      RR7      V n        \        VP                  R4      '       g   V P                  ^,          MVP                  P                  ^,          V n        \        P"                  ! \%        V4       Uu. uFc  p\        P"                  ! \'        V P                  V P                  V P                  V P                  4      \)        V P                   V4      .4      NKe  	  up4      V n        \        P,                  ! V P                  4      V n        R# u upi )aÿ  
Instantiates a Perceiver Resampler that operates over a sequence of embeddings (say from a ResNet or ViT or
MAE) of a given dimension, performs `depth` blocks of cross-attention with a fixed `n_latents` inputs, then
returns a Tensor of shape [bsz, n_latents, embed_dim]. :param embed_dim: Dimensionality of embeddings being fed
to the Perceiver Resampler (also dimensionality of latent embeddings *returned* by the Perceiver Resampler.
Could be e.g., VIT embed_dim, ResNet pool dim, and so on.

Args:
    config (`IdeficsConfig`): config object
    embed_dim (`int`): The size of each embedding vector
    depth (`int`): Depth of the Perceiver Resampler (Transformer w/ cross attention). Should be shallow (< 3).
    n_heads (`int`): Number of heads in each Transformer block (for multi-headed self-attention).
    head_dim (`int`): Dimensionality of each head projection in the Transformer block.
    n_latents (`int`):
        Number of latent embeddings to resample ("compress") the input sequence to (usually < 128).

T)Úrequires_gradr	   N)ÚsuperÚ__init__r	   r   r   r   Úperceiver_configÚqk_layer_norms_perceiverÚqk_layer_normsÚnnÚ	ParameterÚtorchÚrandnÚlatentsÚhasattrÚvision_configÚintermediate_dimÚ
ModuleListÚrangeÚIdeficsPerceiverAttentionÚ
IdeficsMLPÚblocksÚ	LayerNormÚ
layer_norm)	Úselfr   r	   r
   r   r   r   Ú_Ú	__class__s	   &&&&&&& €r   r   Ú"IdeficsPerceiverResampler.__init__/   s;  ø€ ô( 	‰ÑÔØFOÐZbÐFmÑCˆŒ˜œ d¤m°T´^Ø$×5Ñ5×NÑNˆÔô —|’|¤E§K¢K°·±ÀÇÁÓ$OÐ_cÔdˆŒô ˜6×/Ñ/°×=Ò=ð �N‰N˜QÖà×%Ñ%×/Ñ/°!Õ3ð 	Ôô —m’mô ˜uœóñ &�Aô —’ä1°$·.±.À$Ç,Á,ÐPT×P]ÑP]Ð_c×_rÑ_rÓsÜ" 4×#8Ñ#8¸&ÓAðöñ &ñó

ˆŒô Ÿ,š, t§~¡~Ó6ˆŽùòs   Ã<A)Fc                óN   <€ V ^8„  d   QhRS[ P                  RS[ P                  /# )r   Úcontextr   ©r   ÚTensor)r   r   s   "€r   r   r   ]   s#   ø€ ÷ 
(ñ 
(™uŸ|™|ð 
(±·±ñ 
(r   c                óæ   € V P                   P                  VP                  ^ ,          ^^4      pV P                   F#  w  r4V! W4      V,           pV! V4      V,           pK%  	  V P	                  V4      # )zWResample arbitrary length context & *compress* down to self.n_latents latent embeddings)r!   ÚrepeatÚshaper)   r+   )r,   r1   r!   ÚattnÚffs   &&   r   ÚforwardÚ!IdeficsPerceiverResampler.forward]   se   € ð —,‘,×%Ñ% g§m¡m°AÕ&6¸¸1Ó=ˆð Ÿœ‰HˆDÙ˜7Ó,¨wÕ6ˆGÙ˜“k GÕ+ŠGñ $ð �‰˜wÓ'Ð'r   )	r)   r	   r   r$   r!   r+   r   r   r   ©	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r9   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r.   r   s   @@r   r   r   .   s   ù‡ € ÷,7ó ,7÷\
(÷ 
(ð 
(r   r   c                   óP   a a€ ] tR t^jt oV3R lV 3R lltV3R lR ltRtVtV ;t# )r'   c          
      ó6   <€ V ^8„  d   QhRS[ RS[ RS[ RS[RR/# )r   r	   r   r   r   r   N)r   Úbool)r   r   s   "€r   r   Ú&IdeficsPerceiverAttention.__annotate__k   s9   ø€ ÷ Zñ Z¡#ð Z±ð Z¹sð ZÑTXð ZÐ]añ Zr   c                óÚ  <€ \         SV `  4        WVuV n        V n        V n        W@n        \        P                  ! V P                  4      V n        \        P                  ! V P                  4      V n	        V P
                  '       dK   \        P                  ! V P                  4      V n
        \        P                  ! V P                  4      V n        V P                  R,          V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  V P                  ,          RR7      V n        \        P                  ! V P                  V P                  ,          VRR7      V n        R# )ziPerceiver Cross-Attention Module --> let long-form inputs be `context`, resampled embeddings be `latents`F©ÚbiasNg      à¿)r   r   r	   r   r   r   r   r*   Úcontext_layer_normÚlatents_layer_normÚq_layer_normÚk_layer_normÚqk_scaleÚLinearÚq_projÚk_projÚv_projÚoutput_proj)r,   r	   r   r   r   r.   s   &&&&&€r   r   Ú"IdeficsPerceiverAttention.__init__k   s/  ø€ ä‰ÑÔØ6?È(Ð3ˆŒ˜œ d¤mØ,Ôä"$§,¢,¨t¯~©~Ó">ˆÔÜ"$§,¢,¨t¯~©~Ó">ˆÔØ××ÐÜ "§¢¨T¯]©]Ó ;ˆDÔÜ "§¢¨T¯]©]Ó ;ˆDÔàŸ™ tÕ+ˆŒô —i’i §¡°·±¸t¿}¹}Õ0LÐSXÔYˆŒÜ—i’i §¡°·±¸t¿}¹}Õ0LÐSXÔYˆŒÜ—i’i §¡°·±¸t¿}¹}Õ0LÐSXÔYˆŒäŸ9š9 T§\¡\°D·M±MÕ%AÀ9ÐSXÔYˆÖr   c                óh   <€ V ^8„  d   QhRS[ P                  RS[ P                  RS[ P                  /# )r   r1   r!   r   r2   )r   r   s   "€r   r   rG   €   s3   ø€ ÷ (Gñ (G™uŸ|™|ð (G±e·l±lð (GÁuÇ|Á|ñ (Gr   c           	     ó®  € V P                  V4      pV P                  V4      pVP                  R,          w  r4pV P                  V4      pV P	                  \
        P                  ! W.RR7      4      pV P                  \
        P                  ! W.RR7      4      pWgV3 U	u. uFJ  q™P                  W9P                  ^,          V P                  V P                  4      P                  ^^4      NKL  	  up	w  rgpV P                  '       d#   V P                  V4      pV P                  V4      p\
        P                  ! RW`P                   ,          V4      p
WªP#                  RRR7      P%                  4       ,
          pVP'                  RR7      p\
        P                  ! RWÈ4      pV P)                  VP                  ^^4      P+                  R4      4      # u up	i )	aö  
Runs Perceiver Self-Attention, with special (context, latents) appended along the `seq` dimension!

Args:
    context (`torch.Tensor`):
        Tensor of shape `[bsz, seq, embed_dim]` representing long-form context to resample.
    latents (`torch.Tensor`):
        Tensor of shape `[bsz, n_latents, embed_dim]` representing fixed length latents to compress to.

Returns:
    `torch.Tensor`: Tensor of shape `[bsz, n_latents, embed_dim]` representing attention over latents w/ cross
    from context.
:Né   N)Údimz... i d, ... j d -> ... i jT)rY   Úkeepdimz... i j, ... j d -> ... i déþÿÿÿéÿÿÿÿ)rK   rL   r6   rQ   rR   r   ÚcatrS   Úreshaper   r   Ú	transposer   rM   rN   ÚeinsumrO   ÚamaxÚdetachÚsoftmaxrT   Úflatten)r,   r1   r!   Ú
batch_sizeÚ
seq_lengthr	   ÚqÚkÚvÚxÚscoresÚstabilized_scoresr7   Ú	resampleds   &&&           r   r9   Ú!IdeficsPerceiverAttention.forward€   s‰  € ð ×)Ñ)¨'Ó2ˆØ×)Ñ)¨'Ó2ˆØ,3¯M©M¸"Õ,=Ñ)ˆ
 	ð �K‰K˜Ó ˆØ�K‰KœŸ	š	 7Ð"4¸"Ô=Ó>ˆØ�K‰KœŸ	š	 7Ð"4¸"Ô=Ó>ˆð
 mnÐrsÑktÓuÑktÐfg—9‘9˜Z¯©°­°T·\±\À4Ç=Á=ÓQ×[Ñ[Ð\]Ð_`ÖaÑktÑu‰ˆˆaà××ÐØ×!Ñ! !Ó$ˆAØ×!Ñ! !Ó$ˆAä—’Ð;¸QÇÁÕ=NÐPQÓRˆØ"§k¡k°bÀ$ kÓ&G×&NÑ&NÓ&PÕQÐØ ×(Ñ(¨RÐ(Ó0ˆô —L’LÐ!>ÀÓHˆ	à×Ñ 	× 3Ñ 3°A°qÓ 9× AÑ AÀ"Ó EÓFÐFùò vs   Â AG)rK   r	   r   rN   rR   rL   r   rT   rM   rQ   r   rO   rS   r;   rC   s   @@r   r'   r'   j   s#   ù‡ € ÷Zó Z÷*(G÷ (Gð (Gr   r'   c                   óP   a a€ ] tR t^«t oV3R lV 3R lltV3R lR ltRtVtV ;t# )r(   c                ó    <€ V ^8„  d   QhRS[ /# )r   r   r   )r   r   s   "€r   r   ÚIdeficsMLP.__annotate__¬   s   ø€ ÷ Oñ O±-ñ Or   c                óv  <€ \         SV `  4        VP                  P                  V n        \        P
                  ! V P                  4      V n        \        P                  ! V P                  VRR7      V n        \        P                  ! 4       V n
        \        P                  ! WP                  RR7      V n        R# )z:Simple MLP block with intermediate_size and embedding sizeFrI   N)r   r   r#   r	   r   r*   ÚlnrP   ÚfcÚReLUÚactÚc_proj)r,   Úintermediate_sizer   r.   s   &&&€r   r   ÚIdeficsMLP.__init__¬   sr   ø€ ä‰ÑÔØ×-Ñ-×7Ñ7ˆŒÜ—,’,˜tŸ~™~Ó.ˆŒÜ—)’)˜DŸN™NÐ,=ÀEÔJˆŒÜ—7’7“9ˆŒÜ—i’iÐ 1·>±>ÈÔNˆŽr   c                ól   <€ V ^8„  d   QhRS[ S[P                  ,          R,          RS[P                  /# )r   Úhidden_statesNr   )Útupler   ÚFloatTensor)r   r   s   "€r   r   rq   µ   s1   ø€ ÷ ñ ¡U©5×+<Ñ+<Õ%=ÀÕ%Dð É×IZÑIZñ r   c                óŽ   € V P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV# )N)rs   rt   rv   rw   )r,   r{   s   &&r   r9   ÚIdeficsMLP.forwardµ   s@   € ØŸ™ Ó.ˆØŸ™ Ó.ˆØŸ™ Ó/ˆØŸ™ MÓ2ˆàÐr   )rv   rw   r	   rt   rs   r;   rC   s   @@r   r(   r(   «   s    ù‡ € ÷Oó O÷÷ ð r   r(   )
Ú__doc__r   Útorch.nnr   Úconfiguration_ideficsr   ÚModuler   r'   r(   © r   r   Ú<module>r…      sI   ðñ4ó Ý å 0ô9( §	¡	ô 9(ôx>G §	¡	ô >GôB�—‘ö r   