+
    NV-jé/  ã                   óú   € ^ RI t ^ RIHtHtHt ^ RIHt ^ RIH	t	 ^ RI
Ht ^ RIHt ^ RIHt ^ RIHt ^ RIHt  ! R	 R
]4      t ! R R]4      t ! R R]4      t ! R R]4      t ! R R]4      t ! R R]4      tR# )é    N)ÚAnyÚCallableÚOptional)Úrelu)ÚModule)ÚDropout)ÚLinear)Ú	LayerNorm)Ú
checkpointc                   ó†   a a€ ] tR t^t oRtRV3R lV 3R llltR	R lt]]P                  3V3R lR ll4       t
RtVtV ;t# )
ÚMultiHeadAttentiona¡  Implements the scaled dot product attention with multiple heads.

Given inputs for queries, keys and values the ``MultiHeadAttention``
produces new values by aggregating information from the input values
according to the similarities of the input queries and keys.

All inputs as well as the output are linearly projected without biases by
default.

``MultiHeadAttention`` also takes an optional additive attention mask that
should be broadcastable with ``(batch, num_heads, # queries, # keys)``. The
mask should have ``-inf`` or very large negative numbers at the positions
that should *not* be attended to.

Args:
    dims (int): The model dimensions. This is also the default
        value for the queries, keys, values, and the output.
    num_heads (int): The number of attention heads to use.
    query_input_dims (int, optional): The input dimensions of the queries.
        Default: ``dims``.
    key_input_dims (int, optional): The input dimensions of the keys.
        Default: ``dims``.
    value_input_dims (int, optional): The input dimensions of the values.
        Default: ``key_input_dims``.
    value_dims (int, optional): The dimensions of the values after the
        projection. Default: ``dims``.
    value_output_dims (int, optional): The dimensions the new values will
        be projected to. Default: ``dims``.
    bias (bool, optional): Whether or not to use a bias in the projections.
        Default: ``False``.
c                óš   <€ V ^8„  d   QhRS[ RS[ RS[S[ ,          RS[S[ ,          RS[S[ ,          RS[S[ ,          RS[S[ ,          RS[/# )	é   ÚdimsÚ	num_headsÚquery_input_dimsÚkey_input_dimsÚvalue_input_dimsÚ
value_dimsÚvalue_output_dimsÚbias)Úintr   Úbool)ÚformatÚ__classdict__s   "€Új/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx/nn/layers/transformer.pyÚ__annotate__ÚMultiHeadAttention.__annotate__0   sy   ø€ ÷ Iñ IáðIñ ðIñ #¡3�-ð	Iñ
 !¡�ðIñ #¡3�-ðIñ ™S•MðIñ $¡C�=ðIñ ñIó    c	                óv  <€ \         S	V `  4        W,          ^ 8w  d   \        RV RV R24      hT;'       g    TpT;'       g    TpT;'       g    TpT;'       g    TpT;'       g    TpW n        \	        W1VR7      V n        \	        WAVR7      V n        \	        WVVR7      V n        \	        WgVR7      V n        R# )r   zIThe input feature dimensions should be divisible by the number of heads (z % z) != 0)r   N)	ÚsuperÚ__init__Ú
ValueErrorr   r	   Ú
query_projÚkey_projÚ
value_projÚout_proj)
Úselfr   r   r   r   r   r   r   r   Ú	__class__s
   &&&&&&&&&€r   r"   ÚMultiHeadAttention.__init__0   sÄ   ø€ ô 	‰ÑÔàÕ Ô"Üð$Ø$( 6¨¨Y¨K°vð?óð ð
 ,×3Ð3¨tÐØ'×/Ð/¨4ˆØ+×=Ð=¨~ÐØ×'Ð' 4ˆ
Ø-×5Ð5°Ðà"ŒÜ Ð!1¸dÔCˆŒÜ˜~¸$Ô?ˆŒÜ Ð!1ÀDÔIˆŒÜ˜zÀ4ÔHˆŽr   c                ó’  € V P                  V4      pV P                  V4      pV P                  V4      pV P                  p\        P
                  ! VRVR34      P                  ^ ^^^4      p\        P
                  ! VRVR34      P                  ^ ^^^4      p\        P
                  ! VRVR34      P                  ^ ^^^4      p\        P                  ! ^VP                  R,          ,          4      p\        P                  P                  WW6VR7      pVP                  ^ ^^^4      P                  RR4      pV P                  V4      # )é   )ÚscaleÚmaskéÿÿÿÿéþÿÿÿ)r$   r%   r&   r   ÚmxÚ	unflattenÚ	transposeÚmathÚsqrtÚshapeÚfastÚscaled_dot_product_attentionÚflattenr'   )r(   ÚqueriesÚkeysÚvaluesr.   r   r-   Úoutputs   &&&&&   r   Ú__call__ÚMultiHeadAttention.__call__O   s  € Ø—/‘/ 'Ó*ˆØ�}‰}˜TÓ"ˆØ—‘ Ó(ˆà—N‘Nˆ	Ü—,’,˜w¨¨Y¸¨OÓ<×FÑFÀqÈ!ÈQÐPQÓRˆÜ�|Š|˜D " y°" oÓ6×@Ñ@ÀÀAÀqÈ!ÓLˆÜ—’˜f b¨9°b¨/Ó:×DÑDÀQÈÈ1ÈaÓPˆÜ—	’	˜!˜gŸm™m¨BÕ/Õ/Ó0ˆÜ—‘×5Ñ5Ø˜6°Tð 6ó 
ˆð ×!Ñ! ! Q¨¨1Ó-×5Ñ5°b¸"Ó=ˆØ�}‰}˜VÓ$Ð$r   c                ó:   <€ V ^8„  d   QhRS[ RS[P                  /# )r   ÚNÚdtype)r   r1   ÚDtype)r   r   s   "€r   r   r   `   s   ø€ ÷ ñ ¡sð ±2·8±8ñ r   c                óÄ   € \         P                  ! V 4      pVR,          VR,          8  pVP                  V4      \         P                  ! V4      P                  ,          pV# )ºNNNN)rE   N)r1   ÚarangeÚastypeÚfinfoÚmin)rA   rB   Úindicesr.   s   &&  r   Úcreate_additive_causal_maskÚ.MultiHeadAttention.create_additive_causal_mask_   sH   € ä—)’)˜A“,ˆØ�wÕ '¨$¥-Ñ/ˆØ�{‰{˜5Ó!¤B§H¢H¨U£O×$7Ñ$7Õ7ˆØˆr   )r%   r   r'   r$   r&   )NNNNNF©N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r"   r>   Ústaticmethodr1   Úfloat32rK   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r)   r   s   @@r   r   r      sA   ù‡ € ñ÷@Iõ Iô>%ð  Ø>@¿j¹j÷ ñ ó ÷ð r   r   c                   óP   a a€ ] tR t^gt oRR]R3V3R lV 3R llltR tRtVtV ;t	# )ÚTransformerEncoderLayerNç        Tc                óf   <€ V ^8„  d   QhRS[ RS[ RS[S[ ,          RS[RS[S[.S[3,          RS[/# ©r   r   r   Úmlp_dimsÚdropoutÚ
activationÚ
norm_first©r   r   Úfloatr   r   r   )r   r   s   "€r   r   Ú$TransformerEncoderLayer.__annotate__h   sV   ø€ ÷ %ñ %áð%ñ ð%ñ ™3•-ð	%ñ
 ð%ñ ™c˜U¡C˜ZÕ(ð%ñ ñ%r   c                óB  <€ \         SV `  4        T;'       g
    V^,          p\        W4      V n        \	        V4      V n        \	        V4      V n        \        W4      V n        \        W14      V n	        \        V4      V n        \        V4      V n        WPn        W`n        R# ©é   N)r!   r"   r   Ú	attentionr
   Úln1Úln2r	   Úlinear1Úlinear2r   Údropout1Údropout2r`   ra   ©r(   r   r   r^   r_   r`   ra   r)   s   &&&&&&&€r   r"   Ú TransformerEncoderLayer.__init__h   s}   ø€ ô 	‰ÑÔØ×'Ð'˜t a�xˆÜ+¨DÓ<ˆŒÜ˜T“?ˆŒÜ˜T“?ˆŒÜ˜dÓ-ˆŒÜ˜hÓ-ˆŒÜ Ó(ˆŒÜ Ó(ˆŒØ$ŒØ$Žr   c                óŠ  € V P                   '       dœ   V P                  V4      pV P                  W3W24      pV P                  V4      pW,           pV P	                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pW,           pV# V P                  WW4      pV P                  V4      pV P                  W,           4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P	                  W,           4      pV# rM   )	ra   ri   rh   rm   rj   rk   r`   rn   rl   )r(   Úxr.   Úys   &&& r   r>   Ú TransformerEncoderLayer.__call__}   s  € Ø�?�?ˆ?Ø—‘˜“ˆAØ—‘˜q QÓ-ˆAØ—‘˜aÓ ˆAØ•ˆAà—‘˜“ˆAØ—‘˜Q“ˆAØ—‘ Ó"ˆAØ—‘˜aÓ ˆAØ—‘˜Q“ˆAØ•ˆAð ˆð —‘˜q QÓ-ˆAØ—‘˜aÓ ˆAØ—‘˜�“ˆAà—‘˜Q“ˆAØ—‘ Ó"ˆAØ—‘˜aÓ ˆAØ—‘˜Q“ˆAØ—‘˜�“ˆAàˆr   )	r`   rh   rm   rn   rk   rl   ri   rj   ra   ©
rN   rO   rP   rQ   r   r"   r>   rU   rV   rW   rX   s   @@r   rZ   rZ   g   s'   ù‡ € ð
 #'ØØ+/Ø÷%õ %÷*ò r   rZ   c                   óR   a a€ ] tR t^™t oRR]RR3V3R lV 3R llltR tRtVtV ;t	# )	ÚTransformerEncoderNr[   TFc                óT   <€ V ^8„  d   QhRS[ RS[ RS[ RS[S[ ,          RS[RS[RS[/# ©r   Ú
num_layersr   r   r^   r_   ra   r   ©r   r   rc   r   )r   r   s   "€r   r   ÚTransformerEncoder.__annotate__š   óU   ø€ ÷ %ñ %áð%ñ ð%ñ ð	%ñ
 ™3•-ð%ñ ð%ñ ð%ñ ñ%r   c	                ó°   <€ \         S
V `  4        \        V4       U	u. uF  p	\        W#WEWg4      NK  	  up	V n        \        V4      V n        W€n        R # u up	i rM   )r!   r"   ÚrangerZ   Úlayersr
   Úlnr   ©r(   rz   r   r   r^   r_   r`   ra   r   Úir)   s   &&&&&&&&& €r   r"   ÚTransformerEncoder.__init__š   ó]   ø€ ô 	‰ÑÔô
 ˜:Ô&ó	
ñ '�ô $Ø °Jöñ 'ñ	
ˆŒô ˜D“/ˆŒØ$Žùò
ó   �Ac                ó˜   € V P                    F*  pV P                  '       d   \        V4      MTpV! W4      pK,  	  V P                  V4      # rM   ©r€   r   r�   )r(   rr   r.   Úls   &&& r   r>   ÚTransformerEncoder.__call__¯   s<   € Ø—”ˆAØ!%§§ ”
˜1”°aˆAÙ�!“
ŠAñ ð �w‰w�q‹zÐr   ©r   r€   r�   ru   rX   s   @@r   rw   rw   ™   ó*   ù‡ € ð #'ØØØØ ÷%õ %÷*ò r   rw   c                   óP   a a€ ] tR t^¶t oRR]R3V3R lV 3R llltR tRtVtV ;t	# )ÚTransformerDecoderLayerNr[   Tc                óf   <€ V ^8„  d   QhRS[ RS[ RS[S[ ,          RS[RS[S[.S[3,          RS[/# r]   rb   )r   r   s   "€r   r   Ú$TransformerDecoderLayer.__annotate__·   sV   ø€ ÷ %ñ %áð%ñ ð%ñ ™3•-ð	%ñ
 ð%ñ ™c˜U¡C˜ZÕ(ð%ñ ñ%r   c                ó¢  <€ \         SV `  4        T;'       g
    V^,          p\        W4      V n        \        W4      V n        \        V4      V n        \        V4      V n        \        V4      V n        \        W4      V n
        \        W14      V n        \        V4      V n        \        V4      V n        \        V4      V n        WPn        W`n        R# rf   )r!   r"   r   Úself_attentionÚcross_attentionr
   ri   rj   Úln3r	   rk   rl   r   rm   rn   Údropout3r`   ra   ro   s   &&&&&&&€r   r"   Ú TransformerDecoderLayer.__init__·   s¢   ø€ ô 	‰ÑÔØ×'Ð'˜t a�xˆÜ0°ÓAˆÔÜ1°$ÓBˆÔÜ˜T“?ˆŒÜ˜T“?ˆŒÜ˜T“?ˆŒÜ˜dÓ-ˆŒÜ˜hÓ-ˆŒÜ Ó(ˆŒÜ Ó(ˆŒÜ Ó(ˆŒØ$ŒØ$Žr   c                óv  € V P                   '       dØ   V P                  V4      pV P                  WUWS4      pV P                  V4      pW,           pV P	                  V4      pV P                  WRW$4      pV P                  V4      pW,           pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pW,           pV# V P                  WW4      pV P                  V4      pV P                  W,           4      pV P                  WW$4      pV P                  V4      pV P	                  W,           4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  V4      pV P                  W,           4      pV# rM   )ra   ri   r’   rm   rj   r“   rn   r”   rk   r`   r•   rl   )r(   rr   ÚmemoryÚx_maskÚmemory_maskrs   s   &&&&& r   r>   Ú TransformerDecoderLayer.__call__Ï   sn  € Ø�?�?ˆ?Ø—‘˜“ˆAØ×#Ñ# A¨!Ó4ˆAØ—‘˜aÓ ˆAØ•ˆAà—‘˜“ˆAØ×$Ñ$ Q°ÓDˆAØ—‘˜aÓ ˆAØ•ˆAà—‘˜“ˆAØ—‘˜Q“ˆAØ—‘ Ó"ˆAØ—‘˜aÓ ˆAØ—‘˜Q“ˆAØ•ˆAð" ˆð ×#Ñ# A¨!Ó4ˆAØ—‘˜aÓ ˆAØ—‘˜�“ˆAà×$Ñ$ Q°ÓDˆAØ—‘˜aÓ ˆAØ—‘˜�“ˆAà—‘˜Q“ˆAØ—‘ Ó"ˆAØ—‘˜aÓ ˆAØ—‘˜Q“ˆAØ—‘˜�“ˆAàˆr   )r`   r“   rm   rn   r•   rk   rl   ri   rj   r”   ra   r’   ru   rX   s   @@r   rŽ   rŽ   ¶   s'   ù‡ € ð
 #'ØØ+/Ø÷%õ %÷0"ò "r   rŽ   c                   óR   a a€ ] tR t^ôt oRR]RR3V3R lV 3R llltR tRtVtV ;t	# )	ÚTransformerDecoderNr[   TFc                óT   <€ V ^8„  d   QhRS[ RS[ RS[ RS[S[ ,          RS[RS[RS[/# ry   r{   )r   r   s   "€r   r   ÚTransformerDecoder.__annotate__õ   r}   r   c	                ó°   <€ \         S
V `  4        \        V4       U	u. uF  p	\        W#WEWg4      NK  	  up	V n        \        V4      V n        W€n        R # u up	i rM   )r!   r"   r   rŽ   r€   r
   r�   r   r‚   s   &&&&&&&&& €r   r"   ÚTransformerDecoder.__init__õ   r…   r†   c                óš   € V P                    F+  pV P                  '       d   \        V4      MTpV! WW44      pK-  	  V P                  V4      # rM   rˆ   )r(   rr   r˜   r™   rš   r‰   s   &&&&& r   r>   ÚTransformerDecoder.__call__
  s?   € Ø—”ˆAØ!%§§ ”
˜1”°aˆAÙ�!˜VÓ1ŠAñ ð �w‰w�q‹zÐr   r‹   ru   rX   s   @@r   r�   r�   ô   rŒ   r   r�   c                   ób   a a€ ] tR tRt oRtR^^^RR]RRRR3V3R lV 3R	 llltR
 tRtVt	V ;t
# )ÚTransformeri  aŸ  
Implements a standard Transformer model.

The implementation is based on `Attention Is All You Need
<https://arxiv.org/abs/1706.03762>`_.

The Transformer model contains an encoder and a decoder. The encoder
processes the input sequence and the decoder generates the output sequence.
The interaction between encoder and decoder happens through the attention
mechanism.

Args:
    dims (int, optional): The number of expected features in the
        encoder/decoder inputs. Default: ``512``.
    num_heads (int, optional): The number of attention heads. Default:
        ``8``.
    num_encoder_layers (int, optional): The number of encoder layers in the
        Transformer encoder. Default: ``6``.
    num_decoder_layers (int, optional): The number of decoder layers in the
        Transformer decoder. Default: ``6``.
    mlp_dims (int, optional): The hidden dimension of the MLP block in each
        Transformer layer. Defaults to ``4*dims`` if not provided. Default:
        ``None``.
    dropout (float, optional): The dropout value for the Transformer
        encoder and decoder. Dropout is used after each attention layer and
        the activation in the MLP layer. Default: ``0.0``.
    activation (function, optional): the activation function for the MLP
        hidden layer. Default: :func:`mlx.nn.relu`.
    custom_encoder (nn.Module, optional): A custom encoder to replace the
        standard Transformer encoder. Default: ``None``.
    custom_decoder (nn.Module, optional): A custom decoder to replace the
        standard Transformer decoder. Default: ``None``.
    norm_first (bool, optional): if ``True``, encoder and decoder layers
        will perform layer normalization before attention and MLP
        operations, otherwise after. Default: ``True``.
    checkpoint (bool, optional): if ``True`` perform gradient checkpointing
        to reduce the memory usage at the expense of more computation.
        Default: ``False``.
i   Nr[   TFc                ó¤   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ RS[S[ ,          RS[RS[S[.S[3,          RS[S[,          R	S[S[,          R
S[RS[/# )r   r   r   Únum_encoder_layersÚnum_decoder_layersr^   r_   r`   Úcustom_encoderÚcustom_decoderra   r   rb   )r   r   s   "€r   r   ÚTransformer.__annotate__:  s�   ø€ ÷ $
ñ $
áð$
ñ ð$
ñ  ð	$
ñ
  ð$
ñ ™3•-ð$
ñ ð$
ñ ™c˜U¡C˜ZÕ(ð$
ñ !¡�ð$
ñ !¡�ð$
ñ ð$
ñ ñ$
r   c           
     ó¨   <€ \         SV `  4        T;'       g    \        VVVVVVV
V4      V n        T	;'       g    \	        VVVVVVV
V4      V n        R # rM   )r!   r"   rw   Úencoderr�   Údecoder)r(   r   r   r§   r¨   r^   r_   r`   r©   rª   ra   r   r)   s   &&&&&&&&&&&&€r   r"   ÚTransformer.__init__:  st   ø€ ô 	‰ÑÔà%÷ 	
ð 	
Ô);ØØØØØØØØó	*
ˆŒð &÷ 	
ð 	
Ô);ØØØØØØØØó	*
ˆŽr   c                óH   € V P                  W4      pV P                  W&WE4      # rM   )r­   r®   )r(   ÚsrcÚtgtÚsrc_maskÚtgt_maskrš   r˜   s   &&&&&& r   r>   ÚTransformer.__call__`  s!   € Ø—‘˜cÓ,ˆØ�|‰|˜C¨Ó?Ð?r   )r®   r­   )rN   rO   rP   rQ   rR   r   r"   r>   rU   rV   rW   rX   s   @@r   r¥   r¥     sE   ù‡ € ñ&ðT ØØ"#Ø"#Ø"&ØØ+/Ø(,Ø(,ØØ ÷$
õ $
÷L@ò @r   r¥   )r4   Útypingr   r   r   Úmlx.coreÚcorer1   Úmlx.nn.layers.activationsr   Úmlx.nn.layers.baser   Úmlx.nn.layers.dropoutr   Úmlx.nn.layers.linearr	   Úmlx.nn.layers.normalizationr
   Úmlx.nn.utilsr   r   rZ   rw   rŽ   r�   r¥   © r   r   Ú<module>rÀ      sv   ðó ß *Ñ *å Ý *Ý %Ý )Ý 'Ý 1Ý #ôU˜ô Uôp/˜fô /ôd˜ô ô:;˜fô ;ô|˜ô ô:Q@�&ö Q@r   