+
    NV-je  ã                   ót   € ^ RI t ^ RIHt ^ RIHt ^ RIHt  ! R R]4      t ! R R]4      t	 ! R R	]4      t
R# )
é    N)ÚOptional)ÚModulec                   ób   a a€ ] tR t^
t oRtRV3R lV 3R llltR tR	V3R lR lltRtVt	V ;t
# )
ÚRoPEa}  Implements the rotary positional encoding.

The traditional implementation rotates consecutive pairs of elements in the
feature dimension while the default implementation rotates pairs with
stride half the feature dimensions for efficiency.

For more details see `RoFormer: Enhanced Transformer with Rotary Position
Embedding <https://arxiv.org/abs/2104.09864>`_.

Args:
    dims (int): The feature dimensions to be rotated. If the input feature
        is larger than dims then the rest is left unchanged.
    traditional (bool, optional): If set to ``True`` choose the traditional
        implementation which is slightly less efficient. Default: ``False``.
    base (float, optional): The base used to compute angular frequency for
        each dimension in the positional encodings. Default: ``10000``.
    scale (float, optional): The scale used to scale the positions. Default: ``1.0``.
c                ó2   <€ V ^8„  d   QhRS[ RS[RS[RS[/# )é   ÚdimsÚtraditionalÚbaseÚscale)ÚintÚboolÚfloat)ÚformatÚ__classdict__s   "€Úr/Volumes/fast/ai/experiments/ui-tars-smoke/.venv/lib/python3.14/site-packages/mlx/nn/layers/positional_encoding.pyÚ__annotate__ÚRoPE.__annotate__   s3   ø€ ÷ ñ áðñ ðñ ð	ñ
 ñó    c                óT   <€ \         SV `  4        Wn        W n        W0n        W@n        R # )N)ÚsuperÚ__init__r	   r
   r   r   )Úselfr	   r
   r   r   Ú	__class__s   &&&&&€r   r   ÚRoPE.__init__   s%   ø€ ô 	‰ÑÔØŒ	Ø&ÔØŒ	ØŽ
r   c                ó8   € V P                    R V P                   2# )z, traditional=)r	   r
   )r   s   &r   Ú_extra_reprÚRoPE._extra_repr+   s   € Ø—)‘)�˜N¨4×+;Ñ+;Ð*<Ð=Ð=r   c                ó    <€ V ^8„  d   QhRS[ /# )r   Úoffset©r   )r   r   s   "€r   r   r   .   s   ø€ ÷ 
ñ 
¡#ñ 
r   c           	     óœ   € \         P                  P                  VV P                  V P                  V P
                  V P                  VR 7      # ))r
   r   r   r    )ÚmxÚfastÚroper	   r
   r   r   )r   Úxr    s   &&&r   Ú__call__ÚRoPE.__call__.   s@   € Ü�w‰w�|‰|ØØ�I‰IØ×(Ñ(Ø—‘Ø—*‘*Øð ó 
ð 	
r   )r   r	   r   r
   )Fi'  g      ð?)r   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r   r   r'   Ú__static_attributes__Ú__classdictcell__Ú__classcell__©r   r   s   @@r   r   r   
   s(   ù‡ € ñ÷&õ ò>÷
÷ 
ò 
r   r   c                   óL   a a€ ] tR t^9t oRtRV3R lV 3R llltR tRtVtV ;t	# )ÚSinusoidalPositionalEncodingaô  Implements sinusoidal positional encoding.

For more details see the paper `Attention Is All You Need
<https://arxiv.org/abs/1706.03762>`_.

Args:
    dims (int): The dimensionality of the resulting positional embeddings.
    min_freq (float, optional): The minimum frequency expected. Default:
        ``0.0001``.
    max_freq (float, optional): The maximum frequency expected. Default:
        ``1``.
    scale (float, optional): A multiplicative scale for the embeddings.
        Default: ``sqrt(2/dims)``.
    cos_first (bool, optional): If ``True`` embed using ``[cos(x); sin(x)]``
        instead of the reverse. Default: ``False``.
    full_turns (bool, optional): If ``True`` multiply the frequencies with
        :math:`2\pi`. Default: ``False``.
c                óN   <€ V ^8„  d   QhRS[ RS[RS[RS[S[,          RS[RS[/# )r   r	   Úmin_freqÚmax_freqr   Ú	cos_firstÚ
full_turns)r   r   r   r   )r   r   s   "€r   r   Ú)SinusoidalPositionalEncoding.__annotate__M   sK   ø€ ÷ #ñ #áð#ñ ð#ñ ð	#ñ
 ™�ð#ñ ð#ñ ñ#r   c                ó  <€ \         SV `  4        ^\        P                  ! ^ V^,          4      V^,          ^,
          ,          ,
          p\        P
                  ! V4      p\        P
                  ! V4      p\        P                  ! WsV,
          ,          V,           4      V n        V'       d.   V P                  ^\        P                  ,          ,          V n        T;'       g    ^V,          R,          V n	        WPn
        R# )é   g      à?N)r   r   r#   ÚarangeÚmathÚlogÚexpÚ_sigmasÚpir   r7   )	r   r	   r5   r6   r   r7   r8   Úone_zeror   s	   &&&&&&& €r   r   Ú%SinusoidalPositionalEncoding.__init__M   s«   ø€ ô 	‰ÑÔà”r—y’y  D¨A¥IÓ.°$¸!µ)¸aµ-Õ@Õ@ˆÜ—8’8˜HÓ%ˆÜ—8’8˜HÓ%ˆô —v’v˜h°XÕ*=Õ>ÀÕIÓJˆŒßØŸ<™<¨1¬t¯w©w­;Õ7ˆDŒLð ×/Ð/˜q 4�x¨CÕ/ˆŒ
Ø"Žr   c                ób  € VR,          V P                   ,          p\        P                  ! V4      p\        P                  ! V4      pV P                  '       d   \        P
                  ! W4.RR7      pM\        P
                  ! WC.RR7      pV P                  ^8w  d   W P                  ,          pV# ).©Úaxis).Néÿÿÿÿ)r@   r#   ÚcosÚsinr7   Úconcatenater   )r   r&   ÚyÚcosyÚsinys   &&   r   r'   Ú%SinusoidalPositionalEncoding.__call__e   sv   € Øˆi�L˜4Ÿ<™<Õ'ˆÜ�vŠv�a‹yˆÜ�vŠv�a‹yˆà�>�>ˆ>Ü—’ ˜|°"Ô5‰Aä—’ ˜|°"Ô5ˆAà�:‰:˜Œ?Ø—J‘J•ˆAàˆr   )r@   r7   r   )g-Cëâ6?r;   NFF)
r)   r*   r+   r,   r-   r   r'   r.   r/   r0   r1   s   @@r   r3   r3   9   s   ù‡ € ñ÷&#õ #÷0ò r   r3   c                   óp   a € ] tR t^ut o ]]P                  3V 3R lR ll4       t]R 4       tRR lt	Rt
V tR# )ÚALiBic                ó2   <€ V ^8„  d   QhRS[ RS[ RS[ RS[ /# )r   Úq_sequence_lengthÚk_sequence_lengthÚ	num_headsr    r!   )r   r   s   "€r   r   ÚALiBi.__annotate__w   s1   ø€ ÷ ñ Ùðáðñ ðñ ñ	r   c                ó>  € \         P                  ! W04      p\         P                  ! ^ V4      p\         P                  ! \         P                  ! VR,          VR,          ,
          RR7      4      ) p\        P                  W$R7      pWx,          P                  V4      p	V	# )r   rE   )rT   Údtype)ºNNNN)NrX   )r   r;   )r#   r<   ÚabsÚexpand_dimsrP   Úcreate_alibi_slopeÚastype)
rR   rS   rT   r    rW   Úx1Úx2Údistance_matrixÚalibi_slopeÚ
alibi_masks
   &&&&&     r   Úcreate_alibi_matrixÚALiBi.create_alibi_matrixv   s   € ô �YŠY�vÓ1ˆÜ�YŠY�qÐ+Ó,ˆÜŸ6š6Ü�NŠN˜2˜g�;¨¨G­Õ4¸6ÔBó
ð 
ˆô ×.Ñ.¸Ð.ÓPˆØ%Õ3×;Ñ;¸EÓBˆ
ØÐr   c                ó„   a€ R  V3R lloS! V 4      p\         P                  ! W!R7      p\         P                  ! VRR7      # )c                ó$   € V ^8„  d   QhR\         /# )r   Únr!   )r   s   "r   r   Ú.ALiBi.create_alibi_slope.<locals>.__annotate__‰   s   € ÷ 		ñ 		œ#ñ 		r   c                 óÄ  <€ \         P                  ! V 4      P                  4       '       dU   ^^\         P                  ! V 4      ^,
          ) ,          ) ,          p\        V 4       Uu. uF  q!W,          ,          NK  	  up# ^\         P                  ! \         P                  ! V 4      4      ,          pS! V4      S! ^V,          4      R,          RW,
           ,           # u upi )r   :r   Nr   N)r=   Úlog2Ú
is_integerÚrangeÚfloor)rf   ÚstartÚiÚclosest_power_of_2Ú
get_slopess   &   €r   rp   Ú,ALiBi.create_alibi_slope.<locals>.get_slopes‰   s°   ø€ Ü�yŠy˜‹|×&Ñ&×(Ò(Ø ¤t§y¢y°£|°aÕ'7Ð%8Õ 8Ð9Õ:�Ü27¸´(Ó;±(¨Q ¥×(Ð(±(Ñ;Ð;à%&¬$¯*ª*´T·Y²Y¸q³\Ó*BÕ%BÐ"áÐ1Ó2Ù  Ð%7Õ!7Ó8¸Õ>Ð?WÀÕAWÐXõYðùò <s   Á&C)rW   rE   )rG   éþÿÿÿ)r#   ÚarrayrZ   )rT   rW   ÚslopesÚoutrp   s   &&  @r   r[   ÚALiBi.create_alibi_slope‡   s8   ø€ ÷		ð 		ñ ˜IÓ&ˆÜ�hŠh�vÔ+ˆÜ�~Š~˜c¨Ô1Ð1r   Nc                óæ   € \         P                  VP                  R,          V,           VP                  R,          VP                  ^,          VVP                  R7      pVe	   WC,           pW,           # )r   )rR   rS   rT   r    rW   rr   rG   )rP   rb   ÚshaperW   )r   Úattention_scoresr    Úmaskra   s   &&&& r   r'   ÚALiBi.__call__˜   sk   € Ü×.Ñ.Ø.×4Ñ4°RÕ8¸6ÕAØ.×4Ñ4°RÕ8Ø&×,Ñ,¨QÕ/ØØ"×(Ñ(ð /ó 
ˆ
ð ÒØ#Õ*ˆJØÕ,Ð,r   © )r   N)r)   r*   r+   r,   Ústaticmethodr#   Úfloat32rb   r[   r'   r.   r/   )r   s   @r   rP   rP   u   sC   ø‡ € Øð �j‰j÷ñ ó ðð  ñ2ó ð2÷ 
-ò 
-r   rP   )r=   Útypingr   Úmlx.coreÚcorer#   Úmlx.nn.layers.baser   r   r3   rP   r|   r   r   Ú<module>rƒ      s9   ðó Ý å Ý %ô,
ˆ6ô ,
ô^9 6ô 9ôx--ˆFö --r   