Ë
    îÿæi¸#  ã                   óØ   — d dl Z d dlmZmZmZ d dlmZ d dlm	Z	  G d„ de	j                  «      Z G d„ de	j                  «      Z G d„ de	j                  «      Z	 	 dd	ee   d
ee   fd„Zy)é    N)ÚListÚOptionalÚUnionc                   óœ   ‡ — e Zd Z	 	 	 	 	 	 	 ddededededeee   ef   deee   ef   dedefˆ fd	„Zdd
eeej                  f   fd„Z
ˆ xZS )ÚSuScaledRoPEÚdimsÚbaseÚmax_position_embeddingsÚ original_max_position_embeddingsÚshort_factorÚlong_factorÚshort_mscaleÚlong_mscalec	                 óD  •‡— t         ‰| �  «        ‰| _        || _        |t	        j
                  d|dt        j                  ¬«      |z  z  }	t	        j                  |t        j                  ¬«      |	z  | _        ˆfd„}
|‰z  }|xs |dk  rd| _	        y |
|«      | _	        y)a  
        Su Scaled Rotary Embedding layer.

        Args:
            dims (int): The feature dimensions to be rotated.
            base (int, optional): Base for the exponential scaling.
            max_position_embeddings (int, optional): The maximum sequence
              length that this model was trained with. This is used to determine
              the size of the original RoPE embeddings when using long scaling.
              Default: ``131072``.
            original_max_position_embeddings (int, optional): The maximum
              sequence length that this model was trained with. This is used to
              determine the size of the original RoPE embeddings when using long
              scaling. Default: ``4096``.
            short_factor (float or list[float], optional): List of scaling
              factors for sequences of length lesser than
              ``original_max_position_embeddings``. Default: ``1.0``.
            long_factor (float or list[float], optional): List of scaling
              factors for sequences of length greater than
              ``original_max_position_embeddings``.  Default: ``1.0``.
            short_mscale (float, optional): Scale the input prior to embedding.
            long_mscale (float, optional): Scale the input prior to embedding.
        r   é   ©Údtypec                 ó†   •— t        j                  dt        j                  | «      t        j                  ‰«      z  z   «      S )Né   )ÚmathÚsqrtÚlog)Úfactorr   s    €úm/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/mlx_lm/models/rope_utils.pyÚdefault_scalez,SuScaledRoPE.__init__.<locals>.default_scale4   s4   ø€ Ü—9‘9Ø”D—H‘H˜VÓ$¤t§x¡xÐ0PÓ'QÑQÑQóð ó    ç      ð?N)
ÚsuperÚ__init__r   ÚdimÚmxÚarangeÚfloat32ÚarrayÚ_freqsÚ_scale)Úselfr   r	   r
   r   r   r   r   r   Úfreqsr   r   Ú	__class__s       `       €r   r   zSuScaledRoPE.__init__   sŒ   ù€ ôD 	‰ÑÔØ0PˆÔ-ØˆŒàœŸ™ 1 d¨A´R·Z±ZÔ@À4ÑGÑHˆÜ—h‘h˜{´"·*±*Ô=ÀÑEˆŒô	ð
 )Ð+KÑKˆØ!ÒV¨V°sª] cˆ�ÁÈfÓ@Uˆ�r   Úoffsetc           	      óØ   — | j                   |dd | j                  …f   z  |dd | j                  …f<   t        j                  j	                  || j                  dd d|| j
                  ¬«      S )N.Fr   ©Útraditionalr	   Úscaler*   r(   )r&   r    r!   ÚfastÚroper%   ©r'   Úxr*   s      r   Ú__call__zSuScaledRoPE.__call__<   si   € Ø!Ÿ[™[¨1¨S°*°D·H±H°*¨_Ñ+=Ñ=ˆˆ#ˆz�—‘ˆzˆ/ÑÜ�w‰w�|‰|ØØ�H‰HØØØØØ—+‘+ð ó 
ð 	
r   )g     ˆÃ@i   é   r   r   NN©r   )Ú__name__Ú
__module__Ú__qualname__ÚintÚfloatr   r   r   r!   r$   r3   Ú__classcell__©r)   s   @r   r   r   
   s¬   ø„ ð Ø'-Ø04Ø25Ø14Ø"Ø!ñ/Wàð/Wð ð/Wð "%ð	/Wð
 +.ð/Wð ˜D ™K¨Ð.Ñ/ð/Wð ˜4 ™;¨Ð-Ñ.ð/Wð ð/Wð õ/Wñb

 %¨¨R¯X©X¨Ñ"6÷ 

r   r   c                   óR   ‡ — e Zd Z	 	 	 	 d
dededededef
ˆ fd„Zd„ Zddefd	„Z	ˆ xZ
S )Ú
Llama3RoPEr   r
   r-   r	   Úscaling_configc                 óü  •— t         ‰| �  «        || _        || _        || _        |d   }|j                  dd«      }|j                  dd«      }|j                  dd«      }	|	|z  }
|	|z  }|t        j                  d|d	«      |z  z  }d	t        j                  z  |z  }t        j                  ||
kD  ||z  |«      }||kD  ||
k  z  }|	|z  |z
  ||z
  z  }|d
|z
  |z  |z   z  }t        j                  |||«      | _
        y )Nr   Úlow_freq_factorr   Úhigh_freq_factorg      @r   i    r   r   r   )r   r   r   r
   r-   Úgetr!   r"   ÚpiÚwherer%   )r'   r   r
   r-   r	   r?   r   rA   rB   Úold_context_lenÚlow_freq_wavelenÚhigh_freq_wavelenr(   ÚwavelensÚis_medium_freqÚsmooth_factorsÚsmooth_freqsr)   s                    €r   r   zLlama3RoPE.__init__K   s2  ø€ ô 	‰ÑÔØˆŒ	Ø'>ˆÔ$Ø&ˆÔà Ñ)ˆØ(×,Ñ,Ð->ÀÓDˆØ)×-Ñ-Ð.@À#ÓFÐØ(×,Ñ,Ø.Øó
ˆð
 +¨_Ñ<ÐØ+Ð.>Ñ>ÐàœŸ™ 1 d¨AÓ.°Ñ5Ñ6ˆØ”r—u‘u‘9˜uÑ$ˆä—‘˜Ð$4Ñ4°e¸f±nÀeÓLˆØ"Ð%6Ñ6¸8ÐFVÑ;VÑWˆØ)¨HÑ4°ÑFØ˜Ñ.ñ
ˆð   ^Ñ!3°vÑ =ÀÑ NÑOˆÜ—h‘h˜~¨|¸UÓCˆ�r   c                 óR   — | j                   › d| j                  › d| j                  › �S )Nz, traditional=z, max_position_embeddings=)r   r-   r
   )r'   s    r   Ú
extra_reprzLlama3RoPE.extra_reprn   s4   € à�y‰yˆk˜¨×(8Ñ(8Ð'9ð :'Ø'+×'CÑ'CÐ&DðFð	
r   r*   c           	      óŠ   — t         j                  j                  || j                  | j                  d d|| j
                  ¬«      S )Nr   r,   )r!   r/   r0   r   r-   r%   r1   s      r   r3   zLlama3RoPE.__call__t   s?   € Ü�w‰w�|‰|ØØ�I‰IØ×(Ñ(ØØØØ—+‘+ð ó 
ð 	
r   )é   Fé'  Nr5   )r6   r7   r8   r9   Úboolr:   Údictr   rN   r3   r;   r<   s   @r   r>   r>   I   s^   ø„ ð
 (,Ø!ØØ#ñ!Dàð!Dð "%ð!Dð ð	!Dð
 ð!Dð õ!DòF
ñ	
 #÷ 	
r   r>   c                   ó:   ‡ — e Zd Z	 	 	 	 	 	 	 	 	 dˆ fd„	Zdd„Zˆ xZS )ÚYarnRoPEc                 óp  •‡‡‡‡‡‡— t         ‰| �  «        ˆˆˆfd„Šˆˆˆˆfd„}d
d„}d„ } |||	«       |||
«      z  | _        ‰t        j                  d‰dt        j
                  ¬«      ‰z  z  }||z  } |«       \  }}d	 |||‰dz  «      z
  }||z  ||z  |d|z
  z  z   z  | _        ‰| _        || _        y )Nc                 ó”   •— ‰t        j                  ‰| dz  t         j                  z  z  «      z  dt        j                  ‰«      z  z  S )Nr   )r   r   rD   )Únum_rotationsr	   r   r   s    €€€r   Úyarn_find_correction_dimz3YarnRoPE.__init__.<locals>.yarn_find_correction_dim�   sI   ø€ àÜ—(‘(Ø4¸ÈÑ8IÌDÏGÉGÑ8SÑTóñð ”T—X‘X˜d“^Ñ#ñ%ð %r   c                  ó¦   •— t        j                   ‰‰«      «      } t        j                   ‰‰«      «      }t        | d«      t	        |‰dz
  «      fS )Nr   r   )r   ÚfloorÚceilÚmaxÚmin)ÚlowÚhighÚ	beta_fastÚ	beta_slowr   rY   s     €€€€r   Úyarn_find_correction_rangez5YarnRoPE.__init__.<locals>.yarn_find_correction_range˜   sG   ø€ Ü—*‘*Ñ5°iÓ@ÓAˆCÜ—9‘9Ñ5°iÓ@ÓAˆDÜ�s˜A“;¤ D¨$°©(Ó 3Ð3Ð3r   r   c                 óJ   — | dk  ryd|z  t        j                  | «      z  dz   S )Nr   r   gš™™™™™¹?)r   r   )r.   Úmscales     r   Úyarn_get_mscalez*YarnRoPE.__init__.<locals>.yarn_get_mscale�   s(   € Ø˜ŠzØØ˜‘<¤$§(¡(¨5£/Ñ1°CÑ7Ð7r   c                 ó    — | |k(  r|dz  }t        j                  |t         j                  ¬«      | z
  || z
  z  }t        j                  |dd«      S )Ngü©ñÒMbP?r   r   r   )r!   r"   r#   Úclip)Úmin_valÚmax_valr    Úlinear_funcs       r   Úyarn_linear_ramp_maskz0YarnRoPE.__init__.<locals>.yarn_linear_ramp_mask¢   sO   € Ø˜'Ò!Ø˜5Ñ �äŸ9™9 S´·
±
Ô;¸gÑEØ˜'Ñ!ñˆKô —7‘7˜;¨¨1Ó-Ð-r   r   r   r   r   )r   r   )	r   r   re   r!   r"   r#   r%   r   r-   )r'   r   r-   r
   r	   Úscaling_factorr   ra   rb   re   Úmscale_all_dimrc   rf   rl   Ú
freq_extraÚ
freq_interr_   r`   Ú	freq_maskrY   r)   s    `  ` ```          @€r   r   zYarnRoPE.__init__�   sÐ   þ€ ô 	‰ÑÔö	%÷	4ó
	8ò
	.ñ & n°fÓ=ÁØ˜NóA
ñ 
ˆŒð œbŸi™i¨¨4°¼"¿*¹*ÔEÈÑLÑMˆ
Ø# jÑ0ˆ
Ù.Ó0‰	ˆˆTØÑ/°°T¸4À1¹9ÓEÑEˆ	Ø! JÑ.Ø˜Ñ" Z°1°y±=Ñ%AÑAñ
ˆŒð ˆŒ	Ø&ˆÕr   c           	      ó
  — | j                   dk7  r1| j                   |dd | j                  …f   z  |dd | j                  …f<   t        j                  j	                  || j                  | j
                  d d|| j                  ¬«      S )Nr   .r,   )re   r   r!   r/   r0   r-   r%   r1   s      r   r3   zYarnRoPE.__call__¸   s}   € Ø�;‰;˜#ÒØ"&§+¡+°°#°{¸¿¹°{Ð2BÑ0CÑ"CˆAˆc�;�T—Y‘Y�;ÐÑÜ�w‰w�|‰|ØØ�I‰IØ×(Ñ(ØØØØ—+‘+ð ó 
ð 	
r   )	FrP   rQ   r   r4   é    r   r   r   r5   )r6   r7   r8   r   r3   r;   r<   s   @r   rU   rU   €   s+   ø„ ð Ø $ØØØ)-ØØØØõ5'÷n
r   rU   r?   r
   c           	      ó@  — |�&|j                  d«      xs |j                  dd«      }nd}|dv r(|dk(  rd|d   z  nd}t        j                  | |||¬	«      S |d
k(  rt        | ||||¬«      S |dv r-|d   }dD �ci c]  }||v r|||   “Œ }	}t	        d| ||||dœ|	¤ŽS |dk(  rt        | |||d   |d   |d   ¬«      S |dk(  rL|j                  dg «      }
t        |
«      dk(  sJ dt        |
«      › d�«       ‚t        j                  | ||¬«      S t        d|› �«      ‚c c}w )NÚtypeÚ	rope_typeÚdefault)rw   Úlinearrx   r   r   r   )r-   r	   r.   Úllama3)r   r
   r-   r	   r?   )ÚyarnÚdeepseek_yarnztelechat3-yarn)r   ra   rb   re   rn   )r   r
   r-   rm   r	   Úlongroper   r   r   )r   r	   r
   r   r   r   ÚmropeÚmrope_sectioné   z.MRoPE currently only supports 3 sections, got Ú.)r-   r	   zUnsupported RoPE type © )rC   ÚnnÚRoPEr>   rU   r   ÚlenÚ
ValueError)r   r	   r-   r?   r
   rv   r.   rm   ÚkeyÚrope_kwargsr~   s              r   Úinitialize_roperˆ   Æ   sÁ  € ð Ð!Ø"×&Ñ& vÓ.ò 
°.×2DÑ2DØ˜ó3
‰	ð ˆ	àÐ)Ñ)Ø09¸XÒ0E��N 8Ñ,Ò,È3ˆÜ�w‰w�t¨¸4ÀuÔMÐMà	�hÒ	ÜØØ$;Ø#ØØ)ô
ð 	
ð 
ÐAÑ	AØ'¨Ñ1ˆñó

ñ�ð �nÑ$ð � Ñ$Ñ$ðð 	ð 

ô ð 
ØØ$;Ø#Ø)Øñ
ð ñ
ð 	
ð 
�jÒ	 ÜØØØ$;Ø-;Ø2ñ.ð (¨Ñ7Ø& }Ñ5ô	
ð 		
ð 
�gÒ	Ø&×*Ñ*¨?¸BÓ?ˆä�Ó !Ò#ð	Rà;¼CÀÓ<NÐ;OÈqÐQó	RØ#ä�w‰w�t¨¸4Ô@Ð@äÐ1°)°Ð=Ó>Ð>ùòK

s   Á9D)NN)r   Útypingr   r   r   Úmlx.coreÚcorer!   Úmlx.nnr‚   ÚModuler   r>   rU   rS   r9   rˆ   r�   r   r   Ú<module>rŽ      sw   ðó ß (Ñ (å Ý ô<
�2—9‘9ô <
ô~4
�—‘ô 4
ônC
ˆr�y‰yô C
ðT &*Ø-1ñA?ð ˜T‘Nð	A?ð
 & c™]ôA?r   