
    iY;                    D   d Z ddlmZ ddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlmZ dddZdddZ	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd	Z G d
 de	j"                        Z G d de	j"                        Z G d de	j"                        Z G d de	j"                        Zy)z=
MLX transformer blocks for HTDemucs (dense attention only).
    )annotationsN   )
LayerScale)ScaledEmbeddingc                j   |dz  dk7  rt        d      |t        j                  |       j                  ddd      z   }|dz  }t        j                  |      j                  ddd      }||||dz
  z  z  z  }t        j                  t        j
                  |      t        j                  |      gd      S )N   r   dim must be evenr   axis)
ValueErrormxarangereshapeconcatenatecossin)lengthdimshift
max_periodposhalf_dimadimphases           o/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/demucs_mlx/mlx_transformer.pycreate_sin_embeddingr      s    
Qw!|+,,
"))F#++B15
5CaxH99X&&q!R0D:$(Q,"789E>>266%="&&-8rBB    c                6   | dz  dk7  rt        d      | dz  }t        j                  t        j                  d|d      t	        j
                  |      |z   z        }t        j                  d|      j                  dd      }t        j                  d|      j                  dd      }t        j                  ||z        j                  dd      j                  dd|      }t        j                  ||j                  d   ||f      }t        j                  ||z        j                  dd      j                  dd|      }	t        j                  |	|	j                  d   ||f      }	t        j                  ||	gd      j                  d||      }
t        j                  ||z        j                  dd      j                  d|d      }t        j                  ||j                  d   ||f      }t        j                  ||z        j                  dd      j                  d|d      }t        j                  ||j                  d   ||f      }t        j                  ||gd      j                  d||      }t        j                  |
|gd      }|d	d	d	f   S )
z
    Vectorized creation of 2D sinusoidal embeddings.
    Avoids Python loops for interleaving sin/cos, providing a significant speedup.
       r   zd_model must be divisible by 4r           r
   r   r   N)r   r   expr   mathlogr   r   	transposebroadcast_toshaper   stackr   )d_modelheightwidthr   halfdiv_termpos_wpos_hsin_wcos_wpe_wsin_hcos_hpe_hpes                  r   create_2d_sin_embeddingr7      s:   
 {a9::a<DvvbiiT1-$((:2F2M0NNOHIIc5!))"a0EIIc6"**2q1E FF58#$..q!4<<REJEOOEEKKNFE#BCEFF58#$..q!4<<REJEOOEEKKNFE#BCE 88UEN+33BFD FF58#$..q!4<<RKEOOEEKKNFE#BCEFF58#$..q!4<<RKEOOEEKKNFE#BCE 88UEN+33BFD 
t1	-BdAg;r   c	                   |dz  dk7  rt        d      t        j                  |       j                  ddd      j	                  |d      }	|r|	t        j
                  |	dd      z
  }	|rt        j                  j                  | |d|df	      }
t        j                  j                  | || |df	      }t        j                  |      }t        j                  j                  | |d|df	      }|	|
z   |z   t        j                  |      z  }	|dz  }t        j                  |      j                  ddd      }|	|||dz
  z  z  z  }t        j                  t        j                  |      t        j                  |      gd      S )
Nr   r   r	   r
   r   r   T)r   keepdims)r'   )r   r   r   r   repeatmeanrandomuniformr#   r$   r"   r   r   r   )r   r   
batch_sizemean_normalizeaugmentmax_global_shiftmax_local_shift	max_scaler   r   deltadelta_locallog_max_scalelog_lambdasr   r   r   s                    r   create_sin_embedding_caperH   C   so    Qw!|+,,
))F

#
#B1
-
4
4Za
4
HCBGGCa$77		!!#3"35EaQ[]^M_!`ii''o:q) ( + +ii''aQ[]^M_'`U{[(BFF;,??axH99X&&q!R0D:$(Q,"789E>>266%="&&-8rBBr   c                  *     e Zd Zdd fdZddZ xZS )MyGroupNormc                ^    t         |           t        j                  |||d      | _        y )NT)epspytorch_compatible)super__init__nn	GroupNormgn)self
num_groupsnum_channelsrL   	__class__s       r   rO   zMyGroupNorm.__init__b   s%    ,,z<SUYZr   c                $    | j                  |      S N)rR   )rS   xs     r   __call__zMyGroupNorm.__call__f   s    wwqzr   )gh㈵>)rT   intrU   r[   rL   float)rY   mx.arrayreturnr]   __name__
__module____qualname__rO   rZ   __classcell__rV   s   @r   rJ   rJ   a   s    [r   rJ   c                  V     e Zd Z	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fdZdddZ xZS )TransformerEncoderLayerc                   t         |           t        j                  ||d      | _        t        j
                  ||      | _        t        j
                  ||      | _        t        j                  |      | _	        t        j                  |      | _
        || _        || _        |r5t        t        |      |      | _        t        t        |      |      | _        n4t        j"                  |      | _        t        j"                  |      | _        |r|rt        t        |      |      nd | _        |	rt'        ||
d      nt        j(                         | _        |	rt'        ||
d      | _        y t        j(                         | _        y NT)bias)rN   rO   rP   MultiHeadAttentionattnLinearlinear1linear2Dropoutdropout1dropout2
activation
norm_firstrJ   r[   norm1norm2	LayerNormnorm_outr   Identitygamma_1gamma_2rS   r)   nheaddim_feedforwarddropoutrr   rs   
group_normrw   layer_scaleinit_valuesrV   s              r   rO   z TransformerEncoderLayer.__init__k   s    	))'5tD	yy/:yy':

7+

7+$$$S_g>DJ$S_g>DJg.DJg.DJ@JxCM7;^bALz';=RTR]R]R_ALz';=RTR]R]R_r   c                   | j                   r| j                  |      }| j                  ||||      }| j                  |      }|| j	                  |      z   }| j                  | j                  | j                  | j                  | j                  |                              }|| j                  |      z   }| j                  r| j                  |      }|S | j                  ||||      }| j                  |      }| j                  || j	                  |      z         }| j                  | j                  | j                  | j                  |                        }| j                  || j                  |      z         }|S N)mask)rs   rt   rk   rp   ry   rn   rq   rr   rm   ru   rz   rw   )rS   rY   	attn_maskx_normedys        r   rZ   z TransformerEncoderLayer.__call__   s5   ??zz!}H		(HhY	GAa ADLLO#AT]]4??4<<

ST;V+WXYADLLO#A}}MM!$  		!Q		2Aa A

1t||A./AT]]4??4<<?+KLMA

1t||A./Ar   r   FFg-C6?r)   r[   r|   r[   r}   r[   r~   r\   rs   boolr   r[   rw   r   r   r   r   r\   rX   )rY   r]   r_   rd   s   @r   rf   rf   j   s     !!`` ` 	`
 ` ` ` ` ` `> r   rf   c                  V     e Zd Z	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fdZdddZ xZS )CrossTransformerEncoderLayerc                v   t         |           t        j                  ||d      | _        t        j
                  ||      | _        t        j
                  ||      | _        t        j                  |      | _	        t        j                  |      | _
        || _        || _        |rOt        t        |      |      | _        t        t        |      |      | _        t        t        |      |      | _        nNt        j$                  |      | _        t        j$                  |      | _        t        j$                  |      | _        |r|rt        t        |      |      nd | _        |	rt)        ||
d      nt        j*                         | _        |	rt)        ||
d      | _        y t        j*                         | _        y rh   )rN   rO   rP   rj   
cross_attnrl   rm   rn   ro   rp   rq   rr   rs   rJ   r[   rt   ru   norm3rv   rw   r   rx   ry   rz   r{   s              r   rO   z%CrossTransformerEncoderLayer.__init__   s:    	//TJyy/:yy':

7+

7+$$$S_g>DJ$S_g>DJ$S_g>DJg.DJg.DJg.DJ@JxCM7;^bALz';=RTR]R]R_ALz';=RTR]R]R_r   c                   | j                   r| j                  |      }| j                  | j                  |      |||      }|| j	                  | j                  |            z   }| j                  | j                  | j                  | j                  | j                  |                              }|| j                  |      z   }| j                  r| j                  |      }|S | j                  ||||      }| j                  || j	                  | j                  |            z         }| j                  | j                  | j                  | j                  |                        }| j                  || j                  |      z         }|S r   )rs   ru   r   rt   ry   rp   rn   rq   rr   rm   r   rz   rw   )rS   qkr   k_normedattn_outrY   ff_outs           r   rZ   z%CrossTransformerEncoderLayer.__call__   sH   ??zz!}H

1x	 ' CHDLLx!899A\\doodll4::a=.IJKMFDLL((A}}MM!$  q!QY?H

DLLx!899;A\\doodll1o>?AF

1t||F334Ar   r   r   rX   )r   r]   r   r]   r_   rd   s   @r   r   r      s     !!`` ` 	`
 ` ` ` ` ` `B r   r   c                       e Zd Zddddddddddddd	d
dddddg dddf	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fdZd Zd ZddZ xZS )CrossTransformerEncoderr   g      @      Fr!   i  T     @      ?r   )g     @r   gffffff?c                b   t         |           |s|rt        d      ||z  dk7  rt        d      t	        ||z        }|| _        |rdnd| _        || _        || _        || _	        || _
        i | _        |dk(  r|| _        || _        || _        |dk(  rt        ||d      | _        |rd	 nd
 }|	r5t#        j$                  |      | _        t#        j$                  |      | _        ni|
r5t+        t	        |
      |      | _        t+        t	        |
      |      | _        n2t#        j,                         | _        t#        j,                         | _        g | _        g | _        t3        |      D ]  }|dz  | j                  k(  rY| j.                  j5                  t7        |||||||||	             | j0                  j5                  t7        |||||||||	             n| j.                  j5                  t9        |||||||||	             | j0                  j5                  t9        |||||||||	              y )Nz1Sparse attention is not supported in MLX backend.r   z"dim must be divisible by num_headsr   capescaledg?)scalec                ,    t        j                  |       S rX   )rP   gelurY   s    r   <lambda>z2CrossTransformerEncoder.__init__.<locals>.<lambda>  s    
r   c                .    t        j                  | d      S )Nr   )r   maximumr   s    r   r   z2CrossTransformerEncoder.__init__.<locals>.<lambda>  s    BJJqRSDTr   r   )rN   rO   RuntimeErrorr   r[   
num_layersclassic_parityembr   weight_pos_embedsin_random_shift	_pe_cachecape_mean_normalizecape_augmentcape_glob_loc_scaler   position_embeddingsrP   rv   norm_in	norm_in_trJ   rx   layerslayers_trangeappendrf   r   )rS   r   r   hidden_scale	num_headsr   cross_firstr~   max_positionsr   norm_in_groupr   rs   rw   r   r   r   r   r   r   r   r   sparse_self_attnsparse_cross_attnkwargs
hidden_dimrr   idxrV   s                               r   rO   z CrossTransformerEncoder.__init__   s6   6 	0RSS?aABB|+,
$#.aA$ 0 0 &=':D$ ,D':D$(?'6}cQT'UD$/3*:T
<<,DL\\#.DN&s='93?DL(]);SADN;;=DL[[]DN$CQw$---""+!"""" #
 $$+!"""" #
 ""0!"""" #
 $$0!"""" #
U %r   c                "   | j                   dk(  r;t        j                  | j                  dz         }t	        |||| j
                        S | j                   dk(  r| j                  rYt        |||| j                  | j                  | j                  d   | j                  d   | j                  d   | j
                  	      S t        |||| j                  d| j
                  	      S t        j                  |      }| j                  |      d d d f   S )
Nr   r   )r   r   r   r   r   )r?   r@   rA   rB   rC   r   F)r?   r@   r   )r   r<   	randranger   r   r   trainingrH   r   r   r   r   r   r   )rS   TBCr   r   s         r   _get_pos_embeddingz*CrossTransformerEncoder._get_pos_embeddingJ  s    88u$$T%:%:Q%>?E'1EdooVV88v}}0#'#;#; --%)%=%=a%@$($<$<Q$?"66q9#
 
 -#77??  iil'',QW55r   c                    |||| j                   f}|| j                  v r| j                  |   S t        |||| j                         }|| j                  |<   |S )zGet cached or new 2D embedding.)r   r   r7   )rS   r   FrT1keyr6   s         r   _get_2d_embeddingz)CrossTransformerEncoder._get_2d_embeddingg  sX    "b$//*$.. >>#&& %QB@ s	r   c                   |j                   \  }}}}| j                  |||      }|j                  d|||      }t        j                  |||||f      }|j                  dddd      j                  |||z  |      }|j                  dddd      j                  |||z  |      }| j                  |      }|| j                  |z  z   }|j                   \  }}}|j                  ddd      }| j                  |||      }	|	j                  ddd      }	| j                  |      }|| j                  |	z  z   }t        | j                        D ]m  }
|
dz  | j                  k(  r+ | j                  |
   |      } | j                  |
   |      }@|} | j                  |
   ||      } | j                  |
   ||      }o |j                  ||||      j                  dddd      }|j                  ddd      }||fS )Nr   r      r   )r'   r   r   r   r&   r%   r   r   r   r   r   r   r   r   r   )rS   rY   xtr   r   r   r   
pos_emb_2dT2pos_embr   old_xs               r   rZ   z CrossTransformerEncoder.__call__r  s   ww1b" ++Ar26
  ''1b"5
 __Z!QB@
  ))!Q15==ab!L
KK1a#++ArBw:LLO%%
22881b\\!Q"))"a3##Aq!,^^B$'''11)CQw$---$DKK$Q''T]]3'+$DKK$Q+'T]]3'E2 * IIaR#--aAq9\\!Q""ur   ).r   r[   r   strr   r\   r   r[   r   r[   r   r   r~   r\   r   r[   r   r   r   r   r   r[   rs   r   rw   r   r   r\   r   r\   r   r   r   r   r   r[   r   r   r   r   r   listr   r   r   r   )rY   r]   r   r]   )r`   ra   rb   rO   r   r   rZ   rc   rd   s   @r   r   r      sX    !!!# #"%! !$(!$6!&"'1ss s 	s
 s s s s s s s s s s s   !s" #s$ %s& 's( ")s* +s, "-s. /s0  1sj6:	%r   r   )r   r   )r   r[   r   r[   r   r[   r   r\   )r   )r)   r[   r*   r[   r+   r[   r   r\   )r!   r!   r   r   )r   r[   r   r[   r>   r[   r?   r   r@   r   rA   r\   rB   r\   rC   r\   r   r\   )__doc__
__future__r   r#   r<   mlx.corecorer   mlx.nnrP   
mlx_demucsr   mlx_hdemucsr   r   r7   rH   ModulerJ   rf   r   r    r   r   <module>r      s    #     " (C&^ " CC	C C 	C
 C C C C C<")) 0bii 0f4299 4nCbii Cr   