
     i2                        d dl mZ d dlmZ d dlZd dlmZ d dlmc mZ	 d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZ 	 d d
lmZ d dlmZ dZ	 d dlmZ dZ  G d de      Z!y# e$ r dZY w xY w# e$ r dZ Y "w xY w)    )	lru_cache)OptionalN)make_enc_dec)pairwise)Model)Task)
merge_dict)conv1d_num_framesconv1d_receptive_field_centerconv1d_receptive_field_size)DPRNN)
pad_x_to_yTF)	AutoModelc                   @    e Zd ZdZdddddZdddZd	d
d
dddddZddiZ	 	 	 	 	 	 	 	 	 	 	 d)dede	e   de	e   dede
de
de	e   de
dededef fdZede
fd       Zd  Zed!e
de
fd"       Zd*d#e
de
fd$Zd+d%e
de
fd&Zd'ej.                  dej.                  fd(Z xZS ),	ToTaToNetu  ToTaToNet joint speaker diarization and speech separation model

                        /--------------\
    Conv1D Encoder --------+--- DPRNN --X------- Conv1D Decoder
    WavLM -- upsampling --/                 \--- Avg pool -- Linear -- Classifier


    Parameters
    ----------
    sample_rate : int, optional
        Audio sample rate. Defaults to 16kHz (16000).
    num_channels : int, optional
        Number of channels. Defaults to mono (1).
    sincnet : dict, optional
        Keyword arugments passed to the SincNet block.
        Defaults to {"stride": 1}.
    linear : dict, optional
        Keyword arugments used to initialize linear layers
        See ToTaToNet.LINEAR_DEFAULTS for default values.
    diar : dict, optional
        Keyword arguments used to initalize the average pooling in the diarization branch.
        See ToTaToNet.DIAR_DEFAULTS for default values.
    encoder_decoder : dict, optional
        Keyword arguments used to initalize the encoder and decoder.
        See ToTaToNet.ENCODER_DECODER_DEFAULTS for default values.
    dprnn : dict, optional
        Keyword arguments used to initalize the DPRNN model.
        See ToTaToNet.DPRNN_DEFAULTS for default values.
    sample_rate : int, optional
        Audio sample rate. Defaults to 16000.
    num_channels : int, optional
        Number of channels. Defaults to 1.
    task : Task, optional
        Task to perform. Defaults to None.
    n_sources : int, optional
        Number of separated sources. Defaults to 3.
    use_wavlm : bool, optional
        Whether to use the WavLM large model for feature extraction. Defaults to True.
    wavlm_frozen : bool, optional
        Whether to freeze the WavLM model. Defaults to False.
    gradient_clip_val : float, optional
        Gradient clipping value. Required when fine-tuning the WavLM model and thus using two different optimizers.
        Defaults to 5.0.

    References
    ----------
    Joonas Kalda, Clément Pagés, Ricard Marxer, Tanel Alumäe, and Hervé Bredin.
    "PixIT: Joint Training of Speaker Diarization and Speech Separation
    from Real-world Multi-speaker Recordings"
    Odyssey 2024. https://arxiv.org/abs/2403.02288
    free    @      )fb_namekernel_size	n_filtersstride   )hidden_size
num_layers      d   gLNreluLSTM)	n_repeatsbn_chanhid_size
chunk_size	norm_typemask_actrnn_typeframes_per_second}   encoder_decoderlineardiardprnnsample_ratenum_channelstask	n_sources	use_wavlmwavlm_frozengradient_clip_valc           
         t         st        d      t        st        d      t        |   |||       t        | j                  |      }t        | j                  |      }t        | j                  |      }t        | j                  |      }|	| _
        | j                  ddddd       || _        |d	   d
k(  r|d   }n+|d	   dk(  rt        d|d   dz  dz   z        }nt        d      t        dd|i| j                   j"                  \  | _        | _        | j                  rt)        j*                  d      | _        | j,                  j/                         D ]
  }|
 |_         d}| j,                  j2                  j4                  D ]C  }t7        |j8                  t:        j<                        s(||j8                  j>                  d   z  }E t        ||d   z        | _         tC        |d   | j,                  jD                  jF                  jH                  z   f|d   |d| j                   jJ                  | _&        n.tC        |d   f|d   |d| j                   jJ                  | _&        t        ||d   z  |d   z        | _'        t;        jP                  | jN                  | jN                        | _)        |}|d   dkD  rt;        jT                  tW        |g| j                   jX                  d   g| j                   jX                  d   z  z         D cg c]  \  }}t;        jZ                  ||       c}}      | _,        || _.        |
| _/        y c c}}w )Nzw'asteroid' must be installed to use ToTaToNet separation. `pip install pyannote-audio[separation]` should do the trick.z{'transformers' must be installed to use ToTaToNet separation. `pip install pyannote-audio[separation]` should do the trick.)r0   r1   r2   r,   r-   r/   r.   r5   r   r   r   stftr      zFilterbank type not recognized.r0   zmicrosoft/wavlm-larger   r   )out_chann_srcr*   )r   r   r    )0ASTEROID_IS_AVAILABLEImportErrorTRANSFORMERS_IS_AVAILABLEsuper__init__r	   LINEAR_DEFAULTSDPRNN_DEFAULTSENCODER_DECODER_DEFAULTSDIAR_DEFAULTSr4   save_hyperparametersr3   int
ValueErrorr   hparamsr,   encoderdecoderr   from_pretrainedwavlm
parametersrequires_gradfeature_extractorconv_layers
isinstanceconvnnConv1dr   wavlm_scalingr   feature_projection
projectionout_featuresr/   maskerdiarization_scaling	AvgPool1daverage_pool
ModuleListr   r-   Linearr6   automatic_optimization)selfr,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   n_feats_outparamdownsampling_factor
conv_layerlinaer_input_featuresin_featuresrY   	__class__s                      /Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/pyannote/audio/models/separation/ToTaToNet.pyrA   zToTaToNet.__init__   sf    %P 
 )P 
 	[|RVWD00&94..6$T%B%BOT$,,d3"!!x&.	
 #9%/)+6KY'61a?;#?!#Ca#GHIK>??%1 &
#&
'+||'C'C&
"dl >>"223JKDJ..0*6&6# 1"#"jj::FF
jooryy9':??+A+A!+DD' G "%%8?8;T%T!UD,**//::GGH )5	
 ,,$$DK  ,(5 ,,$$	DK $'$233oh6OO$
  LL$$T-E-E
 !,,!#-- 6>1  <<..}=>,,--l;<<6	61\ IIk<86	DK "3&2#	s    M
returnc                      y)zDimension of outputr9   r<   ra   s    ri   	dimensionzToTaToNet.dimension   s         c                     | j                   j                  d   dkD  r&t        j                  d| j                        | _        n%t        j                  d| j                        | _        | j                         | _        y )Nr   r   r   r9   )rI   r-   rT   r_   rm   
classifierdefault_activation
activationrl   s    ri   buildzToTaToNet.build   sU    <<|,q0 iiDNN;DO ii4>>:DO113rn   num_samplesc                     | j                   | j                  j                  d   z  }| j                   | j                  j                  d   z  }t        |||      S )zCompute number of output frames

        Parameters
        ----------
        num_samples : int
            Number of input samples.

        Returns
        -------
        num_frames : int
            Number of output frames.
        r   r   r   r   )r[   rI   r,   r
   )ra   rt   equivalent_strideequivalent_kernel_sizes       ri   
num_frameszToTaToNet.num_frames   sb      $$t||'C'CH'MM 	 $$t||'C'CM'RR 	 !%;DU
 	
rn   ry   c                     | j                   | j                  j                  d   z  }| j                   | j                  j                  d   z  }t        |||      S )a
  Compute size of receptive field

        Parameters
        ----------
        num_frames : int, optional
            Number of frames in the output signal

        Returns
        -------
        receptive_field_size : int
            Receptive field size.
        r   r   rv   )r[   rI   r,   r   )ra   ry   rw   rx   s       ri   receptive_field_sizezToTaToNet.receptive_field_size  sb     $$t||'C'CH'MM 	 $$t||'C'CM'RR 	 +$:CT
 	
rn   framec                     | j                   | j                  j                  d   z  }| j                   | j                  j                  d   z  }t        |||      S )zCompute center of receptive field

        Parameters
        ----------
        frame : int, optional
            Frame index

        Returns
        -------
        receptive_field_center : int
            Index of receptive field center.
        r   r   rv   )r[   rI   r,   r   )ra   r|   rw   rx   s       ri   receptive_field_centerz ToTaToNet.receptive_field_center$  sb     $$t||'C'CH'MM 	 $$t||'C'CM'RR 	 -5>O
 	
rn   	waveformsc                 V   |j                   d   }| j                  |      }| j                  r| j                  |j	                  d            j
                  }|j                  dd      }|j                  | j                  d      }t        ||      }t        j                  ||fd      }| j                  |      }n| j                  |      }||j                  d      z  }| j                  |      }t        ||      }|j                  dd      }t        j                  |dd      }| j!                  |      }|j                  dd      }| j"                  j$                  d   dkD  r,| j$                  D ]  }	t'        j(                   |	|            } | j"                  j$                  d   dk(  r$|dz  j+                  d      j                  d      }| j-                  |      }|j/                  || j0                  d      }|j                  dd      } | j2                  d   |      |fS )zPass forward

        Parameters
        ----------
        waveforms : (batch, channel, sample)

        Returns
        -------
        scores : (batch, frame, classes)
        sources : (batch, sample, n_sources)
        r   r9   r   )dim)	start_dimend_dimr   )shaperJ   r4   rM   squeezelast_hidden_state	transposerepeat_interleaverV   r   torchcatrZ   	unsqueezerK   flattenr]   rI   r-   F
leaky_relusumrp   reshaper3   rr   )
ra   r   bsztf_rep	wavlm_repmasksmasked_tf_repdecoded_sourcesoutputsr-   s
             ri   forwardzToTaToNet.forward=  s    ooa i(>>

9#4#4Q#78JJI!++Aq1I!33D4F4FB3OI"9f5I		69"51=IKK	*EKK'E 0 0 33,,}5$_i@)33Aq9--AF##G,##Aq)<<|,q0++,,vg7 &<<|,1z&&1&-77;G//'*//#t~~r:##Aq)!tq!'*O;;rn   )NNNNi>  r9   N   TFg      @)r9   )r   )__name__
__module____qualname____doc__rD   rB   rC   rE   dictr   rG   r   boolfloatrA   propertyrm   rs   r   ry   r{   r~   r   Tensorr   __classcell__)rh   s   @ri   r   r   <   s   2j 	  '):ON )#.M !%!%# #"#&`3`3 `3 tn	`3
 `3 `3 `3 tn`3 `3 `3 `3 !`3D 3  4 
c 
c 
 
2
s 
3 
2
C 
 
2*< *<%,, *<rn   r   )"	functoolsr   typingr   r   torch.nnrT   torch.nn.functional
functionalr   asteroid_filterbanksr   pyannote.core.utils.generatorsr   pyannote.audio.core.modelr   pyannote.audio.core.taskr   pyannote.audio.utils.paramsr	   $pyannote.audio.utils.receptive_fieldr
   r   r   asteroid.masknnr   asteroid.utils.torch_utilsr   r=   r>   transformersr   r?   r   r<   rn   ri   <module>r      s   2        - 3 + ) 2 "%5 
&& $
k< k<  "!"  & %&s$   	A, A9 ,A65A69BB