
     is                     B   d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	m
Z
mZmZmZ ddlZddlZddlmZ ddlmZmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZm Z m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z) ddl*m+Z+ ddl,m-Z-m.Z.m/Z/ ddl0m1Z1 dde2fdZ3 G d de e      Z4y)zSpeech separation pipelines    N)Path)CallableOptionalTextTupleUnion)	rearrange)Audio	InferenceModelPipeline)	AudioFile)
Clustering)PretrainedSpeakerEmbedding)PipelineModelSpeakerDiarizationMixin	get_model)set_num_speakers)binarize)
AnnotationSlidingWindowSlidingWindowFeature)GreedyDiarizationErrorRate)Categorical	ParamDictUniform)binary_dilation
batch_sizec                 J    t        |       g|z  }t        j                  |d|iS )zBatchify iterable	fillvalue)iter	itertoolszip_longest)iterabler   r    argss       /Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/pyannote/audio/pipelines/speech_separation.pybatchifyr'   6   s+     Nj(D  $<)<<    c                       e Zd ZdZ	 	 	 	 	 	 	 	 	 	 d%dedededededed	ed
e	e
   deedf   deeedf   f fdZedefd       Zej"                  defd       Zd Zd Zed        Z	 d&deeef   fdZ	 	 d'dedede	e   fdZdedej6                  dedefdZ	 	 	 	 	 d(dede	e   d e	e   d!e	e   d"ede	e   defd#Zde fd$Z! xZ"S ))SpeechSeparationu>  Speech separation pipeline

    Parameters
    ----------
    segmentation : Model, str, or dict, optional
        Pretrained segmentation model and separation model.
        See pyannote.audio.pipelines.utils.get_model for supported format.
    segmentation_step: float, optional
        The segmentation model is applied on a window sliding over the whole audio file.
        `segmentation_step` controls the step of this window, provided as a ratio of its
        duration. Defaults to 0.1 (i.e. 90% overlap between two consecuive windows).
    embedding : Model, str, or dict, optional
        Pretrained embedding model. Defaults to "speechbrain/spkrec-ecapa-voxceleb@5c0be38".
        See pyannote.audio.pipelines.utils.get_model for supported format.
    embedding_exclude_overlap : bool, optional
        Exclude overlapping speech regions when extracting embeddings.
        Defaults (False) to use the whole speech.
    clustering : str, optional
        Clustering algorithm. See pyannote.audio.pipelines.clustering.Clustering
        for available options. Defaults to "AgglomerativeClustering".
    segmentation_batch_size : int, optional
        Batch size used for speaker segmentation. Defaults to 1.
    embedding_batch_size : int, optional
        Batch size used for speaker embedding. Defaults to 1.
    der_variant : dict, optional
        Optimize for a variant of diarization error rate.
        Defaults to {"collar": 0.0, "skip_overlap": False}. This is used in `get_metric`
        when instantiating the metric: GreedyDiarizationErrorRate(**der_variant).
    token : str or bool, optional
        Huggingface token to be used for downloading from Huggingface hub.
    cache_dir: Path or str, optional
        Path to the folder where files downloaded from Huggingface hub are stored.

    Usage
    -----
    >>> pipeline = SpeechSeparation()
    >>> diarization, separation = pipeline("/path/to/audio.wav")
    >>> diarization, separation = pipeline("/path/to/audio.wav", num_speakers=4)
    >>> diarization, separation = pipeline("/path/to/audio.wav", min_speakers=2, max_speakers=10)

    Hyper-parameters
    ----------------
    segmentation.min_duration_off : float
        Fill intra-speaker gaps shorter than that many seconds.
    segmentation.threshold : float
        Mark speaker has active when their probability is higher than this.
    clustering.method : {'centroid', 'average', ...}
        Linkage used for agglomerative clustering
    clustering.min_cluster_size : int
        Minium cluster size.
    clustering.threshold : float
        Clustering threshold used to stop merging clusters.
    separation.leakage_removal : bool
        Zero-out sources when speaker is inactive.
    separation.asr_collar
        When using leakage removal, keep that many seconds before and after each speaker turn

    References
    ----------
    Joonas Kalda, Clément Pagés, Ricard Marxer, Tanel Alumäe, and Hervé Bredin.
    "PixIT: Joint Training of Speaker Diarization and Speech Separation
    from Real-world Multi-speaker Recordings"
    Odyssey 2024. https://arxiv.org/abs/2403.02288
    Nsegmentationsegmentation_step	embeddingembedding_exclude_overlap
clusteringembedding_batch_sizesegmentation_batch_sizeder_varianttoken	cache_dirc           	         t         |           || _        t        ||	|
      }|| _        || _        || _        || _        || _        |xs ddd| _	        |j                  d   j                  }t        ||| j                  |z  d|      | _        | j                  j                  j                  d   j                  rt!        t#        dd      	      | _        n&t!        t#        d
d      t#        dd            | _        | j                  dk(  rd}nYt'        | j
                  |	|
      | _        t+        | j(                  j,                  d      | _        | j(                  j0                  }	 t2        |   }|j?                  |      | _         t!        tC        ddg      t#        dd            | _"        y # t4        $ r6 t7        ddj9                  t;        t2        j<                               d      w xY w)N)r3   r4           F)collarskip_overlapr   T)durationstepskip_aggregationr         ?)min_duration_off皙?g?)	thresholdr=   OracleClusteringnot_applicabledownmix)sample_ratemonozclustering must be one of [, ])metric)leakage_removal
asr_collar)#super__init__segmentation_modelr   r,   r-   r0   r.   
klusteringr2   specificationsr9   r   _segmentationmodelpowersetr   r   r+   r   
_embeddingr
   rC   _audiorG   r   KeyError
ValueErrorjoinlist__members__valuer/   r   
separation)selfr+   r,   r-   r.   r/   r0   r1   r2   r3   r4   rP   segmentation_durationrG   
Klustering	__class__s                  r&   rK   zSpeechSeparation.__init__   s    	". UiP!2"$8!)B&$&PS%*P % 4 4Q 7 @ @&*''*??!.
 ##2215>> )!(c!2!D
 !*!#s+!(c!2!D
 ??00%F 9eyDO  DOO,G,GiXDK__++F	#J/J
 %**&*9#'u6sC(
  	-diiZ=S=S8T.U-VVWX 	s   ,	F4 4?G3returnc                 .    | j                   j                  S NrO   r   r[   s    r&   r1   z(SpeechSeparation.segmentation_batch_size   s    !!,,,r(   r   c                 &    || j                   _        y ra   rb   )r[   r   s     r&   r1   z(SpeechSeparation.segmentation_batch_size   s    (2%r(   c                     t               ra   )NotImplementedErrorrc   s    r&   default_parametersz#SpeechSeparation.default_parameters   s    !##r(   c              #   ,   K   d}	 d|d |dz  }w)Nr   SPEAKER_02d    )r[   speakers     r&   classeszSpeechSeparation.classes   s+     WSM**qLG s   c                      y)Nztraining_cache/segmentationrl   rc   s    r&   CACHED_SEGMENTATIONz$SpeechSeparation.CACHED_SEGMENTATION   s    ,r(   c                    |t        j                  |dd      }| j                  rO| j                  |v r|| j                     \  }}||fS | j	                  ||      \  }}||f|| j                  <   ||fS | j	                  ||      \  }}||fS )aI  Apply segmentation model

        Parameter
        ---------
        file : AudioFile
        hook : Optional[Callable]

        Returns
        -------
        segmentations : (num_chunks, num_frames, num_speakers) SlidingWindowFeature
        separations : (num_chunks, num_samples, num_speakers) SlidingWindowFeature
        Nr+   hook)	functoolspartialtrainingrp   rO   )r[   filers   segmentationsseparationss        r&   get_segmentationsz"SpeechSeparation.get_segmentations   s      $$T>4@D==''4/-1$2J2J-K*{ k)) .2-?-?4-?-P*{2?1MT--. k)) *.););Dt);)L&M;k))r(   binary_segmentationsexclude_overlaprs   c                      j                   rlj                  dt                     }d|v rN j                  j                  j
                  d   j                  s|d    j                  j                  k(  r|d   S j                  j                  }j                  j                  \  }}}	|r j                  j                  }
| j                  j                  z  }t!        j"                  ||
z  |z        dt%        j&                  j                  dd      dk  z  }t)        j                  |z  j                        n"d	t)        j                  j                         fd
}t+         |        j,                  d      }t!        j"                  ||	z   j,                  z        }g }| |dd|d       t/        |d      D ]x  \  }}t1        t3        d |       \  }}t5        j6                  |      }t5        j6                  |      } j                  ||      }|j9                  |       |m |d|||       z t%        j6                  |      }t;        |d|      } j                   rR j                  j                  j
                  d   j                  r	d|id<   |S  j                  j                  |dd<   |S )a  Extract embeddings for each (chunk, speaker) pair

        Parameters
        ----------
        file : AudioFile
        binary_segmentations : (num_chunks, num_frames, num_speakers) SlidingWindowFeature
            Binarized segmentation.
        exclude_overlap : bool, optional
            Exclude overlapping speech regions when extracting embeddings.
            In case non-overlapping speech is too short, use the whole speech.
        hook: Optional[Callable]
            Called during embeddings after every batch to report the progress

        Returns
        -------
        embeddings : (num_chunks, num_speakers, dimension) array
        ztraining_cache/embeddings
embeddingsr   segmentation.thresholdr<      Taxiskeepdimsc               3     K   t        	      D ]  \  \  } }\  }}j                  j                  
| d      \  }}t        j                  |d      j                  t        j                        }t        j                  |d      j                  t        j                        }t        |j                  |j                        D ]A  \  }}t        j                  |      kD  r|}n|}|d    t        j                  |      d    f C  y w)Npad)moder6   )nan)ziprS   cropnp
nan_to_numastypefloat32Tsumtorch
from_numpy)chunkmasks_clean_maskswaveformspeaker_activation_with_context
clean_mask	used_maskr{   clean_segmentationsrw   min_num_framesr[   s           r&   iter_waveform_and_maskz?SpeechSeparation.get_embeddings.<locals>.iter_waveform_and_mask<  s     47$&950 0K #kk.. / ! e5<<RZZH mmKSAHHTCFGG[]]D?3Z
 vvj)N:$.	$C	"4.%*:*:9*Ed*KKKD#5s   D	D)NN)r   r    N)total	completedrk   c                     | d   d uS )Nr   rl   )bs    r&   <lambda>z1SpeechSeparation.get_embeddings.<locals>.<lambda>j  s    QqT5Er(   )r   z(c s) d -> c s d)c)r   r~   )rv   getdictrO   rP   rN   rQ   r+   r?   sliding_windowr9   datashaperR   min_num_samplesrC   mathceilr   r   r   r'   r0   	enumerater   filterr   vstackappendr	   )r[   rw   r{   r|   rs   cacher9   
num_chunks
num_framesnum_speakersr   num_samplesclean_framesr   batchesbatch_countembedding_batchesibatch	waveformsr   waveform_batch
mask_batchembedding_batchr~   r   r   s   ```                      @@r&   get_embeddingszSpeechSeparation.get_embeddings   s   : == HH8$&AE%""((77:CC23t7H7H7R7RR\**'66??/C/H/H/N/N,
J #oo==O #T__%@%@@K!YYzO'Ck'QRN +00q4H1LL #7$))L8$33#  N"6$))+?+N+N#	L 	L@ "$00"
 ii
\ 9D<U<U UVt;!D!'1-HAu"F+Eu$MNIu"\\)4N e,J +///j +: +O
 $$_5\?+QRS# .& II&7802D
S
 ==!!''66q9BB *501 	 /3.?.?.I.I",501
 r(   rx   hard_clusterscountc                    |j                   j                  \  }}}t        j                  |      dz   }t        j                  t        j
                  |||f      z  }t        t        ||            D ]T  \  }	\  }
\  }}t        j                  |
      D ]1  }|dk(  r	t        j                  |dd|
|k(  f   d      ||	dd|f<   3 V t        ||j                        }|S )a;  Build final discrete diarization out of clustered segmentation

        Parameters
        ----------
        segmentations : (num_chunks, num_frames, num_speakers) SlidingWindowFeature
            Raw speaker segmentation.
        hard_clusters : (num_chunks, num_speakers) array
            Output of clustering step.
        count : (total_num_frames, 1) SlidingWindowFeature
            Instantaneous number of active speakers.

        Returns
        -------
        discrete_diarization : SlidingWindowFeature
            Discrete (0s and 1s) diarization.
        rk   Nr   )r   r   r   maxr   zerosr   r   uniquer   r   )r[   rx   r   r   r   r   local_num_speakersnum_clustersclustered_segmentationsr   clusterr   r+   ks                 r&   reconstructzSpeechSeparation.reconstruct  s    . 6C5G5G5M5M2
J 2vvm,q0"$&&288\2,
 #
 4=}-4
/A/.%
 YYw'7 4666 GqL14'1a0 (4
 #7#]%A%A#
 '&r(   rw   r   min_speakersmax_speakersreturn_embeddingsc                    | j                  ||      }t        |||      \  }}}| j                  ||      \  }} |d|        |d|       | j                  j                  j
                  d   j                  r|}	n"t        || j                  j                  d      }	| j                  |	| j                  j                  j                  d	      }
 |d
|
       t        j                  |
j                        dk(  rCt        |d         }|r.|dt        j                   d| j"                  j$                  f      fS |dfS | j&                  dk(  r|sd}n(| j)                  ||	| j*                  |      } |d|       | j-                  ||	||||| j                  j                  j                        \  }}}t        j.                  |      dz   }||k  s||kD  r5t1        j2                  t5        j6                  d| d| d| d| d	             t        j8                  |
j                  |      j;                  t        j<                        |
_        t        j>                  |	j                  d      dk(  }d||<   | jA                  |||
      }| jC                  ||
      }t        j>                  |d      dkD  }|j                  dd|f   |_        |j                  jD                  \  }} |d|       | jA                  |||
      }|jF                  jH                  |j                  jD                  d   z  }tK        |d|z        }tM        jN                  ||ddd      }|j                  jD                  \  }}t        jP                  |j                  d dt/        d||z
        ff      |_        |j                  dd|f   |_        | jR                  jT                  r6tW        | j                  j                  jY                  t[        | jR                  j\                  | j^                  j`                  z                    }|dkD  rt        jb                  |j                        }te        |      D ]_  }|j                  jf                  |   }|dk7  }ti        |dgd|z  z        }|j;                  t        j<                        |jf                  |<   a tk        |jF                        }|j                  |jm                  |      j                  z  |_        |j                  t        j.                  t        jn                  |j                        dd!      d"z   z  |_        | jq                  |d| j                  jr                  #      }|d   |_:        d$|v rN|d$   rI| jw                  |d$   |d%      \  }} |jy                         D !ci c]  }!|!| j{                  |!|!       } }!n;t}        |jy                         | j                               D "#ci c]  \  }"}#|"|#
 } }"}#|j                  | &      }| j                         D $"ci c]  \  }$}"|"|$
 }%}$}"|j                  dd|jy                         D "cg c]  }"|%|"   	 c}"f   |_        |s||fS |||dfS t        |jy                               |jD                  d   kD  rAt        jP                  |dt        |jy                               |jD                  d   z
  fd f      }||jy                         D "cg c]  }"|%|"   	 c}"   }|||fS c c}!w c c}#}"w c c}"}$w c c}"w c c}"w )'a  Apply speaker diarization

        Parameters
        ----------
        file : AudioFile
            Processed file.
        num_speakers : int, optional
            Number of speakers, when known.
        min_speakers : int, optional
            Minimum number of speakers. Has no effect when `num_speakers` is provided.
        max_speakers : int, optional
            Maximum number of speakers. Has no effect when `num_speakers` is provided.
        return_embeddings : bool, optional
            Return representative speaker embeddings.
        hook : callable, optional
            Callback called after each major steps of the pipeline as follows:
                hook(step_name,      # human-readable name of current step
                     step_artefact,  # artifact generated by current step
                     file=file)      # file being processed
            Time-consuming steps call `hook` multiple times with the same `step_name`
            and additional `completed` and `total` keyword arguments usable to track
            progress of current step.

        Returns
        -------
        diarization : Annotation
            Speaker diarization
        sources : SlidingWindowFeature
            Separated sources
        embeddings : np.array, optional
            Representative speaker embeddings such that `embeddings[i]` is the
            speaker embedding for i-th speaker in diarization.labels().
            Only returned when `return_embeddings` is True.
        rr   )r   r   r   r+   ry   r   F)onsetinitial_state)r6   r6   )warm_upspeaker_countingr6   uri)r   Nr@   )r|   rs   r~   )r~   rx   r   min_clustersmax_clustersrw   framesrk   z2
                The detected number of speakers (z/) is outside
                the given bounds [rE   zS]. This can happen if the
                given audio file is too short to contain zh or more speakers.
                Try to lower the desired minimal number of speakers.
                r   r   discrete_diarizationr   )r:   r9   T)r   hammingmissingskip_average)r   r   r   g:0yE>)min_duration_onr=   
annotation)return_mapping)mapping)C
setup_hookr   rz   rO   rP   rN   rQ   r   r+   r?   speaker_countreceptive_fieldr   nanmaxr   r   r   rR   	dimensionrM   r   r.   r/   r   warningswarntextwrapdedentminimumr   int8r   r   to_diarizationr   r   r9   r   r   	aggregater   rZ   rH   intr   roundrI   rS   rC   
zeros_likeranger   r   r   alignabsto_annotationr=   r   optimal_mappinglabelsr   r   rn   rename_labelsitemslen)&r[   rw   r   r   r   r   rs   rx   ry   binarized_segmentationsr   diarizationr~   r   r   	centroidsnum_different_speakersinactive_speakersr   active_speakersr   clustered_separationsframe_durationr   sourcesnum_sourcesasr_collar_framesdilated_speaker_activationsr   speaker_activation
non_silentdilated_non_silentr   keylabelexpected_labelindexinverse_mappings&                                         r&   applyzSpeechSeparation.apply  se   Z t$/3C%%%4
0lL &*%;%;Dt%;%L"{^]+]K( ##2215>>&3#<D''11#=# ""#$$44 # 

 	' 99UZZ C'$e5K "D"((At7P7P3Q*RRR$$??009JJ,,' $ > >	 - J z* '+oo!1%%%%%++;; '6 '
#q) "$!6!: #\1%4MM22H1I J##/.<. A::F H	 ZZ

L9@@I

 FF#:#?#?aHAM ,.'(#// 

  $223GO&&!5A>B$8$=$=a>P$Q!#7#<#<#B#B 
L#%9: $ 0 0mU S$33<<{?O?O?U?UVW?XXNQ=OP%%!
 !++;
 vvLL6As1l[.H'I#JK

 ||A$67 ??** #""((33$//44t{{7N7NNO!
 !1$.0mm<P<U<U.V+|,A)=)B)B)D)DQ)G&!3q!8J)8"TFa2C.C$D*& 8J7P7P8/11!4 - +?+-A-P-P+' :@@INNN L ||FF266',,'a$?$F

 (( !..?? ) 

 u+
 4D$6 --\"K . JAw >I=O=O=QR=QcsGKKS11=QGR .11C1C1Et||~-V-V)E> ~%-V  
 "///@ =DMMOLOLE55%<OL||K4F4F4HI4H5&4HII
 !'' -- {!!#$yq'99QK$6$6$8 9IOOA<N NOQWXI 1<1C1C1EF1E_U#1EF
	 GY..] S
 MI. Gs   \'\, \20\8\=c                 ,    t        di | j                  S )Nrl   )r   r2   rc   s    r&   
get_metriczSpeechSeparation.get_metric  s    )=D,<,<==r(   )
zpyannote/separation-ami-1.0r>   zJspeechbrain/spkrec-ecapa-voxceleb@5c0be3875fda05e81f3c004ed8c7c06be308de1eFAgglomerativeClusteringrk   rk   NNNra   )FN)NNNFN)#__name__
__module____qualname____doc__r   floatboolstrr   r   r   r   r   r   rK   propertyr1   setterrg   rn   rp   r   r   rz   r   r   r   ndarrayr   r   r   r  r   r  __classcell__)r^   s   @r&   r*   r*   =   s   ?F 'D#&#o*/3$%'(&*#'-1E
#E
 !E
 !	E

 $(E
 E
 "E
 "%E
 d^E
 T4Z E
 tT)*E
N - - - ##3# 3 $3$ - - *	#%99	:*D !&#'T 3T 	T
 x Tl/'+/' zz/' $	/'
 
/'h '+&*&*"'#'Z/Z/ smZ/ sm	Z/
 smZ/  Z/ x Z/ 
Z/x>6 >r(   r*   )    N)5r  rt   r"   r   r   r   pathlibr   typingr   r   r   r   r   numpyr   r   einopsr	   pyannote.audior
   r   r   r   pyannote.audio.core.ior   #pyannote.audio.pipelines.clusteringr   -pyannote.audio.pipelines.speaker_verificationr   pyannote.audio.pipelines.utilsr   r   r   *pyannote.audio.pipelines.utils.diarizationr   pyannote.audio.utils.signalr   pyannote.corer   r   r   pyannote.metrics.diarizationr   pyannote.pipeline.parameterr   r   r   scipy.ndimager   r   r'   r*   rl   r(   r&   <module>r.     s|   0 "       9 9    < < , : T 
 H 0 I I C G G )=3 =`
>. `
>r(   