
      i^                        d Z ddlZddlZddlZddlZddlZddlZddl	m	Z	 ddl
mZ ddl
mZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddl!m"Z" ddl!m#Z# ddl!m$Z$ ddl!m%Z% ddl&m'Z' ejP                  Z)d Z(e(e_(        de*defdZ+d Z,d)dZ-d  Z.d! Z/d*d"Z0d+d#Z1	 d,d$Z2d*d%Z3d-d&Z4d' Z5e6d(k(  r e5        yy).aq  
Evaluation

Usage:
  pyannote-metrics detection [--subset=<subset> --collar=<seconds> --skip-overlap] <database.task.protocol> <hypothesis.rttm>
  pyannote-metrics segmentation [--subset=<subset> --tolerance=<seconds>] <database.task.protocol> <hypothesis.rttm>
  pyannote-metrics overlap [--subset=<subset> --collar=<seconds>] <database.task.protocol> <hypothesis.rttm>
  pyannote-metrics diarization [--subset=<subset> --greedy --collar=<seconds> --skip-overlap] <database.task.protocol> <hypothesis.rttm>
  pyannote-metrics identification [--subset=<subset> --collar=<seconds> --skip-overlap] <database.task.protocol> <hypothesis.rttm>
  pyannote-metrics spotting [--subset=<subset> --latency=<seconds>... --filter=<expression>...] <database.task.protocol> <hypothesis.json>
  pyannote-metrics -h | --help
  pyannote-metrics --version

Options:
  <database.task.protocol>   Set evaluation protocol (e.g. "Etape.SpeakerDiarization.TV")
  --subset=<subset>          Evaluated subset (train|developement|test) [default: test]
  --collar=<seconds>         Collar, in seconds [default: 0.0].
  --skip-overlap             Do not evaluate overlap regions.
  --tolerance=<seconds>      Tolerance, in seconds [default: 0.5].
  --greedy                   Use greedy diarization error rate.
  --latency=<seconds>        Evaluate with fixed latency.
  --filter=<expression>      Filter out target trials that do not match the
                             expression; e.g. use --filter="speech>10" to skip
                             target trials with less than 10s of speech from
                             the target.
  -h --help                  Show this screen.
  --version                  Show version.

All modes but "spotting" expect hypothesis using the RTTM file format.
RTTM files contain one line per speech turn, using the following convention:

SPEAKER {uri} 1 {start_time} {duration} <NA> <NA> {speaker_id} <NA> <NA>

    * uri: file identifier (as given by pyannote.database protocols)
    * start_time: speech turn start time in seconds
    * duration: speech turn duration in seconds
    * speaker_id: speaker identifier

"spotting" mode expects hypothesis using the following JSON file format.
It should contain a list of trial hypothesis, using the same trial order as
pyannote.database speaker spotting protocols (e.g. protocol.test_trial())

[
    {'uri': '<uri>', 'model_id': '<model_id>', 'scores': [[<t1>, <v1>], [<t2>, <v2>], ... [<tn>, <vn>]]},
    {'uri': '<uri>', 'model_id': '<model_id>', 'scores': [[<t1>, <v1>], [<t2>, <v2>], ... [<tn>, <vn>]]},
    {'uri': '<uri>', 'model_id': '<model_id>', 'scores': [[<t1>, <v1>], [<t2>, <v2>], ... [<tn>, <vn>]]},
    ...
    {'uri': '<uri>', 'model_id': '<model_id>', 'scores': [[<t1>, <v1>], [<t2>, <v2>], ... [<tn>, <vn>]]},
]

    * uri: file identifier (as given by pyannote.database protocols)
    * model_id: target identifier (as given by pyannote.database protocols)
    * [ti, vi]: [time, value] pair indicating that the system has output the
                score vi at time ti (e.g. [10.2, 0.2] means that the system
                gave a score of 0.2 at time 10.2s).

Calling "spotting" mode will create a bunch of files.
* <hypothesis.det.txt> contains DET curve using the following raw file format:
    <threshold> <fpr> <fnr>
* <hypothesis.lcy.txt> contains latency curves using this format:
    <threshold> <fpr> <fnr> <speaker_latency> <absolute_latency>

    N)docopt)
Annotation)Timeline)get_protocol)get_annotated)	load_rttm)tabulate)DetectionAccuracy)DetectionErrorRate)DetectionPrecision)DetectionRecall)DiarizationCoverage)DiarizationErrorRate)DiarizationPurity)GreedyDiarizationErrorRate)IdentificationErrorRate)IdentificationPrecision)IdentificationRecall)SegmentationCoverage)SegmentationPrecision)SegmentationPurity)SegmentationRecall)LowLatencySpeakerSpottingc                 H    t        |j                  dz   t        |              y )N:)print__name__str)messagecategoryargskwargss       i/Volumes/fast/ai/experiments/voice-extract-mac/.venv/lib/python3.12/site-packages/pyannote/metrics/cli.pyshowwarningr$      s    	(

c
!3w<0    current_filereturnc                    | d   }t        |j                        }|j                  |      D ]3  \  \  }}\  }}|||f   }|||f   }||k(  r |j                  ||z         5 |j	                         j                         S )a  Get overlapped speech reference annotation

    Parameters
    ----------
    current_file : `dict`
        File yielded by pyannote.database protocols.

    Returns
    -------
    overlap : `pyannote.core.Annotation`
        Overlapped speech reference.
    
annotation)uri)r   r*   co_iteraddsupportto_annotation)	r&   	referenceoverlaps1t1s2t2l1l2s	            r#   
to_overlapr7      s     \*I9==)G'//	:R(2rr2vr2v8BG ; ??**,,r%   c                 D   |d   }|| v r| |   S | D cg c]	  }||v s| }}t        |      dk(  r(d| d}t        j                  |       t        |d      S t        |      dk(  r| |d      }||_        |S d| d	| d
}t        |j                  ||            c c}w )an  Get hypothesis for given file

    Parameters
    ----------
    hypotheses : `dict`
        Speaker diarization hypothesis provided by `load_rttm`.
    current_file : `dict`
        File description as given by pyannote.database protocols.

    Returns
    -------
    hypothesis : `pyannote.core.Annotation`
        Hypothesis corresponding to `current_file`.
    r*   r   z$Could not find hypothesis for file "z"; assuming empty file.speaker)r*   modality   z)Found too many hypotheses matching file "z" (z).)r*   uris)lenwarningswarnr   r*   
ValueErrorformat)
hypothesesr&   r*   utmp_urimsg
hypothesiss          r#   get_hypothesisrG      s      u
C
j# %1*QSq*G1 7|q4SE9PQccI66 7|q
+

 6cU#gYb
IC
SZZCgZ6
77! 2s
   	BBc           
          | d   }t        ||       }t        |       }|j                         D ci c]  \  }}| ||||       c}}S c c}}w )Nr)   )uem)rG   r   items)itemrB   metricsr/   rF   rI   keymetrics           r#   process_onerO      s[    \"I
D1J

CGN}}GVVVIzs33  s   A
c                     t        j                  t        ||      } t        | |             D ]
  } ||        |j	                         D ci c]  \  }}||j                  d       c}}S c c}}w )N)rB   rL   Fdisplay)	functoolspartialrO   getattrrJ   report)protocolsubsetrB   rL   processrK   rM   rN   s           r#   get_reportsrZ      si    
GTG)&)+ , BIQ+#vCu--QQQs   A/c                     t        | j                        }|j                  d      }| j                  |d| ||dz   d z   dgz         S )z.Reindex report so that 'TOTAL' is the last rowTOTALNr;   )listindexreindex)rV   r^   is      r#   r_   r_      sI    EGA>>%)eAEGn4y@AAr%   c                 L   ||d}t        di |t        di |t        di |t        di |d}t	        | |||      }|d   j                  d      }|d   j                  d      }	|d   j                  d      }
|d   j                  d      }|	|d   j                  d	f   |d
<   |
|d   j                  d	f   |d<   ||d   j                  d	f   |d<   t        |      }t        |j                        }||d   g|dd  z   |dd z      }dj                  d|z  |rdnd      }|gt        d      D cg c]  }|j                  |   d    c}z   |j                  dd  D cg c]  }|d   d	k(  rd	n|d    c}z   }t        t        ||ddddddd	             y c c}w c c}w )Ncollarskip_overlap)erroraccuracy	precisionrecallre   FrQ   rf   rg   rh   %)rf   ri   rg   ri   rh   ri   r   r;   z Detection (collar = {0:g} ms{1})  , no overlap    simple.2fdecimalleftdefaultheaderstablefmtfloatfmtnumalignstralign
missingval	showindexdisable_numparse )r   r
   r   r   rZ   rV   namer_   r]   columnsrA   ranger   r	   )rW   rX   rB   rc   rd   optionsrL   reportsrV   rf   rg   rh   r   summaryr`   crw   s                    r#   	detectionr      s   >G $.g.%00'2'2!,G,	G (FJ@GW$$U$3Fz"))%)8H$++E+:IX%%e%4F&wz':'?'?'DEF?()=)B)BC)GHF"78#4#9#93#>?F=V_F6>>"GWQZL723</'!B-?@F077v~2G
 
	).q
2A6>>!Q
2	3171C
D1CA!A$#+31Q4'1C
D	E  
"
	
	 3
Ds   5F#F!c                 n   d|i}t        di |t        di |t        di |t        di |d}t	        | |||      }|d   j                  d      }|d   j                  d      }|d   j                  d      }	|d   j                  d      }
||d   j                     }||d   j                     }|	|d   j                     }	|
|d   j                     }
t        j                  |||	|
gd	
      }t        |      }dj                  d|z        ddddg}t        t        ||ddddddd	             y )N	tolerance)coveragepurityrg   rh   r   FrQ   r   rg   rh   r;   )axisz#Segmentation (tolerance = {0:g} ms)rm   rq   rr   rs   rt   ro   ru   rv   r   )r   r   r   r   rZ   rV   r   pdconcatr_   rA   r   r	   )rW   rX   rB   r   r   rL   r   r   r   rg   rh   rV   rw   s                r#   segmentationr     s~   I&G )373$/w/*5W5$/w/	G (FJ@Gz"))%)8HX%%e%4F$++E+:IX%%e%4F
+001HGH%**+F'+.334IGH%**+FYY&)V<1EFV_F 	.44TI5EFG 
"
	
r%   c                    ||d}t        di |t        di |d}|rt        di ||d<   nt        di ||d<   t	        | |||      }|d   j                  d      }	|d   j                  d      }
|d   j                  d      }|
|d   j                  df   |	d	<   ||d   j                  df   |	d
<   t        |	j                        }|	|d   g|dd  z   |dd z      }	t        |	      }	dj                  |rdndd|z  |rdnd      }|gt        d      D cg c]  }|	j                  |   d    c}z   |	j                  dd  D cg c]  }|d   dk(  rdn|d    c}z   }t        t        |	|ddddddd	             y c c}w c c}w )Nrb   )r   r   re   FrQ   r   r   ri   )r   ri   )r   ri   r   r;   z'Diarization ({0:s}collar = {1:g} ms{2})zgreedy, ro   rm   rn      rq   rr   rs   rt   ru   rv   r   )r   r   r   r   rZ   rV   r   r]   r   r_   rA   r   r   r	   )rW   rX   rB   greedyrc   rd   r   rL   r   rV   r   r   r   r   r`   r   rw   s                    r#   diarizationr   G  s     >G $.g.'2'2G
 5@@/:':(FJ@GW$$U$3FX%%e%4Fz"))%)8H"78#4#9#93#>?F=&wz':'?'?'DEF?6>>"GWQZL723</'!B-?@FV_F7>>
"v&BG 
	).q
2A6>>!Q
2	3171C
D1CA!A$#+31Q4'1C
D	E  
"
	
	 3
Ds   F Fc                    ||d}t        di |t        di |t        di |d}t        | |||      }|d   j	                  d      }|d   j	                  d      }	|d   j	                  d      }
|	|d   j
                  df   |d	<   |
|d   j
                  df   |d
<   t        |j                        }||d   g|dd  z   |dd z      }t        |      }dj                  d|z  |rdnd      }|gt        d      D cg c]  }|j                  |   d    c}z   |j                  dd  D cg c]  }|d   dk(  rdn|d    c}z   }t        t        ||ddddddd	             y c c}w c c}w )Nrb   )re   rg   rh   re   FrQ   rg   rh   ri   rj   rk   r   r   r;   z%Identification (collar = {0:g} ms{1})rm   rn   ro   r   rq   rr   rs   rt   ru   rv   r   )r   r   r   rZ   rV   r   r]   r   r_   rA   r   r   r	   )rW   rX   rB   rc   rd   r   rL   r   rV   rg   rh   r   r   r`   r   rw   s                   r#   identificationr     s   >G )373,7w7&11G (FJ@GW$$U$3F$++E+:IX%%e%4F()=)B)BC)GHF"78#4#9#93#>?F=6>>"GWQZL723</'!B-?@FV_F5<<v~2G
 
	).q
2A6>>!Q
2	3171C
D1CA!A$#+31Q4'1C
D	E  
"
	
	 3
Ds   ?E&-E+c                 R   |sg }d| _          t        | dj                  |                   }t        t	        ||            D ]|  \  }\  }	}
	 |	d   |
d   k(  sJ 	 	 |	d   |
d   k(  sJ 	 	 t        |
d	         d
kD  sJ 	 t	        |
d	    \  }}|sj                  |       |	d   }	 t        |      |j                  k\  sJ ~ |st        j                        }t        j                  t        ddd      D cg c]  }t        dd      D ]  }|d| z  z    c}}      }t        j                  |t        j                  ddd      d|d d d   z
  g      }t        j                   ||      }|st#              }nt#        |      } t        | dj                  |                   }t        t	        ||            D ]>  \  }\  }	}
|#|	d   j%                         }|d
kD  }|r	 ||      r.|	d   } |||
d	          @ |s|j'                  d      \  }}}}}dj                  |      }d}t)        |d      5 }|j+                  d       t	        |||      D ]+  \  }} }|j                  || |       }!|j+                  |!       - 	 d d d        t-        d!j                  |"             |j'                  d#      \  }}}}}}"}#d$j                  |      }$d%}%t)        |$d      5 }|j+                  d&       t	        ||||"|#      D ]K  \  }} }}&}'| dk(  rt        j.                  |&      r%|%j                  || ||&|''      }!|j+                  |!       M 	 d d d        t-        d(j                  |$)             t-                t-        d*j                  d|z  +             y |j'                         }(g })t1        |(      D ]  }*|(|*   }+d,|*i},|D ]  }-|+|-   \  }}}}}||,|-<   d-j                  ||*|-.      }d}t)        |d      5 }|j+                  d       t	        |||      D ]+  \  }} }|j                  || |       }!|j+                  |!       - 	 d d d         |)j                  |,       d/j                  ||*0      }t-        d!j                  |"              t-                dt2        j4                  j7                  |)      j9                  d,      |   z  }.t-        t;        |.d1d,g|D /cg c]  }/d2j                  |/3       c}/z   d4d5d6d7d8d9	             y # t
        $ r*}d}t        |j                  ||
d   |	d               d }~ww xY w# t
        $ r*}d}t        |j                  ||
d   |	d               d }~ww xY w# t
        $ r"}d}t        |j                  |            d }~ww xY w# t
        $ r7}d}t        |j                  |t        |      |j                              d }~ww xY wc c}}w # 1 sw Y   wxY w# 1 sw Y   xY w# 1 sw Y   TxY wc c}/w ):NFz{subset}_trial)rX   model_idzFtarget mismatch in trial #{i} (found: {found}, should be: {should_be}))r`   found	should_ber*   zDfile mismatch in trial #{i} (found: {found}, should be: {should_be})scoresr   z#empty list of scores in trial #{i}.)r`   try_withzQincorrect timestamp in trial #{i} (found: {found:g}, should be: >= {should_be:g})rp   r;   
   g?g      Y@d   )
thresholds)	latenciesr/   )return_latencyz{output_prefix}.det.txt)output_prefixz{t:.9f} {p:.9f} {n:.9f}
wmodez4# threshold false_positive_rate false_negative_rate
)tpnz> {det_path})det_pathTz{output_prefix}.lcy.txtz({t:.9f} {p:.9f} {n:.9f} {s:.6f} {a:.6f}
zU# threshold false_positive_rate false_negative_rate speaker_latency absolute_latency
)r   r   r   saz> {lcy_path})lcy_pathzEER% = {eer:.2f})eerlatencyz*{output_prefix}.det.{key}.{latency:g}s.txt)r   rM   r   z!{output_prefix}.det.{key}.XXs.txt)r   rM   rq   zEER% @ {l:g}s)lrr   rs   rt   ro   ru   )rx   rw   ry   rz   r{   r|   r}   r~   )r   rU   rA   	enumeratezipAssertionErrorr@   r=   appendminstartnpconcatenatearrayr   arange
percentiler   duration	det_curveopenwriter   isnansortedr   	DataFrame	from_dict	set_indexr	   )0rW   rX   r   rB   r   filter_funcScorestrialsr`   current_trialrF   erE   
timestampsr   r   r   epsilonsr   r   rN   speechtarget_trialr/   fprfnrr   _r   det_tmplfpr   r   linespeaker_lcyabsolute_lcyr   lcy_tmplr   r   resultslogsrM   resultlogr   dfr   s0                                                   r#   spottingr     s    HFWX/66f6EFHF*3C
4K*L&&M:	 ,
:0FFFF		 ':e+<<<<	.z(+,q000
 !*X"67
FMM&! !,		z?hnn444[ +Mn '88%*1a_K_eArlQ^l^_K
 ^^ryyeS13$B$3GH

 ]]6:6
*jA +Y?FWX/66f6EFHF*3C
4K*L&&M:"";/88:F!A:LF 3!+.	y*X./ +M '-'7'7u'7'M$
Cc1 -33-3P.(%HHLMz341aa15 5 & 	n##X#67@F@P@P AQ A
=
CaK
 -33-3P>(%HHh "%Zc;!U1aA688A;a1Q? "V	 & 	n##X#67 ''C#I'67 ""$'?CS\Fc"C$/5g,
Cc1  #GGNN"/S' O  7(-HHTU#&z3#<1a'a1= $= .- % KK:AA+ B H .''':;/ #2 	2<<))$/99)DYOO!"I&VIq'='='='BI&VV"#!&
	
k  	;  

$Z0+J7   	   	;  

Qj&7=QVCW
X 	  	.7CSZZ!Z_--	.  	B  

Qc*o
P 	 L@ &% &%F .-" 'Ws   TU*V%V47!W7
:AW=&A/X
AX,X$	U#%UU	V%U>>V	V1V,,V14	W4=2W//W4=X
XX!c            	      ,   t        t        d      } t        | d         }| d   }t        | d         }| d   }t               }| d   r!|rd}t	        j
                  |       d	t        i}t        ||
      }| d   }| d   r| d   }	t        |	d      5 }
t        j                  |
      }d d d        |	d d }| d   D cg c]  }t        |       }}| d   }|r8ddlm}m}m}  |d      }g |D cg c]  } ||g ||             c}fd}nd }t        |||||       t	        j
                  d       | d   }	 t!        |      }| d   rt%        ||||       | d   rt%        ||||       | d   rt'        |||        | d!   r| d"   }t)        |||||#       | d$   rt+        ||||       y y # 1 sw Y   xY wc c}w c c}w # t"        $ r d| d}t	        j
                  |       Y  d| d}t	        j
                  |       Y xY w)%N
Evaluation)versionz--collarz--skip-overlapz--tolerancez<database.task.protocol>r0   zSOption --skip-overlap is not supported when evaluating overlapped speech detection.r)   )preprocessorsz--subsetr   z<hypothesis.json>rr   z	--latencyz--filterr   )sympifylambdifysymbolsr   c                 .     t         fdD              S )Nc              3   0   K   | ]  } |         y wNr   ).0funcr   s     r#   	<genexpr>z)main.<locals>.<lambda>.<locals>.<genexpr>  s     ,T|td6l]|s   )any)r   filter_funcss   `r#   <lambda>zmain.<locals>.<lambda>  s    ,T|,T)Tr%   )r   z<hypothesis.rttm>zCould not find file .zFailed to load z:, please check its format (only RTTM files are supported).r   rb   r   )r   r   z--greedy)r   rc   rd   r   )r   __doc__floatdictsysexitr7   r   r   jsonloadsympyr   r   r   r   r   FileNotFoundErrorr   r   r   r   )	argumentsrc   rd   r   protocol_namer   rE   rW   rX   hypothesis_jsonr   rB   r   r   r   filtersr   r   r   r   
expressionr   hypothesis_rttmr   r   s                           @r#   mainr   _  s   w5I9Z()F-.Li./I 89MFM?  HHSM%z2MGH z"F#$78/,2J - (,'0'=>'=!U1X'=	>J'88X&FLJQJQJ&7:#67'L UKK#	
 	 34O/
 fjl	
 fjl	
  XvzYG:&%	
 !"fjl	
 #O -,
 ?.  $_$5Q7o. // 0 	 	s*   F8<G/G
=G 8G$H5H__main__)NN)        F)g      ?)Fr  Fr   )7r   rS   r   r   r>   numpyr   pandasr   r   pyannote.corer   r   pyannote.databaser   pyannote.database.utilr   r   r	   pyannote.metrics.detectionr
   r   r   r   pyannote.metrics.diarizationr   r   r   r   pyannote.metrics.identificationr   r   r   pyannote.metrics.segmentationr   r   r   r   pyannote.metrics.spottingr   r$   showwarning_origr   r7   rG   rO   rZ   r_   r   r   r   r   r   r   r   r   r%   r#   <module>r     s   :>@   
     $ " * 0 ,  8 9 9 6 < = : C C C @ > ? < < ?'' 1 # -T -j -2'8TRB0f,` JO5p-`m
`f
R zF r%   