#!/usr/bin/env python
# encoding: utf-8
# The MIT License (MIT)
# Copyright (c) 2012-2019 CNRS
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr
# Benjamin MAURICE - maurice@limsi.fr
import numpy as np
from scipy.optimize import linear_sum_assignment
from ..matcher import LabelMatcher
from pyannote.core import Annotation
from ..matcher import MATCH_CORRECT, MATCH_CONFUSION, \
MATCH_MISSED_DETECTION, MATCH_FALSE_ALARM
from ..identification import UEMSupportMixin
REFERENCE_TOTAL = 'reference'
HYPOTHESIS_TOTAL = 'hypothesis'
REGRESSION = 'regression'
IMPROVEMENT = 'improvement'
BOTH_CORRECT = 'both_correct'
BOTH_INCORRECT = 'both_incorrect'
[docs]class IdentificationErrorAnalysis(UEMSupportMixin, object):
"""
Parameters
----------
collar : float, optional
Duration (in seconds) of collars removed from evaluation around
boundaries of reference segments.
skip_overlap : bool, optional
Set to True to not evaluate overlap regions.
Defaults to False (i.e. keep overlap regions).
"""
def __init__(self, collar=0., skip_overlap=False):
super(IdentificationErrorAnalysis, self).__init__()
self.matcher = LabelMatcher()
self.collar = collar
self.skip_overlap = skip_overlap
[docs] def difference(self, reference, hypothesis, uem=None, uemified=False):
"""Get error analysis as `Annotation`
Labels are (status, reference_label, hypothesis_label) tuples.
`status` is either 'correct', 'confusion', 'missed detection' or
'false alarm'.
`reference_label` is None in case of 'false alarm'.
`hypothesis_label` is None in case of 'missed detection'.
Parameters
----------
uemified : bool, optional
Returns "uemified" version of reference and hypothesis.
Defaults to False.
Returns
-------
errors : `Annotation`
"""
R, H, common_timeline = self.uemify(
reference, hypothesis, uem=uem,
collar=self.collar, skip_overlap=self.skip_overlap,
returns_timeline=True)
errors = Annotation(uri=reference.uri, modality=reference.modality)
# loop on all segments
for segment in common_timeline:
# list of labels in reference segment
rlabels = R.get_labels(segment, unique=False)
# list of labels in hypothesis segment
hlabels = H.get_labels(segment, unique=False)
_, details = self.matcher(rlabels, hlabels)
for r, h in details[MATCH_CORRECT]:
track = errors.new_track(segment, prefix=MATCH_CORRECT)
errors[segment, track] = (MATCH_CORRECT, r, h)
for r, h in details[MATCH_CONFUSION]:
track = errors.new_track(segment, prefix=MATCH_CONFUSION)
errors[segment, track] = (MATCH_CONFUSION, r, h)
for r in details[MATCH_MISSED_DETECTION]:
track = errors.new_track(segment,
prefix=MATCH_MISSED_DETECTION)
errors[segment, track] = (MATCH_MISSED_DETECTION, r, None)
for h in details[MATCH_FALSE_ALARM]:
track = errors.new_track(segment, prefix=MATCH_FALSE_ALARM)
errors[segment, track] = (MATCH_FALSE_ALARM, None, h)
if uemified:
return reference, hypothesis, errors
else:
return errors
def _match_errors(self, before, after):
b_type, b_ref, b_hyp = before
a_type, a_ref, a_hyp = after
return (b_ref == a_ref) * (1 + (b_type == a_type) + (b_hyp == a_hyp))
def regression(self, reference, before, after, uem=None, uemified=False):
_, before, errors_before = self.difference(
reference, before, uem=uem, uemified=True)
reference, after, errors_after = self.difference(
reference, after, uem=uem, uemified=True)
behaviors = Annotation(uri=reference.uri, modality=reference.modality)
# common (up-sampled) timeline
common_timeline = errors_after.get_timeline().union(
errors_before.get_timeline())
common_timeline = common_timeline.segmentation()
# align 'before' errors on common timeline
B = self._tagger(errors_before, common_timeline)
# align 'after' errors on common timeline
A = self._tagger(errors_after, common_timeline)
for segment in common_timeline:
old_errors = B.get_labels(segment, unique=False)
new_errors = A.get_labels(segment, unique=False)
n1 = len(old_errors)
n2 = len(new_errors)
n = max(n1, n2)
match = np.zeros((n, n), dtype=int)
for i1, e1 in enumerate(old_errors):
for i2, e2 in enumerate(new_errors):
match[i1, i2] = self._match_errors(e1, e2)
for i1, i2 in zip(*linear_sum_assignment(-match)):
if i1 >= n1:
track = behaviors.new_track(segment,
candidate=REGRESSION,
prefix=REGRESSION)
behaviors[segment, track] = (
REGRESSION, None, new_errors[i2])
elif i2 >= n2:
track = behaviors.new_track(segment,
candidate=IMPROVEMENT,
prefix=IMPROVEMENT)
behaviors[segment, track] = (
IMPROVEMENT, old_errors[i1], None)
elif old_errors[i1][0] == MATCH_CORRECT:
if new_errors[i2][0] == MATCH_CORRECT:
track = behaviors.new_track(segment,
candidate=BOTH_CORRECT,
prefix=BOTH_CORRECT)
behaviors[segment, track] = (
BOTH_CORRECT, old_errors[i1], new_errors[i2])
else:
track = behaviors.new_track(segment,
candidate=REGRESSION,
prefix=REGRESSION)
behaviors[segment, track] = (
REGRESSION, old_errors[i1], new_errors[i2])
else:
if new_errors[i2][0] == MATCH_CORRECT:
track = behaviors.new_track(segment,
candidate=IMPROVEMENT,
prefix=IMPROVEMENT)
behaviors[segment, track] = (
IMPROVEMENT, old_errors[i1], new_errors[i2])
else:
track = behaviors.new_track(segment,
candidate=BOTH_INCORRECT,
prefix=BOTH_INCORRECT)
behaviors[segment, track] = (
BOTH_INCORRECT, old_errors[i1], new_errors[i2])
behaviors = behaviors.support()
if uemified:
return reference, before, after, behaviors
else:
return behaviors
def matrix(self, reference, hypothesis, uem=None):
reference, hypothesis, errors = self.difference(
reference, hypothesis, uem=uem, uemified=True)
chart = errors.chart()
# rLabels contains reference labels
# hLabels contains hypothesis labels confused with a reference label
# falseAlarmLabels contains false alarm hypothesis labels that do not
# exist in reference labels // corner case //
falseAlarmLabels = set(hypothesis.labels()) - set(reference.labels())
hLabels = set(reference.labels()) | set(hypothesis.labels())
rLabels = set(reference.labels())
# sort these sets of labels
cmp_func = reference._cmp_labels
falseAlarmLabels = sorted(falseAlarmLabels, cmp=cmp_func)
rLabels = sorted(rLabels, cmp=cmp_func)
hLabels = sorted(hLabels, cmp=cmp_func)
# append false alarm labels as last 'reference' labels
# (make sure to mark them as such)
rLabels = rLabels + [(MATCH_FALSE_ALARM, hLabel)
for hLabel in falseAlarmLabels]
# prepend duration columns before the detailed confusion matrix
hLabels = [
REFERENCE_TOTAL, HYPOTHESIS_TOTAL,
MATCH_CORRECT, MATCH_CONFUSION,
MATCH_FALSE_ALARM, MATCH_MISSED_DETECTION
] + hLabels
# initialize empty matrix
try:
from xarray import DataArray
except ImportError:
msg = (
"Please install xarray dependency to use class "
"'IdentificationErrorAnalysis'."
)
raise ImportError(msg)
matrix = DataArray(
np.zeros((len(rLabels), len(hLabels))),
coords=[('reference', rLabels), ('hypothesis', hLabels)])
# loop on chart
for (status, rLabel, hLabel), duration in chart:
# increment correct
if status == MATCH_CORRECT:
matrix.loc[rLabel, hLabel] += duration
matrix.loc[rLabel, MATCH_CORRECT] += duration
# increment confusion matrix
if status == MATCH_CONFUSION:
matrix.loc[rLabel, hLabel] += duration
matrix.loc[rLabel, MATCH_CONFUSION] += duration
if hLabel in falseAlarmLabels:
matrix.loc[(MATCH_FALSE_ALARM, hLabel), rLabel] += duration
matrix.loc[(MATCH_FALSE_ALARM, hLabel), MATCH_CONFUSION] += duration
else:
matrix.loc[hLabel, rLabel] += duration
matrix.loc[hLabel, MATCH_CONFUSION] += duration
if status == MATCH_FALSE_ALARM:
# hLabel is also a reference label
if hLabel in falseAlarmLabels:
matrix.loc[(MATCH_FALSE_ALARM, hLabel), MATCH_FALSE_ALARM] += duration
else:
matrix.loc[hLabel, MATCH_FALSE_ALARM] += duration
if status == MATCH_MISSED_DETECTION:
matrix.loc[rLabel, MATCH_MISSED_DETECTION] += duration
# total reference and hypothesis duration
for rLabel in rLabels:
if isinstance(rLabel, tuple) and rLabel[0] == MATCH_FALSE_ALARM:
r = 0.
h = hypothesis.label_duration(rLabel[1])
else:
r = reference.label_duration(rLabel)
h = hypothesis.label_duration(rLabel)
matrix.loc[rLabel, REFERENCE_TOTAL] = r
matrix.loc[rLabel, HYPOTHESIS_TOTAL] = h
return matrix