mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-14 22:30:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Remove ClippingPredictorEvaluator

Browse source 
Bug: webrtc:7494
Change-Id: Idba27a5dbe72726f9e1469e955c5958558d93a4b
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/278403
Reviewed-by: Alessio Bazzica <alessiob@webrtc.org>
Commit-Queue: Hanna Silen <silen@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38321}
This commit is contained in:
Hanna Silen 2022-10-07 13:58:16 +02:00 committed by WebRTC LUCI CQ
parent 3609a5aeb6
commit b37a9c5f88
4 changed files with 0 additions and 1120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
modules/audio_processing/agc2/BUILD.gn
View file

@ -95,25 +95,6 @@ rtc_library("clipping_predictor") {
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
rtc_library("clipping_predictor_evaluator") {
visibility = [
"../agc:agc",
"./*",
]
sources = [
"clipping_predictor_evaluator.cc",
"clipping_predictor_evaluator.h",
]
deps = [
"../../../rtc_base:checks",
"../../../rtc_base:logging",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
rtc_source_set("common") {
sources = [ "agc2_common.h" ]
}
@ -331,7 +312,6 @@ rtc_library("fixed_digital_unittests") {
rtc_library("input_volume_controller_unittests") {
testonly = true
sources = [
"clipping_predictor_evaluator_unittest.cc",
"clipping_predictor_level_buffer_unittest.cc",
"clipping_predictor_unittest.cc",
"speech_probability_buffer_unittest.cc",
@ -341,7 +321,6 @@ rtc_library("input_volume_controller_unittests") {
deps = [
":clipping_predictor",
":clipping_predictor_evaluator",
":gain_map",
":input_volume_controller",
"../../../rtc_base:checks",

214
modules/audio_processing/agc2/clipping_predictor_evaluator.cc
View file

@ -1,214 +0,0 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/agc2/clipping_predictor_evaluator.h"
#include <algorithm>
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
namespace webrtc {
namespace {
// Returns the index of the oldest item in the ring buffer for a non-empty
// ring buffer with give `size`, `tail` index and `capacity`.
int OldestExpectedDetectionIndex(int size, int tail, int capacity) {
RTC_DCHECK_GT(size, 0);
return tail - size + (tail < size ? capacity : 0);
}
} // namespace
ClippingPredictorEvaluator::ClippingPredictorEvaluator(int history_size)
: history_size_(history_size),
ring_buffer_capacity_(history_size + 1),
ring_buffer_(ring_buffer_capacity_) {
RTC_DCHECK_GT(history_size_, 0);
Reset();
counters_.true_positives = 0;
counters_.true_negatives = 0;
counters_.false_positives = 0;
counters_.false_negatives = 0;
}
ClippingPredictorEvaluator::~ClippingPredictorEvaluator() = default;
absl::optional<int> ClippingPredictorEvaluator::Observe(
bool clipping_detected,
bool clipping_predicted) {
RTC_DCHECK_GE(ring_buffer_size_, 0);
RTC_DCHECK_LE(ring_buffer_size_, ring_buffer_capacity_);
RTC_DCHECK_GE(ring_buffer_tail_, 0);
RTC_DCHECK_LT(ring_buffer_tail_, ring_buffer_capacity_);
DecreaseTimesToLive();
// Clipping is expected if there are expected detections regardless of
// whether all the expected detections have been previously matched - i.e.,
// `ExpectedDetection::detected` is true.
const bool clipping_expected = ring_buffer_size_ > 0;
absl::optional<int> prediction_interval;
if (clipping_expected && clipping_detected) {
prediction_interval = FindEarliestPredictionInterval();
// Add a true positive for each unexpired expected detection.
const int num_modified_items = MarkExpectedDetectionAsDetected();
counters_.true_positives += num_modified_items;
RTC_DCHECK(prediction_interval.has_value() || num_modified_items == 0);
RTC_DCHECK(!prediction_interval.has_value() || num_modified_items > 0);
} else if (clipping_expected && !clipping_detected) {
// Add a false positive if there is one expected detection that has expired
// and that has never been matched before. Note that there is at most one
// unmatched expired detection.
if (HasExpiredUnmatchedExpectedDetection()) {
counters_.false_positives++;
}
} else if (!clipping_expected && clipping_detected) {
counters_.false_negatives++;
} else {
RTC_DCHECK(!clipping_expected && !clipping_detected);
counters_.true_negatives++;
}
if (clipping_predicted) {
// TODO(bugs.webrtc.org/12874): Use designated initializers one fixed.
Push(/*expected_detection=*/{/*ttl=*/history_size_, /*detected=*/false});
}
return prediction_interval;
}
void ClippingPredictorEvaluator::RemoveExpectations() {
// Empty the ring buffer of expected detections.
ring_buffer_tail_ = 0;
ring_buffer_size_ = 0;
}
void ClippingPredictorEvaluator::Reset() {
counters_.true_positives = 0;
counters_.true_negatives = 0;
counters_.false_positives = 0;
counters_.false_negatives = 0;
RemoveExpectations();
}
// Cost: O(1).
void ClippingPredictorEvaluator::Push(ExpectedDetection value) {
ring_buffer_[ring_buffer_tail_] = value;
ring_buffer_tail_++;
if (ring_buffer_tail_ == ring_buffer_capacity_) {
ring_buffer_tail_ = 0;
}
ring_buffer_size_ = std::min(ring_buffer_capacity_, ring_buffer_size_ + 1);
}
// Cost: O(N).
void ClippingPredictorEvaluator::DecreaseTimesToLive() {
bool expired_found = false;
for (int i = ring_buffer_tail_ - ring_buffer_size_; i < ring_buffer_tail_;
++i) {
int index = i >= 0 ? i : ring_buffer_capacity_ + i;
RTC_DCHECK_GE(index, 0);
RTC_DCHECK_LT(index, ring_buffer_.size());
RTC_DCHECK_GE(ring_buffer_[index].ttl, 0);
if (ring_buffer_[index].ttl == 0) {
RTC_DCHECK(!expired_found)
<< "There must be at most one expired item in the ring buffer.";
expired_found = true;
RTC_DCHECK_EQ(index, OldestExpectedDetectionIndex(ring_buffer_size_,
ring_buffer_tail_,
ring_buffer_capacity_))
<< "The expired item must be the oldest in the ring buffer.";
}
ring_buffer_[index].ttl--;
}
if (expired_found) {
ring_buffer_size_--;
}
}
// Cost: O(N).
absl::optional<int> ClippingPredictorEvaluator::FindEarliestPredictionInterval()
const {
absl::optional<int> prediction_interval;
for (int i = ring_buffer_tail_ - ring_buffer_size_; i < ring_buffer_tail_;
++i) {
int index = i >= 0 ? i : ring_buffer_capacity_ + i;
RTC_DCHECK_GE(index, 0);
RTC_DCHECK_LT(index, ring_buffer_.size());
if (!ring_buffer_[index].detected) {
prediction_interval = std::max(prediction_interval.value_or(0),
history_size_ - ring_buffer_[index].ttl);
}
}
return prediction_interval;
}
// Cost: O(N).
int ClippingPredictorEvaluator::MarkExpectedDetectionAsDetected() {
int num_modified_items = 0;
for (int i = ring_buffer_tail_ - ring_buffer_size_; i < ring_buffer_tail_;
++i) {
int index = i >= 0 ? i : ring_buffer_capacity_ + i;
RTC_DCHECK_GE(index, 0);
RTC_DCHECK_LT(index, ring_buffer_.size());
if (!ring_buffer_[index].detected) {
num_modified_items++;
}
ring_buffer_[index].detected = true;
}
return num_modified_items;
}
// Cost: O(1).
bool ClippingPredictorEvaluator::HasExpiredUnmatchedExpectedDetection() const {
if (ring_buffer_size_ == 0) {
return false;
}
// If an expired item, that is `ttl` equal to 0, exists, it must be the
// oldest.
const int oldest_index = OldestExpectedDetectionIndex(
ring_buffer_size_, ring_buffer_tail_, ring_buffer_capacity_);
RTC_DCHECK_GE(oldest_index, 0);
RTC_DCHECK_LT(oldest_index, ring_buffer_.size());
return ring_buffer_[oldest_index].ttl == 0 &&
!ring_buffer_[oldest_index].detected;
}
absl::optional<ClippingPredictionMetrics> ComputeClippingPredictionMetrics(
const ClippingPredictionCounters& counters) {
RTC_DCHECK_GE(counters.true_positives, 0);
RTC_DCHECK_GE(counters.true_negatives, 0);
RTC_DCHECK_GE(counters.false_positives, 0);
RTC_DCHECK_GE(counters.false_negatives, 0);
if (counters.true_positives == 0) {
// Both precision and recall are zero in this case and hence the F1 score
// is undefined.
return absl::nullopt;
}
int precision_denominator =
counters.true_positives + counters.false_positives;
int recall_denominator = counters.true_positives + counters.false_negatives;
if (precision_denominator == 0 || recall_denominator == 0) {
// Both precision and recall must be defined.
return absl::nullopt;
}
ClippingPredictionMetrics metrics;
float true_positives = counters.true_positives;
metrics.precision = true_positives / precision_denominator;
metrics.recall = true_positives / recall_denominator;
float f1_score_denominator = metrics.precision + metrics.recall;
RTC_DCHECK_GT(f1_score_denominator, 0.0f);
metrics.f1_score =
2 * metrics.precision * metrics.recall / f1_score_denominator;
return metrics;
}
} // namespace webrtc

122
modules/audio_processing/agc2/clipping_predictor_evaluator.h
View file

@ -1,122 +0,0 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_PROCESSING_AGC2_CLIPPING_PREDICTOR_EVALUATOR_H_
#define MODULES_AUDIO_PROCESSING_AGC2_CLIPPING_PREDICTOR_EVALUATOR_H_
#include <vector>
#include "absl/types/optional.h"
namespace webrtc {
// Clipping prediction counters.
struct ClippingPredictionCounters {
int true_positives; // TP.
int true_negatives; // TN.
int false_positives; // FP.
int false_negatives; // FN.
};
// Counts true/false positives/negatives while observing sequences of flag pairs
// that indicate whether clipping has been detected and/or if clipping is
// predicted. When a true positive is found measures the time interval between
// prediction and detection events.
// After a prediction is observed and for a period equal to
// `history_size` calls to `Observe()`, one or more detections are expected. If
// the expectation is met, a true positive is added and the time interval
// between the earliest prediction and the detection is recorded; otherwise,
// when the deadline is reached, a false positive is added. Note that one
// detection matches all the expected detections that have not expired - i.e.,
// one detection counts as multiple true positives.
// If a detection is observed, but no prediction has been observed over the past
// `history_size` calls to `Observe()`, then a false negative is added;
// otherwise, a true negative is added.
class ClippingPredictorEvaluator {
public:
// Ctor. `history_size` indicates how long to wait for a call to `Observe()`
// having `clipping_detected` set to true from the time clipping is predicted.
explicit ClippingPredictorEvaluator(int history_size);
ClippingPredictorEvaluator(const ClippingPredictorEvaluator&) = delete;
ClippingPredictorEvaluator& operator=(const ClippingPredictorEvaluator&) =
delete;
~ClippingPredictorEvaluator();
// Observes whether clipping has been detected and/or if clipping is
// predicted. When predicted one or more detections are expected in the next
// `history_size_` calls of `Observe()`. When true positives are found returns
// the prediction interval between the earliest prediction and the detection.
absl::optional<int> Observe(bool clipping_detected, bool clipping_predicted);
// Removes any expectation recently set after a call to `Observe()` having
// `clipping_predicted` set to true. Counters won't be reset.
void RemoveExpectations();
// Resets counters and removes any expectation (see `RemoveExpectations()`).
void Reset();
ClippingPredictionCounters counters() const { return counters_; }
private:
const int history_size_;
// State of a detection expected to be observed after a prediction.
struct ExpectedDetection {
// Time to live (TTL); remaining number of `Observe()` calls to match a call
// having `clipping_detected` set to true.
int ttl;
// True if an `Observe()` call having `clipping_detected` set to true has
// been observed.
bool detected;
};
// Ring buffer of expected detections.
const int ring_buffer_capacity_;
std::vector<ExpectedDetection> ring_buffer_;
int ring_buffer_tail_;
int ring_buffer_size_;
// Pushes `expected_detection` into `expected_matches_ring_buffer_`.
void Push(ExpectedDetection expected_detection);
// Decreased the TTLs in `expected_matches_ring_buffer_` and removes expired
// items.
void DecreaseTimesToLive();
// Returns the prediction interval for the earliest unexpired expected
// detection if any.
absl::optional<int> FindEarliestPredictionInterval() const;
// Marks all the items in `expected_matches_ring_buffer_` as `detected` and
// returns the number of updated items.
int MarkExpectedDetectionAsDetected();
// Returns true if `expected_matches_ring_buffer_` has an item having `ttl`
// equal to 0 (expired) and `detected` equal to false (unmatched).
bool HasExpiredUnmatchedExpectedDetection() const;
// Counters.
ClippingPredictionCounters counters_;
};
// Clipping prediction metrics derived from the clipping prediction counters.
struct ClippingPredictionMetrics {
// Precision (P) is defined as TP / (TP + FP).
float precision;
// Recall (R) is defined as TP / (TP + FN).
float recall;
// The F1 score is defined as 2 * P * R / (P + R).
float f1_score;
};
// Derives clipping prediction metrics from the true/false positives/negatives
// `counters`. Returns an unspecified value if one or more metrics are not
// defined.
absl::optional<ClippingPredictionMetrics> ComputeClippingPredictionMetrics(
const ClippingPredictionCounters& counters);
} // namespace webrtc
#endif // MODULES_AUDIO_PROCESSING_AGC2_CLIPPING_PREDICTOR_EVALUATOR_H_

763
modules/audio_processing/agc2/clipping_predictor_evaluator_unittest.cc
View file

@ -1,763 +0,0 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/agc2/clipping_predictor_evaluator.h"
#include <cstdint>
#include <memory>
#include <tuple>
#include <vector>
#include "absl/types/optional.h"
#include "rtc_base/numerics/safe_conversions.h"
#include "rtc_base/random.h"
#include "test/gmock.h"
#include "test/gtest.h"
namespace webrtc {
namespace {
using testing::Eq;
using testing::Field;
using testing::Optional;
constexpr bool kDetected = true;
constexpr bool kNotDetected = false;
constexpr bool kPredicted = true;
constexpr bool kNotPredicted = false;
ClippingPredictionCounters operator-(const ClippingPredictionCounters& lhs,
const ClippingPredictionCounters& rhs) {
return {
lhs.true_positives - rhs.true_positives,
lhs.true_negatives - rhs.true_negatives,
lhs.false_positives - rhs.false_positives,
lhs.false_negatives - rhs.false_negatives,
};
}
// Checks the metrics after init - i.e., no call to `Observe()`.
TEST(ClippingPredictionEvalTest, Init) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
EXPECT_EQ(evaluator.counters().true_positives, 0);
EXPECT_EQ(evaluator.counters().true_negatives, 0);
EXPECT_EQ(evaluator.counters().false_positives, 0);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
class ClippingPredictorEvaluatorParameterization
: public ::testing::TestWithParam<std::tuple<int, int>> {
protected:
uint64_t seed() const {
return rtc::checked_cast<uint64_t>(std::get<0>(GetParam()));
}
int history_size() const { return std::get<1>(GetParam()); }
};
// Checks that after each call to `Observe()` at most one metric changes.
TEST_P(ClippingPredictorEvaluatorParameterization, AtMostOneMetricChanges) {
constexpr int kNumCalls = 123;
Random random_generator(seed());
ClippingPredictorEvaluator evaluator(history_size());
for (int i = 0; i < kNumCalls; ++i) {
SCOPED_TRACE(i);
// Read metrics before `Observe()` is called.
const auto pre = evaluator.counters();
// `Observe()` a random observation.
bool clipping_detected = random_generator.Rand<bool>();
bool clipping_predicted = random_generator.Rand<bool>();
evaluator.Observe(clipping_detected, clipping_predicted);
// Check that at most one metric has changed.
const auto post = evaluator.counters();
int num_changes = 0;
num_changes += pre.true_positives == post.true_positives ? 0 : 1;
num_changes += pre.true_negatives == post.true_negatives ? 0 : 1;
num_changes += pre.false_positives == post.false_positives ? 0 : 1;
num_changes += pre.false_negatives == post.false_negatives ? 0 : 1;
EXPECT_GE(num_changes, 0);
EXPECT_LE(num_changes, 1);
}
}
// Checks that after each call to `Observe()` each metric either remains
// unchanged or grows.
TEST_P(ClippingPredictorEvaluatorParameterization, MetricsAreWeaklyMonotonic) {
constexpr int kNumCalls = 123;
Random random_generator(seed());
ClippingPredictorEvaluator evaluator(history_size());
for (int i = 0; i < kNumCalls; ++i) {
SCOPED_TRACE(i);
// Read metrics before `Observe()` is called.
const auto pre = evaluator.counters();
// `Observe()` a random observation.
bool clipping_detected = random_generator.Rand<bool>();
bool clipping_predicted = random_generator.Rand<bool>();
evaluator.Observe(clipping_detected, clipping_predicted);
// Check that metrics are weakly monotonic.
const auto post = evaluator.counters();
EXPECT_GE(post.true_positives, pre.true_positives);
EXPECT_GE(post.true_negatives, pre.true_negatives);
EXPECT_GE(post.false_positives, pre.false_positives);
EXPECT_GE(post.false_negatives, pre.false_negatives);
}
}
// Checks that after each call to `Observe()` the growth speed of each metrics
// is bounded.
TEST_P(ClippingPredictorEvaluatorParameterization, BoundedMetricsGrowth) {
constexpr int kNumCalls = 123;
Random random_generator(seed());
ClippingPredictorEvaluator evaluator(history_size());
for (int i = 0; i < kNumCalls; ++i) {
SCOPED_TRACE(i);
// Read metrics before `Observe()` is called.
const auto pre = evaluator.counters();
// `Observe()` a random observation.
bool clipping_detected = random_generator.Rand<bool>();
bool clipping_predicted = random_generator.Rand<bool>();
evaluator.Observe(clipping_detected, clipping_predicted);
const auto diff = evaluator.counters() - pre;
// Check that TPs grow by at most `history_size() + 1`. Such an upper bound
// is reached when multiple predictions are matched by a single detection.
EXPECT_LE(diff.true_positives, history_size() + 1);
// Check that TNs, FPs and FNs grow by at most one.
EXPECT_LE(diff.true_negatives, 1);
EXPECT_LE(diff.false_positives, 1);
EXPECT_LE(diff.false_negatives, 1);
}
}
// Checks that `Observe()` returns a prediction interval if and only if one or
// more true positives are found.
TEST_P(ClippingPredictorEvaluatorParameterization,
PredictionIntervalIfAndOnlyIfTruePositives) {
constexpr int kNumCalls = 123;
Random random_generator(seed());
ClippingPredictorEvaluator evaluator(history_size());
for (int i = 0; i < kNumCalls; ++i) {
SCOPED_TRACE(i);
// Read true positives before `Observe()` is called.
const int last_tp = evaluator.counters().true_positives;
// `Observe()` a random observation.
bool clipping_detected = random_generator.Rand<bool>();
bool clipping_predicted = random_generator.Rand<bool>();
absl::optional<int> prediction_interval =
evaluator.Observe(clipping_detected, clipping_predicted);
// Check that the prediction interval is returned when a true positive is
// found.
if (evaluator.counters().true_positives == last_tp) {
EXPECT_FALSE(prediction_interval.has_value());
} else {
EXPECT_TRUE(prediction_interval.has_value());
}
}
}
INSTANTIATE_TEST_SUITE_P(
ClippingPredictionEvalTest,
ClippingPredictorEvaluatorParameterization,
::testing::Combine(::testing::Values(4, 8, 15, 16, 23, 42),
::testing::Values(1, 10, 21)));
// Checks that after initialization, when no detection is expected,
// observing no detection and no prediction produces a true negative.
TEST(ClippingPredictionEvalTest, TrueNegativeWithNoDetectNoPredictAfterInit) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 0);
EXPECT_EQ(evaluator.counters().true_negatives, 1);
EXPECT_EQ(evaluator.counters().false_positives, 0);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
// Checks that after initialization, when no detection is expected,
// observing no detection and prediction produces a true negative.
TEST(ClippingPredictionEvalTest, TrueNegativeWithNoDetectPredictAfterInit) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 0);
EXPECT_EQ(evaluator.counters().true_negatives, 1);
EXPECT_EQ(evaluator.counters().false_positives, 0);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
// Checks that after initialization, when no detection is expected,
// observing a detection and no prediction produces a false negative.
TEST(ClippingPredictionEvalTest, FalseNegativeWithDetectNoPredictAfterInit) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 0);
EXPECT_EQ(evaluator.counters().true_negatives, 0);
EXPECT_EQ(evaluator.counters().false_positives, 0);
EXPECT_EQ(evaluator.counters().false_negatives, 1);
}
// Checks that after initialization, when no detection is expected,
// simultaneously observing a detection and a prediction produces a false
// negative.
TEST(ClippingPredictionEvalTest, FalseNegativeWithDetectPredictAfterInit) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kDetected, kPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 0);
EXPECT_EQ(evaluator.counters().true_negatives, 0);
EXPECT_EQ(evaluator.counters().false_positives, 0);
EXPECT_EQ(evaluator.counters().false_negatives, 1);
}
// Checks that, after removing existing expectations, observing no detection and
// no prediction produces a true negative.
TEST(ClippingPredictionEvalTest,
TrueNegativeWithNoDetectNoPredictAfterRemoveExpectations) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
// Set an expectation, then remove it.
evaluator.Observe(kNotDetected, kPredicted);
evaluator.RemoveExpectations();
const auto pre = evaluator.counters();
evaluator.Observe(kNotDetected, kNotPredicted);
const auto diff = evaluator.counters() - pre;
EXPECT_EQ(diff.true_positives, 0);
EXPECT_EQ(diff.true_negatives, 1);
EXPECT_EQ(diff.false_positives, 0);
EXPECT_EQ(diff.false_negatives, 0);
}
// Checks that, after removing existing expectations, observing no detection and
// a prediction produces a true negative.
TEST(ClippingPredictionEvalTest,
TrueNegativeWithNoDetectPredictAfterRemoveExpectations) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
// Set an expectation, then remove it.
evaluator.Observe(kNotDetected, kPredicted);
evaluator.RemoveExpectations();
const auto pre = evaluator.counters();
evaluator.Observe(kNotDetected, kPredicted);
const auto diff = evaluator.counters() - pre;
EXPECT_EQ(diff.true_positives, 0);
EXPECT_EQ(diff.true_negatives, 1);
EXPECT_EQ(diff.false_positives, 0);
EXPECT_EQ(diff.false_negatives, 0);
}
// Checks that, after removing existing expectations, observing a detection and
// no prediction produces a false negative.
TEST(ClippingPredictionEvalTest,
FalseNegativeWithDetectNoPredictAfterRemoveExpectations) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
// Set an expectation, then remove it.
evaluator.Observe(kNotDetected, kPredicted);
evaluator.RemoveExpectations();
const auto pre = evaluator.counters();
evaluator.Observe(kDetected, kNotPredicted);
const auto diff = evaluator.counters() - pre;
EXPECT_EQ(diff.true_positives, 0);
EXPECT_EQ(diff.true_negatives, 0);
EXPECT_EQ(diff.false_positives, 0);
EXPECT_EQ(diff.false_negatives, 1);
}
// Checks that, after removing existing expectations, simultaneously observing a
// detection and a prediction produces a false negative.
TEST(ClippingPredictionEvalTest,
FalseNegativeWithDetectPredictAfterRemoveExpectations) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
// Set an expectation, then remove it.
evaluator.Observe(kNotDetected, kPredicted);
evaluator.RemoveExpectations();
const auto pre = evaluator.counters();
evaluator.Observe(kDetected, kPredicted);
const auto diff = evaluator.counters() - pre;
EXPECT_EQ(diff.false_negatives, 1);
EXPECT_EQ(diff.true_positives, 0);
EXPECT_EQ(diff.true_negatives, 0);
EXPECT_EQ(diff.false_positives, 0);
}
// Checks that the evaluator detects true negatives when clipping is neither
// predicted nor detected.
TEST(ClippingPredictionEvalTest, TrueNegativesWhenNeverDetectedOrPredicted) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kNotPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_negatives, 4);
}
// Checks that, until the observation period expires, the evaluator does not
// count a false positive when clipping is predicted and not detected.
TEST(ClippingPredictionEvalTest, PredictedOnceAndNeverDetectedBeforeDeadline) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().false_positives, 0);
evaluator.Observe(kNotDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().false_positives, 0);
evaluator.Observe(kNotDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 0);
EXPECT_EQ(evaluator.counters().false_positives, 1);
}
// Checks that, after the observation period expires, the evaluator detects a
// false positive when clipping is predicted and detected.
TEST(ClippingPredictionEvalTest, PredictedOnceButDetectedAfterDeadline) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
evaluator.Observe(kDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 0);
EXPECT_EQ(evaluator.counters().false_positives, 1);
}
// Checks that a prediction followed by a detection counts as true positive.
TEST(ClippingPredictionEvalTest, PredictedOnceAndThenImmediatelyDetected) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted);
evaluator.Observe(kDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 1);
EXPECT_EQ(evaluator.counters().false_positives, 0);
}
// Checks that a prediction followed by a delayed detection counts as true
// positive if the delay is within the observation period.
TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedBeforeDeadline) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
evaluator.Observe(kDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 1);
EXPECT_EQ(evaluator.counters().false_positives, 0);
}
// Checks that a prediction followed by a delayed detection counts as true
// positive if the delay equals the observation period.
TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedAtDeadline) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
evaluator.Observe(kDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 1);
EXPECT_EQ(evaluator.counters().false_positives, 0);
}
// Checks that a prediction followed by a multiple adjacent detections within
// the deadline counts as a single true positive and that, after the deadline,
// a detection counts as a false negative.
TEST(ClippingPredictionEvalTest, PredictedOnceAndDetectedMultipleTimes) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
// Multiple detections.
evaluator.Observe(kDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 1);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
EXPECT_EQ(evaluator.counters().false_positives, 0);
evaluator.Observe(kDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 1);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
EXPECT_EQ(evaluator.counters().false_positives, 0);
// A detection outside of the observation period counts as false negative.
evaluator.Observe(kDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_positives, 1);
EXPECT_EQ(evaluator.counters().false_negatives, 1);
EXPECT_EQ(evaluator.counters().false_positives, 0);
}
// Checks that when clipping is predicted multiple times, a prediction that is
// observed too early counts as a false positive, whereas the other predictions
// that are matched to a detection count as true positives.
TEST(ClippingPredictionEvalTest,
PredictedMultipleTimesAndDetectedOnceAfterDeadline) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted); // ---+
evaluator.Observe(kNotDetected, kPredicted); // |
evaluator.Observe(kNotDetected, kPredicted); // |
evaluator.Observe(kNotDetected, kPredicted); // <--+ Not matched.
// The time to match a detection after the first prediction expired.
EXPECT_EQ(evaluator.counters().false_positives, 1);
evaluator.Observe(kDetected, kNotPredicted);
// The detection above does not match the first prediction because it happened
// after the deadline of the 1st prediction.
EXPECT_EQ(evaluator.counters().false_positives, 1);
// However, the detection matches all the other predictions.
EXPECT_EQ(evaluator.counters().true_positives, 3);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
// Checks that multiple consecutive predictions match the first detection
// observed before the expected detection deadline expires.
TEST(ClippingPredictionEvalTest, PredictedMultipleTimesAndDetectedOnce) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted); // --+
evaluator.Observe(kNotDetected, kPredicted); // | --+
evaluator.Observe(kNotDetected, kPredicted); // | | --+
evaluator.Observe(kDetected, kNotPredicted); // <-+ <-+ <-+
EXPECT_EQ(evaluator.counters().true_positives, 3);
// The following observations do not generate any true negatives as they
// belong to the observation period of the last prediction - for which a
// detection has already been matched.
const int true_negatives = evaluator.counters().true_negatives;
evaluator.Observe(kNotDetected, kNotPredicted);
evaluator.Observe(kNotDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_negatives, true_negatives);
EXPECT_EQ(evaluator.counters().false_positives, 0);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
// Checks that multiple consecutive predictions match the multiple detections
// observed before the expected detection deadline expires.
TEST(ClippingPredictionEvalTest,
PredictedMultipleTimesAndDetectedMultipleTimes) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted); // --+
evaluator.Observe(kNotDetected, kPredicted); // | --+
evaluator.Observe(kNotDetected, kPredicted); // | | --+
evaluator.Observe(kDetected, kNotPredicted); // <-+ <-+ <-+
evaluator.Observe(kDetected, kNotPredicted); // <-+ <-+
EXPECT_EQ(evaluator.counters().true_positives, 3);
// The following observation does not generate a true negative as it belongs
// to the observation period of the last prediction - for which two detections
// have already been matched.
const int true_negatives = evaluator.counters().true_negatives;
evaluator.Observe(kNotDetected, kNotPredicted);
EXPECT_EQ(evaluator.counters().true_negatives, true_negatives);
EXPECT_EQ(evaluator.counters().false_positives, 0);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
// Checks that multiple consecutive predictions match all the detections
// observed before the expected detection deadline expires.
TEST(ClippingPredictionEvalTest, PredictedMultipleTimesAndAllDetected) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted); // --+
evaluator.Observe(kNotDetected, kPredicted); // | --+
evaluator.Observe(kNotDetected, kPredicted); // | | --+
evaluator.Observe(kDetected, kNotPredicted); // <-+ <-+ <-+
evaluator.Observe(kDetected, kNotPredicted); // <-+ <-+
evaluator.Observe(kDetected, kNotPredicted); // <-+
EXPECT_EQ(evaluator.counters().true_positives, 3);
EXPECT_EQ(evaluator.counters().false_positives, 0);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
// Checks that multiple non-consecutive predictions match all the detections
// observed before the expected detection deadline expires.
TEST(ClippingPredictionEvalTest, PredictedMultipleTimesWithGapAndAllDetected) {
ClippingPredictorEvaluator evaluator(/*history_size=*/3);
evaluator.Observe(kNotDetected, kPredicted); // --+
evaluator.Observe(kNotDetected, kNotPredicted); // |
evaluator.Observe(kNotDetected, kPredicted); // | --+
evaluator.Observe(kDetected, kNotPredicted); // <-+ <-+
evaluator.Observe(kDetected, kNotPredicted); // <-+
evaluator.Observe(kDetected, kNotPredicted); // <-+
EXPECT_EQ(evaluator.counters().true_positives, 2);
EXPECT_EQ(evaluator.counters().false_positives, 0);
EXPECT_EQ(evaluator.counters().false_negatives, 0);
}
class ClippingPredictorEvaluatorPredictionIntervalParameterization
: public ::testing::TestWithParam<std::tuple<int, int>> {
protected:
int num_extra_observe_calls() const { return std::get<0>(GetParam()); }
int history_size() const { return std::get<1>(GetParam()); }
};
// Checks that the minimum prediction interval is returned if clipping is
// correctly predicted just before clipping is detected - i.e., smallest
// anticipation.
TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization,
MinimumPredictionInterval) {
ClippingPredictorEvaluator evaluator(history_size());
for (int i = 0; i < num_extra_observe_calls(); ++i) {
EXPECT_EQ(evaluator.Observe(kNotDetected, kNotPredicted), absl::nullopt);
}
EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
EXPECT_THAT(evaluator.Observe(kDetected, kNotPredicted), Optional(Eq(1)));
}
// Checks that a prediction interval between the minimum and the maximum is
// returned if clipping is correctly predicted before it is detected but not as
// early as possible.
TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization,
IntermediatePredictionInterval) {
ClippingPredictorEvaluator evaluator(history_size());
for (int i = 0; i < num_extra_observe_calls(); ++i) {
EXPECT_EQ(evaluator.Observe(kNotDetected, kNotPredicted), absl::nullopt);
}
EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
EXPECT_THAT(evaluator.Observe(kDetected, kNotPredicted), Optional(Eq(3)));
}
// Checks that the maximum prediction interval is returned if clipping is
// correctly predicted as early as possible.
TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization,
MaximumPredictionInterval) {
ClippingPredictorEvaluator evaluator(history_size());
for (int i = 0; i < num_extra_observe_calls(); ++i) {
EXPECT_EQ(evaluator.Observe(kNotDetected, kNotPredicted), absl::nullopt);
}
for (int i = 0; i < history_size(); ++i) {
EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
}
EXPECT_THAT(evaluator.Observe(kDetected, kNotPredicted),
Optional(Eq(history_size())));
}
// Checks that `Observe()` returns the prediction interval as soon as a true
// positive is found and never again while ongoing detections are matched to a
// previously observed prediction.
TEST_P(ClippingPredictorEvaluatorPredictionIntervalParameterization,
PredictionIntervalReturnedOnce) {
ASSERT_LT(num_extra_observe_calls(), history_size());
ClippingPredictorEvaluator evaluator(history_size());
// Observe predictions before detection.
for (int i = 0; i < num_extra_observe_calls(); ++i) {
EXPECT_EQ(evaluator.Observe(kNotDetected, kPredicted), absl::nullopt);
}
// Observe a detection.
absl::optional<int> prediction_interval =
evaluator.Observe(kDetected, kNotPredicted);
EXPECT_TRUE(prediction_interval.has_value());
// `Observe()` does not return a prediction interval anymore during ongoing
// detections observed while a detection is still expected.
for (int i = 0; i < history_size(); ++i) {
EXPECT_EQ(evaluator.Observe(kDetected, kNotPredicted), absl::nullopt);
}
}
INSTANTIATE_TEST_SUITE_P(
ClippingPredictionEvalTest,
ClippingPredictorEvaluatorPredictionIntervalParameterization,
::testing::Combine(::testing::Values(1, 3, 5), ::testing::Values(7, 11)));
// Checks that, when a detection is expected, the expectation is not removed
// before the detection deadline expires unless `RemoveExpectations()` is
// called.
TEST(ClippingPredictionEvalTest, NoFalsePositivesAfterRemoveExpectations) {
constexpr int kHistorySize = 2;
// Case 1: `RemoveExpectations()` is NOT called.
ClippingPredictorEvaluator e1(kHistorySize);
e1.Observe(kNotDetected, kPredicted);
ASSERT_EQ(e1.counters().true_negatives, 1);
e1.Observe(kNotDetected, kNotPredicted);
e1.Observe(kNotDetected, kNotPredicted);
EXPECT_EQ(e1.counters().true_positives, 0);
EXPECT_EQ(e1.counters().true_negatives, 1);
EXPECT_EQ(e1.counters().false_positives, 1);
EXPECT_EQ(e1.counters().false_negatives, 0);
// Case 2: `RemoveExpectations()` is called.
ClippingPredictorEvaluator e2(kHistorySize);
e2.Observe(kNotDetected, kPredicted);
ASSERT_EQ(e2.counters().true_negatives, 1);
e2.RemoveExpectations();
e2.Observe(kNotDetected, kNotPredicted);
e2.Observe(kNotDetected, kNotPredicted);
EXPECT_EQ(e2.counters().true_positives, 0);
EXPECT_EQ(e2.counters().true_negatives, 3);
EXPECT_EQ(e2.counters().false_positives, 0);
EXPECT_EQ(e2.counters().false_negatives, 0);
}
class ComputeClippingPredictionMetricsParameterization
: public ::testing::TestWithParam<int> {
protected:
int true_negatives() const { return GetParam(); }
};
// Checks that `ComputeClippingPredictionMetrics()` does not return metrics if
// precision cannot be defined - i.e., TP + FP is zero.
TEST_P(ComputeClippingPredictionMetricsParameterization,
NoMetricsWithUndefinedPrecision) {
EXPECT_EQ(ComputeClippingPredictionMetrics(
/*counters=*/{/*true_positives=*/0,
/*true_negatives=*/true_negatives(),
/*false_positives=*/0,
/*false_negatives=*/0}),
absl::nullopt);
EXPECT_EQ(ComputeClippingPredictionMetrics(
/*counters=*/{/*true_positives=*/0,
/*true_negatives=*/true_negatives(),
/*false_positives=*/0,
/*false_negatives=*/1}),
absl::nullopt);
}
// Checks that `ComputeClippingPredictionMetrics()` does not return metrics if
// recall cannot be defined - i.e., TP + FN is zero.
TEST_P(ComputeClippingPredictionMetricsParameterization,
NoMetricsWithUndefinedRecall) {
EXPECT_EQ(ComputeClippingPredictionMetrics(
/*counters=*/{/*true_positives=*/0,
/*true_negatives=*/true_negatives(),
/*false_positives=*/0,
/*false_negatives=*/0}),
absl::nullopt);
EXPECT_EQ(ComputeClippingPredictionMetrics(
/*counters=*/{/*true_positives=*/0,
/*true_negatives=*/true_negatives(),
/*false_positives=*/1,
/*false_negatives=*/0}),
absl::nullopt);
}
// Checks that `ComputeClippingPredictionMetrics()` does not return metrics if
// the F1 score cannot be defined - i.e., P + R is zero.
TEST_P(ComputeClippingPredictionMetricsParameterization,
NoMetricsWithUndefinedF1Score) {
EXPECT_EQ(ComputeClippingPredictionMetrics(
/*counters=*/{/*true_positives=*/0,
/*true_negatives=*/true_negatives(),
/*false_positives=*/1,
/*false_negatives=*/1}),
absl::nullopt);
}
// Checks that the highest precision is reached when there are no false
// positives.
TEST_P(ComputeClippingPredictionMetricsParameterization, HighestPrecision) {
EXPECT_THAT(ComputeClippingPredictionMetrics(
/*counters=*/{/*true_positives=*/1,
/*true_negatives=*/true_negatives(),
/*false_positives=*/0,
/*false_negatives=*/1}),
Optional(Field(&ClippingPredictionMetrics::precision, Eq(1.0f))));
}
// Checks that the highest recall is reached when there are no false
// negatives.
TEST_P(ComputeClippingPredictionMetricsParameterization, HighestRecall) {
EXPECT_THAT(ComputeClippingPredictionMetrics(
/*counters=*/{/*true_positives=*/1,
/*true_negatives=*/true_negatives(),
/*false_positives=*/1,
/*false_negatives=*/0}),
Optional(Field(&ClippingPredictionMetrics::recall, Eq(1.0f))));
}
// Checks that 50% precision and 50% recall is reached when the number of true
// positives, false positives and false negatives are the same.
TEST_P(ComputeClippingPredictionMetricsParameterization,
PrecisionAndRecall50Percent) {
absl::optional<ClippingPredictionMetrics> metrics =
ComputeClippingPredictionMetrics(
/*counters=*/{/*true_positives=*/42,
/*true_negatives=*/true_negatives(),
/*false_positives=*/42,
/*false_negatives=*/42});
ASSERT_TRUE(metrics.has_value());
EXPECT_EQ(metrics->precision, 0.5f);
EXPECT_EQ(metrics->recall, 0.5f);
EXPECT_EQ(metrics->f1_score, 0.5f);
}
// Checks that the highest precision, recall and F1 score are jointly reached
// when there are no false positives and no false negatives.
TEST_P(ComputeClippingPredictionMetricsParameterization,
HighestPrecisionRecallF1Score) {
absl::optional<ClippingPredictionMetrics> metrics =
ComputeClippingPredictionMetrics(
/*counters=*/{/*true_positives=*/123,
/*true_negatives=*/true_negatives(),
/*false_positives=*/0,
/*false_negatives=*/0});
ASSERT_TRUE(metrics.has_value());
EXPECT_EQ(metrics->precision, 1.0f);
EXPECT_EQ(metrics->recall, 1.0f);
EXPECT_EQ(metrics->f1_score, 1.0f);
}
// Checks that precision is lower than recall when there are more false
// positives than false negatives.
TEST_P(ComputeClippingPredictionMetricsParameterization,
PrecisionLowerThanRecall) {
absl::optional<ClippingPredictionMetrics> metrics =
ComputeClippingPredictionMetrics(
/*counters=*/{/*true_positives=*/1,
/*true_negatives=*/true_negatives(),
/*false_positives=*/8,
/*false_negatives=*/1});
ASSERT_TRUE(metrics.has_value());
EXPECT_LT(metrics->precision, metrics->recall);
}
// Checks that precision is greater than recall when there are less false
// positives than false negatives.
TEST_P(ComputeClippingPredictionMetricsParameterization,
PrecisionGreaterThanRecall) {
absl::optional<ClippingPredictionMetrics> metrics =
ComputeClippingPredictionMetrics(
/*counters=*/{/*true_positives=*/1,
/*true_negatives=*/true_negatives(),
/*false_positives=*/1,
/*false_negatives=*/8});
ASSERT_TRUE(metrics.has_value());
EXPECT_GT(metrics->precision, metrics->recall);
}
// Checks that swapping precision and recall does not change the F1 score.
TEST_P(ComputeClippingPredictionMetricsParameterization, SameF1Score) {
absl::optional<ClippingPredictionMetrics> m1 =
ComputeClippingPredictionMetrics(
/*counters=*/{/*true_positives=*/1,
/*true_negatives=*/true_negatives(),
/*false_positives=*/8,
/*false_negatives=*/1});
absl::optional<ClippingPredictionMetrics> m2 =
ComputeClippingPredictionMetrics(
/*counters=*/{/*true_positives=*/1,
/*true_negatives=*/true_negatives(),
/*false_positives=*/1,
/*false_negatives=*/8});
// Preconditions.
ASSERT_TRUE(m1.has_value());
ASSERT_TRUE(m2.has_value());
ASSERT_EQ(m1->precision, m2->recall);
ASSERT_EQ(m1->recall, m2->precision);
// Same F1 score.
EXPECT_EQ(m1->f1_score, m2->f1_score);
}
INSTANTIATE_TEST_SUITE_P(ClippingPredictionEvalTest,
ComputeClippingPredictionMetricsParameterization,
::testing::Values(0, 1, 11));
} // namespace
} // namespace webrtc