mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-13 05:40:42 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

[PCLF] Propagate relevant metadata to all metrics

Browse source 
Bug: None
Change-Id: Ifcb67a59b68cc3468dd06e932a2a3da7b40d9845
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/281680
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38545}
This commit is contained in:
Artem Titov 2022-11-03 16:40:53 +01:00 committed by WebRTC LUCI CQ
parent 42e5ed38a7
commit 15b97d6d90
13 changed files with 350 additions and 165 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
api/test/track_id_stream_info_map.h
View file

@ -11,6 +11,8 @@
#ifndef API_TEST_TRACK_ID_STREAM_INFO_MAP_H_
#define API_TEST_TRACK_ID_STREAM_INFO_MAP_H_
#include <string>
#include "absl/strings/string_view.h"
namespace webrtc {
@ -20,19 +22,19 @@ namespace webrtc_pc_e2e {
// are useful to associate stats reports track_ids to the remote stream info.
class TrackIdStreamInfoMap {
public:
struct StreamInfo {
std::string receiver_peer;
std::string stream_label;
std::string sync_group;
};
virtual ~TrackIdStreamInfoMap() = default;
// These methods must be called on the same thread where
// StatsObserverInterface::OnStatsReports is invoked.
// Returns a reference to a stream label owned by the TrackIdStreamInfoMap.
// Precondition: `track_id` must be already mapped to stream label.
virtual absl::string_view GetStreamLabelFromTrackId(
absl::string_view track_id) const = 0;
// Returns a reference to a sync group name owned by the TrackIdStreamInfoMap.
// Precondition: `track_id` must be already mapped to sync group.
virtual absl::string_view GetSyncGroupLabelFromTrackId(
// Precondition: `track_id` must be already mapped to stream info.
virtual StreamInfo GetStreamInfoFromTrackId(
absl::string_view track_id) const = 0;
};

9
test/pc/e2e/BUILD.gn
View file

@ -488,6 +488,7 @@ if (!build_with_chromium) {
":default_audio_quality_analyzer",
":default_video_quality_analyzer",
":media_helper",
":metric_metadata_keys",
":peer_configurer",
":peer_connection_quality_test_params",
":sdp_changer",
@ -832,7 +833,10 @@ if (!build_with_chromium) {
"../../../api:track_id_stream_info_map",
"../../../rtc_base:macromagic",
]
absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
absl_deps = [
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
]
}
rtc_library("default_audio_quality_analyzer") {
@ -844,6 +848,7 @@ if (!build_with_chromium) {
]
deps = [
":metric_metadata_keys",
"../..:perf_test",
"../../../api:audio_quality_analyzer_api",
"../../../api:rtc_stats_api",
@ -1066,6 +1071,7 @@ if (!build_with_chromium) {
"stats_based_network_quality_metrics_reporter.h",
]
deps = [
":metric_metadata_keys",
"../..:perf_test",
"../../../api:array_view",
"../../../api:network_emulation_manager_api",
@ -1098,6 +1104,7 @@ if (!build_with_chromium) {
"cross_media_metrics_reporter.h",
]
deps = [
":metric_metadata_keys",
"../..:perf_test",
"../../../api:network_emulation_manager_api",
"../../../api:peer_connection_quality_test_fixture_api",

42
test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc
View file

@ -13,8 +13,10 @@
#include "api/stats/rtc_stats.h"
#include "api/stats/rtcstats_objects.h"
#include "api/test/metrics/metric.h"
#include "api/test/track_id_stream_info_map.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "test/pc/e2e/metric_metadata_keys.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@ -62,11 +64,12 @@ void DefaultAudioQualityAnalyzer::OnStatsReports(
sample.jitter_buffer_emitted_count =
stat->jitter_buffer_emitted_count.ValueOrDefault(0ul);
const std::string stream_label = std::string(
analyzer_helper_->GetStreamLabelFromTrackId(*stat->track_identifier));
TrackIdStreamInfoMap::StreamInfo stream_info =
analyzer_helper_->GetStreamInfoFromTrackId(*stat->track_identifier);
MutexLock lock(&lock_);
StatsSample prev_sample = last_stats_sample_[stream_label];
stream_info_.emplace(stream_info.stream_label, stream_info);
StatsSample prev_sample = last_stats_sample_[stream_info.stream_label];
RTC_CHECK_GE(sample.total_samples_received,
prev_sample.total_samples_received);
double total_samples_diff = static_cast<double>(
@ -75,7 +78,8 @@ void DefaultAudioQualityAnalyzer::OnStatsReports(
return;
}
AudioStreamStats& audio_stream_stats = streams_stats_[stream_label];
AudioStreamStats& audio_stream_stats =
streams_stats_[stream_info.stream_label];
audio_stream_stats.expand_rate.AddSample(
(sample.concealed_samples - prev_sample.concealed_samples) /
total_samples_diff);
@ -113,7 +117,7 @@ void DefaultAudioQualityAnalyzer::OnStatsReports(
jitter_buffer_emitted_count_diff);
}
last_stats_sample_[stream_label] = sample;
last_stats_sample_[stream_info.stream_label] = sample;
}
}
@ -125,27 +129,37 @@ std::string DefaultAudioQualityAnalyzer::GetTestCaseName(
void DefaultAudioQualityAnalyzer::Stop() {
MutexLock lock(&lock_);
for (auto& item : streams_stats_) {
const TrackIdStreamInfoMap::StreamInfo& stream_info =
stream_info_[item.first];
std::map<std::string, std::string> metric_metadata{
{MetricMetadataKey::kAudioStreamMetadataKey, item.first},
{MetricMetadataKey::kPeerMetadataKey, stream_info.receiver_peer},
{MetricMetadataKey::kReceiverMetadataKey, stream_info.receiver_peer}};
metrics_logger_->LogMetric("expand_rate", GetTestCaseName(item.first),
item.second.expand_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
ImprovementDirection::kSmallerIsBetter,
metric_metadata);
metrics_logger_->LogMetric("accelerate_rate", GetTestCaseName(item.first),
item.second.accelerate_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
ImprovementDirection::kSmallerIsBetter,
metric_metadata);
metrics_logger_->LogMetric("preemptive_rate", GetTestCaseName(item.first),
item.second.preemptive_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("speech_expand_rate",
GetTestCaseName(item.first),
item.second.speech_expand_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
ImprovementDirection::kSmallerIsBetter,
metric_metadata);
metrics_logger_->LogMetric(
"speech_expand_rate", GetTestCaseName(item.first),
item.second.speech_expand_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter, metric_metadata);
metrics_logger_->LogMetric(
"average_jitter_buffer_delay_ms", GetTestCaseName(item.first),
item.second.average_jitter_buffer_delay_ms, Unit::kMilliseconds,
ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter, metric_metadata);
metrics_logger_->LogMetric(
"preferred_buffer_size_ms", GetTestCaseName(item.first),
item.second.preferred_buffer_size_ms, Unit::kMilliseconds,
ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter, metric_metadata);
}
}

2
test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h
View file

@ -70,6 +70,8 @@ class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface {
mutable Mutex lock_;
std::map<std::string, AudioStreamStats> streams_stats_ RTC_GUARDED_BY(lock_);
std::map<std::string, TrackIdStreamInfoMap::StreamInfo> stream_info_
RTC_GUARDED_BY(lock_);
std::map<std::string, StatsSample> last_stats_sample_ RTC_GUARDED_BY(lock_);
};

3
test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
View file

@ -942,7 +942,8 @@ void DefaultVideoQualityAnalyzer::ReportResults(
std::string test_case_name = GetTestCaseName(ToMetricName(key));
std::map<std::string, std::string> metric_metadata{
{MetricMetadataKey::kPeerMetadataKey, peers_->name(key.sender)},
{MetricMetadataKey::kStreamMetadataKey, streams_.name(key.stream)},
{MetricMetadataKey::kVideoStreamMetadataKey, streams_.name(key.stream)},
{MetricMetadataKey::kSenderMetadataKey, peers_->name(key.sender)},
{MetricMetadataKey::kReceiverMetadataKey, peers_->name(key.receiver)}};
double sum_squared_interframe_delays_secs = 0;

28
test/pc/e2e/analyzer_helper.cc
View file

@ -10,6 +10,7 @@
#include "test/pc/e2e/analyzer_helper.h"
#include <string>
#include <utility>
namespace webrtc {
@ -19,6 +20,21 @@ AnalyzerHelper::AnalyzerHelper() {
signaling_sequence_checker_.Detach();
}
void AnalyzerHelper::AddTrackToStreamMapping(
absl::string_view track_id,
absl::string_view receiver_peer,
absl::string_view stream_label,
absl::optional<std::string> sync_group) {
RTC_DCHECK_RUN_ON(&signaling_sequence_checker_);
track_to_stream_map_.insert(
{std::string(track_id),
StreamInfo{.receiver_peer = std::string(receiver_peer),
.stream_label = std::string(stream_label),
.sync_group = sync_group.has_value()
? *sync_group
: std::string(stream_label)}});
}
void AnalyzerHelper::AddTrackToStreamMapping(std::string track_id,
std::string stream_label) {
RTC_DCHECK_RUN_ON(&signaling_sequence_checker_);
@ -35,7 +51,7 @@ void AnalyzerHelper::AddTrackToStreamMapping(std::string track_id,
StreamInfo{std::move(stream_label), std::move(sync_group)}});
}
const AnalyzerHelper::StreamInfo& AnalyzerHelper::GetStreamInfoFromTrackId(
AnalyzerHelper::StreamInfo AnalyzerHelper::GetStreamInfoFromTrackId(
absl::string_view track_id) const {
RTC_DCHECK_RUN_ON(&signaling_sequence_checker_);
auto track_to_stream_pair = track_to_stream_map_.find(std::string(track_id));
@ -43,15 +59,5 @@ const AnalyzerHelper::StreamInfo& AnalyzerHelper::GetStreamInfoFromTrackId(
return track_to_stream_pair->second;
}
absl::string_view AnalyzerHelper::GetStreamLabelFromTrackId(
absl::string_view track_id) const {
return GetStreamInfoFromTrackId(track_id).stream_label;
}
absl::string_view AnalyzerHelper::GetSyncGroupLabelFromTrackId(
absl::string_view track_id) const {
return GetStreamInfoFromTrackId(track_id).sync_group;
}
} // namespace webrtc_pc_e2e
} // namespace webrtc

19
test/pc/e2e/analyzer_helper.h
View file

@ -15,6 +15,7 @@
#include <string>
#include "absl/strings/string_view.h"
#include "absl/types/optional.h"
#include "api/sequence_checker.h"
#include "api/test/track_id_stream_info_map.h"
#include "rtc_base/thread_annotations.h"
@ -31,30 +32,24 @@ namespace webrtc_pc_e2e {
// AddTrackToStreamMapping, GetStreamLabelFromTrackId and
// GetSyncGroupLabelFromTrackId must be invoked from the signaling thread. Get
// methods should be invoked only after all data is added. Mixing Get methods
// with adding new data may lead to undefined behaviour.
// with adding new data may lead to undefined behavior.
class AnalyzerHelper : public TrackIdStreamInfoMap {
public:
AnalyzerHelper();
void AddTrackToStreamMapping(absl::string_view track_id,
absl::string_view receiver_peer,
absl::string_view stream_label,
absl::optional<std::string> sync_group);
void AddTrackToStreamMapping(std::string track_id, std::string stream_label);
void AddTrackToStreamMapping(std::string track_id,
std::string stream_label,
std::string sync_group);
absl::string_view GetStreamLabelFromTrackId(
absl::string_view track_id) const override;
absl::string_view GetSyncGroupLabelFromTrackId(
StreamInfo GetStreamInfoFromTrackId(
absl::string_view track_id) const override;
private:
struct StreamInfo {
std::string stream_label;
std::string sync_group;
};
const StreamInfo& GetStreamInfoFromTrackId(absl::string_view track_id) const;
SequenceChecker signaling_sequence_checker_;
std::map<std::string, StreamInfo> track_to_stream_map_
RTC_GUARDED_BY(signaling_sequence_checker_);

48
test/pc/e2e/cross_media_metrics_reporter.cc
View file

@ -19,6 +19,7 @@
#include "rtc_base/checks.h"
#include "rtc_base/event.h"
#include "system_wrappers/include/field_trial.h"
#include "test/pc/e2e/metric_metadata_keys.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@ -43,15 +44,17 @@ void CrossMediaMetricsReporter::OnStatsReports(
absl::string_view pc_label,
const rtc::scoped_refptr<const RTCStatsReport>& report) {
auto inbound_stats = report->GetStatsOfType<RTCInboundRTPStreamStats>();
std::map<absl::string_view, std::vector<const RTCInboundRTPStreamStats*>>
std::map<std::string, std::vector<const RTCInboundRTPStreamStats*>>
sync_group_stats;
for (const auto& stat : inbound_stats) {
auto media_source_stat =
report->GetAs<DEPRECATED_RTCMediaStreamTrackStats>(*stat->track_id);
if (stat->estimated_playout_timestamp.ValueOrDefault(0.) > 0 &&
media_source_stat->track_identifier.is_defined()) {
sync_group_stats[reporter_helper_->GetSyncGroupLabelFromTrackId(
*media_source_stat->track_identifier)]
sync_group_stats[reporter_helper_
->GetStreamInfoFromTrackId(
*media_source_stat->track_identifier)
.sync_group]
.push_back(stat);
}
}
@ -85,12 +88,12 @@ void CrossMediaMetricsReporter::OnStatsReports(
*video_stat->track_id);
// *_source_stat->track_identifier is always defined here because we
// checked it while grouping stats.
stats_info_[sync_group].audio_stream_label =
std::string(reporter_helper_->GetStreamLabelFromTrackId(
*audio_source_stat->track_identifier));
stats_info_[sync_group].video_stream_label =
std::string(reporter_helper_->GetStreamLabelFromTrackId(
*video_source_stat->track_identifier));
stats_info_[sync_group].audio_stream_info =
reporter_helper_->GetStreamInfoFromTrackId(
*audio_source_stat->track_identifier);
stats_info_[sync_group].video_stream_info =
reporter_helper_->GetStreamInfoFromTrackId(
*video_source_stat->track_identifier);
}
double audio_video_playout_diff = *audio_stat->estimated_playout_timestamp -
@ -111,16 +114,35 @@ void CrossMediaMetricsReporter::StopAndReportResults() {
MutexLock lock(&mutex_);
for (const auto& pair : stats_info_) {
const std::string& sync_group = pair.first;
std::map<std::string, std::string> audio_metric_metadata{
{MetricMetadataKey::kPeerSyncGroupMetadataKey, sync_group},
{MetricMetadataKey::kAudioStreamMetadataKey,
pair.second.audio_stream_info.stream_label},
{MetricMetadataKey::kPeerMetadataKey,
pair.second.audio_stream_info.receiver_peer},
{MetricMetadataKey::kReceiverMetadataKey,
pair.second.audio_stream_info.receiver_peer}};
metrics_logger_->LogMetric(
"audio_ahead_ms",
GetTestCaseName(pair.second.audio_stream_label, sync_group),
GetTestCaseName(pair.second.audio_stream_info.stream_label, sync_group),
pair.second.audio_ahead_ms, Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter);
webrtc::test::ImprovementDirection::kSmallerIsBetter,
std::move(audio_metric_metadata));
std::map<std::string, std::string> video_metric_metadata{
{MetricMetadataKey::kPeerSyncGroupMetadataKey, sync_group},
{MetricMetadataKey::kAudioStreamMetadataKey,
pair.second.video_stream_info.stream_label},
{MetricMetadataKey::kPeerMetadataKey,
pair.second.video_stream_info.receiver_peer},
{MetricMetadataKey::kReceiverMetadataKey,
pair.second.video_stream_info.receiver_peer}};
metrics_logger_->LogMetric(
"video_ahead_ms",
GetTestCaseName(pair.second.video_stream_label, sync_group),
GetTestCaseName(pair.second.video_stream_info.stream_label, sync_group),
pair.second.video_ahead_ms, Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter);
webrtc::test::ImprovementDirection::kSmallerIsBetter,
std::move(video_metric_metadata));
}
}

2
test/pc/e2e/cross_media_metrics_reporter.h
View file

@ -44,6 +44,8 @@ class CrossMediaMetricsReporter
SamplesStatsCounter audio_ahead_ms;
SamplesStatsCounter video_ahead_ms;
TrackIdStreamInfoMap::StreamInfo audio_stream_info;
TrackIdStreamInfoMap::StreamInfo video_stream_info;
std::string audio_stream_label;
std::string video_stream_label;
};

14
test/pc/e2e/metric_metadata_keys.h
View file

@ -13,18 +13,30 @@
namespace webrtc {
namespace webrtc_pc_e2e {
// All metadata fields are present only if applicable for particular metric.
class MetricMetadataKey {
public:
// Represents on peer with whom the metric is associated.
static constexpr char kPeerMetadataKey[] = "peer";
static constexpr char kStreamMetadataKey[] = "stream";
// Represents sender of the media stream.
static constexpr char kSenderMetadataKey[] = "sender";
// Represents receiver of the media stream.
static constexpr char kReceiverMetadataKey[] = "receiver";
// Represents name of the audio stream.
static constexpr char kAudioStreamMetadataKey[] = "audio_stream";
// Represents name of the video stream.
static constexpr char kVideoStreamMetadataKey[] = "video_stream";
// Represents name of the sync group to which stream belongs.
static constexpr char kPeerSyncGroupMetadataKey[] = "peer_sync_group";
private:
MetricMetadataKey() = default;
};
// All metadata fields are presented only if applicable for particular metric.
class SampleMetadataKey {
public:
// Represents a frame ID with which data point is associated.
static constexpr char kFrameIdMetadataKey[] = "frame_id";
private:

9
test/pc/e2e/peer_connection_quality_test.cc
View file

@ -38,6 +38,7 @@
#include "test/pc/e2e/analyzer/video/video_frame_tracking_id_injector.h"
#include "test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h"
#include "test/pc/e2e/cross_media_metrics_reporter.h"
#include "test/pc/e2e/metric_metadata_keys.h"
#include "test/pc/e2e/stats_poller.h"
#include "test/pc/e2e/test_peer_factory.h"
#include "test/testsupport/file_utils.h"
@ -463,7 +464,7 @@ void PeerConnectionE2EQualityTest::OnTrackCallback(
<< "Expected 2 stream ids: 1st - sync group, 2nd - unique stream label";
std::string sync_group = transceiver->receiver()->stream_ids()[0];
std::string stream_label = transceiver->receiver()->stream_ids()[1];
analyzer_helper_.AddTrackToStreamMapping(track->id(), stream_label,
analyzer_helper_.AddTrackToStreamMapping(track->id(), peer_name, stream_label,
sync_group);
if (track->kind() != MediaStreamTrackInterface::kVideoKind) {
return;
@ -744,10 +745,12 @@ void PeerConnectionE2EQualityTest::TearDownCall() {
void PeerConnectionE2EQualityTest::ReportGeneralTestResults() {
metrics_logger_->LogSingleValueMetric(
*alice_->params().name + "_connected", test_case_name_, alice_connected_,
Unit::kUnitless, ImprovementDirection::kBiggerIsBetter);
Unit::kUnitless, ImprovementDirection::kBiggerIsBetter,
{{MetricMetadataKey::kPeerMetadataKey, *alice_->params().name}});
metrics_logger_->LogSingleValueMetric(
*bob_->params().name + "_connected", test_case_name_, bob_connected_,
Unit::kUnitless, ImprovementDirection::kBiggerIsBetter);
Unit::kUnitless, ImprovementDirection::kBiggerIsBetter,
{{MetricMetadataKey::kPeerMetadataKey, *bob_->params().name}});
}
Timestamp PeerConnectionE2EQualityTest::Now() const {

272
test/pc/e2e/peer_connection_quality_test_metric_names_test.cc
View file

@ -163,13 +163,13 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.name = "alice_connected",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case",
.name = "bob_connected",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
// Metrics from DefaultAudioQualityAnalyzer
MetricValidationInfo{
@ -177,73 +177,109 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.name = "expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"alice_audio"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "accelerate_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"alice_audio"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "preemptive_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"alice_audio"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "speech_expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"alice_audio"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "average_jitter_buffer_delay_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"alice_audio"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_audio",
.name = "preferred_buffer_size_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"alice_audio"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"bob_audio"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "accelerate_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"bob_audio"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "preemptive_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"bob_audio"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "speech_expand_rate",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"bob_audio"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "average_jitter_buffer_delay_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"bob_audio"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_audio",
.name = "preferred_buffer_size_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kAudioStreamMetadataKey,
"bob_audio"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
// Metrics from DefaultVideoQualityAnalyzer
MetricValidationInfo{
@ -252,8 +288,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -261,8 +298,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -270,8 +308,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -279,8 +318,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -288,8 +328,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -297,8 +338,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -306,8 +348,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -315,8 +358,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -324,8 +368,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -333,8 +378,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -342,8 +388,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -351,8 +398,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -360,8 +408,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -369,8 +418,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -378,8 +428,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -387,8 +438,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -396,8 +448,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -405,8 +458,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -414,8 +468,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_video",
@ -423,8 +478,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kStreamMetadataKey,
{MetricMetadataKey::kVideoStreamMetadataKey,
"alice_video"},
{MetricMetadataKey::kSenderMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -432,7 +488,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -440,7 +498,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -448,7 +508,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -456,7 +518,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -464,7 +528,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -472,7 +538,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -480,7 +548,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kHertz,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -488,7 +558,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -496,7 +568,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -504,7 +578,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -512,7 +588,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -520,7 +598,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -528,7 +608,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -536,7 +618,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -544,7 +628,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -552,7 +638,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -560,7 +648,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kBiggerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -568,7 +658,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kCount,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -576,7 +668,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_video",
@ -584,7 +678,9 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kVideoStreamMetadataKey,
"bob_video"},
{MetricMetadataKey::kSenderMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case",
@ -599,133 +695,133 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.name = "bytes_discarded_no_receiver",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "packets_discarded_no_receiver",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "payload_bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "payload_bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "packets_sent",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "average_send_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "packets_received",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "average_receive_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/alice",
.name = "sent_packets_loss",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "bytes_discarded_no_receiver",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "packets_discarded_no_receiver",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "payload_bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "payload_bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "bytes_sent",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "packets_sent",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "average_send_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "bytes_received",
.unit = Unit::kBytes,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "packets_received",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "average_receive_rate",
.unit = Unit::kKilobitsPerSecond,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob",
.name = "sent_packets_loss",
.unit = Unit::kUnitless,
.improvement_direction = ImprovementDirection::kNeitherIsBetter,
.metadata = {}},
.metadata = {{MetricMetadataKey::kPeerMetadataKey, "bob"}}},
// Metrics from VideoQualityMetricsReporter
MetricValidationInfo{
@ -771,25 +867,41 @@ TEST(PeerConnectionE2EQualityTestMetricNamesTest,
.name = "audio_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
.metadata =
{{MetricMetadataKey::kAudioStreamMetadataKey, "alice_audio"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kPeerSyncGroupMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/alice_alice_video",
.name = "video_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
.metadata =
{{MetricMetadataKey::kAudioStreamMetadataKey, "alice_video"},
{MetricMetadataKey::kPeerMetadataKey, "bob"},
{MetricMetadataKey::kPeerSyncGroupMetadataKey, "alice"},
{MetricMetadataKey::kReceiverMetadataKey, "bob"}}},
MetricValidationInfo{
.test_case = "test_case/bob_bob_audio",
.name = "audio_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}},
.metadata =
{{MetricMetadataKey::kAudioStreamMetadataKey, "bob_audio"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kPeerSyncGroupMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}},
MetricValidationInfo{
.test_case = "test_case/bob_bob_video",
.name = "video_ahead_ms",
.unit = Unit::kMilliseconds,
.improvement_direction = ImprovementDirection::kSmallerIsBetter,
.metadata = {}}));
.metadata = {
{MetricMetadataKey::kAudioStreamMetadataKey, "bob_video"},
{MetricMetadataKey::kPeerMetadataKey, "alice"},
{MetricMetadataKey::kPeerSyncGroupMetadataKey, "bob"},
{MetricMetadataKey::kReceiverMetadataKey, "alice"}}}));
}
} // namespace

49
test/pc/e2e/stats_based_network_quality_metrics_reporter.cc
View file

@ -35,6 +35,7 @@
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/synchronization/mutex.h"
#include "system_wrappers/include/field_trial.h"
#include "test/pc/e2e/metric_metadata_keys.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@ -252,48 +253,52 @@ void StatsBasedNetworkQualityMetricsReporter::ReportStats(
const NetworkLayerStats& network_layer_stats,
int64_t packet_loss,
const Timestamp& end_time) {
std::map<std::string, std::string> metric_metadata{
{MetricMetadataKey::kPeerMetadataKey, pc_label}};
metrics_logger_->LogSingleValueMetric(
"bytes_discarded_no_receiver", GetTestCaseName(pc_label),
network_layer_stats.stats->BytesDropped().bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"packets_discarded_no_receiver", GetTestCaseName(pc_label),
network_layer_stats.stats->PacketsDropped(), Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"payload_bytes_received", GetTestCaseName(pc_label),
pc_stats.payload_received.bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"payload_bytes_sent", GetTestCaseName(pc_label),
pc_stats.payload_sent.bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"bytes_sent", GetTestCaseName(pc_label), pc_stats.total_sent.bytes(),
Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
Unit::kBytes, ImprovementDirection::kNeitherIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"packets_sent", GetTestCaseName(pc_label), pc_stats.packets_sent,
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"average_send_rate", GetTestCaseName(pc_label),
(pc_stats.total_sent / (end_time - start_time_)).kbps<double>(),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter,
metric_metadata);
metrics_logger_->LogSingleValueMetric(
"bytes_received", GetTestCaseName(pc_label),
pc_stats.total_received.bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"packets_received", GetTestCaseName(pc_label), pc_stats.packets_received,
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter, metric_metadata);
metrics_logger_->LogSingleValueMetric(
"average_receive_rate", GetTestCaseName(pc_label),
(pc_stats.total_received / (end_time - start_time_)).kbps<double>(),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter,
metric_metadata);
metrics_logger_->LogSingleValueMetric(
"sent_packets_loss", GetTestCaseName(pc_label), packet_loss,
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter, metric_metadata);
}
std::string StatsBasedNetworkQualityMetricsReporter::GetTestCaseName(
@ -312,6 +317,8 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
DataRate average_receive_rate = stats.stats->PacketsReceived() >= 2
? stats.stats->AverageReceiveRate()
: DataRate::Zero();
std::map<std::string, std::string> metric_metadata{
{MetricMetadataKey::kPeerMetadataKey, peer_name}};
rtc::StringBuilder log;
log << "Raw network layer statistic for [" << peer_name << "]:\n"
<< "Local IPs:\n";
@ -320,28 +327,28 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
log << " " << local_ips[i].ToString() << "\n";
}
if (!stats.stats->SentPacketsSizeCounter().IsEmpty()) {
metrics_logger_->LogMetric("sent_packets_size", GetTestCaseName(peer_name),
stats.stats->SentPacketsSizeCounter(),
Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"sent_packets_size", GetTestCaseName(peer_name),
stats.stats->SentPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter, metric_metadata);
}
if (!stats.stats->ReceivedPacketsSizeCounter().IsEmpty()) {
metrics_logger_->LogMetric(
"received_packets_size", GetTestCaseName(peer_name),
stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter, metric_metadata);
}
if (!stats.stats->DroppedPacketsSizeCounter().IsEmpty()) {
metrics_logger_->LogMetric(
"dropped_packets_size", GetTestCaseName(peer_name),
stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter, metric_metadata);
}
if (!stats.stats->SentPacketsQueueWaitTimeUs().IsEmpty()) {
metrics_logger_->LogMetric(
"sent_packets_queue_wait_time_us", GetTestCaseName(peer_name),
stats.stats->SentPacketsQueueWaitTimeUs(), Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter, metric_metadata);
}
log << "Send statistic:\n"
@ -365,7 +372,7 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
"sent_packets_size",
GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->SentPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter, metric_metadata);
}
}
@ -392,14 +399,14 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
"received_packets_size",
GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter, metric_metadata);
}
if (!entry.second->DroppedPacketsSizeCounter().IsEmpty()) {
metrics_logger_->LogMetric(
"dropped_packets_size",
GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter, metric_metadata);
}
}