mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-16 07:10:38 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

[PCLF] Fully switch to new metrics export API

Browse source 
Bug: b/246095034
Change-Id: I9d588d53320e4eb19cb569db2b97dddc013c22bd
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276621
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Tomas Gunnarsson <tommi@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38188}
This commit is contained in:
Artem Titov 2022-09-24 16:56:07 +02:00 committed by WebRTC LUCI CQ
parent 62b40efe5a
commit c45f4e4a3d
21 changed files with 370 additions and 680 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
audio/BUILD.gn
View file

@ -224,6 +224,7 @@ if (rtc_include_tests) {
"../api:peer_connection_quality_test_fixture_api",
"../api:simulated_network_api",
"../api:time_controller",
"../api/test/metrics:global_metrics_logger_and_exporter",
"../call:simulated_network",
"../common_audio",
"../system_wrappers",

4
audio/test/pc_low_bandwidth_audio_test.cc
View file

@ -15,6 +15,7 @@
#include "absl/strings/string_view.h"
#include "api/test/create_network_emulation_manager.h"
#include "api/test/create_peerconnection_quality_test_fixture.h"
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "api/test/network_emulation_manager.h"
#include "api/test/peerconnection_quality_test_fixture.h"
#include "api/test/simulated_network.h"
@ -88,7 +89,8 @@ CreateTestFixture(absl::string_view test_case_name,
bob_configurer);
fixture->AddQualityMetricsReporter(
std::make_unique<webrtc_pc_e2e::NetworkQualityMetricsReporter>(
network_links.first, network_links.second));
network_links.first, network_links.second,
test::GetGlobalMetricsLogger()));
return fixture;
}

1
pc/BUILD.gn
View file

@ -2708,6 +2708,7 @@ if (rtc_include_tests && !build_with_chromium) {
"../api:peer_connection_quality_test_fixture_api",
"../api:simulated_network_api",
"../api:time_controller",
"../api/test/metrics:global_metrics_logger_and_exporter",
"../api/video_codecs:video_codecs_api",
"../call:simulated_network",
"../modules/video_coding:webrtc_vp9",

2
pc/test/svc_e2e_tests.cc
View file

@ -17,6 +17,7 @@
#include "api/test/create_peer_connection_quality_test_frame_generator.h"
#include "api/test/create_peerconnection_quality_test_fixture.h"
#include "api/test/frame_generator_interface.h"
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "api/test/network_emulation_manager.h"
#include "api/test/peerconnection_quality_test_fixture.h"
#include "api/test/simulated_network.h"
@ -185,6 +186,7 @@ class SvcVideoQualityAnalyzer : public DefaultVideoQualityAnalyzer {
explicit SvcVideoQualityAnalyzer(webrtc::Clock* clock)
: DefaultVideoQualityAnalyzer(clock,
test::GetGlobalMetricsLogger(),
DefaultVideoQualityAnalyzerOptions{
.compute_psnr = false,
.compute_ssim = false,

6
test/pc/e2e/BUILD.gn
View file

@ -581,6 +581,7 @@ if (!build_with_chromium) {
"../..:test_support",
"../../../api:create_frame_generator",
"../../../api:rtp_packet_info",
"../../../api/test/metrics:global_metrics_logger_and_exporter",
"../../../api/video:encoded_image",
"../../../api/video:video_frame",
"../../../common_video",
@ -700,6 +701,7 @@ if (!build_with_chromium) {
"../../../api/test/metrics:metrics_logger_and_exporter",
"../../../api/units:time_delta",
"../../../api/units:timestamp",
"../../../rtc_base:checks",
"../../../rtc_base:criticalsection",
"../../../rtc_base:logging",
"../../../rtc_base:rtc_numerics",
@ -749,6 +751,7 @@ if (!build_with_chromium) {
"../../../api/units:data_size",
"../../../api/units:time_delta",
"../../../api/units:timestamp",
"../../../rtc_base:checks",
"../../../rtc_base:criticalsection",
"../../../rtc_base:rtc_numerics",
"../../../rtc_base/synchronization:mutex",
@ -887,6 +890,7 @@ if (!build_with_chromium) {
"../../../api/test/metrics:metrics_logger",
"../../../api/test/metrics:metrics_logger_and_exporter",
"../../../api/units:data_size",
"../../../rtc_base:checks",
"../../../rtc_base:criticalsection",
"../../../rtc_base:rtc_event",
"../../../rtc_base/synchronization:mutex",
@ -918,6 +922,7 @@ if (!build_with_chromium) {
"../../../api/units:data_size",
"../../../api/units:timestamp",
"../../../rtc_base",
"../../../rtc_base:checks",
"../../../rtc_base:ip_address",
"../../../rtc_base:rtc_event",
"../../../rtc_base:stringutils",
@ -945,6 +950,7 @@ if (!build_with_chromium) {
"../../../api/test/metrics:metrics_logger",
"../../../api/test/metrics:metrics_logger_and_exporter",
"../../../api/units:timestamp",
"../../../rtc_base:checks",
"../../../rtc_base:criticalsection",
"../../../rtc_base:rtc_event",
"../../../rtc_base:rtc_numerics",

81
test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc
View file

@ -13,6 +13,7 @@
#include "api/stats/rtc_stats.h"
#include "api/stats/rtcstats_objects.h"
#include "api/test/metrics/metric.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
namespace webrtc {
@ -21,6 +22,12 @@ namespace webrtc_pc_e2e {
using ::webrtc::test::ImprovementDirection;
using ::webrtc::test::Unit;
DefaultAudioQualityAnalyzer::DefaultAudioQualityAnalyzer(
test::MetricsLogger* const metrics_logger)
: metrics_logger_(metrics_logger) {
RTC_CHECK(metrics_logger_);
}
void DefaultAudioQualityAnalyzer::Start(std::string test_case_name,
TrackIdStreamInfoMap* analyzer_helper) {
test_case_name_ = std::move(test_case_name);
@ -116,48 +123,29 @@ std::string DefaultAudioQualityAnalyzer::GetTestCaseName(
}
void DefaultAudioQualityAnalyzer::Stop() {
using ::webrtc::test::ImproveDirection;
MutexLock lock(&lock_);
for (auto& item : streams_stats_) {
if (metrics_logger_ == nullptr) {
ReportResult("expand_rate", item.first, item.second.expand_rate,
"unitless", ImproveDirection::kSmallerIsBetter);
ReportResult("accelerate_rate", item.first, item.second.accelerate_rate,
"unitless", ImproveDirection::kSmallerIsBetter);
ReportResult("preemptive_rate", item.first, item.second.preemptive_rate,
"unitless", ImproveDirection::kSmallerIsBetter);
ReportResult("speech_expand_rate", item.first,
item.second.speech_expand_rate, "unitless",
ImproveDirection::kSmallerIsBetter);
ReportResult("average_jitter_buffer_delay_ms", item.first,
item.second.average_jitter_buffer_delay_ms, "ms",
ImproveDirection::kNone);
ReportResult("preferred_buffer_size_ms", item.first,
item.second.preferred_buffer_size_ms, "ms",
ImproveDirection::kNone);
} else {
metrics_logger_->LogMetric("expand_rate", GetTestCaseName(item.first),
item.second.expand_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("accelerate_rate", GetTestCaseName(item.first),
item.second.accelerate_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("preemptive_rate", GetTestCaseName(item.first),
item.second.preemptive_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric(
"speech_expand_rate", GetTestCaseName(item.first),
item.second.speech_expand_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric(
"average_jitter_buffer_delay_ms", GetTestCaseName(item.first),
item.second.average_jitter_buffer_delay_ms, Unit::kMilliseconds,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"preferred_buffer_size_ms", GetTestCaseName(item.first),
item.second.preferred_buffer_size_ms, Unit::kMilliseconds,
ImprovementDirection::kNeitherIsBetter);
}
metrics_logger_->LogMetric("expand_rate", GetTestCaseName(item.first),
item.second.expand_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("accelerate_rate", GetTestCaseName(item.first),
item.second.accelerate_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("preemptive_rate", GetTestCaseName(item.first),
item.second.preemptive_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("speech_expand_rate",
GetTestCaseName(item.first),
item.second.speech_expand_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric(
"average_jitter_buffer_delay_ms", GetTestCaseName(item.first),
item.second.average_jitter_buffer_delay_ms, Unit::kMilliseconds,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"preferred_buffer_size_ms", GetTestCaseName(item.first),
item.second.preferred_buffer_size_ms, Unit::kMilliseconds,
ImprovementDirection::kNeitherIsBetter);
}
}
@ -167,18 +155,5 @@ DefaultAudioQualityAnalyzer::GetAudioStreamsStats() const {
return streams_stats_;
}
void DefaultAudioQualityAnalyzer::ReportResult(
const std::string& metric_name,
const std::string& stream_label,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) const {
test::PrintResultMeanAndError(
metric_name, /*modifier=*/"", GetTestCaseName(stream_label),
counter.IsEmpty() ? 0 : counter.GetAverage(),
counter.IsEmpty() ? 0 : counter.GetStandardDeviation(), unit,
/*important=*/false, improve_direction);
}
} // namespace webrtc_pc_e2e
} // namespace webrtc

11
test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h
View file

@ -21,7 +21,6 @@
#include "api/test/track_id_stream_info_map.h"
#include "api/units/time_delta.h"
#include "rtc_base/synchronization/mutex.h"
#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@ -37,11 +36,8 @@ struct AudioStreamStats {
class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface {
public:
DefaultAudioQualityAnalyzer()
: DefaultAudioQualityAnalyzer(/*metrics_logger=*/nullptr) {}
explicit DefaultAudioQualityAnalyzer(
test::MetricsLogger* const metrics_logger)
: metrics_logger_(metrics_logger) {}
test::MetricsLogger* const metrics_logger);
void Start(std::string test_case_name,
TrackIdStreamInfoMap* analyzer_helper) override;
@ -66,11 +62,6 @@ class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface {
};
std::string GetTestCaseName(const std::string& stream_label) const;
void ReportResult(const std::string& metric_name,
const std::string& stream_label,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) const;
test::MetricsLogger* const metrics_logger_;

299
test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
View file

@ -23,6 +23,7 @@
#include "api/video/i420_buffer.h"
#include "api/video/video_frame.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/platform_thread.h"
#include "rtc_base/strings/string_builder.h"
@ -124,13 +125,6 @@ SamplesStatsCounter::StatsSample StatsSample(double value,
} // namespace
DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
DefaultVideoQualityAnalyzerOptions options)
: DefaultVideoQualityAnalyzer(clock,
/*metrics_logger=*/nullptr,
std::move(options)) {}
DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
test::MetricsLogger* metrics_logger,
@ -138,7 +132,9 @@ DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
: options_(options),
clock_(clock),
metrics_logger_(metrics_logger),
frames_comparator_(clock, cpu_measurer_, options) {}
frames_comparator_(clock, cpu_measurer_, options) {
RTC_CHECK(metrics_logger_);
}
DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() {
Stop();
@ -891,22 +887,14 @@ void DefaultVideoQualityAnalyzer::
}
void DefaultVideoQualityAnalyzer::ReportResults() {
using ::webrtc::test::ImproveDirection;
MutexLock lock(&mutex_);
for (auto& item : frames_comparator_.stream_stats()) {
ReportResults(GetTestCaseName(ToMetricName(item.first)), item.second,
stream_frame_counters_.at(item.first));
}
if (metrics_logger_ == nullptr) {
test::PrintResult("cpu_usage", "", test_label_.c_str(),
GetCpuUsagePercent(), "%", false,
ImproveDirection::kSmallerIsBetter);
} else {
metrics_logger_->LogSingleValueMetric(
"cpu_usage_%", test_label_, GetCpuUsagePercent(), Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
}
metrics_logger_->LogSingleValueMetric("cpu_usage_%", test_label_,
GetCpuUsagePercent(), Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
LogFrameCounters("Global", frame_counters_);
if (!unknown_sender_frame_counters_.empty()) {
RTC_LOG(LS_INFO) << "Received frame counters with unknown frame id:";
@ -943,7 +931,6 @@ void DefaultVideoQualityAnalyzer::ReportResults(
const std::string& test_case_name,
const StreamStats& stats,
const FrameCounters& frame_counters) {
using ::webrtc::test::ImproveDirection;
TimeDelta test_duration = Now() - start_time_;
double sum_squared_interframe_delays_secs = 0;
@ -972,199 +959,101 @@ void DefaultVideoQualityAnalyzer::ReportResults(
video_duration.seconds<double>() / sum_squared_interframe_delays_secs;
}
if (metrics_logger_ == nullptr) {
// Report results through old performance metrics API.
ReportResult("psnr", test_case_name, stats.psnr, "dB",
ImproveDirection::kBiggerIsBetter);
ReportResult("ssim", test_case_name, stats.ssim, "unitless",
ImproveDirection::kBiggerIsBetter);
ReportResult("transport_time", test_case_name, stats.transport_time_ms,
"ms", ImproveDirection::kSmallerIsBetter);
ReportResult("total_delay_incl_transport", test_case_name,
stats.total_delay_incl_transport_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("time_between_rendered_frames", test_case_name,
stats.time_between_rendered_frames_ms, "ms",
ImproveDirection::kSmallerIsBetter);
test::PrintResult("harmonic_framerate", "", test_case_name,
harmonic_framerate_fps, "Hz", /*important=*/false,
ImproveDirection::kBiggerIsBetter);
test::PrintResult("encode_frame_rate", "", test_case_name,
stats.encode_frame_rate.IsEmpty()
? 0
: stats.encode_frame_rate.GetEventsPerSecond(),
"Hz", /*important=*/false,
ImproveDirection::kBiggerIsBetter);
ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("time_between_freezes", test_case_name,
stats.time_between_freezes_ms, "ms",
ImproveDirection::kBiggerIsBetter);
ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("pixels_per_frame", test_case_name,
stats.resolution_of_rendered_frame, "count",
ImproveDirection::kBiggerIsBetter);
test::PrintResult("min_psnr", "", test_case_name,
stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("receive_to_render_time", test_case_name,
stats.receive_to_render_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
test::PrintResult("dropped_frames", "", test_case_name,
frame_counters.dropped, "count",
/*important=*/false, ImproveDirection::kSmallerIsBetter);
test::PrintResult("frames_in_flight", "", test_case_name,
frame_counters.captured - frame_counters.rendered -
frame_counters.dropped,
"count",
/*important=*/false, ImproveDirection::kSmallerIsBetter);
test::PrintResult("rendered_frames", "", test_case_name,
frame_counters.rendered, "count", /*important=*/false,
ImproveDirection::kBiggerIsBetter);
ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered,
"count", ImproveDirection::kSmallerIsBetter);
ReportResult("target_encode_bitrate", test_case_name,
stats.target_encode_bitrate / kBitsInByte, "bytesPerSecond",
ImproveDirection::kNone);
test::PrintResult("actual_encode_bitrate", "", test_case_name,
static_cast<double>(stats.total_encoded_images_payload) /
test_duration.seconds<double>(),
"bytesPerSecond", /*important=*/false,
ImproveDirection::kNone);
metrics_logger_->LogMetric("psnr_dB", test_case_name, stats.psnr,
Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("ssim", test_case_name, stats.ssim,
Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("transport_time", test_case_name,
stats.transport_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("total_delay_incl_transport", test_case_name,
stats.total_delay_incl_transport_ms,
Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("time_between_rendered_frames", test_case_name,
stats.time_between_rendered_frames_ms,
Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric("harmonic_framerate", test_case_name,
harmonic_framerate_fps, Unit::kHertz,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"encode_frame_rate", test_case_name,
stats.encode_frame_rate.IsEmpty()
? 0
: stats.encode_frame_rate.GetEventsPerSecond(),
Unit::kHertz, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("encode_time", test_case_name,
stats.encode_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("time_between_freezes", test_case_name,
stats.time_between_freezes_ms, Unit::kMilliseconds,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("freeze_time_ms", test_case_name,
stats.freeze_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("pixels_per_frame", test_case_name,
stats.resolution_of_rendered_frame, Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"min_psnr_dB", test_case_name,
stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("decode_time", test_case_name,
stats.decode_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric(
"receive_to_render_time", test_case_name, stats.receive_to_render_time_ms,
Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric("dropped_frames", test_case_name,
frame_counters.dropped, Unit::kCount,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric(
"frames_in_flight", test_case_name,
frame_counters.captured - frame_counters.rendered -
frame_counters.dropped,
Unit::kCount, ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric("rendered_frames", test_case_name,
frame_counters.rendered, Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("max_skipped", test_case_name,
stats.skipped_between_rendered, Unit::kCount,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("target_encode_bitrate", test_case_name,
stats.target_encode_bitrate / 1000,
Unit::kKilobitsPerSecond,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"actual_encode_bitrate", test_case_name,
static_cast<double>(stats.total_encoded_images_payload) /
test_duration.seconds<double>() * kBitsInByte / 1000,
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
if (options_.report_detailed_frame_stats) {
test::PrintResult("num_encoded_frames", "", test_case_name,
frame_counters.encoded, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_decoded_frames", "", test_case_name,
frame_counters.decoded, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_send_key_frames", "", test_case_name,
stats.num_send_key_frames, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_recv_key_frames", "", test_case_name,
stats.num_recv_key_frames, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
ReportResult("recv_key_frame_size_bytes", test_case_name,
stats.recv_key_frame_size_bytes, "count",
ImproveDirection::kBiggerIsBetter);
ReportResult("recv_delta_frame_size_bytes", test_case_name,
stats.recv_delta_frame_size_bytes, "count",
ImproveDirection::kBiggerIsBetter);
}
} else {
metrics_logger_->LogMetric("psnr_dB", test_case_name, stats.psnr,
Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("ssim", test_case_name, stats.ssim,
Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("transport_time", test_case_name,
stats.transport_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("total_delay_incl_transport", test_case_name,
stats.total_delay_incl_transport_ms,
Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("time_between_rendered_frames", test_case_name,
stats.time_between_rendered_frames_ms,
Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
if (options_.report_detailed_frame_stats) {
metrics_logger_->LogSingleValueMetric(
"harmonic_framerate", test_case_name, harmonic_framerate_fps,
Unit::kHertz, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"encode_frame_rate", test_case_name,
stats.encode_frame_rate.IsEmpty()
? 0
: stats.encode_frame_rate.GetEventsPerSecond(),
Unit::kHertz, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("encode_time", test_case_name,
stats.encode_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric(
"time_between_freezes", test_case_name, stats.time_between_freezes_ms,
Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("freeze_time_ms", test_case_name,
stats.freeze_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("pixels_per_frame", test_case_name,
stats.resolution_of_rendered_frame, Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"min_psnr_dB", test_case_name,
stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("decode_time", test_case_name,
stats.decode_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("receive_to_render_time", test_case_name,
stats.receive_to_render_time_ms,
Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric(
"dropped_frames", test_case_name, frame_counters.dropped, Unit::kCount,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric(
"frames_in_flight", test_case_name,
frame_counters.captured - frame_counters.rendered -
frame_counters.dropped,
Unit::kCount, ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric(
"rendered_frames", test_case_name, frame_counters.rendered,
"num_encoded_frames", test_case_name, frame_counters.encoded,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("max_skipped", test_case_name,
stats.skipped_between_rendered, Unit::kCount,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("target_encode_bitrate", test_case_name,
stats.target_encode_bitrate / 1000,
Unit::kKilobitsPerSecond,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"actual_encode_bitrate", test_case_name,
static_cast<double>(stats.total_encoded_images_payload) /
test_duration.seconds<double>() * kBitsInByte / 1000,
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
"num_decoded_frames", test_case_name, frame_counters.decoded,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"num_send_key_frames", test_case_name, stats.num_send_key_frames,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"num_recv_key_frames", test_case_name, stats.num_recv_key_frames,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
if (options_.report_detailed_frame_stats) {
metrics_logger_->LogSingleValueMetric(
"num_encoded_frames", test_case_name, frame_counters.encoded,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"num_decoded_frames", test_case_name, frame_counters.decoded,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"num_send_key_frames", test_case_name, stats.num_send_key_frames,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"num_recv_key_frames", test_case_name, stats.num_recv_key_frames,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("recv_key_frame_size_bytes", test_case_name,
stats.recv_key_frame_size_bytes, Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name,
stats.recv_delta_frame_size_bytes,
Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
}
metrics_logger_->LogMetric("recv_key_frame_size_bytes", test_case_name,
stats.recv_key_frame_size_bytes, Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name,
stats.recv_delta_frame_size_bytes, Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
}
}
void DefaultVideoQualityAnalyzer::ReportResult(
const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) {
test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit,
/*important=*/false, improve_direction);
}
std::string DefaultVideoQualityAnalyzer::GetTestCaseName(
const std::string& stream_label) const {
return test_label_ + "/" + stream_label;

11
test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
View file

@ -41,15 +41,11 @@
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h"
#include "test/pc/e2e/analyzer/video/names_collection.h"
#include "test/testsupport/perf_test.h"
namespace webrtc {
class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
public:
explicit DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
DefaultVideoQualityAnalyzerOptions options = {});
DefaultVideoQualityAnalyzer(webrtc::Clock* clock,
test::MetricsLogger* metrics_logger,
DefaultVideoQualityAnalyzerOptions options = {});
@ -131,13 +127,6 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
const StreamStats& stats,
const FrameCounters& frame_counters)
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
// Report result for single metric for specified stream.
static void ReportResult(const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction =
webrtc::test::ImproveDirection::kNone);
// Returns name of current test case for reporting.
std::string GetTestCaseName(const std::string& stream_label) const;
Timestamp Now();

65
test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc
View file

@ -18,6 +18,7 @@
#include "api/rtp_packet_info.h"
#include "api/rtp_packet_infos.h"
#include "api/test/create_frame_generator.h"
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "api/video/encoded_image.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_frame.h"
@ -148,6 +149,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest());
analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -199,6 +201,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest());
analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -280,6 +283,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest());
analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -330,6 +334,7 @@ TEST(DefaultVideoQualityAnalyzerTest, NormalScenario) {
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest());
analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -388,6 +393,7 @@ TEST(DefaultVideoQualityAnalyzerTest, OneFrameReceivedTwice) {
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest());
analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -444,6 +450,7 @@ TEST(DefaultVideoQualityAnalyzerTest, NormalScenario2Receivers) {
constexpr char kCharlie[] = "charlie";
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest());
analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie},
kAnalyzerMaxThreadsCount);
@ -574,6 +581,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
constexpr char kCharlie[] = "charlie";
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest());
analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie},
kAnalyzerMaxThreadsCount);
@ -633,6 +641,7 @@ TEST(DefaultVideoQualityAnalyzerTest, HeavyQualityMetricsFromEqualFrames) {
analyzer_options.max_frames_in_flight_per_stream_count =
kMaxFramesInFlightPerStream;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
analyzer_options);
analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -692,6 +701,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
analyzer_options.max_frames_in_flight_per_stream_count =
kMaxFramesInFlightPerStream;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
analyzer_options);
analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -750,6 +760,7 @@ TEST(DefaultVideoQualityAnalyzerTest, CpuUsage) {
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest());
analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -813,6 +824,7 @@ TEST(DefaultVideoQualityAnalyzerTest, RuntimeParticipantsAdding) {
constexpr int kTwoThirdFrames = 2 * kOneThirdFrames;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest());
analyzer.Start("test_case", {}, kAnalyzerMaxThreadsCount);
@ -958,6 +970,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest());
constexpr char kAlice[] = "alice";
constexpr char kBob[] = "bob";
@ -1020,7 +1033,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
kAnalyzerMaxThreadsCount);
@ -1115,7 +1129,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
kAnalyzerMaxThreadsCount);
@ -1208,6 +1223,7 @@ TEST(DefaultVideoQualityAnalyzerTest, CodecTrackedCorrectly) {
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest());
analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -1278,7 +1294,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
kAnalyzerMaxThreadsCount);
@ -1370,7 +1387,8 @@ TEST(
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
kAnalyzerMaxThreadsCount);
@ -1474,7 +1492,8 @@ TEST(DefaultVideoQualityAnalyzerTest, GetStreamFrames) {
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount);
@ -1518,7 +1537,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverReceivedFramesWhenSenderRemoved) {
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount);
@ -1561,7 +1581,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount);
@ -1604,7 +1625,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount);
@ -1647,7 +1669,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount);
@ -1702,7 +1725,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeCapturing2ndFrame) {
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount);
@ -1742,7 +1766,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforePreEncoded) {
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount);
@ -1784,7 +1809,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeEncoded) {
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount);
@ -1827,7 +1853,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount);
@ -1873,7 +1900,8 @@ TEST(DefaultVideoQualityAnalyzerTest, UnregisterOneAndRegisterAnother) {
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case",
std::vector<std::string>{"alice", "bob", "charlie"},
kAnalyzerMaxThreadsCount);
@ -1934,7 +1962,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case",
std::vector<std::string>{"alice", "bob", "charlie"},
kAnalyzerMaxThreadsCount);
@ -1989,7 +2018,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount);
@ -2026,7 +2056,8 @@ TEST_P(DefaultVideoQualityAnalyzerTimeBetweenFreezesTest,
/*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount);

51
test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc
View file

@ -16,6 +16,7 @@
#include "api/units/data_rate.h"
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "rtc_base/checks.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@ -30,6 +31,13 @@ SamplesStatsCounter BytesPerSecondToKbps(const SamplesStatsCounter& counter) {
} // namespace
VideoQualityMetricsReporter::VideoQualityMetricsReporter(
Clock* const clock,
test::MetricsLogger* const metrics_logger)
: clock_(clock), metrics_logger_(metrics_logger) {
RTC_CHECK(metrics_logger_);
}
void VideoQualityMetricsReporter::Start(
absl::string_view test_case_name,
const TrackIdStreamInfoMap* /*reporter_helper*/) {
@ -122,37 +130,18 @@ std::string VideoQualityMetricsReporter::GetTestCaseName(
void VideoQualityMetricsReporter::ReportVideoBweResults(
const std::string& test_case_name,
const VideoBweStats& video_bwe_stats) {
if (metrics_logger_ == nullptr) {
ReportResult("available_send_bandwidth", test_case_name,
video_bwe_stats.available_send_bandwidth, "bytesPerSecond");
ReportResult("transmission_bitrate", test_case_name,
video_bwe_stats.transmission_bitrate, "bytesPerSecond");
ReportResult("retransmission_bitrate", test_case_name,
video_bwe_stats.retransmission_bitrate, "bytesPerSecond");
} else {
metrics_logger_->LogMetric(
"available_send_bandwidth", test_case_name,
BytesPerSecondToKbps(video_bwe_stats.available_send_bandwidth),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"transmission_bitrate", test_case_name,
BytesPerSecondToKbps(video_bwe_stats.transmission_bitrate),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"retransmission_bitrate", test_case_name,
BytesPerSecondToKbps(video_bwe_stats.retransmission_bitrate),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
}
}
void VideoQualityMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) {
test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit,
/*important=*/false, improve_direction);
metrics_logger_->LogMetric(
"available_send_bandwidth", test_case_name,
BytesPerSecondToKbps(video_bwe_stats.available_send_bandwidth),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"transmission_bitrate", test_case_name,
BytesPerSecondToKbps(video_bwe_stats.transmission_bitrate),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"retransmission_bitrate", test_case_name,
BytesPerSecondToKbps(video_bwe_stats.retransmission_bitrate),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
}
} // namespace webrtc_pc_e2e

16
test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h
View file

@ -22,7 +22,6 @@
#include "api/units/data_size.h"
#include "api/units/timestamp.h"
#include "rtc_base/synchronization/mutex.h"
#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@ -36,12 +35,8 @@ struct VideoBweStats {
class VideoQualityMetricsReporter
: public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
public:
explicit VideoQualityMetricsReporter(Clock* const clock)
: VideoQualityMetricsReporter(clock, /*metrics_logger=*/nullptr) {}
explicit VideoQualityMetricsReporter(
Clock* const clock,
test::MetricsLogger* const metrics_logger)
: clock_(clock), metrics_logger_(metrics_logger) {}
VideoQualityMetricsReporter(Clock* const clock,
test::MetricsLogger* const metrics_logger);
~VideoQualityMetricsReporter() override = default;
void Start(absl::string_view test_case_name,
@ -63,13 +58,6 @@ class VideoQualityMetricsReporter
std::string GetTestCaseName(const std::string& stream_label) const;
void ReportVideoBweResults(const std::string& test_case_name,
const VideoBweStats& video_bwe_stats);
// Report result for single metric for specified stream.
static void ReportResult(const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction =
webrtc::test::ImproveDirection::kNone);
Timestamp Now() const { return clock_->CurrentTime(); }
Clock* const clock_;

48
test/pc/e2e/cross_media_metrics_reporter.cc
View file

@ -16,6 +16,7 @@
#include "api/stats/rtcstats_objects.h"
#include "api/test/metrics/metric.h"
#include "api/units/timestamp.h"
#include "rtc_base/checks.h"
#include "rtc_base/event.h"
#include "system_wrappers/include/field_trial.h"
@ -25,6 +26,12 @@ namespace webrtc_pc_e2e {
using ::webrtc::test::ImprovementDirection;
using ::webrtc::test::Unit;
CrossMediaMetricsReporter::CrossMediaMetricsReporter(
test::MetricsLogger* metrics_logger)
: metrics_logger_(metrics_logger) {
RTC_CHECK(metrics_logger_);
}
void CrossMediaMetricsReporter::Start(
absl::string_view test_case_name,
const TrackIdStreamInfoMap* reporter_helper) {
@ -102,40 +109,19 @@ void CrossMediaMetricsReporter::StopAndReportResults() {
MutexLock lock(&mutex_);
for (const auto& pair : stats_info_) {
const std::string& sync_group = pair.first;
if (metrics_logger_ == nullptr) {
ReportResult("audio_ahead_ms",
GetTestCaseName(pair.second.audio_stream_label, sync_group),
pair.second.audio_ahead_ms, "ms",
webrtc::test::ImproveDirection::kSmallerIsBetter);
ReportResult("video_ahead_ms",
GetTestCaseName(pair.second.video_stream_label, sync_group),
pair.second.video_ahead_ms, "ms",
webrtc::test::ImproveDirection::kSmallerIsBetter);
} else {
metrics_logger_->LogMetric(
"audio_ahead_ms",
GetTestCaseName(pair.second.audio_stream_label, sync_group),
pair.second.audio_ahead_ms, Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric(
"video_ahead_ms",
GetTestCaseName(pair.second.video_stream_label, sync_group),
pair.second.video_ahead_ms, Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter);
}
metrics_logger_->LogMetric(
"audio_ahead_ms",
GetTestCaseName(pair.second.audio_stream_label, sync_group),
pair.second.audio_ahead_ms, Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric(
"video_ahead_ms",
GetTestCaseName(pair.second.video_stream_label, sync_group),
pair.second.video_ahead_ms, Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter);
}
}
void CrossMediaMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) {
test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit,
/*important=*/false, improve_direction);
}
std::string CrossMediaMetricsReporter::GetTestCaseName(
const std::string& stream_label,
const std::string& sync_group) const {

12
test/pc/e2e/cross_media_metrics_reporter.h
View file

@ -22,7 +22,6 @@
#include "api/test/track_id_stream_info_map.h"
#include "api/units/timestamp.h"
#include "rtc_base/synchronization/mutex.h"
#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@ -30,10 +29,7 @@ namespace webrtc_pc_e2e {
class CrossMediaMetricsReporter
: public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
public:
CrossMediaMetricsReporter()
: CrossMediaMetricsReporter(/*metrics_logger=*/nullptr) {}
explicit CrossMediaMetricsReporter(test::MetricsLogger* metrics_logger)
: metrics_logger_(metrics_logger) {}
explicit CrossMediaMetricsReporter(test::MetricsLogger* metrics_logger);
~CrossMediaMetricsReporter() override = default;
void Start(absl::string_view test_case_name,
@ -52,12 +48,6 @@ class CrossMediaMetricsReporter
std::string video_stream_label;
};
static void ReportResult(const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction =
webrtc::test::ImproveDirection::kNone);
std::string GetTestCaseName(const std::string& stream_label,
const std::string& sync_group) const;

135
test/pc/e2e/network_quality_metrics_reporter.cc
View file

@ -14,9 +14,9 @@
#include "api/stats/rtc_stats.h"
#include "api/stats/rtcstats_objects.h"
#include "api/test/metrics/metric.h"
#include "rtc_base/checks.h"
#include "rtc_base/event.h"
#include "system_wrappers/include/field_trial.h"
#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@ -31,6 +31,17 @@ constexpr TimeDelta kStatsWaitTimeout = TimeDelta::Seconds(1);
// sent/received per stream. If enabled, padding and headers are not included
// in bytes sent or received.
constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats";
} // namespace
NetworkQualityMetricsReporter::NetworkQualityMetricsReporter(
EmulatedNetworkManagerInterface* alice_network,
EmulatedNetworkManagerInterface* bob_network,
test::MetricsLogger* metrics_logger)
: alice_network_(alice_network),
bob_network_(bob_network),
metrics_logger_(metrics_logger) {
RTC_CHECK(metrics_logger_);
}
void NetworkQualityMetricsReporter::Start(
@ -113,93 +124,49 @@ void NetworkQualityMetricsReporter::ReportStats(
const std::string& network_label,
std::unique_ptr<EmulatedNetworkStats> stats,
int64_t packet_loss) {
if (metrics_logger_ == nullptr) {
ReportResult("bytes_sent", network_label, stats->BytesSent().bytes(),
"sizeInBytes");
ReportResult("packets_sent", network_label, stats->PacketsSent(),
"unitless");
ReportResult("average_send_rate", network_label,
stats->PacketsSent() >= 2
? stats->AverageSendRate().bytes_per_sec()
: 0,
"bytesPerSecond");
ReportResult("bytes_discarded_no_receiver", network_label,
stats->BytesDropped().bytes(), "sizeInBytes");
ReportResult("packets_discarded_no_receiver", network_label,
stats->PacketsDropped(), "unitless");
ReportResult("bytes_received", network_label,
stats->BytesReceived().bytes(), "sizeInBytes");
ReportResult("packets_received", network_label, stats->PacketsReceived(),
"unitless");
ReportResult("average_receive_rate", network_label,
stats->PacketsReceived() >= 2
? stats->AverageReceiveRate().bytes_per_sec()
: 0,
"bytesPerSecond");
ReportResult("sent_packets_loss", network_label, packet_loss, "unitless");
} else {
metrics_logger_->LogSingleValueMetric(
"bytes_sent", GetTestCaseName(network_label),
stats->BytesSent().bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_sent", GetTestCaseName(network_label), stats->PacketsSent(),
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"average_send_rate", GetTestCaseName(network_label),
stats->PacketsSent() >= 2 ? stats->AverageSendRate().kbps() : 0,
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"bytes_discarded_no_receiver", GetTestCaseName(network_label),
stats->BytesDropped().bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_discarded_no_receiver", GetTestCaseName(network_label),
stats->PacketsDropped(), Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"bytes_received", GetTestCaseName(network_label),
stats->BytesReceived().bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_received", GetTestCaseName(network_label),
stats->PacketsReceived(), Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"average_receive_rate", GetTestCaseName(network_label),
stats->PacketsReceived() >= 2 ? stats->AverageReceiveRate().kbps() : 0,
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"sent_packets_loss", GetTestCaseName(network_label), packet_loss,
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
}
metrics_logger_->LogSingleValueMetric(
"bytes_sent", GetTestCaseName(network_label), stats->BytesSent().bytes(),
Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_sent", GetTestCaseName(network_label), stats->PacketsSent(),
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"average_send_rate", GetTestCaseName(network_label),
stats->PacketsSent() >= 2 ? stats->AverageSendRate().kbps() : 0,
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"bytes_discarded_no_receiver", GetTestCaseName(network_label),
stats->BytesDropped().bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_discarded_no_receiver", GetTestCaseName(network_label),
stats->PacketsDropped(), Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"bytes_received", GetTestCaseName(network_label),
stats->BytesReceived().bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_received", GetTestCaseName(network_label),
stats->PacketsReceived(), Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"average_receive_rate", GetTestCaseName(network_label),
stats->PacketsReceived() >= 2 ? stats->AverageReceiveRate().kbps() : 0,
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"sent_packets_loss", GetTestCaseName(network_label), packet_loss,
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
}
void NetworkQualityMetricsReporter::ReportPCStats(const std::string& pc_label,
const PCStats& stats) {
if (metrics_logger_ == nullptr) {
ReportResult("payload_bytes_received", pc_label,
stats.payload_received.bytes(), "sizeInBytes");
ReportResult("payload_bytes_sent", pc_label, stats.payload_sent.bytes(),
"sizeInBytes");
} else {
metrics_logger_->LogSingleValueMetric(
"payload_bytes_received", pc_label, stats.payload_received.bytes(),
Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"payload_bytes_sent", pc_label, stats.payload_sent.bytes(),
Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
}
}
void NetworkQualityMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& network_label,
const double value,
const std::string& unit) const {
test::PrintResult(metric_name, /*modifier=*/"",
GetTestCaseName(network_label), value, unit,
/*important=*/false);
metrics_logger_->LogSingleValueMetric(
"payload_bytes_received", pc_label, stats.payload_received.bytes(),
Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"payload_bytes_sent", pc_label, stats.payload_sent.bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
std::string NetworkQualityMetricsReporter::GetTestCaseName(

14
test/pc/e2e/network_quality_metrics_reporter.h
View file

@ -28,17 +28,9 @@ namespace webrtc_pc_e2e {
class NetworkQualityMetricsReporter
: public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
public:
NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network,
EmulatedNetworkManagerInterface* bob_network)
: NetworkQualityMetricsReporter(alice_network,
bob_network,
/*metrics_logger=*/nullptr) {}
NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network,
EmulatedNetworkManagerInterface* bob_network,
test::MetricsLogger* metrics_logger)
: alice_network_(alice_network),
bob_network_(bob_network),
metrics_logger_(metrics_logger) {}
test::MetricsLogger* metrics_logger);
~NetworkQualityMetricsReporter() override = default;
// Network stats must be empty when this method will be invoked.
@ -63,10 +55,6 @@ class NetworkQualityMetricsReporter
std::unique_ptr<EmulatedNetworkStats> stats,
int64_t packet_loss);
void ReportPCStats(const std::string& pc_label, const PCStats& stats);
void ReportResult(const std::string& metric_name,
const std::string& network_label,
double value,
const std::string& unit) const;
std::string GetTestCaseName(const std::string& network_label) const;
std::string test_case_name_;

25
test/pc/e2e/peer_connection_quality_test.cc
View file

@ -41,7 +41,6 @@
#include "test/pc/e2e/stats_poller.h"
#include "test/pc/e2e/test_peer_factory.h"
#include "test/testsupport/file_utils.h"
#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@ -736,24 +735,12 @@ void PeerConnectionE2EQualityTest::TearDownCall() {
}
void PeerConnectionE2EQualityTest::ReportGeneralTestResults() {
if (metrics_logger_ == nullptr) {
test::PrintResult(*alice_->params().name + "_connected", "",
test_case_name_, alice_connected_, "unitless",
/*important=*/false,
test::ImproveDirection::kBiggerIsBetter);
test::PrintResult(*bob_->params().name + "_connected", "", test_case_name_,
bob_connected_, "unitless",
/*important=*/false,
test::ImproveDirection::kBiggerIsBetter);
} else {
metrics_logger_->LogSingleValueMetric(
*alice_->params().name + "_connected", test_case_name_,
alice_connected_, Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
*bob_->params().name + "_connected", test_case_name_, bob_connected_,
Unit::kUnitless, ImprovementDirection::kBiggerIsBetter);
}
metrics_logger_->LogSingleValueMetric(
*alice_->params().name + "_connected", test_case_name_, alice_connected_,
Unit::kUnitless, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
*bob_->params().name + "_connected", test_case_name_, bob_connected_,
Unit::kUnitless, ImprovementDirection::kBiggerIsBetter);
}
Timestamp PeerConnectionE2EQualityTest::Now() const {

244
test/pc/e2e/stats_based_network_quality_metrics_reporter.cc
View file

@ -29,12 +29,12 @@
#include "api/test/network_emulation_manager.h"
#include "api/units/data_rate.h"
#include "api/units/timestamp.h"
#include "rtc_base/checks.h"
#include "rtc_base/event.h"
#include "rtc_base/ip_address.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/synchronization/mutex.h"
#include "system_wrappers/include/field_trial.h"
#include "test/testsupport/perf_test.h"
namespace webrtc {
namespace webrtc_pc_e2e {
@ -81,6 +81,17 @@ std::map<rtc::IPAddress, std::string> PopulateIpToPeer(
} // namespace
StatsBasedNetworkQualityMetricsReporter::
StatsBasedNetworkQualityMetricsReporter(
std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
NetworkEmulationManager* network_emulation,
test::MetricsLogger* metrics_logger)
: collector_(std::move(peer_endpoints), network_emulation),
clock_(network_emulation->time_controller()->GetClock()),
metrics_logger_(metrics_logger) {
RTC_CHECK(metrics_logger_);
}
StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector::
NetworkLayerStatsCollector(
std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
@ -241,99 +252,48 @@ void StatsBasedNetworkQualityMetricsReporter::ReportStats(
const NetworkLayerStats& network_layer_stats,
int64_t packet_loss,
const Timestamp& end_time) {
if (metrics_logger_ == nullptr) {
ReportResult("bytes_discarded_no_receiver", pc_label,
network_layer_stats.stats->BytesDropped().bytes(),
"sizeInBytes");
ReportResult("packets_discarded_no_receiver", pc_label,
network_layer_stats.stats->PacketsDropped(), "unitless");
metrics_logger_->LogSingleValueMetric(
"bytes_discarded_no_receiver", GetTestCaseName(pc_label),
network_layer_stats.stats->BytesDropped().bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_discarded_no_receiver", GetTestCaseName(pc_label),
network_layer_stats.stats->PacketsDropped(), Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
ReportResult("payload_bytes_received", pc_label,
pc_stats.payload_received.bytes(), "sizeInBytes");
ReportResult("payload_bytes_sent", pc_label, pc_stats.payload_sent.bytes(),
"sizeInBytes");
metrics_logger_->LogSingleValueMetric(
"payload_bytes_received", GetTestCaseName(pc_label),
pc_stats.payload_received.bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"payload_bytes_sent", GetTestCaseName(pc_label),
pc_stats.payload_sent.bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
ReportResult("bytes_sent", pc_label, pc_stats.total_sent.bytes(),
"sizeInBytes");
ReportResult("packets_sent", pc_label, pc_stats.packets_sent, "unitless");
ReportResult(
"average_send_rate", pc_label,
(pc_stats.total_sent / (end_time - start_time_)).bytes_per_sec(),
"bytesPerSecond");
ReportResult("bytes_received", pc_label, pc_stats.total_received.bytes(),
"sizeInBytes");
ReportResult("packets_received", pc_label, pc_stats.packets_received,
"unitless");
ReportResult(
"average_receive_rate", pc_label,
(pc_stats.total_received / (end_time - start_time_)).bytes_per_sec(),
"bytesPerSecond");
ReportResult("sent_packets_loss", pc_label, packet_loss, "unitless");
} else {
metrics_logger_->LogSingleValueMetric(
"bytes_discarded_no_receiver", GetTestCaseName(pc_label),
network_layer_stats.stats->BytesDropped().bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_discarded_no_receiver", GetTestCaseName(pc_label),
network_layer_stats.stats->PacketsDropped(), Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"payload_bytes_received", GetTestCaseName(pc_label),
pc_stats.payload_received.bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"payload_bytes_sent", GetTestCaseName(pc_label),
pc_stats.payload_sent.bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"bytes_sent", GetTestCaseName(pc_label), pc_stats.total_sent.bytes(),
Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_sent", GetTestCaseName(pc_label), pc_stats.packets_sent,
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"average_send_rate", GetTestCaseName(pc_label),
(pc_stats.total_sent / (end_time - start_time_)).kbps(),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"bytes_received", GetTestCaseName(pc_label),
pc_stats.total_received.bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_received", GetTestCaseName(pc_label),
pc_stats.packets_received, Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"average_receive_rate", GetTestCaseName(pc_label),
(pc_stats.total_received / (end_time - start_time_)).kbps(),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"sent_packets_loss", GetTestCaseName(pc_label), packet_loss,
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
}
}
void StatsBasedNetworkQualityMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& network_label,
const double value,
const std::string& unit) const {
test::PrintResult(metric_name, /*modifier=*/"",
GetTestCaseName(network_label), value, unit,
/*important=*/false);
}
void StatsBasedNetworkQualityMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& network_label,
const SamplesStatsCounter& value,
const std::string& unit) const {
test::PrintResult(metric_name, /*modifier=*/"",
GetTestCaseName(network_label), value, unit,
/*important=*/false);
metrics_logger_->LogSingleValueMetric(
"bytes_sent", GetTestCaseName(pc_label), pc_stats.total_sent.bytes(),
Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_sent", GetTestCaseName(pc_label), pc_stats.packets_sent,
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"average_send_rate", GetTestCaseName(pc_label),
(pc_stats.total_sent / (end_time - start_time_)).kbps(),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"bytes_received", GetTestCaseName(pc_label),
pc_stats.total_received.bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_received", GetTestCaseName(pc_label), pc_stats.packets_received,
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"average_receive_rate", GetTestCaseName(pc_label),
(pc_stats.total_received / (end_time - start_time_)).kbps(),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"sent_packets_loss", GetTestCaseName(pc_label), packet_loss,
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
}
std::string StatsBasedNetworkQualityMetricsReporter::GetTestCaseName(
@ -360,48 +320,28 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
log << " " << local_ips[i].ToString() << "\n";
}
if (!stats.stats->SentPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) {
ReportResult("sent_packets_size", peer_name,
stats.stats->SentPacketsSizeCounter(), "sizeInBytes");
} else {
metrics_logger_->LogMetric(
"sent_packets_size", GetTestCaseName(peer_name),
stats.stats->SentPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
metrics_logger_->LogMetric("sent_packets_size", GetTestCaseName(peer_name),
stats.stats->SentPacketsSizeCounter(),
Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
if (!stats.stats->ReceivedPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) {
ReportResult("received_packets_size", peer_name,
stats.stats->ReceivedPacketsSizeCounter(), "sizeInBytes");
} else {
metrics_logger_->LogMetric(
"received_packets_size", GetTestCaseName(peer_name),
stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
metrics_logger_->LogMetric(
"received_packets_size", GetTestCaseName(peer_name),
stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
if (!stats.stats->DroppedPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) {
ReportResult("dropped_packets_size", peer_name,
stats.stats->DroppedPacketsSizeCounter(), "sizeInBytes");
} else {
metrics_logger_->LogMetric(
"dropped_packets_size", GetTestCaseName(peer_name),
stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
metrics_logger_->LogMetric(
"dropped_packets_size", GetTestCaseName(peer_name),
stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
if (!stats.stats->SentPacketsQueueWaitTimeUs().IsEmpty()) {
if (metrics_logger_ == nullptr) {
ReportResult("sent_packets_queue_wait_time_us", peer_name,
stats.stats->SentPacketsQueueWaitTimeUs(), "unitless");
} else {
metrics_logger_->LogMetric(
"sent_packets_queue_wait_time_us", GetTestCaseName(peer_name),
stats.stats->SentPacketsQueueWaitTimeUs(), Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
}
metrics_logger_->LogMetric(
"sent_packets_queue_wait_time_us", GetTestCaseName(peer_name),
stats.stats->SentPacketsQueueWaitTimeUs(), Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
}
log << "Send statistic:\n"
@ -421,17 +361,11 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
<< " avg_rate (bytes/sec): " << source_average_send_rate.bytes_per_sec()
<< " avg_rate (bps): " << source_average_send_rate.bps() << "\n";
if (!entry.second->SentPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) {
ReportResult("sent_packets_size",
peer_name + "/" + entry.first.ToString(),
stats.stats->SentPacketsSizeCounter(), "sizeInBytes");
} else {
metrics_logger_->LogMetric(
"sent_packets_size",
GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->SentPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
metrics_logger_->LogMetric(
"sent_packets_size",
GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->SentPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
}
@ -454,30 +388,18 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
<< source_average_receive_rate.bytes_per_sec()
<< " avg_rate (bps): " << source_average_receive_rate.bps() << "\n";
if (!entry.second->ReceivedPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) {
ReportResult("received_packets_size",
peer_name + "/" + entry.first.ToString(),
stats.stats->ReceivedPacketsSizeCounter(), "sizeInBytes");
} else {
metrics_logger_->LogMetric(
"received_packets_size",
GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
metrics_logger_->LogMetric(
"received_packets_size",
GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
if (!entry.second->DroppedPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) {
ReportResult("dropped_packets_size",
peer_name + "/" + entry.first.ToString(),
stats.stats->DroppedPacketsSizeCounter(), "sizeInBytes");
} else {
metrics_logger_->LogMetric(
"dropped_packets_size",
GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
metrics_logger_->LogMetric(
"dropped_packets_size",
GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
}

19
test/pc/e2e/stats_based_network_quality_metrics_reporter.h
View file

@ -39,19 +39,10 @@ class StatsBasedNetworkQualityMetricsReporter
public:
// `networks` map peer name to network to report network layer stability stats
// and to log network layer metrics.
StatsBasedNetworkQualityMetricsReporter(
std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
NetworkEmulationManager* network_emulation)
: StatsBasedNetworkQualityMetricsReporter(std::move(peer_endpoints),
network_emulation,
/*metrics_logger=*/nullptr) {}
StatsBasedNetworkQualityMetricsReporter(
std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
NetworkEmulationManager* network_emulation,
test::MetricsLogger* metrics_logger)
: collector_(std::move(peer_endpoints), network_emulation),
clock_(network_emulation->time_controller()->GetClock()),
metrics_logger_(metrics_logger) {}
test::MetricsLogger* metrics_logger);
~StatsBasedNetworkQualityMetricsReporter() override = default;
void AddPeer(absl::string_view peer_name,
@ -111,14 +102,6 @@ class StatsBasedNetworkQualityMetricsReporter
const NetworkLayerStats& network_layer_stats,
int64_t packet_loss,
const Timestamp& end_time);
void ReportResult(const std::string& metric_name,
const std::string& network_label,
double value,
const std::string& unit) const;
void ReportResult(const std::string& metric_name,
const std::string& network_label,
const SamplesStatsCounter& value,
const std::string& unit) const;
std::string GetTestCaseName(absl::string_view network_label) const;
void LogNetworkLayerStats(const std::string& peer_name,
const NetworkLayerStats& stats) const;

1
video/BUILD.gn
View file

@ -585,6 +585,7 @@ if (rtc_include_tests) {
"../api:peer_connection_quality_test_fixture_api",
"../api:simulated_network_api",
"../api:time_controller",
"../api/test/metrics:global_metrics_logger_and_exporter",
"../api/video_codecs:video_codecs_api",
"../call:simulated_network",
"../modules/video_coding:webrtc_vp9",

4
video/pc_full_stack_tests.cc
View file

@ -17,6 +17,7 @@
#include "api/test/create_peer_connection_quality_test_frame_generator.h"
#include "api/test/create_peerconnection_quality_test_fixture.h"
#include "api/test/frame_generator_interface.h"
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "api/test/network_emulation_manager.h"
#include "api/test/peerconnection_quality_test_fixture.h"
#include "api/test/simulated_network.h"
@ -93,7 +94,8 @@ CreateTestFixture(const std::string& test_case_name,
bob_configurer);
fixture->AddQualityMetricsReporter(
std::make_unique<webrtc_pc_e2e::NetworkQualityMetricsReporter>(
network_links.first, network_links.second));
network_links.first, network_links.second,
test::GetGlobalMetricsLogger()));
return fixture;
}