mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-16 23:30:48 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

[PCLF] Fully switch to new metrics export API

Browse source 
Bug: b/246095034
Change-Id: I9d588d53320e4eb19cb569db2b97dddc013c22bd
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276621
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Tomas Gunnarsson <tommi@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38188}
This commit is contained in:
Artem Titov 2022-09-24 16:56:07 +02:00 committed by WebRTC LUCI CQ
parent 62b40efe5a
commit c45f4e4a3d
21 changed files with 370 additions and 680 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
audio/BUILD.gn
View file

@ -224,6 +224,7 @@ if (rtc_include_tests) {
"../api:peer_connection_quality_test_fixture_api", "../api:peer_connection_quality_test_fixture_api",
"../api:simulated_network_api", "../api:simulated_network_api",
"../api:time_controller", "../api:time_controller",
"../api/test/metrics:global_metrics_logger_and_exporter",
"../call:simulated_network", "../call:simulated_network",
"../common_audio", "../common_audio",
"../system_wrappers", "../system_wrappers",

4
audio/test/pc_low_bandwidth_audio_test.cc
View file

@ -15,6 +15,7 @@
#include "absl/strings/string_view.h" #include "absl/strings/string_view.h"
#include "api/test/create_network_emulation_manager.h" #include "api/test/create_network_emulation_manager.h"
#include "api/test/create_peerconnection_quality_test_fixture.h" #include "api/test/create_peerconnection_quality_test_fixture.h"
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "api/test/network_emulation_manager.h" #include "api/test/network_emulation_manager.h"
#include "api/test/peerconnection_quality_test_fixture.h" #include "api/test/peerconnection_quality_test_fixture.h"
#include "api/test/simulated_network.h" #include "api/test/simulated_network.h"
@ -88,7 +89,8 @@ CreateTestFixture(absl::string_view test_case_name,
bob_configurer); bob_configurer);
fixture->AddQualityMetricsReporter( fixture->AddQualityMetricsReporter(
std::make_unique<webrtc_pc_e2e::NetworkQualityMetricsReporter>( std::make_unique<webrtc_pc_e2e::NetworkQualityMetricsReporter>(
network_links.first, network_links.second)); network_links.first, network_links.second,
test::GetGlobalMetricsLogger()));
return fixture; return fixture;
} }

1
pc/BUILD.gn
View file

@ -2708,6 +2708,7 @@ if (rtc_include_tests && !build_with_chromium) {
"../api:peer_connection_quality_test_fixture_api", "../api:peer_connection_quality_test_fixture_api",
"../api:simulated_network_api", "../api:simulated_network_api",
"../api:time_controller", "../api:time_controller",
"../api/test/metrics:global_metrics_logger_and_exporter",
"../api/video_codecs:video_codecs_api", "../api/video_codecs:video_codecs_api",
"../call:simulated_network", "../call:simulated_network",
"../modules/video_coding:webrtc_vp9", "../modules/video_coding:webrtc_vp9",

2
pc/test/svc_e2e_tests.cc
View file

@ -17,6 +17,7 @@
#include "api/test/create_peer_connection_quality_test_frame_generator.h" #include "api/test/create_peer_connection_quality_test_frame_generator.h"
#include "api/test/create_peerconnection_quality_test_fixture.h" #include "api/test/create_peerconnection_quality_test_fixture.h"
#include "api/test/frame_generator_interface.h" #include "api/test/frame_generator_interface.h"
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "api/test/network_emulation_manager.h" #include "api/test/network_emulation_manager.h"
#include "api/test/peerconnection_quality_test_fixture.h" #include "api/test/peerconnection_quality_test_fixture.h"
#include "api/test/simulated_network.h" #include "api/test/simulated_network.h"
@ -185,6 +186,7 @@ class SvcVideoQualityAnalyzer : public DefaultVideoQualityAnalyzer {
explicit SvcVideoQualityAnalyzer(webrtc::Clock* clock) explicit SvcVideoQualityAnalyzer(webrtc::Clock* clock)
: DefaultVideoQualityAnalyzer(clock, : DefaultVideoQualityAnalyzer(clock,
test::GetGlobalMetricsLogger(),
DefaultVideoQualityAnalyzerOptions{ DefaultVideoQualityAnalyzerOptions{
.compute_psnr = false, .compute_psnr = false,
.compute_ssim = false, .compute_ssim = false,

6
test/pc/e2e/BUILD.gn
View file

@ -581,6 +581,7 @@ if (!build_with_chromium) {
"../..:test_support", "../..:test_support",
"../../../api:create_frame_generator", "../../../api:create_frame_generator",
"../../../api:rtp_packet_info", "../../../api:rtp_packet_info",
"../../../api/test/metrics:global_metrics_logger_and_exporter",
"../../../api/video:encoded_image", "../../../api/video:encoded_image",
"../../../api/video:video_frame", "../../../api/video:video_frame",
"../../../common_video", "../../../common_video",
@ -700,6 +701,7 @@ if (!build_with_chromium) {
"../../../api/test/metrics:metrics_logger_and_exporter", "../../../api/test/metrics:metrics_logger_and_exporter",
"../../../api/units:time_delta", "../../../api/units:time_delta",
"../../../api/units:timestamp", "../../../api/units:timestamp",
"../../../rtc_base:checks",
"../../../rtc_base:criticalsection", "../../../rtc_base:criticalsection",
"../../../rtc_base:logging", "../../../rtc_base:logging",
"../../../rtc_base:rtc_numerics", "../../../rtc_base:rtc_numerics",
@ -749,6 +751,7 @@ if (!build_with_chromium) {
"../../../api/units:data_size", "../../../api/units:data_size",
"../../../api/units:time_delta", "../../../api/units:time_delta",
"../../../api/units:timestamp", "../../../api/units:timestamp",
"../../../rtc_base:checks",
"../../../rtc_base:criticalsection", "../../../rtc_base:criticalsection",
"../../../rtc_base:rtc_numerics", "../../../rtc_base:rtc_numerics",
"../../../rtc_base/synchronization:mutex", "../../../rtc_base/synchronization:mutex",
@ -887,6 +890,7 @@ if (!build_with_chromium) {
"../../../api/test/metrics:metrics_logger", "../../../api/test/metrics:metrics_logger",
"../../../api/test/metrics:metrics_logger_and_exporter", "../../../api/test/metrics:metrics_logger_and_exporter",
"../../../api/units:data_size", "../../../api/units:data_size",
"../../../rtc_base:checks",
"../../../rtc_base:criticalsection", "../../../rtc_base:criticalsection",
"../../../rtc_base:rtc_event", "../../../rtc_base:rtc_event",
"../../../rtc_base/synchronization:mutex", "../../../rtc_base/synchronization:mutex",
@ -918,6 +922,7 @@ if (!build_with_chromium) {
"../../../api/units:data_size", "../../../api/units:data_size",
"../../../api/units:timestamp", "../../../api/units:timestamp",
"../../../rtc_base", "../../../rtc_base",
"../../../rtc_base:checks",
"../../../rtc_base:ip_address", "../../../rtc_base:ip_address",
"../../../rtc_base:rtc_event", "../../../rtc_base:rtc_event",
"../../../rtc_base:stringutils", "../../../rtc_base:stringutils",
@ -945,6 +950,7 @@ if (!build_with_chromium) {
"../../../api/test/metrics:metrics_logger", "../../../api/test/metrics:metrics_logger",
"../../../api/test/metrics:metrics_logger_and_exporter", "../../../api/test/metrics:metrics_logger_and_exporter",
"../../../api/units:timestamp", "../../../api/units:timestamp",
"../../../rtc_base:checks",
"../../../rtc_base:criticalsection", "../../../rtc_base:criticalsection",
"../../../rtc_base:rtc_event", "../../../rtc_base:rtc_event",
"../../../rtc_base:rtc_numerics", "../../../rtc_base:rtc_numerics",

81
test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.cc
View file

@ -13,6 +13,7 @@
#include "api/stats/rtc_stats.h" #include "api/stats/rtc_stats.h"
#include "api/stats/rtcstats_objects.h" #include "api/stats/rtcstats_objects.h"
#include "api/test/metrics/metric.h" #include "api/test/metrics/metric.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h" #include "rtc_base/logging.h"
namespace webrtc { namespace webrtc {
@ -21,6 +22,12 @@ namespace webrtc_pc_e2e {
using ::webrtc::test::ImprovementDirection; using ::webrtc::test::ImprovementDirection;
using ::webrtc::test::Unit; using ::webrtc::test::Unit;
DefaultAudioQualityAnalyzer::DefaultAudioQualityAnalyzer(
test::MetricsLogger* const metrics_logger)
: metrics_logger_(metrics_logger) {
RTC_CHECK(metrics_logger_);
}
void DefaultAudioQualityAnalyzer::Start(std::string test_case_name, void DefaultAudioQualityAnalyzer::Start(std::string test_case_name,
TrackIdStreamInfoMap* analyzer_helper) { TrackIdStreamInfoMap* analyzer_helper) {
test_case_name_ = std::move(test_case_name); test_case_name_ = std::move(test_case_name);
@ -116,48 +123,29 @@ std::string DefaultAudioQualityAnalyzer::GetTestCaseName(
} }
void DefaultAudioQualityAnalyzer::Stop() { void DefaultAudioQualityAnalyzer::Stop() {
using ::webrtc::test::ImproveDirection;
MutexLock lock(&lock_); MutexLock lock(&lock_);
for (auto& item : streams_stats_) { for (auto& item : streams_stats_) {
if (metrics_logger_ == nullptr) { metrics_logger_->LogMetric("expand_rate", GetTestCaseName(item.first),
ReportResult("expand_rate", item.first, item.second.expand_rate, item.second.expand_rate, Unit::kUnitless,
"unitless", ImproveDirection::kSmallerIsBetter); ImprovementDirection::kSmallerIsBetter);
ReportResult("accelerate_rate", item.first, item.second.accelerate_rate, metrics_logger_->LogMetric("accelerate_rate", GetTestCaseName(item.first),
"unitless", ImproveDirection::kSmallerIsBetter); item.second.accelerate_rate, Unit::kUnitless,
ReportResult("preemptive_rate", item.first, item.second.preemptive_rate, ImprovementDirection::kSmallerIsBetter);
"unitless", ImproveDirection::kSmallerIsBetter); metrics_logger_->LogMetric("preemptive_rate", GetTestCaseName(item.first),
ReportResult("speech_expand_rate", item.first, item.second.preemptive_rate, Unit::kUnitless,
item.second.speech_expand_rate, "unitless", ImprovementDirection::kSmallerIsBetter);
ImproveDirection::kSmallerIsBetter); metrics_logger_->LogMetric("speech_expand_rate",
ReportResult("average_jitter_buffer_delay_ms", item.first, GetTestCaseName(item.first),
item.second.average_jitter_buffer_delay_ms, "ms", item.second.speech_expand_rate, Unit::kUnitless,
ImproveDirection::kNone); ImprovementDirection::kSmallerIsBetter);
ReportResult("preferred_buffer_size_ms", item.first, metrics_logger_->LogMetric(
item.second.preferred_buffer_size_ms, "ms", "average_jitter_buffer_delay_ms", GetTestCaseName(item.first),
ImproveDirection::kNone); item.second.average_jitter_buffer_delay_ms, Unit::kMilliseconds,
} else { ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric("expand_rate", GetTestCaseName(item.first), metrics_logger_->LogMetric(
item.second.expand_rate, Unit::kUnitless, "preferred_buffer_size_ms", GetTestCaseName(item.first),
ImprovementDirection::kSmallerIsBetter); item.second.preferred_buffer_size_ms, Unit::kMilliseconds,
metrics_logger_->LogMetric("accelerate_rate", GetTestCaseName(item.first), ImprovementDirection::kNeitherIsBetter);
item.second.accelerate_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("preemptive_rate", GetTestCaseName(item.first),
item.second.preemptive_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric(
"speech_expand_rate", GetTestCaseName(item.first),
item.second.speech_expand_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric(
"average_jitter_buffer_delay_ms", GetTestCaseName(item.first),
item.second.average_jitter_buffer_delay_ms, Unit::kMilliseconds,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"preferred_buffer_size_ms", GetTestCaseName(item.first),
item.second.preferred_buffer_size_ms, Unit::kMilliseconds,
ImprovementDirection::kNeitherIsBetter);
}
} }
} }
@ -167,18 +155,5 @@ DefaultAudioQualityAnalyzer::GetAudioStreamsStats() const {
return streams_stats_; return streams_stats_;
} }
void DefaultAudioQualityAnalyzer::ReportResult(
const std::string& metric_name,
const std::string& stream_label,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) const {
test::PrintResultMeanAndError(
metric_name, /*modifier=*/"", GetTestCaseName(stream_label),
counter.IsEmpty() ? 0 : counter.GetAverage(),
counter.IsEmpty() ? 0 : counter.GetStandardDeviation(), unit,
/*important=*/false, improve_direction);
}
} // namespace webrtc_pc_e2e } // namespace webrtc_pc_e2e
} // namespace webrtc } // namespace webrtc

11
test/pc/e2e/analyzer/audio/default_audio_quality_analyzer.h
View file

@ -21,7 +21,6 @@
#include "api/test/track_id_stream_info_map.h" #include "api/test/track_id_stream_info_map.h"
#include "api/units/time_delta.h" #include "api/units/time_delta.h"
#include "rtc_base/synchronization/mutex.h" #include "rtc_base/synchronization/mutex.h"
#include "test/testsupport/perf_test.h"
namespace webrtc { namespace webrtc {
namespace webrtc_pc_e2e { namespace webrtc_pc_e2e {
@ -37,11 +36,8 @@ struct AudioStreamStats {
class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface { class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface {
public: public:
DefaultAudioQualityAnalyzer()
: DefaultAudioQualityAnalyzer(/*metrics_logger=*/nullptr) {}
explicit DefaultAudioQualityAnalyzer( explicit DefaultAudioQualityAnalyzer(
test::MetricsLogger* const metrics_logger) test::MetricsLogger* const metrics_logger);
: metrics_logger_(metrics_logger) {}
void Start(std::string test_case_name, void Start(std::string test_case_name,
TrackIdStreamInfoMap* analyzer_helper) override; TrackIdStreamInfoMap* analyzer_helper) override;
@ -66,11 +62,6 @@ class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface {
}; };
std::string GetTestCaseName(const std::string& stream_label) const; std::string GetTestCaseName(const std::string& stream_label) const;
void ReportResult(const std::string& metric_name,
const std::string& stream_label,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) const;
test::MetricsLogger* const metrics_logger_; test::MetricsLogger* const metrics_logger_;

299
test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc
View file

@ -23,6 +23,7 @@
#include "api/video/i420_buffer.h" #include "api/video/i420_buffer.h"
#include "api/video/video_frame.h" #include "api/video/video_frame.h"
#include "common_video/libyuv/include/webrtc_libyuv.h" #include "common_video/libyuv/include/webrtc_libyuv.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h" #include "rtc_base/logging.h"
#include "rtc_base/platform_thread.h" #include "rtc_base/platform_thread.h"
#include "rtc_base/strings/string_builder.h" #include "rtc_base/strings/string_builder.h"
@ -124,13 +125,6 @@ SamplesStatsCounter::StatsSample StatsSample(double value,
} // namespace } // namespace
DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
DefaultVideoQualityAnalyzerOptions options)
: DefaultVideoQualityAnalyzer(clock,
/*metrics_logger=*/nullptr,
std::move(options)) {}
DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer( DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
webrtc::Clock* clock, webrtc::Clock* clock,
test::MetricsLogger* metrics_logger, test::MetricsLogger* metrics_logger,
@ -138,7 +132,9 @@ DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
: options_(options), : options_(options),
clock_(clock), clock_(clock),
metrics_logger_(metrics_logger), metrics_logger_(metrics_logger),
frames_comparator_(clock, cpu_measurer_, options) {} frames_comparator_(clock, cpu_measurer_, options) {
RTC_CHECK(metrics_logger_);
}
DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() { DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() {
Stop(); Stop();
@ -891,22 +887,14 @@ void DefaultVideoQualityAnalyzer::
} }
void DefaultVideoQualityAnalyzer::ReportResults() { void DefaultVideoQualityAnalyzer::ReportResults() {
using ::webrtc::test::ImproveDirection;
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
for (auto& item : frames_comparator_.stream_stats()) { for (auto& item : frames_comparator_.stream_stats()) {
ReportResults(GetTestCaseName(ToMetricName(item.first)), item.second, ReportResults(GetTestCaseName(ToMetricName(item.first)), item.second,
stream_frame_counters_.at(item.first)); stream_frame_counters_.at(item.first));
} }
if (metrics_logger_ == nullptr) { metrics_logger_->LogSingleValueMetric("cpu_usage_%", test_label_,
test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(), Unit::kUnitless,
GetCpuUsagePercent(), "%", false, ImprovementDirection::kSmallerIsBetter);
ImproveDirection::kSmallerIsBetter);
} else {
metrics_logger_->LogSingleValueMetric(
"cpu_usage_%", test_label_, GetCpuUsagePercent(), Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter);
}
LogFrameCounters("Global", frame_counters_); LogFrameCounters("Global", frame_counters_);
if (!unknown_sender_frame_counters_.empty()) { if (!unknown_sender_frame_counters_.empty()) {
RTC_LOG(LS_INFO) << "Received frame counters with unknown frame id:"; RTC_LOG(LS_INFO) << "Received frame counters with unknown frame id:";
@ -943,7 +931,6 @@ void DefaultVideoQualityAnalyzer::ReportResults(
const std::string& test_case_name, const std::string& test_case_name,
const StreamStats& stats, const StreamStats& stats,
const FrameCounters& frame_counters) { const FrameCounters& frame_counters) {
using ::webrtc::test::ImproveDirection;
TimeDelta test_duration = Now() - start_time_; TimeDelta test_duration = Now() - start_time_;
double sum_squared_interframe_delays_secs = 0; double sum_squared_interframe_delays_secs = 0;
@ -972,199 +959,101 @@ void DefaultVideoQualityAnalyzer::ReportResults(
video_duration.seconds<double>() / sum_squared_interframe_delays_secs; video_duration.seconds<double>() / sum_squared_interframe_delays_secs;
} }
if (metrics_logger_ == nullptr) { metrics_logger_->LogMetric("psnr_dB", test_case_name, stats.psnr,
// Report results through old performance metrics API. Unit::kUnitless,
ReportResult("psnr", test_case_name, stats.psnr, "dB", ImprovementDirection::kBiggerIsBetter);
ImproveDirection::kBiggerIsBetter); metrics_logger_->LogMetric("ssim", test_case_name, stats.ssim,
ReportResult("ssim", test_case_name, stats.ssim, "unitless", Unit::kUnitless,
ImproveDirection::kBiggerIsBetter); ImprovementDirection::kBiggerIsBetter);
ReportResult("transport_time", test_case_name, stats.transport_time_ms, metrics_logger_->LogMetric("transport_time", test_case_name,
"ms", ImproveDirection::kSmallerIsBetter); stats.transport_time_ms, Unit::kMilliseconds,
ReportResult("total_delay_incl_transport", test_case_name, ImprovementDirection::kSmallerIsBetter);
stats.total_delay_incl_transport_ms, "ms", metrics_logger_->LogMetric("total_delay_incl_transport", test_case_name,
ImproveDirection::kSmallerIsBetter); stats.total_delay_incl_transport_ms,
ReportResult("time_between_rendered_frames", test_case_name, Unit::kMilliseconds,
stats.time_between_rendered_frames_ms, "ms", ImprovementDirection::kSmallerIsBetter);
ImproveDirection::kSmallerIsBetter); metrics_logger_->LogMetric("time_between_rendered_frames", test_case_name,
test::PrintResult("harmonic_framerate", "", test_case_name, stats.time_between_rendered_frames_ms,
harmonic_framerate_fps, "Hz", /*important=*/false, Unit::kMilliseconds,
ImproveDirection::kBiggerIsBetter); ImprovementDirection::kSmallerIsBetter);
test::PrintResult("encode_frame_rate", "", test_case_name, metrics_logger_->LogSingleValueMetric("harmonic_framerate", test_case_name,
stats.encode_frame_rate.IsEmpty() harmonic_framerate_fps, Unit::kHertz,
? 0 ImprovementDirection::kBiggerIsBetter);
: stats.encode_frame_rate.GetEventsPerSecond(), metrics_logger_->LogSingleValueMetric(
"Hz", /*important=*/false, "encode_frame_rate", test_case_name,
ImproveDirection::kBiggerIsBetter); stats.encode_frame_rate.IsEmpty()
ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms", ? 0
ImproveDirection::kSmallerIsBetter); : stats.encode_frame_rate.GetEventsPerSecond(),
ReportResult("time_between_freezes", test_case_name, Unit::kHertz, ImprovementDirection::kBiggerIsBetter);
stats.time_between_freezes_ms, "ms", metrics_logger_->LogMetric("encode_time", test_case_name,
ImproveDirection::kBiggerIsBetter); stats.encode_time_ms, Unit::kMilliseconds,
ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms", ImprovementDirection::kSmallerIsBetter);
ImproveDirection::kSmallerIsBetter); metrics_logger_->LogMetric("time_between_freezes", test_case_name,
ReportResult("pixels_per_frame", test_case_name, stats.time_between_freezes_ms, Unit::kMilliseconds,
stats.resolution_of_rendered_frame, "count", ImprovementDirection::kBiggerIsBetter);
ImproveDirection::kBiggerIsBetter); metrics_logger_->LogMetric("freeze_time_ms", test_case_name,
test::PrintResult("min_psnr", "", test_case_name, stats.freeze_time_ms, Unit::kMilliseconds,
stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB", ImprovementDirection::kSmallerIsBetter);
/*important=*/false, ImproveDirection::kBiggerIsBetter); metrics_logger_->LogMetric("pixels_per_frame", test_case_name,
ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms", stats.resolution_of_rendered_frame, Unit::kCount,
ImproveDirection::kSmallerIsBetter); ImprovementDirection::kBiggerIsBetter);
ReportResult("receive_to_render_time", test_case_name, metrics_logger_->LogSingleValueMetric(
stats.receive_to_render_time_ms, "ms", "min_psnr_dB", test_case_name,
ImproveDirection::kSmallerIsBetter); stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), Unit::kUnitless,
test::PrintResult("dropped_frames", "", test_case_name, ImprovementDirection::kBiggerIsBetter);
frame_counters.dropped, "count", metrics_logger_->LogMetric("decode_time", test_case_name,
/*important=*/false, ImproveDirection::kSmallerIsBetter); stats.decode_time_ms, Unit::kMilliseconds,
test::PrintResult("frames_in_flight", "", test_case_name, ImprovementDirection::kSmallerIsBetter);
frame_counters.captured - frame_counters.rendered - metrics_logger_->LogMetric(
frame_counters.dropped, "receive_to_render_time", test_case_name, stats.receive_to_render_time_ms,
"count", Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter);
/*important=*/false, ImproveDirection::kSmallerIsBetter); metrics_logger_->LogSingleValueMetric("dropped_frames", test_case_name,
test::PrintResult("rendered_frames", "", test_case_name, frame_counters.dropped, Unit::kCount,
frame_counters.rendered, "count", /*important=*/false, ImprovementDirection::kSmallerIsBetter);
ImproveDirection::kBiggerIsBetter); metrics_logger_->LogSingleValueMetric(
ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered, "frames_in_flight", test_case_name,
"count", ImproveDirection::kSmallerIsBetter); frame_counters.captured - frame_counters.rendered -
ReportResult("target_encode_bitrate", test_case_name, frame_counters.dropped,
stats.target_encode_bitrate / kBitsInByte, "bytesPerSecond", Unit::kCount, ImprovementDirection::kSmallerIsBetter);
ImproveDirection::kNone); metrics_logger_->LogSingleValueMetric("rendered_frames", test_case_name,
test::PrintResult("actual_encode_bitrate", "", test_case_name, frame_counters.rendered, Unit::kCount,
static_cast<double>(stats.total_encoded_images_payload) / ImprovementDirection::kBiggerIsBetter);
test_duration.seconds<double>(), metrics_logger_->LogMetric("max_skipped", test_case_name,
"bytesPerSecond", /*important=*/false, stats.skipped_between_rendered, Unit::kCount,
ImproveDirection::kNone); ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("target_encode_bitrate", test_case_name,
stats.target_encode_bitrate / 1000,
Unit::kKilobitsPerSecond,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"actual_encode_bitrate", test_case_name,
static_cast<double>(stats.total_encoded_images_payload) /
test_duration.seconds<double>() * kBitsInByte / 1000,
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
if (options_.report_detailed_frame_stats) { if (options_.report_detailed_frame_stats) {
test::PrintResult("num_encoded_frames", "", test_case_name,
frame_counters.encoded, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_decoded_frames", "", test_case_name,
frame_counters.decoded, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_send_key_frames", "", test_case_name,
stats.num_send_key_frames, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_recv_key_frames", "", test_case_name,
stats.num_recv_key_frames, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
ReportResult("recv_key_frame_size_bytes", test_case_name,
stats.recv_key_frame_size_bytes, "count",
ImproveDirection::kBiggerIsBetter);
ReportResult("recv_delta_frame_size_bytes", test_case_name,
stats.recv_delta_frame_size_bytes, "count",
ImproveDirection::kBiggerIsBetter);
}
} else {
metrics_logger_->LogMetric("psnr_dB", test_case_name, stats.psnr,
Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("ssim", test_case_name, stats.ssim,
Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("transport_time", test_case_name,
stats.transport_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("total_delay_incl_transport", test_case_name,
stats.total_delay_incl_transport_ms,
Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("time_between_rendered_frames", test_case_name,
stats.time_between_rendered_frames_ms,
Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric(
"harmonic_framerate", test_case_name, harmonic_framerate_fps, "num_encoded_frames", test_case_name, frame_counters.encoded,
Unit::kHertz, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"encode_frame_rate", test_case_name,
stats.encode_frame_rate.IsEmpty()
? 0
: stats.encode_frame_rate.GetEventsPerSecond(),
Unit::kHertz, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("encode_time", test_case_name,
stats.encode_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric(
"time_between_freezes", test_case_name, stats.time_between_freezes_ms,
Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("freeze_time_ms", test_case_name,
stats.freeze_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("pixels_per_frame", test_case_name,
stats.resolution_of_rendered_frame, Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"min_psnr_dB", test_case_name,
stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("decode_time", test_case_name,
stats.decode_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("receive_to_render_time", test_case_name,
stats.receive_to_render_time_ms,
Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric(
"dropped_frames", test_case_name, frame_counters.dropped, Unit::kCount,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric(
"frames_in_flight", test_case_name,
frame_counters.captured - frame_counters.rendered -
frame_counters.dropped,
Unit::kCount, ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric(
"rendered_frames", test_case_name, frame_counters.rendered,
Unit::kCount, ImprovementDirection::kBiggerIsBetter); Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("max_skipped", test_case_name,
stats.skipped_between_rendered, Unit::kCount,
ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("target_encode_bitrate", test_case_name,
stats.target_encode_bitrate / 1000,
Unit::kKilobitsPerSecond,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric(
"actual_encode_bitrate", test_case_name, "num_decoded_frames", test_case_name, frame_counters.decoded,
static_cast<double>(stats.total_encoded_images_payload) / Unit::kCount, ImprovementDirection::kBiggerIsBetter);
test_duration.seconds<double>() * kBitsInByte / 1000, metrics_logger_->LogSingleValueMetric(
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); "num_send_key_frames", test_case_name, stats.num_send_key_frames,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"num_recv_key_frames", test_case_name, stats.num_recv_key_frames,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
if (options_.report_detailed_frame_stats) { metrics_logger_->LogMetric("recv_key_frame_size_bytes", test_case_name,
metrics_logger_->LogSingleValueMetric( stats.recv_key_frame_size_bytes, Unit::kCount,
"num_encoded_frames", test_case_name, frame_counters.encoded, ImprovementDirection::kBiggerIsBetter);
Unit::kCount, ImprovementDirection::kBiggerIsBetter); metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name,
metrics_logger_->LogSingleValueMetric( stats.recv_delta_frame_size_bytes, Unit::kCount,
"num_decoded_frames", test_case_name, frame_counters.decoded, ImprovementDirection::kBiggerIsBetter);
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"num_send_key_frames", test_case_name, stats.num_send_key_frames,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
"num_recv_key_frames", test_case_name, stats.num_recv_key_frames,
Unit::kCount, ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("recv_key_frame_size_bytes", test_case_name,
stats.recv_key_frame_size_bytes, Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name,
stats.recv_delta_frame_size_bytes,
Unit::kCount,
ImprovementDirection::kBiggerIsBetter);
}
} }
} }
void DefaultVideoQualityAnalyzer::ReportResult(
const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) {
test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit,
/*important=*/false, improve_direction);
}
std::string DefaultVideoQualityAnalyzer::GetTestCaseName( std::string DefaultVideoQualityAnalyzer::GetTestCaseName(
const std::string& stream_label) const { const std::string& stream_label) const {
return test_label_ + "/" + stream_label; return test_label_ + "/" + stream_label;

11
test/pc/e2e/analyzer/video/default_video_quality_analyzer.h
View file

@ -41,15 +41,11 @@
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h"
#include "test/pc/e2e/analyzer/video/names_collection.h" #include "test/pc/e2e/analyzer/video/names_collection.h"
#include "test/testsupport/perf_test.h"
namespace webrtc { namespace webrtc {
class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
public: public:
explicit DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
DefaultVideoQualityAnalyzerOptions options = {});
DefaultVideoQualityAnalyzer(webrtc::Clock* clock, DefaultVideoQualityAnalyzer(webrtc::Clock* clock,
test::MetricsLogger* metrics_logger, test::MetricsLogger* metrics_logger,
DefaultVideoQualityAnalyzerOptions options = {}); DefaultVideoQualityAnalyzerOptions options = {});
@ -131,13 +127,6 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
const StreamStats& stats, const StreamStats& stats,
const FrameCounters& frame_counters) const FrameCounters& frame_counters)
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
// Report result for single metric for specified stream.
static void ReportResult(const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction =
webrtc::test::ImproveDirection::kNone);
// Returns name of current test case for reporting. // Returns name of current test case for reporting.
std::string GetTestCaseName(const std::string& stream_label) const; std::string GetTestCaseName(const std::string& stream_label) const;
Timestamp Now(); Timestamp Now();

65
test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc
View file

@ -18,6 +18,7 @@
#include "api/rtp_packet_info.h" #include "api/rtp_packet_info.h"
#include "api/rtp_packet_infos.h" #include "api/rtp_packet_infos.h"
#include "api/test/create_frame_generator.h" #include "api/test/create_frame_generator.h"
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "api/video/encoded_image.h" #include "api/video/encoded_image.h"
#include "api/video/i420_buffer.h" #include "api/video/i420_buffer.h"
#include "api/video/video_frame.h" #include "api/video/video_frame.h"
@ -148,6 +149,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -199,6 +201,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -280,6 +283,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -330,6 +334,7 @@ TEST(DefaultVideoQualityAnalyzerTest, NormalScenario) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -388,6 +393,7 @@ TEST(DefaultVideoQualityAnalyzerTest, OneFrameReceivedTwice) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -444,6 +450,7 @@ TEST(DefaultVideoQualityAnalyzerTest, NormalScenario2Receivers) {
constexpr char kCharlie[] = "charlie"; constexpr char kCharlie[] = "charlie";
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie}, analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -574,6 +581,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
constexpr char kCharlie[] = "charlie"; constexpr char kCharlie[] = "charlie";
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie}, analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -633,6 +641,7 @@ TEST(DefaultVideoQualityAnalyzerTest, HeavyQualityMetricsFromEqualFrames) {
analyzer_options.max_frames_in_flight_per_stream_count = analyzer_options.max_frames_in_flight_per_stream_count =
kMaxFramesInFlightPerStream; kMaxFramesInFlightPerStream;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
analyzer_options); analyzer_options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -692,6 +701,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
analyzer_options.max_frames_in_flight_per_stream_count = analyzer_options.max_frames_in_flight_per_stream_count =
kMaxFramesInFlightPerStream; kMaxFramesInFlightPerStream;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
analyzer_options); analyzer_options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -750,6 +760,7 @@ TEST(DefaultVideoQualityAnalyzerTest, CpuUsage) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -813,6 +824,7 @@ TEST(DefaultVideoQualityAnalyzerTest, RuntimeParticipantsAdding) {
constexpr int kTwoThirdFrames = 2 * kOneThirdFrames; constexpr int kTwoThirdFrames = 2 * kOneThirdFrames;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", {}, kAnalyzerMaxThreadsCount); analyzer.Start("test_case", {}, kAnalyzerMaxThreadsCount);
@ -958,6 +970,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
constexpr char kAlice[] = "alice"; constexpr char kAlice[] = "alice";
constexpr char kBob[] = "bob"; constexpr char kBob[] = "bob";
@ -1020,7 +1033,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true; options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1115,7 +1129,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true; options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1208,6 +1223,7 @@ TEST(DefaultVideoQualityAnalyzerTest, CodecTrackedCorrectly) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -1278,7 +1294,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1370,7 +1387,8 @@ TEST(
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true; options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1474,7 +1492,8 @@ TEST(DefaultVideoQualityAnalyzerTest, GetStreamFrames) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1518,7 +1537,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverReceivedFramesWhenSenderRemoved) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1561,7 +1581,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true; options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1604,7 +1625,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true; options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1647,7 +1669,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true; options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1702,7 +1725,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeCapturing2ndFrame) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1742,7 +1766,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforePreEncoded) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1784,7 +1809,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeEncoded) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1827,7 +1853,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1873,7 +1900,8 @@ TEST(DefaultVideoQualityAnalyzerTest, UnregisterOneAndRegisterAnother) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{"alice", "bob", "charlie"}, std::vector<std::string>{"alice", "bob", "charlie"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1934,7 +1962,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{"alice", "bob", "charlie"}, std::vector<std::string>{"alice", "bob", "charlie"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1989,7 +2018,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -2026,7 +2056,8 @@ TEST_P(DefaultVideoQualityAnalyzerTimeBetweenFreezesTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);

51
test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc
View file

@ -16,6 +16,7 @@
#include "api/units/data_rate.h" #include "api/units/data_rate.h"
#include "api/units/time_delta.h" #include "api/units/time_delta.h"
#include "api/units/timestamp.h" #include "api/units/timestamp.h"
#include "rtc_base/checks.h"
namespace webrtc { namespace webrtc {
namespace webrtc_pc_e2e { namespace webrtc_pc_e2e {
@ -30,6 +31,13 @@ SamplesStatsCounter BytesPerSecondToKbps(const SamplesStatsCounter& counter) {
} // namespace } // namespace
VideoQualityMetricsReporter::VideoQualityMetricsReporter(
Clock* const clock,
test::MetricsLogger* const metrics_logger)
: clock_(clock), metrics_logger_(metrics_logger) {
RTC_CHECK(metrics_logger_);
}
void VideoQualityMetricsReporter::Start( void VideoQualityMetricsReporter::Start(
absl::string_view test_case_name, absl::string_view test_case_name,
const TrackIdStreamInfoMap* /*reporter_helper*/) { const TrackIdStreamInfoMap* /*reporter_helper*/) {
@ -122,37 +130,18 @@ std::string VideoQualityMetricsReporter::GetTestCaseName(
void VideoQualityMetricsReporter::ReportVideoBweResults( void VideoQualityMetricsReporter::ReportVideoBweResults(
const std::string& test_case_name, const std::string& test_case_name,
const VideoBweStats& video_bwe_stats) { const VideoBweStats& video_bwe_stats) {
if (metrics_logger_ == nullptr) { metrics_logger_->LogMetric(
ReportResult("available_send_bandwidth", test_case_name, "available_send_bandwidth", test_case_name,
video_bwe_stats.available_send_bandwidth, "bytesPerSecond"); BytesPerSecondToKbps(video_bwe_stats.available_send_bandwidth),
ReportResult("transmission_bitrate", test_case_name, Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
video_bwe_stats.transmission_bitrate, "bytesPerSecond"); metrics_logger_->LogMetric(
ReportResult("retransmission_bitrate", test_case_name, "transmission_bitrate", test_case_name,
video_bwe_stats.retransmission_bitrate, "bytesPerSecond"); BytesPerSecondToKbps(video_bwe_stats.transmission_bitrate),
} else { Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric( metrics_logger_->LogMetric(
"available_send_bandwidth", test_case_name, "retransmission_bitrate", test_case_name,
BytesPerSecondToKbps(video_bwe_stats.available_send_bandwidth), BytesPerSecondToKbps(video_bwe_stats.retransmission_bitrate),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"transmission_bitrate", test_case_name,
BytesPerSecondToKbps(video_bwe_stats.transmission_bitrate),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"retransmission_bitrate", test_case_name,
BytesPerSecondToKbps(video_bwe_stats.retransmission_bitrate),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
}
}
void VideoQualityMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) {
test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit,
/*important=*/false, improve_direction);
} }
} // namespace webrtc_pc_e2e } // namespace webrtc_pc_e2e

16
test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h
View file

@ -22,7 +22,6 @@
#include "api/units/data_size.h" #include "api/units/data_size.h"
#include "api/units/timestamp.h" #include "api/units/timestamp.h"
#include "rtc_base/synchronization/mutex.h" #include "rtc_base/synchronization/mutex.h"
#include "test/testsupport/perf_test.h"
namespace webrtc { namespace webrtc {
namespace webrtc_pc_e2e { namespace webrtc_pc_e2e {
@ -36,12 +35,8 @@ struct VideoBweStats {
class VideoQualityMetricsReporter class VideoQualityMetricsReporter
: public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
public: public:
explicit VideoQualityMetricsReporter(Clock* const clock) VideoQualityMetricsReporter(Clock* const clock,
: VideoQualityMetricsReporter(clock, /*metrics_logger=*/nullptr) {} test::MetricsLogger* const metrics_logger);
explicit VideoQualityMetricsReporter(
Clock* const clock,
test::MetricsLogger* const metrics_logger)
: clock_(clock), metrics_logger_(metrics_logger) {}
~VideoQualityMetricsReporter() override = default; ~VideoQualityMetricsReporter() override = default;
void Start(absl::string_view test_case_name, void Start(absl::string_view test_case_name,
@ -63,13 +58,6 @@ class VideoQualityMetricsReporter
std::string GetTestCaseName(const std::string& stream_label) const; std::string GetTestCaseName(const std::string& stream_label) const;
void ReportVideoBweResults(const std::string& test_case_name, void ReportVideoBweResults(const std::string& test_case_name,
const VideoBweStats& video_bwe_stats); const VideoBweStats& video_bwe_stats);
// Report result for single metric for specified stream.
static void ReportResult(const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction =
webrtc::test::ImproveDirection::kNone);
Timestamp Now() const { return clock_->CurrentTime(); } Timestamp Now() const { return clock_->CurrentTime(); }
Clock* const clock_; Clock* const clock_;

48
test/pc/e2e/cross_media_metrics_reporter.cc
View file

@ -16,6 +16,7 @@
#include "api/stats/rtcstats_objects.h" #include "api/stats/rtcstats_objects.h"
#include "api/test/metrics/metric.h" #include "api/test/metrics/metric.h"
#include "api/units/timestamp.h" #include "api/units/timestamp.h"
#include "rtc_base/checks.h"
#include "rtc_base/event.h" #include "rtc_base/event.h"
#include "system_wrappers/include/field_trial.h" #include "system_wrappers/include/field_trial.h"
@ -25,6 +26,12 @@ namespace webrtc_pc_e2e {
using ::webrtc::test::ImprovementDirection; using ::webrtc::test::ImprovementDirection;
using ::webrtc::test::Unit; using ::webrtc::test::Unit;
CrossMediaMetricsReporter::CrossMediaMetricsReporter(
test::MetricsLogger* metrics_logger)
: metrics_logger_(metrics_logger) {
RTC_CHECK(metrics_logger_);
}
void CrossMediaMetricsReporter::Start( void CrossMediaMetricsReporter::Start(
absl::string_view test_case_name, absl::string_view test_case_name,
const TrackIdStreamInfoMap* reporter_helper) { const TrackIdStreamInfoMap* reporter_helper) {
@ -102,40 +109,19 @@ void CrossMediaMetricsReporter::StopAndReportResults() {
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
for (const auto& pair : stats_info_) { for (const auto& pair : stats_info_) {
const std::string& sync_group = pair.first; const std::string& sync_group = pair.first;
if (metrics_logger_ == nullptr) { metrics_logger_->LogMetric(
ReportResult("audio_ahead_ms", "audio_ahead_ms",
GetTestCaseName(pair.second.audio_stream_label, sync_group), GetTestCaseName(pair.second.audio_stream_label, sync_group),
pair.second.audio_ahead_ms, "ms", pair.second.audio_ahead_ms, Unit::kMilliseconds,
webrtc::test::ImproveDirection::kSmallerIsBetter); webrtc::test::ImprovementDirection::kSmallerIsBetter);
ReportResult("video_ahead_ms", metrics_logger_->LogMetric(
GetTestCaseName(pair.second.video_stream_label, sync_group), "video_ahead_ms",
pair.second.video_ahead_ms, "ms", GetTestCaseName(pair.second.video_stream_label, sync_group),
webrtc::test::ImproveDirection::kSmallerIsBetter); pair.second.video_ahead_ms, Unit::kMilliseconds,
} else { webrtc::test::ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric(
"audio_ahead_ms",
GetTestCaseName(pair.second.audio_stream_label, sync_group),
pair.second.audio_ahead_ms, Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric(
"video_ahead_ms",
GetTestCaseName(pair.second.video_stream_label, sync_group),
pair.second.video_ahead_ms, Unit::kMilliseconds,
webrtc::test::ImprovementDirection::kSmallerIsBetter);
}
} }
} }
void CrossMediaMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) {
test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit,
/*important=*/false, improve_direction);
}
std::string CrossMediaMetricsReporter::GetTestCaseName( std::string CrossMediaMetricsReporter::GetTestCaseName(
const std::string& stream_label, const std::string& stream_label,
const std::string& sync_group) const { const std::string& sync_group) const {

12
test/pc/e2e/cross_media_metrics_reporter.h
View file

@ -22,7 +22,6 @@
#include "api/test/track_id_stream_info_map.h" #include "api/test/track_id_stream_info_map.h"
#include "api/units/timestamp.h" #include "api/units/timestamp.h"
#include "rtc_base/synchronization/mutex.h" #include "rtc_base/synchronization/mutex.h"
#include "test/testsupport/perf_test.h"
namespace webrtc { namespace webrtc {
namespace webrtc_pc_e2e { namespace webrtc_pc_e2e {
@ -30,10 +29,7 @@ namespace webrtc_pc_e2e {
class CrossMediaMetricsReporter class CrossMediaMetricsReporter
: public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
public: public:
CrossMediaMetricsReporter() explicit CrossMediaMetricsReporter(test::MetricsLogger* metrics_logger);
: CrossMediaMetricsReporter(/*metrics_logger=*/nullptr) {}
explicit CrossMediaMetricsReporter(test::MetricsLogger* metrics_logger)
: metrics_logger_(metrics_logger) {}
~CrossMediaMetricsReporter() override = default; ~CrossMediaMetricsReporter() override = default;
void Start(absl::string_view test_case_name, void Start(absl::string_view test_case_name,
@ -52,12 +48,6 @@ class CrossMediaMetricsReporter
std::string video_stream_label; std::string video_stream_label;
}; };
static void ReportResult(const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction =
webrtc::test::ImproveDirection::kNone);
std::string GetTestCaseName(const std::string& stream_label, std::string GetTestCaseName(const std::string& stream_label,
const std::string& sync_group) const; const std::string& sync_group) const;

135
test/pc/e2e/network_quality_metrics_reporter.cc
View file

@ -14,9 +14,9 @@
#include "api/stats/rtc_stats.h" #include "api/stats/rtc_stats.h"
#include "api/stats/rtcstats_objects.h" #include "api/stats/rtcstats_objects.h"
#include "api/test/metrics/metric.h" #include "api/test/metrics/metric.h"
#include "rtc_base/checks.h"
#include "rtc_base/event.h" #include "rtc_base/event.h"
#include "system_wrappers/include/field_trial.h" #include "system_wrappers/include/field_trial.h"
#include "test/testsupport/perf_test.h"
namespace webrtc { namespace webrtc {
namespace webrtc_pc_e2e { namespace webrtc_pc_e2e {
@ -31,6 +31,17 @@ constexpr TimeDelta kStatsWaitTimeout = TimeDelta::Seconds(1);
// sent/received per stream. If enabled, padding and headers are not included // sent/received per stream. If enabled, padding and headers are not included
// in bytes sent or received. // in bytes sent or received.
constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats"; constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats";
} // namespace
NetworkQualityMetricsReporter::NetworkQualityMetricsReporter(
EmulatedNetworkManagerInterface* alice_network,
EmulatedNetworkManagerInterface* bob_network,
test::MetricsLogger* metrics_logger)
: alice_network_(alice_network),
bob_network_(bob_network),
metrics_logger_(metrics_logger) {
RTC_CHECK(metrics_logger_);
} }
void NetworkQualityMetricsReporter::Start( void NetworkQualityMetricsReporter::Start(
@ -113,93 +124,49 @@ void NetworkQualityMetricsReporter::ReportStats(
const std::string& network_label, const std::string& network_label,
std::unique_ptr<EmulatedNetworkStats> stats, std::unique_ptr<EmulatedNetworkStats> stats,
int64_t packet_loss) { int64_t packet_loss) {
if (metrics_logger_ == nullptr) { metrics_logger_->LogSingleValueMetric(
ReportResult("bytes_sent", network_label, stats->BytesSent().bytes(), "bytes_sent", GetTestCaseName(network_label), stats->BytesSent().bytes(),
"sizeInBytes"); Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
ReportResult("packets_sent", network_label, stats->PacketsSent(), metrics_logger_->LogSingleValueMetric(
"unitless"); "packets_sent", GetTestCaseName(network_label), stats->PacketsSent(),
ReportResult("average_send_rate", network_label, Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
stats->PacketsSent() >= 2 metrics_logger_->LogSingleValueMetric(
? stats->AverageSendRate().bytes_per_sec() "average_send_rate", GetTestCaseName(network_label),
: 0, stats->PacketsSent() >= 2 ? stats->AverageSendRate().kbps() : 0,
"bytesPerSecond"); Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
ReportResult("bytes_discarded_no_receiver", network_label, metrics_logger_->LogSingleValueMetric(
stats->BytesDropped().bytes(), "sizeInBytes"); "bytes_discarded_no_receiver", GetTestCaseName(network_label),
ReportResult("packets_discarded_no_receiver", network_label, stats->BytesDropped().bytes(), Unit::kBytes,
stats->PacketsDropped(), "unitless"); ImprovementDirection::kNeitherIsBetter);
ReportResult("bytes_received", network_label, metrics_logger_->LogSingleValueMetric(
stats->BytesReceived().bytes(), "sizeInBytes"); "packets_discarded_no_receiver", GetTestCaseName(network_label),
ReportResult("packets_received", network_label, stats->PacketsReceived(), stats->PacketsDropped(), Unit::kUnitless,
"unitless"); ImprovementDirection::kNeitherIsBetter);
ReportResult("average_receive_rate", network_label, metrics_logger_->LogSingleValueMetric(
stats->PacketsReceived() >= 2 "bytes_received", GetTestCaseName(network_label),
? stats->AverageReceiveRate().bytes_per_sec() stats->BytesReceived().bytes(), Unit::kBytes,
: 0, ImprovementDirection::kNeitherIsBetter);
"bytesPerSecond"); metrics_logger_->LogSingleValueMetric(
ReportResult("sent_packets_loss", network_label, packet_loss, "unitless"); "packets_received", GetTestCaseName(network_label),
} else { stats->PacketsReceived(), Unit::kUnitless,
metrics_logger_->LogSingleValueMetric( ImprovementDirection::kNeitherIsBetter);
"bytes_sent", GetTestCaseName(network_label), metrics_logger_->LogSingleValueMetric(
stats->BytesSent().bytes(), Unit::kBytes, "average_receive_rate", GetTestCaseName(network_label),
ImprovementDirection::kNeitherIsBetter); stats->PacketsReceived() >= 2 ? stats->AverageReceiveRate().kbps() : 0,
metrics_logger_->LogSingleValueMetric( Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
"packets_sent", GetTestCaseName(network_label), stats->PacketsSent(), metrics_logger_->LogSingleValueMetric(
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); "sent_packets_loss", GetTestCaseName(network_label), packet_loss,
metrics_logger_->LogSingleValueMetric( Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
"average_send_rate", GetTestCaseName(network_label),
stats->PacketsSent() >= 2 ? stats->AverageSendRate().kbps() : 0,
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"bytes_discarded_no_receiver", GetTestCaseName(network_label),
stats->BytesDropped().bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_discarded_no_receiver", GetTestCaseName(network_label),
stats->PacketsDropped(), Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"bytes_received", GetTestCaseName(network_label),
stats->BytesReceived().bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_received", GetTestCaseName(network_label),
stats->PacketsReceived(), Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"average_receive_rate", GetTestCaseName(network_label),
stats->PacketsReceived() >= 2 ? stats->AverageReceiveRate().kbps() : 0,
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"sent_packets_loss", GetTestCaseName(network_label), packet_loss,
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
}
} }
void NetworkQualityMetricsReporter::ReportPCStats(const std::string& pc_label, void NetworkQualityMetricsReporter::ReportPCStats(const std::string& pc_label,
const PCStats& stats) { const PCStats& stats) {
if (metrics_logger_ == nullptr) { metrics_logger_->LogSingleValueMetric(
ReportResult("payload_bytes_received", pc_label, "payload_bytes_received", pc_label, stats.payload_received.bytes(),
stats.payload_received.bytes(), "sizeInBytes"); Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
ReportResult("payload_bytes_sent", pc_label, stats.payload_sent.bytes(), metrics_logger_->LogSingleValueMetric(
"sizeInBytes"); "payload_bytes_sent", pc_label, stats.payload_sent.bytes(), Unit::kBytes,
} else { ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"payload_bytes_received", pc_label, stats.payload_received.bytes(),
Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"payload_bytes_sent", pc_label, stats.payload_sent.bytes(),
Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
}
}
void NetworkQualityMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& network_label,
const double value,
const std::string& unit) const {
test::PrintResult(metric_name, /*modifier=*/"",
GetTestCaseName(network_label), value, unit,
/*important=*/false);
} }
std::string NetworkQualityMetricsReporter::GetTestCaseName( std::string NetworkQualityMetricsReporter::GetTestCaseName(

14
test/pc/e2e/network_quality_metrics_reporter.h
View file

@ -28,17 +28,9 @@ namespace webrtc_pc_e2e {
class NetworkQualityMetricsReporter class NetworkQualityMetricsReporter
: public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
public: public:
NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network,
EmulatedNetworkManagerInterface* bob_network)
: NetworkQualityMetricsReporter(alice_network,
bob_network,
/*metrics_logger=*/nullptr) {}
NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network, NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network,
EmulatedNetworkManagerInterface* bob_network, EmulatedNetworkManagerInterface* bob_network,
test::MetricsLogger* metrics_logger) test::MetricsLogger* metrics_logger);
: alice_network_(alice_network),
bob_network_(bob_network),
metrics_logger_(metrics_logger) {}
~NetworkQualityMetricsReporter() override = default; ~NetworkQualityMetricsReporter() override = default;
// Network stats must be empty when this method will be invoked. // Network stats must be empty when this method will be invoked.
@ -63,10 +55,6 @@ class NetworkQualityMetricsReporter
std::unique_ptr<EmulatedNetworkStats> stats, std::unique_ptr<EmulatedNetworkStats> stats,
int64_t packet_loss); int64_t packet_loss);
void ReportPCStats(const std::string& pc_label, const PCStats& stats); void ReportPCStats(const std::string& pc_label, const PCStats& stats);
void ReportResult(const std::string& metric_name,
const std::string& network_label,
double value,
const std::string& unit) const;
std::string GetTestCaseName(const std::string& network_label) const; std::string GetTestCaseName(const std::string& network_label) const;
std::string test_case_name_; std::string test_case_name_;

25
test/pc/e2e/peer_connection_quality_test.cc
View file

@ -41,7 +41,6 @@
#include "test/pc/e2e/stats_poller.h" #include "test/pc/e2e/stats_poller.h"
#include "test/pc/e2e/test_peer_factory.h" #include "test/pc/e2e/test_peer_factory.h"
#include "test/testsupport/file_utils.h" #include "test/testsupport/file_utils.h"
#include "test/testsupport/perf_test.h"
namespace webrtc { namespace webrtc {
namespace webrtc_pc_e2e { namespace webrtc_pc_e2e {
@ -736,24 +735,12 @@ void PeerConnectionE2EQualityTest::TearDownCall() {
} }
void PeerConnectionE2EQualityTest::ReportGeneralTestResults() { void PeerConnectionE2EQualityTest::ReportGeneralTestResults() {
if (metrics_logger_ == nullptr) { metrics_logger_->LogSingleValueMetric(
test::PrintResult(*alice_->params().name + "_connected", "", *alice_->params().name + "_connected", test_case_name_, alice_connected_,
test_case_name_, alice_connected_, "unitless", Unit::kUnitless, ImprovementDirection::kBiggerIsBetter);
/*important=*/false, metrics_logger_->LogSingleValueMetric(
test::ImproveDirection::kBiggerIsBetter); *bob_->params().name + "_connected", test_case_name_, bob_connected_,
test::PrintResult(*bob_->params().name + "_connected", "", test_case_name_, Unit::kUnitless, ImprovementDirection::kBiggerIsBetter);
bob_connected_, "unitless",
/*important=*/false,
test::ImproveDirection::kBiggerIsBetter);
} else {
metrics_logger_->LogSingleValueMetric(
*alice_->params().name + "_connected", test_case_name_,
alice_connected_, Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric(
*bob_->params().name + "_connected", test_case_name_, bob_connected_,
Unit::kUnitless, ImprovementDirection::kBiggerIsBetter);
}
} }
Timestamp PeerConnectionE2EQualityTest::Now() const { Timestamp PeerConnectionE2EQualityTest::Now() const {

244
test/pc/e2e/stats_based_network_quality_metrics_reporter.cc
View file

@ -29,12 +29,12 @@
#include "api/test/network_emulation_manager.h" #include "api/test/network_emulation_manager.h"
#include "api/units/data_rate.h" #include "api/units/data_rate.h"
#include "api/units/timestamp.h" #include "api/units/timestamp.h"
#include "rtc_base/checks.h"
#include "rtc_base/event.h" #include "rtc_base/event.h"
#include "rtc_base/ip_address.h" #include "rtc_base/ip_address.h"
#include "rtc_base/strings/string_builder.h" #include "rtc_base/strings/string_builder.h"
#include "rtc_base/synchronization/mutex.h" #include "rtc_base/synchronization/mutex.h"
#include "system_wrappers/include/field_trial.h" #include "system_wrappers/include/field_trial.h"
#include "test/testsupport/perf_test.h"
namespace webrtc { namespace webrtc {
namespace webrtc_pc_e2e { namespace webrtc_pc_e2e {
@ -81,6 +81,17 @@ std::map<rtc::IPAddress, std::string> PopulateIpToPeer(
} // namespace } // namespace
StatsBasedNetworkQualityMetricsReporter::
StatsBasedNetworkQualityMetricsReporter(
std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
NetworkEmulationManager* network_emulation,
test::MetricsLogger* metrics_logger)
: collector_(std::move(peer_endpoints), network_emulation),
clock_(network_emulation->time_controller()->GetClock()),
metrics_logger_(metrics_logger) {
RTC_CHECK(metrics_logger_);
}
StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector:: StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector::
NetworkLayerStatsCollector( NetworkLayerStatsCollector(
std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints, std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
@ -241,99 +252,48 @@ void StatsBasedNetworkQualityMetricsReporter::ReportStats(
const NetworkLayerStats& network_layer_stats, const NetworkLayerStats& network_layer_stats,
int64_t packet_loss, int64_t packet_loss,
const Timestamp& end_time) { const Timestamp& end_time) {
if (metrics_logger_ == nullptr) { metrics_logger_->LogSingleValueMetric(
ReportResult("bytes_discarded_no_receiver", pc_label, "bytes_discarded_no_receiver", GetTestCaseName(pc_label),
network_layer_stats.stats->BytesDropped().bytes(), network_layer_stats.stats->BytesDropped().bytes(), Unit::kBytes,
"sizeInBytes"); ImprovementDirection::kNeitherIsBetter);
ReportResult("packets_discarded_no_receiver", pc_label, metrics_logger_->LogSingleValueMetric(
network_layer_stats.stats->PacketsDropped(), "unitless"); "packets_discarded_no_receiver", GetTestCaseName(pc_label),
network_layer_stats.stats->PacketsDropped(), Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
ReportResult("payload_bytes_received", pc_label, metrics_logger_->LogSingleValueMetric(
pc_stats.payload_received.bytes(), "sizeInBytes"); "payload_bytes_received", GetTestCaseName(pc_label),
ReportResult("payload_bytes_sent", pc_label, pc_stats.payload_sent.bytes(), pc_stats.payload_received.bytes(), Unit::kBytes,
"sizeInBytes"); ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"payload_bytes_sent", GetTestCaseName(pc_label),
pc_stats.payload_sent.bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
ReportResult("bytes_sent", pc_label, pc_stats.total_sent.bytes(), metrics_logger_->LogSingleValueMetric(
"sizeInBytes"); "bytes_sent", GetTestCaseName(pc_label), pc_stats.total_sent.bytes(),
ReportResult("packets_sent", pc_label, pc_stats.packets_sent, "unitless"); Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
ReportResult( metrics_logger_->LogSingleValueMetric(
"average_send_rate", pc_label, "packets_sent", GetTestCaseName(pc_label), pc_stats.packets_sent,
(pc_stats.total_sent / (end_time - start_time_)).bytes_per_sec(), Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
"bytesPerSecond"); metrics_logger_->LogSingleValueMetric(
ReportResult("bytes_received", pc_label, pc_stats.total_received.bytes(), "average_send_rate", GetTestCaseName(pc_label),
"sizeInBytes"); (pc_stats.total_sent / (end_time - start_time_)).kbps(),
ReportResult("packets_received", pc_label, pc_stats.packets_received, Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
"unitless"); metrics_logger_->LogSingleValueMetric(
ReportResult( "bytes_received", GetTestCaseName(pc_label),
"average_receive_rate", pc_label, pc_stats.total_received.bytes(), Unit::kBytes,
(pc_stats.total_received / (end_time - start_time_)).bytes_per_sec(), ImprovementDirection::kNeitherIsBetter);
"bytesPerSecond"); metrics_logger_->LogSingleValueMetric(
ReportResult("sent_packets_loss", pc_label, packet_loss, "unitless"); "packets_received", GetTestCaseName(pc_label), pc_stats.packets_received,
} else { Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric(
"bytes_discarded_no_receiver", GetTestCaseName(pc_label), "average_receive_rate", GetTestCaseName(pc_label),
network_layer_stats.stats->BytesDropped().bytes(), Unit::kBytes, (pc_stats.total_received / (end_time - start_time_)).kbps(),
ImprovementDirection::kNeitherIsBetter); Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric(
"packets_discarded_no_receiver", GetTestCaseName(pc_label), "sent_packets_loss", GetTestCaseName(pc_label), packet_loss,
network_layer_stats.stats->PacketsDropped(), Unit::kUnitless, Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"payload_bytes_received", GetTestCaseName(pc_label),
pc_stats.payload_received.bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"payload_bytes_sent", GetTestCaseName(pc_label),
pc_stats.payload_sent.bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"bytes_sent", GetTestCaseName(pc_label), pc_stats.total_sent.bytes(),
Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_sent", GetTestCaseName(pc_label), pc_stats.packets_sent,
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"average_send_rate", GetTestCaseName(pc_label),
(pc_stats.total_sent / (end_time - start_time_)).kbps(),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"bytes_received", GetTestCaseName(pc_label),
pc_stats.total_received.bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"packets_received", GetTestCaseName(pc_label),
pc_stats.packets_received, Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"average_receive_rate", GetTestCaseName(pc_label),
(pc_stats.total_received / (end_time - start_time_)).kbps(),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric(
"sent_packets_loss", GetTestCaseName(pc_label), packet_loss,
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
}
}
void StatsBasedNetworkQualityMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& network_label,
const double value,
const std::string& unit) const {
test::PrintResult(metric_name, /*modifier=*/"",
GetTestCaseName(network_label), value, unit,
/*important=*/false);
}
void StatsBasedNetworkQualityMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& network_label,
const SamplesStatsCounter& value,
const std::string& unit) const {
test::PrintResult(metric_name, /*modifier=*/"",
GetTestCaseName(network_label), value, unit,
/*important=*/false);
} }
std::string StatsBasedNetworkQualityMetricsReporter::GetTestCaseName( std::string StatsBasedNetworkQualityMetricsReporter::GetTestCaseName(
@ -360,48 +320,28 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
log << " " << local_ips[i].ToString() << "\n"; log << " " << local_ips[i].ToString() << "\n";
} }
if (!stats.stats->SentPacketsSizeCounter().IsEmpty()) { if (!stats.stats->SentPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) { metrics_logger_->LogMetric("sent_packets_size", GetTestCaseName(peer_name),
ReportResult("sent_packets_size", peer_name, stats.stats->SentPacketsSizeCounter(),
stats.stats->SentPacketsSizeCounter(), "sizeInBytes"); Unit::kBytes,
} else { ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"sent_packets_size", GetTestCaseName(peer_name),
stats.stats->SentPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
} }
if (!stats.stats->ReceivedPacketsSizeCounter().IsEmpty()) { if (!stats.stats->ReceivedPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) { metrics_logger_->LogMetric(
ReportResult("received_packets_size", peer_name, "received_packets_size", GetTestCaseName(peer_name),
stats.stats->ReceivedPacketsSizeCounter(), "sizeInBytes"); stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes,
} else { ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"received_packets_size", GetTestCaseName(peer_name),
stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
} }
if (!stats.stats->DroppedPacketsSizeCounter().IsEmpty()) { if (!stats.stats->DroppedPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) { metrics_logger_->LogMetric(
ReportResult("dropped_packets_size", peer_name, "dropped_packets_size", GetTestCaseName(peer_name),
stats.stats->DroppedPacketsSizeCounter(), "sizeInBytes"); stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes,
} else { ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"dropped_packets_size", GetTestCaseName(peer_name),
stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
} }
if (!stats.stats->SentPacketsQueueWaitTimeUs().IsEmpty()) { if (!stats.stats->SentPacketsQueueWaitTimeUs().IsEmpty()) {
if (metrics_logger_ == nullptr) { metrics_logger_->LogMetric(
ReportResult("sent_packets_queue_wait_time_us", peer_name, "sent_packets_queue_wait_time_us", GetTestCaseName(peer_name),
stats.stats->SentPacketsQueueWaitTimeUs(), "unitless"); stats.stats->SentPacketsQueueWaitTimeUs(), Unit::kUnitless,
} else { ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"sent_packets_queue_wait_time_us", GetTestCaseName(peer_name),
stats.stats->SentPacketsQueueWaitTimeUs(), Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter);
}
} }
log << "Send statistic:\n" log << "Send statistic:\n"
@ -421,17 +361,11 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
<< " avg_rate (bytes/sec): " << source_average_send_rate.bytes_per_sec() << " avg_rate (bytes/sec): " << source_average_send_rate.bytes_per_sec()
<< " avg_rate (bps): " << source_average_send_rate.bps() << "\n"; << " avg_rate (bps): " << source_average_send_rate.bps() << "\n";
if (!entry.second->SentPacketsSizeCounter().IsEmpty()) { if (!entry.second->SentPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) { metrics_logger_->LogMetric(
ReportResult("sent_packets_size", "sent_packets_size",
peer_name + "/" + entry.first.ToString(), GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->SentPacketsSizeCounter(), "sizeInBytes"); stats.stats->SentPacketsSizeCounter(), Unit::kBytes,
} else { ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"sent_packets_size",
GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->SentPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
} }
} }
@ -454,30 +388,18 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
<< source_average_receive_rate.bytes_per_sec() << source_average_receive_rate.bytes_per_sec()
<< " avg_rate (bps): " << source_average_receive_rate.bps() << "\n"; << " avg_rate (bps): " << source_average_receive_rate.bps() << "\n";
if (!entry.second->ReceivedPacketsSizeCounter().IsEmpty()) { if (!entry.second->ReceivedPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) { metrics_logger_->LogMetric(
ReportResult("received_packets_size", "received_packets_size",
peer_name + "/" + entry.first.ToString(), GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->ReceivedPacketsSizeCounter(), "sizeInBytes"); stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes,
} else { ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"received_packets_size",
GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
} }
if (!entry.second->DroppedPacketsSizeCounter().IsEmpty()) { if (!entry.second->DroppedPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) { metrics_logger_->LogMetric(
ReportResult("dropped_packets_size", "dropped_packets_size",
peer_name + "/" + entry.first.ToString(), GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->DroppedPacketsSizeCounter(), "sizeInBytes"); stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes,
} else { ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogMetric(
"dropped_packets_size",
GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter);
}
} }
} }

19
test/pc/e2e/stats_based_network_quality_metrics_reporter.h
View file

@ -39,19 +39,10 @@ class StatsBasedNetworkQualityMetricsReporter
public: public:
// `networks` map peer name to network to report network layer stability stats // `networks` map peer name to network to report network layer stability stats
// and to log network layer metrics. // and to log network layer metrics.
StatsBasedNetworkQualityMetricsReporter(
std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
NetworkEmulationManager* network_emulation)
: StatsBasedNetworkQualityMetricsReporter(std::move(peer_endpoints),
network_emulation,
/*metrics_logger=*/nullptr) {}
StatsBasedNetworkQualityMetricsReporter( StatsBasedNetworkQualityMetricsReporter(
std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints, std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
NetworkEmulationManager* network_emulation, NetworkEmulationManager* network_emulation,
test::MetricsLogger* metrics_logger) test::MetricsLogger* metrics_logger);
: collector_(std::move(peer_endpoints), network_emulation),
clock_(network_emulation->time_controller()->GetClock()),
metrics_logger_(metrics_logger) {}
~StatsBasedNetworkQualityMetricsReporter() override = default; ~StatsBasedNetworkQualityMetricsReporter() override = default;
void AddPeer(absl::string_view peer_name, void AddPeer(absl::string_view peer_name,
@ -111,14 +102,6 @@ class StatsBasedNetworkQualityMetricsReporter
const NetworkLayerStats& network_layer_stats, const NetworkLayerStats& network_layer_stats,
int64_t packet_loss, int64_t packet_loss,
const Timestamp& end_time); const Timestamp& end_time);
void ReportResult(const std::string& metric_name,
const std::string& network_label,
double value,
const std::string& unit) const;
void ReportResult(const std::string& metric_name,
const std::string& network_label,
const SamplesStatsCounter& value,
const std::string& unit) const;
std::string GetTestCaseName(absl::string_view network_label) const; std::string GetTestCaseName(absl::string_view network_label) const;
void LogNetworkLayerStats(const std::string& peer_name, void LogNetworkLayerStats(const std::string& peer_name,
const NetworkLayerStats& stats) const; const NetworkLayerStats& stats) const;

1
video/BUILD.gn
View file

@ -585,6 +585,7 @@ if (rtc_include_tests) {
"../api:peer_connection_quality_test_fixture_api", "../api:peer_connection_quality_test_fixture_api",
"../api:simulated_network_api", "../api:simulated_network_api",
"../api:time_controller", "../api:time_controller",
"../api/test/metrics:global_metrics_logger_and_exporter",
"../api/video_codecs:video_codecs_api", "../api/video_codecs:video_codecs_api",
"../call:simulated_network", "../call:simulated_network",
"../modules/video_coding:webrtc_vp9", "../modules/video_coding:webrtc_vp9",

4
video/pc_full_stack_tests.cc
View file

@ -17,6 +17,7 @@
#include "api/test/create_peer_connection_quality_test_frame_generator.h" #include "api/test/create_peer_connection_quality_test_frame_generator.h"
#include "api/test/create_peerconnection_quality_test_fixture.h" #include "api/test/create_peerconnection_quality_test_fixture.h"
#include "api/test/frame_generator_interface.h" #include "api/test/frame_generator_interface.h"
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "api/test/network_emulation_manager.h" #include "api/test/network_emulation_manager.h"
#include "api/test/peerconnection_quality_test_fixture.h" #include "api/test/peerconnection_quality_test_fixture.h"
#include "api/test/simulated_network.h" #include "api/test/simulated_network.h"
@ -93,7 +94,8 @@ CreateTestFixture(const std::string& test_case_name,
bob_configurer); bob_configurer);
fixture->AddQualityMetricsReporter( fixture->AddQualityMetricsReporter(
std::make_unique<webrtc_pc_e2e::NetworkQualityMetricsReporter>( std::make_unique<webrtc_pc_e2e::NetworkQualityMetricsReporter>(
network_links.first, network_links.second)); network_links.first, network_links.second,
test::GetGlobalMetricsLogger()));
return fixture; return fixture;
} }