[PCLF] Fully switch to new metrics export API

Bug: b/246095034
Change-Id: I9d588d53320e4eb19cb569db2b97dddc013c22bd
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276621
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Tomas Gunnarsson <tommi@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38188}
This commit is contained in:
Artem Titov 2022-09-24 16:56:07 +02:00 committed by WebRTC LUCI CQ
parent 62b40efe5a
commit c45f4e4a3d
21 changed files with 370 additions and 680 deletions

View file

@ -224,6 +224,7 @@ if (rtc_include_tests) {
"../api:peer_connection_quality_test_fixture_api", "../api:peer_connection_quality_test_fixture_api",
"../api:simulated_network_api", "../api:simulated_network_api",
"../api:time_controller", "../api:time_controller",
"../api/test/metrics:global_metrics_logger_and_exporter",
"../call:simulated_network", "../call:simulated_network",
"../common_audio", "../common_audio",
"../system_wrappers", "../system_wrappers",

View file

@ -15,6 +15,7 @@
#include "absl/strings/string_view.h" #include "absl/strings/string_view.h"
#include "api/test/create_network_emulation_manager.h" #include "api/test/create_network_emulation_manager.h"
#include "api/test/create_peerconnection_quality_test_fixture.h" #include "api/test/create_peerconnection_quality_test_fixture.h"
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "api/test/network_emulation_manager.h" #include "api/test/network_emulation_manager.h"
#include "api/test/peerconnection_quality_test_fixture.h" #include "api/test/peerconnection_quality_test_fixture.h"
#include "api/test/simulated_network.h" #include "api/test/simulated_network.h"
@ -88,7 +89,8 @@ CreateTestFixture(absl::string_view test_case_name,
bob_configurer); bob_configurer);
fixture->AddQualityMetricsReporter( fixture->AddQualityMetricsReporter(
std::make_unique<webrtc_pc_e2e::NetworkQualityMetricsReporter>( std::make_unique<webrtc_pc_e2e::NetworkQualityMetricsReporter>(
network_links.first, network_links.second)); network_links.first, network_links.second,
test::GetGlobalMetricsLogger()));
return fixture; return fixture;
} }

View file

@ -2708,6 +2708,7 @@ if (rtc_include_tests && !build_with_chromium) {
"../api:peer_connection_quality_test_fixture_api", "../api:peer_connection_quality_test_fixture_api",
"../api:simulated_network_api", "../api:simulated_network_api",
"../api:time_controller", "../api:time_controller",
"../api/test/metrics:global_metrics_logger_and_exporter",
"../api/video_codecs:video_codecs_api", "../api/video_codecs:video_codecs_api",
"../call:simulated_network", "../call:simulated_network",
"../modules/video_coding:webrtc_vp9", "../modules/video_coding:webrtc_vp9",

View file

@ -17,6 +17,7 @@
#include "api/test/create_peer_connection_quality_test_frame_generator.h" #include "api/test/create_peer_connection_quality_test_frame_generator.h"
#include "api/test/create_peerconnection_quality_test_fixture.h" #include "api/test/create_peerconnection_quality_test_fixture.h"
#include "api/test/frame_generator_interface.h" #include "api/test/frame_generator_interface.h"
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "api/test/network_emulation_manager.h" #include "api/test/network_emulation_manager.h"
#include "api/test/peerconnection_quality_test_fixture.h" #include "api/test/peerconnection_quality_test_fixture.h"
#include "api/test/simulated_network.h" #include "api/test/simulated_network.h"
@ -185,6 +186,7 @@ class SvcVideoQualityAnalyzer : public DefaultVideoQualityAnalyzer {
explicit SvcVideoQualityAnalyzer(webrtc::Clock* clock) explicit SvcVideoQualityAnalyzer(webrtc::Clock* clock)
: DefaultVideoQualityAnalyzer(clock, : DefaultVideoQualityAnalyzer(clock,
test::GetGlobalMetricsLogger(),
DefaultVideoQualityAnalyzerOptions{ DefaultVideoQualityAnalyzerOptions{
.compute_psnr = false, .compute_psnr = false,
.compute_ssim = false, .compute_ssim = false,

View file

@ -581,6 +581,7 @@ if (!build_with_chromium) {
"../..:test_support", "../..:test_support",
"../../../api:create_frame_generator", "../../../api:create_frame_generator",
"../../../api:rtp_packet_info", "../../../api:rtp_packet_info",
"../../../api/test/metrics:global_metrics_logger_and_exporter",
"../../../api/video:encoded_image", "../../../api/video:encoded_image",
"../../../api/video:video_frame", "../../../api/video:video_frame",
"../../../common_video", "../../../common_video",
@ -700,6 +701,7 @@ if (!build_with_chromium) {
"../../../api/test/metrics:metrics_logger_and_exporter", "../../../api/test/metrics:metrics_logger_and_exporter",
"../../../api/units:time_delta", "../../../api/units:time_delta",
"../../../api/units:timestamp", "../../../api/units:timestamp",
"../../../rtc_base:checks",
"../../../rtc_base:criticalsection", "../../../rtc_base:criticalsection",
"../../../rtc_base:logging", "../../../rtc_base:logging",
"../../../rtc_base:rtc_numerics", "../../../rtc_base:rtc_numerics",
@ -749,6 +751,7 @@ if (!build_with_chromium) {
"../../../api/units:data_size", "../../../api/units:data_size",
"../../../api/units:time_delta", "../../../api/units:time_delta",
"../../../api/units:timestamp", "../../../api/units:timestamp",
"../../../rtc_base:checks",
"../../../rtc_base:criticalsection", "../../../rtc_base:criticalsection",
"../../../rtc_base:rtc_numerics", "../../../rtc_base:rtc_numerics",
"../../../rtc_base/synchronization:mutex", "../../../rtc_base/synchronization:mutex",
@ -887,6 +890,7 @@ if (!build_with_chromium) {
"../../../api/test/metrics:metrics_logger", "../../../api/test/metrics:metrics_logger",
"../../../api/test/metrics:metrics_logger_and_exporter", "../../../api/test/metrics:metrics_logger_and_exporter",
"../../../api/units:data_size", "../../../api/units:data_size",
"../../../rtc_base:checks",
"../../../rtc_base:criticalsection", "../../../rtc_base:criticalsection",
"../../../rtc_base:rtc_event", "../../../rtc_base:rtc_event",
"../../../rtc_base/synchronization:mutex", "../../../rtc_base/synchronization:mutex",
@ -918,6 +922,7 @@ if (!build_with_chromium) {
"../../../api/units:data_size", "../../../api/units:data_size",
"../../../api/units:timestamp", "../../../api/units:timestamp",
"../../../rtc_base", "../../../rtc_base",
"../../../rtc_base:checks",
"../../../rtc_base:ip_address", "../../../rtc_base:ip_address",
"../../../rtc_base:rtc_event", "../../../rtc_base:rtc_event",
"../../../rtc_base:stringutils", "../../../rtc_base:stringutils",
@ -945,6 +950,7 @@ if (!build_with_chromium) {
"../../../api/test/metrics:metrics_logger", "../../../api/test/metrics:metrics_logger",
"../../../api/test/metrics:metrics_logger_and_exporter", "../../../api/test/metrics:metrics_logger_and_exporter",
"../../../api/units:timestamp", "../../../api/units:timestamp",
"../../../rtc_base:checks",
"../../../rtc_base:criticalsection", "../../../rtc_base:criticalsection",
"../../../rtc_base:rtc_event", "../../../rtc_base:rtc_event",
"../../../rtc_base:rtc_numerics", "../../../rtc_base:rtc_numerics",

View file

@ -13,6 +13,7 @@
#include "api/stats/rtc_stats.h" #include "api/stats/rtc_stats.h"
#include "api/stats/rtcstats_objects.h" #include "api/stats/rtcstats_objects.h"
#include "api/test/metrics/metric.h" #include "api/test/metrics/metric.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h" #include "rtc_base/logging.h"
namespace webrtc { namespace webrtc {
@ -21,6 +22,12 @@ namespace webrtc_pc_e2e {
using ::webrtc::test::ImprovementDirection; using ::webrtc::test::ImprovementDirection;
using ::webrtc::test::Unit; using ::webrtc::test::Unit;
DefaultAudioQualityAnalyzer::DefaultAudioQualityAnalyzer(
test::MetricsLogger* const metrics_logger)
: metrics_logger_(metrics_logger) {
RTC_CHECK(metrics_logger_);
}
void DefaultAudioQualityAnalyzer::Start(std::string test_case_name, void DefaultAudioQualityAnalyzer::Start(std::string test_case_name,
TrackIdStreamInfoMap* analyzer_helper) { TrackIdStreamInfoMap* analyzer_helper) {
test_case_name_ = std::move(test_case_name); test_case_name_ = std::move(test_case_name);
@ -116,26 +123,8 @@ std::string DefaultAudioQualityAnalyzer::GetTestCaseName(
} }
void DefaultAudioQualityAnalyzer::Stop() { void DefaultAudioQualityAnalyzer::Stop() {
using ::webrtc::test::ImproveDirection;
MutexLock lock(&lock_); MutexLock lock(&lock_);
for (auto& item : streams_stats_) { for (auto& item : streams_stats_) {
if (metrics_logger_ == nullptr) {
ReportResult("expand_rate", item.first, item.second.expand_rate,
"unitless", ImproveDirection::kSmallerIsBetter);
ReportResult("accelerate_rate", item.first, item.second.accelerate_rate,
"unitless", ImproveDirection::kSmallerIsBetter);
ReportResult("preemptive_rate", item.first, item.second.preemptive_rate,
"unitless", ImproveDirection::kSmallerIsBetter);
ReportResult("speech_expand_rate", item.first,
item.second.speech_expand_rate, "unitless",
ImproveDirection::kSmallerIsBetter);
ReportResult("average_jitter_buffer_delay_ms", item.first,
item.second.average_jitter_buffer_delay_ms, "ms",
ImproveDirection::kNone);
ReportResult("preferred_buffer_size_ms", item.first,
item.second.preferred_buffer_size_ms, "ms",
ImproveDirection::kNone);
} else {
metrics_logger_->LogMetric("expand_rate", GetTestCaseName(item.first), metrics_logger_->LogMetric("expand_rate", GetTestCaseName(item.first),
item.second.expand_rate, Unit::kUnitless, item.second.expand_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter); ImprovementDirection::kSmallerIsBetter);
@ -145,8 +134,8 @@ void DefaultAudioQualityAnalyzer::Stop() {
metrics_logger_->LogMetric("preemptive_rate", GetTestCaseName(item.first), metrics_logger_->LogMetric("preemptive_rate", GetTestCaseName(item.first),
item.second.preemptive_rate, Unit::kUnitless, item.second.preemptive_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter); ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric( metrics_logger_->LogMetric("speech_expand_rate",
"speech_expand_rate", GetTestCaseName(item.first), GetTestCaseName(item.first),
item.second.speech_expand_rate, Unit::kUnitless, item.second.speech_expand_rate, Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter); ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric( metrics_logger_->LogMetric(
@ -159,7 +148,6 @@ void DefaultAudioQualityAnalyzer::Stop() {
ImprovementDirection::kNeitherIsBetter); ImprovementDirection::kNeitherIsBetter);
} }
} }
}
std::map<std::string, AudioStreamStats> std::map<std::string, AudioStreamStats>
DefaultAudioQualityAnalyzer::GetAudioStreamsStats() const { DefaultAudioQualityAnalyzer::GetAudioStreamsStats() const {
@ -167,18 +155,5 @@ DefaultAudioQualityAnalyzer::GetAudioStreamsStats() const {
return streams_stats_; return streams_stats_;
} }
void DefaultAudioQualityAnalyzer::ReportResult(
const std::string& metric_name,
const std::string& stream_label,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) const {
test::PrintResultMeanAndError(
metric_name, /*modifier=*/"", GetTestCaseName(stream_label),
counter.IsEmpty() ? 0 : counter.GetAverage(),
counter.IsEmpty() ? 0 : counter.GetStandardDeviation(), unit,
/*important=*/false, improve_direction);
}
} // namespace webrtc_pc_e2e } // namespace webrtc_pc_e2e
} // namespace webrtc } // namespace webrtc

View file

@ -21,7 +21,6 @@
#include "api/test/track_id_stream_info_map.h" #include "api/test/track_id_stream_info_map.h"
#include "api/units/time_delta.h" #include "api/units/time_delta.h"
#include "rtc_base/synchronization/mutex.h" #include "rtc_base/synchronization/mutex.h"
#include "test/testsupport/perf_test.h"
namespace webrtc { namespace webrtc {
namespace webrtc_pc_e2e { namespace webrtc_pc_e2e {
@ -37,11 +36,8 @@ struct AudioStreamStats {
class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface { class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface {
public: public:
DefaultAudioQualityAnalyzer()
: DefaultAudioQualityAnalyzer(/*metrics_logger=*/nullptr) {}
explicit DefaultAudioQualityAnalyzer( explicit DefaultAudioQualityAnalyzer(
test::MetricsLogger* const metrics_logger) test::MetricsLogger* const metrics_logger);
: metrics_logger_(metrics_logger) {}
void Start(std::string test_case_name, void Start(std::string test_case_name,
TrackIdStreamInfoMap* analyzer_helper) override; TrackIdStreamInfoMap* analyzer_helper) override;
@ -66,11 +62,6 @@ class DefaultAudioQualityAnalyzer : public AudioQualityAnalyzerInterface {
}; };
std::string GetTestCaseName(const std::string& stream_label) const; std::string GetTestCaseName(const std::string& stream_label) const;
void ReportResult(const std::string& metric_name,
const std::string& stream_label,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) const;
test::MetricsLogger* const metrics_logger_; test::MetricsLogger* const metrics_logger_;

View file

@ -23,6 +23,7 @@
#include "api/video/i420_buffer.h" #include "api/video/i420_buffer.h"
#include "api/video/video_frame.h" #include "api/video/video_frame.h"
#include "common_video/libyuv/include/webrtc_libyuv.h" #include "common_video/libyuv/include/webrtc_libyuv.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h" #include "rtc_base/logging.h"
#include "rtc_base/platform_thread.h" #include "rtc_base/platform_thread.h"
#include "rtc_base/strings/string_builder.h" #include "rtc_base/strings/string_builder.h"
@ -124,13 +125,6 @@ SamplesStatsCounter::StatsSample StatsSample(double value,
} // namespace } // namespace
DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
DefaultVideoQualityAnalyzerOptions options)
: DefaultVideoQualityAnalyzer(clock,
/*metrics_logger=*/nullptr,
std::move(options)) {}
DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer( DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
webrtc::Clock* clock, webrtc::Clock* clock,
test::MetricsLogger* metrics_logger, test::MetricsLogger* metrics_logger,
@ -138,7 +132,9 @@ DefaultVideoQualityAnalyzer::DefaultVideoQualityAnalyzer(
: options_(options), : options_(options),
clock_(clock), clock_(clock),
metrics_logger_(metrics_logger), metrics_logger_(metrics_logger),
frames_comparator_(clock, cpu_measurer_, options) {} frames_comparator_(clock, cpu_measurer_, options) {
RTC_CHECK(metrics_logger_);
}
DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() { DefaultVideoQualityAnalyzer::~DefaultVideoQualityAnalyzer() {
Stop(); Stop();
@ -891,22 +887,14 @@ void DefaultVideoQualityAnalyzer::
} }
void DefaultVideoQualityAnalyzer::ReportResults() { void DefaultVideoQualityAnalyzer::ReportResults() {
using ::webrtc::test::ImproveDirection;
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
for (auto& item : frames_comparator_.stream_stats()) { for (auto& item : frames_comparator_.stream_stats()) {
ReportResults(GetTestCaseName(ToMetricName(item.first)), item.second, ReportResults(GetTestCaseName(ToMetricName(item.first)), item.second,
stream_frame_counters_.at(item.first)); stream_frame_counters_.at(item.first));
} }
if (metrics_logger_ == nullptr) { metrics_logger_->LogSingleValueMetric("cpu_usage_%", test_label_,
test::PrintResult("cpu_usage", "", test_label_.c_str(), GetCpuUsagePercent(), Unit::kUnitless,
GetCpuUsagePercent(), "%", false,
ImproveDirection::kSmallerIsBetter);
} else {
metrics_logger_->LogSingleValueMetric(
"cpu_usage_%", test_label_, GetCpuUsagePercent(), Unit::kUnitless,
ImprovementDirection::kSmallerIsBetter); ImprovementDirection::kSmallerIsBetter);
}
LogFrameCounters("Global", frame_counters_); LogFrameCounters("Global", frame_counters_);
if (!unknown_sender_frame_counters_.empty()) { if (!unknown_sender_frame_counters_.empty()) {
RTC_LOG(LS_INFO) << "Received frame counters with unknown frame id:"; RTC_LOG(LS_INFO) << "Received frame counters with unknown frame id:";
@ -943,7 +931,6 @@ void DefaultVideoQualityAnalyzer::ReportResults(
const std::string& test_case_name, const std::string& test_case_name,
const StreamStats& stats, const StreamStats& stats,
const FrameCounters& frame_counters) { const FrameCounters& frame_counters) {
using ::webrtc::test::ImproveDirection;
TimeDelta test_duration = Now() - start_time_; TimeDelta test_duration = Now() - start_time_;
double sum_squared_interframe_delays_secs = 0; double sum_squared_interframe_delays_secs = 0;
@ -972,91 +959,6 @@ void DefaultVideoQualityAnalyzer::ReportResults(
video_duration.seconds<double>() / sum_squared_interframe_delays_secs; video_duration.seconds<double>() / sum_squared_interframe_delays_secs;
} }
if (metrics_logger_ == nullptr) {
// Report results through old performance metrics API.
ReportResult("psnr", test_case_name, stats.psnr, "dB",
ImproveDirection::kBiggerIsBetter);
ReportResult("ssim", test_case_name, stats.ssim, "unitless",
ImproveDirection::kBiggerIsBetter);
ReportResult("transport_time", test_case_name, stats.transport_time_ms,
"ms", ImproveDirection::kSmallerIsBetter);
ReportResult("total_delay_incl_transport", test_case_name,
stats.total_delay_incl_transport_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("time_between_rendered_frames", test_case_name,
stats.time_between_rendered_frames_ms, "ms",
ImproveDirection::kSmallerIsBetter);
test::PrintResult("harmonic_framerate", "", test_case_name,
harmonic_framerate_fps, "Hz", /*important=*/false,
ImproveDirection::kBiggerIsBetter);
test::PrintResult("encode_frame_rate", "", test_case_name,
stats.encode_frame_rate.IsEmpty()
? 0
: stats.encode_frame_rate.GetEventsPerSecond(),
"Hz", /*important=*/false,
ImproveDirection::kBiggerIsBetter);
ReportResult("encode_time", test_case_name, stats.encode_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("time_between_freezes", test_case_name,
stats.time_between_freezes_ms, "ms",
ImproveDirection::kBiggerIsBetter);
ReportResult("freeze_time_ms", test_case_name, stats.freeze_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("pixels_per_frame", test_case_name,
stats.resolution_of_rendered_frame, "count",
ImproveDirection::kBiggerIsBetter);
test::PrintResult("min_psnr", "", test_case_name,
stats.psnr.IsEmpty() ? 0 : stats.psnr.GetMin(), "dB",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
ReportResult("decode_time", test_case_name, stats.decode_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
ReportResult("receive_to_render_time", test_case_name,
stats.receive_to_render_time_ms, "ms",
ImproveDirection::kSmallerIsBetter);
test::PrintResult("dropped_frames", "", test_case_name,
frame_counters.dropped, "count",
/*important=*/false, ImproveDirection::kSmallerIsBetter);
test::PrintResult("frames_in_flight", "", test_case_name,
frame_counters.captured - frame_counters.rendered -
frame_counters.dropped,
"count",
/*important=*/false, ImproveDirection::kSmallerIsBetter);
test::PrintResult("rendered_frames", "", test_case_name,
frame_counters.rendered, "count", /*important=*/false,
ImproveDirection::kBiggerIsBetter);
ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered,
"count", ImproveDirection::kSmallerIsBetter);
ReportResult("target_encode_bitrate", test_case_name,
stats.target_encode_bitrate / kBitsInByte, "bytesPerSecond",
ImproveDirection::kNone);
test::PrintResult("actual_encode_bitrate", "", test_case_name,
static_cast<double>(stats.total_encoded_images_payload) /
test_duration.seconds<double>(),
"bytesPerSecond", /*important=*/false,
ImproveDirection::kNone);
if (options_.report_detailed_frame_stats) {
test::PrintResult("num_encoded_frames", "", test_case_name,
frame_counters.encoded, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_decoded_frames", "", test_case_name,
frame_counters.decoded, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_send_key_frames", "", test_case_name,
stats.num_send_key_frames, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
test::PrintResult("num_recv_key_frames", "", test_case_name,
stats.num_recv_key_frames, "count",
/*important=*/false, ImproveDirection::kBiggerIsBetter);
ReportResult("recv_key_frame_size_bytes", test_case_name,
stats.recv_key_frame_size_bytes, "count",
ImproveDirection::kBiggerIsBetter);
ReportResult("recv_delta_frame_size_bytes", test_case_name,
stats.recv_delta_frame_size_bytes, "count",
ImproveDirection::kBiggerIsBetter);
}
} else {
metrics_logger_->LogMetric("psnr_dB", test_case_name, stats.psnr, metrics_logger_->LogMetric("psnr_dB", test_case_name, stats.psnr,
Unit::kUnitless, Unit::kUnitless,
ImprovementDirection::kBiggerIsBetter); ImprovementDirection::kBiggerIsBetter);
@ -1074,9 +976,9 @@ void DefaultVideoQualityAnalyzer::ReportResults(
stats.time_between_rendered_frames_ms, stats.time_between_rendered_frames_ms,
Unit::kMilliseconds, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter); ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric("harmonic_framerate", test_case_name,
"harmonic_framerate", test_case_name, harmonic_framerate_fps, harmonic_framerate_fps, Unit::kHertz,
Unit::kHertz, ImprovementDirection::kBiggerIsBetter); ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric(
"encode_frame_rate", test_case_name, "encode_frame_rate", test_case_name,
stats.encode_frame_rate.IsEmpty() stats.encode_frame_rate.IsEmpty()
@ -1086,9 +988,9 @@ void DefaultVideoQualityAnalyzer::ReportResults(
metrics_logger_->LogMetric("encode_time", test_case_name, metrics_logger_->LogMetric("encode_time", test_case_name,
stats.encode_time_ms, Unit::kMilliseconds, stats.encode_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter); ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric( metrics_logger_->LogMetric("time_between_freezes", test_case_name,
"time_between_freezes", test_case_name, stats.time_between_freezes_ms, stats.time_between_freezes_ms, Unit::kMilliseconds,
Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter); ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("freeze_time_ms", test_case_name, metrics_logger_->LogMetric("freeze_time_ms", test_case_name,
stats.freeze_time_ms, Unit::kMilliseconds, stats.freeze_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter); ImprovementDirection::kSmallerIsBetter);
@ -1102,21 +1004,20 @@ void DefaultVideoQualityAnalyzer::ReportResults(
metrics_logger_->LogMetric("decode_time", test_case_name, metrics_logger_->LogMetric("decode_time", test_case_name,
stats.decode_time_ms, Unit::kMilliseconds, stats.decode_time_ms, Unit::kMilliseconds,
ImprovementDirection::kSmallerIsBetter); ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogMetric("receive_to_render_time", test_case_name, metrics_logger_->LogMetric(
stats.receive_to_render_time_ms, "receive_to_render_time", test_case_name, stats.receive_to_render_time_ms,
Unit::kMilliseconds, Unit::kMilliseconds, ImprovementDirection::kSmallerIsBetter);
ImprovementDirection::kSmallerIsBetter); metrics_logger_->LogSingleValueMetric("dropped_frames", test_case_name,
metrics_logger_->LogSingleValueMetric( frame_counters.dropped, Unit::kCount,
"dropped_frames", test_case_name, frame_counters.dropped, Unit::kCount,
ImprovementDirection::kSmallerIsBetter); ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric(
"frames_in_flight", test_case_name, "frames_in_flight", test_case_name,
frame_counters.captured - frame_counters.rendered - frame_counters.captured - frame_counters.rendered -
frame_counters.dropped, frame_counters.dropped,
Unit::kCount, ImprovementDirection::kSmallerIsBetter); Unit::kCount, ImprovementDirection::kSmallerIsBetter);
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric("rendered_frames", test_case_name,
"rendered_frames", test_case_name, frame_counters.rendered, frame_counters.rendered, Unit::kCount,
Unit::kCount, ImprovementDirection::kBiggerIsBetter); ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("max_skipped", test_case_name, metrics_logger_->LogMetric("max_skipped", test_case_name,
stats.skipped_between_rendered, Unit::kCount, stats.skipped_between_rendered, Unit::kCount,
ImprovementDirection::kSmallerIsBetter); ImprovementDirection::kSmallerIsBetter);
@ -1148,22 +1049,10 @@ void DefaultVideoQualityAnalyzer::ReportResults(
stats.recv_key_frame_size_bytes, Unit::kCount, stats.recv_key_frame_size_bytes, Unit::kCount,
ImprovementDirection::kBiggerIsBetter); ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name, metrics_logger_->LogMetric("recv_delta_frame_size_bytes", test_case_name,
stats.recv_delta_frame_size_bytes, stats.recv_delta_frame_size_bytes, Unit::kCount,
Unit::kCount,
ImprovementDirection::kBiggerIsBetter); ImprovementDirection::kBiggerIsBetter);
} }
} }
}
void DefaultVideoQualityAnalyzer::ReportResult(
const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) {
test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit,
/*important=*/false, improve_direction);
}
std::string DefaultVideoQualityAnalyzer::GetTestCaseName( std::string DefaultVideoQualityAnalyzer::GetTestCaseName(
const std::string& stream_label) const { const std::string& stream_label) const {

View file

@ -41,15 +41,11 @@
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
#include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h" #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_stream_state.h"
#include "test/pc/e2e/analyzer/video/names_collection.h" #include "test/pc/e2e/analyzer/video/names_collection.h"
#include "test/testsupport/perf_test.h"
namespace webrtc { namespace webrtc {
class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface { class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
public: public:
explicit DefaultVideoQualityAnalyzer(
webrtc::Clock* clock,
DefaultVideoQualityAnalyzerOptions options = {});
DefaultVideoQualityAnalyzer(webrtc::Clock* clock, DefaultVideoQualityAnalyzer(webrtc::Clock* clock,
test::MetricsLogger* metrics_logger, test::MetricsLogger* metrics_logger,
DefaultVideoQualityAnalyzerOptions options = {}); DefaultVideoQualityAnalyzerOptions options = {});
@ -131,13 +127,6 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
const StreamStats& stats, const StreamStats& stats,
const FrameCounters& frame_counters) const FrameCounters& frame_counters)
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
// Report result for single metric for specified stream.
static void ReportResult(const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction =
webrtc::test::ImproveDirection::kNone);
// Returns name of current test case for reporting. // Returns name of current test case for reporting.
std::string GetTestCaseName(const std::string& stream_label) const; std::string GetTestCaseName(const std::string& stream_label) const;
Timestamp Now(); Timestamp Now();

View file

@ -18,6 +18,7 @@
#include "api/rtp_packet_info.h" #include "api/rtp_packet_info.h"
#include "api/rtp_packet_infos.h" #include "api/rtp_packet_infos.h"
#include "api/test/create_frame_generator.h" #include "api/test/create_frame_generator.h"
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "api/video/encoded_image.h" #include "api/video/encoded_image.h"
#include "api/video/i420_buffer.h" #include "api/video/i420_buffer.h"
#include "api/video/video_frame.h" #include "api/video/video_frame.h"
@ -148,6 +149,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -199,6 +201,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -280,6 +283,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -330,6 +334,7 @@ TEST(DefaultVideoQualityAnalyzerTest, NormalScenario) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -388,6 +393,7 @@ TEST(DefaultVideoQualityAnalyzerTest, OneFrameReceivedTwice) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -444,6 +450,7 @@ TEST(DefaultVideoQualityAnalyzerTest, NormalScenario2Receivers) {
constexpr char kCharlie[] = "charlie"; constexpr char kCharlie[] = "charlie";
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie}, analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -574,6 +581,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
constexpr char kCharlie[] = "charlie"; constexpr char kCharlie[] = "charlie";
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie}, analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -633,6 +641,7 @@ TEST(DefaultVideoQualityAnalyzerTest, HeavyQualityMetricsFromEqualFrames) {
analyzer_options.max_frames_in_flight_per_stream_count = analyzer_options.max_frames_in_flight_per_stream_count =
kMaxFramesInFlightPerStream; kMaxFramesInFlightPerStream;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
analyzer_options); analyzer_options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -692,6 +701,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
analyzer_options.max_frames_in_flight_per_stream_count = analyzer_options.max_frames_in_flight_per_stream_count =
kMaxFramesInFlightPerStream; kMaxFramesInFlightPerStream;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
analyzer_options); analyzer_options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -750,6 +760,7 @@ TEST(DefaultVideoQualityAnalyzerTest, CpuUsage) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -813,6 +824,7 @@ TEST(DefaultVideoQualityAnalyzerTest, RuntimeParticipantsAdding) {
constexpr int kTwoThirdFrames = 2 * kOneThirdFrames; constexpr int kTwoThirdFrames = 2 * kOneThirdFrames;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", {}, kAnalyzerMaxThreadsCount); analyzer.Start("test_case", {}, kAnalyzerMaxThreadsCount);
@ -958,6 +970,7 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
constexpr char kAlice[] = "alice"; constexpr char kAlice[] = "alice";
constexpr char kBob[] = "bob"; constexpr char kBob[] = "bob";
@ -1020,7 +1033,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true; options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1115,7 +1129,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true; options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1208,6 +1223,7 @@ TEST(DefaultVideoQualityAnalyzerTest, CodecTrackedCorrectly) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(),
AnalyzerOptionsForTest()); AnalyzerOptionsForTest());
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
@ -1278,7 +1294,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1370,7 +1387,8 @@ TEST(
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true; options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{kSenderPeerName, kReceiverPeerName}, std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1474,7 +1492,8 @@ TEST(DefaultVideoQualityAnalyzerTest, GetStreamFrames) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1518,7 +1537,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverReceivedFramesWhenSenderRemoved) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1561,7 +1581,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true; options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1604,7 +1625,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true; options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1647,7 +1669,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
options.enable_receive_own_stream = true; options.enable_receive_own_stream = true;
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1702,7 +1725,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeCapturing2ndFrame) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1742,7 +1766,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforePreEncoded) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1784,7 +1809,8 @@ TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeEncoded) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1827,7 +1853,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1873,7 +1900,8 @@ TEST(DefaultVideoQualityAnalyzerTest, UnregisterOneAndRegisterAnother) {
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{"alice", "bob", "charlie"}, std::vector<std::string>{"alice", "bob", "charlie"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1934,7 +1962,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", analyzer.Start("test_case",
std::vector<std::string>{"alice", "bob", "charlie"}, std::vector<std::string>{"alice", "bob", "charlie"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -1989,7 +2018,8 @@ TEST(DefaultVideoQualityAnalyzerTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);
@ -2026,7 +2056,8 @@ TEST_P(DefaultVideoQualityAnalyzerTimeBetweenFreezesTest,
/*num_squares=*/absl::nullopt); /*num_squares=*/absl::nullopt);
DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest(); DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(), options); DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
test::GetGlobalMetricsLogger(), options);
analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"}, analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
kAnalyzerMaxThreadsCount); kAnalyzerMaxThreadsCount);

View file

@ -16,6 +16,7 @@
#include "api/units/data_rate.h" #include "api/units/data_rate.h"
#include "api/units/time_delta.h" #include "api/units/time_delta.h"
#include "api/units/timestamp.h" #include "api/units/timestamp.h"
#include "rtc_base/checks.h"
namespace webrtc { namespace webrtc {
namespace webrtc_pc_e2e { namespace webrtc_pc_e2e {
@ -30,6 +31,13 @@ SamplesStatsCounter BytesPerSecondToKbps(const SamplesStatsCounter& counter) {
} // namespace } // namespace
VideoQualityMetricsReporter::VideoQualityMetricsReporter(
Clock* const clock,
test::MetricsLogger* const metrics_logger)
: clock_(clock), metrics_logger_(metrics_logger) {
RTC_CHECK(metrics_logger_);
}
void VideoQualityMetricsReporter::Start( void VideoQualityMetricsReporter::Start(
absl::string_view test_case_name, absl::string_view test_case_name,
const TrackIdStreamInfoMap* /*reporter_helper*/) { const TrackIdStreamInfoMap* /*reporter_helper*/) {
@ -122,14 +130,6 @@ std::string VideoQualityMetricsReporter::GetTestCaseName(
void VideoQualityMetricsReporter::ReportVideoBweResults( void VideoQualityMetricsReporter::ReportVideoBweResults(
const std::string& test_case_name, const std::string& test_case_name,
const VideoBweStats& video_bwe_stats) { const VideoBweStats& video_bwe_stats) {
if (metrics_logger_ == nullptr) {
ReportResult("available_send_bandwidth", test_case_name,
video_bwe_stats.available_send_bandwidth, "bytesPerSecond");
ReportResult("transmission_bitrate", test_case_name,
video_bwe_stats.transmission_bitrate, "bytesPerSecond");
ReportResult("retransmission_bitrate", test_case_name,
video_bwe_stats.retransmission_bitrate, "bytesPerSecond");
} else {
metrics_logger_->LogMetric( metrics_logger_->LogMetric(
"available_send_bandwidth", test_case_name, "available_send_bandwidth", test_case_name,
BytesPerSecondToKbps(video_bwe_stats.available_send_bandwidth), BytesPerSecondToKbps(video_bwe_stats.available_send_bandwidth),
@ -143,17 +143,6 @@ void VideoQualityMetricsReporter::ReportVideoBweResults(
BytesPerSecondToKbps(video_bwe_stats.retransmission_bitrate), BytesPerSecondToKbps(video_bwe_stats.retransmission_bitrate),
Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter); Unit::kKilobitsPerSecond, ImprovementDirection::kNeitherIsBetter);
} }
}
void VideoQualityMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) {
test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit,
/*important=*/false, improve_direction);
}
} // namespace webrtc_pc_e2e } // namespace webrtc_pc_e2e
} // namespace webrtc } // namespace webrtc

View file

@ -22,7 +22,6 @@
#include "api/units/data_size.h" #include "api/units/data_size.h"
#include "api/units/timestamp.h" #include "api/units/timestamp.h"
#include "rtc_base/synchronization/mutex.h" #include "rtc_base/synchronization/mutex.h"
#include "test/testsupport/perf_test.h"
namespace webrtc { namespace webrtc {
namespace webrtc_pc_e2e { namespace webrtc_pc_e2e {
@ -36,12 +35,8 @@ struct VideoBweStats {
class VideoQualityMetricsReporter class VideoQualityMetricsReporter
: public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
public: public:
explicit VideoQualityMetricsReporter(Clock* const clock) VideoQualityMetricsReporter(Clock* const clock,
: VideoQualityMetricsReporter(clock, /*metrics_logger=*/nullptr) {} test::MetricsLogger* const metrics_logger);
explicit VideoQualityMetricsReporter(
Clock* const clock,
test::MetricsLogger* const metrics_logger)
: clock_(clock), metrics_logger_(metrics_logger) {}
~VideoQualityMetricsReporter() override = default; ~VideoQualityMetricsReporter() override = default;
void Start(absl::string_view test_case_name, void Start(absl::string_view test_case_name,
@ -63,13 +58,6 @@ class VideoQualityMetricsReporter
std::string GetTestCaseName(const std::string& stream_label) const; std::string GetTestCaseName(const std::string& stream_label) const;
void ReportVideoBweResults(const std::string& test_case_name, void ReportVideoBweResults(const std::string& test_case_name,
const VideoBweStats& video_bwe_stats); const VideoBweStats& video_bwe_stats);
// Report result for single metric for specified stream.
static void ReportResult(const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction =
webrtc::test::ImproveDirection::kNone);
Timestamp Now() const { return clock_->CurrentTime(); } Timestamp Now() const { return clock_->CurrentTime(); }
Clock* const clock_; Clock* const clock_;

View file

@ -16,6 +16,7 @@
#include "api/stats/rtcstats_objects.h" #include "api/stats/rtcstats_objects.h"
#include "api/test/metrics/metric.h" #include "api/test/metrics/metric.h"
#include "api/units/timestamp.h" #include "api/units/timestamp.h"
#include "rtc_base/checks.h"
#include "rtc_base/event.h" #include "rtc_base/event.h"
#include "system_wrappers/include/field_trial.h" #include "system_wrappers/include/field_trial.h"
@ -25,6 +26,12 @@ namespace webrtc_pc_e2e {
using ::webrtc::test::ImprovementDirection; using ::webrtc::test::ImprovementDirection;
using ::webrtc::test::Unit; using ::webrtc::test::Unit;
CrossMediaMetricsReporter::CrossMediaMetricsReporter(
test::MetricsLogger* metrics_logger)
: metrics_logger_(metrics_logger) {
RTC_CHECK(metrics_logger_);
}
void CrossMediaMetricsReporter::Start( void CrossMediaMetricsReporter::Start(
absl::string_view test_case_name, absl::string_view test_case_name,
const TrackIdStreamInfoMap* reporter_helper) { const TrackIdStreamInfoMap* reporter_helper) {
@ -102,16 +109,6 @@ void CrossMediaMetricsReporter::StopAndReportResults() {
MutexLock lock(&mutex_); MutexLock lock(&mutex_);
for (const auto& pair : stats_info_) { for (const auto& pair : stats_info_) {
const std::string& sync_group = pair.first; const std::string& sync_group = pair.first;
if (metrics_logger_ == nullptr) {
ReportResult("audio_ahead_ms",
GetTestCaseName(pair.second.audio_stream_label, sync_group),
pair.second.audio_ahead_ms, "ms",
webrtc::test::ImproveDirection::kSmallerIsBetter);
ReportResult("video_ahead_ms",
GetTestCaseName(pair.second.video_stream_label, sync_group),
pair.second.video_ahead_ms, "ms",
webrtc::test::ImproveDirection::kSmallerIsBetter);
} else {
metrics_logger_->LogMetric( metrics_logger_->LogMetric(
"audio_ahead_ms", "audio_ahead_ms",
GetTestCaseName(pair.second.audio_stream_label, sync_group), GetTestCaseName(pair.second.audio_stream_label, sync_group),
@ -124,17 +121,6 @@ void CrossMediaMetricsReporter::StopAndReportResults() {
webrtc::test::ImprovementDirection::kSmallerIsBetter); webrtc::test::ImprovementDirection::kSmallerIsBetter);
} }
} }
}
void CrossMediaMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction) {
test::PrintResult(metric_name, /*modifier=*/"", test_case_name, counter, unit,
/*important=*/false, improve_direction);
}
std::string CrossMediaMetricsReporter::GetTestCaseName( std::string CrossMediaMetricsReporter::GetTestCaseName(
const std::string& stream_label, const std::string& stream_label,

View file

@ -22,7 +22,6 @@
#include "api/test/track_id_stream_info_map.h" #include "api/test/track_id_stream_info_map.h"
#include "api/units/timestamp.h" #include "api/units/timestamp.h"
#include "rtc_base/synchronization/mutex.h" #include "rtc_base/synchronization/mutex.h"
#include "test/testsupport/perf_test.h"
namespace webrtc { namespace webrtc {
namespace webrtc_pc_e2e { namespace webrtc_pc_e2e {
@ -30,10 +29,7 @@ namespace webrtc_pc_e2e {
class CrossMediaMetricsReporter class CrossMediaMetricsReporter
: public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
public: public:
CrossMediaMetricsReporter() explicit CrossMediaMetricsReporter(test::MetricsLogger* metrics_logger);
: CrossMediaMetricsReporter(/*metrics_logger=*/nullptr) {}
explicit CrossMediaMetricsReporter(test::MetricsLogger* metrics_logger)
: metrics_logger_(metrics_logger) {}
~CrossMediaMetricsReporter() override = default; ~CrossMediaMetricsReporter() override = default;
void Start(absl::string_view test_case_name, void Start(absl::string_view test_case_name,
@ -52,12 +48,6 @@ class CrossMediaMetricsReporter
std::string video_stream_label; std::string video_stream_label;
}; };
static void ReportResult(const std::string& metric_name,
const std::string& test_case_name,
const SamplesStatsCounter& counter,
const std::string& unit,
webrtc::test::ImproveDirection improve_direction =
webrtc::test::ImproveDirection::kNone);
std::string GetTestCaseName(const std::string& stream_label, std::string GetTestCaseName(const std::string& stream_label,
const std::string& sync_group) const; const std::string& sync_group) const;

View file

@ -14,9 +14,9 @@
#include "api/stats/rtc_stats.h" #include "api/stats/rtc_stats.h"
#include "api/stats/rtcstats_objects.h" #include "api/stats/rtcstats_objects.h"
#include "api/test/metrics/metric.h" #include "api/test/metrics/metric.h"
#include "rtc_base/checks.h"
#include "rtc_base/event.h" #include "rtc_base/event.h"
#include "system_wrappers/include/field_trial.h" #include "system_wrappers/include/field_trial.h"
#include "test/testsupport/perf_test.h"
namespace webrtc { namespace webrtc {
namespace webrtc_pc_e2e { namespace webrtc_pc_e2e {
@ -31,6 +31,17 @@ constexpr TimeDelta kStatsWaitTimeout = TimeDelta::Seconds(1);
// sent/received per stream. If enabled, padding and headers are not included // sent/received per stream. If enabled, padding and headers are not included
// in bytes sent or received. // in bytes sent or received.
constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats"; constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats";
} // namespace
NetworkQualityMetricsReporter::NetworkQualityMetricsReporter(
EmulatedNetworkManagerInterface* alice_network,
EmulatedNetworkManagerInterface* bob_network,
test::MetricsLogger* metrics_logger)
: alice_network_(alice_network),
bob_network_(bob_network),
metrics_logger_(metrics_logger) {
RTC_CHECK(metrics_logger_);
} }
void NetworkQualityMetricsReporter::Start( void NetworkQualityMetricsReporter::Start(
@ -113,35 +124,9 @@ void NetworkQualityMetricsReporter::ReportStats(
const std::string& network_label, const std::string& network_label,
std::unique_ptr<EmulatedNetworkStats> stats, std::unique_ptr<EmulatedNetworkStats> stats,
int64_t packet_loss) { int64_t packet_loss) {
if (metrics_logger_ == nullptr) {
ReportResult("bytes_sent", network_label, stats->BytesSent().bytes(),
"sizeInBytes");
ReportResult("packets_sent", network_label, stats->PacketsSent(),
"unitless");
ReportResult("average_send_rate", network_label,
stats->PacketsSent() >= 2
? stats->AverageSendRate().bytes_per_sec()
: 0,
"bytesPerSecond");
ReportResult("bytes_discarded_no_receiver", network_label,
stats->BytesDropped().bytes(), "sizeInBytes");
ReportResult("packets_discarded_no_receiver", network_label,
stats->PacketsDropped(), "unitless");
ReportResult("bytes_received", network_label,
stats->BytesReceived().bytes(), "sizeInBytes");
ReportResult("packets_received", network_label, stats->PacketsReceived(),
"unitless");
ReportResult("average_receive_rate", network_label,
stats->PacketsReceived() >= 2
? stats->AverageReceiveRate().bytes_per_sec()
: 0,
"bytesPerSecond");
ReportResult("sent_packets_loss", network_label, packet_loss, "unitless");
} else {
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric(
"bytes_sent", GetTestCaseName(network_label), "bytes_sent", GetTestCaseName(network_label), stats->BytesSent().bytes(),
stats->BytesSent().bytes(), Unit::kBytes, Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric(
"packets_sent", GetTestCaseName(network_label), stats->PacketsSent(), "packets_sent", GetTestCaseName(network_label), stats->PacketsSent(),
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
@ -173,33 +158,15 @@ void NetworkQualityMetricsReporter::ReportStats(
"sent_packets_loss", GetTestCaseName(network_label), packet_loss, "sent_packets_loss", GetTestCaseName(network_label), packet_loss,
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
} }
}
void NetworkQualityMetricsReporter::ReportPCStats(const std::string& pc_label, void NetworkQualityMetricsReporter::ReportPCStats(const std::string& pc_label,
const PCStats& stats) { const PCStats& stats) {
if (metrics_logger_ == nullptr) {
ReportResult("payload_bytes_received", pc_label,
stats.payload_received.bytes(), "sizeInBytes");
ReportResult("payload_bytes_sent", pc_label, stats.payload_sent.bytes(),
"sizeInBytes");
} else {
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric(
"payload_bytes_received", pc_label, stats.payload_received.bytes(), "payload_bytes_received", pc_label, stats.payload_received.bytes(),
Unit::kBytes, ImprovementDirection::kNeitherIsBetter); Unit::kBytes, ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric(
"payload_bytes_sent", pc_label, stats.payload_sent.bytes(), "payload_bytes_sent", pc_label, stats.payload_sent.bytes(), Unit::kBytes,
Unit::kBytes, ImprovementDirection::kNeitherIsBetter); ImprovementDirection::kNeitherIsBetter);
}
}
void NetworkQualityMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& network_label,
const double value,
const std::string& unit) const {
test::PrintResult(metric_name, /*modifier=*/"",
GetTestCaseName(network_label), value, unit,
/*important=*/false);
} }
std::string NetworkQualityMetricsReporter::GetTestCaseName( std::string NetworkQualityMetricsReporter::GetTestCaseName(

View file

@ -28,17 +28,9 @@ namespace webrtc_pc_e2e {
class NetworkQualityMetricsReporter class NetworkQualityMetricsReporter
: public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter { : public PeerConnectionE2EQualityTestFixture::QualityMetricsReporter {
public: public:
NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network,
EmulatedNetworkManagerInterface* bob_network)
: NetworkQualityMetricsReporter(alice_network,
bob_network,
/*metrics_logger=*/nullptr) {}
NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network, NetworkQualityMetricsReporter(EmulatedNetworkManagerInterface* alice_network,
EmulatedNetworkManagerInterface* bob_network, EmulatedNetworkManagerInterface* bob_network,
test::MetricsLogger* metrics_logger) test::MetricsLogger* metrics_logger);
: alice_network_(alice_network),
bob_network_(bob_network),
metrics_logger_(metrics_logger) {}
~NetworkQualityMetricsReporter() override = default; ~NetworkQualityMetricsReporter() override = default;
// Network stats must be empty when this method will be invoked. // Network stats must be empty when this method will be invoked.
@ -63,10 +55,6 @@ class NetworkQualityMetricsReporter
std::unique_ptr<EmulatedNetworkStats> stats, std::unique_ptr<EmulatedNetworkStats> stats,
int64_t packet_loss); int64_t packet_loss);
void ReportPCStats(const std::string& pc_label, const PCStats& stats); void ReportPCStats(const std::string& pc_label, const PCStats& stats);
void ReportResult(const std::string& metric_name,
const std::string& network_label,
double value,
const std::string& unit) const;
std::string GetTestCaseName(const std::string& network_label) const; std::string GetTestCaseName(const std::string& network_label) const;
std::string test_case_name_; std::string test_case_name_;

View file

@ -41,7 +41,6 @@
#include "test/pc/e2e/stats_poller.h" #include "test/pc/e2e/stats_poller.h"
#include "test/pc/e2e/test_peer_factory.h" #include "test/pc/e2e/test_peer_factory.h"
#include "test/testsupport/file_utils.h" #include "test/testsupport/file_utils.h"
#include "test/testsupport/perf_test.h"
namespace webrtc { namespace webrtc {
namespace webrtc_pc_e2e { namespace webrtc_pc_e2e {
@ -736,25 +735,13 @@ void PeerConnectionE2EQualityTest::TearDownCall() {
} }
void PeerConnectionE2EQualityTest::ReportGeneralTestResults() { void PeerConnectionE2EQualityTest::ReportGeneralTestResults() {
if (metrics_logger_ == nullptr) {
test::PrintResult(*alice_->params().name + "_connected", "",
test_case_name_, alice_connected_, "unitless",
/*important=*/false,
test::ImproveDirection::kBiggerIsBetter);
test::PrintResult(*bob_->params().name + "_connected", "", test_case_name_,
bob_connected_, "unitless",
/*important=*/false,
test::ImproveDirection::kBiggerIsBetter);
} else {
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric(
*alice_->params().name + "_connected", test_case_name_, *alice_->params().name + "_connected", test_case_name_, alice_connected_,
alice_connected_, Unit::kUnitless, Unit::kUnitless, ImprovementDirection::kBiggerIsBetter);
ImprovementDirection::kBiggerIsBetter);
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric(
*bob_->params().name + "_connected", test_case_name_, bob_connected_, *bob_->params().name + "_connected", test_case_name_, bob_connected_,
Unit::kUnitless, ImprovementDirection::kBiggerIsBetter); Unit::kUnitless, ImprovementDirection::kBiggerIsBetter);
} }
}
Timestamp PeerConnectionE2EQualityTest::Now() const { Timestamp PeerConnectionE2EQualityTest::Now() const {
return time_controller_.GetClock()->CurrentTime(); return time_controller_.GetClock()->CurrentTime();

View file

@ -29,12 +29,12 @@
#include "api/test/network_emulation_manager.h" #include "api/test/network_emulation_manager.h"
#include "api/units/data_rate.h" #include "api/units/data_rate.h"
#include "api/units/timestamp.h" #include "api/units/timestamp.h"
#include "rtc_base/checks.h"
#include "rtc_base/event.h" #include "rtc_base/event.h"
#include "rtc_base/ip_address.h" #include "rtc_base/ip_address.h"
#include "rtc_base/strings/string_builder.h" #include "rtc_base/strings/string_builder.h"
#include "rtc_base/synchronization/mutex.h" #include "rtc_base/synchronization/mutex.h"
#include "system_wrappers/include/field_trial.h" #include "system_wrappers/include/field_trial.h"
#include "test/testsupport/perf_test.h"
namespace webrtc { namespace webrtc {
namespace webrtc_pc_e2e { namespace webrtc_pc_e2e {
@ -81,6 +81,17 @@ std::map<rtc::IPAddress, std::string> PopulateIpToPeer(
} // namespace } // namespace
StatsBasedNetworkQualityMetricsReporter::
StatsBasedNetworkQualityMetricsReporter(
std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
NetworkEmulationManager* network_emulation,
test::MetricsLogger* metrics_logger)
: collector_(std::move(peer_endpoints), network_emulation),
clock_(network_emulation->time_controller()->GetClock()),
metrics_logger_(metrics_logger) {
RTC_CHECK(metrics_logger_);
}
StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector:: StatsBasedNetworkQualityMetricsReporter::NetworkLayerStatsCollector::
NetworkLayerStatsCollector( NetworkLayerStatsCollector(
std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints, std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
@ -241,35 +252,6 @@ void StatsBasedNetworkQualityMetricsReporter::ReportStats(
const NetworkLayerStats& network_layer_stats, const NetworkLayerStats& network_layer_stats,
int64_t packet_loss, int64_t packet_loss,
const Timestamp& end_time) { const Timestamp& end_time) {
if (metrics_logger_ == nullptr) {
ReportResult("bytes_discarded_no_receiver", pc_label,
network_layer_stats.stats->BytesDropped().bytes(),
"sizeInBytes");
ReportResult("packets_discarded_no_receiver", pc_label,
network_layer_stats.stats->PacketsDropped(), "unitless");
ReportResult("payload_bytes_received", pc_label,
pc_stats.payload_received.bytes(), "sizeInBytes");
ReportResult("payload_bytes_sent", pc_label, pc_stats.payload_sent.bytes(),
"sizeInBytes");
ReportResult("bytes_sent", pc_label, pc_stats.total_sent.bytes(),
"sizeInBytes");
ReportResult("packets_sent", pc_label, pc_stats.packets_sent, "unitless");
ReportResult(
"average_send_rate", pc_label,
(pc_stats.total_sent / (end_time - start_time_)).bytes_per_sec(),
"bytesPerSecond");
ReportResult("bytes_received", pc_label, pc_stats.total_received.bytes(),
"sizeInBytes");
ReportResult("packets_received", pc_label, pc_stats.packets_received,
"unitless");
ReportResult(
"average_receive_rate", pc_label,
(pc_stats.total_received / (end_time - start_time_)).bytes_per_sec(),
"bytesPerSecond");
ReportResult("sent_packets_loss", pc_label, packet_loss, "unitless");
} else {
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric(
"bytes_discarded_no_receiver", GetTestCaseName(pc_label), "bytes_discarded_no_receiver", GetTestCaseName(pc_label),
network_layer_stats.stats->BytesDropped().bytes(), Unit::kBytes, network_layer_stats.stats->BytesDropped().bytes(), Unit::kBytes,
@ -303,9 +285,8 @@ void StatsBasedNetworkQualityMetricsReporter::ReportStats(
pc_stats.total_received.bytes(), Unit::kBytes, pc_stats.total_received.bytes(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter); ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric(
"packets_received", GetTestCaseName(pc_label), "packets_received", GetTestCaseName(pc_label), pc_stats.packets_received,
pc_stats.packets_received, Unit::kUnitless, Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
ImprovementDirection::kNeitherIsBetter);
metrics_logger_->LogSingleValueMetric( metrics_logger_->LogSingleValueMetric(
"average_receive_rate", GetTestCaseName(pc_label), "average_receive_rate", GetTestCaseName(pc_label),
(pc_stats.total_received / (end_time - start_time_)).kbps(), (pc_stats.total_received / (end_time - start_time_)).kbps(),
@ -314,27 +295,6 @@ void StatsBasedNetworkQualityMetricsReporter::ReportStats(
"sent_packets_loss", GetTestCaseName(pc_label), packet_loss, "sent_packets_loss", GetTestCaseName(pc_label), packet_loss,
Unit::kUnitless, ImprovementDirection::kNeitherIsBetter); Unit::kUnitless, ImprovementDirection::kNeitherIsBetter);
} }
}
void StatsBasedNetworkQualityMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& network_label,
const double value,
const std::string& unit) const {
test::PrintResult(metric_name, /*modifier=*/"",
GetTestCaseName(network_label), value, unit,
/*important=*/false);
}
void StatsBasedNetworkQualityMetricsReporter::ReportResult(
const std::string& metric_name,
const std::string& network_label,
const SamplesStatsCounter& value,
const std::string& unit) const {
test::PrintResult(metric_name, /*modifier=*/"",
GetTestCaseName(network_label), value, unit,
/*important=*/false);
}
std::string StatsBasedNetworkQualityMetricsReporter::GetTestCaseName( std::string StatsBasedNetworkQualityMetricsReporter::GetTestCaseName(
absl::string_view network_label) const { absl::string_view network_label) const {
@ -360,49 +320,29 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
log << " " << local_ips[i].ToString() << "\n"; log << " " << local_ips[i].ToString() << "\n";
} }
if (!stats.stats->SentPacketsSizeCounter().IsEmpty()) { if (!stats.stats->SentPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) { metrics_logger_->LogMetric("sent_packets_size", GetTestCaseName(peer_name),
ReportResult("sent_packets_size", peer_name, stats.stats->SentPacketsSizeCounter(),
stats.stats->SentPacketsSizeCounter(), "sizeInBytes"); Unit::kBytes,
} else {
metrics_logger_->LogMetric(
"sent_packets_size", GetTestCaseName(peer_name),
stats.stats->SentPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter); ImprovementDirection::kNeitherIsBetter);
} }
}
if (!stats.stats->ReceivedPacketsSizeCounter().IsEmpty()) { if (!stats.stats->ReceivedPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) {
ReportResult("received_packets_size", peer_name,
stats.stats->ReceivedPacketsSizeCounter(), "sizeInBytes");
} else {
metrics_logger_->LogMetric( metrics_logger_->LogMetric(
"received_packets_size", GetTestCaseName(peer_name), "received_packets_size", GetTestCaseName(peer_name),
stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes, stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter); ImprovementDirection::kNeitherIsBetter);
} }
}
if (!stats.stats->DroppedPacketsSizeCounter().IsEmpty()) { if (!stats.stats->DroppedPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) {
ReportResult("dropped_packets_size", peer_name,
stats.stats->DroppedPacketsSizeCounter(), "sizeInBytes");
} else {
metrics_logger_->LogMetric( metrics_logger_->LogMetric(
"dropped_packets_size", GetTestCaseName(peer_name), "dropped_packets_size", GetTestCaseName(peer_name),
stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes, stats.stats->DroppedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter); ImprovementDirection::kNeitherIsBetter);
} }
}
if (!stats.stats->SentPacketsQueueWaitTimeUs().IsEmpty()) { if (!stats.stats->SentPacketsQueueWaitTimeUs().IsEmpty()) {
if (metrics_logger_ == nullptr) {
ReportResult("sent_packets_queue_wait_time_us", peer_name,
stats.stats->SentPacketsQueueWaitTimeUs(), "unitless");
} else {
metrics_logger_->LogMetric( metrics_logger_->LogMetric(
"sent_packets_queue_wait_time_us", GetTestCaseName(peer_name), "sent_packets_queue_wait_time_us", GetTestCaseName(peer_name),
stats.stats->SentPacketsQueueWaitTimeUs(), Unit::kUnitless, stats.stats->SentPacketsQueueWaitTimeUs(), Unit::kUnitless,
ImprovementDirection::kNeitherIsBetter); ImprovementDirection::kNeitherIsBetter);
} }
}
log << "Send statistic:\n" log << "Send statistic:\n"
<< " packets: " << stats.stats->PacketsSent() << " packets: " << stats.stats->PacketsSent()
@ -421,11 +361,6 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
<< " avg_rate (bytes/sec): " << source_average_send_rate.bytes_per_sec() << " avg_rate (bytes/sec): " << source_average_send_rate.bytes_per_sec()
<< " avg_rate (bps): " << source_average_send_rate.bps() << "\n"; << " avg_rate (bps): " << source_average_send_rate.bps() << "\n";
if (!entry.second->SentPacketsSizeCounter().IsEmpty()) { if (!entry.second->SentPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) {
ReportResult("sent_packets_size",
peer_name + "/" + entry.first.ToString(),
stats.stats->SentPacketsSizeCounter(), "sizeInBytes");
} else {
metrics_logger_->LogMetric( metrics_logger_->LogMetric(
"sent_packets_size", "sent_packets_size",
GetTestCaseName(peer_name + "/" + entry.first.ToString()), GetTestCaseName(peer_name + "/" + entry.first.ToString()),
@ -433,7 +368,6 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
ImprovementDirection::kNeitherIsBetter); ImprovementDirection::kNeitherIsBetter);
} }
} }
}
log << "Receive statistic:\n" log << "Receive statistic:\n"
<< " packets: " << stats.stats->PacketsReceived() << " packets: " << stats.stats->PacketsReceived()
@ -454,24 +388,13 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
<< source_average_receive_rate.bytes_per_sec() << source_average_receive_rate.bytes_per_sec()
<< " avg_rate (bps): " << source_average_receive_rate.bps() << "\n"; << " avg_rate (bps): " << source_average_receive_rate.bps() << "\n";
if (!entry.second->ReceivedPacketsSizeCounter().IsEmpty()) { if (!entry.second->ReceivedPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) {
ReportResult("received_packets_size",
peer_name + "/" + entry.first.ToString(),
stats.stats->ReceivedPacketsSizeCounter(), "sizeInBytes");
} else {
metrics_logger_->LogMetric( metrics_logger_->LogMetric(
"received_packets_size", "received_packets_size",
GetTestCaseName(peer_name + "/" + entry.first.ToString()), GetTestCaseName(peer_name + "/" + entry.first.ToString()),
stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes, stats.stats->ReceivedPacketsSizeCounter(), Unit::kBytes,
ImprovementDirection::kNeitherIsBetter); ImprovementDirection::kNeitherIsBetter);
} }
}
if (!entry.second->DroppedPacketsSizeCounter().IsEmpty()) { if (!entry.second->DroppedPacketsSizeCounter().IsEmpty()) {
if (metrics_logger_ == nullptr) {
ReportResult("dropped_packets_size",
peer_name + "/" + entry.first.ToString(),
stats.stats->DroppedPacketsSizeCounter(), "sizeInBytes");
} else {
metrics_logger_->LogMetric( metrics_logger_->LogMetric(
"dropped_packets_size", "dropped_packets_size",
GetTestCaseName(peer_name + "/" + entry.first.ToString()), GetTestCaseName(peer_name + "/" + entry.first.ToString()),
@ -479,7 +402,6 @@ void StatsBasedNetworkQualityMetricsReporter::LogNetworkLayerStats(
ImprovementDirection::kNeitherIsBetter); ImprovementDirection::kNeitherIsBetter);
} }
} }
}
RTC_LOG(LS_INFO) << log.str(); RTC_LOG(LS_INFO) << log.str();
} }

View file

@ -39,19 +39,10 @@ class StatsBasedNetworkQualityMetricsReporter
public: public:
// `networks` map peer name to network to report network layer stability stats // `networks` map peer name to network to report network layer stability stats
// and to log network layer metrics. // and to log network layer metrics.
StatsBasedNetworkQualityMetricsReporter(
std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
NetworkEmulationManager* network_emulation)
: StatsBasedNetworkQualityMetricsReporter(std::move(peer_endpoints),
network_emulation,
/*metrics_logger=*/nullptr) {}
StatsBasedNetworkQualityMetricsReporter( StatsBasedNetworkQualityMetricsReporter(
std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints, std::map<std::string, std::vector<EmulatedEndpoint*>> peer_endpoints,
NetworkEmulationManager* network_emulation, NetworkEmulationManager* network_emulation,
test::MetricsLogger* metrics_logger) test::MetricsLogger* metrics_logger);
: collector_(std::move(peer_endpoints), network_emulation),
clock_(network_emulation->time_controller()->GetClock()),
metrics_logger_(metrics_logger) {}
~StatsBasedNetworkQualityMetricsReporter() override = default; ~StatsBasedNetworkQualityMetricsReporter() override = default;
void AddPeer(absl::string_view peer_name, void AddPeer(absl::string_view peer_name,
@ -111,14 +102,6 @@ class StatsBasedNetworkQualityMetricsReporter
const NetworkLayerStats& network_layer_stats, const NetworkLayerStats& network_layer_stats,
int64_t packet_loss, int64_t packet_loss,
const Timestamp& end_time); const Timestamp& end_time);
void ReportResult(const std::string& metric_name,
const std::string& network_label,
double value,
const std::string& unit) const;
void ReportResult(const std::string& metric_name,
const std::string& network_label,
const SamplesStatsCounter& value,
const std::string& unit) const;
std::string GetTestCaseName(absl::string_view network_label) const; std::string GetTestCaseName(absl::string_view network_label) const;
void LogNetworkLayerStats(const std::string& peer_name, void LogNetworkLayerStats(const std::string& peer_name,
const NetworkLayerStats& stats) const; const NetworkLayerStats& stats) const;

View file

@ -585,6 +585,7 @@ if (rtc_include_tests) {
"../api:peer_connection_quality_test_fixture_api", "../api:peer_connection_quality_test_fixture_api",
"../api:simulated_network_api", "../api:simulated_network_api",
"../api:time_controller", "../api:time_controller",
"../api/test/metrics:global_metrics_logger_and_exporter",
"../api/video_codecs:video_codecs_api", "../api/video_codecs:video_codecs_api",
"../call:simulated_network", "../call:simulated_network",
"../modules/video_coding:webrtc_vp9", "../modules/video_coding:webrtc_vp9",

View file

@ -17,6 +17,7 @@
#include "api/test/create_peer_connection_quality_test_frame_generator.h" #include "api/test/create_peer_connection_quality_test_frame_generator.h"
#include "api/test/create_peerconnection_quality_test_fixture.h" #include "api/test/create_peerconnection_quality_test_fixture.h"
#include "api/test/frame_generator_interface.h" #include "api/test/frame_generator_interface.h"
#include "api/test/metrics/global_metrics_logger_and_exporter.h"
#include "api/test/network_emulation_manager.h" #include "api/test/network_emulation_manager.h"
#include "api/test/peerconnection_quality_test_fixture.h" #include "api/test/peerconnection_quality_test_fixture.h"
#include "api/test/simulated_network.h" #include "api/test/simulated_network.h"
@ -93,7 +94,8 @@ CreateTestFixture(const std::string& test_case_name,
bob_configurer); bob_configurer);
fixture->AddQualityMetricsReporter( fixture->AddQualityMetricsReporter(
std::make_unique<webrtc_pc_e2e::NetworkQualityMetricsReporter>( std::make_unique<webrtc_pc_e2e::NetworkQualityMetricsReporter>(
network_links.first, network_links.second)); network_links.first, network_links.second,
test::GetGlobalMetricsLogger()));
return fixture; return fixture;
} }