diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 5b59df7dac..48dac93bd9 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -517,6 +517,7 @@ if (rtc_include_tests) { "../../test:test_support", "../../test:video_test_common", "../../test:video_test_support", + "../rtp_rtcp:rtp_rtcp_format", ] } @@ -578,6 +579,7 @@ if (rtc_include_tests) { "../../rtc_base:rtc_base_tests_utils", "../../system_wrappers", "../../test:field_trial", + "../../test:test_common", "../../test:test_support", "../../test:video_test_common", "../../test:video_test_support", diff --git a/modules/video_coding/codecs/test/plot_webrtc_test_logs.py b/modules/video_coding/codecs/test/plot_webrtc_test_logs.py index ae5621f44a..e25ea3e010 100755 --- a/modules/video_coding/codecs/test/plot_webrtc_test_logs.py +++ b/modules/video_coding/codecs/test/plot_webrtc_test_logs.py @@ -24,6 +24,7 @@ EVENT_END = 'OK ] CodecSettings/VideoProcessorIntegrationTestParameterized.' # Metrics to plot, tuple: (name to parse in file, label to use when plotting). BITRATE = ('Target bitrate', 'target bitrate (kbps)') +FRAMERATE = ('Target framerate', 'fps') WIDTH = ('Width', 'width') HEIGHT = ('Height', 'height') FILENAME = ('Filename', 'clip') @@ -35,24 +36,22 @@ CORES = ('# CPU cores used', 'CPU cores used') DENOISING = ('Denoising', 'denoising') RESILIENCE = ('Resilience', 'resilience') ERROR_CONCEALMENT = ('Error concealment', 'error concealment') -QP = ('Average QP', 'avg QP') +QP = ('QP', 'QP avg') CPU_USAGE = ('CPU usage %', 'CPU usage (%)') -PSNR = ('PSNR avg', 'PSNR (dB)') -SSIM = ('SSIM avg', 'SSIM') +PSNR = ('PSNR', 'PSNR (dB)') +SSIM = ('SSIM', 'SSIM') ENC_BITRATE = ('Encoded bitrate', 'encoded bitrate (kbps)') -FRAMERATE = ('Frame rate', 'fps') -NUM_FRAMES = ('# processed frames', 'num frames') +NUM_FRAMES = ('# input frames', 'num frames') NUM_DROPPED_FRAMES = ('# dropped frames', 'num dropped frames') -NUM_FRAMES_TO_TARGET = ('# frames to convergence', - 'frames to reach target rate') +TIME_TO_TARGET = ('Time to reach target bitrate', + 'time to reach target rate (sec)') ENCODE_TIME = ('Encoding time', 'encode time (us)') -ENCODE_TIME_AVG = ('Encoding time', 'encode time (us) avg') +ENCODE_TIME_AVG = ('Frame encoding time', 'encode time (us) avg') DECODE_TIME = ('Decoding time', 'decode time (us)') -DECODE_TIME_AVG = ('Decoding time', 'decode time (us) avg') -FRAME_SIZE = ('Frame sizes', 'frame size (bytes)') -FRAME_SIZE_AVG = ('Frame sizes', 'frame size (bytes) avg') -AVG_KEY_FRAME_SIZE = ('Average key frame size', 'avg key frame size (bytes)') -AVG_NON_KEY_FRAME_SIZE = ('Average non-key frame size', +DECODE_TIME_AVG = ('Frame decoding time', 'decode time (us) avg') +FRAME_SIZE = ('Frame size', 'frame size (bytes)') +AVG_KEY_FRAME_SIZE = ('Avg key frame size', 'avg key frame size (bytes)') +AVG_NON_KEY_FRAME_SIZE = ('Avg delta frame size', 'avg non-key frame size (bytes)') # Settings. @@ -90,7 +89,7 @@ RESULTS = [ SSIM, ENC_BITRATE, NUM_DROPPED_FRAMES, - NUM_FRAMES_TO_TARGET, + TIME_TO_TARGET, ENCODE_TIME_AVG, DECODE_TIME_AVG, QP, @@ -235,7 +234,7 @@ def TryFindMetric(parsed, line, settings_file): found, maximum = GetMetric("Max", settings_file.readline()) if not found: return - found, average = GetMetric("Average", settings_file.readline()) + found, average = GetMetric("Avg", settings_file.readline()) if not found: return diff --git a/modules/video_coding/codecs/test/stats.cc b/modules/video_coding/codecs/test/stats.cc index a3199ab6fa..0aab931b52 100644 --- a/modules/video_coding/codecs/test/stats.cc +++ b/modules/video_coding/codecs/test/stats.cc @@ -9,59 +9,35 @@ */ #include "modules/video_coding/codecs/test/stats.h" - -#include - -#include - #include "rtc_base/checks.h" -#include "rtc_base/format_macros.h" namespace webrtc { namespace test { -namespace { - -bool LessForEncodeTime(const FrameStatistic& s1, const FrameStatistic& s2) { - RTC_DCHECK_NE(s1.frame_number, s2.frame_number); - return s1.encode_time_us < s2.encode_time_us; +std::string FrameStatistic::ToString() const { + std::stringstream ss; + ss << "frame " << frame_number; + ss << " " << decoded_width << "x" << decoded_height; + ss << " sl " << simulcast_svc_idx; + ss << " tl " << temporal_layer_idx; + ss << " type " << frame_type; + ss << " length " << encoded_frame_size_bytes; + ss << " qp " << qp; + ss << " psnr " << psnr; + ss << " ssim " << ssim; + ss << " enc_time_us " << encode_time_us; + ss << " dec_time_us " << decode_time_us; + ss << " rtp_ts " << rtp_timestamp; + ss << " bitrate_kbps " << target_bitrate_kbps; + return ss.str(); } -bool LessForDecodeTime(const FrameStatistic& s1, const FrameStatistic& s2) { - RTC_DCHECK_NE(s1.frame_number, s2.frame_number); - return s1.decode_time_us < s2.decode_time_us; -} - -bool LessForEncodedSize(const FrameStatistic& s1, const FrameStatistic& s2) { - RTC_DCHECK_NE(s1.frame_number, s2.frame_number); - return s1.encoded_frame_size_bytes < s2.encoded_frame_size_bytes; -} - -bool LessForBitRate(const FrameStatistic& s1, const FrameStatistic& s2) { - RTC_DCHECK_NE(s1.frame_number, s2.frame_number); - return s1.bitrate_kbps < s2.bitrate_kbps; -} - -bool LessForPsnr(const FrameStatistic& s1, const FrameStatistic& s2) { - RTC_DCHECK_NE(s1.frame_number, s2.frame_number); - return s1.psnr < s2.psnr; -} - -bool LessForSsim(const FrameStatistic& s1, const FrameStatistic& s2) { - RTC_DCHECK_NE(s1.frame_number, s2.frame_number); - return s1.ssim < s2.ssim; -} - -} // namespace - FrameStatistic* Stats::AddFrame() { - // We don't expect more frames than what can be stored in an int. - stats_.emplace_back(static_cast(stats_.size())); + stats_.emplace_back(stats_.size()); return &stats_.back(); } -FrameStatistic* Stats::GetFrame(int frame_number) { - RTC_CHECK_GE(frame_number, 0); +FrameStatistic* Stats::GetFrame(size_t frame_number) { RTC_CHECK_LT(frame_number, stats_.size()); return &stats_[frame_number]; } @@ -70,153 +46,5 @@ size_t Stats::size() const { return stats_.size(); } -void Stats::PrintSummary() const { - if (stats_.empty()) { - printf("No frame statistics have been logged yet.\n"); - return; - } - - printf("Encode/decode statistics\n==\n"); - - // Calculate min, max, average and total encoding time. - int total_encoding_time_us = 0; - int total_decoding_time_us = 0; - size_t total_encoded_frame_size_bytes = 0; - size_t total_encoded_key_frame_size_bytes = 0; - size_t total_encoded_delta_frame_size_bytes = 0; - size_t num_key_frames = 0; - size_t num_delta_frames = 0; - int num_encode_failures = 0; - double total_psnr = 0.0; - double total_ssim = 0.0; - - for (const FrameStatistic& stat : stats_) { - total_encoding_time_us += stat.encode_time_us; - total_decoding_time_us += stat.decode_time_us; - total_encoded_frame_size_bytes += stat.encoded_frame_size_bytes; - if (stat.frame_type == webrtc::kVideoFrameKey) { - total_encoded_key_frame_size_bytes += stat.encoded_frame_size_bytes; - ++num_key_frames; - } else { - total_encoded_delta_frame_size_bytes += stat.encoded_frame_size_bytes; - ++num_delta_frames; - } - if (stat.encode_return_code != 0) { - ++num_encode_failures; - } - if (stat.decoding_successful) { - total_psnr += stat.psnr; - total_ssim += stat.ssim; - } - } - - // Encoding stats. - printf("# Encoded frame failures: %d\n", num_encode_failures); - printf("Encoding time:\n"); - auto frame_it = - std::min_element(stats_.begin(), stats_.end(), LessForEncodeTime); - printf(" Min : %7d us (frame %d)\n", frame_it->encode_time_us, - frame_it->frame_number); - frame_it = std::max_element(stats_.begin(), stats_.end(), LessForEncodeTime); - printf(" Max : %7d us (frame %d)\n", frame_it->encode_time_us, - frame_it->frame_number); - printf(" Average : %7d us\n", - static_cast(total_encoding_time_us / stats_.size())); - - // Decoding stats. - printf("Decoding time:\n"); - // Only consider successfully decoded frames (packet loss may cause failures). - std::vector decoded_frames; - for (const FrameStatistic& stat : stats_) { - if (stat.decoding_successful) { - decoded_frames.push_back(stat); - } - } - if (decoded_frames.empty()) { - printf("No successfully decoded frames exist in this statistics.\n"); - } else { - frame_it = std::min_element(decoded_frames.begin(), decoded_frames.end(), - LessForDecodeTime); - printf(" Min : %7d us (frame %d)\n", frame_it->decode_time_us, - frame_it->frame_number); - frame_it = std::max_element(decoded_frames.begin(), decoded_frames.end(), - LessForDecodeTime); - printf(" Max : %7d us (frame %d)\n", frame_it->decode_time_us, - frame_it->frame_number); - printf(" Average : %7d us\n", - static_cast(total_decoding_time_us / decoded_frames.size())); - printf(" Failures: %d frames failed to decode.\n", - static_cast(stats_.size() - decoded_frames.size())); - } - - // Frame size stats. - printf("Frame sizes:\n"); - frame_it = std::min_element(stats_.begin(), stats_.end(), LessForEncodedSize); - printf(" Min : %7" PRIuS " bytes (frame %d)\n", - frame_it->encoded_frame_size_bytes, frame_it->frame_number); - frame_it = std::max_element(stats_.begin(), stats_.end(), LessForEncodedSize); - printf(" Max : %7" PRIuS " bytes (frame %d)\n", - frame_it->encoded_frame_size_bytes, frame_it->frame_number); - printf(" Average : %7" PRIuS " bytes\n", - total_encoded_frame_size_bytes / stats_.size()); - if (num_key_frames > 0) { - printf(" Average key frame size : %7" PRIuS " bytes (%" PRIuS - " keyframes)\n", - total_encoded_key_frame_size_bytes / num_key_frames, num_key_frames); - } - if (num_delta_frames > 0) { - printf(" Average non-key frame size: %7" PRIuS " bytes (%" PRIuS - " frames)\n", - total_encoded_delta_frame_size_bytes / num_delta_frames, - num_delta_frames); - } - - // Bitrate stats. - printf("Bitrates:\n"); - frame_it = std::min_element(stats_.begin(), stats_.end(), LessForBitRate); - printf(" Min bitrate: %7d kbps (frame %d)\n", frame_it->bitrate_kbps, - frame_it->frame_number); - frame_it = std::max_element(stats_.begin(), stats_.end(), LessForBitRate); - printf(" Max bitrate: %7d kbps (frame %d)\n", frame_it->bitrate_kbps, - frame_it->frame_number); - - // Quality. - printf("Quality:\n"); - if (decoded_frames.empty()) { - printf("No successfully decoded frames exist in this statistics.\n"); - } else { - frame_it = std::min_element(decoded_frames.begin(), decoded_frames.end(), - LessForPsnr); - printf(" PSNR min: %f (frame %d)\n", frame_it->psnr, - frame_it->frame_number); - printf(" PSNR avg: %f\n", total_psnr / decoded_frames.size()); - - frame_it = std::min_element(decoded_frames.begin(), decoded_frames.end(), - LessForSsim); - printf(" SSIM min: %f (frame %d)\n", frame_it->ssim, - frame_it->frame_number); - printf(" SSIM avg: %f\n", total_ssim / decoded_frames.size()); - } - - printf("\n"); - printf("Total encoding time : %7d ms.\n", total_encoding_time_us / 1000); - printf("Total decoding time : %7d ms.\n", total_decoding_time_us / 1000); - printf("Total processing time: %7d ms.\n", - (total_encoding_time_us + total_decoding_time_us) / 1000); - - // QP stats. - int total_qp = 0; - int total_qp_count = 0; - for (const FrameStatistic& stat : stats_) { - if (stat.qp >= 0) { - total_qp += stat.qp; - ++total_qp_count; - } - } - int avg_qp = (total_qp_count > 0) ? (total_qp / total_qp_count) : -1; - printf("Average QP: %d\n", avg_qp); - printf("\n"); -} - } // namespace test } // namespace webrtc diff --git a/modules/video_coding/codecs/test/stats.h b/modules/video_coding/codecs/test/stats.h index 02ca641041..42a399c016 100644 --- a/modules/video_coding/codecs/test/stats.h +++ b/modules/video_coding/codecs/test/stats.h @@ -11,6 +11,7 @@ #ifndef MODULES_VIDEO_CODING_CODECS_TEST_STATS_H_ #define MODULES_VIDEO_CODING_CODECS_TEST_STATS_H_ +#include #include #include "common_types.h" // NOLINT(build/include) @@ -20,34 +21,42 @@ namespace test { // Statistics for one processed frame. struct FrameStatistic { - explicit FrameStatistic(int frame_number) : frame_number(frame_number) {} - const int frame_number = 0; + explicit FrameStatistic(size_t frame_number) : frame_number(frame_number) {} + + std::string ToString() const; + + size_t frame_number = 0; + size_t rtp_timestamp = 0; // Encoding. int64_t encode_start_ns = 0; int encode_return_code = 0; bool encoding_successful = false; - int encode_time_us = 0; - int bitrate_kbps = 0; + size_t encode_time_us = 0; + size_t target_bitrate_kbps = 0; size_t encoded_frame_size_bytes = 0; webrtc::FrameType frame_type = kVideoFrameDelta; + // Layering. + size_t temporal_layer_idx = 0; + size_t simulcast_svc_idx = 0; + // H264 specific. - rtc::Optional max_nalu_length; + size_t max_nalu_size_bytes = 0; // Decoding. int64_t decode_start_ns = 0; int decode_return_code = 0; bool decoding_successful = false; - int decode_time_us = 0; - int decoded_width = 0; - int decoded_height = 0; + size_t decode_time_us = 0; + size_t decoded_width = 0; + size_t decoded_height = 0; // Quantization. int qp = -1; // How many packets were discarded of the encoded frame data (if any). - int packets_dropped = 0; + size_t packets_dropped = 0; size_t total_packets = 0; size_t manipulated_length = 0; @@ -66,13 +75,10 @@ class Stats { FrameStatistic* AddFrame(); // Returns the FrameStatistic corresponding to |frame_number|. - FrameStatistic* GetFrame(int frame_number); + FrameStatistic* GetFrame(size_t frame_number); size_t size() const; - // TODO(brandtr): Add output as CSV. - void PrintSummary() const; - private: std::vector stats_; }; diff --git a/modules/video_coding/codecs/test/stats_unittest.cc b/modules/video_coding/codecs/test/stats_unittest.cc index 87727e2889..08fe56fc6b 100644 --- a/modules/video_coding/codecs/test/stats_unittest.cc +++ b/modules/video_coding/codecs/test/stats_unittest.cc @@ -15,28 +15,21 @@ namespace webrtc { namespace test { -TEST(StatsTest, TestEmptyObject) { - Stats stats; - stats.PrintSummary(); // Should not crash. -} - TEST(StatsTest, AddSingleFrame) { Stats stats; FrameStatistic* frame_stat = stats.AddFrame(); - EXPECT_EQ(0, frame_stat->frame_number); + EXPECT_EQ(0ull, frame_stat->frame_number); EXPECT_EQ(1u, stats.size()); } TEST(StatsTest, AddMultipleFrames) { Stats stats; - const int kNumFrames = 1000; - for (int i = 0; i < kNumFrames; ++i) { + const size_t kNumFrames = 1000; + for (size_t i = 0; i < kNumFrames; ++i) { FrameStatistic* frame_stat = stats.AddFrame(); EXPECT_EQ(i, frame_stat->frame_number); } - EXPECT_EQ(kNumFrames, static_cast(stats.size())); - - stats.PrintSummary(); // Should not crash. + EXPECT_EQ(kNumFrames, stats.size()); } } // namespace test diff --git a/modules/video_coding/codecs/test/test_config.cc b/modules/video_coding/codecs/test/test_config.cc index a289c619a5..0c2e7acf8c 100644 --- a/modules/video_coding/codecs/test/test_config.cc +++ b/modules/video_coding/codecs/test/test_config.cc @@ -27,34 +27,36 @@ std::string CodecSpecificToString(const webrtc::VideoCodec& codec) { std::stringstream ss; switch (codec.codecType) { case kVideoCodecVP8: - ss << "\n Complexity : " << codec.VP8().complexity; - ss << "\n Resilience : " << codec.VP8().resilience; - ss << "\n # temporal layers : " + ss << "\n Complexity : " << codec.VP8().complexity; + ss << "\n Resilience : " << codec.VP8().resilience; + ss << "\n # temporal layers : " << static_cast(codec.VP8().numberOfTemporalLayers); - ss << "\n Denoising : " << codec.VP8().denoisingOn; - ss << "\n Error concealment : " << codec.VP8().errorConcealmentOn; - ss << "\n Automatic resize : " << codec.VP8().automaticResizeOn; - ss << "\n Frame dropping : " << codec.VP8().frameDroppingOn; - ss << "\n Key frame interval: " << codec.VP8().keyFrameInterval; + ss << "\n Denoising : " << codec.VP8().denoisingOn; + ss << "\n Error concealment : " << codec.VP8().errorConcealmentOn; + ss << "\n Automatic resize : " << codec.VP8().automaticResizeOn; + ss << "\n Frame dropping : " << codec.VP8().frameDroppingOn; + ss << "\n Key frame interval : " << codec.VP8().keyFrameInterval; break; case kVideoCodecVP9: - ss << "\n Complexity : " << codec.VP9().complexity; - ss << "\n Resilience : " << codec.VP9().resilienceOn; - ss << "\n # temporal layers : " + ss << "\n Complexity : " << codec.VP9().complexity; + ss << "\n Resilience : " << codec.VP9().resilienceOn; + ss << "\n # temporal layers : " << static_cast(codec.VP9().numberOfTemporalLayers); - ss << "\n Denoising : " << codec.VP9().denoisingOn; - ss << "\n Frame dropping : " << codec.VP9().frameDroppingOn; - ss << "\n Key frame interval: " << codec.VP9().keyFrameInterval; - ss << "\n Adaptive QP mode : " << codec.VP9().adaptiveQpMode; - ss << "\n Automatic resize : " << codec.VP9().automaticResizeOn; - ss << "\n # spatial layers : " + ss << "\n # spatial layers : " << static_cast(codec.VP9().numberOfSpatialLayers); - ss << "\n Flexible mode : " << codec.VP9().flexibleMode; + ss << "\n Denoising : " << codec.VP9().denoisingOn; + ss << "\n Frame dropping : " << codec.VP9().frameDroppingOn; + ss << "\n Key frame interval : " << codec.VP9().keyFrameInterval; + ss << "\n Adaptive QP mode : " << codec.VP9().adaptiveQpMode; + ss << "\n Automatic resize : " << codec.VP9().automaticResizeOn; + ss << "\n # spatial layers : " + << static_cast(codec.VP9().numberOfSpatialLayers); + ss << "\n Flexible mode : " << codec.VP9().flexibleMode; break; case kVideoCodecH264: - ss << "\n Frame dropping : " << codec.H264().frameDroppingOn; - ss << "\n Key frame interval: " << codec.H264().keyFrameInterval; - ss << "\n Profile : " << codec.H264().profile; + ss << "\n Frame dropping : " << codec.H264().frameDroppingOn; + ss << "\n Key frame interval : " << codec.H264().keyFrameInterval; + ss << "\n Profile : " << codec.H264().profile; break; default: break; @@ -65,26 +67,27 @@ std::string CodecSpecificToString(const webrtc::VideoCodec& codec) { } // namespace void TestConfig::SetCodecSettings(VideoCodecType codec_type, - int num_temporal_layers, + size_t num_temporal_layers, bool error_concealment_on, bool denoising_on, bool frame_dropper_on, bool spatial_resize_on, bool resilience_on, - int width, - int height) { + size_t width, + size_t height) { webrtc::test::CodecSettings(codec_type, &codec_settings); // TODO(brandtr): Move the setting of |width| and |height| to the tests, and // DCHECK that they are set before initializing the codec instead. - codec_settings.width = width; - codec_settings.height = height; + codec_settings.width = static_cast(width); + codec_settings.height = static_cast(height); switch (codec_settings.codecType) { case kVideoCodecVP8: codec_settings.VP8()->resilience = resilience_on ? kResilientStream : kResilienceOff; - codec_settings.VP8()->numberOfTemporalLayers = num_temporal_layers; + codec_settings.VP8()->numberOfTemporalLayers = + static_cast(num_temporal_layers); codec_settings.VP8()->denoisingOn = denoising_on; codec_settings.VP8()->errorConcealmentOn = error_concealment_on; codec_settings.VP8()->automaticResizeOn = spatial_resize_on; @@ -93,7 +96,8 @@ void TestConfig::SetCodecSettings(VideoCodecType codec_type, break; case kVideoCodecVP9: codec_settings.VP9()->resilienceOn = resilience_on; - codec_settings.VP9()->numberOfTemporalLayers = num_temporal_layers; + codec_settings.VP9()->numberOfTemporalLayers = + static_cast(num_temporal_layers); codec_settings.VP9()->denoisingOn = denoising_on; codec_settings.VP9()->frameDroppingOn = frame_dropper_on; codec_settings.VP9()->keyFrameInterval = kBaseKeyFrameInterval; @@ -109,11 +113,11 @@ void TestConfig::SetCodecSettings(VideoCodecType codec_type, } } -int TestConfig::NumberOfCores() const { +size_t TestConfig::NumberOfCores() const { return use_single_core ? 1 : CpuInfo::DetectNumberOfCores(); } -int TestConfig::NumberOfTemporalLayers() const { +size_t TestConfig::NumberOfTemporalLayers() const { if (codec_settings.codecType == kVideoCodecVP8) { return codec_settings.VP8().numberOfTemporalLayers; } else if (codec_settings.codecType == kVideoCodecVP9) { @@ -123,8 +127,16 @@ int TestConfig::NumberOfTemporalLayers() const { } } -int TestConfig::TemporalLayerForFrame(int frame_idx) const { - int tl = -1; +size_t TestConfig::NumberOfSpatialLayers() const { + if (codec_settings.codecType == kVideoCodecVP9) { + return codec_settings.VP9().numberOfSpatialLayers; + } else { + return 1; + } +} + +size_t TestConfig::TemporalLayerForFrame(size_t frame_idx) const { + size_t tl = 0; switch (NumberOfTemporalLayers()) { case 1: tl = 0; @@ -153,7 +165,7 @@ int TestConfig::TemporalLayerForFrame(int frame_idx) const { return tl; } -std::vector TestConfig::FrameTypeForFrame(int frame_idx) const { +std::vector TestConfig::FrameTypeForFrame(size_t frame_idx) const { if (keyframe_interval > 0 && (frame_idx % keyframe_interval == 0)) { return {kVideoFrameKey}; } @@ -163,17 +175,19 @@ std::vector TestConfig::FrameTypeForFrame(int frame_idx) const { std::string TestConfig::ToString() const { std::string codec_type = CodecTypeToPayloadString(codec_settings.codecType); std::stringstream ss; - ss << "\n Filename : " << filename; - ss << "\n # CPU cores used : " << NumberOfCores(); + ss << "\n Filename : " << filename; + ss << "\n # CPU cores used : " << NumberOfCores(); ss << "\n General:"; - ss << "\n Codec type : " << codec_type; - ss << "\n Start bitrate : " << codec_settings.startBitrate << " kbps"; - ss << "\n Max bitrate : " << codec_settings.maxBitrate << " kbps"; - ss << "\n Min bitrate : " << codec_settings.minBitrate << " kbps"; - ss << "\n Width : " << codec_settings.width; - ss << "\n Height : " << codec_settings.height; - ss << "\n Max frame rate : " << codec_settings.maxFramerate; - ss << "\n QPmax : " << codec_settings.qpMax; + ss << "\n Codec type : " << codec_type; + ss << "\n Start bitrate : " << codec_settings.startBitrate << " kbps"; + ss << "\n Max bitrate : " << codec_settings.maxBitrate << " kbps"; + ss << "\n Min bitrate : " << codec_settings.minBitrate << " kbps"; + ss << "\n Width : " << codec_settings.width; + ss << "\n Height : " << codec_settings.height; + ss << "\n Max frame rate : " << codec_settings.maxFramerate; + ss << "\n QPmax : " << codec_settings.qpMax; + ss << "\n # simulcast streams : " + << static_cast(codec_settings.numberOfSimulcastStreams); ss << "\n " << codec_type << " specific: "; ss << CodecSpecificToString(codec_settings); return ss.str(); diff --git a/modules/video_coding/codecs/test/test_config.h b/modules/video_coding/codecs/test/test_config.h index 04a687878c..849967c35a 100644 --- a/modules/video_coding/codecs/test/test_config.h +++ b/modules/video_coding/codecs/test/test_config.h @@ -42,19 +42,20 @@ struct TestConfig { }; void SetCodecSettings(VideoCodecType codec_type, - int num_temporal_layers, + size_t num_temporal_layers, bool error_concealment_on, bool denoising_on, bool frame_dropper_on, bool spatial_resize_on, bool resilience_on, - int width, - int height); + size_t width, + size_t height); - int NumberOfCores() const; - int NumberOfTemporalLayers() const; - int TemporalLayerForFrame(int frame_idx) const; - std::vector FrameTypeForFrame(int frame_idx) const; + size_t NumberOfCores() const; + size_t NumberOfTemporalLayers() const; + size_t NumberOfSpatialLayers() const; + size_t TemporalLayerForFrame(size_t frame_idx) const; + std::vector FrameTypeForFrame(size_t frame_idx) const; std::string ToString() const; std::string CodecName() const; std::string FilenameWithParams() const; @@ -70,7 +71,7 @@ struct TestConfig { std::string output_filename; // Number of frames to process. - int num_frames = 0; + size_t num_frames = 0; // Configurations related to networking. NetworkingConfig networking_config; @@ -96,7 +97,7 @@ struct TestConfig { // to this setting. Forcing key frames may also affect encoder planning // optimizations in a negative way, since it will suddenly be forced to // produce an expensive key frame. - int keyframe_interval = 0; + size_t keyframe_interval = 0; // Codec settings to use. webrtc::VideoCodec codec_settings; @@ -118,6 +119,9 @@ struct TestConfig { // Custom checker that will be called for each frame. const EncodedFrameChecker* encoded_frame_checker = nullptr; + + // Print out frame level stats. + bool print_frame_level_stats = false; }; } // namespace test diff --git a/modules/video_coding/codecs/test/test_config_unittest.cc b/modules/video_coding/codecs/test/test_config_unittest.cc index 968e1f3993..966f3e9f5a 100644 --- a/modules/video_coding/codecs/test/test_config_unittest.cc +++ b/modules/video_coding/codecs/test/test_config_unittest.cc @@ -19,25 +19,25 @@ namespace webrtc { namespace test { namespace { -const int kNumTemporalLayers = 2; +const size_t kNumTemporalLayers = 2; } // namespace TEST(TestConfig, NumberOfCoresWithUseSingleCore) { TestConfig config; config.use_single_core = true; - EXPECT_EQ(1, config.NumberOfCores()); + EXPECT_EQ(1u, config.NumberOfCores()); } TEST(TestConfig, NumberOfCoresWithoutUseSingleCore) { TestConfig config; config.use_single_core = false; - EXPECT_GE(config.NumberOfCores(), 1); + EXPECT_GE(config.NumberOfCores(), 1u); } TEST(TestConfig, NumberOfTemporalLayersIsOne) { TestConfig config; webrtc::test::CodecSettings(kVideoCodecH264, &config.codec_settings); - EXPECT_EQ(1, config.NumberOfTemporalLayers()); + EXPECT_EQ(1u, config.NumberOfTemporalLayers()); } TEST(TestConfig, NumberOfTemporalLayers_Vp8) { @@ -58,33 +58,33 @@ TEST(TestConfig, TemporalLayersForFrame_OneLayer) { TestConfig config; webrtc::test::CodecSettings(kVideoCodecVP8, &config.codec_settings); config.codec_settings.VP8()->numberOfTemporalLayers = 1; - EXPECT_EQ(0, config.TemporalLayerForFrame(0)); - EXPECT_EQ(0, config.TemporalLayerForFrame(1)); - EXPECT_EQ(0, config.TemporalLayerForFrame(2)); + EXPECT_EQ(0u, config.TemporalLayerForFrame(0)); + EXPECT_EQ(0u, config.TemporalLayerForFrame(1)); + EXPECT_EQ(0u, config.TemporalLayerForFrame(2)); } TEST(TestConfig, TemporalLayersForFrame_TwoLayers) { TestConfig config; webrtc::test::CodecSettings(kVideoCodecVP8, &config.codec_settings); config.codec_settings.VP8()->numberOfTemporalLayers = 2; - EXPECT_EQ(0, config.TemporalLayerForFrame(0)); - EXPECT_EQ(1, config.TemporalLayerForFrame(1)); - EXPECT_EQ(0, config.TemporalLayerForFrame(2)); - EXPECT_EQ(1, config.TemporalLayerForFrame(3)); + EXPECT_EQ(0u, config.TemporalLayerForFrame(0)); + EXPECT_EQ(1u, config.TemporalLayerForFrame(1)); + EXPECT_EQ(0u, config.TemporalLayerForFrame(2)); + EXPECT_EQ(1u, config.TemporalLayerForFrame(3)); } TEST(TestConfig, TemporalLayersForFrame_ThreeLayers) { TestConfig config; webrtc::test::CodecSettings(kVideoCodecVP8, &config.codec_settings); config.codec_settings.VP8()->numberOfTemporalLayers = 3; - EXPECT_EQ(0, config.TemporalLayerForFrame(0)); - EXPECT_EQ(2, config.TemporalLayerForFrame(1)); - EXPECT_EQ(1, config.TemporalLayerForFrame(2)); - EXPECT_EQ(2, config.TemporalLayerForFrame(3)); - EXPECT_EQ(0, config.TemporalLayerForFrame(4)); - EXPECT_EQ(2, config.TemporalLayerForFrame(5)); - EXPECT_EQ(1, config.TemporalLayerForFrame(6)); - EXPECT_EQ(2, config.TemporalLayerForFrame(7)); + EXPECT_EQ(0u, config.TemporalLayerForFrame(0)); + EXPECT_EQ(2u, config.TemporalLayerForFrame(1)); + EXPECT_EQ(1u, config.TemporalLayerForFrame(2)); + EXPECT_EQ(2u, config.TemporalLayerForFrame(3)); + EXPECT_EQ(0u, config.TemporalLayerForFrame(4)); + EXPECT_EQ(2u, config.TemporalLayerForFrame(5)); + EXPECT_EQ(1u, config.TemporalLayerForFrame(6)); + EXPECT_EQ(2u, config.TemporalLayerForFrame(7)); } TEST(TestConfig, ForcedKeyFrameIntervalOff) { @@ -126,26 +126,27 @@ TEST(TestConfig, ToString_Vp8) { config.codec_settings.VP8()->keyFrameInterval = 999; EXPECT_EQ( - "\n Filename : yuvfile" - "\n # CPU cores used : 1" + "\n Filename : yuvfile" + "\n # CPU cores used : 1" "\n General:" - "\n Codec type : VP8" - "\n Start bitrate : 400 kbps" - "\n Max bitrate : 500 kbps" - "\n Min bitrate : 70 kbps" - "\n Width : 320" - "\n Height : 180" - "\n Max frame rate : 35" - "\n QPmax : 66" + "\n Codec type : VP8" + "\n Start bitrate : 400 kbps" + "\n Max bitrate : 500 kbps" + "\n Min bitrate : 70 kbps" + "\n Width : 320" + "\n Height : 180" + "\n Max frame rate : 35" + "\n QPmax : 66" + "\n # simulcast streams : 0" "\n VP8 specific: " - "\n Complexity : 0" - "\n Resilience : 0" - "\n # temporal layers : 2" - "\n Denoising : 0" - "\n Error concealment : 1" - "\n Automatic resize : 1" - "\n Frame dropping : 0" - "\n Key frame interval: 999\n", + "\n Complexity : 0" + "\n Resilience : 0" + "\n # temporal layers : 2" + "\n Denoising : 0" + "\n Error concealment : 1" + "\n Automatic resize : 1" + "\n Frame dropping : 0" + "\n Key frame interval : 999\n", config.ToString()); } diff --git a/modules/video_coding/codecs/test/videoprocessor.cc b/modules/video_coding/codecs/test/videoprocessor.cc index 06475e196d..f7d1fc9832 100644 --- a/modules/video_coding/codecs/test/videoprocessor.cc +++ b/modules/video_coding/codecs/test/videoprocessor.cc @@ -17,6 +17,7 @@ #include "api/video/i420_buffer.h" #include "common_types.h" // NOLINT(build/include) #include "common_video/h264/h264_common.h" +#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h" #include "modules/video_coding/include/video_codec_initializer.h" #include "modules/video_coding/utility/default_video_bitrate_allocator.h" @@ -29,8 +30,6 @@ namespace test { namespace { -const int kRtpClockRateHz = 90000; - std::unique_ptr CreateBitrateAllocator( TestConfig* config) { std::unique_ptr tl_factory; @@ -43,10 +42,10 @@ std::unique_ptr CreateBitrateAllocator( std::move(tl_factory))); } -rtc::Optional GetMaxNaluLength(const EncodedImage& encoded_frame, - const TestConfig& config) { +size_t GetMaxNaluSizeBytes(const EncodedImage& encoded_frame, + const TestConfig& config) { if (config.codec_settings.codecType != kVideoCodecH264) - return rtc::nullopt; + return 0; std::vector nalu_indices = webrtc::H264::FindNaluIndices(encoded_frame._buffer, @@ -54,11 +53,11 @@ rtc::Optional GetMaxNaluLength(const EncodedImage& encoded_frame, RTC_CHECK(!nalu_indices.empty()); - size_t max_length = 0; + size_t max_size = 0; for (const webrtc::H264::NaluIndex& index : nalu_indices) - max_length = std::max(max_length, index.payload_size); + max_size = std::max(max_size, index.payload_size); - return max_length; + return max_size; } int GetElapsedTimeMicroseconds(int64_t start_ns, int64_t stop_ns) { @@ -113,13 +112,14 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder, analysis_frame_reader_(analysis_frame_reader), encoded_frame_writer_(encoded_frame_writer), decoded_frame_writer_(decoded_frame_writer), - last_inputed_frame_num_(-1), - last_encoded_frame_num_(-1), - last_decoded_frame_num_(-1), + last_inputed_frame_num_(0), + last_encoded_frame_num_(0), + last_decoded_frame_num_(0), + num_encoded_frames_(0), + num_decoded_frames_(0), first_key_frame_has_been_excluded_(false), last_decoded_frame_buffer_(analysis_frame_reader->FrameLength()), - stats_(stats), - rate_update_index_(-1) { + stats_(stats) { RTC_DCHECK(encoder); RTC_DCHECK(decoder); RTC_DCHECK(packet_manipulator); @@ -134,12 +134,13 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder, // Initialize the encoder and decoder. RTC_CHECK_EQ( - encoder_->InitEncode(&config_.codec_settings, config_.NumberOfCores(), + encoder_->InitEncode(&config_.codec_settings, + static_cast(config_.NumberOfCores()), config_.networking_config.max_payload_size_in_bytes), WEBRTC_VIDEO_CODEC_OK); - RTC_CHECK_EQ( - decoder_->InitDecode(&config_.codec_settings, config_.NumberOfCores()), - WEBRTC_VIDEO_CODEC_OK); + RTC_CHECK_EQ(decoder_->InitDecode(&config_.codec_settings, + static_cast(config_.NumberOfCores())), + WEBRTC_VIDEO_CODEC_OK); } VideoProcessor::~VideoProcessor() { @@ -154,7 +155,7 @@ VideoProcessor::~VideoProcessor() { void VideoProcessor::ProcessFrame() { RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); - const int frame_number = ++last_inputed_frame_num_; + const size_t frame_number = last_inputed_frame_num_++; // Get frame from file. rtc::scoped_refptr buffer( @@ -163,18 +164,20 @@ void VideoProcessor::ProcessFrame() { // Use the frame number as the basis for timestamp to identify frames. Let the // first timestamp be non-zero, to not make the IvfFileWriter believe that we // want to use capture timestamps in the IVF files. - const uint32_t rtp_timestamp = (frame_number + 1) * kRtpClockRateHz / - config_.codec_settings.maxFramerate; + const size_t rtp_timestamp = (frame_number + 1) * kVideoPayloadTypeFrequency / + config_.codec_settings.maxFramerate; const int64_t render_time_ms = (frame_number + 1) * rtc::kNumMillisecsPerSec / config_.codec_settings.maxFramerate; rtp_timestamp_to_frame_num_[rtp_timestamp] = frame_number; - input_frames_[frame_number] = rtc::MakeUnique( - buffer, rtp_timestamp, render_time_ms, webrtc::kVideoRotation_0); + input_frames_[frame_number] = + rtc::MakeUnique(buffer, static_cast(rtp_timestamp), + render_time_ms, webrtc::kVideoRotation_0); std::vector frame_types = config_.FrameTypeForFrame(frame_number); // Create frame statistics object used for aggregation at end of test run. FrameStatistic* frame_stat = stats_->AddFrame(); + frame_stat->rtp_timestamp = rtp_timestamp; // For the highest measurement accuracy of the encode time, the start/stop // time recordings should wrap the Encode call as tightly as possible. @@ -183,27 +186,16 @@ void VideoProcessor::ProcessFrame() { encoder_->Encode(*input_frames_[frame_number], nullptr, &frame_types); } -void VideoProcessor::SetRates(int bitrate_kbps, int framerate_fps) { +void VideoProcessor::SetRates(size_t bitrate_kbps, size_t framerate_fps) { RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); - config_.codec_settings.maxFramerate = framerate_fps; - int set_rates_result = encoder_->SetRateAllocation( - bitrate_allocator_->GetAllocation(bitrate_kbps * 1000, framerate_fps), - framerate_fps); + config_.codec_settings.maxFramerate = static_cast(framerate_fps); + bitrate_allocation_ = bitrate_allocator_->GetAllocation( + static_cast(bitrate_kbps * 1000), + static_cast(framerate_fps)); + const int set_rates_result = encoder_->SetRateAllocation( + bitrate_allocation_, static_cast(framerate_fps)); RTC_DCHECK_GE(set_rates_result, 0) << "Failed to update encoder with new rate " << bitrate_kbps << "."; - ++rate_update_index_; - num_dropped_frames_.push_back(0); - num_spatial_resizes_.push_back(0); -} - -std::vector VideoProcessor::NumberDroppedFramesPerRateUpdate() const { - RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); - return num_dropped_frames_; -} - -std::vector VideoProcessor::NumberSpatialResizesPerRateUpdate() const { - RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_); - return num_spatial_resizes_; } void VideoProcessor::FrameEncoded(webrtc::VideoCodecType codec, @@ -218,20 +210,17 @@ void VideoProcessor::FrameEncoded(webrtc::VideoCodecType codec, config_.encoded_frame_checker->CheckEncodedFrame(codec, encoded_image); } - const int frame_number = + const size_t frame_number = rtp_timestamp_to_frame_num_[encoded_image._timeStamp]; // Ensure strict monotonicity. - RTC_CHECK_GT(frame_number, last_encoded_frame_num_); + if (num_encoded_frames_ > 0) { + RTC_CHECK_GT(frame_number, last_encoded_frame_num_); + } // Check for dropped frames. bool last_frame_missing = false; if (frame_number > 0) { - int num_dropped_from_last_encode = - frame_number - last_encoded_frame_num_ - 1; - RTC_DCHECK_GE(num_dropped_from_last_encode, 0); - RTC_CHECK_GE(rate_update_index_, 0); - num_dropped_frames_[rate_update_index_] += num_dropped_from_last_encode; const FrameStatistic* last_encoded_frame_stat = stats_->GetFrame(last_encoded_frame_num_); last_frame_missing = (last_encoded_frame_stat->manipulated_length == 0); @@ -245,13 +234,14 @@ void VideoProcessor::FrameEncoded(webrtc::VideoCodecType codec, frame_stat->encoding_successful = true; frame_stat->encoded_frame_size_bytes = encoded_image._length; frame_stat->frame_type = encoded_image._frameType; + frame_stat->temporal_layer_idx = config_.TemporalLayerForFrame(frame_number); frame_stat->qp = encoded_image.qp_; - frame_stat->bitrate_kbps = static_cast( - encoded_image._length * config_.codec_settings.maxFramerate * 8 / 1000); + frame_stat->target_bitrate_kbps = + bitrate_allocation_.GetSpatialLayerSum(0) / 1000; frame_stat->total_packets = encoded_image._length / config_.networking_config.packet_size_in_bytes + 1; - frame_stat->max_nalu_length = GetMaxNaluLength(encoded_image, config_); + frame_stat->max_nalu_size_bytes = GetMaxNaluSizeBytes(encoded_image, config_); // Make a raw copy of |encoded_image| to feed to the decoder. size_t copied_buffer_size = encoded_image._length + @@ -278,6 +268,8 @@ void VideoProcessor::FrameEncoded(webrtc::VideoCodecType codec, if (encoded_frame_writer_) { RTC_CHECK(encoded_frame_writer_->WriteFrame(encoded_image, codec)); } + + ++num_encoded_frames_; } void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame) { @@ -288,7 +280,7 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame) { int64_t decode_stop_ns = rtc::TimeNanos(); // Update frame statistics. - const int frame_number = + const size_t frame_number = rtp_timestamp_to_frame_num_[decoded_frame.timestamp()]; FrameStatistic* frame_stat = stats_->GetFrame(frame_number); frame_stat->decoded_width = decoded_frame.width(); @@ -298,26 +290,21 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame) { frame_stat->decoding_successful = true; // Ensure strict monotonicity. - RTC_CHECK_GT(frame_number, last_decoded_frame_num_); + if (num_decoded_frames_ > 0) { + RTC_CHECK_GT(frame_number, last_decoded_frame_num_); + } // Check if the codecs have resized the frame since previously decoded frame. if (frame_number > 0) { - if (decoded_frame_writer_ && last_decoded_frame_num_ >= 0) { + if (decoded_frame_writer_ && num_decoded_frames_ > 0) { // For dropped/lost frames, write out the last decoded frame to make it // look like a freeze at playback. - const int num_dropped_frames = frame_number - last_decoded_frame_num_; - for (int i = 0; i < num_dropped_frames; i++) { + const size_t num_dropped_frames = + frame_number - last_decoded_frame_num_ - 1; + for (size_t i = 0; i < num_dropped_frames; i++) { WriteDecodedFrameToFile(&last_decoded_frame_buffer_); } } - // TODO(ssilkin): move to FrameEncoded when webm:1474 is implemented. - const FrameStatistic* last_decoded_frame_stat = - stats_->GetFrame(last_decoded_frame_num_); - if (decoded_frame.width() != last_decoded_frame_stat->decoded_width || - decoded_frame.height() != last_decoded_frame_stat->decoded_height) { - RTC_CHECK_GE(rate_update_index_, 0); - ++num_spatial_resizes_[rate_update_index_]; - } } last_decoded_frame_num_ = frame_number; @@ -331,10 +318,8 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame) { // Delay erasing of input frames by one frame. The current frame might // still be needed for other simulcast stream or spatial layer. - const int frame_number_to_erase = frame_number - 1; - if (frame_number_to_erase >= 0) { - auto input_frame_erase_to = - input_frames_.lower_bound(frame_number_to_erase); + if (frame_number > 0) { + auto input_frame_erase_to = input_frames_.lower_bound(frame_number - 1); input_frames_.erase(input_frames_.begin(), input_frame_erase_to); } @@ -344,6 +329,8 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame) { &last_decoded_frame_buffer_); WriteDecodedFrameToFile(&last_decoded_frame_buffer_); } + + ++num_decoded_frames_; } void VideoProcessor::WriteDecodedFrameToFile(rtc::Buffer* buffer) { diff --git a/modules/video_coding/codecs/test/videoprocessor.h b/modules/video_coding/codecs/test/videoprocessor.h index 62a12ef871..190b6a3134 100644 --- a/modules/video_coding/codecs/test/videoprocessor.h +++ b/modules/video_coding/codecs/test/videoprocessor.h @@ -76,13 +76,7 @@ class VideoProcessor { void ProcessFrame(); // Updates the encoder with target rates. Must be called at least once. - void SetRates(int bitrate_kbps, int framerate_fps); - - // Returns the number of dropped frames. - std::vector NumberDroppedFramesPerRateUpdate() const; - - // Returns the number of spatial resizes. - std::vector NumberSpatialResizesPerRateUpdate() const; + void SetRates(size_t bitrate_kbps, size_t framerate_fps); private: class VideoProcessorEncodeCompleteCallback @@ -190,6 +184,7 @@ class VideoProcessor { webrtc::VideoEncoder* const encoder_; webrtc::VideoDecoder* const decoder_; const std::unique_ptr bitrate_allocator_; + BitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(sequence_checker_); // Adapters for the codec callbacks. VideoProcessorEncodeCompleteCallback encode_callback_; @@ -202,7 +197,7 @@ class VideoProcessor { // Async codecs might queue frames. To handle that we keep input frame // and release it after corresponding coded frame is decoded and quality // measurement is done. - std::map> input_frames_ + std::map> input_frames_ RTC_GUARDED_BY(sequence_checker_); // These (mandatory) file manipulators are used for, e.g., objective PSNR and @@ -217,13 +212,15 @@ class VideoProcessor { FrameWriter* const decoded_frame_writer_; // Keep track of inputed/encoded/decoded frames, so we can detect frame drops. - int last_inputed_frame_num_ RTC_GUARDED_BY(sequence_checker_); - int last_encoded_frame_num_ RTC_GUARDED_BY(sequence_checker_); - int last_decoded_frame_num_ RTC_GUARDED_BY(sequence_checker_); + size_t last_inputed_frame_num_ RTC_GUARDED_BY(sequence_checker_); + size_t last_encoded_frame_num_ RTC_GUARDED_BY(sequence_checker_); + size_t last_decoded_frame_num_ RTC_GUARDED_BY(sequence_checker_); + size_t num_encoded_frames_ RTC_GUARDED_BY(sequence_checker_); + size_t num_decoded_frames_ RTC_GUARDED_BY(sequence_checker_); // Store an RTP timestamp -> frame number map, since the timestamps are // based off of the frame rate, which can change mid-test. - std::map rtp_timestamp_to_frame_num_ + std::map rtp_timestamp_to_frame_num_ RTC_GUARDED_BY(sequence_checker_); // Keep track of if we have excluded the first key frame from packet loss. @@ -235,9 +232,6 @@ class VideoProcessor { // Statistics. Stats* stats_; - std::vector num_dropped_frames_ RTC_GUARDED_BY(sequence_checker_); - std::vector num_spatial_resizes_ RTC_GUARDED_BY(sequence_checker_); - int rate_update_index_ RTC_GUARDED_BY(sequence_checker_); rtc::SequencedTaskChecker sequence_checker_; diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc index dc9b146560..f9b0a72c62 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -37,6 +37,7 @@ #include "rtc_base/file.h" #include "rtc_base/ptr_util.h" #include "system_wrappers/include/sleep.h" +#include "test/statistics.h" #include "test/testsupport/fileutils.h" #include "test/testsupport/metrics/video_metrics.h" @@ -45,12 +46,9 @@ namespace test { namespace { -const int kMaxBitrateMismatchPercent = 20; +const int kRtpClockRateHz = 90000; -// Parameters from VP8 wrapper, which control target size of key frames. -const float kInitialBufferSize = 0.5f; -const float kOptimalBufferSize = 0.6f; -const float kScaleKeyFrameSize = 0.5f; +const int kMaxBitrateMismatchPercent = 20; bool RunEncodeInRealTime(const TestConfig& config) { if (config.measure_cpu) { @@ -173,118 +171,210 @@ VideoProcessorIntegrationTest::~VideoProcessorIntegrationTest() = default; void VideoProcessorIntegrationTest::ProcessFramesAndMaybeVerify( const std::vector& rate_profiles, const std::vector* rc_thresholds, - const QualityThresholds* quality_thresholds, + const std::vector* quality_thresholds, const BitstreamThresholds* bs_thresholds, const VisualizationParams* visualization_params) { RTC_DCHECK(!rate_profiles.empty()); // The Android HW codec needs to be run on a task queue, so we simply always // run the test on a task queue. rtc::TaskQueue task_queue("VidProc TQ"); - rtc::Event sync_event(false, false); - SetUpAndInitObjects(&task_queue, rate_profiles[0].target_kbps, - rate_profiles[0].input_fps, visualization_params); + SetUpAndInitObjects( + &task_queue, static_cast(rate_profiles[0].target_kbps), + static_cast(rate_profiles[0].input_fps), visualization_params); PrintSettings(); + ProcessAllFrames(&task_queue, rate_profiles); + + ReleaseAndCloseObjects(&task_queue); + + AnalyzeAllFrames(rate_profiles, rc_thresholds, quality_thresholds, + bs_thresholds); +} + +void VideoProcessorIntegrationTest::ProcessAllFrames( + rtc::TaskQueue* task_queue, + const std::vector& rate_profiles) { + // Process all frames. + size_t rate_update_index = 0; + // Set initial rates. - int rate_update_index = 0; - task_queue.PostTask([this, &rate_profiles, rate_update_index] { + task_queue->PostTask([this, &rate_profiles, rate_update_index] { processor_->SetRates(rate_profiles[rate_update_index].target_kbps, rate_profiles[rate_update_index].input_fps); }); cpu_process_time_->Start(); - // Process all frames. - int frame_number = 0; - const int num_frames = config_.num_frames; - RTC_DCHECK_GE(num_frames, 1); - while (frame_number < num_frames) { - if (RunEncodeInRealTime(config_)) { - // Roughly pace the frames. - SleepMs(rtc::kNumMillisecsPerSec / - rate_profiles[rate_update_index].input_fps); - } - - task_queue.PostTask([this] { processor_->ProcessFrame(); }); - ++frame_number; - + for (size_t frame_number = 0; frame_number < config_.num_frames; + ++frame_number) { if (frame_number == rate_profiles[rate_update_index].frame_index_rate_update) { ++rate_update_index; RTC_DCHECK_GT(rate_profiles.size(), rate_update_index); - task_queue.PostTask([this, &rate_profiles, rate_update_index] { + task_queue->PostTask([this, &rate_profiles, rate_update_index] { processor_->SetRates(rate_profiles[rate_update_index].target_kbps, rate_profiles[rate_update_index].input_fps); }); } + + task_queue->PostTask([this] { processor_->ProcessFrame(); }); + + if (RunEncodeInRealTime(config_)) { + // Roughly pace the frames. + size_t frame_duration_ms = + rtc::kNumMillisecsPerSec / rate_profiles[rate_update_index].input_fps; + SleepMs(static_cast(frame_duration_ms)); + } } + rtc::Event sync_event(false, false); + task_queue->PostTask([&sync_event] { sync_event.Set(); }); + sync_event.Wait(rtc::Event::kForever); + // Give the VideoProcessor pipeline some time to process the last frame, // and then release the codecs. if (config_.hw_encoder || config_.hw_decoder) { SleepMs(1 * rtc::kNumMillisecsPerSec); } + cpu_process_time_->Stop(); +} - std::vector num_dropped_frames; - std::vector num_spatial_resizes; - sync_event.Reset(); - task_queue.PostTask( - [this, &num_dropped_frames, &num_spatial_resizes, &sync_event]() { - num_dropped_frames = processor_->NumberDroppedFramesPerRateUpdate(); - num_spatial_resizes = processor_->NumberSpatialResizesPerRateUpdate(); - sync_event.Set(); - }); - sync_event.Wait(rtc::Event::kForever); +void VideoProcessorIntegrationTest::AnalyzeAllFrames( + const std::vector& rate_profiles, + const std::vector* rc_thresholds, + const std::vector* quality_thresholds, + const BitstreamThresholds* bs_thresholds) { + const bool is_svc = config_.NumberOfSpatialLayers() > 1; + const size_t number_of_simulcast_or_spatial_layers = + std::max(std::size_t{1}, + std::max(config_.NumberOfSpatialLayers(), + static_cast( + config_.codec_settings.numberOfSimulcastStreams))); + const size_t number_of_temporal_layers = config_.NumberOfTemporalLayers(); + printf("Rate control statistics\n==\n"); + for (size_t rate_update_index = 0; rate_update_index < rate_profiles.size(); + ++rate_update_index) { + const size_t first_frame_number = + (rate_update_index == 0) + ? 0 + : rate_profiles[rate_update_index - 1].frame_index_rate_update; + const size_t last_frame_number = + rate_profiles[rate_update_index].frame_index_rate_update - 1; + RTC_CHECK(last_frame_number >= first_frame_number); + const size_t number_of_frames = last_frame_number - first_frame_number + 1; + const float input_duration_sec = + 1.0 * number_of_frames / rate_profiles[rate_update_index].input_fps; - ReleaseAndCloseObjects(&task_queue); + std::vector overall_stats = + ExtractLayerStats(number_of_simulcast_or_spatial_layers - 1, + number_of_temporal_layers - 1, first_frame_number, + last_frame_number, true); - // Calculate and print rate control statistics. - rate_update_index = 0; - frame_number = 0; - quality_ = QualityMetrics(); - ResetRateControlMetrics(rate_update_index, rate_profiles); - while (frame_number < num_frames) { - UpdateRateControlMetrics(frame_number); + printf("Rate update #%zu:\n", rate_update_index); - if (quality_thresholds) { - UpdateQualityMetrics(frame_number); + const RateControlThresholds* rc_threshold = + rc_thresholds ? &(*rc_thresholds)[rate_update_index] : nullptr; + const QualityThresholds* quality_threshold = + quality_thresholds ? &(*quality_thresholds)[rate_update_index] + : nullptr; + AnalyzeAndPrintStats( + overall_stats, rate_profiles[rate_update_index].target_kbps, + rate_profiles[rate_update_index].input_fps, input_duration_sec, + rc_threshold, quality_threshold, bs_thresholds); + + if (config_.print_frame_level_stats) { + PrintFrameLevelStats(overall_stats); } - if (bs_thresholds) { - VerifyBitstream(frame_number, *bs_thresholds); - } + for (size_t spatial_layer_number = 0; + spatial_layer_number < number_of_simulcast_or_spatial_layers; + ++spatial_layer_number) { + for (size_t temporal_layer_number = 0; + temporal_layer_number < number_of_temporal_layers; + ++temporal_layer_number) { + std::vector layer_stats = + ExtractLayerStats(spatial_layer_number, temporal_layer_number, + first_frame_number, last_frame_number, is_svc); - ++frame_number; + const size_t target_bitrate_kbps = layer_stats[0].target_bitrate_kbps; + const float target_framerate_fps = + 1.0 * rate_profiles[rate_update_index].input_fps / + (1 << (number_of_temporal_layers - temporal_layer_number - 1)); - if (frame_number == - rate_profiles[rate_update_index].frame_index_rate_update) { - PrintRateControlMetrics(rate_update_index, num_dropped_frames, - num_spatial_resizes); - VerifyRateControlMetrics(rate_update_index, rc_thresholds, - num_dropped_frames, num_spatial_resizes); - ++rate_update_index; - ResetRateControlMetrics(rate_update_index, rate_profiles); + printf("Spatial %zu temporal %zu:\n", spatial_layer_number, + temporal_layer_number); + AnalyzeAndPrintStats(layer_stats, target_bitrate_kbps, + target_framerate_fps, input_duration_sec, nullptr, + nullptr, nullptr); + + if (config_.print_frame_level_stats) { + PrintFrameLevelStats(layer_stats); + } + } } } - PrintRateControlMetrics(rate_update_index, num_dropped_frames, - num_spatial_resizes); - VerifyRateControlMetrics(rate_update_index, rc_thresholds, num_dropped_frames, - num_spatial_resizes); - - if (quality_thresholds) { - VerifyQualityMetrics(*quality_thresholds); - } - - // Calculate and print other statistics. - EXPECT_EQ(num_frames, static_cast(stats_.size())); - stats_.PrintSummary(); cpu_process_time_->Print(); } +std::vector VideoProcessorIntegrationTest::ExtractLayerStats( + size_t target_spatial_layer_number, + size_t target_temporal_layer_number, + size_t first_frame_number, + size_t last_frame_number, + bool combine_layers_stats) { + size_t target_bitrate_kbps = 0; + std::vector layer_stats; + + for (size_t frame_number = first_frame_number; + frame_number <= last_frame_number; ++frame_number) { + // TODO(ssilkin): Add layering support + // FrameStatistic superframe_stat = + // *stats_[target_spatial_layer_number].GetFrame(frame_number); + FrameStatistic superframe_stat = *stats_.GetFrame(frame_number); + const size_t tl_idx = superframe_stat.temporal_layer_idx; + if (tl_idx <= target_temporal_layer_number) { + if (combine_layers_stats) { + for (size_t spatial_layer_number = 0; + spatial_layer_number < target_spatial_layer_number; + ++spatial_layer_number) { + // TODO(ssilkin): Add layering support + // const FrameStatistic* frame_stat = + // stats_[spatial_layer_number].GetFrame(frame_number); + const FrameStatistic* frame_stat = stats_.GetFrame(frame_number); + superframe_stat.encoded_frame_size_bytes += + frame_stat->encoded_frame_size_bytes; + superframe_stat.encode_time_us = std::max( + superframe_stat.encode_time_us, frame_stat->encode_time_us); + superframe_stat.decode_time_us = std::max( + superframe_stat.decode_time_us, frame_stat->decode_time_us); + } + } + + target_bitrate_kbps = + std::max(target_bitrate_kbps, superframe_stat.target_bitrate_kbps); + + if (superframe_stat.encoding_successful) { + RTC_CHECK(superframe_stat.target_bitrate_kbps <= target_bitrate_kbps || + tl_idx == target_temporal_layer_number); + RTC_CHECK(superframe_stat.target_bitrate_kbps == target_bitrate_kbps || + tl_idx < target_temporal_layer_number); + } + + layer_stats.push_back(superframe_stat); + } + } + + for (auto& frame_stat : layer_stats) { + frame_stat.target_bitrate_kbps = target_bitrate_kbps; + } + + return layer_stats; +} + void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() { std::unique_ptr encoder_factory; if (config_.hw_encoder) { @@ -436,139 +526,10 @@ void VideoProcessorIntegrationTest::ReleaseAndCloseObjects( } } -// For every encoded frame, update the rate control metrics. -void VideoProcessorIntegrationTest::UpdateRateControlMetrics(int frame_number) { - RTC_CHECK_GE(frame_number, 0); - - const int tl_idx = config_.TemporalLayerForFrame(frame_number); - ++actual_.num_frames_layer[tl_idx]; - ++actual_.num_frames; - - const FrameStatistic* frame_stat = stats_.GetFrame(frame_number); - FrameType frame_type = frame_stat->frame_type; - float framesize_kbits = frame_stat->encoded_frame_size_bytes * 8.0f / 1000.0f; - - // Update rate mismatch relative to per-frame bandwidth. - if (frame_type == kVideoFrameDelta) { - // TODO(marpan): Should we count dropped (zero size) frames in mismatch? - actual_.sum_delta_framesize_mismatch_layer[tl_idx] += - fabs(framesize_kbits - target_.framesize_kbits_layer[tl_idx]) / - target_.framesize_kbits_layer[tl_idx]; - } else { - float key_framesize_kbits = (frame_number == 0) - ? target_.key_framesize_kbits_initial - : target_.key_framesize_kbits; - actual_.sum_key_framesize_mismatch += - fabs(framesize_kbits - key_framesize_kbits) / key_framesize_kbits; - ++actual_.num_key_frames; - } - actual_.sum_framesize_kbits += framesize_kbits; - actual_.sum_framesize_kbits_layer[tl_idx] += framesize_kbits; - - // Encoded bitrate: from the start of the update/run to current frame. - actual_.kbps = actual_.sum_framesize_kbits * target_.fps / actual_.num_frames; - actual_.kbps_layer[tl_idx] = actual_.sum_framesize_kbits_layer[tl_idx] * - target_.fps_layer[tl_idx] / - actual_.num_frames_layer[tl_idx]; - - // Number of frames to hit target bitrate. - if (actual_.BitrateMismatchPercent(target_.kbps) < - kMaxBitrateMismatchPercent) { - actual_.num_frames_to_hit_target = - std::min(actual_.num_frames, actual_.num_frames_to_hit_target); - } -} - -// Verify expected behavior of rate control. -void VideoProcessorIntegrationTest::VerifyRateControlMetrics( - int rate_update_index, - const std::vector* rc_thresholds, - const std::vector& num_dropped_frames, - const std::vector& num_spatial_resizes) const { - if (!rc_thresholds) - return; - - const RateControlThresholds& rc_threshold = - (*rc_thresholds)[rate_update_index]; - - EXPECT_LE(num_dropped_frames[rate_update_index], - rc_threshold.max_num_dropped_frames); - EXPECT_EQ(rc_threshold.num_spatial_resizes, - num_spatial_resizes[rate_update_index]); - - EXPECT_LE(actual_.num_frames_to_hit_target, - rc_threshold.max_num_frames_to_hit_target); - EXPECT_EQ(rc_threshold.num_key_frames, actual_.num_key_frames); - EXPECT_LE(actual_.KeyFrameSizeMismatchPercent(), - rc_threshold.max_key_framesize_mismatch_percent); - EXPECT_LE(actual_.BitrateMismatchPercent(target_.kbps), - rc_threshold.max_bitrate_mismatch_percent); - - const int num_temporal_layers = config_.NumberOfTemporalLayers(); - for (int i = 0; i < num_temporal_layers; ++i) { - EXPECT_LE(actual_.DeltaFrameSizeMismatchPercent(i), - rc_threshold.max_delta_framesize_mismatch_percent); - EXPECT_LE(actual_.BitrateMismatchPercent(i, target_.kbps_layer[i]), - rc_threshold.max_bitrate_mismatch_percent); - } -} - -void VideoProcessorIntegrationTest::UpdateQualityMetrics(int frame_number) { - FrameStatistic* frame_stat = stats_.GetFrame(frame_number); - if (frame_stat->decoding_successful) { - ++quality_.num_decoded_frames; - quality_.total_psnr += frame_stat->psnr; - quality_.total_ssim += frame_stat->ssim; - if (frame_stat->psnr < quality_.min_psnr) - quality_.min_psnr = frame_stat->psnr; - if (frame_stat->ssim < quality_.min_ssim) - quality_.min_ssim = frame_stat->ssim; - } -} - -void VideoProcessorIntegrationTest::PrintRateControlMetrics( - int rate_update_index, - const std::vector& num_dropped_frames, - const std::vector& num_spatial_resizes) const { - if (rate_update_index == 0) { - printf("Rate control statistics\n==\n"); - } - - printf("Rate update #%d:\n", rate_update_index); - printf(" Target bitrate : %d\n", target_.kbps); - printf(" Encoded bitrate : %f\n", actual_.kbps); - printf(" Frame rate : %d\n", target_.fps); - printf(" # processed frames : %d\n", actual_.num_frames); - printf(" # frames to convergence : %d\n", actual_.num_frames_to_hit_target); - printf(" # dropped frames : %d\n", - num_dropped_frames[rate_update_index]); - printf(" # spatial resizes : %d\n", - num_spatial_resizes[rate_update_index]); - printf(" # key frames : %d\n", actual_.num_key_frames); - printf(" Key frame rate mismatch : %d\n", - actual_.KeyFrameSizeMismatchPercent()); - - const int num_temporal_layers = config_.NumberOfTemporalLayers(); - for (int i = 0; i < num_temporal_layers; ++i) { - printf(" Temporal layer #%d:\n", i); - printf(" TL%d target bitrate : %f\n", i, target_.kbps_layer[i]); - printf(" TL%d encoded bitrate : %f\n", i, actual_.kbps_layer[i]); - printf(" TL%d frame rate : %f\n", i, target_.fps_layer[i]); - printf(" TL%d # processed frames : %d\n", i, - actual_.num_frames_layer[i]); - printf(" TL%d frame size %% mismatch : %d\n", i, - actual_.DeltaFrameSizeMismatchPercent(i)); - printf(" TL%d bitrate %% mismatch : %d\n", i, - actual_.BitrateMismatchPercent(i, target_.kbps_layer[i])); - printf(" TL%d per-frame bitrate : %f\n", i, - target_.framesize_kbits_layer[i]); - } - printf("\n"); -} - void VideoProcessorIntegrationTest::PrintSettings() const { printf("VideoProcessor settings\n==\n"); - printf(" Total # of frames: %d", analysis_frame_reader_->NumberOfFrames()); + printf(" Total # of frames : %d", + analysis_frame_reader_->NumberOfFrames()); printf("%s\n", config_.ToString().c_str()); printf("VideoProcessorIntegrationTest settings\n==\n"); @@ -577,87 +538,192 @@ void VideoProcessorIntegrationTest::PrintSettings() const { const char* decoder_name = decoder_->ImplementationName(); printf(" Decoder implementation name: %s\n", decoder_name); if (strcmp(encoder_name, decoder_name) == 0) { - printf(" Codec implementation name : %s_%s\n", config_.CodecName().c_str(), + printf(" Codec implementation name : %s_%s\n", config_.CodecName().c_str(), encoder_name); } printf("\n"); } -void VideoProcessorIntegrationTest::VerifyBitstream( - int frame_number, - const BitstreamThresholds& bs_thresholds) { - RTC_CHECK_GE(frame_number, 0); - const FrameStatistic* frame_stat = stats_.GetFrame(frame_number); - EXPECT_LE(*(frame_stat->max_nalu_length), bs_thresholds.max_nalu_length); -} +void VideoProcessorIntegrationTest::AnalyzeAndPrintStats( + const std::vector& stats, + const float target_bitrate_kbps, + const float target_framerate_fps, + const float input_duration_sec, + const RateControlThresholds* rc_thresholds, + const QualityThresholds* quality_thresholds, + const BitstreamThresholds* bs_thresholds) { + const size_t num_input_frames = stats.size(); + size_t num_dropped_frames = 0; + size_t num_decoded_frames = 0; + size_t num_spatial_resizes = 0; + size_t num_key_frames = 0; + size_t max_nalu_size_bytes = 0; -void VideoProcessorIntegrationTest::VerifyQualityMetrics( - const QualityThresholds& quality_thresholds) { - EXPECT_GT(quality_.num_decoded_frames, 0); - EXPECT_GT(quality_.total_psnr / quality_.num_decoded_frames, - quality_thresholds.min_avg_psnr); - EXPECT_GT(quality_.min_psnr, quality_thresholds.min_min_psnr); - EXPECT_GT(quality_.total_ssim / quality_.num_decoded_frames, - quality_thresholds.min_avg_ssim); - EXPECT_GT(quality_.min_ssim, quality_thresholds.min_min_ssim); -} + size_t encoded_bytes = 0; + float buffer_level_kbits = 0.0; + float time_to_reach_target_bitrate_sec = -1.0; -// Reset quantities before each encoder rate update. -void VideoProcessorIntegrationTest::ResetRateControlMetrics( - int rate_update_index, - const std::vector& rate_profiles) { - RTC_DCHECK_GT(rate_profiles.size(), rate_update_index); - // Set new rates. - target_.kbps = rate_profiles[rate_update_index].target_kbps; - target_.fps = rate_profiles[rate_update_index].input_fps; - SetRatesPerTemporalLayer(); + Statistics buffer_level_sec; + Statistics key_frame_size_bytes; + Statistics delta_frame_size_bytes; - // Set key frame target sizes. - if (rate_update_index == 0) { - target_.key_framesize_kbits_initial = - 0.5 * kInitialBufferSize * target_.kbps_layer[0]; - } + Statistics encoding_time_us; + Statistics decoding_time_us; + Statistics psnr; + Statistics ssim; - // Set maximum size of key frames, following setting in the VP8 wrapper. - float max_key_size = kScaleKeyFrameSize * kOptimalBufferSize * target_.fps; - // We don't know exact target size of the key frames (except for first one), - // but the minimum in libvpx is ~|3 * per_frame_bandwidth| and maximum is - // set by |max_key_size_ * per_frame_bandwidth|. Take middle point/average - // as reference for mismatch. Note key frames always correspond to base - // layer frame in this test. - target_.key_framesize_kbits = - 0.5 * (3 + max_key_size) * target_.framesize_kbits_layer[0]; + Statistics qp; - // Reset rate control metrics. - actual_ = TestResults(); - actual_.num_frames_to_hit_target = // Set to max number of frames. - rate_profiles[rate_update_index].frame_index_rate_update; -} + FrameStatistic last_successfully_decoded_frame(0); + for (size_t frame_idx = 0; frame_idx < stats.size(); ++frame_idx) { + const FrameStatistic& frame_stat = stats[frame_idx]; -void VideoProcessorIntegrationTest::SetRatesPerTemporalLayer() { - const int num_temporal_layers = config_.NumberOfTemporalLayers(); - RTC_DCHECK_LE(num_temporal_layers, kMaxNumTemporalLayers); + const float time_since_first_input_sec = + frame_idx == 0 + ? 0.0 + : 1.0 * (frame_stat.rtp_timestamp - stats[0].rtp_timestamp) / + kRtpClockRateHz; + const float time_since_last_input_sec = + frame_idx == 0 ? 0.0 + : 1.0 * + (frame_stat.rtp_timestamp - + stats[frame_idx - 1].rtp_timestamp) / + kRtpClockRateHz; - for (int i = 0; i < num_temporal_layers; ++i) { - float bitrate_ratio; - if (i > 0) { - bitrate_ratio = kVp8LayerRateAlloction[num_temporal_layers - 1][i] - - kVp8LayerRateAlloction[num_temporal_layers - 1][i - 1]; + // Testing framework uses constant input framerate. This guarantees even + // sampling, which is important, of buffer level. + buffer_level_kbits -= time_since_last_input_sec * target_bitrate_kbps; + buffer_level_kbits = std::max(0.0f, buffer_level_kbits); + buffer_level_kbits += 8.0 * frame_stat.encoded_frame_size_bytes / 1000; + buffer_level_sec.AddSample(buffer_level_kbits / target_bitrate_kbps); + + encoded_bytes += frame_stat.encoded_frame_size_bytes; + if (frame_stat.encoded_frame_size_bytes == 0) { + ++num_dropped_frames; } else { - bitrate_ratio = kVp8LayerRateAlloction[num_temporal_layers - 1][i]; + if (frame_stat.frame_type == kVideoFrameKey) { + key_frame_size_bytes.AddSample(frame_stat.encoded_frame_size_bytes); + ++num_key_frames; + } else { + delta_frame_size_bytes.AddSample(frame_stat.encoded_frame_size_bytes); + } + + encoding_time_us.AddSample(frame_stat.encode_time_us); + qp.AddSample(frame_stat.qp); + + max_nalu_size_bytes = + std::max(max_nalu_size_bytes, frame_stat.max_nalu_size_bytes); + } + + if (frame_stat.decoding_successful) { + psnr.AddSample(frame_stat.psnr); + ssim.AddSample(frame_stat.ssim); + if (num_decoded_frames > 0) { + if (last_successfully_decoded_frame.decoded_width != + frame_stat.decoded_width || + last_successfully_decoded_frame.decoded_height != + frame_stat.decoded_height) { + ++num_spatial_resizes; + } + } + decoding_time_us.AddSample(frame_stat.decode_time_us); + last_successfully_decoded_frame = frame_stat; + ++num_decoded_frames; + } + + if (time_to_reach_target_bitrate_sec < 0 && frame_idx > 0) { + const float curr_bitrate_kbps = + (8.0 * encoded_bytes / 1000) / time_since_first_input_sec; + const float bitrate_mismatch_percent = + 100 * std::fabs(curr_bitrate_kbps - target_bitrate_kbps) / + target_bitrate_kbps; + if (bitrate_mismatch_percent < kMaxBitrateMismatchPercent) { + time_to_reach_target_bitrate_sec = time_since_first_input_sec; + } } - target_.kbps_layer[i] = target_.kbps * bitrate_ratio; - target_.fps_layer[i] = - target_.fps / static_cast(1 << (num_temporal_layers - 1)); - } - if (num_temporal_layers == 3) { - target_.fps_layer[2] = target_.fps / 2.0f; } - // Update layer per-frame-bandwidth. - for (int i = 0; i < num_temporal_layers; ++i) { - target_.framesize_kbits_layer[i] = - target_.kbps_layer[i] / target_.fps_layer[i]; + const float encoded_bitrate_kbps = + 8 * encoded_bytes / input_duration_sec / 1000; + const float bitrate_mismatch_percent = + 100 * std::fabs(encoded_bitrate_kbps - target_bitrate_kbps) / + target_bitrate_kbps; + const size_t num_encoded_frames = num_input_frames - num_dropped_frames; + const float encoded_framerate_fps = num_encoded_frames / input_duration_sec; + const float decoded_framerate_fps = num_decoded_frames / input_duration_sec; + const float framerate_mismatch_percent = + 100 * std::fabs(decoded_framerate_fps - target_framerate_fps) / + target_framerate_fps; + const float max_key_frame_delay_sec = + 8 * key_frame_size_bytes.Max() / 1000 / target_bitrate_kbps; + const float max_delta_frame_delay_sec = + 8 * delta_frame_size_bytes.Max() / 1000 / target_bitrate_kbps; + + printf("Target bitrate : %f kbps\n", target_bitrate_kbps); + printf("Encoded bitrate : %f kbps\n", encoded_bitrate_kbps); + printf("Bitrate mismatch : %f %%\n", bitrate_mismatch_percent); + printf("Time to reach target bitrate : %f sec\n", + time_to_reach_target_bitrate_sec); + printf("Target framerate : %f fps\n", target_framerate_fps); + printf("Encoding framerate : %f fps\n", encoded_framerate_fps); + printf("Decoding framerate : %f fps\n", decoded_framerate_fps); + printf("Frame encoding time : %f us\n", encoding_time_us.Mean()); + printf("Frame decoding time : %f us\n", decoding_time_us.Mean()); + printf("Framerate mismatch percent : %f %%\n", + framerate_mismatch_percent); + printf("Avg buffer level : %f sec\n", buffer_level_sec.Mean()); + printf("Max key frame delay : %f sec\n", max_key_frame_delay_sec); + printf("Max delta frame delay : %f sec\n", + max_delta_frame_delay_sec); + printf("Avg key frame size : %f bytes\n", + key_frame_size_bytes.Mean()); + printf("Avg delta frame size : %f bytes\n", + delta_frame_size_bytes.Mean()); + printf("Avg QP : %f\n", qp.Mean()); + printf("Avg PSNR : %f dB\n", psnr.Mean()); + printf("Min PSNR : %f dB\n", psnr.Min()); + printf("Avg SSIM : %f\n", ssim.Mean()); + printf("Min SSIM : %f\n", ssim.Min()); + printf("# input frames : %zu\n", num_input_frames); + printf("# encoded frames : %zu\n", num_encoded_frames); + printf("# decoded frames : %zu\n", num_decoded_frames); + printf("# dropped frames : %zu\n", num_dropped_frames); + printf("# key frames : %zu\n", num_key_frames); + printf("# encoded bytes : %zu\n", encoded_bytes); + printf("# spatial resizes : %zu\n", num_spatial_resizes); + + if (rc_thresholds) { + EXPECT_LE(bitrate_mismatch_percent, + rc_thresholds->max_avg_bitrate_mismatch_percent); + EXPECT_LE(time_to_reach_target_bitrate_sec, + rc_thresholds->max_time_to_reach_target_bitrate_sec); + EXPECT_LE(framerate_mismatch_percent, + rc_thresholds->max_avg_framerate_mismatch_percent); + EXPECT_LE(buffer_level_sec.Mean(), rc_thresholds->max_avg_buffer_level_sec); + EXPECT_LE(max_key_frame_delay_sec, + rc_thresholds->max_max_key_frame_delay_sec); + EXPECT_LE(max_delta_frame_delay_sec, + rc_thresholds->max_max_delta_frame_delay_sec); + EXPECT_LE(num_spatial_resizes, rc_thresholds->max_num_spatial_resizes); + EXPECT_LE(num_key_frames, rc_thresholds->max_num_key_frames); + } + + if (quality_thresholds) { + EXPECT_GT(psnr.Mean(), quality_thresholds->min_avg_psnr); + EXPECT_GT(psnr.Min(), quality_thresholds->min_min_psnr); + EXPECT_GT(ssim.Mean(), quality_thresholds->min_avg_ssim); + EXPECT_GT(ssim.Min(), quality_thresholds->min_min_ssim); + } + + if (bs_thresholds) { + EXPECT_LE(max_nalu_size_bytes, bs_thresholds->max_max_nalu_size_bytes); + } +} + +void VideoProcessorIntegrationTest::PrintFrameLevelStats( + const std::vector& stats) const { + for (auto& frame_stat : stats) { + printf("%s\n", frame_stat.ToString().c_str()); } } diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest.h b/modules/video_coding/codecs/test/videoprocessor_integrationtest.h index 10677da2bc..adcae54527 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest.h +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest.h @@ -36,35 +36,24 @@ namespace test { // Rates for the encoder and the frame number when to change profile. struct RateProfile { - int target_kbps; - int input_fps; - int frame_index_rate_update; + size_t target_kbps; + size_t input_fps; + size_t frame_index_rate_update; }; -// Thresholds for the rate control metrics. The thresholds are defined for each -// rate update sequence. |max_num_frames_to_hit_target| is defined as number of -// frames, after a rate update is made to the encoder, for the encoder to reach -// |kMaxBitrateMismatchPercent| of new target rate. struct RateControlThresholds { - int max_num_dropped_frames; - int max_key_framesize_mismatch_percent; - int max_delta_framesize_mismatch_percent; - int max_bitrate_mismatch_percent; - int max_num_frames_to_hit_target; - int num_spatial_resizes; - int num_key_frames; + double max_avg_bitrate_mismatch_percent; + double max_time_to_reach_target_bitrate_sec; + // TODO(ssilkin): Use absolute threshold for framerate. + double max_avg_framerate_mismatch_percent; + double max_avg_buffer_level_sec; + double max_max_key_frame_delay_sec; + double max_max_delta_frame_delay_sec; + size_t max_num_spatial_resizes; + size_t max_num_key_frames; }; -// Thresholds for the quality metrics. struct QualityThresholds { - QualityThresholds(double min_avg_psnr, - double min_min_psnr, - double min_avg_ssim, - double min_min_ssim) - : min_avg_psnr(min_avg_psnr), - min_min_psnr(min_min_psnr), - min_avg_ssim(min_avg_ssim), - min_min_ssim(min_min_ssim) {} double min_avg_psnr; double min_min_psnr; double min_avg_ssim; @@ -72,9 +61,7 @@ struct QualityThresholds { }; struct BitstreamThresholds { - explicit BitstreamThresholds(size_t max_nalu_length) - : max_nalu_length(max_nalu_length) {} - size_t max_nalu_length; + size_t max_max_nalu_size_bytes; }; // Should video files be saved persistently to disk for post-run visualization? @@ -83,15 +70,10 @@ struct VisualizationParams { bool save_decoded_y4m; }; -// Integration test for video processor. Encodes+decodes a clip and -// writes it to the output directory. After completion, quality metrics -// (PSNR and SSIM) and rate control metrics are computed and compared to given -// thresholds, to verify that the quality and encoder response is acceptable. -// The rate control tests allow us to verify the behavior for changing bit rate, -// changing frame rate, frame dropping/spatial resize, and temporal layers. -// The thresholds for the rate control metrics are set to be fairly -// conservative, so failure should only happen when some significant regression -// or breakdown occurs. +// Integration test for video processor. It does rate control and frame quality +// analysis using frame statistics collected by video processor and logs the +// results. If thresholds are specified it checks that corresponding metrics +// are in desirable range. class VideoProcessorIntegrationTest : public testing::Test { protected: // Verifies that all H.264 keyframes contain SPS/PPS/IDR NALUs. @@ -107,7 +89,7 @@ class VideoProcessorIntegrationTest : public testing::Test { void ProcessFramesAndMaybeVerify( const std::vector& rate_profiles, const std::vector* rc_thresholds, - const QualityThresholds* quality_thresholds, + const std::vector* quality_thresholds, const BitstreamThresholds* bs_thresholds, const VisualizationParams* visualization_params); @@ -119,54 +101,6 @@ class VideoProcessorIntegrationTest : public testing::Test { private: class CpuProcessTime; - static const int kMaxNumTemporalLayers = 3; - - struct TestResults { - int KeyFrameSizeMismatchPercent() const { - if (num_key_frames == 0) { - return -1; - } - return 100 * sum_key_framesize_mismatch / num_key_frames; - } - int DeltaFrameSizeMismatchPercent(int i) const { - return 100 * sum_delta_framesize_mismatch_layer[i] / num_frames_layer[i]; - } - int BitrateMismatchPercent(float target_kbps) const { - return 100 * std::fabs(kbps - target_kbps) / target_kbps; - } - int BitrateMismatchPercent(int i, float target_kbps_layer) const { - return 100 * std::fabs(kbps_layer[i] - target_kbps_layer) / - target_kbps_layer; - } - int num_frames = 0; - int num_frames_layer[kMaxNumTemporalLayers] = {0}; - int num_key_frames = 0; - int num_frames_to_hit_target = 0; - float sum_framesize_kbits = 0.0f; - float sum_framesize_kbits_layer[kMaxNumTemporalLayers] = {0}; - float kbps = 0.0f; - float kbps_layer[kMaxNumTemporalLayers] = {0}; - float sum_key_framesize_mismatch = 0.0f; - float sum_delta_framesize_mismatch_layer[kMaxNumTemporalLayers] = {0}; - }; - - struct TargetRates { - int kbps; - int fps; - float kbps_layer[kMaxNumTemporalLayers]; - float fps_layer[kMaxNumTemporalLayers]; - float framesize_kbits_layer[kMaxNumTemporalLayers]; - float key_framesize_kbits_initial; - float key_framesize_kbits; - }; - - struct QualityMetrics { - int num_decoded_frames = 0; - double total_psnr = 0.0; - double total_ssim = 0.0; - double min_psnr = std::numeric_limits::max(); - double min_ssim = std::numeric_limits::max(); - }; void CreateEncoderAndDecoder(); void DestroyEncoderAndDecoder(); @@ -176,26 +110,29 @@ class VideoProcessorIntegrationTest : public testing::Test { const VisualizationParams* visualization_params); void ReleaseAndCloseObjects(rtc::TaskQueue* task_queue); - // Rate control metrics. - void ResetRateControlMetrics(int rate_update_index, - const std::vector& rate_profiles); - void SetRatesPerTemporalLayer(); - void UpdateRateControlMetrics(int frame_number); - void PrintRateControlMetrics( - int rate_update_index, - const std::vector& num_dropped_frames, - const std::vector& num_spatial_resizes) const; - void VerifyRateControlMetrics( - int rate_update_index, + void ProcessAllFrames(rtc::TaskQueue* task_queue, + const std::vector& rate_profiles); + void AnalyzeAllFrames( + const std::vector& rate_profiles, const std::vector* rc_thresholds, - const std::vector& num_dropped_frames, - const std::vector& num_spatial_resizes) const; + const std::vector* quality_thresholds, + const BitstreamThresholds* bs_thresholds); - void VerifyBitstream(int frame_number, - const BitstreamThresholds& bs_thresholds); + std::vector ExtractLayerStats( + size_t target_spatial_layer_number, + size_t target_temporal_layer_number, + size_t first_frame_number, + size_t last_frame_number, + bool combine_layers); - void UpdateQualityMetrics(int frame_number); - void VerifyQualityMetrics(const QualityThresholds& quality_thresholds); + void AnalyzeAndPrintStats(const std::vector& stats, + float target_bitrate_kbps, + float target_framerate_fps, + float input_duration_sec, + const RateControlThresholds* rc_thresholds, + const QualityThresholds* quality_thresholds, + const BitstreamThresholds* bs_thresholds); + void PrintFrameLevelStats(const std::vector& stats) const; void PrintSettings() const; @@ -213,14 +150,6 @@ class VideoProcessorIntegrationTest : public testing::Test { Stats stats_; std::unique_ptr processor_; std::unique_ptr cpu_process_time_; - - // Quantities updated for every encoded frame. - TestResults actual_; - - // Rates set for every encoder rate update. - TargetRates target_; - - QualityMetrics quality_; }; } // namespace test diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc index ee4a8bb6ca..ce35ce5220 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc @@ -75,110 +75,96 @@ class VideoProcessorIntegrationTestLibvpx #if !defined(WEBRTC_IOS) #if !defined(RTC_DISABLE_VP9) -// VP9: Run with no packet loss and fixed bitrate. Quality should be very high. -// One key frame (first frame only) in sequence. -TEST_F(VideoProcessorIntegrationTestLibvpx, Process0PercentPacketLossVP9) { +TEST_F(VideoProcessorIntegrationTestLibvpx, HighBitrateVP9) { config_.SetCodecSettings(kVideoCodecVP9, 1, false, false, true, false, kResilienceOn, kCifWidth, kCifHeight); config_.num_frames = kNumFramesShort; - std::vector rate_profiles = {{500, 30, kNumFramesShort + 1}}; + std::vector rate_profiles = {{500, 30, kNumFramesShort}}; std::vector rc_thresholds = { - {0, 40, 20, 10, 20, 0, 1}}; + {5, 1, 0, 0.1, 0.3, 0.1, 0, 1}}; - QualityThresholds quality_thresholds(37.0, 36.0, 0.93, 0.92); + std::vector quality_thresholds = {{37, 36, 0.94, 0.92}}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, kNoVisualizationParams); } -// VP9: Run with no packet loss, with varying bitrate (3 rate updates): -// low to high to medium. Check that quality and encoder response to the new -// target rate/per-frame bandwidth (for each rate update) is within limits. -// One key frame (first frame only) in sequence. -TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessNoLossChangeBitRateVP9) { +TEST_F(VideoProcessorIntegrationTestLibvpx, ChangeBitrateVP9) { config_.SetCodecSettings(kVideoCodecVP9, 1, false, false, true, false, kResilienceOn, kCifWidth, kCifHeight); std::vector rate_profiles = { {200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update {700, 30, 200}, - {500, 30, kNumFramesLong + 1}}; + {500, 30, kNumFramesLong}}; - std::vector rc_thresholds = {{0, 35, 20, 20, 35, 0, 1}, - {2, 0, 20, 20, 60, 0, 0}, - {0, 0, 25, 20, 40, 0, 0}}; + std::vector rc_thresholds = { + {5, 1, 0, 0.15, 0.5, 0.1, 0, 1}, + {15, 2, 0, 0.2, 0.5, 0.1, 0, 0}, + {10, 1, 0, 0.3, 0.5, 0.1, 0, 0}}; - QualityThresholds quality_thresholds(35.5, 30.0, 0.90, 0.85); + std::vector quality_thresholds = { + {34, 33, 0.90, 0.88}, {38, 35, 0.95, 0.91}, {35, 34, 0.93, 0.90}}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, kNoVisualizationParams); } -// VP9: Run with no packet loss, with an update (decrease) in frame rate. -// Lower frame rate means higher per-frame-bandwidth, so easier to encode. -// At the low bitrate in this test, this means better rate control after the -// update(s) to lower frame rate. So expect less frame drops, and max values -// for the rate control metrics can be lower. One key frame (first frame only). -// Note: quality after update should be higher but we currently compute quality -// metrics averaged over whole sequence run. -TEST_F(VideoProcessorIntegrationTestLibvpx, - ProcessNoLossChangeFrameRateFrameDropVP9) { +TEST_F(VideoProcessorIntegrationTestLibvpx, ChangeFramerateVP9) { config_.SetCodecSettings(kVideoCodecVP9, 1, false, false, true, false, kResilienceOn, kCifWidth, kCifHeight); std::vector rate_profiles = { {100, 24, 100}, // target_kbps, input_fps, frame_index_rate_update {100, 15, 200}, - {100, 10, kNumFramesLong + 1}}; + {100, 10, kNumFramesLong}}; + // Framerate mismatch should be lower for lower framerate. std::vector rc_thresholds = { - {45, 50, 95, 15, 45, 0, 1}, - {20, 0, 50, 10, 30, 0, 0}, - {5, 0, 30, 5, 25, 0, 0}}; + {10, 2, 40, 0.4, 0.5, 0.2, 0, 1}, + {8, 2, 5, 0.2, 0.5, 0.2, 0, 0}, + {5, 2, 0, 0.2, 0.5, 0.3, 0, 0}}; - QualityThresholds quality_thresholds(31.5, 18.0, 0.80, 0.43); + // Quality should be higher for lower framerates for the same content. + std::vector quality_thresholds = { + {33, 32, 0.89, 0.87}, {33.5, 32, 0.90, 0.86}, {33.5, 31.5, 0.90, 0.85}}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, kNoVisualizationParams); } -// VP9: Run with no packet loss and denoiser on. One key frame (first frame). -TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessNoLossDenoiserOnVP9) { +TEST_F(VideoProcessorIntegrationTestLibvpx, DenoiserOnVP9) { config_.SetCodecSettings(kVideoCodecVP9, 1, false, true, true, false, kResilienceOn, kCifWidth, kCifHeight); config_.num_frames = kNumFramesShort; - std::vector rate_profiles = {{500, 30, kNumFramesShort + 1}}; + std::vector rate_profiles = {{500, 30, kNumFramesShort}}; std::vector rc_thresholds = { - {0, 40, 20, 10, 20, 0, 1}}; + {5, 1, 0, 0.1, 0.3, 0.1, 0, 1}}; - QualityThresholds quality_thresholds(36.8, 35.8, 0.92, 0.91); + std::vector quality_thresholds = {{37.5, 36, 0.94, 0.93}}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, kNoVisualizationParams); } -// Run with no packet loss, at low bitrate. -// spatial_resize is on, for this low bitrate expect one resize in sequence. -// Resize happens on delta frame. Expect only one key frame (first frame). -TEST_F(VideoProcessorIntegrationTestLibvpx, - DISABLED_ProcessNoLossSpatialResizeFrameDropVP9) { +TEST_F(VideoProcessorIntegrationTestLibvpx, VeryLowBitrateVP9) { config_.SetCodecSettings(kVideoCodecVP9, 1, false, false, true, true, kResilienceOn, kCifWidth, kCifHeight); - std::vector rate_profiles = {{50, 30, kNumFramesLong + 1}}; + std::vector rate_profiles = {{50, 30, kNumFramesLong}}; std::vector rc_thresholds = { - {228, 70, 160, 15, 80, 1, 1}}; + {15, 3, 75, 1.0, 0.5, 0.4, 1, 1}}; - QualityThresholds quality_thresholds(24.0, 13.0, 0.65, 0.37); + std::vector quality_thresholds = {{28, 25, 0.80, 0.65}}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, @@ -190,20 +176,20 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, #endif // !defined(RTC_DISABLE_VP9) -// VP8: Run with no packet loss and fixed bitrate. Quality should be very high. -// One key frame (first frame only) in sequence. Setting |key_frame_interval| -// to -1 below means no periodic key frames in test. -TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessZeroPacketLoss) { +TEST_F(VideoProcessorIntegrationTestLibvpx, HighBitrateVP8) { config_.SetCodecSettings(kVideoCodecVP8, 1, false, true, true, false, kResilienceOn, kCifWidth, kCifHeight); config_.num_frames = kNumFramesShort; - std::vector rate_profiles = {{500, 30, kNumFramesShort + 1}}; + std::vector rate_profiles = {{500, 30, kNumFramesShort}}; std::vector rc_thresholds = { - {0, 40, 20, 10, 15, 0, 1}}; + {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; - QualityThresholds quality_thresholds(34.95, 33.0, 0.90, 0.89); + // std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; + // TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse + // than quality of x86 version. Use lower thresholds for now. + std::vector quality_thresholds = {{35, 33, 0.91, 0.89}}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, @@ -221,10 +207,6 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessZeroPacketLoss) { // disabled on Android. Some quality parameter in the above test has been // adjusted to also pass for |cpu_speed| <= 12. -// VP8: Run with no packet loss, with varying bitrate (3 rate updates): -// low to high to medium. Check that quality and encoder response to the new -// target rate/per-frame bandwidth (for each rate update) is within limits. -// One key frame (first frame only) in sequence. // Too slow to finish before timeout on iOS. See webrtc:4755. #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) #define MAYBE_ProcessNoLossChangeBitRateVP8 \ @@ -232,34 +214,32 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessZeroPacketLoss) { #else #define MAYBE_ProcessNoLossChangeBitRateVP8 ProcessNoLossChangeBitRateVP8 #endif -TEST_F(VideoProcessorIntegrationTestLibvpx, - MAYBE_ProcessNoLossChangeBitRateVP8) { +TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeBitrateVP8) { config_.SetCodecSettings(kVideoCodecVP8, 1, false, true, true, false, kResilienceOn, kCifWidth, kCifHeight); std::vector rate_profiles = { {200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update {800, 30, 200}, - {500, 30, kNumFramesLong + 1}}; + {500, 30, kNumFramesLong}}; - std::vector rc_thresholds = {{0, 45, 20, 10, 15, 0, 1}, - {0, 0, 25, 20, 10, 0, 0}, - {0, 0, 25, 15, 10, 0, 0}}; + std::vector rc_thresholds = { + {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}, + {15, 1, 0, 0.1, 0.2, 0.1, 0, 0}, + {15, 1, 0, 0.3, 0.2, 0.1, 0, 0}}; - QualityThresholds quality_thresholds(34.0, 32.0, 0.85, 0.80); + // std::vector quality_thresholds = { + // {33, 32, 0.89, 0.88}, {38, 36, 0.94, 0.93}, {35, 34, 0.92, 0.91}}; + // TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse + // than quality of x86 version. Use lower thresholds for now. + std::vector quality_thresholds = { + {31.8, 31, 0.86, 0.85}, {36, 34.8, 0.92, 0.90}, {33.5, 32, 0.90, 0.88}}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, kNoVisualizationParams); } -// VP8: Run with no packet loss, with an update (decrease) in frame rate. -// Lower frame rate means higher per-frame-bandwidth, so easier to encode. -// At the bitrate in this test, this means better rate control after the -// update(s) to lower frame rate. So expect less frame drops, and max values -// for the rate control metrics can be lower. One key frame (first frame only). -// Note: quality after update should be higher but we currently compute quality -// metrics averaged over whole sequence run. // Too slow to finish before timeout on iOS. See webrtc:4755. #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) #define MAYBE_ProcessNoLossChangeFrameRateFrameDropVP8 \ @@ -268,33 +248,38 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, #define MAYBE_ProcessNoLossChangeFrameRateFrameDropVP8 \ ProcessNoLossChangeFrameRateFrameDropVP8 #endif -TEST_F(VideoProcessorIntegrationTestLibvpx, - MAYBE_ProcessNoLossChangeFrameRateFrameDropVP8) { +TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeFramerateVP8) { config_.SetCodecSettings(kVideoCodecVP8, 1, false, true, true, false, kResilienceOn, kCifWidth, kCifHeight); std::vector rate_profiles = { {80, 24, 100}, // target_kbps, input_fps, frame_index_rate_update {80, 15, 200}, - {80, 10, kNumFramesLong + 1}}; + {80, 10, kNumFramesLong}}; + // std::vector rc_thresholds = { + // {10, 2, 20, 0.4, 0.3, 0.1, 0, 1}, + // {5, 2, 5, 0.3, 0.3, 0.1, 0, 0}, + // {4, 2, 1, 0.2, 0.3, 0.2, 0, 0}}; + // TODO(webrtc:8757): AMR VP8 drops more frames than x86 version. Use lower + // thresholds for now. std::vector rc_thresholds = { - {40, 20, 75, 15, 60, 0, 1}, - {10, 0, 25, 10, 35, 0, 0}, - {0, 0, 20, 10, 15, 0, 0}}; + {10, 2, 60, 0.5, 0.3, 0.3, 0, 1}, + {10, 2, 30, 0.3, 0.3, 0.3, 0, 0}, + {10, 2, 10, 0.2, 0.3, 0.2, 0, 0}}; - QualityThresholds quality_thresholds(31.0, 22.0, 0.80, 0.65); + // std::vector quality_thresholds = { + // {31, 30, 0.87, 0.86}, {32, 31, 0.89, 0.86}, {32, 30, 0.87, 0.82}}; + // TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse + // than quality of x86 version. Use lower thresholds for now. + std::vector quality_thresholds = { + {31, 30, 0.85, 0.84}, {31.5, 30.5, 0.86, 0.84}, {30.5, 29, 0.83, 0.78}}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, kNoVisualizationParams); } -// VP8: Run with no packet loss, with 3 temporal layers, with a rate update in -// the middle of the sequence. The max values for the frame size mismatch and -// encoding rate mismatch are applied to each layer. -// No dropped frames in this test, and internal spatial resizer is off. -// One key frame (first frame only) in sequence, so no spatial resizing. // Too slow to finish before timeout on iOS. See webrtc:4755. #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) #define MAYBE_ProcessNoLossTemporalLayersVP8 \ @@ -302,18 +287,27 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, #else #define MAYBE_ProcessNoLossTemporalLayersVP8 ProcessNoLossTemporalLayersVP8 #endif -TEST_F(VideoProcessorIntegrationTestLibvpx, - MAYBE_ProcessNoLossTemporalLayersVP8) { +TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_TemporalLayersVP8) { config_.SetCodecSettings(kVideoCodecVP8, 3, false, true, true, false, kResilienceOn, kCifWidth, kCifHeight); std::vector rate_profiles = {{200, 30, 150}, - {400, 30, kNumFramesLong + 1}}; + {400, 30, kNumFramesLong}}; - std::vector rc_thresholds = {{0, 20, 30, 10, 10, 0, 1}, - {0, 0, 30, 15, 10, 0, 0}}; + // std::vector rc_thresholds = { + // {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}, {10, 2, 0, 0.1, 0.2, 0.1, 0, 1}}; + // TODO(webrtc:8757): AMR VP8 drops more frames than x86 version. Use lower + // thresholds for now. + std::vector rc_thresholds = { + {10, 1, 2, 0.3, 0.2, 0.1, 0, 1}, {12, 2, 3, 0.1, 0.2, 0.1, 0, 1}}; - QualityThresholds quality_thresholds(32.5, 30.0, 0.85, 0.80); + // Min SSIM drops because of high motion scene with complex backgound (trees). + // std::vector quality_thresholds = {{32, 30, 0.88, 0.85}, + // {33, 30, 0.89, 0.83}}; + // TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse + // than quality of x86 version. Use lower thresholds for now. + std::vector quality_thresholds = {{31, 30, 0.85, 0.84}, + {31, 28, 0.85, 0.75}}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc index 7f6b40b0b9..936049c0b7 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc @@ -44,15 +44,15 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsVp8) { config_.SetCodecSettings(kVideoCodecVP8, 1, false, false, false, false, false, 352, 288); - std::vector rate_profiles = {{500, 30, kForemanNumFrames + 1}}; + std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; // The thresholds below may have to be tweaked to let even poor MediaCodec // implementations pass. If this test fails on the bots, disable it and // ping brandtr@. std::vector rc_thresholds = { - {20, 95, 22, 11, 50, 0, 1}}; + {10, 1, 1, 0.1, 0.2, 0.1, 0, 1}}; - QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39); + std::vector quality_thresholds = {{36, 31, 0.92, 0.86}}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, @@ -64,15 +64,15 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsH264CBP) { config_.SetCodecSettings(kVideoCodecH264, 1, false, false, false, false, false, 352, 288); - std::vector rate_profiles = {{500, 30, kForemanNumFrames + 1}}; + std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; // The thresholds below may have to be tweaked to let even poor MediaCodec // implementations pass. If this test fails on the bots, disable it and // ping brandtr@. std::vector rc_thresholds = { - {20, 95, 22, 11, 20, 0, 1}}; + {10, 1, 1, 0.1, 0.2, 0.1, 0, 1}}; - QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39); + std::vector quality_thresholds = {{36, 31, 0.92, 0.86}}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, @@ -90,14 +90,15 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, config_.SetCodecSettings(kVideoCodecH264, 1, false, false, false, false, false, 352, 288); - std::vector rate_profiles = {{500, 30, kForemanNumFrames + 1}}; + std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; // The thresholds below may have to be tweaked to let even poor MediaCodec // implementations pass. If this test fails on the bots, disable it and // ping brandtr@. - std::vector rc_thresholds = {{5, 60, 20, 5, 15, 0, 1}}; + std::vector rc_thresholds = { + {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; - QualityThresholds quality_thresholds(33.0, 30.0, 0.90, 0.85); + std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc index 9967fbddfa..45a1fb0a91 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc @@ -49,21 +49,16 @@ class VideoProcessorIntegrationTestOpenH264 } }; -// H264: Run with no packet loss and fixed bitrate. Quality should be very high. -// Note(hbos): The PacketManipulatorImpl code used to simulate packet loss in -// these unittests appears to drop "packets" in a way that is not compatible -// with H264. Therefore ProcessXPercentPacketLossH264, X != 0, unittests have -// not been added. -TEST_F(VideoProcessorIntegrationTestOpenH264, Process0PercentPacketLoss) { +TEST_F(VideoProcessorIntegrationTestOpenH264, ConstantHighBitrate) { config_.SetCodecSettings(kVideoCodecH264, 1, false, false, true, false, kResilienceOn, kCifWidth, kCifHeight); - std::vector rate_profiles = {{500, 30, kNumFrames + 1}}; + std::vector rate_profiles = {{500, 30, kNumFrames}}; std::vector rc_thresholds = { - {2, 60, 20, 10, 20, 0, 1}}; + {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; - QualityThresholds quality_thresholds(35.0, 25.0, 0.93, 0.70); + std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, @@ -72,22 +67,22 @@ TEST_F(VideoProcessorIntegrationTestOpenH264, Process0PercentPacketLoss) { // H264: Enable SingleNalUnit packetization mode. Encoder should split // large frames into multiple slices and limit length of NAL units. -TEST_F(VideoProcessorIntegrationTestOpenH264, ProcessNoLossSingleNalUnit) { +TEST_F(VideoProcessorIntegrationTestOpenH264, SingleNalUnit) { config_.h264_codec_settings.packetization_mode = H264PacketizationMode::SingleNalUnit; config_.networking_config.max_payload_size_in_bytes = 500; config_.SetCodecSettings(kVideoCodecH264, 1, false, false, true, false, kResilienceOn, kCifWidth, kCifHeight); - std::vector rate_profiles = {{500, 30, kNumFrames + 1}}; + std::vector rate_profiles = {{500, 30, kNumFrames}}; std::vector rc_thresholds = { - {2, 60, 30, 10, 20, 0, 1}}; + {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; - QualityThresholds quality_thresholds(35.0, 25.0, 0.93, 0.70); + std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; - BitstreamThresholds bs_thresholds( - config_.networking_config.max_payload_size_in_bytes); + BitstreamThresholds bs_thresholds = { + config_.networking_config.max_payload_size_in_bytes}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, &bs_thresholds, diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc index b058e2992a..550479e761 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc @@ -18,13 +18,13 @@ namespace test { namespace { // Loop variables. -const int kBitrates[] = {500}; +const size_t kBitrates[] = {500}; const VideoCodecType kVideoCodecType[] = {kVideoCodecVP8}; const bool kHwCodec[] = {false}; // Codec settings. -const bool kResilienceOn = false; const int kNumTemporalLayers = 1; +const bool kResilienceOn = kNumTemporalLayers > 1; const bool kDenoisingOn = false; const bool kErrorConcealmentOn = false; const bool kSpatialResizeOn = false; @@ -46,7 +46,7 @@ const int kNumFrames = 30; class VideoProcessorIntegrationTestParameterized : public VideoProcessorIntegrationTest, public ::testing::WithParamInterface< - ::testing::tuple> { + ::testing::tuple> { protected: VideoProcessorIntegrationTestParameterized() : bitrate_(::testing::get<0>(GetParam())), @@ -54,9 +54,9 @@ class VideoProcessorIntegrationTestParameterized hw_codec_(::testing::get<2>(GetParam())) {} ~VideoProcessorIntegrationTestParameterized() override = default; - void RunTest(int width, - int height, - int framerate, + void RunTest(size_t width, + size_t height, + size_t framerate, const std::string& filename) { config_.filename = filename; config_.input_filename = ResourcePath(filename, "yuv"); @@ -72,13 +72,13 @@ class VideoProcessorIntegrationTestParameterized kSpatialResizeOn, kResilienceOn, width, height); std::vector rate_profiles = { - {bitrate_, framerate, kNumFrames + 1}}; + {bitrate_, framerate, kNumFrames}}; ProcessFramesAndMaybeVerify(rate_profiles, nullptr, nullptr, nullptr, &kVisualizationParams); } - const int bitrate_; + const size_t bitrate_; const VideoCodecType codec_type_; const bool hw_codec_; }; diff --git a/modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc b/modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc index 2e040a4964..adcdab5f6b 100644 --- a/modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc +++ b/modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc @@ -48,12 +48,12 @@ TEST_F(VideoProcessorIntegrationTestVideoToolbox, config_.SetCodecSettings(kVideoCodecH264, 1, false, false, false, false, false, 352, 288); - std::vector rate_profiles = {{500, 30, kForemanNumFrames + 1}}; + std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; std::vector rc_thresholds = { - {20, 95, 60, 60, 10, 0, 1}}; + {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; - QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39); + std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, @@ -68,11 +68,12 @@ TEST_F(VideoProcessorIntegrationTestVideoToolbox, config_.SetCodecSettings(kVideoCodecH264, 1, false, false, false, false, false, 352, 288); - std::vector rate_profiles = {{500, 30, kForemanNumFrames + 1}}; + std::vector rate_profiles = {{500, 30, kForemanNumFrames}}; - std::vector rc_thresholds = {{5, 75, 65, 60, 6, 0, 1}}; + std::vector rc_thresholds = { + {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}}; - QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39); + std::vector quality_thresholds = {{37, 35, 0.93, 0.91}}; ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr, diff --git a/modules/video_coding/codecs/test/videoprocessor_unittest.cc b/modules/video_coding/codecs/test/videoprocessor_unittest.cc index 1a51df223d..f8caa05e19 100644 --- a/modules/video_coding/codecs/test/videoprocessor_unittest.cc +++ b/modules/video_coding/codecs/test/videoprocessor_unittest.cc @@ -151,10 +151,6 @@ TEST_F(VideoProcessorTest, SetRates) { kFramerateFps)) .Times(1); video_processor_->SetRates(kBitrateKbps, kFramerateFps); - EXPECT_THAT(video_processor_->NumberDroppedFramesPerRateUpdate(), - ElementsAre(0)); - EXPECT_THAT(video_processor_->NumberSpatialResizesPerRateUpdate(), - ElementsAre(0)); const int kNewBitrateKbps = 456; const int kNewFramerateFps = 34; @@ -164,10 +160,6 @@ TEST_F(VideoProcessorTest, SetRates) { kNewFramerateFps)) .Times(1); video_processor_->SetRates(kNewBitrateKbps, kNewFramerateFps); - EXPECT_THAT(video_processor_->NumberDroppedFramesPerRateUpdate(), - ElementsAre(0, 0)); - EXPECT_THAT(video_processor_->NumberSpatialResizesPerRateUpdate(), - ElementsAre(0, 0)); ExpectRelease(); } diff --git a/test/statistics.cc b/test/statistics.cc index c43dde96fa..192366aef8 100644 --- a/test/statistics.cc +++ b/test/statistics.cc @@ -11,23 +11,40 @@ #include +#include + namespace webrtc { namespace test { -Statistics::Statistics() : sum_(0.0), sum_squared_(0.0), count_(0) {} +Statistics::Statistics() + : sum_(0.0), + sum_squared_(0.0), + max_(std::numeric_limits::min()), + min_(std::numeric_limits::max()), + count_(0) {} void Statistics::AddSample(double sample) { sum_ += sample; sum_squared_ += sample * sample; + max_ = std::max(max_, sample); + min_ = std::min(min_, sample); ++count_; } +double Statistics::Max() const { + return max_; +} + double Statistics::Mean() const { if (count_ == 0) return 0.0; return sum_ / count_; } +double Statistics::Min() const { + return min_; +} + double Statistics::Variance() const { if (count_ == 0) return 0.0; diff --git a/test/statistics.h b/test/statistics.h index d52d92dc11..0389fd1232 100644 --- a/test/statistics.h +++ b/test/statistics.h @@ -21,13 +21,17 @@ class Statistics { void AddSample(double sample); + double Max() const; double Mean() const; + double Min() const; double Variance() const; double StandardDeviation() const; private: double sum_; double sum_squared_; + double max_; + double min_; uint64_t count_; }; } // namespace test