diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn index 759b9410cf..70e9a13f61 100644 --- a/webrtc/modules/BUILD.gn +++ b/webrtc/modules/BUILD.gn @@ -465,6 +465,7 @@ if (rtc_include_tests) { "video_coding/utility/frame_dropper_unittest.cc", "video_coding/utility/h264_bitstream_parser_unittest.cc", "video_coding/utility/ivf_file_writer_unittest.cc", + "video_coding/utility/moving_average_unittest.cc", "video_coding/utility/quality_scaler_unittest.cc", "video_coding/utility/simulcast_rate_allocator_unittest.cc", "video_coding/video_coding_robustness_unittest.cc", diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp index f829233e06..c40fafcbd3 100644 --- a/webrtc/modules/modules.gyp +++ b/webrtc/modules/modules.gyp @@ -389,6 +389,7 @@ 'video_coding/utility/frame_dropper_unittest.cc', 'video_coding/utility/h264_bitstream_parser_unittest.cc', 'video_coding/utility/ivf_file_writer_unittest.cc', + 'video_coding/utility/moving_average_unittest.cc', 'video_coding/utility/quality_scaler_unittest.cc', 'video_coding/utility/simulcast_rate_allocator_unittest.cc', 'video_coding/video_coding_robustness_unittest.cc', diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn index b73b6e703f..979355d229 100644 --- a/webrtc/modules/video_coding/BUILD.gn +++ b/webrtc/modules/video_coding/BUILD.gn @@ -103,6 +103,7 @@ rtc_source_set("video_coding_utility") { "utility/h264_bitstream_parser.h", "utility/ivf_file_writer.cc", "utility/ivf_file_writer.h", + "utility/moving_average.cc", "utility/moving_average.h", "utility/qp_parser.cc", "utility/qp_parser.h", diff --git a/webrtc/modules/video_coding/utility/moving_average.cc b/webrtc/modules/video_coding/utility/moving_average.cc new file mode 100644 index 0000000000..34e7bb60e8 --- /dev/null +++ b/webrtc/modules/video_coding/utility/moving_average.cc @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/video_coding/utility/moving_average.h" + +#include + +namespace webrtc { + +MovingAverage::MovingAverage(size_t s) : sum_history_(s + 1, 0) {} + +void MovingAverage::AddSample(int sample) { + count_++; + sum_ += sample; + sum_history_[count_ % sum_history_.size()] = sum_; +} + +rtc::Optional MovingAverage::GetAverage() const { + return GetAverage(size()); +} + +rtc::Optional MovingAverage::GetAverage(size_t num_samples) const { + if (num_samples > size() || num_samples == 0) + return rtc::Optional(); + int sum = sum_ - sum_history_[(count_ - num_samples) % sum_history_.size()]; + return rtc::Optional(sum / static_cast(num_samples)); +} + +void MovingAverage::Reset() { + count_ = 0; + sum_ = 0; + std::fill(sum_history_.begin(), sum_history_.end(), 0); +} + +size_t MovingAverage::size() const { + return std::min(count_, sum_history_.size() - 1); +} + +} // namespace webrtc diff --git a/webrtc/modules/video_coding/utility/moving_average.h b/webrtc/modules/video_coding/utility/moving_average.h index cdad50f0f9..dd42385038 100644 --- a/webrtc/modules/video_coding/utility/moving_average.h +++ b/webrtc/modules/video_coding/utility/moving_average.h @@ -11,63 +11,25 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_MOVING_AVERAGE_H_ #define WEBRTC_MODULES_VIDEO_CODING_UTILITY_MOVING_AVERAGE_H_ -#include +#include -#include - -#include "webrtc/typedefs.h" +#include "webrtc/base/optional.h" namespace webrtc { -template class MovingAverage { public: - MovingAverage(); - void AddSample(T sample); - bool GetAverage(size_t num_samples, T* average); + explicit MovingAverage(size_t s); + void AddSample(int sample); + rtc::Optional GetAverage() const; + rtc::Optional GetAverage(size_t num_samples) const; void Reset(); - int size(); + size_t size() const; private: - T sum_; - std::list samples_; + size_t count_ = 0; + int sum_ = 0; + std::vector sum_history_; }; - -template -MovingAverage::MovingAverage() - : sum_(static_cast(0)) {} - -template -void MovingAverage::AddSample(T sample) { - samples_.push_back(sample); - sum_ += sample; -} - -template -bool MovingAverage::GetAverage(size_t num_samples, T* avg) { - if (num_samples > samples_.size()) - return false; - - // Remove old samples. - while (num_samples < samples_.size()) { - sum_ -= samples_.front(); - samples_.pop_front(); - } - - *avg = sum_ / static_cast(num_samples); - return true; -} - -template -void MovingAverage::Reset() { - sum_ = static_cast(0); - samples_.clear(); -} - -template -int MovingAverage::size() { - return samples_.size(); -} - } // namespace webrtc #endif // WEBRTC_MODULES_VIDEO_CODING_UTILITY_MOVING_AVERAGE_H_ diff --git a/webrtc/modules/video_coding/utility/moving_average_unittest.cc b/webrtc/modules/video_coding/utility/moving_average_unittest.cc new file mode 100644 index 0000000000..fa32a45e46 --- /dev/null +++ b/webrtc/modules/video_coding/utility/moving_average_unittest.cc @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/video_coding/utility/moving_average.h" + +#include "testing/gtest/include/gtest/gtest.h" + +TEST(MovingAverageTest, EmptyAverage) { + webrtc::MovingAverage moving_average(1); + EXPECT_EQ(0u, moving_average.size()); + EXPECT_FALSE(moving_average.GetAverage(0)); +} + +// Test single value. +TEST(MovingAverageTest, OneElement) { + webrtc::MovingAverage moving_average(1); + moving_average.AddSample(3); + EXPECT_EQ(1u, moving_average.size()); + EXPECT_EQ(3, *moving_average.GetAverage()); + EXPECT_EQ(3, *moving_average.GetAverage(1)); + EXPECT_FALSE(moving_average.GetAverage(2)); +} + +TEST(MovingAverageTest, GetAverage) { + webrtc::MovingAverage moving_average(1024); + moving_average.AddSample(1); + moving_average.AddSample(1); + moving_average.AddSample(3); + moving_average.AddSample(3); + EXPECT_EQ(*moving_average.GetAverage(4), 2); + EXPECT_EQ(*moving_average.GetAverage(2), 3); + EXPECT_FALSE(moving_average.GetAverage(0)); +} + +TEST(MovingAverageTest, Reset) { + webrtc::MovingAverage moving_average(5); + moving_average.AddSample(1); + EXPECT_EQ(1, *moving_average.GetAverage(1)); + moving_average.Reset(); + EXPECT_FALSE(moving_average.GetAverage(1)); + EXPECT_FALSE(moving_average.GetAverage(6)); +} + +TEST(MovingAverageTest, ManySamples) { + webrtc::MovingAverage moving_average(10); + for (int i = 1; i < 11; i++) { + moving_average.AddSample(i); + } + EXPECT_EQ(*moving_average.GetAverage(), 5); + moving_average.Reset(); + for (int i = 1; i < 2001; i++) { + moving_average.AddSample(i); + } + EXPECT_EQ(*moving_average.GetAverage(), 1995); +} diff --git a/webrtc/modules/video_coding/utility/quality_scaler.cc b/webrtc/modules/video_coding/utility/quality_scaler.cc index c509e84364..99bc6dad22 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler.cc @@ -10,10 +10,12 @@ #include "webrtc/modules/video_coding/utility/quality_scaler.h" +#include +#include + namespace webrtc { namespace { -static const int kMinFps = 5; // Threshold constant used until first downscale (to permit fast rampup). static const int kMeasureSecondsFastUpscale = 2; static const int kMeasureSecondsUpscale = 5; @@ -46,7 +48,11 @@ const int QualityScaler::kLowH264QpThreshold = 24; const int QualityScaler::kBadH264QpThreshold = 37; #endif -QualityScaler::QualityScaler() : low_qp_threshold_(-1) {} +// Default values. Should immediately get set to something more sensible. +QualityScaler::QualityScaler() + : average_qp_(kMeasureSecondsUpscale * 30), + framedrop_percent_(kMeasureSecondsUpscale * 30), + low_qp_threshold_(-1) {} void QualityScaler::Init(int low_qp_threshold, int high_qp_threshold, @@ -54,14 +60,15 @@ void QualityScaler::Init(int low_qp_threshold, int width, int height, int fps) { - ClearSamples(); low_qp_threshold_ = low_qp_threshold; high_qp_threshold_ = high_qp_threshold; downscale_shift_ = 0; - // Use a faster window for upscaling initially (but be more graceful later). - // This enables faster initial rampups without risking strong up-down - // behavior later. - measure_seconds_upscale_ = kMeasureSecondsFastUpscale; + + fast_rampup_ = true; + + ClearSamples(); + ReportFramerate(fps); + const int init_width = width; const int init_height = height; if (initial_bitrate_kbps > 0) { @@ -76,24 +83,28 @@ void QualityScaler::Init(int low_qp_threshold, height /= 2; } } - - // Zero out width/height so they can be checked against inside - // UpdateTargetResolution. - res_.width = res_.height = 0; UpdateTargetResolution(init_width, init_height); ReportFramerate(fps); } // Report framerate(fps) to estimate # of samples. void QualityScaler::ReportFramerate(int framerate) { - framerate_ = framerate; - UpdateSampleCounts(); + // Use a faster window for upscaling initially. + // This enables faster initial rampups without risking strong up-down + // behavior later. + num_samples_upscale_ = framerate * (fast_rampup_ ? kMeasureSecondsFastUpscale + : kMeasureSecondsUpscale); + num_samples_downscale_ = framerate * kMeasureSecondsDownscale; + + average_qp_ = + MovingAverage(std::max(num_samples_upscale_, num_samples_downscale_)); + framedrop_percent_ = + MovingAverage(std::max(num_samples_upscale_, num_samples_downscale_)); } void QualityScaler::ReportQP(int qp) { framedrop_percent_.AddSample(0); - average_qp_downscale_.AddSample(qp); - average_qp_upscale_.AddSample(qp); + average_qp_.AddSample(qp); } void QualityScaler::ReportDroppedFrame() { @@ -103,34 +114,58 @@ void QualityScaler::ReportDroppedFrame() { void QualityScaler::OnEncodeFrame(int width, int height) { // Should be set through InitEncode -> Should be set by now. RTC_DCHECK_GE(low_qp_threshold_, 0); - RTC_DCHECK_GT(num_samples_upscale_, 0u); - RTC_DCHECK_GT(num_samples_downscale_, 0u); - - // Update scale factor. - int avg_drop = 0; - int avg_qp = 0; - - if ((framedrop_percent_.GetAverage(num_samples_downscale_, &avg_drop) && - avg_drop >= kFramedropPercentThreshold) || - (average_qp_downscale_.GetAverage(num_samples_downscale_, &avg_qp) && - avg_qp > high_qp_threshold_)) { - AdjustScale(false); - } else if (average_qp_upscale_.GetAverage(num_samples_upscale_, &avg_qp) && - avg_qp <= low_qp_threshold_) { - AdjustScale(true); + if (target_res_.width != width || target_res_.height != height) { + UpdateTargetResolution(width, height); + } + + // Check if we should scale down due to high frame drop. + const auto drop_rate = framedrop_percent_.GetAverage(num_samples_downscale_); + if (drop_rate && *drop_rate >= kFramedropPercentThreshold) { + ScaleDown(); + return; + } + + // Check if we should scale up or down based on QP. + const auto avg_qp_down = average_qp_.GetAverage(num_samples_downscale_); + if (avg_qp_down && *avg_qp_down > high_qp_threshold_) { + ScaleDown(); + return; + } + const auto avg_qp_up = average_qp_.GetAverage(num_samples_upscale_); + if (avg_qp_up && *avg_qp_up <= low_qp_threshold_) { + // QP has been low. We want to try a higher resolution. + ScaleUp(); + return; + } +} + +void QualityScaler::ScaleUp() { + downscale_shift_ = std::max(0, downscale_shift_ - 1); + ClearSamples(); +} + +void QualityScaler::ScaleDown() { + downscale_shift_ = std::min(maximum_shift_, downscale_shift_ + 1); + ClearSamples(); + // If we've scaled down, wait longer before scaling up again. + if (fast_rampup_) { + fast_rampup_ = false; + num_samples_upscale_ = (num_samples_upscale_ / kMeasureSecondsFastUpscale) * + kMeasureSecondsUpscale; } - UpdateTargetResolution(width, height); } QualityScaler::Resolution QualityScaler::GetScaledResolution() const { - return res_; + const int frame_width = target_res_.width >> downscale_shift_; + const int frame_height = target_res_.height >> downscale_shift_; + return Resolution{frame_width, frame_height}; } rtc::scoped_refptr QualityScaler::GetScaledBuffer( const rtc::scoped_refptr& frame) { Resolution res = GetScaledResolution(); - int src_width = frame->width(); - int src_height = frame->height(); + const int src_width = frame->width(); + const int src_height = frame->height(); if (res.width == src_width && res.height == src_height) return frame; @@ -142,50 +177,20 @@ rtc::scoped_refptr QualityScaler::GetScaledBuffer( return scaled_buffer; } -void QualityScaler::UpdateTargetResolution(int frame_width, int frame_height) { - RTC_DCHECK_GE(downscale_shift_, 0); - int shifts_performed = 0; - for (int shift = downscale_shift_; - shift > 0 && (frame_width / 2 >= kMinDownscaleDimension) && - (frame_height / 2 >= kMinDownscaleDimension); - --shift, ++shifts_performed) { - frame_width /= 2; - frame_height /= 2; +void QualityScaler::UpdateTargetResolution(int width, int height) { + if (width < kMinDownscaleDimension || height < kMinDownscaleDimension) { + maximum_shift_ = 0; + } else { + maximum_shift_ = static_cast( + std::log2(std::min(width, height) / kMinDownscaleDimension)); } - // Clamp to number of shifts actually performed to not be stuck trying to - // scale way beyond QVGA. - downscale_shift_ = shifts_performed; - if (res_.width == frame_width && res_.height == frame_height) { - // No reset done/needed, using same resolution. - return; - } - res_.width = frame_width; - res_.height = frame_height; - ClearSamples(); + target_res_ = Resolution{width, height}; } void QualityScaler::ClearSamples() { framedrop_percent_.Reset(); - average_qp_downscale_.Reset(); - average_qp_upscale_.Reset(); + average_qp_.Reset(); } -void QualityScaler::UpdateSampleCounts() { - num_samples_downscale_ = static_cast( - kMeasureSecondsDownscale * (framerate_ < kMinFps ? kMinFps : framerate_)); - num_samples_upscale_ = static_cast( - measure_seconds_upscale_ * (framerate_ < kMinFps ? kMinFps : framerate_)); -} - -void QualityScaler::AdjustScale(bool up) { - downscale_shift_ += up ? -1 : 1; - if (downscale_shift_ < 0) - downscale_shift_ = 0; - if (!up) { - // First downscale hit, start using a slower threshold for going up. - measure_seconds_upscale_ = kMeasureSecondsUpscale; - UpdateSampleCounts(); - } -} } // namespace webrtc diff --git a/webrtc/modules/video_coding/utility/quality_scaler.h b/webrtc/modules/video_coding/utility/quality_scaler.h index 7176d4966e..c1ae50b06e 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler.h +++ b/webrtc/modules/video_coding/utility/quality_scaler.h @@ -48,26 +48,25 @@ class QualityScaler { static const int kBadH264QpThreshold; private: - void AdjustScale(bool up); - void UpdateTargetResolution(int frame_width, int frame_height); void ClearSamples(); - void UpdateSampleCounts(); + void ScaleUp(); + void ScaleDown(); + void UpdateTargetResolution(int width, int height); I420BufferPool pool_; size_t num_samples_downscale_; size_t num_samples_upscale_; - int measure_seconds_upscale_; - MovingAverage average_qp_upscale_; - MovingAverage average_qp_downscale_; + bool fast_rampup_; + MovingAverage average_qp_; + MovingAverage framedrop_percent_; - int framerate_; int low_qp_threshold_; int high_qp_threshold_; - MovingAverage framedrop_percent_; - Resolution res_; + Resolution target_res_; int downscale_shift_; + int maximum_shift_; }; } // namespace webrtc diff --git a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc index 27ee25cd94..2bbd26d760 100644 --- a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc +++ b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc @@ -43,7 +43,7 @@ class QualityScalerTest : public ::testing::Test { QualityScalerTest() { input_frame_ = I420Buffer::Create(kWidth, kHeight); - qs_.Init(kLowQpThreshold, kHighQp, 0, 0, 0, kFramerate); + qs_.Init(kLowQpThreshold, kHighQp, 0, kWidth, kHeight, kFramerate); qs_.OnEncodeFrame(input_frame_->width(), input_frame_->height()); } diff --git a/webrtc/modules/video_coding/utility/video_coding_utility.gyp b/webrtc/modules/video_coding/utility/video_coding_utility.gyp index c189abcfda..e4ed78f211 100644 --- a/webrtc/modules/video_coding/utility/video_coding_utility.gyp +++ b/webrtc/modules/video_coding/utility/video_coding_utility.gyp @@ -25,6 +25,7 @@ 'h264_bitstream_parser.h', 'ivf_file_writer.cc', 'ivf_file_writer.h', + 'moving_average.cc', 'moving_average.h', 'qp_parser.cc', 'qp_parser.h',