From f2707706798c072c63a7be5a4ed5850912a201d8 Mon Sep 17 00:00:00 2001 From: Shuhai Peng Date: Wed, 29 Sep 2021 17:19:44 +0800 Subject: [PATCH] video: Implement bandwidth based scaler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The |slice_qp_detla| reported by the hardware is not credible, which causing the quality scaler cannot work properly,the resolution cannot be adjusted correctly. To fix this issue, this CL implements a bandwidth scaler which is used for adjust resolution, this scaler will be used when QP based quality scaler is not working due to untrusted QP reported by HW AVC encoder. Bug: webrtc:12942 Change-Id: I2fc5f07a5400ec7e5ead2c2c502faee84d7f2a76 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/228860 Reviewed-by: Erik Språng Reviewed-by: Evan Shrubsole Commit-Queue: Erik Språng Cr-Commit-Position: refs/heads/main@{#35120} --- AUTHORS | 1 + modules/video_coding/BUILD.gn | 6 + .../utility/bandwidth_quality_scaler.cc | 150 ++++++++++ .../utility/bandwidth_quality_scaler.h | 95 ++++++ .../bandwidth_quality_scaler_unittest.cc | 279 ++++++++++++++++++ rtc_base/experiments/BUILD.gn | 17 ++ .../bandwidth_quality_scaler_settings.cc | 43 +++ .../bandwidth_quality_scaler_settings.h | 35 +++ ...dwidth_quality_scaler_settings_unittest.cc | 49 +++ rtc_base/experiments/encoder_info_settings.cc | 94 +++++- rtc_base/experiments/encoder_info_settings.h | 9 + video/adaptation/BUILD.gn | 2 + .../bandwidth_quality_scaler_resource.cc | 85 ++++++ .../bandwidth_quality_scaler_resource.h | 64 ++++ .../video_stream_encoder_resource_manager.cc | 53 +++- .../video_stream_encoder_resource_manager.h | 13 +- video/video_stream_encoder.cc | 136 ++++++--- video/video_stream_encoder_unittest.cc | 267 +++++++++++++++-- 18 files changed, 1322 insertions(+), 76 deletions(-) create mode 100644 modules/video_coding/utility/bandwidth_quality_scaler.cc create mode 100644 modules/video_coding/utility/bandwidth_quality_scaler.h create mode 100644 modules/video_coding/utility/bandwidth_quality_scaler_unittest.cc create mode 100644 rtc_base/experiments/bandwidth_quality_scaler_settings.cc create mode 100644 rtc_base/experiments/bandwidth_quality_scaler_settings.h create mode 100644 rtc_base/experiments/bandwidth_quality_scaler_settings_unittest.cc create mode 100644 video/adaptation/bandwidth_quality_scaler_resource.cc create mode 100644 video/adaptation/bandwidth_quality_scaler_resource.h diff --git a/AUTHORS b/AUTHORS index 11da0f4b1e..4ab41e03b6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -97,6 +97,7 @@ Sarah Thompson Satender Saroha Saul Kravitz Sergio Garcia Murillo +Shuhai Peng Silviu Caragea Stefan Gula Stephan Hartmann diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 129d4eab5c..f92dc63734 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -364,6 +364,8 @@ rtc_source_set("codec_globals_headers") { rtc_library("video_coding_utility") { visibility = [ "*" ] sources = [ + "utility/bandwidth_quality_scaler.cc", + "utility/bandwidth_quality_scaler.h", "utility/decoded_frames_history.cc", "utility/decoded_frames_history.h", "utility/frame_dropper.cc", @@ -410,6 +412,8 @@ rtc_library("video_coding_utility") { "../../rtc_base:rtc_numerics", "../../rtc_base:rtc_task_queue", "../../rtc_base:weak_ptr", + "../../rtc_base/experiments:bandwidth_quality_scaler_settings", + "../../rtc_base/experiments:encoder_info_settings", "../../rtc_base/experiments:quality_scaler_settings", "../../rtc_base/experiments:quality_scaling_experiment", "../../rtc_base/experiments:rate_control_settings", @@ -1031,6 +1035,7 @@ if (rtc_include_tests) { "timestamp_map_unittest.cc", "timing_unittest.cc", "unique_timestamp_counter_unittest.cc", + "utility/bandwidth_quality_scaler_unittest.cc", "utility/decoded_frames_history_unittest.cc", "utility/frame_dropper_unittest.cc", "utility/framerate_controller_deprecated_unittest.cc", @@ -1106,6 +1111,7 @@ if (rtc_include_tests) { "../../rtc_base:rtc_numerics", "../../rtc_base:rtc_task_queue", "../../rtc_base:task_queue_for_test", + "../../rtc_base/experiments:encoder_info_settings", "../../rtc_base/experiments:jitter_upper_bound_experiment", "../../rtc_base/synchronization:mutex", "../../rtc_base/system:unused", diff --git a/modules/video_coding/utility/bandwidth_quality_scaler.cc b/modules/video_coding/utility/bandwidth_quality_scaler.cc new file mode 100644 index 0000000000..0d6501aa4b --- /dev/null +++ b/modules/video_coding/utility/bandwidth_quality_scaler.cc @@ -0,0 +1,150 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/bandwidth_quality_scaler.h" + +#include +#include +#include +#include + +#include "api/video/video_adaptation_reason.h" +#include "api/video_codecs/video_encoder.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/bandwidth_quality_scaler_settings.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/exp_filter.h" +#include "rtc_base/task_queue.h" +#include "rtc_base/task_utils/to_queued_task.h" +#include "rtc_base/time_utils.h" +#include "rtc_base/weak_ptr.h" + +namespace webrtc { + +namespace { + +constexpr int kDefaultMaxWindowSizeMs = 5000; +constexpr float kHigherMaxBitrateTolerationFactor = 0.95; +constexpr float kLowerMinBitrateTolerationFactor = 0.8; +constexpr int kDefaultBitrateStateUpdateIntervalSeconds = 5; +} // namespace + +BandwidthQualityScaler::BandwidthQualityScaler( + BandwidthQualityScalerUsageHandlerInterface* handler) + : kBitrateStateUpdateInterval(TimeDelta::Seconds( + BandwidthQualityScalerSettings::ParseFromFieldTrials() + .BitrateStateUpdateInterval() + .value_or(kDefaultBitrateStateUpdateIntervalSeconds))), + handler_(handler), + encoded_bitrate_(kDefaultMaxWindowSizeMs, RateStatistics::kBpsScale), + weak_ptr_factory_(this) { + RTC_DCHECK_RUN_ON(&task_checker_); + RTC_DCHECK(handler_ != nullptr); + + StartCheckForBitrate(); +} + +BandwidthQualityScaler::~BandwidthQualityScaler() { + RTC_DCHECK_RUN_ON(&task_checker_); +} + +void BandwidthQualityScaler::StartCheckForBitrate() { + RTC_DCHECK_RUN_ON(&task_checker_); + TaskQueueBase::Current()->PostDelayedTask( + ToQueuedTask([this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] { + if (!this_weak_ptr) { + // The caller BandwidthQualityScaler has been deleted. + return; + } + RTC_DCHECK_RUN_ON(&task_checker_); + switch (CheckBitrate()) { + case BandwidthQualityScaler::CheckBitrateResult::kHighBitRate: { + handler_->OnReportUsageBandwidthHigh(); + last_frame_size_pixels_.reset(); + break; + } + case BandwidthQualityScaler::CheckBitrateResult::kLowBitRate: { + handler_->OnReportUsageBandwidthLow(); + last_frame_size_pixels_.reset(); + break; + } + case BandwidthQualityScaler::CheckBitrateResult::kNormalBitrate: { + break; + } + case BandwidthQualityScaler::CheckBitrateResult:: + kInsufficientSamples: { + break; + } + } + StartCheckForBitrate(); + }), + kBitrateStateUpdateInterval.ms()); +} + +void BandwidthQualityScaler::ReportEncodeInfo(int frame_size_bytes, + int64_t time_sent_in_ms, + uint32_t encoded_width, + uint32_t encoded_height) { + RTC_DCHECK_RUN_ON(&task_checker_); + last_time_sent_in_ms_ = time_sent_in_ms; + last_frame_size_pixels_ = encoded_width * encoded_height; + encoded_bitrate_.Update(frame_size_bytes, time_sent_in_ms); +} + +void BandwidthQualityScaler::SetResolutionBitrateLimits( + const std::vector& + resolution_bitrate_limits) { + if (resolution_bitrate_limits.empty()) { + resolution_bitrate_limits_ = EncoderInfoSettings:: + GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted(); + } else { + resolution_bitrate_limits_ = resolution_bitrate_limits; + } +} + +BandwidthQualityScaler::CheckBitrateResult +BandwidthQualityScaler::CheckBitrate() { + RTC_DCHECK_RUN_ON(&task_checker_); + if (!last_frame_size_pixels_.has_value() || + !last_time_sent_in_ms_.has_value()) { + return BandwidthQualityScaler::CheckBitrateResult::kInsufficientSamples; + } + + absl::optional current_bitrate_bps = + encoded_bitrate_.Rate(last_time_sent_in_ms_.value()); + if (!current_bitrate_bps.has_value()) { + // We can't get a valid bitrate due to not enough data points. + return BandwidthQualityScaler::CheckBitrateResult::kInsufficientSamples; + } + absl::optional suitable_bitrate_limit = + EncoderInfoSettings:: + GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted( + last_frame_size_pixels_, resolution_bitrate_limits_); + + if (!suitable_bitrate_limit.has_value()) { + return BandwidthQualityScaler::CheckBitrateResult::kInsufficientSamples; + } + + // Multiply by toleration factor to solve the frequent adaptation due to + // critical value. + if (current_bitrate_bps > suitable_bitrate_limit->max_bitrate_bps * + kHigherMaxBitrateTolerationFactor) { + return BandwidthQualityScaler::CheckBitrateResult::kLowBitRate; + } else if (current_bitrate_bps < + suitable_bitrate_limit->min_start_bitrate_bps * + kLowerMinBitrateTolerationFactor) { + return BandwidthQualityScaler::CheckBitrateResult::kHighBitRate; + } + return BandwidthQualityScaler::CheckBitrateResult::kNormalBitrate; +} + +BandwidthQualityScalerUsageHandlerInterface:: + ~BandwidthQualityScalerUsageHandlerInterface() {} +} // namespace webrtc diff --git a/modules/video_coding/utility/bandwidth_quality_scaler.h b/modules/video_coding/utility/bandwidth_quality_scaler.h new file mode 100644 index 0000000000..bb356dd560 --- /dev/null +++ b/modules/video_coding/utility/bandwidth_quality_scaler.h @@ -0,0 +1,95 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_BANDWIDTH_QUALITY_SCALER_H_ +#define MODULES_VIDEO_CODING_UTILITY_BANDWIDTH_QUALITY_SCALER_H_ + +#include +#include + +#include +#include + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/sequence_checker.h" +#include "api/video_codecs/video_encoder.h" +#include "rtc_base/experiments/encoder_info_settings.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/exp_filter.h" +#include "rtc_base/rate_statistics.h" +#include "rtc_base/ref_count.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/system/no_unique_address.h" +#include "rtc_base/task_queue.h" +#include "rtc_base/weak_ptr.h" + +namespace webrtc { + +class BandwidthQualityScalerUsageHandlerInterface { + public: + virtual ~BandwidthQualityScalerUsageHandlerInterface(); + + virtual void OnReportUsageBandwidthHigh() = 0; + virtual void OnReportUsageBandwidthLow() = 0; +}; + +// BandwidthQualityScaler runs asynchronously and monitors bandwidth values of +// encoded frames. It holds a reference to a +// BandwidthQualityScalerUsageHandlerInterface implementation to signal an +// overuse or underuse of bandwidth (which indicate a desire to scale the video +// stream down or up). +class BandwidthQualityScaler { + public: + explicit BandwidthQualityScaler( + BandwidthQualityScalerUsageHandlerInterface* handler); + virtual ~BandwidthQualityScaler(); + + void ReportEncodeInfo(int frame_size_bytes, + int64_t time_sent_in_ms, + uint32_t encoded_width, + uint32_t encoded_height); + + // We prioritise to using the |resolution_bitrate_limits| provided by the + // current decoder. If not provided, we will use the default data by + // GetDefaultResolutionBitrateLimits(). + void SetResolutionBitrateLimits( + const std::vector& + resolution_bitrate_limits); + + const TimeDelta kBitrateStateUpdateInterval; + + private: + enum class CheckBitrateResult { + kInsufficientSamples, + kNormalBitrate, + kHighBitRate, + kLowBitRate, + }; + + // We will periodically check encode bitrate, this function will make + // resolution up or down decisions and report the decision to the adapter. + void StartCheckForBitrate(); + CheckBitrateResult CheckBitrate(); + + RTC_NO_UNIQUE_ADDRESS SequenceChecker task_checker_; + BandwidthQualityScalerUsageHandlerInterface* const handler_ + RTC_GUARDED_BY(&task_checker_); + + absl::optional last_time_sent_in_ms_ RTC_GUARDED_BY(&task_checker_); + RateStatistics encoded_bitrate_ RTC_GUARDED_BY(&task_checker_); + absl::optional last_frame_size_pixels_ RTC_GUARDED_BY(&task_checker_); + rtc::WeakPtrFactory weak_ptr_factory_; + + std::vector resolution_bitrate_limits_; +}; + +} // namespace webrtc +#endif // MODULES_VIDEO_CODING_UTILITY_BANDWIDTH_QUALITY_SCALER_H_ diff --git a/modules/video_coding/utility/bandwidth_quality_scaler_unittest.cc b/modules/video_coding/utility/bandwidth_quality_scaler_unittest.cc new file mode 100644 index 0000000000..30af1f96bd --- /dev/null +++ b/modules/video_coding/utility/bandwidth_quality_scaler_unittest.cc @@ -0,0 +1,279 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/bandwidth_quality_scaler.h" + +#include +#include + +#include "rtc_base/checks.h" +#include "rtc_base/event.h" +#include "rtc_base/experiments/encoder_info_settings.h" +#include "rtc_base/task_queue_for_test.h" +#include "rtc_base/time_utils.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { +constexpr int kFramerateFps = 30; +constexpr int kDefaultBitrateStateUpdateIntervalSeconds = 5; +constexpr int kDefaultEncodeDeltaTimeMs = 33; // 1/30(s) => 33(ms) + +} // namespace + +class FakeBandwidthQualityScalerHandler + : public BandwidthQualityScalerUsageHandlerInterface { + public: + ~FakeBandwidthQualityScalerHandler() override = default; + void OnReportUsageBandwidthHigh() override { + adapt_down_event_count_++; + event_.Set(); + } + + void OnReportUsageBandwidthLow() override { + adapt_up_event_count_++; + event_.Set(); + } + + rtc::Event event_; + int adapt_up_event_count_ = 0; + int adapt_down_event_count_ = 0; +}; + +class BandwidthQualityScalerUnderTest : public BandwidthQualityScaler { + public: + explicit BandwidthQualityScalerUnderTest( + BandwidthQualityScalerUsageHandlerInterface* handler) + : BandwidthQualityScaler(handler) {} + + int GetBitrateStateUpdateIntervalMs() { + return this->kBitrateStateUpdateInterval.ms() + 200; + } +}; + +class BandwidthQualityScalerTest + : public ::testing::Test, + public ::testing::WithParamInterface { + protected: + enum ScaleDirection { + kKeepScaleNormalBandwidth, + kKeepScaleAboveMaxBandwidth, + kKeepScaleUnderMinBandwidth, + }; + + enum FrameType { + kKeyFrame, + kNormalFrame, + kNormalFrame_Overuse, + kNormalFrame_Underuse, + }; + struct FrameConfig { + FrameConfig(int frame_num, + FrameType frame_type, + int actual_width, + int actual_height) + : frame_num(frame_num), + frame_type(frame_type), + actual_width(actual_width), + actual_height(actual_height) {} + + int frame_num; + FrameType frame_type; + int actual_width; + int actual_height; + }; + + BandwidthQualityScalerTest() + : scoped_field_trial_(GetParam()), + task_queue_("BandwidthQualityScalerTestQueue"), + handler_(std::make_unique()) { + task_queue_.SendTask( + [this] { + bandwidth_quality_scaler_ = + std::unique_ptr( + new BandwidthQualityScalerUnderTest(handler_.get())); + bandwidth_quality_scaler_->SetResolutionBitrateLimits( + EncoderInfoSettings:: + GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted()); + // Only for testing. Set first_timestamp_ in RateStatistics to 0. + bandwidth_quality_scaler_->ReportEncodeInfo(0, 0, 0, 0); + }, + RTC_FROM_HERE); + } + + ~BandwidthQualityScalerTest() { + task_queue_.SendTask([this] { bandwidth_quality_scaler_ = nullptr; }, + RTC_FROM_HERE); + } + + int GetFrameSizeBytes( + const FrameConfig& config, + const VideoEncoder::ResolutionBitrateLimits& bitrate_limits) { + int scale = 8 * kFramerateFps; + switch (config.frame_type) { + case FrameType::kKeyFrame: { + // 4 is experimental value. Based on the test, the number of bytes of + // the key frame is about four times of the normal frame + return bitrate_limits.max_bitrate_bps * 4 / scale; + } + case FrameType::kNormalFrame_Overuse: { + return bitrate_limits.max_bitrate_bps * 3 / 2 / scale; + } + case FrameType::kNormalFrame_Underuse: { + return bitrate_limits.min_start_bitrate_bps * 3 / 4 / scale; + } + case FrameType::kNormalFrame: { + return (bitrate_limits.max_bitrate_bps + + bitrate_limits.min_start_bitrate_bps) / + 2 / scale; + } + } + return -1; + } + + absl::optional + GetDefaultSuitableBitrateLimit(int frame_size_pixels) { + return EncoderInfoSettings:: + GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted( + frame_size_pixels, + EncoderInfoSettings:: + GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted()); + } + + void TriggerBandwidthQualityScalerTest( + const std::vector& frame_configs) { + task_queue_.SendTask( + [frame_configs, this] { + RTC_CHECK(!frame_configs.empty()); + + int total_frame_nums = 0; + for (const FrameConfig& frame_config : frame_configs) { + total_frame_nums += frame_config.frame_num; + } + + EXPECT_EQ(kFramerateFps * kDefaultBitrateStateUpdateIntervalSeconds, + total_frame_nums); + + uint32_t time_send_to_scaler_ms_ = rtc::TimeMillis(); + for (size_t i = 0; i < frame_configs.size(); ++i) { + const FrameConfig& config = frame_configs[i]; + absl::optional + suitable_bitrate = GetDefaultSuitableBitrateLimit( + config.actual_width * config.actual_height); + EXPECT_TRUE(suitable_bitrate); + for (int j = 0; j <= config.frame_num; ++j) { + time_send_to_scaler_ms_ += kDefaultEncodeDeltaTimeMs; + int frame_size_bytes = + GetFrameSizeBytes(config, suitable_bitrate.value()); + RTC_CHECK(frame_size_bytes > 0); + bandwidth_quality_scaler_->ReportEncodeInfo( + frame_size_bytes, time_send_to_scaler_ms_, + config.actual_width, config.actual_height); + } + } + }, + RTC_FROM_HERE); + } + + test::ScopedFieldTrials scoped_field_trial_; + TaskQueueForTest task_queue_; + std::unique_ptr bandwidth_quality_scaler_; + std::unique_ptr handler_; +}; + +INSTANTIATE_TEST_SUITE_P( + FieldTrials, + BandwidthQualityScalerTest, + ::testing::Values("WebRTC-Video-BandwidthQualityScalerSettings/" + "bitrate_state_update_interval_s_:1/", + "WebRTC-Video-BandwidthQualityScalerSettings/" + "bitrate_state_update_interval_s_:2/")); + +TEST_P(BandwidthQualityScalerTest, AllNormalFrame_640x360) { + const std::vector frame_configs{ + FrameConfig(150, FrameType::kNormalFrame, 640, 360)}; + TriggerBandwidthQualityScalerTest(frame_configs); + + // When resolution is 640*360, experimental working bitrate range is + // [500000,800000] bps. Encoded bitrate is 654253, so it falls in the range + // without any operation(up/down). + EXPECT_FALSE(handler_->event_.Wait( + bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs())); + EXPECT_EQ(0, handler_->adapt_down_event_count_); + EXPECT_EQ(0, handler_->adapt_up_event_count_); +} + +TEST_P(BandwidthQualityScalerTest, AllNoramlFrame_AboveMaxBandwidth_640x360) { + const std::vector frame_configs{ + FrameConfig(150, FrameType::kNormalFrame_Overuse, 640, 360)}; + TriggerBandwidthQualityScalerTest(frame_configs); + + // When resolution is 640*360, experimental working bitrate range is + // [500000,800000] bps. Encoded bitrate is 1208000 > 800000 * 0.95, so it + // triggers adapt_up_event_count_. + EXPECT_TRUE(handler_->event_.Wait( + bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs())); + EXPECT_EQ(0, handler_->adapt_down_event_count_); + EXPECT_EQ(1, handler_->adapt_up_event_count_); +} + +TEST_P(BandwidthQualityScalerTest, AllNormalFrame_Underuse_640x360) { + const std::vector frame_configs{ + FrameConfig(150, FrameType::kNormalFrame_Underuse, 640, 360)}; + TriggerBandwidthQualityScalerTest(frame_configs); + + // When resolution is 640*360, experimental working bitrate range is + // [500000,800000] bps. Encoded bitrate is 377379 < 500000 * 0.8, so it + // triggers adapt_down_event_count_. + EXPECT_TRUE(handler_->event_.Wait( + bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs())); + EXPECT_EQ(1, handler_->adapt_down_event_count_); + EXPECT_EQ(0, handler_->adapt_up_event_count_); +} + +TEST_P(BandwidthQualityScalerTest, FixedFrameTypeTest1_640x360) { + const std::vector frame_configs{ + FrameConfig(5, FrameType::kNormalFrame_Underuse, 640, 360), + FrameConfig(110, FrameType::kNormalFrame, 640, 360), + FrameConfig(20, FrameType::kNormalFrame_Overuse, 640, 360), + FrameConfig(15, FrameType::kKeyFrame, 640, 360), + }; + TriggerBandwidthQualityScalerTest(frame_configs); + + // When resolution is 640*360, experimental working bitrate range is + // [500000,800000] bps. Encoded bitrate is 1059462 > 800000 * 0.95, so it + // triggers adapt_up_event_count_. + EXPECT_TRUE(handler_->event_.Wait( + bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs())); + EXPECT_EQ(0, handler_->adapt_down_event_count_); + EXPECT_EQ(1, handler_->adapt_up_event_count_); +} + +TEST_P(BandwidthQualityScalerTest, FixedFrameTypeTest2_640x360) { + const std::vector frame_configs{ + FrameConfig(10, FrameType::kNormalFrame_Underuse, 640, 360), + FrameConfig(50, FrameType::kNormalFrame, 640, 360), + FrameConfig(5, FrameType::kKeyFrame, 640, 360), + FrameConfig(85, FrameType::kNormalFrame_Overuse, 640, 360), + }; + TriggerBandwidthQualityScalerTest(frame_configs); + + // When resolution is 640*360, experimental working bitrate range is + // [500000,800000] bps. Encoded bitrate is 1059462 > 800000 * 0.95, so it + // triggers adapt_up_event_count_. + EXPECT_TRUE(handler_->event_.Wait( + bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs())); + EXPECT_EQ(0, handler_->adapt_down_event_count_); + EXPECT_EQ(1, handler_->adapt_up_event_count_); +} + +} // namespace webrtc diff --git a/rtc_base/experiments/BUILD.gn b/rtc_base/experiments/BUILD.gn index b0a729abfe..56d5000869 100644 --- a/rtc_base/experiments/BUILD.gn +++ b/rtc_base/experiments/BUILD.gn @@ -78,6 +78,21 @@ rtc_library("quality_scaler_settings") { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } +rtc_library("bandwidth_quality_scaler_settings") { + sources = [ + "bandwidth_quality_scaler_settings.cc", + "bandwidth_quality_scaler_settings.h", + ] + deps = [ + ":field_trial_parser", + "../:rtc_base_approved", + "../../api/transport:field_trial_based_config", + "../../api/transport:webrtc_key_value_config", + "../../system_wrappers:field_trial", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + rtc_library("quality_scaling_experiment") { sources = [ "quality_scaling_experiment.cc", @@ -237,6 +252,7 @@ if (rtc_include_tests && !build_with_chromium) { sources = [ "balanced_degradation_settings_unittest.cc", + "bandwidth_quality_scaler_settings_unittest.cc", "cpu_speed_experiment_unittest.cc", "encoder_info_settings_unittest.cc", "field_trial_list_unittest.cc", @@ -255,6 +271,7 @@ if (rtc_include_tests && !build_with_chromium) { ] deps = [ ":balanced_degradation_settings", + ":bandwidth_quality_scaler_settings", ":cpu_speed_experiment", ":encoder_info_settings", ":field_trial_parser", diff --git a/rtc_base/experiments/bandwidth_quality_scaler_settings.cc b/rtc_base/experiments/bandwidth_quality_scaler_settings.cc new file mode 100644 index 0000000000..332ab6be4b --- /dev/null +++ b/rtc_base/experiments/bandwidth_quality_scaler_settings.cc @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/experiments/bandwidth_quality_scaler_settings.h" + +#include "api/transport/field_trial_based_config.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +BandwidthQualityScalerSettings::BandwidthQualityScalerSettings( + const WebRtcKeyValueConfig* const key_value_config) + : bitrate_state_update_interval_s_("bitrate_state_update_interval_s_") { + ParseFieldTrial( + {&bitrate_state_update_interval_s_}, + key_value_config->Lookup("WebRTC-Video-BandwidthQualityScalerSettings")); +} + +BandwidthQualityScalerSettings +BandwidthQualityScalerSettings::ParseFromFieldTrials() { + FieldTrialBasedConfig field_trial_config; + return BandwidthQualityScalerSettings(&field_trial_config); +} + +absl::optional +BandwidthQualityScalerSettings::BitrateStateUpdateInterval() const { + if (bitrate_state_update_interval_s_ && + bitrate_state_update_interval_s_.Value() <= 0) { + RTC_LOG(LS_WARNING) + << "Unsupported bitrate_state_update_interval_s_ value, ignored."; + return absl::nullopt; + } + return bitrate_state_update_interval_s_.GetOptional(); +} + +} // namespace webrtc diff --git a/rtc_base/experiments/bandwidth_quality_scaler_settings.h b/rtc_base/experiments/bandwidth_quality_scaler_settings.h new file mode 100644 index 0000000000..959aea5bd3 --- /dev/null +++ b/rtc_base/experiments/bandwidth_quality_scaler_settings.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_EXPERIMENTS_BANDWIDTH_QUALITY_SCALER_SETTINGS_H_ +#define RTC_BASE_EXPERIMENTS_BANDWIDTH_QUALITY_SCALER_SETTINGS_H_ + +#include "absl/types/optional.h" +#include "api/transport/webrtc_key_value_config.h" +#include "rtc_base/experiments/field_trial_parser.h" + +namespace webrtc { + +class BandwidthQualityScalerSettings final { + public: + static BandwidthQualityScalerSettings ParseFromFieldTrials(); + + absl::optional BitrateStateUpdateInterval() const; + + private: + explicit BandwidthQualityScalerSettings( + const WebRtcKeyValueConfig* const key_value_config); + + FieldTrialOptional bitrate_state_update_interval_s_; +}; + +} // namespace webrtc + +#endif // RTC_BASE_EXPERIMENTS_BANDWIDTH_QUALITY_SCALER_SETTINGS_H_ diff --git a/rtc_base/experiments/bandwidth_quality_scaler_settings_unittest.cc b/rtc_base/experiments/bandwidth_quality_scaler_settings_unittest.cc new file mode 100644 index 0000000000..fab22cede0 --- /dev/null +++ b/rtc_base/experiments/bandwidth_quality_scaler_settings_unittest.cc @@ -0,0 +1,49 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/experiments/bandwidth_quality_scaler_settings.h" + +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +TEST(BandwidthQualityScalerSettingsTest, ValuesNotSetByDefault) { + const auto settings = BandwidthQualityScalerSettings::ParseFromFieldTrials(); + EXPECT_FALSE(settings.BitrateStateUpdateInterval()); +} + +TEST(BandwidthQualityScalerSettingsTest, ParseBitrateStateUpdateInterval) { + test::ScopedFieldTrials field_trials( + "WebRTC-Video-BandwidthQualityScalerSettings/" + "bitrate_state_update_interval_s_:100/"); + EXPECT_EQ(100u, BandwidthQualityScalerSettings::ParseFromFieldTrials() + .BitrateStateUpdateInterval()); +} + +TEST(BandwidthQualityScalerSettingsTest, ParseAll) { + test::ScopedFieldTrials field_trials( + "WebRTC-Video-BandwidthQualityScalerSettings/" + "bitrate_state_update_interval_s_:100/"); + EXPECT_EQ(100u, BandwidthQualityScalerSettings::ParseFromFieldTrials() + .BitrateStateUpdateInterval()); +} + +TEST(BandwidthQualityScalerSettingsTest, DoesNotParseIncorrectValue) { + test::ScopedFieldTrials field_trials( + "WebRTC-Video-BandwidthQualityScalerSettings/" + "bitrate_state_update_interval_s_:??/"); + const auto settings = BandwidthQualityScalerSettings::ParseFromFieldTrials(); + EXPECT_FALSE(settings.BitrateStateUpdateInterval()); +} + +} // namespace +} // namespace webrtc diff --git a/rtc_base/experiments/encoder_info_settings.cc b/rtc_base/experiments/encoder_info_settings.cc index 9e1a5190a3..b39c68468f 100644 --- a/rtc_base/experiments/encoder_info_settings.cc +++ b/rtc_base/experiments/encoder_info_settings.cc @@ -29,7 +29,7 @@ std::vector ToResolutionBitrateLimits( } return result; } - +constexpr float kDefaultMinBitratebps = 30000; } // namespace // Default bitrate limits for simulcast with one active stream: @@ -63,6 +63,98 @@ EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution( return info.GetEncoderBitrateLimitsForResolution(frame_size_pixels); } +// Return the suitable bitrate limits for specified resolution when qp is +// untrusted, they are experimental values. +// TODO(bugs.webrtc.org/12942): Maybe we need to add other codecs(VP8/VP9) +// experimental values. +std::vector +EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted() { + // Specific limits for H264/AVC + return {{0 * 0, 0, 0, 0}, + {320 * 180, 0, 30000, 300000}, + {480 * 270, 300000, 30000, 500000}, + {640 * 360, 500000, 30000, 800000}, + {960 * 540, 800000, 30000, 1500000}, + {1280 * 720, 1500000, 30000, 2500000}, + {1920 * 1080, 2500000, 30000, 4000000}}; +} + +// Through linear interpolation, return the bitrate limit corresponding to the +// specified |frame_size_pixels|. +absl::optional +EncoderInfoSettings::GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted( + absl::optional frame_size_pixels, + const std::vector& + resolution_bitrate_limits) { + if (!frame_size_pixels.has_value() || frame_size_pixels.value() <= 0) { + return absl::nullopt; + } + + std::vector bitrate_limits = + resolution_bitrate_limits; + + // Sort the list of bitrate limits by resolution. + sort(bitrate_limits.begin(), bitrate_limits.end(), + [](const VideoEncoder::ResolutionBitrateLimits& lhs, + const VideoEncoder::ResolutionBitrateLimits& rhs) { + return lhs.frame_size_pixels < rhs.frame_size_pixels; + }); + + if (bitrate_limits.empty()) { + return absl::nullopt; + } + + int interpolation_index = -1; + for (size_t i = 0; i < bitrate_limits.size(); ++i) { + if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels.value()) { + interpolation_index = i; + break; + } + } + + // -1 means that the maximum resolution is exceeded, we will select the + // largest data as the return result. + if (interpolation_index == -1) { + return *bitrate_limits.rbegin(); + } + + // If we have a matching resolution, return directly without interpolation. + if (bitrate_limits[interpolation_index].frame_size_pixels == + frame_size_pixels.value()) { + return bitrate_limits[interpolation_index]; + } + + // No matching resolution, do a linear interpolate. + int lower_pixel_count = + bitrate_limits[interpolation_index - 1].frame_size_pixels; + int upper_pixel_count = bitrate_limits[interpolation_index].frame_size_pixels; + float alpha = (frame_size_pixels.value() - lower_pixel_count) * 1.0 / + (upper_pixel_count - lower_pixel_count); + int min_start_bitrate_bps = static_cast( + bitrate_limits[interpolation_index].min_start_bitrate_bps * alpha + + bitrate_limits[interpolation_index - 1].min_start_bitrate_bps * + (1.0 - alpha)); + int max_bitrate_bps = static_cast( + bitrate_limits[interpolation_index].max_bitrate_bps * alpha + + bitrate_limits[interpolation_index - 1].max_bitrate_bps * (1.0 - alpha)); + + if (max_bitrate_bps >= min_start_bitrate_bps) { + return VideoEncoder::ResolutionBitrateLimits( + frame_size_pixels.value(), min_start_bitrate_bps, kDefaultMinBitratebps, + max_bitrate_bps); + } else { + RTC_LOG(LS_WARNING) + << "BitRate interpolation calculating result is abnormal. " + << " lower_pixel_count = " << lower_pixel_count + << " upper_pixel_count = " << upper_pixel_count + << " frame_size_pixels = " << frame_size_pixels.value() + << " min_start_bitrate_bps = " << min_start_bitrate_bps + << " min_bitrate_bps = " << kDefaultMinBitratebps + << " max_bitrate_bps = " << max_bitrate_bps; + return absl::nullopt; + } +} + EncoderInfoSettings::EncoderInfoSettings(std::string name) : requested_resolution_alignment_("requested_resolution_alignment"), apply_alignment_to_all_simulcast_layers_( diff --git a/rtc_base/experiments/encoder_info_settings.h b/rtc_base/experiments/encoder_info_settings.h index 9cbb5875bb..e4dc459fcf 100644 --- a/rtc_base/experiments/encoder_info_settings.h +++ b/rtc_base/experiments/encoder_info_settings.h @@ -48,6 +48,15 @@ class EncoderInfoSettings { GetDefaultSinglecastBitrateLimitsForResolution(VideoCodecType codec_type, int frame_size_pixels); + static std::vector + GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted(); + + static absl::optional + GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted( + absl::optional frame_size_pixels, + const std::vector& + resolution_bitrate_limits); + protected: explicit EncoderInfoSettings(std::string name); diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn index 20a2370b57..11962c04e3 100644 --- a/video/adaptation/BUILD.gn +++ b/video/adaptation/BUILD.gn @@ -12,6 +12,8 @@ rtc_library("video_adaptation") { sources = [ "balanced_constraint.cc", "balanced_constraint.h", + "bandwidth_quality_scaler_resource.cc", + "bandwidth_quality_scaler_resource.h", "bitrate_constraint.cc", "bitrate_constraint.h", "encode_usage_resource.cc", diff --git a/video/adaptation/bandwidth_quality_scaler_resource.cc b/video/adaptation/bandwidth_quality_scaler_resource.cc new file mode 100644 index 0000000000..6f491358f1 --- /dev/null +++ b/video/adaptation/bandwidth_quality_scaler_resource.cc @@ -0,0 +1,85 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/bandwidth_quality_scaler_resource.h" + +#include + +#include "rtc_base/checks.h" +#include "rtc_base/experiments/balanced_degradation_settings.h" +#include "rtc_base/logging.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/task_utils/to_queued_task.h" +#include "rtc_base/time_utils.h" + +namespace webrtc { + +// static +rtc::scoped_refptr +BandwidthQualityScalerResource::Create() { + return rtc::make_ref_counted(); +} + +BandwidthQualityScalerResource::BandwidthQualityScalerResource() + : VideoStreamEncoderResource("BandwidthQualityScalerResource"), + bandwidth_quality_scaler_(nullptr) {} + +BandwidthQualityScalerResource::~BandwidthQualityScalerResource() { + RTC_DCHECK(!bandwidth_quality_scaler_); +} + +bool BandwidthQualityScalerResource::is_started() const { + RTC_DCHECK_RUN_ON(encoder_queue()); + return bandwidth_quality_scaler_.get(); +} + +void BandwidthQualityScalerResource::StartCheckForOveruse( + const std::vector& + resolution_bitrate_limits) { + RTC_DCHECK_RUN_ON(encoder_queue()); + RTC_DCHECK(!is_started()); + bandwidth_quality_scaler_ = std::make_unique(this); + + // If the configuration parameters more than one, we should define and + // declare the function BandwidthQualityScaler::Initialize() and call it. + bandwidth_quality_scaler_->SetResolutionBitrateLimits( + resolution_bitrate_limits); +} + +void BandwidthQualityScalerResource::StopCheckForOveruse() { + RTC_DCHECK_RUN_ON(encoder_queue()); + RTC_DCHECK(is_started()); + // Ensure we have no pending callbacks. This makes it safe to destroy the + // BandwidthQualityScaler and even task queues with tasks in-flight. + bandwidth_quality_scaler_.reset(); +} + +void BandwidthQualityScalerResource::OnReportUsageBandwidthHigh() { + OnResourceUsageStateMeasured(ResourceUsageState::kOveruse); +} + +void BandwidthQualityScalerResource::OnReportUsageBandwidthLow() { + OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); +} + +void BandwidthQualityScalerResource::OnEncodeCompleted( + const EncodedImage& encoded_image, + int64_t time_sent_in_us, + int64_t encoded_image_size_bytes) { + RTC_DCHECK_RUN_ON(encoder_queue()); + + if (bandwidth_quality_scaler_) { + bandwidth_quality_scaler_->ReportEncodeInfo( + encoded_image_size_bytes, time_sent_in_us / 1000, + encoded_image._encodedWidth, encoded_image._encodedHeight); + } +} + +} // namespace webrtc diff --git a/video/adaptation/bandwidth_quality_scaler_resource.h b/video/adaptation/bandwidth_quality_scaler_resource.h new file mode 100644 index 0000000000..f33ce97f1d --- /dev/null +++ b/video/adaptation/bandwidth_quality_scaler_resource.h @@ -0,0 +1,64 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_ +#define VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_ + +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/video/video_adaptation_reason.h" +#include "api/video_codecs/video_encoder.h" +#include "call/adaptation/degradation_preference_provider.h" +#include "call/adaptation/resource_adaptation_processor_interface.h" +#include "modules/video_coding/utility/bandwidth_quality_scaler.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/task_queue.h" +#include "video/adaptation/video_stream_encoder_resource.h" + +namespace webrtc { + +// Handles interaction with the BandwidthQualityScaler. +class BandwidthQualityScalerResource + : public VideoStreamEncoderResource, + public BandwidthQualityScalerUsageHandlerInterface { + public: + static rtc::scoped_refptr Create(); + + BandwidthQualityScalerResource(); + ~BandwidthQualityScalerResource() override; + + bool is_started() const; + + void OnEncodeCompleted(const EncodedImage& encoded_image, + int64_t time_sent_in_us, + int64_t encoded_image_size_bytes); + + void StartCheckForOveruse( + const std::vector& + resolution_bitrate_limits); + void StopCheckForOveruse(); + + // BandwidthScalerQpUsageHandlerInterface implementation. + void OnReportUsageBandwidthHigh() override; + void OnReportUsageBandwidthLow() override; + + private: + std::unique_ptr bandwidth_quality_scaler_ + RTC_GUARDED_BY(encoder_queue()); +}; + +} // namespace webrtc + +#endif // VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_ diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index 0b2fa89339..6a1e9215a6 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -275,6 +275,8 @@ VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( EncodeUsageResource::Create(std::move(overuse_detector))), quality_scaler_resource_(QualityScalerResource::Create()), pixel_limit_resource_(nullptr), + bandwidth_quality_scaler_resource_( + BandwidthQualityScalerResource::Create()), encoder_queue_(nullptr), input_state_provider_(input_state_provider), adaptation_processor_(nullptr), @@ -307,6 +309,8 @@ void VideoStreamEncoderResourceManager::Initialize( encoder_queue_ = encoder_queue; encode_usage_resource_->RegisterEncoderTaskQueue(encoder_queue_->Get()); quality_scaler_resource_->RegisterEncoderTaskQueue(encoder_queue_->Get()); + bandwidth_quality_scaler_resource_->RegisterEncoderTaskQueue( + encoder_queue_->Get()); } void VideoStreamEncoderResourceManager::SetAdaptationProcessor( @@ -385,6 +389,10 @@ void VideoStreamEncoderResourceManager::StopManagedResources() { RemoveResource(pixel_limit_resource_); pixel_limit_resource_ = nullptr; } + if (bandwidth_quality_scaler_resource_->is_started()) { + bandwidth_quality_scaler_resource_->StopCheckForOveruse(); + RemoveResource(bandwidth_quality_scaler_resource_); + } } void VideoStreamEncoderResourceManager::AddResource( @@ -491,7 +499,8 @@ void VideoStreamEncoderResourceManager::OnEncodeStarted( void VideoStreamEncoderResourceManager::OnEncodeCompleted( const EncodedImage& encoded_image, int64_t time_sent_in_us, - absl::optional encode_duration_us) { + absl::optional encode_duration_us, + DataSize frame_size) { RTC_DCHECK_RUN_ON(encoder_queue_); // Inform `encode_usage_resource_` of the encode completed event. uint32_t timestamp = encoded_image.Timestamp(); @@ -500,6 +509,8 @@ void VideoStreamEncoderResourceManager::OnEncodeCompleted( encode_usage_resource_->OnEncodeCompleted( timestamp, time_sent_in_us, capture_time_us, encode_duration_us); quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us); + bandwidth_quality_scaler_resource_->OnEncodeCompleted( + encoded_image, time_sent_in_us, frame_size.bytes()); } void VideoStreamEncoderResourceManager::OnFrameDropped( @@ -556,6 +567,29 @@ void VideoStreamEncoderResourceManager::UpdateQualityScalerSettings( initial_frame_dropper_->OnQualityScalerSettingsUpdated(); } +void VideoStreamEncoderResourceManager::UpdateBandwidthQualityScalerSettings( + bool bandwidth_quality_scaling_allowed, + const std::vector& + resolution_bitrate_limits) { + RTC_DCHECK_RUN_ON(encoder_queue_); + + if (!bandwidth_quality_scaling_allowed) { + if (bandwidth_quality_scaler_resource_->is_started()) { + bandwidth_quality_scaler_resource_->StopCheckForOveruse(); + RemoveResource(bandwidth_quality_scaler_resource_); + } + } else { + if (!bandwidth_quality_scaler_resource_->is_started()) { + // Before executing "StartCheckForOveruse",we must execute "AddResource" + // firstly,because it can make the listener valid. + AddResource(bandwidth_quality_scaler_resource_, + webrtc::VideoAdaptationReason::kQuality); + bandwidth_quality_scaler_resource_->StartCheckForOveruse( + resolution_bitrate_limits); + } + } +} + void VideoStreamEncoderResourceManager::ConfigureQualityScaler( const VideoEncoder::EncoderInfo& encoder_info) { RTC_DCHECK_RUN_ON(encoder_queue_); @@ -601,6 +635,20 @@ void VideoStreamEncoderResourceManager::ConfigureQualityScaler( UpdateStatsAdaptationSettings(); } +void VideoStreamEncoderResourceManager::ConfigureBandwidthQualityScaler( + const VideoEncoder::EncoderInfo& encoder_info) { + RTC_DCHECK_RUN_ON(encoder_queue_); + const bool bandwidth_quality_scaling_allowed = + IsResolutionScalingEnabled(degradation_preference_) && + (encoder_settings_.has_value() && + encoder_settings_->encoder_config().is_quality_scaling_allowed) && + !encoder_info.is_qp_trusted.value_or(true); + + UpdateBandwidthQualityScalerSettings(bandwidth_quality_scaling_allowed, + encoder_info.resolution_bitrate_limits); + UpdateStatsAdaptationSettings(); +} + VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource( rtc::scoped_refptr resource) const { RTC_DCHECK_RUN_ON(encoder_queue_); @@ -727,7 +775,8 @@ void VideoStreamEncoderResourceManager::UpdateStatsAdaptationSettings() const { IsFramerateScalingEnabled(degradation_preference_)); VideoStreamEncoderObserver::AdaptationSettings quality_settings = - quality_scaler_resource_->is_started() + (quality_scaler_resource_->is_started() || + bandwidth_quality_scaler_resource_->is_started()) ? cpu_settings : VideoStreamEncoderObserver::AdaptationSettings(); encoder_stats_observer_->UpdateAdaptationSettings(cpu_settings, diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index 2f5dfcd280..f1bc8854b2 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -43,6 +43,7 @@ #include "rtc_base/thread_annotations.h" #include "system_wrappers/include/clock.h" #include "video/adaptation/balanced_constraint.h" +#include "video/adaptation/bandwidth_quality_scaler_resource.h" #include "video/adaptation/bitrate_constraint.h" #include "video/adaptation/encode_usage_resource.h" #include "video/adaptation/overuse_frame_detector.h" @@ -109,6 +110,8 @@ class VideoStreamEncoderResourceManager // TODO(https://crbug.com/webrtc/11338): This can be made private if we // configure on SetDegredationPreference and SetEncoderSettings. void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info); + void ConfigureBandwidthQualityScaler( + const VideoEncoder::EncoderInfo& encoder_info); // Methods corresponding to different points in the encoding pipeline. void OnFrameDroppedDueToSize(); @@ -117,7 +120,8 @@ class VideoStreamEncoderResourceManager int64_t time_when_first_seen_us); void OnEncodeCompleted(const EncodedImage& encoded_image, int64_t time_sent_in_us, - absl::optional encode_duration_us); + absl::optional encode_duration_us, + DataSize frame_size); void OnFrameDropped(EncodedImageCallback::DropReason reason); // Resources need to be mapped to an AdaptReason (kCpu or kQuality) in order @@ -166,6 +170,11 @@ class VideoStreamEncoderResourceManager void UpdateQualityScalerSettings( absl::optional qp_thresholds); + void UpdateBandwidthQualityScalerSettings( + bool bandwidth_quality_scaling_allowed, + const std::vector& + resolution_bitrate_limits); + void UpdateStatsAdaptationSettings() const; static std::string ActiveCountsToString( @@ -180,6 +189,8 @@ class VideoStreamEncoderResourceManager const rtc::scoped_refptr encode_usage_resource_; const rtc::scoped_refptr quality_scaler_resource_; rtc::scoped_refptr pixel_limit_resource_; + const rtc::scoped_refptr + bandwidth_quality_scaler_resource_; rtc::TaskQueue* encoder_queue_; VideoStreamInputStateProvider* const input_state_provider_ diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 8ea6750b87..8774ff7b72 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -73,6 +73,8 @@ const int64_t kParameterUpdateIntervalMs = 1000; // Animation is capped to 720p. constexpr int kMaxAnimationPixels = 1280 * 720; +constexpr int kDefaultMinScreenSharebps = 1200000; + bool RequiresEncoderReset(const VideoCodec& prev_send_codec, const VideoCodec& new_send_codec, bool was_encode_called_since_last_initialization) { @@ -783,6 +785,8 @@ void VideoStreamEncoder::SetSource( if (encoder_) { stream_resource_manager_.ConfigureQualityScaler( encoder_->GetEncoderInfo()); + stream_resource_manager_.ConfigureBandwidthQualityScaler( + encoder_->GetEncoderInfo()); } }); } @@ -914,50 +918,97 @@ void VideoStreamEncoder::ReconfigureEncoder() { crop_width_ = last_frame_info_->width - highest_stream_width; crop_height_ = last_frame_info_->height - highest_stream_height; - absl::optional encoder_bitrate_limits = - encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution( - last_frame_info_->width * last_frame_info_->height); + if (!encoder_->GetEncoderInfo().is_qp_trusted.value_or(true)) { + // when qp is not trusted, we priorities to using the + // |resolution_bitrate_limits| provided by the decoder. + const std::vector& bitrate_limits = + encoder_->GetEncoderInfo().resolution_bitrate_limits.empty() + ? EncoderInfoSettings:: + GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted() + : encoder_->GetEncoderInfo().resolution_bitrate_limits; - if (encoder_bitrate_limits) { - if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) { - // Bitrate limits can be set by app (in SDP or RtpEncodingParameters) - // or/and can be provided by encoder. In presence of both set of limits, - // the final set is derived as their intersection. - int min_bitrate_bps; - if (encoder_config_.simulcast_layers.empty() || - encoder_config_.simulcast_layers[0].min_bitrate_bps <= 0) { - min_bitrate_bps = encoder_bitrate_limits->min_bitrate_bps; - } else { - min_bitrate_bps = std::max(encoder_bitrate_limits->min_bitrate_bps, - streams.back().min_bitrate_bps); - } + // For BandwidthQualityScaler, its implement based on a certain pixel_count + // correspond a certain bps interval. In fact, WebRTC default max_bps is + // 2500Kbps when width * height > 960 * 540. For example, we assume: + // 1.the camera support 1080p. + // 2.ResolutionBitrateLimits set 720p bps interval is [1500Kbps,2000Kbps]. + // 3.ResolutionBitrateLimits set 1080p bps interval is [2000Kbps,2500Kbps]. + // We will never be stable at 720p due to actual encoding bps of 720p and + // 1080p are both 2500Kbps. So it is necessary to do a linear interpolation + // to get a certain bitrate for certain pixel_count. It also doesn't work + // for 960*540 and 640*520, we will nerver be stable at 640*520 due to their + // |target_bitrate_bps| are both 2000Kbps. + absl::optional + qp_untrusted_bitrate_limit = EncoderInfoSettings:: + GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted( + last_frame_info_->width * last_frame_info_->height, + bitrate_limits); - int max_bitrate_bps; - // We don't check encoder_config_.simulcast_layers[0].max_bitrate_bps - // here since encoder_config_.max_bitrate_bps is derived from it (as - // well as from other inputs). - if (encoder_config_.max_bitrate_bps <= 0) { - max_bitrate_bps = encoder_bitrate_limits->max_bitrate_bps; - } else { - max_bitrate_bps = std::min(encoder_bitrate_limits->max_bitrate_bps, - streams.back().max_bitrate_bps); - } - - if (min_bitrate_bps < max_bitrate_bps) { - streams.back().min_bitrate_bps = min_bitrate_bps; - streams.back().max_bitrate_bps = max_bitrate_bps; + if (qp_untrusted_bitrate_limit) { + // bandwidth_quality_scaler is only used for singlecast. + if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) { + streams.back().min_bitrate_bps = + qp_untrusted_bitrate_limit->min_bitrate_bps; + streams.back().max_bitrate_bps = + qp_untrusted_bitrate_limit->max_bitrate_bps; + // If it is screen share mode, the minimum value of max_bitrate should + // be greater than/equal to 1200kbps. + if (encoder_config_.content_type == + VideoEncoderConfig::ContentType::kScreen) { + streams.back().max_bitrate_bps = std::max( + streams.back().max_bitrate_bps, kDefaultMinScreenSharebps); + } streams.back().target_bitrate_bps = - std::min(streams.back().target_bitrate_bps, - encoder_bitrate_limits->max_bitrate_bps); - } else { - RTC_LOG(LS_WARNING) - << "Bitrate limits provided by encoder" - << " (min=" << encoder_bitrate_limits->min_bitrate_bps - << ", max=" << encoder_bitrate_limits->max_bitrate_bps - << ") do not intersect with limits set by app" - << " (min=" << streams.back().min_bitrate_bps - << ", max=" << encoder_config_.max_bitrate_bps - << "). The app bitrate limits will be used."; + qp_untrusted_bitrate_limit->max_bitrate_bps; + } + } + } else { + absl::optional + encoder_bitrate_limits = + encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution( + last_frame_info_->width * last_frame_info_->height); + + if (encoder_bitrate_limits) { + if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) { + // Bitrate limits can be set by app (in SDP or RtpEncodingParameters) + // or/and can be provided by encoder. In presence of both set of + // limits, the final set is derived as their intersection. + int min_bitrate_bps; + if (encoder_config_.simulcast_layers.empty() || + encoder_config_.simulcast_layers[0].min_bitrate_bps <= 0) { + min_bitrate_bps = encoder_bitrate_limits->min_bitrate_bps; + } else { + min_bitrate_bps = std::max(encoder_bitrate_limits->min_bitrate_bps, + streams.back().min_bitrate_bps); + } + + int max_bitrate_bps; + // We don't check encoder_config_.simulcast_layers[0].max_bitrate_bps + // here since encoder_config_.max_bitrate_bps is derived from it (as + // well as from other inputs). + if (encoder_config_.max_bitrate_bps <= 0) { + max_bitrate_bps = encoder_bitrate_limits->max_bitrate_bps; + } else { + max_bitrate_bps = std::min(encoder_bitrate_limits->max_bitrate_bps, + streams.back().max_bitrate_bps); + } + + if (min_bitrate_bps < max_bitrate_bps) { + streams.back().min_bitrate_bps = min_bitrate_bps; + streams.back().max_bitrate_bps = max_bitrate_bps; + streams.back().target_bitrate_bps = + std::min(streams.back().target_bitrate_bps, + encoder_bitrate_limits->max_bitrate_bps); + } else { + RTC_LOG(LS_WARNING) + << "Bitrate limits provided by encoder" + << " (min=" << encoder_bitrate_limits->min_bitrate_bps + << ", max=" << encoder_bitrate_limits->max_bitrate_bps + << ") do not intersect with limits set by app" + << " (min=" << streams.back().min_bitrate_bps + << ", max=" << encoder_config_.max_bitrate_bps + << "). The app bitrate limits will be used."; + } } } } @@ -1195,6 +1246,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { encoder_config_.min_transmit_bitrate_bps); stream_resource_manager_.ConfigureQualityScaler(info); + stream_resource_manager_.ConfigureBandwidthQualityScaler(info); } void VideoStreamEncoder::OnEncoderSettingsChanged() { @@ -2140,7 +2192,7 @@ void VideoStreamEncoder::RunPostEncode(const EncodedImage& encoded_image, } stream_resource_manager_.OnEncodeCompleted(encoded_image, time_sent_us, - encode_duration_us); + encode_duration_us, frame_size); if (bitrate_adjuster_) { bitrate_adjuster_->OnEncodedFrame( frame_size, encoded_image.SpatialIndex().value_or(0), temporal_index); diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index d64341901e..d9f11363dd 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -8210,36 +8210,6 @@ TEST_F(VideoStreamEncoderTest, video_stream_encoder_->Stop(); } -TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) { - VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); - - // Disable scaling settings in encoder info. - fake_encoder_.SetQualityScaling(false); - // Set QP not trusted in encoder info. - fake_encoder_.SetIsQpTrusted(false); - // Enable quality scaling in encoder config. - video_encoder_config.is_quality_scaling_allowed = true; - ConfigureEncoder(std::move(video_encoder_config)); - - video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( - DataRate::BitsPerSec(kTargetBitrateBps), - DataRate::BitsPerSec(kTargetBitrateBps), - DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); - - test::FrameForwarder source; - video_stream_encoder_->SetSource( - &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); - EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants()); - EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); - - source.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); - WaitForEncodedFrame(1); - video_stream_encoder_->TriggerQualityLow(); - EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); - - video_stream_encoder_->Stop(); -} - TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) { VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); @@ -8270,6 +8240,147 @@ TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) { video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, + QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) { + VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); + + // Disable scaling settings in encoder info. + fake_encoder_.SetQualityScaling(false); + // Set QP trusted in encoder info. + fake_encoder_.SetIsQpTrusted(true); + // Enable quality scaling in encoder config. + video_encoder_config.is_quality_scaling_allowed = false; + ConfigureEncoder(std::move(video_encoder_config)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + test::FrameForwarder source; + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants()); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + source.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) { + VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); + + // Disable scaling settings in encoder info. + fake_encoder_.SetQualityScaling(false); + // Set QP trusted in encoder info. + fake_encoder_.SetIsQpTrusted(false); + // Enable quality scaling in encoder config. + video_encoder_config.is_quality_scaling_allowed = false; + ConfigureEncoder(std::move(video_encoder_config)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + test::FrameForwarder source; + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants()); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + source.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) { + // Set QP trusted in encoder info. + fake_encoder_.SetIsQpTrusted(false); + + const int MinEncBitrateKbps = 30; + const int MaxEncBitrateKbps = 100; + const int MinStartBitrateKbp = 50; + const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits( + /*frame_size_pixels=*/codec_width_ * codec_height_, + /*min_start_bitrate_bps=*/MinStartBitrateKbp, + /*min_bitrate_bps=*/MinEncBitrateKbps * 1000, + /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits}); + + VideoEncoderConfig video_encoder_config; + test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config); + video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000; + video_encoder_config.simulcast_layers[0].min_bitrate_bps = + MinEncBitrateKbps * 1000; + video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(), + kMaxPayloadLength); + + video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr)); + WaitForEncodedFrame(1); + EXPECT_EQ( + MaxEncBitrateKbps, + static_cast(bitrate_allocator_factory_.codec_config().maxBitrate)); + EXPECT_EQ( + MinEncBitrateKbps, + static_cast(bitrate_allocator_factory_.codec_config().minBitrate)); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) { + // Set QP trusted in encoder info. + fake_encoder_.SetIsQpTrusted(false); + + absl::optional suitable_bitrate_limit = + EncoderInfoSettings:: + GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted( + codec_width_ * codec_height_, + EncoderInfoSettings:: + GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted()); + EXPECT_TRUE(suitable_bitrate_limit.has_value()); + + const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps; + const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps; + const int TargetEncBitrate = MaxEncBitrate; + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(TargetEncBitrate), + DataRate::BitsPerSec(TargetEncBitrate), + DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0); + + VideoEncoderConfig video_encoder_config; + test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config); + video_encoder_config.max_bitrate_bps = MaxEncBitrate; + video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate; + video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(), + kMaxPayloadLength); + + video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr)); + WaitForEncodedFrame(1); + EXPECT_EQ( + MaxEncBitrate / 1000, + static_cast(bitrate_allocator_factory_.codec_config().maxBitrate)); + EXPECT_EQ( + MinEncBitrate / 1000, + static_cast(bitrate_allocator_factory_.codec_config().minBitrate)); + + video_stream_encoder_->Stop(); +} + #if !defined(WEBRTC_IOS) // TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is // disabled by default on iOS. @@ -8330,6 +8441,102 @@ TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) { video_stream_encoder_->Stop(); } + +TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) { + VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); + + // Disable scaling settings in encoder info. + fake_encoder_.SetQualityScaling(false); + // Set QP not trusted in encoder info. + fake_encoder_.SetIsQpTrusted(false); + // Enable quality scaling in encoder config. + video_encoder_config.is_quality_scaling_allowed = true; + ConfigureEncoder(std::move(video_encoder_config)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + test::FrameForwarder source; + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants()); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + source.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + // When quality_scaler doesn't work and is_quality_scaling_allowed is + // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true. + EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) { + VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); + + // Disable scaling settings in encoder info. + fake_encoder_.SetQualityScaling(false); + // Set QP trusted in encoder info. + fake_encoder_.SetIsQpTrusted(true); + // Enable quality scaling in encoder config. + video_encoder_config.is_quality_scaling_allowed = true; + ConfigureEncoder(std::move(video_encoder_config)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + test::FrameForwarder source; + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants()); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + source.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + // bandwidth_quality_scaler isn't working, but quality_scaler is working. + EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) { + VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); + + // Disable scaling settings in encoder info. + fake_encoder_.SetQualityScaling(false); + // Set QP trusted in encoder info. + fake_encoder_.SetIsQpTrusted(false); + // Enable quality scaling in encoder config. + video_encoder_config.is_quality_scaling_allowed = true; + ConfigureEncoder(std::move(video_encoder_config)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + test::FrameForwarder source; + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants()); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + source.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution); + + video_stream_encoder_->Stop(); +} + #endif // Test parameters: (VideoCodecType codec, bool allow_i420_conversion)