diff --git a/AUTHORS b/AUTHORS index 11da0f4b1e..4ab41e03b6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -97,6 +97,7 @@ Sarah Thompson Satender Saroha Saul Kravitz Sergio Garcia Murillo +Shuhai Peng Silviu Caragea Stefan Gula Stephan Hartmann diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 129d4eab5c..f92dc63734 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -364,6 +364,8 @@ rtc_source_set("codec_globals_headers") { rtc_library("video_coding_utility") { visibility = [ "*" ] sources = [ + "utility/bandwidth_quality_scaler.cc", + "utility/bandwidth_quality_scaler.h", "utility/decoded_frames_history.cc", "utility/decoded_frames_history.h", "utility/frame_dropper.cc", @@ -410,6 +412,8 @@ rtc_library("video_coding_utility") { "../../rtc_base:rtc_numerics", "../../rtc_base:rtc_task_queue", "../../rtc_base:weak_ptr", + "../../rtc_base/experiments:bandwidth_quality_scaler_settings", + "../../rtc_base/experiments:encoder_info_settings", "../../rtc_base/experiments:quality_scaler_settings", "../../rtc_base/experiments:quality_scaling_experiment", "../../rtc_base/experiments:rate_control_settings", @@ -1031,6 +1035,7 @@ if (rtc_include_tests) { "timestamp_map_unittest.cc", "timing_unittest.cc", "unique_timestamp_counter_unittest.cc", + "utility/bandwidth_quality_scaler_unittest.cc", "utility/decoded_frames_history_unittest.cc", "utility/frame_dropper_unittest.cc", "utility/framerate_controller_deprecated_unittest.cc", @@ -1106,6 +1111,7 @@ if (rtc_include_tests) { "../../rtc_base:rtc_numerics", "../../rtc_base:rtc_task_queue", "../../rtc_base:task_queue_for_test", + "../../rtc_base/experiments:encoder_info_settings", "../../rtc_base/experiments:jitter_upper_bound_experiment", "../../rtc_base/synchronization:mutex", "../../rtc_base/system:unused", diff --git a/modules/video_coding/utility/bandwidth_quality_scaler.cc b/modules/video_coding/utility/bandwidth_quality_scaler.cc new file mode 100644 index 0000000000..0d6501aa4b --- /dev/null +++ b/modules/video_coding/utility/bandwidth_quality_scaler.cc @@ -0,0 +1,150 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/bandwidth_quality_scaler.h" + +#include +#include +#include +#include + +#include "api/video/video_adaptation_reason.h" +#include "api/video_codecs/video_encoder.h" +#include "rtc_base/checks.h" +#include "rtc_base/experiments/bandwidth_quality_scaler_settings.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/exp_filter.h" +#include "rtc_base/task_queue.h" +#include "rtc_base/task_utils/to_queued_task.h" +#include "rtc_base/time_utils.h" +#include "rtc_base/weak_ptr.h" + +namespace webrtc { + +namespace { + +constexpr int kDefaultMaxWindowSizeMs = 5000; +constexpr float kHigherMaxBitrateTolerationFactor = 0.95; +constexpr float kLowerMinBitrateTolerationFactor = 0.8; +constexpr int kDefaultBitrateStateUpdateIntervalSeconds = 5; +} // namespace + +BandwidthQualityScaler::BandwidthQualityScaler( + BandwidthQualityScalerUsageHandlerInterface* handler) + : kBitrateStateUpdateInterval(TimeDelta::Seconds( + BandwidthQualityScalerSettings::ParseFromFieldTrials() + .BitrateStateUpdateInterval() + .value_or(kDefaultBitrateStateUpdateIntervalSeconds))), + handler_(handler), + encoded_bitrate_(kDefaultMaxWindowSizeMs, RateStatistics::kBpsScale), + weak_ptr_factory_(this) { + RTC_DCHECK_RUN_ON(&task_checker_); + RTC_DCHECK(handler_ != nullptr); + + StartCheckForBitrate(); +} + +BandwidthQualityScaler::~BandwidthQualityScaler() { + RTC_DCHECK_RUN_ON(&task_checker_); +} + +void BandwidthQualityScaler::StartCheckForBitrate() { + RTC_DCHECK_RUN_ON(&task_checker_); + TaskQueueBase::Current()->PostDelayedTask( + ToQueuedTask([this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] { + if (!this_weak_ptr) { + // The caller BandwidthQualityScaler has been deleted. + return; + } + RTC_DCHECK_RUN_ON(&task_checker_); + switch (CheckBitrate()) { + case BandwidthQualityScaler::CheckBitrateResult::kHighBitRate: { + handler_->OnReportUsageBandwidthHigh(); + last_frame_size_pixels_.reset(); + break; + } + case BandwidthQualityScaler::CheckBitrateResult::kLowBitRate: { + handler_->OnReportUsageBandwidthLow(); + last_frame_size_pixels_.reset(); + break; + } + case BandwidthQualityScaler::CheckBitrateResult::kNormalBitrate: { + break; + } + case BandwidthQualityScaler::CheckBitrateResult:: + kInsufficientSamples: { + break; + } + } + StartCheckForBitrate(); + }), + kBitrateStateUpdateInterval.ms()); +} + +void BandwidthQualityScaler::ReportEncodeInfo(int frame_size_bytes, + int64_t time_sent_in_ms, + uint32_t encoded_width, + uint32_t encoded_height) { + RTC_DCHECK_RUN_ON(&task_checker_); + last_time_sent_in_ms_ = time_sent_in_ms; + last_frame_size_pixels_ = encoded_width * encoded_height; + encoded_bitrate_.Update(frame_size_bytes, time_sent_in_ms); +} + +void BandwidthQualityScaler::SetResolutionBitrateLimits( + const std::vector& + resolution_bitrate_limits) { + if (resolution_bitrate_limits.empty()) { + resolution_bitrate_limits_ = EncoderInfoSettings:: + GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted(); + } else { + resolution_bitrate_limits_ = resolution_bitrate_limits; + } +} + +BandwidthQualityScaler::CheckBitrateResult +BandwidthQualityScaler::CheckBitrate() { + RTC_DCHECK_RUN_ON(&task_checker_); + if (!last_frame_size_pixels_.has_value() || + !last_time_sent_in_ms_.has_value()) { + return BandwidthQualityScaler::CheckBitrateResult::kInsufficientSamples; + } + + absl::optional current_bitrate_bps = + encoded_bitrate_.Rate(last_time_sent_in_ms_.value()); + if (!current_bitrate_bps.has_value()) { + // We can't get a valid bitrate due to not enough data points. + return BandwidthQualityScaler::CheckBitrateResult::kInsufficientSamples; + } + absl::optional suitable_bitrate_limit = + EncoderInfoSettings:: + GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted( + last_frame_size_pixels_, resolution_bitrate_limits_); + + if (!suitable_bitrate_limit.has_value()) { + return BandwidthQualityScaler::CheckBitrateResult::kInsufficientSamples; + } + + // Multiply by toleration factor to solve the frequent adaptation due to + // critical value. + if (current_bitrate_bps > suitable_bitrate_limit->max_bitrate_bps * + kHigherMaxBitrateTolerationFactor) { + return BandwidthQualityScaler::CheckBitrateResult::kLowBitRate; + } else if (current_bitrate_bps < + suitable_bitrate_limit->min_start_bitrate_bps * + kLowerMinBitrateTolerationFactor) { + return BandwidthQualityScaler::CheckBitrateResult::kHighBitRate; + } + return BandwidthQualityScaler::CheckBitrateResult::kNormalBitrate; +} + +BandwidthQualityScalerUsageHandlerInterface:: + ~BandwidthQualityScalerUsageHandlerInterface() {} +} // namespace webrtc diff --git a/modules/video_coding/utility/bandwidth_quality_scaler.h b/modules/video_coding/utility/bandwidth_quality_scaler.h new file mode 100644 index 0000000000..bb356dd560 --- /dev/null +++ b/modules/video_coding/utility/bandwidth_quality_scaler.h @@ -0,0 +1,95 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_BANDWIDTH_QUALITY_SCALER_H_ +#define MODULES_VIDEO_CODING_UTILITY_BANDWIDTH_QUALITY_SCALER_H_ + +#include +#include + +#include +#include + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/sequence_checker.h" +#include "api/video_codecs/video_encoder.h" +#include "rtc_base/experiments/encoder_info_settings.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/exp_filter.h" +#include "rtc_base/rate_statistics.h" +#include "rtc_base/ref_count.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/system/no_unique_address.h" +#include "rtc_base/task_queue.h" +#include "rtc_base/weak_ptr.h" + +namespace webrtc { + +class BandwidthQualityScalerUsageHandlerInterface { + public: + virtual ~BandwidthQualityScalerUsageHandlerInterface(); + + virtual void OnReportUsageBandwidthHigh() = 0; + virtual void OnReportUsageBandwidthLow() = 0; +}; + +// BandwidthQualityScaler runs asynchronously and monitors bandwidth values of +// encoded frames. It holds a reference to a +// BandwidthQualityScalerUsageHandlerInterface implementation to signal an +// overuse or underuse of bandwidth (which indicate a desire to scale the video +// stream down or up). +class BandwidthQualityScaler { + public: + explicit BandwidthQualityScaler( + BandwidthQualityScalerUsageHandlerInterface* handler); + virtual ~BandwidthQualityScaler(); + + void ReportEncodeInfo(int frame_size_bytes, + int64_t time_sent_in_ms, + uint32_t encoded_width, + uint32_t encoded_height); + + // We prioritise to using the |resolution_bitrate_limits| provided by the + // current decoder. If not provided, we will use the default data by + // GetDefaultResolutionBitrateLimits(). + void SetResolutionBitrateLimits( + const std::vector& + resolution_bitrate_limits); + + const TimeDelta kBitrateStateUpdateInterval; + + private: + enum class CheckBitrateResult { + kInsufficientSamples, + kNormalBitrate, + kHighBitRate, + kLowBitRate, + }; + + // We will periodically check encode bitrate, this function will make + // resolution up or down decisions and report the decision to the adapter. + void StartCheckForBitrate(); + CheckBitrateResult CheckBitrate(); + + RTC_NO_UNIQUE_ADDRESS SequenceChecker task_checker_; + BandwidthQualityScalerUsageHandlerInterface* const handler_ + RTC_GUARDED_BY(&task_checker_); + + absl::optional last_time_sent_in_ms_ RTC_GUARDED_BY(&task_checker_); + RateStatistics encoded_bitrate_ RTC_GUARDED_BY(&task_checker_); + absl::optional last_frame_size_pixels_ RTC_GUARDED_BY(&task_checker_); + rtc::WeakPtrFactory weak_ptr_factory_; + + std::vector resolution_bitrate_limits_; +}; + +} // namespace webrtc +#endif // MODULES_VIDEO_CODING_UTILITY_BANDWIDTH_QUALITY_SCALER_H_ diff --git a/modules/video_coding/utility/bandwidth_quality_scaler_unittest.cc b/modules/video_coding/utility/bandwidth_quality_scaler_unittest.cc new file mode 100644 index 0000000000..30af1f96bd --- /dev/null +++ b/modules/video_coding/utility/bandwidth_quality_scaler_unittest.cc @@ -0,0 +1,279 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/bandwidth_quality_scaler.h" + +#include +#include + +#include "rtc_base/checks.h" +#include "rtc_base/event.h" +#include "rtc_base/experiments/encoder_info_settings.h" +#include "rtc_base/task_queue_for_test.h" +#include "rtc_base/time_utils.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { +constexpr int kFramerateFps = 30; +constexpr int kDefaultBitrateStateUpdateIntervalSeconds = 5; +constexpr int kDefaultEncodeDeltaTimeMs = 33; // 1/30(s) => 33(ms) + +} // namespace + +class FakeBandwidthQualityScalerHandler + : public BandwidthQualityScalerUsageHandlerInterface { + public: + ~FakeBandwidthQualityScalerHandler() override = default; + void OnReportUsageBandwidthHigh() override { + adapt_down_event_count_++; + event_.Set(); + } + + void OnReportUsageBandwidthLow() override { + adapt_up_event_count_++; + event_.Set(); + } + + rtc::Event event_; + int adapt_up_event_count_ = 0; + int adapt_down_event_count_ = 0; +}; + +class BandwidthQualityScalerUnderTest : public BandwidthQualityScaler { + public: + explicit BandwidthQualityScalerUnderTest( + BandwidthQualityScalerUsageHandlerInterface* handler) + : BandwidthQualityScaler(handler) {} + + int GetBitrateStateUpdateIntervalMs() { + return this->kBitrateStateUpdateInterval.ms() + 200; + } +}; + +class BandwidthQualityScalerTest + : public ::testing::Test, + public ::testing::WithParamInterface { + protected: + enum ScaleDirection { + kKeepScaleNormalBandwidth, + kKeepScaleAboveMaxBandwidth, + kKeepScaleUnderMinBandwidth, + }; + + enum FrameType { + kKeyFrame, + kNormalFrame, + kNormalFrame_Overuse, + kNormalFrame_Underuse, + }; + struct FrameConfig { + FrameConfig(int frame_num, + FrameType frame_type, + int actual_width, + int actual_height) + : frame_num(frame_num), + frame_type(frame_type), + actual_width(actual_width), + actual_height(actual_height) {} + + int frame_num; + FrameType frame_type; + int actual_width; + int actual_height; + }; + + BandwidthQualityScalerTest() + : scoped_field_trial_(GetParam()), + task_queue_("BandwidthQualityScalerTestQueue"), + handler_(std::make_unique()) { + task_queue_.SendTask( + [this] { + bandwidth_quality_scaler_ = + std::unique_ptr( + new BandwidthQualityScalerUnderTest(handler_.get())); + bandwidth_quality_scaler_->SetResolutionBitrateLimits( + EncoderInfoSettings:: + GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted()); + // Only for testing. Set first_timestamp_ in RateStatistics to 0. + bandwidth_quality_scaler_->ReportEncodeInfo(0, 0, 0, 0); + }, + RTC_FROM_HERE); + } + + ~BandwidthQualityScalerTest() { + task_queue_.SendTask([this] { bandwidth_quality_scaler_ = nullptr; }, + RTC_FROM_HERE); + } + + int GetFrameSizeBytes( + const FrameConfig& config, + const VideoEncoder::ResolutionBitrateLimits& bitrate_limits) { + int scale = 8 * kFramerateFps; + switch (config.frame_type) { + case FrameType::kKeyFrame: { + // 4 is experimental value. Based on the test, the number of bytes of + // the key frame is about four times of the normal frame + return bitrate_limits.max_bitrate_bps * 4 / scale; + } + case FrameType::kNormalFrame_Overuse: { + return bitrate_limits.max_bitrate_bps * 3 / 2 / scale; + } + case FrameType::kNormalFrame_Underuse: { + return bitrate_limits.min_start_bitrate_bps * 3 / 4 / scale; + } + case FrameType::kNormalFrame: { + return (bitrate_limits.max_bitrate_bps + + bitrate_limits.min_start_bitrate_bps) / + 2 / scale; + } + } + return -1; + } + + absl::optional + GetDefaultSuitableBitrateLimit(int frame_size_pixels) { + return EncoderInfoSettings:: + GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted( + frame_size_pixels, + EncoderInfoSettings:: + GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted()); + } + + void TriggerBandwidthQualityScalerTest( + const std::vector& frame_configs) { + task_queue_.SendTask( + [frame_configs, this] { + RTC_CHECK(!frame_configs.empty()); + + int total_frame_nums = 0; + for (const FrameConfig& frame_config : frame_configs) { + total_frame_nums += frame_config.frame_num; + } + + EXPECT_EQ(kFramerateFps * kDefaultBitrateStateUpdateIntervalSeconds, + total_frame_nums); + + uint32_t time_send_to_scaler_ms_ = rtc::TimeMillis(); + for (size_t i = 0; i < frame_configs.size(); ++i) { + const FrameConfig& config = frame_configs[i]; + absl::optional + suitable_bitrate = GetDefaultSuitableBitrateLimit( + config.actual_width * config.actual_height); + EXPECT_TRUE(suitable_bitrate); + for (int j = 0; j <= config.frame_num; ++j) { + time_send_to_scaler_ms_ += kDefaultEncodeDeltaTimeMs; + int frame_size_bytes = + GetFrameSizeBytes(config, suitable_bitrate.value()); + RTC_CHECK(frame_size_bytes > 0); + bandwidth_quality_scaler_->ReportEncodeInfo( + frame_size_bytes, time_send_to_scaler_ms_, + config.actual_width, config.actual_height); + } + } + }, + RTC_FROM_HERE); + } + + test::ScopedFieldTrials scoped_field_trial_; + TaskQueueForTest task_queue_; + std::unique_ptr bandwidth_quality_scaler_; + std::unique_ptr handler_; +}; + +INSTANTIATE_TEST_SUITE_P( + FieldTrials, + BandwidthQualityScalerTest, + ::testing::Values("WebRTC-Video-BandwidthQualityScalerSettings/" + "bitrate_state_update_interval_s_:1/", + "WebRTC-Video-BandwidthQualityScalerSettings/" + "bitrate_state_update_interval_s_:2/")); + +TEST_P(BandwidthQualityScalerTest, AllNormalFrame_640x360) { + const std::vector frame_configs{ + FrameConfig(150, FrameType::kNormalFrame, 640, 360)}; + TriggerBandwidthQualityScalerTest(frame_configs); + + // When resolution is 640*360, experimental working bitrate range is + // [500000,800000] bps. Encoded bitrate is 654253, so it falls in the range + // without any operation(up/down). + EXPECT_FALSE(handler_->event_.Wait( + bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs())); + EXPECT_EQ(0, handler_->adapt_down_event_count_); + EXPECT_EQ(0, handler_->adapt_up_event_count_); +} + +TEST_P(BandwidthQualityScalerTest, AllNoramlFrame_AboveMaxBandwidth_640x360) { + const std::vector frame_configs{ + FrameConfig(150, FrameType::kNormalFrame_Overuse, 640, 360)}; + TriggerBandwidthQualityScalerTest(frame_configs); + + // When resolution is 640*360, experimental working bitrate range is + // [500000,800000] bps. Encoded bitrate is 1208000 > 800000 * 0.95, so it + // triggers adapt_up_event_count_. + EXPECT_TRUE(handler_->event_.Wait( + bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs())); + EXPECT_EQ(0, handler_->adapt_down_event_count_); + EXPECT_EQ(1, handler_->adapt_up_event_count_); +} + +TEST_P(BandwidthQualityScalerTest, AllNormalFrame_Underuse_640x360) { + const std::vector frame_configs{ + FrameConfig(150, FrameType::kNormalFrame_Underuse, 640, 360)}; + TriggerBandwidthQualityScalerTest(frame_configs); + + // When resolution is 640*360, experimental working bitrate range is + // [500000,800000] bps. Encoded bitrate is 377379 < 500000 * 0.8, so it + // triggers adapt_down_event_count_. + EXPECT_TRUE(handler_->event_.Wait( + bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs())); + EXPECT_EQ(1, handler_->adapt_down_event_count_); + EXPECT_EQ(0, handler_->adapt_up_event_count_); +} + +TEST_P(BandwidthQualityScalerTest, FixedFrameTypeTest1_640x360) { + const std::vector frame_configs{ + FrameConfig(5, FrameType::kNormalFrame_Underuse, 640, 360), + FrameConfig(110, FrameType::kNormalFrame, 640, 360), + FrameConfig(20, FrameType::kNormalFrame_Overuse, 640, 360), + FrameConfig(15, FrameType::kKeyFrame, 640, 360), + }; + TriggerBandwidthQualityScalerTest(frame_configs); + + // When resolution is 640*360, experimental working bitrate range is + // [500000,800000] bps. Encoded bitrate is 1059462 > 800000 * 0.95, so it + // triggers adapt_up_event_count_. + EXPECT_TRUE(handler_->event_.Wait( + bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs())); + EXPECT_EQ(0, handler_->adapt_down_event_count_); + EXPECT_EQ(1, handler_->adapt_up_event_count_); +} + +TEST_P(BandwidthQualityScalerTest, FixedFrameTypeTest2_640x360) { + const std::vector frame_configs{ + FrameConfig(10, FrameType::kNormalFrame_Underuse, 640, 360), + FrameConfig(50, FrameType::kNormalFrame, 640, 360), + FrameConfig(5, FrameType::kKeyFrame, 640, 360), + FrameConfig(85, FrameType::kNormalFrame_Overuse, 640, 360), + }; + TriggerBandwidthQualityScalerTest(frame_configs); + + // When resolution is 640*360, experimental working bitrate range is + // [500000,800000] bps. Encoded bitrate is 1059462 > 800000 * 0.95, so it + // triggers adapt_up_event_count_. + EXPECT_TRUE(handler_->event_.Wait( + bandwidth_quality_scaler_->GetBitrateStateUpdateIntervalMs())); + EXPECT_EQ(0, handler_->adapt_down_event_count_); + EXPECT_EQ(1, handler_->adapt_up_event_count_); +} + +} // namespace webrtc diff --git a/rtc_base/experiments/BUILD.gn b/rtc_base/experiments/BUILD.gn index b0a729abfe..56d5000869 100644 --- a/rtc_base/experiments/BUILD.gn +++ b/rtc_base/experiments/BUILD.gn @@ -78,6 +78,21 @@ rtc_library("quality_scaler_settings") { absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] } +rtc_library("bandwidth_quality_scaler_settings") { + sources = [ + "bandwidth_quality_scaler_settings.cc", + "bandwidth_quality_scaler_settings.h", + ] + deps = [ + ":field_trial_parser", + "../:rtc_base_approved", + "../../api/transport:field_trial_based_config", + "../../api/transport:webrtc_key_value_config", + "../../system_wrappers:field_trial", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + rtc_library("quality_scaling_experiment") { sources = [ "quality_scaling_experiment.cc", @@ -237,6 +252,7 @@ if (rtc_include_tests && !build_with_chromium) { sources = [ "balanced_degradation_settings_unittest.cc", + "bandwidth_quality_scaler_settings_unittest.cc", "cpu_speed_experiment_unittest.cc", "encoder_info_settings_unittest.cc", "field_trial_list_unittest.cc", @@ -255,6 +271,7 @@ if (rtc_include_tests && !build_with_chromium) { ] deps = [ ":balanced_degradation_settings", + ":bandwidth_quality_scaler_settings", ":cpu_speed_experiment", ":encoder_info_settings", ":field_trial_parser", diff --git a/rtc_base/experiments/bandwidth_quality_scaler_settings.cc b/rtc_base/experiments/bandwidth_quality_scaler_settings.cc new file mode 100644 index 0000000000..332ab6be4b --- /dev/null +++ b/rtc_base/experiments/bandwidth_quality_scaler_settings.cc @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/experiments/bandwidth_quality_scaler_settings.h" + +#include "api/transport/field_trial_based_config.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +BandwidthQualityScalerSettings::BandwidthQualityScalerSettings( + const WebRtcKeyValueConfig* const key_value_config) + : bitrate_state_update_interval_s_("bitrate_state_update_interval_s_") { + ParseFieldTrial( + {&bitrate_state_update_interval_s_}, + key_value_config->Lookup("WebRTC-Video-BandwidthQualityScalerSettings")); +} + +BandwidthQualityScalerSettings +BandwidthQualityScalerSettings::ParseFromFieldTrials() { + FieldTrialBasedConfig field_trial_config; + return BandwidthQualityScalerSettings(&field_trial_config); +} + +absl::optional +BandwidthQualityScalerSettings::BitrateStateUpdateInterval() const { + if (bitrate_state_update_interval_s_ && + bitrate_state_update_interval_s_.Value() <= 0) { + RTC_LOG(LS_WARNING) + << "Unsupported bitrate_state_update_interval_s_ value, ignored."; + return absl::nullopt; + } + return bitrate_state_update_interval_s_.GetOptional(); +} + +} // namespace webrtc diff --git a/rtc_base/experiments/bandwidth_quality_scaler_settings.h b/rtc_base/experiments/bandwidth_quality_scaler_settings.h new file mode 100644 index 0000000000..959aea5bd3 --- /dev/null +++ b/rtc_base/experiments/bandwidth_quality_scaler_settings.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_EXPERIMENTS_BANDWIDTH_QUALITY_SCALER_SETTINGS_H_ +#define RTC_BASE_EXPERIMENTS_BANDWIDTH_QUALITY_SCALER_SETTINGS_H_ + +#include "absl/types/optional.h" +#include "api/transport/webrtc_key_value_config.h" +#include "rtc_base/experiments/field_trial_parser.h" + +namespace webrtc { + +class BandwidthQualityScalerSettings final { + public: + static BandwidthQualityScalerSettings ParseFromFieldTrials(); + + absl::optional BitrateStateUpdateInterval() const; + + private: + explicit BandwidthQualityScalerSettings( + const WebRtcKeyValueConfig* const key_value_config); + + FieldTrialOptional bitrate_state_update_interval_s_; +}; + +} // namespace webrtc + +#endif // RTC_BASE_EXPERIMENTS_BANDWIDTH_QUALITY_SCALER_SETTINGS_H_ diff --git a/rtc_base/experiments/bandwidth_quality_scaler_settings_unittest.cc b/rtc_base/experiments/bandwidth_quality_scaler_settings_unittest.cc new file mode 100644 index 0000000000..fab22cede0 --- /dev/null +++ b/rtc_base/experiments/bandwidth_quality_scaler_settings_unittest.cc @@ -0,0 +1,49 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/experiments/bandwidth_quality_scaler_settings.h" + +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +TEST(BandwidthQualityScalerSettingsTest, ValuesNotSetByDefault) { + const auto settings = BandwidthQualityScalerSettings::ParseFromFieldTrials(); + EXPECT_FALSE(settings.BitrateStateUpdateInterval()); +} + +TEST(BandwidthQualityScalerSettingsTest, ParseBitrateStateUpdateInterval) { + test::ScopedFieldTrials field_trials( + "WebRTC-Video-BandwidthQualityScalerSettings/" + "bitrate_state_update_interval_s_:100/"); + EXPECT_EQ(100u, BandwidthQualityScalerSettings::ParseFromFieldTrials() + .BitrateStateUpdateInterval()); +} + +TEST(BandwidthQualityScalerSettingsTest, ParseAll) { + test::ScopedFieldTrials field_trials( + "WebRTC-Video-BandwidthQualityScalerSettings/" + "bitrate_state_update_interval_s_:100/"); + EXPECT_EQ(100u, BandwidthQualityScalerSettings::ParseFromFieldTrials() + .BitrateStateUpdateInterval()); +} + +TEST(BandwidthQualityScalerSettingsTest, DoesNotParseIncorrectValue) { + test::ScopedFieldTrials field_trials( + "WebRTC-Video-BandwidthQualityScalerSettings/" + "bitrate_state_update_interval_s_:??/"); + const auto settings = BandwidthQualityScalerSettings::ParseFromFieldTrials(); + EXPECT_FALSE(settings.BitrateStateUpdateInterval()); +} + +} // namespace +} // namespace webrtc diff --git a/rtc_base/experiments/encoder_info_settings.cc b/rtc_base/experiments/encoder_info_settings.cc index 9e1a5190a3..b39c68468f 100644 --- a/rtc_base/experiments/encoder_info_settings.cc +++ b/rtc_base/experiments/encoder_info_settings.cc @@ -29,7 +29,7 @@ std::vector ToResolutionBitrateLimits( } return result; } - +constexpr float kDefaultMinBitratebps = 30000; } // namespace // Default bitrate limits for simulcast with one active stream: @@ -63,6 +63,98 @@ EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution( return info.GetEncoderBitrateLimitsForResolution(frame_size_pixels); } +// Return the suitable bitrate limits for specified resolution when qp is +// untrusted, they are experimental values. +// TODO(bugs.webrtc.org/12942): Maybe we need to add other codecs(VP8/VP9) +// experimental values. +std::vector +EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted() { + // Specific limits for H264/AVC + return {{0 * 0, 0, 0, 0}, + {320 * 180, 0, 30000, 300000}, + {480 * 270, 300000, 30000, 500000}, + {640 * 360, 500000, 30000, 800000}, + {960 * 540, 800000, 30000, 1500000}, + {1280 * 720, 1500000, 30000, 2500000}, + {1920 * 1080, 2500000, 30000, 4000000}}; +} + +// Through linear interpolation, return the bitrate limit corresponding to the +// specified |frame_size_pixels|. +absl::optional +EncoderInfoSettings::GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted( + absl::optional frame_size_pixels, + const std::vector& + resolution_bitrate_limits) { + if (!frame_size_pixels.has_value() || frame_size_pixels.value() <= 0) { + return absl::nullopt; + } + + std::vector bitrate_limits = + resolution_bitrate_limits; + + // Sort the list of bitrate limits by resolution. + sort(bitrate_limits.begin(), bitrate_limits.end(), + [](const VideoEncoder::ResolutionBitrateLimits& lhs, + const VideoEncoder::ResolutionBitrateLimits& rhs) { + return lhs.frame_size_pixels < rhs.frame_size_pixels; + }); + + if (bitrate_limits.empty()) { + return absl::nullopt; + } + + int interpolation_index = -1; + for (size_t i = 0; i < bitrate_limits.size(); ++i) { + if (bitrate_limits[i].frame_size_pixels >= frame_size_pixels.value()) { + interpolation_index = i; + break; + } + } + + // -1 means that the maximum resolution is exceeded, we will select the + // largest data as the return result. + if (interpolation_index == -1) { + return *bitrate_limits.rbegin(); + } + + // If we have a matching resolution, return directly without interpolation. + if (bitrate_limits[interpolation_index].frame_size_pixels == + frame_size_pixels.value()) { + return bitrate_limits[interpolation_index]; + } + + // No matching resolution, do a linear interpolate. + int lower_pixel_count = + bitrate_limits[interpolation_index - 1].frame_size_pixels; + int upper_pixel_count = bitrate_limits[interpolation_index].frame_size_pixels; + float alpha = (frame_size_pixels.value() - lower_pixel_count) * 1.0 / + (upper_pixel_count - lower_pixel_count); + int min_start_bitrate_bps = static_cast( + bitrate_limits[interpolation_index].min_start_bitrate_bps * alpha + + bitrate_limits[interpolation_index - 1].min_start_bitrate_bps * + (1.0 - alpha)); + int max_bitrate_bps = static_cast( + bitrate_limits[interpolation_index].max_bitrate_bps * alpha + + bitrate_limits[interpolation_index - 1].max_bitrate_bps * (1.0 - alpha)); + + if (max_bitrate_bps >= min_start_bitrate_bps) { + return VideoEncoder::ResolutionBitrateLimits( + frame_size_pixels.value(), min_start_bitrate_bps, kDefaultMinBitratebps, + max_bitrate_bps); + } else { + RTC_LOG(LS_WARNING) + << "BitRate interpolation calculating result is abnormal. " + << " lower_pixel_count = " << lower_pixel_count + << " upper_pixel_count = " << upper_pixel_count + << " frame_size_pixels = " << frame_size_pixels.value() + << " min_start_bitrate_bps = " << min_start_bitrate_bps + << " min_bitrate_bps = " << kDefaultMinBitratebps + << " max_bitrate_bps = " << max_bitrate_bps; + return absl::nullopt; + } +} + EncoderInfoSettings::EncoderInfoSettings(std::string name) : requested_resolution_alignment_("requested_resolution_alignment"), apply_alignment_to_all_simulcast_layers_( diff --git a/rtc_base/experiments/encoder_info_settings.h b/rtc_base/experiments/encoder_info_settings.h index 9cbb5875bb..e4dc459fcf 100644 --- a/rtc_base/experiments/encoder_info_settings.h +++ b/rtc_base/experiments/encoder_info_settings.h @@ -48,6 +48,15 @@ class EncoderInfoSettings { GetDefaultSinglecastBitrateLimitsForResolution(VideoCodecType codec_type, int frame_size_pixels); + static std::vector + GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted(); + + static absl::optional + GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted( + absl::optional frame_size_pixels, + const std::vector& + resolution_bitrate_limits); + protected: explicit EncoderInfoSettings(std::string name); diff --git a/video/adaptation/BUILD.gn b/video/adaptation/BUILD.gn index 20a2370b57..11962c04e3 100644 --- a/video/adaptation/BUILD.gn +++ b/video/adaptation/BUILD.gn @@ -12,6 +12,8 @@ rtc_library("video_adaptation") { sources = [ "balanced_constraint.cc", "balanced_constraint.h", + "bandwidth_quality_scaler_resource.cc", + "bandwidth_quality_scaler_resource.h", "bitrate_constraint.cc", "bitrate_constraint.h", "encode_usage_resource.cc", diff --git a/video/adaptation/bandwidth_quality_scaler_resource.cc b/video/adaptation/bandwidth_quality_scaler_resource.cc new file mode 100644 index 0000000000..6f491358f1 --- /dev/null +++ b/video/adaptation/bandwidth_quality_scaler_resource.cc @@ -0,0 +1,85 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "video/adaptation/bandwidth_quality_scaler_resource.h" + +#include + +#include "rtc_base/checks.h" +#include "rtc_base/experiments/balanced_degradation_settings.h" +#include "rtc_base/logging.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/task_utils/to_queued_task.h" +#include "rtc_base/time_utils.h" + +namespace webrtc { + +// static +rtc::scoped_refptr +BandwidthQualityScalerResource::Create() { + return rtc::make_ref_counted(); +} + +BandwidthQualityScalerResource::BandwidthQualityScalerResource() + : VideoStreamEncoderResource("BandwidthQualityScalerResource"), + bandwidth_quality_scaler_(nullptr) {} + +BandwidthQualityScalerResource::~BandwidthQualityScalerResource() { + RTC_DCHECK(!bandwidth_quality_scaler_); +} + +bool BandwidthQualityScalerResource::is_started() const { + RTC_DCHECK_RUN_ON(encoder_queue()); + return bandwidth_quality_scaler_.get(); +} + +void BandwidthQualityScalerResource::StartCheckForOveruse( + const std::vector& + resolution_bitrate_limits) { + RTC_DCHECK_RUN_ON(encoder_queue()); + RTC_DCHECK(!is_started()); + bandwidth_quality_scaler_ = std::make_unique(this); + + // If the configuration parameters more than one, we should define and + // declare the function BandwidthQualityScaler::Initialize() and call it. + bandwidth_quality_scaler_->SetResolutionBitrateLimits( + resolution_bitrate_limits); +} + +void BandwidthQualityScalerResource::StopCheckForOveruse() { + RTC_DCHECK_RUN_ON(encoder_queue()); + RTC_DCHECK(is_started()); + // Ensure we have no pending callbacks. This makes it safe to destroy the + // BandwidthQualityScaler and even task queues with tasks in-flight. + bandwidth_quality_scaler_.reset(); +} + +void BandwidthQualityScalerResource::OnReportUsageBandwidthHigh() { + OnResourceUsageStateMeasured(ResourceUsageState::kOveruse); +} + +void BandwidthQualityScalerResource::OnReportUsageBandwidthLow() { + OnResourceUsageStateMeasured(ResourceUsageState::kUnderuse); +} + +void BandwidthQualityScalerResource::OnEncodeCompleted( + const EncodedImage& encoded_image, + int64_t time_sent_in_us, + int64_t encoded_image_size_bytes) { + RTC_DCHECK_RUN_ON(encoder_queue()); + + if (bandwidth_quality_scaler_) { + bandwidth_quality_scaler_->ReportEncodeInfo( + encoded_image_size_bytes, time_sent_in_us / 1000, + encoded_image._encodedWidth, encoded_image._encodedHeight); + } +} + +} // namespace webrtc diff --git a/video/adaptation/bandwidth_quality_scaler_resource.h b/video/adaptation/bandwidth_quality_scaler_resource.h new file mode 100644 index 0000000000..f33ce97f1d --- /dev/null +++ b/video/adaptation/bandwidth_quality_scaler_resource.h @@ -0,0 +1,64 @@ +/* + * Copyright 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_ +#define VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_ + +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/video/video_adaptation_reason.h" +#include "api/video_codecs/video_encoder.h" +#include "call/adaptation/degradation_preference_provider.h" +#include "call/adaptation/resource_adaptation_processor_interface.h" +#include "modules/video_coding/utility/bandwidth_quality_scaler.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/task_queue.h" +#include "video/adaptation/video_stream_encoder_resource.h" + +namespace webrtc { + +// Handles interaction with the BandwidthQualityScaler. +class BandwidthQualityScalerResource + : public VideoStreamEncoderResource, + public BandwidthQualityScalerUsageHandlerInterface { + public: + static rtc::scoped_refptr Create(); + + BandwidthQualityScalerResource(); + ~BandwidthQualityScalerResource() override; + + bool is_started() const; + + void OnEncodeCompleted(const EncodedImage& encoded_image, + int64_t time_sent_in_us, + int64_t encoded_image_size_bytes); + + void StartCheckForOveruse( + const std::vector& + resolution_bitrate_limits); + void StopCheckForOveruse(); + + // BandwidthScalerQpUsageHandlerInterface implementation. + void OnReportUsageBandwidthHigh() override; + void OnReportUsageBandwidthLow() override; + + private: + std::unique_ptr bandwidth_quality_scaler_ + RTC_GUARDED_BY(encoder_queue()); +}; + +} // namespace webrtc + +#endif // VIDEO_ADAPTATION_BANDWIDTH_QUALITY_SCALER_RESOURCE_H_ diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index 0b2fa89339..6a1e9215a6 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -275,6 +275,8 @@ VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager( EncodeUsageResource::Create(std::move(overuse_detector))), quality_scaler_resource_(QualityScalerResource::Create()), pixel_limit_resource_(nullptr), + bandwidth_quality_scaler_resource_( + BandwidthQualityScalerResource::Create()), encoder_queue_(nullptr), input_state_provider_(input_state_provider), adaptation_processor_(nullptr), @@ -307,6 +309,8 @@ void VideoStreamEncoderResourceManager::Initialize( encoder_queue_ = encoder_queue; encode_usage_resource_->RegisterEncoderTaskQueue(encoder_queue_->Get()); quality_scaler_resource_->RegisterEncoderTaskQueue(encoder_queue_->Get()); + bandwidth_quality_scaler_resource_->RegisterEncoderTaskQueue( + encoder_queue_->Get()); } void VideoStreamEncoderResourceManager::SetAdaptationProcessor( @@ -385,6 +389,10 @@ void VideoStreamEncoderResourceManager::StopManagedResources() { RemoveResource(pixel_limit_resource_); pixel_limit_resource_ = nullptr; } + if (bandwidth_quality_scaler_resource_->is_started()) { + bandwidth_quality_scaler_resource_->StopCheckForOveruse(); + RemoveResource(bandwidth_quality_scaler_resource_); + } } void VideoStreamEncoderResourceManager::AddResource( @@ -491,7 +499,8 @@ void VideoStreamEncoderResourceManager::OnEncodeStarted( void VideoStreamEncoderResourceManager::OnEncodeCompleted( const EncodedImage& encoded_image, int64_t time_sent_in_us, - absl::optional encode_duration_us) { + absl::optional encode_duration_us, + DataSize frame_size) { RTC_DCHECK_RUN_ON(encoder_queue_); // Inform `encode_usage_resource_` of the encode completed event. uint32_t timestamp = encoded_image.Timestamp(); @@ -500,6 +509,8 @@ void VideoStreamEncoderResourceManager::OnEncodeCompleted( encode_usage_resource_->OnEncodeCompleted( timestamp, time_sent_in_us, capture_time_us, encode_duration_us); quality_scaler_resource_->OnEncodeCompleted(encoded_image, time_sent_in_us); + bandwidth_quality_scaler_resource_->OnEncodeCompleted( + encoded_image, time_sent_in_us, frame_size.bytes()); } void VideoStreamEncoderResourceManager::OnFrameDropped( @@ -556,6 +567,29 @@ void VideoStreamEncoderResourceManager::UpdateQualityScalerSettings( initial_frame_dropper_->OnQualityScalerSettingsUpdated(); } +void VideoStreamEncoderResourceManager::UpdateBandwidthQualityScalerSettings( + bool bandwidth_quality_scaling_allowed, + const std::vector& + resolution_bitrate_limits) { + RTC_DCHECK_RUN_ON(encoder_queue_); + + if (!bandwidth_quality_scaling_allowed) { + if (bandwidth_quality_scaler_resource_->is_started()) { + bandwidth_quality_scaler_resource_->StopCheckForOveruse(); + RemoveResource(bandwidth_quality_scaler_resource_); + } + } else { + if (!bandwidth_quality_scaler_resource_->is_started()) { + // Before executing "StartCheckForOveruse",we must execute "AddResource" + // firstly,because it can make the listener valid. + AddResource(bandwidth_quality_scaler_resource_, + webrtc::VideoAdaptationReason::kQuality); + bandwidth_quality_scaler_resource_->StartCheckForOveruse( + resolution_bitrate_limits); + } + } +} + void VideoStreamEncoderResourceManager::ConfigureQualityScaler( const VideoEncoder::EncoderInfo& encoder_info) { RTC_DCHECK_RUN_ON(encoder_queue_); @@ -601,6 +635,20 @@ void VideoStreamEncoderResourceManager::ConfigureQualityScaler( UpdateStatsAdaptationSettings(); } +void VideoStreamEncoderResourceManager::ConfigureBandwidthQualityScaler( + const VideoEncoder::EncoderInfo& encoder_info) { + RTC_DCHECK_RUN_ON(encoder_queue_); + const bool bandwidth_quality_scaling_allowed = + IsResolutionScalingEnabled(degradation_preference_) && + (encoder_settings_.has_value() && + encoder_settings_->encoder_config().is_quality_scaling_allowed) && + !encoder_info.is_qp_trusted.value_or(true); + + UpdateBandwidthQualityScalerSettings(bandwidth_quality_scaling_allowed, + encoder_info.resolution_bitrate_limits); + UpdateStatsAdaptationSettings(); +} + VideoAdaptationReason VideoStreamEncoderResourceManager::GetReasonFromResource( rtc::scoped_refptr resource) const { RTC_DCHECK_RUN_ON(encoder_queue_); @@ -727,7 +775,8 @@ void VideoStreamEncoderResourceManager::UpdateStatsAdaptationSettings() const { IsFramerateScalingEnabled(degradation_preference_)); VideoStreamEncoderObserver::AdaptationSettings quality_settings = - quality_scaler_resource_->is_started() + (quality_scaler_resource_->is_started() || + bandwidth_quality_scaler_resource_->is_started()) ? cpu_settings : VideoStreamEncoderObserver::AdaptationSettings(); encoder_stats_observer_->UpdateAdaptationSettings(cpu_settings, diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index 2f5dfcd280..f1bc8854b2 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -43,6 +43,7 @@ #include "rtc_base/thread_annotations.h" #include "system_wrappers/include/clock.h" #include "video/adaptation/balanced_constraint.h" +#include "video/adaptation/bandwidth_quality_scaler_resource.h" #include "video/adaptation/bitrate_constraint.h" #include "video/adaptation/encode_usage_resource.h" #include "video/adaptation/overuse_frame_detector.h" @@ -109,6 +110,8 @@ class VideoStreamEncoderResourceManager // TODO(https://crbug.com/webrtc/11338): This can be made private if we // configure on SetDegredationPreference and SetEncoderSettings. void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info); + void ConfigureBandwidthQualityScaler( + const VideoEncoder::EncoderInfo& encoder_info); // Methods corresponding to different points in the encoding pipeline. void OnFrameDroppedDueToSize(); @@ -117,7 +120,8 @@ class VideoStreamEncoderResourceManager int64_t time_when_first_seen_us); void OnEncodeCompleted(const EncodedImage& encoded_image, int64_t time_sent_in_us, - absl::optional encode_duration_us); + absl::optional encode_duration_us, + DataSize frame_size); void OnFrameDropped(EncodedImageCallback::DropReason reason); // Resources need to be mapped to an AdaptReason (kCpu or kQuality) in order @@ -166,6 +170,11 @@ class VideoStreamEncoderResourceManager void UpdateQualityScalerSettings( absl::optional qp_thresholds); + void UpdateBandwidthQualityScalerSettings( + bool bandwidth_quality_scaling_allowed, + const std::vector& + resolution_bitrate_limits); + void UpdateStatsAdaptationSettings() const; static std::string ActiveCountsToString( @@ -180,6 +189,8 @@ class VideoStreamEncoderResourceManager const rtc::scoped_refptr encode_usage_resource_; const rtc::scoped_refptr quality_scaler_resource_; rtc::scoped_refptr pixel_limit_resource_; + const rtc::scoped_refptr + bandwidth_quality_scaler_resource_; rtc::TaskQueue* encoder_queue_; VideoStreamInputStateProvider* const input_state_provider_ diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 8ea6750b87..8774ff7b72 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -73,6 +73,8 @@ const int64_t kParameterUpdateIntervalMs = 1000; // Animation is capped to 720p. constexpr int kMaxAnimationPixels = 1280 * 720; +constexpr int kDefaultMinScreenSharebps = 1200000; + bool RequiresEncoderReset(const VideoCodec& prev_send_codec, const VideoCodec& new_send_codec, bool was_encode_called_since_last_initialization) { @@ -783,6 +785,8 @@ void VideoStreamEncoder::SetSource( if (encoder_) { stream_resource_manager_.ConfigureQualityScaler( encoder_->GetEncoderInfo()); + stream_resource_manager_.ConfigureBandwidthQualityScaler( + encoder_->GetEncoderInfo()); } }); } @@ -914,50 +918,97 @@ void VideoStreamEncoder::ReconfigureEncoder() { crop_width_ = last_frame_info_->width - highest_stream_width; crop_height_ = last_frame_info_->height - highest_stream_height; - absl::optional encoder_bitrate_limits = - encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution( - last_frame_info_->width * last_frame_info_->height); + if (!encoder_->GetEncoderInfo().is_qp_trusted.value_or(true)) { + // when qp is not trusted, we priorities to using the + // |resolution_bitrate_limits| provided by the decoder. + const std::vector& bitrate_limits = + encoder_->GetEncoderInfo().resolution_bitrate_limits.empty() + ? EncoderInfoSettings:: + GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted() + : encoder_->GetEncoderInfo().resolution_bitrate_limits; - if (encoder_bitrate_limits) { - if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) { - // Bitrate limits can be set by app (in SDP or RtpEncodingParameters) - // or/and can be provided by encoder. In presence of both set of limits, - // the final set is derived as their intersection. - int min_bitrate_bps; - if (encoder_config_.simulcast_layers.empty() || - encoder_config_.simulcast_layers[0].min_bitrate_bps <= 0) { - min_bitrate_bps = encoder_bitrate_limits->min_bitrate_bps; - } else { - min_bitrate_bps = std::max(encoder_bitrate_limits->min_bitrate_bps, - streams.back().min_bitrate_bps); - } + // For BandwidthQualityScaler, its implement based on a certain pixel_count + // correspond a certain bps interval. In fact, WebRTC default max_bps is + // 2500Kbps when width * height > 960 * 540. For example, we assume: + // 1.the camera support 1080p. + // 2.ResolutionBitrateLimits set 720p bps interval is [1500Kbps,2000Kbps]. + // 3.ResolutionBitrateLimits set 1080p bps interval is [2000Kbps,2500Kbps]. + // We will never be stable at 720p due to actual encoding bps of 720p and + // 1080p are both 2500Kbps. So it is necessary to do a linear interpolation + // to get a certain bitrate for certain pixel_count. It also doesn't work + // for 960*540 and 640*520, we will nerver be stable at 640*520 due to their + // |target_bitrate_bps| are both 2000Kbps. + absl::optional + qp_untrusted_bitrate_limit = EncoderInfoSettings:: + GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted( + last_frame_info_->width * last_frame_info_->height, + bitrate_limits); - int max_bitrate_bps; - // We don't check encoder_config_.simulcast_layers[0].max_bitrate_bps - // here since encoder_config_.max_bitrate_bps is derived from it (as - // well as from other inputs). - if (encoder_config_.max_bitrate_bps <= 0) { - max_bitrate_bps = encoder_bitrate_limits->max_bitrate_bps; - } else { - max_bitrate_bps = std::min(encoder_bitrate_limits->max_bitrate_bps, - streams.back().max_bitrate_bps); - } - - if (min_bitrate_bps < max_bitrate_bps) { - streams.back().min_bitrate_bps = min_bitrate_bps; - streams.back().max_bitrate_bps = max_bitrate_bps; + if (qp_untrusted_bitrate_limit) { + // bandwidth_quality_scaler is only used for singlecast. + if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) { + streams.back().min_bitrate_bps = + qp_untrusted_bitrate_limit->min_bitrate_bps; + streams.back().max_bitrate_bps = + qp_untrusted_bitrate_limit->max_bitrate_bps; + // If it is screen share mode, the minimum value of max_bitrate should + // be greater than/equal to 1200kbps. + if (encoder_config_.content_type == + VideoEncoderConfig::ContentType::kScreen) { + streams.back().max_bitrate_bps = std::max( + streams.back().max_bitrate_bps, kDefaultMinScreenSharebps); + } streams.back().target_bitrate_bps = - std::min(streams.back().target_bitrate_bps, - encoder_bitrate_limits->max_bitrate_bps); - } else { - RTC_LOG(LS_WARNING) - << "Bitrate limits provided by encoder" - << " (min=" << encoder_bitrate_limits->min_bitrate_bps - << ", max=" << encoder_bitrate_limits->max_bitrate_bps - << ") do not intersect with limits set by app" - << " (min=" << streams.back().min_bitrate_bps - << ", max=" << encoder_config_.max_bitrate_bps - << "). The app bitrate limits will be used."; + qp_untrusted_bitrate_limit->max_bitrate_bps; + } + } + } else { + absl::optional + encoder_bitrate_limits = + encoder_->GetEncoderInfo().GetEncoderBitrateLimitsForResolution( + last_frame_info_->width * last_frame_info_->height); + + if (encoder_bitrate_limits) { + if (streams.size() == 1 && encoder_config_.simulcast_layers.size() == 1) { + // Bitrate limits can be set by app (in SDP or RtpEncodingParameters) + // or/and can be provided by encoder. In presence of both set of + // limits, the final set is derived as their intersection. + int min_bitrate_bps; + if (encoder_config_.simulcast_layers.empty() || + encoder_config_.simulcast_layers[0].min_bitrate_bps <= 0) { + min_bitrate_bps = encoder_bitrate_limits->min_bitrate_bps; + } else { + min_bitrate_bps = std::max(encoder_bitrate_limits->min_bitrate_bps, + streams.back().min_bitrate_bps); + } + + int max_bitrate_bps; + // We don't check encoder_config_.simulcast_layers[0].max_bitrate_bps + // here since encoder_config_.max_bitrate_bps is derived from it (as + // well as from other inputs). + if (encoder_config_.max_bitrate_bps <= 0) { + max_bitrate_bps = encoder_bitrate_limits->max_bitrate_bps; + } else { + max_bitrate_bps = std::min(encoder_bitrate_limits->max_bitrate_bps, + streams.back().max_bitrate_bps); + } + + if (min_bitrate_bps < max_bitrate_bps) { + streams.back().min_bitrate_bps = min_bitrate_bps; + streams.back().max_bitrate_bps = max_bitrate_bps; + streams.back().target_bitrate_bps = + std::min(streams.back().target_bitrate_bps, + encoder_bitrate_limits->max_bitrate_bps); + } else { + RTC_LOG(LS_WARNING) + << "Bitrate limits provided by encoder" + << " (min=" << encoder_bitrate_limits->min_bitrate_bps + << ", max=" << encoder_bitrate_limits->max_bitrate_bps + << ") do not intersect with limits set by app" + << " (min=" << streams.back().min_bitrate_bps + << ", max=" << encoder_config_.max_bitrate_bps + << "). The app bitrate limits will be used."; + } } } } @@ -1195,6 +1246,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { encoder_config_.min_transmit_bitrate_bps); stream_resource_manager_.ConfigureQualityScaler(info); + stream_resource_manager_.ConfigureBandwidthQualityScaler(info); } void VideoStreamEncoder::OnEncoderSettingsChanged() { @@ -2140,7 +2192,7 @@ void VideoStreamEncoder::RunPostEncode(const EncodedImage& encoded_image, } stream_resource_manager_.OnEncodeCompleted(encoded_image, time_sent_us, - encode_duration_us); + encode_duration_us, frame_size); if (bitrate_adjuster_) { bitrate_adjuster_->OnEncodedFrame( frame_size, encoded_image.SpatialIndex().value_or(0), temporal_index); diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index d64341901e..d9f11363dd 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -8210,36 +8210,6 @@ TEST_F(VideoStreamEncoderTest, video_stream_encoder_->Stop(); } -TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) { - VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); - - // Disable scaling settings in encoder info. - fake_encoder_.SetQualityScaling(false); - // Set QP not trusted in encoder info. - fake_encoder_.SetIsQpTrusted(false); - // Enable quality scaling in encoder config. - video_encoder_config.is_quality_scaling_allowed = true; - ConfigureEncoder(std::move(video_encoder_config)); - - video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( - DataRate::BitsPerSec(kTargetBitrateBps), - DataRate::BitsPerSec(kTargetBitrateBps), - DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); - - test::FrameForwarder source; - video_stream_encoder_->SetSource( - &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); - EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants()); - EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); - - source.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); - WaitForEncodedFrame(1); - video_stream_encoder_->TriggerQualityLow(); - EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); - - video_stream_encoder_->Stop(); -} - TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) { VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); @@ -8270,6 +8240,147 @@ TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) { video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, + QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) { + VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); + + // Disable scaling settings in encoder info. + fake_encoder_.SetQualityScaling(false); + // Set QP trusted in encoder info. + fake_encoder_.SetIsQpTrusted(true); + // Enable quality scaling in encoder config. + video_encoder_config.is_quality_scaling_allowed = false; + ConfigureEncoder(std::move(video_encoder_config)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + test::FrameForwarder source; + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants()); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + source.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) { + VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); + + // Disable scaling settings in encoder info. + fake_encoder_.SetQualityScaling(false); + // Set QP trusted in encoder info. + fake_encoder_.SetIsQpTrusted(false); + // Enable quality scaling in encoder config. + video_encoder_config.is_quality_scaling_allowed = false; + ConfigureEncoder(std::move(video_encoder_config)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + test::FrameForwarder source; + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants()); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + source.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) { + // Set QP trusted in encoder info. + fake_encoder_.SetIsQpTrusted(false); + + const int MinEncBitrateKbps = 30; + const int MaxEncBitrateKbps = 100; + const int MinStartBitrateKbp = 50; + const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits( + /*frame_size_pixels=*/codec_width_ * codec_height_, + /*min_start_bitrate_bps=*/MinStartBitrateKbp, + /*min_bitrate_bps=*/MinEncBitrateKbps * 1000, + /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits}); + + VideoEncoderConfig video_encoder_config; + test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config); + video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000; + video_encoder_config.simulcast_layers[0].min_bitrate_bps = + MinEncBitrateKbps * 1000; + video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(), + kMaxPayloadLength); + + video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr)); + WaitForEncodedFrame(1); + EXPECT_EQ( + MaxEncBitrateKbps, + static_cast(bitrate_allocator_factory_.codec_config().maxBitrate)); + EXPECT_EQ( + MinEncBitrateKbps, + static_cast(bitrate_allocator_factory_.codec_config().minBitrate)); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) { + // Set QP trusted in encoder info. + fake_encoder_.SetIsQpTrusted(false); + + absl::optional suitable_bitrate_limit = + EncoderInfoSettings:: + GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted( + codec_width_ * codec_height_, + EncoderInfoSettings:: + GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted()); + EXPECT_TRUE(suitable_bitrate_limit.has_value()); + + const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps; + const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps; + const int TargetEncBitrate = MaxEncBitrate; + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(TargetEncBitrate), + DataRate::BitsPerSec(TargetEncBitrate), + DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0); + + VideoEncoderConfig video_encoder_config; + test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config); + video_encoder_config.max_bitrate_bps = MaxEncBitrate; + video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate; + video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(), + kMaxPayloadLength); + + video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr)); + WaitForEncodedFrame(1); + EXPECT_EQ( + MaxEncBitrate / 1000, + static_cast(bitrate_allocator_factory_.codec_config().maxBitrate)); + EXPECT_EQ( + MinEncBitrate / 1000, + static_cast(bitrate_allocator_factory_.codec_config().minBitrate)); + + video_stream_encoder_->Stop(); +} + #if !defined(WEBRTC_IOS) // TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is // disabled by default on iOS. @@ -8330,6 +8441,102 @@ TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) { video_stream_encoder_->Stop(); } + +TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) { + VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); + + // Disable scaling settings in encoder info. + fake_encoder_.SetQualityScaling(false); + // Set QP not trusted in encoder info. + fake_encoder_.SetIsQpTrusted(false); + // Enable quality scaling in encoder config. + video_encoder_config.is_quality_scaling_allowed = true; + ConfigureEncoder(std::move(video_encoder_config)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + test::FrameForwarder source; + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants()); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + source.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + // When quality_scaler doesn't work and is_quality_scaling_allowed is + // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true. + EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) { + VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); + + // Disable scaling settings in encoder info. + fake_encoder_.SetQualityScaling(false); + // Set QP trusted in encoder info. + fake_encoder_.SetIsQpTrusted(true); + // Enable quality scaling in encoder config. + video_encoder_config.is_quality_scaling_allowed = true; + ConfigureEncoder(std::move(video_encoder_config)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + test::FrameForwarder source; + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants()); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + source.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + // bandwidth_quality_scaler isn't working, but quality_scaler is working. + EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) { + VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy(); + + // Disable scaling settings in encoder info. + fake_encoder_.SetQualityScaling(false); + // Set QP trusted in encoder info. + fake_encoder_.SetIsQpTrusted(false); + // Enable quality scaling in encoder config. + video_encoder_config.is_quality_scaling_allowed = true; + ConfigureEncoder(std::move(video_encoder_config)); + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), + DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0); + + test::FrameForwarder source; + video_stream_encoder_->SetSource( + &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE); + EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants()); + EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution); + + source.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution); + + video_stream_encoder_->Stop(); +} + #endif // Test parameters: (VideoCodecType codec, bool allow_i420_conversion)