diff --git a/api/video/video_stream_encoder_interface.h b/api/video/video_stream_encoder_interface.h index 0cd9ddcead..7181143c05 100644 --- a/api/video/video_stream_encoder_interface.h +++ b/api/video/video_stream_encoder_interface.h @@ -42,6 +42,7 @@ class VideoStreamEncoderInterface : public rtc::VideoSinkInterface { public: virtual void OnEncoderConfigurationChanged( std::vector streams, + VideoEncoderConfig::ContentType content_type, int min_transmit_bitrate_bps) = 0; }; diff --git a/modules/video_coding/utility/simulcast_rate_allocator.cc b/modules/video_coding/utility/simulcast_rate_allocator.cc index b6edfd1b9b..e54248fc23 100644 --- a/modules/video_coding/utility/simulcast_rate_allocator.cc +++ b/modules/video_coding/utility/simulcast_rate_allocator.cc @@ -58,11 +58,8 @@ float SimulcastRateAllocator::GetTemporalRateAllocation(int num_layers, SimulcastRateAllocator::SimulcastRateAllocator(const VideoCodec& codec) : codec_(codec), - hysteresis_factor_(codec.mode == VideoCodecMode::kScreensharing - ? RateControlSettings::ParseFromFieldTrials() - .GetSimulcastScreenshareHysteresisFactor() - : RateControlSettings::ParseFromFieldTrials() - .GetSimulcastVideoHysteresisFactor()) {} + hysteresis_factor_(RateControlSettings::ParseFromFieldTrials() + .GetSimulcastHysteresisFactor(codec.mode)) {} SimulcastRateAllocator::~SimulcastRateAllocator() = default; diff --git a/rtc_base/experiments/BUILD.gn b/rtc_base/experiments/BUILD.gn index 6b283b43b5..80fa70b27d 100644 --- a/rtc_base/experiments/BUILD.gn +++ b/rtc_base/experiments/BUILD.gn @@ -125,6 +125,7 @@ rtc_static_library("rate_control_settings") { "../:rtc_base_approved", "../../api/transport:field_trial_based_config", "../../api/transport:webrtc_key_value_config", + "../../api/video_codecs:video_codecs_api", "../../system_wrappers:field_trial", "//third_party/abseil-cpp/absl/memory:memory", "//third_party/abseil-cpp/absl/types:optional", @@ -141,6 +142,7 @@ if (rtc_include_tests) { "field_trial_units_unittest.cc", "normalize_simulcast_size_experiment_unittest.cc", "quality_scaling_experiment_unittest.cc", + "rate_control_settings_unittest.cc", "rtt_mult_experiment_unittest.cc", ] deps = [ @@ -148,10 +150,12 @@ if (rtc_include_tests) { ":field_trial_parser", ":normalize_simulcast_size_experiment", ":quality_scaling_experiment", + ":rate_control_settings", ":rtt_mult_experiment", "..:gunit_helpers", "../:rtc_base_tests_main", "../:rtc_base_tests_utils", + "../../api/video_codecs:video_codecs_api", "../../system_wrappers:field_trial", "../../test:field_trial", "../../test:test_support", diff --git a/rtc_base/experiments/rate_control_settings.cc b/rtc_base/experiments/rate_control_settings.cc index f465933e54..cb6e0dead4 100644 --- a/rtc_base/experiments/rate_control_settings.cc +++ b/rtc_base/experiments/rate_control_settings.cc @@ -178,6 +178,22 @@ bool RateControlSettings::LibvpxVp9TrustedRateController() const { return trust_vp9_.Get(); } +double RateControlSettings::GetSimulcastHysteresisFactor( + VideoCodecMode mode) const { + if (mode == VideoCodecMode::kScreensharing) { + return GetSimulcastScreenshareHysteresisFactor(); + } + return GetSimulcastVideoHysteresisFactor(); +} + +double RateControlSettings::GetSimulcastHysteresisFactor( + VideoEncoderConfig::ContentType content_type) const { + if (content_type == VideoEncoderConfig::ContentType::kScreen) { + return GetSimulcastScreenshareHysteresisFactor(); + } + return GetSimulcastVideoHysteresisFactor(); +} + double RateControlSettings::GetSimulcastVideoHysteresisFactor() const { return video_hysteresis_.Get(); } diff --git a/rtc_base/experiments/rate_control_settings.h b/rtc_base/experiments/rate_control_settings.h index cbc2e69d79..e7dc868591 100644 --- a/rtc_base/experiments/rate_control_settings.h +++ b/rtc_base/experiments/rate_control_settings.h @@ -13,6 +13,8 @@ #include "absl/types/optional.h" #include "api/transport/webrtc_key_value_config.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder_config.h" #include "rtc_base/experiments/field_trial_parser.h" #include "rtc_base/experiments/field_trial_units.h" @@ -41,8 +43,11 @@ class RateControlSettings final { bool LibvpxVp8TrustedRateController() const; bool LibvpxVp9TrustedRateController() const; - double GetSimulcastVideoHysteresisFactor() const; - double GetSimulcastScreenshareHysteresisFactor() const; + // TODO(bugs.webrtc.org/10272): Remove one of these when we have merged + // VideoCodecMode and VideoEncoderConfig::ContentType. + double GetSimulcastHysteresisFactor(VideoCodecMode mode) const; + double GetSimulcastHysteresisFactor( + VideoEncoderConfig::ContentType content_type) const; bool TriggerProbeOnMaxAllocatedBitrateChange() const; @@ -50,6 +55,9 @@ class RateControlSettings final { explicit RateControlSettings( const WebRtcKeyValueConfig* const key_value_config); + double GetSimulcastVideoHysteresisFactor() const; + double GetSimulcastScreenshareHysteresisFactor() const; + FieldTrialOptional congestion_window_; FieldTrialOptional congestion_window_pushback_; FieldTrialOptional pacing_factor_; diff --git a/rtc_base/experiments/rate_control_settings_unittest.cc b/rtc_base/experiments/rate_control_settings_unittest.cc new file mode 100644 index 0000000000..d049bc7a6d --- /dev/null +++ b/rtc_base/experiments/rate_control_settings_unittest.cc @@ -0,0 +1,55 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/experiments/rate_control_settings.h" + +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder_config.h" +#include "test/field_trial.h" +#include "test/gtest.h" + +namespace webrtc { + +namespace { + +TEST(RateControlSettingsTest, LibvpxTrustedRateController) { + test::ScopedFieldTrials field_trials( + "WebRTC-VideoRateControl/trust_vp8:1,trust_vp9:0/"); + const RateControlSettings rate_control_settings = + RateControlSettings::ParseFromFieldTrials(); + + EXPECT_TRUE(rate_control_settings.LibvpxVp8TrustedRateController()); + EXPECT_FALSE(rate_control_settings.LibvpxVp9TrustedRateController()); +} + +TEST(RateControlSettingsTest, GetSimulcastHysteresisFactor) { + test::ScopedFieldTrials field_trials( + "WebRTC-VideoRateControl/" + "video_hysteresis:1.2,screenshare_hysteresis:1.4/"); + const RateControlSettings rate_control_settings = + RateControlSettings::ParseFromFieldTrials(); + + EXPECT_EQ(rate_control_settings.GetSimulcastHysteresisFactor( + VideoCodecMode::kRealtimeVideo), + 1.2); + EXPECT_EQ(rate_control_settings.GetSimulcastHysteresisFactor( + VideoEncoderConfig::ContentType::kRealtimeVideo), + 1.2); + EXPECT_EQ(rate_control_settings.GetSimulcastHysteresisFactor( + VideoCodecMode::kScreensharing), + 1.4); + EXPECT_EQ(rate_control_settings.GetSimulcastHysteresisFactor( + VideoEncoderConfig::ContentType::kScreen), + 1.4); +} + +} // namespace + +} // namespace webrtc diff --git a/video/video_send_stream_impl.cc b/video/video_send_stream_impl.cc index 563555044f..266d27fc0a 100644 --- a/video/video_send_stream_impl.cc +++ b/video/video_send_stream_impl.cc @@ -89,6 +89,7 @@ int GetEncoderMinBitrateBps() { // Calculate max padding bitrate for a multi layer codec. int CalculateMaxPadBitrateBps(const std::vector& streams, + VideoEncoderConfig::ContentType content_type, int min_transmit_bitrate_bps, bool pad_to_min_bitrate, bool alr_probing) { @@ -107,12 +108,23 @@ int CalculateMaxPadBitrateBps(const std::vector& streams, // probing will handle the rest of the rampup. pad_up_to_bitrate_bps = active_streams[0].min_bitrate_bps; } else { - // Pad to min bitrate of the highest layer. - pad_up_to_bitrate_bps = - active_streams[active_streams.size() - 1].min_bitrate_bps; - // Add target_bitrate_bps of the lower layers. - for (size_t i = 0; i < active_streams.size() - 1; ++i) + // Without alr probing, pad up to start bitrate of the + // highest active stream. + const double hysteresis_factor = + RateControlSettings::ParseFromFieldTrials() + .GetSimulcastHysteresisFactor(content_type); + const size_t top_active_stream_idx = active_streams.size() - 1; + pad_up_to_bitrate_bps = std::min( + static_cast( + hysteresis_factor * + active_streams[top_active_stream_idx].min_bitrate_bps + + 0.5), + active_streams[top_active_stream_idx].target_bitrate_bps); + + // Add target_bitrate_bps of the lower active streams. + for (size_t i = 0; i < top_active_stream_idx; ++i) { pad_up_to_bitrate_bps += active_streams[i].target_bitrate_bps; + } } } else if (!active_streams.empty() && pad_to_min_bitrate) { pad_up_to_bitrate_bps = active_streams[0].min_bitrate_bps; @@ -497,14 +509,17 @@ void VideoSendStreamImpl::SignalEncoderActive() { void VideoSendStreamImpl::OnEncoderConfigurationChanged( std::vector streams, + VideoEncoderConfig::ContentType content_type, int min_transmit_bitrate_bps) { if (!worker_queue_->IsCurrent()) { rtc::WeakPtr send_stream = weak_ptr_; - worker_queue_->PostTask([send_stream, streams, min_transmit_bitrate_bps]() { - if (send_stream) - send_stream->OnEncoderConfigurationChanged(std::move(streams), - min_transmit_bitrate_bps); - }); + worker_queue_->PostTask( + [send_stream, streams, content_type, min_transmit_bitrate_bps]() { + if (send_stream) { + send_stream->OnEncoderConfigurationChanged( + std::move(streams), content_type, min_transmit_bitrate_bps); + } + }); return; } RTC_DCHECK_GE(config_->rtp.ssrcs.size(), streams.size()); @@ -530,6 +545,7 @@ void VideoSendStreamImpl::OnEncoderConfigurationChanged( std::max(static_cast(encoder_min_bitrate_bps_), encoder_max_bitrate_bps_); + // TODO(bugs.webrtc.org/10266): Query the VideoBitrateAllocator instead. const VideoCodecType codec_type = PayloadStringToCodecType(config_->rtp.payload_name); if (codec_type == kVideoCodecVP9) { @@ -537,8 +553,8 @@ void VideoSendStreamImpl::OnEncoderConfigurationChanged( : streams[0].target_bitrate_bps; } else { max_padding_bitrate_ = CalculateMaxPadBitrateBps( - streams, min_transmit_bitrate_bps, config_->suspend_below_min_bitrate, - has_alr_probing_); + streams, content_type, min_transmit_bitrate_bps, + config_->suspend_below_min_bitrate, has_alr_probing_); } // Clear stats for disabled layers. diff --git a/video/video_send_stream_impl.h b/video/video_send_stream_impl.h index 050da3e8ec..4c5e3e4634 100644 --- a/video/video_send_stream_impl.h +++ b/video/video_send_stream_impl.h @@ -112,8 +112,10 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver, // Implements BitrateAllocatorObserver. uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) override; - void OnEncoderConfigurationChanged(std::vector streams, - int min_transmit_bitrate_bps) override; + void OnEncoderConfigurationChanged( + std::vector streams, + VideoEncoderConfig::ContentType content_type, + int min_transmit_bitrate_bps) override; // Implements EncodedImageCallback. The implementation routes encoded frames // to the |payload_router_| and |config.pre_encode_callback| if set. diff --git a/video/video_send_stream_impl_unittest.cc b/video/video_send_stream_impl_unittest.cc index e57e36ddc8..41c039710b 100644 --- a/video/video_send_stream_impl_unittest.cc +++ b/video/video_send_stream_impl_unittest.cc @@ -228,6 +228,7 @@ TEST_F(VideoSendStreamImplTest, UpdatesObserverOnConfigurationChange) { static_cast(vss_impl.get()) ->OnEncoderConfigurationChanged( std::vector{qvga_stream, vga_stream}, + VideoEncoderConfig::ContentType::kRealtimeVideo, min_transmit_bitrate_bps); vss_impl->Stop(); }); @@ -292,7 +293,64 @@ TEST_F(VideoSendStreamImplTest, UpdatesObserverOnConfigurationChangeWithAlr) { static_cast(vss_impl.get()) ->OnEncoderConfigurationChanged( std::vector{low_stream, high_stream}, - min_transmit_bitrate_bps); + VideoEncoderConfig::ContentType::kScreen, min_transmit_bitrate_bps); + vss_impl->Stop(); + }); +} + +TEST_F(VideoSendStreamImplTest, + UpdatesObserverOnConfigurationChangeWithSimulcastVideoHysteresis) { + test::ScopedFieldTrials hysteresis_experiment( + "WebRTC-VideoRateControl/video_hysteresis:1.25/"); + + test_queue_.SendTask([this] { + auto vss_impl = CreateVideoSendStreamImpl( + kDefaultInitialBitrateBps, kDefaultBitratePriority, + VideoEncoderConfig::ContentType::kRealtimeVideo); + vss_impl->Start(); + + // 2-layer video simulcast. + VideoStream low_stream; + low_stream.width = 320; + low_stream.height = 240; + low_stream.max_framerate = 30; + low_stream.min_bitrate_bps = 30000; + low_stream.target_bitrate_bps = 100000; + low_stream.max_bitrate_bps = 200000; + low_stream.max_qp = 56; + low_stream.bitrate_priority = 1; + + VideoStream high_stream; + high_stream.width = 640; + high_stream.height = 480; + high_stream.max_framerate = 30; + high_stream.min_bitrate_bps = 150000; + high_stream.target_bitrate_bps = 500000; + high_stream.max_bitrate_bps = 750000; + high_stream.max_qp = 56; + high_stream.bitrate_priority = 1; + + config_.rtp.ssrcs.emplace_back(1); + config_.rtp.ssrcs.emplace_back(2); + + EXPECT_CALL(bitrate_allocator_, AddObserver(vss_impl.get(), _)) + .WillOnce(Invoke([&](BitrateAllocatorObserver*, + MediaStreamAllocationConfig config) { + EXPECT_EQ(config.min_bitrate_bps, + static_cast(low_stream.min_bitrate_bps)); + EXPECT_EQ(config.max_bitrate_bps, + static_cast(low_stream.max_bitrate_bps + + high_stream.max_bitrate_bps)); + EXPECT_EQ(config.pad_up_bitrate_bps, + static_cast(low_stream.target_bitrate_bps + + 1.25 * high_stream.min_bitrate_bps)); + })); + + static_cast(vss_impl.get()) + ->OnEncoderConfigurationChanged( + std::vector{low_stream, high_stream}, + VideoEncoderConfig::ContentType::kRealtimeVideo, + /*min_transmit_bitrate_bps=*/0); vss_impl->Stop(); }); } diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index ddb7631941..ea59b0aba0 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -660,7 +660,8 @@ void VideoStreamEncoder::ReconfigureEncoder() { pending_encoder_reconfiguration_ = false; sink_->OnEncoderConfigurationChanged( - std::move(streams), encoder_config_.min_transmit_bitrate_bps); + std::move(streams), encoder_config_.content_type, + encoder_config_.min_transmit_bitrate_bps); // Get the current target framerate, ie the maximum framerate as specified by // the current codec configuration, or any limit imposed by cpu adaption in diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 9d275adae5..fba99fe8db 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -733,8 +733,10 @@ class VideoStreamEncoderTest : public ::testing::Test { return Result(Result::OK, last_timestamp_); } - void OnEncoderConfigurationChanged(std::vector streams, - int min_transmit_bitrate_bps) override { + void OnEncoderConfigurationChanged( + std::vector streams, + VideoEncoderConfig::ContentType content_type, + int min_transmit_bitrate_bps) override { rtc::CriticalSection crit_; ++number_of_reconfigurations_; min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;