diff --git a/rtc_base/experiments/rate_control_settings.cc b/rtc_base/experiments/rate_control_settings.cc index b181432e88..ce300ef8fa 100644 --- a/rtc_base/experiments/rate_control_settings.cc +++ b/rtc_base/experiments/rate_control_settings.cc @@ -83,16 +83,18 @@ RateControlSettings::RateControlSettings( kDefaultScreenshareHysteresisFactor)), probe_max_allocation_("probe_max_allocation", true), bitrate_adjuster_("bitrate_adjuster", false), + adjuster_use_headroom_("adjuster_use_headroom", false), vp8_s0_boost_("vp8_s0_boost", true), vp8_dynamic_rate_("vp8_dynamic_rate", false), vp9_dynamic_rate_("vp9_dynamic_rate", false) { ParseFieldTrial({&congestion_window_, &congestion_window_pushback_}, key_value_config->Lookup("WebRTC-CongestionWindow")); - ParseFieldTrial({&pacing_factor_, &alr_probing_, &trust_vp8_, &trust_vp9_, - &video_hysteresis_, &screenshare_hysteresis_, - &probe_max_allocation_, &bitrate_adjuster_, &vp8_s0_boost_, - &vp8_dynamic_rate_, &vp9_dynamic_rate_}, - key_value_config->Lookup("WebRTC-VideoRateControl")); + ParseFieldTrial( + {&pacing_factor_, &alr_probing_, &trust_vp8_, &trust_vp9_, + &video_hysteresis_, &screenshare_hysteresis_, &probe_max_allocation_, + &bitrate_adjuster_, &adjuster_use_headroom_, &vp8_s0_boost_, + &vp8_dynamic_rate_, &vp9_dynamic_rate_}, + key_value_config->Lookup("WebRTC-VideoRateControl")); } RateControlSettings::~RateControlSettings() = default; @@ -188,4 +190,8 @@ bool RateControlSettings::UseEncoderBitrateAdjuster() const { return bitrate_adjuster_.Get(); } +bool RateControlSettings::BitrateAdjusterCanUseNetworkHeadroom() const { + return adjuster_use_headroom_.Get(); +} + } // namespace webrtc diff --git a/rtc_base/experiments/rate_control_settings.h b/rtc_base/experiments/rate_control_settings.h index c907b33a0b..a36d9be3ff 100644 --- a/rtc_base/experiments/rate_control_settings.h +++ b/rtc_base/experiments/rate_control_settings.h @@ -54,6 +54,7 @@ class RateControlSettings final { bool TriggerProbeOnMaxAllocatedBitrateChange() const; bool UseEncoderBitrateAdjuster() const; + bool BitrateAdjusterCanUseNetworkHeadroom() const; private: explicit RateControlSettings( @@ -72,6 +73,7 @@ class RateControlSettings final { FieldTrialParameter screenshare_hysteresis_; FieldTrialParameter probe_max_allocation_; FieldTrialParameter bitrate_adjuster_; + FieldTrialParameter adjuster_use_headroom_; FieldTrialParameter vp8_s0_boost_; FieldTrialParameter vp8_dynamic_rate_; FieldTrialParameter vp9_dynamic_rate_; diff --git a/video/encoder_bitrate_adjuster.cc b/video/encoder_bitrate_adjuster.cc index 97fe363c08..34921282a9 100644 --- a/video/encoder_bitrate_adjuster.cc +++ b/video/encoder_bitrate_adjuster.cc @@ -11,19 +11,42 @@ #include "video/encoder_bitrate_adjuster.h" #include +#include #include "absl/memory/memory.h" +#include "rtc_base/experiments/rate_control_settings.h" #include "rtc_base/logging.h" #include "rtc_base/time_utils.h" +#include "system_wrappers/include/field_trial.h" namespace webrtc { +namespace { +// Helper struct with metadata for a single spatial layer. +struct LayerRateInfo { + double link_utilization_factor = 0.0; + double media_utilization_factor = 0.0; + DataRate target_rate = DataRate::Zero(); + DataRate WantedOvershoot() const { + // If there is headroom, allow bitrate to go up to media rate limit. + // Still limit media utilization to 1.0, so we don't overshoot over long + // runs even if we have headroom. + const double max_media_utilization = + std::max(1.0, media_utilization_factor); + if (link_utilization_factor > max_media_utilization) { + return (link_utilization_factor - max_media_utilization) * target_rate; + } + return DataRate::Zero(); + } +}; +} // namespace constexpr int64_t EncoderBitrateAdjuster::kWindowSizeMs; constexpr size_t EncoderBitrateAdjuster::kMinFramesSinceLayoutChange; constexpr double EncoderBitrateAdjuster::kDefaultUtilizationFactor; EncoderBitrateAdjuster::EncoderBitrateAdjuster(const VideoCodec& codec_settings) - : current_total_framerate_fps_(0), + : utilize_bandwidth_headroom_(RateControlSettings::ParseFromFieldTrials() + .BitrateAdjusterCanUseNetworkHeadroom()), frames_since_layout_change_(0), min_bitrates_bps_{} { if (codec_settings.codecType == VideoCodecType::kVideoCodecVP9) { @@ -48,10 +71,8 @@ EncoderBitrateAdjuster::EncoderBitrateAdjuster(const VideoCodec& codec_settings) EncoderBitrateAdjuster::~EncoderBitrateAdjuster() = default; VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation( - const VideoBitrateAllocation& bitrate_allocation, - int framerate_fps) { - current_bitrate_allocation_ = bitrate_allocation; - current_total_framerate_fps_ = framerate_fps; + const VideoEncoder::RateControlParameters& rates) { + current_rate_control_parameters_ = rates; // First check that overshoot detectors exist, and store per spatial layer // how many active temporal layers we have. @@ -60,7 +81,7 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation( active_tls_[si] = 0; for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { // Layer is enabled iff it has both positive bitrate and framerate target. - if (bitrate_allocation.GetBitrate(si, ti) > 0 && + if (rates.bitrate.GetBitrate(si, ti) > 0 && current_fps_allocation_[si].size() > ti && current_fps_allocation_[si][ti] > 0) { ++active_tls_[si]; @@ -80,85 +101,157 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation( // Next poll the overshoot detectors and populate the adjusted allocation. const int64_t now_ms = rtc::TimeMillis(); VideoBitrateAllocation adjusted_allocation; + std::vector layer_infos; + DataRate wanted_overshoot_sum = DataRate::Zero(); + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { - const uint32_t spatial_layer_bitrate_bps = - bitrate_allocation.GetSpatialLayerSum(si); + layer_infos.emplace_back(); + LayerRateInfo& layer_info = layer_infos.back(); + + layer_info.target_rate = + DataRate::bps(rates.bitrate.GetSpatialLayerSum(si)); // Adjustment is done per spatial layer only (not per temporal layer). - double utilization_factor; if (frames_since_layout_change_ < kMinFramesSinceLayoutChange) { - utilization_factor = kDefaultUtilizationFactor; - } else if (active_tls_[si] == 0 || spatial_layer_bitrate_bps == 0) { + layer_info.link_utilization_factor = kDefaultUtilizationFactor; + layer_info.media_utilization_factor = kDefaultUtilizationFactor; + } else if (active_tls_[si] == 0 || + layer_info.target_rate == DataRate::Zero()) { // No signaled temporal layers, or no bitrate set. Could either be unused // spatial layer or bitrate dynamic mode; pass bitrate through without any // change. - utilization_factor = 1.0; + layer_info.link_utilization_factor = 1.0; + layer_info.media_utilization_factor = 1.0; } else if (active_tls_[si] == 1) { // A single active temporal layer, this might mean single layer or that // encoder does not support temporal layers. Merge target bitrates for // this spatial layer. RTC_DCHECK(overshoot_detectors_[si][0]); - utilization_factor = overshoot_detectors_[si][0] - ->GetNetworkRateUtilizationFactor(now_ms) - .value_or(kDefaultUtilizationFactor); - } else if (spatial_layer_bitrate_bps > 0) { + layer_info.link_utilization_factor = + overshoot_detectors_[si][0] + ->GetNetworkRateUtilizationFactor(now_ms) + .value_or(kDefaultUtilizationFactor); + layer_info.media_utilization_factor = + overshoot_detectors_[si][0] + ->GetMediaRateUtilizationFactor(now_ms) + .value_or(kDefaultUtilizationFactor); + } else if (layer_info.target_rate > DataRate::Zero()) { // Multiple temporal layers enabled for this spatial layer. Update rate // for each of them and make a weighted average of utilization factors, // with bitrate fraction used as weight. // If any layer is missing a utilization factor, fall back to default. - utilization_factor = 0.0; + layer_info.link_utilization_factor = 0.0; + layer_info.media_utilization_factor = 0.0; for (size_t ti = 0; ti < active_tls_[si]; ++ti) { RTC_DCHECK(overshoot_detectors_[si][ti]); - const absl::optional ti_utilization_factor = + const absl::optional ti_link_utilization_factor = overshoot_detectors_[si][ti]->GetNetworkRateUtilizationFactor( now_ms); - if (!ti_utilization_factor) { - utilization_factor = kDefaultUtilizationFactor; + const absl::optional ti_media_utilization_factor = + overshoot_detectors_[si][ti]->GetMediaRateUtilizationFactor(now_ms); + if (!ti_link_utilization_factor || !ti_media_utilization_factor) { + layer_info.link_utilization_factor = kDefaultUtilizationFactor; + layer_info.media_utilization_factor = kDefaultUtilizationFactor; break; } const double weight = - static_cast(bitrate_allocation.GetBitrate(si, ti)) / - spatial_layer_bitrate_bps; - utilization_factor += weight * ti_utilization_factor.value(); + static_cast(rates.bitrate.GetBitrate(si, ti)) / + layer_info.target_rate.bps(); + layer_info.link_utilization_factor += + weight * ti_link_utilization_factor.value(); + layer_info.media_utilization_factor += + weight * ti_media_utilization_factor.value(); } } else { RTC_NOTREACHED(); } - // Don't boost target bitrate if encoder is under-using. - utilization_factor = std::max(utilization_factor, 1.0); + if (layer_info.link_utilization_factor < 1.0) { + // TODO(sprang): Consider checking underuse and allowing it to cancel some + // potential overuse by other streams. - // Don't reduce encoder target below 50%, in which case the frame dropper - // should kick in instead. - utilization_factor = std::min(utilization_factor, 2.0); + // Don't boost target bitrate if encoder is under-using. + layer_info.link_utilization_factor = 1.0; + } else { + // Don't reduce encoder target below 50%, in which case the frame dropper + // should kick in instead. + layer_info.link_utilization_factor = + std::min(layer_info.link_utilization_factor, 2.0); - if (min_bitrates_bps_[si] > 0 && spatial_layer_bitrate_bps > 0 && - min_bitrates_bps_[si] < spatial_layer_bitrate_bps) { - // Make sure rate adjuster doesn't push target bitrate below minimum. - utilization_factor = std::min( - utilization_factor, static_cast(spatial_layer_bitrate_bps) / - min_bitrates_bps_[si]); + // Keep track of sum of desired overshoot bitrate. + wanted_overshoot_sum += layer_info.WantedOvershoot(); + } + } + + // Available link headroom that can be used to fill wanted overshoot. + DataRate available_headroom = DataRate::Zero(); + if (utilize_bandwidth_headroom_) { + available_headroom = + rates.bandwidth_allocation - DataRate::bps(rates.bitrate.get_sum_bps()); + } + + // All wanted overshoots are satisfied in the same proportion based on + // available headroom. + const double granted_overshoot_ratio = + wanted_overshoot_sum == DataRate::Zero() + ? 0.0 + : std::min(1.0, available_headroom.bps() / + wanted_overshoot_sum.bps()); + + for (size_t si = 0; si < kMaxSpatialLayers; ++si) { + LayerRateInfo& layer_info = layer_infos[si]; + double utilization_factor = layer_info.link_utilization_factor; + DataRate allowed_overshoot = + granted_overshoot_ratio * layer_info.WantedOvershoot(); + if (allowed_overshoot > DataRate::Zero()) { + // Pretend the target bitrate is higher by the allowed overshoot. + // Since utilization_factor = actual_bitrate / target_bitrate, it can be + // done by multiplying by old_target_bitrate / new_target_bitrate. + utilization_factor *= layer_info.target_rate.bps() / + (allowed_overshoot.bps() + + layer_info.target_rate.bps()); } - if (spatial_layer_bitrate_bps > 0) { - RTC_LOG(LS_VERBOSE) << "Utilization factor for spatial index " << si - << ": " << utilization_factor; + if (min_bitrates_bps_[si] > 0 && + layer_info.target_rate > DataRate::Zero() && + DataRate::bps(min_bitrates_bps_[si]) < layer_info.target_rate) { + // Make sure rate adjuster doesn't push target bitrate below minimum. + utilization_factor = + std::min(utilization_factor, layer_info.target_rate.bps() / + min_bitrates_bps_[si]); + } + + if (layer_info.target_rate > DataRate::Zero()) { + RTC_LOG(LS_VERBOSE) << "Utilization factors for spatial index " << si + << ": link = " << layer_info.link_utilization_factor + << ", media = " << layer_info.media_utilization_factor + << ", wanted overshoot = " + << layer_info.WantedOvershoot().bps() + << " bps, available headroom = " + << available_headroom.bps() + << " bps, total utilization factor = " + << utilization_factor; } // Populate the adjusted allocation with determined utilization factor. if (active_tls_[si] == 1 && - spatial_layer_bitrate_bps > bitrate_allocation.GetBitrate(si, 0)) { + layer_info.target_rate > + DataRate::bps(rates.bitrate.GetBitrate(si, 0))) { // Bitrate allocation indicates temporal layer usage, but encoder // does not seem to support it. Pipe all bitrate into a single // overshoot detector. - uint32_t adjusted_layer_bitrate_bps = static_cast( - spatial_layer_bitrate_bps / utilization_factor + 0.5); + uint32_t adjusted_layer_bitrate_bps = + std::min(static_cast( + layer_info.target_rate.bps() / utilization_factor + 0.5), + layer_info.target_rate.bps()); adjusted_allocation.SetBitrate(si, 0, adjusted_layer_bitrate_bps); } else { for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { - if (bitrate_allocation.HasBitrate(si, ti)) { - uint32_t adjusted_layer_bitrate_bps = static_cast( - bitrate_allocation.GetBitrate(si, ti) / utilization_factor + 0.5); + if (rates.bitrate.HasBitrate(si, ti)) { + uint32_t adjusted_layer_bitrate_bps = std::min( + static_cast( + rates.bitrate.GetBitrate(si, ti) / utilization_factor + 0.5), + rates.bitrate.GetBitrate(si, ti)); adjusted_allocation.SetBitrate(si, ti, adjusted_layer_bitrate_bps); } } @@ -168,7 +261,7 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation( // constraint has been met. const uint32_t adjusted_spatial_layer_sum = adjusted_allocation.GetSpatialLayerSum(si); - if (spatial_layer_bitrate_bps > 0 && + if (layer_info.target_rate > DataRate::Zero() && adjusted_spatial_layer_sum < min_bitrates_bps_[si]) { adjusted_allocation.SetBitrate(si, 0, adjusted_allocation.GetBitrate(si, 0) + @@ -191,7 +284,7 @@ VideoBitrateAllocation EncoderBitrateAdjuster::AdjustRateAllocation( overshoot_detectors_[si][ti]->SetTargetRate( DataRate::bps(layer_bitrate_bps), - fps_fraction * current_total_framerate_fps_, now_ms); + fps_fraction * rates.framerate_fps, now_ms); } } } @@ -207,8 +300,7 @@ void EncoderBitrateAdjuster::OnEncoderInfo( } // Trigger re-allocation so that overshoot detectors have correct targets. - AdjustRateAllocation(current_bitrate_allocation_, - current_total_framerate_fps_); + AdjustRateAllocation(current_rate_control_parameters_); } void EncoderBitrateAdjuster::OnEncodedFrame(const EncodedImage& encoded_image, @@ -231,8 +323,7 @@ void EncoderBitrateAdjuster::Reset() { } // Call AdjustRateAllocation() with the last know bitrate allocation, so that // the appropriate overuse detectors are immediately re-created. - AdjustRateAllocation(current_bitrate_allocation_, - current_total_framerate_fps_); + AdjustRateAllocation(current_rate_control_parameters_); } } // namespace webrtc diff --git a/video/encoder_bitrate_adjuster.h b/video/encoder_bitrate_adjuster.h index 8901ad4034..b142519b4e 100644 --- a/video/encoder_bitrate_adjuster.h +++ b/video/encoder_bitrate_adjuster.h @@ -40,8 +40,7 @@ class EncoderBitrateAdjuster { // Adjusts the given rate allocation to make it paceable within the target // rates. VideoBitrateAllocation AdjustRateAllocation( - const VideoBitrateAllocation& bitrate_allocation, - int framerate_fps); + const VideoEncoder::RateControlParameters& rates); // Updated overuse detectors with data about the encoder, specifically about // the temporal layer frame rate allocation. @@ -53,8 +52,9 @@ class EncoderBitrateAdjuster { void Reset(); private: - VideoBitrateAllocation current_bitrate_allocation_; - int current_total_framerate_fps_; + const bool utilize_bandwidth_headroom_; + + VideoEncoder::RateControlParameters current_rate_control_parameters_; // FPS allocation of temporal layers, per spatial layer. Represented as a Q8 // fraction; 0 = 0%, 255 = 100%. See VideoEncoder::EncoderInfo.fps_allocation. absl::InlinedVector diff --git a/video/encoder_bitrate_adjuster_unittest.cc b/video/encoder_bitrate_adjuster_unittest.cc index 312fde7cb3..27a38c4b8a 100644 --- a/video/encoder_bitrate_adjuster_unittest.cc +++ b/video/encoder_bitrate_adjuster_unittest.cc @@ -16,19 +16,28 @@ #include "api/units/data_rate.h" #include "rtc_base/fake_clock.h" #include "rtc_base/numerics/safe_conversions.h" +#include "test/field_trial.h" #include "test/gtest.h" namespace webrtc { +namespace test { class EncoderBitrateAdjusterTest : public ::testing::Test { public: static constexpr int64_t kWindowSizeMs = 3000; static constexpr int kDefaultBitrateBps = 300000; static constexpr int kDefaultFrameRateFps = 30; + // For network utilization higher than media utilization, loop over a + // sequence where the first half undershoots and the second half overshoots + // by the same amount. + static constexpr int kSequenceLength = 4; + static_assert(kSequenceLength % 2 == 0, "Sequence length must be even."); + EncoderBitrateAdjusterTest() : target_bitrate_(DataRate::bps(kDefaultBitrateBps)), target_framerate_fps_(kDefaultFrameRateFps), - tl_pattern_idx_{} {} + tl_pattern_idx_{}, + sequence_idx_{} {} protected: void SetUpAdjuster(size_t num_spatial_layers, @@ -73,12 +82,24 @@ class EncoderBitrateAdjusterTest : public ::testing::Test { adjuster_ = absl::make_unique(codec_); adjuster_->OnEncoderInfo(encoder_info_); - current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( - current_input_allocation_, target_framerate_fps_); + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_)); } - void InsertFrames(std::vector> utilization_factors, + void InsertFrames(std::vector> media_utilization_factors, int64_t duration_ms) { + InsertFrames(media_utilization_factors, media_utilization_factors, + duration_ms); + } + + void InsertFrames( + std::vector> media_utilization_factors, + std::vector> network_utilization_factors, + int64_t duration_ms) { + RTC_DCHECK_EQ(media_utilization_factors.size(), + network_utilization_factors.size()); + constexpr size_t kMaxFrameSize = 100000; uint8_t buffer[kMaxFrameSize]; @@ -107,19 +128,52 @@ class EncoderBitrateAdjusterTest : public ::testing::Test { (target_framerate_fps_ * layer_fps_fraction) / VideoEncoder::EncoderInfo::kMaxFramerateFraction; } - double utilization_factor = 1.0; - if (utilization_factors.size() > si && - utilization_factors[si].size() > ti) { - utilization_factor = utilization_factors[si][ti]; + double media_utilization_factor = 1.0; + double network_utilization_factor = 1.0; + if (media_utilization_factors.size() > si) { + RTC_DCHECK_EQ(media_utilization_factors[si].size(), + network_utilization_factors[si].size()); + if (media_utilization_factors[si].size() > ti) { + media_utilization_factor = media_utilization_factors[si][ti]; + network_utilization_factor = network_utilization_factors[si][ti]; + } } - size_t frame_size_bytes = utilization_factor * - (layer_bitrate_bps / 8.0) / - layer_framerate_fps; + RTC_DCHECK_GE(network_utilization_factor, media_utilization_factor); + + // Frame size based on constant (media) overshoot. + const size_t media_frame_size = media_utilization_factor * + (layer_bitrate_bps / 8.0) / + layer_framerate_fps; + + constexpr int kFramesWithPenalty = (kSequenceLength / 2) - 1; + RTC_DCHECK_GT(kFramesWithPenalty, 0); + + // The positive/negative size diff needed to achieve network rate but + // not media rate penalty is the difference between the utilization + // factors times the media rate frame size, then scaled by the fraction + // between total frames and penalized frames in the sequence. + // Cap to media frame size to avoid negative size undershoot. + const size_t network_frame_size_diff_bytes = std::min( + media_frame_size, + static_cast( + (((network_utilization_factor - media_utilization_factor) * + media_frame_size) * + kSequenceLength) / + kFramesWithPenalty + + 0.5)); + + int sequence_idx = sequence_idx_[si][ti]; + sequence_idx_[si][ti] = (sequence_idx_[si][ti] + 1) % kSequenceLength; + const size_t frame_size_bytes = + (sequence_idx < kSequenceLength / 2) + ? media_frame_size - network_frame_size_diff_bytes + : media_frame_size + network_frame_size_diff_bytes; EncodedImage image(buffer, 0, kMaxFrameSize); image.set_size(frame_size_bytes); image.SetSpatialIndex(si); adjuster_->OnEncodedFrame(image, ti); + sequence_idx = ++sequence_idx % kSequenceLength; } } } @@ -184,6 +238,7 @@ class EncoderBitrateAdjusterTest : public ::testing::Test { DataRate target_bitrate_; double target_framerate_fps_; int tl_pattern_idx_[kMaxSpatialLayers]; + int sequence_idx_[kMaxSpatialLayers][kMaxTemporalStreams]; const std::vector kTlPatterns[kMaxTemporalStreams] = { {0}, @@ -198,8 +253,9 @@ TEST_F(EncoderBitrateAdjusterTest, SingleLayerOptimal) { target_framerate_fps_ = 30; SetUpAdjuster(1, 1, false); InsertFrames({{1.0}}, kWindowSizeMs); - current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( - current_input_allocation_, target_framerate_fps_); + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_)); // Adjusted allocation near input. Allow 1% error margin due to rounding // errors etc. ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.01); @@ -211,8 +267,9 @@ TEST_F(EncoderBitrateAdjusterTest, SingleLayerOveruse) { target_framerate_fps_ = 30; SetUpAdjuster(1, 1, false); InsertFrames({{1.2}}, kWindowSizeMs); - current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( - current_input_allocation_, target_framerate_fps_); + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_)); // Adjusted allocation lowered by 20%. ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.2), current_adjusted_allocation_, 0.01); @@ -224,8 +281,9 @@ TEST_F(EncoderBitrateAdjusterTest, SingleLayerUnderuse) { target_framerate_fps_ = 30; SetUpAdjuster(1, 1, false); InsertFrames({{0.5}}, kWindowSizeMs); - current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( - current_input_allocation_, target_framerate_fps_); + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_)); // Undershoot, adjusted should exactly match input. ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.00); } @@ -238,8 +296,9 @@ TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersOptimalSize) { target_framerate_fps_ = 30; SetUpAdjuster(1, 3, false); InsertFrames({{1.0, 1.0, 1.0}}, kWindowSizeMs); - current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( - current_input_allocation_, target_framerate_fps_); + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_)); ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.01); } @@ -252,8 +311,9 @@ TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersOvershoot) { target_framerate_fps_ = 30; SetUpAdjuster(1, 3, false); InsertFrames({{1.1, 1.1, 1.1}}, kWindowSizeMs); - current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( - current_input_allocation_, target_framerate_fps_); + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_)); // Adjusted allocation lowered by 10%. ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.1), current_adjusted_allocation_, 0.01); @@ -267,8 +327,9 @@ TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersUndershoot) { target_framerate_fps_ = 30; SetUpAdjuster(1, 3, false); InsertFrames({{0.8, 0.8, 0.8}}, kWindowSizeMs); - current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( - current_input_allocation_, target_framerate_fps_); + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_)); // Adjusted allocation identical since we don't boost bitrates. ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.0); } @@ -282,8 +343,9 @@ TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersSkewedOvershoot) { target_framerate_fps_ = 30; SetUpAdjuster(1, 3, false); InsertFrames({{1.1, 1.2, 1.2}}, kWindowSizeMs); - current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( - current_input_allocation_, target_framerate_fps_); + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_)); // Expected overshoot is weighted by bitrate: // (0.6 * 1.1 + 0.2 * 1.2 + 0.2 * 1.2) = 1.14 ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.14), @@ -300,8 +362,9 @@ TEST_F(EncoderBitrateAdjusterTest, FourTemporalLayersSkewedOvershoot) { target_framerate_fps_ = 30; SetUpAdjuster(1, 4, false); InsertFrames({{1.1, 1.2, 1.2, 1.2}}, kWindowSizeMs); - current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( - current_input_allocation_, target_framerate_fps_); + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_)); // Expected overshoot is weighted by bitrate: // (0.4 * 1.1 + 0.3 * 1.2 + 0.15 * 1.2 + 0.15 * 1.2) = 1.16 ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.16), @@ -317,8 +380,9 @@ TEST_F(EncoderBitrateAdjusterTest, ThreeTemporalLayersNonLayeredEncoder) { target_framerate_fps_ = 30; SetUpAdjuster(1, 1, false); InsertFrames({{1.1}}, kWindowSizeMs); - current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( - current_input_allocation_, target_framerate_fps_); + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_)); // Expect the actual 10% overuse to be detected and the allocation to // only contain the one entry. VideoBitrateAllocation expected_allocation; @@ -339,8 +403,9 @@ TEST_F(EncoderBitrateAdjusterTest, IgnoredStream) { adjuster_->OnEncoderInfo(encoder_info_); InsertFrames({{1.1}}, kWindowSizeMs); - current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( - current_input_allocation_, target_framerate_fps_); + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_)); // Values passed through. ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.00); @@ -360,8 +425,9 @@ TEST_F(EncoderBitrateAdjusterTest, DifferentSpatialOvershoots) { for (int i = 0; i < 2; ++i) { SetUpAdjuster(2, 3, i == 0); InsertFrames({{1.05, 1.05, 1.05}, {1.25, 1.25, 1.25}}, kWindowSizeMs); - current_adjusted_allocation_ = adjuster_->AdjustRateAllocation( - current_input_allocation_, target_framerate_fps_); + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_)); VideoBitrateAllocation expected_allocation; for (size_t ti = 0; ti < 3; ++ti) { expected_allocation.SetBitrate( @@ -377,4 +443,89 @@ TEST_F(EncoderBitrateAdjusterTest, DifferentSpatialOvershoots) { } } +TEST_F(EncoderBitrateAdjusterTest, HeadroomAllowsOvershootToMediaRate) { + // Two streams, both with three temporal layers. + // Media rate is 1.0, but network rate is higher. + ScopedFieldTrials field_trial( + "WebRTC-VideoRateControl/adjuster_use_headroom:true/"); + + const uint32_t kS0Bitrate = 300000; + const uint32_t kS1Bitrate = 900000; + current_input_allocation_.SetBitrate(0, 0, kS0Bitrate / 3); + current_input_allocation_.SetBitrate(0, 1, kS0Bitrate / 3); + current_input_allocation_.SetBitrate(0, 2, kS0Bitrate / 3); + current_input_allocation_.SetBitrate(1, 0, kS1Bitrate / 3); + current_input_allocation_.SetBitrate(1, 1, kS1Bitrate / 3); + current_input_allocation_.SetBitrate(1, 2, kS1Bitrate / 3); + + target_framerate_fps_ = 30; + + // Run twice, once configured as simulcast and once as VP9 SVC. + for (int i = 0; i < 2; ++i) { + SetUpAdjuster(2, 3, i == 0); + // Network rate has 10% overshoot, but media rate is correct at 1.0. + InsertFrames({{1.0, 1.0, 1.0}, {1.0, 1.0, 1.0}}, + {{1.1, 1.1, 1.1}, {1.1, 1.1, 1.1}}, + kWindowSizeMs * kSequenceLength); + + // Push back by 10%. + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_)); + ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.1), + current_adjusted_allocation_, 0.01); + + // Add 10% link headroom, overshoot is now allowed. + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_, + DataRate::bps(current_input_allocation_.get_sum_bps() * 1.1))); + ExpectNear(current_input_allocation_, current_adjusted_allocation_, 0.01); + } +} + +TEST_F(EncoderBitrateAdjusterTest, DontExceedMediaRateEvenWithHeadroom) { + // Two streams, both with three temporal layers. + // Media rate is 1.1, but network rate is higher. + ScopedFieldTrials field_trial( + "WebRTC-VideoRateControl/adjuster_use_headroom:true/"); + + const uint32_t kS0Bitrate = 300000; + const uint32_t kS1Bitrate = 900000; + current_input_allocation_.SetBitrate(0, 0, kS0Bitrate / 3); + current_input_allocation_.SetBitrate(0, 1, kS0Bitrate / 3); + current_input_allocation_.SetBitrate(0, 2, kS0Bitrate / 3); + current_input_allocation_.SetBitrate(1, 0, kS1Bitrate / 3); + current_input_allocation_.SetBitrate(1, 1, kS1Bitrate / 3); + current_input_allocation_.SetBitrate(1, 2, kS1Bitrate / 3); + + target_framerate_fps_ = 30; + + // Run twice, once configured as simulcast and once as VP9 SVC. + for (int i = 0; i < 2; ++i) { + SetUpAdjuster(2, 3, i == 0); + // Network rate has 30% overshoot, media rate has 10% overshoot. + InsertFrames({{1.1, 1.1, 1.1}, {1.1, 1.1, 1.1}}, + {{1.3, 1.3, 1.3}, {1.3, 1.3, 1.3}}, + kWindowSizeMs * kSequenceLength); + + // Push back by 30%. + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_)); + // The up-down causes a bit more noise, allow slightly more error margin. + ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.3), + current_adjusted_allocation_, 0.015); + + // Add 100% link headroom, overshoot from network to media rate is allowed. + current_adjusted_allocation_ = + adjuster_->AdjustRateAllocation(VideoEncoder::RateControlParameters( + current_input_allocation_, target_framerate_fps_, + DataRate::bps(current_input_allocation_.get_sum_bps() * 2))); + ExpectNear(MultiplyAllocation(current_input_allocation_, 1 / 1.1), + current_adjusted_allocation_, 0.015); + } +} + +} // namespace test } // namespace webrtc diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 9c184bad42..dbe2d00234 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -1042,21 +1042,20 @@ VideoStreamEncoder::UpdateBitrateAllocationAndNotifyObserver( } } + EncoderRateSettings new_rate_settings = rate_settings; + new_rate_settings.bitrate = new_allocation; + if (bitrate_adjuster_) { VideoBitrateAllocation adjusted_allocation = - bitrate_adjuster_->AdjustRateAllocation( - new_allocation, - static_cast(rate_settings.framerate_fps + 0.5)); + bitrate_adjuster_->AdjustRateAllocation(new_rate_settings); RTC_LOG(LS_VERBOSE) << "Adjusting allocation, fps = " << rate_settings.framerate_fps << ", from " << new_allocation.ToString() << ", to " << adjusted_allocation.ToString(); - new_allocation = adjusted_allocation; + new_rate_settings.bitrate = adjusted_allocation; } - return EncoderRateSettings(new_allocation, rate_settings.framerate_fps, - rate_settings.bandwidth_allocation, - rate_settings.encoder_target); + return new_rate_settings; } uint32_t VideoStreamEncoder::GetInputFramerateFps() {