Apply resolution-bitrate limits collected from field trial (cl/294600) for AV1.

Bug: webrtc:14931
Change-Id: I1e8471a499bc884cb9479609a2b093de90f638d8
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/296120
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Commit-Queue: Michael Horowitz <mhoro@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#39582}

call/adaptation/BUILD.gn

@@ -44,6 +44,7 @@ rtc_library("resource_adaptation") {
     "../../api/video:video_stream_encoder",
     "../../api/video_codecs:video_codecs_api",
     "../../modules/video_coding:video_coding_utility",
+    "../../modules/video_coding/svc:scalability_mode_util",
     "../../rtc_base:checks",
     "../../rtc_base:logging",
     "../../rtc_base:macromagic",

call/adaptation/video_stream_adapter.cc

@@ -22,6 +22,7 @@
 #include "api/video_codecs/video_encoder.h"
 #include "call/adaptation/video_source_restrictions.h"
 #include "call/adaptation/video_stream_input_state.h"
+#include "modules/video_coding/svc/scalability_mode_util.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/logging.h"
 #include "rtc_base/numerics/safe_conversions.h"
@@ -720,7 +721,17 @@ absl::optional<uint32_t> VideoStreamAdapter::GetSingleActiveLayerPixels(
     const VideoCodec& codec) {
   int num_active = 0;
   absl::optional<uint32_t> pixels;
-  if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
+  if (codec.codecType == VideoCodecType::kVideoCodecAV1 &&
+      codec.GetScalabilityMode().has_value()) {
+    for (int i = 0;
+         i < ScalabilityModeToNumSpatialLayers(*(codec.GetScalabilityMode()));
+         ++i) {
+      if (codec.spatialLayers[i].active) {
+        ++num_active;
+        pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height;
+      }
+    }
+  } else if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
     for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) {
       if (codec.spatialLayers[i].active) {
         ++num_active;

rtc_base/experiments/encoder_info_settings.cc

@@ -38,7 +38,8 @@ constexpr float kDefaultMinBitratebps = 30000;
 std::vector<VideoEncoder::ResolutionBitrateLimits>
 EncoderInfoSettings::GetDefaultSinglecastBitrateLimits(
     VideoCodecType codec_type) {
-  // Specific limits for VP9. Other codecs use VP8 limits.
+  // Specific limits for VP9. Determining specific limits for AV1 via
+  // field trial experiment is a work in progress. Other codecs use VP8 limits.
   if (codec_type == kVideoCodecVP9) {
     return {{320 * 180, 0, 30000, 150000},
             {480 * 270, 120000, 30000, 300000},
@@ -46,6 +47,10 @@ EncoderInfoSettings::GetDefaultSinglecastBitrateLimits(
             {960 * 540, 350000, 30000, 1000000},
             {1280 * 720, 480000, 30000, 1500000}};
   }
+  // Don't override existing AV1 limits with default values.
+  if (codec_type == kVideoCodecAV1) {
+    return {};
+  }
 
   return {{320 * 180, 0, 30000, 300000},
           {480 * 270, 200000, 30000, 500000},

video/BUILD.gn

@@ -403,6 +403,7 @@ rtc_library("video_stream_encoder_impl") {
     "../modules/video_coding:video_codec_interface",
     "../modules/video_coding:video_coding_utility",
     "../modules/video_coding:webrtc_vp9_helpers",
+    "../modules/video_coding/svc:scalability_mode_util",
     "../modules/video_coding/svc:scalability_structures",
     "../modules/video_coding/svc:svc_rate_allocator",
     "../rtc_base:checks",

video/end_to_end_tests/resolution_bitrate_limits_tests.cc

@@ -41,7 +41,18 @@ struct BitrateLimits {
 };
 
 BitrateLimits GetLayerBitrateLimits(int pixels, const VideoCodec& codec) {
-  if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
+  if (codec.codecType == VideoCodecType::kVideoCodecAV1) {
+    EXPECT_TRUE(codec.GetScalabilityMode().has_value());
+    for (int i = 0;
+         i < ScalabilityModeToNumSpatialLayers(*(codec.GetScalabilityMode()));
+         ++i) {
+      if (codec.spatialLayers[i].width * codec.spatialLayers[i].height ==
+          pixels) {
+        return {DataRate::KilobitsPerSec(codec.spatialLayers[i].minBitrate),
+                DataRate::KilobitsPerSec(codec.spatialLayers[i].maxBitrate)};
+      }
+    }
+  } else if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
     for (size_t i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) {
       if (codec.spatialLayers[i].width * codec.spatialLayers[i].height ==
           pixels) {
@@ -62,6 +73,10 @@ BitrateLimits GetLayerBitrateLimits(int pixels, const VideoCodec& codec) {
   return BitrateLimits();
 }
 
+bool SupportsSpatialLayers(const std::string& payload_name) {
+  return payload_name == "VP9" || payload_name == "AV1";
+}
+
 }  // namespace
 
 class ResolutionBitrateLimitsWithScalabilityModeTest : public test::CallTest {};
@@ -122,7 +137,7 @@ class InitEncodeTest : public test::EndToEndTest,
                           const rtc::VideoSinkWants& wants) override {}
 
   size_t GetNumVideoStreams() const override {
-    return (payload_name_ == "VP9") ? 1 : configs_.size();
+    return SupportsSpatialLayers(payload_name_) ? 1 : configs_.size();
   }
 
   void ModifyVideoConfigs(
@@ -142,7 +157,7 @@ class InitEncodeTest : public test::EndToEndTest,
     encoder_config->max_bitrate_bps = -1;
     if (configs_.size() == 1 && configs_[0].bitrate.max)
       encoder_config->max_bitrate_bps = configs_[0].bitrate.max->bps();
-    if (payload_name_ == "VP9") {
+    if (SupportsSpatialLayers(payload_name_)) {
       // Simulcast layers indicates which spatial layers are active.
       encoder_config->simulcast_layers.resize(configs_.size());
     }
@@ -156,7 +171,7 @@ class InitEncodeTest : public test::EndToEndTest,
       if (configs_[i].bitrate.max)
         stream.max_bitrate_bps = configs_[i].bitrate.max->bps();
       stream.scale_resolution_down_by = scale_factor;
-      scale_factor *= (payload_name_ == "VP9") ? 1.0 : 2.0;
+      scale_factor *= SupportsSpatialLayers(payload_name_) ? 1.0 : 2.0;
     }
     SetEncoderSpecific(encoder_config, codec_type, configs_.size());
   }
@@ -458,6 +473,67 @@ TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,
   RunBaseTest(&test);
 }
 
+TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,
+       OneStreamLimitsAppliedForAv1OneSpatialLayer) {
+  webrtc::test::ScopedFieldTrials field_trials(
+      "WebRTC-GetEncoderInfoOverride/"
+      "frame_size_pixels:921600,"
+      "min_start_bitrate_bps:0,"
+      "min_bitrate_bps:32000,"
+      "max_bitrate_bps:133000/");
+
+  InitEncodeTest test(
+      "AV1", {{.active = true, .scalability_mode = ScalabilityMode::kL1T1}},
+      // Expectations:
+      {{.pixels = 1280 * 720,
+        .eq_bitrate = {DataRate::KilobitsPerSec(32),
+                       DataRate::KilobitsPerSec(133)}}});
+  RunBaseTest(&test);
+}
+
+TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,
+       LimitsAppliedForAv1Simulcast) {
+  webrtc::test::ScopedFieldTrials field_trials(
+      "WebRTC-GetEncoderInfoOverride/"
+      "frame_size_pixels:230400|921600,"
+      "min_start_bitrate_bps:0|0,"
+      "min_bitrate_bps:25000|80000,"
+      "max_bitrate_bps:400000|1200000/");
+
+  InitEncodeTest test(
+      "AV1",
+      {{.active = true, .scalability_mode = ScalabilityMode::kL1T1},
+       {.active = false}},
+      // Expectations:
+      {{.pixels = 1280 * 720,
+        .eq_bitrate = {DataRate::KilobitsPerSec(80),
+                       DataRate::KilobitsPerSec(1200)}}});
+  RunBaseTest(&test);
+}
+
+TEST_F(ResolutionBitrateLimitsWithScalabilityModeTest,
+       LimitsNotAppliedForAv1MultipleSpatialLayers) {
+  webrtc::test::ScopedFieldTrials field_trials(
+      "WebRTC-GetEncoderInfoOverride/"
+      "frame_size_pixels:230400|921600,"
+      "min_start_bitrate_bps:0|0,"
+      "min_bitrate_bps:20000|25000,"
+      "max_bitrate_bps:900000|1333000/");
+
+  InitEncodeTest test(
+      "AV1",
+      {{.active = true, .scalability_mode = ScalabilityMode::kL2T1},
+       {.active = false}},
+      // Expectations:
+      {{.pixels = 640 * 360,
+        .ne_bitrate = {DataRate::KilobitsPerSec(20),
+                       DataRate::KilobitsPerSec(900)}},
+       {.pixels = 1280 * 720,
+        .ne_bitrate = {DataRate::KilobitsPerSec(25),
+                       DataRate::KilobitsPerSec(1333)}}});
+  RunBaseTest(&test);
+}
+
 TEST_P(ResolutionBitrateLimitsTest, LimitsNotAppliedSimulcast) {
   webrtc::test::ScopedFieldTrials field_trials(
       "WebRTC-GetEncoderInfoOverride/"

video/video_send_stream_impl.cc

@@ -52,6 +52,9 @@ constexpr TimeDelta kEncoderTimeOut = TimeDelta::Seconds(2);
 constexpr double kVideoHysteresis = 1.2;
 constexpr double kScreenshareHysteresis = 1.35;
 
+constexpr int kMinDefaultAv1BitrateBps =
+    15000;  // This value acts as an absolute minimum AV1 bitrate limit.
+
 // When send-side BWE is used a stricter 1.1x pacing factor is used, rather than
 // the 2.5x which is used with receive-side BWE. Provides a more careful
 // bandwidth rampup with less risk of overshoots causing adverse effects like
@@ -194,6 +197,13 @@ uint32_t GetInitialEncoderMaxBitrate(int initial_encoder_max_bitrate) {
   return kFallbackMaxBitrateBps;
 }
 
+int GetDefaultMinVideoBitrateBps(VideoCodecType codec_type) {
+  if (codec_type == VideoCodecType::kVideoCodecAV1) {
+    return kMinDefaultAv1BitrateBps;
+  }
+  return kDefaultMinVideoBitrateBps;
+}
+
 }  // namespace
 
 PacingConfig::PacingConfig(const FieldTrialsView& field_trials)
@@ -485,7 +495,8 @@ void VideoSendStreamImpl::OnEncoderConfigurationChanged(
     encoder_min_bitrate_bps_ =
         experimental_min_bitrate
             ? experimental_min_bitrate->bps()
-            : std::max(streams[0].min_bitrate_bps, kDefaultMinVideoBitrateBps);
+            : std::max(streams[0].min_bitrate_bps,
+                       GetDefaultMinVideoBitrateBps(codec_type));
 
     encoder_max_bitrate_bps_ = 0;
     double stream_bitrate_priority_sum = 0;

video/video_stream_encoder.cc

@@ -37,6 +37,7 @@
 #include "call/adaptation/video_stream_adapter.h"
 #include "media/base/media_channel.h"
 #include "modules/video_coding/include/video_codec_initializer.h"
+#include "modules/video_coding/svc/scalability_mode_util.h"
 #include "modules/video_coding/svc/svc_rate_allocator.h"
 #include "modules/video_coding/utility/vp8_constants.h"
 #include "rtc_base/arraysize.h"
@@ -82,6 +83,17 @@ constexpr int kMaxAnimationPixels = 1280 * 720;
 
 constexpr int kDefaultMinScreenSharebps = 1200000;
 
+int GetNumSpatialLayers(const VideoCodec& codec) {
+  if (codec.codecType == kVideoCodecVP9) {
+    return codec.VP9().numberOfSpatialLayers;
+  } else if (codec.codecType == kVideoCodecAV1 &&
+             codec.GetScalabilityMode().has_value()) {
+    return ScalabilityModeToNumSpatialLayers(*(codec.GetScalabilityMode()));
+  } else {
+    return 0;
+  }
+}
+
 bool RequiresEncoderReset(const VideoCodec& prev_send_codec,
                           const VideoCodec& new_send_codec,
                           bool was_encode_called_since_last_initialization) {
@@ -383,13 +395,18 @@ int NumActiveStreams(const std::vector<VideoStream>& streams) {
   return num_active;
 }
 
-void ApplyVp9BitrateLimits(const VideoEncoder::EncoderInfo& encoder_info,
+void ApplySpatialLayerBitrateLimits(
+    const VideoEncoder::EncoderInfo& encoder_info,
     const VideoEncoderConfig& encoder_config,
     VideoCodec* codec) {
-  if (codec->codecType != VideoCodecType::kVideoCodecVP9 ||
-      encoder_config.simulcast_layers.size() <= 1 ||
-      VideoStreamEncoderResourceManager::IsSimulcastOrMultipleSpatialLayers(
-          encoder_config)) {
+  if (!(GetNumSpatialLayers(*codec) > 0)) {
+    // ApplySpatialLayerBitrateLimits() supports VP9 and AV1 (the latter with
+    // scalability mode set) only.
+    return;
+  }
+  if (VideoStreamEncoderResourceManager::IsSimulcastOrMultipleSpatialLayers(
+          encoder_config) ||
+      encoder_config.simulcast_layers.size() <= 1) {
     // Resolution bitrate limits usage is restricted to singlecast.
     return;
   }
@@ -405,7 +422,6 @@ void ApplyVp9BitrateLimits(const VideoEncoder::EncoderInfo& encoder_info,
   if (!bitrate_limits.has_value()) {
     return;
   }
-
   // Index for the active stream.
   absl::optional<size_t> index;
   for (size_t i = 0; i < encoder_config.simulcast_layers.size(); ++i) {
@@ -415,7 +431,6 @@ void ApplyVp9BitrateLimits(const VideoEncoder::EncoderInfo& encoder_info,
   if (!index.has_value()) {
     return;
   }
-
   int min_bitrate_bps;
   if (encoder_config.simulcast_layers[*index].min_bitrate_bps <= 0) {
     min_bitrate_bps = bitrate_limits->min_bitrate_bps;
@@ -439,7 +454,7 @@ void ApplyVp9BitrateLimits(const VideoEncoder::EncoderInfo& encoder_info,
     return;
   }
 
-  for (int i = 0; i < codec->VP9()->numberOfSpatialLayers; ++i) {
+  for (int i = 0; i < GetNumSpatialLayers(*codec); ++i) {
     if (codec->spatialLayers[i].active) {
       codec->spatialLayers[i].minBitrate = min_bitrate_bps / 1000;
       codec->spatialLayers[i].maxBitrate = max_bitrate_bps / 1000;
@@ -1141,13 +1156,15 @@ void VideoStreamEncoder::ReconfigureEncoder() {
     RTC_LOG(LS_ERROR) << "Failed to create encoder configuration.";
   }
 
-  if (encoder_config_.codec_type == kVideoCodecVP9) {
+  if (encoder_config_.codec_type == kVideoCodecVP9 ||
+      encoder_config_.codec_type == kVideoCodecAV1) {
     // Spatial layers configuration might impose some parity restrictions,
     // thus some cropping might be needed.
     crop_width_ = last_frame_info_->width - codec.width;
     crop_height_ = last_frame_info_->height - codec.height;
-    ApplyVp9BitrateLimits(GetEncoderInfoWithBitrateLimitUpdate(
-                              encoder_->GetEncoderInfo(), encoder_config_,
+    ApplySpatialLayerBitrateLimits(
+        GetEncoderInfoWithBitrateLimitUpdate(encoder_->GetEncoderInfo(),
+                                             encoder_config_,
                                              default_limits_allowed_),
         encoder_config_, &codec);
   }
@@ -1168,10 +1185,10 @@ void VideoStreamEncoder::ReconfigureEncoder() {
                << " active: "
                << (codec.simulcastStream[i].active ? "true" : "false") << "\n";
   }
-  if (encoder_config_.codec_type == kVideoCodecVP9) {
-    size_t num_spatial_layers = codec.VP9()->numberOfSpatialLayers;
+  if (encoder_config_.codec_type == kVideoCodecVP9 ||
+      encoder_config_.codec_type == kVideoCodecAV1) {
     log_stream << "Spatial layers:\n";
-    for (size_t i = 0; i < num_spatial_layers; ++i) {
+    for (int i = 0; i < GetNumSpatialLayers(codec); ++i) {
       log_stream << i << ": " << codec.spatialLayers[i].width << "x"
                  << codec.spatialLayers[i].height
                  << " min_kbps: " << codec.spatialLayers[i].minBitrate
@@ -1331,6 +1348,10 @@ void VideoStreamEncoder::ReconfigureEncoder() {
     num_layers = codec.VP8()->numberOfTemporalLayers;
   } else if (codec.codecType == kVideoCodecVP9) {
     num_layers = codec.VP9()->numberOfTemporalLayers;
+  } else if (codec.codecType == kVideoCodecAV1 &&
+             codec.GetScalabilityMode().has_value()) {
+    num_layers =
+        ScalabilityModeToNumTemporalLayers(*(codec.GetScalabilityMode()));
   } else if (codec.codecType == kVideoCodecH264) {
     num_layers = codec.H264()->numberOfTemporalLayers;
   } else if (codec.codecType == kVideoCodecGeneric &&
@@ -1375,8 +1396,9 @@ void VideoStreamEncoder::ReconfigureEncoder() {
 
   bool is_svc = false;
   // Set min_bitrate_bps, max_bitrate_bps, and max padding bit rate for VP9
-  // and leave only one stream containing all necessary information.
-  if (encoder_config_.codec_type == kVideoCodecVP9 &&
+  // and AV1 and leave only one stream containing all necessary information.
+  if ((encoder_config_.codec_type == kVideoCodecVP9 ||
+       encoder_config_.codec_type == kVideoCodecAV1) &&
       encoder_config_.number_of_streams == 1) {
     // Lower max bitrate to the level codec actually can produce.
     streams[0].max_bitrate_bps =
@@ -1388,7 +1410,7 @@ void VideoStreamEncoder::ReconfigureEncoder() {
         SvcRateAllocator::GetPaddingBitrate(codec).bps<int>();
     streams[0].width = streams.back().width;
     streams[0].height = streams.back().height;
-    is_svc = codec.VP9()->numberOfSpatialLayers > 1;
+    is_svc = GetNumSpatialLayers(codec) > 1;
     streams.resize(1);
   }