In Av1 encoder propagate zero bitrate as inactive decode target

Bug: webrtc:10342
Change-Id: I019acb6cca433db1551c22dd139a735ef976cff5
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/178101
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Commit-Queue: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31567}

common_video/generic_frame_descriptor/generic_frame_info.h

@@ -11,6 +11,7 @@
 #ifndef COMMON_VIDEO_GENERIC_FRAME_DESCRIPTOR_GENERIC_FRAME_INFO_H_
 #define COMMON_VIDEO_GENERIC_FRAME_DESCRIPTOR_GENERIC_FRAME_INFO_H_
 
+#include <bitset>
 #include <initializer_list>
 #include <vector>
 
@@ -40,6 +41,7 @@ struct GenericFrameInfo : public FrameDependencyTemplate {
 
   absl::InlinedVector<CodecBufferUsage, kMaxEncoderBuffers> encoder_buffers;
   std::vector<bool> part_of_chain;
+  std::bitset<32> active_decode_targets = ~uint32_t{0};
 };
 
 class GenericFrameInfo::Builder {

modules/video_coding/codecs/av1/BUILD.gn

@@ -42,6 +42,7 @@ rtc_source_set("scalable_video_controller") {
   ]
   deps = [
     "../../../../api/transport/rtp:dependency_descriptor",
+    "../../../../api/video:video_bitrate_allocation",
     "../../../../common_video/generic_frame_descriptor",
     "../../../../rtc_base:checks",
   ]

modules/video_coding/codecs/av1/libaom_av1_encoder.cc

@@ -544,6 +544,7 @@ void LibaomAv1Encoder::SetRates(const RateControlParameters& parameters) {
     RTC_DCHECK_LE(rc_target_bitrate_kbps, encoder_settings_.maxBitrate);
   RTC_DCHECK_GE(rc_target_bitrate_kbps, encoder_settings_.minBitrate);
 
+  svc_controller_->OnRatesUpdated(parameters.bitrate);
   // Set target bit rate.
   cfg_.rc_target_bitrate = rc_target_bitrate_kbps;
 

modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc

@@ -11,15 +11,38 @@
 #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
 
 #include <memory>
+#include <vector>
 
+#include "absl/types/optional.h"
 #include "api/video_codecs/video_codec.h"
 #include "api/video_codecs/video_encoder.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_l1t2.h"
+#include "modules/video_coding/codecs/test/encoded_video_frame_producer.h"
 #include "modules/video_coding/include/video_error_codes.h"
+#include "test/gmock.h"
 #include "test/gtest.h"
 
 namespace webrtc {
 namespace {
 
+using ::testing::SizeIs;
+
+VideoCodec DefaultCodecSettings() {
+  VideoCodec codec_settings;
+  codec_settings.width = 320;
+  codec_settings.height = 180;
+  codec_settings.maxFramerate = 30;
+  codec_settings.maxBitrate = 1000;
+  codec_settings.qpMax = 63;
+  return codec_settings;
+}
+
+VideoEncoder::Settings DefaultEncoderSettings() {
+  return VideoEncoder::Settings(
+      VideoEncoder::Capabilities(/*loss_notification=*/false),
+      /*number_of_cores=*/1, /*max_payload_size=*/1200);
+}
+
 TEST(LibaomAv1EncoderTest, CanCreate) {
   std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder();
   EXPECT_TRUE(encoder);
@@ -28,18 +51,37 @@ TEST(LibaomAv1EncoderTest, CanCreate) {
 TEST(LibaomAv1EncoderTest, InitAndRelease) {
   std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder();
   ASSERT_TRUE(encoder);
-  VideoCodec codec_settings;
-  codec_settings.width = 1280;
-  codec_settings.height = 720;
-  codec_settings.maxFramerate = 30;
-  codec_settings.qpMax = 63;
-  VideoEncoder::Capabilities capabilities(/*loss_notification=*/false);
-  VideoEncoder::Settings encoder_settings(capabilities, /*number_of_cores=*/1,
-                                          /*max_payload_size=*/1200);
-  EXPECT_EQ(encoder->InitEncode(&codec_settings, encoder_settings),
+  VideoCodec codec_settings = DefaultCodecSettings();
+  EXPECT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()),
             WEBRTC_VIDEO_CODEC_OK);
   EXPECT_EQ(encoder->Release(), WEBRTC_VIDEO_CODEC_OK);
 }
 
+TEST(LibaomAv1EncoderTest, NoBitrateOnTopLayerRefecltedInActiveDecodeTargets) {
+  // Configure encoder with 2 temporal layers.
+  std::unique_ptr<VideoEncoder> encoder =
+      CreateLibaomAv1Encoder(std::make_unique<ScalabilityStructureL1T2>());
+  VideoCodec codec_settings = DefaultCodecSettings();
+  ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()),
+            WEBRTC_VIDEO_CODEC_OK);
+
+  VideoEncoder::RateControlParameters rate_parameters;
+  rate_parameters.framerate_fps = 30;
+  rate_parameters.bitrate.SetBitrate(0, /*temporal_index=*/0, 300'000);
+  rate_parameters.bitrate.SetBitrate(0, /*temporal_index=*/1, 0);
+  encoder->SetRates(rate_parameters);
+
+  std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames =
+      EncodedVideoFrameProducer(*encoder).SetNumInputFrames(1).Encode();
+  ASSERT_THAT(encoded_frames, SizeIs(1));
+  ASSERT_NE(encoded_frames[0].codec_specific_info.generic_frame_info,
+            absl::nullopt);
+  // Assuming L1T2 structure uses 1st decode target for T0 and 2nd decode target
+  // for T0+T1 frames, expect only 1st decode target is active.
+  EXPECT_EQ(encoded_frames[0]
+                .codec_specific_info.generic_frame_info->active_decode_targets,
+            0b01);
+}
+
 }  // namespace
 }  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l1t2.cc

@@ -56,8 +56,15 @@ FrameDependencyStructure ScalabilityStructureL1T2::DependencyStructure() const {
 
 std::vector<ScalableVideoController::LayerFrameConfig>
 ScalabilityStructureL1T2::NextFrameConfig(bool restart) {
+  if (!active_decode_targets_[0]) {
+    RTC_LOG(LS_WARNING) << "No bitrate allocated for temporal layer 0, yet "
+                           "frame is requested. No frame will be encoded.";
+    return {};
+  }
   if (restart) {
     next_pattern_ = kKeyFrame;
+  } else if (!active_decode_targets_[1]) {
+    next_pattern_ = kDeltaFrameT0;
   }
   std::vector<LayerFrameConfig> result(1);
 
@@ -97,7 +104,20 @@ absl::optional<GenericFrameInfo> ScalabilityStructureL1T2::OnEncodeDone(
   frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]),
                                                std::end(kDtis[config.Id()]));
   frame_info->part_of_chain = {config.TemporalId() == 0};
+  frame_info->active_decode_targets = active_decode_targets_;
   return frame_info;
 }
 
+void ScalabilityStructureL1T2::OnRatesUpdated(
+    const VideoBitrateAllocation& bitrates) {
+  if (bitrates.GetBitrate(0, 0) == 0) {
+    // It is unclear what frame can be produced when base layer is disabled,
+    // so mark all decode targets as inactive to produce no frames.
+    active_decode_targets_.reset();
+    return;
+  }
+  active_decode_targets_.set(0, true);
+  active_decode_targets_.set(1, bitrates.GetBitrate(0, 1) > 0);
+}
+
 }  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l1t2.h

@@ -10,6 +10,7 @@
 #ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T2_H_
 #define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T2_H_
 
+#include <bitset>
 #include <vector>
 
 #include "api/transport/rtp/dependency_descriptor.h"
@@ -29,6 +30,8 @@ class ScalabilityStructureL1T2 : public ScalableVideoController {
   absl::optional<GenericFrameInfo> OnEncodeDone(
       LayerFrameConfig config) override;
 
+  void OnRatesUpdated(const VideoBitrateAllocation& bitrates) override;
+
  private:
   enum FramePattern {
     kKeyFrame,
@@ -37,6 +40,7 @@ class ScalabilityStructureL1T2 : public ScalableVideoController {
   };
 
   FramePattern next_pattern_ = kKeyFrame;
+  std::bitset<32> active_decode_targets_ = 0b11;
 };
 
 }  // namespace webrtc

modules/video_coding/codecs/av1/scalable_video_controller.h

@@ -15,6 +15,7 @@
 #include "absl/container/inlined_vector.h"
 #include "absl/types/optional.h"
 #include "api/transport/rtp/dependency_descriptor.h"
+#include "api/video/video_bitrate_allocation.h"
 #include "common_video/generic_frame_descriptor/generic_frame_info.h"
 
 namespace webrtc {
@@ -77,6 +78,13 @@ class ScalableVideoController {
   // dependency descriptor rtp header extension.
   virtual FrameDependencyStructure DependencyStructure() const = 0;
 
+  // Notifies Controller with updated bitrates per layer. In particular notifies
+  // when certain layers should be disabled.
+  // Controller shouldn't produce LayerFrameConfig for disabled layers.
+  // TODO(bugs.webrtc.org/11404): Make pure virtual when implemented by all
+  // structures.
+  virtual void OnRatesUpdated(const VideoBitrateAllocation& bitrates) {}
+
   // When `restart` is true, first `LayerFrameConfig` should have `is_keyframe`
   // set to true.
   // Returned vector shouldn't be empty.