Count disabled due to low bw streams or layers as bw limited quality in GetStats

Bug: webrtc:11015
Change-Id: I65cd890706f765366d89ded8c21fa7507797fc23
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/155964
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29421}

api/video/video_bitrate_allocation.cc

@@ -18,7 +18,8 @@
 
 namespace webrtc {
 
-VideoBitrateAllocation::VideoBitrateAllocation() : sum_(0) {}
+VideoBitrateAllocation::VideoBitrateAllocation()
+    : sum_(0), is_bw_limited_(false) {}
 
 bool VideoBitrateAllocation::SetBitrate(size_t spatial_index,
                                         size_t temporal_index,

api/video/video_bitrate_allocation.h

@@ -80,9 +80,15 @@ class RTC_EXPORT VideoBitrateAllocation {
 
   std::string ToString() const;
 
+  // Indicates if the allocation has some layers/streams disabled due to
+  // low available bandwidth.
+  void set_bw_limited(bool limited) { is_bw_limited_ = limited; }
+  bool is_bw_limited() const { return is_bw_limited_; }
+
  private:
   uint32_t sum_;
   absl::optional<uint32_t> bitrates_[kMaxSpatialLayers][kMaxTemporalStreams];
+  bool is_bw_limited_;
 };
 
 }  // namespace webrtc

modules/video_coding/codecs/vp9/svc_rate_allocator.cc

@@ -211,7 +211,8 @@ VideoBitrateAllocation SvcRateAllocator::Allocate(
   }
 
   const size_t first_active_layer = GetFirstActiveLayer(codec_);
-  size_t num_spatial_layers = GetNumActiveSpatialLayers(codec_);
+  const size_t num_active_layers = GetNumActiveSpatialLayers(codec_);
+  size_t num_spatial_layers = num_active_layers;
 
   if (num_spatial_layers == 0) {
     return VideoBitrateAllocation();  // All layers are deactivated.
@@ -244,13 +245,16 @@ VideoBitrateAllocation SvcRateAllocator::Allocate(
   }
   last_active_layer_count_ = num_spatial_layers;
 
+  VideoBitrateAllocation allocation;
   if (codec_.mode == VideoCodecMode::kRealtimeVideo) {
-    return GetAllocationNormalVideo(total_bitrate, first_active_layer,
-                                    num_spatial_layers);
+    allocation = GetAllocationNormalVideo(total_bitrate, first_active_layer,
+                                          num_spatial_layers);
   } else {
-    return GetAllocationScreenSharing(total_bitrate, first_active_layer,
-                                      num_spatial_layers);
+    allocation = GetAllocationScreenSharing(total_bitrate, first_active_layer,
+                                            num_spatial_layers);
   }
+  allocation.set_bw_limited(num_spatial_layers < num_active_layers);
+  return allocation;
 }
 
 VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo(

modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc

@@ -224,6 +224,24 @@ TEST(SvcRateAllocatorTest, DeactivateLowerLayers) {
   }
 }
 
+TEST(SvcRateAllocatorTest, SignalsBwLimited) {
+  VideoCodec codec = Configure(1280, 720, 3, 1, false);
+  SvcRateAllocator allocator = SvcRateAllocator(codec);
+
+  // Rough estimate calculated by hand.
+  uint32_t min_to_enable_all = 900000;
+
+  EXPECT_TRUE(
+      allocator
+          .Allocate(VideoBitrateAllocationParameters(min_to_enable_all / 2, 30))
+          .is_bw_limited());
+
+  EXPECT_FALSE(
+      allocator
+          .Allocate(VideoBitrateAllocationParameters(min_to_enable_all, 30))
+          .is_bw_limited());
+}
+
 TEST(SvcRateAllocatorTest, NoPaddingIfAllLayersAreDeactivated) {
   VideoCodec codec = Configure(1280, 720, 3, 1, false);
   EXPECT_EQ(codec.VP9()->numberOfSpatialLayers, 3U);

modules/video_coding/utility/simulcast_rate_allocator.cc

@@ -166,6 +166,7 @@ void SimulcastRateAllocator::DistributeAllocationToSimulcastLayers(
       min_bitrate = std::min(hysteresis_factor * min_bitrate, target_bitrate);
     }
     if (left_in_stable_allocation < min_bitrate) {
+      allocated_bitrates->set_bw_limited(true);
       break;
     }
 

modules/video_coding/utility/simulcast_rate_allocator_unittest.cc

@@ -221,6 +221,27 @@ TEST_F(SimulcastRateAllocatorTest, SingleSimulcastBelowMin) {
   ExpectEqual(expected, GetAllocation(0));
 }
 
+TEST_F(SimulcastRateAllocatorTest, SignalsBwLimited) {
+  // Enough to enable all layers.
+  const int kVeryBigBitrate = 100000;
+  // With simulcast, use the min bitrate from the ss spec instead of the global.
+  SetupCodec3SL3TL({true, true, true});
+  CreateAllocator();
+
+  EXPECT_TRUE(
+      GetAllocation(codec_.simulcastStream[0].minBitrate - 10).is_bw_limited());
+  EXPECT_TRUE(
+      GetAllocation(codec_.simulcastStream[0].targetBitrate).is_bw_limited());
+  EXPECT_TRUE(GetAllocation(codec_.simulcastStream[0].targetBitrate +
+                            codec_.simulcastStream[1].minBitrate)
+                  .is_bw_limited());
+  EXPECT_FALSE(GetAllocation(codec_.simulcastStream[0].targetBitrate +
+                             codec_.simulcastStream[1].targetBitrate +
+                             codec_.simulcastStream[2].minBitrate)
+                   .is_bw_limited());
+  EXPECT_FALSE(GetAllocation(kVeryBigBitrate).is_bw_limited());
+}
+
 TEST_F(SimulcastRateAllocatorTest, SingleSimulcastAboveMax) {
   codec_.numberOfSimulcastStreams = 1;
   codec_.simulcastStream[0].minBitrate = kMinBitrateKbps;
@@ -655,6 +676,7 @@ TEST_P(ScreenshareRateAllocationTest, BitrateBelowTl0) {
   EXPECT_EQ(kLegacyScreenshareTargetBitrateKbps, allocation.get_sum_kbps());
   EXPECT_EQ(kLegacyScreenshareTargetBitrateKbps,
             allocation.GetBitrate(0, 0) / 1000);
+  EXPECT_EQ(allocation.is_bw_limited(), GetParam());
 }
 
 TEST_P(ScreenshareRateAllocationTest, BitrateAboveTl0) {
@@ -674,6 +696,7 @@ TEST_P(ScreenshareRateAllocationTest, BitrateAboveTl0) {
             allocation.GetBitrate(0, 0) / 1000);
   EXPECT_EQ(target_bitrate_kbps - kLegacyScreenshareTargetBitrateKbps,
             allocation.GetBitrate(0, 1) / 1000);
+  EXPECT_EQ(allocation.is_bw_limited(), GetParam());
 }
 
 TEST_F(ScreenshareRateAllocationTest, BitrateAboveTl1) {
@@ -692,6 +715,7 @@ TEST_F(ScreenshareRateAllocationTest, BitrateAboveTl1) {
   EXPECT_EQ(
       kLegacyScreenshareMaxBitrateKbps - kLegacyScreenshareTargetBitrateKbps,
       allocation.GetBitrate(0, 1) / 1000);
+  EXPECT_FALSE(allocation.is_bw_limited());
 }
 
 // This tests when the screenshare is inactive it should be allocated 0 bitrate

video/send_statistics_proxy.cc

@@ -147,6 +147,7 @@ SendStatisticsProxy::SendStatisticsProxy(
       last_num_spatial_layers_(0),
       last_num_simulcast_streams_(0),
       last_spatial_layer_use_{},
+      bw_limited_layers_(false),
       uma_container_(
           new UmaSamplesContainer(GetUmaPrefix(content_type_), stats_, clock)) {
 }
@@ -1073,10 +1074,21 @@ void SendStatisticsProxy::OnAdaptationChanged(
       break;
   }
 
-  bool is_cpu_limited = cpu_counts.num_resolution_reductions > 0 ||
-                        cpu_counts.num_framerate_reductions > 0;
-  bool is_bandwidth_limited = quality_counts.num_resolution_reductions > 0 ||
-                              quality_counts.num_framerate_reductions > 0;
+  cpu_downscales_ = cpu_counts.num_resolution_reductions.value_or(-1);
+  quality_downscales_ = quality_counts.num_resolution_reductions.value_or(-1);
+
+  cpu_counts_ = cpu_counts;
+  quality_counts_ = quality_counts;
+
+  UpdateAdaptationStats();
+}
+
+void SendStatisticsProxy::UpdateAdaptationStats() {
+  bool is_cpu_limited = cpu_counts_.num_resolution_reductions > 0 ||
+                        cpu_counts_.num_framerate_reductions > 0;
+  bool is_bandwidth_limited = quality_counts_.num_resolution_reductions > 0 ||
+                              quality_counts_.num_framerate_reductions > 0 ||
+                              bw_limited_layers_;
   if (is_bandwidth_limited) {
     // We may be both CPU limited and bandwidth limited at the same time but
     // there is no way to express this in standardized stats. Heuristically,
@@ -1092,21 +1104,27 @@ void SendStatisticsProxy::OnAdaptationChanged(
         QualityLimitationReason::kNone);
   }
 
-  UpdateAdaptationStats(cpu_counts, quality_counts);
-}
-
-void SendStatisticsProxy::UpdateAdaptationStats(
-    const AdaptationSteps& cpu_counts,
-    const AdaptationSteps& quality_counts) {
-  cpu_downscales_ = cpu_counts.num_resolution_reductions.value_or(-1);
-  quality_downscales_ = quality_counts.num_resolution_reductions.value_or(-1);
-
-  stats_.cpu_limited_resolution = cpu_counts.num_resolution_reductions > 0;
-  stats_.cpu_limited_framerate = cpu_counts.num_framerate_reductions > 0;
-  stats_.bw_limited_resolution = quality_counts.num_resolution_reductions > 0;
-  stats_.bw_limited_framerate = quality_counts.num_framerate_reductions > 0;
+  stats_.cpu_limited_resolution = cpu_counts_.num_resolution_reductions > 0;
+  stats_.cpu_limited_framerate = cpu_counts_.num_framerate_reductions > 0;
+  stats_.bw_limited_resolution = quality_counts_.num_resolution_reductions > 0;
+  stats_.bw_limited_framerate = quality_counts_.num_framerate_reductions > 0;
+  // If bitrate allocator has disabled some layers frame-rate or resolution are
+  // limited depending on the encoder configuration.
+  if (bw_limited_layers_) {
+    switch (content_type_) {
+      case VideoEncoderConfig::ContentType::kRealtimeVideo: {
+        stats_.bw_limited_resolution = true;
+        break;
+      }
+      case VideoEncoderConfig::ContentType::kScreen: {
+        stats_.bw_limited_framerate = true;
+        break;
+      }
+    }
+  }
   stats_.quality_limitation_reason =
       quality_limitation_reason_tracker_.current_reason();
+
   // |stats_.quality_limitation_durations_ms| depends on the current time
   // when it is polled; it is updated in SendStatisticsProxy::GetStats().
 }
@@ -1134,6 +1152,9 @@ void SendStatisticsProxy::OnBitrateAllocationUpdated(
 
   rtc::CritScope lock(&crit_);
 
+  bw_limited_layers_ = allocation.is_bw_limited();
+  UpdateAdaptationStats();
+
   if (spatial_layers != last_spatial_layer_use_) {
     // If the number of spatial layers has changed, the resolution change is
     // not due to quality limitations, it is because the configuration

video/send_statistics_proxy.h

@@ -223,9 +223,7 @@ class SendStatisticsProxy : public VideoStreamEncoderObserver,
 
   void SetAdaptTimer(const AdaptationSteps& counts, StatsTimer* timer)
       RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
-  void UpdateAdaptationStats(const AdaptationSteps& cpu_counts,
-                             const AdaptationSteps& quality_counts)
-      RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
+  void UpdateAdaptationStats() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
   void TryUpdateInitialQualityResolutionAdaptUp(
       const AdaptationSteps& quality_counts)
       RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
@@ -263,6 +261,11 @@ class SendStatisticsProxy : public VideoStreamEncoderObserver,
   int last_num_simulcast_streams_ RTC_GUARDED_BY(crit_);
   std::array<bool, kMaxSpatialLayers> last_spatial_layer_use_
       RTC_GUARDED_BY(crit_);
+  // Indicates if the latest bitrate allocation had layers disabled by low
+  // available bandwidth.
+  bool bw_limited_layers_ RTC_GUARDED_BY(crit_);
+  AdaptationSteps cpu_counts_ RTC_GUARDED_BY(crit_);
+  AdaptationSteps quality_counts_ RTC_GUARDED_BY(crit_);
 
   struct EncoderChangeEvent {
     std::string previous_encoder_implementation;

video/send_statistics_proxy_unittest.cc

@@ -1371,6 +1371,79 @@ TEST_F(SendStatisticsProxyTest,
       0u, statistics_proxy_->GetStats().quality_limitation_resolution_changes);
 }
 
+TEST_F(SendStatisticsProxyTest,
+       QualityLimitationReasonsAreCorrectForContentType) {
+  // Realtime case.
+  // Configure two streams.
+  VideoEncoderConfig config;
+  config.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
+  config.number_of_streams = 2;
+  VideoStream stream1;
+  stream1.width = kWidth / 2;
+  stream1.height = kHeight / 2;
+  VideoStream stream2;
+  stream2.width = kWidth;
+  stream2.height = kHeight;
+  statistics_proxy_->OnEncoderReconfigured(config, {stream1, stream2});
+  EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_framerate);
+  EXPECT_EQ(statistics_proxy_->GetStats().quality_limitation_reason,
+            QualityLimitationReason::kNone);
+  // Bw disabled one layer.
+  VideoCodec codec;
+  codec.numberOfSimulcastStreams = 2;
+  codec.simulcastStream[0].active = true;
+  codec.simulcastStream[1].active = true;
+  VideoBitrateAllocation allocation;
+  // Some positive bitrate only on the first stream.
+  allocation.SetBitrate(0, 0, 10000);
+  allocation.SetBitrate(1, 0, 0);
+  allocation.set_bw_limited(true);
+  statistics_proxy_->OnBitrateAllocationUpdated(codec, allocation);
+  EXPECT_TRUE(statistics_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_framerate);
+  EXPECT_EQ(statistics_proxy_->GetStats().quality_limitation_reason,
+            QualityLimitationReason::kBandwidth);
+  // Bw enabled all layers.
+  allocation.SetBitrate(1, 0, 10000);
+  allocation.set_bw_limited(false);
+  statistics_proxy_->OnBitrateAllocationUpdated(codec, allocation);
+  EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_framerate);
+  EXPECT_EQ(statistics_proxy_->GetStats().quality_limitation_reason,
+            QualityLimitationReason::kNone);
+
+  // Screencast case
+  // Configure two streams.
+  config.content_type = VideoEncoderConfig::ContentType::kScreen;
+  config.number_of_streams = 2;
+  stream1.width = kWidth;
+  stream1.height = kHeight;
+  statistics_proxy_->OnEncoderReconfigured(config, {stream1, stream2});
+  EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_framerate);
+  EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_EQ(statistics_proxy_->GetStats().quality_limitation_reason,
+            QualityLimitationReason::kNone);
+  // Bw disabled one layer.
+  // Some positive bitrate only on the second stream.
+  allocation.SetBitrate(0, 0, 10000);
+  allocation.SetBitrate(1, 0, 0);
+  allocation.set_bw_limited(true);
+  statistics_proxy_->OnBitrateAllocationUpdated(codec, allocation);
+  EXPECT_TRUE(statistics_proxy_->GetStats().bw_limited_framerate);
+  EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_EQ(statistics_proxy_->GetStats().quality_limitation_reason,
+            QualityLimitationReason::kBandwidth);
+  // Bw enabled all layers.
+  allocation.SetBitrate(1, 0, 10000);
+  allocation.set_bw_limited(false);
+  statistics_proxy_->OnBitrateAllocationUpdated(codec, allocation);
+  EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution);
+  EXPECT_EQ(statistics_proxy_->GetStats().quality_limitation_reason,
+            QualityLimitationReason::kNone);
+}
+
 TEST_F(SendStatisticsProxyTest, SwitchContentTypeUpdatesHistograms) {
   for (int i = 0; i < SendStatisticsProxy::kMinRequiredMetricsSamples; ++i)
     statistics_proxy_->OnIncomingFrame(kWidth, kHeight);
@@ -1982,6 +2055,7 @@ TEST_F(SendStatisticsProxyTest, GetStatsReportsBandwidthLimitedResolution) {
   // Configure two streams.
   VideoEncoderConfig config;
   config.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
+  config.number_of_streams = 2;
   VideoStream stream1;
   stream1.width = kWidth / 2;
   stream1.height = kHeight / 2;
@@ -2044,6 +2118,26 @@ TEST_F(SendStatisticsProxyTest, GetStatsReportsBandwidthLimitedResolution) {
       quality_counts);
   statistics_proxy_->OnSendEncodedImage(encoded_image, nullptr);
   EXPECT_TRUE(statistics_proxy_->GetStats().bw_limited_resolution);
+
+  // Adapt up.
+  quality_counts.num_resolution_reductions = 0;
+  statistics_proxy_->OnAdaptationChanged(
+      VideoStreamEncoderObserver::AdaptationReason::kQuality, cpu_counts,
+      quality_counts);
+  statistics_proxy_->OnSendEncodedImage(encoded_image, nullptr);
+  EXPECT_FALSE(statistics_proxy_->GetStats().bw_limited_resolution);
+
+  // Bw disabled one layer.
+  VideoCodec codec;
+  codec.numberOfSimulcastStreams = 2;
+  codec.simulcastStream[0].active = true;
+  codec.simulcastStream[1].active = true;
+  VideoBitrateAllocation allocation;
+  // Some positive bitrate only on the second stream.
+  allocation.SetBitrate(1, 0, 10000);
+  allocation.set_bw_limited(true);
+  statistics_proxy_->OnBitrateAllocationUpdated(codec, allocation);
+  EXPECT_TRUE(statistics_proxy_->GetStats().bw_limited_resolution);
 }
 
 TEST_F(SendStatisticsProxyTest, GetStatsReportsTargetMediaBitrate) {