Move calculation of target_encode_bitrate to DefaultVideoQualityAnalyzer

To migrate on new GetStats API and properly support target encode bitrate
for regular, simulcast and svc cases we need to calculate it inside video
quality analyzer getting values from SetRates in VideoEncoder.

Bug: webrtc:11381
Change-Id: Ia37acac764ed3c30f64cdbfda8906d543fa03ae2
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171501
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Patrik Höglund <phoglund@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30881}

api/test/video_quality_analyzer_interface.h

@@ -53,6 +53,20 @@ namespace webrtc_pc_e2e {
 // The analyzer will be injected in all points from A to F.
 class VideoQualityAnalyzerInterface : public StatsObserverInterface {
  public:
+  // Contains extra statistic provided by video encoder.
+  struct EncoderStats {
+    // TODO(hbos) https://crbug.com/webrtc/9547,
+    // https://crbug.com/webrtc/11443: improve stats API to make available
+    // there.
+    uint32_t target_encode_bitrate;
+  };
+  // Contains extra statistic provided by video decoder.
+  struct DecoderStats {
+    // Decode time provided by decoder itself. If decoder doesn’t produce such
+    // information can be omitted.
+    absl::optional<int32_t> decode_time_ms;
+  };
+
   ~VideoQualityAnalyzerInterface() override = default;
 
   // Will be called by framework before test.
@@ -74,18 +88,16 @@ class VideoQualityAnalyzerInterface : public StatsObserverInterface {
   // VideoFrame can produce multiple EncodedImages. Each encoded image will
   // have id from VideoFrame.
   virtual void OnFrameEncoded(uint16_t frame_id,
-                              const EncodedImage& encoded_image) {}
+                              const EncodedImage& encoded_image,
+                              const EncoderStats& stats) {}
   // Will be called for each frame dropped by encoder.
   virtual void OnFrameDropped(EncodedImageCallback::DropReason reason) {}
   // Will be called before calling the decoder.
   virtual void OnFramePreDecode(uint16_t frame_id,
                                 const EncodedImage& encoded_image) {}
-  // Will be called after decoding the frame. |decode_time_ms| is a decode
+  // Will be called after decoding the frame.
-  // time provided by decoder itself. If decoder doesn’t produce such
-  // information can be omitted.
   virtual void OnFrameDecoded(const VideoFrame& frame,
-                              absl::optional<int32_t> decode_time_ms,
+                              const DecoderStats& stats) {}
-                              absl::optional<uint8_t> qp) {}
   // Will be called when frame will be obtained from PeerConnection stack.
   virtual void OnFrameRendered(const VideoFrame& frame) {}
   // Will be called if encoder return not WEBRTC_VIDEO_CODEC_OK.

test/pc/e2e/analyzer/video/default_video_quality_analyzer.cc

@@ -26,6 +26,7 @@ namespace {
 constexpr int kMaxActiveComparisons = 10;
 constexpr int kFreezeThresholdMs = 150;
 constexpr int kMicrosPerSecond = 1000000;
+constexpr int kBitsInByte = 8;
 
 void LogFrameCounters(const std::string& name, const FrameCounters& counters) {
   RTC_LOG(INFO) << "[" << name << "] Captured    : " << counters.captured;
@@ -180,7 +181,8 @@ void DefaultVideoQualityAnalyzer::OnFramePreEncode(
 
 void DefaultVideoQualityAnalyzer::OnFrameEncoded(
     uint16_t frame_id,
-    const webrtc::EncodedImage& encoded_image) {
+    const webrtc::EncodedImage& encoded_image,
+    const EncoderStats& stats) {
   rtc::CritScope crit(&lock_);
   auto it = frame_stats_.find(frame_id);
   RTC_DCHECK(it != frame_stats_.end());
@@ -193,6 +195,7 @@ void DefaultVideoQualityAnalyzer::OnFrameEncoded(
   }
   it->second.encoded_time = Now();
   it->second.encoded_image_size = encoded_image.size();
+  it->second.target_encode_bitrate = stats.target_encode_bitrate;
 }
 
 void DefaultVideoQualityAnalyzer::OnFrameDropped(
@@ -226,8 +229,7 @@ void DefaultVideoQualityAnalyzer::OnFramePreDecode(
 
 void DefaultVideoQualityAnalyzer::OnFrameDecoded(
     const webrtc::VideoFrame& frame,
-    absl::optional<int32_t> decode_time_ms,
+    const DecoderStats& stats) {
-    absl::optional<uint8_t> qp) {
   rtc::CritScope crit(&lock_);
   auto it = frame_stats_.find(frame.id());
   RTC_DCHECK(it != frame_stats_.end());
@@ -517,6 +519,7 @@ void DefaultVideoQualityAnalyzer::ProcessComparison(
         (frame_stats.encoded_time - frame_stats.pre_encode_time).ms());
     stats->encode_frame_rate.AddEvent(frame_stats.encoded_time);
     stats->total_encoded_images_payload += frame_stats.encoded_image_size;
+    stats->target_encode_bitrate.AddSample(frame_stats.target_encode_bitrate);
   } else {
     if (frame_stats.pre_encode_time.IsFinite()) {
       stats->dropped_by_encoder++;
@@ -670,6 +673,9 @@ void DefaultVideoQualityAnalyzer::ReportResults(
                     /*important=*/false, ImproveDirection::kSmallerIsBetter);
   ReportResult("max_skipped", test_case_name, stats.skipped_between_rendered,
                "count", ImproveDirection::kSmallerIsBetter);
+  ReportResult("target_encode_bitrate", test_case_name,
+               stats.target_encode_bitrate / kBitsInByte, "bytesPerSecond",
+               ImproveDirection::kNone);
   test::PrintResult(
       "actual_encode_bitrate", "", test_case_name,
       static_cast<double>(stats.total_encoded_images_payload) /

test/pc/e2e/analyzer/video/default_video_quality_analyzer.h

@@ -101,6 +101,7 @@ struct StreamStats {
   // Mean time between one freeze end and next freeze start.
   SamplesStatsCounter time_between_freezes_ms;
   SamplesStatsCounter resolution_of_rendered_frame;
+  SamplesStatsCounter target_encode_bitrate;
 
   int64_t total_encoded_images_payload = 0;
   int64_t dropped_by_encoder = 0;
@@ -138,13 +139,13 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
                            const VideoFrame& frame) override;
   void OnFramePreEncode(const VideoFrame& frame) override;
   void OnFrameEncoded(uint16_t frame_id,
-                      const EncodedImage& encoded_image) override;
+                      const EncodedImage& encoded_image,
+                      const EncoderStats& stats) override;
   void OnFrameDropped(EncodedImageCallback::DropReason reason) override;
   void OnFramePreDecode(uint16_t frame_id,
                         const EncodedImage& input_image) override;
   void OnFrameDecoded(const VideoFrame& frame,
-                      absl::optional<int32_t> decode_time_ms,
+                      const DecoderStats& stats) override;
-                      absl::optional<uint8_t> qp) override;
   void OnFrameRendered(const VideoFrame& frame) override;
   void OnEncoderError(const VideoFrame& frame, int32_t error_code) override;
   void OnDecoderError(uint16_t frame_id, int32_t error_code) override;
@@ -181,6 +182,8 @@ class DefaultVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
     Timestamp rendered_time = Timestamp::MinusInfinity();
     Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity();
 
+    uint32_t target_encode_bitrate = 0;
+
     absl::optional<int> rendered_frame_width = absl::nullopt;
     absl::optional<int> rendered_frame_height = absl::nullopt;
 

test/pc/e2e/analyzer/video/default_video_quality_analyzer_test.cc

@@ -83,14 +83,15 @@ TEST(DefaultVideoQualityAnalyzerTest,
     frames_order.push_back(frame.id());
     captured_frames.insert({frame.id(), frame});
     analyzer.OnFramePreEncode(frame);
-    analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame));
+    analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame),
+                            VideoQualityAnalyzerInterface::EncoderStats());
   }
 
   for (const uint16_t& frame_id : frames_order) {
     VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
     analyzer.OnFramePreDecode(received_frame.id(), FakeEncode(received_frame));
-    analyzer.OnFrameDecoded(received_frame, /*decode_time_ms=*/absl::nullopt,
+    analyzer.OnFrameDecoded(received_frame,
-                            /*qp=*/absl::nullopt);
+                            VideoQualityAnalyzerInterface::DecoderStats());
     analyzer.OnFrameRendered(received_frame);
   }
 
@@ -129,15 +130,16 @@ TEST(DefaultVideoQualityAnalyzerTest,
     frames_order.push_back(frame.id());
     captured_frames.insert({frame.id(), frame});
     analyzer.OnFramePreEncode(frame);
-    analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame));
+    analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame),
+                            VideoQualityAnalyzerInterface::EncoderStats());
   }
 
   for (size_t i = kMaxFramesInFlightPerStream; i < frames_order.size(); ++i) {
     uint16_t frame_id = frames_order.at(i);
     VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
     analyzer.OnFramePreDecode(received_frame.id(), FakeEncode(received_frame));
-    analyzer.OnFrameDecoded(received_frame, /*decode_time_ms=*/absl::nullopt,
+    analyzer.OnFrameDecoded(received_frame,
-                            /*qp=*/absl::nullopt);
+                            VideoQualityAnalyzerInterface::DecoderStats());
     analyzer.OnFrameRendered(received_frame);
   }
 
@@ -174,15 +176,16 @@ TEST(DefaultVideoQualityAnalyzerTest, NormalScenario) {
     frames_order.push_back(frame.id());
     captured_frames.insert({frame.id(), frame});
     analyzer.OnFramePreEncode(frame);
-    analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame));
+    analyzer.OnFrameEncoded(frame.id(), FakeEncode(frame),
+                            VideoQualityAnalyzerInterface::EncoderStats());
   }
 
   for (size_t i = 1; i < frames_order.size(); i += 2) {
     uint16_t frame_id = frames_order.at(i);
     VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
     analyzer.OnFramePreDecode(received_frame.id(), FakeEncode(received_frame));
-    analyzer.OnFrameDecoded(received_frame, /*decode_time_ms=*/absl::nullopt,
+    analyzer.OnFrameDecoded(received_frame,
-                            /*qp=*/absl::nullopt);
+                            VideoQualityAnalyzerInterface::DecoderStats());
     analyzer.OnFrameRendered(received_frame);
   }
 

test/pc/e2e/analyzer/video/example_video_quality_analyzer.cc

@@ -52,7 +52,8 @@ void ExampleVideoQualityAnalyzer::OnFramePreEncode(
 
 void ExampleVideoQualityAnalyzer::OnFrameEncoded(
     uint16_t frame_id,
-    const webrtc::EncodedImage& encoded_image) {
+    const webrtc::EncodedImage& encoded_image,
+    const EncoderStats& stats) {
   rtc::CritScope crit(&lock_);
   ++frames_encoded_;
 }
@@ -73,8 +74,7 @@ void ExampleVideoQualityAnalyzer::OnFramePreDecode(
 
 void ExampleVideoQualityAnalyzer::OnFrameDecoded(
     const webrtc::VideoFrame& frame,
-    absl::optional<int32_t> decode_time_ms,
+    const DecoderStats& stats) {
-    absl::optional<uint8_t> qp) {
   rtc::CritScope crit(&lock_);
   ++frames_decoded_;
 }

test/pc/e2e/analyzer/video/example_video_quality_analyzer.h

@@ -38,13 +38,13 @@ class ExampleVideoQualityAnalyzer : public VideoQualityAnalyzerInterface {
                            const VideoFrame& frame) override;
   void OnFramePreEncode(const VideoFrame& frame) override;
   void OnFrameEncoded(uint16_t frame_id,
-                      const EncodedImage& encoded_image) override;
+                      const EncodedImage& encoded_image,
+                      const EncoderStats& stats) override;
   void OnFrameDropped(EncodedImageCallback::DropReason reason) override;
   void OnFramePreDecode(uint16_t frame_id,
                         const EncodedImage& encoded_image) override;
   void OnFrameDecoded(const VideoFrame& frame,
-                      absl::optional<int32_t> decode_time_ms,
+                      const DecoderStats& stats) override;
-                      absl::optional<uint8_t> qp) override;
   void OnFrameRendered(const VideoFrame& frame) override;
   void OnEncoderError(const VideoFrame& frame, int32_t error_code) override;
   void OnDecoderError(uint16_t frame_id, int32_t error_code) override;

test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.cc

@@ -222,7 +222,9 @@ void QualityAnalyzingVideoDecoder::OnFrameDecoded(
   // Set frame id to the value, that was extracted from corresponding encoded
   // image.
   frame->set_id(frame_id);
-  analyzer_->OnFrameDecoded(*frame, decode_time_ms, qp);
+  VideoQualityAnalyzerInterface::DecoderStats stats;
+  stats.decode_time_ms = decode_time_ms;
+  analyzer_->OnFrameDecoded(*frame, stats);
 }
 
 QualityAnalyzingVideoDecoderFactory::QualityAnalyzingVideoDecoderFactory(

test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.cc

@@ -161,6 +161,10 @@ void QualityAnalyzingVideoEncoder::SetRates(
     const VideoEncoder::RateControlParameters& parameters) {
   RTC_DCHECK_GT(bitrate_multiplier_, 0.0);
   if (fabs(bitrate_multiplier_ - kNoMultiplier) < kEps) {
+    {
+      rtc::CritScope crit(&lock_);
+      bitrate_allocation_ = parameters.bitrate;
+    }
     return delegate_->SetRates(parameters);
   }
 
@@ -200,6 +204,10 @@ void QualityAnalyzingVideoEncoder::SetRates(
 
   RateControlParameters adjusted_params = parameters;
   adjusted_params.bitrate = multiplied_allocation;
+  {
+    rtc::CritScope crit(&lock_);
+    bitrate_allocation_ = adjusted_params.bitrate;
+  }
   return delegate_->SetRates(adjusted_params);
 }
 
@@ -226,6 +234,7 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage(
     const RTPFragmentationHeader* fragmentation) {
   uint16_t frame_id;
   bool discard = false;
+  uint32_t target_encode_bitrate = 0;
   {
     rtc::CritScope crit(&lock_);
     std::pair<uint32_t, uint16_t> timestamp_frame_id;
@@ -257,11 +266,20 @@ EncodedImageCallback::Result QualityAnalyzingVideoEncoder::OnEncodedImage(
     frame_id = timestamp_frame_id.second;
 
     discard = ShouldDiscard(frame_id, encoded_image);
+    if (!discard) {
+      std::string stream_label = analyzer_->GetStreamLabel(frame_id);
+      absl::optional<int> required_spatial_index =
+          stream_required_spatial_index_[stream_label];
+      target_encode_bitrate = bitrate_allocation_.GetSpatialLayerSum(
+          required_spatial_index.value_or(0));
+    }
   }
 
   if (!discard) {
     // Analyzer should see only encoded images, that weren't discarded.
-    analyzer_->OnFrameEncoded(frame_id, encoded_image);
+    VideoQualityAnalyzerInterface::EncoderStats stats;
+    stats.target_encode_bitrate = target_encode_bitrate;
+    analyzer_->OnFrameEncoded(frame_id, encoded_image, stats);
   }
 
   // Image data injector injects frame id and discard flag into provided

test/pc/e2e/analyzer/video/quality_analyzing_video_encoder.h

@@ -150,6 +150,7 @@ class QualityAnalyzingVideoEncoder : public VideoEncoder,
   EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(lock_);
   std::list<std::pair<uint32_t, uint16_t>> timestamp_to_frame_id_list_
       RTC_GUARDED_BY(lock_);
+  VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(lock_);
 };
 
 // Produces QualityAnalyzingVideoEncoder, which hold decoders, produced by

test/pc/e2e/analyzer/video/video_quality_metrics_reporter.cc

@@ -42,13 +42,9 @@ void VideoQualityMetricsReporter::OnStatsReports(
     const webrtc::StatsReport::Value* transmission_bitrate =
         stats_report->FindValue(
             StatsReport::StatsValueName::kStatsValueNameTransmitBitrate);
-    const webrtc::StatsReport::Value* target_encode_bitrate =
-        stats_report->FindValue(
-            StatsReport::StatsValueName::kStatsValueNameTargetEncBitrate);
     RTC_CHECK(available_send_bandwidth);
     RTC_CHECK(retransmission_bitrate);
     RTC_CHECK(transmission_bitrate);
-    RTC_CHECK(target_encode_bitrate);
 
     rtc::CritScope crit(&video_bwe_stats_lock_);
     VideoBweStats& video_bwe_stats = video_bwe_stats_[pc_label];
@@ -58,8 +54,6 @@ void VideoQualityMetricsReporter::OnStatsReports(
         transmission_bitrate->int_val());
     video_bwe_stats.retransmission_bitrate.AddSample(
         retransmission_bitrate->int_val());
-    video_bwe_stats.target_encode_bitrate.AddSample(
-        target_encode_bitrate->int_val());
   }
 }
 
@@ -87,9 +81,6 @@ void VideoQualityMetricsReporter::ReportVideoBweResults(
   ReportResult("retransmission_bitrate", test_case_name,
                video_bwe_stats.retransmission_bitrate / kBitsInByte,
                "bytesPerSecond");
-  ReportResult("target_encode_bitrate", test_case_name,
-               video_bwe_stats.target_encode_bitrate / kBitsInByte,
-               "bytesPerSecond");
 }
 
 void VideoQualityMetricsReporter::ReportResult(

test/pc/e2e/analyzer/video/video_quality_metrics_reporter.h

@@ -26,7 +26,6 @@ struct VideoBweStats {
   SamplesStatsCounter available_send_bandwidth;
   SamplesStatsCounter transmission_bitrate;
   SamplesStatsCounter retransmission_bitrate;
-  SamplesStatsCounter target_encode_bitrate;
 };
 
 class VideoQualityMetricsReporter