Add helper to calculate frame dependencies based on encoder buffer usage

Bug: webrtc:10342
Change-Id: I1d856d060c2defcd10310f0d8639ce8a9554fff3
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/168194
Commit-Queue: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30458}

common_video/generic_frame_descriptor/generic_frame_info.h

@@ -22,7 +22,7 @@ namespace webrtc {
 
 // Describes how a certain encoder buffer was used when encoding a frame.
 struct CodecBufferUsage {
-  CodecBufferUsage(int id, bool referenced, bool updated)
+  constexpr CodecBufferUsage(int id, bool referenced, bool updated)
       : id(id), referenced(referenced), updated(updated) {}
 
   int id = 0;

modules/video_coding/BUILD.gn

@@ -34,6 +34,26 @@ rtc_library("encoded_frame") {
   ]
 }
 
+rtc_library("frame_dependencies_calculator") {
+  sources = [
+    "frame_dependencies_calculator.cc",
+    "frame_dependencies_calculator.h",
+  ]
+
+  deps = [
+    "../../api:array_view",
+    "../../api/video:video_frame_type",
+    "../../common_video/generic_frame_descriptor",
+    "../../rtc_base:checks",
+    "../../rtc_base:logging",
+    "../../rtc_base:macromagic",
+    "../../rtc_base/synchronization:sequence_checker",
+    "//third_party/abseil-cpp/absl/algorithm:container",
+    "//third_party/abseil-cpp/absl/container:inlined_vector",
+    "//third_party/abseil-cpp/absl/types:optional",
+  ]
+}
+
 rtc_library("nack_module") {
   visibility = [ "*" ]
   sources = [
@@ -827,6 +847,7 @@ if (rtc_include_tests) {
       "decoding_state_unittest.cc",
       "fec_controller_unittest.cc",
       "frame_buffer2_unittest.cc",
+      "frame_dependencies_calculator_unittest.cc",
       "generic_decoder_unittest.cc",
       "h264_sprop_parameter_sets_unittest.cc",
       "h264_sps_pps_tracker_unittest.cc",
@@ -864,6 +885,7 @@ if (rtc_include_tests) {
     deps = [
       ":codec_globals_headers",
       ":encoded_frame",
+      ":frame_dependencies_calculator",
       ":nack_module",
       ":simulcast_test_fixture_impl",
       ":video_codec_interface",
@@ -897,10 +919,12 @@ if (rtc_include_tests) {
       "../../api/video:video_bitrate_allocator_factory",
       "../../api/video:video_frame",
       "../../api/video:video_frame_i420",
+      "../../api/video:video_frame_type",
       "../../api/video:video_rtp_headers",
       "../../api/video_codecs:video_codecs_api",
       "../../api/video_codecs:vp8_temporal_layers_factory",
       "../../common_video",
+      "../../common_video/generic_frame_descriptor",
       "../../common_video/test:utilities",
       "../../media:rtc_media_base",
       "../../rtc_base",

modules/video_coding/frame_dependencies_calculator.cc

@@ -0,0 +1,81 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/frame_dependencies_calculator.h"
+
+#include <stdint.h>
+
+#include <iterator>
+#include <set>
+
+#include "absl/algorithm/container.h"
+#include "absl/container/inlined_vector.h"
+#include "api/array_view.h"
+#include "api/video/video_frame_type.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+#include "rtc_base/synchronization/sequence_checker.h"
+
+namespace webrtc {
+
+absl::InlinedVector<int64_t, 5> FrameDependenciesCalculator::FromBuffersUsage(
+    VideoFrameType frame_type,
+    int64_t frame_id,
+    rtc::ArrayView<const CodecBufferUsage> buffers_usage) {
+  RTC_DCHECK_RUN_ON(&checker_);
+
+  absl::InlinedVector<int64_t, 5> dependencies;
+  RTC_DCHECK_GT(buffers_usage.size(), 0);
+  for (const CodecBufferUsage& buffer_usage : buffers_usage) {
+    RTC_CHECK_GE(buffer_usage.id, 0);
+    if (buffers_.size() <= static_cast<size_t>(buffer_usage.id)) {
+      buffers_.resize(buffer_usage.id + 1);
+    }
+  }
+  std::set<int64_t> direct_depenendencies;
+  std::set<int64_t> indirect_depenendencies;
+  if (frame_type == VideoFrameType::kVideoFrameDelta) {
+    for (const CodecBufferUsage& buffer_usage : buffers_usage) {
+      if (!buffer_usage.referenced) {
+        continue;
+      }
+      const BufferUsage& buffer = buffers_[buffer_usage.id];
+      if (buffer.frame_id == absl::nullopt) {
+        RTC_LOG(LS_ERROR) << "Odd configuration: frame " << frame_id
+                          << " references buffer #" << buffer_usage.id
+                          << " that was never updated.";
+        continue;
+      }
+      direct_depenendencies.insert(*buffer.frame_id);
+      indirect_depenendencies.insert(buffer.dependencies.begin(),
+                                     buffer.dependencies.end());
+    }
+    // Reduce references: if frame #3 depends on frame #2 and #1, and frame #2
+    // depends on frame #1, then frame #3 needs to depend just on frame #2.
+    // Though this set diff removes only 1 level of indirection, it seems
+    // enough for all currently used structures.
+    absl::c_set_difference(direct_depenendencies, indirect_depenendencies,
+                           std::back_inserter(dependencies));
+  }
+
+  // Update buffers.
+  for (const CodecBufferUsage& buffer_usage : buffers_usage) {
+    if (!buffer_usage.updated) {
+      continue;
+    }
+    BufferUsage& buffer = buffers_[buffer_usage.id];
+    buffer.frame_id = frame_id;
+    buffer.dependencies.assign(direct_depenendencies.begin(),
+                               direct_depenendencies.end());
+  }
+
+  return dependencies;
+}
+
+}  // namespace webrtc

modules/video_coding/frame_dependencies_calculator.h

@@ -0,0 +1,53 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_FRAME_DEPENDENCIES_CALCULATOR_H_
+#define MODULES_VIDEO_CODING_FRAME_DEPENDENCIES_CALCULATOR_H_
+
+#include <stdint.h>
+
+#include <vector>
+
+#include "absl/container/inlined_vector.h"
+#include "absl/types/optional.h"
+#include "api/array_view.h"
+#include "api/video/video_frame_type.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "rtc_base/synchronization/sequence_checker.h"
+#include "rtc_base/thread_annotations.h"
+
+namespace webrtc {
+
+class FrameDependenciesCalculator {
+ public:
+  FrameDependenciesCalculator() = default;
+  FrameDependenciesCalculator(FrameDependenciesCalculator&&) = default;
+  FrameDependenciesCalculator& operator=(FrameDependenciesCalculator&&) =
+      default;
+
+  // Calculates frame dependencies based on previous encoder buffer usage.
+  absl::InlinedVector<int64_t, 5> FromBuffersUsage(
+      VideoFrameType frame_type,
+      int64_t frame_id,
+      rtc::ArrayView<const CodecBufferUsage> buffers_usage);
+
+ private:
+  struct BufferUsage {
+    absl::optional<int64_t> frame_id;
+    absl::InlinedVector<int64_t, 4> dependencies;
+  };
+
+  SequenceChecker checker_;
+  absl::InlinedVector<BufferUsage, 4> buffers_ RTC_GUARDED_BY(checker_);
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_FRAME_DEPENDENCIES_CALCULATOR_H_

modules/video_coding/frame_dependencies_calculator_unittest.cc

@@ -0,0 +1,153 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/frame_dependencies_calculator.h"
+
+#include "api/video/video_frame_type.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+using ::testing::ElementsAre;
+using ::testing::IsEmpty;
+using ::testing::UnorderedElementsAre;
+
+constexpr VideoFrameType kVideoFrameKey = VideoFrameType::kVideoFrameKey;
+constexpr VideoFrameType kVideoFrameDelta = VideoFrameType::kVideoFrameDelta;
+
+constexpr CodecBufferUsage ReferenceAndUpdate(int id) {
+  return CodecBufferUsage(id, /*referenced=*/true, /*updated=*/true);
+}
+constexpr CodecBufferUsage Reference(int id) {
+  return CodecBufferUsage(id, /*referenced=*/true, /*updated=*/false);
+}
+constexpr CodecBufferUsage Update(int id) {
+  return CodecBufferUsage(id, /*referenced=*/false, /*updated=*/true);
+}
+
+TEST(FrameDependenciesCalculatorTest, SingleLayer) {
+  CodecBufferUsage pattern[] = {ReferenceAndUpdate(0)};
+  FrameDependenciesCalculator calculator;
+
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameKey, /*frame_id=*/1, pattern),
+      IsEmpty());
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/3, pattern),
+      ElementsAre(1));
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/6, pattern),
+      ElementsAre(3));
+}
+
+TEST(FrameDependenciesCalculatorTest, TwoTemporalLayers) {
+  // Shortened 4-frame pattern:
+  // T1:  2---4   6---8 ...
+  //      /   /   /   /
+  // T0: 1---3---5---7 ...
+  CodecBufferUsage pattern0[] = {ReferenceAndUpdate(0)};
+  CodecBufferUsage pattern1[] = {Reference(0), Update(1)};
+  CodecBufferUsage pattern2[] = {ReferenceAndUpdate(0)};
+  CodecBufferUsage pattern3[] = {Reference(0), Reference(1)};
+  FrameDependenciesCalculator calculator;
+
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameKey, /*frame_id=*/1, pattern0),
+      IsEmpty());
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/2, pattern1),
+      ElementsAre(1));
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/3, pattern2),
+      ElementsAre(1));
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/4, pattern3),
+      UnorderedElementsAre(2, 3));
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/5, pattern0),
+      ElementsAre(3));
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/6, pattern1),
+      ElementsAre(5));
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/7, pattern2),
+      ElementsAre(5));
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/8, pattern3),
+      UnorderedElementsAre(6, 7));
+}
+
+TEST(FrameDependenciesCalculatorTest, ThreeTemporalLayers4FramePattern) {
+  // T2:   2---4   6---8 ...
+  //      /   /   /   /
+  // T1:  |  3    |  7   ...
+  //      /_/     /_/
+  // T0: 1-------5-----  ...
+  CodecBufferUsage pattern0[] = {ReferenceAndUpdate(0)};
+  CodecBufferUsage pattern1[] = {Reference(0), Update(2)};
+  CodecBufferUsage pattern2[] = {Reference(0), Update(1)};
+  CodecBufferUsage pattern3[] = {Reference(0), Reference(1), Reference(2)};
+  FrameDependenciesCalculator calculator;
+
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameKey, /*frame_id=*/1, pattern0),
+      IsEmpty());
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/2, pattern1),
+      ElementsAre(1));
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/3, pattern2),
+      ElementsAre(1));
+  // Note that frame#4 references buffer#0 that is updated by frame#1,
+  // yet there is no direct dependency from frame#4 to frame#1.
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/4, pattern3),
+      UnorderedElementsAre(2, 3));
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/5, pattern0),
+      ElementsAre(1));
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/6, pattern1),
+      ElementsAre(5));
+}
+
+TEST(FrameDependenciesCalculatorTest, SimulcastWith2Layers) {
+  // S1: 2---4---6-  ...
+  //
+  // S0: 1---3---5-  ...
+  CodecBufferUsage pattern0[] = {ReferenceAndUpdate(0)};
+  CodecBufferUsage pattern1[] = {ReferenceAndUpdate(1)};
+  FrameDependenciesCalculator calculator;
+
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameKey, /*frame_id=*/1, pattern0),
+      IsEmpty());
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameKey, /*frame_id=*/2, pattern1),
+      IsEmpty());
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/3, pattern0),
+      ElementsAre(1));
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/4, pattern1),
+      ElementsAre(2));
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/5, pattern0),
+      ElementsAre(3));
+  EXPECT_THAT(
+      calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/6, pattern1),
+      ElementsAre(4));
+}
+
+}  // namespace
+}  // namespace webrtc