diff --git a/common_video/generic_frame_descriptor/generic_frame_info.h b/common_video/generic_frame_descriptor/generic_frame_info.h index ce3ee6c4b3..b602ee06a6 100644 --- a/common_video/generic_frame_descriptor/generic_frame_info.h +++ b/common_video/generic_frame_descriptor/generic_frame_info.h @@ -22,7 +22,7 @@ namespace webrtc { // Describes how a certain encoder buffer was used when encoding a frame. struct CodecBufferUsage { - CodecBufferUsage(int id, bool referenced, bool updated) + constexpr CodecBufferUsage(int id, bool referenced, bool updated) : id(id), referenced(referenced), updated(updated) {} int id = 0; diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 22bb142c6d..4ae65dfe2b 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -34,6 +34,26 @@ rtc_library("encoded_frame") { ] } +rtc_library("frame_dependencies_calculator") { + sources = [ + "frame_dependencies_calculator.cc", + "frame_dependencies_calculator.h", + ] + + deps = [ + "../../api:array_view", + "../../api/video:video_frame_type", + "../../common_video/generic_frame_descriptor", + "../../rtc_base:checks", + "../../rtc_base:logging", + "../../rtc_base:macromagic", + "../../rtc_base/synchronization:sequence_checker", + "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/container:inlined_vector", + "//third_party/abseil-cpp/absl/types:optional", + ] +} + rtc_library("nack_module") { visibility = [ "*" ] sources = [ @@ -827,6 +847,7 @@ if (rtc_include_tests) { "decoding_state_unittest.cc", "fec_controller_unittest.cc", "frame_buffer2_unittest.cc", + "frame_dependencies_calculator_unittest.cc", "generic_decoder_unittest.cc", "h264_sprop_parameter_sets_unittest.cc", "h264_sps_pps_tracker_unittest.cc", @@ -864,6 +885,7 @@ if (rtc_include_tests) { deps = [ ":codec_globals_headers", ":encoded_frame", + ":frame_dependencies_calculator", ":nack_module", ":simulcast_test_fixture_impl", ":video_codec_interface", @@ -897,10 +919,12 @@ if (rtc_include_tests) { "../../api/video:video_bitrate_allocator_factory", "../../api/video:video_frame", "../../api/video:video_frame_i420", + "../../api/video:video_frame_type", "../../api/video:video_rtp_headers", "../../api/video_codecs:video_codecs_api", "../../api/video_codecs:vp8_temporal_layers_factory", "../../common_video", + "../../common_video/generic_frame_descriptor", "../../common_video/test:utilities", "../../media:rtc_media_base", "../../rtc_base", diff --git a/modules/video_coding/frame_dependencies_calculator.cc b/modules/video_coding/frame_dependencies_calculator.cc new file mode 100644 index 0000000000..c3042d9fad --- /dev/null +++ b/modules/video_coding/frame_dependencies_calculator.cc @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/frame_dependencies_calculator.h" + +#include + +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/container/inlined_vector.h" +#include "api/array_view.h" +#include "api/video/video_frame_type.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/synchronization/sequence_checker.h" + +namespace webrtc { + +absl::InlinedVector FrameDependenciesCalculator::FromBuffersUsage( + VideoFrameType frame_type, + int64_t frame_id, + rtc::ArrayView buffers_usage) { + RTC_DCHECK_RUN_ON(&checker_); + + absl::InlinedVector dependencies; + RTC_DCHECK_GT(buffers_usage.size(), 0); + for (const CodecBufferUsage& buffer_usage : buffers_usage) { + RTC_CHECK_GE(buffer_usage.id, 0); + if (buffers_.size() <= static_cast(buffer_usage.id)) { + buffers_.resize(buffer_usage.id + 1); + } + } + std::set direct_depenendencies; + std::set indirect_depenendencies; + if (frame_type == VideoFrameType::kVideoFrameDelta) { + for (const CodecBufferUsage& buffer_usage : buffers_usage) { + if (!buffer_usage.referenced) { + continue; + } + const BufferUsage& buffer = buffers_[buffer_usage.id]; + if (buffer.frame_id == absl::nullopt) { + RTC_LOG(LS_ERROR) << "Odd configuration: frame " << frame_id + << " references buffer #" << buffer_usage.id + << " that was never updated."; + continue; + } + direct_depenendencies.insert(*buffer.frame_id); + indirect_depenendencies.insert(buffer.dependencies.begin(), + buffer.dependencies.end()); + } + // Reduce references: if frame #3 depends on frame #2 and #1, and frame #2 + // depends on frame #1, then frame #3 needs to depend just on frame #2. + // Though this set diff removes only 1 level of indirection, it seems + // enough for all currently used structures. + absl::c_set_difference(direct_depenendencies, indirect_depenendencies, + std::back_inserter(dependencies)); + } + + // Update buffers. + for (const CodecBufferUsage& buffer_usage : buffers_usage) { + if (!buffer_usage.updated) { + continue; + } + BufferUsage& buffer = buffers_[buffer_usage.id]; + buffer.frame_id = frame_id; + buffer.dependencies.assign(direct_depenendencies.begin(), + direct_depenendencies.end()); + } + + return dependencies; +} + +} // namespace webrtc diff --git a/modules/video_coding/frame_dependencies_calculator.h b/modules/video_coding/frame_dependencies_calculator.h new file mode 100644 index 0000000000..f723d0f031 --- /dev/null +++ b/modules/video_coding/frame_dependencies_calculator.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_FRAME_DEPENDENCIES_CALCULATOR_H_ +#define MODULES_VIDEO_CODING_FRAME_DEPENDENCIES_CALCULATOR_H_ + +#include + +#include + +#include "absl/container/inlined_vector.h" +#include "absl/types/optional.h" +#include "api/array_view.h" +#include "api/video/video_frame_type.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "rtc_base/synchronization/sequence_checker.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +class FrameDependenciesCalculator { + public: + FrameDependenciesCalculator() = default; + FrameDependenciesCalculator(FrameDependenciesCalculator&&) = default; + FrameDependenciesCalculator& operator=(FrameDependenciesCalculator&&) = + default; + + // Calculates frame dependencies based on previous encoder buffer usage. + absl::InlinedVector FromBuffersUsage( + VideoFrameType frame_type, + int64_t frame_id, + rtc::ArrayView buffers_usage); + + private: + struct BufferUsage { + absl::optional frame_id; + absl::InlinedVector dependencies; + }; + + SequenceChecker checker_; + absl::InlinedVector buffers_ RTC_GUARDED_BY(checker_); +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_FRAME_DEPENDENCIES_CALCULATOR_H_ diff --git a/modules/video_coding/frame_dependencies_calculator_unittest.cc b/modules/video_coding/frame_dependencies_calculator_unittest.cc new file mode 100644 index 0000000000..81f774b227 --- /dev/null +++ b/modules/video_coding/frame_dependencies_calculator_unittest.cc @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/frame_dependencies_calculator.h" + +#include "api/video/video_frame_type.h" +#include "common_video/generic_frame_descriptor/generic_frame_info.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using ::testing::ElementsAre; +using ::testing::IsEmpty; +using ::testing::UnorderedElementsAre; + +constexpr VideoFrameType kVideoFrameKey = VideoFrameType::kVideoFrameKey; +constexpr VideoFrameType kVideoFrameDelta = VideoFrameType::kVideoFrameDelta; + +constexpr CodecBufferUsage ReferenceAndUpdate(int id) { + return CodecBufferUsage(id, /*referenced=*/true, /*updated=*/true); +} +constexpr CodecBufferUsage Reference(int id) { + return CodecBufferUsage(id, /*referenced=*/true, /*updated=*/false); +} +constexpr CodecBufferUsage Update(int id) { + return CodecBufferUsage(id, /*referenced=*/false, /*updated=*/true); +} + +TEST(FrameDependenciesCalculatorTest, SingleLayer) { + CodecBufferUsage pattern[] = {ReferenceAndUpdate(0)}; + FrameDependenciesCalculator calculator; + + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameKey, /*frame_id=*/1, pattern), + IsEmpty()); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/3, pattern), + ElementsAre(1)); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/6, pattern), + ElementsAre(3)); +} + +TEST(FrameDependenciesCalculatorTest, TwoTemporalLayers) { + // Shortened 4-frame pattern: + // T1: 2---4 6---8 ... + // / / / / + // T0: 1---3---5---7 ... + CodecBufferUsage pattern0[] = {ReferenceAndUpdate(0)}; + CodecBufferUsage pattern1[] = {Reference(0), Update(1)}; + CodecBufferUsage pattern2[] = {ReferenceAndUpdate(0)}; + CodecBufferUsage pattern3[] = {Reference(0), Reference(1)}; + FrameDependenciesCalculator calculator; + + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameKey, /*frame_id=*/1, pattern0), + IsEmpty()); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/2, pattern1), + ElementsAre(1)); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/3, pattern2), + ElementsAre(1)); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/4, pattern3), + UnorderedElementsAre(2, 3)); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/5, pattern0), + ElementsAre(3)); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/6, pattern1), + ElementsAre(5)); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/7, pattern2), + ElementsAre(5)); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/8, pattern3), + UnorderedElementsAre(6, 7)); +} + +TEST(FrameDependenciesCalculatorTest, ThreeTemporalLayers4FramePattern) { + // T2: 2---4 6---8 ... + // / / / / + // T1: | 3 | 7 ... + // /_/ /_/ + // T0: 1-------5----- ... + CodecBufferUsage pattern0[] = {ReferenceAndUpdate(0)}; + CodecBufferUsage pattern1[] = {Reference(0), Update(2)}; + CodecBufferUsage pattern2[] = {Reference(0), Update(1)}; + CodecBufferUsage pattern3[] = {Reference(0), Reference(1), Reference(2)}; + FrameDependenciesCalculator calculator; + + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameKey, /*frame_id=*/1, pattern0), + IsEmpty()); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/2, pattern1), + ElementsAre(1)); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/3, pattern2), + ElementsAre(1)); + // Note that frame#4 references buffer#0 that is updated by frame#1, + // yet there is no direct dependency from frame#4 to frame#1. + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/4, pattern3), + UnorderedElementsAre(2, 3)); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/5, pattern0), + ElementsAre(1)); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/6, pattern1), + ElementsAre(5)); +} + +TEST(FrameDependenciesCalculatorTest, SimulcastWith2Layers) { + // S1: 2---4---6- ... + // + // S0: 1---3---5- ... + CodecBufferUsage pattern0[] = {ReferenceAndUpdate(0)}; + CodecBufferUsage pattern1[] = {ReferenceAndUpdate(1)}; + FrameDependenciesCalculator calculator; + + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameKey, /*frame_id=*/1, pattern0), + IsEmpty()); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameKey, /*frame_id=*/2, pattern1), + IsEmpty()); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/3, pattern0), + ElementsAre(1)); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/4, pattern1), + ElementsAre(2)); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/5, pattern0), + ElementsAre(3)); + EXPECT_THAT( + calculator.FromBuffersUsage(kVideoFrameDelta, /*frame_id=*/6, pattern1), + ElementsAre(4)); +} + +} // namespace +} // namespace webrtc