From c46385c346d5a98b83ecef16bd4b4aa219a61d71 Mon Sep 17 00:00:00 2001 From: Danil Chapovalov Date: Wed, 11 Mar 2020 10:45:57 +0100 Subject: [PATCH] Add Av1 Decoder wrapper behind a build flag Bug: webrtc:11404 Change-Id: I090ffd173d667e8845de1b986af462516b7c76e6 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169452 Reviewed-by: Ilya Nikolaevskiy Reviewed-by: Michael Horowitz Reviewed-by: Mirko Bonadei Commit-Queue: Danil Chapovalov Cr-Commit-Position: refs/heads/master@{#30757} --- .gn | 2 + DEPS | 5 + media/BUILD.gn | 2 + media/base/media_constants.cc | 1 + media/base/media_constants.h | 1 + media/engine/internal_decoder_factory.cc | 6 + .../internal_decoder_factory_unittest.cc | 18 ++ modules/video_coding/codecs/av1/BUILD.gn | 37 ++++ modules/video_coding/codecs/av1/DEPS | 3 + .../codecs/av1/libaom_av1_decoder.cc | 191 ++++++++++++++++++ .../codecs/av1/libaom_av1_decoder.h | 26 +++ .../codecs/av1/libaom_av1_decoder_absent.cc | 24 +++ tools_webrtc/libs/generate_licenses.py | 1 + 13 files changed, 317 insertions(+) create mode 100644 modules/video_coding/codecs/av1/BUILD.gn create mode 100644 modules/video_coding/codecs/av1/DEPS create mode 100644 modules/video_coding/codecs/av1/libaom_av1_decoder.cc create mode 100644 modules/video_coding/codecs/av1/libaom_av1_decoder.h create mode 100644 modules/video_coding/codecs/av1/libaom_av1_decoder_absent.cc diff --git a/.gn b/.gn index 09f1b650a1..59dd7d68d8 100644 --- a/.gn +++ b/.gn @@ -72,5 +72,7 @@ default_args = { # for unittests, it can be disabled (see third_party/libyuv/BUILD.gn) libyuv_use_gflags = false + enable_libaom_decoder = true + gtest_enable_absl_printers = true } diff --git a/DEPS b/DEPS index a8bd168fef..c22ac35180 100644 --- a/DEPS +++ b/DEPS @@ -1,5 +1,8 @@ # This file contains dependencies for WebRTC. +gclient_gn_args_file = 'src/build/config/gclient_args.gni' +gclient_gn_args = [] + vars = { # By default, we should check out everything needed to run on the main # chromium waterfalls. More info at: crbug.com/570091. @@ -192,6 +195,8 @@ deps = { 'https://chromium.googlesource.com/chromium/deps/libjpeg_turbo.git@ce0e57e8e636f5132fe6f0590a4dba91f92fd935', 'src/third_party/libsrtp': 'https://chromium.googlesource.com/chromium/deps/libsrtp.git@650611720ecc23e0e6b32b0e3100f8b4df91696c', + 'src/third_party/libaom/source/libaom': + 'https://aomedia.googlesource.com/aom.git@f83d681765cd2aefa9a70ce771af48edd1dbf416', 'src/third_party/libvpx/source/libvpx': 'https://chromium.googlesource.com/webm/libvpx.git@5532775efe808cb0942e7b99bf2f232c6ce99fee', 'src/third_party/libyuv': diff --git a/media/BUILD.gn b/media/BUILD.gn index 8c2d8c0d0c..3b45cf217f 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -246,6 +246,7 @@ rtc_library("rtc_internal_video_codecs") { "../modules/video_coding:webrtc_multiplex", "../modules/video_coding:webrtc_vp8", "../modules/video_coding:webrtc_vp9", + "../modules/video_coding/codecs/av1:libaom_av1_decoder", "../rtc_base:checks", "../rtc_base:deprecation", "../rtc_base:rtc_base_approved", @@ -573,6 +574,7 @@ if (rtc_include_tests) { "../modules/video_coding:simulcast_test_fixture_impl", "../modules/video_coding:video_codec_interface", "../modules/video_coding:webrtc_vp8", + "../modules/video_coding/codecs/av1:libaom_av1_decoder", "../p2p:p2p_test_utils", "../rtc_base", "../rtc_base:checks", diff --git a/media/base/media_constants.cc b/media/base/media_constants.cc index 5bd4b754d2..5144a6ea65 100644 --- a/media/base/media_constants.cc +++ b/media/base/media_constants.cc @@ -105,6 +105,7 @@ const char kComfortNoiseCodecName[] = "CN"; const char kVp8CodecName[] = "VP8"; const char kVp9CodecName[] = "VP9"; +const char kAv1CodecName[] = "AV1X"; const char kH264CodecName[] = "H264"; // RFC 6184 RTP Payload Format for H.264 video diff --git a/media/base/media_constants.h b/media/base/media_constants.h index 136e9f19a0..b9b8a336f7 100644 --- a/media/base/media_constants.h +++ b/media/base/media_constants.h @@ -134,6 +134,7 @@ extern const char kComfortNoiseCodecName[]; RTC_EXPORT extern const char kVp8CodecName[]; RTC_EXPORT extern const char kVp9CodecName[]; +RTC_EXPORT extern const char kAv1CodecName[]; RTC_EXPORT extern const char kH264CodecName[]; // RFC 6184 RTP Payload Format for H.264 video diff --git a/media/engine/internal_decoder_factory.cc b/media/engine/internal_decoder_factory.cc index 5180b28917..e68bb369b5 100644 --- a/media/engine/internal_decoder_factory.cc +++ b/media/engine/internal_decoder_factory.cc @@ -14,6 +14,7 @@ #include "api/video_codecs/sdp_video_format.h" #include "media/base/codec.h" #include "media/base/media_constants.h" +#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h" #include "modules/video_coding/codecs/h264/include/h264.h" #include "modules/video_coding/codecs/vp8/include/vp8.h" #include "modules/video_coding/codecs/vp9/include/vp9.h" @@ -47,6 +48,8 @@ std::vector InternalDecoderFactory::GetSupportedFormats() formats.push_back(format); for (const SdpVideoFormat& h264_format : SupportedH264Codecs()) formats.push_back(h264_format); + if (kIsLibaomAv1DecoderSupported) + formats.push_back(SdpVideoFormat(cricket::kAv1CodecName)); return formats; } @@ -63,6 +66,9 @@ std::unique_ptr InternalDecoderFactory::CreateVideoDecoder( return VP9Decoder::Create(); if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName)) return H264Decoder::Create(); + if (kIsLibaomAv1DecoderSupported && + absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName)) + return CreateLibaomAv1Decoder(); RTC_NOTREACHED(); return nullptr; diff --git a/media/engine/internal_decoder_factory_unittest.cc b/media/engine/internal_decoder_factory_unittest.cc index 5e2bfbf9ec..705933d439 100644 --- a/media/engine/internal_decoder_factory_unittest.cc +++ b/media/engine/internal_decoder_factory_unittest.cc @@ -13,10 +13,16 @@ #include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/video_decoder.h" #include "media/base/media_constants.h" +#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h" +#include "test/gmock.h" #include "test/gtest.h" namespace webrtc { +using ::testing::Contains; +using ::testing::Field; +using ::testing::Not; + TEST(InternalDecoderFactory, TestVP8) { InternalDecoderFactory factory; std::unique_ptr decoder = @@ -24,4 +30,16 @@ TEST(InternalDecoderFactory, TestVP8) { EXPECT_TRUE(decoder); } +TEST(InternalDecoderFactory, Av1) { + InternalDecoderFactory factory; + if (kIsLibaomAv1DecoderSupported) { + EXPECT_THAT(factory.GetSupportedFormats(), + Contains(Field(&SdpVideoFormat::name, "AV1X"))); + EXPECT_TRUE(factory.CreateVideoDecoder(SdpVideoFormat("AV1X"))); + } else { + EXPECT_THAT(factory.GetSupportedFormats(), + Not(Contains(Field(&SdpVideoFormat::name, "AV1X")))); + } +} + } // namespace webrtc diff --git a/modules/video_coding/codecs/av1/BUILD.gn b/modules/video_coding/codecs/av1/BUILD.gn new file mode 100644 index 0000000000..561c81f49f --- /dev/null +++ b/modules/video_coding/codecs/av1/BUILD.gn @@ -0,0 +1,37 @@ +# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import("//third_party/libaom/options.gni") +import("../../../../webrtc.gni") + +rtc_library("libaom_av1_decoder") { + visibility = [ "*" ] + poisonous = [ "software_video_codecs" ] + public = [ "libaom_av1_decoder.h" ] + deps = [ + "../../../../api/video_codecs:video_codecs_api", + "//third_party/abseil-cpp/absl/base:core_headers", + ] + + if (enable_libaom_decoder) { + sources = [ "libaom_av1_decoder.cc" ] + deps += [ + "../..:video_codec_interface", + "../../../../api:scoped_refptr", + "../../../../api/video:encoded_image", + "../../../../api/video:video_frame_i420", + "../../../../common_video", + "../../../../rtc_base:logging", + "//third_party/abseil-cpp/absl/types:optional", + "//third_party/libaom", + "//third_party/libyuv", + ] + } else { + sources = [ "libaom_av1_decoder_absent.cc" ] + } +} diff --git a/modules/video_coding/codecs/av1/DEPS b/modules/video_coding/codecs/av1/DEPS new file mode 100644 index 0000000000..25779919a7 --- /dev/null +++ b/modules/video_coding/codecs/av1/DEPS @@ -0,0 +1,3 @@ +include_rules = [ + "+third_party/libaom", +] diff --git a/modules/video_coding/codecs/av1/libaom_av1_decoder.cc b/modules/video_coding/codecs/av1/libaom_av1_decoder.cc new file mode 100644 index 0000000000..122f214a5c --- /dev/null +++ b/modules/video_coding/codecs/av1/libaom_av1_decoder.cc @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h" + +#include + +#include + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "api/video/encoded_image.h" +#include "api/video/i420_buffer.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_decoder.h" +#include "common_video/include/i420_buffer_pool.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "rtc_base/logging.h" +#include "third_party/libaom/source/libaom/aom/aom_decoder.h" +#include "third_party/libaom/source/libaom/aom/aomdx.h" +#include "third_party/libyuv/include/libyuv/convert.h" + +namespace webrtc { +namespace { + +constexpr int kConfigLowBitDepth = 1; // 8-bits per luma/chroma sample. +constexpr int kDecFlags = 0; // 0 signals no post processing. + +class LibaomAv1Decoder final : public VideoDecoder { + public: + LibaomAv1Decoder(); + LibaomAv1Decoder(const LibaomAv1Decoder&) = delete; + LibaomAv1Decoder& operator=(const LibaomAv1Decoder&) = delete; + ~LibaomAv1Decoder(); + + // Implements VideoDecoder. + int32_t InitDecode(const VideoCodec* codec_settings, + int number_of_cores) override; + + // Decode an encoded video frame. + int32_t Decode(const EncodedImage& encoded_image, + bool missing_frames, + int64_t render_time_ms) override; + + int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) override; + + int32_t Release() override; + + private: + aom_codec_ctx_t context_; + bool inited_; + // Pool of memory buffers to store decoded image data for application access. + I420BufferPool buffer_pool_; + DecodedImageCallback* decode_complete_callback_; +}; + +LibaomAv1Decoder::LibaomAv1Decoder() + : context_(), // Force value initialization instead of default one. + inited_(false), + buffer_pool_(false, /*max_number_of_buffers=*/150), + decode_complete_callback_(nullptr) {} + +LibaomAv1Decoder::~LibaomAv1Decoder() { + Release(); +} + +int32_t LibaomAv1Decoder::InitDecode(const VideoCodec* codec_settings, + int number_of_cores) { + aom_codec_dec_cfg_t config = { + static_cast(number_of_cores), // Max # of threads. + 0, // Frame width set after decode. + 0, // Frame height set after decode. + kConfigLowBitDepth}; // Enable low-bit-depth code path. + + aom_codec_err_t ret = + aom_codec_dec_init(&context_, aom_codec_av1_dx(), &config, kDecFlags); + if (ret != AOM_CODEC_OK) { + RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::InitDecode returned " << ret + << " on aom_codec_dec_init."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + inited_ = true; + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t LibaomAv1Decoder::Decode(const EncodedImage& encoded_image, + bool missing_frames, + int64_t /*render_time_ms*/) { + if (!inited_) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (decode_complete_callback_ == nullptr) { + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + // Decode one video frame. + aom_codec_err_t ret = + aom_codec_decode(&context_, encoded_image.data(), encoded_image.size(), + /*user_priv=*/nullptr); + if (ret != AOM_CODEC_OK) { + RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode returned " << ret + << " on aom_codec_decode."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // Get decoded frame data. + int corrupted_frame = 0; + aom_codec_iter_t iter = nullptr; + while (aom_image_t* decoded_image = aom_codec_get_frame(&context_, &iter)) { + if (aom_codec_control(&context_, AOMD_GET_FRAME_CORRUPTED, + &corrupted_frame)) { + RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode " + "AOM_GET_FRAME_CORRUPTED."; + } + // Check that decoded image format is I420 and has 8-bit depth. + if (decoded_image->fmt != AOM_IMG_FMT_I420) { + RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode invalid image format"; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // Return decoded frame data. + int qp; + ret = aom_codec_control_(&context_, AOMD_GET_LAST_QUANTIZER, &qp); + if (ret != AOM_CODEC_OK) { + RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode returned " << ret + << " on control AOME_GET_LAST_QUANTIZER."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // Allocate memory for decoded frame. + rtc::scoped_refptr buffer = + buffer_pool_.CreateBuffer(decoded_image->d_w, decoded_image->d_h); + if (!buffer.get()) { + // Pool has too many pending frames. + RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode returned due to lack of" + " space in decoded frame buffer pool."; + return WEBRTC_VIDEO_CODEC_ERROR; + } + + // Copy decoded_image to decoded_frame. + libyuv::I420Copy( + decoded_image->planes[AOM_PLANE_Y], decoded_image->stride[AOM_PLANE_Y], + decoded_image->planes[AOM_PLANE_U], decoded_image->stride[AOM_PLANE_U], + decoded_image->planes[AOM_PLANE_V], decoded_image->stride[AOM_PLANE_V], + buffer->MutableDataY(), buffer->StrideY(), buffer->MutableDataU(), + buffer->StrideU(), buffer->MutableDataV(), buffer->StrideV(), + decoded_image->d_w, decoded_image->d_h); + VideoFrame decoded_frame = VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_timestamp_rtp(encoded_image.Timestamp()) + .set_ntp_time_ms(encoded_image.ntp_time_ms_) + .set_color_space(encoded_image.ColorSpace()) + .build(); + + decode_complete_callback_->Decoded(decoded_frame, absl::nullopt, + absl::nullopt); + } + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t LibaomAv1Decoder::RegisterDecodeCompleteCallback( + DecodedImageCallback* decode_complete_callback) { + decode_complete_callback_ = decode_complete_callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t LibaomAv1Decoder::Release() { + if (aom_codec_destroy(&context_) != AOM_CODEC_OK) { + return WEBRTC_VIDEO_CODEC_MEMORY; + } + buffer_pool_.Release(); + inited_ = false; + return WEBRTC_VIDEO_CODEC_OK; +} + +} // namespace + +const bool kIsLibaomAv1DecoderSupported = true; + +std::unique_ptr CreateLibaomAv1Decoder() { + return std::make_unique(); +} + +} // namespace webrtc diff --git a/modules/video_coding/codecs/av1/libaom_av1_decoder.h b/modules/video_coding/codecs/av1/libaom_av1_decoder.h new file mode 100644 index 0000000000..9b01285c73 --- /dev/null +++ b/modules/video_coding/codecs/av1/libaom_av1_decoder.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_VIDEO_CODING_CODECS_AV1_LIBAOM_AV1_DECODER_H_ +#define MODULES_VIDEO_CODING_CODECS_AV1_LIBAOM_AV1_DECODER_H_ + +#include + +#include "absl/base/attributes.h" +#include "api/video_codecs/video_decoder.h" + +namespace webrtc { + +ABSL_CONST_INIT extern const bool kIsLibaomAv1DecoderSupported; + +std::unique_ptr CreateLibaomAv1Decoder(); + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_AV1_LIBAOM_AV1_DECODER_H_ diff --git a/modules/video_coding/codecs/av1/libaom_av1_decoder_absent.cc b/modules/video_coding/codecs/av1/libaom_av1_decoder_absent.cc new file mode 100644 index 0000000000..b97b68b33f --- /dev/null +++ b/modules/video_coding/codecs/av1/libaom_av1_decoder_absent.cc @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h" + +#include + +#include "api/video_codecs/video_decoder.h" + +namespace webrtc { + +const bool kIsLibaomAv1DecoderSupported = false; + +std::unique_ptr CreateLibaomAv1Decoder() { + return nullptr; +} + +} // namespace webrtc diff --git a/tools_webrtc/libs/generate_licenses.py b/tools_webrtc/libs/generate_licenses.py index 2202188339..b8894ed39b 100755 --- a/tools_webrtc/libs/generate_licenses.py +++ b/tools_webrtc/libs/generate_licenses.py @@ -46,6 +46,7 @@ LIB_TO_LICENSES_DICT = { 'guava': ['third_party/guava/LICENSE'], 'ijar': ['third_party/ijar/LICENSE'], 'jsoncpp': ['third_party/jsoncpp/LICENSE'], + 'libaom': ['third_party/libaom/source/libaom/LICENSE'], 'libc++': ['buildtools/third_party/libc++/trunk/LICENSE.TXT'], 'libc++abi': ['buildtools/third_party/libc++abi/trunk/LICENSE.TXT'], 'libevent': ['base/third_party/libevent/LICENSE'],