Add Av1 Decoder wrapper behind a build flag

Bug: webrtc:11404
Change-Id: I090ffd173d667e8845de1b986af462516b7c76e6
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169452
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Michael Horowitz <mhoro@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30757}

.gn

@@ -72,5 +72,7 @@ default_args = {
   # for unittests, it can be disabled (see third_party/libyuv/BUILD.gn)
   libyuv_use_gflags = false
 
+  enable_libaom_decoder = true
+
   gtest_enable_absl_printers = true
 }

DEPS

@@ -1,5 +1,8 @@
 # This file contains dependencies for WebRTC.
 
+gclient_gn_args_file = 'src/build/config/gclient_args.gni'
+gclient_gn_args = []
+
 vars = {
   # By default, we should check out everything needed to run on the main
   # chromium waterfalls. More info at: crbug.com/570091.
@@ -192,6 +195,8 @@ deps = {
     'https://chromium.googlesource.com/chromium/deps/libjpeg_turbo.git@ce0e57e8e636f5132fe6f0590a4dba91f92fd935',
   'src/third_party/libsrtp':
     'https://chromium.googlesource.com/chromium/deps/libsrtp.git@650611720ecc23e0e6b32b0e3100f8b4df91696c',
+  'src/third_party/libaom/source/libaom':
+    'https://aomedia.googlesource.com/aom.git@f83d681765cd2aefa9a70ce771af48edd1dbf416',
   'src/third_party/libvpx/source/libvpx':
     'https://chromium.googlesource.com/webm/libvpx.git@5532775efe808cb0942e7b99bf2f232c6ce99fee',
   'src/third_party/libyuv':

media/BUILD.gn

@@ -246,6 +246,7 @@ rtc_library("rtc_internal_video_codecs") {
     "../modules/video_coding:webrtc_multiplex",
     "../modules/video_coding:webrtc_vp8",
     "../modules/video_coding:webrtc_vp9",
+    "../modules/video_coding/codecs/av1:libaom_av1_decoder",
     "../rtc_base:checks",
     "../rtc_base:deprecation",
     "../rtc_base:rtc_base_approved",
@@ -573,6 +574,7 @@ if (rtc_include_tests) {
       "../modules/video_coding:simulcast_test_fixture_impl",
       "../modules/video_coding:video_codec_interface",
       "../modules/video_coding:webrtc_vp8",
+      "../modules/video_coding/codecs/av1:libaom_av1_decoder",
       "../p2p:p2p_test_utils",
       "../rtc_base",
       "../rtc_base:checks",

media/base/media_constants.cc

@@ -105,6 +105,7 @@ const char kComfortNoiseCodecName[] = "CN";
 
 const char kVp8CodecName[] = "VP8";
 const char kVp9CodecName[] = "VP9";
+const char kAv1CodecName[] = "AV1X";
 const char kH264CodecName[] = "H264";
 
 // RFC 6184 RTP Payload Format for H.264 video

media/base/media_constants.h

@@ -134,6 +134,7 @@ extern const char kComfortNoiseCodecName[];
 
 RTC_EXPORT extern const char kVp8CodecName[];
 RTC_EXPORT extern const char kVp9CodecName[];
+RTC_EXPORT extern const char kAv1CodecName[];
 RTC_EXPORT extern const char kH264CodecName[];
 
 // RFC 6184 RTP Payload Format for H.264 video

media/engine/internal_decoder_factory.cc

@@ -14,6 +14,7 @@
 #include "api/video_codecs/sdp_video_format.h"
 #include "media/base/codec.h"
 #include "media/base/media_constants.h"
+#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h"
 #include "modules/video_coding/codecs/h264/include/h264.h"
 #include "modules/video_coding/codecs/vp8/include/vp8.h"
 #include "modules/video_coding/codecs/vp9/include/vp9.h"
@@ -47,6 +48,8 @@ std::vector<SdpVideoFormat> InternalDecoderFactory::GetSupportedFormats()
     formats.push_back(format);
   for (const SdpVideoFormat& h264_format : SupportedH264Codecs())
     formats.push_back(h264_format);
+  if (kIsLibaomAv1DecoderSupported)
+    formats.push_back(SdpVideoFormat(cricket::kAv1CodecName));
   return formats;
 }
 
@@ -63,6 +66,9 @@ std::unique_ptr<VideoDecoder> InternalDecoderFactory::CreateVideoDecoder(
     return VP9Decoder::Create();
   if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName))
     return H264Decoder::Create();
+  if (kIsLibaomAv1DecoderSupported &&
+      absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName))
+    return CreateLibaomAv1Decoder();
 
   RTC_NOTREACHED();
   return nullptr;

media/engine/internal_decoder_factory_unittest.cc

@@ -13,10 +13,16 @@
 #include "api/video_codecs/sdp_video_format.h"
 #include "api/video_codecs/video_decoder.h"
 #include "media/base/media_constants.h"
+#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h"
+#include "test/gmock.h"
 #include "test/gtest.h"
 
 namespace webrtc {
 
+using ::testing::Contains;
+using ::testing::Field;
+using ::testing::Not;
+
 TEST(InternalDecoderFactory, TestVP8) {
   InternalDecoderFactory factory;
   std::unique_ptr<VideoDecoder> decoder =
@@ -24,4 +30,16 @@ TEST(InternalDecoderFactory, TestVP8) {
   EXPECT_TRUE(decoder);
 }
 
+TEST(InternalDecoderFactory, Av1) {
+  InternalDecoderFactory factory;
+  if (kIsLibaomAv1DecoderSupported) {
+    EXPECT_THAT(factory.GetSupportedFormats(),
+                Contains(Field(&SdpVideoFormat::name, "AV1X")));
+    EXPECT_TRUE(factory.CreateVideoDecoder(SdpVideoFormat("AV1X")));
+  } else {
+    EXPECT_THAT(factory.GetSupportedFormats(),
+                Not(Contains(Field(&SdpVideoFormat::name, "AV1X"))));
+  }
+}
+
 }  // namespace webrtc

modules/video_coding/codecs/av1/BUILD.gn

@@ -0,0 +1,37 @@
+# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import("//third_party/libaom/options.gni")
+import("../../../../webrtc.gni")
+
+rtc_library("libaom_av1_decoder") {
+  visibility = [ "*" ]
+  poisonous = [ "software_video_codecs" ]
+  public = [ "libaom_av1_decoder.h" ]
+  deps = [
+    "../../../../api/video_codecs:video_codecs_api",
+    "//third_party/abseil-cpp/absl/base:core_headers",
+  ]
+
+  if (enable_libaom_decoder) {
+    sources = [ "libaom_av1_decoder.cc" ]
+    deps += [
+      "../..:video_codec_interface",
+      "../../../../api:scoped_refptr",
+      "../../../../api/video:encoded_image",
+      "../../../../api/video:video_frame_i420",
+      "../../../../common_video",
+      "../../../../rtc_base:logging",
+      "//third_party/abseil-cpp/absl/types:optional",
+      "//third_party/libaom",
+      "//third_party/libyuv",
+    ]
+  } else {
+    sources = [ "libaom_av1_decoder_absent.cc" ]
+  }
+}

modules/video_coding/codecs/av1/DEPS

@@ -0,0 +1,3 @@
+include_rules = [
+  "+third_party/libaom",
+]

modules/video_coding/codecs/av1/libaom_av1_decoder.cc

@@ -0,0 +1,191 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h"
+
+#include <stdint.h>
+
+#include <memory>
+
+#include "absl/types/optional.h"
+#include "api/scoped_refptr.h"
+#include "api/video/encoded_image.h"
+#include "api/video/i420_buffer.h"
+#include "api/video_codecs/video_codec.h"
+#include "api/video_codecs/video_decoder.h"
+#include "common_video/include/i420_buffer_pool.h"
+#include "modules/video_coding/include/video_error_codes.h"
+#include "rtc_base/logging.h"
+#include "third_party/libaom/source/libaom/aom/aom_decoder.h"
+#include "third_party/libaom/source/libaom/aom/aomdx.h"
+#include "third_party/libyuv/include/libyuv/convert.h"
+
+namespace webrtc {
+namespace {
+
+constexpr int kConfigLowBitDepth = 1;  // 8-bits per luma/chroma sample.
+constexpr int kDecFlags = 0;           // 0 signals no post processing.
+
+class LibaomAv1Decoder final : public VideoDecoder {
+ public:
+  LibaomAv1Decoder();
+  LibaomAv1Decoder(const LibaomAv1Decoder&) = delete;
+  LibaomAv1Decoder& operator=(const LibaomAv1Decoder&) = delete;
+  ~LibaomAv1Decoder();
+
+  // Implements VideoDecoder.
+  int32_t InitDecode(const VideoCodec* codec_settings,
+                     int number_of_cores) override;
+
+  // Decode an encoded video frame.
+  int32_t Decode(const EncodedImage& encoded_image,
+                 bool missing_frames,
+                 int64_t render_time_ms) override;
+
+  int32_t RegisterDecodeCompleteCallback(
+      DecodedImageCallback* callback) override;
+
+  int32_t Release() override;
+
+ private:
+  aom_codec_ctx_t context_;
+  bool inited_;
+  // Pool of memory buffers to store decoded image data for application access.
+  I420BufferPool buffer_pool_;
+  DecodedImageCallback* decode_complete_callback_;
+};
+
+LibaomAv1Decoder::LibaomAv1Decoder()
+    : context_(),  // Force value initialization instead of default one.
+      inited_(false),
+      buffer_pool_(false, /*max_number_of_buffers=*/150),
+      decode_complete_callback_(nullptr) {}
+
+LibaomAv1Decoder::~LibaomAv1Decoder() {
+  Release();
+}
+
+int32_t LibaomAv1Decoder::InitDecode(const VideoCodec* codec_settings,
+                                     int number_of_cores) {
+  aom_codec_dec_cfg_t config = {
+      static_cast<unsigned int>(number_of_cores),  // Max # of threads.
+      0,                    // Frame width set after decode.
+      0,                    // Frame height set after decode.
+      kConfigLowBitDepth};  // Enable low-bit-depth code path.
+
+  aom_codec_err_t ret =
+      aom_codec_dec_init(&context_, aom_codec_av1_dx(), &config, kDecFlags);
+  if (ret != AOM_CODEC_OK) {
+    RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::InitDecode returned " << ret
+                        << " on aom_codec_dec_init.";
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  inited_ = true;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t LibaomAv1Decoder::Decode(const EncodedImage& encoded_image,
+                                 bool missing_frames,
+                                 int64_t /*render_time_ms*/) {
+  if (!inited_) {
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  }
+  if (decode_complete_callback_ == nullptr) {
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  }
+
+  // Decode one video frame.
+  aom_codec_err_t ret =
+      aom_codec_decode(&context_, encoded_image.data(), encoded_image.size(),
+                       /*user_priv=*/nullptr);
+  if (ret != AOM_CODEC_OK) {
+    RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode returned " << ret
+                        << " on aom_codec_decode.";
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
+  // Get decoded frame data.
+  int corrupted_frame = 0;
+  aom_codec_iter_t iter = nullptr;
+  while (aom_image_t* decoded_image = aom_codec_get_frame(&context_, &iter)) {
+    if (aom_codec_control(&context_, AOMD_GET_FRAME_CORRUPTED,
+                          &corrupted_frame)) {
+      RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode "
+                             "AOM_GET_FRAME_CORRUPTED.";
+    }
+    // Check that decoded image format is I420 and has 8-bit depth.
+    if (decoded_image->fmt != AOM_IMG_FMT_I420) {
+      RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode invalid image format";
+      return WEBRTC_VIDEO_CODEC_ERROR;
+    }
+
+    // Return decoded frame data.
+    int qp;
+    ret = aom_codec_control_(&context_, AOMD_GET_LAST_QUANTIZER, &qp);
+    if (ret != AOM_CODEC_OK) {
+      RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode returned " << ret
+                          << " on control AOME_GET_LAST_QUANTIZER.";
+      return WEBRTC_VIDEO_CODEC_ERROR;
+    }
+
+    // Allocate memory for decoded frame.
+    rtc::scoped_refptr<I420Buffer> buffer =
+        buffer_pool_.CreateBuffer(decoded_image->d_w, decoded_image->d_h);
+    if (!buffer.get()) {
+      // Pool has too many pending frames.
+      RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode returned due to lack of"
+                             " space in decoded frame buffer pool.";
+      return WEBRTC_VIDEO_CODEC_ERROR;
+    }
+
+    // Copy decoded_image to decoded_frame.
+    libyuv::I420Copy(
+        decoded_image->planes[AOM_PLANE_Y], decoded_image->stride[AOM_PLANE_Y],
+        decoded_image->planes[AOM_PLANE_U], decoded_image->stride[AOM_PLANE_U],
+        decoded_image->planes[AOM_PLANE_V], decoded_image->stride[AOM_PLANE_V],
+        buffer->MutableDataY(), buffer->StrideY(), buffer->MutableDataU(),
+        buffer->StrideU(), buffer->MutableDataV(), buffer->StrideV(),
+        decoded_image->d_w, decoded_image->d_h);
+    VideoFrame decoded_frame = VideoFrame::Builder()
+                                   .set_video_frame_buffer(buffer)
+                                   .set_timestamp_rtp(encoded_image.Timestamp())
+                                   .set_ntp_time_ms(encoded_image.ntp_time_ms_)
+                                   .set_color_space(encoded_image.ColorSpace())
+                                   .build();
+
+    decode_complete_callback_->Decoded(decoded_frame, absl::nullopt,
+                                       absl::nullopt);
+  }
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t LibaomAv1Decoder::RegisterDecodeCompleteCallback(
+    DecodedImageCallback* decode_complete_callback) {
+  decode_complete_callback_ = decode_complete_callback;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t LibaomAv1Decoder::Release() {
+  if (aom_codec_destroy(&context_) != AOM_CODEC_OK) {
+    return WEBRTC_VIDEO_CODEC_MEMORY;
+  }
+  buffer_pool_.Release();
+  inited_ = false;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+}  // namespace
+
+const bool kIsLibaomAv1DecoderSupported = true;
+
+std::unique_ptr<VideoDecoder> CreateLibaomAv1Decoder() {
+  return std::make_unique<LibaomAv1Decoder>();
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/libaom_av1_decoder.h

@@ -0,0 +1,26 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_LIBAOM_AV1_DECODER_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_LIBAOM_AV1_DECODER_H_
+
+#include <memory>
+
+#include "absl/base/attributes.h"
+#include "api/video_codecs/video_decoder.h"
+
+namespace webrtc {
+
+ABSL_CONST_INIT extern const bool kIsLibaomAv1DecoderSupported;
+
+std::unique_ptr<VideoDecoder> CreateLibaomAv1Decoder();
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_LIBAOM_AV1_DECODER_H_

modules/video_coding/codecs/av1/libaom_av1_decoder_absent.cc

@@ -0,0 +1,24 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h"
+
+#include <memory>
+
+#include "api/video_codecs/video_decoder.h"
+
+namespace webrtc {
+
+const bool kIsLibaomAv1DecoderSupported = false;
+
+std::unique_ptr<VideoDecoder> CreateLibaomAv1Decoder() {
+  return nullptr;
+}
+
+}  // namespace webrtc

tools_webrtc/libs/generate_licenses.py

@@ -46,6 +46,7 @@ LIB_TO_LICENSES_DICT = {
     'guava': ['third_party/guava/LICENSE'],
     'ijar': ['third_party/ijar/LICENSE'],
     'jsoncpp': ['third_party/jsoncpp/LICENSE'],
+    'libaom': ['third_party/libaom/source/libaom/LICENSE'],
     'libc++': ['buildtools/third_party/libc++/trunk/LICENSE.TXT'],
     'libc++abi': ['buildtools/third_party/libc++abi/trunk/LICENSE.TXT'],
     'libevent': ['base/third_party/libevent/LICENSE'],