Add dav1d decoder to WebRTC.

Bug: none
Change-Id: I7642f42e592dcf510679f881f118bc4dab93b31c
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/237504
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Magnus Flodman <mflodman@webrtc.org>
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35394}

DEPS

@@ -219,6 +219,8 @@ deps = {
     'https://chromium.googlesource.com/chromium/deps/libjpeg_turbo.git@49836d72bd22c7a78bc0250483f04162278cdc6a',
   'src/third_party/libsrtp':
     'https://chromium.googlesource.com/chromium/deps/libsrtp.git@5b7c744eb8310250ccc534f3f86a2015b3887a0a',
+  'src/third_party/dav1d/libdav1d':
+    'https://chromium.googlesource.com/external/github.com/videolan/dav1d.git@692c0ce873d7d823f2255968e32b233d71d88b43',
   'src/third_party/libaom/source/libaom':
     'https://aomedia.googlesource.com/aom.git@b5719d38f3eb67e405b9fd7c90945f0a7ece10c0',
   'src/third_party/libunwindstack': {

media/BUILD.gn

@@ -210,11 +210,13 @@ rtc_library("rtc_internal_video_codecs") {
     "../modules/video_coding:webrtc_multiplex",
     "../modules/video_coding:webrtc_vp8",
     "../modules/video_coding:webrtc_vp9",
+    "../modules/video_coding/codecs/av1:dav1d_decoder",
     "../modules/video_coding/codecs/av1:libaom_av1_decoder",
     "../modules/video_coding/codecs/av1:libaom_av1_encoder",
     "../rtc_base:checks",
     "../rtc_base:rtc_base_approved",
     "../rtc_base/system:rtc_export",
+    "../system_wrappers:field_trial",
     "../test:fake_video_codecs",
   ]
   absl_deps = [

media/engine/internal_decoder_factory.cc

@@ -15,14 +15,19 @@
 #include "api/video_codecs/video_codec.h"
 #include "media/base/codec.h"
 #include "media/base/media_constants.h"
+#include "modules/video_coding/codecs/av1/dav1d_decoder.h"
 #include "modules/video_coding/codecs/av1/libaom_av1_decoder.h"
 #include "modules/video_coding/codecs/h264/include/h264.h"
 #include "modules/video_coding/codecs/vp8/include/vp8.h"
 #include "modules/video_coding/codecs/vp9/include/vp9.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/logging.h"
+#include "system_wrappers/include/field_trial.h"
 
 namespace webrtc {
+namespace {
+constexpr char kDav1dFieldTrial[] = "WebRTC-Dav1dDecoder";
+}  // namespace
 
 std::vector<SdpVideoFormat> InternalDecoderFactory::GetSupportedFormats()
     const {
@@ -32,8 +37,10 @@ std::vector<SdpVideoFormat> InternalDecoderFactory::GetSupportedFormats()
     formats.push_back(format);
   for (const SdpVideoFormat& h264_format : SupportedH264Codecs())
     formats.push_back(h264_format);
-  if (kIsLibaomAv1DecoderSupported)
+  if (field_trial::IsEnabled(kDav1dFieldTrial) ||
+      kIsLibaomAv1DecoderSupported) {
     formats.push_back(SdpVideoFormat(cricket::kAv1CodecName));
+  }
   return formats;
 }
 
@@ -69,9 +76,14 @@ std::unique_ptr<VideoDecoder> InternalDecoderFactory::CreateVideoDecoder(
     return VP9Decoder::Create();
   if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName))
     return H264Decoder::Create();
-  if (kIsLibaomAv1DecoderSupported &&
+  if (absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName) &&
-      absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName))
+      field_trial::IsEnabled(kDav1dFieldTrial)) {
+    return CreateDav1dDecoder();
+  }
+  if (absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName) &&
+      kIsLibaomAv1DecoderSupported) {
     return CreateLibaomAv1Decoder();
+  }
 
   RTC_DCHECK_NOTREACHED();
   return nullptr;

modules/video_coding/BUILD.gn

@@ -915,6 +915,7 @@ if (rtc_include_tests) {
       "codecs/h264/test/h264_impl_unittest.cc",
       "codecs/multiplex/test/multiplex_adapter_unittest.cc",
       "codecs/test/video_encoder_decoder_instantiation_tests.cc",
+      "codecs/test/videocodec_test_av1.cc",
       "codecs/test/videocodec_test_libvpx.cc",
       "codecs/vp8/test/vp8_impl_unittest.cc",
     ]
@@ -923,10 +924,6 @@ if (rtc_include_tests) {
       sources += [ "codecs/vp9/test/vp9_impl_unittest.cc" ]
     }
 
-    # TODO(jianj): Fix crash on iOS and re-enable
-    if (enable_libaom && !is_ios) {
-      sources += [ "codecs/test/videocodec_test_libaom.cc" ]
-    }
     if (rtc_use_h264) {
       sources += [ "codecs/test/videocodec_test_openh264.cc" ]
     }
@@ -970,6 +967,7 @@ if (rtc_include_tests) {
       "../../test:test_support",
       "../../test:video_test_common",
       "../rtp_rtcp:rtp_rtcp_format",
+      "codecs/av1:libaom_av1_decoder",
       "//third_party/libyuv",
     ]
     absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]

modules/video_coding/codecs/av1/BUILD.gn

@@ -23,6 +23,25 @@ rtc_library("av1_svc_config") {
   ]
 }
 
+rtc_library("dav1d_decoder") {
+  poisonous = [ "software_video_codecs" ]
+  public = [ "dav1d_decoder.h" ]
+  sources = [ "dav1d_decoder.cc" ]
+
+  deps = [
+    "../..:video_codec_interface",
+    "../../../../api:scoped_refptr",
+    "../../../../api/video:encoded_image",
+    "../../../../api/video:video_frame",
+    "../../../../api/video_codecs:video_codecs_api",
+    "../../../../common_video",
+    "../../../../rtc_base:logging",
+    "//third_party/dav1d",
+    "//third_party/libyuv",
+  ]
+  absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
 rtc_library("libaom_av1_decoder") {
   visibility = [ "*" ]
   poisonous = [ "software_video_codecs" ]

modules/video_coding/codecs/av1/DEPS

@@ -1,3 +1,4 @@
 include_rules = [
   "+third_party/libaom",
+  "+third_party/dav1d",
 ]

modules/video_coding/codecs/av1/dav1d_decoder.cc

@@ -0,0 +1,197 @@
+/*
+ *  Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/av1/dav1d_decoder.h"
+
+#include <algorithm>
+
+#include "api/scoped_refptr.h"
+#include "api/video/encoded_image.h"
+#include "api/video/i420_buffer.h"
+#include "common_video/include/video_frame_buffer_pool.h"
+#include "modules/video_coding/include/video_error_codes.h"
+#include "rtc_base/logging.h"
+#include "third_party/dav1d/libdav1d/include/dav1d/dav1d.h"
+#include "third_party/libyuv/include/libyuv/convert.h"
+
+namespace webrtc {
+namespace {
+
+class Dav1dDecoder : public VideoDecoder {
+ public:
+  Dav1dDecoder();
+  Dav1dDecoder(const Dav1dDecoder&) = delete;
+  Dav1dDecoder& operator=(const Dav1dDecoder&) = delete;
+
+  ~Dav1dDecoder() override;
+
+  bool Configure(const Settings& settings) override;
+  int32_t Decode(const EncodedImage& encoded_image,
+                 bool missing_frames,
+                 int64_t render_time_ms) override;
+  int32_t RegisterDecodeCompleteCallback(
+      DecodedImageCallback* callback) override;
+  int32_t Release() override;
+  DecoderInfo GetDecoderInfo() const override;
+  const char* ImplementationName() const override;
+
+ private:
+  VideoFrameBufferPool buffer_pool_;
+  Dav1dContext* context_ = nullptr;
+  DecodedImageCallback* decode_complete_callback_ = nullptr;
+};
+
+class ScopedDav1dData {
+ public:
+  ~ScopedDav1dData() { dav1d_data_unref(&data_); }
+
+  Dav1dData& Data() { return data_; }
+
+ private:
+  Dav1dData data_ = {};
+};
+
+class ScopedDav1dPicture {
+ public:
+  ~ScopedDav1dPicture() { dav1d_picture_unref(&picture_); }
+
+  Dav1dPicture& Picture() { return picture_; }
+
+ private:
+  Dav1dPicture picture_ = {};
+};
+
+constexpr char kDav1dName[] = "dav1d";
+
+// Calling `dav1d_data_wrap` requires a `free_callback` to be registered.
+void NullFreeCallback(const uint8_t* buffer, void* opaque) {}
+
+Dav1dDecoder::Dav1dDecoder()
+    : buffer_pool_(/*zero_initialize=*/false, /*max_number_of_buffers=*/150) {}
+
+Dav1dDecoder::~Dav1dDecoder() {
+  Release();
+}
+
+bool Dav1dDecoder::Configure(const Settings& settings) {
+  Dav1dSettings s;
+  dav1d_default_settings(&s);
+
+  s.n_threads = std::max(2, settings.number_of_cores());
+  s.max_frame_delay = 1;   // For low latency decoding.
+  s.all_layers = 0;        // Don't output a frame for every spatial layer.
+  s.operating_point = 31;  // Decode all operating points.
+
+  return dav1d_open(&context_, &s) == 0;
+}
+
+int32_t Dav1dDecoder::RegisterDecodeCompleteCallback(
+    DecodedImageCallback* decode_complete_callback) {
+  decode_complete_callback_ = decode_complete_callback;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+int32_t Dav1dDecoder::Release() {
+  dav1d_close(&context_);
+  if (context_ != nullptr) {
+    return WEBRTC_VIDEO_CODEC_MEMORY;
+  }
+  buffer_pool_.Release();
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+VideoDecoder::DecoderInfo Dav1dDecoder::GetDecoderInfo() const {
+  DecoderInfo info;
+  info.implementation_name = kDav1dName;
+  info.is_hardware_accelerated = false;
+  return info;
+}
+
+const char* Dav1dDecoder::ImplementationName() const {
+  return kDav1dName;
+}
+
+int32_t Dav1dDecoder::Decode(const EncodedImage& encoded_image,
+                             bool /*missing_frames*/,
+                             int64_t /*render_time_ms*/) {
+  if (!context_ || decode_complete_callback_ == nullptr) {
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  }
+
+  ScopedDav1dData scoped_dav1d_data;
+  Dav1dData& dav1d_data = scoped_dav1d_data.Data();
+  dav1d_data_wrap(&dav1d_data, encoded_image.data(), encoded_image.size(),
+                  /*free_callback=*/&NullFreeCallback,
+                  /*user_data=*/nullptr);
+
+  if (int decode_res = dav1d_send_data(context_, &dav1d_data)) {
+    RTC_LOG(LS_WARNING)
+        << "Dav1dDecoder::Decode decoding failed with error code "
+        << decode_res;
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
+  ScopedDav1dPicture scoped_dav1d_picture;
+  Dav1dPicture& dav1d_picture = scoped_dav1d_picture.Picture();
+  if (int get_picture_res = dav1d_get_picture(context_, &dav1d_picture)) {
+    RTC_LOG(LS_WARNING)
+        << "Dav1dDecoder::Decode getting picture failed with error code "
+        << get_picture_res;
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
+  // Only accept I420 pixel format and 8 bit depth.
+  if (dav1d_picture.p.layout != DAV1D_PIXEL_LAYOUT_I420 ||
+      dav1d_picture.p.bpc != 8) {
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
+  rtc::scoped_refptr<I420Buffer> buffer =
+      buffer_pool_.CreateI420Buffer(dav1d_picture.p.w, dav1d_picture.p.h);
+  if (!buffer.get()) {
+    RTC_LOG(LS_WARNING)
+        << "Dav1dDecoder::Decode failed to get frame from the buffer pool.";
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
+  uint8_t* y_data = static_cast<uint8_t*>(dav1d_picture.data[0]);
+  uint8_t* u_data = static_cast<uint8_t*>(dav1d_picture.data[1]);
+  uint8_t* v_data = static_cast<uint8_t*>(dav1d_picture.data[2]);
+  int y_stride = dav1d_picture.stride[0];
+  int uv_stride = dav1d_picture.stride[1];
+  libyuv::I420Copy(y_data, y_stride,                           //
+                   u_data, uv_stride,                          //
+                   v_data, uv_stride,                          //
+                   buffer->MutableDataY(), buffer->StrideY(),  //
+                   buffer->MutableDataU(), buffer->StrideU(),  //
+                   buffer->MutableDataV(), buffer->StrideV(),  //
+                   dav1d_picture.p.w,                          //
+                   dav1d_picture.p.h);                         //
+
+  VideoFrame decoded_frame = VideoFrame::Builder()
+                                 .set_video_frame_buffer(buffer)
+                                 .set_timestamp_rtp(encoded_image.Timestamp())
+                                 .set_ntp_time_ms(encoded_image.ntp_time_ms_)
+                                 .set_color_space(encoded_image.ColorSpace())
+                                 .build();
+
+  decode_complete_callback_->Decoded(decoded_frame, absl::nullopt,
+                                     absl::nullopt);
+
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+}  // namespace
+
+std::unique_ptr<VideoDecoder> CreateDav1dDecoder() {
+  return std::make_unique<Dav1dDecoder>();
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/dav1d_decoder.h

@@ -0,0 +1,23 @@
+/*
+ *  Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_DAV1D_DECODER_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_DAV1D_DECODER_H_
+
+#include <memory>
+
+#include "api/video_codecs/video_decoder.h"
+
+namespace webrtc {
+
+std::unique_ptr<VideoDecoder> CreateDav1dDecoder();
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_DAV1D_DECODER_H_

modules/video_coding/codecs/test/videocodec_test_av1.cc

@@ -18,6 +18,8 @@
 #include "media/engine/internal_decoder_factory.h"
 #include "media/engine/internal_encoder_factory.h"
 #include "media/engine/simulcast_encoder_adapter.h"
+#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h"
+#include "test/field_trial.h"
 #include "test/gtest.h"
 #include "test/testsupport/file_utils.h"
 
@@ -38,7 +40,15 @@ VideoCodecTestFixture::Config CreateConfig(std::string filename) {
   return config;
 }
 
-TEST(VideoCodecTestLibaom, HighBitrateAV1) {
+class VideoCodecTestAv1 : public ::testing::TestWithParam<std::string> {
+ public:
+  VideoCodecTestAv1() : scoped_field_trial_(GetParam()) {}
+
+ private:
+  ScopedFieldTrials scoped_field_trial_;
+};
+
+TEST_P(VideoCodecTestAv1, HighBitrate) {
   auto config = CreateConfig("foreman_cif");
   config.SetCodecSettings(cricket::kAv1CodecName, 1, 1, 1, false, true, true,
                           kCifWidth, kCifHeight);
@@ -56,7 +66,7 @@ TEST(VideoCodecTestLibaom, HighBitrateAV1) {
   fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
 }
 
-TEST(VideoCodecTestLibaom, VeryLowBitrateAV1) {
+TEST_P(VideoCodecTestAv1, VeryLowBitrate) {
   auto config = CreateConfig("foreman_cif");
   config.SetCodecSettings(cricket::kAv1CodecName, 1, 1, 1, false, true, true,
                           kCifWidth, kCifHeight);
@@ -76,7 +86,7 @@ TEST(VideoCodecTestLibaom, VeryLowBitrateAV1) {
 #if !defined(WEBRTC_ANDROID)
 constexpr int kHdWidth = 1280;
 constexpr int kHdHeight = 720;
-TEST(VideoCodecTestLibaom, HdAV1) {
+TEST_P(VideoCodecTestAv1, Hd) {
   auto config = CreateConfig("ConferenceMotion_1280_720_50");
   config.SetCodecSettings(cricket::kAv1CodecName, 1, 1, 1, false, true, true,
                           kHdWidth, kHdHeight);
@@ -95,6 +105,21 @@ TEST(VideoCodecTestLibaom, HdAV1) {
 }
 #endif
 
+std::vector<std::string> GetTestValues() {
+  std::vector<std::string> field_trial_values;
+  field_trial_values.push_back("WebRTC-Dav1dDecoder/Enabled/");
+  if (kIsLibaomAv1DecoderSupported) {
+    // As long as the field trial doesn't enable dav1d the libaom decoder will
+    // be used instead.
+    field_trial_values.push_back("");
+  }
+  return field_trial_values;
+}
+
+INSTANTIATE_TEST_SUITE_P(Decoder,
+                         VideoCodecTestAv1,
+                         testing::ValuesIn(GetTestValues()));
+
 }  // namespace
 }  // namespace test
 }  // namespace webrtc