mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-13 05:40:42 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Added support for H264 YUV444 (I444) decoding.

Browse source 
Added Nutanix Inc. to the AUTHORS file.

PS#1 is a reland of "Added support for H264 YUV444 (I444) decoding." https://webrtc-review.googlesource.com/c/src/+/234540

Bug: chromium:1251096
Change-Id: I99a1b1e4d8b60192ff96f92334a430240875c66c
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/235340
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Commit-Queue: Harald Alvestrand <hta@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35684}
This commit is contained in:
Stefan Mitic 2021-12-17 07:05:51 -08:00 committed by WebRTC LUCI CQ
parent 6b667a8fe2
commit 3babb8af23
19 changed files with 683 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
AUTHORS
View file

@ -138,6 +138,7 @@ Microsoft Corporation <*@microsoft.com>
MIPS Technologies <*@mips.com>
Mozilla Foundation <*@mozilla.com>
Netgem S.A. <*@netgem.com>
Nutanix Inc. <*@nutanix.com>
NVIDIA Corporation <*@nvidia.com>
Opera Software ASA <*@opera.com>
Optical Tone Ltd <*@opticaltone.com>

2
api/video/BUILD.gn
View file

@ -43,6 +43,8 @@ rtc_library("video_frame") {
sources = [
"i420_buffer.cc",
"i420_buffer.h",
"i444_buffer.cc",
"i444_buffer.h",
"nv12_buffer.cc",
"nv12_buffer.h",
"video_codec_type.h",

4
api/video/DEPS
View file

@ -18,6 +18,10 @@ specific_include_rules = {
"+rtc_base/memory/aligned_malloc.h",
],
"i444_buffer\.h": [
"+rtc_base/memory/aligned_malloc.h",
],
"nv12_buffer\.h": [
"+rtc_base/memory/aligned_malloc.h",
],

211
api/video/i444_buffer.cc Normal file
View file

@ -0,0 +1,211 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/video/i444_buffer.h"
#include <string.h>
#include <algorithm>
#include <utility>
#include "api/video/i420_buffer.h"
#include "rtc_base/checks.h"
#include "rtc_base/ref_counted_object.h"
#include "third_party/libyuv/include/libyuv/convert.h"
#include "third_party/libyuv/include/libyuv/planar_functions.h"
#include "third_party/libyuv/include/libyuv/scale.h"
// Aligning pointer to 64 bytes for improved performance, e.g. use SIMD.
static const int kBufferAlignment = 64;
namespace webrtc {
namespace {
int I444DataSize(int height, int stride_y, int stride_u, int stride_v) {
return stride_y * height + stride_u * height + stride_v * height;
}
} // namespace
I444Buffer::I444Buffer(int width, int height)
: I444Buffer(width, height, width, (width), (width)) {}
I444Buffer::I444Buffer(int width,
int height,
int stride_y,
int stride_u,
int stride_v)
: width_(width),
height_(height),
stride_y_(stride_y),
stride_u_(stride_u),
stride_v_(stride_v),
data_(static_cast<uint8_t*>(
AlignedMalloc(I444DataSize(height, stride_y, stride_u, stride_v),
kBufferAlignment))) {
RTC_DCHECK_GT(width, 0);
RTC_DCHECK_GT(height, 0);
RTC_DCHECK_GE(stride_y, width);
RTC_DCHECK_GE(stride_u, (width));
RTC_DCHECK_GE(stride_v, (width));
}
I444Buffer::~I444Buffer() {}
// static
rtc::scoped_refptr<I444Buffer> I444Buffer::Create(int width, int height) {
return new rtc::RefCountedObject<I444Buffer>(width, height);
}
// static
rtc::scoped_refptr<I444Buffer> I444Buffer::Create(int width,
int height,
int stride_y,
int stride_u,
int stride_v) {
return new rtc::RefCountedObject<I444Buffer>(width, height, stride_y,
stride_u, stride_v);
}
// static
rtc::scoped_refptr<I444Buffer> I444Buffer::Copy(
const I444BufferInterface& source) {
return Copy(source.width(), source.height(), source.DataY(), source.StrideY(),
source.DataU(), source.StrideU(), source.DataV(),
source.StrideV());
}
// static
rtc::scoped_refptr<I444Buffer> I444Buffer::Copy(int width,
int height,
const uint8_t* data_y,
int stride_y,
const uint8_t* data_u,
int stride_u,
const uint8_t* data_v,
int stride_v) {
// Note: May use different strides than the input data.
rtc::scoped_refptr<I444Buffer> buffer = Create(width, height);
RTC_CHECK_EQ(0, libyuv::I444Copy(data_y, stride_y, data_u, stride_u, data_v,
stride_v, buffer->MutableDataY(),
buffer->StrideY(), buffer->MutableDataU(),
buffer->StrideU(), buffer->MutableDataV(),
buffer->StrideV(), width, height));
return buffer;
}
// static
rtc::scoped_refptr<I444Buffer> I444Buffer::Rotate(
const I444BufferInterface& src,
VideoRotation rotation) {
RTC_CHECK(src.DataY());
RTC_CHECK(src.DataU());
RTC_CHECK(src.DataV());
int rotated_width = src.width();
int rotated_height = src.height();
if (rotation == webrtc::kVideoRotation_90 ||
rotation == webrtc::kVideoRotation_270) {
std::swap(rotated_width, rotated_height);
}
rtc::scoped_refptr<webrtc::I444Buffer> buffer =
I444Buffer::Create(rotated_width, rotated_height);
RTC_CHECK_EQ(0,
libyuv::I444Rotate(
src.DataY(), src.StrideY(), src.DataU(), src.StrideU(),
src.DataV(), src.StrideV(), buffer->MutableDataY(),
buffer->StrideY(), buffer->MutableDataU(), buffer->StrideU(),
buffer->MutableDataV(), buffer->StrideV(), src.width(),
src.height(), static_cast<libyuv::RotationMode>(rotation)));
return buffer;
}
rtc::scoped_refptr<I420BufferInterface> I444Buffer::ToI420() {
rtc::scoped_refptr<I420Buffer> i420_buffer =
I420Buffer::Create(width(), height());
libyuv::I444ToI420(DataY(), StrideY(), DataU(), StrideU(), DataV(), StrideV(),
i420_buffer->MutableDataY(), i420_buffer->StrideY(),
i420_buffer->MutableDataU(), i420_buffer->StrideU(),
i420_buffer->MutableDataV(), i420_buffer->StrideV(),
width(), height());
return i420_buffer;
}
void I444Buffer::InitializeData() {
memset(data_.get(), 0,
I444DataSize(height_, stride_y_, stride_u_, stride_v_));
}
int I444Buffer::width() const {
return width_;
}
int I444Buffer::height() const {
return height_;
}
const uint8_t* I444Buffer::DataY() const {
return data_.get();
}
const uint8_t* I444Buffer::DataU() const {
return data_.get() + stride_y_ * height_;
}
const uint8_t* I444Buffer::DataV() const {
return data_.get() + stride_y_ * height_ + stride_u_ * ((height_));
}
int I444Buffer::StrideY() const {
return stride_y_;
}
int I444Buffer::StrideU() const {
return stride_u_;
}
int I444Buffer::StrideV() const {
return stride_v_;
}
uint8_t* I444Buffer::MutableDataY() {
return const_cast<uint8_t*>(DataY());
}
uint8_t* I444Buffer::MutableDataU() {
return const_cast<uint8_t*>(DataU());
}
uint8_t* I444Buffer::MutableDataV() {
return const_cast<uint8_t*>(DataV());
}
void I444Buffer::CropAndScaleFrom(const I444BufferInterface& src,
int offset_x,
int offset_y,
int crop_width,
int crop_height) {
RTC_CHECK_LE(crop_width, src.width());
RTC_CHECK_LE(crop_height, src.height());
RTC_CHECK_LE(crop_width + offset_x, src.width());
RTC_CHECK_LE(crop_height + offset_y, src.height());
RTC_CHECK_GE(offset_x, 0);
RTC_CHECK_GE(offset_y, 0);
const uint8_t* y_plane = src.DataY() + src.StrideY() * offset_y + offset_x;
const uint8_t* u_plane = src.DataU() + src.StrideU() * offset_y + offset_x;
const uint8_t* v_plane = src.DataV() + src.StrideV() * offset_y + offset_x;
int res =
libyuv::I444Scale(y_plane, src.StrideY(), u_plane, src.StrideU(), v_plane,
src.StrideV(), crop_width, crop_height, MutableDataY(),
StrideY(), MutableDataU(), StrideU(), MutableDataV(),
StrideV(), width(), height(), libyuv::kFilterBox);
RTC_DCHECK_EQ(res, 0);
}
} // namespace webrtc

104
api/video/i444_buffer.h Normal file
View file

@ -0,0 +1,104 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef API_VIDEO_I444_BUFFER_H_
#define API_VIDEO_I444_BUFFER_H_
#include <stdint.h>
#include <memory>
#include "api/scoped_refptr.h"
#include "api/video/video_frame_buffer.h"
#include "api/video/video_rotation.h"
#include "rtc_base/memory/aligned_malloc.h"
#include "rtc_base/system/rtc_export.h"
namespace webrtc {
// Plain I444 buffer in standard memory.
// I444 represents an image with in YUV format withouth any chroma subsampling.
// https://en.wikipedia.org/wiki/Chroma_subsampling#4:4:4
class RTC_EXPORT I444Buffer : public I444BufferInterface {
public:
static rtc::scoped_refptr<I444Buffer> Create(int width, int height);
static rtc::scoped_refptr<I444Buffer> Create(int width,
int height,
int stride_y,
int stride_u,
int stride_v);
// Create a new buffer and copy the pixel data.
static rtc::scoped_refptr<I444Buffer> Copy(const I444BufferInterface& buffer);
static rtc::scoped_refptr<I444Buffer> Copy(int width,
int height,
const uint8_t* data_y,
int stride_y,
const uint8_t* data_u,
int stride_u,
const uint8_t* data_v,
int stride_v);
// Returns a rotated copy of |src|.
static rtc::scoped_refptr<I444Buffer> Rotate(const I444BufferInterface& src,
VideoRotation rotation);
rtc::scoped_refptr<I420BufferInterface> ToI420() final;
const I420BufferInterface* GetI420() const final { return nullptr; }
// Sets all three planes to all zeros. Used to work around for
// quirks in memory checkers
// (https://bugs.chromium.org/p/libyuv/issues/detail?id=377) and
// ffmpeg (http://crbug.com/390941).
// TODO(nisse): Deprecated. Should be deleted if/when those issues
// are resolved in a better way. Or in the mean time, use SetBlack.
void InitializeData();
int width() const override;
int height() const override;
const uint8_t* DataY() const override;
const uint8_t* DataU() const override;
const uint8_t* DataV() const override;
int StrideY() const override;
int StrideU() const override;
int StrideV() const override;
uint8_t* MutableDataY();
uint8_t* MutableDataU();
uint8_t* MutableDataV();
// Scale the cropped area of |src| to the size of |this| buffer, and
// write the result into |this|.
void CropAndScaleFrom(const I444BufferInterface& src,
int offset_x,
int offset_y,
int crop_width,
int crop_height);
protected:
I444Buffer(int width, int height);
I444Buffer(int width, int height, int stride_y, int stride_u, int stride_v);
~I444Buffer() override;
private:
const int width_;
const int height_;
const int stride_y_;
const int stride_u_;
const int stride_v_;
const std::unique_ptr<uint8_t, AlignedFreeDeleter> data_;
};
} // namespace webrtc
#endif // API_VIDEO_I444_BUFFER_H_

1
api/video/test/BUILD.gn
View file

@ -12,6 +12,7 @@ rtc_library("rtc_api_video_unittests") {
testonly = true
sources = [
"color_space_unittest.cc",
"i444_buffer_unittest.cc",
"nv12_buffer_unittest.cc",
"video_adaptation_counters_unittest.cc",
"video_bitrate_allocation_unittest.cc",

112
api/video/test/i444_buffer_unittest.cc Normal file
View file

@ -0,0 +1,112 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/video/i444_buffer.h"
#include "api/video/i420_buffer.h"
#include "test/frame_utils.h"
#include "test/gmock.h"
#include "test/gtest.h"
namespace webrtc {
namespace {
int GetY(rtc::scoped_refptr<I444BufferInterface> buf, int col, int row) {
return buf->DataY()[row * buf->StrideY() + col];
}
int GetU(rtc::scoped_refptr<I444BufferInterface> buf, int col, int row) {
return buf->DataU()[row * buf->StrideU() + col];
}
int GetV(rtc::scoped_refptr<I444BufferInterface> buf, int col, int row) {
return buf->DataV()[row * buf->StrideV() + col];
}
void FillI444Buffer(rtc::scoped_refptr<I444Buffer> buf) {
const uint8_t Y = 1;
const uint8_t U = 2;
const uint8_t V = 3;
for (int row = 0; row < buf->height(); ++row) {
for (int col = 0; col < buf->width(); ++col) {
buf->MutableDataY()[row * buf->StrideY() + col] = Y;
buf->MutableDataU()[row * buf->StrideU() + col] = U;
buf->MutableDataV()[row * buf->StrideV() + col] = V;
}
}
}
} // namespace
TEST(I444BufferTest, InitialData) {
constexpr int stride = 3;
constexpr int width = 3;
constexpr int height = 3;
rtc::scoped_refptr<I444Buffer> i444_buffer(I444Buffer::Create(width, height));
EXPECT_EQ(width, i444_buffer->width());
EXPECT_EQ(height, i444_buffer->height());
EXPECT_EQ(stride, i444_buffer->StrideY());
EXPECT_EQ(stride, i444_buffer->StrideU());
EXPECT_EQ(stride, i444_buffer->StrideV());
EXPECT_EQ(3, i444_buffer->ChromaWidth());
EXPECT_EQ(3, i444_buffer->ChromaHeight());
}
TEST(I444BufferTest, ReadPixels) {
constexpr int width = 3;
constexpr int height = 3;
rtc::scoped_refptr<I444Buffer> i444_buffer(I444Buffer::Create(width, height));
// Y = 1, U = 2, V = 3.
FillI444Buffer(i444_buffer);
for (int row = 0; row < height; row++) {
for (int col = 0; col < width; col++) {
EXPECT_EQ(1, GetY(i444_buffer, col, row));
EXPECT_EQ(2, GetU(i444_buffer, col, row));
EXPECT_EQ(3, GetV(i444_buffer, col, row));
}
}
}
TEST(I444BufferTest, ToI420) {
constexpr int width = 3;
constexpr int height = 3;
constexpr int size_y = width * height;
constexpr int size_u = (width + 1) / 2 * (height + 1) / 2;
constexpr int size_v = (width + 1) / 2 * (height + 1) / 2;
rtc::scoped_refptr<I420Buffer> reference(I420Buffer::Create(width, height));
memset(reference->MutableDataY(), 8, size_y);
memset(reference->MutableDataU(), 4, size_u);
memset(reference->MutableDataV(), 2, size_v);
rtc::scoped_refptr<I444Buffer> i444_buffer(I444Buffer::Create(width, height));
// Convert the reference buffer to I444.
memset(i444_buffer->MutableDataY(), 8, size_y);
memset(i444_buffer->MutableDataU(), 4, size_y);
memset(i444_buffer->MutableDataV(), 2, size_y);
// Confirm YUV values are as expected.
for (int row = 0; row < height; row++) {
for (int col = 0; col < width; col++) {
EXPECT_EQ(8, GetY(i444_buffer, col, row));
EXPECT_EQ(4, GetU(i444_buffer, col, row));
EXPECT_EQ(2, GetV(i444_buffer, col, row));
}
}
rtc::scoped_refptr<I420BufferInterface> i420_buffer(i444_buffer->ToI420());
EXPECT_EQ(height, i420_buffer->height());
EXPECT_EQ(width, i420_buffer->width());
EXPECT_TRUE(test::FrameBufsEqual(reference, i420_buffer));
}
} // namespace webrtc

14
api/video/video_frame_buffer.cc
View file

@ -11,6 +11,7 @@
#include "api/video/video_frame_buffer.h"
#include "api/video/i420_buffer.h"
#include "api/video/i444_buffer.h"
#include "api/video/nv12_buffer.h"
#include "rtc_base/checks.h"
@ -117,6 +118,19 @@ int I444BufferInterface::ChromaHeight() const {
return height();
}
rtc::scoped_refptr<VideoFrameBuffer> I444BufferInterface::CropAndScale(
int offset_x,
int offset_y,
int crop_width,
int crop_height,
int scaled_width,
int scaled_height) {
rtc::scoped_refptr<I444Buffer> result =
I444Buffer::Create(scaled_width, scaled_height);
result->CropAndScaleFrom(*this, offset_x, offset_y, crop_width, crop_height);
return result;
}
VideoFrameBuffer::Type I010BufferInterface::type() const {
return Type::kI010;
}

7
api/video/video_frame_buffer.h
View file

@ -184,6 +184,13 @@ class I444BufferInterface : public PlanarYuv8Buffer {
int ChromaWidth() const final;
int ChromaHeight() const final;
rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x,
int offset_y,
int crop_width,
int crop_height,
int scaled_width,
int scaled_height) override;
protected:
~I444BufferInterface() override {}
};

6
api/video_codecs/h264_profile_level_id.cc
View file

@ -68,7 +68,8 @@ constexpr ProfilePattern kProfilePatterns[] = {
{0x58, BitPattern("10xx0000"), H264Profile::kProfileBaseline},
{0x4D, BitPattern("0x0x0000"), H264Profile::kProfileMain},
{0x64, BitPattern("00000000"), H264Profile::kProfileHigh},
{0x64, BitPattern("00001100"), H264Profile::kProfileConstrainedHigh}};
{0x64, BitPattern("00001100"), H264Profile::kProfileConstrainedHigh},
{0xF4, BitPattern("00000000"), H264Profile::kProfilePredictiveHigh444}};
struct LevelConstraint {
const int max_macroblocks_per_second;
@ -228,6 +229,9 @@ absl::optional<std::string> H264ProfileLevelIdToString(
case H264Profile::kProfileHigh:
profile_idc_iop_string = "6400";
break;
case H264Profile::kProfilePredictiveHigh444:
profile_idc_iop_string = "f400";
break;
// Unrecognized profile.
default:
return absl::nullopt;

1
api/video_codecs/h264_profile_level_id.h
View file

@ -25,6 +25,7 @@ enum class H264Profile {
kProfileMain,
kProfileConstrainedHigh,
kProfileHigh,
kProfilePredictiveHigh444,
};
// All values are equal to ten times the level number, except level 1b which is

2
common_video/include/video_frame_buffer_pool.h
View file

@ -17,6 +17,7 @@
#include "api/scoped_refptr.h"
#include "api/video/i420_buffer.h"
#include "api/video/i444_buffer.h"
#include "api/video/nv12_buffer.h"
#include "rtc_base/race_checker.h"
#include "rtc_base/ref_counted_object.h"
@ -43,6 +44,7 @@ class VideoFrameBufferPool {
// and there are less than `max_number_of_buffers` pending, a buffer is
// created. Returns null otherwise.
rtc::scoped_refptr<I420Buffer> CreateI420Buffer(int width, int height);
rtc::scoped_refptr<I444Buffer> CreateI444Buffer(int width, int height);
rtc::scoped_refptr<NV12Buffer> CreateNV12Buffer(int width, int height);
// Changes the max amount of buffers in the pool to the new value.

38
common_video/video_frame_buffer_pool.cc
View file

@ -20,12 +20,17 @@ namespace {
bool HasOneRef(const rtc::scoped_refptr<VideoFrameBuffer>& buffer) {
// Cast to rtc::RefCountedObject is safe because this function is only called
// on locally created VideoFrameBuffers, which are either
// `rtc::RefCountedObject<I420Buffer>` or `rtc::RefCountedObject<NV12Buffer>`.
// `rtc::RefCountedObject<I420Buffer>`, `rtc::RefCountedObject<I444Buffer>` or
// `rtc::RefCountedObject<NV12Buffer>`.
switch (buffer->type()) {
case VideoFrameBuffer::Type::kI420: {
return static_cast<rtc::RefCountedObject<I420Buffer>*>(buffer.get())
->HasOneRef();
}
case VideoFrameBuffer::Type::kI444: {
return static_cast<rtc::RefCountedObject<I444Buffer>*>(buffer.get())
->HasOneRef();
}
case VideoFrameBuffer::Type::kNV12: {
return static_cast<rtc::RefCountedObject<NV12Buffer>*>(buffer.get())
->HasOneRef();
@ -116,6 +121,37 @@ rtc::scoped_refptr<I420Buffer> VideoFrameBufferPool::CreateI420Buffer(
return buffer;
}
rtc::scoped_refptr<I444Buffer> VideoFrameBufferPool::CreateI444Buffer(
int width,
int height) {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
rtc::scoped_refptr<VideoFrameBuffer> existing_buffer =
GetExistingBuffer(width, height, VideoFrameBuffer::Type::kI444);
if (existing_buffer) {
// Cast is safe because the only way kI444 buffer is created is
// in the same function below, where |RefCountedObject<I444Buffer>|
// is created.
rtc::RefCountedObject<I444Buffer>* raw_buffer =
static_cast<rtc::RefCountedObject<I444Buffer>*>(existing_buffer.get());
// Creates a new scoped_refptr, which is also pointing to the same
// RefCountedObject as buffer, increasing ref count.
return rtc::scoped_refptr<I444Buffer>(raw_buffer);
}
if (buffers_.size() >= max_number_of_buffers_)
return nullptr;
// Allocate new buffer.
rtc::scoped_refptr<I444Buffer> buffer =
new rtc::RefCountedObject<I444Buffer>(width, height);
if (zero_initialize_)
buffer->InitializeData();
buffers_.push_back(buffer);
return buffer;
}
rtc::scoped_refptr<NV12Buffer> VideoFrameBufferPool::CreateNV12Buffer(
int width,
int height) {

2
media/engine/internal_decoder_factory.cc
View file

@ -47,7 +47,7 @@ std::vector<SdpVideoFormat> InternalDecoderFactory::GetSupportedFormats()
formats.push_back(SdpVideoFormat(cricket::kVp8CodecName));
for (const SdpVideoFormat& format : SupportedVP9DecoderCodecs())
formats.push_back(format);
for (const SdpVideoFormat& h264_format : SupportedH264Codecs())
for (const SdpVideoFormat& h264_format : SupportedH264DecoderCodecs())
formats.push_back(h264_format);
if (kIsLibaomAv1DecoderSupported ||

40
modules/video_coding/codecs/h264/h264.cc
View file

@ -80,18 +80,34 @@ std::vector<SdpVideoFormat> SupportedH264Codecs() {
//
// We support both packetization modes 0 (mandatory) and 1 (optional,
// preferred).
return {CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1,
"1"),
CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1,
"0"),
CreateH264Format(H264Profile::kProfileConstrainedBaseline,
H264Level::kLevel3_1, "1"),
CreateH264Format(H264Profile::kProfileConstrainedBaseline,
H264Level::kLevel3_1, "0"),
CreateH264Format(H264Profile::kProfileMain,
H264Level::kLevel3_1, "1"),
CreateH264Format(H264Profile::kProfileMain,
H264Level::kLevel3_1, "0")};
return {
CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1,
"1"),
CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1,
"0"),
CreateH264Format(H264Profile::kProfileConstrainedBaseline,
H264Level::kLevel3_1, "1"),
CreateH264Format(H264Profile::kProfileConstrainedBaseline,
H264Level::kLevel3_1, "0"),
CreateH264Format(H264Profile::kProfileMain, H264Level::kLevel3_1, "1"),
CreateH264Format(H264Profile::kProfileMain, H264Level::kLevel3_1, "0")};
}
std::vector<SdpVideoFormat> SupportedH264DecoderCodecs() {
TRACE_EVENT0("webrtc", __func__);
if (!IsH264CodecSupported())
return std::vector<SdpVideoFormat>();
std::vector<SdpVideoFormat> supportedCodecs = SupportedH264Codecs();
// OpenH264 doesn't yet support High Predictive 4:4:4 encoding but it does
// support decoding.
supportedCodecs.push_back(CreateH264Format(
H264Profile::kProfilePredictiveHigh444, H264Level::kLevel3_1, "1"));
supportedCodecs.push_back(CreateH264Format(
H264Profile::kProfilePredictiveHigh444, H264Level::kLevel3_1, "0"));
return supportedCodecs;
}
std::unique_ptr<H264Encoder> H264Encoder::Create(

181
modules/video_coding/codecs/h264/h264_decoder_impl.cc
View file

@ -41,8 +41,10 @@ namespace webrtc {
namespace {
const AVPixelFormat kPixelFormatDefault = AV_PIX_FMT_YUV420P;
const AVPixelFormat kPixelFormatFullRange = AV_PIX_FMT_YUVJ420P;
constexpr std::array<AVPixelFormat, 2> kPixelFormatsDefault = {
AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV444P};
constexpr std::array<AVPixelFormat, 2> kPixelFormatsFullRange = {
AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ444P};
const size_t kYPlaneIndex = 0;
const size_t kUPlaneIndex = 1;
const size_t kVPlaneIndex = 2;
@ -76,9 +78,17 @@ int H264DecoderImpl::AVGetBuffer2(AVCodecContext* context,
// Necessary capability to be allowed to provide our own buffers.
RTC_DCHECK(context->codec->capabilities | AV_CODEC_CAP_DR1);
// Limited or full range YUV420 or YUV444 is expected.
auto pixelFormatDefault = std::find_if(
kPixelFormatsDefault.begin(), kPixelFormatsDefault.end(),
[context](AVPixelFormat format) { return context->pix_fmt == format; });
auto pixelFormatFullRange = std::find_if(
kPixelFormatsFullRange.begin(), kPixelFormatsFullRange.end(),
[context](AVPixelFormat format) { return context->pix_fmt == format; });
// Limited or full range YUV420 is expected.
RTC_CHECK(context->pix_fmt == kPixelFormatDefault ||
context->pix_fmt == kPixelFormatFullRange);
RTC_CHECK(pixelFormatDefault != kPixelFormatsDefault.end() ||
pixelFormatFullRange != kPixelFormatsFullRange.end());
// `av_frame->width` and `av_frame->height` are set by FFmpeg. These are the
// actual image's dimensions and may be different from `context->width` and
@ -112,8 +122,43 @@ int H264DecoderImpl::AVGetBuffer2(AVCodecContext* context,
// http://crbug.com/390941. Our pool is set up to zero-initialize new buffers.
// TODO(nisse): Delete that feature from the video pool, instead add
// an explicit call to InitializeData here.
rtc::scoped_refptr<I420Buffer> frame_buffer =
decoder->ffmpeg_buffer_pool_.CreateI420Buffer(width, height);
rtc::scoped_refptr<PlanarYuv8Buffer> frame_buffer;
rtc::scoped_refptr<I444Buffer> i444_buffer;
rtc::scoped_refptr<I420Buffer> i420_buffer;
switch (context->pix_fmt) {
case AV_PIX_FMT_YUV420P:
case AV_PIX_FMT_YUVJ420P:
i420_buffer =
decoder->ffmpeg_buffer_pool_.CreateI420Buffer(width, height);
// Set `av_frame` members as required by FFmpeg.
av_frame->data[kYPlaneIndex] = i420_buffer->MutableDataY();
av_frame->linesize[kYPlaneIndex] = i420_buffer->StrideY();
av_frame->data[kUPlaneIndex] = i420_buffer->MutableDataU();
av_frame->linesize[kUPlaneIndex] = i420_buffer->StrideU();
av_frame->data[kVPlaneIndex] = i420_buffer->MutableDataV();
av_frame->linesize[kVPlaneIndex] = i420_buffer->StrideV();
RTC_DCHECK_EQ(av_frame->extended_data, av_frame->data);
frame_buffer = i420_buffer;
break;
case AV_PIX_FMT_YUV444P:
case AV_PIX_FMT_YUVJ444P:
i444_buffer =
decoder->ffmpeg_buffer_pool_.CreateI444Buffer(width, height);
// Set `av_frame` members as required by FFmpeg.
av_frame->data[kYPlaneIndex] = i444_buffer->MutableDataY();
av_frame->linesize[kYPlaneIndex] = i444_buffer->StrideY();
av_frame->data[kUPlaneIndex] = i444_buffer->MutableDataU();
av_frame->linesize[kUPlaneIndex] = i444_buffer->StrideU();
av_frame->data[kVPlaneIndex] = i444_buffer->MutableDataV();
av_frame->linesize[kVPlaneIndex] = i444_buffer->StrideV();
frame_buffer = i444_buffer;
break;
default:
RTC_LOG(LS_ERROR) << "Unsupported buffer type " << context->pix_fmt
<< ". Check supported supported pixel formats!";
decoder->ReportError();
return -1;
}
int y_size = width * height;
int uv_size = frame_buffer->ChromaWidth() * frame_buffer->ChromaHeight();
@ -125,15 +170,6 @@ int H264DecoderImpl::AVGetBuffer2(AVCodecContext* context,
av_frame->format = context->pix_fmt;
av_frame->reordered_opaque = context->reordered_opaque;
// Set `av_frame` members as required by FFmpeg.
av_frame->data[kYPlaneIndex] = frame_buffer->MutableDataY();
av_frame->linesize[kYPlaneIndex] = frame_buffer->StrideY();
av_frame->data[kUPlaneIndex] = frame_buffer->MutableDataU();
av_frame->linesize[kUPlaneIndex] = frame_buffer->StrideU();
av_frame->data[kVPlaneIndex] = frame_buffer->MutableDataV();
av_frame->linesize[kVPlaneIndex] = frame_buffer->StrideV();
RTC_DCHECK_EQ(av_frame->extended_data, av_frame->data);
// Create a VideoFrame object, to keep a reference to the buffer.
// TODO(nisse): The VideoFrame's timestamp and rotation info is not used.
// Refactor to do not use a VideoFrame object at all.
@ -197,7 +233,6 @@ bool H264DecoderImpl::Configure(const Settings& settings) {
av_context_->coded_width = resolution.Width();
av_context_->coded_height = resolution.Height();
}
av_context_->pix_fmt = kPixelFormatDefault;
av_context_->extradata = nullptr;
av_context_->extradata_size = 0;
@ -317,47 +352,103 @@ int32_t H264DecoderImpl::Decode(const EncodedImage& input_image,
RTC_DCHECK(input_frame);
rtc::scoped_refptr<VideoFrameBuffer> frame_buffer =
input_frame->video_frame_buffer();
const webrtc::I420BufferInterface* i420_buffer = frame_buffer->GetI420();
// Instantiate Planar YUV8 buffer according to video frame buffer type
const webrtc::PlanarYuv8Buffer* planar_yuv8_buffer = nullptr;
VideoFrameBuffer::Type video_frame_buffer_type = frame_buffer->type();
switch (video_frame_buffer_type) {
case VideoFrameBuffer::Type::kI420:
planar_yuv8_buffer = frame_buffer->GetI420();
break;
case VideoFrameBuffer::Type::kI444:
planar_yuv8_buffer = frame_buffer->GetI444();
break;
default:
// If this code is changed to allow other video frame buffer type,
// make sure that the code below which wraps I420/I444 buffer and
// code which converts to NV12 is changed
// to work with new video frame buffer type
RTC_LOG(LS_ERROR) << "frame_buffer type: "
<< static_cast<int32_t>(video_frame_buffer_type)
<< " is not supported!";
ReportError();
return WEBRTC_VIDEO_CODEC_ERROR;
}
// When needed, FFmpeg applies cropping by moving plane pointers and adjusting
// frame width/height. Ensure that cropped buffers lie within the allocated
// memory.
RTC_DCHECK_LE(av_frame_->width, i420_buffer->width());
RTC_DCHECK_LE(av_frame_->height, i420_buffer->height());
RTC_DCHECK_GE(av_frame_->data[kYPlaneIndex], i420_buffer->DataY());
RTC_DCHECK_LE(
av_frame_->data[kYPlaneIndex] +
av_frame_->linesize[kYPlaneIndex] * av_frame_->height,
i420_buffer->DataY() + i420_buffer->StrideY() * i420_buffer->height());
RTC_DCHECK_GE(av_frame_->data[kUPlaneIndex], i420_buffer->DataU());
RTC_DCHECK_LE(av_frame_->width, planar_yuv8_buffer->width());
RTC_DCHECK_LE(av_frame_->height, planar_yuv8_buffer->height());
RTC_DCHECK_GE(av_frame_->data[kYPlaneIndex], planar_yuv8_buffer->DataY());
RTC_DCHECK_LE(av_frame_->data[kYPlaneIndex] +
av_frame_->linesize[kYPlaneIndex] * av_frame_->height,
planar_yuv8_buffer->DataY() + planar_yuv8_buffer->StrideY() *
planar_yuv8_buffer->height());
RTC_DCHECK_GE(av_frame_->data[kUPlaneIndex], planar_yuv8_buffer->DataU());
RTC_DCHECK_LE(av_frame_->data[kUPlaneIndex] +
av_frame_->linesize[kUPlaneIndex] * av_frame_->height / 2,
i420_buffer->DataU() +
i420_buffer->StrideU() * i420_buffer->height() / 2);
RTC_DCHECK_GE(av_frame_->data[kVPlaneIndex], i420_buffer->DataV());
planar_yuv8_buffer->DataU() + planar_yuv8_buffer->StrideU() *
planar_yuv8_buffer->height() /
2);
RTC_DCHECK_GE(av_frame_->data[kVPlaneIndex], planar_yuv8_buffer->DataV());
RTC_DCHECK_LE(av_frame_->data[kVPlaneIndex] +
av_frame_->linesize[kVPlaneIndex] * av_frame_->height / 2,
i420_buffer->DataV() +
i420_buffer->StrideV() * i420_buffer->height() / 2);
planar_yuv8_buffer->DataV() + planar_yuv8_buffer->StrideV() *
planar_yuv8_buffer->height() /
2);
rtc::scoped_refptr<webrtc::VideoFrameBuffer> cropped_buffer = WrapI420Buffer(
av_frame_->width, av_frame_->height, av_frame_->data[kYPlaneIndex],
av_frame_->linesize[kYPlaneIndex], av_frame_->data[kUPlaneIndex],
av_frame_->linesize[kUPlaneIndex], av_frame_->data[kVPlaneIndex],
av_frame_->linesize[kVPlaneIndex],
// To keep reference alive.
[frame_buffer] {});
rtc::scoped_refptr<webrtc::VideoFrameBuffer> cropped_buffer;
if (video_frame_buffer_type == VideoFrameBuffer::Type::kI420) {
cropped_buffer = WrapI420Buffer(
av_frame_->width, av_frame_->height, av_frame_->data[kYPlaneIndex],
av_frame_->linesize[kYPlaneIndex], av_frame_->data[kUPlaneIndex],
av_frame_->linesize[kUPlaneIndex], av_frame_->data[kVPlaneIndex],
av_frame_->linesize[kVPlaneIndex],
// To keep reference alive.
[frame_buffer] {});
} else {
cropped_buffer = WrapI444Buffer(
av_frame_->width, av_frame_->height, av_frame_->data[kYPlaneIndex],
av_frame_->linesize[kYPlaneIndex], av_frame_->data[kUPlaneIndex],
av_frame_->linesize[kUPlaneIndex], av_frame_->data[kVPlaneIndex],
av_frame_->linesize[kVPlaneIndex],
// To keep reference alive.
[frame_buffer] {});
}
if (preferred_output_format_ == VideoFrameBuffer::Type::kNV12) {
const I420BufferInterface* cropped_i420 = cropped_buffer->GetI420();
auto nv12_buffer = output_buffer_pool_.CreateNV12Buffer(
cropped_i420->width(), cropped_i420->height());
libyuv::I420ToNV12(cropped_i420->DataY(), cropped_i420->StrideY(),
cropped_i420->DataU(), cropped_i420->StrideU(),
cropped_i420->DataV(), cropped_i420->StrideV(),
nv12_buffer->MutableDataY(), nv12_buffer->StrideY(),
nv12_buffer->MutableDataUV(), nv12_buffer->StrideUV(),
i420_buffer->width(), i420_buffer->height());
cropped_buffer->width(), cropped_buffer->height());
const PlanarYuv8Buffer* cropped_planar_yuv8_buffer = nullptr;
if (video_frame_buffer_type == VideoFrameBuffer::Type::kI420) {
cropped_planar_yuv8_buffer = cropped_buffer->GetI420();
libyuv::I420ToNV12(cropped_planar_yuv8_buffer->DataY(),
cropped_planar_yuv8_buffer->StrideY(),
cropped_planar_yuv8_buffer->DataU(),
cropped_planar_yuv8_buffer->StrideU(),
cropped_planar_yuv8_buffer->DataV(),
cropped_planar_yuv8_buffer->StrideV(),
nv12_buffer->MutableDataY(), nv12_buffer->StrideY(),
nv12_buffer->MutableDataUV(), nv12_buffer->StrideUV(),
planar_yuv8_buffer->width(),
planar_yuv8_buffer->height());
} else {
cropped_planar_yuv8_buffer = cropped_buffer->GetI444();
libyuv::I444ToNV12(cropped_planar_yuv8_buffer->DataY(),
cropped_planar_yuv8_buffer->StrideY(),
cropped_planar_yuv8_buffer->DataU(),
cropped_planar_yuv8_buffer->StrideU(),
cropped_planar_yuv8_buffer->DataV(),
cropped_planar_yuv8_buffer->StrideV(),
nv12_buffer->MutableDataY(), nv12_buffer->StrideY(),
nv12_buffer->MutableDataUV(), nv12_buffer->StrideUV(),
planar_yuv8_buffer->width(),
planar_yuv8_buffer->height());
}
cropped_buffer = nv12_buffer;
}

7
modules/video_coding/codecs/h264/include/h264.h
View file

@ -38,10 +38,15 @@ CreateH264Format(H264Profile profile,
// and is not thread-safe.
RTC_EXPORT void DisableRtcUseH264();
// Returns a vector with all supported internal H264 profiles that we can
// Returns a vector with all supported internal H264 encode profiles that we can
// negotiate in SDP, in order of preference.
std::vector<SdpVideoFormat> SupportedH264Codecs();
// Returns a vector with all supported internal H264 decode profiles that we can
// negotiate in SDP, in order of preference. This will be available for receive
// only connections.
std::vector<SdpVideoFormat> SupportedH264DecoderCodecs();
class RTC_EXPORT H264Encoder : public VideoEncoder {
public:
static std::unique_ptr<H264Encoder> Create(const cricket::VideoCodec& codec);

17
pc/media_session.cc
View file

@ -2902,8 +2902,12 @@ void MediaSessionDescriptionFactory::ComputeVideoCodecsIntersectionAndUnion() {
video_sendrecv_codecs_.clear();
all_video_codecs_.clear();
// Compute the video codecs union.
// Keep track of payload types to avoid collisions.
UsedPayloadTypes used_payload_types;
for (const VideoCodec& send : video_send_codecs_) {
all_video_codecs_.push_back(send);
VideoCodec send_mutable = send;
used_payload_types.FindAndSetIdUsed(&send_mutable);
all_video_codecs_.push_back(send_mutable);
if (!FindMatchingCodec<VideoCodec>(video_send_codecs_, video_recv_codecs_,
send, nullptr)) {
// TODO(kron): This check is violated by the unit test:
@ -2915,12 +2919,11 @@ void MediaSessionDescriptionFactory::ComputeVideoCodecsIntersectionAndUnion() {
// RTC_DCHECK(!IsRtxCodec(send));
}
}
for (const VideoCodec& recv : video_recv_codecs_) {
if (!FindMatchingCodec<VideoCodec>(video_recv_codecs_, video_send_codecs_,
recv, nullptr)) {
all_video_codecs_.push_back(recv);
}
}
// Use MergeCodecs to merge the second half of our list as it already checks
// and fixes problems with duplicate payload types.
MergeCodecs<VideoCodec>(video_recv_codecs_, &all_video_codecs_,
&used_payload_types);
// Use NegotiateCodecs to merge our codec lists, since the operation is
// essentially the same. Put send_codecs as the offered_codecs, which is the
// order we'd like to follow. The reasoning is that encoding is usually more

1
sdk/objc/components/video_codec/RTCVideoEncoderH264.mm
View file

@ -240,6 +240,7 @@ CFStringRef ExtractProfile(const webrtc::H264ProfileLevelId &profile_level_id) {
case webrtc::H264Profile::kProfileConstrainedHigh:
case webrtc::H264Profile::kProfileHigh:
case webrtc::H264Profile::kProfilePredictiveHigh444:
switch (profile_level_id.level) {
case webrtc::H264Level::kLevel3:
return kVTProfileLevel_H264_High_3_0;