mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-13 05:40:42 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Move EncodeResultCallback to FrameEncodeSettings.

Browse source 
Bug: b/336978562
Change-Id: I25cb6662a6b66fba9d22d974e1b6da48e137d0ea
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/349225
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42284}
This commit is contained in:
philipel 2024-04-26 16:32:07 +02:00 committed by WebRTC LUCI CQ
parent 28a4ec36a0
commit cf4764e062
7 changed files with 250 additions and 228 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
api/video_codecs/BUILD.gn
View file

@ -342,6 +342,7 @@ rtc_library("simple_encoder_wrapper") {
absl_deps = [ absl_deps = [
"//third_party/abseil-cpp/absl/algorithm:container", "//third_party/abseil-cpp/absl/algorithm:container",
"//third_party/abseil-cpp/absl/functional:any_invocable", "//third_party/abseil-cpp/absl/functional:any_invocable",
"//third_party/abseil-cpp/absl/types:variant",
] ]
} }
@ -397,6 +398,8 @@ rtc_library("libaom_av1_encoder_factory_test") {
"../../test:test_support", "../../test:test_support",
"../../test:video_test_support", "../../test:video_test_support",
] ]
absl_deps = [ "//third_party/abseil-cpp/absl/types:variant" ]
} }
rtc_library("vp8_temporal_layers_factory") { rtc_library("vp8_temporal_layers_factory") {

55
api/video_codecs/libaom_av1_encoder_factory.cc
View file

@ -27,7 +27,7 @@
#define SET_OR_DO_ERROR_CALLBACK_AND_RETURN(param_id, param_value) \ #define SET_OR_DO_ERROR_CALLBACK_AND_RETURN(param_id, param_value) \
do { \ do { \
if (!SetEncoderControlParameters(&ctx_, param_id, param_value)) { \ if (!SetEncoderControlParameters(&ctx_, param_id, param_value)) { \
encode_result_callback({}); \ DoErrorCallback(frame_settings); \
return; \ return; \
} \ } \
} while (0) } while (0)
@ -41,8 +41,9 @@
namespace webrtc { namespace webrtc {
using Cbr = VideoEncoderInterface::FrameEncodeSettings::Cbr; using FrameEncodeSettings = VideoEncoderInterface::FrameEncodeSettings;
using Cqp = VideoEncoderInterface::FrameEncodeSettings::Cqp; using Cbr = FrameEncodeSettings::Cbr;
using Cqp = FrameEncodeSettings::Cqp;
using aom_img_ptr = std::unique_ptr<aom_image_t, decltype(&aom_img_free)>; using aom_img_ptr = std::unique_ptr<aom_image_t, decltype(&aom_img_free)>;
namespace { namespace {
@ -87,8 +88,7 @@ class LibaomAv1Encoder : public VideoEncoderInterface {
void Encode(rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer, void Encode(rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer,
const TemporalUnitSettings& tu_settings, const TemporalUnitSettings& tu_settings,
const std::vector<FrameEncodeSettings>& frame_settings, std::vector<FrameEncodeSettings> frame_settings) override;
EncodeResultCallback encode_result_callback) override;
private: private:
aom_img_ptr image_to_encode_ = aom_img_ptr(nullptr, aom_img_free); aom_img_ptr image_to_encode_ = aom_img_ptr(nullptr, aom_img_free);
@ -276,6 +276,11 @@ bool ValidateEncodeParams(
const VideoEncoderInterface::FrameEncodeSettings& settings = const VideoEncoderInterface::FrameEncodeSettings& settings =
frame_settings[i]; frame_settings[i];
if (!settings.result_callback) {
RTC_LOG(LS_ERROR) << "No result callback function provided.";
return false;
}
if (!in_range(0, kMaxSpatialLayersWtf, settings.spatial_id)) { if (!in_range(0, kMaxSpatialLayersWtf, settings.spatial_id)) {
RTC_LOG(LS_ERROR) << "invalid spatial id " << settings.spatial_id; RTC_LOG(LS_ERROR) << "invalid spatial id " << settings.spatial_id;
return false; return false;
@ -611,14 +616,23 @@ aom_svc_params_t GetSvcParams(
return svc_params; return svc_params;
} }
void DoErrorCallback(std::vector<FrameEncodeSettings>& frame_settings) {
for (FrameEncodeSettings& settings : frame_settings) {
if (settings.result_callback) {
settings.result_callback({});
// To avoid invoking any callback more than once.
settings.result_callback = {};
}
}
}
void LibaomAv1Encoder::Encode( void LibaomAv1Encoder::Encode(
rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer, rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer,
const TemporalUnitSettings& tu_settings, const TemporalUnitSettings& tu_settings,
const std::vector<FrameEncodeSettings>& frame_settings, std::vector<FrameEncodeSettings> frame_settings) {
EncodeResultCallback encode_result_callback) {
if (!ValidateEncodeParams(*frame_buffer, tu_settings, frame_settings, if (!ValidateEncodeParams(*frame_buffer, tu_settings, frame_settings,
last_resolution_in_buffer_, cfg_.rc_end_usage)) { last_resolution_in_buffer_, cfg_.rc_end_usage)) {
encode_result_callback({}); DoErrorCallback(frame_settings);
return; return;
} }
@ -681,7 +695,7 @@ void LibaomAv1Encoder::Encode(
if (aom_codec_err_t ret = aom_codec_enc_config_set(&ctx_, &cfg_); if (aom_codec_err_t ret = aom_codec_enc_config_set(&ctx_, &cfg_);
ret != AOM_CODEC_OK) { ret != AOM_CODEC_OK) {
RTC_LOG(LS_ERROR) << "aom_codec_enc_config_set returned " << ret; RTC_LOG(LS_ERROR) << "aom_codec_enc_config_set returned " << ret;
encode_result_callback({}); DoErrorCallback(frame_settings);
return; return;
} }
aom_svc_params_t svc_params = GetSvcParams(*frame_buffer, frame_settings); aom_svc_params_t svc_params = GetSvcParams(*frame_buffer, frame_settings);
@ -689,20 +703,20 @@ void LibaomAv1Encoder::Encode(
// The libaom AV1 encoder requires that `aom_codec_encode` is called for // The libaom AV1 encoder requires that `aom_codec_encode` is called for
// every spatial layer, even if no frame should be encoded for that layer. // every spatial layer, even if no frame should be encoded for that layer.
std::array<const FrameEncodeSettings*, kMaxSpatialLayersWtf> std::array<FrameEncodeSettings*, kMaxSpatialLayersWtf>
settings_for_spatial_id; settings_for_spatial_id;
settings_for_spatial_id.fill(nullptr); settings_for_spatial_id.fill(nullptr);
FrameEncodeSettings settings_for_unused_layer; FrameEncodeSettings settings_for_unused_layer;
for (const FrameEncodeSettings& settings : frame_settings) { for (FrameEncodeSettings& settings : frame_settings) {
settings_for_spatial_id[settings.spatial_id] = &settings; settings_for_spatial_id[settings.spatial_id] = &settings;
} }
for (int sid = frame_settings[0].spatial_id; for (int sid = frame_settings[0].spatial_id;
sid < svc_params.number_spatial_layers; ++sid) { sid < svc_params.number_spatial_layers; ++sid) {
const bool layer_enabled = settings_for_spatial_id[sid] != nullptr; const bool layer_enabled = settings_for_spatial_id[sid] != nullptr;
const FrameEncodeSettings& settings = layer_enabled FrameEncodeSettings& settings = layer_enabled
? *settings_for_spatial_id[sid] ? *settings_for_spatial_id[sid]
: settings_for_unused_layer; : settings_for_unused_layer;
aom_svc_layer_id_t layer_id = { aom_svc_layer_id_t layer_id = {
.spatial_layer_id = sid, .spatial_layer_id = sid,
@ -713,7 +727,6 @@ void LibaomAv1Encoder::Encode(
SET_OR_DO_ERROR_CALLBACK_AND_RETURN(AV1E_SET_SVC_REF_FRAME_CONFIG, SET_OR_DO_ERROR_CALLBACK_AND_RETURN(AV1E_SET_SVC_REF_FRAME_CONFIG,
&ref_config); &ref_config);
// TD: Why does the libaom have both `encode_timestamp_` and `duration`?
// TD: Duration can't be zero, what does it matter when the layer is // TD: Duration can't be zero, what does it matter when the layer is
// not being encoded? // not being encoded?
TimeDelta duration = TimeDelta::Millis(1); TimeDelta duration = TimeDelta::Millis(1);
@ -738,7 +751,7 @@ void LibaomAv1Encoder::Encode(
settings.frame_type == FrameType::kKeyframe ? AOM_EFLAG_FORCE_KF : 0); settings.frame_type == FrameType::kKeyframe ? AOM_EFLAG_FORCE_KF : 0);
if (ret != AOM_CODEC_OK) { if (ret != AOM_CODEC_OK) {
RTC_LOG(LS_WARNING) << "aom_codec_encode returned " << ret; RTC_LOG(LS_WARNING) << "aom_codec_encode returned " << ret;
encode_result_callback({}); DoErrorCallback(frame_settings);
return; return;
} }
@ -766,17 +779,18 @@ void LibaomAv1Encoder::Encode(
: FrameType::kDeltaFrame; : FrameType::kDeltaFrame;
result.bitstream_data = EncodedImageBuffer::Create( result.bitstream_data = EncodedImageBuffer::Create(
static_cast<uint8_t*>(pkt->data.frame.buf), pkt->data.frame.sz); static_cast<uint8_t*>(pkt->data.frame.buf), pkt->data.frame.sz);
result.spatial_id = sid;
break; break;
} }
} }
if (result.bitstream_data == nullptr) { if (result.bitstream_data == nullptr) {
// TD: How should error callbacks be handled, only call once? DoErrorCallback(frame_settings);
encode_result_callback({});
return; return;
} else { } else {
encode_result_callback(result); RTC_CHECK(settings.result_callback);
settings.result_callback(result);
// To avoid invoking any callback more than once.
settings.result_callback = {};
} }
} }
} }
@ -786,7 +800,6 @@ std::string LibaomAv1EncoderFactory::CodecName() const {
return "AV1"; return "AV1";
} }
// TD: it should also possible to expose SW/HW/driver version.
std::string LibaomAv1EncoderFactory::ImplementationName() const { std::string LibaomAv1EncoderFactory::ImplementationName() const {
return "Libaom"; return "Libaom";
} }

323
api/video_codecs/libaom_av1_encoder_factory_test.cc
View file

@ -14,6 +14,7 @@
#include <utility> #include <utility>
#include <vector> #include <vector>
#include "absl/types/variant.h"
#include "api/video/i420_buffer.h" #include "api/video/i420_buffer.h"
#include "api/video_codecs/video_decoder.h" #include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_encoder_interface.h" #include "api/video_codecs/video_encoder_interface.h"
@ -59,7 +60,7 @@ std::string OutPath() {
class EncodeResults { class EncodeResults {
public: public:
EncodeResultCallback CallBack() { EncodeResultCallback Cb() {
return [&](const EncodeResult& result) { results_.push_back(result); }; return [&](const EncodeResult& result) { results_.push_back(result); };
} }
@ -68,7 +69,7 @@ class EncodeResults {
RTC_CHECK(false); RTC_CHECK(false);
return nullptr; return nullptr;
} }
return std::get_if<EncodedData>(&results_[index]); return absl::get_if<EncodedData>(&results_[index]);
} }
private: private:
@ -172,8 +173,13 @@ class FrameEncoderSettingsBuilder {
return *this; return *this;
} }
VideoEncoderInterface::FrameEncodeSettings Build() { FrameEncoderSettingsBuilder& Cb(EncodeResultCallback cb) {
return frame_encode_settings_; frame_encode_settings_.result_callback = std::move(cb);
return *this;
}
operator VideoEncoderInterface::FrameEncodeSettings&&() {
return std::move(frame_encode_settings_);
} }
private: private:
@ -182,6 +188,16 @@ class FrameEncoderSettingsBuilder {
using Fb = FrameEncoderSettingsBuilder; using Fb = FrameEncoderSettingsBuilder;
// Since FrameEncodeSettings is move only, initalizer-list initialization won't
// work, so instead a C-style array can be used to do aggregate initialization.
template <int N>
std::vector<VideoEncoderInterface::FrameEncodeSettings> ToVec(
VideoEncoderInterface::FrameEncodeSettings (&&settings)[N]) {
return std::vector<VideoEncoderInterface::FrameEncodeSettings>(
std::make_move_iterator(std::begin(settings)),
std::make_move_iterator(std::end(settings)));
}
// For reasonable debug printout when an EXPECT fail. // For reasonable debug printout when an EXPECT fail.
struct Resolution { struct Resolution {
explicit Resolution(const VideoFrame& frame) explicit Resolution(const VideoFrame& frame)
@ -253,15 +269,14 @@ TEST(LibaomAv1Encoder, KeyframeUpdatesSpecifiedBuffer) {
auto raw_delta = frame_reader->PullFrame(); auto raw_delta = frame_reader->PullFrame();
enc->Encode(raw_key, {.presentation_timestamp = Timestamp::Millis(0)}, enc->Encode(raw_key, {.presentation_timestamp = Timestamp::Millis(0)},
{Fb().Rate(kCbr).Res(640, 360).Upd(5).Key().Build()}, ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(5).Key().Cb(res.Cb())}));
res.CallBack());
ASSERT_THAT(res.FrameAt(0), NotNull()); ASSERT_THAT(res.FrameAt(0), NotNull());
VideoFrame decoded_key = dec.Decode(*res.FrameAt(0)); VideoFrame decoded_key = dec.Decode(*res.FrameAt(0));
EXPECT_THAT(Resolution(decoded_key), ResolutionIs(640, 360)); EXPECT_THAT(Resolution(decoded_key), ResolutionIs(640, 360));
EXPECT_THAT(Psnr(raw_key, decoded_key), Gt(40)); EXPECT_THAT(Psnr(raw_key, decoded_key), Gt(40));
enc->Encode(raw_delta, {.presentation_timestamp = Timestamp::Millis(100)}, enc->Encode(raw_delta, {.presentation_timestamp = Timestamp::Millis(100)},
{Fb().Rate(kCbr).Res(640, 360).Ref({0}).Build()}, res.CallBack()); ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Cb(res.Cb())}));
ASSERT_THAT(res.FrameAt(1), Eq(nullptr)); ASSERT_THAT(res.FrameAt(1), Eq(nullptr));
} }
@ -271,20 +286,20 @@ TEST(LibaomAv1Encoder, MidTemporalUnitKeyframeResetsBuffers) {
EncodeResults res; EncodeResults res;
Av1Decoder dec; Av1Decoder dec;
enc->Encode(frame_reader->PullFrame(), enc->Encode(
{.presentation_timestamp = Timestamp::Millis(0)}, frame_reader->PullFrame(),
{Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Build(), {.presentation_timestamp = Timestamp::Millis(0)},
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Build(), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0}).Build()}, Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0}).Cb(res.Cb())}));
ASSERT_THAT(res.FrameAt(2), NotNull()); ASSERT_THAT(res.FrameAt(2), NotNull());
enc->Encode(frame_reader->PullFrame(), enc->Encode(
{.presentation_timestamp = Timestamp::Millis(100)}, frame_reader->PullFrame(),
{Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Ref({0}).Build(), {.presentation_timestamp = Timestamp::Millis(100)},
Fb().Rate(kCbr).Res(320, 180).S(1).Upd(1).Key().Build(), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Ref({0}).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0}).Build()}, Fb().Rate(kCbr).Res(320, 180).S(1).Upd(1).Key().Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({0}).Cb(res.Cb())}));
ASSERT_THAT(res.FrameAt(3), Eq(nullptr)); ASSERT_THAT(res.FrameAt(3), Eq(nullptr));
} }
@ -295,20 +310,15 @@ TEST(LibaomAv1Encoder, ResolutionSwitching) {
rtc::scoped_refptr<I420Buffer> in0 = frame_reader->PullFrame(); rtc::scoped_refptr<I420Buffer> in0 = frame_reader->PullFrame();
enc->Encode(in0, {.presentation_timestamp = Timestamp::Millis(0)}, enc->Encode(in0, {.presentation_timestamp = Timestamp::Millis(0)},
{Fb().Rate(kCbr).Res(320, 180).Upd(0).Key().Build()}, ToVec({Fb().Rate(kCbr).Res(320, 180).Upd(0).Key().Cb(res.Cb())}));
res.CallBack());
rtc::scoped_refptr<I420Buffer> in1 = frame_reader->PullFrame(); rtc::scoped_refptr<I420Buffer> in1 = frame_reader->PullFrame();
enc->Encode(in1, {.presentation_timestamp = Timestamp::Millis(100)}, enc->Encode(in1, {.presentation_timestamp = Timestamp::Millis(100)},
{Fb().Rate(kCbr).Res(640, 360).Ref({0}).Build()}, res.CallBack()); ToVec({Fb().Rate(kCbr).Res(640, 360).Ref({0}).Cb(res.Cb())}));
rtc::scoped_refptr<I420Buffer> in2 = frame_reader->PullFrame(); rtc::scoped_refptr<I420Buffer> in2 = frame_reader->PullFrame();
enc->Encode(in2, {.presentation_timestamp = Timestamp::Millis(200)}, enc->Encode(in2, {.presentation_timestamp = Timestamp::Millis(200)},
{Fb().Rate(kCbr).Res(160, 90).Ref({0}).Build()}, res.CallBack()); ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Cb(res.Cb())}));
EXPECT_THAT(res.FrameAt(0), Field(&EncodedData::spatial_id, 0));
EXPECT_THAT(res.FrameAt(1), Field(&EncodedData::spatial_id, 0));
EXPECT_THAT(res.FrameAt(2), Field(&EncodedData::spatial_id, 0));
Av1Decoder dec; Av1Decoder dec;
VideoFrame f0 = dec.Decode(*res.FrameAt(0)); VideoFrame f0 = dec.Decode(*res.FrameAt(0));
@ -333,26 +343,21 @@ TEST(LibaomAv1Encoder, InputResolutionSwitching) {
rtc::scoped_refptr<I420Buffer> in0 = frame_reader->PullFrame(); rtc::scoped_refptr<I420Buffer> in0 = frame_reader->PullFrame();
enc->Encode(in0, {.presentation_timestamp = Timestamp::Millis(0)}, enc->Encode(in0, {.presentation_timestamp = Timestamp::Millis(0)},
{Fb().Rate(kCbr).Res(160, 90).Upd(0).Key().Build()}, ToVec({Fb().Rate(kCbr).Res(160, 90).Upd(0).Key().Cb(res.Cb())}));
res.CallBack());
rtc::scoped_refptr<I420Buffer> in1 = frame_reader->PullFrame( rtc::scoped_refptr<I420Buffer> in1 = frame_reader->PullFrame(
/*frame_num=*/nullptr, /*frame_num=*/nullptr,
/*resolution=*/{320, 180}, /*resolution=*/{320, 180},
/*framerate_scale=*/{1, 1}); /*framerate_scale=*/{1, 1});
enc->Encode(in1, {.presentation_timestamp = Timestamp::Millis(100)}, enc->Encode(in1, {.presentation_timestamp = Timestamp::Millis(100)},
{Fb().Rate(kCbr).Res(160, 90).Ref({0}).Build()}, res.CallBack()); ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Cb(res.Cb())}));
rtc::scoped_refptr<I420Buffer> in2 = frame_reader->PullFrame( rtc::scoped_refptr<I420Buffer> in2 = frame_reader->PullFrame(
/*frame_num=*/nullptr, /*frame_num=*/nullptr,
/*resolution=*/{160, 90}, /*resolution=*/{160, 90},
/*framerate_scale=*/{1, 1}); /*framerate_scale=*/{1, 1});
enc->Encode(in2, {.presentation_timestamp = Timestamp::Millis(200)}, enc->Encode(in2, {.presentation_timestamp = Timestamp::Millis(200)},
{Fb().Rate(kCbr).Res(160, 90).Ref({0}).Build()}, res.CallBack()); ToVec({Fb().Rate(kCbr).Res(160, 90).Ref({0}).Cb(res.Cb())}));
EXPECT_THAT(res.FrameAt(0), Field(&EncodedData::spatial_id, 0));
EXPECT_THAT(res.FrameAt(1), Field(&EncodedData::spatial_id, 0));
EXPECT_THAT(res.FrameAt(2), Field(&EncodedData::spatial_id, 0));
Av1Decoder dec; Av1Decoder dec;
VideoFrame f0 = dec.Decode(*res.FrameAt(0)); VideoFrame f0 = dec.Decode(*res.FrameAt(0));
@ -380,24 +385,27 @@ TEST(LibaomAv1Encoder, TempoSpatial) {
const Cbr k20Fps{.duration = TimeDelta::Millis(50), const Cbr k20Fps{.duration = TimeDelta::Millis(50),
.target_bitrate = DataRate::KilobitsPerSec(500)}; .target_bitrate = DataRate::KilobitsPerSec(500)};
enc->Encode(frame_reader->PullFrame(), enc->Encode(
{.presentation_timestamp = Timestamp::Millis(0)}, frame_reader->PullFrame(),
{Fb().Rate(k10Fps).Res(160, 90).S(0).Upd(0).Key().Build(), {.presentation_timestamp = Timestamp::Millis(0)},
Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0}).Upd(1).Build(), ToVec(
Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1}).Upd(2).Build()}, {Fb().Rate(k10Fps).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()),
res.CallBack()); Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()),
Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())}));
enc->Encode(frame_reader->PullFrame(), enc->Encode(frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(50)}, {.presentation_timestamp = Timestamp::Millis(50)},
{Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({2}).Upd(2).Build()}, ToVec({Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(
res.CallBack()); res.Cb())}));
rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame(); rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame();
enc->Encode(frame, {.presentation_timestamp = Timestamp::Millis(100)}, enc->Encode(
{Fb().Rate(k10Fps).Res(160, 90).S(0).Ref({0}).Upd(0).Build(), frame, {.presentation_timestamp = Timestamp::Millis(100)},
Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Build(), ToVec(
Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Build()}, {Fb().Rate(k10Fps).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
res.CallBack()); Fb().Rate(k10Fps).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Cb(res.Cb()),
Fb().Rate(k20Fps).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Cb(
res.Cb())}));
Av1Decoder dec; Av1Decoder dec;
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(160, 90));
@ -418,22 +426,23 @@ TEST(DISABLED_LibaomAv1Encoder, InvertedTempoSpatial) {
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res; EncodeResults res;
enc->Encode(frame_reader->PullFrame(), enc->Encode(
{.presentation_timestamp = Timestamp::Millis(0)}, frame_reader->PullFrame(),
{Fb().Rate(kCbr).Res(320, 180).S(0).Upd(0).Key().Build(), {.presentation_timestamp = Timestamp::Millis(0)},
Fb().Rate(kCbr).Res(640, 360).S(1).Ref({0}).Upd(1).Build()}, ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Upd(0).Key().Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(1).Ref({0}).Upd(1).Cb(res.Cb())}));
enc->Encode(frame_reader->PullFrame(), enc->Encode(
{.presentation_timestamp = Timestamp::Millis(100)}, frame_reader->PullFrame(),
{Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Build()}, {.presentation_timestamp = Timestamp::Millis(100)},
res.CallBack()); ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Cb(res.Cb())}));
rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame(); rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame();
enc->Encode(frame, {.presentation_timestamp = Timestamp::Millis(200)}, enc->Encode(
{Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Build(), frame, {.presentation_timestamp = Timestamp::Millis(200)},
Fb().Rate(kCbr).Res(640, 360).S(1).Ref({1, 0}).Upd(1).Build()}, ToVec({Fb().Rate(kCbr).Res(320, 180).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(1).Ref({1, 0}).Upd(1).Cb(
res.Cb())}));
Av1Decoder dec; Av1Decoder dec;
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(320, 180)); EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(320, 180));
@ -448,25 +457,26 @@ TEST(LibaomAv1Encoder, SkipMidLayer) {
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res; EncodeResults res;
enc->Encode(frame_reader->PullFrame(), enc->Encode(
{.presentation_timestamp = Timestamp::Millis(0)}, frame_reader->PullFrame(),
{Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Build(), {.presentation_timestamp = Timestamp::Millis(0)},
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Build(), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Build()}, Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())}));
enc->Encode(frame_reader->PullFrame(), enc->Encode(
{.presentation_timestamp = Timestamp::Millis(100)}, frame_reader->PullFrame(),
{Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Build(), {.presentation_timestamp = Timestamp::Millis(100)},
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Build()}, ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())}));
rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame(); rtc::scoped_refptr<I420Buffer> frame = frame_reader->PullFrame();
enc->Encode(frame, {.presentation_timestamp = Timestamp::Millis(200)}, enc->Encode(
{Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Build(), frame, {.presentation_timestamp = Timestamp::Millis(200)},
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Build(), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Build()}, Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0, 1}).Upd(1).Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1, 2}).Upd(2).Cb(
res.Cb())}));
Av1Decoder dec; Av1Decoder dec;
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(160, 90));
@ -486,26 +496,26 @@ TEST(LibaomAv1Encoder, L3T1) {
auto frame_reader = CreateFrameReader(); auto frame_reader = CreateFrameReader();
auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {}); auto enc = LibaomAv1EncoderFactory().CreateEncoder(kCbrEncoderSettings, {});
EncodeResults res; EncodeResults res;
Av1Decoder dec; Av1Decoder dec;
enc->Encode(frame_reader->PullFrame(), enc->Encode(
{.presentation_timestamp = Timestamp::Millis(0)}, frame_reader->PullFrame(),
{Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Build(), {.presentation_timestamp = Timestamp::Millis(0)},
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Build(), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Build()}, Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())}));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(0))), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(1))), ResolutionIs(320, 180)); EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(1))), ResolutionIs(320, 180));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(2))), ResolutionIs(640, 360)); EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(2))), ResolutionIs(640, 360));
auto tu1_frame = frame_reader->PullFrame(); auto tu1_frame = frame_reader->PullFrame();
enc->Encode(tu1_frame, {.presentation_timestamp = Timestamp::Millis(100)}, enc->Encode(
{Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Build(), tu1_frame, {.presentation_timestamp = Timestamp::Millis(100)},
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Build(), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Build()}, Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Cb(
res.Cb())}));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(3))), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(3))), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(4))), ResolutionIs(320, 180)); EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(4))), ResolutionIs(320, 180));
@ -515,11 +525,12 @@ TEST(LibaomAv1Encoder, L3T1) {
EXPECT_THAT(Psnr(tu1_frame, f_tu1), Gt(40)); EXPECT_THAT(Psnr(tu1_frame, f_tu1), Gt(40));
auto tu2_frame = frame_reader->PullFrame(); auto tu2_frame = frame_reader->PullFrame();
enc->Encode(tu2_frame, {.presentation_timestamp = Timestamp::Millis(200)}, enc->Encode(
{Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Build(), tu2_frame, {.presentation_timestamp = Timestamp::Millis(200)},
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Build(), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Build()}, Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1, 0}).Upd(1).Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2, 1}).Upd(2).Cb(
res.Cb())}));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(6))), ResolutionIs(160, 90)); EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(6))), ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(7))), ResolutionIs(320, 180)); EXPECT_THAT(Resolution(dec.Decode(*res.FrameAt(7))), ResolutionIs(320, 180));
@ -538,12 +549,12 @@ TEST(LibaomAv1Encoder, L3T1_KEY) {
Av1Decoder dec_s1; Av1Decoder dec_s1;
Av1Decoder dec_s2; Av1Decoder dec_s2;
enc->Encode(frame_reader->PullFrame(), enc->Encode(
{.presentation_timestamp = Timestamp::Millis(0)}, frame_reader->PullFrame(),
{Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Build(), {.presentation_timestamp = Timestamp::Millis(0)},
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Build(), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Upd(0).Key().Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Build()}, Fb().Rate(kCbr).Res(320, 180).S(1).Ref({0}).Upd(1).Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({1}).Upd(2).Cb(res.Cb())}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(0))), EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(0))),
ResolutionIs(160, 90)); ResolutionIs(160, 90));
@ -557,12 +568,12 @@ TEST(LibaomAv1Encoder, L3T1_KEY) {
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(2))), EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(2))),
ResolutionIs(640, 360)); ResolutionIs(640, 360));
enc->Encode(frame_reader->PullFrame(), enc->Encode(
{.presentation_timestamp = Timestamp::Millis(100)}, frame_reader->PullFrame(),
{Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Build(), {.presentation_timestamp = Timestamp::Millis(100)},
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Build(), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Build()}, Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(3))), EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(3))),
ResolutionIs(160, 90)); ResolutionIs(160, 90));
@ -571,12 +582,12 @@ TEST(LibaomAv1Encoder, L3T1_KEY) {
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(5))), EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(5))),
ResolutionIs(640, 360)); ResolutionIs(640, 360));
enc->Encode(frame_reader->PullFrame(), enc->Encode(
{.presentation_timestamp = Timestamp::Millis(200)}, frame_reader->PullFrame(),
{Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Build(), {.presentation_timestamp = Timestamp::Millis(200)},
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Build(), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Build()}, Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(6))), EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(6))),
ResolutionIs(160, 90)); ResolutionIs(160, 90));
@ -595,12 +606,12 @@ TEST(LibaomAv1Encoder, S3T1) {
Av1Decoder dec_s1; Av1Decoder dec_s1;
Av1Decoder dec_s2; Av1Decoder dec_s2;
enc->Encode(frame_reader->PullFrame(), enc->Encode(
{.presentation_timestamp = Timestamp::Millis(0)}, frame_reader->PullFrame(),
{Fb().Rate(kCbr).Res(160, 90).S(0).Start().Upd(0).Build(), {.presentation_timestamp = Timestamp::Millis(0)},
Fb().Rate(kCbr).Res(320, 180).S(1).Start().Upd(1).Build(), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Start().Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Start().Upd(2).Build()}, Fb().Rate(kCbr).Res(320, 180).S(1).Start().Upd(1).Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(2).Start().Upd(2).Cb(res.Cb())}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(0))), EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(0))),
ResolutionIs(160, 90)); ResolutionIs(160, 90));
EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(1))), EXPECT_THAT(Resolution(dec_s1.Decode(*res.FrameAt(1))),
@ -608,12 +619,12 @@ TEST(LibaomAv1Encoder, S3T1) {
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(2))), EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(2))),
ResolutionIs(640, 360)); ResolutionIs(640, 360));
enc->Encode(frame_reader->PullFrame(), enc->Encode(
{.presentation_timestamp = Timestamp::Millis(100)}, frame_reader->PullFrame(),
{Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Build(), {.presentation_timestamp = Timestamp::Millis(100)},
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Build(), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Build()}, Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(3))), EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(3))),
ResolutionIs(160, 90)); ResolutionIs(160, 90));
@ -622,12 +633,12 @@ TEST(LibaomAv1Encoder, S3T1) {
EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(5))), EXPECT_THAT(Resolution(dec_s2.Decode(*res.FrameAt(5))),
ResolutionIs(640, 360)); ResolutionIs(640, 360));
enc->Encode(frame_reader->PullFrame(), enc->Encode(
{.presentation_timestamp = Timestamp::Millis(200)}, frame_reader->PullFrame(),
{Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Build(), {.presentation_timestamp = Timestamp::Millis(200)},
Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Build(), ToVec({Fb().Rate(kCbr).Res(160, 90).S(0).Ref({0}).Upd(0).Cb(res.Cb()),
Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Build()}, Fb().Rate(kCbr).Res(320, 180).S(1).Ref({1}).Upd(1).Cb(res.Cb()),
res.CallBack()); Fb().Rate(kCbr).Res(640, 360).S(2).Ref({2}).Upd(2).Cb(res.Cb())}));
EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(6))), EXPECT_THAT(Resolution(dec_s0.Decode(*res.FrameAt(6))),
ResolutionIs(160, 90)); ResolutionIs(160, 90));
@ -653,7 +664,7 @@ TEST(LibaomAv1Encoder, HigherEffortLevelYieldsHigherQualityFrames) {
enc->Encode( enc->Encode(
frame_in, frame_in,
{.presentation_timestamp = Timestamp::Millis(0), .effort_level = i}, {.presentation_timestamp = Timestamp::Millis(0), .effort_level = i},
{Fb().Rate(kCbr).Res(640, 360).Upd(0).Key().Build()}, res.CallBack()); ToVec({Fb().Rate(kCbr).Res(640, 360).Upd(0).Key().Cb(res.Cb())}));
double psnr = Psnr(frame_in, dec.Decode(*res.FrameAt(0))); double psnr = Psnr(frame_in, dec.Decode(*res.FrameAt(0)));
EXPECT_THAT(psnr, Gt(psnr_last)); EXPECT_THAT(psnr, Gt(psnr_last));
psnr_last = psnr; psnr_last = psnr;
@ -689,12 +700,12 @@ TEST(LibaomAv1Encoder, KeyframeAndStartrameAreApproximatelyEqual) {
auto frame_in = frame_reader->PullFrame(); auto frame_in = frame_reader->PullFrame();
enc_key->Encode( enc_key->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(0)}, frame_in, {.presentation_timestamp = Timestamp::Millis(0)},
{Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Build()}, ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Cb(
res_key.CallBack()); res_key.Cb())}));
enc_start->Encode( enc_start->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(0)}, frame_in, {.presentation_timestamp = Timestamp::Millis(0)},
{Fb().Rate(kRate).Res(640, 360).S(sid).Start().Upd(0).Build()}, ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Start().Upd(0).Cb(
res_start.CallBack()); res_start.Cb())}));
total_size_key += total_size_key +=
DataSize::Bytes(res_key.FrameAt(0)->bitstream_data->size()); DataSize::Bytes(res_key.FrameAt(0)->bitstream_data->size());
total_size_start += total_size_start +=
@ -710,12 +721,12 @@ TEST(LibaomAv1Encoder, KeyframeAndStartrameAreApproximatelyEqual) {
frame_in = frame_reader->PullFrame(); frame_in = frame_reader->PullFrame();
enc_key->Encode( enc_key->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(f * 100)}, frame_in, {.presentation_timestamp = Timestamp::Millis(f * 100)},
{Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Build()}, ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Cb(
res_key.CallBack()); res_key.Cb())}));
enc_start->Encode( enc_start->Encode(
frame_in, {.presentation_timestamp = Timestamp::Millis(f * 100)}, frame_in, {.presentation_timestamp = Timestamp::Millis(f * 100)},
{Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Build()}, ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Cb(
res_start.CallBack()); res_start.Cb())}));
total_size_key += total_size_key +=
DataSize::Bytes(res_key.FrameAt(f)->bitstream_data->size()); DataSize::Bytes(res_key.FrameAt(f)->bitstream_data->size());
total_size_start += total_size_start +=
@ -751,8 +762,8 @@ TEST(LibaomAv1Encoder, BitrateConsistentAcrossSpatialLayers) {
EncodeResults res; EncodeResults res;
enc->Encode(frame_reader->PullFrame(), enc->Encode(frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)}, {.presentation_timestamp = Timestamp::Millis(0)},
{Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Build()}, ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Upd(0).Key().Cb(
res.CallBack()); res.Cb())}));
total_size += DataSize::Bytes(res.FrameAt(0)->bitstream_data->size()); total_size += DataSize::Bytes(res.FrameAt(0)->bitstream_data->size());
total_duration += kRate.duration; total_duration += kRate.duration;
dec.Decode(*res.FrameAt(0)); dec.Decode(*res.FrameAt(0));
@ -761,8 +772,8 @@ TEST(LibaomAv1Encoder, BitrateConsistentAcrossSpatialLayers) {
enc->Encode( enc->Encode(
frame_reader->PullFrame(), frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(f * 100)}, {.presentation_timestamp = Timestamp::Millis(f * 100)},
{Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Build()}, ToVec({Fb().Rate(kRate).Res(640, 360).S(sid).Ref({0}).Upd(0).Cb(
res.CallBack()); res.Cb())}));
total_size += DataSize::Bytes(res.FrameAt(f)->bitstream_data->size()); total_size += DataSize::Bytes(res.FrameAt(f)->bitstream_data->size());
total_duration += kRate.duration; total_duration += kRate.duration;
dec.Decode(*res.FrameAt(f)); dec.Decode(*res.FrameAt(f));
@ -791,13 +802,12 @@ TEST(LibaomAv1Encoder, ConstantQp) {
enc->Encode(frame_reader->PullFrame(), enc->Encode(frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(0)}, {.presentation_timestamp = Timestamp::Millis(0)},
{Fb().Rate(Cqp{.target_qp = kQp}) ToVec({Fb().Rate(Cqp{.target_qp = kQp})
.Res(640, 360) .Res(640, 360)
.S(sid) .S(sid)
.Upd(0) .Upd(0)
.Key() .Key()
.Build()}, .Cb(res.Cb())}));
res.CallBack());
EXPECT_THAT(res.FrameAt(0)->encoded_qp, Eq(kQp)); EXPECT_THAT(res.FrameAt(0)->encoded_qp, Eq(kQp));
total_size += DataSize::Bytes(res.FrameAt(0)->bitstream_data->size()); total_size += DataSize::Bytes(res.FrameAt(0)->bitstream_data->size());
dec.Decode(*res.FrameAt(0)); dec.Decode(*res.FrameAt(0));
@ -805,13 +815,12 @@ TEST(LibaomAv1Encoder, ConstantQp) {
for (int f = 1; f < 10; ++f) { for (int f = 1; f < 10; ++f) {
enc->Encode(frame_reader->PullFrame(), enc->Encode(frame_reader->PullFrame(),
{.presentation_timestamp = Timestamp::Millis(f * 100)}, {.presentation_timestamp = Timestamp::Millis(f * 100)},
{Fb().Rate(Cqp{.target_qp = kQp - f}) ToVec({Fb().Rate(Cqp{.target_qp = kQp - f})
.Res(640, 360) .Res(640, 360)
.S(sid) .S(sid)
.Ref({0}) .Ref({0})
.Upd(0) .Upd(0)
.Build()}, .Cb(res.Cb())}));
res.CallBack());
EXPECT_THAT(res.FrameAt(f)->encoded_qp, Eq(kQp - f)); EXPECT_THAT(res.FrameAt(f)->encoded_qp, Eq(kQp - f));
dec.Decode(*res.FrameAt(f)); dec.Decode(*res.FrameAt(f));
} }

55
api/video_codecs/simple_encoder_wrapper.cc
View file

@ -17,6 +17,7 @@
#include <vector> #include <vector>
#include "absl/algorithm/container.h" #include "absl/algorithm/container.h"
#include "absl/types/variant.h"
#include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/scalability_mode.h"
#include "api/video_codecs/scalability_mode_helper.h" #include "api/video_codecs/scalability_mode_helper.h"
#include "api/video_codecs/video_encoder_factory_interface.h" #include "api/video_codecs/video_encoder_factory_interface.h"
@ -146,7 +147,6 @@ void SimpleEncoderWrapper::Encode(
svc_controller_->NextFrameConfig(force_keyframe); svc_controller_->NextFrameConfig(force_keyframe);
std::vector<FrameEncodeSettings> encode_settings; std::vector<FrameEncodeSettings> encode_settings;
std::vector<GenericFrameInfo> frame_infos; std::vector<GenericFrameInfo> frame_infos;
bool include_dependency_structure = false; bool include_dependency_structure = false;
for (size_t s = 0; s < configs.size(); ++s) { for (size_t s = 0; s < configs.size(); ++s) {
@ -178,38 +178,39 @@ void SimpleEncoderWrapper::Encode(
settings.frame_type = FrameType::kKeyframe; settings.frame_type = FrameType::kKeyframe;
include_dependency_structure = true; include_dependency_structure = true;
} }
}
absl::optional<FrameDependencyStructure> dependency_structure; absl::optional<FrameDependencyStructure> dependency_structure;
if (include_dependency_structure) { if (include_dependency_structure) {
dependency_structure = svc_controller_->DependencyStructure(); dependency_structure = svc_controller_->DependencyStructure();
} }
VideoEncoderInterface::EncodeResultCallback callback_internal = settings.result_callback =
[cb = std::move(callback), ds = std::move(dependency_structure), [cb = callback, ds = std::move(dependency_structure),
infos = std::move(frame_infos)]( info = std::move(frame_infos[settings.spatial_id])](
const VideoEncoderInterface::EncodeResult& result) mutable { const VideoEncoderInterface::EncodeResult& result) mutable {
auto* data = std::get_if<VideoEncoderInterface::EncodedData>(&result); auto* data =
EncodeResult res; absl::get_if<VideoEncoderInterface::EncodedData>(&result);
if (!data || data->spatial_id >= static_cast<int>(infos.size())) {
res.oh_no = true; EncodeResult res;
if (!data) {
res.oh_no = true;
cb(res);
return;
}
res.frame_type = data->frame_type;
res.bitstream_data = std::move(data->bitstream_data);
res.generic_frame_info = info;
if (res.frame_type == FrameType::kKeyframe) {
res.dependency_structure = ds;
}
cb(res); cb(res);
return; };
} }
res.frame_type = data->frame_type;
res.bitstream_data = std::move(data->bitstream_data);
res.generic_frame_info = infos[data->spatial_id];
if (res.frame_type == FrameType::kKeyframe) {
// Keyframe
res.dependency_structure = ds;
}
cb(res);
};
encoder_->Encode(std::move(frame_buffer), encoder_->Encode(std::move(frame_buffer),
{.presentation_timestamp = presentation_timestamp_}, {.presentation_timestamp = presentation_timestamp_},
encode_settings, std::move(callback_internal)); std::move(encode_settings));
presentation_timestamp_ += 1 / Frequency::Hertz(fps_); presentation_timestamp_ += 1 / Frequency::Hertz(fps_);
} }

3
api/video_codecs/simple_encoder_wrapper.h
View file

@ -33,8 +33,7 @@ class SimpleEncoderWrapper {
absl::optional<FrameDependencyStructure> dependency_structure; absl::optional<FrameDependencyStructure> dependency_structure;
}; };
using EncodeResultCallback = using EncodeResultCallback = std::function<void(const EncodeResult& result)>;
absl::AnyInvocable<void(const EncodeResult& result)>;
static std::vector<std::string> SupportedWebrtcSvcModes( static std::vector<std::string> SupportedWebrtcSvcModes(
const VideoEncoderFactoryInterface::Capabilities::PredictionConstraints& const VideoEncoderFactoryInterface::Capabilities::PredictionConstraints&

8
api/video_codecs/simple_encoder_wrapper_unittests.cc
View file

@ -125,8 +125,8 @@ TEST(SimpleEncoderWrapper, SupportedSvcModesUpToL3T3KeyWithHScaling) {
"S3T1", "S3T1h", "S3T2", "S3T2h", "S3T3", "S3T3h")); "S3T1", "S3T1h", "S3T2", "S3T2h", "S3T3", "S3T3h"));
} }
// TD: The encoder wrapper shouldn't really use an actual encoder implementation // TD: The encoder wrapper shouldn't really use an actual encoder
// for testing, but hey, this is just a PoC. // implementation for testing, but hey, this is just a PoC.
TEST(SimpleEncoderWrapper, EncodeL1T1) { TEST(SimpleEncoderWrapper, EncodeL1T1) {
auto encoder = LibaomAv1EncoderFactory().CreateEncoder( auto encoder = LibaomAv1EncoderFactory().CreateEncoder(
{.max_encode_dimensions = {1080, 720}, {.max_encode_dimensions = {1080, 720},
@ -171,7 +171,7 @@ TEST(SimpleEncoderWrapper, EncodeL1T1) {
}); });
} }
TEST(SimpleEncoderWrapper, DISABLED_EncodeL2T2_KEY) { TEST(SimpleEncoderWrapper, EncodeL2T2_KEY) {
auto encoder = LibaomAv1EncoderFactory().CreateEncoder( auto encoder = LibaomAv1EncoderFactory().CreateEncoder(
{.max_encode_dimensions = {1080, 720}, {.max_encode_dimensions = {1080, 720},
.encoding_format = {.sub_sampling = EncodingFormat::k420, .encoding_format = {.sub_sampling = EncodingFormat::k420,
@ -231,7 +231,7 @@ TEST(SimpleEncoderWrapper, DISABLED_EncodeL2T2_KEY) {
EXPECT_THAT(num_callbacks, Eq(4)); EXPECT_THAT(num_callbacks, Eq(4));
} }
TEST(SimpleEncoderWrapper, DISABLED_EncodeL1T3ForceKeyframe) { TEST(SimpleEncoderWrapper, EncodeL1T3ForceKeyframe) {
auto encoder = LibaomAv1EncoderFactory().CreateEncoder( auto encoder = LibaomAv1EncoderFactory().CreateEncoder(
{.max_encode_dimensions = {1080, 720}, {.max_encode_dimensions = {1080, 720},
.encoding_format = {.sub_sampling = EncodingFormat::k420, .encoding_format = {.sub_sampling = EncodingFormat::k420,

31
api/video_codecs/video_encoder_interface.h
View file

@ -13,7 +13,6 @@
#include <map> #include <map>
#include <memory> #include <memory>
// #include <unordered_set>
#include <string> #include <string>
#include <utility> #include <utility>
#include <vector> #include <vector>
@ -44,6 +43,17 @@ class VideoEncoderInterface {
int effort_level = 0; int effort_level = 0;
}; };
// Results from calling Encode. Called once for each configured frame.
struct EncodingError {};
struct EncodedData {
rtc::scoped_refptr<EncodedImageBufferInterface> bitstream_data;
FrameType frame_type;
int encoded_qp;
};
using EncodeResult = absl::variant<EncodingError, EncodedData>;
using EncodeResultCallback =
absl::AnyInvocable<void(const EncodeResult& result)>;
struct FrameEncodeSettings { struct FrameEncodeSettings {
struct Cbr { struct Cbr {
TimeDelta duration; TimeDelta duration;
@ -62,26 +72,13 @@ class VideoEncoderInterface {
Resolution resolution; Resolution resolution;
std::vector<int> reference_buffers; std::vector<int> reference_buffers;
absl::optional<int> update_buffer; absl::optional<int> update_buffer;
EncodeResultCallback result_callback;
}; };
// Results from calling Encode. Called once for each configured frame.
struct EncodingError {};
struct EncodedData {
rtc::scoped_refptr<EncodedImageBufferInterface> bitstream_data;
FrameType frame_type;
int spatial_id;
int encoded_qp;
};
using EncodeResult = std::variant<EncodingError, EncodedData>;
using EncodeResultCallback =
absl::AnyInvocable<void(const EncodeResult& result)>;
virtual void Encode(rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer, virtual void Encode(rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer,
const TemporalUnitSettings& settings, const TemporalUnitSettings& settings,
const std::vector<FrameEncodeSettings>& frame_settings, std::vector<FrameEncodeSettings> frame_settings) = 0;
EncodeResultCallback encode_result_callback) = 0;
}; };
} // namespace webrtc } // namespace webrtc