mirror of
https://github.com/mollyim/webrtc.git
synced 2025-05-13 05:40:42 +01:00
10d9d59db1
Encoded frames are preserved and decoded after all layers are encoded. Each spatial layer is decoded with separate decoder. For quality evaluation of lowres layers original input frame is downscaled with bilinear interpolation. Encoded and decoded frames are dumped into separate files. For async codecs encoded frames are passed to decoder in encode callback, as before. Bug: webrtc:8524 Change-Id: Idb0c92c7274c1915cff9a011a2794f1cf4bc8cb1 Reviewed-on: https://webrtc-review.googlesource.com/43381 Commit-Queue: Sergey Silkin <ssilkin@webrtc.org> Reviewed-by: Rasmus Brandt <brandtr@webrtc.org> Cr-Commit-Position: refs/heads/master@{#21844}
362 lines
14 KiB
C++
362 lines
14 KiB
C++
/*
|
|
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "modules/video_coding/codecs/test/videoprocessor_integrationtest.h"
|
|
|
|
#include <vector>
|
|
|
|
#include "modules/video_coding/codecs/test/test_config.h"
|
|
#include "rtc_base/ptr_util.h"
|
|
#include "test/testsupport/fileutils.h"
|
|
|
|
namespace webrtc {
|
|
namespace test {
|
|
|
|
namespace {
|
|
|
|
// Codec settings.
|
|
const bool kResilienceOn = true;
|
|
const int kCifWidth = 352;
|
|
const int kCifHeight = 288;
|
|
#if !defined(WEBRTC_IOS)
|
|
const int kNumFramesShort = 100;
|
|
#endif
|
|
const int kNumFramesLong = 300;
|
|
|
|
const std::nullptr_t kNoVisualizationParams = nullptr;
|
|
|
|
} // namespace
|
|
|
|
class VideoProcessorIntegrationTestLibvpx
|
|
: public VideoProcessorIntegrationTest {
|
|
protected:
|
|
VideoProcessorIntegrationTestLibvpx() {
|
|
config_.filename = "foreman_cif";
|
|
config_.input_filename = ResourcePath(config_.filename, "yuv");
|
|
config_.output_filename =
|
|
TempFilename(OutputPath(), "videoprocessor_integrationtest_libvpx");
|
|
config_.num_frames = kNumFramesLong;
|
|
// Only allow encoder/decoder to use single core, for predictability.
|
|
config_.use_single_core = true;
|
|
config_.hw_encoder = false;
|
|
config_.hw_decoder = false;
|
|
config_.encoded_frame_checker = &qp_frame_checker_;
|
|
}
|
|
|
|
private:
|
|
// Verify that the QP parser returns the same QP as the encoder does.
|
|
const class QpFrameChecker : public TestConfig::EncodedFrameChecker {
|
|
public:
|
|
void CheckEncodedFrame(webrtc::VideoCodecType codec,
|
|
const EncodedImage& encoded_frame) const override {
|
|
int qp;
|
|
if (codec == kVideoCodecVP8) {
|
|
EXPECT_TRUE(
|
|
vp8::GetQp(encoded_frame._buffer, encoded_frame._length, &qp));
|
|
} else if (codec == kVideoCodecVP9) {
|
|
EXPECT_TRUE(
|
|
vp9::GetQp(encoded_frame._buffer, encoded_frame._length, &qp));
|
|
} else {
|
|
RTC_NOTREACHED();
|
|
}
|
|
EXPECT_EQ(encoded_frame.qp_, qp) << "Encoder QP != parsed bitstream QP.";
|
|
}
|
|
} qp_frame_checker_;
|
|
};
|
|
|
|
// Fails on iOS. See webrtc:4755.
|
|
#if !defined(WEBRTC_IOS)
|
|
|
|
#if !defined(RTC_DISABLE_VP9)
|
|
TEST_F(VideoProcessorIntegrationTestLibvpx, HighBitrateVP9) {
|
|
config_.SetCodecSettings(kVideoCodecVP9, 1, 1, 1, false, false, true, false,
|
|
kResilienceOn, kCifWidth, kCifHeight);
|
|
config_.num_frames = kNumFramesShort;
|
|
|
|
std::vector<RateProfile> rate_profiles = {{500, 30, kNumFramesShort}};
|
|
|
|
std::vector<RateControlThresholds> rc_thresholds = {
|
|
{5, 1, 0, 0.1, 0.3, 0.1, 0, 1}};
|
|
|
|
std::vector<QualityThresholds> quality_thresholds = {{37, 36, 0.94, 0.92}};
|
|
|
|
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
|
|
&quality_thresholds, nullptr,
|
|
kNoVisualizationParams);
|
|
}
|
|
|
|
TEST_F(VideoProcessorIntegrationTestLibvpx, ChangeBitrateVP9) {
|
|
config_.SetCodecSettings(kVideoCodecVP9, 1, 1, 1, false, false, true, false,
|
|
kResilienceOn, kCifWidth, kCifHeight);
|
|
|
|
std::vector<RateProfile> rate_profiles = {
|
|
{200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update
|
|
{700, 30, 200},
|
|
{500, 30, kNumFramesLong}};
|
|
|
|
std::vector<RateControlThresholds> rc_thresholds = {
|
|
{5, 1, 0, 0.15, 0.5, 0.1, 0, 1},
|
|
{15, 2, 0, 0.2, 0.5, 0.1, 0, 0},
|
|
{10, 1, 0, 0.3, 0.5, 0.1, 0, 0}};
|
|
|
|
std::vector<QualityThresholds> quality_thresholds = {
|
|
{34, 33, 0.90, 0.88}, {38, 35, 0.95, 0.91}, {35, 34, 0.93, 0.90}};
|
|
|
|
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
|
|
&quality_thresholds, nullptr,
|
|
kNoVisualizationParams);
|
|
}
|
|
|
|
TEST_F(VideoProcessorIntegrationTestLibvpx, ChangeFramerateVP9) {
|
|
config_.SetCodecSettings(kVideoCodecVP9, 1, 1, 1, false, false, true, false,
|
|
kResilienceOn, kCifWidth, kCifHeight);
|
|
|
|
std::vector<RateProfile> rate_profiles = {
|
|
{100, 24, 100}, // target_kbps, input_fps, frame_index_rate_update
|
|
{100, 15, 200},
|
|
{100, 10, kNumFramesLong}};
|
|
|
|
// Framerate mismatch should be lower for lower framerate.
|
|
std::vector<RateControlThresholds> rc_thresholds = {
|
|
{10, 2, 40, 0.4, 0.5, 0.2, 0, 1},
|
|
{8, 2, 5, 0.2, 0.5, 0.2, 0, 0},
|
|
{5, 2, 0, 0.2, 0.5, 0.3, 0, 0}};
|
|
|
|
// Quality should be higher for lower framerates for the same content.
|
|
std::vector<QualityThresholds> quality_thresholds = {
|
|
{33, 32, 0.89, 0.87}, {33.5, 32, 0.90, 0.86}, {33.5, 31.5, 0.90, 0.85}};
|
|
|
|
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
|
|
&quality_thresholds, nullptr,
|
|
kNoVisualizationParams);
|
|
}
|
|
|
|
TEST_F(VideoProcessorIntegrationTestLibvpx, DenoiserOnVP9) {
|
|
config_.SetCodecSettings(kVideoCodecVP9, 1, 1, 1, false, true, true, false,
|
|
kResilienceOn, kCifWidth, kCifHeight);
|
|
config_.num_frames = kNumFramesShort;
|
|
|
|
std::vector<RateProfile> rate_profiles = {{500, 30, kNumFramesShort}};
|
|
|
|
std::vector<RateControlThresholds> rc_thresholds = {
|
|
{5, 1, 0, 0.1, 0.3, 0.1, 0, 1}};
|
|
|
|
std::vector<QualityThresholds> quality_thresholds = {{37.5, 36, 0.94, 0.93}};
|
|
|
|
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
|
|
&quality_thresholds, nullptr,
|
|
kNoVisualizationParams);
|
|
}
|
|
|
|
TEST_F(VideoProcessorIntegrationTestLibvpx, VeryLowBitrateVP9) {
|
|
config_.SetCodecSettings(kVideoCodecVP9, 1, 1, 1, false, false, true, true,
|
|
kResilienceOn, kCifWidth, kCifHeight);
|
|
|
|
std::vector<RateProfile> rate_profiles = {{50, 30, kNumFramesLong}};
|
|
|
|
std::vector<RateControlThresholds> rc_thresholds = {
|
|
{15, 3, 75, 1.0, 0.5, 0.4, 1, 1}};
|
|
|
|
std::vector<QualityThresholds> quality_thresholds = {{28, 25, 0.80, 0.65}};
|
|
|
|
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
|
|
&quality_thresholds, nullptr,
|
|
kNoVisualizationParams);
|
|
}
|
|
|
|
// TODO(marpan): Add temporal layer test for VP9, once changes are in
|
|
// vp9 wrapper for this.
|
|
|
|
#endif // !defined(RTC_DISABLE_VP9)
|
|
|
|
TEST_F(VideoProcessorIntegrationTestLibvpx, HighBitrateVP8) {
|
|
config_.SetCodecSettings(kVideoCodecVP8, 1, 1, 1, false, true, true, false,
|
|
kResilienceOn, kCifWidth, kCifHeight);
|
|
config_.num_frames = kNumFramesShort;
|
|
|
|
std::vector<RateProfile> rate_profiles = {{500, 30, kNumFramesShort}};
|
|
|
|
std::vector<RateControlThresholds> rc_thresholds = {
|
|
{5, 1, 0, 0.1, 0.2, 0.1, 0, 1}};
|
|
|
|
// std::vector<QualityThresholds> quality_thresholds = {{37, 35, 0.93, 0.91}};
|
|
// TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse
|
|
// than quality of x86 version. Use lower thresholds for now.
|
|
std::vector<QualityThresholds> quality_thresholds = {{35, 33, 0.91, 0.89}};
|
|
|
|
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
|
|
&quality_thresholds, nullptr,
|
|
kNoVisualizationParams);
|
|
}
|
|
|
|
#endif // !defined(WEBRTC_IOS)
|
|
|
|
// The tests below are currently disabled for Android. For ARM, the encoder
|
|
// uses |cpu_speed| = 12, as opposed to default |cpu_speed| <= 6 for x86,
|
|
// which leads to significantly different quality. The quality and rate control
|
|
// settings in the tests below are defined for encoder speed setting
|
|
// |cpu_speed| <= ~6. A number of settings would need to be significantly
|
|
// modified for the |cpu_speed| = 12 case. For now, keep the tests below
|
|
// disabled on Android. Some quality parameter in the above test has been
|
|
// adjusted to also pass for |cpu_speed| <= 12.
|
|
|
|
// Too slow to finish before timeout on iOS. See webrtc:4755.
|
|
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
|
|
#define MAYBE_ChangeBitrateVP8 DISABLED_ChangeBitrateVP8
|
|
#else
|
|
#define MAYBE_ChangeBitrateVP8 ChangeBitrateVP8
|
|
#endif
|
|
TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeBitrateVP8) {
|
|
config_.SetCodecSettings(kVideoCodecVP8, 1, 1, 1, false, true, true, false,
|
|
kResilienceOn, kCifWidth, kCifHeight);
|
|
|
|
std::vector<RateProfile> rate_profiles = {
|
|
{200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update
|
|
{800, 30, 200},
|
|
{500, 30, kNumFramesLong}};
|
|
|
|
std::vector<RateControlThresholds> rc_thresholds = {
|
|
{5, 1, 0, 0.1, 0.2, 0.1, 0, 1},
|
|
{15, 1, 0, 0.1, 0.2, 0.1, 0, 0},
|
|
{15, 1, 0, 0.3, 0.2, 0.1, 0, 0}};
|
|
|
|
// std::vector<QualityThresholds> quality_thresholds = {
|
|
// {33, 32, 0.89, 0.88}, {38, 36, 0.94, 0.93}, {35, 34, 0.92, 0.91}};
|
|
// TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse
|
|
// than quality of x86 version. Use lower thresholds for now.
|
|
std::vector<QualityThresholds> quality_thresholds = {
|
|
{31.8, 31, 0.86, 0.85}, {36, 34.8, 0.92, 0.90}, {33.5, 32, 0.90, 0.88}};
|
|
|
|
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
|
|
&quality_thresholds, nullptr,
|
|
kNoVisualizationParams);
|
|
}
|
|
|
|
// Too slow to finish before timeout on iOS. See webrtc:4755.
|
|
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
|
|
#define MAYBE_ChangeFramerateVP8 DISABLED_ChangeFramerateVP8
|
|
#else
|
|
#define MAYBE_ChangeFramerateVP8 ChangeFramerateVP8
|
|
#endif
|
|
TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeFramerateVP8) {
|
|
config_.SetCodecSettings(kVideoCodecVP8, 1, 1, 1, false, true, true, false,
|
|
kResilienceOn, kCifWidth, kCifHeight);
|
|
|
|
std::vector<RateProfile> rate_profiles = {
|
|
{80, 24, 100}, // target_kbps, input_fps, frame_index_rate_update
|
|
{80, 15, 200},
|
|
{80, 10, kNumFramesLong}};
|
|
|
|
// std::vector<RateControlThresholds> rc_thresholds = {
|
|
// {10, 2, 20, 0.4, 0.3, 0.1, 0, 1},
|
|
// {5, 2, 5, 0.3, 0.3, 0.1, 0, 0},
|
|
// {4, 2, 1, 0.2, 0.3, 0.2, 0, 0}};
|
|
// TODO(webrtc:8757): AMR VP8 drops more frames than x86 version. Use lower
|
|
// thresholds for now.
|
|
std::vector<RateControlThresholds> rc_thresholds = {
|
|
{10, 2, 60, 0.5, 0.3, 0.3, 0, 1},
|
|
{10, 2, 30, 0.3, 0.3, 0.3, 0, 0},
|
|
{10, 2, 10, 0.2, 0.3, 0.2, 0, 0}};
|
|
|
|
// std::vector<QualityThresholds> quality_thresholds = {
|
|
// {31, 30, 0.87, 0.86}, {32, 31, 0.89, 0.86}, {32, 30, 0.87, 0.82}};
|
|
// TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse
|
|
// than quality of x86 version. Use lower thresholds for now.
|
|
std::vector<QualityThresholds> quality_thresholds = {
|
|
{31, 30, 0.85, 0.84}, {31.5, 30.5, 0.86, 0.84}, {30.5, 29, 0.83, 0.78}};
|
|
|
|
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
|
|
&quality_thresholds, nullptr,
|
|
kNoVisualizationParams);
|
|
}
|
|
|
|
// Too slow to finish before timeout on iOS. See webrtc:4755.
|
|
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
|
|
#define MAYBE_TemporalLayersVP8 DISABLED_TemporalLayersVP8
|
|
#else
|
|
#define MAYBE_TemporalLayersVP8 TemporalLayersVP8
|
|
#endif
|
|
TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_TemporalLayersVP8) {
|
|
config_.SetCodecSettings(kVideoCodecVP8, 1, 1, 3, false, true, true, false,
|
|
kResilienceOn, kCifWidth, kCifHeight);
|
|
|
|
std::vector<RateProfile> rate_profiles = {{200, 30, 150},
|
|
{400, 30, kNumFramesLong}};
|
|
|
|
// std::vector<RateControlThresholds> rc_thresholds = {
|
|
// {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}, {10, 2, 0, 0.1, 0.2, 0.1, 0, 1}};
|
|
// TODO(webrtc:8757): AMR VP8 drops more frames than x86 version. Use lower
|
|
// thresholds for now.
|
|
std::vector<RateControlThresholds> rc_thresholds = {
|
|
{10, 1, 2, 0.3, 0.2, 0.1, 0, 1}, {12, 2, 3, 0.1, 0.2, 0.1, 0, 1}};
|
|
|
|
// Min SSIM drops because of high motion scene with complex backgound (trees).
|
|
// std::vector<QualityThresholds> quality_thresholds = {{32, 30, 0.88, 0.85},
|
|
// {33, 30, 0.89, 0.83}};
|
|
// TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse
|
|
// than quality of x86 version. Use lower thresholds for now.
|
|
std::vector<QualityThresholds> quality_thresholds = {{31, 30, 0.85, 0.84},
|
|
{31, 28, 0.85, 0.75}};
|
|
|
|
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
|
|
&quality_thresholds, nullptr,
|
|
kNoVisualizationParams);
|
|
}
|
|
|
|
// Might be too slow on mobile platforms.
|
|
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
|
|
#define MAYBE_SimulcastVP8 DISABLED_SimulcastVP8
|
|
#else
|
|
#define MAYBE_SimulcastVP8 SimulcastVP8
|
|
#endif
|
|
TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_SimulcastVP8) {
|
|
config_.filename = "ConferenceMotion_1280_720_50";
|
|
config_.input_filename = ResourcePath(config_.filename, "yuv");
|
|
config_.num_frames = 100;
|
|
config_.SetCodecSettings(kVideoCodecVP8, 3, 1, 3, false, true, true, false,
|
|
kResilienceOn, 1280, 720);
|
|
|
|
std::vector<RateProfile> rate_profiles = {{1500, 30, config_.num_frames}};
|
|
|
|
std::vector<RateControlThresholds> rc_thresholds = {
|
|
{5, 1, 5, 0.2, 0.3, 0.1, 0, 1}};
|
|
std::vector<QualityThresholds> quality_thresholds = {{34, 32, 0.90, 0.88}};
|
|
|
|
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
|
|
&quality_thresholds, nullptr,
|
|
kNoVisualizationParams);
|
|
}
|
|
|
|
// Might be too slow on mobile platforms.
|
|
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
|
|
#define MAYBE_SvcVP9 DISABLED_SvcVP9
|
|
#else
|
|
#define MAYBE_SvcVP9 SvcVP9
|
|
#endif
|
|
TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_SvcVP9) {
|
|
config_.filename = "ConferenceMotion_1280_720_50";
|
|
config_.input_filename = ResourcePath(config_.filename, "yuv");
|
|
config_.num_frames = 100;
|
|
config_.SetCodecSettings(kVideoCodecVP9, 1, 3, 3, false, true, true, false,
|
|
kResilienceOn, 1280, 720);
|
|
|
|
std::vector<RateProfile> rate_profiles = {{1500, 30, config_.num_frames}};
|
|
|
|
std::vector<RateControlThresholds> rc_thresholds = {
|
|
{5, 1, 5, 0.2, 0.3, 0.1, 0, 1}};
|
|
std::vector<QualityThresholds> quality_thresholds = {{36, 34, 0.93, 0.91}};
|
|
|
|
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
|
|
&quality_thresholds, nullptr,
|
|
kNoVisualizationParams);
|
|
}
|
|
|
|
} // namespace test
|
|
} // namespace webrtc
|