mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-13 13:50:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Reland "Updated analysis in videoprocessor."

Browse source 
This is a reland of 1880c7162b
Original change's description:
> Updated analysis in videoprocessor.
>
> - Run analysis after all frames are processed. Before part of it was
> done at bitrate change points;
> - Analysis is done for whole stream as well as for each rate update
> interval;
> - Changed units from number of frames to time units for some metrics
> and thresholds. E.g. 'num frames to hit tagret bitrate' is changed to
> 'time to reach target bitrate, sec';
> - Changed data type of FrameStatistic::max_nalu_length (renamed to
> max_nalu_size_bytes) from rtc::Optional to size_t. There it no need to
> use such advanced data type in such low level data structure.
>
> Bug: webrtc:8524
> Change-Id: Ic9f6eab5b15ee12a80324b1f9c101de1bf3c702f
> Reviewed-on: https://webrtc-review.googlesource.com/31901
> Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
> Reviewed-by: Stefan Holmer <stefan@webrtc.org>
> Reviewed-by: Åsa Persson <asapersson@webrtc.org>
> Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
> Cr-Commit-Position: refs/heads/master@{#21653}

TBR=brandtr@webrtc.org, stefan@webrtc.org

Bug: webrtc:8524
Change-Id: Ie0ad7790689422ffa61da294967fc492a13b75ae
Reviewed-on: https://webrtc-review.googlesource.com/40202
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#21668}
This commit is contained in:
Sergey Silkin 2018-01-17 15:11:44 +01:00 committed by Commit Bot
parent e2a931886f
commit 3be2a55e7f
20 changed files with 735 additions and 908 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
modules/video_coding/BUILD.gn
View file

@ -517,6 +517,7 @@ if (rtc_include_tests) {
"../../test:test_support",
"../../test:video_test_common",
"../../test:video_test_support",
"../rtp_rtcp:rtp_rtcp_format",
]
}
@ -578,6 +579,7 @@ if (rtc_include_tests) {
"../../rtc_base:rtc_base_tests_utils",
"../../system_wrappers",
"../../test:field_trial",
"../../test:test_common",
"../../test:test_support",
"../../test:video_test_common",
"../../test:video_test_support",

29
modules/video_coding/codecs/test/plot_webrtc_test_logs.py
View file

@ -24,6 +24,7 @@ EVENT_END = 'OK ] CodecSettings/VideoProcessorIntegrationTestParameterized.'
# Metrics to plot, tuple: (name to parse in file, label to use when plotting).
BITRATE = ('Target bitrate', 'target bitrate (kbps)')
FRAMERATE = ('Target framerate', 'fps')
WIDTH = ('Width', 'width')
HEIGHT = ('Height', 'height')
FILENAME = ('Filename', 'clip')
@ -35,24 +36,22 @@ CORES = ('# CPU cores used', 'CPU cores used')
DENOISING = ('Denoising', 'denoising')
RESILIENCE = ('Resilience', 'resilience')
ERROR_CONCEALMENT = ('Error concealment', 'error concealment')
QP = ('Average QP', 'avg QP')
QP = ('QP', 'QP avg')
CPU_USAGE = ('CPU usage %', 'CPU usage (%)')
PSNR = ('PSNR avg', 'PSNR (dB)')
SSIM = ('SSIM avg', 'SSIM')
PSNR = ('PSNR', 'PSNR (dB)')
SSIM = ('SSIM', 'SSIM')
ENC_BITRATE = ('Encoded bitrate', 'encoded bitrate (kbps)')
FRAMERATE = ('Frame rate', 'fps')
NUM_FRAMES = ('# processed frames', 'num frames')
NUM_FRAMES = ('# input frames', 'num frames')
NUM_DROPPED_FRAMES = ('# dropped frames', 'num dropped frames')
NUM_FRAMES_TO_TARGET = ('# frames to convergence',
'frames to reach target rate')
TIME_TO_TARGET = ('Time to reach target bitrate',
'time to reach target rate (sec)')
ENCODE_TIME = ('Encoding time', 'encode time (us)')
ENCODE_TIME_AVG = ('Encoding time', 'encode time (us) avg')
ENCODE_TIME_AVG = ('Frame encoding time', 'encode time (us) avg')
DECODE_TIME = ('Decoding time', 'decode time (us)')
DECODE_TIME_AVG = ('Decoding time', 'decode time (us) avg')
FRAME_SIZE = ('Frame sizes', 'frame size (bytes)')
FRAME_SIZE_AVG = ('Frame sizes', 'frame size (bytes) avg')
AVG_KEY_FRAME_SIZE = ('Average key frame size', 'avg key frame size (bytes)')
AVG_NON_KEY_FRAME_SIZE = ('Average non-key frame size',
DECODE_TIME_AVG = ('Frame decoding time', 'decode time (us) avg')
FRAME_SIZE = ('Frame size', 'frame size (bytes)')
AVG_KEY_FRAME_SIZE = ('Avg key frame size', 'avg key frame size (bytes)')
AVG_NON_KEY_FRAME_SIZE = ('Avg delta frame size',
'avg non-key frame size (bytes)')
# Settings.
@ -90,7 +89,7 @@ RESULTS = [
SSIM,
ENC_BITRATE,
NUM_DROPPED_FRAMES,
NUM_FRAMES_TO_TARGET,
TIME_TO_TARGET,
ENCODE_TIME_AVG,
DECODE_TIME_AVG,
QP,
@ -235,7 +234,7 @@ def TryFindMetric(parsed, line, settings_file):
found, maximum = GetMetric("Max", settings_file.readline())
if not found:
return
found, average = GetMetric("Average", settings_file.readline())
found, average = GetMetric("Avg", settings_file.readline())
if not found:
return

208
modules/video_coding/codecs/test/stats.cc
View file

@ -9,59 +9,35 @@
*/
#include "modules/video_coding/codecs/test/stats.h"
#include <stdio.h>
#include <algorithm>
#include "rtc_base/checks.h"
#include "rtc_base/format_macros.h"
namespace webrtc {
namespace test {
namespace {
bool LessForEncodeTime(const FrameStatistic& s1, const FrameStatistic& s2) {
RTC_DCHECK_NE(s1.frame_number, s2.frame_number);
return s1.encode_time_us < s2.encode_time_us;
std::string FrameStatistic::ToString() const {
std::stringstream ss;
ss << "frame " << frame_number;
ss << " " << decoded_width << "x" << decoded_height;
ss << " sl " << simulcast_svc_idx;
ss << " tl " << temporal_layer_idx;
ss << " type " << frame_type;
ss << " length " << encoded_frame_size_bytes;
ss << " qp " << qp;
ss << " psnr " << psnr;
ss << " ssim " << ssim;
ss << " enc_time_us " << encode_time_us;
ss << " dec_time_us " << decode_time_us;
ss << " rtp_ts " << rtp_timestamp;
ss << " bitrate_kbps " << target_bitrate_kbps;
return ss.str();
}
bool LessForDecodeTime(const FrameStatistic& s1, const FrameStatistic& s2) {
RTC_DCHECK_NE(s1.frame_number, s2.frame_number);
return s1.decode_time_us < s2.decode_time_us;
}
bool LessForEncodedSize(const FrameStatistic& s1, const FrameStatistic& s2) {
RTC_DCHECK_NE(s1.frame_number, s2.frame_number);
return s1.encoded_frame_size_bytes < s2.encoded_frame_size_bytes;
}
bool LessForBitRate(const FrameStatistic& s1, const FrameStatistic& s2) {
RTC_DCHECK_NE(s1.frame_number, s2.frame_number);
return s1.bitrate_kbps < s2.bitrate_kbps;
}
bool LessForPsnr(const FrameStatistic& s1, const FrameStatistic& s2) {
RTC_DCHECK_NE(s1.frame_number, s2.frame_number);
return s1.psnr < s2.psnr;
}
bool LessForSsim(const FrameStatistic& s1, const FrameStatistic& s2) {
RTC_DCHECK_NE(s1.frame_number, s2.frame_number);
return s1.ssim < s2.ssim;
}
} // namespace
FrameStatistic* Stats::AddFrame() {
// We don't expect more frames than what can be stored in an int.
stats_.emplace_back(static_cast<int>(stats_.size()));
stats_.emplace_back(stats_.size());
return &stats_.back();
}
FrameStatistic* Stats::GetFrame(int frame_number) {
RTC_CHECK_GE(frame_number, 0);
FrameStatistic* Stats::GetFrame(size_t frame_number) {
RTC_CHECK_LT(frame_number, stats_.size());
return &stats_[frame_number];
}
@ -70,153 +46,5 @@ size_t Stats::size() const {
return stats_.size();
}
void Stats::PrintSummary() const {
if (stats_.empty()) {
printf("No frame statistics have been logged yet.\n");
return;
}
printf("Encode/decode statistics\n==\n");
// Calculate min, max, average and total encoding time.
int total_encoding_time_us = 0;
int total_decoding_time_us = 0;
size_t total_encoded_frame_size_bytes = 0;
size_t total_encoded_key_frame_size_bytes = 0;
size_t total_encoded_delta_frame_size_bytes = 0;
size_t num_key_frames = 0;
size_t num_delta_frames = 0;
int num_encode_failures = 0;
double total_psnr = 0.0;
double total_ssim = 0.0;
for (const FrameStatistic& stat : stats_) {
total_encoding_time_us += stat.encode_time_us;
total_decoding_time_us += stat.decode_time_us;
total_encoded_frame_size_bytes += stat.encoded_frame_size_bytes;
if (stat.frame_type == webrtc::kVideoFrameKey) {
total_encoded_key_frame_size_bytes += stat.encoded_frame_size_bytes;
++num_key_frames;
} else {
total_encoded_delta_frame_size_bytes += stat.encoded_frame_size_bytes;
++num_delta_frames;
}
if (stat.encode_return_code != 0) {
++num_encode_failures;
}
if (stat.decoding_successful) {
total_psnr += stat.psnr;
total_ssim += stat.ssim;
}
}
// Encoding stats.
printf("# Encoded frame failures: %d\n", num_encode_failures);
printf("Encoding time:\n");
auto frame_it =
std::min_element(stats_.begin(), stats_.end(), LessForEncodeTime);
printf(" Min : %7d us (frame %d)\n", frame_it->encode_time_us,
frame_it->frame_number);
frame_it = std::max_element(stats_.begin(), stats_.end(), LessForEncodeTime);
printf(" Max : %7d us (frame %d)\n", frame_it->encode_time_us,
frame_it->frame_number);
printf(" Average : %7d us\n",
static_cast<int>(total_encoding_time_us / stats_.size()));
// Decoding stats.
printf("Decoding time:\n");
// Only consider successfully decoded frames (packet loss may cause failures).
std::vector<FrameStatistic> decoded_frames;
for (const FrameStatistic& stat : stats_) {
if (stat.decoding_successful) {
decoded_frames.push_back(stat);
}
}
if (decoded_frames.empty()) {
printf("No successfully decoded frames exist in this statistics.\n");
} else {
frame_it = std::min_element(decoded_frames.begin(), decoded_frames.end(),
LessForDecodeTime);
printf(" Min : %7d us (frame %d)\n", frame_it->decode_time_us,
frame_it->frame_number);
frame_it = std::max_element(decoded_frames.begin(), decoded_frames.end(),
LessForDecodeTime);
printf(" Max : %7d us (frame %d)\n", frame_it->decode_time_us,
frame_it->frame_number);
printf(" Average : %7d us\n",
static_cast<int>(total_decoding_time_us / decoded_frames.size()));
printf(" Failures: %d frames failed to decode.\n",
static_cast<int>(stats_.size() - decoded_frames.size()));
}
// Frame size stats.
printf("Frame sizes:\n");
frame_it = std::min_element(stats_.begin(), stats_.end(), LessForEncodedSize);
printf(" Min : %7" PRIuS " bytes (frame %d)\n",
frame_it->encoded_frame_size_bytes, frame_it->frame_number);
frame_it = std::max_element(stats_.begin(), stats_.end(), LessForEncodedSize);
printf(" Max : %7" PRIuS " bytes (frame %d)\n",
frame_it->encoded_frame_size_bytes, frame_it->frame_number);
printf(" Average : %7" PRIuS " bytes\n",
total_encoded_frame_size_bytes / stats_.size());
if (num_key_frames > 0) {
printf(" Average key frame size : %7" PRIuS " bytes (%" PRIuS
" keyframes)\n",
total_encoded_key_frame_size_bytes / num_key_frames, num_key_frames);
}
if (num_delta_frames > 0) {
printf(" Average non-key frame size: %7" PRIuS " bytes (%" PRIuS
" frames)\n",
total_encoded_delta_frame_size_bytes / num_delta_frames,
num_delta_frames);
}
// Bitrate stats.
printf("Bitrates:\n");
frame_it = std::min_element(stats_.begin(), stats_.end(), LessForBitRate);
printf(" Min bitrate: %7d kbps (frame %d)\n", frame_it->bitrate_kbps,
frame_it->frame_number);
frame_it = std::max_element(stats_.begin(), stats_.end(), LessForBitRate);
printf(" Max bitrate: %7d kbps (frame %d)\n", frame_it->bitrate_kbps,
frame_it->frame_number);
// Quality.
printf("Quality:\n");
if (decoded_frames.empty()) {
printf("No successfully decoded frames exist in this statistics.\n");
} else {
frame_it = std::min_element(decoded_frames.begin(), decoded_frames.end(),
LessForPsnr);
printf(" PSNR min: %f (frame %d)\n", frame_it->psnr,
frame_it->frame_number);
printf(" PSNR avg: %f\n", total_psnr / decoded_frames.size());
frame_it = std::min_element(decoded_frames.begin(), decoded_frames.end(),
LessForSsim);
printf(" SSIM min: %f (frame %d)\n", frame_it->ssim,
frame_it->frame_number);
printf(" SSIM avg: %f\n", total_ssim / decoded_frames.size());
}
printf("\n");
printf("Total encoding time : %7d ms.\n", total_encoding_time_us / 1000);
printf("Total decoding time : %7d ms.\n", total_decoding_time_us / 1000);
printf("Total processing time: %7d ms.\n",
(total_encoding_time_us + total_decoding_time_us) / 1000);
// QP stats.
int total_qp = 0;
int total_qp_count = 0;
for (const FrameStatistic& stat : stats_) {
if (stat.qp >= 0) {
total_qp += stat.qp;
++total_qp_count;
}
}
int avg_qp = (total_qp_count > 0) ? (total_qp / total_qp_count) : -1;
printf("Average QP: %d\n", avg_qp);
printf("\n");
}
} // namespace test
} // namespace webrtc

32
modules/video_coding/codecs/test/stats.h
View file

@ -11,6 +11,7 @@
#ifndef MODULES_VIDEO_CODING_CODECS_TEST_STATS_H_
#define MODULES_VIDEO_CODING_CODECS_TEST_STATS_H_
#include <string>
#include <vector>
#include "common_types.h" // NOLINT(build/include)
@ -20,34 +21,42 @@ namespace test {
// Statistics for one processed frame.
struct FrameStatistic {
explicit FrameStatistic(int frame_number) : frame_number(frame_number) {}
const int frame_number = 0;
explicit FrameStatistic(size_t frame_number) : frame_number(frame_number) {}
std::string ToString() const;
size_t frame_number = 0;
size_t rtp_timestamp = 0;
// Encoding.
int64_t encode_start_ns = 0;
int encode_return_code = 0;
bool encoding_successful = false;
int encode_time_us = 0;
int bitrate_kbps = 0;
size_t encode_time_us = 0;
size_t target_bitrate_kbps = 0;
size_t encoded_frame_size_bytes = 0;
webrtc::FrameType frame_type = kVideoFrameDelta;
// Layering.
size_t temporal_layer_idx = 0;
size_t simulcast_svc_idx = 0;
// H264 specific.
rtc::Optional<size_t> max_nalu_length;
size_t max_nalu_size_bytes = 0;
// Decoding.
int64_t decode_start_ns = 0;
int decode_return_code = 0;
bool decoding_successful = false;
int decode_time_us = 0;
int decoded_width = 0;
int decoded_height = 0;
size_t decode_time_us = 0;
size_t decoded_width = 0;
size_t decoded_height = 0;
// Quantization.
int qp = -1;
// How many packets were discarded of the encoded frame data (if any).
int packets_dropped = 0;
size_t packets_dropped = 0;
size_t total_packets = 0;
size_t manipulated_length = 0;
@ -66,13 +75,10 @@ class Stats {
FrameStatistic* AddFrame();
// Returns the FrameStatistic corresponding to |frame_number|.
FrameStatistic* GetFrame(int frame_number);
FrameStatistic* GetFrame(size_t frame_number);
size_t size() const;
// TODO(brandtr): Add output as CSV.
void PrintSummary() const;
private:
std::vector<FrameStatistic> stats_;
};

15
modules/video_coding/codecs/test/stats_unittest.cc
View file

@ -15,28 +15,21 @@
namespace webrtc {
namespace test {
TEST(StatsTest, TestEmptyObject) {
Stats stats;
stats.PrintSummary(); // Should not crash.
}
TEST(StatsTest, AddSingleFrame) {
Stats stats;
FrameStatistic* frame_stat = stats.AddFrame();
EXPECT_EQ(0, frame_stat->frame_number);
EXPECT_EQ(0ull, frame_stat->frame_number);
EXPECT_EQ(1u, stats.size());
}
TEST(StatsTest, AddMultipleFrames) {
Stats stats;
const int kNumFrames = 1000;
for (int i = 0; i < kNumFrames; ++i) {
const size_t kNumFrames = 1000;
for (size_t i = 0; i < kNumFrames; ++i) {
FrameStatistic* frame_stat = stats.AddFrame();
EXPECT_EQ(i, frame_stat->frame_number);
}
EXPECT_EQ(kNumFrames, static_cast<int>(stats.size()));
stats.PrintSummary(); // Should not crash.
EXPECT_EQ(kNumFrames, stats.size());
}
} // namespace test

100
modules/video_coding/codecs/test/test_config.cc
View file

@ -27,34 +27,36 @@ std::string CodecSpecificToString(const webrtc::VideoCodec& codec) {
std::stringstream ss;
switch (codec.codecType) {
case kVideoCodecVP8:
ss << "\n Complexity : " << codec.VP8().complexity;
ss << "\n Resilience : " << codec.VP8().resilience;
ss << "\n # temporal layers : "
ss << "\n Complexity : " << codec.VP8().complexity;
ss << "\n Resilience : " << codec.VP8().resilience;
ss << "\n # temporal layers : "
<< static_cast<int>(codec.VP8().numberOfTemporalLayers);
ss << "\n Denoising : " << codec.VP8().denoisingOn;
ss << "\n Error concealment : " << codec.VP8().errorConcealmentOn;
ss << "\n Automatic resize : " << codec.VP8().automaticResizeOn;
ss << "\n Frame dropping : " << codec.VP8().frameDroppingOn;
ss << "\n Key frame interval: " << codec.VP8().keyFrameInterval;
ss << "\n Denoising : " << codec.VP8().denoisingOn;
ss << "\n Error concealment : " << codec.VP8().errorConcealmentOn;
ss << "\n Automatic resize : " << codec.VP8().automaticResizeOn;
ss << "\n Frame dropping : " << codec.VP8().frameDroppingOn;
ss << "\n Key frame interval : " << codec.VP8().keyFrameInterval;
break;
case kVideoCodecVP9:
ss << "\n Complexity : " << codec.VP9().complexity;
ss << "\n Resilience : " << codec.VP9().resilienceOn;
ss << "\n # temporal layers : "
ss << "\n Complexity : " << codec.VP9().complexity;
ss << "\n Resilience : " << codec.VP9().resilienceOn;
ss << "\n # temporal layers : "
<< static_cast<int>(codec.VP9().numberOfTemporalLayers);
ss << "\n Denoising : " << codec.VP9().denoisingOn;
ss << "\n Frame dropping : " << codec.VP9().frameDroppingOn;
ss << "\n Key frame interval: " << codec.VP9().keyFrameInterval;
ss << "\n Adaptive QP mode : " << codec.VP9().adaptiveQpMode;
ss << "\n Automatic resize : " << codec.VP9().automaticResizeOn;
ss << "\n # spatial layers : "
ss << "\n # spatial layers : "
<< static_cast<int>(codec.VP9().numberOfSpatialLayers);
ss << "\n Flexible mode : " << codec.VP9().flexibleMode;
ss << "\n Denoising : " << codec.VP9().denoisingOn;
ss << "\n Frame dropping : " << codec.VP9().frameDroppingOn;
ss << "\n Key frame interval : " << codec.VP9().keyFrameInterval;
ss << "\n Adaptive QP mode : " << codec.VP9().adaptiveQpMode;
ss << "\n Automatic resize : " << codec.VP9().automaticResizeOn;
ss << "\n # spatial layers : "
<< static_cast<int>(codec.VP9().numberOfSpatialLayers);
ss << "\n Flexible mode : " << codec.VP9().flexibleMode;
break;
case kVideoCodecH264:
ss << "\n Frame dropping : " << codec.H264().frameDroppingOn;
ss << "\n Key frame interval: " << codec.H264().keyFrameInterval;
ss << "\n Profile : " << codec.H264().profile;
ss << "\n Frame dropping : " << codec.H264().frameDroppingOn;
ss << "\n Key frame interval : " << codec.H264().keyFrameInterval;
ss << "\n Profile : " << codec.H264().profile;
break;
default:
break;
@ -65,26 +67,27 @@ std::string CodecSpecificToString(const webrtc::VideoCodec& codec) {
} // namespace
void TestConfig::SetCodecSettings(VideoCodecType codec_type,
int num_temporal_layers,
size_t num_temporal_layers,
bool error_concealment_on,
bool denoising_on,
bool frame_dropper_on,
bool spatial_resize_on,
bool resilience_on,
int width,
int height) {
size_t width,
size_t height) {
webrtc::test::CodecSettings(codec_type, &codec_settings);
// TODO(brandtr): Move the setting of |width| and |height| to the tests, and
// DCHECK that they are set before initializing the codec instead.
codec_settings.width = width;
codec_settings.height = height;
codec_settings.width = static_cast<uint16_t>(width);
codec_settings.height = static_cast<uint16_t>(height);
switch (codec_settings.codecType) {
case kVideoCodecVP8:
codec_settings.VP8()->resilience =
resilience_on ? kResilientStream : kResilienceOff;
codec_settings.VP8()->numberOfTemporalLayers = num_temporal_layers;
codec_settings.VP8()->numberOfTemporalLayers =
static_cast<uint8_t>(num_temporal_layers);
codec_settings.VP8()->denoisingOn = denoising_on;
codec_settings.VP8()->errorConcealmentOn = error_concealment_on;
codec_settings.VP8()->automaticResizeOn = spatial_resize_on;
@ -93,7 +96,8 @@ void TestConfig::SetCodecSettings(VideoCodecType codec_type,
break;
case kVideoCodecVP9:
codec_settings.VP9()->resilienceOn = resilience_on;
codec_settings.VP9()->numberOfTemporalLayers = num_temporal_layers;
codec_settings.VP9()->numberOfTemporalLayers =
static_cast<uint8_t>(num_temporal_layers);
codec_settings.VP9()->denoisingOn = denoising_on;
codec_settings.VP9()->frameDroppingOn = frame_dropper_on;
codec_settings.VP9()->keyFrameInterval = kBaseKeyFrameInterval;
@ -109,11 +113,11 @@ void TestConfig::SetCodecSettings(VideoCodecType codec_type,
}
}
int TestConfig::NumberOfCores() const {
size_t TestConfig::NumberOfCores() const {
return use_single_core ? 1 : CpuInfo::DetectNumberOfCores();
}
int TestConfig::NumberOfTemporalLayers() const {
size_t TestConfig::NumberOfTemporalLayers() const {
if (codec_settings.codecType == kVideoCodecVP8) {
return codec_settings.VP8().numberOfTemporalLayers;
} else if (codec_settings.codecType == kVideoCodecVP9) {
@ -123,8 +127,16 @@ int TestConfig::NumberOfTemporalLayers() const {
}
}
int TestConfig::TemporalLayerForFrame(int frame_idx) const {
int tl = -1;
size_t TestConfig::NumberOfSpatialLayers() const {
if (codec_settings.codecType == kVideoCodecVP9) {
return codec_settings.VP9().numberOfSpatialLayers;
} else {
return 1;
}
}
size_t TestConfig::TemporalLayerForFrame(size_t frame_idx) const {
size_t tl = 0;
switch (NumberOfTemporalLayers()) {
case 1:
tl = 0;
@ -153,7 +165,7 @@ int TestConfig::TemporalLayerForFrame(int frame_idx) const {
return tl;
}
std::vector<FrameType> TestConfig::FrameTypeForFrame(int frame_idx) const {
std::vector<FrameType> TestConfig::FrameTypeForFrame(size_t frame_idx) const {
if (keyframe_interval > 0 && (frame_idx % keyframe_interval == 0)) {
return {kVideoFrameKey};
}
@ -163,17 +175,19 @@ std::vector<FrameType> TestConfig::FrameTypeForFrame(int frame_idx) const {
std::string TestConfig::ToString() const {
std::string codec_type = CodecTypeToPayloadString(codec_settings.codecType);
std::stringstream ss;
ss << "\n Filename : " << filename;
ss << "\n # CPU cores used : " << NumberOfCores();
ss << "\n Filename : " << filename;
ss << "\n # CPU cores used : " << NumberOfCores();
ss << "\n General:";
ss << "\n Codec type : " << codec_type;
ss << "\n Start bitrate : " << codec_settings.startBitrate << " kbps";
ss << "\n Max bitrate : " << codec_settings.maxBitrate << " kbps";
ss << "\n Min bitrate : " << codec_settings.minBitrate << " kbps";
ss << "\n Width : " << codec_settings.width;
ss << "\n Height : " << codec_settings.height;
ss << "\n Max frame rate : " << codec_settings.maxFramerate;
ss << "\n QPmax : " << codec_settings.qpMax;
ss << "\n Codec type : " << codec_type;
ss << "\n Start bitrate : " << codec_settings.startBitrate << " kbps";
ss << "\n Max bitrate : " << codec_settings.maxBitrate << " kbps";
ss << "\n Min bitrate : " << codec_settings.minBitrate << " kbps";
ss << "\n Width : " << codec_settings.width;
ss << "\n Height : " << codec_settings.height;
ss << "\n Max frame rate : " << codec_settings.maxFramerate;
ss << "\n QPmax : " << codec_settings.qpMax;
ss << "\n # simulcast streams : "
<< static_cast<int>(codec_settings.numberOfSimulcastStreams);
ss << "\n " << codec_type << " specific: ";
ss << CodecSpecificToString(codec_settings);
return ss.str();

22
modules/video_coding/codecs/test/test_config.h
View file

@ -42,19 +42,20 @@ struct TestConfig {
};
void SetCodecSettings(VideoCodecType codec_type,
int num_temporal_layers,
size_t num_temporal_layers,
bool error_concealment_on,
bool denoising_on,
bool frame_dropper_on,
bool spatial_resize_on,
bool resilience_on,
int width,
int height);
size_t width,
size_t height);
int NumberOfCores() const;
int NumberOfTemporalLayers() const;
int TemporalLayerForFrame(int frame_idx) const;
std::vector<FrameType> FrameTypeForFrame(int frame_idx) const;
size_t NumberOfCores() const;
size_t NumberOfTemporalLayers() const;
size_t NumberOfSpatialLayers() const;
size_t TemporalLayerForFrame(size_t frame_idx) const;
std::vector<FrameType> FrameTypeForFrame(size_t frame_idx) const;
std::string ToString() const;
std::string CodecName() const;
std::string FilenameWithParams() const;
@ -70,7 +71,7 @@ struct TestConfig {
std::string output_filename;
// Number of frames to process.
int num_frames = 0;
size_t num_frames = 0;
// Configurations related to networking.
NetworkingConfig networking_config;
@ -96,7 +97,7 @@ struct TestConfig {
// to this setting. Forcing key frames may also affect encoder planning
// optimizations in a negative way, since it will suddenly be forced to
// produce an expensive key frame.
int keyframe_interval = 0;
size_t keyframe_interval = 0;
// Codec settings to use.
webrtc::VideoCodec codec_settings;
@ -118,6 +119,9 @@ struct TestConfig {
// Custom checker that will be called for each frame.
const EncodedFrameChecker* encoded_frame_checker = nullptr;
// Print out frame level stats.
bool print_frame_level_stats = false;
};
} // namespace test

75
modules/video_coding/codecs/test/test_config_unittest.cc
View file

@ -19,25 +19,25 @@ namespace webrtc {
namespace test {
namespace {
const int kNumTemporalLayers = 2;
const size_t kNumTemporalLayers = 2;
} // namespace
TEST(TestConfig, NumberOfCoresWithUseSingleCore) {
TestConfig config;
config.use_single_core = true;
EXPECT_EQ(1, config.NumberOfCores());
EXPECT_EQ(1u, config.NumberOfCores());
}
TEST(TestConfig, NumberOfCoresWithoutUseSingleCore) {
TestConfig config;
config.use_single_core = false;
EXPECT_GE(config.NumberOfCores(), 1);
EXPECT_GE(config.NumberOfCores(), 1u);
}
TEST(TestConfig, NumberOfTemporalLayersIsOne) {
TestConfig config;
webrtc::test::CodecSettings(kVideoCodecH264, &config.codec_settings);
EXPECT_EQ(1, config.NumberOfTemporalLayers());
EXPECT_EQ(1u, config.NumberOfTemporalLayers());
}
TEST(TestConfig, NumberOfTemporalLayers_Vp8) {
@ -58,33 +58,33 @@ TEST(TestConfig, TemporalLayersForFrame_OneLayer) {
TestConfig config;
webrtc::test::CodecSettings(kVideoCodecVP8, &config.codec_settings);
config.codec_settings.VP8()->numberOfTemporalLayers = 1;
EXPECT_EQ(0, config.TemporalLayerForFrame(0));
EXPECT_EQ(0, config.TemporalLayerForFrame(1));
EXPECT_EQ(0, config.TemporalLayerForFrame(2));
EXPECT_EQ(0u, config.TemporalLayerForFrame(0));
EXPECT_EQ(0u, config.TemporalLayerForFrame(1));
EXPECT_EQ(0u, config.TemporalLayerForFrame(2));
}
TEST(TestConfig, TemporalLayersForFrame_TwoLayers) {
TestConfig config;
webrtc::test::CodecSettings(kVideoCodecVP8, &config.codec_settings);
config.codec_settings.VP8()->numberOfTemporalLayers = 2;
EXPECT_EQ(0, config.TemporalLayerForFrame(0));
EXPECT_EQ(1, config.TemporalLayerForFrame(1));
EXPECT_EQ(0, config.TemporalLayerForFrame(2));
EXPECT_EQ(1, config.TemporalLayerForFrame(3));
EXPECT_EQ(0u, config.TemporalLayerForFrame(0));
EXPECT_EQ(1u, config.TemporalLayerForFrame(1));
EXPECT_EQ(0u, config.TemporalLayerForFrame(2));
EXPECT_EQ(1u, config.TemporalLayerForFrame(3));
}
TEST(TestConfig, TemporalLayersForFrame_ThreeLayers) {
TestConfig config;
webrtc::test::CodecSettings(kVideoCodecVP8, &config.codec_settings);
config.codec_settings.VP8()->numberOfTemporalLayers = 3;
EXPECT_EQ(0, config.TemporalLayerForFrame(0));
EXPECT_EQ(2, config.TemporalLayerForFrame(1));
EXPECT_EQ(1, config.TemporalLayerForFrame(2));
EXPECT_EQ(2, config.TemporalLayerForFrame(3));
EXPECT_EQ(0, config.TemporalLayerForFrame(4));
EXPECT_EQ(2, config.TemporalLayerForFrame(5));
EXPECT_EQ(1, config.TemporalLayerForFrame(6));
EXPECT_EQ(2, config.TemporalLayerForFrame(7));
EXPECT_EQ(0u, config.TemporalLayerForFrame(0));
EXPECT_EQ(2u, config.TemporalLayerForFrame(1));
EXPECT_EQ(1u, config.TemporalLayerForFrame(2));
EXPECT_EQ(2u, config.TemporalLayerForFrame(3));
EXPECT_EQ(0u, config.TemporalLayerForFrame(4));
EXPECT_EQ(2u, config.TemporalLayerForFrame(5));
EXPECT_EQ(1u, config.TemporalLayerForFrame(6));
EXPECT_EQ(2u, config.TemporalLayerForFrame(7));
}
TEST(TestConfig, ForcedKeyFrameIntervalOff) {
@ -126,26 +126,27 @@ TEST(TestConfig, ToString_Vp8) {
config.codec_settings.VP8()->keyFrameInterval = 999;
EXPECT_EQ(
"\n Filename : yuvfile"
"\n # CPU cores used : 1"
"\n Filename : yuvfile"
"\n # CPU cores used : 1"
"\n General:"
"\n Codec type : VP8"
"\n Start bitrate : 400 kbps"
"\n Max bitrate : 500 kbps"
"\n Min bitrate : 70 kbps"
"\n Width : 320"
"\n Height : 180"
"\n Max frame rate : 35"
"\n QPmax : 66"
"\n Codec type : VP8"
"\n Start bitrate : 400 kbps"
"\n Max bitrate : 500 kbps"
"\n Min bitrate : 70 kbps"
"\n Width : 320"
"\n Height : 180"
"\n Max frame rate : 35"
"\n QPmax : 66"
"\n # simulcast streams : 0"
"\n VP8 specific: "
"\n Complexity : 0"
"\n Resilience : 0"
"\n # temporal layers : 2"
"\n Denoising : 0"
"\n Error concealment : 1"
"\n Automatic resize : 1"
"\n Frame dropping : 0"
"\n Key frame interval: 999\n",
"\n Complexity : 0"
"\n Resilience : 0"
"\n # temporal layers : 2"
"\n Denoising : 0"
"\n Error concealment : 1"
"\n Automatic resize : 1"
"\n Frame dropping : 0"
"\n Key frame interval : 999\n",
config.ToString());
}

121
modules/video_coding/codecs/test/videoprocessor.cc
View file

@ -17,6 +17,7 @@
#include "api/video/i420_buffer.h"
#include "common_types.h" // NOLINT(build/include)
#include "common_video/h264/h264_common.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/video_coding/codecs/vp8/simulcast_rate_allocator.h"
#include "modules/video_coding/include/video_codec_initializer.h"
#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
@ -29,8 +30,6 @@ namespace test {
namespace {
const int kRtpClockRateHz = 90000;
std::unique_ptr<VideoBitrateAllocator> CreateBitrateAllocator(
TestConfig* config) {
std::unique_ptr<TemporalLayersFactory> tl_factory;
@ -43,10 +42,10 @@ std::unique_ptr<VideoBitrateAllocator> CreateBitrateAllocator(
std::move(tl_factory)));
}
rtc::Optional<size_t> GetMaxNaluLength(const EncodedImage& encoded_frame,
const TestConfig& config) {
size_t GetMaxNaluSizeBytes(const EncodedImage& encoded_frame,
const TestConfig& config) {
if (config.codec_settings.codecType != kVideoCodecH264)
return rtc::nullopt;
return 0;
std::vector<webrtc::H264::NaluIndex> nalu_indices =
webrtc::H264::FindNaluIndices(encoded_frame._buffer,
@ -54,11 +53,11 @@ rtc::Optional<size_t> GetMaxNaluLength(const EncodedImage& encoded_frame,
RTC_CHECK(!nalu_indices.empty());
size_t max_length = 0;
size_t max_size = 0;
for (const webrtc::H264::NaluIndex& index : nalu_indices)
max_length = std::max(max_length, index.payload_size);
max_size = std::max(max_size, index.payload_size);
return max_length;
return max_size;
}
int GetElapsedTimeMicroseconds(int64_t start_ns, int64_t stop_ns) {
@ -113,13 +112,14 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder,
analysis_frame_reader_(analysis_frame_reader),
encoded_frame_writer_(encoded_frame_writer),
decoded_frame_writer_(decoded_frame_writer),
last_inputed_frame_num_(-1),
last_encoded_frame_num_(-1),
last_decoded_frame_num_(-1),
last_inputed_frame_num_(0),
last_encoded_frame_num_(0),
last_decoded_frame_num_(0),
num_encoded_frames_(0),
num_decoded_frames_(0),
first_key_frame_has_been_excluded_(false),
last_decoded_frame_buffer_(analysis_frame_reader->FrameLength()),
stats_(stats),
rate_update_index_(-1) {
stats_(stats) {
RTC_DCHECK(encoder);
RTC_DCHECK(decoder);
RTC_DCHECK(packet_manipulator);
@ -134,12 +134,13 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder,
// Initialize the encoder and decoder.
RTC_CHECK_EQ(
encoder_->InitEncode(&config_.codec_settings, config_.NumberOfCores(),
encoder_->InitEncode(&config_.codec_settings,
static_cast<int>(config_.NumberOfCores()),
config_.networking_config.max_payload_size_in_bytes),
WEBRTC_VIDEO_CODEC_OK);
RTC_CHECK_EQ(
decoder_->InitDecode(&config_.codec_settings, config_.NumberOfCores()),
WEBRTC_VIDEO_CODEC_OK);
RTC_CHECK_EQ(decoder_->InitDecode(&config_.codec_settings,
static_cast<int>(config_.NumberOfCores())),
WEBRTC_VIDEO_CODEC_OK);
}
VideoProcessor::~VideoProcessor() {
@ -154,7 +155,7 @@ VideoProcessor::~VideoProcessor() {
void VideoProcessor::ProcessFrame() {
RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
const int frame_number = ++last_inputed_frame_num_;
const size_t frame_number = last_inputed_frame_num_++;
// Get frame from file.
rtc::scoped_refptr<I420BufferInterface> buffer(
@ -163,18 +164,20 @@ void VideoProcessor::ProcessFrame() {
// Use the frame number as the basis for timestamp to identify frames. Let the
// first timestamp be non-zero, to not make the IvfFileWriter believe that we
// want to use capture timestamps in the IVF files.
const uint32_t rtp_timestamp = (frame_number + 1) * kRtpClockRateHz /
config_.codec_settings.maxFramerate;
const size_t rtp_timestamp = (frame_number + 1) * kVideoPayloadTypeFrequency /
config_.codec_settings.maxFramerate;
const int64_t render_time_ms = (frame_number + 1) * rtc::kNumMillisecsPerSec /
config_.codec_settings.maxFramerate;
rtp_timestamp_to_frame_num_[rtp_timestamp] = frame_number;
input_frames_[frame_number] = rtc::MakeUnique<VideoFrame>(
buffer, rtp_timestamp, render_time_ms, webrtc::kVideoRotation_0);
input_frames_[frame_number] =
rtc::MakeUnique<VideoFrame>(buffer, static_cast<uint32_t>(rtp_timestamp),
render_time_ms, webrtc::kVideoRotation_0);
std::vector<FrameType> frame_types = config_.FrameTypeForFrame(frame_number);
// Create frame statistics object used for aggregation at end of test run.
FrameStatistic* frame_stat = stats_->AddFrame();
frame_stat->rtp_timestamp = rtp_timestamp;
// For the highest measurement accuracy of the encode time, the start/stop
// time recordings should wrap the Encode call as tightly as possible.
@ -183,27 +186,16 @@ void VideoProcessor::ProcessFrame() {
encoder_->Encode(*input_frames_[frame_number], nullptr, &frame_types);
}
void VideoProcessor::SetRates(int bitrate_kbps, int framerate_fps) {
void VideoProcessor::SetRates(size_t bitrate_kbps, size_t framerate_fps) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
config_.codec_settings.maxFramerate = framerate_fps;
int set_rates_result = encoder_->SetRateAllocation(
bitrate_allocator_->GetAllocation(bitrate_kbps * 1000, framerate_fps),
framerate_fps);
config_.codec_settings.maxFramerate = static_cast<uint32_t>(framerate_fps);
bitrate_allocation_ = bitrate_allocator_->GetAllocation(
static_cast<uint32_t>(bitrate_kbps * 1000),
static_cast<uint32_t>(framerate_fps));
const int set_rates_result = encoder_->SetRateAllocation(
bitrate_allocation_, static_cast<uint32_t>(framerate_fps));
RTC_DCHECK_GE(set_rates_result, 0)
<< "Failed to update encoder with new rate " << bitrate_kbps << ".";
++rate_update_index_;
num_dropped_frames_.push_back(0);
num_spatial_resizes_.push_back(0);
}
std::vector<int> VideoProcessor::NumberDroppedFramesPerRateUpdate() const {
RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
return num_dropped_frames_;
}
std::vector<int> VideoProcessor::NumberSpatialResizesPerRateUpdate() const {
RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
return num_spatial_resizes_;
}
void VideoProcessor::FrameEncoded(webrtc::VideoCodecType codec,
@ -218,20 +210,17 @@ void VideoProcessor::FrameEncoded(webrtc::VideoCodecType codec,
config_.encoded_frame_checker->CheckEncodedFrame(codec, encoded_image);
}
const int frame_number =
const size_t frame_number =
rtp_timestamp_to_frame_num_[encoded_image._timeStamp];
// Ensure strict monotonicity.
RTC_CHECK_GT(frame_number, last_encoded_frame_num_);
if (num_encoded_frames_ > 0) {
RTC_CHECK_GT(frame_number, last_encoded_frame_num_);
}
// Check for dropped frames.
bool last_frame_missing = false;
if (frame_number > 0) {
int num_dropped_from_last_encode =
frame_number - last_encoded_frame_num_ - 1;
RTC_DCHECK_GE(num_dropped_from_last_encode, 0);
RTC_CHECK_GE(rate_update_index_, 0);
num_dropped_frames_[rate_update_index_] += num_dropped_from_last_encode;
const FrameStatistic* last_encoded_frame_stat =
stats_->GetFrame(last_encoded_frame_num_);
last_frame_missing = (last_encoded_frame_stat->manipulated_length == 0);
@ -245,13 +234,14 @@ void VideoProcessor::FrameEncoded(webrtc::VideoCodecType codec,
frame_stat->encoding_successful = true;
frame_stat->encoded_frame_size_bytes = encoded_image._length;
frame_stat->frame_type = encoded_image._frameType;
frame_stat->temporal_layer_idx = config_.TemporalLayerForFrame(frame_number);
frame_stat->qp = encoded_image.qp_;
frame_stat->bitrate_kbps = static_cast<int>(
encoded_image._length * config_.codec_settings.maxFramerate * 8 / 1000);
frame_stat->target_bitrate_kbps =
bitrate_allocation_.GetSpatialLayerSum(0) / 1000;
frame_stat->total_packets =
encoded_image._length / config_.networking_config.packet_size_in_bytes +
1;
frame_stat->max_nalu_length = GetMaxNaluLength(encoded_image, config_);
frame_stat->max_nalu_size_bytes = GetMaxNaluSizeBytes(encoded_image, config_);
// Make a raw copy of |encoded_image| to feed to the decoder.
size_t copied_buffer_size = encoded_image._length +
@ -278,6 +268,8 @@ void VideoProcessor::FrameEncoded(webrtc::VideoCodecType codec,
if (encoded_frame_writer_) {
RTC_CHECK(encoded_frame_writer_->WriteFrame(encoded_image, codec));
}
++num_encoded_frames_;
}
void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame) {
@ -288,7 +280,7 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame) {
int64_t decode_stop_ns = rtc::TimeNanos();
// Update frame statistics.
const int frame_number =
const size_t frame_number =
rtp_timestamp_to_frame_num_[decoded_frame.timestamp()];
FrameStatistic* frame_stat = stats_->GetFrame(frame_number);
frame_stat->decoded_width = decoded_frame.width();
@ -298,26 +290,21 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame) {
frame_stat->decoding_successful = true;
// Ensure strict monotonicity.
RTC_CHECK_GT(frame_number, last_decoded_frame_num_);
if (num_decoded_frames_ > 0) {
RTC_CHECK_GT(frame_number, last_decoded_frame_num_);
}
// Check if the codecs have resized the frame since previously decoded frame.
if (frame_number > 0) {
if (decoded_frame_writer_ && last_decoded_frame_num_ >= 0) {
if (decoded_frame_writer_ && num_decoded_frames_ > 0) {
// For dropped/lost frames, write out the last decoded frame to make it
// look like a freeze at playback.
const int num_dropped_frames = frame_number - last_decoded_frame_num_;
for (int i = 0; i < num_dropped_frames; i++) {
const size_t num_dropped_frames =
frame_number - last_decoded_frame_num_ - 1;
for (size_t i = 0; i < num_dropped_frames; i++) {
WriteDecodedFrameToFile(&last_decoded_frame_buffer_);
}
}
// TODO(ssilkin): move to FrameEncoded when webm:1474 is implemented.
const FrameStatistic* last_decoded_frame_stat =
stats_->GetFrame(last_decoded_frame_num_);
if (decoded_frame.width() != last_decoded_frame_stat->decoded_width ||
decoded_frame.height() != last_decoded_frame_stat->decoded_height) {
RTC_CHECK_GE(rate_update_index_, 0);
++num_spatial_resizes_[rate_update_index_];
}
}
last_decoded_frame_num_ = frame_number;
@ -331,10 +318,8 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame) {
// Delay erasing of input frames by one frame. The current frame might
// still be needed for other simulcast stream or spatial layer.
const int frame_number_to_erase = frame_number - 1;
if (frame_number_to_erase >= 0) {
auto input_frame_erase_to =
input_frames_.lower_bound(frame_number_to_erase);
if (frame_number > 0) {
auto input_frame_erase_to = input_frames_.lower_bound(frame_number - 1);
input_frames_.erase(input_frames_.begin(), input_frame_erase_to);
}
@ -344,6 +329,8 @@ void VideoProcessor::FrameDecoded(const VideoFrame& decoded_frame) {
&last_decoded_frame_buffer_);
WriteDecodedFrameToFile(&last_decoded_frame_buffer_);
}
++num_decoded_frames_;
}
void VideoProcessor::WriteDecodedFrameToFile(rtc::Buffer* buffer) {

24
modules/video_coding/codecs/test/videoprocessor.h
View file

@ -76,13 +76,7 @@ class VideoProcessor {
void ProcessFrame();
// Updates the encoder with target rates. Must be called at least once.
void SetRates(int bitrate_kbps, int framerate_fps);
// Returns the number of dropped frames.
std::vector<int> NumberDroppedFramesPerRateUpdate() const;
// Returns the number of spatial resizes.
std::vector<int> NumberSpatialResizesPerRateUpdate() const;
void SetRates(size_t bitrate_kbps, size_t framerate_fps);
private:
class VideoProcessorEncodeCompleteCallback
@ -190,6 +184,7 @@ class VideoProcessor {
webrtc::VideoEncoder* const encoder_;
webrtc::VideoDecoder* const decoder_;
const std::unique_ptr<VideoBitrateAllocator> bitrate_allocator_;
BitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(sequence_checker_);
// Adapters for the codec callbacks.
VideoProcessorEncodeCompleteCallback encode_callback_;
@ -202,7 +197,7 @@ class VideoProcessor {
// Async codecs might queue frames. To handle that we keep input frame
// and release it after corresponding coded frame is decoded and quality
// measurement is done.
std::map<int, std::unique_ptr<VideoFrame>> input_frames_
std::map<size_t, std::unique_ptr<VideoFrame>> input_frames_
RTC_GUARDED_BY(sequence_checker_);
// These (mandatory) file manipulators are used for, e.g., objective PSNR and
@ -217,13 +212,15 @@ class VideoProcessor {
FrameWriter* const decoded_frame_writer_;
// Keep track of inputed/encoded/decoded frames, so we can detect frame drops.
int last_inputed_frame_num_ RTC_GUARDED_BY(sequence_checker_);
int last_encoded_frame_num_ RTC_GUARDED_BY(sequence_checker_);
int last_decoded_frame_num_ RTC_GUARDED_BY(sequence_checker_);
size_t last_inputed_frame_num_ RTC_GUARDED_BY(sequence_checker_);
size_t last_encoded_frame_num_ RTC_GUARDED_BY(sequence_checker_);
size_t last_decoded_frame_num_ RTC_GUARDED_BY(sequence_checker_);
size_t num_encoded_frames_ RTC_GUARDED_BY(sequence_checker_);
size_t num_decoded_frames_ RTC_GUARDED_BY(sequence_checker_);
// Store an RTP timestamp -> frame number map, since the timestamps are
// based off of the frame rate, which can change mid-test.
std::map<uint32_t, int> rtp_timestamp_to_frame_num_
std::map<size_t, size_t> rtp_timestamp_to_frame_num_
RTC_GUARDED_BY(sequence_checker_);
// Keep track of if we have excluded the first key frame from packet loss.
@ -235,9 +232,6 @@ class VideoProcessor {
// Statistics.
Stats* stats_;
std::vector<int> num_dropped_frames_ RTC_GUARDED_BY(sequence_checker_);
std::vector<int> num_spatial_resizes_ RTC_GUARDED_BY(sequence_checker_);
int rate_update_index_ RTC_GUARDED_BY(sequence_checker_);
rtc::SequencedTaskChecker sequence_checker_;

598
modules/video_coding/codecs/test/videoprocessor_integrationtest.cc
View file

@ -37,6 +37,7 @@
#include "rtc_base/file.h"
#include "rtc_base/ptr_util.h"
#include "system_wrappers/include/sleep.h"
#include "test/statistics.h"
#include "test/testsupport/fileutils.h"
#include "test/testsupport/metrics/video_metrics.h"
@ -45,12 +46,9 @@ namespace test {
namespace {
const int kMaxBitrateMismatchPercent = 20;
const int kRtpClockRateHz = 90000;
// Parameters from VP8 wrapper, which control target size of key frames.
const float kInitialBufferSize = 0.5f;
const float kOptimalBufferSize = 0.6f;
const float kScaleKeyFrameSize = 0.5f;
const int kMaxBitrateMismatchPercent = 20;
bool RunEncodeInRealTime(const TestConfig& config) {
if (config.measure_cpu) {
@ -173,118 +171,210 @@ VideoProcessorIntegrationTest::~VideoProcessorIntegrationTest() = default;
void VideoProcessorIntegrationTest::ProcessFramesAndMaybeVerify(
const std::vector<RateProfile>& rate_profiles,
const std::vector<RateControlThresholds>* rc_thresholds,
const QualityThresholds* quality_thresholds,
const std::vector<QualityThresholds>* quality_thresholds,
const BitstreamThresholds* bs_thresholds,
const VisualizationParams* visualization_params) {
RTC_DCHECK(!rate_profiles.empty());
// The Android HW codec needs to be run on a task queue, so we simply always
// run the test on a task queue.
rtc::TaskQueue task_queue("VidProc TQ");
rtc::Event sync_event(false, false);
SetUpAndInitObjects(&task_queue, rate_profiles[0].target_kbps,
rate_profiles[0].input_fps, visualization_params);
SetUpAndInitObjects(
&task_queue, static_cast<const int>(rate_profiles[0].target_kbps),
static_cast<const int>(rate_profiles[0].input_fps), visualization_params);
PrintSettings();
ProcessAllFrames(&task_queue, rate_profiles);
ReleaseAndCloseObjects(&task_queue);
AnalyzeAllFrames(rate_profiles, rc_thresholds, quality_thresholds,
bs_thresholds);
}
void VideoProcessorIntegrationTest::ProcessAllFrames(
rtc::TaskQueue* task_queue,
const std::vector<RateProfile>& rate_profiles) {
// Process all frames.
size_t rate_update_index = 0;
// Set initial rates.
int rate_update_index = 0;
task_queue.PostTask([this, &rate_profiles, rate_update_index] {
task_queue->PostTask([this, &rate_profiles, rate_update_index] {
processor_->SetRates(rate_profiles[rate_update_index].target_kbps,
rate_profiles[rate_update_index].input_fps);
});
cpu_process_time_->Start();
// Process all frames.
int frame_number = 0;
const int num_frames = config_.num_frames;
RTC_DCHECK_GE(num_frames, 1);
while (frame_number < num_frames) {
if (RunEncodeInRealTime(config_)) {
// Roughly pace the frames.
SleepMs(rtc::kNumMillisecsPerSec /
rate_profiles[rate_update_index].input_fps);
}
task_queue.PostTask([this] { processor_->ProcessFrame(); });
++frame_number;
for (size_t frame_number = 0; frame_number < config_.num_frames;
++frame_number) {
if (frame_number ==
rate_profiles[rate_update_index].frame_index_rate_update) {
++rate_update_index;
RTC_DCHECK_GT(rate_profiles.size(), rate_update_index);
task_queue.PostTask([this, &rate_profiles, rate_update_index] {
task_queue->PostTask([this, &rate_profiles, rate_update_index] {
processor_->SetRates(rate_profiles[rate_update_index].target_kbps,
rate_profiles[rate_update_index].input_fps);
});
}
task_queue->PostTask([this] { processor_->ProcessFrame(); });
if (RunEncodeInRealTime(config_)) {
// Roughly pace the frames.
size_t frame_duration_ms =
rtc::kNumMillisecsPerSec / rate_profiles[rate_update_index].input_fps;
SleepMs(static_cast<int>(frame_duration_ms));
}
}
rtc::Event sync_event(false, false);
task_queue->PostTask([&sync_event] { sync_event.Set(); });
sync_event.Wait(rtc::Event::kForever);
// Give the VideoProcessor pipeline some time to process the last frame,
// and then release the codecs.
if (config_.hw_encoder || config_.hw_decoder) {
SleepMs(1 * rtc::kNumMillisecsPerSec);
}
cpu_process_time_->Stop();
}
std::vector<int> num_dropped_frames;
std::vector<int> num_spatial_resizes;
sync_event.Reset();
task_queue.PostTask(
[this, &num_dropped_frames, &num_spatial_resizes, &sync_event]() {
num_dropped_frames = processor_->NumberDroppedFramesPerRateUpdate();
num_spatial_resizes = processor_->NumberSpatialResizesPerRateUpdate();
sync_event.Set();
});
sync_event.Wait(rtc::Event::kForever);
void VideoProcessorIntegrationTest::AnalyzeAllFrames(
const std::vector<RateProfile>& rate_profiles,
const std::vector<RateControlThresholds>* rc_thresholds,
const std::vector<QualityThresholds>* quality_thresholds,
const BitstreamThresholds* bs_thresholds) {
const bool is_svc = config_.NumberOfSpatialLayers() > 1;
const size_t number_of_simulcast_or_spatial_layers =
std::max(std::size_t{1},
std::max(config_.NumberOfSpatialLayers(),
static_cast<size_t>(
config_.codec_settings.numberOfSimulcastStreams)));
const size_t number_of_temporal_layers = config_.NumberOfTemporalLayers();
printf("Rate control statistics\n==\n");
for (size_t rate_update_index = 0; rate_update_index < rate_profiles.size();
++rate_update_index) {
const size_t first_frame_number =
(rate_update_index == 0)
? 0
: rate_profiles[rate_update_index - 1].frame_index_rate_update;
const size_t last_frame_number =
rate_profiles[rate_update_index].frame_index_rate_update - 1;
RTC_CHECK(last_frame_number >= first_frame_number);
const size_t number_of_frames = last_frame_number - first_frame_number + 1;
const float input_duration_sec =
1.0 * number_of_frames / rate_profiles[rate_update_index].input_fps;
ReleaseAndCloseObjects(&task_queue);
std::vector<FrameStatistic> overall_stats =
ExtractLayerStats(number_of_simulcast_or_spatial_layers - 1,
number_of_temporal_layers - 1, first_frame_number,
last_frame_number, true);
// Calculate and print rate control statistics.
rate_update_index = 0;
frame_number = 0;
quality_ = QualityMetrics();
ResetRateControlMetrics(rate_update_index, rate_profiles);
while (frame_number < num_frames) {
UpdateRateControlMetrics(frame_number);
printf("Rate update #%zu:\n", rate_update_index);
if (quality_thresholds) {
UpdateQualityMetrics(frame_number);
const RateControlThresholds* rc_threshold =
rc_thresholds ? &(*rc_thresholds)[rate_update_index] : nullptr;
const QualityThresholds* quality_threshold =
quality_thresholds ? &(*quality_thresholds)[rate_update_index]
: nullptr;
AnalyzeAndPrintStats(
overall_stats, rate_profiles[rate_update_index].target_kbps,
rate_profiles[rate_update_index].input_fps, input_duration_sec,
rc_threshold, quality_threshold, bs_thresholds);
if (config_.print_frame_level_stats) {
PrintFrameLevelStats(overall_stats);
}
if (bs_thresholds) {
VerifyBitstream(frame_number, *bs_thresholds);
}
for (size_t spatial_layer_number = 0;
spatial_layer_number < number_of_simulcast_or_spatial_layers;
++spatial_layer_number) {
for (size_t temporal_layer_number = 0;
temporal_layer_number < number_of_temporal_layers;
++temporal_layer_number) {
std::vector<FrameStatistic> layer_stats =
ExtractLayerStats(spatial_layer_number, temporal_layer_number,
first_frame_number, last_frame_number, is_svc);
++frame_number;
const size_t target_bitrate_kbps = layer_stats[0].target_bitrate_kbps;
const float target_framerate_fps =
1.0 * rate_profiles[rate_update_index].input_fps /
(1 << (number_of_temporal_layers - temporal_layer_number - 1));
if (frame_number ==
rate_profiles[rate_update_index].frame_index_rate_update) {
PrintRateControlMetrics(rate_update_index, num_dropped_frames,
num_spatial_resizes);
VerifyRateControlMetrics(rate_update_index, rc_thresholds,
num_dropped_frames, num_spatial_resizes);
++rate_update_index;
ResetRateControlMetrics(rate_update_index, rate_profiles);
printf("Spatial %zu temporal %zu:\n", spatial_layer_number,
temporal_layer_number);
AnalyzeAndPrintStats(layer_stats, target_bitrate_kbps,
target_framerate_fps, input_duration_sec, nullptr,
nullptr, nullptr);
if (config_.print_frame_level_stats) {
PrintFrameLevelStats(layer_stats);
}
}
}
}
PrintRateControlMetrics(rate_update_index, num_dropped_frames,
num_spatial_resizes);
VerifyRateControlMetrics(rate_update_index, rc_thresholds, num_dropped_frames,
num_spatial_resizes);
if (quality_thresholds) {
VerifyQualityMetrics(*quality_thresholds);
}
// Calculate and print other statistics.
EXPECT_EQ(num_frames, static_cast<int>(stats_.size()));
stats_.PrintSummary();
cpu_process_time_->Print();
}
std::vector<FrameStatistic> VideoProcessorIntegrationTest::ExtractLayerStats(
size_t target_spatial_layer_number,
size_t target_temporal_layer_number,
size_t first_frame_number,
size_t last_frame_number,
bool combine_layers_stats) {
size_t target_bitrate_kbps = 0;
std::vector<FrameStatistic> layer_stats;
for (size_t frame_number = first_frame_number;
frame_number <= last_frame_number; ++frame_number) {
// TODO(ssilkin): Add layering support
// FrameStatistic superframe_stat =
// *stats_[target_spatial_layer_number].GetFrame(frame_number);
FrameStatistic superframe_stat = *stats_.GetFrame(frame_number);
const size_t tl_idx = superframe_stat.temporal_layer_idx;
if (tl_idx <= target_temporal_layer_number) {
if (combine_layers_stats) {
for (size_t spatial_layer_number = 0;
spatial_layer_number < target_spatial_layer_number;
++spatial_layer_number) {
// TODO(ssilkin): Add layering support
// const FrameStatistic* frame_stat =
// stats_[spatial_layer_number].GetFrame(frame_number);
const FrameStatistic* frame_stat = stats_.GetFrame(frame_number);
superframe_stat.encoded_frame_size_bytes +=
frame_stat->encoded_frame_size_bytes;
superframe_stat.encode_time_us = std::max(
superframe_stat.encode_time_us, frame_stat->encode_time_us);
superframe_stat.decode_time_us = std::max(
superframe_stat.decode_time_us, frame_stat->decode_time_us);
}
}
target_bitrate_kbps =
std::max(target_bitrate_kbps, superframe_stat.target_bitrate_kbps);
if (superframe_stat.encoding_successful) {
RTC_CHECK(superframe_stat.target_bitrate_kbps <= target_bitrate_kbps ||
tl_idx == target_temporal_layer_number);
RTC_CHECK(superframe_stat.target_bitrate_kbps == target_bitrate_kbps ||
tl_idx < target_temporal_layer_number);
}
layer_stats.push_back(superframe_stat);
}
}
for (auto& frame_stat : layer_stats) {
frame_stat.target_bitrate_kbps = target_bitrate_kbps;
}
return layer_stats;
}
void VideoProcessorIntegrationTest::CreateEncoderAndDecoder() {
std::unique_ptr<VideoEncoderFactory> encoder_factory;
if (config_.hw_encoder) {
@ -436,139 +526,10 @@ void VideoProcessorIntegrationTest::ReleaseAndCloseObjects(
}
}
// For every encoded frame, update the rate control metrics.
void VideoProcessorIntegrationTest::UpdateRateControlMetrics(int frame_number) {
RTC_CHECK_GE(frame_number, 0);
const int tl_idx = config_.TemporalLayerForFrame(frame_number);
++actual_.num_frames_layer[tl_idx];
++actual_.num_frames;
const FrameStatistic* frame_stat = stats_.GetFrame(frame_number);
FrameType frame_type = frame_stat->frame_type;
float framesize_kbits = frame_stat->encoded_frame_size_bytes * 8.0f / 1000.0f;
// Update rate mismatch relative to per-frame bandwidth.
if (frame_type == kVideoFrameDelta) {
// TODO(marpan): Should we count dropped (zero size) frames in mismatch?
actual_.sum_delta_framesize_mismatch_layer[tl_idx] +=
fabs(framesize_kbits - target_.framesize_kbits_layer[tl_idx]) /
target_.framesize_kbits_layer[tl_idx];
} else {
float key_framesize_kbits = (frame_number == 0)
? target_.key_framesize_kbits_initial
: target_.key_framesize_kbits;
actual_.sum_key_framesize_mismatch +=
fabs(framesize_kbits - key_framesize_kbits) / key_framesize_kbits;
++actual_.num_key_frames;
}
actual_.sum_framesize_kbits += framesize_kbits;
actual_.sum_framesize_kbits_layer[tl_idx] += framesize_kbits;
// Encoded bitrate: from the start of the update/run to current frame.
actual_.kbps = actual_.sum_framesize_kbits * target_.fps / actual_.num_frames;
actual_.kbps_layer[tl_idx] = actual_.sum_framesize_kbits_layer[tl_idx] *
target_.fps_layer[tl_idx] /
actual_.num_frames_layer[tl_idx];
// Number of frames to hit target bitrate.
if (actual_.BitrateMismatchPercent(target_.kbps) <
kMaxBitrateMismatchPercent) {
actual_.num_frames_to_hit_target =
std::min(actual_.num_frames, actual_.num_frames_to_hit_target);
}
}
// Verify expected behavior of rate control.
void VideoProcessorIntegrationTest::VerifyRateControlMetrics(
int rate_update_index,
const std::vector<RateControlThresholds>* rc_thresholds,
const std::vector<int>& num_dropped_frames,
const std::vector<int>& num_spatial_resizes) const {
if (!rc_thresholds)
return;
const RateControlThresholds& rc_threshold =
(*rc_thresholds)[rate_update_index];
EXPECT_LE(num_dropped_frames[rate_update_index],
rc_threshold.max_num_dropped_frames);
EXPECT_EQ(rc_threshold.num_spatial_resizes,
num_spatial_resizes[rate_update_index]);
EXPECT_LE(actual_.num_frames_to_hit_target,
rc_threshold.max_num_frames_to_hit_target);
EXPECT_EQ(rc_threshold.num_key_frames, actual_.num_key_frames);
EXPECT_LE(actual_.KeyFrameSizeMismatchPercent(),
rc_threshold.max_key_framesize_mismatch_percent);
EXPECT_LE(actual_.BitrateMismatchPercent(target_.kbps),
rc_threshold.max_bitrate_mismatch_percent);
const int num_temporal_layers = config_.NumberOfTemporalLayers();
for (int i = 0; i < num_temporal_layers; ++i) {
EXPECT_LE(actual_.DeltaFrameSizeMismatchPercent(i),
rc_threshold.max_delta_framesize_mismatch_percent);
EXPECT_LE(actual_.BitrateMismatchPercent(i, target_.kbps_layer[i]),
rc_threshold.max_bitrate_mismatch_percent);
}
}
void VideoProcessorIntegrationTest::UpdateQualityMetrics(int frame_number) {
FrameStatistic* frame_stat = stats_.GetFrame(frame_number);
if (frame_stat->decoding_successful) {
++quality_.num_decoded_frames;
quality_.total_psnr += frame_stat->psnr;
quality_.total_ssim += frame_stat->ssim;
if (frame_stat->psnr < quality_.min_psnr)
quality_.min_psnr = frame_stat->psnr;
if (frame_stat->ssim < quality_.min_ssim)
quality_.min_ssim = frame_stat->ssim;
}
}
void VideoProcessorIntegrationTest::PrintRateControlMetrics(
int rate_update_index,
const std::vector<int>& num_dropped_frames,
const std::vector<int>& num_spatial_resizes) const {
if (rate_update_index == 0) {
printf("Rate control statistics\n==\n");
}
printf("Rate update #%d:\n", rate_update_index);
printf(" Target bitrate : %d\n", target_.kbps);
printf(" Encoded bitrate : %f\n", actual_.kbps);
printf(" Frame rate : %d\n", target_.fps);
printf(" # processed frames : %d\n", actual_.num_frames);
printf(" # frames to convergence : %d\n", actual_.num_frames_to_hit_target);
printf(" # dropped frames : %d\n",
num_dropped_frames[rate_update_index]);
printf(" # spatial resizes : %d\n",
num_spatial_resizes[rate_update_index]);
printf(" # key frames : %d\n", actual_.num_key_frames);
printf(" Key frame rate mismatch : %d\n",
actual_.KeyFrameSizeMismatchPercent());
const int num_temporal_layers = config_.NumberOfTemporalLayers();
for (int i = 0; i < num_temporal_layers; ++i) {
printf(" Temporal layer #%d:\n", i);
printf(" TL%d target bitrate : %f\n", i, target_.kbps_layer[i]);
printf(" TL%d encoded bitrate : %f\n", i, actual_.kbps_layer[i]);
printf(" TL%d frame rate : %f\n", i, target_.fps_layer[i]);
printf(" TL%d # processed frames : %d\n", i,
actual_.num_frames_layer[i]);
printf(" TL%d frame size %% mismatch : %d\n", i,
actual_.DeltaFrameSizeMismatchPercent(i));
printf(" TL%d bitrate %% mismatch : %d\n", i,
actual_.BitrateMismatchPercent(i, target_.kbps_layer[i]));
printf(" TL%d per-frame bitrate : %f\n", i,
target_.framesize_kbits_layer[i]);
}
printf("\n");
}
void VideoProcessorIntegrationTest::PrintSettings() const {
printf("VideoProcessor settings\n==\n");
printf(" Total # of frames: %d", analysis_frame_reader_->NumberOfFrames());
printf(" Total # of frames : %d",
analysis_frame_reader_->NumberOfFrames());
printf("%s\n", config_.ToString().c_str());
printf("VideoProcessorIntegrationTest settings\n==\n");
@ -577,87 +538,192 @@ void VideoProcessorIntegrationTest::PrintSettings() const {
const char* decoder_name = decoder_->ImplementationName();
printf(" Decoder implementation name: %s\n", decoder_name);
if (strcmp(encoder_name, decoder_name) == 0) {
printf(" Codec implementation name : %s_%s\n", config_.CodecName().c_str(),
printf(" Codec implementation name : %s_%s\n", config_.CodecName().c_str(),
encoder_name);
}
printf("\n");
}
void VideoProcessorIntegrationTest::VerifyBitstream(
int frame_number,
const BitstreamThresholds& bs_thresholds) {
RTC_CHECK_GE(frame_number, 0);
const FrameStatistic* frame_stat = stats_.GetFrame(frame_number);
EXPECT_LE(*(frame_stat->max_nalu_length), bs_thresholds.max_nalu_length);
}
void VideoProcessorIntegrationTest::AnalyzeAndPrintStats(
const std::vector<FrameStatistic>& stats,
const float target_bitrate_kbps,
const float target_framerate_fps,
const float input_duration_sec,
const RateControlThresholds* rc_thresholds,
const QualityThresholds* quality_thresholds,
const BitstreamThresholds* bs_thresholds) {
const size_t num_input_frames = stats.size();
size_t num_dropped_frames = 0;
size_t num_decoded_frames = 0;
size_t num_spatial_resizes = 0;
size_t num_key_frames = 0;
size_t max_nalu_size_bytes = 0;
void VideoProcessorIntegrationTest::VerifyQualityMetrics(
const QualityThresholds& quality_thresholds) {
EXPECT_GT(quality_.num_decoded_frames, 0);
EXPECT_GT(quality_.total_psnr / quality_.num_decoded_frames,
quality_thresholds.min_avg_psnr);
EXPECT_GT(quality_.min_psnr, quality_thresholds.min_min_psnr);
EXPECT_GT(quality_.total_ssim / quality_.num_decoded_frames,
quality_thresholds.min_avg_ssim);
EXPECT_GT(quality_.min_ssim, quality_thresholds.min_min_ssim);
}
size_t encoded_bytes = 0;
float buffer_level_kbits = 0.0;
float time_to_reach_target_bitrate_sec = -1.0;
// Reset quantities before each encoder rate update.
void VideoProcessorIntegrationTest::ResetRateControlMetrics(
int rate_update_index,
const std::vector<RateProfile>& rate_profiles) {
RTC_DCHECK_GT(rate_profiles.size(), rate_update_index);
// Set new rates.
target_.kbps = rate_profiles[rate_update_index].target_kbps;
target_.fps = rate_profiles[rate_update_index].input_fps;
SetRatesPerTemporalLayer();
Statistics buffer_level_sec;
Statistics key_frame_size_bytes;
Statistics delta_frame_size_bytes;
// Set key frame target sizes.
if (rate_update_index == 0) {
target_.key_framesize_kbits_initial =
0.5 * kInitialBufferSize * target_.kbps_layer[0];
}
Statistics encoding_time_us;
Statistics decoding_time_us;
Statistics psnr;
Statistics ssim;
// Set maximum size of key frames, following setting in the VP8 wrapper.
float max_key_size = kScaleKeyFrameSize * kOptimalBufferSize * target_.fps;
// We don't know exact target size of the key frames (except for first one),
// but the minimum in libvpx is ~|3 * per_frame_bandwidth| and maximum is
// set by |max_key_size_ * per_frame_bandwidth|. Take middle point/average
// as reference for mismatch. Note key frames always correspond to base
// layer frame in this test.
target_.key_framesize_kbits =
0.5 * (3 + max_key_size) * target_.framesize_kbits_layer[0];
Statistics qp;
// Reset rate control metrics.
actual_ = TestResults();
actual_.num_frames_to_hit_target = // Set to max number of frames.
rate_profiles[rate_update_index].frame_index_rate_update;
}
FrameStatistic last_successfully_decoded_frame(0);
for (size_t frame_idx = 0; frame_idx < stats.size(); ++frame_idx) {
const FrameStatistic& frame_stat = stats[frame_idx];
void VideoProcessorIntegrationTest::SetRatesPerTemporalLayer() {
const int num_temporal_layers = config_.NumberOfTemporalLayers();
RTC_DCHECK_LE(num_temporal_layers, kMaxNumTemporalLayers);
const float time_since_first_input_sec =
frame_idx == 0
? 0.0
: 1.0 * (frame_stat.rtp_timestamp - stats[0].rtp_timestamp) /
kRtpClockRateHz;
const float time_since_last_input_sec =
frame_idx == 0 ? 0.0
: 1.0 *
(frame_stat.rtp_timestamp -
stats[frame_idx - 1].rtp_timestamp) /
kRtpClockRateHz;
for (int i = 0; i < num_temporal_layers; ++i) {
float bitrate_ratio;
if (i > 0) {
bitrate_ratio = kVp8LayerRateAlloction[num_temporal_layers - 1][i] -
kVp8LayerRateAlloction[num_temporal_layers - 1][i - 1];
// Testing framework uses constant input framerate. This guarantees even
// sampling, which is important, of buffer level.
buffer_level_kbits -= time_since_last_input_sec * target_bitrate_kbps;
buffer_level_kbits = std::max(0.0f, buffer_level_kbits);
buffer_level_kbits += 8.0 * frame_stat.encoded_frame_size_bytes / 1000;
buffer_level_sec.AddSample(buffer_level_kbits / target_bitrate_kbps);
encoded_bytes += frame_stat.encoded_frame_size_bytes;
if (frame_stat.encoded_frame_size_bytes == 0) {
++num_dropped_frames;
} else {
bitrate_ratio = kVp8LayerRateAlloction[num_temporal_layers - 1][i];
if (frame_stat.frame_type == kVideoFrameKey) {
key_frame_size_bytes.AddSample(frame_stat.encoded_frame_size_bytes);
++num_key_frames;
} else {
delta_frame_size_bytes.AddSample(frame_stat.encoded_frame_size_bytes);
}
encoding_time_us.AddSample(frame_stat.encode_time_us);
qp.AddSample(frame_stat.qp);
max_nalu_size_bytes =
std::max(max_nalu_size_bytes, frame_stat.max_nalu_size_bytes);
}
if (frame_stat.decoding_successful) {
psnr.AddSample(frame_stat.psnr);
ssim.AddSample(frame_stat.ssim);
if (num_decoded_frames > 0) {
if (last_successfully_decoded_frame.decoded_width !=
frame_stat.decoded_width ||
last_successfully_decoded_frame.decoded_height !=
frame_stat.decoded_height) {
++num_spatial_resizes;
}
}
decoding_time_us.AddSample(frame_stat.decode_time_us);
last_successfully_decoded_frame = frame_stat;
++num_decoded_frames;
}
if (time_to_reach_target_bitrate_sec < 0 && frame_idx > 0) {
const float curr_bitrate_kbps =
(8.0 * encoded_bytes / 1000) / time_since_first_input_sec;
const float bitrate_mismatch_percent =
100 * std::fabs(curr_bitrate_kbps - target_bitrate_kbps) /
target_bitrate_kbps;
if (bitrate_mismatch_percent < kMaxBitrateMismatchPercent) {
time_to_reach_target_bitrate_sec = time_since_first_input_sec;
}
}
target_.kbps_layer[i] = target_.kbps * bitrate_ratio;
target_.fps_layer[i] =
target_.fps / static_cast<float>(1 << (num_temporal_layers - 1));
}
if (num_temporal_layers == 3) {
target_.fps_layer[2] = target_.fps / 2.0f;
}
// Update layer per-frame-bandwidth.
for (int i = 0; i < num_temporal_layers; ++i) {
target_.framesize_kbits_layer[i] =
target_.kbps_layer[i] / target_.fps_layer[i];
const float encoded_bitrate_kbps =
8 * encoded_bytes / input_duration_sec / 1000;
const float bitrate_mismatch_percent =
100 * std::fabs(encoded_bitrate_kbps - target_bitrate_kbps) /
target_bitrate_kbps;
const size_t num_encoded_frames = num_input_frames - num_dropped_frames;
const float encoded_framerate_fps = num_encoded_frames / input_duration_sec;
const float decoded_framerate_fps = num_decoded_frames / input_duration_sec;
const float framerate_mismatch_percent =
100 * std::fabs(decoded_framerate_fps - target_framerate_fps) /
target_framerate_fps;
const float max_key_frame_delay_sec =
8 * key_frame_size_bytes.Max() / 1000 / target_bitrate_kbps;
const float max_delta_frame_delay_sec =
8 * delta_frame_size_bytes.Max() / 1000 / target_bitrate_kbps;
printf("Target bitrate : %f kbps\n", target_bitrate_kbps);
printf("Encoded bitrate : %f kbps\n", encoded_bitrate_kbps);
printf("Bitrate mismatch : %f %%\n", bitrate_mismatch_percent);
printf("Time to reach target bitrate : %f sec\n",
time_to_reach_target_bitrate_sec);
printf("Target framerate : %f fps\n", target_framerate_fps);
printf("Encoding framerate : %f fps\n", encoded_framerate_fps);
printf("Decoding framerate : %f fps\n", decoded_framerate_fps);
printf("Frame encoding time : %f us\n", encoding_time_us.Mean());
printf("Frame decoding time : %f us\n", decoding_time_us.Mean());
printf("Framerate mismatch percent : %f %%\n",
framerate_mismatch_percent);
printf("Avg buffer level : %f sec\n", buffer_level_sec.Mean());
printf("Max key frame delay : %f sec\n", max_key_frame_delay_sec);
printf("Max delta frame delay : %f sec\n",
max_delta_frame_delay_sec);
printf("Avg key frame size : %f bytes\n",
key_frame_size_bytes.Mean());
printf("Avg delta frame size : %f bytes\n",
delta_frame_size_bytes.Mean());
printf("Avg QP : %f\n", qp.Mean());
printf("Avg PSNR : %f dB\n", psnr.Mean());
printf("Min PSNR : %f dB\n", psnr.Min());
printf("Avg SSIM : %f\n", ssim.Mean());
printf("Min SSIM : %f\n", ssim.Min());
printf("# input frames : %zu\n", num_input_frames);
printf("# encoded frames : %zu\n", num_encoded_frames);
printf("# decoded frames : %zu\n", num_decoded_frames);
printf("# dropped frames : %zu\n", num_dropped_frames);
printf("# key frames : %zu\n", num_key_frames);
printf("# encoded bytes : %zu\n", encoded_bytes);
printf("# spatial resizes : %zu\n", num_spatial_resizes);
if (rc_thresholds) {
EXPECT_LE(bitrate_mismatch_percent,
rc_thresholds->max_avg_bitrate_mismatch_percent);
EXPECT_LE(time_to_reach_target_bitrate_sec,
rc_thresholds->max_time_to_reach_target_bitrate_sec);
EXPECT_LE(framerate_mismatch_percent,
rc_thresholds->max_avg_framerate_mismatch_percent);
EXPECT_LE(buffer_level_sec.Mean(), rc_thresholds->max_avg_buffer_level_sec);
EXPECT_LE(max_key_frame_delay_sec,
rc_thresholds->max_max_key_frame_delay_sec);
EXPECT_LE(max_delta_frame_delay_sec,
rc_thresholds->max_max_delta_frame_delay_sec);
EXPECT_LE(num_spatial_resizes, rc_thresholds->max_num_spatial_resizes);
EXPECT_LE(num_key_frames, rc_thresholds->max_num_key_frames);
}
if (quality_thresholds) {
EXPECT_GT(psnr.Mean(), quality_thresholds->min_avg_psnr);
EXPECT_GT(psnr.Min(), quality_thresholds->min_min_psnr);
EXPECT_GT(ssim.Mean(), quality_thresholds->min_avg_ssim);
EXPECT_GT(ssim.Min(), quality_thresholds->min_min_ssim);
}
if (bs_thresholds) {
EXPECT_LE(max_nalu_size_bytes, bs_thresholds->max_max_nalu_size_bytes);
}
}
void VideoProcessorIntegrationTest::PrintFrameLevelStats(
const std::vector<FrameStatistic>& stats) const {
for (auto& frame_stat : stats) {
printf("%s\n", frame_stat.ToString().c_str());
}
}

147
modules/video_coding/codecs/test/videoprocessor_integrationtest.h
View file

@ -36,35 +36,24 @@ namespace test {
// Rates for the encoder and the frame number when to change profile.
struct RateProfile {
int target_kbps;
int input_fps;
int frame_index_rate_update;
size_t target_kbps;
size_t input_fps;
size_t frame_index_rate_update;
};
// Thresholds for the rate control metrics. The thresholds are defined for each
// rate update sequence. |max_num_frames_to_hit_target| is defined as number of
// frames, after a rate update is made to the encoder, for the encoder to reach
// |kMaxBitrateMismatchPercent| of new target rate.
struct RateControlThresholds {
int max_num_dropped_frames;
int max_key_framesize_mismatch_percent;
int max_delta_framesize_mismatch_percent;
int max_bitrate_mismatch_percent;
int max_num_frames_to_hit_target;
int num_spatial_resizes;
int num_key_frames;
double max_avg_bitrate_mismatch_percent;
double max_time_to_reach_target_bitrate_sec;
// TODO(ssilkin): Use absolute threshold for framerate.
double max_avg_framerate_mismatch_percent;
double max_avg_buffer_level_sec;
double max_max_key_frame_delay_sec;
double max_max_delta_frame_delay_sec;
size_t max_num_spatial_resizes;
size_t max_num_key_frames;
};
// Thresholds for the quality metrics.
struct QualityThresholds {
QualityThresholds(double min_avg_psnr,
double min_min_psnr,
double min_avg_ssim,
double min_min_ssim)
: min_avg_psnr(min_avg_psnr),
min_min_psnr(min_min_psnr),
min_avg_ssim(min_avg_ssim),
min_min_ssim(min_min_ssim) {}
double min_avg_psnr;
double min_min_psnr;
double min_avg_ssim;
@ -72,9 +61,7 @@ struct QualityThresholds {
};
struct BitstreamThresholds {
explicit BitstreamThresholds(size_t max_nalu_length)
: max_nalu_length(max_nalu_length) {}
size_t max_nalu_length;
size_t max_max_nalu_size_bytes;
};
// Should video files be saved persistently to disk for post-run visualization?
@ -83,15 +70,10 @@ struct VisualizationParams {
bool save_decoded_y4m;
};
// Integration test for video processor. Encodes+decodes a clip and
// writes it to the output directory. After completion, quality metrics
// (PSNR and SSIM) and rate control metrics are computed and compared to given
// thresholds, to verify that the quality and encoder response is acceptable.
// The rate control tests allow us to verify the behavior for changing bit rate,
// changing frame rate, frame dropping/spatial resize, and temporal layers.
// The thresholds for the rate control metrics are set to be fairly
// conservative, so failure should only happen when some significant regression
// or breakdown occurs.
// Integration test for video processor. It does rate control and frame quality
// analysis using frame statistics collected by video processor and logs the
// results. If thresholds are specified it checks that corresponding metrics
// are in desirable range.
class VideoProcessorIntegrationTest : public testing::Test {
protected:
// Verifies that all H.264 keyframes contain SPS/PPS/IDR NALUs.
@ -107,7 +89,7 @@ class VideoProcessorIntegrationTest : public testing::Test {
void ProcessFramesAndMaybeVerify(
const std::vector<RateProfile>& rate_profiles,
const std::vector<RateControlThresholds>* rc_thresholds,
const QualityThresholds* quality_thresholds,
const std::vector<QualityThresholds>* quality_thresholds,
const BitstreamThresholds* bs_thresholds,
const VisualizationParams* visualization_params);
@ -119,54 +101,6 @@ class VideoProcessorIntegrationTest : public testing::Test {
private:
class CpuProcessTime;
static const int kMaxNumTemporalLayers = 3;
struct TestResults {
int KeyFrameSizeMismatchPercent() const {
if (num_key_frames == 0) {
return -1;
}
return 100 * sum_key_framesize_mismatch / num_key_frames;
}
int DeltaFrameSizeMismatchPercent(int i) const {
return 100 * sum_delta_framesize_mismatch_layer[i] / num_frames_layer[i];
}
int BitrateMismatchPercent(float target_kbps) const {
return 100 * std::fabs(kbps - target_kbps) / target_kbps;
}
int BitrateMismatchPercent(int i, float target_kbps_layer) const {
return 100 * std::fabs(kbps_layer[i] - target_kbps_layer) /
target_kbps_layer;
}
int num_frames = 0;
int num_frames_layer[kMaxNumTemporalLayers] = {0};
int num_key_frames = 0;
int num_frames_to_hit_target = 0;
float sum_framesize_kbits = 0.0f;
float sum_framesize_kbits_layer[kMaxNumTemporalLayers] = {0};
float kbps = 0.0f;
float kbps_layer[kMaxNumTemporalLayers] = {0};
float sum_key_framesize_mismatch = 0.0f;
float sum_delta_framesize_mismatch_layer[kMaxNumTemporalLayers] = {0};
};
struct TargetRates {
int kbps;
int fps;
float kbps_layer[kMaxNumTemporalLayers];
float fps_layer[kMaxNumTemporalLayers];
float framesize_kbits_layer[kMaxNumTemporalLayers];
float key_framesize_kbits_initial;
float key_framesize_kbits;
};
struct QualityMetrics {
int num_decoded_frames = 0;
double total_psnr = 0.0;
double total_ssim = 0.0;
double min_psnr = std::numeric_limits<double>::max();
double min_ssim = std::numeric_limits<double>::max();
};
void CreateEncoderAndDecoder();
void DestroyEncoderAndDecoder();
@ -176,26 +110,29 @@ class VideoProcessorIntegrationTest : public testing::Test {
const VisualizationParams* visualization_params);
void ReleaseAndCloseObjects(rtc::TaskQueue* task_queue);
// Rate control metrics.
void ResetRateControlMetrics(int rate_update_index,
const std::vector<RateProfile>& rate_profiles);
void SetRatesPerTemporalLayer();
void UpdateRateControlMetrics(int frame_number);
void PrintRateControlMetrics(
int rate_update_index,
const std::vector<int>& num_dropped_frames,
const std::vector<int>& num_spatial_resizes) const;
void VerifyRateControlMetrics(
int rate_update_index,
void ProcessAllFrames(rtc::TaskQueue* task_queue,
const std::vector<RateProfile>& rate_profiles);
void AnalyzeAllFrames(
const std::vector<RateProfile>& rate_profiles,
const std::vector<RateControlThresholds>* rc_thresholds,
const std::vector<int>& num_dropped_frames,
const std::vector<int>& num_spatial_resizes) const;
const std::vector<QualityThresholds>* quality_thresholds,
const BitstreamThresholds* bs_thresholds);
void VerifyBitstream(int frame_number,
const BitstreamThresholds& bs_thresholds);
std::vector<FrameStatistic> ExtractLayerStats(
size_t target_spatial_layer_number,
size_t target_temporal_layer_number,
size_t first_frame_number,
size_t last_frame_number,
bool combine_layers);
void UpdateQualityMetrics(int frame_number);
void VerifyQualityMetrics(const QualityThresholds& quality_thresholds);
void AnalyzeAndPrintStats(const std::vector<FrameStatistic>& stats,
float target_bitrate_kbps,
float target_framerate_fps,
float input_duration_sec,
const RateControlThresholds* rc_thresholds,
const QualityThresholds* quality_thresholds,
const BitstreamThresholds* bs_thresholds);
void PrintFrameLevelStats(const std::vector<FrameStatistic>& stats) const;
void PrintSettings() const;
@ -213,14 +150,6 @@ class VideoProcessorIntegrationTest : public testing::Test {
Stats stats_;
std::unique_ptr<VideoProcessor> processor_;
std::unique_ptr<CpuProcessTime> cpu_process_time_;
// Quantities updated for every encoded frame.
TestResults actual_;
// Rates set for every encoder rate update.
TargetRates target_;
QualityMetrics quality_;
};
} // namespace test

166
modules/video_coding/codecs/test/videoprocessor_integrationtest_libvpx.cc
View file

@ -75,110 +75,96 @@ class VideoProcessorIntegrationTestLibvpx
#if !defined(WEBRTC_IOS)
#if !defined(RTC_DISABLE_VP9)
// VP9: Run with no packet loss and fixed bitrate. Quality should be very high.
// One key frame (first frame only) in sequence.
TEST_F(VideoProcessorIntegrationTestLibvpx, Process0PercentPacketLossVP9) {
TEST_F(VideoProcessorIntegrationTestLibvpx, HighBitrateVP9) {
config_.SetCodecSettings(kVideoCodecVP9, 1, false, false, true, false,
kResilienceOn, kCifWidth, kCifHeight);
config_.num_frames = kNumFramesShort;
std::vector<RateProfile> rate_profiles = {{500, 30, kNumFramesShort + 1}};
std::vector<RateProfile> rate_profiles = {{500, 30, kNumFramesShort}};
std::vector<RateControlThresholds> rc_thresholds = {
{0, 40, 20, 10, 20, 0, 1}};
{5, 1, 0, 0.1, 0.3, 0.1, 0, 1}};
QualityThresholds quality_thresholds(37.0, 36.0, 0.93, 0.92);
std::vector<QualityThresholds> quality_thresholds = {{37, 36, 0.94, 0.92}};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, nullptr,
kNoVisualizationParams);
}
// VP9: Run with no packet loss, with varying bitrate (3 rate updates):
// low to high to medium. Check that quality and encoder response to the new
// target rate/per-frame bandwidth (for each rate update) is within limits.
// One key frame (first frame only) in sequence.
TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessNoLossChangeBitRateVP9) {
TEST_F(VideoProcessorIntegrationTestLibvpx, ChangeBitrateVP9) {
config_.SetCodecSettings(kVideoCodecVP9, 1, false, false, true, false,
kResilienceOn, kCifWidth, kCifHeight);
std::vector<RateProfile> rate_profiles = {
{200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update
{700, 30, 200},
{500, 30, kNumFramesLong + 1}};
{500, 30, kNumFramesLong}};
std::vector<RateControlThresholds> rc_thresholds = {{0, 35, 20, 20, 35, 0, 1},
{2, 0, 20, 20, 60, 0, 0},
{0, 0, 25, 20, 40, 0, 0}};
std::vector<RateControlThresholds> rc_thresholds = {
{5, 1, 0, 0.15, 0.5, 0.1, 0, 1},
{15, 2, 0, 0.2, 0.5, 0.1, 0, 0},
{10, 1, 0, 0.3, 0.5, 0.1, 0, 0}};
QualityThresholds quality_thresholds(35.5, 30.0, 0.90, 0.85);
std::vector<QualityThresholds> quality_thresholds = {
{34, 33, 0.90, 0.88}, {38, 35, 0.95, 0.91}, {35, 34, 0.93, 0.90}};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, nullptr,
kNoVisualizationParams);
}
// VP9: Run with no packet loss, with an update (decrease) in frame rate.
// Lower frame rate means higher per-frame-bandwidth, so easier to encode.
// At the low bitrate in this test, this means better rate control after the
// update(s) to lower frame rate. So expect less frame drops, and max values
// for the rate control metrics can be lower. One key frame (first frame only).
// Note: quality after update should be higher but we currently compute quality
// metrics averaged over whole sequence run.
TEST_F(VideoProcessorIntegrationTestLibvpx,
ProcessNoLossChangeFrameRateFrameDropVP9) {
TEST_F(VideoProcessorIntegrationTestLibvpx, ChangeFramerateVP9) {
config_.SetCodecSettings(kVideoCodecVP9, 1, false, false, true, false,
kResilienceOn, kCifWidth, kCifHeight);
std::vector<RateProfile> rate_profiles = {
{100, 24, 100}, // target_kbps, input_fps, frame_index_rate_update
{100, 15, 200},
{100, 10, kNumFramesLong + 1}};
{100, 10, kNumFramesLong}};
// Framerate mismatch should be lower for lower framerate.
std::vector<RateControlThresholds> rc_thresholds = {
{45, 50, 95, 15, 45, 0, 1},
{20, 0, 50, 10, 30, 0, 0},
{5, 0, 30, 5, 25, 0, 0}};
{10, 2, 40, 0.4, 0.5, 0.2, 0, 1},
{8, 2, 5, 0.2, 0.5, 0.2, 0, 0},
{5, 2, 0, 0.2, 0.5, 0.3, 0, 0}};
QualityThresholds quality_thresholds(31.5, 18.0, 0.80, 0.43);
// Quality should be higher for lower framerates for the same content.
std::vector<QualityThresholds> quality_thresholds = {
{33, 32, 0.89, 0.87}, {33.5, 32, 0.90, 0.86}, {33.5, 31.5, 0.90, 0.85}};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, nullptr,
kNoVisualizationParams);
}
// VP9: Run with no packet loss and denoiser on. One key frame (first frame).
TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessNoLossDenoiserOnVP9) {
TEST_F(VideoProcessorIntegrationTestLibvpx, DenoiserOnVP9) {
config_.SetCodecSettings(kVideoCodecVP9, 1, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
config_.num_frames = kNumFramesShort;
std::vector<RateProfile> rate_profiles = {{500, 30, kNumFramesShort + 1}};
std::vector<RateProfile> rate_profiles = {{500, 30, kNumFramesShort}};
std::vector<RateControlThresholds> rc_thresholds = {
{0, 40, 20, 10, 20, 0, 1}};
{5, 1, 0, 0.1, 0.3, 0.1, 0, 1}};
QualityThresholds quality_thresholds(36.8, 35.8, 0.92, 0.91);
std::vector<QualityThresholds> quality_thresholds = {{37.5, 36, 0.94, 0.93}};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, nullptr,
kNoVisualizationParams);
}
// Run with no packet loss, at low bitrate.
// spatial_resize is on, for this low bitrate expect one resize in sequence.
// Resize happens on delta frame. Expect only one key frame (first frame).
TEST_F(VideoProcessorIntegrationTestLibvpx,
DISABLED_ProcessNoLossSpatialResizeFrameDropVP9) {
TEST_F(VideoProcessorIntegrationTestLibvpx, VeryLowBitrateVP9) {
config_.SetCodecSettings(kVideoCodecVP9, 1, false, false, true, true,
kResilienceOn, kCifWidth, kCifHeight);
std::vector<RateProfile> rate_profiles = {{50, 30, kNumFramesLong + 1}};
std::vector<RateProfile> rate_profiles = {{50, 30, kNumFramesLong}};
std::vector<RateControlThresholds> rc_thresholds = {
{228, 70, 160, 15, 80, 1, 1}};
{15, 3, 75, 1.0, 0.5, 0.4, 1, 1}};
QualityThresholds quality_thresholds(24.0, 13.0, 0.65, 0.37);
std::vector<QualityThresholds> quality_thresholds = {{28, 25, 0.80, 0.65}};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, nullptr,
@ -190,20 +176,20 @@ TEST_F(VideoProcessorIntegrationTestLibvpx,
#endif // !defined(RTC_DISABLE_VP9)
// VP8: Run with no packet loss and fixed bitrate. Quality should be very high.
// One key frame (first frame only) in sequence. Setting |key_frame_interval|
// to -1 below means no periodic key frames in test.
TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessZeroPacketLoss) {
TEST_F(VideoProcessorIntegrationTestLibvpx, HighBitrateVP8) {
config_.SetCodecSettings(kVideoCodecVP8, 1, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
config_.num_frames = kNumFramesShort;
std::vector<RateProfile> rate_profiles = {{500, 30, kNumFramesShort + 1}};
std::vector<RateProfile> rate_profiles = {{500, 30, kNumFramesShort}};
std::vector<RateControlThresholds> rc_thresholds = {
{0, 40, 20, 10, 15, 0, 1}};
{5, 1, 0, 0.1, 0.2, 0.1, 0, 1}};
QualityThresholds quality_thresholds(34.95, 33.0, 0.90, 0.89);
// std::vector<QualityThresholds> quality_thresholds = {{37, 35, 0.93, 0.91}};
// TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse
// than quality of x86 version. Use lower thresholds for now.
std::vector<QualityThresholds> quality_thresholds = {{35, 33, 0.91, 0.89}};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, nullptr,
@ -221,10 +207,6 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessZeroPacketLoss) {
// disabled on Android. Some quality parameter in the above test has been
// adjusted to also pass for |cpu_speed| <= 12.
// VP8: Run with no packet loss, with varying bitrate (3 rate updates):
// low to high to medium. Check that quality and encoder response to the new
// target rate/per-frame bandwidth (for each rate update) is within limits.
// One key frame (first frame only) in sequence.
// Too slow to finish before timeout on iOS. See webrtc:4755.
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
#define MAYBE_ProcessNoLossChangeBitRateVP8 \
@ -232,34 +214,32 @@ TEST_F(VideoProcessorIntegrationTestLibvpx, ProcessZeroPacketLoss) {
#else
#define MAYBE_ProcessNoLossChangeBitRateVP8 ProcessNoLossChangeBitRateVP8
#endif
TEST_F(VideoProcessorIntegrationTestLibvpx,
MAYBE_ProcessNoLossChangeBitRateVP8) {
TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeBitrateVP8) {
config_.SetCodecSettings(kVideoCodecVP8, 1, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
std::vector<RateProfile> rate_profiles = {
{200, 30, 100}, // target_kbps, input_fps, frame_index_rate_update
{800, 30, 200},
{500, 30, kNumFramesLong + 1}};
{500, 30, kNumFramesLong}};
std::vector<RateControlThresholds> rc_thresholds = {{0, 45, 20, 10, 15, 0, 1},
{0, 0, 25, 20, 10, 0, 0},
{0, 0, 25, 15, 10, 0, 0}};
std::vector<RateControlThresholds> rc_thresholds = {
{5, 1, 0, 0.1, 0.2, 0.1, 0, 1},
{15, 1, 0, 0.1, 0.2, 0.1, 0, 0},
{15, 1, 0, 0.3, 0.2, 0.1, 0, 0}};
QualityThresholds quality_thresholds(34.0, 32.0, 0.85, 0.80);
// std::vector<QualityThresholds> quality_thresholds = {
// {33, 32, 0.89, 0.88}, {38, 36, 0.94, 0.93}, {35, 34, 0.92, 0.91}};
// TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse
// than quality of x86 version. Use lower thresholds for now.
std::vector<QualityThresholds> quality_thresholds = {
{31.8, 31, 0.86, 0.85}, {36, 34.8, 0.92, 0.90}, {33.5, 32, 0.90, 0.88}};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, nullptr,
kNoVisualizationParams);
}
// VP8: Run with no packet loss, with an update (decrease) in frame rate.
// Lower frame rate means higher per-frame-bandwidth, so easier to encode.
// At the bitrate in this test, this means better rate control after the
// update(s) to lower frame rate. So expect less frame drops, and max values
// for the rate control metrics can be lower. One key frame (first frame only).
// Note: quality after update should be higher but we currently compute quality
// metrics averaged over whole sequence run.
// Too slow to finish before timeout on iOS. See webrtc:4755.
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
#define MAYBE_ProcessNoLossChangeFrameRateFrameDropVP8 \
@ -268,33 +248,38 @@ TEST_F(VideoProcessorIntegrationTestLibvpx,
#define MAYBE_ProcessNoLossChangeFrameRateFrameDropVP8 \
ProcessNoLossChangeFrameRateFrameDropVP8
#endif
TEST_F(VideoProcessorIntegrationTestLibvpx,
MAYBE_ProcessNoLossChangeFrameRateFrameDropVP8) {
TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_ChangeFramerateVP8) {
config_.SetCodecSettings(kVideoCodecVP8, 1, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
std::vector<RateProfile> rate_profiles = {
{80, 24, 100}, // target_kbps, input_fps, frame_index_rate_update
{80, 15, 200},
{80, 10, kNumFramesLong + 1}};
{80, 10, kNumFramesLong}};
// std::vector<RateControlThresholds> rc_thresholds = {
// {10, 2, 20, 0.4, 0.3, 0.1, 0, 1},
// {5, 2, 5, 0.3, 0.3, 0.1, 0, 0},
// {4, 2, 1, 0.2, 0.3, 0.2, 0, 0}};
// TODO(webrtc:8757): AMR VP8 drops more frames than x86 version. Use lower
// thresholds for now.
std::vector<RateControlThresholds> rc_thresholds = {
{40, 20, 75, 15, 60, 0, 1},
{10, 0, 25, 10, 35, 0, 0},
{0, 0, 20, 10, 15, 0, 0}};
{10, 2, 60, 0.5, 0.3, 0.3, 0, 1},
{10, 2, 30, 0.3, 0.3, 0.3, 0, 0},
{10, 2, 10, 0.2, 0.3, 0.2, 0, 0}};
QualityThresholds quality_thresholds(31.0, 22.0, 0.80, 0.65);
// std::vector<QualityThresholds> quality_thresholds = {
// {31, 30, 0.87, 0.86}, {32, 31, 0.89, 0.86}, {32, 30, 0.87, 0.82}};
// TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse
// than quality of x86 version. Use lower thresholds for now.
std::vector<QualityThresholds> quality_thresholds = {
{31, 30, 0.85, 0.84}, {31.5, 30.5, 0.86, 0.84}, {30.5, 29, 0.83, 0.78}};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, nullptr,
kNoVisualizationParams);
}
// VP8: Run with no packet loss, with 3 temporal layers, with a rate update in
// the middle of the sequence. The max values for the frame size mismatch and
// encoding rate mismatch are applied to each layer.
// No dropped frames in this test, and internal spatial resizer is off.
// One key frame (first frame only) in sequence, so no spatial resizing.
// Too slow to finish before timeout on iOS. See webrtc:4755.
#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS)
#define MAYBE_ProcessNoLossTemporalLayersVP8 \
@ -302,18 +287,27 @@ TEST_F(VideoProcessorIntegrationTestLibvpx,
#else
#define MAYBE_ProcessNoLossTemporalLayersVP8 ProcessNoLossTemporalLayersVP8
#endif
TEST_F(VideoProcessorIntegrationTestLibvpx,
MAYBE_ProcessNoLossTemporalLayersVP8) {
TEST_F(VideoProcessorIntegrationTestLibvpx, MAYBE_TemporalLayersVP8) {
config_.SetCodecSettings(kVideoCodecVP8, 3, false, true, true, false,
kResilienceOn, kCifWidth, kCifHeight);
std::vector<RateProfile> rate_profiles = {{200, 30, 150},
{400, 30, kNumFramesLong + 1}};
{400, 30, kNumFramesLong}};
std::vector<RateControlThresholds> rc_thresholds = {{0, 20, 30, 10, 10, 0, 1},
{0, 0, 30, 15, 10, 0, 0}};
// std::vector<RateControlThresholds> rc_thresholds = {
// {5, 1, 0, 0.1, 0.2, 0.1, 0, 1}, {10, 2, 0, 0.1, 0.2, 0.1, 0, 1}};
// TODO(webrtc:8757): AMR VP8 drops more frames than x86 version. Use lower
// thresholds for now.
std::vector<RateControlThresholds> rc_thresholds = {
{10, 1, 2, 0.3, 0.2, 0.1, 0, 1}, {12, 2, 3, 0.1, 0.2, 0.1, 0, 1}};
QualityThresholds quality_thresholds(32.5, 30.0, 0.85, 0.80);
// Min SSIM drops because of high motion scene with complex backgound (trees).
// std::vector<QualityThresholds> quality_thresholds = {{32, 30, 0.88, 0.85},
// {33, 30, 0.89, 0.83}};
// TODO(webrtc:8757): AMR VP8 encoder's quality is significantly worse
// than quality of x86 version. Use lower thresholds for now.
std::vector<QualityThresholds> quality_thresholds = {{31, 30, 0.85, 0.84},
{31, 28, 0.85, 0.75}};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, nullptr,

19
modules/video_coding/codecs/test/videoprocessor_integrationtest_mediacodec.cc
View file

@ -44,15 +44,15 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsVp8) {
config_.SetCodecSettings(kVideoCodecVP8, 1, false, false, false, false, false,
352, 288);
std::vector<RateProfile> rate_profiles = {{500, 30, kForemanNumFrames + 1}};
std::vector<RateProfile> rate_profiles = {{500, 30, kForemanNumFrames}};
// The thresholds below may have to be tweaked to let even poor MediaCodec
// implementations pass. If this test fails on the bots, disable it and
// ping brandtr@.
std::vector<RateControlThresholds> rc_thresholds = {
{20, 95, 22, 11, 50, 0, 1}};
{10, 1, 1, 0.1, 0.2, 0.1, 0, 1}};
QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39);
std::vector<QualityThresholds> quality_thresholds = {{36, 31, 0.92, 0.86}};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, nullptr,
@ -64,15 +64,15 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec, ForemanCif500kbpsH264CBP) {
config_.SetCodecSettings(kVideoCodecH264, 1, false, false, false, false,
false, 352, 288);
std::vector<RateProfile> rate_profiles = {{500, 30, kForemanNumFrames + 1}};
std::vector<RateProfile> rate_profiles = {{500, 30, kForemanNumFrames}};
// The thresholds below may have to be tweaked to let even poor MediaCodec
// implementations pass. If this test fails on the bots, disable it and
// ping brandtr@.
std::vector<RateControlThresholds> rc_thresholds = {
{20, 95, 22, 11, 20, 0, 1}};
{10, 1, 1, 0.1, 0.2, 0.1, 0, 1}};
QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39);
std::vector<QualityThresholds> quality_thresholds = {{36, 31, 0.92, 0.86}};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, nullptr,
@ -90,14 +90,15 @@ TEST_F(VideoProcessorIntegrationTestMediaCodec,
config_.SetCodecSettings(kVideoCodecH264, 1, false, false, false, false,
false, 352, 288);
std::vector<RateProfile> rate_profiles = {{500, 30, kForemanNumFrames + 1}};
std::vector<RateProfile> rate_profiles = {{500, 30, kForemanNumFrames}};
// The thresholds below may have to be tweaked to let even poor MediaCodec
// implementations pass. If this test fails on the bots, disable it and
// ping brandtr@.
std::vector<RateControlThresholds> rc_thresholds = {{5, 60, 20, 5, 15, 0, 1}};
std::vector<RateControlThresholds> rc_thresholds = {
{5, 1, 0, 0.1, 0.2, 0.1, 0, 1}};
QualityThresholds quality_thresholds(33.0, 30.0, 0.90, 0.85);
std::vector<QualityThresholds> quality_thresholds = {{37, 35, 0.93, 0.91}};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, nullptr,

25
modules/video_coding/codecs/test/videoprocessor_integrationtest_openh264.cc
View file

@ -49,21 +49,16 @@ class VideoProcessorIntegrationTestOpenH264
}
};
// H264: Run with no packet loss and fixed bitrate. Quality should be very high.
// Note(hbos): The PacketManipulatorImpl code used to simulate packet loss in
// these unittests appears to drop "packets" in a way that is not compatible
// with H264. Therefore ProcessXPercentPacketLossH264, X != 0, unittests have
// not been added.
TEST_F(VideoProcessorIntegrationTestOpenH264, Process0PercentPacketLoss) {
TEST_F(VideoProcessorIntegrationTestOpenH264, ConstantHighBitrate) {
config_.SetCodecSettings(kVideoCodecH264, 1, false, false, true, false,
kResilienceOn, kCifWidth, kCifHeight);
std::vector<RateProfile> rate_profiles = {{500, 30, kNumFrames + 1}};
std::vector<RateProfile> rate_profiles = {{500, 30, kNumFrames}};
std::vector<RateControlThresholds> rc_thresholds = {
{2, 60, 20, 10, 20, 0, 1}};
{5, 1, 0, 0.1, 0.2, 0.1, 0, 1}};
QualityThresholds quality_thresholds(35.0, 25.0, 0.93, 0.70);
std::vector<QualityThresholds> quality_thresholds = {{37, 35, 0.93, 0.91}};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, nullptr,
@ -72,22 +67,22 @@ TEST_F(VideoProcessorIntegrationTestOpenH264, Process0PercentPacketLoss) {
// H264: Enable SingleNalUnit packetization mode. Encoder should split
// large frames into multiple slices and limit length of NAL units.
TEST_F(VideoProcessorIntegrationTestOpenH264, ProcessNoLossSingleNalUnit) {
TEST_F(VideoProcessorIntegrationTestOpenH264, SingleNalUnit) {
config_.h264_codec_settings.packetization_mode =
H264PacketizationMode::SingleNalUnit;
config_.networking_config.max_payload_size_in_bytes = 500;
config_.SetCodecSettings(kVideoCodecH264, 1, false, false, true, false,
kResilienceOn, kCifWidth, kCifHeight);
std::vector<RateProfile> rate_profiles = {{500, 30, kNumFrames + 1}};
std::vector<RateProfile> rate_profiles = {{500, 30, kNumFrames}};
std::vector<RateControlThresholds> rc_thresholds = {
{2, 60, 30, 10, 20, 0, 1}};
{5, 1, 0, 0.1, 0.2, 0.1, 0, 1}};
QualityThresholds quality_thresholds(35.0, 25.0, 0.93, 0.70);
std::vector<QualityThresholds> quality_thresholds = {{37, 35, 0.93, 0.91}};
BitstreamThresholds bs_thresholds(
config_.networking_config.max_payload_size_in_bytes);
BitstreamThresholds bs_thresholds = {
config_.networking_config.max_payload_size_in_bytes};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, &bs_thresholds,

16
modules/video_coding/codecs/test/videoprocessor_integrationtest_parameterized.cc
View file

@ -18,13 +18,13 @@ namespace test {
namespace {
// Loop variables.
const int kBitrates[] = {500};
const size_t kBitrates[] = {500};
const VideoCodecType kVideoCodecType[] = {kVideoCodecVP8};
const bool kHwCodec[] = {false};
// Codec settings.
const bool kResilienceOn = false;
const int kNumTemporalLayers = 1;
const bool kResilienceOn = kNumTemporalLayers > 1;
const bool kDenoisingOn = false;
const bool kErrorConcealmentOn = false;
const bool kSpatialResizeOn = false;
@ -46,7 +46,7 @@ const int kNumFrames = 30;
class VideoProcessorIntegrationTestParameterized
: public VideoProcessorIntegrationTest,
public ::testing::WithParamInterface<
::testing::tuple<int, VideoCodecType, bool>> {
::testing::tuple<size_t, VideoCodecType, bool>> {
protected:
VideoProcessorIntegrationTestParameterized()
: bitrate_(::testing::get<0>(GetParam())),
@ -54,9 +54,9 @@ class VideoProcessorIntegrationTestParameterized
hw_codec_(::testing::get<2>(GetParam())) {}
~VideoProcessorIntegrationTestParameterized() override = default;
void RunTest(int width,
int height,
int framerate,
void RunTest(size_t width,
size_t height,
size_t framerate,
const std::string& filename) {
config_.filename = filename;
config_.input_filename = ResourcePath(filename, "yuv");
@ -72,13 +72,13 @@ class VideoProcessorIntegrationTestParameterized
kSpatialResizeOn, kResilienceOn, width, height);
std::vector<RateProfile> rate_profiles = {
{bitrate_, framerate, kNumFrames + 1}};
{bitrate_, framerate, kNumFrames}};
ProcessFramesAndMaybeVerify(rate_profiles, nullptr, nullptr, nullptr,
&kVisualizationParams);
}
const int bitrate_;
const size_t bitrate_;
const VideoCodecType codec_type_;
const bool hw_codec_;
};

13
modules/video_coding/codecs/test/videoprocessor_integrationtest_videotoolbox.cc
View file

@ -48,12 +48,12 @@ TEST_F(VideoProcessorIntegrationTestVideoToolbox,
config_.SetCodecSettings(kVideoCodecH264, 1, false, false, false, false,
false, 352, 288);
std::vector<RateProfile> rate_profiles = {{500, 30, kForemanNumFrames + 1}};
std::vector<RateProfile> rate_profiles = {{500, 30, kForemanNumFrames}};
std::vector<RateControlThresholds> rc_thresholds = {
{20, 95, 60, 60, 10, 0, 1}};
{5, 1, 0, 0.1, 0.2, 0.1, 0, 1}};
QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39);
std::vector<QualityThresholds> quality_thresholds = {{37, 35, 0.93, 0.91}};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, nullptr,
@ -68,11 +68,12 @@ TEST_F(VideoProcessorIntegrationTestVideoToolbox,
config_.SetCodecSettings(kVideoCodecH264, 1, false, false, false, false,
false, 352, 288);
std::vector<RateProfile> rate_profiles = {{500, 30, kForemanNumFrames + 1}};
std::vector<RateProfile> rate_profiles = {{500, 30, kForemanNumFrames}};
std::vector<RateControlThresholds> rc_thresholds = {{5, 75, 65, 60, 6, 0, 1}};
std::vector<RateControlThresholds> rc_thresholds = {
{5, 1, 0, 0.1, 0.2, 0.1, 0, 1}};
QualityThresholds quality_thresholds(30.0, 14.0, 0.86, 0.39);
std::vector<QualityThresholds> quality_thresholds = {{37, 35, 0.93, 0.91}};
ProcessFramesAndMaybeVerify(rate_profiles, &rc_thresholds,
&quality_thresholds, nullptr,

8
modules/video_coding/codecs/test/videoprocessor_unittest.cc
View file

@ -151,10 +151,6 @@ TEST_F(VideoProcessorTest, SetRates) {
kFramerateFps))
.Times(1);
video_processor_->SetRates(kBitrateKbps, kFramerateFps);
EXPECT_THAT(video_processor_->NumberDroppedFramesPerRateUpdate(),
ElementsAre(0));
EXPECT_THAT(video_processor_->NumberSpatialResizesPerRateUpdate(),
ElementsAre(0));
const int kNewBitrateKbps = 456;
const int kNewFramerateFps = 34;
@ -164,10 +160,6 @@ TEST_F(VideoProcessorTest, SetRates) {
kNewFramerateFps))
.Times(1);
video_processor_->SetRates(kNewBitrateKbps, kNewFramerateFps);
EXPECT_THAT(video_processor_->NumberDroppedFramesPerRateUpdate(),
ElementsAre(0, 0));
EXPECT_THAT(video_processor_->NumberSpatialResizesPerRateUpdate(),
ElementsAre(0, 0));
ExpectRelease();
}

19
test/statistics.cc
View file

@ -11,23 +11,40 @@
#include <math.h>
#include <algorithm>
namespace webrtc {
namespace test {
Statistics::Statistics() : sum_(0.0), sum_squared_(0.0), count_(0) {}
Statistics::Statistics()
: sum_(0.0),
sum_squared_(0.0),
max_(std::numeric_limits<double>::min()),
min_(std::numeric_limits<double>::max()),
count_(0) {}
void Statistics::AddSample(double sample) {
sum_ += sample;
sum_squared_ += sample * sample;
max_ = std::max(max_, sample);
min_ = std::min(min_, sample);
++count_;
}
double Statistics::Max() const {
return max_;
}
double Statistics::Mean() const {
if (count_ == 0)
return 0.0;
return sum_ / count_;
}
double Statistics::Min() const {
return min_;
}
double Statistics::Variance() const {
if (count_ == 0)
return 0.0;

4
test/statistics.h
View file

@ -21,13 +21,17 @@ class Statistics {
void AddSample(double sample);
double Max() const;
double Mean() const;
double Min() const;
double Variance() const;
double StandardDeviation() const;
private:
double sum_;
double sum_squared_;
double max_;
double min_;
uint64_t count_;
};
} // namespace test