diff --git a/rtc_tools/BUILD.gn b/rtc_tools/BUILD.gn index 6f60ab1e92..5214bc3bd0 100644 --- a/rtc_tools/BUILD.gn +++ b/rtc_tools/BUILD.gn @@ -92,6 +92,8 @@ rtc_static_library("video_quality_analysis") { "frame_analyzer/linear_least_squares.h", "frame_analyzer/video_color_aligner.cc", "frame_analyzer/video_color_aligner.h", + "frame_analyzer/video_geometry_aligner.cc", + "frame_analyzer/video_geometry_aligner.h", "frame_analyzer/video_quality_analysis.cc", "frame_analyzer/video_quality_analysis.h", "frame_analyzer/video_temporal_aligner.cc", @@ -100,6 +102,7 @@ rtc_static_library("video_quality_analysis") { deps = [ ":video_file_reader", "../api:array_view", + "../api/video:video_frame", "../api/video:video_frame_i420", "../common_video", "../rtc_base:checks", @@ -343,6 +346,7 @@ if (rtc_include_tests) { "frame_analyzer/linear_least_squares_unittest.cc", "frame_analyzer/reference_less_video_analysis_unittest.cc", "frame_analyzer/video_color_aligner_unittest.cc", + "frame_analyzer/video_geometry_aligner_unittest.cc", "frame_analyzer/video_quality_analysis_unittest.cc", "frame_analyzer/video_temporal_aligner_unittest.cc", "frame_editing/frame_editing_unittest.cc", diff --git a/rtc_tools/frame_analyzer/video_geometry_aligner.cc b/rtc_tools/frame_analyzer/video_geometry_aligner.cc new file mode 100644 index 0000000000..d3cbc051c9 --- /dev/null +++ b/rtc_tools/frame_analyzer/video_geometry_aligner.cc @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_tools/frame_analyzer/video_geometry_aligner.h" + +#include + +#include "api/video/i420_buffer.h" +#include "rtc_base/checks.h" +#include "rtc_base/refcountedobject.h" +#include "rtc_tools/frame_analyzer/video_quality_analysis.h" +#include "third_party/libyuv/include/libyuv/scale.h" + +namespace webrtc { +namespace test { + +namespace { + +bool IsValidRegion(const CropRegion& region, + const rtc::scoped_refptr& frame) { + return region.left >= 0 && region.right >= 0 && region.top >= 0 && + region.bottom >= 0 && region.left + region.right < frame->width() && + region.top + region.bottom < frame->height(); +} + +} // namespace + +rtc::scoped_refptr CropAndZoom( + const CropRegion& crop_region, + const rtc::scoped_refptr& frame) { + RTC_CHECK(IsValidRegion(crop_region, frame)); + + const int uv_crop_left = crop_region.left / 2; + const int uv_crop_top = crop_region.top / 2; + + const int cropped_width = + frame->width() - crop_region.left - crop_region.right; + const int cropped_height = + frame->height() - crop_region.top - crop_region.bottom; + + // Crop by only adjusting pointers. + const uint8_t* y_plane = + frame->DataY() + frame->StrideY() * crop_region.top + crop_region.left; + const uint8_t* u_plane = + frame->DataU() + frame->StrideU() * uv_crop_top + uv_crop_left; + const uint8_t* v_plane = + frame->DataV() + frame->StrideV() * uv_crop_top + uv_crop_left; + + // Stretch the cropped frame to the original size using libyuv. + rtc::scoped_refptr adjusted_frame = + I420Buffer::Create(frame->width(), frame->height()); + libyuv::I420Scale(y_plane, frame->StrideY(), u_plane, frame->StrideU(), + v_plane, frame->StrideV(), cropped_width, cropped_height, + adjusted_frame->MutableDataY(), adjusted_frame->StrideY(), + adjusted_frame->MutableDataU(), adjusted_frame->StrideU(), + adjusted_frame->MutableDataV(), adjusted_frame->StrideV(), + frame->width(), frame->height(), libyuv::kFilterBilinear); + + return adjusted_frame; +} + +CropRegion CalculateCropRegion( + const rtc::scoped_refptr& reference_frame, + const rtc::scoped_refptr& test_frame) { + RTC_CHECK_EQ(reference_frame->width(), test_frame->width()); + RTC_CHECK_EQ(reference_frame->height(), test_frame->height()); + + CropRegion best_region; + double best_ssim = Ssim(reference_frame, test_frame); + + typedef int CropRegion::*CropParameter; + CropParameter crop_parameters[4] = {&CropRegion::left, &CropRegion::top, + &CropRegion::right, &CropRegion::bottom}; + + while (true) { + // Find the parameter in which direction SSIM improves the most. + CropParameter best_parameter = nullptr; + const CropRegion prev_best_region = best_region; + + for (CropParameter crop_parameter : crop_parameters) { + CropRegion test_region = prev_best_region; + ++(test_region.*crop_parameter); + + if (!IsValidRegion(test_region, reference_frame)) + continue; + + const double ssim = + Ssim(CropAndZoom(test_region, reference_frame), test_frame); + + if (ssim > best_ssim) { + best_ssim = ssim; + best_parameter = crop_parameter; + best_region = test_region; + } + } + + // No improvement among any direction, stop iteration. + if (best_parameter == nullptr) + break; + + // Iterate in the best direction as long as it improves SSIM. + for (CropRegion test_region = best_region; + IsValidRegion(test_region, reference_frame); + ++(test_region.*best_parameter)) { + const double ssim = + Ssim(CropAndZoom(test_region, reference_frame), test_frame); + if (ssim <= best_ssim) + break; + + best_ssim = ssim; + best_region = test_region; + } + } + + return best_region; +} + +rtc::scoped_refptr AdjustCropping( + const rtc::scoped_refptr& reference_frame, + const rtc::scoped_refptr& test_frame) { + return CropAndZoom(CalculateCropRegion(reference_frame, test_frame), + reference_frame); +} + +rtc::scoped_refptr