Add tool for aligning cropped region of video files

This class adds logic for aligning what part of a test video has been
encoded from a reference video. It does that by cropping and zooming in
on a region of the reference video that most closely matches the test
video. A small cropping does not have much impact on human perception,
but it has a big impact on PSNR and SSIM calculations.

For example, if the test video is cropped with one row in the top and
bottom, adjusting for this improves average PSNR from 27.7146 to
29.3357 and average SSIM from 0.934891 to 0.95318 in an example test
video.

TBR=phoglund

Bug: webrtc:9642
Change-Id: I02cfe0e2261fb58df8cdb1e15ba93285e3dc4538
Reviewed-on: https://webrtc-review.googlesource.com/c/99480
Commit-Queue: Magnus Jedvert <magjed@webrtc.org>
Reviewed-by: Patrik Höglund <phoglund@google.com>
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25755}

rtc_tools/BUILD.gn

@@ -92,6 +92,8 @@ rtc_static_library("video_quality_analysis") {
     "frame_analyzer/linear_least_squares.h",
     "frame_analyzer/video_color_aligner.cc",
     "frame_analyzer/video_color_aligner.h",
+    "frame_analyzer/video_geometry_aligner.cc",
+    "frame_analyzer/video_geometry_aligner.h",
     "frame_analyzer/video_quality_analysis.cc",
     "frame_analyzer/video_quality_analysis.h",
     "frame_analyzer/video_temporal_aligner.cc",
@@ -100,6 +102,7 @@ rtc_static_library("video_quality_analysis") {
   deps = [
     ":video_file_reader",
     "../api:array_view",
+    "../api/video:video_frame",
     "../api/video:video_frame_i420",
     "../common_video",
     "../rtc_base:checks",
@@ -343,6 +346,7 @@ if (rtc_include_tests) {
       "frame_analyzer/linear_least_squares_unittest.cc",
       "frame_analyzer/reference_less_video_analysis_unittest.cc",
       "frame_analyzer/video_color_aligner_unittest.cc",
+      "frame_analyzer/video_geometry_aligner_unittest.cc",
       "frame_analyzer/video_quality_analysis_unittest.cc",
       "frame_analyzer/video_temporal_aligner_unittest.cc",
       "frame_editing/frame_editing_unittest.cc",

rtc_tools/frame_analyzer/video_geometry_aligner.cc

@@ -0,0 +1,178 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
+
+#include <map>
+
+#include "api/video/i420_buffer.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/refcountedobject.h"
+#include "rtc_tools/frame_analyzer/video_quality_analysis.h"
+#include "third_party/libyuv/include/libyuv/scale.h"
+
+namespace webrtc {
+namespace test {
+
+namespace {
+
+bool IsValidRegion(const CropRegion& region,
+                   const rtc::scoped_refptr<I420BufferInterface>& frame) {
+  return region.left >= 0 && region.right >= 0 && region.top >= 0 &&
+         region.bottom >= 0 && region.left + region.right < frame->width() &&
+         region.top + region.bottom < frame->height();
+}
+
+}  // namespace
+
+rtc::scoped_refptr<I420BufferInterface> CropAndZoom(
+    const CropRegion& crop_region,
+    const rtc::scoped_refptr<I420BufferInterface>& frame) {
+  RTC_CHECK(IsValidRegion(crop_region, frame));
+
+  const int uv_crop_left = crop_region.left / 2;
+  const int uv_crop_top = crop_region.top / 2;
+
+  const int cropped_width =
+      frame->width() - crop_region.left - crop_region.right;
+  const int cropped_height =
+      frame->height() - crop_region.top - crop_region.bottom;
+
+  // Crop by only adjusting pointers.
+  const uint8_t* y_plane =
+      frame->DataY() + frame->StrideY() * crop_region.top + crop_region.left;
+  const uint8_t* u_plane =
+      frame->DataU() + frame->StrideU() * uv_crop_top + uv_crop_left;
+  const uint8_t* v_plane =
+      frame->DataV() + frame->StrideV() * uv_crop_top + uv_crop_left;
+
+  // Stretch the cropped frame to the original size using libyuv.
+  rtc::scoped_refptr<I420Buffer> adjusted_frame =
+      I420Buffer::Create(frame->width(), frame->height());
+  libyuv::I420Scale(y_plane, frame->StrideY(), u_plane, frame->StrideU(),
+                    v_plane, frame->StrideV(), cropped_width, cropped_height,
+                    adjusted_frame->MutableDataY(), adjusted_frame->StrideY(),
+                    adjusted_frame->MutableDataU(), adjusted_frame->StrideU(),
+                    adjusted_frame->MutableDataV(), adjusted_frame->StrideV(),
+                    frame->width(), frame->height(), libyuv::kFilterBilinear);
+
+  return adjusted_frame;
+}
+
+CropRegion CalculateCropRegion(
+    const rtc::scoped_refptr<I420BufferInterface>& reference_frame,
+    const rtc::scoped_refptr<I420BufferInterface>& test_frame) {
+  RTC_CHECK_EQ(reference_frame->width(), test_frame->width());
+  RTC_CHECK_EQ(reference_frame->height(), test_frame->height());
+
+  CropRegion best_region;
+  double best_ssim = Ssim(reference_frame, test_frame);
+
+  typedef int CropRegion::*CropParameter;
+  CropParameter crop_parameters[4] = {&CropRegion::left, &CropRegion::top,
+                                      &CropRegion::right, &CropRegion::bottom};
+
+  while (true) {
+    // Find the parameter in which direction SSIM improves the most.
+    CropParameter best_parameter = nullptr;
+    const CropRegion prev_best_region = best_region;
+
+    for (CropParameter crop_parameter : crop_parameters) {
+      CropRegion test_region = prev_best_region;
+      ++(test_region.*crop_parameter);
+
+      if (!IsValidRegion(test_region, reference_frame))
+        continue;
+
+      const double ssim =
+          Ssim(CropAndZoom(test_region, reference_frame), test_frame);
+
+      if (ssim > best_ssim) {
+        best_ssim = ssim;
+        best_parameter = crop_parameter;
+        best_region = test_region;
+      }
+    }
+
+    // No improvement among any direction, stop iteration.
+    if (best_parameter == nullptr)
+      break;
+
+    // Iterate in the best direction as long as it improves SSIM.
+    for (CropRegion test_region = best_region;
+         IsValidRegion(test_region, reference_frame);
+         ++(test_region.*best_parameter)) {
+      const double ssim =
+          Ssim(CropAndZoom(test_region, reference_frame), test_frame);
+      if (ssim <= best_ssim)
+        break;
+
+      best_ssim = ssim;
+      best_region = test_region;
+    }
+  }
+
+  return best_region;
+}
+
+rtc::scoped_refptr<I420BufferInterface> AdjustCropping(
+    const rtc::scoped_refptr<I420BufferInterface>& reference_frame,
+    const rtc::scoped_refptr<I420BufferInterface>& test_frame) {
+  return CropAndZoom(CalculateCropRegion(reference_frame, test_frame),
+                     reference_frame);
+}
+
+rtc::scoped_refptr<Video> AdjustCropping(
+    const rtc::scoped_refptr<Video>& reference_video,
+    const rtc::scoped_refptr<Video>& test_video) {
+  class CroppedVideo : public rtc::RefCountedObject<Video> {
+   public:
+    CroppedVideo(const rtc::scoped_refptr<Video>& reference_video,
+                 const rtc::scoped_refptr<Video>& test_video)
+        : reference_video_(reference_video), test_video_(test_video) {
+      RTC_CHECK_EQ(reference_video->number_of_frames(),
+                   test_video->number_of_frames());
+      RTC_CHECK_EQ(reference_video->width(), test_video->width());
+      RTC_CHECK_EQ(reference_video->height(), test_video->height());
+    }
+
+    int width() const override { return test_video_->width(); }
+    int height() const override { return test_video_->height(); }
+    size_t number_of_frames() const override {
+      return test_video_->number_of_frames();
+    }
+
+    rtc::scoped_refptr<I420BufferInterface> GetFrame(
+        size_t index) const override {
+      const rtc::scoped_refptr<I420BufferInterface> reference_frame =
+          reference_video_->GetFrame(index);
+
+      // Only calculate cropping region once per frame since it's expensive.
+      if (!crop_regions_.count(index)) {
+        crop_regions_[index] =
+            CalculateCropRegion(reference_frame, test_video_->GetFrame(index));
+      }
+
+      return CropAndZoom(crop_regions_[index], reference_frame);
+    }
+
+   private:
+    const rtc::scoped_refptr<Video> reference_video_;
+    const rtc::scoped_refptr<Video> test_video_;
+    // Mutable since this is a cache that affects performance and not logical
+    // behavior.
+    mutable std::map<size_t, CropRegion> crop_regions_;
+  };
+
+  return new CroppedVideo(reference_video, test_video);
+}
+
+}  // namespace test
+}  // namespace webrtc

rtc_tools/frame_analyzer/video_geometry_aligner.h

@@ -0,0 +1,57 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef RTC_TOOLS_FRAME_ANALYZER_VIDEO_GEOMETRY_ALIGNER_H_
+#define RTC_TOOLS_FRAME_ANALYZER_VIDEO_GEOMETRY_ALIGNER_H_
+
+#include "api/video/video_frame_buffer.h"
+#include "rtc_tools/video_file_reader.h"
+
+namespace webrtc {
+namespace test {
+
+struct CropRegion {
+  // Each value represents how much to crop from each side. Left is where x=0,
+  // and top is where y=0. All values equal to zero represents no cropping.
+  int left = 0;
+  int right = 0;
+  int top = 0;
+  int bottom = 0;
+};
+
+// Crops and zooms in on the cropped region so that the returned frame has the
+// same resolution as the input frame.
+rtc::scoped_refptr<I420BufferInterface> CropAndZoom(
+    const CropRegion& crop_region,
+    const rtc::scoped_refptr<I420BufferInterface>& frame);
+
+// Calculate the optimal cropping region on the reference frame to maximize SSIM
+// to the test frame.
+CropRegion CalculateCropRegion(
+    const rtc::scoped_refptr<I420BufferInterface>& reference_frame,
+    const rtc::scoped_refptr<I420BufferInterface>& test_frame);
+
+// Returns a cropped and zoomed version of the reference frame that matches up
+// to the test frame. This is a simple helper function on top of
+// CalculateCropRegion() and CropAndZoom().
+rtc::scoped_refptr<I420BufferInterface> AdjustCropping(
+    const rtc::scoped_refptr<I420BufferInterface>& reference_frame,
+    const rtc::scoped_refptr<I420BufferInterface>& test_frame);
+
+// Returns a cropped and zoomed version of the reference video that matches up
+// to the test video. Frames are individually adjusted for cropping.
+rtc::scoped_refptr<Video> AdjustCropping(
+    const rtc::scoped_refptr<Video>& reference_video,
+    const rtc::scoped_refptr<Video>& test_video);
+
+}  // namespace test
+}  // namespace webrtc
+
+#endif  // RTC_TOOLS_FRAME_ANALYZER_VIDEO_GEOMETRY_ALIGNER_H_

rtc_tools/frame_analyzer/video_geometry_aligner_unittest.cc

@@ -0,0 +1,150 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
+
+#include <vector>
+
+#include "rtc_tools/frame_analyzer/video_quality_analysis.h"
+#include "rtc_tools/video_file_reader.h"
+#include "test/gtest.h"
+#include "test/testsupport/fileutils.h"
+
+namespace webrtc {
+namespace test {
+
+class VideoGeometryAlignerTest : public ::testing::Test {
+ protected:
+  void SetUp() {
+    reference_video_ =
+        OpenYuvFile(ResourcePath("foreman_128x96", "yuv"), 128, 96);
+    ASSERT_TRUE(reference_video_);
+
+    // Very simple 4x4 frame used for verying CropAndZoom.
+    const uint8_t data_y[] = {0, 1, 2,  3,  4,  5,  6,  7,
+                              8, 9, 10, 11, 12, 13, 14, 15};
+    const uint8_t data_u[] = {0, 1, 2, 3};
+    const uint8_t data_v[] = {0, 1, 2, 3};
+    test_frame_ = I420Buffer::Copy(
+        /* width= */ 4, /* height= */ 4, data_y, /* stride_y= */ 4, data_u,
+        /* stride_u= */ 2, data_v, /* stride_v= */ 2);
+  }
+
+  rtc::scoped_refptr<Video> reference_video_;
+  rtc::scoped_refptr<I420BufferInterface> test_frame_;
+};
+
+// Teach gtest how to compare CropRegions.
+bool operator==(const CropRegion& a, const CropRegion& b) {
+  return a.left == b.left && a.top == b.top && a.right == b.right &&
+         a.bottom == b.bottom;
+}
+
+TEST_F(VideoGeometryAlignerTest, CropAndZoomIdentity) {
+  const rtc::scoped_refptr<I420BufferInterface> frame =
+      reference_video_->GetFrame(0);
+
+  // Assume perfect match, i.e. SSIM == 1.
+  CropRegion identity_region;
+  EXPECT_EQ(1.0, Ssim(frame, CropAndZoom(identity_region, frame)));
+}
+
+TEST_F(VideoGeometryAlignerTest, CropAndZoomLeft) {
+  CropRegion region;
+  region.left = 2;
+  const rtc::scoped_refptr<I420BufferInterface> cropped_frame =
+      CropAndZoom(region, test_frame_);
+  EXPECT_EQ(std::vector<uint8_t>(
+                {2, 2, 3, 3, 6, 6, 7, 7, 10, 10, 11, 11, 14, 14, 15, 15}),
+            std::vector<uint8_t>(cropped_frame->DataY(),
+                                 cropped_frame->DataY() + 16));
+  EXPECT_EQ(
+      std::vector<uint8_t>({1, 1, 3, 3}),
+      std::vector<uint8_t>(cropped_frame->DataU(), cropped_frame->DataU() + 4));
+  EXPECT_EQ(
+      std::vector<uint8_t>({1, 1, 3, 3}),
+      std::vector<uint8_t>(cropped_frame->DataV(), cropped_frame->DataV() + 4));
+}
+
+TEST_F(VideoGeometryAlignerTest, CropAndZoomTop) {
+  CropRegion region;
+  region.top = 2;
+  const rtc::scoped_refptr<I420BufferInterface> cropped_frame =
+      CropAndZoom(region, test_frame_);
+  EXPECT_EQ(std::vector<uint8_t>(
+                {8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15, 12, 13, 14, 15}),
+            std::vector<uint8_t>(cropped_frame->DataY(),
+                                 cropped_frame->DataY() + 16));
+  EXPECT_EQ(
+      std::vector<uint8_t>({2, 3, 2, 3}),
+      std::vector<uint8_t>(cropped_frame->DataU(), cropped_frame->DataU() + 4));
+  EXPECT_EQ(
+      std::vector<uint8_t>({2, 3, 2, 3}),
+      std::vector<uint8_t>(cropped_frame->DataV(), cropped_frame->DataV() + 4));
+}
+
+TEST_F(VideoGeometryAlignerTest, CropAndZoomRight) {
+  CropRegion region;
+  region.right = 2;
+  const rtc::scoped_refptr<I420BufferInterface> cropped_frame =
+      CropAndZoom(region, test_frame_);
+  EXPECT_EQ(std::vector<uint8_t>(
+                {0, 0, 1, 1, 4, 4, 5, 5, 8, 8, 9, 9, 12, 12, 13, 13}),
+            std::vector<uint8_t>(cropped_frame->DataY(),
+                                 cropped_frame->DataY() + 16));
+  EXPECT_EQ(
+      std::vector<uint8_t>({0, 0, 2, 2}),
+      std::vector<uint8_t>(cropped_frame->DataU(), cropped_frame->DataU() + 4));
+  EXPECT_EQ(
+      std::vector<uint8_t>({0, 0, 2, 2}),
+      std::vector<uint8_t>(cropped_frame->DataV(), cropped_frame->DataV() + 4));
+}
+
+TEST_F(VideoGeometryAlignerTest, CropAndZoomBottom) {
+  CropRegion region;
+  region.bottom = 2;
+  const rtc::scoped_refptr<I420BufferInterface> cropped_frame =
+      CropAndZoom(region, test_frame_);
+  EXPECT_EQ(
+      std::vector<uint8_t>({0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 4, 5, 6, 7}),
+      std::vector<uint8_t>(cropped_frame->DataY(),
+                           cropped_frame->DataY() + 16));
+  EXPECT_EQ(
+      std::vector<uint8_t>({0, 1, 0, 1}),
+      std::vector<uint8_t>(cropped_frame->DataU(), cropped_frame->DataU() + 4));
+  EXPECT_EQ(
+      std::vector<uint8_t>({0, 1, 0, 1}),
+      std::vector<uint8_t>(cropped_frame->DataV(), cropped_frame->DataV() + 4));
+}
+
+TEST_F(VideoGeometryAlignerTest, CalculateCropRegionIdentity) {
+  const rtc::scoped_refptr<I420BufferInterface> frame =
+      reference_video_->GetFrame(0);
+  CropRegion identity_region;
+  EXPECT_EQ(identity_region, CalculateCropRegion(frame, frame));
+}
+
+TEST_F(VideoGeometryAlignerTest, CalculateCropRegionArbitrary) {
+  // Arbitrary crop region.
+  CropRegion crop_region;
+  crop_region.left = 2;
+  crop_region.top = 4;
+  crop_region.right = 5;
+  crop_region.bottom = 3;
+
+  const rtc::scoped_refptr<I420BufferInterface> frame =
+      reference_video_->GetFrame(0);
+
+  EXPECT_EQ(crop_region,
+            CalculateCropRegion(frame, CropAndZoom(crop_region, frame)));
+}
+
+}  // namespace test
+}  // namespace webrtc