Add support for NV12 frame generator

Bug: b/240540204
Change-Id: Id2205e8bd0dfd59476dcd68c32c4981f98b51422
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/278402
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38322}

api/test/create_frame_generator.cc

@@ -47,6 +47,23 @@ std::unique_ptr<FrameGeneratorInterface> CreateFromYuvFileFrameGenerator(
                                             frame_repeat_count);
 }
 
+std::unique_ptr<FrameGeneratorInterface> CreateFromNV12FileFrameGenerator(
+    std::vector<std::string> filenames,
+    size_t width,
+    size_t height,
+    int frame_repeat_count) {
+  RTC_DCHECK(!filenames.empty());
+  std::vector<FILE*> files;
+  for (const std::string& filename : filenames) {
+    FILE* file = fopen(filename.c_str(), "rb");
+    RTC_DCHECK(file != nullptr) << "Failed to open: '" << filename << "'\n";
+    files.push_back(file);
+  }
+
+  return std::make_unique<NV12FileGenerator>(files, width, height,
+                                             frame_repeat_count);
+}
+
 std::unique_ptr<FrameGeneratorInterface> CreateFromIvfFileFrameGenerator(
     std::string filename) {
   return std::make_unique<IvfVideoFrameGenerator>(std::move(filename));

api/test/create_frame_generator.h

@@ -41,6 +41,15 @@ std::unique_ptr<FrameGeneratorInterface> CreateFromYuvFileFrameGenerator(
     size_t height,
     int frame_repeat_count);
 
+// Creates a frame generator that repeatedly plays a set of nv12 files.
+// The frame_repeat_count determines how many times each frame is shown,
+// with 1 = show each frame once, etc.
+std::unique_ptr<FrameGeneratorInterface> CreateFromNV12FileFrameGenerator(
+    std::vector<std::string> filenames,
+    size_t width,
+    size_t height,
+    int frame_repeat_count = 1);
+
 // Creates a frame generator that repeatedly plays an ivf file.
 std::unique_ptr<FrameGeneratorInterface> CreateFromIvfFileFrameGenerator(
     std::string filename);

common_video/libyuv/include/webrtc_libyuv.h

@@ -39,6 +39,7 @@ enum class VideoType {
   kUYVY,
   kMJPEG,
   kBGRA,
+  kNV12,
 };
 
 // This is the max PSNR value our algorithms can return.

common_video/libyuv/webrtc_libyuv.cc

@@ -26,7 +26,8 @@ size_t CalcBufferSize(VideoType type, int width, int height) {
   switch (type) {
     case VideoType::kI420:
     case VideoType::kIYUV:
-    case VideoType::kYV12: {
+    case VideoType::kYV12:
+    case VideoType::kNV12: {
       int half_width = (width + 1) >> 1;
       int half_height = (height + 1) >> 1;
       buffer_size = width * height + half_width * half_height * 2;
@@ -105,6 +106,8 @@ int ConvertVideoType(VideoType video_type) {
       return libyuv::FOURCC_ARGB;
     case VideoType::kBGRA:
       return libyuv::FOURCC_BGRA;
+    case VideoType::kNV12:
+      return libyuv::FOURCC_NV12;
   }
   RTC_DCHECK_NOTREACHED();
   return libyuv::FOURCC_ANY;

test/frame_generator.cc

@@ -205,6 +205,68 @@ bool YuvFileGenerator::ReadNextFrame() {
   return frame_index_ != prev_frame_index || file_index_ != prev_file_index;
 }
 
+NV12FileGenerator::NV12FileGenerator(std::vector<FILE*> files,
+                                     size_t width,
+                                     size_t height,
+                                     int frame_repeat_count)
+    : file_index_(0),
+      frame_index_(std::numeric_limits<size_t>::max()),
+      files_(files),
+      width_(width),
+      height_(height),
+      frame_size_(CalcBufferSize(VideoType::kNV12,
+                                 static_cast<int>(width_),
+                                 static_cast<int>(height_))),
+      frame_buffer_(new uint8_t[frame_size_]),
+      frame_display_count_(frame_repeat_count),
+      current_display_count_(0) {
+  RTC_DCHECK_GT(width, 0);
+  RTC_DCHECK_GT(height, 0);
+  RTC_DCHECK_GT(frame_repeat_count, 0);
+}
+
+NV12FileGenerator::~NV12FileGenerator() {
+  for (FILE* file : files_)
+    fclose(file);
+}
+
+FrameGeneratorInterface::VideoFrameData NV12FileGenerator::NextFrame() {
+  // Empty update by default.
+  VideoFrame::UpdateRect update_rect{0, 0, 0, 0};
+  if (current_display_count_ == 0) {
+    const bool got_new_frame = ReadNextFrame();
+    // Full update on a new frame from file.
+    if (got_new_frame) {
+      update_rect = VideoFrame::UpdateRect{0, 0, static_cast<int>(width_),
+                                           static_cast<int>(height_)};
+    }
+  }
+  if (++current_display_count_ >= frame_display_count_)
+    current_display_count_ = 0;
+
+  return VideoFrameData(last_read_buffer_, update_rect);
+}
+
+bool NV12FileGenerator::ReadNextFrame() {
+  size_t prev_frame_index = frame_index_;
+  size_t prev_file_index = file_index_;
+  last_read_buffer_ = test::ReadNV12Buffer(
+      static_cast<int>(width_), static_cast<int>(height_), files_[file_index_]);
+  ++frame_index_;
+  if (!last_read_buffer_) {
+    // No more frames to read in this file, rewind and move to next file.
+    rewind(files_[file_index_]);
+
+    frame_index_ = 0;
+    file_index_ = (file_index_ + 1) % files_.size();
+    last_read_buffer_ =
+        test::ReadNV12Buffer(static_cast<int>(width_),
+                             static_cast<int>(height_), files_[file_index_]);
+    RTC_CHECK(last_read_buffer_);
+  }
+  return frame_index_ != prev_frame_index || file_index_ != prev_file_index;
+}
+
 SlideGenerator::SlideGenerator(int width, int height, int frame_repeat_count)
     : width_(width),
       height_(height),

test/frame_generator.h

@@ -17,6 +17,7 @@
 #include "api/scoped_refptr.h"
 #include "api/test/frame_generator_interface.h"
 #include "api/video/i420_buffer.h"
+#include "api/video/nv12_buffer.h"
 #include "api/video/video_frame.h"
 #include "api/video/video_frame_buffer.h"
 #include "api/video/video_source_interface.h"
@@ -76,8 +77,7 @@ class YuvFileGenerator : public FrameGeneratorInterface {
 
   VideoFrameData NextFrame() override;
   void ChangeResolution(size_t width, size_t height) override {
-    RTC_LOG(LS_WARNING)
-        << "ScrollingImageFrameGenerator::ChangeResolution not implemented";
+    RTC_LOG(LS_WARNING) << "YuvFileGenerator::ChangeResolution not implemented";
   }
 
  private:
@@ -97,6 +97,38 @@ class YuvFileGenerator : public FrameGeneratorInterface {
   rtc::scoped_refptr<I420Buffer> last_read_buffer_;
 };
 
+class NV12FileGenerator : public FrameGeneratorInterface {
+ public:
+  NV12FileGenerator(std::vector<FILE*> files,
+                    size_t width,
+                    size_t height,
+                    int frame_repeat_count);
+
+  ~NV12FileGenerator();
+
+  VideoFrameData NextFrame() override;
+  void ChangeResolution(size_t width, size_t height) override {
+    RTC_LOG(LS_WARNING)
+        << "NV12FileGenerator::ChangeResolution not implemented";
+  }
+
+ private:
+  // Returns true if the new frame was loaded.
+  // False only in case of a single file with a single frame in it.
+  bool ReadNextFrame();
+
+  size_t file_index_;
+  size_t frame_index_;
+  const std::vector<FILE*> files_;
+  const size_t width_;
+  const size_t height_;
+  const size_t frame_size_;
+  const std::unique_ptr<uint8_t[]> frame_buffer_;
+  const int frame_display_count_;
+  int current_display_count_;
+  rtc::scoped_refptr<NV12Buffer> last_read_buffer_;
+};
+
 // SlideGenerator works similarly to YuvFileGenerator but it fills the frames
 // with randomly sized and colored squares instead of reading their content
 // from files.

test/frame_generator_unittest.cc

@@ -27,28 +27,44 @@
 namespace webrtc {
 namespace test {
 
-static const int kFrameWidth = 4;
-static const int kFrameHeight = 4;
+constexpr int kFrameWidth = 4;
+constexpr int kFrameHeight = 4;
+constexpr int y_size = kFrameWidth * kFrameHeight;
+constexpr int uv_size = ((kFrameHeight + 1) / 2) * ((kFrameWidth + 1) / 2);
 
 class FrameGeneratorTest : public ::testing::Test {
  public:
   void SetUp() override {
-    two_frame_filename_ =
+    two_frame_yuv_filename_ =
         test::TempFilename(test::OutputPath(), "2_frame_yuv_file");
-    one_frame_filename_ =
+    one_frame_yuv_filename_ =
         test::TempFilename(test::OutputPath(), "1_frame_yuv_file");
+    two_frame_nv12_filename_ =
+        test::TempFilename(test::OutputPath(), "2_frame_nv12_file");
+    one_frame_nv12_filename_ =
+        test::TempFilename(test::OutputPath(), "1_frame_nv12_file");
 
-    FILE* file = fopen(two_frame_filename_.c_str(), "wb");
+    FILE* file = fopen(two_frame_yuv_filename_.c_str(), "wb");
     WriteYuvFile(file, 0, 0, 0);
-    WriteYuvFile(file, 127, 127, 127);
+    WriteYuvFile(file, 127, 128, 129);
     fclose(file);
-    file = fopen(one_frame_filename_.c_str(), "wb");
+    file = fopen(one_frame_yuv_filename_.c_str(), "wb");
     WriteYuvFile(file, 255, 255, 255);
     fclose(file);
+    file = fopen(two_frame_nv12_filename_.c_str(), "wb");
+    WriteNV12File(file, 0, 0, 0);
+    WriteNV12File(file, 127, 128, 129);
+    fclose(file);
+    file = fopen(one_frame_nv12_filename_.c_str(), "wb");
+    WriteNV12File(file, 255, 255, 255);
+    fclose(file);
   }
+
   void TearDown() override {
-    remove(one_frame_filename_.c_str());
-    remove(two_frame_filename_.c_str());
+    remove(one_frame_yuv_filename_.c_str());
+    remove(two_frame_yuv_filename_.c_str());
+    remove(one_frame_nv12_filename_.c_str());
+    remove(two_frame_nv12_filename_.c_str());
   }
 
  protected:
@@ -63,6 +79,19 @@ class FrameGeneratorTest : public ::testing::Test {
     fwrite(plane_buffer.get(), 1, uv_size, file);
   }
 
+  void WriteNV12File(FILE* file, uint8_t y, uint8_t u, uint8_t v) {
+    RTC_DCHECK(file);
+    uint8_t plane_buffer[y_size];
+
+    memset(&plane_buffer, y, y_size);
+    fwrite(&plane_buffer, 1, y_size, file);
+    for (size_t i = 0; i < uv_size; ++i) {
+      plane_buffer[2 * i] = u;
+      plane_buffer[2 * i + 1] = v;
+    }
+    fwrite(&plane_buffer, 1, 2 * uv_size, file);
+  }
+
   void CheckFrameAndMutate(const FrameGeneratorInterface::VideoFrameData& frame,
                            uint8_t y,
                            uint8_t u,
@@ -102,69 +131,131 @@ class FrameGeneratorTest : public ::testing::Test {
     return hash;
   }
 
-  std::string two_frame_filename_;
-  std::string one_frame_filename_;
-  const int y_size = kFrameWidth * kFrameHeight;
-  const int uv_size = ((kFrameHeight + 1) / 2) * ((kFrameWidth + 1) / 2);
+  std::string two_frame_yuv_filename_;
+  std::string one_frame_yuv_filename_;
+  std::string two_frame_nv12_filename_;
+  std::string one_frame_nv12_filename_;
 };
 
-TEST_F(FrameGeneratorTest, SingleFrameFile) {
+TEST_F(FrameGeneratorTest, SingleFrameYuvFile) {
   std::unique_ptr<FrameGeneratorInterface> generator(
       CreateFromYuvFileFrameGenerator(
-          std::vector<std::string>(1, one_frame_filename_), kFrameWidth,
+          std::vector<std::string>(1, one_frame_yuv_filename_), kFrameWidth,
           kFrameHeight, 1));
   CheckFrameAndMutate(generator->NextFrame(), 255, 255, 255);
   CheckFrameAndMutate(generator->NextFrame(), 255, 255, 255);
 }
 
-TEST_F(FrameGeneratorTest, TwoFrameFile) {
+TEST_F(FrameGeneratorTest, TwoFrameYuvFile) {
   std::unique_ptr<FrameGeneratorInterface> generator(
       CreateFromYuvFileFrameGenerator(
-          std::vector<std::string>(1, two_frame_filename_), kFrameWidth,
+          std::vector<std::string>(1, two_frame_yuv_filename_), kFrameWidth,
           kFrameHeight, 1));
   CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);
-  CheckFrameAndMutate(generator->NextFrame(), 127, 127, 127);
+  CheckFrameAndMutate(generator->NextFrame(), 127, 128, 129);
   CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);
 }
 
-TEST_F(FrameGeneratorTest, MultipleFrameFiles) {
+TEST_F(FrameGeneratorTest, MultipleFrameYuvFiles) {
   std::vector<std::string> files;
-  files.push_back(two_frame_filename_);
-  files.push_back(one_frame_filename_);
+  files.push_back(two_frame_yuv_filename_);
+  files.push_back(one_frame_yuv_filename_);
 
   std::unique_ptr<FrameGeneratorInterface> generator(
       CreateFromYuvFileFrameGenerator(files, kFrameWidth, kFrameHeight, 1));
   CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);
-  CheckFrameAndMutate(generator->NextFrame(), 127, 127, 127);
+  CheckFrameAndMutate(generator->NextFrame(), 127, 128, 129);
   CheckFrameAndMutate(generator->NextFrame(), 255, 255, 255);
   CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);
 }
 
-TEST_F(FrameGeneratorTest, TwoFrameFileWithRepeat) {
+TEST_F(FrameGeneratorTest, TwoFrameYuvFileWithRepeat) {
   const int kRepeatCount = 3;
   std::unique_ptr<FrameGeneratorInterface> generator(
       CreateFromYuvFileFrameGenerator(
-          std::vector<std::string>(1, two_frame_filename_), kFrameWidth,
+          std::vector<std::string>(1, two_frame_yuv_filename_), kFrameWidth,
           kFrameHeight, kRepeatCount));
   for (int i = 0; i < kRepeatCount; ++i)
     CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);
   for (int i = 0; i < kRepeatCount; ++i)
-    CheckFrameAndMutate(generator->NextFrame(), 127, 127, 127);
+    CheckFrameAndMutate(generator->NextFrame(), 127, 128, 129);
   CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);
 }
 
-TEST_F(FrameGeneratorTest, MultipleFrameFilesWithRepeat) {
+TEST_F(FrameGeneratorTest, MultipleFrameYuvFilesWithRepeat) {
   const int kRepeatCount = 3;
   std::vector<std::string> files;
-  files.push_back(two_frame_filename_);
-  files.push_back(one_frame_filename_);
+  files.push_back(two_frame_yuv_filename_);
+  files.push_back(one_frame_yuv_filename_);
   std::unique_ptr<FrameGeneratorInterface> generator(
       CreateFromYuvFileFrameGenerator(files, kFrameWidth, kFrameHeight,
                                       kRepeatCount));
   for (int i = 0; i < kRepeatCount; ++i)
     CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);
   for (int i = 0; i < kRepeatCount; ++i)
-    CheckFrameAndMutate(generator->NextFrame(), 127, 127, 127);
+    CheckFrameAndMutate(generator->NextFrame(), 127, 128, 129);
+  for (int i = 0; i < kRepeatCount; ++i)
+    CheckFrameAndMutate(generator->NextFrame(), 255, 255, 255);
+  CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);
+}
+
+TEST_F(FrameGeneratorTest, SingleFrameNV12File) {
+  std::unique_ptr<FrameGeneratorInterface> generator(
+      CreateFromNV12FileFrameGenerator(
+          std::vector<std::string>(1, one_frame_nv12_filename_), kFrameWidth,
+          kFrameHeight, 1));
+  CheckFrameAndMutate(generator->NextFrame(), 255, 255, 255);
+  CheckFrameAndMutate(generator->NextFrame(), 255, 255, 255);
+}
+
+TEST_F(FrameGeneratorTest, TwoFrameNV12File) {
+  std::unique_ptr<FrameGeneratorInterface> generator(
+      CreateFromNV12FileFrameGenerator(
+          std::vector<std::string>(1, two_frame_nv12_filename_), kFrameWidth,
+          kFrameHeight, 1));
+  CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);
+  CheckFrameAndMutate(generator->NextFrame(), 127, 128, 129);
+  CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);
+}
+
+TEST_F(FrameGeneratorTest, MultipleFrameNV12Files) {
+  std::vector<std::string> files;
+  files.push_back(two_frame_nv12_filename_);
+  files.push_back(one_frame_nv12_filename_);
+
+  std::unique_ptr<FrameGeneratorInterface> generator(
+      CreateFromNV12FileFrameGenerator(files, kFrameWidth, kFrameHeight, 1));
+  CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);
+  CheckFrameAndMutate(generator->NextFrame(), 127, 128, 129);
+  CheckFrameAndMutate(generator->NextFrame(), 255, 255, 255);
+  CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);
+}
+
+TEST_F(FrameGeneratorTest, TwoFrameNV12FileWithRepeat) {
+  const int kRepeatCount = 3;
+  std::unique_ptr<FrameGeneratorInterface> generator(
+      CreateFromNV12FileFrameGenerator(
+          std::vector<std::string>(1, two_frame_nv12_filename_), kFrameWidth,
+          kFrameHeight, kRepeatCount));
+  for (int i = 0; i < kRepeatCount; ++i)
+    CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);
+  for (int i = 0; i < kRepeatCount; ++i)
+    CheckFrameAndMutate(generator->NextFrame(), 127, 128, 129);
+  CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);
+}
+
+TEST_F(FrameGeneratorTest, MultipleFrameNV12FilesWithRepeat) {
+  const int kRepeatCount = 3;
+  std::vector<std::string> files;
+  files.push_back(two_frame_nv12_filename_);
+  files.push_back(one_frame_nv12_filename_);
+  std::unique_ptr<FrameGeneratorInterface> generator(
+      CreateFromNV12FileFrameGenerator(files, kFrameWidth, kFrameHeight,
+                                       kRepeatCount));
+  for (int i = 0; i < kRepeatCount; ++i)
+    CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);
+  for (int i = 0; i < kRepeatCount; ++i)
+    CheckFrameAndMutate(generator->NextFrame(), 127, 128, 129);
   for (int i = 0; i < kRepeatCount; ++i)
     CheckFrameAndMutate(generator->NextFrame(), 255, 255, 255);
   CheckFrameAndMutate(generator->NextFrame(), 0, 0, 0);

test/frame_utils.cc

@@ -14,6 +14,7 @@
 #include <string.h>
 
 #include "api/video/i420_buffer.h"
+#include "api/video/nv12_buffer.h"
 #include "api/video/video_frame.h"
 
 namespace webrtc {
@@ -87,5 +88,17 @@ rtc::scoped_refptr<I420Buffer> ReadI420Buffer(int width, int height, FILE* f) {
   return buffer;
 }
 
+rtc::scoped_refptr<NV12Buffer> ReadNV12Buffer(int width, int height, FILE* f) {
+  rtc::scoped_refptr<NV12Buffer> buffer(NV12Buffer::Create(width, height));
+  size_t size_y = static_cast<size_t>(width) * height;
+  size_t size_uv = static_cast<size_t>(width + width % 2) * ((height + 1) / 2);
+
+  if (fread(buffer->MutableDataY(), 1, size_y, f) < size_y)
+    return nullptr;
+  if (fread(buffer->MutableDataUV(), 1, size_uv, f) < size_uv)
+    return nullptr;
+  return buffer;
+}
+
 }  // namespace test
 }  // namespace webrtc

test/frame_utils.h

@@ -13,6 +13,7 @@
 #include <stdint.h>
 
 #include "api/scoped_refptr.h"
+#include "api/video/nv12_buffer.h"
 
 namespace webrtc {
 class I420Buffer;
@@ -42,6 +43,8 @@ bool FrameBufsEqual(const rtc::scoped_refptr<webrtc::VideoFrameBuffer>& f1,
 
 rtc::scoped_refptr<I420Buffer> ReadI420Buffer(int width, int height, FILE*);
 
+rtc::scoped_refptr<NV12Buffer> ReadNV12Buffer(int width, int height, FILE*);
+
 }  // namespace test
 }  // namespace webrtc