Reland "Implement Optimized CropAndScale for ObjCFrameBuffer"

This is a reland of commit 9204302248

Original change's description:
> Implement Optimized CropAndScale for ObjCFrameBuffer
>
> The default implementation of CropAndScale uses ToI420() and then Scale,
> and this implementation behaves inefficiently with RTCCVPixelBuffer.
>
> Bug: None
> Change-Id: I422ef80d124db0354a2e696892e882a78db445bb
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/271140
> Reviewed-by: Kári Helgason <kthelgason@webrtc.org>
> Commit-Queue: Daniel.L (Byoungchan) Lee <daniel.l@hpcnt.com>
> Auto-Submit: Daniel.L (Byoungchan) Lee <daniel.l@hpcnt.com>
> Commit-Queue: Kári Helgason <kthelgason@webrtc.org>
> Cr-Commit-Position: refs/heads/main@{#37877}

Bug: None
Change-Id: Ie74146a33c1f54d0c988978bd925671afe699d05
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/272740
Reviewed-by: Kári Helgason <kthelgason@webrtc.org>
Commit-Queue: Daniel.L (Byoungchan) Lee <daniel.l@hpcnt.com>
Cr-Commit-Position: refs/heads/main@{#37887}

sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer.mm

@@ -99,6 +99,22 @@
   return _i420Buffer->DataV();
 }
 
+- (id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)>)cropAndScaleWith:(int)offsetX
+                                                   offsetY:(int)offsetY
+                                                 cropWidth:(int)cropWidth
+                                                cropHeight:(int)cropHeight
+                                                scaleWidth:(int)scaleWidth
+                                               scaleHeight:(int)scaleHeight {
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer =
+      _i420Buffer->CropAndScale(offsetX, offsetY, cropWidth, cropHeight, scaleWidth, scaleHeight);
+  RTC_DCHECK_EQ(scaled_buffer->type(), webrtc::VideoFrameBuffer::Type::kI420);
+  // Calling ToI420() doesn't do any conversions.
+  rtc::scoped_refptr<webrtc::I420BufferInterface> buffer = scaled_buffer->ToI420();
+  RTC_OBJC_TYPE(RTCI420Buffer) *result =
+      [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:buffer];
+  return result;
+}
+
 - (id<RTC_OBJC_TYPE(RTCI420Buffer)>)toI420 {
   return self;
 }

sdk/objc/base/RTCVideoFrameBuffer.h

@@ -27,6 +27,14 @@ RTC_OBJC_EXPORT
 
 - (id<RTC_OBJC_TYPE(RTCI420Buffer)>)toI420;
 
+@optional
+- (id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)>)cropAndScaleWith:(int)offsetX
+                                                   offsetY:(int)offsetY
+                                                 cropWidth:(int)cropWidth
+                                                cropHeight:(int)cropHeight
+                                                scaleWidth:(int)scaleWidth
+                                               scaleHeight:(int)scaleHeight;
+
 @end
 
 NS_ASSUME_NONNULL_END

sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.mm

@@ -153,6 +153,21 @@
 
   return YES;
 }
+- (id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)>)cropAndScaleWith:(int)offsetX
+                                                   offsetY:(int)offsetY
+                                                 cropWidth:(int)cropWidth
+                                                cropHeight:(int)cropHeight
+                                                scaleWidth:(int)scaleWidth
+                                               scaleHeight:(int)scaleHeight {
+  return [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc]
+      initWithPixelBuffer:_pixelBuffer
+             adaptedWidth:scaleWidth
+            adaptedHeight:scaleHeight
+                cropWidth:cropWidth * _cropWidth / _width
+               cropHeight:cropHeight * _cropHeight / _height
+                    cropX:_cropX + offsetX * _cropWidth / _width
+                    cropY:_cropY + offsetY * _cropHeight / _height];
+}
 
 - (id<RTC_OBJC_TYPE(RTCI420Buffer)>)toI420 {
   const OSType pixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer);

sdk/objc/native/src/objc_frame_buffer.h

@@ -33,6 +33,12 @@ class ObjCFrameBuffer : public VideoFrameBuffer {
   int height() const override;
 
   rtc::scoped_refptr<I420BufferInterface> ToI420() override;
+  rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x,
+                                                    int offset_y,
+                                                    int crop_width,
+                                                    int crop_height,
+                                                    int scaled_width,
+                                                    int scaled_height) override;
 
   id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> wrapped_frame_buffer() const;
 

sdk/objc/native/src/objc_frame_buffer.mm

@@ -70,6 +70,27 @@ rtc::scoped_refptr<I420BufferInterface> ObjCFrameBuffer::ToI420() {
   return rtc::make_ref_counted<ObjCI420FrameBuffer>([frame_buffer_ toI420]);
 }
 
+rtc::scoped_refptr<VideoFrameBuffer> ObjCFrameBuffer::CropAndScale(int offset_x,
+                                                                   int offset_y,
+                                                                   int crop_width,
+                                                                   int crop_height,
+                                                                   int scaled_width,
+                                                                   int scaled_height) {
+  if ([frame_buffer_ respondsToSelector:@selector
+                     (cropAndScaleWith:offsetY:cropWidth:cropHeight:scaleWidth:scaleHeight:)]) {
+    return rtc::make_ref_counted<ObjCFrameBuffer>([frame_buffer_ cropAndScaleWith:offset_x
+                                                                          offsetY:offset_y
+                                                                        cropWidth:crop_width
+                                                                       cropHeight:crop_height
+                                                                       scaleWidth:scaled_width
+                                                                      scaleHeight:scaled_height]);
+  }
+
+  // Use the default implementation.
+  return VideoFrameBuffer::CropAndScale(
+      offset_x, offset_y, crop_width, crop_height, scaled_width, scaled_height);
+}
+
 id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> ObjCFrameBuffer::wrapped_frame_buffer() const {
   return frame_buffer_;
 }

sdk/objc/unittests/RTCCVPixelBuffer_xctest.mm

@@ -21,6 +21,44 @@
 #include "common_video/libyuv/include/webrtc_libyuv.h"
 #include "third_party/libyuv/include/libyuv.h"
 
+namespace {
+
+struct ToI420WithCropAndScaleSetting {
+  int inputWidth;
+  int inputHeight;
+  int offsetX;
+  int offsetY;
+  int cropWidth;
+  int cropHeight;
+  int scaleWidth;
+  int scaleHeight;
+};
+
+constexpr const ToI420WithCropAndScaleSetting kToI420WithCropAndScaleSettings[] = {
+    ToI420WithCropAndScaleSetting{
+        .inputWidth = 640,
+        .inputHeight = 360,
+        .offsetX = 0,
+        .offsetY = 0,
+        .cropWidth = 640,
+        .cropHeight = 360,
+        .scaleWidth = 320,
+        .scaleHeight = 180,
+    },
+    ToI420WithCropAndScaleSetting{
+        .inputWidth = 640,
+        .inputHeight = 360,
+        .offsetX = 160,
+        .offsetY = 90,
+        .cropWidth = 160,
+        .cropHeight = 90,
+        .scaleWidth = 320,
+        .scaleHeight = 180,
+    },
+};
+
+}  // namespace
+
 @interface RTCCVPixelBufferTests : XCTestCase
 @end
 
@@ -183,6 +221,76 @@
   [self toI420WithPixelFormat:kCVPixelFormatType_32ARGB];
 }
 
+- (void)testToI420WithCropAndScale_NV12 {
+  for (const auto &setting : kToI420WithCropAndScaleSettings) {
+    [self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
+                                        setting:setting];
+  }
+}
+
+- (void)testToI420WithCropAndScale_32BGRA {
+  for (const auto &setting : kToI420WithCropAndScaleSettings) {
+    [self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_32BGRA setting:setting];
+  }
+}
+
+- (void)testToI420WithCropAndScale_32ARGB {
+  for (const auto &setting : kToI420WithCropAndScaleSettings) {
+    [self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_32ARGB setting:setting];
+  }
+}
+
+- (void)testScaleBufferTest {
+  CVPixelBufferRef pixelBufferRef = NULL;
+  CVPixelBufferCreate(
+      NULL, 1920, 1080, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef);
+
+  rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer = CreateI420Gradient(1920, 1080);
+  CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef);
+
+  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
+      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
+
+  XCTAssertEqual(buffer.width, 1920);
+  XCTAssertEqual(buffer.height, 1080);
+  XCTAssertEqual(buffer.cropX, 0);
+  XCTAssertEqual(buffer.cropY, 0);
+  XCTAssertEqual(buffer.cropWidth, 1920);
+  XCTAssertEqual(buffer.cropHeight, 1080);
+
+  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer2 =
+      (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)[buffer cropAndScaleWith:320
+                                                          offsetY:180
+                                                        cropWidth:1280
+                                                       cropHeight:720
+                                                       scaleWidth:960
+                                                      scaleHeight:540];
+
+  XCTAssertEqual(buffer2.width, 960);
+  XCTAssertEqual(buffer2.height, 540);
+  XCTAssertEqual(buffer2.cropX, 320);
+  XCTAssertEqual(buffer2.cropY, 180);
+  XCTAssertEqual(buffer2.cropWidth, 1280);
+  XCTAssertEqual(buffer2.cropHeight, 720);
+
+  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer3 =
+      (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)[buffer2 cropAndScaleWith:240
+                                                           offsetY:135
+                                                         cropWidth:480
+                                                        cropHeight:270
+                                                        scaleWidth:320
+                                                       scaleHeight:180];
+
+  XCTAssertEqual(buffer3.width, 320);
+  XCTAssertEqual(buffer3.height, 180);
+  XCTAssertEqual(buffer3.cropX, 640);
+  XCTAssertEqual(buffer3.cropY, 360);
+  XCTAssertEqual(buffer3.cropWidth, 640);
+  XCTAssertEqual(buffer3.cropHeight, 360);
+
+  CVBufferRelease(pixelBufferRef);
+}
+
 #pragma mark - Shared test code
 
 - (void)cropAndScaleTestWithNV12 {
@@ -305,4 +413,49 @@
   CVBufferRelease(pixelBufferRef);
 }
 
+- (void)toI420WithCropAndScaleWithPixelFormat:(OSType)pixelFormat
+                                      setting:(const ToI420WithCropAndScaleSetting &)setting {
+  rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer =
+      CreateI420Gradient(setting.inputWidth, setting.inputHeight);
+
+  CVPixelBufferRef pixelBufferRef = NULL;
+  CVPixelBufferCreate(
+      NULL, setting.inputWidth, setting.inputHeight, pixelFormat, NULL, &pixelBufferRef);
+
+  CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef);
+
+  RTC_OBJC_TYPE(RTCI420Buffer) *objcI420Buffer =
+      [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:i420Buffer];
+  RTC_OBJC_TYPE(RTCI420Buffer) *scaledObjcI420Buffer =
+      (RTC_OBJC_TYPE(RTCI420Buffer) *)[objcI420Buffer cropAndScaleWith:setting.offsetX
+                                                               offsetY:setting.offsetY
+                                                             cropWidth:setting.cropWidth
+                                                            cropHeight:setting.cropHeight
+                                                            scaleWidth:setting.scaleWidth
+                                                           scaleHeight:setting.scaleHeight];
+  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
+      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
+  id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> scaledBuffer =
+      [buffer cropAndScaleWith:setting.offsetX
+                       offsetY:setting.offsetY
+                     cropWidth:setting.cropWidth
+                    cropHeight:setting.cropHeight
+                    scaleWidth:setting.scaleWidth
+                   scaleHeight:setting.scaleHeight];
+  XCTAssertTrue([scaledBuffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]);
+
+  RTC_OBJC_TYPE(RTCI420Buffer) *fromCVPixelBuffer = [scaledBuffer toI420];
+
+  double psnr =
+      I420PSNR(*[scaledObjcI420Buffer nativeI420Buffer], *[fromCVPixelBuffer nativeI420Buffer]);
+  double target = webrtc::kPerfectPSNR;
+  if (pixelFormat != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
+    // libyuv's I420ToRGB functions seem to lose some quality.
+    target = 19.0;
+  }
+  XCTAssertGreaterThanOrEqual(psnr, target);
+
+  CVBufferRelease(pixelBufferRef);
+}
+
 @end