mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-13 13:50:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Reland "Implement Optimized CropAndScale for ObjCFrameBuffer"

Browse source 
This is a reland of commit 9204302248

Original change's description:
> Implement Optimized CropAndScale for ObjCFrameBuffer
>
> The default implementation of CropAndScale uses ToI420() and then Scale,
> and this implementation behaves inefficiently with RTCCVPixelBuffer.
>
> Bug: None
> Change-Id: I422ef80d124db0354a2e696892e882a78db445bb
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/271140
> Reviewed-by: Kári Helgason <kthelgason@webrtc.org>
> Commit-Queue: Daniel.L (Byoungchan) Lee <daniel.l@hpcnt.com>
> Auto-Submit: Daniel.L (Byoungchan) Lee <daniel.l@hpcnt.com>
> Commit-Queue: Kári Helgason <kthelgason@webrtc.org>
> Cr-Commit-Position: refs/heads/main@{#37877}

Bug: None
Change-Id: Ie74146a33c1f54d0c988978bd925671afe699d05
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/272740
Reviewed-by: Kári Helgason <kthelgason@webrtc.org>
Commit-Queue: Daniel.L (Byoungchan) Lee <daniel.l@hpcnt.com>
Cr-Commit-Position: refs/heads/main@{#37887}
This commit is contained in:
Byoungchan Lee 2022-08-24 01:49:29 +09:00 committed by WebRTC LUCI CQ
parent a7dcd29c1f
commit 5aa3b073ad
6 changed files with 219 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
sdk/objc/api/video_frame_buffer/RTCNativeI420Buffer.mm
View file

@ -99,6 +99,22 @@
return _i420Buffer->DataV();
}
- (id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)>)cropAndScaleWith:(int)offsetX
offsetY:(int)offsetY
cropWidth:(int)cropWidth
cropHeight:(int)cropHeight
scaleWidth:(int)scaleWidth
scaleHeight:(int)scaleHeight {
rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer =
_i420Buffer->CropAndScale(offsetX, offsetY, cropWidth, cropHeight, scaleWidth, scaleHeight);
RTC_DCHECK_EQ(scaled_buffer->type(), webrtc::VideoFrameBuffer::Type::kI420);
// Calling ToI420() doesn't do any conversions.
rtc::scoped_refptr<webrtc::I420BufferInterface> buffer = scaled_buffer->ToI420();
RTC_OBJC_TYPE(RTCI420Buffer) *result =
[[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:buffer];
return result;
}
- (id<RTC_OBJC_TYPE(RTCI420Buffer)>)toI420 {
return self;
}

8
sdk/objc/base/RTCVideoFrameBuffer.h
View file

@ -27,6 +27,14 @@ RTC_OBJC_EXPORT
- (id<RTC_OBJC_TYPE(RTCI420Buffer)>)toI420;
@optional
- (id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)>)cropAndScaleWith:(int)offsetX
offsetY:(int)offsetY
cropWidth:(int)cropWidth
cropHeight:(int)cropHeight
scaleWidth:(int)scaleWidth
scaleHeight:(int)scaleHeight;
@end
NS_ASSUME_NONNULL_END

15
sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.mm
View file

@ -153,6 +153,21 @@
return YES;
}
- (id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)>)cropAndScaleWith:(int)offsetX
offsetY:(int)offsetY
cropWidth:(int)cropWidth
cropHeight:(int)cropHeight
scaleWidth:(int)scaleWidth
scaleHeight:(int)scaleHeight {
return [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc]
initWithPixelBuffer:_pixelBuffer
adaptedWidth:scaleWidth
adaptedHeight:scaleHeight
cropWidth:cropWidth * _cropWidth / _width
cropHeight:cropHeight * _cropHeight / _height
cropX:_cropX + offsetX * _cropWidth / _width
cropY:_cropY + offsetY * _cropHeight / _height];
}
- (id<RTC_OBJC_TYPE(RTCI420Buffer)>)toI420 {
const OSType pixelFormat = CVPixelBufferGetPixelFormatType(_pixelBuffer);

6
sdk/objc/native/src/objc_frame_buffer.h
View file

@ -33,6 +33,12 @@ class ObjCFrameBuffer : public VideoFrameBuffer {
int height() const override;
rtc::scoped_refptr<I420BufferInterface> ToI420() override;
rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(int offset_x,
int offset_y,
int crop_width,
int crop_height,
int scaled_width,
int scaled_height) override;
id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> wrapped_frame_buffer() const;

21
sdk/objc/native/src/objc_frame_buffer.mm
View file

@ -70,6 +70,27 @@ rtc::scoped_refptr<I420BufferInterface> ObjCFrameBuffer::ToI420() {
return rtc::make_ref_counted<ObjCI420FrameBuffer>([frame_buffer_ toI420]);
}
rtc::scoped_refptr<VideoFrameBuffer> ObjCFrameBuffer::CropAndScale(int offset_x,
int offset_y,
int crop_width,
int crop_height,
int scaled_width,
int scaled_height) {
if ([frame_buffer_ respondsToSelector:@selector
(cropAndScaleWith:offsetY:cropWidth:cropHeight:scaleWidth:scaleHeight:)]) {
return rtc::make_ref_counted<ObjCFrameBuffer>([frame_buffer_ cropAndScaleWith:offset_x
offsetY:offset_y
cropWidth:crop_width
cropHeight:crop_height
scaleWidth:scaled_width
scaleHeight:scaled_height]);
}
// Use the default implementation.
return VideoFrameBuffer::CropAndScale(
offset_x, offset_y, crop_width, crop_height, scaled_width, scaled_height);
}
id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> ObjCFrameBuffer::wrapped_frame_buffer() const {
return frame_buffer_;
}

153
sdk/objc/unittests/RTCCVPixelBuffer_xctest.mm
View file

@ -21,6 +21,44 @@
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "third_party/libyuv/include/libyuv.h"
namespace {
struct ToI420WithCropAndScaleSetting {
int inputWidth;
int inputHeight;
int offsetX;
int offsetY;
int cropWidth;
int cropHeight;
int scaleWidth;
int scaleHeight;
};
constexpr const ToI420WithCropAndScaleSetting kToI420WithCropAndScaleSettings[] = {
ToI420WithCropAndScaleSetting{
.inputWidth = 640,
.inputHeight = 360,
.offsetX = 0,
.offsetY = 0,
.cropWidth = 640,
.cropHeight = 360,
.scaleWidth = 320,
.scaleHeight = 180,
},
ToI420WithCropAndScaleSetting{
.inputWidth = 640,
.inputHeight = 360,
.offsetX = 160,
.offsetY = 90,
.cropWidth = 160,
.cropHeight = 90,
.scaleWidth = 320,
.scaleHeight = 180,
},
};
} // namespace
@interface RTCCVPixelBufferTests : XCTestCase
@end
@ -183,6 +221,76 @@
[self toI420WithPixelFormat:kCVPixelFormatType_32ARGB];
}
- (void)testToI420WithCropAndScale_NV12 {
for (const auto &setting : kToI420WithCropAndScaleSettings) {
[self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
setting:setting];
}
}
- (void)testToI420WithCropAndScale_32BGRA {
for (const auto &setting : kToI420WithCropAndScaleSettings) {
[self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_32BGRA setting:setting];
}
}
- (void)testToI420WithCropAndScale_32ARGB {
for (const auto &setting : kToI420WithCropAndScaleSettings) {
[self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_32ARGB setting:setting];
}
}
- (void)testScaleBufferTest {
CVPixelBufferRef pixelBufferRef = NULL;
CVPixelBufferCreate(
NULL, 1920, 1080, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef);
rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer = CreateI420Gradient(1920, 1080);
CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef);
RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
[[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
XCTAssertEqual(buffer.width, 1920);
XCTAssertEqual(buffer.height, 1080);
XCTAssertEqual(buffer.cropX, 0);
XCTAssertEqual(buffer.cropY, 0);
XCTAssertEqual(buffer.cropWidth, 1920);
XCTAssertEqual(buffer.cropHeight, 1080);
RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer2 =
(RTC_OBJC_TYPE(RTCCVPixelBuffer) *)[buffer cropAndScaleWith:320
offsetY:180
cropWidth:1280
cropHeight:720
scaleWidth:960
scaleHeight:540];
XCTAssertEqual(buffer2.width, 960);
XCTAssertEqual(buffer2.height, 540);
XCTAssertEqual(buffer2.cropX, 320);
XCTAssertEqual(buffer2.cropY, 180);
XCTAssertEqual(buffer2.cropWidth, 1280);
XCTAssertEqual(buffer2.cropHeight, 720);
RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer3 =
(RTC_OBJC_TYPE(RTCCVPixelBuffer) *)[buffer2 cropAndScaleWith:240
offsetY:135
cropWidth:480
cropHeight:270
scaleWidth:320
scaleHeight:180];
XCTAssertEqual(buffer3.width, 320);
XCTAssertEqual(buffer3.height, 180);
XCTAssertEqual(buffer3.cropX, 640);
XCTAssertEqual(buffer3.cropY, 360);
XCTAssertEqual(buffer3.cropWidth, 640);
XCTAssertEqual(buffer3.cropHeight, 360);
CVBufferRelease(pixelBufferRef);
}
#pragma mark - Shared test code
- (void)cropAndScaleTestWithNV12 {
@ -305,4 +413,49 @@
CVBufferRelease(pixelBufferRef);
}
- (void)toI420WithCropAndScaleWithPixelFormat:(OSType)pixelFormat
setting:(const ToI420WithCropAndScaleSetting &)setting {
rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer =
CreateI420Gradient(setting.inputWidth, setting.inputHeight);
CVPixelBufferRef pixelBufferRef = NULL;
CVPixelBufferCreate(
NULL, setting.inputWidth, setting.inputHeight, pixelFormat, NULL, &pixelBufferRef);
CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef);
RTC_OBJC_TYPE(RTCI420Buffer) *objcI420Buffer =
[[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:i420Buffer];
RTC_OBJC_TYPE(RTCI420Buffer) *scaledObjcI420Buffer =
(RTC_OBJC_TYPE(RTCI420Buffer) *)[objcI420Buffer cropAndScaleWith:setting.offsetX
offsetY:setting.offsetY
cropWidth:setting.cropWidth
cropHeight:setting.cropHeight
scaleWidth:setting.scaleWidth
scaleHeight:setting.scaleHeight];
RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
[[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> scaledBuffer =
[buffer cropAndScaleWith:setting.offsetX
offsetY:setting.offsetY
cropWidth:setting.cropWidth
cropHeight:setting.cropHeight
scaleWidth:setting.scaleWidth
scaleHeight:setting.scaleHeight];
XCTAssertTrue([scaledBuffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]);
RTC_OBJC_TYPE(RTCI420Buffer) *fromCVPixelBuffer = [scaledBuffer toI420];
double psnr =
I420PSNR(*[scaledObjcI420Buffer nativeI420Buffer], *[fromCVPixelBuffer nativeI420Buffer]);
double target = webrtc::kPerfectPSNR;
if (pixelFormat != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
// libyuv's I420ToRGB functions seem to lose some quality.
target = 19.0;
}
XCTAssertGreaterThanOrEqual(psnr, target);
CVBufferRelease(pixelBufferRef);
}
@end