From 1c4bbba4919234b2373b16b4763a79b4e65f456f Mon Sep 17 00:00:00 2001 From: Julien Isorce Date: Mon, 9 Apr 2018 15:57:55 -0700 Subject: [PATCH] Can use the given IOSurfaceRef to reach higher capture framerate on Mac The given IOSurfaceRef was ignored until now. Wrap it into the new DesktopFrameIOSurface. The new DesktopFrameProvider object is there to manage them as it has to be done per display id. From initial measurement this speed-up the frame capture by 2. Disabled by default for now but it can be enabled by calling options.set_use_iosurface. This CL will allow to do some advanced tests. Bug: webrtc:8652 Change-Id: Ia9ac0b69b30098774941cb378804b45cb1710119 Reviewed-on: https://webrtc-review.googlesource.com/33014 Commit-Queue: Zijie He Reviewed-by: Zijie He Cr-Commit-Position: refs/heads/master@{#22801} --- modules/desktop_capture/BUILD.gn | 6 ++ modules/desktop_capture/DEPS | 6 ++ .../desktop_capture/desktop_capture_options.h | 4 ++ .../mac/desktop_frame_iosurface.h | 44 ++++++++++++ .../mac/desktop_frame_iosurface.mm | 59 ++++++++++++++++ .../mac/desktop_frame_provider.h | 58 ++++++++++++++++ .../mac/desktop_frame_provider.mm | 69 +++++++++++++++++++ .../desktop_capture/mac/screen_capturer_mac.h | 15 ++-- .../mac/screen_capturer_mac.mm | 28 +++++--- .../desktop_capture/screen_capturer_darwin.mm | 4 +- 10 files changed, 279 insertions(+), 14 deletions(-) create mode 100644 modules/desktop_capture/mac/desktop_frame_iosurface.h create mode 100644 modules/desktop_capture/mac/desktop_frame_iosurface.mm create mode 100644 modules/desktop_capture/mac/desktop_frame_provider.h create mode 100644 modules/desktop_capture/mac/desktop_frame_provider.mm diff --git a/modules/desktop_capture/BUILD.gn b/modules/desktop_capture/BUILD.gn index 778d155c5f..7c07c9e3b2 100644 --- a/modules/desktop_capture/BUILD.gn +++ b/modules/desktop_capture/BUILD.gn @@ -177,6 +177,10 @@ if (is_mac) { "mac/desktop_configuration.mm", "mac/desktop_frame_cgimage.h", "mac/desktop_frame_cgimage.mm", + "mac/desktop_frame_iosurface.h", + "mac/desktop_frame_iosurface.mm", + "mac/desktop_frame_provider.h", + "mac/desktop_frame_provider.mm", "mac/screen_capturer_mac.h", "mac/screen_capturer_mac.mm", "mouse_cursor_monitor_mac.mm", @@ -191,11 +195,13 @@ if (is_mac) { "../../rtc_base:checks", "../../rtc_base:rtc_base", "../../rtc_base:rtc_base_approved", + "../../rtc_base/synchronization:rw_lock_wrapper", "../../sdk:common_objc", ] libs = [ "AppKit.framework", "IOKit.framework", + "IOSurface.framework", ] } } diff --git a/modules/desktop_capture/DEPS b/modules/desktop_capture/DEPS index 61bf938fb6..8c894c4430 100644 --- a/modules/desktop_capture/DEPS +++ b/modules/desktop_capture/DEPS @@ -7,6 +7,12 @@ specific_include_rules = { "desktop_frame_cgimage\.h": [ "+sdk/objc", ], + "desktop_frame_iosurface\.h": [ + "+sdk/objc", + ], + "desktop_frame_provider\.h": [ + "+sdk/objc", + ], "screen_capturer_mac\.mm": [ "+sdk/objc", ], diff --git a/modules/desktop_capture/desktop_capture_options.h b/modules/desktop_capture/desktop_capture_options.h index fec5ff0ac5..a945d27264 100644 --- a/modules/desktop_capture/desktop_capture_options.h +++ b/modules/desktop_capture/desktop_capture_options.h @@ -72,6 +72,9 @@ class DesktopCaptureOptions { rtc::scoped_refptr detector) { full_screen_window_detector_ = detector; } + + bool allow_iosurface() const { return allow_iosurface_; } + void set_allow_iosurface(bool allow) { allow_iosurface_ = allow; } #endif // Flag indicating that the capturer should use screen change notifications. @@ -122,6 +125,7 @@ class DesktopCaptureOptions { rtc::scoped_refptr configuration_monitor_; rtc::scoped_refptr full_screen_window_detector_; + bool allow_iosurface_ = false; #endif #if defined(WEBRTC_WIN) diff --git a/modules/desktop_capture/mac/desktop_frame_iosurface.h b/modules/desktop_capture/mac/desktop_frame_iosurface.h new file mode 100644 index 0000000000..ced60f8057 --- /dev/null +++ b/modules/desktop_capture/mac/desktop_frame_iosurface.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_IOSURFACE_H_ +#define MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_IOSURFACE_H_ + +#include +#include + +#include + +#include "modules/desktop_capture/desktop_frame.h" +#include "sdk/objc/Framework/Classes/Common/scoped_cftyperef.h" + +namespace webrtc { + +class DesktopFrameIOSurface final : public DesktopFrame { + public: + // Lock an IOSurfaceRef containing a snapshot of a display. Return NULL if + // failed to lock. + static std::unique_ptr Wrap( + rtc::ScopedCFTypeRef io_surface); + + ~DesktopFrameIOSurface() override; + + private: + // This constructor expects |io_surface| to hold a non-null IOSurfaceRef. + explicit DesktopFrameIOSurface(rtc::ScopedCFTypeRef io_surface); + + const rtc::ScopedCFTypeRef io_surface_; + + RTC_DISALLOW_COPY_AND_ASSIGN(DesktopFrameIOSurface); +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_IOSURFACE_H_ diff --git a/modules/desktop_capture/mac/desktop_frame_iosurface.mm b/modules/desktop_capture/mac/desktop_frame_iosurface.mm new file mode 100644 index 0000000000..7df01b7c85 --- /dev/null +++ b/modules/desktop_capture/mac/desktop_frame_iosurface.mm @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/mac/desktop_frame_iosurface.h" + +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +// static +std::unique_ptr DesktopFrameIOSurface::Wrap( + rtc::ScopedCFTypeRef io_surface) { + if (!io_surface) { + return nullptr; + } + + IOSurfaceIncrementUseCount(io_surface.get()); + IOReturn status = IOSurfaceLock(io_surface.get(), kIOSurfaceLockReadOnly, nullptr); + if (status != kIOReturnSuccess) { + RTC_LOG(LS_ERROR) << "Failed to lock the IOSurface with status " << status; + IOSurfaceDecrementUseCount(io_surface.get()); + return nullptr; + } + + // Verify that the image has 32-bit depth. + int bytes_per_pixel = IOSurfaceGetBytesPerElement(io_surface.get()); + if (bytes_per_pixel != DesktopFrame::kBytesPerPixel) { + RTC_LOG(LS_ERROR) << "CGDisplayStream handler returned IOSurface with " << (8 * bytes_per_pixel) + << " bits per pixel. Only 32-bit depth is supported."; + return nullptr; + } + + return std::unique_ptr(new DesktopFrameIOSurface(io_surface)); +} + +DesktopFrameIOSurface::DesktopFrameIOSurface(rtc::ScopedCFTypeRef io_surface) + : DesktopFrame( + DesktopSize(IOSurfaceGetWidth(io_surface.get()), IOSurfaceGetHeight(io_surface.get())), + IOSurfaceGetBytesPerRow(io_surface.get()), + static_cast(IOSurfaceGetBaseAddress(io_surface.get())), + nullptr), + io_surface_(io_surface) { + RTC_DCHECK(io_surface_); +} + +DesktopFrameIOSurface::~DesktopFrameIOSurface() { + IOSurfaceUnlock(io_surface_.get(), kIOSurfaceLockReadOnly, nullptr); + IOSurfaceDecrementUseCount(io_surface_.get()); +} + +} // namespace webrtc diff --git a/modules/desktop_capture/mac/desktop_frame_provider.h b/modules/desktop_capture/mac/desktop_frame_provider.h new file mode 100644 index 0000000000..d4bbdb1690 --- /dev/null +++ b/modules/desktop_capture/mac/desktop_frame_provider.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_PROVIDER_H_ +#define MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_PROVIDER_H_ + +#include +#include + +#include +#include + +#include "modules/desktop_capture/desktop_frame.h" +#include "rtc_base/synchronization/rw_lock_wrapper.h" +#include "sdk/objc/Framework/Classes/Common/scoped_cftyperef.h" + +namespace webrtc { + +class DesktopFrameProvider { + public: + explicit DesktopFrameProvider(bool allow_iosurface); + ~DesktopFrameProvider(); + + // The caller takes ownership of the returned desktop frame. Otherwise + // returns null if |display_id| is invalid or not ready. + std::unique_ptr TakeLatestFrameForDisplay( + CGDirectDisplayID display_id); + + // OS sends the latest IOSurfaceRef through + // CGDisplayStreamFrameAvailableHandler callback; we store it here. + void InvalidateIOSurface(CGDirectDisplayID display_id, + rtc::ScopedCFTypeRef io_surface); + + // Expected to be called before stopping the CGDisplayStreamRef streams. + void Release(); + + private: + const bool allow_iosurface_; + + // A lock protecting |io_surfaces_| across threads. + const std::unique_ptr io_surfaces_lock_; + + // Most recent IOSurface that contains a capture of matching display. + std::map> io_surfaces_; + + RTC_DISALLOW_COPY_AND_ASSIGN(DesktopFrameProvider); +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_PROVIDER_H_ diff --git a/modules/desktop_capture/mac/desktop_frame_provider.mm b/modules/desktop_capture/mac/desktop_frame_provider.mm new file mode 100644 index 0000000000..3e13125682 --- /dev/null +++ b/modules/desktop_capture/mac/desktop_frame_provider.mm @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/mac/desktop_frame_provider.h" + +#include + +#include "modules/desktop_capture/mac/desktop_frame_cgimage.h" +#include "modules/desktop_capture/mac/desktop_frame_iosurface.h" + +namespace webrtc { + +DesktopFrameProvider::DesktopFrameProvider(bool allow_iosurface) + : allow_iosurface_(allow_iosurface), io_surfaces_lock_(RWLockWrapper::CreateRWLock()) {} + +DesktopFrameProvider::~DesktopFrameProvider() { + // Might be called from a thread which is not the one running the CGDisplayStream + // handler. Indeed chromium's content destroys it from a dedicated thread. + Release(); +} + +std::unique_ptr DesktopFrameProvider::TakeLatestFrameForDisplay( + CGDirectDisplayID display_id) { + if (!allow_iosurface_) { + // Regenerate a snapshot. + return DesktopFrameCGImage::CreateForDisplay(display_id); + } + + // Might be called from a thread which is not the one running the CGDisplayStream + // handler. Indeed chromium's content uses a dedicates thread. + WriteLockScoped scoped_io_surfaces_lock(*io_surfaces_lock_); + if (io_surfaces_[display_id]) { + return std::move(io_surfaces_[display_id]); + } + + return nullptr; +} + +void DesktopFrameProvider::InvalidateIOSurface(CGDirectDisplayID display_id, + rtc::ScopedCFTypeRef io_surface) { + if (!allow_iosurface_) { + return; + } + + std::unique_ptr desktop_frame_iosurface = + DesktopFrameIOSurface::Wrap(io_surface); + + // Call from the thread which runs the CGDisplayStream handler. + WriteLockScoped scoped_io_surfaces_lock(*io_surfaces_lock_); + io_surfaces_[display_id] = std::move(desktop_frame_iosurface); +} + +void DesktopFrameProvider::Release() { + if (!allow_iosurface_) { + return; + } + + WriteLockScoped scoped_io_surfaces_lock(*io_surfaces_lock_); + io_surfaces_.clear(); +} + +} // namespace webrtc diff --git a/modules/desktop_capture/mac/screen_capturer_mac.h b/modules/desktop_capture/mac/screen_capturer_mac.h index 0191317062..8a406700ee 100644 --- a/modules/desktop_capture/mac/screen_capturer_mac.h +++ b/modules/desktop_capture/mac/screen_capturer_mac.h @@ -22,6 +22,7 @@ #include "modules/desktop_capture/desktop_region.h" #include "modules/desktop_capture/mac/desktop_configuration.h" #include "modules/desktop_capture/mac/desktop_configuration_monitor.h" +#include "modules/desktop_capture/mac/desktop_frame_provider.h" #include "modules/desktop_capture/screen_capture_frame_queue.h" #include "modules/desktop_capture/screen_capturer_helper.h" #include "modules/desktop_capture/shared_desktop_frame.h" @@ -33,9 +34,10 @@ class DisplayStreamManager; // A class to perform video frame capturing for mac. class ScreenCapturerMac final : public DesktopCapturer { public: - explicit ScreenCapturerMac( + ScreenCapturerMac( rtc::scoped_refptr desktop_config_monitor, - bool detect_updated_region); + bool detect_updated_region, + bool allow_iosurface); ~ScreenCapturerMac() override; // TODO(julien.isorce): Remove Init() or make it private. @@ -58,9 +60,11 @@ class ScreenCapturerMac final : public DesktopCapturer { bool RegisterRefreshAndMoveHandlers(); void UnregisterRefreshAndMoveHandlers(); - void ScreenRefresh(CGRectCount count, + void ScreenRefresh(CGDirectDisplayID display_id, + CGRectCount count, const CGRect* rect_array, - DesktopVector display_origin); + DesktopVector display_origin, + IOSurfaceRef io_surface); void ReleaseBuffers(); std::unique_ptr CreateFrame(); @@ -101,6 +105,9 @@ class ScreenCapturerMac final : public DesktopCapturer { // all display streams have been destroyed.. DisplayStreamManager* display_stream_manager_; + // Container holding latest state of the snapshot per displays. + DesktopFrameProvider desktop_frame_provider_; + RTC_DISALLOW_COPY_AND_ASSIGN(ScreenCapturerMac); }; diff --git a/modules/desktop_capture/mac/screen_capturer_mac.mm b/modules/desktop_capture/mac/screen_capturer_mac.mm index df18777226..c06723c1a5 100644 --- a/modules/desktop_capture/mac/screen_capturer_mac.mm +++ b/modules/desktop_capture/mac/screen_capturer_mac.mm @@ -12,7 +12,7 @@ #include "modules/desktop_capture/mac/screen_capturer_mac.h" -#include "modules/desktop_capture/mac/desktop_frame_cgimage.h" +#include "modules/desktop_capture/mac/desktop_frame_provider.h" #include "rtc_base/checks.h" #include "rtc_base/constructormagic.h" #include "rtc_base/logging.h" @@ -215,10 +215,14 @@ rtc::ScopedCFTypeRef CreateExcludedWindowRegionImage(const DesktopRe ScreenCapturerMac::ScreenCapturerMac( rtc::scoped_refptr desktop_config_monitor, - bool detect_updated_region) + bool detect_updated_region, + bool allow_iosurface) : detect_updated_region_(detect_updated_region), - desktop_config_monitor_(desktop_config_monitor) { + desktop_config_monitor_(desktop_config_monitor), + desktop_frame_provider_(allow_iosurface) { display_stream_manager_ = new DisplayStreamManager; + + RTC_LOG(LS_INFO) << "Allow IOSurface: " << allow_iosurface; } ScreenCapturerMac::~ScreenCapturerMac() { @@ -406,8 +410,8 @@ bool ScreenCapturerMac::CgBlit(const DesktopFrame& frame, const DesktopRegion& r } } - std::unique_ptr frame_source = - DesktopFrameCGImage::CreateForDisplay(display_config.id); + std::unique_ptr frame_source = + desktop_frame_provider_.TakeLatestFrameForDisplay(display_config.id); if (!frame_source) { continue; } @@ -518,7 +522,7 @@ bool ScreenCapturerMac::RegisterRefreshAndMoveHandlers() { if (count != 0) { // According to CGDisplayStream.h, it's safe to call // CGDisplayStreamStop() from within the callback. - ScreenRefresh(count, rects, display_origin); + ScreenRefresh(display_id, count, rects, display_origin, frame_surface); } }; @@ -547,12 +551,17 @@ bool ScreenCapturerMac::RegisterRefreshAndMoveHandlers() { } void ScreenCapturerMac::UnregisterRefreshAndMoveHandlers() { + // Release obsolete io surfaces. + desktop_frame_provider_.Release(); + display_stream_manager_->UnregisterActiveStreams(); } -void ScreenCapturerMac::ScreenRefresh(CGRectCount count, +void ScreenCapturerMac::ScreenRefresh(CGDirectDisplayID display_id, + CGRectCount count, const CGRect* rect_array, - DesktopVector display_origin) { + DesktopVector display_origin, + IOSurfaceRef io_surface) { if (screen_pixel_bounds_.is_empty()) ScreenConfigurationChanged(); // The refresh rects are in display coordinates. We want to translate to @@ -576,6 +585,9 @@ void ScreenCapturerMac::ScreenRefresh(CGRectCount count, } helper_.InvalidateRegion(region); + + desktop_frame_provider_.InvalidateIOSurface( + display_id, rtc::ScopedCFTypeRef(io_surface, rtc::RetainPolicy::RETAIN)); } std::unique_ptr ScreenCapturerMac::CreateFrame() { diff --git a/modules/desktop_capture/screen_capturer_darwin.mm b/modules/desktop_capture/screen_capturer_darwin.mm index ca26f32896..d5a7bb0522 100644 --- a/modules/desktop_capture/screen_capturer_darwin.mm +++ b/modules/desktop_capture/screen_capturer_darwin.mm @@ -21,8 +21,8 @@ std::unique_ptr DesktopCapturer::CreateRawScreenCapturer( return nullptr; } - std::unique_ptr capturer( - new ScreenCapturerMac(options.configuration_monitor(), options.detect_updated_region())); + std::unique_ptr capturer(new ScreenCapturerMac( + options.configuration_monitor(), options.detect_updated_region(), options.allow_iosurface())); if (!capturer.get()->Init()) { return nullptr; }