Can use the given IOSurfaceRef to reach higher capture framerate on Mac

The given IOSurfaceRef was ignored until now. Wrap it into the new
DesktopFrameIOSurface. The new DesktopFrameProvider object is there
to manage them as it has to be done per display id.

From initial measurement this speed-up the frame capture by 2.

Disabled by default for now but it can be enabled by calling
options.set_use_iosurface. This CL will allow to do some advanced
tests.

Bug: webrtc:8652
Change-Id: Ia9ac0b69b30098774941cb378804b45cb1710119
Reviewed-on: https://webrtc-review.googlesource.com/33014
Commit-Queue: Zijie He <zijiehe@chromium.org>
Reviewed-by: Zijie He <zijiehe@chromium.org>
Cr-Commit-Position: refs/heads/master@{#22801}

modules/desktop_capture/BUILD.gn

@@ -177,6 +177,10 @@ if (is_mac) {
       "mac/desktop_configuration.mm",
       "mac/desktop_frame_cgimage.h",
       "mac/desktop_frame_cgimage.mm",
+      "mac/desktop_frame_iosurface.h",
+      "mac/desktop_frame_iosurface.mm",
+      "mac/desktop_frame_provider.h",
+      "mac/desktop_frame_provider.mm",
       "mac/screen_capturer_mac.h",
       "mac/screen_capturer_mac.mm",
       "mouse_cursor_monitor_mac.mm",
@@ -191,11 +195,13 @@ if (is_mac) {
       "../../rtc_base:checks",
       "../../rtc_base:rtc_base",
       "../../rtc_base:rtc_base_approved",
+      "../../rtc_base/synchronization:rw_lock_wrapper",
       "../../sdk:common_objc",
     ]
     libs = [
       "AppKit.framework",
       "IOKit.framework",
+      "IOSurface.framework",
     ]
   }
 }

modules/desktop_capture/DEPS

@@ -7,6 +7,12 @@ specific_include_rules = {
   "desktop_frame_cgimage\.h": [
     "+sdk/objc",
   ],
+  "desktop_frame_iosurface\.h": [
+    "+sdk/objc",
+  ],
+  "desktop_frame_provider\.h": [
+    "+sdk/objc",
+  ],
   "screen_capturer_mac\.mm": [
     "+sdk/objc",
   ],

modules/desktop_capture/desktop_capture_options.h

@@ -72,6 +72,9 @@ class DesktopCaptureOptions {
       rtc::scoped_refptr<FullScreenChromeWindowDetector> detector) {
     full_screen_window_detector_ = detector;
   }
+
+  bool allow_iosurface() const { return allow_iosurface_; }
+  void set_allow_iosurface(bool allow) { allow_iosurface_ = allow; }
 #endif
 
   // Flag indicating that the capturer should use screen change notifications.
@@ -122,6 +125,7 @@ class DesktopCaptureOptions {
   rtc::scoped_refptr<DesktopConfigurationMonitor> configuration_monitor_;
   rtc::scoped_refptr<FullScreenChromeWindowDetector>
       full_screen_window_detector_;
+  bool allow_iosurface_ = false;
 #endif
 
 #if defined(WEBRTC_WIN)

modules/desktop_capture/mac/desktop_frame_iosurface.h

@@ -0,0 +1,44 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_IOSURFACE_H_
+#define MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_IOSURFACE_H_
+
+#include <CoreGraphics/CoreGraphics.h>
+#include <IOSurface/IOSurface.h>
+
+#include <memory>
+
+#include "modules/desktop_capture/desktop_frame.h"
+#include "sdk/objc/Framework/Classes/Common/scoped_cftyperef.h"
+
+namespace webrtc {
+
+class DesktopFrameIOSurface final : public DesktopFrame {
+ public:
+  // Lock an IOSurfaceRef containing a snapshot of a display. Return NULL if
+  // failed to lock.
+  static std::unique_ptr<DesktopFrameIOSurface> Wrap(
+      rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface);
+
+  ~DesktopFrameIOSurface() override;
+
+ private:
+  // This constructor expects |io_surface| to hold a non-null IOSurfaceRef.
+  explicit DesktopFrameIOSurface(rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface);
+
+  const rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface_;
+
+  RTC_DISALLOW_COPY_AND_ASSIGN(DesktopFrameIOSurface);
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_IOSURFACE_H_

modules/desktop_capture/mac/desktop_frame_iosurface.mm

@@ -0,0 +1,59 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/desktop_capture/mac/desktop_frame_iosurface.h"
+
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+// static
+std::unique_ptr<DesktopFrameIOSurface> DesktopFrameIOSurface::Wrap(
+    rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface) {
+  if (!io_surface) {
+    return nullptr;
+  }
+
+  IOSurfaceIncrementUseCount(io_surface.get());
+  IOReturn status = IOSurfaceLock(io_surface.get(), kIOSurfaceLockReadOnly, nullptr);
+  if (status != kIOReturnSuccess) {
+    RTC_LOG(LS_ERROR) << "Failed to lock the IOSurface with status " << status;
+    IOSurfaceDecrementUseCount(io_surface.get());
+    return nullptr;
+  }
+
+  // Verify that the image has 32-bit depth.
+  int bytes_per_pixel = IOSurfaceGetBytesPerElement(io_surface.get());
+  if (bytes_per_pixel != DesktopFrame::kBytesPerPixel) {
+    RTC_LOG(LS_ERROR) << "CGDisplayStream handler returned IOSurface with " << (8 * bytes_per_pixel)
+                      << " bits per pixel. Only 32-bit depth is supported.";
+    return nullptr;
+  }
+
+  return std::unique_ptr<DesktopFrameIOSurface>(new DesktopFrameIOSurface(io_surface));
+}
+
+DesktopFrameIOSurface::DesktopFrameIOSurface(rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface)
+    : DesktopFrame(
+          DesktopSize(IOSurfaceGetWidth(io_surface.get()), IOSurfaceGetHeight(io_surface.get())),
+          IOSurfaceGetBytesPerRow(io_surface.get()),
+          static_cast<uint8_t*>(IOSurfaceGetBaseAddress(io_surface.get())),
+          nullptr),
+      io_surface_(io_surface) {
+  RTC_DCHECK(io_surface_);
+}
+
+DesktopFrameIOSurface::~DesktopFrameIOSurface() {
+  IOSurfaceUnlock(io_surface_.get(), kIOSurfaceLockReadOnly, nullptr);
+  IOSurfaceDecrementUseCount(io_surface_.get());
+}
+
+}  // namespace webrtc

modules/desktop_capture/mac/desktop_frame_provider.h

@@ -0,0 +1,58 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_PROVIDER_H_
+#define MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_PROVIDER_H_
+
+#include <CoreGraphics/CoreGraphics.h>
+#include <IOSurface/IOSurface.h>
+
+#include <map>
+#include <memory>
+
+#include "modules/desktop_capture/desktop_frame.h"
+#include "rtc_base/synchronization/rw_lock_wrapper.h"
+#include "sdk/objc/Framework/Classes/Common/scoped_cftyperef.h"
+
+namespace webrtc {
+
+class DesktopFrameProvider {
+ public:
+  explicit DesktopFrameProvider(bool allow_iosurface);
+  ~DesktopFrameProvider();
+
+  // The caller takes ownership of the returned desktop frame. Otherwise
+  // returns null if |display_id| is invalid or not ready.
+  std::unique_ptr<DesktopFrame> TakeLatestFrameForDisplay(
+      CGDirectDisplayID display_id);
+
+  // OS sends the latest IOSurfaceRef through
+  // CGDisplayStreamFrameAvailableHandler callback; we store it here.
+  void InvalidateIOSurface(CGDirectDisplayID display_id,
+                           rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface);
+
+  // Expected to be called before stopping the CGDisplayStreamRef streams.
+  void Release();
+
+ private:
+  const bool allow_iosurface_;
+
+  // A lock protecting |io_surfaces_| across threads.
+  const std::unique_ptr<RWLockWrapper> io_surfaces_lock_;
+
+  // Most recent IOSurface that contains a capture of matching display.
+  std::map<CGDirectDisplayID, std::unique_ptr<DesktopFrame>> io_surfaces_;
+
+  RTC_DISALLOW_COPY_AND_ASSIGN(DesktopFrameProvider);
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_DESKTOP_CAPTURE_MAC_DESKTOP_FRAME_PROVIDER_H_

modules/desktop_capture/mac/desktop_frame_provider.mm

@@ -0,0 +1,69 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/desktop_capture/mac/desktop_frame_provider.h"
+
+#include <utility>
+
+#include "modules/desktop_capture/mac/desktop_frame_cgimage.h"
+#include "modules/desktop_capture/mac/desktop_frame_iosurface.h"
+
+namespace webrtc {
+
+DesktopFrameProvider::DesktopFrameProvider(bool allow_iosurface)
+    : allow_iosurface_(allow_iosurface), io_surfaces_lock_(RWLockWrapper::CreateRWLock()) {}
+
+DesktopFrameProvider::~DesktopFrameProvider() {
+  // Might be called from a thread which is not the one running the CGDisplayStream
+  // handler. Indeed chromium's content destroys it from a dedicated thread.
+  Release();
+}
+
+std::unique_ptr<DesktopFrame> DesktopFrameProvider::TakeLatestFrameForDisplay(
+    CGDirectDisplayID display_id) {
+  if (!allow_iosurface_) {
+    // Regenerate a snapshot.
+    return DesktopFrameCGImage::CreateForDisplay(display_id);
+  }
+
+  // Might be called from a thread which is not the one running the CGDisplayStream
+  // handler. Indeed chromium's content uses a dedicates thread.
+  WriteLockScoped scoped_io_surfaces_lock(*io_surfaces_lock_);
+  if (io_surfaces_[display_id]) {
+    return std::move(io_surfaces_[display_id]);
+  }
+
+  return nullptr;
+}
+
+void DesktopFrameProvider::InvalidateIOSurface(CGDirectDisplayID display_id,
+                                               rtc::ScopedCFTypeRef<IOSurfaceRef> io_surface) {
+  if (!allow_iosurface_) {
+    return;
+  }
+
+  std::unique_ptr<DesktopFrameIOSurface> desktop_frame_iosurface =
+      DesktopFrameIOSurface::Wrap(io_surface);
+
+  // Call from the thread which runs the CGDisplayStream handler.
+  WriteLockScoped scoped_io_surfaces_lock(*io_surfaces_lock_);
+  io_surfaces_[display_id] = std::move(desktop_frame_iosurface);
+}
+
+void DesktopFrameProvider::Release() {
+  if (!allow_iosurface_) {
+    return;
+  }
+
+  WriteLockScoped scoped_io_surfaces_lock(*io_surfaces_lock_);
+  io_surfaces_.clear();
+}
+
+}  // namespace webrtc

modules/desktop_capture/mac/screen_capturer_mac.h

@@ -22,6 +22,7 @@
 #include "modules/desktop_capture/desktop_region.h"
 #include "modules/desktop_capture/mac/desktop_configuration.h"
 #include "modules/desktop_capture/mac/desktop_configuration_monitor.h"
+#include "modules/desktop_capture/mac/desktop_frame_provider.h"
 #include "modules/desktop_capture/screen_capture_frame_queue.h"
 #include "modules/desktop_capture/screen_capturer_helper.h"
 #include "modules/desktop_capture/shared_desktop_frame.h"
@@ -33,9 +34,10 @@ class DisplayStreamManager;
 // A class to perform video frame capturing for mac.
 class ScreenCapturerMac final : public DesktopCapturer {
  public:
-  explicit ScreenCapturerMac(
+  ScreenCapturerMac(
       rtc::scoped_refptr<DesktopConfigurationMonitor> desktop_config_monitor,
-      bool detect_updated_region);
+      bool detect_updated_region,
+      bool allow_iosurface);
   ~ScreenCapturerMac() override;
 
   // TODO(julien.isorce): Remove Init() or make it private.
@@ -58,9 +60,11 @@ class ScreenCapturerMac final : public DesktopCapturer {
   bool RegisterRefreshAndMoveHandlers();
   void UnregisterRefreshAndMoveHandlers();
 
-  void ScreenRefresh(CGRectCount count,
+  void ScreenRefresh(CGDirectDisplayID display_id,
+                     CGRectCount count,
                      const CGRect* rect_array,
-                     DesktopVector display_origin);
+                     DesktopVector display_origin,
+                     IOSurfaceRef io_surface);
   void ReleaseBuffers();
 
   std::unique_ptr<DesktopFrame> CreateFrame();
@@ -101,6 +105,9 @@ class ScreenCapturerMac final : public DesktopCapturer {
   // all display streams have been destroyed..
   DisplayStreamManager* display_stream_manager_;
 
+  // Container holding latest state of the snapshot per displays.
+  DesktopFrameProvider desktop_frame_provider_;
+
   RTC_DISALLOW_COPY_AND_ASSIGN(ScreenCapturerMac);
 };
 

modules/desktop_capture/mac/screen_capturer_mac.mm

@@ -12,7 +12,7 @@
 
 #include "modules/desktop_capture/mac/screen_capturer_mac.h"
 
-#include "modules/desktop_capture/mac/desktop_frame_cgimage.h"
+#include "modules/desktop_capture/mac/desktop_frame_provider.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/constructormagic.h"
 #include "rtc_base/logging.h"
@@ -215,10 +215,14 @@ rtc::ScopedCFTypeRef<CGImageRef> CreateExcludedWindowRegionImage(const DesktopRe
 
 ScreenCapturerMac::ScreenCapturerMac(
     rtc::scoped_refptr<DesktopConfigurationMonitor> desktop_config_monitor,
-    bool detect_updated_region)
+    bool detect_updated_region,
+    bool allow_iosurface)
     : detect_updated_region_(detect_updated_region),
-      desktop_config_monitor_(desktop_config_monitor) {
+      desktop_config_monitor_(desktop_config_monitor),
+      desktop_frame_provider_(allow_iosurface) {
   display_stream_manager_ = new DisplayStreamManager;
+
+  RTC_LOG(LS_INFO) << "Allow IOSurface: " << allow_iosurface;
 }
 
 ScreenCapturerMac::~ScreenCapturerMac() {
@@ -406,8 +410,8 @@ bool ScreenCapturerMac::CgBlit(const DesktopFrame& frame, const DesktopRegion& r
       }
     }
 
-    std::unique_ptr<DesktopFrameCGImage> frame_source =
+    std::unique_ptr<DesktopFrame> frame_source =
-        DesktopFrameCGImage::CreateForDisplay(display_config.id);
+        desktop_frame_provider_.TakeLatestFrameForDisplay(display_config.id);
     if (!frame_source) {
       continue;
     }
@@ -518,7 +522,7 @@ bool ScreenCapturerMac::RegisterRefreshAndMoveHandlers() {
       if (count != 0) {
         // According to CGDisplayStream.h, it's safe to call
         // CGDisplayStreamStop() from within the callback.
-        ScreenRefresh(count, rects, display_origin);
+        ScreenRefresh(display_id, count, rects, display_origin, frame_surface);
       }
     };
 
@@ -547,12 +551,17 @@ bool ScreenCapturerMac::RegisterRefreshAndMoveHandlers() {
 }
 
 void ScreenCapturerMac::UnregisterRefreshAndMoveHandlers() {
+  // Release obsolete io surfaces.
+  desktop_frame_provider_.Release();
+
   display_stream_manager_->UnregisterActiveStreams();
 }
 
-void ScreenCapturerMac::ScreenRefresh(CGRectCount count,
+void ScreenCapturerMac::ScreenRefresh(CGDirectDisplayID display_id,
+                                      CGRectCount count,
                                       const CGRect* rect_array,
-                                      DesktopVector display_origin) {
+                                      DesktopVector display_origin,
+                                      IOSurfaceRef io_surface) {
   if (screen_pixel_bounds_.is_empty()) ScreenConfigurationChanged();
 
   // The refresh rects are in display coordinates. We want to translate to
@@ -576,6 +585,9 @@ void ScreenCapturerMac::ScreenRefresh(CGRectCount count,
   }
 
   helper_.InvalidateRegion(region);
+
+  desktop_frame_provider_.InvalidateIOSurface(
+      display_id, rtc::ScopedCFTypeRef<IOSurfaceRef>(io_surface, rtc::RetainPolicy::RETAIN));
 }
 
 std::unique_ptr<DesktopFrame> ScreenCapturerMac::CreateFrame() {

modules/desktop_capture/screen_capturer_darwin.mm

@@ -21,8 +21,8 @@ std::unique_ptr<DesktopCapturer> DesktopCapturer::CreateRawScreenCapturer(
     return nullptr;
   }
 
-  std::unique_ptr<ScreenCapturerMac> capturer(
+  std::unique_ptr<ScreenCapturerMac> capturer(new ScreenCapturerMac(
-      new ScreenCapturerMac(options.configuration_monitor(), options.detect_updated_region()));
+      options.configuration_monitor(), options.detect_updated_region(), options.allow_iosurface()));
   if (!capturer.get()->Init()) {
     return nullptr;
   }