[Window Capturer] Implement scaling in GetWindowBounds()

On Mac OSX system, if retina screen is used, the GetWindowBounds() returns
pre-scaled values instead of system coordinates. So this fix considers
per-monitor scale-factor, and stretchs the DesktopRect.

Bug: chromium:778049
Change-Id: I9dc51e08235eba9b3ef6378eaa15737aa444b0c8
Reviewed-on: https://webrtc-review.googlesource.com/17600
Commit-Queue: Zijie He <zijiehe@chromium.org>
Reviewed-by: Jamie Walch <jamiewalch@chromium.org>
Cr-Commit-Position: refs/heads/master@{#20578}

modules/desktop_capture/BUILD.gn

@@ -293,6 +293,7 @@ rtc_static_library("desktop_capture_generic") {
     "win/window_capture_utils.cc",
     "win/window_capture_utils.h",
     "window_capturer_win.cc",
+    "window_finder.cc",
     "window_finder.h",
     "window_finder_win.cc",
     "window_finder_win.h",

modules/desktop_capture/mac/window_list_utils.cc

@@ -12,6 +12,9 @@
 
 #include <ApplicationServices/ApplicationServices.h>
 
+#include <algorithm>
+#include <iterator>
+
 #include "rtc_base/checks.h"
 #include "rtc_base/macutils.h"
 
@@ -57,6 +60,37 @@ bool GetWindowRef(CGWindowID id,
   return result;
 }
 
+// Scales the |rect| according to the DIP to physical pixel scale of |rect|.
+// |rect| is in unscaled system coordinate, i.e. it's device-independent and the
+// primary monitor starts from (0, 0). If |rect| overlaps multiple monitors, the
+// returned size may not be accurate when monitors have different DIP settings.
+// If |rect| is entirely out of the display, this function returns |rect|.
+DesktopRect ApplyScaleFactorOfRect(
+    const MacDesktopConfiguration& desktop_config,
+    DesktopRect rect) {
+  // TODO(http://crbug.com/778049): How does Mac OSX decide the scale factor
+  // if one window is across two monitors with different DPIs.
+  float scales[] = {
+      GetScaleFactorAtPosition(desktop_config, rect.top_left()),
+      GetScaleFactorAtPosition(desktop_config,
+          DesktopVector(rect.left() + rect.width() / 2,
+                        rect.top() + rect.height() / 2)),
+      GetScaleFactorAtPosition(
+            desktop_config, DesktopVector(rect.right(), rect.bottom())),
+  };
+  // Since GetScaleFactorAtPosition() returns 1 if the position is out of the
+  // display, we always prefer a value which not equals to 1.
+  float scale = *std::max_element(std::begin(scales), std::end(scales));
+  if (scale == 1) {
+    scale = *std::min_element(std::begin(scales), std::end(scales));
+  }
+
+  return DesktopRect::MakeXYWH(rect.left() * scale,
+                               rect.top() * scale,
+                               rect.width() * scale,
+                               rect.height() * scale);
+}
+
 }  // namespace
 
 bool GetWindowList(rtc::FunctionView<bool(CFDictionaryRef)> on_window,
@@ -227,6 +261,19 @@ WindowId GetWindowId(CFDictionaryRef window) {
   return id;
 }
 
+float GetScaleFactorAtPosition(const MacDesktopConfiguration& desktop_config,
+                               DesktopVector position) {
+  // Find the dpi to physical pixel scale for the screen where the mouse cursor
+  // is.
+  for (auto it = desktop_config.displays.begin();
+       it != desktop_config.displays.end(); ++it) {
+    if (it->bounds.Contains(position)) {
+      return it->dip_to_pixel_scale;
+    }
+  }
+  return 1;
+}
+
 DesktopRect GetWindowBounds(CFDictionaryRef window) {
   CFDictionaryRef window_bounds = reinterpret_cast<CFDictionaryRef>(
       CFDictionaryGetValue(window, kCGWindowBounds));
@@ -245,6 +292,12 @@ DesktopRect GetWindowBounds(CFDictionaryRef window) {
                                gc_window_rect.size.height);
 }
 
+DesktopRect GetWindowBounds(const MacDesktopConfiguration& desktop_config,
+                            CFDictionaryRef window) {
+  DesktopRect rect = GetWindowBounds(window);
+  return ApplyScaleFactorOfRect(desktop_config, rect);
+}
+
 DesktopRect GetWindowBounds(CGWindowID id) {
   DesktopRect result;
   if (GetWindowRef(id,
@@ -256,4 +309,10 @@ DesktopRect GetWindowBounds(CGWindowID id) {
   return DesktopRect();
 }
 
+DesktopRect GetWindowBounds(const MacDesktopConfiguration& desktop_config,
+                            CGWindowID id) {
+  DesktopRect rect = GetWindowBounds(id);
+  return ApplyScaleFactorOfRect(desktop_config, rect);
+}
+
 }  // namespace webrtc

modules/desktop_capture/mac/window_list_utils.h

@@ -52,17 +52,38 @@ std::string GetWindowTitle(CFDictionaryRef window);
 // be retrieved, this function returns kNullWindowId.
 WindowId GetWindowId(CFDictionaryRef window);
 
+// Returns the DIP to physical pixel scale at |position|. |position| is in
+// *unscaled* system coordinate, i.e. it's device-independent and the primary
+// monitor starts from (0, 0). If |position| is out of the system display, this
+// function returns 1.
+float GetScaleFactorAtPosition(const MacDesktopConfiguration& desktop_config,
+                               DesktopVector position);
+
 // Returns the bounds of |window|. If |window| is not a window or the bounds
 // cannot be retrieved, this function returns an empty DesktopRect. The returned
 // DesktopRect is in system coordinate, i.e. the primary monitor always starts
 // from (0, 0).
+// Deprecated: This function should be avoided in favor of the overload with
+// MacDesktopConfiguration.
 DesktopRect GetWindowBounds(CFDictionaryRef window);
 
+// Same as GetWindowBounds(CFDictionaryRef), but this function stretches the
+// result with the scale factor.
+DesktopRect GetWindowBounds(const MacDesktopConfiguration& desktop_config,
+                            CFDictionaryRef window);
+
 // Returns the bounds of window with |id|. If |id| does not represent a window
 // or the bounds cannot be retrieved, this function returns an empty
 // DesktopRect. The returned DesktopRect is in system coordinates.
+// Deprecated: This function should be avoided in favor of the overload with
+// MacDesktopConfiguration.
 DesktopRect GetWindowBounds(CGWindowID id);
 
+// Same as GetWindowBounds(CGWindowID), but this function stretches the result
+// with the scale factor.
+DesktopRect GetWindowBounds(const MacDesktopConfiguration& desktop_config,
+                            CGWindowID id);
+
 }  // namespace webrtc
 
 #endif  // MODULES_DESKTOP_CAPTURE_MAC_WINDOW_LIST_UTILS_H_

modules/desktop_capture/mouse_cursor_monitor_mac.mm

@@ -24,6 +24,7 @@
 #include "modules/desktop_capture/mac/desktop_configuration.h"
 #include "modules/desktop_capture/mac/desktop_configuration_monitor.h"
 #include "modules/desktop_capture/mac/full_screen_chrome_window_detector.h"
+#include "modules/desktop_capture/mac/window_list_utils.h"
 #include "modules/desktop_capture/mouse_cursor.h"
 #include "rtc_base/macutils.h"
 #include "rtc_base/scoped_ref_ptr.h"
@@ -122,17 +123,7 @@ void MouseCursorMonitorMac::Capture() {
   MacDesktopConfiguration configuration =
       configuration_monitor_->desktop_configuration();
   configuration_monitor_->Unlock();
-  float scale = 1.0f;
+  float scale = GetScaleFactorAtPosition(configuration, position);
-
-  // Find the dpi to physical pixel scale for the screen where the mouse cursor
-  // is.
-  for (MacDisplayConfigurations::iterator it = configuration.displays.begin();
-       it != configuration.displays.end(); ++it) {
-    if (it->bounds.Contains(position)) {
-      scale = it->dip_to_pixel_scale;
-      break;
-    }
-  }
 
   CaptureImage(scale);
 

modules/desktop_capture/window_capturer_mac.mm

@@ -83,7 +83,8 @@ WindowCapturerMac::WindowCapturerMac(
     rtc::scoped_refptr<DesktopConfigurationMonitor> configuration_monitor)
     : full_screen_chrome_window_detector_(
           std::move(full_screen_chrome_window_detector)),
-      configuration_monitor_(std::move(configuration_monitor)) {}
+      configuration_monitor_(std::move(configuration_monitor)),
+      window_finder_(configuration_monitor_) {}
 
 WindowCapturerMac::~WindowCapturerMac() {}
 
@@ -204,12 +205,15 @@ void WindowCapturerMac::CaptureFrame() {
 
   frame->mutable_updated_region()->SetRect(
       DesktopRect::MakeSize(frame->size()));
-  DesktopVector top_left = GetWindowBounds(on_screen_window).top_left();
+  DesktopVector top_left;
   if (configuration_monitor_) {
     configuration_monitor_->Lock();
     auto configuration = configuration_monitor_->desktop_configuration();
     configuration_monitor_->Unlock();
+    top_left = GetWindowBounds(configuration, on_screen_window).top_left();
     top_left = top_left.subtract(configuration.bounds.top_left());
+  } else {
+    top_left = GetWindowBounds(on_screen_window).top_left();
   }
   frame->set_top_left(top_left);
 

modules/desktop_capture/window_finder.cc

@@ -0,0 +1,20 @@
+/*
+ *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/desktop_capture/window_finder.h"
+
+namespace webrtc {
+
+WindowFinder::Options::Options() = default;
+WindowFinder::Options::~Options() = default;
+WindowFinder::Options::Options(const WindowFinder::Options& other) = default;
+WindowFinder::Options::Options(WindowFinder::Options&& other) = default;
+
+}  // namespace webrtc

modules/desktop_capture/window_finder.h

@@ -15,6 +15,11 @@
 
 #include "modules/desktop_capture/desktop_capture_types.h"
 #include "modules/desktop_capture/desktop_geometry.h"
+#include "rtc_base/scoped_ref_ptr.h"
+
+#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
+#include "modules/desktop_capture/mac/desktop_configuration_monitor.h"
+#endif
 
 namespace webrtc {
 
@@ -35,9 +40,17 @@ class WindowFinder {
   // starts from (0, 0).
   virtual WindowId GetWindowUnderPoint(DesktopVector point) = 0;
 
-  struct Options {
+  struct Options final {
+    Options();
+    ~Options();
+    Options(const Options& other);
+    Options(Options&& other);
+
 #if defined(USE_X11)
     XAtomCache* cache = nullptr;
+#endif
+#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS)
+    rtc::scoped_refptr<DesktopConfigurationMonitor> configuration_monitor;
 #endif
   };
 

modules/desktop_capture/window_finder_mac.h

@@ -12,17 +12,24 @@
 #define MODULES_DESKTOP_CAPTURE_WINDOW_FINDER_MAC_H_
 
 #include "modules/desktop_capture/window_finder.h"
+#include "rtc_base/scoped_ref_ptr.h"
 
 namespace webrtc {
 
+class DesktopConfigurationMonitor;
+
 // The implementation of WindowFinder for Mac OSX.
 class WindowFinderMac final : public WindowFinder {
  public:
-  WindowFinderMac();
+  explicit WindowFinderMac(
+      rtc::scoped_refptr<DesktopConfigurationMonitor> configuration_monitor);
   ~WindowFinderMac() override;
 
   // WindowFinder implementation.
   WindowId GetWindowUnderPoint(DesktopVector point) override;
+
+ private:
+  const rtc::scoped_refptr<DesktopConfigurationMonitor> configuration_monitor_;
 };
 
 }  // namespace webrtc

modules/desktop_capture/window_finder_mac.mm

@@ -12,18 +12,37 @@
 
 #include <CoreFoundation/CoreFoundation.h>
 
+#include <utility>
+
 #include "modules/desktop_capture/mac/window_list_utils.h"
+#include "modules/desktop_capture/mac/desktop_configuration.h"
+#include "modules/desktop_capture/mac/desktop_configuration_monitor.h"
 #include "rtc_base/ptr_util.h"
 
 namespace webrtc {
 
-WindowFinderMac::WindowFinderMac() = default;
+WindowFinderMac::WindowFinderMac(
+    rtc::scoped_refptr<DesktopConfigurationMonitor> configuration_monitor)
+    : configuration_monitor_(std::move(configuration_monitor)) {}
 WindowFinderMac::~WindowFinderMac() = default;
 
 WindowId WindowFinderMac::GetWindowUnderPoint(DesktopVector point) {
   WindowId id = kNullWindowId;
-  GetWindowList([&id, point](CFDictionaryRef window) {
+  MacDesktopConfiguration configuration_holder;
-                  DesktopRect bounds = GetWindowBounds(window);
+  MacDesktopConfiguration* configuration = nullptr;
+  if (configuration_monitor_) {
+    configuration_monitor_->Lock();
+    configuration_holder = configuration_monitor_->desktop_configuration();
+    configuration_monitor_->Unlock();
+    configuration = &configuration_holder;
+  }
+  GetWindowList([&id, point, configuration](CFDictionaryRef window) {
+                  DesktopRect bounds;
+                  if (configuration) {
+                    bounds = GetWindowBounds(*configuration, window);
+                  } else {
+                    bounds = GetWindowBounds(window);
+                  }
                   if (bounds.Contains(point)) {
                     id = GetWindowId(window);
                     return false;
@@ -37,7 +56,7 @@ WindowId WindowFinderMac::GetWindowUnderPoint(DesktopVector point) {
 // static
 std::unique_ptr<WindowFinder> WindowFinder::Create(
     const WindowFinder::Options& options) {
-  return rtc::MakeUnique<WindowFinderMac>();
+  return rtc::MakeUnique<WindowFinderMac>(options.configuration_monitor);
 }
 
 }  // namespace webrtc