Include menus & dialogs in frames captured by WindowCapturerWin

This change adds logic to WindowCapturerWin to capture overlapping
owned/pop-up windows (e.g. menus, dialogs, tooltips). This makes window
capture behavior more consistent regardless of whether
CroppingWindowCapturerWin is used & its conditions for using crop-from-
screen capture are met (in ShouldUseScreenCapturer). (I.e. regardless
of OS version, window shape / translucency, occlusion by another
potentially top-most window, or whether the capturing app has opted in
to using the cropping capturer).

Owned/pop-up windows associated with the selected window are enumerated
then captured individually, with their contents composited into the
final frame.

This change also:
- Crops out the top window border (which exposed a bit of the background
  when using the cropping capturer, and resulted in an inconsistent
  appearance compared to the side & bottom borders being cropped out).

Bug: chromium:980864
Change-Id: I81c504848a0c0e6bf122aeff437b400e44944718
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/148302
Commit-Queue: Jamie Walch <jamiewalch@chromium.org>
Reviewed-by: Jamie Walch <jamiewalch@chromium.org>
Cr-Commit-Position: refs/heads/master@{#28922}

modules/desktop_capture/BUILD.gn

@@ -81,6 +81,7 @@ if (rtc_include_tests) {
       "desktop_and_cursor_composer_unittest.cc",
       "desktop_capturer_differ_wrapper_unittest.cc",
       "desktop_frame_rotation_unittest.cc",
+      "desktop_frame_unittest.cc",
       "desktop_geometry_unittest.cc",
       "desktop_region_unittest.cc",
       "differ_block_unittest.cc",
@@ -347,6 +348,8 @@ rtc_static_library("desktop_capture_generic") {
     "win/screen_capturer_win_gdi.h",
     "win/screen_capturer_win_magnifier.cc",
     "win/screen_capturer_win_magnifier.h",
+    "win/selected_window_context.cc",
+    "win/selected_window_context.h",
     "win/window_capture_utils.cc",
     "win/window_capture_utils.h",
     "window_capturer_win.cc",

modules/desktop_capture/cropping_window_capturer_win.cc

@@ -11,6 +11,7 @@
 #include "modules/desktop_capture/cropping_window_capturer.h"
 #include "modules/desktop_capture/desktop_capturer_differ_wrapper.h"
 #include "modules/desktop_capture/win/screen_capture_utils.h"
+#include "modules/desktop_capture/win/selected_window_context.h"
 #include "modules/desktop_capture/win/window_capture_utils.h"
 #include "rtc_base/logging.h"
 #include "rtc_base/trace_event.h"
@@ -20,34 +21,22 @@ namespace webrtc {
 
 namespace {
 
-const size_t kTitleLength = 256;
+// Used to pass input/output data during the EnumWindows call for verifying if
-
-// Used to pass input/output data during the EnumWindow call for verifying if
 // the selected window is on top.
-struct TopWindowVerifierContext {
+struct TopWindowVerifierContext : public SelectedWindowContext {
   TopWindowVerifierContext(HWND selected_window,
                            HWND excluded_window,
                            DesktopRect selected_window_rect,
                            WindowCaptureHelperWin* window_capture_helper)
-      : selected_window(selected_window),
+      : SelectedWindowContext(selected_window,
+                              selected_window_rect,
+                              window_capture_helper),
         excluded_window(excluded_window),
-        selected_window_rect(selected_window_rect),
-        window_capture_helper(window_capture_helper),
         is_top_window(false) {
     RTC_DCHECK_NE(selected_window, excluded_window);
-
-    GetWindowTextW(selected_window, selected_window_title, kTitleLength);
-    selected_window_thread_id =
-        GetWindowThreadProcessId(selected_window, &selected_window_process_id);
   }
 
-  const HWND selected_window;
   const HWND excluded_window;
-  const DesktopRect selected_window_rect;
-  WindowCaptureHelperWin* window_capture_helper;
-  WCHAR selected_window_title[kTitleLength];
-  DWORD selected_window_process_id;
-  DWORD selected_window_thread_id;
   bool is_top_window;
 };
 
@@ -61,7 +50,9 @@ BOOL CALLBACK TopWindowVerifier(HWND hwnd, LPARAM param) {
   TopWindowVerifierContext* context =
       reinterpret_cast<TopWindowVerifierContext*>(param);
 
-  if (hwnd == context->selected_window) {
+  if (context->IsWindowSelected(hwnd)) {
+    // Windows are enumerated in top-down z-order, so we can stop enumerating
+    // upon reaching the selected window & report it's on top.
     context->is_top_window = true;
     return FALSE;
   }
@@ -72,7 +63,8 @@ BOOL CALLBACK TopWindowVerifier(HWND hwnd, LPARAM param) {
   }
 
   // Ignore invisible window on current desktop.
-  if (!context->window_capture_helper->IsWindowVisibleOnCurrentDesktop(hwnd)) {
+  if (!context->window_capture_helper()->IsWindowVisibleOnCurrentDesktop(
+          hwnd)) {
     return TRUE;
   }
 
@@ -83,33 +75,18 @@ BOOL CALLBACK TopWindowVerifier(HWND hwnd, LPARAM param) {
   // - All notifications from Chrome will be ignored.
   // - This may cause part or whole of notification window being cropped into
   // the capturing of the target window if there is overlapping.
-  if (context->window_capture_helper->IsWindowChromeNotification(hwnd)) {
+  if (context->window_capture_helper()->IsWindowChromeNotification(hwnd)) {
     return TRUE;
   }
 
-  // Ignore descendant/owned windows since we want to capture them. This check
+  // Ignore descendant/owned windows since we want to capture them.
-  // works for drop-down menus, pop-up (dialog) windows, and child (confined)
+  if (context->IsWindowOwned(hwnd)) {
-  // windows. It doesn't work for tooltips or context menus, which are handled
-  // differently below.
-  if (GetAncestor(hwnd, GA_ROOTOWNER) == context->selected_window) {
-    return TRUE;
-  }
-
-  // Ignore windows that belong to the same thread since we want to capture
-  // them. This check works for tooltips & context menus.
-  DWORD enumerated_window_process_id = 0;
-  DWORD enumerated_window_thread_id =
-      GetWindowThreadProcessId(hwnd, &enumerated_window_process_id);
-  if (enumerated_window_thread_id != 0 &&
-      enumerated_window_process_id == context->selected_window_process_id &&
-      enumerated_window_thread_id == context->selected_window_thread_id) {
     return TRUE;
   }
 
   // Checks whether current window |hwnd| intersects with
   // |context|->selected_window.
-  if (context->window_capture_helper->IsWindowIntersectWithSelectedWindow(
+  if (context->IsWindowOverlapping(hwnd)) {
-          hwnd, context->selected_window, context->selected_window_rect)) {
     // If intersection is not empty, the selected window is not on top.
     context->is_top_window = false;
     return FALSE;
@@ -218,6 +195,10 @@ bool CroppingWindowCapturerWin::ShouldUseScreenCapturer() {
   TopWindowVerifierContext context(selected,
                                    reinterpret_cast<HWND>(excluded_window()),
                                    content_rect, &window_capture_helper_);
+  if (!context.IsSelectedWindowValid()) {
+    return false;
+  }
+
   EnumWindows(&TopWindowVerifier, reinterpret_cast<LPARAM>(&context));
   return context.is_top_window;
 }
@@ -227,7 +208,8 @@ DesktopRect CroppingWindowCapturerWin::GetWindowRectInVirtualScreen() {
                "CroppingWindowCapturerWin::GetWindowRectInVirtualScreen");
   DesktopRect window_rect;
   HWND hwnd = reinterpret_cast<HWND>(selected_window());
-  if (!GetCroppedWindowRect(hwnd, &window_rect, /* original_rect */ nullptr)) {
+  if (!GetCroppedWindowRect(hwnd, /*avoid_cropping_border*/ false, &window_rect,
+                            /*original_rect*/ nullptr)) {
     RTC_LOG(LS_WARNING) << "Failed to get window info: " << GetLastError();
     return window_rect;
   }

modules/desktop_capture/desktop_frame.cc

@@ -12,6 +12,7 @@
 
 #include <string.h>
 
+#include <cmath>
 #include <utility>
 
 #include "absl/memory/memory.h"
@@ -61,6 +62,47 @@ void DesktopFrame::CopyPixelsFrom(const DesktopFrame& src_frame,
                  dest_rect);
 }
 
+bool DesktopFrame::CopyIntersectingPixelsFrom(const DesktopFrame& src_frame,
+                                              double horizontal_scale,
+                                              double vertical_scale) {
+  const DesktopVector& origin = top_left();
+  const DesktopVector& src_frame_origin = src_frame.top_left();
+
+  DesktopVector src_frame_offset = src_frame_origin.subtract(origin);
+
+  // Determine the intersection, first adjusting its origin to account for any
+  // DPI scaling.
+  DesktopRect intersection_rect = src_frame.rect();
+  if (horizontal_scale != 1.0 || vertical_scale != 1.0) {
+    DesktopVector origin_adjustment(
+        static_cast<int>(
+            std::round((horizontal_scale - 1.0) * src_frame_offset.x())),
+        static_cast<int>(
+            std::round((vertical_scale - 1.0) * src_frame_offset.y())));
+
+    intersection_rect.Translate(origin_adjustment);
+
+    src_frame_offset = src_frame_offset.add(origin_adjustment);
+  }
+
+  intersection_rect.IntersectWith(rect());
+  if (intersection_rect.is_empty()) {
+    return false;
+  }
+
+  // Translate the intersection rect to be relative to the outer rect.
+  intersection_rect.Translate(-origin.x(), -origin.y());
+
+  // Determine source position for the copy (offsets of outer frame from
+  // source origin, if positive).
+  int32_t src_pos_x = std::max(0, -src_frame_offset.x());
+  int32_t src_pos_y = std::max(0, -src_frame_offset.y());
+
+  CopyPixelsFrom(src_frame, DesktopVector(src_pos_x, src_pos_y),
+                 intersection_rect);
+  return true;
+}
+
 DesktopRect DesktopFrame::rect() const {
   const float scale = scale_factor();
   // Only scale the size.

modules/desktop_capture/desktop_frame.h

@@ -85,6 +85,16 @@ class RTC_EXPORT DesktopFrame {
                       const DesktopVector& src_pos,
                       const DesktopRect& dest_rect);
 
+  // Copies pixels from another frame, with the copied & overwritten regions
+  // representing the intersection between the two frames. Returns true if
+  // pixels were copied, or false if there's no intersection. The scale factors
+  // represent the ratios between pixel space & offset coordinate space (e.g.
+  // 2.0 would indicate the frames are scaled down by 50% for display, so any
+  // offset between their origins should be doubled).
+  bool CopyIntersectingPixelsFrom(const DesktopFrame& src_frame,
+                                  double horizontal_scale,
+                                  double vertical_scale);
+
   // A helper to return the data pointer of a frame at the specified position.
   uint8_t* GetFrameDataAtPos(const DesktopVector& pos) const;
 

modules/desktop_capture/desktop_frame_unittest.cc

@@ -0,0 +1,335 @@
+/*
+ *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/desktop_capture/desktop_frame.h"
+
+#include "absl/memory/memory.h"
+#include "modules/desktop_capture/desktop_region.h"
+#include "modules/desktop_capture/test_utils.h"
+#include "rtc_base/arraysize.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+namespace {
+
+std::unique_ptr<DesktopFrame> CreateTestFrame(DesktopRect rect,
+                                              int pixels_value) {
+  DesktopSize size = rect.size();
+  auto frame = absl::make_unique<BasicDesktopFrame>(size);
+  frame->set_top_left(rect.top_left());
+  memset(frame->data(), pixels_value, frame->stride() * size.height());
+  return frame;
+}
+
+struct TestData {
+  const char* description;
+  DesktopRect dest_frame_rect;
+  DesktopRect src_frame_rect;
+  double horizontal_scale;
+  double vertical_scale;
+  DesktopRect expected_overlap_rect;
+};
+
+void RunTest(const TestData& test) {
+  // Copy a source frame with all bits set into a dest frame with none set.
+  auto dest_frame = CreateTestFrame(test.dest_frame_rect, 0);
+  auto src_frame = CreateTestFrame(test.src_frame_rect, 0xff);
+
+  dest_frame->CopyIntersectingPixelsFrom(
+      *src_frame, test.horizontal_scale, test.vertical_scale);
+
+  // Translate the expected overlap rect to be relative to the dest frame/rect.
+  DesktopVector dest_frame_origin = test.dest_frame_rect.top_left();
+  DesktopRect relative_expected_overlap_rect = test.expected_overlap_rect;
+  relative_expected_overlap_rect.Translate(-dest_frame_origin.x(),
+                                           -dest_frame_origin.y());
+
+  // Confirm bits are now set in the dest frame if & only if they fall in the
+  // expected range.
+  for (int y = 0; y < dest_frame->size().height(); ++y) {
+    SCOPED_TRACE(y);
+
+    for (int x = 0; x < dest_frame->size().width(); ++x) {
+      SCOPED_TRACE(x);
+
+      DesktopVector point(x, y);
+      uint8_t* data = dest_frame->GetFrameDataAtPos(point);
+      uint32_t pixel_value = *reinterpret_cast<uint32_t*>(data);
+      bool was_copied = pixel_value == 0xffffffff;
+      ASSERT_TRUE(was_copied || pixel_value == 0);
+
+      bool expected_to_be_copied =
+          relative_expected_overlap_rect.Contains(point);
+
+      ASSERT_EQ(was_copied, expected_to_be_copied);
+    }
+  }
+}
+
+void RunTests(const TestData* tests, int num_tests) {
+  for (int i = 0; i < num_tests; i++) {
+    const TestData& test = tests[i];
+
+    SCOPED_TRACE(test.description);
+
+    RunTest(test);
+  }
+}
+
+}  // namespace
+
+TEST(DesktopFrameTest, CopyIntersectingPixelsMatchingRects) {
+  const TestData tests[] = {
+    {"0 origin",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     1.0, 1.0,
+     DesktopRect::MakeXYWH(0, 0, 2, 2)},
+
+    {"Negative origin",
+     DesktopRect::MakeXYWH(-1, -1, 2, 2),
+     DesktopRect::MakeXYWH(-1, -1, 2, 2),
+     1.0, 1.0,
+     DesktopRect::MakeXYWH(-1, -1, 2, 2)}
+  };
+
+  RunTests(tests, arraysize(tests));
+}
+
+TEST(DesktopFrameTest, CopyIntersectingPixelsMatchingRectsScaled) {
+  // The scale factors shouldn't affect matching rects (they're only applied
+  // to any difference between the origins)
+  const TestData tests[] = {
+    {"0 origin 2x",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     2.0, 2.0,
+     DesktopRect::MakeXYWH(0, 0, 2, 2)},
+
+    {"0 origin 0.5x",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     0.5, 0.5,
+     DesktopRect::MakeXYWH(0, 0, 2, 2)},
+
+    {"Negative origin 2x",
+     DesktopRect::MakeXYWH(-1, -1, 2, 2),
+     DesktopRect::MakeXYWH(-1, -1, 2, 2),
+     2.0, 2.0,
+     DesktopRect::MakeXYWH(-1, -1, 2, 2)},
+
+    {"Negative origin 0.5x",
+     DesktopRect::MakeXYWH(-1, -1, 2, 2),
+     DesktopRect::MakeXYWH(-1, -1, 2, 2),
+     0.5, 0.5,
+     DesktopRect::MakeXYWH(-1, -1, 2, 2)}
+  };
+
+  RunTests(tests, arraysize(tests));
+}
+
+TEST(DesktopFrameTest, CopyIntersectingPixelsFullyContainedRects) {
+  const TestData tests[] = {
+    {"0 origin top left",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(0, 0, 1, 1),
+     1.0, 1.0,
+     DesktopRect::MakeXYWH(0, 0, 1, 1)},
+
+    {"0 origin bottom right",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(1, 1, 1, 1),
+     1.0, 1.0,
+     DesktopRect::MakeXYWH(1, 1, 1, 1)},
+
+    {"Negative origin bottom left",
+     DesktopRect::MakeXYWH(-1, -1, 2, 2),
+     DesktopRect::MakeXYWH(-1, 0, 1, 1),
+     1.0, 1.0,
+     DesktopRect::MakeXYWH(-1, 0, 1, 1)}
+  };
+
+  RunTests(tests, arraysize(tests));
+}
+
+TEST(DesktopFrameTest, CopyIntersectingPixelsFullyContainedRectsScaled) {
+  const TestData tests[] = {
+    {"0 origin top left 2x",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(0, 0, 1, 1),
+     2.0, 2.0,
+     DesktopRect::MakeXYWH(0, 0, 1, 1)},
+
+    {"0 origin top left 0.5x",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(0, 0, 1, 1),
+     0.5, 0.5,
+     DesktopRect::MakeXYWH(0, 0, 1, 1)},
+
+    {"0 origin bottom left 2x",
+     DesktopRect::MakeXYWH(0, 0, 4, 4),
+     DesktopRect::MakeXYWH(1, 1, 2, 2),
+     2.0, 2.0,
+     DesktopRect::MakeXYWH(2, 2, 2, 2)},
+
+    {"0 origin bottom middle 2x/1x",
+     DesktopRect::MakeXYWH(0, 0, 4, 3),
+     DesktopRect::MakeXYWH(1, 1, 2, 2),
+     2.0, 1.0,
+     DesktopRect::MakeXYWH(2, 1, 2, 2)},
+
+    {"0 origin middle 0.5x",
+     DesktopRect::MakeXYWH(0, 0, 3, 3),
+     DesktopRect::MakeXYWH(2, 2, 1, 1),
+     0.5, 0.5,
+     DesktopRect::MakeXYWH(1, 1, 1, 1)},
+
+    {"Negative origin bottom left 2x",
+     DesktopRect::MakeXYWH(-1, -1, 3, 3),
+     DesktopRect::MakeXYWH(-1, 0, 1, 1),
+     2.0, 2.0,
+     DesktopRect::MakeXYWH(-1, 1, 1, 1)},
+
+    {"Negative origin near middle 0.5x",
+     DesktopRect::MakeXYWH(-2, -2, 2, 2),
+     DesktopRect::MakeXYWH(0, 0, 1, 1),
+     0.5, 0.5,
+     DesktopRect::MakeXYWH(-1, -1, 1, 1)}
+  };
+
+  RunTests(tests, arraysize(tests));
+}
+
+
+TEST(DesktopFrameTest, CopyIntersectingPixelsPartiallyContainedRects) {
+  const TestData tests[] = {
+    {"Top left",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(-1, -1, 2, 2),
+     1.0, 1.0,
+     DesktopRect::MakeXYWH(0, 0, 1, 1)},
+
+    {"Top right",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(1, -1, 2, 2),
+     1.0, 1.0,
+     DesktopRect::MakeXYWH(1, 0, 1, 1)},
+
+    {"Bottom right",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(1, 1, 2, 2),
+     1.0, 1.0,
+     DesktopRect::MakeXYWH(1, 1, 1, 1)},
+
+    {"Bottom left",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(-1, 1, 2, 2),
+     1.0, 1.0,
+     DesktopRect::MakeXYWH(0, 1, 1, 1)}
+  };
+
+  RunTests(tests, arraysize(tests));
+}
+
+TEST(DesktopFrameTest, CopyIntersectingPixelsPartiallyContainedRectsScaled) {
+  const TestData tests[] = {
+    {"Top left 2x",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(-1, -1, 3, 3),
+     2.0, 2.0,
+     DesktopRect::MakeXYWH(0, 0, 1, 1)},
+
+    {"Top right 0.5x",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(2, -2, 2, 2),
+     0.5, 0.5,
+     DesktopRect::MakeXYWH(1, 0, 1, 1)},
+
+    {"Bottom right 2x",
+     DesktopRect::MakeXYWH(0, 0, 3, 3),
+     DesktopRect::MakeXYWH(-1, 1, 3, 3),
+     2.0, 2.0,
+     DesktopRect::MakeXYWH(0, 2, 1, 1)},
+
+    {"Bottom left 0.5x",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(-2, 2, 2, 2),
+     0.5, 0.5,
+     DesktopRect::MakeXYWH(0, 1, 1, 1)}
+  };
+
+  RunTests(tests, arraysize(tests));
+}
+
+
+TEST(DesktopFrameTest, CopyIntersectingPixelsUncontainedRects) {
+  const TestData tests[] = {
+    {"Left",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(-1, 0, 1, 2),
+     1.0, 1.0,
+     DesktopRect::MakeXYWH(0, 0, 0, 0)},
+
+    {"Top",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(0, -1, 2, 1),
+     1.0, 1.0,
+     DesktopRect::MakeXYWH(0, 0, 0, 0)},
+
+    {"Right",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(2, 0, 1, 2),
+     1.0, 1.0,
+     DesktopRect::MakeXYWH(0, 0, 0, 0)},
+
+
+    {"Bottom",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(0, 2, 2, 1),
+     1.0, 1.0,
+     DesktopRect::MakeXYWH(0, 0, 0, 0)}
+  };
+
+  RunTests(tests, arraysize(tests));
+}
+
+TEST(DesktopFrameTest, CopyIntersectingPixelsUncontainedRectsScaled) {
+  const TestData tests[] = {
+    {"Left 2x",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(-1, 0, 2, 2),
+     2.0, 2.0,
+     DesktopRect::MakeXYWH(0, 0, 0, 0)},
+
+    {"Top 0.5x",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(0, -2, 2, 1),
+     0.5, 0.5,
+     DesktopRect::MakeXYWH(0, 0, 0, 0)},
+
+    {"Right 2x",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(1, 0, 1, 2),
+     2.0, 2.0,
+     DesktopRect::MakeXYWH(0, 0, 0, 0)},
+
+
+    {"Bottom 0.5x",
+     DesktopRect::MakeXYWH(0, 0, 2, 2),
+     DesktopRect::MakeXYWH(0, 4, 2, 1),
+     0.5, 0.5,
+     DesktopRect::MakeXYWH(0, 0, 0, 0)}
+  };
+
+  RunTests(tests, arraysize(tests));
+}
+
+}  // namespace webrtc

modules/desktop_capture/desktop_geometry.cc

@@ -11,6 +11,7 @@
 #include "modules/desktop_capture/desktop_geometry.h"
 
 #include <algorithm>
+#include <cmath>
 
 namespace webrtc {
 
@@ -71,8 +72,8 @@ void DesktopRect::Extend(int32_t left_offset,
 }
 
 void DesktopRect::Scale(double horizontal, double vertical) {
-  right_ += width() * (horizontal - 1);
+  right_ += static_cast<int>(std::round(width() * (horizontal - 1)));
-  bottom_ += height() * (vertical - 1);
+  bottom_ += static_cast<int>(std::round(height() * (vertical - 1)));
 }
 
 }  // namespace webrtc

modules/desktop_capture/mouse_cursor_monitor_win.cc

@@ -144,7 +144,8 @@ void MouseCursorMonitorWin::Capture() {
   if (window_) {
     DesktopRect original_rect;
     DesktopRect cropped_rect;
-    if (!GetCroppedWindowRect(window_, &cropped_rect, &original_rect)) {
+    if (!GetCroppedWindowRect(window_, /*avoid_cropping_border*/ false,
+                              &cropped_rect, &original_rect)) {
       position.set(0, 0);
       inside = false;
     } else {

modules/desktop_capture/win/selected_window_context.cc

@@ -0,0 +1,60 @@
+/*
+ *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/desktop_capture/win/selected_window_context.h"
+
+namespace webrtc {
+
+SelectedWindowContext::SelectedWindowContext(
+    HWND selected_window,
+    DesktopRect selected_window_rect,
+    WindowCaptureHelperWin* window_capture_helper)
+    : selected_window_(selected_window),
+      selected_window_rect_(selected_window_rect),
+      window_capture_helper_(window_capture_helper) {
+  selected_window_thread_id_ =
+      GetWindowThreadProcessId(selected_window, &selected_window_process_id_);
+}
+
+bool SelectedWindowContext::IsSelectedWindowValid() const {
+  return selected_window_thread_id_ != 0;
+}
+
+bool SelectedWindowContext::IsWindowSelected(HWND hwnd) const {
+  return hwnd == selected_window_;
+}
+
+bool SelectedWindowContext::IsWindowOwned(HWND hwnd) const {
+  // This check works for drop-down menus & dialog pop-up windows. It doesn't
+  // work for context menus or tooltips, which are handled differently below.
+  if (GetAncestor(hwnd, GA_ROOTOWNER) == selected_window_) {
+    return true;
+  }
+
+  // Some pop-up windows aren't owned (e.g. context menus, tooltips); treat
+  // windows that belong to the same thread as owned.
+  DWORD enumerated_window_process_id = 0;
+  DWORD enumerated_window_thread_id =
+      GetWindowThreadProcessId(hwnd, &enumerated_window_process_id);
+  return enumerated_window_thread_id != 0 &&
+         enumerated_window_process_id == selected_window_process_id_ &&
+         enumerated_window_thread_id == selected_window_thread_id_;
+}
+
+bool SelectedWindowContext::IsWindowOverlapping(HWND hwnd) const {
+  return window_capture_helper_->IsWindowIntersectWithSelectedWindow(
+      hwnd, selected_window_, selected_window_rect_);
+}
+
+WindowCaptureHelperWin* SelectedWindowContext::window_capture_helper() const {
+  return window_capture_helper_;
+}
+
+}  // namespace webrtc

modules/desktop_capture/win/selected_window_context.h

@@ -0,0 +1,45 @@
+/*
+ *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_DESKTOP_CAPTURE_WIN_SELECTED_WINDOW_CONTEXT_H_
+#define MODULES_DESKTOP_CAPTURE_WIN_SELECTED_WINDOW_CONTEXT_H_
+
+#include <windows.h>
+
+#include "modules/desktop_capture/desktop_geometry.h"
+#include "modules/desktop_capture/win/window_capture_utils.h"
+
+namespace webrtc {
+
+class SelectedWindowContext {
+ public:
+  SelectedWindowContext(HWND selected_window,
+                        DesktopRect selected_window_rect,
+                        WindowCaptureHelperWin* window_capture_helper);
+
+  bool IsSelectedWindowValid() const;
+
+  bool IsWindowSelected(HWND hwnd) const;
+  bool IsWindowOwned(HWND hwnd) const;
+  bool IsWindowOverlapping(HWND hwnd) const;
+
+  WindowCaptureHelperWin* window_capture_helper() const;
+
+ private:
+  const HWND selected_window_;
+  const DesktopRect selected_window_rect_;
+  WindowCaptureHelperWin* const window_capture_helper_;
+  DWORD selected_window_thread_id_;
+  DWORD selected_window_process_id_;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_DESKTOP_CAPTURE_WIN_SELECTED_WINDOW_CONTEXT_H_

modules/desktop_capture/win/window_capture_utils.cc

@@ -36,6 +36,7 @@ bool GetWindowRect(HWND window, DesktopRect* result) {
 }
 
 bool GetCroppedWindowRect(HWND window,
+                          bool avoid_cropping_border,
                           DesktopRect* cropped_rect,
                           DesktopRect* original_rect) {
   DesktopRect window_rect;
@@ -53,13 +54,30 @@ bool GetCroppedWindowRect(HWND window,
     return false;
   }
 
-  // After Windows8, transparent borders will be added by OS at
+  // As of Windows8, transparent resize borders are added by the OS at
-  // left/bottom/right sides of a window. If the cropped window
+  // left/bottom/right sides of a resizeable window. If the cropped window
   // doesn't remove these borders, the background will be exposed a bit.
   if (rtc::IsWindows8OrLater() || is_maximized) {
-    const int width = GetSystemMetrics(SM_CXSIZEFRAME);
+    // Only apply this cropping to windows with a resize border (otherwise,
-    const int height = GetSystemMetrics(SM_CYSIZEFRAME);
+    // it'd clip the edges of captured pop-up windows without this border).
-    cropped_rect->Extend(-width, 0, -width, -height);
+    LONG style = GetWindowLong(window, GWL_STYLE);
+    if (style & WS_THICKFRAME || style & DS_MODALFRAME) {
+      int width = GetSystemMetrics(SM_CXSIZEFRAME);
+      int bottom_height = GetSystemMetrics(SM_CYSIZEFRAME);
+      const int visible_border_height = GetSystemMetrics(SM_CYBORDER);
+      int top_height = visible_border_height;
+
+      // If requested, avoid cropping the visible window border. This is used
+      // for pop-up windows to include their border, but not for the outermost
+      // window (where a partially-transparent border may expose the
+      // background a bit).
+      if (avoid_cropping_border) {
+        width = std::max(0, width - GetSystemMetrics(SM_CXBORDER));
+        bottom_height = std::max(0, bottom_height - visible_border_height);
+        top_height = 0;
+      }
+      cropped_rect->Extend(-width, -top_height, -width, -bottom_height);
+    }
   }
 
   return true;

modules/desktop_capture/win/window_capture_utils.h

@@ -8,6 +8,9 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
+#ifndef MODULES_DESKTOP_CAPTURE_WIN_WINDOW_CAPTURE_UTILS_H_
+#define MODULES_DESKTOP_CAPTURE_WIN_WINDOW_CAPTURE_UTILS_H_
+
 #include <shlobj.h>
 #include <windows.h>
 #include <wrl/client.h>
@@ -23,7 +26,10 @@ namespace webrtc {
 bool GetWindowRect(HWND window, DesktopRect* result);
 
 // Outputs the window rect, with the left/right/bottom frame border cropped if
-// the window is maximized. |cropped_rect| is the cropped rect relative to the
+// the window is maximized or has a transparent resize border.
+// |avoid_cropping_border| may be set to true to avoid cropping the visible
+// border when cropping any resize border.
+// |cropped_rect| is the cropped rect relative to the
 // desktop. |original_rect| is the original rect returned from GetWindowRect.
 // Returns true if all API calls succeeded. The returned DesktopRect is in
 // system coordinates, i.e. the primary monitor on the system always starts from
@@ -37,6 +43,7 @@ bool GetWindowRect(HWND window, DesktopRect* result);
 // WindowCapturerWin to crop out the borders or shadow according to their
 // scenarios. But this function is too generic and easy to be misused.
 bool GetCroppedWindowRect(HWND window,
+                          bool avoid_cropping_border,
                           DesktopRect* cropped_rect,
                           DesktopRect* original_rect);
 
@@ -91,3 +98,5 @@ class WindowCaptureHelperWin {
 };
 
 }  // namespace webrtc
+
+#endif  // MODULES_DESKTOP_CAPTURE_WIN_WINDOW_CAPTURE_UTILS_H_

modules/desktop_capture/window_capturer_win.cc

@@ -12,12 +12,15 @@
 
 #include <memory>
 
+#include "absl/memory/memory.h"
 #include "modules/desktop_capture/cropped_desktop_frame.h"
 #include "modules/desktop_capture/desktop_capturer.h"
 #include "modules/desktop_capture/desktop_frame_win.h"
 #include "modules/desktop_capture/win/screen_capture_utils.h"
+#include "modules/desktop_capture/win/selected_window_context.h"
 #include "modules/desktop_capture/win/window_capture_utils.h"
 #include "modules/desktop_capture/window_finder_win.h"
+#include "rtc_base/arraysize.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/constructor_magic.h"
 #include "rtc_base/logging.h"
@@ -92,6 +95,66 @@ BOOL CALLBACK WindowsEnumerationHandler(HWND hwnd, LPARAM param) {
   return TRUE;
 }
 
+// Used to pass input/output data during the EnumWindows call to collect
+// owned/pop-up windows that should be captured.
+struct OwnedWindowCollectorContext : public SelectedWindowContext {
+  OwnedWindowCollectorContext(HWND selected_window,
+                              DesktopRect selected_window_rect,
+                              WindowCaptureHelperWin* window_capture_helper,
+                              std::vector<HWND>* owned_windows)
+      : SelectedWindowContext(selected_window,
+                              selected_window_rect,
+                              window_capture_helper),
+        owned_windows(owned_windows) {}
+
+  std::vector<HWND>* owned_windows;
+};
+
+// Called via EnumWindows for each root window; adds owned/pop-up windows that
+// should be captured to a vector it's passed.
+BOOL CALLBACK OwnedWindowCollector(HWND hwnd, LPARAM param) {
+  OwnedWindowCollectorContext* context =
+      reinterpret_cast<OwnedWindowCollectorContext*>(param);
+  if (context->IsWindowSelected(hwnd)) {
+    // Windows are enumerated in top-down z-order, so we can stop enumerating
+    // upon reaching the selected window.
+    return FALSE;
+  }
+
+  // Skip windows that aren't visible pop-up windows.
+  if (!(GetWindowLong(hwnd, GWL_STYLE) & WS_POPUP) ||
+      !context->window_capture_helper()->IsWindowVisibleOnCurrentDesktop(
+          hwnd)) {
+    return TRUE;
+  }
+
+  // Owned windows that intersect the selected window should be captured.
+  if (context->IsWindowOwned(hwnd) && context->IsWindowOverlapping(hwnd)) {
+    // Skip windows that draw shadows around menus. These "SysShadow" windows
+    // would otherwise be captured as solid black bars with no transparency
+    // gradient (since this capturer doesn't detect / respect variations in the
+    // window alpha channel). Any other semi-transparent owned windows will be
+    // captured fully-opaque. This seems preferable to excluding them (at least
+    // when they have content aside from a solid fill color / visual adornment;
+    // e.g. some tooltips have the transparent style set).
+    if (GetWindowLong(hwnd, GWL_EXSTYLE) & WS_EX_TRANSPARENT) {
+      const WCHAR kSysShadow[] = L"SysShadow";
+      const size_t kClassLength = arraysize(kSysShadow);
+      WCHAR class_name[kClassLength];
+      const int class_name_length =
+          GetClassNameW(hwnd, class_name, kClassLength);
+      if (class_name_length == kClassLength - 1 &&
+          wcscmp(class_name, kSysShadow) == 0) {
+        return TRUE;
+      }
+    }
+
+    context->owned_windows->push_back(hwnd);
+  }
+
+  return TRUE;
+}
+
 class WindowCapturerWin : public DesktopCapturer {
  public:
   WindowCapturerWin();
@@ -106,6 +169,13 @@ class WindowCapturerWin : public DesktopCapturer {
   bool IsOccluded(const DesktopVector& pos) override;
 
  private:
+  struct CaptureResults {
+    Result result;
+    std::unique_ptr<DesktopFrame> frame;
+  };
+
+  CaptureResults CaptureFrame(bool capture_owned_windows);
+
   Callback* callback_ = nullptr;
 
   // HWND and HDC for the currently selected window or nullptr if window is not
@@ -122,6 +192,9 @@ class WindowCapturerWin : public DesktopCapturer {
 
   WindowFinderWin window_finder_;
 
+  std::vector<HWND> owned_windows_;
+  std::unique_ptr<WindowCapturerWin> owned_window_capturer_;
+
   RTC_DISALLOW_COPY_AND_ASSIGN(WindowCapturerWin);
 };
 
@@ -179,8 +252,12 @@ bool WindowCapturerWin::FocusOnSelectedSource() {
 
 bool WindowCapturerWin::IsOccluded(const DesktopVector& pos) {
   DesktopVector sys_pos = pos.add(GetFullscreenRect().top_left());
-  return reinterpret_cast<HWND>(window_finder_.GetWindowUnderPoint(sys_pos)) !=
+  HWND hwnd =
-         window_;
+      reinterpret_cast<HWND>(window_finder_.GetWindowUnderPoint(sys_pos));
+
+  return hwnd != window_ &&
+         std::find(owned_windows_.begin(), owned_windows_.end(), hwnd) ==
+             owned_windows_.end();
 }
 
 void WindowCapturerWin::Start(Callback* callback) {
@@ -191,28 +268,40 @@ void WindowCapturerWin::Start(Callback* callback) {
 }
 
 void WindowCapturerWin::CaptureFrame() {
+  CaptureResults results = CaptureFrame(/*capture_owned_windows*/ true);
+
+  callback_->OnCaptureResult(results.result, std::move(results.frame));
+}
+
+WindowCapturerWin::CaptureResults WindowCapturerWin::CaptureFrame(
+    bool capture_owned_windows) {
   TRACE_EVENT0("webrtc", "WindowCapturerWin::CaptureFrame");
 
   if (!window_) {
     RTC_LOG(LS_ERROR) << "Window hasn't been selected: " << GetLastError();
-    callback_->OnCaptureResult(Result::ERROR_PERMANENT, nullptr);
+    return {Result::ERROR_PERMANENT, nullptr};
-    return;
   }
 
   // Stop capturing if the window has been closed.
   if (!IsWindow(window_)) {
     RTC_LOG(LS_ERROR) << "target window has been closed";
-    callback_->OnCaptureResult(Result::ERROR_PERMANENT, nullptr);
+    return {Result::ERROR_PERMANENT, nullptr};
-    return;
   }
 
+  // Determine the window region excluding any resize border, and including
+  // any visible border if capturing an owned window / dialog. (Don't include
+  // any visible border for the selected window for consistency with
+  // CroppingWindowCapturerWin, which would expose a bit of the background
+  // through the partially-transparent border.)
+  const bool avoid_cropping_border = !capture_owned_windows;
   DesktopRect cropped_rect;
   DesktopRect original_rect;
-  if (!GetCroppedWindowRect(window_, &cropped_rect, &original_rect)) {
+
+  if (!GetCroppedWindowRect(window_, avoid_cropping_border, &cropped_rect,
+                            &original_rect)) {
     RTC_LOG(LS_WARNING) << "Failed to get drawable window area: "
                         << GetLastError();
-    callback_->OnCaptureResult(Result::ERROR_TEMPORARY, nullptr);
+    return {Result::ERROR_TEMPORARY, nullptr};
-    return;
   }
 
   // Return a 1x1 black frame if the window is minimized or invisible on current
@@ -225,17 +314,19 @@ void WindowCapturerWin::CaptureFrame() {
 
     previous_size_ = frame->size();
     window_size_map_[window_] = previous_size_;
-    callback_->OnCaptureResult(Result::SUCCESS, std::move(frame));
+    return {Result::SUCCESS, std::move(frame)};
-    return;
   }
 
   HDC window_dc = GetWindowDC(window_);
   if (!window_dc) {
     RTC_LOG(LS_WARNING) << "Failed to get window DC: " << GetLastError();
-    callback_->OnCaptureResult(Result::ERROR_TEMPORARY, nullptr);
+    return {Result::ERROR_TEMPORARY, nullptr};
-    return;
   }
 
+  DesktopRect unscaled_cropped_rect = cropped_rect;
+  double horizontal_scale = 1.0;
+  double vertical_scale = 1.0;
+
   DesktopSize window_dc_size;
   if (GetDcSize(window_dc, &window_dc_size)) {
     // The |window_dc_size| is used to detect the scaling of the original
@@ -251,12 +342,12 @@ void WindowCapturerWin::CaptureFrame() {
 
     // If |window_dc_size| is smaller than |window_rect|, let's resize both
     // |original_rect| and |cropped_rect| according to the scaling factor.
-    const double vertical_scale =
+    horizontal_scale =
         static_cast<double>(window_dc_size.width()) / original_rect.width();
-    const double horizontal_scale =
+    vertical_scale =
         static_cast<double>(window_dc_size.height()) / original_rect.height();
-    original_rect.Scale(vertical_scale, horizontal_scale);
+    original_rect.Scale(horizontal_scale, vertical_scale);
-    cropped_rect.Scale(vertical_scale, horizontal_scale);
+    cropped_rect.Scale(horizontal_scale, vertical_scale);
   }
 
   std::unique_ptr<DesktopFrameWin> frame(
@@ -264,8 +355,7 @@ void WindowCapturerWin::CaptureFrame() {
   if (!frame.get()) {
     RTC_LOG(LS_WARNING) << "Failed to create frame.";
     ReleaseDC(window_, window_dc);
-    callback_->OnCaptureResult(Result::ERROR_TEMPORARY, nullptr);
+    return {Result::ERROR_TEMPORARY, nullptr};
-    return;
   }
 
   HDC mem_dc = CreateCompatibleDC(window_dc);
@@ -330,7 +420,7 @@ void WindowCapturerWin::CaptureFrame() {
 
   if (!result) {
     RTC_LOG(LS_ERROR) << "Both PrintWindow() and BitBlt() failed.";
-    callback_->OnCaptureResult(Result::ERROR_TEMPORARY, nullptr);
+    return {Result::ERROR_TEMPORARY, nullptr};
   }
 
   // Rect for the data is relative to the first pixel of the frame.
@@ -339,7 +429,53 @@ void WindowCapturerWin::CaptureFrame() {
       CreateCroppedDesktopFrame(std::move(frame), cropped_rect);
   RTC_DCHECK(cropped_frame);
 
-  callback_->OnCaptureResult(Result::SUCCESS, std::move(cropped_frame));
+  if (capture_owned_windows) {
+    // If any owned/pop-up windows overlap the selected window, capture them
+    // and copy/composite their contents into the frame.
+    owned_windows_.clear();
+    OwnedWindowCollectorContext context(window_, unscaled_cropped_rect,
+                                        &window_capture_helper_,
+                                        &owned_windows_);
+
+    if (context.IsSelectedWindowValid()) {
+      EnumWindows(OwnedWindowCollector, reinterpret_cast<LPARAM>(&context));
+
+      if (!owned_windows_.empty()) {
+        if (!owned_window_capturer_) {
+          owned_window_capturer_ = absl::make_unique<WindowCapturerWin>();
+        }
+
+        // Owned windows are stored in top-down z-order, so this iterates in
+        // reverse to capture / draw them in bottom-up z-order
+        for (auto it = owned_windows_.rbegin(); it != owned_windows_.rend();
+             it++) {
+          HWND hwnd = *it;
+          if (owned_window_capturer_->SelectSource(
+                  reinterpret_cast<SourceId>(hwnd))) {
+            CaptureResults results = owned_window_capturer_->CaptureFrame(
+                /*capture_owned_windows*/ false);
+
+            if (results.result != DesktopCapturer::Result::SUCCESS) {
+              // Simply log any error capturing an owned/pop-up window without
+              // bubbling it up to the caller (an expected error here is that
+              // the owned/pop-up window was closed; any unexpected errors won't
+              // fail the outer capture).
+              RTC_LOG(LS_INFO) << "Capturing owned window failed (previous "
+                                  "error/warning pertained to that)";
+            } else {
+              // Copy / composite the captured frame into the outer frame. This
+              // may no-op if they no longer intersect (if the owned window was
+              // moved outside the owner bounds since scheduled for capture.)
+              cropped_frame->CopyIntersectingPixelsFrom(
+                  *results.frame, horizontal_scale, vertical_scale);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return {Result::SUCCESS, std::move(cropped_frame)};
 }
 
 }  // namespace