Implement MouseCursorMonitorPipeWire to track cursor changes separately

Current implementation has mouse cursor as part of the screen itself
which means that everytime a cursor changes location, we have to update
whole screen content, which brings unnecessary load overhead. Using our
own mouse cursor monitor implementation allows us to track only mouse
cursor changes and update them separately for much better performance.

Bug: webrtc:13429
Change-Id: I224e9145f0bc7e45eafe4490de160f2ad4c8b545
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/244507
Reviewed-by: Tomas Gunnarsson <tommi@webrtc.org>
Reviewed-by: Mark Foltz <mfoltz@chromium.org>
Commit-Queue: Mark Foltz <mfoltz@chromium.org>
Cr-Commit-Position: refs/heads/main@{#36011}

modules/desktop_capture/BUILD.gn

@@ -561,6 +561,8 @@ rtc_library("desktop_capture_generic") {
       "linux/wayland/base_capturer_pipewire.h",
       "linux/wayland/egl_dmabuf.cc",
       "linux/wayland/egl_dmabuf.h",
+      "linux/wayland/mouse_cursor_monitor_pipewire.cc",
+      "linux/wayland/mouse_cursor_monitor_pipewire.h",
       "linux/wayland/scoped_glib.cc",
       "linux/wayland/scoped_glib.h",
       "linux/wayland/screencast_portal.cc",

modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.cc

@@ -0,0 +1,59 @@
+/*
+ *  Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.h"
+
+#include <utility>
+
+#include "modules/desktop_capture/desktop_capture_options.h"
+#include "modules/desktop_capture/desktop_capturer.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+MouseCursorMonitorPipeWire::MouseCursorMonitorPipeWire(
+    const DesktopCaptureOptions& options)
+    : options_(options) {
+  sequence_checker_.Detach();
+}
+
+MouseCursorMonitorPipeWire::~MouseCursorMonitorPipeWire() {}
+
+void MouseCursorMonitorPipeWire::Init(Callback* callback, Mode mode) {
+  RTC_DCHECK_RUN_ON(&sequence_checker_);
+  RTC_DCHECK(!callback_);
+  RTC_DCHECK(callback);
+
+  callback_ = callback;
+  mode_ = mode;
+}
+
+void MouseCursorMonitorPipeWire::Capture() {
+  RTC_DCHECK_RUN_ON(&sequence_checker_);
+  RTC_DCHECK(callback_);
+
+  std::unique_ptr<MouseCursor> mouse_cursor =
+      options_.screencast_stream()->CaptureCursor();
+
+  if (mouse_cursor && mouse_cursor->image()->data()) {
+    callback_->OnMouseCursor(mouse_cursor.release());
+  }
+
+  if (mode_ == SHAPE_AND_POSITION) {
+    absl::optional<DesktopVector> mouse_cursor_position =
+        options_.screencast_stream()->CaptureCursorPosition();
+    if (mouse_cursor_position) {
+      callback_->OnMouseCursorPosition(mouse_cursor_position.value());
+    }
+  }
+}
+
+}  // namespace webrtc

modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.h

@@ -0,0 +1,43 @@
+/*
+ *  Copyright 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_DESKTOP_CAPTURE_LINUX_WAYLAND_MOUSE_CURSOR_MONITOR_PIPEWIRE_H_
+#define MODULES_DESKTOP_CAPTURE_LINUX_WAYLAND_MOUSE_CURSOR_MONITOR_PIPEWIRE_H_
+
+#include <memory>
+
+#include "api/scoped_refptr.h"
+#include "api/sequence_checker.h"
+#include "modules/desktop_capture/desktop_capture_options.h"
+#include "modules/desktop_capture/desktop_capture_types.h"
+#include "modules/desktop_capture/linux/wayland/shared_screencast_stream.h"
+#include "modules/desktop_capture/mouse_cursor.h"
+#include "modules/desktop_capture/mouse_cursor_monitor.h"
+
+namespace webrtc {
+
+class MouseCursorMonitorPipeWire : public MouseCursorMonitor {
+ public:
+  explicit MouseCursorMonitorPipeWire(const DesktopCaptureOptions& options);
+  ~MouseCursorMonitorPipeWire() override;
+
+  // MouseCursorMonitor:
+  void Init(Callback* callback, Mode mode) override;
+  void Capture() override;
+
+  DesktopCaptureOptions options_ RTC_GUARDED_BY(sequence_checker_);
+  Callback* callback_ RTC_GUARDED_BY(sequence_checker_) = nullptr;
+  Mode mode_ RTC_GUARDED_BY(sequence_checker_) = SHAPE_AND_POSITION;
+  RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_DESKTOP_CAPTURE_LINUX_WAYLAND_MOUSE_CURSOR_MONITOR_PIPEWIRE_H_

modules/desktop_capture/linux/wayland/screencast_portal.h

@@ -90,7 +90,7 @@ class ScreenCastPortal {
   CaptureSourceType capture_source_type_ =
       ScreenCastPortal::CaptureSourceType::kScreen;
 
-  CursorMode cursor_mode_ = ScreenCastPortal::CursorMode::kEmbedded;
+  CursorMode cursor_mode_ = ScreenCastPortal::CursorMode::kMetadata;
 
   GDBusConnection* connection_ = nullptr;
   GDBusProxy* proxy_ = nullptr;

modules/desktop_capture/linux/wayland/shared_screencast_stream.cc

@@ -57,6 +57,12 @@ const char kDrmLib[] = "libdrm.so.2";
 #define SPA_POD_PROP_FLAG_DONT_FIXATE (1u << 4)
 #endif
 
+constexpr int kCursorBpp = 4;
+constexpr int CursorMetaSize(int w, int h) {
+  return (sizeof(struct spa_meta_cursor) + sizeof(struct spa_meta_bitmap) +
+          w * h * kCursorBpp);
+}
+
 struct PipeWireVersion {
   int major = 0;
   int minor = 0;
@@ -183,6 +189,8 @@ class SharedScreenCastStreamPrivate {
   bool StartScreenCastStream(uint32_t stream_node_id, int fd);
   void StopScreenCastStream();
   std::unique_ptr<DesktopFrame> CaptureFrame();
+  std::unique_ptr<MouseCursor> CaptureCursor();
+  DesktopVector CaptureCursorPosition();
 
  private:
   uint32_t pw_stream_node_id_ = 0;
@@ -194,6 +202,8 @@ class SharedScreenCastStreamPrivate {
   webrtc::Mutex queue_lock_;
   ScreenCaptureFrameQueue<SharedDesktopFrame> queue_
       RTC_GUARDED_BY(&queue_lock_);
+  std::unique_ptr<MouseCursor> mouse_cursor_;
+  DesktopVector mouse_cursor_position_ = DesktopVector(-1, -1);
 
   int64_t modifier_;
   std::unique_ptr<EglDmaBuf> egl_dmabuf_;
@@ -387,6 +397,18 @@ void SharedScreenCastStreamPrivate::OnStreamParamChanged(
       &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type,
       SPA_POD_Id(SPA_META_VideoCrop), SPA_PARAM_META_size,
       SPA_POD_Int(sizeof(struct spa_meta_region)))));
+  params.push_back(reinterpret_cast<spa_pod*>(spa_pod_builder_add_object(
+      &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type,
+      SPA_POD_Id(SPA_META_Cursor), SPA_PARAM_META_size,
+      SPA_POD_CHOICE_RANGE_Int(CursorMetaSize(64, 64), CursorMetaSize(1, 1),
+                               CursorMetaSize(384, 384)))));
+  params.push_back(reinterpret_cast<spa_pod*>(spa_pod_builder_add_object(
+      &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type,
+      SPA_POD_Id(SPA_META_VideoDamage), SPA_PARAM_META_size,
+      SPA_POD_CHOICE_RANGE_Int(sizeof(struct spa_meta_region) * 16,
+                               sizeof(struct spa_meta_region) * 1,
+                               sizeof(struct spa_meta_region) * 16))));
+
   pw_stream_update_params(that->pw_stream_, params.data(), params.size());
 }
 
@@ -604,14 +626,53 @@ std::unique_ptr<DesktopFrame> SharedScreenCastStreamPrivate::CaptureFrame() {
   return std::move(frame);
 }
 
+std::unique_ptr<MouseCursor> SharedScreenCastStreamPrivate::CaptureCursor() {
+  if (!mouse_cursor_) {
+    return nullptr;
+  }
+
+  return std::move(mouse_cursor_);
+}
+
+DesktopVector SharedScreenCastStreamPrivate::CaptureCursorPosition() {
+  return mouse_cursor_position_;
+}
+
 void SharedScreenCastStreamPrivate::ProcessBuffer(pw_buffer* buffer) {
   spa_buffer* spa_buffer = buffer->buffer;
   ScopedBuf map;
   std::unique_ptr<uint8_t[]> src_unique_ptr;
   uint8_t* src = nullptr;
 
+  // Try to update the mouse cursor first, because it can be the only
+  // information carried by the buffer
+  {
+    const struct spa_meta_cursor* cursor =
+        static_cast<struct spa_meta_cursor*>(spa_buffer_find_meta_data(
+            spa_buffer, SPA_META_Cursor, sizeof(*cursor)));
+    if (spa_meta_cursor_is_valid(cursor)) {
+      struct spa_meta_bitmap* bitmap = nullptr;
+
+      if (cursor->bitmap_offset)
+        bitmap =
+            SPA_MEMBER(cursor, cursor->bitmap_offset, struct spa_meta_bitmap);
+
+      if (bitmap && bitmap->size.width > 0 && bitmap->size.height > 0) {
+        const uint8_t* bitmap_data =
+            SPA_MEMBER(bitmap, bitmap->offset, uint8_t);
+        BasicDesktopFrame* mouse_frame = new BasicDesktopFrame(
+            DesktopSize(bitmap->size.width, bitmap->size.height));
+        mouse_frame->CopyPixelsFrom(
+            bitmap_data, bitmap->stride,
+            DesktopRect::MakeWH(bitmap->size.width, bitmap->size.height));
+        mouse_cursor_ = std::make_unique<MouseCursor>(
+            mouse_frame, DesktopVector(cursor->hotspot.x, cursor->hotspot.y));
+      }
+      mouse_cursor_position_.set(cursor->position.x, cursor->position.y);
+    }
+  }
+
   if (spa_buffer->datas[0].chunk->size == 0) {
-    RTC_LOG(LS_ERROR) << "Failed to get video stream: Zero size.";
     return;
   }
 
@@ -674,7 +735,6 @@ void SharedScreenCastStreamPrivate::ProcessBuffer(pw_buffer* buffer) {
   if (!src) {
     return;
   }
-
   struct spa_meta_region* video_metadata =
       static_cast<struct spa_meta_region*>(spa_buffer_find_meta_data(
           spa_buffer, SPA_META_VideoCrop, sizeof(*video_metadata)));
@@ -753,8 +813,8 @@ void SharedScreenCastStreamPrivate::ProcessBuffer(pw_buffer* buffer) {
       spa_video_format_.format == SPA_VIDEO_FORMAT_RGBA) {
     uint8_t* tmp_src = queue_.current_frame()->data();
     for (int i = 0; i < video_size_.height(); ++i) {
-      // If both sides decided to go with the RGBx format we need to convert it
+      // If both sides decided to go with the RGBx format we need to convert
-      // to BGRx to match color format expected by WebRTC.
+      // it to BGRx to match color format expected by WebRTC.
       ConvertRGBxToBGRx(tmp_src, queue_.current_frame()->stride());
       tmp_src += queue_.current_frame()->stride();
     }
@@ -795,4 +855,19 @@ std::unique_ptr<DesktopFrame> SharedScreenCastStream::CaptureFrame() {
   return private_->CaptureFrame();
 }
 
+std::unique_ptr<MouseCursor> SharedScreenCastStream::CaptureCursor() {
+  return private_->CaptureCursor();
+}
+
+absl::optional<DesktopVector> SharedScreenCastStream::CaptureCursorPosition() {
+  DesktopVector position = private_->CaptureCursorPosition();
+
+  // Consider only (x >= 0 and y >= 0) a valid position
+  if (position.x() < 0 || position.y() < 0) {
+    return absl::nullopt;
+  }
+
+  return position;
+}
+
 }  // namespace webrtc

modules/desktop_capture/linux/wayland/shared_screencast_stream.h

@@ -13,9 +13,11 @@
 
 #include <memory>
 
+#include "absl/types/optional.h"
 #include "api/ref_counted_base.h"
 #include "api/scoped_refptr.h"
 #include "modules/desktop_capture/desktop_frame.h"
+#include "modules/desktop_capture/mouse_cursor.h"
 #include "rtc_base/system/rtc_export.h"
 
 namespace webrtc {
@@ -29,8 +31,29 @@ class RTC_EXPORT SharedScreenCastStream
 
   bool StartScreenCastStream(uint32_t stream_node_id, int fd);
   void StopScreenCastStream();
+
+  // Below functions return the most recent information we get from a
+  // PipeWire buffer on each Process() callback. This assumes that we
+  // managed to successfuly connect to a PipeWire stream provided by the
+  // compositor (based on stream parameters). The cursor data are obtained
+  // from spa_meta_cursor stream metadata and therefore the cursor is not
+  // part of actual screen/window frame.
+
+  // Returns the most recent screen/window frame we obtained from PipeWire
+  // buffer. Will return an empty frame in case we didn't manage to get a frame
+  // from PipeWire buffer.
   std::unique_ptr<DesktopFrame> CaptureFrame();
 
+  // Returns the most recent mouse cursor image. Will return an nullptr cursor
+  // in case we didn't manage to get a cursor from PipeWire buffer. NOTE: the
+  // cursor image might not be updated on every cursor location change, but
+  // actually only when its shape changes.
+  std::unique_ptr<MouseCursor> CaptureCursor();
+
+  // Returns the most recent mouse cursor position. Will not return a value in
+  // case we didn't manage to get it from PipeWire buffer.
+  absl::optional<DesktopVector> CaptureCursorPosition();
+
   ~SharedScreenCastStream();
 
  protected:

modules/desktop_capture/mouse_cursor_monitor_linux.cc

@@ -17,6 +17,10 @@
 #include "modules/desktop_capture/linux/x11/mouse_cursor_monitor_x11.h"
 #endif  // defined(WEBRTC_USE_X11)
 
+#if defined(WEBRTC_USE_PIPEWIRE)
+#include "modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.h"
+#endif  // defined(WEBRTC_USE_PIPEWIRE)
+
 namespace webrtc {
 
 // static
@@ -44,6 +48,13 @@ MouseCursorMonitor* MouseCursorMonitor::CreateForScreen(
 // static
 std::unique_ptr<MouseCursorMonitor> MouseCursorMonitor::Create(
     const DesktopCaptureOptions& options) {
+#if defined(WEBRTC_USE_PIPEWIRE)
+  if (options.allow_pipewire() && DesktopCapturer::IsRunningUnderWayland() &&
+      options.screencast_stream()) {
+    return std::make_unique<MouseCursorMonitorPipeWire>(options);
+  }
+#endif  // defined(WEBRTC_USE_PIPEWIRE)
+
 #if defined(WEBRTC_USE_X11)
   return MouseCursorMonitorX11::Create(options);
 #else