diff --git a/modules/desktop_capture/BUILD.gn b/modules/desktop_capture/BUILD.gn index 63a0bb3412..25ea1cb05a 100644 --- a/modules/desktop_capture/BUILD.gn +++ b/modules/desktop_capture/BUILD.gn @@ -561,6 +561,8 @@ rtc_library("desktop_capture_generic") { "linux/wayland/base_capturer_pipewire.h", "linux/wayland/egl_dmabuf.cc", "linux/wayland/egl_dmabuf.h", + "linux/wayland/mouse_cursor_monitor_pipewire.cc", + "linux/wayland/mouse_cursor_monitor_pipewire.h", "linux/wayland/scoped_glib.cc", "linux/wayland/scoped_glib.h", "linux/wayland/screencast_portal.cc", diff --git a/modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.cc b/modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.cc new file mode 100644 index 0000000000..3d33b0fbb8 --- /dev/null +++ b/modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.cc @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.h" + +#include + +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capturer.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +MouseCursorMonitorPipeWire::MouseCursorMonitorPipeWire( + const DesktopCaptureOptions& options) + : options_(options) { + sequence_checker_.Detach(); +} + +MouseCursorMonitorPipeWire::~MouseCursorMonitorPipeWire() {} + +void MouseCursorMonitorPipeWire::Init(Callback* callback, Mode mode) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(!callback_); + RTC_DCHECK(callback); + + callback_ = callback; + mode_ = mode; +} + +void MouseCursorMonitorPipeWire::Capture() { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(callback_); + + std::unique_ptr mouse_cursor = + options_.screencast_stream()->CaptureCursor(); + + if (mouse_cursor && mouse_cursor->image()->data()) { + callback_->OnMouseCursor(mouse_cursor.release()); + } + + if (mode_ == SHAPE_AND_POSITION) { + absl::optional mouse_cursor_position = + options_.screencast_stream()->CaptureCursorPosition(); + if (mouse_cursor_position) { + callback_->OnMouseCursorPosition(mouse_cursor_position.value()); + } + } +} + +} // namespace webrtc diff --git a/modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.h b/modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.h new file mode 100644 index 0000000000..8af4f879f8 --- /dev/null +++ b/modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.h @@ -0,0 +1,43 @@ +/* + * Copyright 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_LINUX_WAYLAND_MOUSE_CURSOR_MONITOR_PIPEWIRE_H_ +#define MODULES_DESKTOP_CAPTURE_LINUX_WAYLAND_MOUSE_CURSOR_MONITOR_PIPEWIRE_H_ + +#include + +#include "api/scoped_refptr.h" +#include "api/sequence_checker.h" +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capture_types.h" +#include "modules/desktop_capture/linux/wayland/shared_screencast_stream.h" +#include "modules/desktop_capture/mouse_cursor.h" +#include "modules/desktop_capture/mouse_cursor_monitor.h" + +namespace webrtc { + +class MouseCursorMonitorPipeWire : public MouseCursorMonitor { + public: + explicit MouseCursorMonitorPipeWire(const DesktopCaptureOptions& options); + ~MouseCursorMonitorPipeWire() override; + + // MouseCursorMonitor: + void Init(Callback* callback, Mode mode) override; + void Capture() override; + + DesktopCaptureOptions options_ RTC_GUARDED_BY(sequence_checker_); + Callback* callback_ RTC_GUARDED_BY(sequence_checker_) = nullptr; + Mode mode_ RTC_GUARDED_BY(sequence_checker_) = SHAPE_AND_POSITION; + RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_LINUX_WAYLAND_MOUSE_CURSOR_MONITOR_PIPEWIRE_H_ diff --git a/modules/desktop_capture/linux/wayland/screencast_portal.h b/modules/desktop_capture/linux/wayland/screencast_portal.h index 00f78af578..fd6bb1d468 100644 --- a/modules/desktop_capture/linux/wayland/screencast_portal.h +++ b/modules/desktop_capture/linux/wayland/screencast_portal.h @@ -90,7 +90,7 @@ class ScreenCastPortal { CaptureSourceType capture_source_type_ = ScreenCastPortal::CaptureSourceType::kScreen; - CursorMode cursor_mode_ = ScreenCastPortal::CursorMode::kEmbedded; + CursorMode cursor_mode_ = ScreenCastPortal::CursorMode::kMetadata; GDBusConnection* connection_ = nullptr; GDBusProxy* proxy_ = nullptr; diff --git a/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc b/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc index f49bf1e325..a8c86e267b 100644 --- a/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc +++ b/modules/desktop_capture/linux/wayland/shared_screencast_stream.cc @@ -57,6 +57,12 @@ const char kDrmLib[] = "libdrm.so.2"; #define SPA_POD_PROP_FLAG_DONT_FIXATE (1u << 4) #endif +constexpr int kCursorBpp = 4; +constexpr int CursorMetaSize(int w, int h) { + return (sizeof(struct spa_meta_cursor) + sizeof(struct spa_meta_bitmap) + + w * h * kCursorBpp); +} + struct PipeWireVersion { int major = 0; int minor = 0; @@ -183,6 +189,8 @@ class SharedScreenCastStreamPrivate { bool StartScreenCastStream(uint32_t stream_node_id, int fd); void StopScreenCastStream(); std::unique_ptr CaptureFrame(); + std::unique_ptr CaptureCursor(); + DesktopVector CaptureCursorPosition(); private: uint32_t pw_stream_node_id_ = 0; @@ -194,6 +202,8 @@ class SharedScreenCastStreamPrivate { webrtc::Mutex queue_lock_; ScreenCaptureFrameQueue queue_ RTC_GUARDED_BY(&queue_lock_); + std::unique_ptr mouse_cursor_; + DesktopVector mouse_cursor_position_ = DesktopVector(-1, -1); int64_t modifier_; std::unique_ptr egl_dmabuf_; @@ -387,6 +397,18 @@ void SharedScreenCastStreamPrivate::OnStreamParamChanged( &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, SPA_POD_Id(SPA_META_VideoCrop), SPA_PARAM_META_size, SPA_POD_Int(sizeof(struct spa_meta_region))))); + params.push_back(reinterpret_cast(spa_pod_builder_add_object( + &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, + SPA_POD_Id(SPA_META_Cursor), SPA_PARAM_META_size, + SPA_POD_CHOICE_RANGE_Int(CursorMetaSize(64, 64), CursorMetaSize(1, 1), + CursorMetaSize(384, 384))))); + params.push_back(reinterpret_cast(spa_pod_builder_add_object( + &builder, SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, SPA_PARAM_META_type, + SPA_POD_Id(SPA_META_VideoDamage), SPA_PARAM_META_size, + SPA_POD_CHOICE_RANGE_Int(sizeof(struct spa_meta_region) * 16, + sizeof(struct spa_meta_region) * 1, + sizeof(struct spa_meta_region) * 16)))); + pw_stream_update_params(that->pw_stream_, params.data(), params.size()); } @@ -604,14 +626,53 @@ std::unique_ptr SharedScreenCastStreamPrivate::CaptureFrame() { return std::move(frame); } +std::unique_ptr SharedScreenCastStreamPrivate::CaptureCursor() { + if (!mouse_cursor_) { + return nullptr; + } + + return std::move(mouse_cursor_); +} + +DesktopVector SharedScreenCastStreamPrivate::CaptureCursorPosition() { + return mouse_cursor_position_; +} + void SharedScreenCastStreamPrivate::ProcessBuffer(pw_buffer* buffer) { spa_buffer* spa_buffer = buffer->buffer; ScopedBuf map; std::unique_ptr src_unique_ptr; uint8_t* src = nullptr; + // Try to update the mouse cursor first, because it can be the only + // information carried by the buffer + { + const struct spa_meta_cursor* cursor = + static_cast(spa_buffer_find_meta_data( + spa_buffer, SPA_META_Cursor, sizeof(*cursor))); + if (spa_meta_cursor_is_valid(cursor)) { + struct spa_meta_bitmap* bitmap = nullptr; + + if (cursor->bitmap_offset) + bitmap = + SPA_MEMBER(cursor, cursor->bitmap_offset, struct spa_meta_bitmap); + + if (bitmap && bitmap->size.width > 0 && bitmap->size.height > 0) { + const uint8_t* bitmap_data = + SPA_MEMBER(bitmap, bitmap->offset, uint8_t); + BasicDesktopFrame* mouse_frame = new BasicDesktopFrame( + DesktopSize(bitmap->size.width, bitmap->size.height)); + mouse_frame->CopyPixelsFrom( + bitmap_data, bitmap->stride, + DesktopRect::MakeWH(bitmap->size.width, bitmap->size.height)); + mouse_cursor_ = std::make_unique( + mouse_frame, DesktopVector(cursor->hotspot.x, cursor->hotspot.y)); + } + mouse_cursor_position_.set(cursor->position.x, cursor->position.y); + } + } + if (spa_buffer->datas[0].chunk->size == 0) { - RTC_LOG(LS_ERROR) << "Failed to get video stream: Zero size."; return; } @@ -674,7 +735,6 @@ void SharedScreenCastStreamPrivate::ProcessBuffer(pw_buffer* buffer) { if (!src) { return; } - struct spa_meta_region* video_metadata = static_cast(spa_buffer_find_meta_data( spa_buffer, SPA_META_VideoCrop, sizeof(*video_metadata))); @@ -753,8 +813,8 @@ void SharedScreenCastStreamPrivate::ProcessBuffer(pw_buffer* buffer) { spa_video_format_.format == SPA_VIDEO_FORMAT_RGBA) { uint8_t* tmp_src = queue_.current_frame()->data(); for (int i = 0; i < video_size_.height(); ++i) { - // If both sides decided to go with the RGBx format we need to convert it - // to BGRx to match color format expected by WebRTC. + // If both sides decided to go with the RGBx format we need to convert + // it to BGRx to match color format expected by WebRTC. ConvertRGBxToBGRx(tmp_src, queue_.current_frame()->stride()); tmp_src += queue_.current_frame()->stride(); } @@ -795,4 +855,19 @@ std::unique_ptr SharedScreenCastStream::CaptureFrame() { return private_->CaptureFrame(); } +std::unique_ptr SharedScreenCastStream::CaptureCursor() { + return private_->CaptureCursor(); +} + +absl::optional SharedScreenCastStream::CaptureCursorPosition() { + DesktopVector position = private_->CaptureCursorPosition(); + + // Consider only (x >= 0 and y >= 0) a valid position + if (position.x() < 0 || position.y() < 0) { + return absl::nullopt; + } + + return position; +} + } // namespace webrtc diff --git a/modules/desktop_capture/linux/wayland/shared_screencast_stream.h b/modules/desktop_capture/linux/wayland/shared_screencast_stream.h index d7105ba7f4..443ec745d5 100644 --- a/modules/desktop_capture/linux/wayland/shared_screencast_stream.h +++ b/modules/desktop_capture/linux/wayland/shared_screencast_stream.h @@ -13,9 +13,11 @@ #include +#include "absl/types/optional.h" #include "api/ref_counted_base.h" #include "api/scoped_refptr.h" #include "modules/desktop_capture/desktop_frame.h" +#include "modules/desktop_capture/mouse_cursor.h" #include "rtc_base/system/rtc_export.h" namespace webrtc { @@ -29,8 +31,29 @@ class RTC_EXPORT SharedScreenCastStream bool StartScreenCastStream(uint32_t stream_node_id, int fd); void StopScreenCastStream(); + + // Below functions return the most recent information we get from a + // PipeWire buffer on each Process() callback. This assumes that we + // managed to successfuly connect to a PipeWire stream provided by the + // compositor (based on stream parameters). The cursor data are obtained + // from spa_meta_cursor stream metadata and therefore the cursor is not + // part of actual screen/window frame. + + // Returns the most recent screen/window frame we obtained from PipeWire + // buffer. Will return an empty frame in case we didn't manage to get a frame + // from PipeWire buffer. std::unique_ptr CaptureFrame(); + // Returns the most recent mouse cursor image. Will return an nullptr cursor + // in case we didn't manage to get a cursor from PipeWire buffer. NOTE: the + // cursor image might not be updated on every cursor location change, but + // actually only when its shape changes. + std::unique_ptr CaptureCursor(); + + // Returns the most recent mouse cursor position. Will not return a value in + // case we didn't manage to get it from PipeWire buffer. + absl::optional CaptureCursorPosition(); + ~SharedScreenCastStream(); protected: diff --git a/modules/desktop_capture/mouse_cursor_monitor_linux.cc b/modules/desktop_capture/mouse_cursor_monitor_linux.cc index 87edb40373..b44500d4e8 100644 --- a/modules/desktop_capture/mouse_cursor_monitor_linux.cc +++ b/modules/desktop_capture/mouse_cursor_monitor_linux.cc @@ -17,6 +17,10 @@ #include "modules/desktop_capture/linux/x11/mouse_cursor_monitor_x11.h" #endif // defined(WEBRTC_USE_X11) +#if defined(WEBRTC_USE_PIPEWIRE) +#include "modules/desktop_capture/linux/wayland/mouse_cursor_monitor_pipewire.h" +#endif // defined(WEBRTC_USE_PIPEWIRE) + namespace webrtc { // static @@ -44,6 +48,13 @@ MouseCursorMonitor* MouseCursorMonitor::CreateForScreen( // static std::unique_ptr MouseCursorMonitor::Create( const DesktopCaptureOptions& options) { +#if defined(WEBRTC_USE_PIPEWIRE) + if (options.allow_pipewire() && DesktopCapturer::IsRunningUnderWayland() && + options.screencast_stream()) { + return std::make_unique(options); + } +#endif // defined(WEBRTC_USE_PIPEWIRE) + #if defined(WEBRTC_USE_X11) return MouseCursorMonitorX11::Create(options); #else