diff --git a/apps/desktop/src/gui/features/devices/session_device_manager.cpp b/apps/desktop/src/gui/features/devices/session_device_manager.cpp index 91257193..bd03fda0 100644 --- a/apps/desktop/src/gui/features/devices/session_device_manager.cpp +++ b/apps/desktop/src/gui/features/devices/session_device_manager.cpp @@ -74,7 +74,8 @@ int SessionDeviceManager::InitializeScreenCapturer() { const int init_ret = screen_capturer_->Init( fps, [this, fps](unsigned char *data, int size, int width, int height, - const char *display_name) { + const char *display_name, + const XNativeVideoFrame *native_frame) { const auto now_time = std::chrono::steady_clock::now(); if (!ShouldSendCapturedFrame(now_time, fps)) { return; @@ -142,6 +143,7 @@ int SessionDeviceManager::InitializeScreenCapturer() { frame.width = width; frame.height = height; frame.captured_timestamp = GetSystemTimeMicros(owner_.peer_); + frame.native_frame = native_frame; for (const std::string &remote_id : connected_remote_ids) { SendVideoFrameToPeer(owner_.peer_, &frame, stream_id.c_str(), remote_id.data(), remote_id.size()); diff --git a/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_drm.cpp b/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_drm.cpp index fc4bc5f0..a87cd2d3 100644 --- a/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_drm.cpp +++ b/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_drm.cpp @@ -429,7 +429,7 @@ bool ScreenCapturerDrm::CaptureOutputFrame(const DrmOutput& output, if (emit_callback && callback_) { callback_(nv12.data(), static_cast(nv12.size()), capture_width, - capture_height, output.stream_id.c_str()); + capture_height, output.stream_id.c_str(), nullptr); } UnmapFramebuffer(mapped_ptr, mapped_size, prime_fd); diff --git a/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_linux.cpp b/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_linux.cpp index cba08f60..8a91cc94 100644 --- a/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_linux.cpp +++ b/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_linux.cpp @@ -50,7 +50,8 @@ int ScreenCapturerLinux::Init(const int fps, cb_desktop_data cb) { fps_ = fps; callback_orig_ = std::move(cb); callback_ = [this](unsigned char* data, int size, int width, int height, - const char* reported_stream_id) { + const char* reported_stream_id, + const XNativeVideoFrame* native_frame) { const std::string mapped_stream_id = MapStreamId(reported_stream_id); if (mapped_stream_id.empty()) { if (!invalid_stream_id_logged_.exchange(true, @@ -64,7 +65,8 @@ int ScreenCapturerLinux::Init(const int fps, cb_desktop_data cb) { } invalid_stream_id_logged_.store(false, std::memory_order_relaxed); if (callback_orig_) { - callback_orig_(data, size, width, height, mapped_stream_id.c_str()); + callback_orig_(data, size, width, height, mapped_stream_id.c_str(), + native_frame); } }; diff --git a/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_wayland_pipewire.cpp b/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_wayland_pipewire.cpp index 9f97fb3d..859a2704 100644 --- a/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_wayland_pipewire.cpp +++ b/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_wayland_pipewire.cpp @@ -1047,7 +1047,7 @@ void ScreenCapturerWayland::HandlePipeWireBuffer() { if (callback_) { const std::string stream_id = MakeDisplayStreamId(0); callback_(nv12.data(), static_cast(nv12.size()), even_width, - even_height, stream_id.c_str()); + even_height, stream_id.c_str(), nullptr); } pipewire_last_frame_ms_.store(NowMs()); diff --git a/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_x11.cpp b/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_x11.cpp index 48c2a939..9665b954 100644 --- a/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_x11.cpp +++ b/apps/desktop/src/platform/linux/screen_capturer/screen_capturer_x11.cpp @@ -321,7 +321,7 @@ void ScreenCapturerX11::OnFrame() { if (callback_) { const std::string stream_id = MakeDisplayStreamId(monitor_index); callback_(nv12.data(), width_ * height_ * 3 / 2, width_, height_, - stream_id.c_str()); + stream_id.c_str(), nullptr); } XDestroyImage(image); diff --git a/apps/desktop/src/platform/macos/screen_capturer/screen_capturer_sck_impl.mm b/apps/desktop/src/platform/macos/screen_capturer/screen_capturer_sck_impl.mm index 304e34c7..33e9f069 100644 --- a/apps/desktop/src/platform/macos/screen_capturer/screen_capturer_sck_impl.mm +++ b/apps/desktop/src/platform/macos/screen_capturer/screen_capturer_sck_impl.mm @@ -42,6 +42,65 @@ static std::string NSErrorToString(NSError *error) { return description ? description : ""; } +static void RetainCapturedPixelBuffer(void *owner) { + if (owner) { + CVPixelBufferRetain(static_cast(owner)); + } +} + +static void ReleaseCapturedPixelBuffer(void *owner) { + if (owner) { + CVPixelBufferRelease(static_cast(owner)); + } +} + +static int CopyCapturedPixelBufferToNv12(void *owner, uint8_t *destination, + size_t destination_size) { + auto pixel_buffer = static_cast(owner); + if (!pixel_buffer || !destination || + !CVPixelBufferIsPlanar(pixel_buffer) || + CVPixelBufferGetPlaneCount(pixel_buffer) < 2) { + return -1; + } + const OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer); + if (pixel_format != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange && + pixel_format != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) { + return -1; + } + + const size_t width = CVPixelBufferGetWidth(pixel_buffer); + const size_t height = CVPixelBufferGetHeight(pixel_buffer); + const size_t required_size = width * height * 3U / 2U; + if (destination_size < required_size || + CVPixelBufferLockBaseAddress(pixel_buffer, + kCVPixelBufferLock_ReadOnly) != + kCVReturnSuccess) { + return -1; + } + + const auto *y_plane = static_cast( + CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0)); + const auto *uv_plane = static_cast( + CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1)); + const size_t y_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0); + const size_t uv_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1); + int result = 0; + if (!y_plane || !uv_plane || y_stride < width || uv_stride < width) { + result = -1; + } else { + for (size_t row = 0; row < height; ++row) { + std::memcpy(destination + row * width, y_plane + row * y_stride, width); + } + uint8_t *uv_destination = destination + width * height; + for (size_t row = 0; row < height / 2U; ++row) { + std::memcpy(uv_destination + row * width, + uv_plane + row * uv_stride, width); + } + } + CVPixelBufferUnlockBaseAddress(pixel_buffer, kCVPixelBufferLock_ReadOnly); + return result; +} + // The ScreenCaptureKit API was available in macOS 12.3, but full-screen capture // was reported to be broken before macOS 13 - see http://crbug.com/40234870. // Also, the `SCContentFilter` fields `contentRect` and `pointPixelScale` were @@ -92,14 +151,11 @@ class API_AVAILABLE(macos(14.0)) ScreenCapturerSckImpl : public ScreenCapturer { private: std::vector display_info_list_; std::map display_id_map_; - unsigned char *nv12_frame_ = nullptr; - size_t nv12_frame_size_ = 0; - int width_ = 0; - int height_ = 0; int fps_ = 60; bool show_cursor_ = false; bool capture_requested_ = false; bool invalid_stream_id_logged_ = false; + bool native_output_logged_ = false; public: // Called by SckHelper when shareable content is returned by ScreenCaptureKit. `content` will be @@ -119,8 +175,8 @@ class API_AVAILABLE(macos(14.0)) ScreenCapturerSckImpl : public ScreenCapturer { // object. The helper may outlive this C++ instance, if a completion-handler is passed to // ScreenCaptureKit APIs and the C++ object is deleted before the handler executes. SckHelper *__strong helper_ = nil; - // Callback for returning captured frames, or errors, to the caller. Only used on the caller's - // thread. + // Callback for returning captured frames to the caller. ScreenCaptureKit may + // invoke it on its sample-handler queue. cb_desktop_data _on_data = nullptr; // Signals that a permanent error occurred. This may be set on any thread, and is read by // CaptureFrame() which runs on the caller's thread. @@ -244,12 +300,6 @@ ScreenCapturerSckImpl::~ScreenCapturerSckImpl() { display_info_list_.clear(); display_id_map_.clear(); - - if (nv12_frame_) { - delete[] nv12_frame_; - nv12_frame_ = nullptr; - nv12_frame_size_ = 0; - } } int ScreenCapturerSckImpl::Init(const int fps, cb_desktop_data cb) { @@ -262,6 +312,7 @@ int ScreenCapturerSckImpl::Init(const int fps, cb_desktop_data cb) { fps_ = fps > 0 ? fps : 60; display_info_list_.clear(); display_id_map_.clear(); + native_output_logged_ = false; if (@available(macOS 10.15, *)) { bool has_permission = CGPreflightScreenCaptureAccess(); @@ -596,81 +647,69 @@ void ScreenCapturerSckImpl::OnNewCVPixelBuffer(CVPixelBufferRef pixelBuffer, return; } - size_t width = CVPixelBufferGetWidth(pixelBuffer); - size_t height = CVPixelBufferGetHeight(pixelBuffer); - if (width == 0 || height == 0 || CVPixelBufferGetPlaneCount(pixelBuffer) < 2) { - LOG_ERROR("Invalid CVPixelBuffer: width={}, height={}, planes={}", width, height, + const size_t width = CVPixelBufferGetWidth(pixelBuffer); + const size_t height = CVPixelBufferGetHeight(pixelBuffer); + const OSType pixel_format = CVPixelBufferGetPixelFormatType(pixelBuffer); + if (width == 0 || height == 0 || (width & 1U) != 0 || + (height & 1U) != 0 || !CVPixelBufferIsPlanar(pixelBuffer) || + CVPixelBufferGetPlaneCount(pixelBuffer) < 2 || + (pixel_format != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange && + pixel_format != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)) { + LOG_ERROR("Invalid CVPixelBuffer: width={}, height={}, format={}, planes={}", + width, height, pixel_format, CVPixelBufferGetPlaneCount(pixelBuffer)); return; } - CVReturn status = CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); - if (status != kCVReturnSuccess) { - LOG_ERROR("Failed to lock CVPixelBuffer base address: %d", status); - return; - } - - size_t required_size = width * height * 3 / 2; + const size_t required_size = width * height * 3U / 2U; if (required_size > static_cast((std::numeric_limits::max)())) { LOG_ERROR("Captured frame is too large: {} bytes", required_size); - CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); return; } - std::lock_guard lock(lock_); - if (!_on_data || !capture_requested_) { - CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); - return; - } - - if (!nv12_frame_ || nv12_frame_size_ < required_size) { - delete[] nv12_frame_; - nv12_frame_ = new unsigned char[required_size]; - nv12_frame_size_ = required_size; - } - width_ = static_cast(width); - height_ = static_cast(height); - - void *base_y = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0); - size_t stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0); - - void *base_uv = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1); - size_t stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1); - if (!base_y || !base_uv || stride_y < width || stride_uv < width) { - LOG_ERROR("Invalid CVPixelBuffer planes: base_y={}, base_uv={}, stride_y={}, stride_uv={}", - base_y != nullptr, base_uv != nullptr, stride_y, stride_uv); - CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); - return; - } - - unsigned char *dst_y = nv12_frame_; - for (size_t row = 0; row < height; ++row) { - memcpy(dst_y + row * width, static_cast(base_y) + row * stride_y, width); - } - - unsigned char *dst_uv = nv12_frame_ + width * height; - for (size_t row = 0; row < height / 2; ++row) { - memcpy(dst_uv + row * width, static_cast(base_uv) + row * stride_uv, width); - } - - const std::string stream_id = ResolveDisplayStreamId( - current_stream_id_.c_str(), display_info_list_.size(), - current_monitor_index_); - if (stream_id.empty()) { - if (!invalid_stream_id_logged_) { - LOG_ERROR("Dropping macOS frames without a registered stream id, " - "display_id={}", - current_display_); - invalid_stream_id_logged_ = true; + cb_desktop_data callback; + std::string stream_id; + bool log_native_output = false; + { + std::lock_guard lock(lock_); + if (!_on_data || !capture_requested_) { + return; + } + stream_id = ResolveDisplayStreamId(current_stream_id_.c_str(), + display_info_list_.size(), + current_monitor_index_); + if (stream_id.empty()) { + if (!invalid_stream_id_logged_) { + LOG_ERROR("Dropping macOS frames without a registered stream id, " + "display_id={}", + current_display_); + invalid_stream_id_logged_ = true; + } + return; + } + invalid_stream_id_logged_ = false; + callback = _on_data; + if (!native_output_logged_) { + native_output_logged_ = true; + log_native_output = true; } - CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); - return; } - invalid_stream_id_logged_ = false; - _on_data(nv12_frame_, static_cast(required_size), static_cast(width), - static_cast(height), stream_id.c_str()); - CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + XNativeVideoFrame native_frame{}; + native_frame.struct_size = sizeof(native_frame); + native_frame.type = XNativeVideoFrameCVPixelBuffer; + native_frame.width = static_cast(width); + native_frame.height = static_cast(height); + native_frame.payload.cv_pixel_buffer = pixelBuffer; + native_frame.owner = pixelBuffer; + native_frame.retain = &RetainCapturedPixelBuffer; + native_frame.release = &ReleaseCapturedPixelBuffer; + native_frame.copy_to_nv12 = &CopyCapturedPixelBufferToNv12; + if (log_native_output) { + LOG_INFO("macOS ScreenCaptureKit native CVPixelBuffer output enabled"); + } + callback(nullptr, static_cast(required_size), static_cast(width), + static_cast(height), stream_id.c_str(), &native_frame); } void ScreenCapturerSckImpl::StartOrReconfigureCapturer() { diff --git a/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_dxgi.cpp b/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_dxgi.cpp index 2e4f25ac..a7061cbf 100644 --- a/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_dxgi.cpp +++ b/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_dxgi.cpp @@ -384,7 +384,7 @@ void ScreenCapturerDxgi::CaptureLoop() { if (idx >= 0 && idx < static_cast(display_info_list_.size())) { const std::string stream_id = MakeDisplayStreamId(idx); callback_(nv12_frame_, nv12_size, even_width, even_height, - stream_id.c_str()); + stream_id.c_str(), nullptr); } else { LOG_ERROR("DXGI: CaptureLoop invalid monitor_index {} (list size {})", idx, display_info_list_.size()); diff --git a/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_gdi.cpp b/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_gdi.cpp index 5c60ad6c..17c0185f 100644 --- a/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_gdi.cpp +++ b/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_gdi.cpp @@ -212,7 +212,8 @@ void ScreenCapturerGdi::CaptureLoop() { if (callback_) { const std::string stream_id = MakeDisplayStreamId(idx); - callback_(nv12_frame_, nv12_size, width, height, stream_id.c_str()); + callback_(nv12_frame_, nv12_size, width, height, stream_id.c_str(), + nullptr); } SelectObject(mem_dc, old); diff --git a/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_wgc.cpp b/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_wgc.cpp index e65bfb00..71d966a2 100644 --- a/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_wgc.cpp +++ b/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_wgc.cpp @@ -406,7 +406,7 @@ void ScreenCapturerWgc::OnFrame(const WgcSession::wgc_session_frame& frame, const std::string stream_id = MakeDisplayStreamId(id); on_data_(nv12_frame_, nv12_size, even_width, even_height, - stream_id.c_str()); + stream_id.c_str(), nullptr); } } diff --git a/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.cpp b/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.cpp index a0c79ae5..f2ec6b51 100644 --- a/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.cpp +++ b/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.cpp @@ -404,7 +404,8 @@ int ScreenCapturerWin::Init(const int fps, cb_desktop_data cb) { fps_ = fps; cb_orig_ = cb; cb_ = [this](unsigned char* data, int size, int w, int h, - const char* reported_stream_id) { + const char* reported_stream_id, + const XNativeVideoFrame* native_frame) { if (secure_desktop_capture_active_.load(std::memory_order_relaxed)) { return; } @@ -450,7 +451,9 @@ int ScreenCapturerWin::Init(const int fps, cb_desktop_data cb) { "elapsed_ms={}", raw_stream_id, mapped_stream_id, w, h, size, elapsed_ms); } - if (cb_orig_) cb_orig_(data, size, w, h, mapped_stream_id.c_str()); + if (cb_orig_) { + cb_orig_(data, size, w, h, mapped_stream_id.c_str(), native_frame); + } }; int ret = -1; @@ -1263,7 +1266,8 @@ void ScreenCapturerWin::SecureDesktopCaptureLoop() { &captured_width, &captured_height, &error_message)) { if (cb_orig_ && !secure_frame.empty()) { cb_orig_(secure_frame.data(), static_cast(secure_frame.size()), - captured_width, captured_height, display_name.c_str()); + captured_width, captured_height, display_name.c_str(), + nullptr); } frame_delivered = true; } @@ -1277,7 +1281,8 @@ void ScreenCapturerWin::SecureDesktopCaptureLoop() { &captured_height, &error_message)) { if (cb_orig_ && !secure_frame.empty()) { cb_orig_(secure_frame.data(), static_cast(secure_frame.size()), - captured_width, captured_height, display_name.c_str()); + captured_width, captured_height, display_name.c_str(), + nullptr); } frame_delivered = true; } diff --git a/apps/desktop/src/screen_capturer/screen_capturer.h b/apps/desktop/src/screen_capturer/screen_capturer.h index d91327cf..19eba26a 100644 --- a/apps/desktop/src/screen_capturer/screen_capturer.h +++ b/apps/desktop/src/screen_capturer/screen_capturer.h @@ -10,14 +10,18 @@ #include #include "display_info.h" +#include "minirtc.h" namespace crossdesk { class ScreenCapturer { public: - // The final callback argument is a logical MiniRTC stream ID (DisplayN), - // not a platform display name or physical handle. - typedef std::function + // |stream_id| is a logical MiniRTC stream ID (DisplayN), not a platform + // display name or physical handle. |native_frame| is borrowed for the + // duration of the callback; retain its owner before using it asynchronously. + typedef std::function cb_desktop_data; public: diff --git a/apps/desktop/xmake/targets.lua b/apps/desktop/xmake/targets.lua index 766ba385..1ad8c0f7 100644 --- a/apps/desktop/xmake/targets.lua +++ b/apps/desktop/xmake/targets.lua @@ -159,6 +159,7 @@ function setup_targets() set_kind("object") add_deps("rd_log", "common", "crossdesk_wire") add_includedirs("apps/desktop/src/screen_capturer", {public = true}) + add_includedirs("deps/submodules/minirtc/src/api", {public = true}) if is_os("windows") then add_packages("libyuv") add_files("apps/desktop/src/platform/windows/screen_capturer/screen_capturer_dxgi.cpp", @@ -367,7 +368,8 @@ function setup_targets() "apps/desktop/src/platform/windows/screen_capturer/wgc_plugin_entry.cpp") add_files("apps/desktop/resources/windows/wgc_plugin.rc") add_includedirs("apps/desktop/src/common", "apps/desktop/src/screen_capturer", - "apps/desktop/src/platform/windows/screen_capturer") + "apps/desktop/src/platform/windows/screen_capturer", + "deps/submodules/minirtc/src/api") target("crossdesk_service") set_kind("binary") diff --git a/deps/submodules/minirtc b/deps/submodules/minirtc index 38b116d2..5f8c4f90 160000 --- a/deps/submodules/minirtc +++ b/deps/submodules/minirtc @@ -1 +1 @@ -Subproject commit 38b116d29eeb00026d3283663eb194da919d5d78 +Subproject commit 5f8c4f90f2069094aafe51e39552748a65177758