From b5bb62bd2278de3bebebe84b62f1da550aa32e9b Mon Sep 17 00:00:00 2001 From: dijunkun Date: Fri, 18 Oct 2024 17:20:52 +0800 Subject: [PATCH] [feat] support new screen capture method by using ScreenCaptureKit on MacOSX --- src/common/platform.cpp | 3 +- .../screen_capturer_avf.cpp | 2 +- .../{ => avfoundation}/screen_capturer_avf.h | 1 - .../core_graphics/screen_capturer_cg.cpp | 104 +++++++ .../macosx/core_graphics/screen_capturer_cg.h | 56 ++++ .../screen_capturer_sck.cpp | 51 ++++ .../screen_capturer_kit/screen_capturer_sck.h | 59 ++++ .../screen_capturer_sck_impl.mm | 256 ++++++++++++++++++ src/screen_capturer/screen_capturer_factory.h | 6 +- thirdparty/projectx | 2 +- xmake.lua | 13 +- 11 files changed, 542 insertions(+), 11 deletions(-) rename src/screen_capturer/macosx/{ => avfoundation}/screen_capturer_avf.cpp (99%) rename src/screen_capturer/macosx/{ => avfoundation}/screen_capturer_avf.h (97%) create mode 100644 src/screen_capturer/macosx/core_graphics/screen_capturer_cg.cpp create mode 100644 src/screen_capturer/macosx/core_graphics/screen_capturer_cg.h create mode 100644 src/screen_capturer/macosx/screen_capturer_kit/screen_capturer_sck.cpp create mode 100644 src/screen_capturer/macosx/screen_capturer_kit/screen_capturer_sck.h create mode 100644 src/screen_capturer/macosx/screen_capturer_kit/screen_capturer_sck_impl.mm diff --git a/src/common/platform.cpp b/src/common/platform.cpp index 354a8b8..f0e5f8e 100644 --- a/src/common/platform.cpp +++ b/src/common/platform.cpp @@ -55,7 +55,8 @@ std::string GetMac() { const unsigned char *base = (const unsigned char *)&dlAddr->sdl_data[dlAddr->sdl_nlen]; for (int i = 0; i < dlAddr->sdl_alen; i++) { - len += sprintf(mac_addr + len, "%.2X", base[i]); + len += + snprintf(mac_addr + len, sizeof(mac_addr) - len, "%.2X", base[i]); } } cursor = cursor->ifa_next; diff --git a/src/screen_capturer/macosx/screen_capturer_avf.cpp b/src/screen_capturer/macosx/avfoundation/screen_capturer_avf.cpp similarity index 99% rename from src/screen_capturer/macosx/screen_capturer_avf.cpp rename to src/screen_capturer/macosx/avfoundation/screen_capturer_avf.cpp index 317f1c1..a9286f5 100644 --- a/src/screen_capturer/macosx/screen_capturer_avf.cpp +++ b/src/screen_capturer/macosx/avfoundation/screen_capturer_avf.cpp @@ -152,7 +152,7 @@ int ScreenCapturerAvf::Destroy() { } int ScreenCapturerAvf::Start() { - if (_running) { + if (running_) { return 0; } diff --git a/src/screen_capturer/macosx/screen_capturer_avf.h b/src/screen_capturer/macosx/avfoundation/screen_capturer_avf.h similarity index 97% rename from src/screen_capturer/macosx/screen_capturer_avf.h rename to src/screen_capturer/macosx/avfoundation/screen_capturer_avf.h index 415aa9d..fc2ba59 100644 --- a/src/screen_capturer/macosx/screen_capturer_avf.h +++ b/src/screen_capturer/macosx/avfoundation/screen_capturer_avf.h @@ -49,7 +49,6 @@ class ScreenCapturerAvf : public ScreenCapturer { void CleanUp(); private: - std::atomic_bool _running; std::atomic_bool _paused; std::atomic_bool _inited; diff --git a/src/screen_capturer/macosx/core_graphics/screen_capturer_cg.cpp b/src/screen_capturer/macosx/core_graphics/screen_capturer_cg.cpp new file mode 100644 index 0000000..d079edb --- /dev/null +++ b/src/screen_capturer/macosx/core_graphics/screen_capturer_cg.cpp @@ -0,0 +1,104 @@ +#include + +#include + +#include "rd_log.h" +#include "screen_capturer_cgd.h" + +ScreenCapturerCg::ScreenCapturerCg() {} + +ScreenCapturerCg::~ScreenCapturerCg() {} + +int ScreenCapturerCg::Init(const int fps, cb_desktop_data cb) { + if (cb) { + _on_data = cb; + } + + size_t pixel_width = 1280; + size_t pixel_height = 720; + CGDirectDisplayID display_id = 0; + + CGDisplayStreamFrameAvailableHandler handler = + ^(CGDisplayStreamFrameStatus status, uint64_t display_time, + IOSurfaceRef frame_surface, CGDisplayStreamUpdateRef updateRef) { + if (status == kCGDisplayStreamFrameStatusStopped) return; + // Only pay attention to frame updates. + if (status != kCGDisplayStreamFrameStatusFrameComplete) return; + + // size_t count = 0; + // const CGRect* rects = CGDisplayStreamUpdateGetRects( + // updateRef, kCGDisplayStreamUpdateDirtyRects, &count); + + // 获取帧数据 + void* frameData = IOSurfaceGetBaseAddressOfPlane(frame_surface, 0); + size_t width = IOSurfaceGetWidthOfPlane(frame_surface, 0); + size_t height = IOSurfaceGetHeightOfPlane(frame_surface, 0); + }; + + CFDictionaryRef properties_dictionary = CFDictionaryCreate( + kCFAllocatorDefault, (const void*[]){kCGDisplayStreamShowCursor}, + (const void*[]){kCFBooleanFalse}, 1, &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + + CGDisplayStreamRef display_stream = + CGDisplayStreamCreate(display_id, pixel_width, pixel_height, 'BGRA', + properties_dictionary, handler); + + if (display_stream) { + CGError error = CGDisplayStreamStart(display_stream); + if (error != kCGErrorSuccess) return -1; + + CFRunLoopSourceRef source = CGDisplayStreamGetRunLoopSource(display_stream); + CFRunLoopAddSource(CFRunLoopGetCurrent(), source, kCFRunLoopCommonModes); + display_streams_.push_back(display_stream); + } + + CFRelease(properties_dictionary); + + return 0; +} + +int ScreenCapturerCg::Destroy() { + running_ = false; + return 0; +} + +int ScreenCapturerCg::Start() { + if (_running) { + return 0; + } + + running_ = true; + capture_thread_ = std::thread([this]() { + while (running_) { + CFRunLoopRun(); + } + }); + + return 0; +} + +int ScreenCapturerCg::Stop() { + running_ = false; + return 0; +} + +int ScreenCapturerCg::Pause() { return 0; } + +int ScreenCapturerCg::Resume() { return 0; } + +void ScreenCapturerCg::OnFrame() {} + +void ScreenCapturerCg::CleanUp() {} + +// + +void ScreenCapturerCg::UnregisterRefreshAndMoveHandlers() { + for (CGDisplayStreamRef stream : display_streams_) { + CFRunLoopSourceRef source = CGDisplayStreamGetRunLoopSource(stream); + CFRunLoopRemoveSource(CFRunLoopGetCurrent(), source, kCFRunLoopCommonModes); + CGDisplayStreamStop(stream); + CFRelease(stream); + } + display_streams_.clear(); +} diff --git a/src/screen_capturer/macosx/core_graphics/screen_capturer_cg.h b/src/screen_capturer/macosx/core_graphics/screen_capturer_cg.h new file mode 100644 index 0000000..f347b56 --- /dev/null +++ b/src/screen_capturer/macosx/core_graphics/screen_capturer_cg.h @@ -0,0 +1,56 @@ + +/* + * @Author: DI JUNKUN + * @Date: 2024-10-16 + * Copyright (c) 2024 by DI JUNKUN, All Rights Reserved. + */ + +#ifndef _SCREEN_CAPTURER_CGD_H_ +#define _SCREEN_CAPTURER_CGD_H_ + +#include + +#include +#include +#include +#include +#include +#include + +#include "screen_capturer.h" + +class ScreenCapturerCg : public ScreenCapturer { + public: + ScreenCapturerCg(); + ~ScreenCapturerCg(); + + public: + virtual int Init(const int fps, cb_desktop_data cb); + + virtual int Destroy(); + + virtual int Start(); + + virtual int Stop(); + + int Pause(); + + int Resume(); + + void OnFrame(); + + protected: + void CleanUp(); + + private: + int _fps; + cb_desktop_data _on_data; + + // thread + std::thread capture_thread_; + std::atomic_bool running_; + + private: +}; + +#endif \ No newline at end of file diff --git a/src/screen_capturer/macosx/screen_capturer_kit/screen_capturer_sck.cpp b/src/screen_capturer/macosx/screen_capturer_kit/screen_capturer_sck.cpp new file mode 100644 index 0000000..f90ea3f --- /dev/null +++ b/src/screen_capturer/macosx/screen_capturer_kit/screen_capturer_sck.cpp @@ -0,0 +1,51 @@ +#include "screen_capturer_sck.h" + +#include "rd_log.h" + +ScreenCapturerSck::ScreenCapturerSck() {} +ScreenCapturerSck::~ScreenCapturerSck() { + // if (inited_ && capture_thread_.joinable()) { + // capture_thread_.join(); + // inited_ = false; + // } +} + +int ScreenCapturerSck::Init(const int fps, cb_desktop_data cb) { + if (cb) { + on_data_ = cb; + } else { + LOG_ERROR("cb is null"); + return -1; + } + + screen_capturer_sck_impl_ = CreateScreenCapturerSck(); + screen_capturer_sck_impl_->Init(fps, on_data_); + + return 0; +} + +int ScreenCapturerSck::Destroy() { return 0; } + +int ScreenCapturerSck::Start() { + // if (running_) { + // return 0; + // } + + // running_ = true; + // capture_thread_ = std::thread([this]() { + // while (running_) { + // } + // }); + + return 0; +} + +int ScreenCapturerSck::Stop() { return 0; } + +int ScreenCapturerSck::Pause() { return 0; } + +int ScreenCapturerSck::Resume() { return 0; } + +void ScreenCapturerSck::OnFrame() {} + +void ScreenCapturerSck::CleanUp() {} \ No newline at end of file diff --git a/src/screen_capturer/macosx/screen_capturer_kit/screen_capturer_sck.h b/src/screen_capturer/macosx/screen_capturer_kit/screen_capturer_sck.h new file mode 100644 index 0000000..392a892 --- /dev/null +++ b/src/screen_capturer/macosx/screen_capturer_kit/screen_capturer_sck.h @@ -0,0 +1,59 @@ +/* + * @Author: DI JUNKUN + * @Date: 2024-10-17 + * Copyright (c) 2024 by DI JUNKUN, All Rights Reserved. + */ + +#ifndef _SCREEN_CAPTURER_SCK_H_ +#define _SCREEN_CAPTURER_SCK_H_ + +#include +#include +#include +#include +#include +#include + +#include "screen_capturer.h" + +class ScreenCapturerSck : public ScreenCapturer { + public: + ScreenCapturerSck(); + ~ScreenCapturerSck(); + + public: + virtual int Init(const int fps, cb_desktop_data cb); + + virtual int Destroy(); + + virtual int Start(); + + virtual int Stop(); + + int Pause(); + + int Resume(); + + void OnFrame(); + + protected: + void CleanUp(); + + private: + std::unique_ptr CreateScreenCapturerSck(); + + private: + int _fps; + cb_desktop_data on_data_; + unsigned char* nv12_frame_ = nullptr; + bool inited_ = false; + + // thread + std::thread capture_thread_; + std::atomic_bool running_; + + private: + std::unique_ptr screen_capturer_sck_impl_; +}; + +#endif \ No newline at end of file diff --git a/src/screen_capturer/macosx/screen_capturer_kit/screen_capturer_sck_impl.mm b/src/screen_capturer/macosx/screen_capturer_kit/screen_capturer_sck_impl.mm new file mode 100644 index 0000000..aaf12cd --- /dev/null +++ b/src/screen_capturer/macosx/screen_capturer_kit/screen_capturer_sck_impl.mm @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2024 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "screen_capturer_sck.h" + +#include "rd_log.h" +#include +#include +#include +#include +#include + +class ScreenCapturerSckImpl; + +// The ScreenCaptureKit API was available in macOS 12.3, but full-screen capture +// was reported to be broken before macOS 13 - see http://crbug.com/40234870. +// Also, the `SCContentFilter` fields `contentRect` and `pointPixelScale` were +// introduced in macOS 14. +API_AVAILABLE(macos(14.0)) +@interface SckHelper : NSObject + +- (instancetype)initWithCapturer:(ScreenCapturerSckImpl *)capturer; + +- (void)onShareableContentCreated:(SCShareableContent *)content; + +// Called just before the capturer is destroyed. This avoids a dangling pointer, +// and prevents any new calls into a deleted capturer. If any method-call on the +// capturer is currently running on a different thread, this blocks until it +// completes. +- (void)releaseCapturer; +@end + +class API_AVAILABLE(macos(14.0)) ScreenCapturerSckImpl : public ScreenCapturer { +public: + explicit ScreenCapturerSckImpl(); + + ScreenCapturerSckImpl(const ScreenCapturerSckImpl &) = delete; + ScreenCapturerSckImpl &operator=(const ScreenCapturerSckImpl &) = delete; + ~ScreenCapturerSckImpl(); + +public: + int Init(const int fps, cb_desktop_data cb); + void OnReceiveContent(SCShareableContent *content); + void OnNewIOSurface(IOSurfaceRef io_surface, CFDictionaryRef attachment); + + virtual int Destroy() { return 0; } + + virtual int Start() { return 0; } + + virtual int Stop() { return 0; } + +private: + SckHelper *__strong helper_; + SCStream *__strong stream_; + + cb_desktop_data _on_data; + unsigned char *nv12_frame_ = nullptr; + bool permanent_error_ = false; + CGDirectDisplayID current_display_ = -1; + std::mutex mtx_; +}; + +@implementation SckHelper { + // This lock is to prevent the capturer being destroyed while an instance + // method is still running on another thread. + std::mutex helper_mtx_; + ScreenCapturerSckImpl *_capturer; +} + +- (instancetype)initWithCapturer:(ScreenCapturerSckImpl *)capturer { + self = [super init]; + if (self) { + _capturer = capturer; + } + return self; +} + +- (void)onShareableContentCreated:(SCShareableContent *)content { + std::lock_guard lock(helper_mtx_); + if (_capturer) { + _capturer->OnReceiveContent(content); + } else { + LOG_ERROR("Invalid capturer"); + } +} + +- (void)stream:(SCStream *)stream + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + ofType:(SCStreamOutputType)type { + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + if (!pixelBuffer) { + return; + } + + IOSurfaceRef ioSurface = CVPixelBufferGetIOSurface(pixelBuffer); + if (!ioSurface) { + return; + } + + CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray( + sampleBuffer, /*createIfNecessary=*/false); + if (!attachmentsArray || CFArrayGetCount(attachmentsArray) <= 0) { + LOG_ERROR("Discarding frame with no attachments"); + return; + } + + CFDictionaryRef attachment = + static_cast(CFArrayGetValueAtIndex(attachmentsArray, 0)); + + std::lock_guard lock(helper_mtx_); + if (_capturer) { + _capturer->OnNewIOSurface(ioSurface, attachment); + } +} + +- (void)releaseCapturer { + std::lock_guard lock(helper_mtx_); + _capturer = nullptr; +} + +@end + +ScreenCapturerSckImpl::ScreenCapturerSckImpl() { + helper_ = [[SckHelper alloc] initWithCapturer:this]; +} + +ScreenCapturerSckImpl::~ScreenCapturerSckImpl() { + [stream_ stopCaptureWithCompletionHandler:nil]; + [helper_ releaseCapturer]; +} + +int ScreenCapturerSckImpl::Init(const int fps, cb_desktop_data cb) { + _on_data = cb; + + SckHelper *local_helper = helper_; + auto handler = ^(SCShareableContent *content, NSError *error) { + [local_helper onShareableContentCreated:content]; + }; + + [SCShareableContent getShareableContentWithCompletionHandler:handler]; + + return 0; +} + +void ScreenCapturerSckImpl::OnReceiveContent(SCShareableContent *content) { + if (!content) { + LOG_ERROR("getShareableContent failed"); + permanent_error_ = true; + return; + } + + if (!content.displays.count) { + LOG_ERROR("getShareableContent returned no displays"); + permanent_error_ = true; + return; + } + + SCDisplay *captured_display; + { + std::lock_guard lock(mtx_); + for (SCDisplay *display in content.displays) { + if (current_display_ == display.displayID) { + captured_display = display; + break; + } + } + if (!captured_display) { + if (-1 == current_display_) { + LOG_ERROR("Full screen capture is not supported, falling back to first " + "display"); + } else { + LOG_ERROR("Display [{}] not found, falling back to first display", + current_display_); + } + captured_display = content.displays.firstObject; + } + } + + SCContentFilter *filter = + [[SCContentFilter alloc] initWithDisplay:captured_display + excludingWindows:@[]]; + SCStreamConfiguration *config = [[SCStreamConfiguration alloc] init]; + config.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; + config.showsCursor = false; + config.width = filter.contentRect.size.width * filter.pointPixelScale; + config.height = filter.contentRect.size.height * filter.pointPixelScale; + config.captureResolution = SCCaptureResolutionNominal; + + std::lock_guard lock(mtx_); + + if (stream_) { + LOG_INFO("Updating stream configuration"); + [stream_ updateContentFilter:filter completionHandler:nil]; + [stream_ updateConfiguration:config completionHandler:nil]; + } else { + stream_ = [[SCStream alloc] initWithFilter:filter + configuration:config + delegate:helper_]; + + // TODO: crbug.com/327458809 - Choose an appropriate sampleHandlerQueue for + // best performance. + NSError *add_stream_output_error; + bool add_stream_output_result = + [stream_ addStreamOutput:helper_ + type:SCStreamOutputTypeScreen + sampleHandlerQueue:nil + error:&add_stream_output_error]; + if (!add_stream_output_result) { + stream_ = nil; + LOG_ERROR("addStreamOutput failed"); + permanent_error_ = true; + return; + } + + auto handler = ^(NSError *error) { + if (error) { + // It should be safe to access `this` here, because the C++ destructor + // calls stopCaptureWithCompletionHandler on the stream, which cancels + // this handler. + permanent_error_ = true; + LOG_ERROR("startCaptureWithCompletionHandler failed"); + } else { + LOG_INFO("Capture started"); + } + }; + + [stream_ startCaptureWithCompletionHandler:handler]; + } +} + +void ScreenCapturerSckImpl::OnNewIOSurface(IOSurfaceRef io_surface, + CFDictionaryRef attachment) { + size_t width = IOSurfaceGetWidth(io_surface); + size_t height = IOSurfaceGetHeight(io_surface); + + uint32_t aseed; + IOSurfaceLock(io_surface, kIOSurfaceLockReadOnly, &aseed); + + nv12_frame_ = + static_cast(IOSurfaceGetBaseAddress(io_surface)); + + _on_data(nv12_frame_, width * height * 3 / 2, width, height); + + IOSurfaceUnlock(io_surface, kIOSurfaceLockReadOnly, &aseed); +} + +std::unique_ptr ScreenCapturerSck::CreateScreenCapturerSck() { + return std::make_unique(); +} \ No newline at end of file diff --git a/src/screen_capturer/screen_capturer_factory.h b/src/screen_capturer/screen_capturer_factory.h index cdd8927..d7f5655 100644 --- a/src/screen_capturer/screen_capturer_factory.h +++ b/src/screen_capturer/screen_capturer_factory.h @@ -12,7 +12,8 @@ #elif __linux__ #include "screen_capturer_x11.h" #elif __APPLE__ -#include "screen_capturer_avf.h" +// #include "screen_capturer_avf.h" +#include "screen_capturer_sck.h" #endif class ScreenCapturerFactory { @@ -26,7 +27,8 @@ class ScreenCapturerFactory { #elif __linux__ return new ScreenCapturerX11(); #elif __APPLE__ - return new ScreenCapturerAvf(); + // return new ScreenCapturerAvf(); + return new ScreenCapturerSck(); #else return nullptr; #endif diff --git a/thirdparty/projectx b/thirdparty/projectx index 9d2e122..35d4f52 160000 --- a/thirdparty/projectx +++ b/thirdparty/projectx @@ -1 +1 @@ -Subproject commit 9d2e122fccda17a0df0c8fe1530d93ccb42c35d3 +Subproject commit 35d4f522c57a26222a82d5b42a76f5c99dbd3ad3 diff --git a/xmake.lua b/xmake.lua index 100da37..391eb6b 100644 --- a/xmake.lua +++ b/xmake.lua @@ -41,7 +41,7 @@ elseif is_os("macosx") then add_packages("libxcb") add_links("SDL2", "SDL2main") add_ldflags("-Wl,-ld_classic") - add_frameworks("OpenGL") + add_frameworks("OpenGL", "IOSurface", "ScreenCaptureKit") end add_packages("spdlog", "imgui") @@ -70,8 +70,11 @@ target("screen_capturer") add_includedirs("src/screen_capturer/windows", {public = true}) elseif is_os("macosx") then add_packages("ffmpeg") - add_files("src/screen_capturer/macosx/*.cpp") - add_includedirs("src/screen_capturer/macosx", {public = true}) + add_files("src/screen_capturer/macosx/avfoundation/*.cpp", + "src/screen_capturer/macosx/screen_capturer_kit/*.cpp", + "src/screen_capturer/macosx/screen_capturer_kit/*.mm") + add_includedirs("src/screen_capturer/macosx/avfoundation", + "src/screen_capturer/macosx/screen_capturer_kit", {public = true}) elseif is_os("linux") then add_packages("ffmpeg") add_files("src/screen_capturer/linux/*.cpp") @@ -138,8 +141,8 @@ target("remote_desk") add_files("icon/app.rc") elseif is_os("macosx") then add_packages("ffmpeg") - add_rules("xcode.application") - add_files("Info.plist") + -- add_rules("xcode.application") + -- add_files("Info.plist") elseif is_os("linux") then add_packages("ffmpeg") end