[feat] support new screen capture method by using ScreenCaptureKit on MacOSX

src/common/platform.cpp

@@ -55,7 +55,8 @@ std::string GetMac() {
         const unsigned char *base =
             (const unsigned char *)&dlAddr->sdl_data[dlAddr->sdl_nlen];
         for (int i = 0; i < dlAddr->sdl_alen; i++) {
-          len += sprintf(mac_addr + len, "%.2X", base[i]);
+          len +=
+              snprintf(mac_addr + len, sizeof(mac_addr) - len, "%.2X", base[i]);
         }
       }
       cursor = cursor->ifa_next;

src/screen_capturer/macosx/avfoundation/screen_capturer_avf.cpp

@@ -152,7 +152,7 @@ int ScreenCapturerAvf::Destroy() {
 }
 
 int ScreenCapturerAvf::Start() {
-  if (_running) {
+  if (running_) {
     return 0;
   }
 

src/screen_capturer/macosx/avfoundation/screen_capturer_avf.h

@@ -49,7 +49,6 @@ class ScreenCapturerAvf : public ScreenCapturer {
   void CleanUp();
 
  private:
-  std::atomic_bool _running;
   std::atomic_bool _paused;
   std::atomic_bool _inited;
 

src/screen_capturer/macosx/core_graphics/screen_capturer_cg.cpp

@@ -0,0 +1,104 @@
+#include <IOSurface/IOSurface.h>
+
+#include <utility>
+
+#include "rd_log.h"
+#include "screen_capturer_cgd.h"
+
+ScreenCapturerCg::ScreenCapturerCg() {}
+
+ScreenCapturerCg::~ScreenCapturerCg() {}
+
+int ScreenCapturerCg::Init(const int fps, cb_desktop_data cb) {
+  if (cb) {
+    _on_data = cb;
+  }
+
+  size_t pixel_width = 1280;
+  size_t pixel_height = 720;
+  CGDirectDisplayID display_id = 0;
+
+  CGDisplayStreamFrameAvailableHandler handler =
+      ^(CGDisplayStreamFrameStatus status, uint64_t display_time,
+        IOSurfaceRef frame_surface, CGDisplayStreamUpdateRef updateRef) {
+        if (status == kCGDisplayStreamFrameStatusStopped) return;
+        // Only pay attention to frame updates.
+        if (status != kCGDisplayStreamFrameStatusFrameComplete) return;
+
+        // size_t count = 0;
+        // const CGRect* rects = CGDisplayStreamUpdateGetRects(
+        //     updateRef, kCGDisplayStreamUpdateDirtyRects, &count);
+
+        // 获取帧数据
+        void* frameData = IOSurfaceGetBaseAddressOfPlane(frame_surface, 0);
+        size_t width = IOSurfaceGetWidthOfPlane(frame_surface, 0);
+        size_t height = IOSurfaceGetHeightOfPlane(frame_surface, 0);
+      };
+
+  CFDictionaryRef properties_dictionary = CFDictionaryCreate(
+      kCFAllocatorDefault, (const void*[]){kCGDisplayStreamShowCursor},
+      (const void*[]){kCFBooleanFalse}, 1, &kCFTypeDictionaryKeyCallBacks,
+      &kCFTypeDictionaryValueCallBacks);
+
+  CGDisplayStreamRef display_stream =
+      CGDisplayStreamCreate(display_id, pixel_width, pixel_height, 'BGRA',
+                            properties_dictionary, handler);
+
+  if (display_stream) {
+    CGError error = CGDisplayStreamStart(display_stream);
+    if (error != kCGErrorSuccess) return -1;
+
+    CFRunLoopSourceRef source = CGDisplayStreamGetRunLoopSource(display_stream);
+    CFRunLoopAddSource(CFRunLoopGetCurrent(), source, kCFRunLoopCommonModes);
+    display_streams_.push_back(display_stream);
+  }
+
+  CFRelease(properties_dictionary);
+
+  return 0;
+}
+
+int ScreenCapturerCg::Destroy() {
+  running_ = false;
+  return 0;
+}
+
+int ScreenCapturerCg::Start() {
+  if (_running) {
+    return 0;
+  }
+
+  running_ = true;
+  capture_thread_ = std::thread([this]() {
+    while (running_) {
+      CFRunLoopRun();
+    }
+  });
+
+  return 0;
+}
+
+int ScreenCapturerCg::Stop() {
+  running_ = false;
+  return 0;
+}
+
+int ScreenCapturerCg::Pause() { return 0; }
+
+int ScreenCapturerCg::Resume() { return 0; }
+
+void ScreenCapturerCg::OnFrame() {}
+
+void ScreenCapturerCg::CleanUp() {}
+
+//
+
+void ScreenCapturerCg::UnregisterRefreshAndMoveHandlers() {
+  for (CGDisplayStreamRef stream : display_streams_) {
+    CFRunLoopSourceRef source = CGDisplayStreamGetRunLoopSource(stream);
+    CFRunLoopRemoveSource(CFRunLoopGetCurrent(), source, kCFRunLoopCommonModes);
+    CGDisplayStreamStop(stream);
+    CFRelease(stream);
+  }
+  display_streams_.clear();
+}

src/screen_capturer/macosx/core_graphics/screen_capturer_cg.h

@@ -0,0 +1,56 @@
+
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2024-10-16
+ * Copyright (c) 2024 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _SCREEN_CAPTURER_CGD_H_
+#define _SCREEN_CAPTURER_CGD_H_
+
+#include <CoreGraphics/CoreGraphics.h>
+
+#include <atomic>
+#include <functional>
+#include <memory>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "screen_capturer.h"
+
+class ScreenCapturerCg : public ScreenCapturer {
+ public:
+  ScreenCapturerCg();
+  ~ScreenCapturerCg();
+
+ public:
+  virtual int Init(const int fps, cb_desktop_data cb);
+
+  virtual int Destroy();
+
+  virtual int Start();
+
+  virtual int Stop();
+
+  int Pause();
+
+  int Resume();
+
+  void OnFrame();
+
+ protected:
+  void CleanUp();
+
+ private:
+  int _fps;
+  cb_desktop_data _on_data;
+
+  // thread
+  std::thread capture_thread_;
+  std::atomic_bool running_;
+
+ private:
+};
+
+#endif

src/screen_capturer/macosx/screen_capturer_kit/screen_capturer_sck.cpp

@@ -0,0 +1,51 @@
+#include "screen_capturer_sck.h"
+
+#include "rd_log.h"
+
+ScreenCapturerSck::ScreenCapturerSck() {}
+ScreenCapturerSck::~ScreenCapturerSck() {
+  // if (inited_ && capture_thread_.joinable()) {
+  //   capture_thread_.join();
+  //   inited_ = false;
+  // }
+}
+
+int ScreenCapturerSck::Init(const int fps, cb_desktop_data cb) {
+  if (cb) {
+    on_data_ = cb;
+  } else {
+    LOG_ERROR("cb is null");
+    return -1;
+  }
+
+  screen_capturer_sck_impl_ = CreateScreenCapturerSck();
+  screen_capturer_sck_impl_->Init(fps, on_data_);
+
+  return 0;
+}
+
+int ScreenCapturerSck::Destroy() { return 0; }
+
+int ScreenCapturerSck::Start() {
+  // if (running_) {
+  //   return 0;
+  // }
+
+  // running_ = true;
+  // capture_thread_ = std::thread([this]() {
+  //   while (running_) {
+  //   }
+  // });
+
+  return 0;
+}
+
+int ScreenCapturerSck::Stop() { return 0; }
+
+int ScreenCapturerSck::Pause() { return 0; }
+
+int ScreenCapturerSck::Resume() { return 0; }
+
+void ScreenCapturerSck::OnFrame() {}
+
+void ScreenCapturerSck::CleanUp() {}

src/screen_capturer/macosx/screen_capturer_kit/screen_capturer_sck.h

@@ -0,0 +1,59 @@
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2024-10-17
+ * Copyright (c) 2024 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _SCREEN_CAPTURER_SCK_H_
+#define _SCREEN_CAPTURER_SCK_H_
+
+#include <atomic>
+#include <functional>
+#include <memory>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "screen_capturer.h"
+
+class ScreenCapturerSck : public ScreenCapturer {
+ public:
+  ScreenCapturerSck();
+  ~ScreenCapturerSck();
+
+ public:
+  virtual int Init(const int fps, cb_desktop_data cb);
+
+  virtual int Destroy();
+
+  virtual int Start();
+
+  virtual int Stop();
+
+  int Pause();
+
+  int Resume();
+
+  void OnFrame();
+
+ protected:
+  void CleanUp();
+
+ private:
+  std::unique_ptr<ScreenCapturer> CreateScreenCapturerSck();
+
+ private:
+  int _fps;
+  cb_desktop_data on_data_;
+  unsigned char* nv12_frame_ = nullptr;
+  bool inited_ = false;
+
+  // thread
+  std::thread capture_thread_;
+  std::atomic_bool running_;
+
+ private:
+  std::unique_ptr<ScreenCapturer> screen_capturer_sck_impl_;
+};
+
+#endif

src/screen_capturer/macosx/screen_capturer_kit/screen_capturer_sck_impl.mm

@@ -0,0 +1,256 @@
+/*
+ *  Copyright (c) 2024 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "screen_capturer_sck.h"
+
+#include "rd_log.h"
+#include <CoreGraphics/CoreGraphics.h>
+#include <IOSurface/IOSurface.h>
+#include <ScreenCaptureKit/ScreenCaptureKit.h>
+#include <atomic>
+#include <mutex>
+
+class ScreenCapturerSckImpl;
+
+// The ScreenCaptureKit API was available in macOS 12.3, but full-screen capture
+// was reported to be broken before macOS 13 - see http://crbug.com/40234870.
+// Also, the `SCContentFilter` fields `contentRect` and `pointPixelScale` were
+// introduced in macOS 14.
+API_AVAILABLE(macos(14.0))
+@interface SckHelper : NSObject <SCStreamDelegate, SCStreamOutput>
+
+- (instancetype)initWithCapturer:(ScreenCapturerSckImpl *)capturer;
+
+- (void)onShareableContentCreated:(SCShareableContent *)content;
+
+// Called just before the capturer is destroyed. This avoids a dangling pointer,
+// and prevents any new calls into a deleted capturer. If any method-call on the
+// capturer is currently running on a different thread, this blocks until it
+// completes.
+- (void)releaseCapturer;
+@end
+
+class API_AVAILABLE(macos(14.0)) ScreenCapturerSckImpl : public ScreenCapturer {
+public:
+  explicit ScreenCapturerSckImpl();
+
+  ScreenCapturerSckImpl(const ScreenCapturerSckImpl &) = delete;
+  ScreenCapturerSckImpl &operator=(const ScreenCapturerSckImpl &) = delete;
+  ~ScreenCapturerSckImpl();
+
+public:
+  int Init(const int fps, cb_desktop_data cb);
+  void OnReceiveContent(SCShareableContent *content);
+  void OnNewIOSurface(IOSurfaceRef io_surface, CFDictionaryRef attachment);
+
+  virtual int Destroy() { return 0; }
+
+  virtual int Start() { return 0; }
+
+  virtual int Stop() { return 0; }
+
+private:
+  SckHelper *__strong helper_;
+  SCStream *__strong stream_;
+
+  cb_desktop_data _on_data;
+  unsigned char *nv12_frame_ = nullptr;
+  bool permanent_error_ = false;
+  CGDirectDisplayID current_display_ = -1;
+  std::mutex mtx_;
+};
+
+@implementation SckHelper {
+  // This lock is to prevent the capturer being destroyed while an instance
+  // method is still running on another thread.
+  std::mutex helper_mtx_;
+  ScreenCapturerSckImpl *_capturer;
+}
+
+- (instancetype)initWithCapturer:(ScreenCapturerSckImpl *)capturer {
+  self = [super init];
+  if (self) {
+    _capturer = capturer;
+  }
+  return self;
+}
+
+- (void)onShareableContentCreated:(SCShareableContent *)content {
+  std::lock_guard<std::mutex> lock(helper_mtx_);
+  if (_capturer) {
+    _capturer->OnReceiveContent(content);
+  } else {
+    LOG_ERROR("Invalid capturer");
+  }
+}
+
+- (void)stream:(SCStream *)stream
+    didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
+                   ofType:(SCStreamOutputType)type {
+  CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+  if (!pixelBuffer) {
+    return;
+  }
+
+  IOSurfaceRef ioSurface = CVPixelBufferGetIOSurface(pixelBuffer);
+  if (!ioSurface) {
+    return;
+  }
+
+  CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(
+      sampleBuffer, /*createIfNecessary=*/false);
+  if (!attachmentsArray || CFArrayGetCount(attachmentsArray) <= 0) {
+    LOG_ERROR("Discarding frame with no attachments");
+    return;
+  }
+
+  CFDictionaryRef attachment =
+      static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachmentsArray, 0));
+
+  std::lock_guard<std::mutex> lock(helper_mtx_);
+  if (_capturer) {
+    _capturer->OnNewIOSurface(ioSurface, attachment);
+  }
+}
+
+- (void)releaseCapturer {
+  std::lock_guard<std::mutex> lock(helper_mtx_);
+  _capturer = nullptr;
+}
+
+@end
+
+ScreenCapturerSckImpl::ScreenCapturerSckImpl() {
+  helper_ = [[SckHelper alloc] initWithCapturer:this];
+}
+
+ScreenCapturerSckImpl::~ScreenCapturerSckImpl() {
+  [stream_ stopCaptureWithCompletionHandler:nil];
+  [helper_ releaseCapturer];
+}
+
+int ScreenCapturerSckImpl::Init(const int fps, cb_desktop_data cb) {
+  _on_data = cb;
+
+  SckHelper *local_helper = helper_;
+  auto handler = ^(SCShareableContent *content, NSError *error) {
+    [local_helper onShareableContentCreated:content];
+  };
+
+  [SCShareableContent getShareableContentWithCompletionHandler:handler];
+
+  return 0;
+}
+
+void ScreenCapturerSckImpl::OnReceiveContent(SCShareableContent *content) {
+  if (!content) {
+    LOG_ERROR("getShareableContent failed");
+    permanent_error_ = true;
+    return;
+  }
+
+  if (!content.displays.count) {
+    LOG_ERROR("getShareableContent returned no displays");
+    permanent_error_ = true;
+    return;
+  }
+
+  SCDisplay *captured_display;
+  {
+    std::lock_guard<std::mutex> lock(mtx_);
+    for (SCDisplay *display in content.displays) {
+      if (current_display_ == display.displayID) {
+        captured_display = display;
+        break;
+      }
+    }
+    if (!captured_display) {
+      if (-1 == current_display_) {
+        LOG_ERROR("Full screen capture is not supported, falling back to first "
+                  "display");
+      } else {
+        LOG_ERROR("Display [{}] not found, falling back to first display",
+                  current_display_);
+      }
+      captured_display = content.displays.firstObject;
+    }
+  }
+
+  SCContentFilter *filter =
+      [[SCContentFilter alloc] initWithDisplay:captured_display
+                              excludingWindows:@[]];
+  SCStreamConfiguration *config = [[SCStreamConfiguration alloc] init];
+  config.pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
+  config.showsCursor = false;
+  config.width = filter.contentRect.size.width * filter.pointPixelScale;
+  config.height = filter.contentRect.size.height * filter.pointPixelScale;
+  config.captureResolution = SCCaptureResolutionNominal;
+
+  std::lock_guard<std::mutex> lock(mtx_);
+
+  if (stream_) {
+    LOG_INFO("Updating stream configuration");
+    [stream_ updateContentFilter:filter completionHandler:nil];
+    [stream_ updateConfiguration:config completionHandler:nil];
+  } else {
+    stream_ = [[SCStream alloc] initWithFilter:filter
+                                 configuration:config
+                                      delegate:helper_];
+
+    // TODO: crbug.com/327458809 - Choose an appropriate sampleHandlerQueue for
+    // best performance.
+    NSError *add_stream_output_error;
+    bool add_stream_output_result =
+        [stream_ addStreamOutput:helper_
+                            type:SCStreamOutputTypeScreen
+              sampleHandlerQueue:nil
+                           error:&add_stream_output_error];
+    if (!add_stream_output_result) {
+      stream_ = nil;
+      LOG_ERROR("addStreamOutput failed");
+      permanent_error_ = true;
+      return;
+    }
+
+    auto handler = ^(NSError *error) {
+      if (error) {
+        // It should be safe to access `this` here, because the C++ destructor
+        // calls stopCaptureWithCompletionHandler on the stream, which cancels
+        // this handler.
+        permanent_error_ = true;
+        LOG_ERROR("startCaptureWithCompletionHandler failed");
+      } else {
+        LOG_INFO("Capture started");
+      }
+    };
+
+    [stream_ startCaptureWithCompletionHandler:handler];
+  }
+}
+
+void ScreenCapturerSckImpl::OnNewIOSurface(IOSurfaceRef io_surface,
+                                           CFDictionaryRef attachment) {
+  size_t width = IOSurfaceGetWidth(io_surface);
+  size_t height = IOSurfaceGetHeight(io_surface);
+
+  uint32_t aseed;
+  IOSurfaceLock(io_surface, kIOSurfaceLockReadOnly, &aseed);
+
+  nv12_frame_ =
+      static_cast<unsigned char *>(IOSurfaceGetBaseAddress(io_surface));
+
+  _on_data(nv12_frame_, width * height * 3 / 2, width, height);
+
+  IOSurfaceUnlock(io_surface, kIOSurfaceLockReadOnly, &aseed);
+}
+
+std::unique_ptr<ScreenCapturer> ScreenCapturerSck::CreateScreenCapturerSck() {
+  return std::make_unique<ScreenCapturerSckImpl>();
+}

src/screen_capturer/screen_capturer_factory.h

@@ -12,7 +12,8 @@
 #elif __linux__
 #include "screen_capturer_x11.h"
 #elif __APPLE__
-#include "screen_capturer_avf.h"
+// #include "screen_capturer_avf.h"
+#include "screen_capturer_sck.h"
 #endif
 
 class ScreenCapturerFactory {
@@ -26,7 +27,8 @@ class ScreenCapturerFactory {
 #elif __linux__
     return new ScreenCapturerX11();
 #elif __APPLE__
-    return new ScreenCapturerAvf();
+    // return new ScreenCapturerAvf();
+    return new ScreenCapturerSck();
 #else
     return nullptr;
 #endif

xmake.lua

@@ -41,7 +41,7 @@ elseif is_os("macosx") then
     add_packages("libxcb")
     add_links("SDL2", "SDL2main")
     add_ldflags("-Wl,-ld_classic")
-    add_frameworks("OpenGL")
+    add_frameworks("OpenGL", "IOSurface", "ScreenCaptureKit")
 end
 
 add_packages("spdlog", "imgui")
@@ -70,8 +70,11 @@ target("screen_capturer")
         add_includedirs("src/screen_capturer/windows", {public = true})
     elseif is_os("macosx") then
         add_packages("ffmpeg")
-        add_files("src/screen_capturer/macosx/*.cpp")
+        add_files("src/screen_capturer/macosx/avfoundation/*.cpp",
-        add_includedirs("src/screen_capturer/macosx", {public = true})
+        "src/screen_capturer/macosx/screen_capturer_kit/*.cpp",
+        "src/screen_capturer/macosx/screen_capturer_kit/*.mm")
+        add_includedirs("src/screen_capturer/macosx/avfoundation",
+        "src/screen_capturer/macosx/screen_capturer_kit", {public = true})
     elseif is_os("linux") then
         add_packages("ffmpeg")
         add_files("src/screen_capturer/linux/*.cpp")
@@ -138,8 +141,8 @@ target("remote_desk")
         add_files("icon/app.rc")
     elseif is_os("macosx") then
         add_packages("ffmpeg")
-        add_rules("xcode.application")
+        -- add_rules("xcode.application")
-        add_files("Info.plist")
+        -- add_files("Info.plist")
     elseif is_os("linux") then
         add_packages("ffmpeg")
     end