[feat] optimize macOS native video encoding

apps/desktop/src/gui/features/devices/session_device_manager.cpp

@@ -74,7 +74,8 @@ int SessionDeviceManager::InitializeScreenCapturer() {
 
   const int init_ret = screen_capturer_->Init(
       fps, [this, fps](unsigned char *data, int size, int width, int height,
-                       const char *display_name) {
+                       const char *display_name,
+                       const XNativeVideoFrame *native_frame) {
         const auto now_time = std::chrono::steady_clock::now();
         if (!ShouldSendCapturedFrame(now_time, fps)) {
           return;
@@ -142,6 +143,7 @@ int SessionDeviceManager::InitializeScreenCapturer() {
         frame.width = width;
         frame.height = height;
         frame.captured_timestamp = GetSystemTimeMicros(owner_.peer_);
+        frame.native_frame = native_frame;
         for (const std::string &remote_id : connected_remote_ids) {
           SendVideoFrameToPeer(owner_.peer_, &frame, stream_id.c_str(),
                                remote_id.data(), remote_id.size());

apps/desktop/src/platform/linux/screen_capturer/screen_capturer_drm.cpp

@@ -429,7 +429,7 @@ bool ScreenCapturerDrm::CaptureOutputFrame(const DrmOutput& output,
 
   if (emit_callback && callback_) {
     callback_(nv12.data(), static_cast<int>(nv12.size()), capture_width,
-              capture_height, output.stream_id.c_str());
+              capture_height, output.stream_id.c_str(), nullptr);
   }
 
   UnmapFramebuffer(mapped_ptr, mapped_size, prime_fd);

apps/desktop/src/platform/linux/screen_capturer/screen_capturer_linux.cpp

@@ -50,7 +50,8 @@ int ScreenCapturerLinux::Init(const int fps, cb_desktop_data cb) {
   fps_ = fps;
   callback_orig_ = std::move(cb);
   callback_ = [this](unsigned char* data, int size, int width, int height,
-                     const char* reported_stream_id) {
+                     const char* reported_stream_id,
+                     const XNativeVideoFrame* native_frame) {
     const std::string mapped_stream_id = MapStreamId(reported_stream_id);
     if (mapped_stream_id.empty()) {
       if (!invalid_stream_id_logged_.exchange(true,
@@ -64,7 +65,8 @@ int ScreenCapturerLinux::Init(const int fps, cb_desktop_data cb) {
     }
     invalid_stream_id_logged_.store(false, std::memory_order_relaxed);
     if (callback_orig_) {
-      callback_orig_(data, size, width, height, mapped_stream_id.c_str());
+      callback_orig_(data, size, width, height, mapped_stream_id.c_str(),
+                     native_frame);
     }
   };
 

apps/desktop/src/platform/linux/screen_capturer/screen_capturer_wayland_pipewire.cpp

@@ -1047,7 +1047,7 @@ void ScreenCapturerWayland::HandlePipeWireBuffer() {
   if (callback_) {
     const std::string stream_id = MakeDisplayStreamId(0);
     callback_(nv12.data(), static_cast<int>(nv12.size()), even_width,
-              even_height, stream_id.c_str());
+              even_height, stream_id.c_str(), nullptr);
   }
   pipewire_last_frame_ms_.store(NowMs());
 

apps/desktop/src/platform/linux/screen_capturer/screen_capturer_x11.cpp

@@ -321,7 +321,7 @@ void ScreenCapturerX11::OnFrame() {
   if (callback_) {
     const std::string stream_id = MakeDisplayStreamId(monitor_index);
     callback_(nv12.data(), width_ * height_ * 3 / 2, width_, height_,
-              stream_id.c_str());
+              stream_id.c_str(), nullptr);
   }
 
   XDestroyImage(image);

apps/desktop/src/platform/macos/screen_capturer/screen_capturer_sck_impl.mm

@@ -42,6 +42,65 @@ static std::string NSErrorToString(NSError *error) {
   return description ? description : "";
 }
 
+static void RetainCapturedPixelBuffer(void *owner) {
+  if (owner) {
+    CVPixelBufferRetain(static_cast<CVPixelBufferRef>(owner));
+  }
+}
+
+static void ReleaseCapturedPixelBuffer(void *owner) {
+  if (owner) {
+    CVPixelBufferRelease(static_cast<CVPixelBufferRef>(owner));
+  }
+}
+
+static int CopyCapturedPixelBufferToNv12(void *owner, uint8_t *destination,
+                                         size_t destination_size) {
+  auto pixel_buffer = static_cast<CVPixelBufferRef>(owner);
+  if (!pixel_buffer || !destination ||
+      !CVPixelBufferIsPlanar(pixel_buffer) ||
+      CVPixelBufferGetPlaneCount(pixel_buffer) < 2) {
+    return -1;
+  }
+  const OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer);
+  if (pixel_format != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange &&
+      pixel_format != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
+    return -1;
+  }
+
+  const size_t width = CVPixelBufferGetWidth(pixel_buffer);
+  const size_t height = CVPixelBufferGetHeight(pixel_buffer);
+  const size_t required_size = width * height * 3U / 2U;
+  if (destination_size < required_size ||
+      CVPixelBufferLockBaseAddress(pixel_buffer,
+                                   kCVPixelBufferLock_ReadOnly) !=
+          kCVReturnSuccess) {
+    return -1;
+  }
+
+  const auto *y_plane = static_cast<const uint8_t *>(
+      CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 0));
+  const auto *uv_plane = static_cast<const uint8_t *>(
+      CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, 1));
+  const size_t y_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 0);
+  const size_t uv_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, 1);
+  int result = 0;
+  if (!y_plane || !uv_plane || y_stride < width || uv_stride < width) {
+    result = -1;
+  } else {
+    for (size_t row = 0; row < height; ++row) {
+      std::memcpy(destination + row * width, y_plane + row * y_stride, width);
+    }
+    uint8_t *uv_destination = destination + width * height;
+    for (size_t row = 0; row < height / 2U; ++row) {
+      std::memcpy(uv_destination + row * width,
+                  uv_plane + row * uv_stride, width);
+    }
+  }
+  CVPixelBufferUnlockBaseAddress(pixel_buffer, kCVPixelBufferLock_ReadOnly);
+  return result;
+}
+
 // The ScreenCaptureKit API was available in macOS 12.3, but full-screen capture
 // was reported to be broken before macOS 13 - see http://crbug.com/40234870.
 // Also, the `SCContentFilter` fields `contentRect` and `pointPixelScale` were
@@ -92,14 +151,11 @@ class API_AVAILABLE(macos(14.0)) ScreenCapturerSckImpl : public ScreenCapturer {
  private:
   std::vector<DisplayInfo> display_info_list_;
   std::map<int, CGDirectDisplayID> display_id_map_;
-  unsigned char *nv12_frame_ = nullptr;
-  size_t nv12_frame_size_ = 0;
-  int width_ = 0;
-  int height_ = 0;
   int fps_ = 60;
   bool show_cursor_ = false;
   bool capture_requested_ = false;
   bool invalid_stream_id_logged_ = false;
+  bool native_output_logged_ = false;
 
  public:
   // Called by SckHelper when shareable content is returned by ScreenCaptureKit. `content` will be
@@ -119,8 +175,8 @@ class API_AVAILABLE(macos(14.0)) ScreenCapturerSckImpl : public ScreenCapturer {
   // object. The helper may outlive this C++ instance, if a completion-handler is passed to
   // ScreenCaptureKit APIs and the C++ object is deleted before the handler executes.
   SckHelper *__strong helper_ = nil;
-  // Callback for returning captured frames, or errors, to the caller. Only used on the caller's
-  // thread.
+  // Callback for returning captured frames to the caller. ScreenCaptureKit may
+  // invoke it on its sample-handler queue.
   cb_desktop_data _on_data = nullptr;
   // Signals that a permanent error occurred. This may be set on any thread, and is read by
   // CaptureFrame() which runs on the caller's thread.
@@ -244,12 +300,6 @@ ScreenCapturerSckImpl::~ScreenCapturerSckImpl() {
 
   display_info_list_.clear();
   display_id_map_.clear();
-
-  if (nv12_frame_) {
-    delete[] nv12_frame_;
-    nv12_frame_ = nullptr;
-    nv12_frame_size_ = 0;
-  }
 }
 
 int ScreenCapturerSckImpl::Init(const int fps, cb_desktop_data cb) {
@@ -262,6 +312,7 @@ int ScreenCapturerSckImpl::Init(const int fps, cb_desktop_data cb) {
   fps_ = fps > 0 ? fps : 60;
   display_info_list_.clear();
   display_id_map_.clear();
+  native_output_logged_ = false;
 
   if (@available(macOS 10.15, *)) {
     bool has_permission = CGPreflightScreenCaptureAccess();
@@ -596,81 +647,69 @@ void ScreenCapturerSckImpl::OnNewCVPixelBuffer(CVPixelBufferRef pixelBuffer,
     return;
   }
 
-  size_t width = CVPixelBufferGetWidth(pixelBuffer);
-  size_t height = CVPixelBufferGetHeight(pixelBuffer);
-  if (width == 0 || height == 0 || CVPixelBufferGetPlaneCount(pixelBuffer) < 2) {
-    LOG_ERROR("Invalid CVPixelBuffer: width={}, height={}, planes={}", width, height,
+  const size_t width = CVPixelBufferGetWidth(pixelBuffer);
+  const size_t height = CVPixelBufferGetHeight(pixelBuffer);
+  const OSType pixel_format = CVPixelBufferGetPixelFormatType(pixelBuffer);
+  if (width == 0 || height == 0 || (width & 1U) != 0 ||
+      (height & 1U) != 0 || !CVPixelBufferIsPlanar(pixelBuffer) ||
+      CVPixelBufferGetPlaneCount(pixelBuffer) < 2 ||
+      (pixel_format != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange &&
+       pixel_format != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)) {
+    LOG_ERROR("Invalid CVPixelBuffer: width={}, height={}, format={}, planes={}",
+              width, height, pixel_format,
               CVPixelBufferGetPlaneCount(pixelBuffer));
     return;
   }
 
-  CVReturn status = CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
-  if (status != kCVReturnSuccess) {
-    LOG_ERROR("Failed to lock CVPixelBuffer base address: %d", status);
-    return;
-  }
-
-  size_t required_size = width * height * 3 / 2;
+  const size_t required_size = width * height * 3U / 2U;
   if (required_size > static_cast<size_t>((std::numeric_limits<int>::max)())) {
     LOG_ERROR("Captured frame is too large: {} bytes", required_size);
-    CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
     return;
   }
 
-  std::lock_guard<std::mutex> lock(lock_);
-  if (!_on_data || !capture_requested_) {
-    CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
-    return;
-  }
-
-  if (!nv12_frame_ || nv12_frame_size_ < required_size) {
-    delete[] nv12_frame_;
-    nv12_frame_ = new unsigned char[required_size];
-    nv12_frame_size_ = required_size;
-  }
-  width_ = static_cast<int>(width);
-  height_ = static_cast<int>(height);
-
-  void *base_y = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
-  size_t stride_y = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
-
-  void *base_uv = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);
-  size_t stride_uv = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
-  if (!base_y || !base_uv || stride_y < width || stride_uv < width) {
-    LOG_ERROR("Invalid CVPixelBuffer planes: base_y={}, base_uv={}, stride_y={}, stride_uv={}",
-              base_y != nullptr, base_uv != nullptr, stride_y, stride_uv);
-    CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
-    return;
-  }
-
-  unsigned char *dst_y = nv12_frame_;
-  for (size_t row = 0; row < height; ++row) {
-    memcpy(dst_y + row * width, static_cast<unsigned char *>(base_y) + row * stride_y, width);
-  }
-
-  unsigned char *dst_uv = nv12_frame_ + width * height;
-  for (size_t row = 0; row < height / 2; ++row) {
-    memcpy(dst_uv + row * width, static_cast<unsigned char *>(base_uv) + row * stride_uv, width);
-  }
-
-  const std::string stream_id = ResolveDisplayStreamId(
-      current_stream_id_.c_str(), display_info_list_.size(),
-      current_monitor_index_);
-  if (stream_id.empty()) {
-    if (!invalid_stream_id_logged_) {
-      LOG_ERROR("Dropping macOS frames without a registered stream id, "
-                "display_id={}",
-                current_display_);
-      invalid_stream_id_logged_ = true;
+  cb_desktop_data callback;
+  std::string stream_id;
+  bool log_native_output = false;
+  {
+    std::lock_guard<std::mutex> lock(lock_);
+    if (!_on_data || !capture_requested_) {
+      return;
+    }
+    stream_id = ResolveDisplayStreamId(current_stream_id_.c_str(),
+                                       display_info_list_.size(),
+                                       current_monitor_index_);
+    if (stream_id.empty()) {
+      if (!invalid_stream_id_logged_) {
+        LOG_ERROR("Dropping macOS frames without a registered stream id, "
+                  "display_id={}",
+                  current_display_);
+        invalid_stream_id_logged_ = true;
+      }
+      return;
+    }
+    invalid_stream_id_logged_ = false;
+    callback = _on_data;
+    if (!native_output_logged_) {
+      native_output_logged_ = true;
+      log_native_output = true;
     }
-    CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
-    return;
   }
-  invalid_stream_id_logged_ = false;
-  _on_data(nv12_frame_, static_cast<int>(required_size), static_cast<int>(width),
-           static_cast<int>(height), stream_id.c_str());
 
-  CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
+  XNativeVideoFrame native_frame{};
+  native_frame.struct_size = sizeof(native_frame);
+  native_frame.type = XNativeVideoFrameCVPixelBuffer;
+  native_frame.width = static_cast<uint32_t>(width);
+  native_frame.height = static_cast<uint32_t>(height);
+  native_frame.payload.cv_pixel_buffer = pixelBuffer;
+  native_frame.owner = pixelBuffer;
+  native_frame.retain = &RetainCapturedPixelBuffer;
+  native_frame.release = &ReleaseCapturedPixelBuffer;
+  native_frame.copy_to_nv12 = &CopyCapturedPixelBufferToNv12;
+  if (log_native_output) {
+    LOG_INFO("macOS ScreenCaptureKit native CVPixelBuffer output enabled");
+  }
+  callback(nullptr, static_cast<int>(required_size), static_cast<int>(width),
+           static_cast<int>(height), stream_id.c_str(), &native_frame);
 }
 
 void ScreenCapturerSckImpl::StartOrReconfigureCapturer() {

apps/desktop/src/platform/windows/screen_capturer/screen_capturer_dxgi.cpp

@@ -384,7 +384,7 @@ void ScreenCapturerDxgi::CaptureLoop() {
       if (idx >= 0 && idx < static_cast<int>(display_info_list_.size())) {
         const std::string stream_id = MakeDisplayStreamId(idx);
         callback_(nv12_frame_, nv12_size, even_width, even_height,
-                  stream_id.c_str());
+                  stream_id.c_str(), nullptr);
       } else {
         LOG_ERROR("DXGI: CaptureLoop invalid monitor_index {} (list size {})",
                   idx, display_info_list_.size());

apps/desktop/src/platform/windows/screen_capturer/screen_capturer_gdi.cpp

@@ -212,7 +212,8 @@ void ScreenCapturerGdi::CaptureLoop() {
 
     if (callback_) {
       const std::string stream_id = MakeDisplayStreamId(idx);
-      callback_(nv12_frame_, nv12_size, width, height, stream_id.c_str());
+      callback_(nv12_frame_, nv12_size, width, height, stream_id.c_str(),
+                nullptr);
     }
 
     SelectObject(mem_dc, old);

apps/desktop/src/platform/windows/screen_capturer/screen_capturer_wgc.cpp

@@ -406,7 +406,7 @@ void ScreenCapturerWgc::OnFrame(const WgcSession::wgc_session_frame& frame,
 
     const std::string stream_id = MakeDisplayStreamId(id);
     on_data_(nv12_frame_, nv12_size, even_width, even_height,
-             stream_id.c_str());
+             stream_id.c_str(), nullptr);
   }
 }
 

apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.cpp

@@ -404,7 +404,8 @@ int ScreenCapturerWin::Init(const int fps, cb_desktop_data cb) {
   fps_ = fps;
   cb_orig_ = cb;
   cb_ = [this](unsigned char* data, int size, int w, int h,
-               const char* reported_stream_id) {
+               const char* reported_stream_id,
+               const XNativeVideoFrame* native_frame) {
     if (secure_desktop_capture_active_.load(std::memory_order_relaxed)) {
       return;
     }
@@ -450,7 +451,9 @@ int ScreenCapturerWin::Init(const int fps, cb_desktop_data cb) {
           "elapsed_ms={}",
           raw_stream_id, mapped_stream_id, w, h, size, elapsed_ms);
     }
-    if (cb_orig_) cb_orig_(data, size, w, h, mapped_stream_id.c_str());
+    if (cb_orig_) {
+      cb_orig_(data, size, w, h, mapped_stream_id.c_str(), native_frame);
+    }
   };
 
   int ret = -1;
@@ -1263,7 +1266,8 @@ void ScreenCapturerWin::SecureDesktopCaptureLoop() {
             &captured_width, &captured_height, &error_message)) {
       if (cb_orig_ && !secure_frame.empty()) {
         cb_orig_(secure_frame.data(), static_cast<int>(secure_frame.size()),
-                 captured_width, captured_height, display_name.c_str());
+                 captured_width, captured_height, display_name.c_str(),
+                 nullptr);
       }
       frame_delivered = true;
     }
@@ -1277,7 +1281,8 @@ void ScreenCapturerWin::SecureDesktopCaptureLoop() {
                                       &captured_height, &error_message)) {
       if (cb_orig_ && !secure_frame.empty()) {
         cb_orig_(secure_frame.data(), static_cast<int>(secure_frame.size()),
-                 captured_width, captured_height, display_name.c_str());
+                 captured_width, captured_height, display_name.c_str(),
+                 nullptr);
       }
       frame_delivered = true;
     }

apps/desktop/src/screen_capturer/screen_capturer.h

@@ -10,14 +10,18 @@
 #include <functional>
 
 #include "display_info.h"
+#include "minirtc.h"
 
 namespace crossdesk {
 
 class ScreenCapturer {
  public:
-  // The final callback argument is a logical MiniRTC stream ID (DisplayN),
-  // not a platform display name or physical handle.
-  typedef std::function<void(unsigned char*, int, int, int, const char*)>
+  // |stream_id| is a logical MiniRTC stream ID (DisplayN), not a platform
+  // display name or physical handle. |native_frame| is borrowed for the
+  // duration of the callback; retain its owner before using it asynchronously.
+  typedef std::function<void(unsigned char* data, int size, int width,
+                             int height, const char* stream_id,
+                             const XNativeVideoFrame* native_frame)>
       cb_desktop_data;
 
  public:

apps/desktop/xmake/targets.lua

@@ -159,6 +159,7 @@ function setup_targets()
         set_kind("object")
         add_deps("rd_log", "common", "crossdesk_wire")
         add_includedirs("apps/desktop/src/screen_capturer", {public = true})
+        add_includedirs("deps/submodules/minirtc/src/api", {public = true})
         if is_os("windows") then
             add_packages("libyuv")
             add_files("apps/desktop/src/platform/windows/screen_capturer/screen_capturer_dxgi.cpp",
@@ -367,7 +368,8 @@ function setup_targets()
                 "apps/desktop/src/platform/windows/screen_capturer/wgc_plugin_entry.cpp")
             add_files("apps/desktop/resources/windows/wgc_plugin.rc")
             add_includedirs("apps/desktop/src/common", "apps/desktop/src/screen_capturer",
-                "apps/desktop/src/platform/windows/screen_capturer")
+                "apps/desktop/src/platform/windows/screen_capturer",
+                "deps/submodules/minirtc/src/api")
 
         target("crossdesk_service")
             set_kind("binary")