[feat] optimize Windows native video encoding

apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.cpp

@@ -15,6 +15,7 @@
 #include <vector>
 
 #include <display_stream_id.h>
+#include "captured_nv12_frame.h"
 #include "interactive_state.h"
 #include "rd_log.h"
 #include "screen_capturer_dxgi.h"
@@ -403,6 +404,14 @@ ScreenCapturerWin::~ScreenCapturerWin() { Destroy(); }
 int ScreenCapturerWin::Init(const int fps, cb_desktop_data cb) {
   fps_ = fps;
   cb_orig_ = cb;
+  native_output_logged_.store(false, std::memory_order_relaxed);
+  native_output_error_logged_.store(false, std::memory_order_relaxed);
+  try {
+    native_frame_pool_ = CapturedNv12FramePool::Create();
+  } catch (...) {
+    LOG_ERROR("Windows capturer: failed to create native NV12 frame pool");
+    return -1;
+  }
   cb_ = [this](unsigned char* data, int size, int w, int h,
                const char* reported_stream_id,
                const XNativeVideoFrame* native_frame) {
@@ -451,9 +460,8 @@ int ScreenCapturerWin::Init(const int fps, cb_desktop_data cb) {
           "elapsed_ms={}",
           raw_stream_id, mapped_stream_id, w, h, size, elapsed_ms);
     }
-    if (cb_orig_) {
-      cb_orig_(data, size, w, h, mapped_stream_id.c_str(), native_frame);
-    }
+    EmitCapturedFrame(data, size, w, h, mapped_stream_id.c_str(),
+                      native_frame);
   };
 
   int ret = -1;
@@ -516,9 +524,52 @@ int ScreenCapturerWin::Destroy() {
     stream_id_alias_.clear();
     handle_to_canonical_index_.clear();
   }
+  native_frame_pool_.reset();
   return 0;
 }
 
+void ScreenCapturerWin::EmitCapturedFrame(
+    unsigned char* data, int size, int width, int height,
+    const char* stream_id, const XNativeVideoFrame* native_frame) {
+  if (!cb_orig_) {
+    return;
+  }
+
+  if (native_frame) {
+    if (!native_output_logged_.exchange(true, std::memory_order_relaxed)) {
+      LOG_INFO("Windows capturer native frame output enabled (type={})",
+               static_cast<uint32_t>(native_frame->type));
+    }
+    cb_orig_(data, size, width, height, stream_id, native_frame);
+    return;
+  }
+
+  CapturedNv12Frame* owned_frame = nullptr;
+  if (native_frame_pool_ && data && size > 0 && width > 0 && height > 0) {
+    owned_frame = native_frame_pool_->CopyFrom(
+        data, static_cast<size_t>(size), static_cast<uint32_t>(width),
+        static_cast<uint32_t>(height));
+  }
+  if (!owned_frame) {
+    if (!native_output_error_logged_.exchange(true,
+                                               std::memory_order_relaxed)) {
+      LOG_WARN(
+          "Windows capturer could not retain a native NV12 frame; falling "
+          "back to copied CPU input (size={}x{}, bytes={})",
+          width, height, size);
+    }
+    cb_orig_(data, size, width, height, stream_id, nullptr);
+    return;
+  }
+
+  if (!native_output_logged_.exchange(true, std::memory_order_relaxed)) {
+    LOG_INFO("Windows capturer native NV12 output enabled");
+  }
+  cb_orig_(nullptr, static_cast<int>(owned_frame->Size()), width, height,
+           stream_id, owned_frame->Descriptor());
+  owned_frame->Release();
+}
+
 int ScreenCapturerWin::Start(bool show_cursor) {
   if (!impl_) return -1;
   if (running_.load(std::memory_order_relaxed)) {
@@ -1264,10 +1315,11 @@ void ScreenCapturerWin::SecureDesktopCaptureLoop() {
         ReadSecureDesktopSharedFrame(
             static_cast<DWORD>(frame_interval_ms + 20), &secure_frame,
             &captured_width, &captured_height, &error_message)) {
-      if (cb_orig_ && !secure_frame.empty()) {
-        cb_orig_(secure_frame.data(), static_cast<int>(secure_frame.size()),
-                 captured_width, captured_height, display_name.c_str(),
-                 nullptr);
+      if (!secure_frame.empty()) {
+        EmitCapturedFrame(secure_frame.data(),
+                          static_cast<int>(secure_frame.size()),
+                          captured_width, captured_height,
+                          display_name.c_str());
       }
       frame_delivered = true;
     }
@@ -1279,10 +1331,11 @@ void ScreenCapturerWin::SecureDesktopCaptureLoop() {
                                       status.interactive_desktop,
                                       &secure_frame, &captured_width,
                                       &captured_height, &error_message)) {
-      if (cb_orig_ && !secure_frame.empty()) {
-        cb_orig_(secure_frame.data(), static_cast<int>(secure_frame.size()),
-                 captured_width, captured_height, display_name.c_str(),
-                 nullptr);
+      if (!secure_frame.empty()) {
+        EmitCapturedFrame(secure_frame.data(),
+                          static_cast<int>(secure_frame.size()),
+                          captured_width, captured_height,
+                          display_name.c_str());
       }
       frame_delivered = true;
     }

apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.h

@@ -14,6 +14,7 @@
 #include <cstdint>
 #include <memory>
 #include <mutex>
+#include <string>
 #include <thread>
 #include <unordered_map>
 #include <vector>
@@ -22,6 +23,8 @@
 
 namespace crossdesk {
 
+class CapturedNv12FramePool;
+
 class ScreenCapturerWin : public ScreenCapturer {
  public:
   ScreenCapturerWin();
@@ -53,6 +56,8 @@ class ScreenCapturerWin : public ScreenCapturer {
   std::mutex alias_mutex_;
   std::vector<DisplayInfo> canonical_displays_;
   std::atomic<bool> invalid_stream_id_logged_{false};
+  std::atomic<bool> native_output_logged_{false};
+  std::atomic<bool> native_output_error_logged_{false};
   std::atomic<bool> running_{false};
   std::atomic<bool> paused_{false};
   std::atomic<bool> show_cursor_{true};
@@ -77,9 +82,13 @@ class ScreenCapturerWin : public ScreenCapturer {
   std::string secure_shared_stage_;
   std::string secure_shared_desktop_;
   bool secure_shared_capture_started_ = false;
+  std::shared_ptr<CapturedNv12FramePool> native_frame_pool_;
 
   void BuildCanonicalFromImpl();
   void RebuildAliasesFromImpl();
+  void EmitCapturedFrame(unsigned char* data, int size, int width, int height,
+                         const char* stream_id,
+                         const XNativeVideoFrame* native_frame = nullptr);
   void StopSecureCaptureThread();
   bool RestartCaptureBackendAfterSecureDesktop();
   void SecureDesktopCaptureLoop();

apps/desktop/src/screen_capturer/captured_nv12_frame.cpp

@@ -0,0 +1,165 @@
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2026-09-04
+ * Copyright (c) 2026 by DI JUNKUN, All Rights Reserved.
+ */
+
+#include "captured_nv12_frame.h"
+
+#include <algorithm>
+#include <cstring>
+#include <limits>
+#include <new>
+#include <utility>
+
+namespace crossdesk {
+namespace {
+
+bool GetNv12Size(uint32_t width, uint32_t height, size_t* size) {
+  if (!size || width == 0 || height == 0 || (width & 1U) != 0 ||
+      (height & 1U) != 0 ||
+      static_cast<size_t>(width) >
+          std::numeric_limits<size_t>::max() / height) {
+    return false;
+  }
+  const size_t pixels = static_cast<size_t>(width) * height;
+  if (pixels > std::numeric_limits<size_t>::max() / 3U * 2U) {
+    return false;
+  }
+  *size = pixels * 3U / 2U;
+  return true;
+}
+
+}  // namespace
+
+std::shared_ptr<CapturedNv12FramePool> CapturedNv12FramePool::Create(
+    size_t max_cached_buffers) {
+  return std::shared_ptr<CapturedNv12FramePool>(
+      new CapturedNv12FramePool(max_cached_buffers));
+}
+
+CapturedNv12FramePool::CapturedNv12FramePool(size_t max_cached_buffers)
+    : max_cached_buffers_(max_cached_buffers) {}
+
+CapturedNv12Frame* CapturedNv12FramePool::CopyFrom(const uint8_t* data,
+                                                   size_t size,
+                                                   uint32_t width,
+                                                   uint32_t height) {
+  size_t required_size = 0;
+  if (!data || !GetNv12Size(width, height, &required_size) ||
+      size < required_size) {
+    return nullptr;
+  }
+
+  std::vector<uint8_t> storage;
+  {
+    std::lock_guard<std::mutex> lock(mutex_);
+    auto best = free_buffers_.end();
+    for (auto it = free_buffers_.begin(); it != free_buffers_.end(); ++it) {
+      if (it->capacity() < required_size) {
+        continue;
+      }
+      if (best == free_buffers_.end() || it->capacity() < best->capacity()) {
+        best = it;
+      }
+    }
+    if (best == free_buffers_.end() && !free_buffers_.empty()) {
+      best = std::max_element(
+          free_buffers_.begin(), free_buffers_.end(),
+          [](const auto& lhs, const auto& rhs) {
+            return lhs.capacity() < rhs.capacity();
+          });
+    }
+    if (best != free_buffers_.end()) {
+      storage = std::move(*best);
+      free_buffers_.erase(best);
+    }
+  }
+
+  try {
+    storage.resize(required_size);
+    std::memcpy(storage.data(), data, required_size);
+    return new CapturedNv12Frame(shared_from_this(), std::move(storage), width,
+                                 height);
+  } catch (...) {
+    return nullptr;
+  }
+}
+
+void CapturedNv12FramePool::Recycle(std::vector<uint8_t> storage) {
+  if (max_cached_buffers_ == 0 || storage.empty()) {
+    return;
+  }
+
+  std::lock_guard<std::mutex> lock(mutex_);
+  if (free_buffers_.size() < max_cached_buffers_) {
+    free_buffers_.push_back(std::move(storage));
+    return;
+  }
+
+  auto smallest = std::min_element(
+      free_buffers_.begin(), free_buffers_.end(),
+      [](const auto& lhs, const auto& rhs) {
+        return lhs.capacity() < rhs.capacity();
+      });
+  if (smallest != free_buffers_.end() &&
+      smallest->capacity() < storage.capacity()) {
+    *smallest = std::move(storage);
+  }
+}
+
+CapturedNv12Frame::CapturedNv12Frame(
+    std::shared_ptr<CapturedNv12FramePool> pool,
+    std::vector<uint8_t> storage, uint32_t width, uint32_t height)
+    : pool_(std::move(pool)),
+      storage_(std::move(storage)),
+      width_(width),
+      height_(height) {
+  descriptor_.struct_size = sizeof(descriptor_);
+  descriptor_.type = XNativeVideoFrameCpuNv12;
+  descriptor_.width = width_;
+  descriptor_.height = height_;
+  descriptor_.payload.cpu_nv12 = {
+      storage_.data(), storage_.data() + static_cast<size_t>(width_) * height_,
+      width_, width_};
+  descriptor_.owner = this;
+  descriptor_.retain = &RetainOwner;
+  descriptor_.release = &ReleaseOwner;
+  descriptor_.copy_to_nv12 = &CopyToNv12;
+}
+
+CapturedNv12Frame::~CapturedNv12Frame() {
+  if (pool_) {
+    pool_->Recycle(std::move(storage_));
+  }
+}
+
+void CapturedNv12Frame::AddRef() {
+  references_.fetch_add(1, std::memory_order_relaxed);
+}
+
+void CapturedNv12Frame::Release() {
+  if (references_.fetch_sub(1, std::memory_order_acq_rel) == 1) {
+    delete this;
+  }
+}
+
+void CapturedNv12Frame::RetainOwner(void* owner) {
+  static_cast<CapturedNv12Frame*>(owner)->AddRef();
+}
+
+void CapturedNv12Frame::ReleaseOwner(void* owner) {
+  static_cast<CapturedNv12Frame*>(owner)->Release();
+}
+
+int CapturedNv12Frame::CopyToNv12(void* owner, uint8_t* destination,
+                                  size_t destination_size) {
+  auto* frame = static_cast<CapturedNv12Frame*>(owner);
+  if (!frame || !destination || destination_size < frame->Size()) {
+    return -1;
+  }
+  std::memcpy(destination, frame->storage_.data(), frame->Size());
+  return 0;
+}
+
+}  // namespace crossdesk

apps/desktop/src/screen_capturer/captured_nv12_frame.h

@@ -0,0 +1,78 @@
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2026-09-04
+ * Copyright (c) 2026 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _CAPTURED_NV12_FRAME_H_
+#define _CAPTURED_NV12_FRAME_H_
+
+#include <atomic>
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <mutex>
+#include <vector>
+
+#include "minirtc.h"
+
+namespace crossdesk {
+
+class CapturedNv12Frame;
+
+// Owns the pixels behind captured native frames. A frame may outlive the
+// capture callback while MiniRTC waits for its asynchronous encoder, so capture
+// buffers cannot be borrowed directly. Released allocations are cached to keep
+// that ownership transfer inexpensive after the pipeline is warm.
+class CapturedNv12FramePool final
+    : public std::enable_shared_from_this<CapturedNv12FramePool> {
+ public:
+  static std::shared_ptr<CapturedNv12FramePool> Create(
+      size_t max_cached_buffers = 4);
+
+  CapturedNv12Frame* CopyFrom(const uint8_t* data, size_t size,
+                              uint32_t width, uint32_t height);
+
+ private:
+  explicit CapturedNv12FramePool(size_t max_cached_buffers);
+  void Recycle(std::vector<uint8_t> storage);
+
+  friend class CapturedNv12Frame;
+
+  const size_t max_cached_buffers_;
+  std::mutex mutex_;
+  std::vector<std::vector<uint8_t>> free_buffers_;
+};
+
+class CapturedNv12Frame final {
+ public:
+  const XNativeVideoFrame* Descriptor() const { return &descriptor_; }
+  size_t Size() const { return storage_.size(); }
+
+  void AddRef();
+  void Release();
+
+ private:
+  friend class CapturedNv12FramePool;
+
+  CapturedNv12Frame(std::shared_ptr<CapturedNv12FramePool> pool,
+                    std::vector<uint8_t> storage, uint32_t width,
+                    uint32_t height);
+  ~CapturedNv12Frame();
+
+  static void RetainOwner(void* owner);
+  static void ReleaseOwner(void* owner);
+  static int CopyToNv12(void* owner, uint8_t* destination,
+                        size_t destination_size);
+
+  std::atomic<uint32_t> references_{1};
+  std::shared_ptr<CapturedNv12FramePool> pool_;
+  std::vector<uint8_t> storage_;
+  uint32_t width_ = 0;
+  uint32_t height_ = 0;
+  XNativeVideoFrame descriptor_{};
+};
+
+}  // namespace crossdesk
+
+#endif

apps/desktop/xmake/targets.lua

@@ -162,6 +162,7 @@ function setup_targets()
         add_includedirs("deps/submodules/minirtc/src/api", {public = true})
         if is_os("windows") then
             add_packages("libyuv")
+            add_files("apps/desktop/src/screen_capturer/captured_nv12_frame.cpp")
             add_files("apps/desktop/src/platform/windows/screen_capturer/screen_capturer_dxgi.cpp",
                 "apps/desktop/src/platform/windows/screen_capturer/screen_capturer_gdi.cpp",
                 "apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.cpp",