From 010064d5b772948ad8fabc5f23c004a0f37a84ee Mon Sep 17 00:00:00 2001 From: dijunkun Date: Fri, 4 Sep 2026 02:23:16 +0800 Subject: [PATCH] [feat] optimize Windows native video encoding --- .../screen_capturer/screen_capturer_win.cpp | 75 ++++++-- .../screen_capturer/screen_capturer_win.h | 9 + .../screen_capturer/captured_nv12_frame.cpp | 165 ++++++++++++++++++ .../src/screen_capturer/captured_nv12_frame.h | 78 +++++++++ apps/desktop/xmake/targets.lua | 1 + deps/submodules/minirtc | 2 +- 6 files changed, 318 insertions(+), 12 deletions(-) create mode 100644 apps/desktop/src/screen_capturer/captured_nv12_frame.cpp create mode 100644 apps/desktop/src/screen_capturer/captured_nv12_frame.h diff --git a/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.cpp b/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.cpp index f2ec6b51..bd373f57 100644 --- a/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.cpp +++ b/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.cpp @@ -15,6 +15,7 @@ #include #include +#include "captured_nv12_frame.h" #include "interactive_state.h" #include "rd_log.h" #include "screen_capturer_dxgi.h" @@ -403,6 +404,14 @@ ScreenCapturerWin::~ScreenCapturerWin() { Destroy(); } int ScreenCapturerWin::Init(const int fps, cb_desktop_data cb) { fps_ = fps; cb_orig_ = cb; + native_output_logged_.store(false, std::memory_order_relaxed); + native_output_error_logged_.store(false, std::memory_order_relaxed); + try { + native_frame_pool_ = CapturedNv12FramePool::Create(); + } catch (...) { + LOG_ERROR("Windows capturer: failed to create native NV12 frame pool"); + return -1; + } cb_ = [this](unsigned char* data, int size, int w, int h, const char* reported_stream_id, const XNativeVideoFrame* native_frame) { @@ -451,9 +460,8 @@ int ScreenCapturerWin::Init(const int fps, cb_desktop_data cb) { "elapsed_ms={}", raw_stream_id, mapped_stream_id, w, h, size, elapsed_ms); } - if (cb_orig_) { - cb_orig_(data, size, w, h, mapped_stream_id.c_str(), native_frame); - } + EmitCapturedFrame(data, size, w, h, mapped_stream_id.c_str(), + native_frame); }; int ret = -1; @@ -516,9 +524,52 @@ int ScreenCapturerWin::Destroy() { stream_id_alias_.clear(); handle_to_canonical_index_.clear(); } + native_frame_pool_.reset(); return 0; } +void ScreenCapturerWin::EmitCapturedFrame( + unsigned char* data, int size, int width, int height, + const char* stream_id, const XNativeVideoFrame* native_frame) { + if (!cb_orig_) { + return; + } + + if (native_frame) { + if (!native_output_logged_.exchange(true, std::memory_order_relaxed)) { + LOG_INFO("Windows capturer native frame output enabled (type={})", + static_cast(native_frame->type)); + } + cb_orig_(data, size, width, height, stream_id, native_frame); + return; + } + + CapturedNv12Frame* owned_frame = nullptr; + if (native_frame_pool_ && data && size > 0 && width > 0 && height > 0) { + owned_frame = native_frame_pool_->CopyFrom( + data, static_cast(size), static_cast(width), + static_cast(height)); + } + if (!owned_frame) { + if (!native_output_error_logged_.exchange(true, + std::memory_order_relaxed)) { + LOG_WARN( + "Windows capturer could not retain a native NV12 frame; falling " + "back to copied CPU input (size={}x{}, bytes={})", + width, height, size); + } + cb_orig_(data, size, width, height, stream_id, nullptr); + return; + } + + if (!native_output_logged_.exchange(true, std::memory_order_relaxed)) { + LOG_INFO("Windows capturer native NV12 output enabled"); + } + cb_orig_(nullptr, static_cast(owned_frame->Size()), width, height, + stream_id, owned_frame->Descriptor()); + owned_frame->Release(); +} + int ScreenCapturerWin::Start(bool show_cursor) { if (!impl_) return -1; if (running_.load(std::memory_order_relaxed)) { @@ -1264,10 +1315,11 @@ void ScreenCapturerWin::SecureDesktopCaptureLoop() { ReadSecureDesktopSharedFrame( static_cast(frame_interval_ms + 20), &secure_frame, &captured_width, &captured_height, &error_message)) { - if (cb_orig_ && !secure_frame.empty()) { - cb_orig_(secure_frame.data(), static_cast(secure_frame.size()), - captured_width, captured_height, display_name.c_str(), - nullptr); + if (!secure_frame.empty()) { + EmitCapturedFrame(secure_frame.data(), + static_cast(secure_frame.size()), + captured_width, captured_height, + display_name.c_str()); } frame_delivered = true; } @@ -1279,10 +1331,11 @@ void ScreenCapturerWin::SecureDesktopCaptureLoop() { status.interactive_desktop, &secure_frame, &captured_width, &captured_height, &error_message)) { - if (cb_orig_ && !secure_frame.empty()) { - cb_orig_(secure_frame.data(), static_cast(secure_frame.size()), - captured_width, captured_height, display_name.c_str(), - nullptr); + if (!secure_frame.empty()) { + EmitCapturedFrame(secure_frame.data(), + static_cast(secure_frame.size()), + captured_width, captured_height, + display_name.c_str()); } frame_delivered = true; } diff --git a/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.h b/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.h index 3a936bf8..3d0f9378 100644 --- a/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.h +++ b/apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,8 @@ namespace crossdesk { +class CapturedNv12FramePool; + class ScreenCapturerWin : public ScreenCapturer { public: ScreenCapturerWin(); @@ -53,6 +56,8 @@ class ScreenCapturerWin : public ScreenCapturer { std::mutex alias_mutex_; std::vector canonical_displays_; std::atomic invalid_stream_id_logged_{false}; + std::atomic native_output_logged_{false}; + std::atomic native_output_error_logged_{false}; std::atomic running_{false}; std::atomic paused_{false}; std::atomic show_cursor_{true}; @@ -77,9 +82,13 @@ class ScreenCapturerWin : public ScreenCapturer { std::string secure_shared_stage_; std::string secure_shared_desktop_; bool secure_shared_capture_started_ = false; + std::shared_ptr native_frame_pool_; void BuildCanonicalFromImpl(); void RebuildAliasesFromImpl(); + void EmitCapturedFrame(unsigned char* data, int size, int width, int height, + const char* stream_id, + const XNativeVideoFrame* native_frame = nullptr); void StopSecureCaptureThread(); bool RestartCaptureBackendAfterSecureDesktop(); void SecureDesktopCaptureLoop(); diff --git a/apps/desktop/src/screen_capturer/captured_nv12_frame.cpp b/apps/desktop/src/screen_capturer/captured_nv12_frame.cpp new file mode 100644 index 00000000..a3ee64c7 --- /dev/null +++ b/apps/desktop/src/screen_capturer/captured_nv12_frame.cpp @@ -0,0 +1,165 @@ +/* + * @Author: DI JUNKUN + * @Date: 2026-09-04 + * Copyright (c) 2026 by DI JUNKUN, All Rights Reserved. + */ + +#include "captured_nv12_frame.h" + +#include +#include +#include +#include +#include + +namespace crossdesk { +namespace { + +bool GetNv12Size(uint32_t width, uint32_t height, size_t* size) { + if (!size || width == 0 || height == 0 || (width & 1U) != 0 || + (height & 1U) != 0 || + static_cast(width) > + std::numeric_limits::max() / height) { + return false; + } + const size_t pixels = static_cast(width) * height; + if (pixels > std::numeric_limits::max() / 3U * 2U) { + return false; + } + *size = pixels * 3U / 2U; + return true; +} + +} // namespace + +std::shared_ptr CapturedNv12FramePool::Create( + size_t max_cached_buffers) { + return std::shared_ptr( + new CapturedNv12FramePool(max_cached_buffers)); +} + +CapturedNv12FramePool::CapturedNv12FramePool(size_t max_cached_buffers) + : max_cached_buffers_(max_cached_buffers) {} + +CapturedNv12Frame* CapturedNv12FramePool::CopyFrom(const uint8_t* data, + size_t size, + uint32_t width, + uint32_t height) { + size_t required_size = 0; + if (!data || !GetNv12Size(width, height, &required_size) || + size < required_size) { + return nullptr; + } + + std::vector storage; + { + std::lock_guard lock(mutex_); + auto best = free_buffers_.end(); + for (auto it = free_buffers_.begin(); it != free_buffers_.end(); ++it) { + if (it->capacity() < required_size) { + continue; + } + if (best == free_buffers_.end() || it->capacity() < best->capacity()) { + best = it; + } + } + if (best == free_buffers_.end() && !free_buffers_.empty()) { + best = std::max_element( + free_buffers_.begin(), free_buffers_.end(), + [](const auto& lhs, const auto& rhs) { + return lhs.capacity() < rhs.capacity(); + }); + } + if (best != free_buffers_.end()) { + storage = std::move(*best); + free_buffers_.erase(best); + } + } + + try { + storage.resize(required_size); + std::memcpy(storage.data(), data, required_size); + return new CapturedNv12Frame(shared_from_this(), std::move(storage), width, + height); + } catch (...) { + return nullptr; + } +} + +void CapturedNv12FramePool::Recycle(std::vector storage) { + if (max_cached_buffers_ == 0 || storage.empty()) { + return; + } + + std::lock_guard lock(mutex_); + if (free_buffers_.size() < max_cached_buffers_) { + free_buffers_.push_back(std::move(storage)); + return; + } + + auto smallest = std::min_element( + free_buffers_.begin(), free_buffers_.end(), + [](const auto& lhs, const auto& rhs) { + return lhs.capacity() < rhs.capacity(); + }); + if (smallest != free_buffers_.end() && + smallest->capacity() < storage.capacity()) { + *smallest = std::move(storage); + } +} + +CapturedNv12Frame::CapturedNv12Frame( + std::shared_ptr pool, + std::vector storage, uint32_t width, uint32_t height) + : pool_(std::move(pool)), + storage_(std::move(storage)), + width_(width), + height_(height) { + descriptor_.struct_size = sizeof(descriptor_); + descriptor_.type = XNativeVideoFrameCpuNv12; + descriptor_.width = width_; + descriptor_.height = height_; + descriptor_.payload.cpu_nv12 = { + storage_.data(), storage_.data() + static_cast(width_) * height_, + width_, width_}; + descriptor_.owner = this; + descriptor_.retain = &RetainOwner; + descriptor_.release = &ReleaseOwner; + descriptor_.copy_to_nv12 = &CopyToNv12; +} + +CapturedNv12Frame::~CapturedNv12Frame() { + if (pool_) { + pool_->Recycle(std::move(storage_)); + } +} + +void CapturedNv12Frame::AddRef() { + references_.fetch_add(1, std::memory_order_relaxed); +} + +void CapturedNv12Frame::Release() { + if (references_.fetch_sub(1, std::memory_order_acq_rel) == 1) { + delete this; + } +} + +void CapturedNv12Frame::RetainOwner(void* owner) { + static_cast(owner)->AddRef(); +} + +void CapturedNv12Frame::ReleaseOwner(void* owner) { + static_cast(owner)->Release(); +} + +int CapturedNv12Frame::CopyToNv12(void* owner, uint8_t* destination, + size_t destination_size) { + auto* frame = static_cast(owner); + if (!frame || !destination || destination_size < frame->Size()) { + return -1; + } + std::memcpy(destination, frame->storage_.data(), frame->Size()); + return 0; +} + +} // namespace crossdesk diff --git a/apps/desktop/src/screen_capturer/captured_nv12_frame.h b/apps/desktop/src/screen_capturer/captured_nv12_frame.h new file mode 100644 index 00000000..d494483b --- /dev/null +++ b/apps/desktop/src/screen_capturer/captured_nv12_frame.h @@ -0,0 +1,78 @@ +/* + * @Author: DI JUNKUN + * @Date: 2026-09-04 + * Copyright (c) 2026 by DI JUNKUN, All Rights Reserved. + */ + +#ifndef _CAPTURED_NV12_FRAME_H_ +#define _CAPTURED_NV12_FRAME_H_ + +#include +#include +#include +#include +#include +#include + +#include "minirtc.h" + +namespace crossdesk { + +class CapturedNv12Frame; + +// Owns the pixels behind captured native frames. A frame may outlive the +// capture callback while MiniRTC waits for its asynchronous encoder, so capture +// buffers cannot be borrowed directly. Released allocations are cached to keep +// that ownership transfer inexpensive after the pipeline is warm. +class CapturedNv12FramePool final + : public std::enable_shared_from_this { + public: + static std::shared_ptr Create( + size_t max_cached_buffers = 4); + + CapturedNv12Frame* CopyFrom(const uint8_t* data, size_t size, + uint32_t width, uint32_t height); + + private: + explicit CapturedNv12FramePool(size_t max_cached_buffers); + void Recycle(std::vector storage); + + friend class CapturedNv12Frame; + + const size_t max_cached_buffers_; + std::mutex mutex_; + std::vector> free_buffers_; +}; + +class CapturedNv12Frame final { + public: + const XNativeVideoFrame* Descriptor() const { return &descriptor_; } + size_t Size() const { return storage_.size(); } + + void AddRef(); + void Release(); + + private: + friend class CapturedNv12FramePool; + + CapturedNv12Frame(std::shared_ptr pool, + std::vector storage, uint32_t width, + uint32_t height); + ~CapturedNv12Frame(); + + static void RetainOwner(void* owner); + static void ReleaseOwner(void* owner); + static int CopyToNv12(void* owner, uint8_t* destination, + size_t destination_size); + + std::atomic references_{1}; + std::shared_ptr pool_; + std::vector storage_; + uint32_t width_ = 0; + uint32_t height_ = 0; + XNativeVideoFrame descriptor_{}; +}; + +} // namespace crossdesk + +#endif diff --git a/apps/desktop/xmake/targets.lua b/apps/desktop/xmake/targets.lua index 1ad8c0f7..b36ae608 100644 --- a/apps/desktop/xmake/targets.lua +++ b/apps/desktop/xmake/targets.lua @@ -162,6 +162,7 @@ function setup_targets() add_includedirs("deps/submodules/minirtc/src/api", {public = true}) if is_os("windows") then add_packages("libyuv") + add_files("apps/desktop/src/screen_capturer/captured_nv12_frame.cpp") add_files("apps/desktop/src/platform/windows/screen_capturer/screen_capturer_dxgi.cpp", "apps/desktop/src/platform/windows/screen_capturer/screen_capturer_gdi.cpp", "apps/desktop/src/platform/windows/screen_capturer/screen_capturer_win.cpp", diff --git a/deps/submodules/minirtc b/deps/submodules/minirtc index 5f8c4f90..454a0c54 160000 --- a/deps/submodules/minirtc +++ b/deps/submodules/minirtc @@ -1 +1 @@ -Subproject commit 5f8c4f90f2069094aafe51e39552748a65177758 +Subproject commit 454a0c544a104dc2a0ddb51f081a116e7b71a358