From 16348444f05855ef4dec6f6393635708a5a31cdf Mon Sep 17 00:00:00 2001 From: dijunkun Date: Thu, 3 Sep 2026 01:53:34 +0800 Subject: [PATCH] [feat] add Windows native video rendering pipeline --- .../gui/rendering/slint_video_presenter.cpp | 14 +- apps/desktop/src/gui/runtime/gui_runtime.cpp | 8 +- .../src/gui/runtime/peer_media_callbacks.cpp | 71 +- .../common/gui/opengl_video_renderer.cpp | 914 ++++++++++++++++-- .../common/gui/opengl_video_renderer.h | 16 +- apps/desktop/src/platform/video_renderer.h | 6 + .../tests/video_receive_scheduler_test.cpp | 200 ++++ .../tests/video_rtp_timestamp_test.cpp | 84 ++ apps/desktop/xmake/targets.lua | 12 + deps/submodules/minirtc | 2 +- 10 files changed, 1210 insertions(+), 117 deletions(-) create mode 100644 apps/desktop/tests/video_receive_scheduler_test.cpp create mode 100644 apps/desktop/tests/video_rtp_timestamp_test.cpp diff --git a/apps/desktop/src/gui/rendering/slint_video_presenter.cpp b/apps/desktop/src/gui/rendering/slint_video_presenter.cpp index 684b14d8..29a2f17e 100644 --- a/apps/desktop/src/gui/rendering/slint_video_presenter.cpp +++ b/apps/desktop/src/gui/rendering/slint_video_presenter.cpp @@ -119,9 +119,21 @@ struct SlintVideoPresenter::Impl { return; } - std::lock_guard lock(gl_mutex); auto* opengl_renderer = dynamic_cast(&renderer); + if (state == slint::RenderingState::AfterRendering && stream && + opengl_renderer && + (*stream)->get_native_video_enabled() && + opengl_renderer->ShouldContinueRendering()) { + // Queue the next frame directly from the display callback. A + // 16/17 ms polling timer can drift across vsync and make Windows + // coalesce every second request, effectively capping video near + // 30 fps even when decode and submission stay near 60 fps. + (*stream)->window().request_redraw(); + return; + } + + std::lock_guard lock(gl_mutex); if (state == slint::RenderingState::RenderingSetup) { GLuint texture = 0; glGenTextures(1, &texture); diff --git a/apps/desktop/src/gui/runtime/gui_runtime.cpp b/apps/desktop/src/gui/runtime/gui_runtime.cpp index f832d7f7..96715c4c 100644 --- a/apps/desktop/src/gui/runtime/gui_runtime.cpp +++ b/apps/desktop/src/gui/runtime/gui_runtime.cpp @@ -141,9 +141,13 @@ int GuiRuntime::CreateConnectionPeer() { sizeof(params_.log_path) - 1); params_.log_path[sizeof(params_.log_path) - 1] = '\0'; params_.hardware_acceleration = config_center_->IsHardwareVideoCodec(); - // The Slint desktop renderer currently consumes packed CPU frames. Native - // renderers can opt into platform-native frame output independently. +#if defined(_WIN32) + // Windows accepts ref-counted CPU NV12 and CUDA device NV12. The OpenGL + // renderer retains the decoded frame until the UI thread has uploaded it. + params_.native_video_output = true; +#else params_.native_video_output = false; +#endif params_.av1_encoding = config_center_->GetVideoEncodeFormat() == ConfigCenter::VIDEO_ENCODE_FORMAT::AV1 ? true diff --git a/apps/desktop/src/gui/runtime/peer_media_callbacks.cpp b/apps/desktop/src/gui/runtime/peer_media_callbacks.cpp index 606adcce..dc01ef6b 100644 --- a/apps/desktop/src/gui/runtime/peer_media_callbacks.cpp +++ b/apps/desktop/src/gui/runtime/peer_media_callbacks.cpp @@ -14,6 +14,23 @@ namespace { constexpr auto kBackgroundSnapshotInterval = std::chrono::seconds(1); +#if defined(_WIN32) +const XWindowsVideoFrame* GetWindowsNativeFrame(const XVideoFrame* frame) { + if (!frame || + frame->native_handle_type != XVideoFrameNativeHandleWindowsNv12 || + !frame->native_handle) { + return nullptr; + } + const auto* native = + static_cast(frame->native_handle); + return native->struct_size >= + static_cast(sizeof(XWindowsVideoFrame)) && + native->owner && native->copy_to_cpu + ? native + : nullptr; +} +#endif + } // namespace void PeerEventHandler::OnReceiveVideoBuffer( @@ -38,9 +55,19 @@ void PeerEventHandler::OnReceiveVideoBuffer( bool background_snapshot_only = false; auto* native_renderer = runtime->video_renderer_.get(); if (native_renderer && native_renderer->IsActive()) { - const auto submit_result = native_renderer->SubmitNv12( - remote_id, reinterpret_cast(video_frame->data), - video_frame->size, video_frame->width, video_frame->height); + VideoRenderer::SubmitResult submit_result = + VideoRenderer::SubmitResult::failed; +#if defined(_WIN32) + if (GetWindowsNativeFrame(video_frame)) { + submit_result = + native_renderer->SubmitNativeFrame(remote_id, *video_frame); + } else +#endif + { + submit_result = native_renderer->SubmitNv12( + remote_id, reinterpret_cast(video_frame->data), + video_frame->size, video_frame->width, video_frame->height); + } if (submit_result == VideoRenderer::SubmitResult::submitted) { std::lock_guard lock(props->video_frame_mutex_); const bool size_changed = (props->video_width_ != video_frame->width) || @@ -88,17 +115,43 @@ void PeerEventHandler::OnReceiveVideoBuffer( return; } + size_t frame_size = video_frame->size; +#if defined(_WIN32) + const XWindowsVideoFrame* windows_native_frame = + GetWindowsNativeFrame(video_frame); + if (windows_native_frame) { + frame_size = static_cast(windows_native_frame->width) * + windows_native_frame->height * 3U / 2U; + } +#endif + if (frame_size == 0) { + return; + } + // Allocate a third buffer only while the UI still owns the old snapshot. if (!props->back_frame_ || props->back_frame_.use_count() != 1) { props->back_frame_ = - std::make_shared>(video_frame->size); + std::make_shared>(frame_size); } - if (props->back_frame_->size() != video_frame->size) { - props->back_frame_->resize(video_frame->size); + if (props->back_frame_->size() != frame_size) { + props->back_frame_->resize(frame_size); } - std::memcpy(props->back_frame_->data(), video_frame->data, - video_frame->size); +#if defined(_WIN32) + if (windows_native_frame) { + if (windows_native_frame->copy_to_cpu( + windows_native_frame->owner, props->back_frame_->data(), + props->back_frame_->size()) != 0) { + return; + } + } else +#endif + { + if (!video_frame->data || video_frame->size < frame_size) { + return; + } + std::memcpy(props->back_frame_->data(), video_frame->data, frame_size); + } const bool size_changed = (props->video_width_ != video_frame->width) || (props->video_height_ != video_frame->height); @@ -108,7 +161,7 @@ void PeerEventHandler::OnReceiveVideoBuffer( props->video_width_ = video_frame->width; props->video_height_ = video_frame->height; - props->video_size_ = video_frame->size; + props->video_size_ = frame_size; props->front_frame_.swap(props->back_frame_); props->thumbnail_frame_ = props->front_frame_; diff --git a/apps/desktop/src/platform/common/gui/opengl_video_renderer.cpp b/apps/desktop/src/platform/common/gui/opengl_video_renderer.cpp index e5775b3f..9d435325 100644 --- a/apps/desktop/src/platform/common/gui/opengl_video_renderer.cpp +++ b/apps/desktop/src/platform/common/gui/opengl_video_renderer.cpp @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include #include #include @@ -26,11 +28,88 @@ #include #include "rd_log.h" +#if defined(_WIN32) && USE_CUDA +#include "nvcodec_api.h" +#endif namespace crossdesk { namespace { constexpr size_t kFrameSlotCount = 3; +constexpr size_t kMaxPendingFrames = 2; +constexpr auto kActiveRenderWindow = std::chrono::milliseconds(250); + +#if defined(_WIN32) +class WindowsNativeFrameRef { +public: + WindowsNativeFrameRef() = default; + + explicit WindowsNativeFrameRef(const XWindowsVideoFrame *frame) + : frame_(frame) { + if (frame_ && frame_->owner && frame_->retain) { + frame_->retain(frame_->owner); + } else { + frame_ = nullptr; + } + } + + WindowsNativeFrameRef(const WindowsNativeFrameRef &other) + : WindowsNativeFrameRef(other.frame_) {} + + WindowsNativeFrameRef(WindowsNativeFrameRef &&other) noexcept + : frame_(std::exchange(other.frame_, nullptr)) {} + + WindowsNativeFrameRef &operator=(WindowsNativeFrameRef other) noexcept { + Swap(other); + return *this; + } + + ~WindowsNativeFrameRef() { Reset(); } + + void Reset() { + if (frame_ && frame_->owner && frame_->release) { + frame_->release(frame_->owner); + } + frame_ = nullptr; + } + + void Swap(WindowsNativeFrameRef &other) noexcept { + std::swap(frame_, other.frame_); + } + + const XWindowsVideoFrame *Get() const { return frame_; } + explicit operator bool() const { return frame_ != nullptr; } + +private: + const XWindowsVideoFrame *frame_ = nullptr; +}; + +const XWindowsVideoFrame *GetWindowsNativeFrame(const XVideoFrame &frame) { + if (frame.native_handle_type != XVideoFrameNativeHandleWindowsNv12 || + !frame.native_handle) { + return nullptr; + } + const auto *native = + static_cast(frame.native_handle); + if (native->struct_size < static_cast(sizeof(XWindowsVideoFrame)) || + !native->owner || !native->retain || !native->release || + !native->copy_to_cpu || native->width == 0 || native->height == 0 || + (native->width & 1U) != 0 || (native->height & 1U) != 0 || + native->y_stride < native->width || native->uv_stride < native->width) { + return nullptr; + } + if (native->memory_type == XWindowsVideoFrameMemoryCpu) { + return native->y_plane && native->uv_plane ? native : nullptr; + } + if (native->memory_type == XWindowsVideoFrameMemoryCuda) { + return native->y_device_pointer != 0 && native->uv_device_pointer != 0 && + native->device_context + ? native + : nullptr; + } + return nullptr; +} +#endif enum class SlotUse { available, @@ -40,6 +119,10 @@ enum class SlotUse { struct FrameSlot { std::vector bytes; +#if defined(_WIN32) + WindowsNativeFrameRef native_frame; +#endif + std::chrono::steady_clock::time_point submitted_at; int width = 0; int height = 0; std::string remote_id; @@ -55,6 +138,76 @@ struct SharedFrameState { uint64_t next_sequence = 1; }; +FrameSlot *SelectSubmissionSlot(SharedFrameState &frames, + std::string_view remote_id, + bool replace_pending, + VideoRenderer::SubmitResult *result) { + if (frames.selected_stream != remote_id) { + *result = VideoRenderer::SubmitResult::not_selected; + return nullptr; + } + + FrameSlot *target = nullptr; + if (replace_pending) { + size_t pending_count = 0; + FrameSlot *oldest_pending = nullptr; + for (auto &slot : frames.slots) { + if (slot.use == SlotUse::pending && slot.remote_id == remote_id) { + ++pending_count; + if (!oldest_pending || slot.sequence < oldest_pending->sequence) { + oldest_pending = &slot; + } + } + } + // Keep a two-frame jitter queue. A second frame that arrives before the + // next display callback is preserved instead of replacing the first one. + // Once the queue is full, replace its oldest frame to keep latency bounded. + if (pending_count >= kMaxPendingFrames) { + target = oldest_pending; + } + } else { + for (const auto &slot : frames.slots) { + if (slot.valid && slot.remote_id == remote_id && + (slot.use == SlotUse::pending || slot.use == SlotUse::uploading)) { + *result = VideoRenderer::SubmitResult::dropped; + return nullptr; + } + } + } + if (!target) { + uint64_t oldest_sequence = std::numeric_limits::max(); + for (auto &slot : frames.slots) { + if (slot.use != SlotUse::available) { + continue; + } + if (!slot.valid) { + target = &slot; + break; + } + if (slot.sequence < oldest_sequence) { + oldest_sequence = slot.sequence; + target = &slot; + } + } + } + if (!target && replace_pending) { + // An upload can temporarily own one slot. If no available slot remains, + // prefer replacing the oldest queued frame over blocking the decode thread. + for (auto &slot : frames.slots) { + if (slot.use == SlotUse::pending && slot.remote_id == remote_id && + (!target || slot.sequence < target->sequence)) { + target = &slot; + } + } + } + if (!target) { + *result = VideoRenderer::SubmitResult::dropped; + return nullptr; + } + *result = VideoRenderer::SubmitResult::submitted; + return target; +} + template bool LoadOpenGlFunction(T *function, const char *name) { #if defined(_WIN32) PROC address = wglGetProcAddress(name); @@ -248,6 +401,388 @@ struct OpenGlVideoRenderer::Impl { int texture_height = 0; std::string uploaded_stream; uint64_t uploaded_sequence = 0; + GLuint cuda_upload_buffer = 0; + std::atomic submitted_frame_total{0}; + std::atomic presented_frame_total{0}; + std::atomic superseded_frame_total{0}; + std::atomic dropped_frame_total{0}; + std::atomic failed_submit_total{0}; + std::atomic render_failed_total{0}; + std::atomic not_selected_total{0}; + std::atomic last_submit_time_ns{0}; + uint64_t render_callback_total = 0; + uint64_t render_time_total_us = 0; + uint64_t upload_sample_total = 0; + uint64_t upload_time_total_us = 0; + uint64_t queue_wait_sample_total = 0; + uint64_t queue_wait_time_total_us = 0; + uint64_t render_gap_sample_total = 0; + uint64_t render_gap_time_total_us = 0; + uint64_t render_time_window_max_us = 0; + uint64_t upload_time_window_max_us = 0; + uint64_t queue_wait_window_max_us = 0; + size_t queue_depth_window_max = 0; + uint64_t render_gap_window_min_us = std::numeric_limits::max(); + uint64_t render_gap_window_max_us = 0; + std::chrono::steady_clock::time_point last_render_started; + std::chrono::steady_clock::time_point pipeline_window_started; + uint64_t pipeline_window_submit_start = 0; + uint64_t pipeline_window_present_start = 0; + uint64_t pipeline_window_render_start = 0; + uint64_t pipeline_window_render_time_start_us = 0; + uint64_t pipeline_window_upload_sample_start = 0; + uint64_t pipeline_window_upload_time_start_us = 0; + uint64_t pipeline_window_queue_wait_sample_start = 0; + uint64_t pipeline_window_queue_wait_time_start_us = 0; + uint64_t pipeline_window_render_gap_sample_start = 0; + uint64_t pipeline_window_render_gap_time_start_us = 0; + + void ResetVideoPipeline() { + submitted_frame_total.store(0, std::memory_order_relaxed); + presented_frame_total.store(0, std::memory_order_relaxed); + superseded_frame_total.store(0, std::memory_order_relaxed); + dropped_frame_total.store(0, std::memory_order_relaxed); + failed_submit_total.store(0, std::memory_order_relaxed); + render_failed_total.store(0, std::memory_order_relaxed); + not_selected_total.store(0, std::memory_order_relaxed); + last_submit_time_ns.store(0, std::memory_order_relaxed); + render_callback_total = 0; + render_time_total_us = 0; + upload_sample_total = 0; + upload_time_total_us = 0; + queue_wait_sample_total = 0; + queue_wait_time_total_us = 0; + render_gap_sample_total = 0; + render_gap_time_total_us = 0; + render_time_window_max_us = 0; + upload_time_window_max_us = 0; + queue_wait_window_max_us = 0; + queue_depth_window_max = 0; + render_gap_window_min_us = std::numeric_limits::max(); + render_gap_window_max_us = 0; + last_render_started = {}; + pipeline_window_started = {}; + pipeline_window_submit_start = 0; + pipeline_window_present_start = 0; + pipeline_window_render_start = 0; + pipeline_window_render_time_start_us = 0; + pipeline_window_upload_sample_start = 0; + pipeline_window_upload_time_start_us = 0; + pipeline_window_queue_wait_sample_start = 0; + pipeline_window_queue_wait_time_start_us = 0; + pipeline_window_render_gap_sample_start = 0; + pipeline_window_render_gap_time_start_us = 0; + } + + void RecordRenderStart(std::chrono::steady_clock::time_point now) { + if (last_render_started != std::chrono::steady_clock::time_point{}) { + const uint64_t gap_us = static_cast( + std::chrono::duration_cast( + now - last_render_started) + .count()); + ++render_gap_sample_total; + render_gap_time_total_us += gap_us; + render_gap_window_min_us = std::min(render_gap_window_min_us, gap_us); + render_gap_window_max_us = std::max(render_gap_window_max_us, gap_us); + } + last_render_started = now; + } + + void MaybeLogVideoPipeline(std::string_view remote_id) { + const auto now = std::chrono::steady_clock::now(); + const uint64_t submitted = + submitted_frame_total.load(std::memory_order_relaxed); + const uint64_t presented = + presented_frame_total.load(std::memory_order_relaxed); + if (pipeline_window_started == std::chrono::steady_clock::time_point{}) { + pipeline_window_started = now; + pipeline_window_submit_start = 0; + pipeline_window_present_start = 0; + return; + } + + const auto elapsed_ms = + std::chrono::duration_cast( + now - pipeline_window_started) + .count(); + if (elapsed_ms < 1000) { + return; + } + + const uint64_t submit_delta = submitted - pipeline_window_submit_start; + const uint64_t present_delta = presented - pipeline_window_present_start; + const uint64_t render_delta = + render_callback_total - pipeline_window_render_start; + const uint64_t render_time_delta_us = + render_time_total_us - pipeline_window_render_time_start_us; + const uint64_t upload_sample_delta = + upload_sample_total - pipeline_window_upload_sample_start; + const uint64_t upload_time_delta_us = + upload_time_total_us - pipeline_window_upload_time_start_us; + const uint64_t queue_wait_sample_delta = + queue_wait_sample_total - pipeline_window_queue_wait_sample_start; + const uint64_t queue_wait_time_delta_us = + queue_wait_time_total_us - pipeline_window_queue_wait_time_start_us; + const uint64_t render_gap_sample_delta = + render_gap_sample_total - pipeline_window_render_gap_sample_start; + const uint64_t render_gap_time_delta_us = + render_gap_time_total_us - pipeline_window_render_gap_time_start_us; + const uint64_t submit_fps = submit_delta * 1000 / elapsed_ms; + const uint64_t present_fps = present_delta * 1000 / elapsed_ms; + const uint64_t render_fps = render_delta * 1000 / elapsed_ms; + const uint64_t render_avg_us = + render_delta > 0 ? render_time_delta_us / render_delta : 0; + const uint64_t upload_avg_us = upload_sample_delta > 0 + ? upload_time_delta_us / upload_sample_delta + : 0; + const uint64_t queue_wait_avg_us = + queue_wait_sample_delta > 0 + ? queue_wait_time_delta_us / queue_wait_sample_delta + : 0; + const uint64_t render_gap_avg_us = + render_gap_sample_delta > 0 + ? render_gap_time_delta_us / render_gap_sample_delta + : 0; + const uint64_t render_gap_min_us = + render_gap_window_min_us == std::numeric_limits::max() + ? 0 + : render_gap_window_min_us; + LOG_INFO("VIDEO_PIPELINE_RENDER stream=[{}] interval_ms={} submit_fps={} " + "present_fps={} submit_total={} present_total={} " + "superseded_total={} dropped_total={} failed_submit_total={} " + "render_failed_total={} not_selected_total={} render_fps={} " + "render_avg_us={} render_max_us={} upload_avg_us={} " + "upload_max_us={} queue_wait_avg_us={} queue_wait_max_us={} " + "queue_depth_max={} render_gap_avg_us={} render_gap_min_us={} " + "render_gap_max_us={}", + remote_id, elapsed_ms, submit_fps, present_fps, submitted, + presented, superseded_frame_total.load(std::memory_order_relaxed), + dropped_frame_total.load(std::memory_order_relaxed), + failed_submit_total.load(std::memory_order_relaxed), + render_failed_total.load(std::memory_order_relaxed), + not_selected_total.load(std::memory_order_relaxed), render_fps, + render_avg_us, render_time_window_max_us, upload_avg_us, + upload_time_window_max_us, queue_wait_avg_us, + queue_wait_window_max_us, queue_depth_window_max, + render_gap_avg_us, render_gap_min_us, render_gap_window_max_us); + + pipeline_window_started = now; + pipeline_window_submit_start = submitted; + pipeline_window_present_start = presented; + pipeline_window_render_start = render_callback_total; + pipeline_window_render_time_start_us = render_time_total_us; + pipeline_window_upload_sample_start = upload_sample_total; + pipeline_window_upload_time_start_us = upload_time_total_us; + pipeline_window_queue_wait_sample_start = queue_wait_sample_total; + pipeline_window_queue_wait_time_start_us = queue_wait_time_total_us; + pipeline_window_render_gap_sample_start = render_gap_sample_total; + pipeline_window_render_gap_time_start_us = render_gap_time_total_us; + render_time_window_max_us = 0; + upload_time_window_max_us = 0; + queue_wait_window_max_us = 0; + queue_depth_window_max = 0; + render_gap_window_min_us = std::numeric_limits::max(); + render_gap_window_max_us = 0; + } + + VideoRenderer::SubmitResult + RecordSubmitResult(std::string_view remote_id, + VideoRenderer::SubmitResult result, + bool superseded = false) { + switch (result) { + case VideoRenderer::SubmitResult::submitted: + submitted_frame_total.fetch_add(1, std::memory_order_relaxed); + last_submit_time_ns.store( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(), + std::memory_order_release); + if (superseded) { + superseded_frame_total.fetch_add(1, std::memory_order_relaxed); + } + break; + case VideoRenderer::SubmitResult::dropped: + dropped_frame_total.fetch_add(1, std::memory_order_relaxed); + break; + case VideoRenderer::SubmitResult::failed: + failed_submit_total.fetch_add(1, std::memory_order_relaxed); + break; + case VideoRenderer::SubmitResult::not_selected: + not_selected_total.fetch_add(1, std::memory_order_relaxed); + break; + } + return result; + } + + void RecordRenderResult(std::string_view remote_id, bool presented_new_frame, + bool failed_new_frame, uint64_t render_time_us, + uint64_t upload_time_us, uint64_t queue_wait_us, + size_t queue_depth, bool sampled_upload) { + ++render_callback_total; + render_time_total_us += render_time_us; + render_time_window_max_us = + std::max(render_time_window_max_us, render_time_us); + if (sampled_upload) { + ++upload_sample_total; + upload_time_total_us += upload_time_us; + upload_time_window_max_us = + std::max(upload_time_window_max_us, upload_time_us); + ++queue_wait_sample_total; + queue_wait_time_total_us += queue_wait_us; + queue_wait_window_max_us = + std::max(queue_wait_window_max_us, queue_wait_us); + queue_depth_window_max = std::max(queue_depth_window_max, queue_depth); + } + if (presented_new_frame) { + presented_frame_total.fetch_add(1, std::memory_order_relaxed); + } + if (failed_new_frame) { + render_failed_total.fetch_add(1, std::memory_order_relaxed); + } + MaybeLogVideoPipeline(remote_id); + } + +#if defined(_WIN32) && USE_CUDA + size_t cuda_upload_capacity = 0; + CUgraphicsResource cuda_upload_resource = nullptr; + CUcontext cuda_context = nullptr; + WindowsNativeFrameRef cuda_context_owner; + bool cuda_interop_disabled = false; + bool cuda_fallback_logged = false; + bool cuda_native_logged = false; + + void ReleaseCudaRegistration() { + if (cuda_upload_resource && cuda_context && + minirtc::cuCtxPushCurrent_ld(cuda_context) == CUDA_SUCCESS) { + const CUresult unregister_result = + minirtc::cuGraphicsUnregisterResource_ld(cuda_upload_resource); + CUcontext popped_context = nullptr; + const CUresult pop_result = minirtc::cuCtxPopCurrent_ld(&popped_context); + if (unregister_result != CUDA_SUCCESS || pop_result != CUDA_SUCCESS) { + LOG_WARN("CUDA/OpenGL unregister failed, cuda={}, pop={}", + static_cast(unregister_result), + static_cast(pop_result)); + } + } + cuda_upload_resource = nullptr; + cuda_context = nullptr; + cuda_context_owner.Reset(); + } + + bool EnsureCudaUploadBuffer(const XWindowsVideoFrame &frame) { + if (cuda_interop_disabled || !frame.device_context || frame.size == 0 || + minirtc::LoadCudaGraphicsInterop() != 0) { + cuda_interop_disabled = true; + return false; + } + + auto frame_context = static_cast(frame.device_context); + if (cuda_upload_resource && cuda_context != frame_context) { + ReleaseCudaRegistration(); + } + + if (cuda_upload_buffer == 0) { + gl.gen_buffers(1, &cuda_upload_buffer); + } + if (cuda_upload_buffer == 0) { + cuda_interop_disabled = true; + return false; + } + + if (cuda_upload_capacity < frame.size) { + ReleaseCudaRegistration(); + GLint previous_unpack_buffer = 0; + glGetIntegerv(GL_PIXEL_UNPACK_BUFFER_BINDING, &previous_unpack_buffer); + gl.bind_buffer(GL_PIXEL_UNPACK_BUFFER, cuda_upload_buffer); + gl.buffer_data(GL_PIXEL_UNPACK_BUFFER, + static_cast(frame.size), nullptr, + GL_STREAM_DRAW); + gl.bind_buffer(GL_PIXEL_UNPACK_BUFFER, + static_cast(previous_unpack_buffer)); + if (glGetError() != GL_NO_ERROR) { + cuda_interop_disabled = true; + return false; + } + cuda_upload_capacity = frame.size; + } + + if (!cuda_upload_resource) { + if (minirtc::cuCtxPushCurrent_ld(frame_context) != CUDA_SUCCESS) { + return false; + } + const CUresult register_result = minirtc::cuGraphicsGLRegisterBuffer_ld( + &cuda_upload_resource, cuda_upload_buffer, + CU_GRAPHICS_REGISTER_FLAGS_WRITE_DISCARD); + CUcontext popped_context = nullptr; + const CUresult pop_result = minirtc::cuCtxPopCurrent_ld(&popped_context); + if (register_result != CUDA_SUCCESS || pop_result != CUDA_SUCCESS) { + cuda_upload_resource = nullptr; + cuda_interop_disabled = true; + LOG_WARN("CUDA/OpenGL buffer registration failed, cuda={}, pop={}", + static_cast(register_result), + static_cast(pop_result)); + return false; + } + cuda_context = frame_context; + } + return true; + } + + bool CopyCudaFrameToUploadBuffer(const WindowsNativeFrameRef &native_frame) { + const XWindowsVideoFrame *frame = native_frame.Get(); + if (!frame || !EnsureCudaUploadBuffer(*frame)) { + return false; + } + cuda_context_owner = native_frame; + + if (minirtc::cuCtxPushCurrent_ld(cuda_context) != CUDA_SUCCESS) { + return false; + } + + CUgraphicsResource resource = cuda_upload_resource; + CUresult result = minirtc::cuGraphicsMapResources_ld(1, &resource, nullptr); + bool mapped = result == CUDA_SUCCESS; + CUdeviceptr destination = 0; + size_t mapped_size = 0; + if (mapped) { + result = minirtc::cuGraphicsResourceGetMappedPointer_ld( + &destination, &mapped_size, cuda_upload_resource); + } + if (result == CUDA_SUCCESS && mapped_size < frame->size) { + result = CUDA_ERROR_INVALID_VALUE; + } + if (result == CUDA_SUCCESS) { + CUDA_MEMCPY2D copy{}; + copy.srcMemoryType = CU_MEMORYTYPE_DEVICE; + copy.srcDevice = static_cast(frame->y_device_pointer); + copy.srcPitch = frame->y_stride; + copy.dstMemoryType = CU_MEMORYTYPE_DEVICE; + copy.dstDevice = destination; + copy.dstPitch = frame->width; + copy.WidthInBytes = frame->width; + copy.Height = frame->height; + result = minirtc::cuMemcpy2DAsync_ld(©, nullptr); + if (result == CUDA_SUCCESS) { + copy.srcDevice = static_cast(frame->uv_device_pointer); + copy.srcPitch = frame->uv_stride; + copy.dstDevice = destination + + static_cast(frame->width) * frame->height; + copy.Height = frame->height / 2U; + result = minirtc::cuMemcpy2DAsync_ld(©, nullptr); + } + } + if (mapped) { + const CUresult unmap_result = + minirtc::cuGraphicsUnmapResources_ld(1, &resource, nullptr); + if (result == CUDA_SUCCESS) { + result = unmap_result; + } + } + CUcontext popped_context = nullptr; + const CUresult pop_result = minirtc::cuCtxPopCurrent_ld(&popped_context); + return result == CUDA_SUCCESS && pop_result == CUDA_SUCCESS; + } +#endif void ResetGlHandles() { program = 0; @@ -269,8 +804,7 @@ struct OpenGlVideoRenderer::Impl { } }; -OpenGlVideoRenderer::OpenGlVideoRenderer() - : impl_(std::make_unique()) {} +OpenGlVideoRenderer::OpenGlVideoRenderer() : impl_(std::make_unique()) {} OpenGlVideoRenderer::~OpenGlVideoRenderer() = default; @@ -456,8 +990,7 @@ void main() { if (impl_->position_location < 0 || impl_->texcoord_location < 0 || impl_->y_texture_location < 0 || impl_->uv_texture_location < 0 || impl_->target_size_location < 0 || impl_->video_rect_location < 0 || - impl_->corner_radius_location < 0 || - impl_->video_enabled_location < 0) { + impl_->corner_radius_location < 0 || impl_->video_enabled_location < 0) { LOG_ERROR("OpenGL NV12 program is missing required shader bindings"); impl_->gl.delete_program(impl_->program); impl_->ResetGlHandles(); @@ -524,6 +1057,17 @@ void main() { void OpenGlVideoRenderer::Teardown() { impl_->ready.store(false, std::memory_order_release); +#if defined(_WIN32) && USE_CUDA + impl_->ReleaseCudaRegistration(); + if (impl_->cuda_upload_buffer != 0 && impl_->gl.delete_buffers) { + impl_->gl.delete_buffers(1, &impl_->cuda_upload_buffer); + } + impl_->cuda_upload_buffer = 0; + impl_->cuda_upload_capacity = 0; + impl_->cuda_interop_disabled = false; + impl_->cuda_fallback_logged = false; + impl_->cuda_native_logged = false; +#endif if (impl_->y_texture != 0) glDeleteTextures(1, &impl_->y_texture); if (impl_->uv_texture != 0) @@ -534,6 +1078,17 @@ void OpenGlVideoRenderer::Teardown() { if (impl_->program != 0 && impl_->gl.delete_program) { impl_->gl.delete_program(impl_->program); } + { + std::lock_guard lock(impl_->frames->mutex); + for (auto &slot : impl_->frames->slots) { + slot.bytes.clear(); +#if defined(_WIN32) + slot.native_frame.Reset(); +#endif + slot.valid = false; + slot.use = SlotUse::available; + } + } impl_->ResetGlHandles(); } @@ -543,6 +1098,25 @@ bool OpenGlVideoRenderer::IsReady() const { bool OpenGlVideoRenderer::IsActive() const { return IsReady(); } +bool OpenGlVideoRenderer::ShouldContinueRendering() const { + if (!IsReady()) { + return false; + } + const int64_t last_submit_ns = + impl_->last_submit_time_ns.load(std::memory_order_acquire); + if (last_submit_ns == 0) { + return false; + } + const int64_t now_ns = + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + return now_ns - last_submit_ns <= + std::chrono::duration_cast( + kActiveRenderWindow) + .count(); +} + bool OpenGlVideoRenderer::SetSelectedStream(std::string remote_id) { std::lock_guard lock(impl_->frames->mutex); if (impl_->frames->selected_stream == remote_id) { @@ -555,6 +1129,7 @@ bool OpenGlVideoRenderer::SetSelectedStream(std::string remote_id) { slot.use = SlotUse::available; } } + impl_->ResetVideoPipeline(); return true; } @@ -565,6 +1140,10 @@ void OpenGlVideoRenderer::DiscardStream(std::string_view remote_id) { slot.valid = false; slot.use = SlotUse::available; slot.remote_id.clear(); + slot.bytes.clear(); +#if defined(_WIN32) + slot.native_frame.Reset(); +#endif } } if (impl_->frames->selected_stream == remote_id) { @@ -573,113 +1152,143 @@ void OpenGlVideoRenderer::DiscardStream(std::string_view remote_id) { } OpenGlVideoRenderer::SubmitResult -OpenGlVideoRenderer::SubmitNv12(std::string_view remote_id, - const uint8_t *data, size_t size, - int width, int height) { +OpenGlVideoRenderer::SubmitNv12(std::string_view remote_id, const uint8_t *data, + size_t size, int width, int height) { return SubmitNv12Internal(remote_id, data, size, width, height, true); } OpenGlVideoRenderer::SubmitResult OpenGlVideoRenderer::SubmitCachedNv12(std::string_view remote_id, - const uint8_t *data, size_t size, - int width, int height) { + const uint8_t *data, size_t size, + int width, int height) { return SubmitNv12Internal(remote_id, data, size, width, height, false); } -OpenGlVideoRenderer::SubmitResult -OpenGlVideoRenderer::SubmitNv12Internal(std::string_view remote_id, - const uint8_t *data, size_t size, - int width, int height, - bool replace_pending) { +OpenGlVideoRenderer::SubmitResult OpenGlVideoRenderer::SubmitNv12Internal( + std::string_view remote_id, const uint8_t *data, size_t size, int width, + int height, bool replace_pending) { if (!IsReady()) { - return SubmitResult::failed; + return impl_->RecordSubmitResult(remote_id, SubmitResult::failed); } if (data == nullptr || width <= 0 || height <= 0 || (width & 1) != 0 || (height & 1) != 0) { - return SubmitResult::failed; + return impl_->RecordSubmitResult(remote_id, SubmitResult::failed); } const size_t y_size = static_cast(width) * height; const size_t required_size = y_size + y_size / 2; if (size < required_size) { - return SubmitResult::failed; + return impl_->RecordSubmitResult(remote_id, SubmitResult::failed); } auto &frames = *impl_->frames; std::lock_guard lock(frames.mutex); - if (frames.selected_stream != remote_id) { - return SubmitResult::not_selected; + if (!IsReady()) { + return impl_->RecordSubmitResult(remote_id, SubmitResult::failed); } - - FrameSlot *target = nullptr; - if (replace_pending) { - for (auto &slot : frames.slots) { - if (slot.use == SlotUse::pending) { - target = &slot; - break; - } - } - } else { - for (const auto &slot : frames.slots) { - if (slot.valid && slot.remote_id == remote_id && - (slot.use == SlotUse::pending || slot.use == SlotUse::uploading)) { - return SubmitResult::dropped; - } - } - } - if (target == nullptr) { - uint64_t oldest_sequence = std::numeric_limits::max(); - for (auto &slot : frames.slots) { - if (slot.use != SlotUse::available) { - continue; - } - if (!slot.valid) { - target = &slot; - break; - } - if (slot.sequence < oldest_sequence) { - oldest_sequence = slot.sequence; - target = &slot; - } - } - } - if (target == nullptr) { - return SubmitResult::dropped; + SubmitResult selection_result = SubmitResult::failed; + FrameSlot *target = SelectSubmissionSlot(frames, remote_id, replace_pending, + &selection_result); + if (!target) { + return impl_->RecordSubmitResult(remote_id, selection_result); } + const bool superseded = target->use == SlotUse::pending; target->bytes.resize(required_size); std::memcpy(target->bytes.data(), data, required_size); +#if defined(_WIN32) + target->native_frame.Reset(); +#endif target->width = width; target->height = height; target->remote_id.assign(remote_id); target->sequence = frames.next_sequence++; + target->submitted_at = std::chrono::steady_clock::now(); target->use = SlotUse::pending; target->valid = true; - return SubmitResult::submitted; + return impl_->RecordSubmitResult(remote_id, SubmitResult::submitted, + superseded); +} + +OpenGlVideoRenderer::SubmitResult +OpenGlVideoRenderer::SubmitNativeFrame(std::string_view remote_id, + const XVideoFrame &frame) { +#if defined(_WIN32) + if (!IsReady()) { + return impl_->RecordSubmitResult(remote_id, SubmitResult::failed); + } + const XWindowsVideoFrame *native = GetWindowsNativeFrame(frame); + if (!native || native->width > static_cast(INT_MAX) || + native->height > static_cast(INT_MAX)) { + return impl_->RecordSubmitResult(remote_id, SubmitResult::failed); + } + + auto &frames = *impl_->frames; + std::lock_guard lock(frames.mutex); + if (!IsReady()) { + return impl_->RecordSubmitResult(remote_id, SubmitResult::failed); + } + SubmitResult selection_result = SubmitResult::failed; + FrameSlot *target = + SelectSubmissionSlot(frames, remote_id, true, &selection_result); + if (!target) { + return impl_->RecordSubmitResult(remote_id, selection_result); + } + const bool superseded = target->use == SlotUse::pending; + + WindowsNativeFrameRef retained(native); + if (!retained) { + return impl_->RecordSubmitResult(remote_id, SubmitResult::failed); + } + target->bytes.clear(); + target->native_frame = std::move(retained); + target->width = static_cast(native->width); + target->height = static_cast(native->height); + target->remote_id.assign(remote_id); + target->sequence = frames.next_sequence++; + target->submitted_at = std::chrono::steady_clock::now(); + target->use = SlotUse::pending; + target->valid = true; + return impl_->RecordSubmitResult(remote_id, SubmitResult::submitted, + superseded); +#else + (void)remote_id; + (void)frame; + return SubmitResult::failed; +#endif } OpenGlVideoRenderer::RenderOutcome -OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, - int target_width, int target_height, - int top_inset_pixels, +OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, int target_width, + int target_height, int top_inset_pixels, int corner_radius_pixels) { if (!IsReady() || target_width <= 0 || target_height <= 0) { return {RenderResult::failed}; } + const auto render_started = std::chrono::steady_clock::now(); + impl_->RecordRenderStart(render_started); size_t slot_index = kFrameSlotCount; uint64_t sequence = 0; + uint64_t queue_wait_us = 0; + size_t queue_depth = 0; int source_width = 0; int source_height = 0; - const unsigned char *frame_data = nullptr; + const unsigned char *y_data = nullptr; + const unsigned char *uv_data = nullptr; +#if defined(_WIN32) + WindowsNativeFrameRef active_native_frame; +#endif { std::lock_guard lock(impl_->frames->mutex); for (size_t index = 0; index < impl_->frames->slots.size(); ++index) { const auto &slot = impl_->frames->slots[index]; if (slot.valid && slot.use == SlotUse::pending && - slot.remote_id == remote_id && - (slot_index == kFrameSlotCount || slot.sequence > sequence)) { - slot_index = index; - sequence = slot.sequence; + slot.remote_id == remote_id) { + ++queue_depth; + if (slot_index == kFrameSlotCount || slot.sequence < sequence) { + slot_index = index; + sequence = slot.sequence; + } } } if (slot_index != kFrameSlotCount) { @@ -687,13 +1296,20 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, selected.use = SlotUse::uploading; source_width = selected.width; source_height = selected.height; - frame_data = selected.bytes.data(); - for (size_t index = 0; index < impl_->frames->slots.size(); ++index) { - auto &slot = impl_->frames->slots[index]; - if (index != slot_index && slot.use == SlotUse::pending && - slot.remote_id == remote_id) { - slot.use = SlotUse::available; - } + if (selected.submitted_at != + std::chrono::steady_clock::time_point{}) { + queue_wait_us = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - selected.submitted_at) + .count()); + } +#if defined(_WIN32) + active_native_frame = selected.native_frame; +#endif + if (!selected.bytes.empty()) { + const size_t y_size = static_cast(source_width) * source_height; + y_data = selected.bytes.data(); + uv_data = selected.bytes.data() + y_size; } } } @@ -701,6 +1317,7 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, GLint previous_program = 0; GLint previous_active_texture = GL_TEXTURE0; GLint previous_array_buffer = 0; + GLint previous_unpack_buffer = 0; GLint previous_unpack_alignment = 4; GLint previous_viewport[4] = {}; GLint previous_scissor_box[4] = {}; @@ -709,6 +1326,7 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, glGetIntegerv(GL_CURRENT_PROGRAM, &previous_program); glGetIntegerv(GL_ACTIVE_TEXTURE, &previous_active_texture); glGetIntegerv(GL_ARRAY_BUFFER_BINDING, &previous_array_buffer); + glGetIntegerv(GL_PIXEL_UNPACK_BUFFER_BINDING, &previous_unpack_buffer); glGetIntegerv(GL_UNPACK_ALIGNMENT, &previous_unpack_alignment); glGetIntegerv(GL_VIEWPORT, previous_viewport); glGetIntegerv(GL_SCISSOR_BOX, previous_scissor_box); @@ -746,35 +1364,93 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, glClear(GL_COLOR_BUFFER_BIT); bool upload_succeeded = true; + uint64_t upload_time_us = 0; if (slot_index != kFrameSlotCount) { + const auto upload_started = std::chrono::steady_clock::now(); while (glGetError() != GL_NO_ERROR) { } const size_t y_size = static_cast(source_width) * source_height; + const size_t required_size = y_size + y_size / 2U; + bool use_pixel_unpack_buffer = false; + std::vector cpu_fallback; +#if defined(_WIN32) + if (const XWindowsVideoFrame *native = active_native_frame.Get()) { + if (native->memory_type == XWindowsVideoFrameMemoryCpu && + native->y_stride == native->width && + native->uv_stride == native->width) { + y_data = native->y_plane; + uv_data = native->uv_plane; + } else if (native->memory_type == XWindowsVideoFrameMemoryCuda) { +#if USE_CUDA + use_pixel_unpack_buffer = + impl_->CopyCudaFrameToUploadBuffer(active_native_frame); + if (use_pixel_unpack_buffer && !impl_->cuda_native_logged) { + LOG_INFO("Windows native NV12 using CUDA/OpenGL device upload"); + impl_->cuda_native_logged = true; + } + if (!use_pixel_unpack_buffer && !impl_->cuda_fallback_logged) { + LOG_WARN("CUDA/OpenGL native upload unavailable; falling back to " + "CUDA-to-CPU copy"); + impl_->cuda_fallback_logged = true; + } +#endif + } + + if (!use_pixel_unpack_buffer && (!y_data || !uv_data)) { + cpu_fallback.resize(required_size); + if (native->copy_to_cpu(native->owner, cpu_fallback.data(), + cpu_fallback.size()) == 0) { + y_data = cpu_fallback.data(); + uv_data = cpu_fallback.data() + y_size; + } else { + upload_succeeded = false; + LOG_WARN("Windows native NV12 CPU fallback failed"); + } + } + } +#endif + if (!use_pixel_unpack_buffer && (!y_data || !uv_data)) { + upload_succeeded = false; + } + glPixelStorei(GL_UNPACK_ALIGNMENT, 1); impl_->gl.active_texture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, impl_->y_texture); impl_->gl.active_texture(GL_TEXTURE1); glBindTexture(GL_TEXTURE_2D, impl_->uv_texture); - if (impl_->texture_width != source_width || - impl_->texture_height != source_height) { + if (use_pixel_unpack_buffer) { + impl_->gl.bind_buffer(GL_PIXEL_UNPACK_BUFFER, impl_->cuda_upload_buffer); + } + const void *y_pixels = + use_pixel_unpack_buffer ? nullptr : static_cast(y_data); + const void *uv_pixels = + use_pixel_unpack_buffer + ? reinterpret_cast(static_cast(y_size)) + : static_cast(uv_data); + if (upload_succeeded && (impl_->texture_width != source_width || + impl_->texture_height != source_height)) { impl_->gl.active_texture(GL_TEXTURE0); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, source_width, source_height, - 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, frame_data); + 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, y_pixels); impl_->gl.active_texture(GL_TEXTURE1); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE_ALPHA, source_width / 2, source_height / 2, 0, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, - frame_data + y_size); - } else { + uv_pixels); + } else if (upload_succeeded) { impl_->gl.active_texture(GL_TEXTURE0); glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, source_width, source_height, - GL_LUMINANCE, GL_UNSIGNED_BYTE, frame_data); + GL_LUMINANCE, GL_UNSIGNED_BYTE, y_pixels); impl_->gl.active_texture(GL_TEXTURE1); glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, source_width / 2, source_height / 2, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, - frame_data + y_size); + uv_pixels); + } + if (use_pixel_unpack_buffer) { + impl_->gl.bind_buffer(GL_PIXEL_UNPACK_BUFFER, + static_cast(previous_unpack_buffer)); } const GLenum upload_error = glGetError(); - upload_succeeded = upload_error == GL_NO_ERROR; + upload_succeeded = upload_succeeded && upload_error == GL_NO_ERROR; if (upload_succeeded) { impl_->texture_width = source_width; impl_->texture_height = source_height; @@ -783,6 +1459,10 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, } else { LOG_WARN("OpenGL NV12 texture upload failed, error={}", upload_error); } + upload_time_us = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - upload_started) + .count()); } { @@ -797,10 +1477,10 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, const int content_y = std::clamp(top_inset_pixels, 0, target_height); const int content_height = target_height - content_y; - const bool has_video = - upload_succeeded && impl_->uploaded_stream == remote_id && - impl_->texture_width > 0 && impl_->texture_height > 0 && - content_height > 0; + const bool has_video = upload_succeeded && + impl_->uploaded_stream == remote_id && + impl_->texture_width > 0 && + impl_->texture_height > 0 && content_height > 0; int video_x = 0; int video_y = content_y; int video_width = target_width; @@ -860,9 +1540,9 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); const GLenum draw_error = glGetError(); RenderOutcome outcome{ - !upload_succeeded || draw_error != GL_NO_ERROR - ? RenderResult::failed - : has_video ? RenderResult::rendered : RenderResult::empty, + !upload_succeeded || draw_error != GL_NO_ERROR ? RenderResult::failed + : has_video ? RenderResult::rendered + : RenderResult::empty, has_video ? source_width : 0, has_video ? source_height : 0, has_video ? sequence : 0}; @@ -872,6 +1552,8 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, texcoord_state); impl_->gl.bind_buffer(GL_ARRAY_BUFFER, static_cast(previous_array_buffer)); + impl_->gl.bind_buffer(GL_PIXEL_UNPACK_BUFFER, + static_cast(previous_unpack_buffer)); impl_->gl.active_texture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, static_cast(previous_texture_0)); impl_->gl.active_texture(GL_TEXTURE1); @@ -899,30 +1581,62 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, outcome.result = RenderResult::failed; } } + const bool had_new_frame = slot_index != kFrameSlotCount; + const uint64_t render_time_us = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - render_started) + .count()); + impl_->RecordRenderResult( + remote_id, had_new_frame && outcome.result == RenderResult::rendered, + had_new_frame && outcome.result == RenderResult::failed, render_time_us, + upload_time_us, queue_wait_us, queue_depth, had_new_frame); return outcome; } -bool OpenGlVideoRenderer::CopyLatestNv12( - std::string_view remote_id, std::vector *output, int *width, - int *height) const { +bool OpenGlVideoRenderer::CopyLatestNv12(std::string_view remote_id, + std::vector *output, + int *width, int *height) const { if (!output || !width || !height) { return false; } - std::lock_guard lock(impl_->frames->mutex); - const FrameSlot *newest = nullptr; - for (const auto &slot : impl_->frames->slots) { - if (slot.valid && slot.remote_id == remote_id && - (!newest || slot.sequence > newest->sequence)) { - newest = &slot; + std::vector cpu_frame; +#if defined(_WIN32) + WindowsNativeFrameRef native_frame; +#endif + { + std::lock_guard lock(impl_->frames->mutex); + const FrameSlot *newest = nullptr; + for (const auto &slot : impl_->frames->slots) { + if (slot.valid && slot.remote_id == remote_id && + (!newest || slot.sequence > newest->sequence)) { + newest = &slot; + } } + if (!newest) { + return false; + } + *width = newest->width; + *height = newest->height; + cpu_frame = newest->bytes; +#if defined(_WIN32) + native_frame = newest->native_frame; +#endif } - if (!newest || newest->bytes.empty()) { - return false; + + if (!cpu_frame.empty()) { + *output = std::move(cpu_frame); + return true; } - *output = newest->bytes; - *width = newest->width; - *height = newest->height; - return true; +#if defined(_WIN32) + if (const XWindowsVideoFrame *native = native_frame.Get()) { + const size_t required_size = + static_cast(native->width) * native->height * 3U / 2U; + output->resize(required_size); + return native->copy_to_cpu(native->owner, output->data(), output->size()) == + 0; + } +#endif + return false; } } // namespace crossdesk diff --git a/apps/desktop/src/platform/common/gui/opengl_video_renderer.h b/apps/desktop/src/platform/common/gui/opengl_video_renderer.h index 316567d6..2f5bdb78 100644 --- a/apps/desktop/src/platform/common/gui/opengl_video_renderer.h +++ b/apps/desktop/src/platform/common/gui/opengl_video_renderer.h @@ -13,10 +13,11 @@ namespace crossdesk { // NV12 underlay for Slint's FemtoVG OpenGL renderer on Windows and Linux. -// MiniRTC submits tightly packed CPU NV12 frames from its decode callback -// thread. The Slint UI thread uploads the newest frame as Y and UV textures and -// performs color conversion and scaling in a fragment shader before Slint -// draws its overlay. +// MiniRTC submits either ref-counted Windows native NV12 frames or packed CPU +// NV12 fallback frames from its decode callback thread. CUDA-backed frames use +// CUDA/OpenGL interop; CPU-backed frames are retained without an intermediate +// queue copy. The Slint UI thread updates Y and UV textures and performs color +// conversion and scaling before Slint draws its overlay. class OpenGlVideoRenderer final : public VideoRenderer { public: OpenGlVideoRenderer(); @@ -32,6 +33,11 @@ public: bool IsReady() const override; bool IsActive() const override; + // Keeps Slint's native surface on a display-driven redraw loop while decoded + // frames are arriving. This avoids timer/vsync phase drift that otherwise + // causes every second redraw request to be coalesced on Windows. + bool ShouldContinueRendering() const; + // Only the selected stream is queued for upload. Frames for other tabs are // retained by GuiRuntime at a throttled rate for thumbnail/tab restoration. bool SetSelectedStream(std::string remote_id) override; @@ -42,6 +48,8 @@ public: size_t size, int width, int height) override; SubmitResult SubmitCachedNv12(std::string_view remote_id, const uint8_t *data, size_t size, int width, int height) override; + SubmitResult SubmitNativeFrame(std::string_view remote_id, + const XVideoFrame &frame) override; // Draws below Slint while its OpenGL context is current. All dimensions are // physical pixels in the window client area. corner_radius_pixels clips the diff --git a/apps/desktop/src/platform/video_renderer.h b/apps/desktop/src/platform/video_renderer.h index 3c06011b..d8c1e6f6 100644 --- a/apps/desktop/src/platform/video_renderer.h +++ b/apps/desktop/src/platform/video_renderer.h @@ -8,6 +8,8 @@ #include #include +#include "minirtc.h" + namespace crossdesk { // Platform-independent contract for the native stream video renderer. @@ -56,6 +58,10 @@ public: virtual SubmitResult SubmitCachedNv12(std::string_view remote_id, const uint8_t* data, size_t size, int width, int height) = 0; + virtual SubmitResult SubmitNativeFrame(std::string_view, + const XVideoFrame&) { + return SubmitResult::failed; + } virtual bool CopyLatestNv12(std::string_view remote_id, std::vector* output, int* width, diff --git a/apps/desktop/tests/video_receive_scheduler_test.cpp b/apps/desktop/tests/video_receive_scheduler_test.cpp new file mode 100644 index 00000000..67753024 --- /dev/null +++ b/apps/desktop/tests/video_receive_scheduler_test.cpp @@ -0,0 +1,200 @@ +#include +#include +#include + +#include "video_receive_scheduler.h" + +namespace { + +std::unique_ptr +MakeFrame(uint32_t rtp_timestamp, int64_t received_us, bool key_frame = false) { + const uint8_t payload = key_frame ? 0x65 : 0x41; + auto frame = + std::make_unique(&payload, sizeof(payload)); + frame->SetRtpTimestamp(rtp_timestamp); + frame->SetReceivedTimestamp(received_us); + frame->SetCapturedTimestamp(received_us - 5000); + frame->SetFrameType(key_frame ? minirtc::kVideoFrameKey + : minirtc::kVideoFrameDelta); + return frame; +} + +bool Expect(bool condition, const char *message) { + if (!condition) { + std::cerr << message << '\n'; + } + return condition; +} + +bool TestOneFramePerPlaybackCycle() { + minirtc::VideoReceiveScheduler scheduler(60); + scheduler.Enqueue(MakeFrame(90000, 1000000), 1000000); + scheduler.Enqueue(MakeFrame(91500, 1016667), 1016667); + + auto first = scheduler.Poll(1016667, false); + bool ok = Expect(first.frame != nullptr, "first frame was not released"); + ok &= Expect(first.frame && first.frame->RtpTimestamp() == 90000, + "scheduler did not preserve RTP order"); + + auto same_cycle = scheduler.Poll(1016667, false); + ok &= Expect(!same_cycle.frame, + "scheduler released more than one frame in a cycle"); + ok &= Expect(same_cycle.next_delay_us > 0, + "scheduler did not schedule the next playback cycle"); + + auto second = scheduler.Poll(1033334, false); + ok &= Expect(second.frame != nullptr, + "second frame was not released on the next cycle"); + ok &= Expect(second.frame && second.frame->RtpTimestamp() == 91500, + "second RTP timestamp was incorrect"); + return ok; +} + +bool TestSevereLagJumpsToNewestKeyFrame() { + minirtc::VideoReceiveScheduler scheduler(60); + for (uint32_t i = 0; i < 10; ++i) { + scheduler.Enqueue(MakeFrame(90000 + i * 1500, 1000000 + i * 16667, i == 8), + 1000000 + i * 16667); + } + + auto result = scheduler.Poll(1200000, false); + const auto metrics = scheduler.GetMetrics(1200000); + bool ok = Expect(result.frame && result.frame->IsKeyFrame(), + "catch-up did not jump to the newest key frame"); + ok &= Expect(result.dropped_frames == 8, + "catch-up dropped an unexpected number of frames"); + ok &= Expect(metrics.catch_up_count == 1, + "catch-up metric was not incremented"); + return ok; +} + +bool TestSevereLagRequestsKeyFrameWhenNoSyncPointExists() { + minirtc::VideoReceiveScheduler scheduler(60); + for (uint32_t i = 0; i < 10; ++i) { + scheduler.Enqueue(MakeFrame(180000 + i * 1500, 2000000 + i * 16667), + 2000000 + i * 16667); + } + + auto result = scheduler.Poll(2200000, false); + const auto metrics = scheduler.GetMetrics(2200000); + bool ok = Expect(!result.frame, + "undecodable delta frame was released during catch-up"); + ok &= + Expect(result.request_key_frame, "catch-up did not request a key frame"); + ok &= Expect(metrics.buffered_frames == 0 && metrics.awaiting_key_frame, + "scheduler did not enter key-frame wait state"); + return ok; +} + +bool TestAdaptiveBufferStaysWithinOneToThreeFrames() { + minirtc::VideoReceiveScheduler scheduler(60); + int64_t arrival_us = 3000000; + for (uint32_t i = 0; i < 40; ++i) { + arrival_us += (i % 2 == 0) ? 5000 : 30000; + scheduler.Enqueue(MakeFrame(270000 + i * 1500, arrival_us), arrival_us); + } + + const auto metrics = scheduler.GetMetrics(arrival_us); + bool ok = Expect(metrics.target_buffer_frames >= 1 && + metrics.target_buffer_frames <= 3, + "adaptive target left the 1-3 frame range"); + ok &= Expect(metrics.target_buffer_frames > 1, + "arrival jitter did not increase the target buffer"); + return ok; +} + +bool TestLegacyMicrosecondTimestampFallsBackToArrivalCadence() { + minirtc::VideoReceiveScheduler scheduler(60); + int64_t arrival_us = 4000000; + for (uint32_t i = 0; i < 20; ++i) { + scheduler.Enqueue(MakeFrame(500000 + i * 16667, arrival_us), arrival_us); + scheduler.Poll(arrival_us, false); + arrival_us += 16667; + } + scheduler.Poll(arrival_us, false); + + const auto metrics = scheduler.GetMetrics(arrival_us); + bool ok = Expect(metrics.estimated_frame_interval_us >= 15000 && + metrics.estimated_frame_interval_us <= 20000, + "legacy RTP units slowed the playback interval"); + ok &= Expect(metrics.timestamp_fallbacks > 0, + "legacy RTP units did not activate timestamp fallback"); + ok &= Expect(metrics.released_frames == 20 && metrics.dropped_frames == 0, + "legacy RTP units caused scheduler frame loss"); + return ok; +} + +bool TestSixtyFpsCadenceDoesNotTriggerCatchUp() { + minirtc::VideoReceiveScheduler scheduler(60); + constexpr int64_t kStartUs = 6000000; + constexpr int64_t kFrameIntervalUs = 16667; + constexpr uint32_t kStartRtpTimestamp = 700000; + constexpr uint32_t kRtpFrameInterval = 1500; + + for (uint32_t i = 0; i < 60; ++i) { + const int64_t now_us = kStartUs + static_cast(i) * + kFrameIntervalUs; + scheduler.Enqueue( + MakeFrame(kStartRtpTimestamp + i * kRtpFrameInterval, now_us), + now_us); + scheduler.Poll(now_us, false); + } + scheduler.Poll(kStartUs + 60 * kFrameIntervalUs, false); + + const auto metrics = + scheduler.GetMetrics(kStartUs + 60 * kFrameIntervalUs); + bool ok = Expect(metrics.released_frames == 60, + "60 fps input was not released at 60 fps cadence"); + ok &= Expect(metrics.dropped_frames == 0 && metrics.catch_up_count == 0, + "steady 60 fps input incorrectly triggered catch-up"); + ok &= Expect(metrics.estimated_frame_interval_us >= 15000 && + metrics.estimated_frame_interval_us <= 18000, + "steady 60 fps interval estimate drifted"); + return ok; +} + +bool TestKeyFrameRequestCooldownKeepsDecodableProgress() { + minirtc::VideoReceiveScheduler scheduler(60); + for (uint32_t i = 0; i < 10; ++i) { + scheduler.Enqueue(MakeFrame(600000 + i * 1500, + 5000000 + static_cast(i) * 16667), + 5000000 + static_cast(i) * 16667); + } + auto first_catch_up = scheduler.Poll(5200000, false); + bool ok = Expect(first_catch_up.request_key_frame, + "initial severe lag did not request a key frame"); + + scheduler.Enqueue(MakeFrame(615000, 5220000, true), 5220000); + auto key_frame = scheduler.Poll(5240000, false); + ok &= Expect(key_frame.frame && key_frame.frame->IsKeyFrame(), + "requested key frame was not released"); + + for (uint32_t i = 0; i < 10; ++i) { + scheduler.Enqueue(MakeFrame(616500 + i * 1500, + 5250000 + static_cast(i) * 16667), + 5250000 + static_cast(i) * 16667); + } + auto cooldown_catch_up = scheduler.Poll(5450000, false); + const auto metrics = scheduler.GetMetrics(5450000); + ok &= Expect(!cooldown_catch_up.request_key_frame, + "key frame request cooldown was ignored"); + ok &= Expect(cooldown_catch_up.frame != nullptr, + "cooldown prevented forward playback progress"); + ok &= Expect(metrics.key_frame_requests == 1, + "key frame request metric ignored cooldown"); + return ok; +} + +} // namespace + +int main() { + bool ok = true; + ok &= TestOneFramePerPlaybackCycle(); + ok &= TestSevereLagJumpsToNewestKeyFrame(); + ok &= TestSevereLagRequestsKeyFrameWhenNoSyncPointExists(); + ok &= TestAdaptiveBufferStaysWithinOneToThreeFrames(); + ok &= TestLegacyMicrosecondTimestampFallsBackToArrivalCadence(); + ok &= TestSixtyFpsCadenceDoesNotTriggerCatchUp(); + ok &= TestKeyFrameRequestCooldownKeepsDecodableProgress(); + return ok ? 0 : 1; +} diff --git a/apps/desktop/tests/video_rtp_timestamp_test.cpp b/apps/desktop/tests/video_rtp_timestamp_test.cpp new file mode 100644 index 00000000..2715e248 --- /dev/null +++ b/apps/desktop/tests/video_rtp_timestamp_test.cpp @@ -0,0 +1,84 @@ +#include +#include +#include +#include + +#include "rtp_defines.h" +#include "rtp_packetizer_av1.h" +#include "rtp_packetizer_h264.h" + +namespace { + +bool Expect(bool condition, const char *message) { + if (!condition) { + std::cerr << message << '\n'; + } + return condition; +} + +bool AllPacketsUseTimestamp( + const std::vector> &packets, + uint32_t expected_timestamp) { + if (packets.empty()) { + return false; + } + for (const auto &packet : packets) { + if (!packet || packet->Timestamp() != expected_timestamp) { + return false; + } + } + return true; +} + +bool TestTimestampConversion() { + bool ok = Expect(minirtc::rtp::VideoTimestampFromMicroseconds(1000000) == + 90000, + "one second did not convert to 90 kHz"); + ok &= Expect(minirtc::rtp::VideoTimestampFromMicroseconds(16667) == 1500, + "60 fps interval did not convert to 1500 RTP ticks"); + ok &= Expect(minirtc::rtp::VideoTimestampToMicroseconds(90000) == 1000000, + "90 kHz timestamp did not convert to one second"); + return ok; +} + +bool TestH264PacketizerPreservesTimestamp() { + constexpr uint32_t kTimestamp = 0x89abcdef; + minirtc::RtpPacketizerH264 packetizer(1234); + std::vector small_nalu = {0x65, 0x01, 0x02}; + auto single_packet = + packetizer.Build(small_nalu.data(), small_nalu.size(), kTimestamp, true); + bool ok = Expect(AllPacketsUseTimestamp(single_packet, kTimestamp), + "H.264 single-NAL packetizer changed RTP timestamp"); + + std::vector large_nalu(MAX_NALU_LEN + 100, 0x55); + large_nalu.front() = 0x65; + auto fragmented = + packetizer.Build(large_nalu.data(), large_nalu.size(), kTimestamp, true); + ok &= Expect(AllPacketsUseTimestamp(fragmented, kTimestamp), + "H.264 FU-A packetizer changed RTP timestamp"); + + auto padding = packetizer.BuildPadding(2000, kTimestamp, true); + ok &= Expect(AllPacketsUseTimestamp(padding, kTimestamp), + "H.264 padding packetizer changed RTP timestamp"); + return ok; +} + +bool TestAv1PacketizerPreservesTimestamp() { + constexpr uint32_t kTimestamp = 0x12345678; + minirtc::RtpPacketizerAv1 packetizer(5678); + // One frame OBU with a one-byte payload. + std::vector obu = {0x32, 0x01, 0x00}; + auto packets = packetizer.Build(obu.data(), obu.size(), kTimestamp, true); + return Expect(AllPacketsUseTimestamp(packets, kTimestamp), + "AV1 packetizer changed RTP timestamp"); +} + +} // namespace + +int main() { + bool ok = true; + ok &= TestTimestampConversion(); + ok &= TestH264PacketizerPreservesTimestamp(); + ok &= TestAv1PacketizerPreservesTimestamp(); + return ok ? 0 : 1; +} diff --git a/apps/desktop/xmake/targets.lua b/apps/desktop/xmake/targets.lua index 766ba385..37cd0161 100644 --- a/apps/desktop/xmake/targets.lua +++ b/apps/desktop/xmake/targets.lua @@ -106,6 +106,18 @@ function setup_targets() set_default(false) add_files("apps/desktop/tests/video_callback_lifetime_test.cpp") + target("video_receive_scheduler_test") + set_kind("binary") + set_default(false) + add_deps("minirtc") + add_files("apps/desktop/tests/video_receive_scheduler_test.cpp") + + target("video_rtp_timestamp_test") + set_kind("binary") + set_default(false) + add_deps("minirtc") + add_files("apps/desktop/tests/video_rtp_timestamp_test.cpp") + target("windows_manifest_resource_test") set_kind("binary") set_default(false) diff --git a/deps/submodules/minirtc b/deps/submodules/minirtc index a1d221da..6738542d 160000 --- a/deps/submodules/minirtc +++ b/deps/submodules/minirtc @@ -1 +1 @@ -Subproject commit a1d221da960439a0719e17bc88be0e94b7462c07 +Subproject commit 6738542d757f6b447f77f5af47f9fc97c35fbcbe