From 18c1c6535ce153ae5443d67b8792f17789fd0018 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Mon, 24 Aug 2026 16:17:22 +0800 Subject: [PATCH] [fix] eliminate 30 fps cap in remote desktop rendering pipeline --- src/gui/application/gui_application.cpp | 301 +++++++++++++++++- src/gui/application/gui_application.h | 5 + .../devices/session_device_manager.cpp | 44 ++- .../features/devices/session_device_manager.h | 6 +- src/gui/runtime/gui_runtime.h | 2 + src/gui/runtime/peer_media_callbacks.cpp | 36 +-- submodules/minirtc | 2 +- xmake/targets.lua | 1 + 8 files changed, 337 insertions(+), 60 deletions(-) diff --git a/src/gui/application/gui_application.cpp b/src/gui/application/gui_application.cpp index e281f87..6aa2cde 100644 --- a/src/gui/application/gui_application.cpp +++ b/src/gui/application/gui_application.cpp @@ -32,6 +32,7 @@ #include "platform.h" #if _WIN32 #include +#include #include "platform/tray/win_tray.h" #elif defined(__APPLE__) @@ -52,6 +53,39 @@ namespace { using namespace std::chrono_literals; +#if _WIN32 +constexpr GLint kGlClampToEdge = 0x812F; +#endif + +struct VideoRenderSize { + int width = 0; + int height = 0; + + bool operator==(const VideoRenderSize&) const = default; +}; + +VideoRenderSize FitVideoToRenderArea(int source_width, int source_height, + int area_width, int area_height) { + if (source_width <= 0 || source_height <= 0 || area_width <= 0 || + area_height <= 0) { + return {}; + } + + const double scale = + std::min({1.0, static_cast(area_width) / source_width, + static_cast(area_height) / source_height}); + int width = std::max(2, static_cast(std::floor(source_width * scale))); + int height = + std::max(2, static_cast(std::floor(source_height * scale))); + + // NV12 chroma samples cover 2x2 pixels. Keep the render buffer even-sized + // so libyuv never needs to read a partial chroma sample at the edge. + width &= ~1; + height &= ~1; + return {std::min(width, source_width & ~1), + std::min(height, source_height & ~1)}; +} + std::string CreatePasswordChangeRequestId(uint64_t sequence) { const auto timestamp = std::chrono::system_clock::now() .time_since_epoch() @@ -825,6 +859,17 @@ struct GuiApplication::SlintUi { controller_name_model = std::make_shared>(); std::unordered_map displayed_frame_sequence; + std::vector scaled_video_frame; +#if _WIN32 + std::mutex video_gl_mutex; + std::vector video_gl_conversion_frame; + std::vector video_gl_pending_frame; + uint32_t video_gl_texture = 0; + VideoRenderSize video_gl_texture_size; + VideoRenderSize video_gl_image_size; + VideoRenderSize video_gl_pending_size; + bool video_gl_pending_dirty = false; +#endif std::vector tab_order; std::vector tab_ids; std::string tab_model_signature; @@ -845,6 +890,7 @@ struct GuiApplication::SlintUi { bool capture_mode = false; std::string capture_page; std::chrono::steady_clock::time_point last_clipboard_poll{}; + slint::Timer video_timer; #if _WIN32 std::unique_ptr tray; #elif defined(__APPLE__) @@ -923,6 +969,7 @@ int GuiApplication::Run() { InitializeUi(); ui_->timer.start(slint::TimerMode::Repeated, 16ms, [this] { Tick(); }); + ScheduleNextVideoFrame(); if (ui_->capture_mode && (ui_->capture_page == "stream" || ui_->capture_page == "server")) { slint::run_event_loop(); @@ -1128,6 +1175,7 @@ void GuiApplication::InitializeUi() { (*ui_->stream)->set_custom_titlebar(use_xwayland_gui_); #endif RegisterFontAwesome((*ui_->stream)->window()); + ConfigureStreamVideoRenderer(); (*ui_->stream)->set_tabs(ui_->tab_model); (*ui_->stream)->set_displays(ui_->display_model); (*ui_->stream)->set_file_transfers(ui_->transfer_model); @@ -2466,6 +2514,7 @@ void GuiApplication::SyncStreamWindow() { (*ui_->stream)->set_custom_titlebar(use_xwayland_gui_); #endif RegisterFontAwesome((*ui_->stream)->window()); + ConfigureStreamVideoRenderer(); // Slint globals belong to a component tree. Force the next localization // pass to initialize the newly-created, independent stream window tree. ui_->localized_language = -1; @@ -2643,7 +2692,6 @@ void GuiApplication::SyncStreamWindow() { append_stats_row(localization::total[localization_language_index_], net.total_inbound_stats, net.total_outbound_stats); ui_->stats_model->set_vector(std::move(stats_rows)); - (*ui_->stream)->set_stats_fps(UiText(std::to_string(props->fps_))); (*ui_->stream) ->set_stats_resolution(UiText(std::to_string(props->video_width_) + "x" + std::to_string(props->video_height_))); @@ -2709,6 +2757,35 @@ void GuiApplication::SyncStreamWindow() { ->set_file_transfer_visible( props->file_transfer_.file_transfer_window_visible_); + const auto fps_now = std::chrono::steady_clock::now(); + if (!props->net_traffic_stats_button_pressed_) { + props->fps_ = 0; + props->frame_count_ = 0; + props->last_time_ = {}; + } else if (props->last_time_.time_since_epoch().count() == 0) { + props->last_time_ = fps_now; + } else { + const auto elapsed = std::chrono::duration_cast( + fps_now - props->last_time_) + .count(); + if (elapsed >= 1000) { + props->fps_ = static_cast(props->frame_count_ * 1000 / elapsed); + props->frame_count_ = 0; + props->last_time_ = fps_now; + } + } + (*ui_->stream)->set_stats_fps(UiText(std::to_string(props->fps_))); +} + +void GuiApplication::SyncStreamVideoFrame() { + if (!ui_ || !ui_->stream) { + return; + } + auto props = SelectedSession(); + if (!props) { + return; + } + std::shared_ptr> frame; int width = 0; int height = 0; @@ -2720,23 +2797,215 @@ void GuiApplication::SyncStreamWindow() { height = props->video_height_; sequence = props->video_frame_sequence_; } + const size_t nv12_size = static_cast(width) * height * 3 / 2; - if (frame && width > 0 && height > 0 && frame->size() >= nv12_size && - ui_->displayed_frame_sequence[props->remote_id_] != sequence) { - slint::SharedPixelBuffer pixels(width, height); - const int result = libyuv::NV12ToRAW( - frame->data(), width, - frame->data() + static_cast(width) * height, width, - reinterpret_cast(pixels.begin()), width * 3, width, height); - if (result == 0) { - (*ui_->stream)->set_frame(slint::Image(std::move(pixels))); - (*ui_->stream)->set_has_frame(true); - (*ui_->stream)->set_receiving_text(""); - ui_->displayed_frame_sequence[props->remote_id_] = sequence; - } - } else if (!frame) { + if (!frame || width <= 0 || height <= 0 || frame->size() < nv12_size) { (*ui_->stream)->set_has_frame(false); + return; } + if (ui_->displayed_frame_sequence[props->remote_id_] == sequence) { + return; + } + + const auto window_size = (*ui_->stream)->window().size(); + const VideoRenderSize render_size = FitVideoToRenderArea( + width, height, static_cast(window_size.width), + static_cast(window_size.height)); + const uint8_t* y_plane = frame->data(); + const uint8_t* uv_plane = + frame->data() + static_cast(width) * height; + int output_width = width; + int output_height = height; + + if (render_size.width > 0 && render_size.height > 0 && + (render_size.width != width || render_size.height != height)) { + const size_t scaled_nv12_size = + static_cast(render_size.width) * render_size.height * 3 / 2; + ui_->scaled_video_frame.resize(scaled_nv12_size); + uint8_t* scaled_y = ui_->scaled_video_frame.data(); + uint8_t* scaled_uv = + scaled_y + + static_cast(render_size.width) * render_size.height; + if (libyuv::NV12Scale(y_plane, width, uv_plane, width, width, height, + scaled_y, render_size.width, scaled_uv, + render_size.width, render_size.width, + render_size.height, libyuv::kFilterBox) == 0) { + y_plane = scaled_y; + uv_plane = scaled_uv; + output_width = render_size.width; + output_height = render_size.height; + } + } + +#if _WIN32 + uint32_t texture_id = 0; + { + std::lock_guard lock(ui_->video_gl_mutex); + texture_id = ui_->video_gl_texture; + } + if (texture_id != 0) { + ui_->video_gl_conversion_frame.resize( + static_cast(output_width) * output_height * 4); + if (libyuv::NV12ToABGR( + y_plane, output_width, uv_plane, output_width, + ui_->video_gl_conversion_frame.data(), output_width * 4, + output_width, output_height) != 0) { + return; + } + { + std::lock_guard lock(ui_->video_gl_mutex); + if (ui_->video_gl_texture == 0) { + return; + } + texture_id = ui_->video_gl_texture; + ui_->video_gl_pending_frame.swap(ui_->video_gl_conversion_frame); + ui_->video_gl_pending_size = {output_width, output_height}; + ui_->video_gl_pending_dirty = true; + } + const VideoRenderSize output_size{output_width, output_height}; + if (ui_->video_gl_image_size != output_size) { + (*ui_->stream) + ->set_frame(slint::Image::create_from_borrowed_gl_2d_rgba_texture( + texture_id, + slint::Size{static_cast(output_width), + static_cast(output_height)})); + ui_->video_gl_image_size = output_size; + } + (*ui_->stream)->set_has_frame(true); + (*ui_->stream)->set_receiving_text(""); + (*ui_->stream)->window().request_redraw(); + ui_->displayed_frame_sequence[props->remote_id_] = sequence; + ++props->frame_count_; + return; + } +#endif + + slint::SharedPixelBuffer pixels(output_width, + output_height); + if (libyuv::NV12ToRAW( + y_plane, output_width, uv_plane, output_width, + reinterpret_cast(pixels.begin()), output_width * 3, + output_width, output_height) != 0) { + return; + } + + (*ui_->stream)->set_frame(slint::Image(std::move(pixels))); + (*ui_->stream)->set_has_frame(true); + (*ui_->stream)->set_receiving_text(""); + ui_->displayed_frame_sequence[props->remote_id_] = sequence; + ++props->frame_count_; +} + +void GuiApplication::ConfigureStreamVideoRenderer() { +#if _WIN32 + if (!ui_ || !ui_->stream) { + return; + } + const auto error = (*ui_->stream)->window().set_rendering_notifier( + [this](slint::RenderingState state, slint::GraphicsAPI graphics_api) { + if (!ui_ || graphics_api != slint::GraphicsAPI::NativeOpenGL) { + return; + } + + std::lock_guard lock(ui_->video_gl_mutex); + if (state == slint::RenderingState::RenderingSetup) { + GLuint texture = 0; + glGenTextures(1, &texture); + if (texture == 0) { + return; + } + GLint previous_texture = 0; + glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture); + glBindTexture(GL_TEXTURE_2D, texture); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, kGlClampToEdge); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, kGlClampToEdge); + glBindTexture(GL_TEXTURE_2D, + static_cast(previous_texture)); + ui_->video_gl_texture = texture; + return; + } + + if (state == slint::RenderingState::BeforeRendering && + ui_->video_gl_texture != 0 && ui_->video_gl_pending_dirty && + !ui_->video_gl_pending_frame.empty()) { + GLint previous_texture = 0; + GLint previous_unpack_alignment = 0; + glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture); + glGetIntegerv(GL_UNPACK_ALIGNMENT, &previous_unpack_alignment); + glBindTexture(GL_TEXTURE_2D, ui_->video_gl_texture); + glPixelStorei(GL_UNPACK_ALIGNMENT, 1); + if (ui_->video_gl_texture_size != ui_->video_gl_pending_size) { + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, + ui_->video_gl_pending_size.width, + ui_->video_gl_pending_size.height, 0, GL_RGBA, + GL_UNSIGNED_BYTE, ui_->video_gl_pending_frame.data()); + ui_->video_gl_texture_size = ui_->video_gl_pending_size; + } else { + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, + ui_->video_gl_pending_size.width, + ui_->video_gl_pending_size.height, GL_RGBA, + GL_UNSIGNED_BYTE, + ui_->video_gl_pending_frame.data()); + } + glPixelStorei(GL_UNPACK_ALIGNMENT, previous_unpack_alignment); + glBindTexture(GL_TEXTURE_2D, + static_cast(previous_texture)); + ui_->video_gl_pending_dirty = false; + return; + } + + if (state == slint::RenderingState::RenderingTeardown) { + if (ui_->video_gl_texture != 0) { + const GLuint texture = ui_->video_gl_texture; + glDeleteTextures(1, &texture); + } + ui_->video_gl_texture = 0; + ui_->video_gl_texture_size = {}; + ui_->video_gl_image_size = {}; + ui_->video_gl_pending_size = {}; + ui_->video_gl_pending_dirty = false; + ui_->video_gl_pending_frame.clear(); + } + }); + if (error.has_value()) { + LOG_WARN("Slint OpenGL video renderer unavailable, using pixel buffers"); + } +#endif +} + +void GuiApplication::ScheduleNextVideoFrame() { + if (!ui_) { + return; + } + + constexpr auto frame_interval = std::chrono::nanoseconds(1'000'000'000 / 60); + const auto now = std::chrono::steady_clock::now(); + if (next_video_frame_time_ == std::chrono::steady_clock::time_point{} || + now - next_video_frame_time_ > 250ms) { + next_video_frame_time_ = now; + } + + if (now >= next_video_frame_time_) { + if (video_frame_dirty_.exchange(false, std::memory_order_acq_rel)) { + SyncStreamVideoFrame(); + } + do { + next_video_frame_time_ += frame_interval; + } while (next_video_frame_time_ <= now); + } + + const auto after_render = std::chrono::steady_clock::now(); + const auto remaining = + std::max(next_video_frame_time_ - after_render, + std::chrono::steady_clock::duration::zero()); + const auto delay_ns = + std::chrono::duration_cast(remaining).count(); + const auto delay_ms = std::max(1, (delay_ns + 999'999) / 1'000'000); + ui_->video_timer.start(slint::TimerMode::SingleShot, + std::chrono::milliseconds(delay_ms), + [this] { ScheduleNextVideoFrame(); }); } void GuiApplication::SyncServerWindow() { @@ -3241,6 +3510,7 @@ void GuiApplication::Cleanup() { return; } ui_->timer.stop(); + ui_->video_timer.stop(); #if _WIN32 && CROSSDESK_PORTABLE JoinPortableWindowsServiceInstallThread(); #endif @@ -3253,6 +3523,7 @@ void GuiApplication::Cleanup() { devices_.DestroyAudioOutput(); if (ui_->stream) { (*ui_->stream)->hide(); + ui_->stream.reset(); } if (ui_->server) { (*ui_->server)->hide(); diff --git a/src/gui/application/gui_application.h b/src/gui/application/gui_application.h index e2dbb0a..9aea77c 100644 --- a/src/gui/application/gui_application.h +++ b/src/gui/application/gui_application.h @@ -1,6 +1,7 @@ #ifndef CROSSDESK_GUI_APPLICATION_H_ #define CROSSDESK_GUI_APPLICATION_H_ +#include #include #include @@ -37,6 +38,9 @@ private: void SyncConnectionDialog(); void SyncPlatformDialogs(); void SyncStreamWindow(); + void SyncStreamVideoFrame(); + void ScheduleNextVideoFrame(); + void ConfigureStreamVideoRenderer(); void SyncStreamKeyboardFocus(); void SetStreamKeyboardFocus(bool focused); void SyncServerWindow(); @@ -66,6 +70,7 @@ private: bool OpenUrl(const std::string &url); std::unique_ptr ui_; + std::chrono::steady_clock::time_point next_video_frame_time_{}; #if defined(__linux__) && !defined(__APPLE__) bool use_xwayland_gui_ = false; bool use_x11_custom_titlebar_ = false; diff --git a/src/gui/features/devices/session_device_manager.cpp b/src/gui/features/devices/session_device_manager.cpp index dea6917..a21013b 100644 --- a/src/gui/features/devices/session_device_manager.cpp +++ b/src/gui/features/devices/session_device_manager.cpp @@ -13,11 +13,29 @@ namespace { constexpr uint64_t kCaptureResumeKeyFrameGapMs = 500; constexpr size_t kMaxCapturedKeyboardInputs = 512; +constexpr auto kFrameDeadlineTolerance = std::chrono::milliseconds(1); } // namespace SessionDeviceManager::SessionDeviceManager(GuiRuntime &owner) : owner_(owner) {} +bool SessionDeviceManager::ShouldSendCapturedFrame( + std::chrono::steady_clock::time_point now, int fps) { + const auto interval = + std::chrono::nanoseconds(std::chrono::seconds(1)) / (fps > 0 ? fps : 1); + if (next_frame_deadline_ == std::chrono::steady_clock::time_point{}) { + next_frame_deadline_ = now; + } + if (now + kFrameDeadlineTolerance < next_frame_deadline_) { + return false; + } + next_frame_deadline_ += interval; + if (next_frame_deadline_ <= now) { + next_frame_deadline_ = now + interval; + } + return true; +} + void SessionDeviceManager::Initialize() { InitializeAudioOutput(); screen_capturer_factory_ = new ScreenCapturerFactory(); @@ -40,9 +58,8 @@ int SessionDeviceManager::InitializeScreenCapturer() { static_cast(screen_capturer_factory_->Create()); } - last_frame_time_ = std::chrono::duration_cast( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + last_frame_time_ = {}; + next_frame_deadline_ = {}; const int fps = owner_.config_center_->GetVideoFrameRate() == ConfigCenter::VIDEO_FRAME_RATE::FPS_30 ? 30 @@ -56,15 +73,20 @@ int SessionDeviceManager::InitializeScreenCapturer() { const int init_ret = screen_capturer_->Init( fps, [this, fps](unsigned char *data, int size, int width, int height, const char *display_name) { - const auto now_time = static_cast( - std::chrono::duration_cast( - std::chrono::steady_clock::now().time_since_epoch()) - .count()); - const auto duration = now_time - last_frame_time_; - if (duration * fps < 1000) { + const auto now_time = std::chrono::steady_clock::now(); + if (!ShouldSendCapturedFrame(now_time, fps)) { return; } + const bool has_previous_frame = + last_frame_time_.time_since_epoch().count() != 0; + const auto duration_ms = + has_previous_frame + ? std::chrono::duration_cast( + now_time - last_frame_time_) + .count() + : 0; + std::vector connected_remote_ids; { std::shared_lock lock(owner_.connection_status_mutex_); @@ -101,14 +123,14 @@ int SessionDeviceManager::InitializeScreenCapturer() { } invalid_video_stream_id_logged_ = false; const bool resumed_after_gap = - last_frame_time_ != 0 && duration >= kCaptureResumeKeyFrameGapMs; + has_previous_frame && duration_ms >= kCaptureResumeKeyFrameGapMs; const bool stream_changed = !last_video_frame_stream_id_.empty() && last_video_frame_stream_id_ != stream_id; if (resumed_after_gap || stream_changed) { if (RequestVideoKeyFrame(owner_.peer_, stream_id.c_str()) == 0) { LOG_INFO("Request video key frame before sending captured frame, " "stream='{}', gap_ms={}, stream_changed={}", - stream_id, duration, stream_changed); + stream_id, duration_ms, stream_changed); } } diff --git a/src/gui/features/devices/session_device_manager.h b/src/gui/features/devices/session_device_manager.h index 9a3064b..293a65e 100644 --- a/src/gui/features/devices/session_device_manager.h +++ b/src/gui/features/devices/session_device_manager.h @@ -3,6 +3,7 @@ #include +#include #include #include #include @@ -66,6 +67,8 @@ private: uint32_t scan_code, bool extended); void DrainCapturedKeyboardInput(); void ClearCapturedKeyboardInput(); + bool ShouldSendCapturedFrame(std::chrono::steady_clock::time_point now, + int fps); GuiRuntime &owner_; SDL_AudioStream *output_stream_ = nullptr; @@ -80,7 +83,8 @@ private: size_t registered_display_stream_count_ = 0; std::deque captured_keyboard_inputs_; std::mutex captured_keyboard_inputs_mutex_; - uint64_t last_frame_time_ = 0; + std::chrono::steady_clock::time_point last_frame_time_{}; + std::chrono::steady_clock::time_point next_frame_deadline_{}; std::string last_video_frame_stream_id_; bool invalid_video_stream_id_logged_ = false; }; diff --git a/src/gui/runtime/gui_runtime.h b/src/gui/runtime/gui_runtime.h index 374534e..76ba424 100644 --- a/src/gui/runtime/gui_runtime.h +++ b/src/gui/runtime/gui_runtime.h @@ -1,6 +1,7 @@ #ifndef CROSSDESK_GUI_RUNTIME_H_ #define CROSSDESK_GUI_RUNTIME_H_ +#include #include #include @@ -81,6 +82,7 @@ class GuiRuntime : protected gui_detail::GuiState { SettingsManager settings_; KeyboardController keyboard_; PeerEventHandler peer_events_; + std::atomic video_frame_dirty_{false}; private: friend class ClipboardController; diff --git a/src/gui/runtime/peer_media_callbacks.cpp b/src/gui/runtime/peer_media_callbacks.cpp index b5a931c..bd65ad8 100644 --- a/src/gui/runtime/peer_media_callbacks.cpp +++ b/src/gui/runtime/peer_media_callbacks.cpp @@ -1,28 +1,12 @@ #include "runtime/peer_event_handler.h" -#include -#include #include -#include #include #include -#include #include -#include -#include +#include -#include "device_controller.h" -#include "file_transfer.h" -#include "localization.h" -#include "platform.h" -#include "rd_log.h" #include "runtime/gui_runtime.h" -#include "runtime/remote_action_codec.h" - -#if _WIN32 -#include "interactive_state.h" -#include "service_host.h" -#endif namespace crossdesk { @@ -48,7 +32,8 @@ void PeerEventHandler::OnReceiveVideoBuffer( { std::lock_guard lock(props->video_frame_mutex_); - if (!props->back_frame_) { + // Allocate a third buffer only while the UI still owns the old snapshot. + if (!props->back_frame_ || props->back_frame_.use_count() != 1) { props->back_frame_ = std::make_shared>(video_frame->size); } @@ -74,20 +59,7 @@ void PeerEventHandler::OnReceiveVideoBuffer( } props->streaming_ = true; - - if (props->net_traffic_stats_button_pressed_) { - props->frame_count_++; - auto now = std::chrono::steady_clock::now(); - auto elapsed = std::chrono::duration_cast( - now - props->last_time_) - .count(); - - if (elapsed >= 1000) { - props->fps_ = props->frame_count_ * 1000 / elapsed; - props->frame_count_ = 0; - props->last_time_ = now; - } - } + runtime->video_frame_dirty_.store(true, std::memory_order_release); } } diff --git a/submodules/minirtc b/submodules/minirtc index e0bd8a3..f4d978e 160000 --- a/submodules/minirtc +++ b/submodules/minirtc @@ -1 +1 @@ -Subproject commit e0bd8a3271a892c7117cfd5268ed6111b686e001 +Subproject commit f4d978e6e38e08c54e2c5b0c13cd9f524be1264f diff --git a/xmake/targets.lua b/xmake/targets.lua index f79edbb..7b89d33 100644 --- a/xmake/targets.lua +++ b/xmake/targets.lua @@ -275,6 +275,7 @@ function setup_targets() add_includedirs("src/gui", {public = true}) if is_os("windows") then add_cxxflags("/bigobj") + add_links("opengl32") add_files("src/gui/platform/tray/win_tray.cpp") add_includedirs("src/service/windows", {public = true}) elseif is_os("macosx") then