[fix] eliminate 30 fps cap in remote desktop rendering pipeline

This commit is contained in:
dijunkun
2026-08-24 16:22:32 +08:00
parent ee92c4ce5f
commit 05885ecd48
8 changed files with 337 additions and 60 deletions
+286 -15
View File
@@ -32,6 +32,7 @@
#include "platform.h"
#if _WIN32
#include <windows.h>
#include <GL/gl.h>
#include "platform/tray/win_tray.h"
#elif defined(__APPLE__)
@@ -52,6 +53,39 @@ namespace {
using namespace std::chrono_literals;
#if _WIN32
constexpr GLint kGlClampToEdge = 0x812F;
#endif
struct VideoRenderSize {
int width = 0;
int height = 0;
bool operator==(const VideoRenderSize&) const = default;
};
VideoRenderSize FitVideoToRenderArea(int source_width, int source_height,
int area_width, int area_height) {
if (source_width <= 0 || source_height <= 0 || area_width <= 0 ||
area_height <= 0) {
return {};
}
const double scale =
std::min({1.0, static_cast<double>(area_width) / source_width,
static_cast<double>(area_height) / source_height});
int width = std::max(2, static_cast<int>(std::floor(source_width * scale)));
int height =
std::max(2, static_cast<int>(std::floor(source_height * scale)));
// NV12 chroma samples cover 2x2 pixels. Keep the render buffer even-sized
// so libyuv never needs to read a partial chroma sample at the edge.
width &= ~1;
height &= ~1;
return {std::min(width, source_width & ~1),
std::min(height, source_height & ~1)};
}
std::string CreatePasswordChangeRequestId(uint64_t sequence) {
const auto timestamp = std::chrono::system_clock::now()
.time_since_epoch()
@@ -825,6 +859,17 @@ struct GuiApplication::SlintUi {
controller_name_model =
std::make_shared<slint::VectorModel<slint::SharedString>>();
std::unordered_map<std::string, uint64_t> displayed_frame_sequence;
std::vector<uint8_t> scaled_video_frame;
#if _WIN32
std::mutex video_gl_mutex;
std::vector<uint8_t> video_gl_conversion_frame;
std::vector<uint8_t> video_gl_pending_frame;
uint32_t video_gl_texture = 0;
VideoRenderSize video_gl_texture_size;
VideoRenderSize video_gl_image_size;
VideoRenderSize video_gl_pending_size;
bool video_gl_pending_dirty = false;
#endif
std::vector<std::string> tab_order;
std::vector<std::string> tab_ids;
std::string tab_model_signature;
@@ -845,6 +890,7 @@ struct GuiApplication::SlintUi {
bool capture_mode = false;
std::string capture_page;
std::chrono::steady_clock::time_point last_clipboard_poll{};
slint::Timer video_timer;
#if _WIN32
std::unique_ptr<WinTray> tray;
#elif defined(__APPLE__)
@@ -923,6 +969,7 @@ int GuiApplication::Run() {
InitializeUi();
ui_->timer.start(slint::TimerMode::Repeated, 16ms, [this] { Tick(); });
ScheduleNextVideoFrame();
if (ui_->capture_mode &&
(ui_->capture_page == "stream" || ui_->capture_page == "server")) {
slint::run_event_loop();
@@ -1128,6 +1175,7 @@ void GuiApplication::InitializeUi() {
(*ui_->stream)->set_custom_titlebar(use_xwayland_gui_);
#endif
RegisterFontAwesome((*ui_->stream)->window());
ConfigureStreamVideoRenderer();
(*ui_->stream)->set_tabs(ui_->tab_model);
(*ui_->stream)->set_displays(ui_->display_model);
(*ui_->stream)->set_file_transfers(ui_->transfer_model);
@@ -2466,6 +2514,7 @@ void GuiApplication::SyncStreamWindow() {
(*ui_->stream)->set_custom_titlebar(use_xwayland_gui_);
#endif
RegisterFontAwesome((*ui_->stream)->window());
ConfigureStreamVideoRenderer();
// Slint globals belong to a component tree. Force the next localization
// pass to initialize the newly-created, independent stream window tree.
ui_->localized_language = -1;
@@ -2643,7 +2692,6 @@ void GuiApplication::SyncStreamWindow() {
append_stats_row(localization::total[localization_language_index_],
net.total_inbound_stats, net.total_outbound_stats);
ui_->stats_model->set_vector(std::move(stats_rows));
(*ui_->stream)->set_stats_fps(UiText(std::to_string(props->fps_)));
(*ui_->stream)
->set_stats_resolution(UiText(std::to_string(props->video_width_) + "x" +
std::to_string(props->video_height_)));
@@ -2709,6 +2757,35 @@ void GuiApplication::SyncStreamWindow() {
->set_file_transfer_visible(
props->file_transfer_.file_transfer_window_visible_);
const auto fps_now = std::chrono::steady_clock::now();
if (!props->net_traffic_stats_button_pressed_) {
props->fps_ = 0;
props->frame_count_ = 0;
props->last_time_ = {};
} else if (props->last_time_.time_since_epoch().count() == 0) {
props->last_time_ = fps_now;
} else {
const auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
fps_now - props->last_time_)
.count();
if (elapsed >= 1000) {
props->fps_ = static_cast<int>(props->frame_count_ * 1000 / elapsed);
props->frame_count_ = 0;
props->last_time_ = fps_now;
}
}
(*ui_->stream)->set_stats_fps(UiText(std::to_string(props->fps_)));
}
void GuiApplication::SyncStreamVideoFrame() {
if (!ui_ || !ui_->stream) {
return;
}
auto props = SelectedSession();
if (!props) {
return;
}
std::shared_ptr<std::vector<unsigned char>> frame;
int width = 0;
int height = 0;
@@ -2720,23 +2797,215 @@ void GuiApplication::SyncStreamWindow() {
height = props->video_height_;
sequence = props->video_frame_sequence_;
}
const size_t nv12_size = static_cast<size_t>(width) * height * 3 / 2;
if (frame && width > 0 && height > 0 && frame->size() >= nv12_size &&
ui_->displayed_frame_sequence[props->remote_id_] != sequence) {
slint::SharedPixelBuffer<slint::Rgb8Pixel> pixels(width, height);
const int result = libyuv::NV12ToRAW(
frame->data(), width,
frame->data() + static_cast<size_t>(width) * height, width,
reinterpret_cast<uint8_t*>(pixels.begin()), width * 3, width, height);
if (result == 0) {
(*ui_->stream)->set_frame(slint::Image(std::move(pixels)));
(*ui_->stream)->set_has_frame(true);
(*ui_->stream)->set_receiving_text("");
ui_->displayed_frame_sequence[props->remote_id_] = sequence;
}
} else if (!frame) {
if (!frame || width <= 0 || height <= 0 || frame->size() < nv12_size) {
(*ui_->stream)->set_has_frame(false);
return;
}
if (ui_->displayed_frame_sequence[props->remote_id_] == sequence) {
return;
}
const auto window_size = (*ui_->stream)->window().size();
const VideoRenderSize render_size = FitVideoToRenderArea(
width, height, static_cast<int>(window_size.width),
static_cast<int>(window_size.height));
const uint8_t* y_plane = frame->data();
const uint8_t* uv_plane =
frame->data() + static_cast<size_t>(width) * height;
int output_width = width;
int output_height = height;
if (render_size.width > 0 && render_size.height > 0 &&
(render_size.width != width || render_size.height != height)) {
const size_t scaled_nv12_size =
static_cast<size_t>(render_size.width) * render_size.height * 3 / 2;
ui_->scaled_video_frame.resize(scaled_nv12_size);
uint8_t* scaled_y = ui_->scaled_video_frame.data();
uint8_t* scaled_uv =
scaled_y +
static_cast<size_t>(render_size.width) * render_size.height;
if (libyuv::NV12Scale(y_plane, width, uv_plane, width, width, height,
scaled_y, render_size.width, scaled_uv,
render_size.width, render_size.width,
render_size.height, libyuv::kFilterBox) == 0) {
y_plane = scaled_y;
uv_plane = scaled_uv;
output_width = render_size.width;
output_height = render_size.height;
}
}
#if _WIN32
uint32_t texture_id = 0;
{
std::lock_guard lock(ui_->video_gl_mutex);
texture_id = ui_->video_gl_texture;
}
if (texture_id != 0) {
ui_->video_gl_conversion_frame.resize(
static_cast<size_t>(output_width) * output_height * 4);
if (libyuv::NV12ToABGR(
y_plane, output_width, uv_plane, output_width,
ui_->video_gl_conversion_frame.data(), output_width * 4,
output_width, output_height) != 0) {
return;
}
{
std::lock_guard lock(ui_->video_gl_mutex);
if (ui_->video_gl_texture == 0) {
return;
}
texture_id = ui_->video_gl_texture;
ui_->video_gl_pending_frame.swap(ui_->video_gl_conversion_frame);
ui_->video_gl_pending_size = {output_width, output_height};
ui_->video_gl_pending_dirty = true;
}
const VideoRenderSize output_size{output_width, output_height};
if (ui_->video_gl_image_size != output_size) {
(*ui_->stream)
->set_frame(slint::Image::create_from_borrowed_gl_2d_rgba_texture(
texture_id,
slint::Size<uint32_t>{static_cast<uint32_t>(output_width),
static_cast<uint32_t>(output_height)}));
ui_->video_gl_image_size = output_size;
}
(*ui_->stream)->set_has_frame(true);
(*ui_->stream)->set_receiving_text("");
(*ui_->stream)->window().request_redraw();
ui_->displayed_frame_sequence[props->remote_id_] = sequence;
++props->frame_count_;
return;
}
#endif
slint::SharedPixelBuffer<slint::Rgb8Pixel> pixels(output_width,
output_height);
if (libyuv::NV12ToRAW(
y_plane, output_width, uv_plane, output_width,
reinterpret_cast<uint8_t*>(pixels.begin()), output_width * 3,
output_width, output_height) != 0) {
return;
}
(*ui_->stream)->set_frame(slint::Image(std::move(pixels)));
(*ui_->stream)->set_has_frame(true);
(*ui_->stream)->set_receiving_text("");
ui_->displayed_frame_sequence[props->remote_id_] = sequence;
++props->frame_count_;
}
void GuiApplication::ConfigureStreamVideoRenderer() {
#if _WIN32
if (!ui_ || !ui_->stream) {
return;
}
const auto error = (*ui_->stream)->window().set_rendering_notifier(
[this](slint::RenderingState state, slint::GraphicsAPI graphics_api) {
if (!ui_ || graphics_api != slint::GraphicsAPI::NativeOpenGL) {
return;
}
std::lock_guard lock(ui_->video_gl_mutex);
if (state == slint::RenderingState::RenderingSetup) {
GLuint texture = 0;
glGenTextures(1, &texture);
if (texture == 0) {
return;
}
GLint previous_texture = 0;
glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture);
glBindTexture(GL_TEXTURE_2D, texture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, kGlClampToEdge);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, kGlClampToEdge);
glBindTexture(GL_TEXTURE_2D,
static_cast<GLuint>(previous_texture));
ui_->video_gl_texture = texture;
return;
}
if (state == slint::RenderingState::BeforeRendering &&
ui_->video_gl_texture != 0 && ui_->video_gl_pending_dirty &&
!ui_->video_gl_pending_frame.empty()) {
GLint previous_texture = 0;
GLint previous_unpack_alignment = 0;
glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture);
glGetIntegerv(GL_UNPACK_ALIGNMENT, &previous_unpack_alignment);
glBindTexture(GL_TEXTURE_2D, ui_->video_gl_texture);
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
if (ui_->video_gl_texture_size != ui_->video_gl_pending_size) {
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA,
ui_->video_gl_pending_size.width,
ui_->video_gl_pending_size.height, 0, GL_RGBA,
GL_UNSIGNED_BYTE, ui_->video_gl_pending_frame.data());
ui_->video_gl_texture_size = ui_->video_gl_pending_size;
} else {
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0,
ui_->video_gl_pending_size.width,
ui_->video_gl_pending_size.height, GL_RGBA,
GL_UNSIGNED_BYTE,
ui_->video_gl_pending_frame.data());
}
glPixelStorei(GL_UNPACK_ALIGNMENT, previous_unpack_alignment);
glBindTexture(GL_TEXTURE_2D,
static_cast<GLuint>(previous_texture));
ui_->video_gl_pending_dirty = false;
return;
}
if (state == slint::RenderingState::RenderingTeardown) {
if (ui_->video_gl_texture != 0) {
const GLuint texture = ui_->video_gl_texture;
glDeleteTextures(1, &texture);
}
ui_->video_gl_texture = 0;
ui_->video_gl_texture_size = {};
ui_->video_gl_image_size = {};
ui_->video_gl_pending_size = {};
ui_->video_gl_pending_dirty = false;
ui_->video_gl_pending_frame.clear();
}
});
if (error.has_value()) {
LOG_WARN("Slint OpenGL video renderer unavailable, using pixel buffers");
}
#endif
}
void GuiApplication::ScheduleNextVideoFrame() {
if (!ui_) {
return;
}
constexpr auto frame_interval = std::chrono::nanoseconds(1'000'000'000 / 60);
const auto now = std::chrono::steady_clock::now();
if (next_video_frame_time_ == std::chrono::steady_clock::time_point{} ||
now - next_video_frame_time_ > 250ms) {
next_video_frame_time_ = now;
}
if (now >= next_video_frame_time_) {
if (video_frame_dirty_.exchange(false, std::memory_order_acq_rel)) {
SyncStreamVideoFrame();
}
do {
next_video_frame_time_ += frame_interval;
} while (next_video_frame_time_ <= now);
}
const auto after_render = std::chrono::steady_clock::now();
const auto remaining =
std::max(next_video_frame_time_ - after_render,
std::chrono::steady_clock::duration::zero());
const auto delay_ns =
std::chrono::duration_cast<std::chrono::nanoseconds>(remaining).count();
const auto delay_ms = std::max<int64_t>(1, (delay_ns + 999'999) / 1'000'000);
ui_->video_timer.start(slint::TimerMode::SingleShot,
std::chrono::milliseconds(delay_ms),
[this] { ScheduleNextVideoFrame(); });
}
void GuiApplication::SyncServerWindow() {
@@ -3241,6 +3510,7 @@ void GuiApplication::Cleanup() {
return;
}
ui_->timer.stop();
ui_->video_timer.stop();
#if _WIN32 && CROSSDESK_PORTABLE
JoinPortableWindowsServiceInstallThread();
#endif
@@ -3253,6 +3523,7 @@ void GuiApplication::Cleanup() {
devices_.DestroyAudioOutput();
if (ui_->stream) {
(*ui_->stream)->hide();
ui_->stream.reset();
}
if (ui_->server) {
(*ui_->server)->hide();
+5
View File
@@ -1,6 +1,7 @@
#ifndef CROSSDESK_GUI_APPLICATION_H_
#define CROSSDESK_GUI_APPLICATION_H_
#include <chrono>
#include <memory>
#include <string>
@@ -37,6 +38,9 @@ private:
void SyncConnectionDialog();
void SyncPlatformDialogs();
void SyncStreamWindow();
void SyncStreamVideoFrame();
void ScheduleNextVideoFrame();
void ConfigureStreamVideoRenderer();
void SyncStreamKeyboardFocus();
void SetStreamKeyboardFocus(bool focused);
void SyncServerWindow();
@@ -66,6 +70,7 @@ private:
bool OpenUrl(const std::string &url);
std::unique_ptr<SlintUi> ui_;
std::chrono::steady_clock::time_point next_video_frame_time_{};
#if defined(__linux__) && !defined(__APPLE__)
bool use_xwayland_gui_ = false;
bool use_x11_custom_titlebar_ = false;
@@ -13,11 +13,29 @@ namespace {
constexpr uint64_t kCaptureResumeKeyFrameGapMs = 500;
constexpr size_t kMaxCapturedKeyboardInputs = 512;
constexpr auto kFrameDeadlineTolerance = std::chrono::milliseconds(1);
} // namespace
SessionDeviceManager::SessionDeviceManager(GuiRuntime &owner) : owner_(owner) {}
bool SessionDeviceManager::ShouldSendCapturedFrame(
std::chrono::steady_clock::time_point now, int fps) {
const auto interval =
std::chrono::nanoseconds(std::chrono::seconds(1)) / (fps > 0 ? fps : 1);
if (next_frame_deadline_ == std::chrono::steady_clock::time_point{}) {
next_frame_deadline_ = now;
}
if (now + kFrameDeadlineTolerance < next_frame_deadline_) {
return false;
}
next_frame_deadline_ += interval;
if (next_frame_deadline_ <= now) {
next_frame_deadline_ = now + interval;
}
return true;
}
void SessionDeviceManager::Initialize() {
InitializeAudioOutput();
screen_capturer_factory_ = new ScreenCapturerFactory();
@@ -40,9 +58,8 @@ int SessionDeviceManager::InitializeScreenCapturer() {
static_cast<ScreenCapturer *>(screen_capturer_factory_->Create());
}
last_frame_time_ = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now().time_since_epoch())
.count();
last_frame_time_ = {};
next_frame_deadline_ = {};
const int fps = owner_.config_center_->GetVideoFrameRate() ==
ConfigCenter::VIDEO_FRAME_RATE::FPS_30
? 30
@@ -56,15 +73,20 @@ int SessionDeviceManager::InitializeScreenCapturer() {
const int init_ret = screen_capturer_->Init(
fps, [this, fps](unsigned char *data, int size, int width, int height,
const char *display_name) {
const auto now_time = static_cast<uint64_t>(
std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now().time_since_epoch())
.count());
const auto duration = now_time - last_frame_time_;
if (duration * fps < 1000) {
const auto now_time = std::chrono::steady_clock::now();
if (!ShouldSendCapturedFrame(now_time, fps)) {
return;
}
const bool has_previous_frame =
last_frame_time_.time_since_epoch().count() != 0;
const auto duration_ms =
has_previous_frame
? std::chrono::duration_cast<std::chrono::milliseconds>(
now_time - last_frame_time_)
.count()
: 0;
std::vector<std::string> connected_remote_ids;
{
std::shared_lock lock(owner_.connection_status_mutex_);
@@ -101,14 +123,14 @@ int SessionDeviceManager::InitializeScreenCapturer() {
}
invalid_video_stream_id_logged_ = false;
const bool resumed_after_gap =
last_frame_time_ != 0 && duration >= kCaptureResumeKeyFrameGapMs;
has_previous_frame && duration_ms >= kCaptureResumeKeyFrameGapMs;
const bool stream_changed = !last_video_frame_stream_id_.empty() &&
last_video_frame_stream_id_ != stream_id;
if (resumed_after_gap || stream_changed) {
if (RequestVideoKeyFrame(owner_.peer_, stream_id.c_str()) == 0) {
LOG_INFO("Request video key frame before sending captured frame, "
"stream='{}', gap_ms={}, stream_changed={}",
stream_id, duration, stream_changed);
stream_id, duration_ms, stream_changed);
}
}
@@ -3,6 +3,7 @@
#include <SDL3/SDL.h>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <deque>
@@ -66,6 +67,8 @@ private:
uint32_t scan_code, bool extended);
void DrainCapturedKeyboardInput();
void ClearCapturedKeyboardInput();
bool ShouldSendCapturedFrame(std::chrono::steady_clock::time_point now,
int fps);
GuiRuntime &owner_;
SDL_AudioStream *output_stream_ = nullptr;
@@ -80,7 +83,8 @@ private:
size_t registered_display_stream_count_ = 0;
std::deque<CapturedKeyboardInput> captured_keyboard_inputs_;
std::mutex captured_keyboard_inputs_mutex_;
uint64_t last_frame_time_ = 0;
std::chrono::steady_clock::time_point last_frame_time_{};
std::chrono::steady_clock::time_point next_frame_deadline_{};
std::string last_video_frame_stream_id_;
bool invalid_video_stream_id_logged_ = false;
};
+2
View File
@@ -1,6 +1,7 @@
#ifndef CROSSDESK_GUI_RUNTIME_H_
#define CROSSDESK_GUI_RUNTIME_H_
#include <atomic>
#include <memory>
#include <string>
@@ -81,6 +82,7 @@ class GuiRuntime : protected gui_detail::GuiState {
SettingsManager settings_;
KeyboardController keyboard_;
PeerEventHandler peer_events_;
std::atomic<bool> video_frame_dirty_{false};
private:
friend class ClipboardController;
+4 -32
View File
@@ -1,28 +1,12 @@
#include "runtime/peer_event_handler.h"
#include <algorithm>
#include <chrono>
#include <cstring>
#include <filesystem>
#include <memory>
#include <mutex>
#include <shared_mutex>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "device_controller.h"
#include "file_transfer.h"
#include "localization.h"
#include "platform.h"
#include "rd_log.h"
#include "runtime/gui_runtime.h"
#include "runtime/remote_action_codec.h"
#if _WIN32
#include "interactive_state.h"
#include "service_host.h"
#endif
namespace crossdesk {
@@ -48,7 +32,8 @@ void PeerEventHandler::OnReceiveVideoBuffer(
{
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
if (!props->back_frame_) {
// Allocate a third buffer only while the UI still owns the old snapshot.
if (!props->back_frame_ || props->back_frame_.use_count() != 1) {
props->back_frame_ =
std::make_shared<std::vector<unsigned char>>(video_frame->size);
}
@@ -74,20 +59,7 @@ void PeerEventHandler::OnReceiveVideoBuffer(
}
props->streaming_ = true;
if (props->net_traffic_stats_button_pressed_) {
props->frame_count_++;
auto now = std::chrono::steady_clock::now();
auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
now - props->last_time_)
.count();
if (elapsed >= 1000) {
props->fps_ = props->frame_count_ * 1000 / elapsed;
props->frame_count_ = 0;
props->last_time_ = now;
}
}
runtime->video_frame_dirty_.store(true, std::memory_order_release);
}
}
+1
View File
@@ -275,6 +275,7 @@ function setup_targets()
add_includedirs("src/gui", {public = true})
if is_os("windows") then
add_cxxflags("/bigobj")
add_links("opengl32")
add_files("src/gui/platform/tray/win_tray.cpp")
add_includedirs("src/service/windows", {public = true})
elseif is_os("macosx") then