YangWX/crossdesk
1
0
Fork 0
mirror of https://github.com/kunkundi/crossdesk.git synced 2026-08-26 13:29:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feat] add OpenGL NV12 underlay rendering for Slint on Windows

Browse Source
This commit is contained in:
dijunkun
2026-08-25 02:16:14 +08:00
parent 20f8137df2
commit 0d9d385d10
11 changed files with 1150 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/gui/runtime/connection_runtime.cpp
+20 -14
View File
@@ -11,6 +11,9 @@
#include "display_stream_id.h"
#include "localization.h"
#include "platform.h"
#if defined(_WIN32)
#include "platform/windows_opengl_video_renderer.h"
#endif
#ifdef __APPLE__
#include "platform/metal_video_renderer.h"
#endif
@@ -226,7 +229,7 @@ void GuiRuntime::CloseRemoteSession(std::shared_ptr<RemoteSession> props) {
frame_snapshot = props->front_frame_;
video_width = props->video_width_;
video_height = props->video_height_;
#ifdef __APPLE__
#if defined(__APPLE__) || defined(_WIN32)
if ((!frame_snapshot || frame_snapshot->empty()) &&
props->thumbnail_frame_ && !props->thumbnail_frame_->empty()) {
frame_snapshot = props->thumbnail_frame_;
@@ -235,15 +238,18 @@ void GuiRuntime::CloseRemoteSession(std::shared_ptr<RemoteSession> props) {
}
#endif
}
#ifdef __APPLE__
if ((!frame_snapshot || frame_snapshot->empty()) &&
mac_metal_video_renderer_) {
auto metal_snapshot =
std::make_shared<std::vector<unsigned char>>();
if (mac_metal_video_renderer_->CopyLatestNv12(
props->remote_id_, metal_snapshot.get(), &video_width,
&video_height)) {
frame_snapshot = std::move(metal_snapshot);
#if defined(__APPLE__) || defined(_WIN32)
#if defined(_WIN32)
auto* native_renderer = windows_opengl_video_renderer_.get();
#else
auto* native_renderer = mac_metal_video_renderer_.get();
#endif
if ((!frame_snapshot || frame_snapshot->empty()) && native_renderer) {
auto native_snapshot = std::make_shared<std::vector<unsigned char>>();
if (native_renderer->CopyLatestNv12(props->remote_id_,
native_snapshot.get(), &video_width,
&video_height)) {
frame_snapshot = std::move(native_snapshot);
}
}
#endif
@@ -270,9 +276,9 @@ void GuiRuntime::CloseRemoteSession(std::shared_ptr<RemoteSession> props) {
}
}
#ifdef __APPLE__
if (mac_metal_video_renderer_) {
mac_metal_video_renderer_->DiscardStream(props->remote_id_);
#if defined(__APPLE__) || defined(_WIN32)
if (native_renderer) {
native_renderer->DiscardStream(props->remote_id_);
video_frame_dirty_.store(true, std::memory_order_release);
}
#endif
@@ -337,7 +343,7 @@ void GuiRuntime::ResetRemoteSessionResources(
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
props->front_frame_.reset();
props->back_frame_.reset();
#ifdef __APPLE__
#if defined(__APPLE__) || defined(_WIN32)
props->thumbnail_frame_.reset();
props->thumbnail_width_ = 0;
props->thumbnail_height_ = 0;
src/gui/runtime/gui_runtime.cpp
+3
View File
@@ -12,6 +12,9 @@
#include "display_stream_id.h"
#include "localization.h"
#if defined(_WIN32)
#include "platform/windows_opengl_video_renderer.h"
#endif
#ifdef __APPLE__
#include "platform/metal_video_renderer.h"
#endif
src/gui/runtime/gui_runtime.h
+6
View File
@@ -15,6 +15,9 @@
namespace crossdesk {
#if defined(_WIN32)
class WindowsOpenGlVideoRenderer;
#endif
#ifdef __APPLE__
class MacMetalVideoRenderer;
#endif
@@ -87,6 +90,9 @@ class GuiRuntime : protected gui_detail::GuiState {
KeyboardController keyboard_;
PeerEventHandler peer_events_;
std::atomic<bool> video_frame_dirty_{false};
#if defined(_WIN32)
std::unique_ptr<WindowsOpenGlVideoRenderer> windows_opengl_video_renderer_;
#endif
#ifdef __APPLE__
std::unique_ptr<MacMetalVideoRenderer> mac_metal_video_renderer_;
#endif
src/gui/runtime/peer_event_handler.cpp
+35 -20
View File
@@ -16,6 +16,9 @@
#include "file_transfer.h"
#include "localization.h"
#include "platform.h"
#if defined(_WIN32)
#include "platform/windows_opengl_video_renderer.h"
#endif
#ifdef __APPLE__
#include "platform/metal_video_renderer.h"
#endif
@@ -316,22 +319,29 @@ void PeerEventHandler::OnConnectionStatus(ConnectionStatus status,
props->enable_mouse_control_ = false;
runtime->ResetRemoteServiceStatus(*props);
#ifdef __APPLE__
std::shared_ptr<std::vector<unsigned char>> metal_snapshot;
int metal_snapshot_width = 0;
int metal_snapshot_height = 0;
bool needs_metal_snapshot = false;
#if defined(__APPLE__) || defined(_WIN32)
std::shared_ptr<std::vector<unsigned char>> native_snapshot;
int native_snapshot_width = 0;
int native_snapshot_height = 0;
bool needs_native_snapshot = false;
{
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
needs_metal_snapshot = !props->thumbnail_frame_ ||
props->thumbnail_frame_->empty();
needs_native_snapshot = !props->thumbnail_frame_ ||
props->thumbnail_frame_->empty();
}
if (needs_metal_snapshot && runtime->mac_metal_video_renderer_) {
if (needs_native_snapshot) {
#if defined(_WIN32)
auto* native_renderer =
runtime->windows_opengl_video_renderer_.get();
#else
auto* native_renderer = runtime->mac_metal_video_renderer_.get();
#endif
auto snapshot = std::make_shared<std::vector<unsigned char>>();
if (runtime->mac_metal_video_renderer_->CopyLatestNv12(
remote_id, snapshot.get(), &metal_snapshot_width,
&metal_snapshot_height)) {
metal_snapshot = std::move(snapshot);
if (native_renderer && native_renderer->CopyLatestNv12(
remote_id, snapshot.get(),
&native_snapshot_width,
&native_snapshot_height)) {
native_snapshot = std::move(snapshot);
}
}
#endif
@@ -339,12 +349,12 @@ void PeerEventHandler::OnConnectionStatus(ConnectionStatus status,
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
props->front_frame_.reset();
props->back_frame_.reset();
#ifdef __APPLE__
if (metal_snapshot &&
#if defined(__APPLE__) || defined(_WIN32)
if (native_snapshot &&
(!props->thumbnail_frame_ || props->thumbnail_frame_->empty())) {
props->thumbnail_frame_ = std::move(metal_snapshot);
props->thumbnail_width_ = metal_snapshot_width;
props->thumbnail_height_ = metal_snapshot_height;
props->thumbnail_frame_ = std::move(native_snapshot);
props->thumbnail_width_ = native_snapshot_width;
props->thumbnail_height_ = native_snapshot_height;
}
#endif
props->video_width_ = 0;
@@ -353,9 +363,14 @@ void PeerEventHandler::OnConnectionStatus(ConnectionStatus status,
props->render_rect_dirty_ = true;
props->stream_cleanup_pending_ = true;
}
#ifdef __APPLE__
if (runtime->mac_metal_video_renderer_) {
runtime->mac_metal_video_renderer_->DiscardStream(remote_id);
#if defined(__APPLE__) || defined(_WIN32)
#if defined(_WIN32)
auto* native_renderer = runtime->windows_opengl_video_renderer_.get();
#else
auto* native_renderer = runtime->mac_metal_video_renderer_.get();
#endif
if (native_renderer) {
native_renderer->DiscardStream(remote_id);
runtime->video_frame_dirty_.store(true,
std::memory_order_release);
}
src/gui/runtime/peer_media_callbacks.cpp
+32 -20
View File
@@ -7,14 +7,18 @@
#include <vector>
#include "runtime/gui_runtime.h"
#ifdef __APPLE__
#if defined(__APPLE__) || defined(_WIN32)
#include <chrono>
#if defined(_WIN32)
#include "platform/windows_opengl_video_renderer.h"
#else
#include "platform/metal_video_renderer.h"
#endif
#endif
namespace crossdesk {
#ifdef __APPLE__
#if defined(__APPLE__) || defined(_WIN32)
namespace {
constexpr auto kBackgroundSnapshotInterval = std::chrono::seconds(1);
@@ -41,17 +45,24 @@ void PeerEventHandler::OnReceiveVideoBuffer(
runtime->remote_sessions_.find(remote_id)->second.get();
if (props->connection_established_) {
#ifdef __APPLE__
#if defined(__APPLE__) || defined(_WIN32)
bool background_snapshot_only = false;
if (runtime->mac_metal_video_renderer_ &&
runtime->mac_metal_video_renderer_->IsReady() &&
runtime->mac_metal_video_renderer_->IsAttached()) {
const auto submit_result =
runtime->mac_metal_video_renderer_->SubmitNv12(
remote_id,
reinterpret_cast<const uint8_t*>(video_frame->data),
video_frame->size, video_frame->width, video_frame->height);
if (submit_result == MacMetalVideoRenderer::SubmitResult::submitted) {
#if defined(_WIN32)
auto* native_renderer = runtime->windows_opengl_video_renderer_.get();
using NativeVideoRenderer = WindowsOpenGlVideoRenderer;
#else
auto* native_renderer = runtime->mac_metal_video_renderer_.get();
using NativeVideoRenderer = MacMetalVideoRenderer;
#endif
if (native_renderer && native_renderer->IsReady()
#if defined(__APPLE__)
&& native_renderer->IsAttached()
#endif
) {
const auto submit_result = native_renderer->SubmitNv12(
remote_id, reinterpret_cast<const uint8_t*>(video_frame->data),
video_frame->size, video_frame->width, video_frame->height);
if (submit_result == NativeVideoRenderer::SubmitResult::submitted) {
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
const bool size_changed =
(props->video_width_ != video_frame->width) ||
@@ -63,7 +74,7 @@ void PeerEventHandler::OnReceiveVideoBuffer(
props->video_height_ = video_frame->height;
props->video_size_ = video_frame->size;
// Once the native renderer owns the current stream, do not leave an
// older CPU frame ahead of the Metal close snapshot.
// older CPU frame ahead of the renderer's close snapshot.
props->front_frame_.reset();
props->back_frame_.reset();
props->thumbnail_frame_.reset();
@@ -75,21 +86,22 @@ void PeerEventHandler::OnReceiveVideoBuffer(
runtime->video_frame_dirty_.store(true, std::memory_order_release);
return;
}
if (submit_result == MacMetalVideoRenderer::SubmitResult::dropped ||
submit_result == MacMetalVideoRenderer::SubmitResult::failed) {
// Keep presenting the last Metal frame. A later decoded frame can
if (submit_result == NativeVideoRenderer::SubmitResult::dropped ||
submit_result == NativeVideoRenderer::SubmitResult::failed) {
// Keep presenting the last native frame. A later decoded frame can
// reuse the renderer without changing ownership mid-window.
props->streaming_ = true;
return;
}
if (submit_result == MacMetalVideoRenderer::SubmitResult::not_selected) {
if (submit_result ==
NativeVideoRenderer::SubmitResult::not_selected) {
background_snapshot_only = true;
}
}
#endif
{
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
#ifdef __APPLE__
#if defined(__APPLE__) || defined(_WIN32)
const auto now = std::chrono::steady_clock::now();
if (background_snapshot_only && props->thumbnail_frame_ &&
!props->thumbnail_frame_->empty() &&
@@ -125,7 +137,7 @@ void PeerEventHandler::OnReceiveVideoBuffer(
props->video_size_ = video_frame->size;
props->front_frame_.swap(props->back_frame_);
#ifdef __APPLE__
#if defined(__APPLE__) || defined(_WIN32)
props->thumbnail_frame_ = props->front_frame_;
props->thumbnail_width_ = video_frame->width;
props->thumbnail_height_ = video_frame->height;
@@ -137,7 +149,7 @@ void PeerEventHandler::OnReceiveVideoBuffer(
}
props->streaming_ = true;
#ifdef __APPLE__
#if defined(__APPLE__) || defined(_WIN32)
if (background_snapshot_only) {
return;
}
src/gui/runtime/remote_session.h
+1 -1
View File
@@ -110,7 +110,7 @@ struct RemoteSession {
std::mutex video_frame_mutex_;
std::shared_ptr<std::vector<unsigned char>> front_frame_;
std::shared_ptr<std::vector<unsigned char>> back_frame_;
#ifdef __APPLE__
#if defined(__APPLE__) || defined(_WIN32)
// Retains the most recent background CPU frame across disconnect cleanup
// so closing an unselected tab can still update its recent-item thumbnail.
std::shared_ptr<std::vector<unsigned char>> thumbnail_frame_;