From 0d9d385d1051da98004a4945196e0646417a5588 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Tue, 25 Aug 2026 02:16:14 +0800 Subject: [PATCH] [feat] add OpenGL NV12 underlay rendering for Slint on Windows --- src/gui/application/gui_application.cpp | 154 +++- src/gui/application/gui_application.h | 4 + .../windows_opengl_video_renderer.cpp | 812 ++++++++++++++++++ .../platform/windows_opengl_video_renderer.h | 86 ++ src/gui/runtime/connection_runtime.cpp | 34 +- src/gui/runtime/gui_runtime.cpp | 3 + src/gui/runtime/gui_runtime.h | 6 + src/gui/runtime/peer_event_handler.cpp | 55 +- src/gui/runtime/peer_media_callbacks.cpp | 52 +- src/gui/runtime/remote_session.h | 2 +- xmake/targets.lua | 3 +- 11 files changed, 1150 insertions(+), 61 deletions(-) create mode 100644 src/gui/platform/windows_opengl_video_renderer.cpp create mode 100644 src/gui/platform/windows_opengl_video_renderer.h diff --git a/src/gui/application/gui_application.cpp b/src/gui/application/gui_application.cpp index e9aa23d..f077881 100644 --- a/src/gui/application/gui_application.cpp +++ b/src/gui/application/gui_application.cpp @@ -35,6 +35,7 @@ #include #include "platform/tray/win_tray.h" +#include "platform/windows_opengl_video_renderer.h" #elif defined(__APPLE__) #include "platform/metal_video_renderer.h" #include "platform/tray/mac_tray.h" @@ -1043,7 +1044,10 @@ void GuiApplication::InitializeModules() { if (modules_inited_) { return; } -#if defined(__APPLE__) +#if defined(_WIN32) + windows_opengl_video_renderer_ = + std::make_unique(); +#elif defined(__APPLE__) // Metal setup can log device, shader, or pipeline failures. Initialize it // here, after Run() has selected the application's log directory, rather // than from GuiRuntime's constructor before InitializeLogger(). @@ -2277,6 +2281,11 @@ void GuiApplication::SyncStreamWindow() { } if (!has_sessions) { SetStreamKeyboardFocus(false); +#if defined(_WIN32) + if (windows_opengl_video_renderer_) { + windows_opengl_video_renderer_->SetSelectedStream({}); + } +#endif #if defined(__APPLE__) void* stream_view = (*ui_->stream)->window().appkit_view(); SetStreamWindowFullscreen(false); @@ -2564,6 +2573,44 @@ void GuiApplication::SyncStreamVideoFrame() { } #endif +#if defined(_WIN32) + if (windows_opengl_video_renderer_ && + windows_opengl_video_renderer_->IsReady()) { + windows_opengl_video_renderer_->SetSelectedStream(props->remote_id_); + SubmitCachedFrameToOpenGl(props); + if (!(*ui_->stream)->get_native_video_enabled()) { + (*ui_->stream)->set_native_video_enabled(true); + } + + int width = 0; + int height = 0; + uint64_t sequence = 0; + { + std::lock_guard lock(props->video_frame_mutex_); + width = props->video_width_; + height = props->video_height_; + sequence = props->video_frame_sequence_; + } + const bool has_frame = width > 0 && height > 0 && sequence > 0; + (*ui_->stream)->set_has_frame(has_frame); + if (has_frame) { + (*ui_->stream)->set_receiving_text(""); + (*ui_->stream) + ->set_stats_resolution(UiText(std::to_string(width) + "x" + + std::to_string(height))); + if (ui_->displayed_frame_sequence[props->remote_id_] != sequence) { + ui_->displayed_frame_sequence[props->remote_id_] = sequence; + ++props->frame_count_; + } + } + (*ui_->stream)->window().request_redraw(); + return; + } + if ((*ui_->stream)->get_native_video_enabled()) { + (*ui_->stream)->set_native_video_enabled(false); + } +#endif + std::shared_ptr> frame; int width = 0; int height = 0; @@ -2736,9 +2783,36 @@ void GuiApplication::ConfigureStreamVideoRenderer() { glBindTexture(GL_TEXTURE_2D, static_cast(previous_texture)); ui_->video_gl_texture = texture; +#if defined(_WIN32) + if (windows_opengl_video_renderer_ && + windows_opengl_video_renderer_->Setup()) { + windows_opengl_video_renderer_->SetSelectedStream( + focused_remote_id_); + video_frame_dirty_.store(true, std::memory_order_release); + } +#endif return; } +#if defined(_WIN32) + if (state == slint::RenderingState::BeforeRendering && + windows_opengl_video_renderer_ && + windows_opengl_video_renderer_->IsReady() && ui_->stream && + (*ui_->stream)->get_native_video_enabled()) { + const auto window_size = (*ui_->stream)->window().size(); + const float scale_factor = + std::max(1.0f, (*ui_->stream)->window().scale_factor()); + const int top_inset = + !fullscreen_button_pressed_ && ui_->tab_ids.size() > 1 + ? static_cast(std::lround(30.0f * scale_factor)) + : 0; + windows_opengl_video_renderer_->RenderLatest( + focused_remote_id_, static_cast(window_size.width), + static_cast(window_size.height), top_inset); + return; + } +#endif + if (state == slint::RenderingState::BeforeRendering && ui_->video_gl_texture != 0 && ui_->video_gl_pending_dirty && !ui_->video_gl_pending_frame.empty()) { @@ -2769,6 +2843,11 @@ void GuiApplication::ConfigureStreamVideoRenderer() { } if (state == slint::RenderingState::RenderingTeardown) { +#if defined(_WIN32) + if (windows_opengl_video_renderer_) { + windows_opengl_video_renderer_->Teardown(); + } +#endif if (ui_->video_gl_texture != 0) { const GLuint texture = ui_->video_gl_texture; glDeleteTextures(1, &texture); @@ -2783,10 +2862,50 @@ void GuiApplication::ConfigureStreamVideoRenderer() { }); if (error.has_value()) { LOG_WARN("Slint OpenGL video renderer unavailable, using pixel buffers"); +#if defined(_WIN32) + } else if (windows_opengl_video_renderer_) { + windows_opengl_video_renderer_->SetSelectedStream(focused_remote_id_); +#endif } #endif } +#if defined(_WIN32) +void GuiApplication::SubmitCachedFrameToOpenGl( + const std::shared_ptr& session) { + if (!session || !windows_opengl_video_renderer_ || + !windows_opengl_video_renderer_->IsReady() || + session->connection_status_.load() != ConnectionStatus::Connected) { + return; + } + + std::shared_ptr> cached_frame; + int cached_width = 0; + int cached_height = 0; + { + std::lock_guard lock(session->video_frame_mutex_); + if (session->front_frame_ && !session->front_frame_->empty()) { + cached_frame = session->front_frame_; + cached_width = session->video_width_; + cached_height = session->video_height_; + } else if (session->thumbnail_frame_ && + !session->thumbnail_frame_->empty()) { + cached_frame = session->thumbnail_frame_; + cached_width = session->thumbnail_width_; + cached_height = session->thumbnail_height_; + } + } + + if (cached_frame && cached_width > 0 && cached_height > 0 && + windows_opengl_video_renderer_->SubmitCachedNv12( + session->remote_id_, cached_frame->data(), cached_frame->size(), + cached_width, cached_height) == + WindowsOpenGlVideoRenderer::SubmitResult::submitted) { + video_frame_dirty_.store(true, std::memory_order_release); + } +} +#endif + #if defined(__APPLE__) void GuiApplication::SubmitCachedFrameToMetal( const std::shared_ptr& session) { @@ -3124,7 +3243,7 @@ void GuiApplication::SelectStreamTab(int index) { return; } const std::string selected_remote_id = ui_->tab_ids[index]; -#if defined(__APPLE__) +#if defined(__APPLE__) || defined(_WIN32) const bool selection_changed = focused_remote_id_ != selected_remote_id; bool renderer_selection_changed = false; #endif @@ -3134,7 +3253,18 @@ void GuiApplication::SelectStreamTab(int index) { } focused_remote_id_ = selected_remote_id; controlled_remote_id_ = selected_remote_id; -#if defined(__APPLE__) +#if defined(_WIN32) + if (windows_opengl_video_renderer_) { + renderer_selection_changed = + windows_opengl_video_renderer_->SetSelectedStream(selected_remote_id); + } + if (renderer_selection_changed) { + video_frame_dirty_.store(true, std::memory_order_release); + } + if (selection_changed && ui_->stream) { + (*ui_->stream)->set_has_frame(false); + } +#elif defined(__APPLE__) if (mac_metal_video_renderer_) { renderer_selection_changed = mac_metal_video_renderer_->SetSelectedStream(selected_remote_id); @@ -3163,7 +3293,11 @@ void GuiApplication::SelectStreamTab(int index) { selected_session->connection_status_.load() == ConnectionStatus::Connected; } -#if defined(__APPLE__) +#if defined(_WIN32) + if (renderer_selection_changed) { + SubmitCachedFrameToOpenGl(selected_session); + } +#elif defined(__APPLE__) if (renderer_selection_changed) { SubmitCachedFrameToMetal(selected_session); } @@ -3215,7 +3349,12 @@ void GuiApplication::CloseStreamTab(const std::string& remote_id) { if (focused_remote_id_ == remote_id) { focused_remote_id_.clear(); controlled_remote_id_.clear(); -#if defined(__APPLE__) +#if defined(_WIN32) + if (windows_opengl_video_renderer_ && + windows_opengl_video_renderer_->SetSelectedStream({})) { + video_frame_dirty_.store(true, std::memory_order_release); + } +#elif defined(__APPLE__) if (mac_metal_video_renderer_ && mac_metal_video_renderer_->SetSelectedStream({})) { video_frame_dirty_.store(true, std::memory_order_release); @@ -3414,6 +3553,11 @@ void GuiApplication::Cleanup() { WaitForThumbnailSaveTasks(); devices_.DestroyAudioOutput(); if (ui_->stream) { +#if defined(_WIN32) + if (windows_opengl_video_renderer_) { + windows_opengl_video_renderer_->SetSelectedStream({}); + } +#endif #if defined(__APPLE__) void* stream_view = (*ui_->stream)->window().appkit_view(); SetStreamWindowFullscreen(false); diff --git a/src/gui/application/gui_application.h b/src/gui/application/gui_application.h index 0204fb2..d0cbd3f 100644 --- a/src/gui/application/gui_application.h +++ b/src/gui/application/gui_application.h @@ -41,6 +41,10 @@ private: void SyncStreamVideoFrame(); void ScheduleNextVideoFrame(); void ConfigureStreamVideoRenderer(); +#if defined(_WIN32) + void SubmitCachedFrameToOpenGl( + const std::shared_ptr& session); +#endif #if defined(__APPLE__) void SubmitCachedFrameToMetal( const std::shared_ptr& session); diff --git a/src/gui/platform/windows_opengl_video_renderer.cpp b/src/gui/platform/windows_opengl_video_renderer.cpp new file mode 100644 index 0000000..c571cab --- /dev/null +++ b/src/gui/platform/windows_opengl_video_renderer.cpp @@ -0,0 +1,812 @@ +#include "platform/windows_opengl_video_renderer.h" + +#if !defined(_WIN32) +#error "WindowsOpenGlVideoRenderer is only available on Windows" +#endif + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rd_log.h" + +namespace crossdesk { +namespace { + +constexpr size_t kFrameSlotCount = 3; + +enum class SlotUse { + available, + pending, + uploading, +}; + +struct FrameSlot { + std::vector bytes; + int width = 0; + int height = 0; + std::string remote_id; + uint64_t sequence = 0; + SlotUse use = SlotUse::available; + bool valid = false; +}; + +struct SharedFrameState { + mutable std::mutex mutex; + std::array slots; + std::string selected_stream; + uint64_t next_sequence = 1; +}; + +template bool LoadOpenGlFunction(T *function, const char *name) { + PROC address = wglGetProcAddress(name); + if (address == nullptr || address == reinterpret_cast(1) || + address == reinterpret_cast(2) || + address == reinterpret_cast(3) || + address == reinterpret_cast(-1)) { + HMODULE module = GetModuleHandleW(L"opengl32.dll"); + address = module ? GetProcAddress(module, name) : nullptr; + } + *function = reinterpret_cast(address); + return *function != nullptr; +} + +struct OpenGlFunctions { + PFNGLACTIVETEXTUREPROC active_texture = nullptr; + PFNGLATTACHSHADERPROC attach_shader = nullptr; + PFNGLBINDBUFFERPROC bind_buffer = nullptr; + PFNGLBUFFERDATAPROC buffer_data = nullptr; + PFNGLCOMPILESHADERPROC compile_shader = nullptr; + PFNGLCREATEPROGRAMPROC create_program = nullptr; + PFNGLCREATESHADERPROC create_shader = nullptr; + PFNGLDELETEBUFFERSPROC delete_buffers = nullptr; + PFNGLDELETEPROGRAMPROC delete_program = nullptr; + PFNGLDELETESHADERPROC delete_shader = nullptr; + PFNGLDISABLEVERTEXATTRIBARRAYPROC disable_vertex_attrib_array = nullptr; + PFNGLENABLEVERTEXATTRIBARRAYPROC enable_vertex_attrib_array = nullptr; + PFNGLGENBUFFERSPROC gen_buffers = nullptr; + PFNGLGETATTRIBLOCATIONPROC get_attrib_location = nullptr; + PFNGLGETPROGRAMINFOLOGPROC get_program_info_log = nullptr; + PFNGLGETPROGRAMIVPROC get_program_iv = nullptr; + PFNGLGETSHADERINFOLOGPROC get_shader_info_log = nullptr; + PFNGLGETSHADERIVPROC get_shader_iv = nullptr; + PFNGLGETUNIFORMLOCATIONPROC get_uniform_location = nullptr; + PFNGLGETVERTEXATTRIBIVPROC get_vertex_attrib_iv = nullptr; + PFNGLGETVERTEXATTRIBPOINTERVPROC get_vertex_attrib_pointer_v = nullptr; + PFNGLLINKPROGRAMPROC link_program = nullptr; + PFNGLSHADERSOURCEPROC shader_source = nullptr; + PFNGLUNIFORM1IPROC uniform_1i = nullptr; + PFNGLUSEPROGRAMPROC use_program = nullptr; + PFNGLVERTEXATTRIBPOINTERPROC vertex_attrib_pointer = nullptr; + + bool Load() { + return LoadOpenGlFunction(&active_texture, "glActiveTexture") && + LoadOpenGlFunction(&attach_shader, "glAttachShader") && + LoadOpenGlFunction(&bind_buffer, "glBindBuffer") && + LoadOpenGlFunction(&buffer_data, "glBufferData") && + LoadOpenGlFunction(&compile_shader, "glCompileShader") && + LoadOpenGlFunction(&create_program, "glCreateProgram") && + LoadOpenGlFunction(&create_shader, "glCreateShader") && + LoadOpenGlFunction(&delete_buffers, "glDeleteBuffers") && + LoadOpenGlFunction(&delete_program, "glDeleteProgram") && + LoadOpenGlFunction(&delete_shader, "glDeleteShader") && + LoadOpenGlFunction(&disable_vertex_attrib_array, + "glDisableVertexAttribArray") && + LoadOpenGlFunction(&enable_vertex_attrib_array, + "glEnableVertexAttribArray") && + LoadOpenGlFunction(&gen_buffers, "glGenBuffers") && + LoadOpenGlFunction(&get_attrib_location, "glGetAttribLocation") && + LoadOpenGlFunction(&get_program_info_log, "glGetProgramInfoLog") && + LoadOpenGlFunction(&get_program_iv, "glGetProgramiv") && + LoadOpenGlFunction(&get_shader_info_log, "glGetShaderInfoLog") && + LoadOpenGlFunction(&get_shader_iv, "glGetShaderiv") && + LoadOpenGlFunction(&get_uniform_location, "glGetUniformLocation") && + LoadOpenGlFunction(&get_vertex_attrib_iv, "glGetVertexAttribiv") && + LoadOpenGlFunction(&get_vertex_attrib_pointer_v, + "glGetVertexAttribPointerv") && + LoadOpenGlFunction(&link_program, "glLinkProgram") && + LoadOpenGlFunction(&shader_source, "glShaderSource") && + LoadOpenGlFunction(&uniform_1i, "glUniform1i") && + LoadOpenGlFunction(&use_program, "glUseProgram") && + LoadOpenGlFunction(&vertex_attrib_pointer, "glVertexAttribPointer"); + } +}; + +GLuint CompileShader(const OpenGlFunctions &gl, GLenum type, + const char *source) { + const GLuint shader = gl.create_shader(type); + if (shader == 0) { + return 0; + } + gl.shader_source(shader, 1, &source, nullptr); + gl.compile_shader(shader); + + GLint compiled = GL_FALSE; + gl.get_shader_iv(shader, GL_COMPILE_STATUS, &compiled); + if (compiled == GL_TRUE) { + return shader; + } + + GLint log_length = 0; + gl.get_shader_iv(shader, GL_INFO_LOG_LENGTH, &log_length); + std::string log(static_cast(std::max(log_length, 1)), '\0'); + GLsizei written = 0; + gl.get_shader_info_log(shader, static_cast(log.size()), &written, + log.data()); + if (written >= 0 && static_cast(written) < log.size()) { + log.resize(static_cast(written)); + } + LOG_ERROR("OpenGL NV12 {} shader compilation failed: {}", + type == GL_VERTEX_SHADER ? "vertex" : "fragment", log); + gl.delete_shader(shader); + return 0; +} + +bool IsEnabled(GLenum capability) { return glIsEnabled(capability) == GL_TRUE; } + +void RestoreCapability(GLenum capability, bool enabled) { + if (enabled) { + glEnable(capability); + } else { + glDisable(capability); + } +} + +struct VertexAttribState { + GLint enabled = GL_FALSE; + GLint size = 4; + GLint stride = 0; + GLint type = GL_FLOAT; + GLint normalized = GL_FALSE; + GLint buffer = 0; + void *pointer = nullptr; +}; + +VertexAttribState CaptureVertexAttrib(const OpenGlFunctions &gl, GLuint index) { + VertexAttribState state; + gl.get_vertex_attrib_iv(index, GL_VERTEX_ATTRIB_ARRAY_ENABLED, + &state.enabled); + gl.get_vertex_attrib_iv(index, GL_VERTEX_ATTRIB_ARRAY_SIZE, &state.size); + gl.get_vertex_attrib_iv(index, GL_VERTEX_ATTRIB_ARRAY_STRIDE, &state.stride); + gl.get_vertex_attrib_iv(index, GL_VERTEX_ATTRIB_ARRAY_TYPE, &state.type); + gl.get_vertex_attrib_iv(index, GL_VERTEX_ATTRIB_ARRAY_NORMALIZED, + &state.normalized); + gl.get_vertex_attrib_iv(index, GL_VERTEX_ATTRIB_ARRAY_BUFFER_BINDING, + &state.buffer); + gl.get_vertex_attrib_pointer_v(index, GL_VERTEX_ATTRIB_ARRAY_POINTER, + &state.pointer); + return state; +} + +void RestoreVertexAttrib(const OpenGlFunctions &gl, GLuint index, + const VertexAttribState &state) { + if (state.buffer != 0 || state.pointer != nullptr) { + gl.bind_buffer(GL_ARRAY_BUFFER, static_cast(state.buffer)); + gl.vertex_attrib_pointer(index, state.size, static_cast(state.type), + static_cast(state.normalized), + state.stride, state.pointer); + } + if (state.enabled == GL_TRUE) { + gl.enable_vertex_attrib_array(index); + } else { + gl.disable_vertex_attrib_array(index); + } +} + +} // namespace + +struct WindowsOpenGlVideoRenderer::Impl { + OpenGlFunctions gl; + std::shared_ptr frames = + std::make_shared(); + std::atomic ready{false}; + GLuint program = 0; + GLuint vertex_buffer = 0; + GLuint y_texture = 0; + GLuint uv_texture = 0; + GLint position_location = -1; + GLint texcoord_location = -1; + GLint y_texture_location = -1; + GLint uv_texture_location = -1; + int texture_width = 0; + int texture_height = 0; + std::string uploaded_stream; + uint64_t uploaded_sequence = 0; + + void ResetGlHandles() { + program = 0; + vertex_buffer = 0; + y_texture = 0; + uv_texture = 0; + position_location = -1; + texcoord_location = -1; + y_texture_location = -1; + uv_texture_location = -1; + texture_width = 0; + texture_height = 0; + uploaded_stream.clear(); + uploaded_sequence = 0; + } +}; + +WindowsOpenGlVideoRenderer::WindowsOpenGlVideoRenderer() + : impl_(std::make_unique()) {} + +WindowsOpenGlVideoRenderer::~WindowsOpenGlVideoRenderer() = default; + +bool WindowsOpenGlVideoRenderer::Setup() { + if (IsReady()) { + return true; + } + impl_->ready.store(false, std::memory_order_release); + impl_->ResetGlHandles(); + if (!impl_->gl.Load()) { + LOG_WARN("OpenGL NV12 underlay unavailable: required functions missing"); + return false; + } + + static constexpr char kGlesVertexShader[] = R"glsl( +#version 100 +attribute vec2 position; +attribute vec2 texcoord; +varying vec2 video_texcoord; +void main() { + video_texcoord = texcoord; + gl_Position = vec4(position, 0.0, 1.0); +} +)glsl"; + static constexpr char kGlesFragmentShader[] = R"glsl( +#version 100 +#ifdef GL_FRAGMENT_PRECISION_HIGH +precision highp float; +#else +precision mediump float; +#endif +uniform sampler2D y_texture; +uniform sampler2D uv_texture; +varying vec2 video_texcoord; +void main() { + float y = 1.16438356 * + (texture2D(y_texture, video_texcoord).r - 16.0 / 255.0); + vec2 uv = texture2D(uv_texture, video_texcoord).ra - vec2(0.5, 0.5); + vec3 rgb = vec3(y + 1.59602678 * uv.y, + y - 0.39176229 * uv.x - 0.81296764 * uv.y, + y + 2.01723214 * uv.x); + gl_FragColor = vec4(clamp(rgb, 0.0, 1.0), 1.0); +} +)glsl"; + static constexpr char kDesktopVertexShader[] = R"glsl( +#version 110 +attribute vec2 position; +attribute vec2 texcoord; +varying vec2 video_texcoord; +void main() { + video_texcoord = texcoord; + gl_Position = vec4(position, 0.0, 1.0); +} +)glsl"; + static constexpr char kDesktopFragmentShader[] = R"glsl( +#version 110 +uniform sampler2D y_texture; +uniform sampler2D uv_texture; +varying vec2 video_texcoord; +void main() { + float y = 1.16438356 * + (texture2D(y_texture, video_texcoord).r - 16.0 / 255.0); + vec2 uv = texture2D(uv_texture, video_texcoord).ra - vec2(0.5, 0.5); + vec3 rgb = vec3(y + 1.59602678 * uv.y, + y - 0.39176229 * uv.x - 0.81296764 * uv.y, + y + 2.01723214 * uv.x); + gl_FragColor = vec4(clamp(rgb, 0.0, 1.0), 1.0); +} +)glsl"; + + const auto *version = reinterpret_cast(glGetString(GL_VERSION)); + const bool is_gles = version && std::strstr(version, "OpenGL ES") != nullptr; + const char *vertex_source = + is_gles ? kGlesVertexShader : kDesktopVertexShader; + const char *fragment_source = + is_gles ? kGlesFragmentShader : kDesktopFragmentShader; + + const GLuint vertex_shader = + CompileShader(impl_->gl, GL_VERTEX_SHADER, vertex_source); + const GLuint fragment_shader = + CompileShader(impl_->gl, GL_FRAGMENT_SHADER, fragment_source); + if (vertex_shader == 0 || fragment_shader == 0) { + if (vertex_shader != 0) + impl_->gl.delete_shader(vertex_shader); + if (fragment_shader != 0) + impl_->gl.delete_shader(fragment_shader); + return false; + } + + impl_->program = impl_->gl.create_program(); + impl_->gl.attach_shader(impl_->program, vertex_shader); + impl_->gl.attach_shader(impl_->program, fragment_shader); + impl_->gl.link_program(impl_->program); + impl_->gl.delete_shader(vertex_shader); + impl_->gl.delete_shader(fragment_shader); + + GLint linked = GL_FALSE; + impl_->gl.get_program_iv(impl_->program, GL_LINK_STATUS, &linked); + if (linked != GL_TRUE) { + GLint log_length = 0; + impl_->gl.get_program_iv(impl_->program, GL_INFO_LOG_LENGTH, &log_length); + std::string log(static_cast(std::max(log_length, 1)), '\0'); + GLsizei written = 0; + impl_->gl.get_program_info_log( + impl_->program, static_cast(log.size()), &written, log.data()); + if (written >= 0 && static_cast(written) < log.size()) { + log.resize(static_cast(written)); + } + LOG_ERROR("OpenGL NV12 program link failed: {}", log); + impl_->gl.delete_program(impl_->program); + impl_->ResetGlHandles(); + return false; + } + + impl_->position_location = + impl_->gl.get_attrib_location(impl_->program, "position"); + impl_->texcoord_location = + impl_->gl.get_attrib_location(impl_->program, "texcoord"); + impl_->y_texture_location = + impl_->gl.get_uniform_location(impl_->program, "y_texture"); + impl_->uv_texture_location = + impl_->gl.get_uniform_location(impl_->program, "uv_texture"); + if (impl_->position_location < 0 || impl_->texcoord_location < 0 || + impl_->y_texture_location < 0 || impl_->uv_texture_location < 0) { + LOG_ERROR("OpenGL NV12 program is missing required shader bindings"); + impl_->gl.delete_program(impl_->program); + impl_->ResetGlHandles(); + return false; + } + + static constexpr GLfloat kVertices[] = { + -1.0f, 1.0f, 0.0f, 0.0f, // top-left + -1.0f, -1.0f, 0.0f, 1.0f, // bottom-left + 1.0f, 1.0f, 1.0f, 0.0f, // top-right + 1.0f, -1.0f, 1.0f, 1.0f, // bottom-right + }; + GLint previous_array_buffer = 0; + glGetIntegerv(GL_ARRAY_BUFFER_BINDING, &previous_array_buffer); + impl_->gl.gen_buffers(1, &impl_->vertex_buffer); + impl_->gl.bind_buffer(GL_ARRAY_BUFFER, impl_->vertex_buffer); + impl_->gl.buffer_data(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices, + GL_STATIC_DRAW); + impl_->gl.bind_buffer(GL_ARRAY_BUFFER, + static_cast(previous_array_buffer)); + + GLint previous_active_texture = GL_TEXTURE0; + glGetIntegerv(GL_ACTIVE_TEXTURE, &previous_active_texture); + impl_->gl.active_texture(GL_TEXTURE0); + GLint previous_texture_0 = 0; + glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture_0); + impl_->gl.active_texture(GL_TEXTURE1); + GLint previous_texture_1 = 0; + glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture_1); + while (glGetError() != GL_NO_ERROR) { + } + glGenTextures(1, &impl_->y_texture); + glGenTextures(1, &impl_->uv_texture); + for (const auto [unit, texture] : + {std::pair{GL_TEXTURE0, impl_->y_texture}, + std::pair{GL_TEXTURE1, impl_->uv_texture}}) { + impl_->gl.active_texture(unit); + glBindTexture(GL_TEXTURE_2D, texture); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + } + impl_->gl.active_texture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, static_cast(previous_texture_0)); + impl_->gl.active_texture(GL_TEXTURE1); + glBindTexture(GL_TEXTURE_2D, static_cast(previous_texture_1)); + impl_->gl.active_texture(static_cast(previous_active_texture)); + + const GLenum setup_error = glGetError(); + if (impl_->program == 0 || impl_->vertex_buffer == 0 || + impl_->y_texture == 0 || impl_->uv_texture == 0 || + setup_error != GL_NO_ERROR) { + LOG_ERROR("OpenGL NV12 resource setup failed, error={}", setup_error); + Teardown(); + return false; + } + + impl_->ready.store(true, std::memory_order_release); + LOG_INFO("OpenGL NV12 underlay initialized ({})", + version ? version : "unknown OpenGL version"); + return true; +} + +void WindowsOpenGlVideoRenderer::Teardown() { + impl_->ready.store(false, std::memory_order_release); + if (impl_->y_texture != 0) + glDeleteTextures(1, &impl_->y_texture); + if (impl_->uv_texture != 0) + glDeleteTextures(1, &impl_->uv_texture); + if (impl_->vertex_buffer != 0 && impl_->gl.delete_buffers) { + impl_->gl.delete_buffers(1, &impl_->vertex_buffer); + } + if (impl_->program != 0 && impl_->gl.delete_program) { + impl_->gl.delete_program(impl_->program); + } + impl_->ResetGlHandles(); +} + +bool WindowsOpenGlVideoRenderer::IsReady() const { + return impl_->ready.load(std::memory_order_acquire); +} + +bool WindowsOpenGlVideoRenderer::SetSelectedStream(std::string remote_id) { + std::lock_guard lock(impl_->frames->mutex); + if (impl_->frames->selected_stream == remote_id) { + return false; + } + impl_->frames->selected_stream = std::move(remote_id); + for (auto &slot : impl_->frames->slots) { + if (slot.use == SlotUse::pending && + slot.remote_id != impl_->frames->selected_stream) { + slot.use = SlotUse::available; + } + } + return true; +} + +void WindowsOpenGlVideoRenderer::DiscardStream(std::string_view remote_id) { + std::lock_guard lock(impl_->frames->mutex); + for (auto &slot : impl_->frames->slots) { + if (slot.remote_id == remote_id && slot.use != SlotUse::uploading) { + slot.valid = false; + slot.use = SlotUse::available; + slot.remote_id.clear(); + } + } + if (impl_->frames->selected_stream == remote_id) { + impl_->frames->selected_stream.clear(); + } +} + +WindowsOpenGlVideoRenderer::SubmitResult +WindowsOpenGlVideoRenderer::SubmitNv12(std::string_view remote_id, + const uint8_t *data, size_t size, + int width, int height) { + return SubmitNv12Internal(remote_id, data, size, width, height, true); +} + +WindowsOpenGlVideoRenderer::SubmitResult +WindowsOpenGlVideoRenderer::SubmitCachedNv12(std::string_view remote_id, + const uint8_t *data, size_t size, + int width, int height) { + return SubmitNv12Internal(remote_id, data, size, width, height, false); +} + +WindowsOpenGlVideoRenderer::SubmitResult +WindowsOpenGlVideoRenderer::SubmitNv12Internal(std::string_view remote_id, + const uint8_t *data, size_t size, + int width, int height, + bool replace_pending) { + if (!IsReady()) { + return SubmitResult::failed; + } + if (data == nullptr || width <= 0 || height <= 0 || (width & 1) != 0 || + (height & 1) != 0) { + return SubmitResult::failed; + } + const size_t y_size = static_cast(width) * height; + const size_t required_size = y_size + y_size / 2; + if (size < required_size) { + return SubmitResult::failed; + } + + auto &frames = *impl_->frames; + std::lock_guard lock(frames.mutex); + if (frames.selected_stream != remote_id) { + return SubmitResult::not_selected; + } + + FrameSlot *target = nullptr; + if (replace_pending) { + for (auto &slot : frames.slots) { + if (slot.use == SlotUse::pending) { + target = &slot; + break; + } + } + } else { + for (const auto &slot : frames.slots) { + if (slot.valid && slot.remote_id == remote_id && + (slot.use == SlotUse::pending || slot.use == SlotUse::uploading)) { + return SubmitResult::dropped; + } + } + } + if (target == nullptr) { + uint64_t oldest_sequence = std::numeric_limits::max(); + for (auto &slot : frames.slots) { + if (slot.use != SlotUse::available) { + continue; + } + if (!slot.valid) { + target = &slot; + break; + } + if (slot.sequence < oldest_sequence) { + oldest_sequence = slot.sequence; + target = &slot; + } + } + } + if (target == nullptr) { + return SubmitResult::dropped; + } + + target->bytes.resize(required_size); + std::memcpy(target->bytes.data(), data, required_size); + target->width = width; + target->height = height; + target->remote_id.assign(remote_id); + target->sequence = frames.next_sequence++; + target->use = SlotUse::pending; + target->valid = true; + return SubmitResult::submitted; +} + +WindowsOpenGlVideoRenderer::RenderOutcome +WindowsOpenGlVideoRenderer::RenderLatest(std::string_view remote_id, + int target_width, int target_height, + int top_inset_pixels) { + if (!IsReady() || target_width <= 0 || target_height <= 0) { + return {RenderResult::failed}; + } + + size_t slot_index = kFrameSlotCount; + uint64_t sequence = 0; + int source_width = 0; + int source_height = 0; + const unsigned char *frame_data = nullptr; + { + std::lock_guard lock(impl_->frames->mutex); + for (size_t index = 0; index < impl_->frames->slots.size(); ++index) { + const auto &slot = impl_->frames->slots[index]; + if (slot.valid && slot.use == SlotUse::pending && + slot.remote_id == remote_id && + (slot_index == kFrameSlotCount || slot.sequence > sequence)) { + slot_index = index; + sequence = slot.sequence; + } + } + if (slot_index != kFrameSlotCount) { + auto &selected = impl_->frames->slots[slot_index]; + selected.use = SlotUse::uploading; + source_width = selected.width; + source_height = selected.height; + frame_data = selected.bytes.data(); + for (size_t index = 0; index < impl_->frames->slots.size(); ++index) { + auto &slot = impl_->frames->slots[index]; + if (index != slot_index && slot.use == SlotUse::pending && + slot.remote_id == remote_id) { + slot.use = SlotUse::available; + } + } + } + } + + GLint previous_program = 0; + GLint previous_active_texture = GL_TEXTURE0; + GLint previous_array_buffer = 0; + GLint previous_unpack_alignment = 4; + GLint previous_viewport[4] = {}; + GLint previous_scissor_box[4] = {}; + GLfloat previous_clear_color[4] = {}; + GLboolean previous_color_mask[4] = {}; + glGetIntegerv(GL_CURRENT_PROGRAM, &previous_program); + glGetIntegerv(GL_ACTIVE_TEXTURE, &previous_active_texture); + glGetIntegerv(GL_ARRAY_BUFFER_BINDING, &previous_array_buffer); + glGetIntegerv(GL_UNPACK_ALIGNMENT, &previous_unpack_alignment); + glGetIntegerv(GL_VIEWPORT, previous_viewport); + glGetIntegerv(GL_SCISSOR_BOX, previous_scissor_box); + glGetFloatv(GL_COLOR_CLEAR_VALUE, previous_clear_color); + glGetBooleanv(GL_COLOR_WRITEMASK, previous_color_mask); + const bool blend_enabled = IsEnabled(GL_BLEND); + const bool cull_enabled = IsEnabled(GL_CULL_FACE); + const bool depth_enabled = IsEnabled(GL_DEPTH_TEST); + const bool scissor_enabled = IsEnabled(GL_SCISSOR_TEST); + const bool stencil_enabled = IsEnabled(GL_STENCIL_TEST); + + impl_->gl.active_texture(GL_TEXTURE0); + GLint previous_texture_0 = 0; + glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture_0); + impl_->gl.active_texture(GL_TEXTURE1); + GLint previous_texture_1 = 0; + glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture_1); + + const auto position_state = CaptureVertexAttrib( + impl_->gl, static_cast(impl_->position_location)); + const auto texcoord_state = CaptureVertexAttrib( + impl_->gl, static_cast(impl_->texcoord_location)); + + glDisable(GL_BLEND); + glDisable(GL_CULL_FACE); + glDisable(GL_DEPTH_TEST); + glDisable(GL_SCISSOR_TEST); + glDisable(GL_STENCIL_TEST); + glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE); + glViewport(0, 0, target_width, target_height); + glClearColor(0.0f, 0.0f, 0.0f, 1.0f); + glClear(GL_COLOR_BUFFER_BIT); + + bool upload_succeeded = true; + if (slot_index != kFrameSlotCount) { + while (glGetError() != GL_NO_ERROR) { + } + const size_t y_size = static_cast(source_width) * source_height; + glPixelStorei(GL_UNPACK_ALIGNMENT, 1); + impl_->gl.active_texture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, impl_->y_texture); + impl_->gl.active_texture(GL_TEXTURE1); + glBindTexture(GL_TEXTURE_2D, impl_->uv_texture); + if (impl_->texture_width != source_width || + impl_->texture_height != source_height) { + impl_->gl.active_texture(GL_TEXTURE0); + glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, source_width, source_height, + 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, frame_data); + impl_->gl.active_texture(GL_TEXTURE1); + glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE_ALPHA, source_width / 2, + source_height / 2, 0, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, + frame_data + y_size); + } else { + impl_->gl.active_texture(GL_TEXTURE0); + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, source_width, source_height, + GL_LUMINANCE, GL_UNSIGNED_BYTE, frame_data); + impl_->gl.active_texture(GL_TEXTURE1); + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, source_width / 2, + source_height / 2, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, + frame_data + y_size); + } + const GLenum upload_error = glGetError(); + upload_succeeded = upload_error == GL_NO_ERROR; + if (upload_succeeded) { + impl_->texture_width = source_width; + impl_->texture_height = source_height; + impl_->uploaded_stream.assign(remote_id); + impl_->uploaded_sequence = sequence; + } else { + LOG_WARN("OpenGL NV12 texture upload failed, error={}", upload_error); + } + } + + { + std::lock_guard lock(impl_->frames->mutex); + if (slot_index != kFrameSlotCount) { + auto &slot = impl_->frames->slots[slot_index]; + if (slot.sequence == sequence && slot.use == SlotUse::uploading) { + slot.use = SlotUse::available; + } + } + } + + RenderOutcome outcome{RenderResult::empty}; + if (upload_succeeded && impl_->uploaded_stream == remote_id && + impl_->texture_width > 0 && impl_->texture_height > 0) { + source_width = impl_->texture_width; + source_height = impl_->texture_height; + sequence = impl_->uploaded_sequence; + const int video_height = std::max( + 0, target_height - std::clamp(top_inset_pixels, 0, target_height)); + if (video_height > 0) { + const double source_aspect = + static_cast(source_width) / source_height; + const double target_aspect = + static_cast(target_width) / video_height; + int viewport_width = target_width; + int viewport_height = video_height; + int viewport_x = 0; + int viewport_y = 0; + if (source_aspect > target_aspect) { + viewport_height = + std::max(1, static_cast(target_width / source_aspect + 0.5)); + viewport_y = (video_height - viewport_height) / 2; + } else { + viewport_width = + std::max(1, static_cast(video_height * source_aspect + 0.5)); + viewport_x = (target_width - viewport_width) / 2; + } + + glViewport(viewport_x, viewport_y, viewport_width, viewport_height); + impl_->gl.use_program(impl_->program); + impl_->gl.active_texture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, impl_->y_texture); + impl_->gl.uniform_1i(impl_->y_texture_location, 0); + impl_->gl.active_texture(GL_TEXTURE1); + glBindTexture(GL_TEXTURE_2D, impl_->uv_texture); + impl_->gl.uniform_1i(impl_->uv_texture_location, 1); + impl_->gl.bind_buffer(GL_ARRAY_BUFFER, impl_->vertex_buffer); + impl_->gl.vertex_attrib_pointer( + static_cast(impl_->position_location), 2, GL_FLOAT, GL_FALSE, + 4 * static_cast(sizeof(GLfloat)), nullptr); + impl_->gl.vertex_attrib_pointer( + static_cast(impl_->texcoord_location), 2, GL_FLOAT, GL_FALSE, + 4 * static_cast(sizeof(GLfloat)), + reinterpret_cast(2 * sizeof(GLfloat))); + impl_->gl.enable_vertex_attrib_array( + static_cast(impl_->position_location)); + impl_->gl.enable_vertex_attrib_array( + static_cast(impl_->texcoord_location)); + glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + outcome = {glGetError() == GL_NO_ERROR ? RenderResult::rendered + : RenderResult::failed, + source_width, source_height, sequence}; + } + } + + RestoreVertexAttrib(impl_->gl, static_cast(impl_->position_location), + position_state); + RestoreVertexAttrib(impl_->gl, static_cast(impl_->texcoord_location), + texcoord_state); + impl_->gl.bind_buffer(GL_ARRAY_BUFFER, + static_cast(previous_array_buffer)); + impl_->gl.active_texture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, static_cast(previous_texture_0)); + impl_->gl.active_texture(GL_TEXTURE1); + glBindTexture(GL_TEXTURE_2D, static_cast(previous_texture_1)); + impl_->gl.active_texture(static_cast(previous_active_texture)); + impl_->gl.use_program(static_cast(previous_program)); + glPixelStorei(GL_UNPACK_ALIGNMENT, previous_unpack_alignment); + glViewport(previous_viewport[0], previous_viewport[1], previous_viewport[2], + previous_viewport[3]); + glScissor(previous_scissor_box[0], previous_scissor_box[1], + previous_scissor_box[2], previous_scissor_box[3]); + glClearColor(previous_clear_color[0], previous_clear_color[1], + previous_clear_color[2], previous_clear_color[3]); + glColorMask(previous_color_mask[0], previous_color_mask[1], + previous_color_mask[2], previous_color_mask[3]); + RestoreCapability(GL_BLEND, blend_enabled); + RestoreCapability(GL_CULL_FACE, cull_enabled); + RestoreCapability(GL_DEPTH_TEST, depth_enabled); + RestoreCapability(GL_SCISSOR_TEST, scissor_enabled); + RestoreCapability(GL_STENCIL_TEST, stencil_enabled); + const GLenum restore_error = glGetError(); + if (restore_error != GL_NO_ERROR) { + LOG_WARN("OpenGL NV12 state restoration failed, error={}", restore_error); + if (outcome.result == RenderResult::rendered) { + outcome.result = RenderResult::failed; + } + } + return outcome; +} + +bool WindowsOpenGlVideoRenderer::CopyLatestNv12( + std::string_view remote_id, std::vector *output, int *width, + int *height) const { + if (!output || !width || !height) { + return false; + } + std::lock_guard lock(impl_->frames->mutex); + const FrameSlot *newest = nullptr; + for (const auto &slot : impl_->frames->slots) { + if (slot.valid && slot.remote_id == remote_id && + (!newest || slot.sequence > newest->sequence)) { + newest = &slot; + } + } + if (!newest || newest->bytes.empty()) { + return false; + } + *output = newest->bytes; + *width = newest->width; + *height = newest->height; + return true; +} + +} // namespace crossdesk diff --git a/src/gui/platform/windows_opengl_video_renderer.h b/src/gui/platform/windows_opengl_video_renderer.h new file mode 100644 index 0000000..fa96284 --- /dev/null +++ b/src/gui/platform/windows_opengl_video_renderer.h @@ -0,0 +1,86 @@ +#ifndef CROSSDESK_GUI_PLATFORM_WINDOWS_OPENGL_VIDEO_RENDERER_H_ +#define CROSSDESK_GUI_PLATFORM_WINDOWS_OPENGL_VIDEO_RENDERER_H_ + +#include +#include +#include +#include +#include +#include + +namespace crossdesk { + +// Windows NV12 underlay for Slint's FemtoVG OpenGL renderer. MiniRTC submits +// tightly packed CPU NV12 frames from its decode callback thread. The Slint UI +// thread uploads the newest frame as Y and UV textures and performs color +// conversion and scaling in a fragment shader before Slint draws its overlay. +class WindowsOpenGlVideoRenderer { +public: + enum class SubmitResult { + submitted, + not_selected, + dropped, + failed, + }; + + enum class RenderResult { + rendered, + idle, + empty, + failed, + }; + + struct RenderOutcome { + RenderResult result = RenderResult::failed; + int width = 0; + int height = 0; + uint64_t sequence = 0; + }; + + WindowsOpenGlVideoRenderer(); + ~WindowsOpenGlVideoRenderer(); + + WindowsOpenGlVideoRenderer(const WindowsOpenGlVideoRenderer &) = delete; + WindowsOpenGlVideoRenderer & + operator=(const WindowsOpenGlVideoRenderer &) = delete; + + // These methods must run while Slint's OpenGL context is current, from the + // RenderingSetup and RenderingTeardown notifier states respectively. + bool Setup(); + void Teardown(); + bool IsReady() const; + + // Only the selected stream is queued for upload. Frames for other tabs are + // retained by GuiRuntime at a throttled rate for thumbnail/tab restoration. + bool SetSelectedStream(std::string remote_id); + void DiscardStream(std::string_view remote_id); + + // Thread-safe; called from MiniRTC's decode callback thread. + SubmitResult SubmitNv12(std::string_view remote_id, const uint8_t *data, + size_t size, int width, int height); + SubmitResult SubmitCachedNv12(std::string_view remote_id, const uint8_t *data, + size_t size, int width, int height); + + // Draws below Slint while its OpenGL context is current. target dimensions + // and top_inset_pixels are physical pixels in the window client area. + RenderOutcome RenderLatest(std::string_view remote_id, int target_width, + int target_height, int top_inset_pixels); + + // Used during disconnect cleanup so thumbnail generation does not require a + // second per-frame CPU copy during normal playback. + bool CopyLatestNv12(std::string_view remote_id, + std::vector *output, int *width, + int *height) const; + +private: + SubmitResult SubmitNv12Internal(std::string_view remote_id, + const uint8_t *data, size_t size, int width, + int height, bool replace_pending); + + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace crossdesk + +#endif // CROSSDESK_GUI_PLATFORM_WINDOWS_OPENGL_VIDEO_RENDERER_H_ diff --git a/src/gui/runtime/connection_runtime.cpp b/src/gui/runtime/connection_runtime.cpp index e3ef122..d6ec5de 100644 --- a/src/gui/runtime/connection_runtime.cpp +++ b/src/gui/runtime/connection_runtime.cpp @@ -11,6 +11,9 @@ #include "display_stream_id.h" #include "localization.h" #include "platform.h" +#if defined(_WIN32) +#include "platform/windows_opengl_video_renderer.h" +#endif #ifdef __APPLE__ #include "platform/metal_video_renderer.h" #endif @@ -226,7 +229,7 @@ void GuiRuntime::CloseRemoteSession(std::shared_ptr props) { frame_snapshot = props->front_frame_; video_width = props->video_width_; video_height = props->video_height_; -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(_WIN32) if ((!frame_snapshot || frame_snapshot->empty()) && props->thumbnail_frame_ && !props->thumbnail_frame_->empty()) { frame_snapshot = props->thumbnail_frame_; @@ -235,15 +238,18 @@ void GuiRuntime::CloseRemoteSession(std::shared_ptr props) { } #endif } -#ifdef __APPLE__ - if ((!frame_snapshot || frame_snapshot->empty()) && - mac_metal_video_renderer_) { - auto metal_snapshot = - std::make_shared>(); - if (mac_metal_video_renderer_->CopyLatestNv12( - props->remote_id_, metal_snapshot.get(), &video_width, - &video_height)) { - frame_snapshot = std::move(metal_snapshot); +#if defined(__APPLE__) || defined(_WIN32) +#if defined(_WIN32) + auto* native_renderer = windows_opengl_video_renderer_.get(); +#else + auto* native_renderer = mac_metal_video_renderer_.get(); +#endif + if ((!frame_snapshot || frame_snapshot->empty()) && native_renderer) { + auto native_snapshot = std::make_shared>(); + if (native_renderer->CopyLatestNv12(props->remote_id_, + native_snapshot.get(), &video_width, + &video_height)) { + frame_snapshot = std::move(native_snapshot); } } #endif @@ -270,9 +276,9 @@ void GuiRuntime::CloseRemoteSession(std::shared_ptr props) { } } -#ifdef __APPLE__ - if (mac_metal_video_renderer_) { - mac_metal_video_renderer_->DiscardStream(props->remote_id_); +#if defined(__APPLE__) || defined(_WIN32) + if (native_renderer) { + native_renderer->DiscardStream(props->remote_id_); video_frame_dirty_.store(true, std::memory_order_release); } #endif @@ -337,7 +343,7 @@ void GuiRuntime::ResetRemoteSessionResources( std::lock_guard lock(props->video_frame_mutex_); props->front_frame_.reset(); props->back_frame_.reset(); -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(_WIN32) props->thumbnail_frame_.reset(); props->thumbnail_width_ = 0; props->thumbnail_height_ = 0; diff --git a/src/gui/runtime/gui_runtime.cpp b/src/gui/runtime/gui_runtime.cpp index a6accd3..9070dd3 100644 --- a/src/gui/runtime/gui_runtime.cpp +++ b/src/gui/runtime/gui_runtime.cpp @@ -12,6 +12,9 @@ #include "display_stream_id.h" #include "localization.h" +#if defined(_WIN32) +#include "platform/windows_opengl_video_renderer.h" +#endif #ifdef __APPLE__ #include "platform/metal_video_renderer.h" #endif diff --git a/src/gui/runtime/gui_runtime.h b/src/gui/runtime/gui_runtime.h index 49937e6..635f184 100644 --- a/src/gui/runtime/gui_runtime.h +++ b/src/gui/runtime/gui_runtime.h @@ -15,6 +15,9 @@ namespace crossdesk { +#if defined(_WIN32) +class WindowsOpenGlVideoRenderer; +#endif #ifdef __APPLE__ class MacMetalVideoRenderer; #endif @@ -87,6 +90,9 @@ class GuiRuntime : protected gui_detail::GuiState { KeyboardController keyboard_; PeerEventHandler peer_events_; std::atomic video_frame_dirty_{false}; +#if defined(_WIN32) + std::unique_ptr windows_opengl_video_renderer_; +#endif #ifdef __APPLE__ std::unique_ptr mac_metal_video_renderer_; #endif diff --git a/src/gui/runtime/peer_event_handler.cpp b/src/gui/runtime/peer_event_handler.cpp index d3dc0e2..d7dcbea 100644 --- a/src/gui/runtime/peer_event_handler.cpp +++ b/src/gui/runtime/peer_event_handler.cpp @@ -16,6 +16,9 @@ #include "file_transfer.h" #include "localization.h" #include "platform.h" +#if defined(_WIN32) +#include "platform/windows_opengl_video_renderer.h" +#endif #ifdef __APPLE__ #include "platform/metal_video_renderer.h" #endif @@ -316,22 +319,29 @@ void PeerEventHandler::OnConnectionStatus(ConnectionStatus status, props->enable_mouse_control_ = false; runtime->ResetRemoteServiceStatus(*props); -#ifdef __APPLE__ - std::shared_ptr> metal_snapshot; - int metal_snapshot_width = 0; - int metal_snapshot_height = 0; - bool needs_metal_snapshot = false; +#if defined(__APPLE__) || defined(_WIN32) + std::shared_ptr> native_snapshot; + int native_snapshot_width = 0; + int native_snapshot_height = 0; + bool needs_native_snapshot = false; { std::lock_guard lock(props->video_frame_mutex_); - needs_metal_snapshot = !props->thumbnail_frame_ || - props->thumbnail_frame_->empty(); + needs_native_snapshot = !props->thumbnail_frame_ || + props->thumbnail_frame_->empty(); } - if (needs_metal_snapshot && runtime->mac_metal_video_renderer_) { + if (needs_native_snapshot) { +#if defined(_WIN32) + auto* native_renderer = + runtime->windows_opengl_video_renderer_.get(); +#else + auto* native_renderer = runtime->mac_metal_video_renderer_.get(); +#endif auto snapshot = std::make_shared>(); - if (runtime->mac_metal_video_renderer_->CopyLatestNv12( - remote_id, snapshot.get(), &metal_snapshot_width, - &metal_snapshot_height)) { - metal_snapshot = std::move(snapshot); + if (native_renderer && native_renderer->CopyLatestNv12( + remote_id, snapshot.get(), + &native_snapshot_width, + &native_snapshot_height)) { + native_snapshot = std::move(snapshot); } } #endif @@ -339,12 +349,12 @@ void PeerEventHandler::OnConnectionStatus(ConnectionStatus status, std::lock_guard lock(props->video_frame_mutex_); props->front_frame_.reset(); props->back_frame_.reset(); -#ifdef __APPLE__ - if (metal_snapshot && +#if defined(__APPLE__) || defined(_WIN32) + if (native_snapshot && (!props->thumbnail_frame_ || props->thumbnail_frame_->empty())) { - props->thumbnail_frame_ = std::move(metal_snapshot); - props->thumbnail_width_ = metal_snapshot_width; - props->thumbnail_height_ = metal_snapshot_height; + props->thumbnail_frame_ = std::move(native_snapshot); + props->thumbnail_width_ = native_snapshot_width; + props->thumbnail_height_ = native_snapshot_height; } #endif props->video_width_ = 0; @@ -353,9 +363,14 @@ void PeerEventHandler::OnConnectionStatus(ConnectionStatus status, props->render_rect_dirty_ = true; props->stream_cleanup_pending_ = true; } -#ifdef __APPLE__ - if (runtime->mac_metal_video_renderer_) { - runtime->mac_metal_video_renderer_->DiscardStream(remote_id); +#if defined(__APPLE__) || defined(_WIN32) +#if defined(_WIN32) + auto* native_renderer = runtime->windows_opengl_video_renderer_.get(); +#else + auto* native_renderer = runtime->mac_metal_video_renderer_.get(); +#endif + if (native_renderer) { + native_renderer->DiscardStream(remote_id); runtime->video_frame_dirty_.store(true, std::memory_order_release); } diff --git a/src/gui/runtime/peer_media_callbacks.cpp b/src/gui/runtime/peer_media_callbacks.cpp index 0e2b214..8964aeb 100644 --- a/src/gui/runtime/peer_media_callbacks.cpp +++ b/src/gui/runtime/peer_media_callbacks.cpp @@ -7,14 +7,18 @@ #include #include "runtime/gui_runtime.h" -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(_WIN32) #include +#if defined(_WIN32) +#include "platform/windows_opengl_video_renderer.h" +#else #include "platform/metal_video_renderer.h" #endif +#endif namespace crossdesk { -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(_WIN32) namespace { constexpr auto kBackgroundSnapshotInterval = std::chrono::seconds(1); @@ -41,17 +45,24 @@ void PeerEventHandler::OnReceiveVideoBuffer( runtime->remote_sessions_.find(remote_id)->second.get(); if (props->connection_established_) { -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(_WIN32) bool background_snapshot_only = false; - if (runtime->mac_metal_video_renderer_ && - runtime->mac_metal_video_renderer_->IsReady() && - runtime->mac_metal_video_renderer_->IsAttached()) { - const auto submit_result = - runtime->mac_metal_video_renderer_->SubmitNv12( - remote_id, - reinterpret_cast(video_frame->data), - video_frame->size, video_frame->width, video_frame->height); - if (submit_result == MacMetalVideoRenderer::SubmitResult::submitted) { +#if defined(_WIN32) + auto* native_renderer = runtime->windows_opengl_video_renderer_.get(); + using NativeVideoRenderer = WindowsOpenGlVideoRenderer; +#else + auto* native_renderer = runtime->mac_metal_video_renderer_.get(); + using NativeVideoRenderer = MacMetalVideoRenderer; +#endif + if (native_renderer && native_renderer->IsReady() +#if defined(__APPLE__) + && native_renderer->IsAttached() +#endif + ) { + const auto submit_result = native_renderer->SubmitNv12( + remote_id, reinterpret_cast(video_frame->data), + video_frame->size, video_frame->width, video_frame->height); + if (submit_result == NativeVideoRenderer::SubmitResult::submitted) { std::lock_guard lock(props->video_frame_mutex_); const bool size_changed = (props->video_width_ != video_frame->width) || @@ -63,7 +74,7 @@ void PeerEventHandler::OnReceiveVideoBuffer( props->video_height_ = video_frame->height; props->video_size_ = video_frame->size; // Once the native renderer owns the current stream, do not leave an - // older CPU frame ahead of the Metal close snapshot. + // older CPU frame ahead of the renderer's close snapshot. props->front_frame_.reset(); props->back_frame_.reset(); props->thumbnail_frame_.reset(); @@ -75,21 +86,22 @@ void PeerEventHandler::OnReceiveVideoBuffer( runtime->video_frame_dirty_.store(true, std::memory_order_release); return; } - if (submit_result == MacMetalVideoRenderer::SubmitResult::dropped || - submit_result == MacMetalVideoRenderer::SubmitResult::failed) { - // Keep presenting the last Metal frame. A later decoded frame can + if (submit_result == NativeVideoRenderer::SubmitResult::dropped || + submit_result == NativeVideoRenderer::SubmitResult::failed) { + // Keep presenting the last native frame. A later decoded frame can // reuse the renderer without changing ownership mid-window. props->streaming_ = true; return; } - if (submit_result == MacMetalVideoRenderer::SubmitResult::not_selected) { + if (submit_result == + NativeVideoRenderer::SubmitResult::not_selected) { background_snapshot_only = true; } } #endif { std::lock_guard lock(props->video_frame_mutex_); -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(_WIN32) const auto now = std::chrono::steady_clock::now(); if (background_snapshot_only && props->thumbnail_frame_ && !props->thumbnail_frame_->empty() && @@ -125,7 +137,7 @@ void PeerEventHandler::OnReceiveVideoBuffer( props->video_size_ = video_frame->size; props->front_frame_.swap(props->back_frame_); -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(_WIN32) props->thumbnail_frame_ = props->front_frame_; props->thumbnail_width_ = video_frame->width; props->thumbnail_height_ = video_frame->height; @@ -137,7 +149,7 @@ void PeerEventHandler::OnReceiveVideoBuffer( } props->streaming_ = true; -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(_WIN32) if (background_snapshot_only) { return; } diff --git a/src/gui/runtime/remote_session.h b/src/gui/runtime/remote_session.h index 0cba811..4305e68 100644 --- a/src/gui/runtime/remote_session.h +++ b/src/gui/runtime/remote_session.h @@ -110,7 +110,7 @@ struct RemoteSession { std::mutex video_frame_mutex_; std::shared_ptr> front_frame_; std::shared_ptr> back_frame_; -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(_WIN32) // Retains the most recent background CPU frame across disconnect cleanup // so closing an unselected tab can still update its recent-item thumbnail. std::shared_ptr> thumbnail_frame_; diff --git a/xmake/targets.lua b/xmake/targets.lua index 3af638e..f14575f 100644 --- a/xmake/targets.lua +++ b/xmake/targets.lua @@ -277,7 +277,8 @@ function setup_targets() if is_os("windows") then add_cxxflags("/bigobj") add_links("opengl32") - add_files("src/gui/platform/tray/win_tray.cpp") + add_files("src/gui/platform/tray/win_tray.cpp", + "src/gui/platform/windows_opengl_video_renderer.cpp") add_includedirs("src/service/windows", {public = true}) elseif is_os("macosx") then add_files("src/gui/runtime/*.mm", "src/gui/platform/tray/*.mm",