From 6f7c5a29ab7d0d1ed31292f7848b2302f26c2fa8 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Wed, 4 Nov 2026 23:07:42 +0800 Subject: [PATCH] [fix] fix OpenGL core rendering and video fallback, refs #138 --- .../gui/rendering/slint_video_presenter.cpp | 86 +++++++++- .../src/gui/runtime/peer_media_callbacks.cpp | 6 +- .../common/gui/opengl_video_renderer.cpp | 157 +++++++++++++----- 3 files changed, 199 insertions(+), 50 deletions(-) diff --git a/apps/desktop/src/gui/rendering/slint_video_presenter.cpp b/apps/desktop/src/gui/rendering/slint_video_presenter.cpp index 4feea74a..0decf305 100644 --- a/apps/desktop/src/gui/rendering/slint_video_presenter.cpp +++ b/apps/desktop/src/gui/rendering/slint_video_presenter.cpp @@ -88,6 +88,11 @@ struct SlintVideoPresenter::Impl { VideoRenderSize gl_pending_size; bool gl_pending_dirty = false; VideoLatencyFrame gl_pending_timing; + bool gl_use_pixel_buffers = false; + VideoRenderer::RenderOutcome gl_last_outcome; + std::string gl_rendered_stream; + std::string gl_requested_stream; + uint64_t gl_requested_sequence = 0; #endif SurfaceState CurrentState() const { @@ -125,6 +130,13 @@ struct SlintVideoPresenter::Impl { auto* opengl_renderer = dynamic_cast(&renderer); if (state == slint::RenderingState::RenderingSetup) { + gl_use_pixel_buffers = false; + gl_last_outcome = {}; + gl_rendered_stream.clear(); + gl_requested_stream.clear(); + gl_requested_sequence = 0; + displayed_frame_sequence.clear(); + seeded_frame_sequence.clear(); GLuint texture = 0; glGenTextures(1, &texture); if (texture == 0) { @@ -165,9 +177,28 @@ struct SlintVideoPresenter::Impl { rounded_window ? static_cast(std::lround( kStreamWindowCornerRadius * scale_factor)) : 0; - opengl_renderer->RenderLatest( + const auto outcome = opengl_renderer->RenderLatest( surface.selected_stream, static_cast(window_size.width), static_cast(window_size.height), top_inset, corner_radius); + if (outcome.result == VideoRenderer::RenderResult::idle) { + return; + } + if (outcome.result == VideoRenderer::RenderResult::failed) { + // RenderLatest stops native submissions. Apply the UI fallback + // on the next UI tick, outside Slint's rendering callback. + gl_use_pixel_buffers = true; + LOG_WARN("Switching Slint video presentation to pixel buffers"); + } + const bool changed = gl_rendered_stream != surface.selected_stream || + gl_last_outcome.result != outcome.result || + gl_last_outcome.sequence != outcome.sequence || + gl_last_outcome.width != outcome.width || + gl_last_outcome.height != outcome.height; + gl_last_outcome = outcome; + gl_rendered_stream = surface.selected_stream; + if (changed) { + MarkDirty(); + } return; } @@ -211,6 +242,8 @@ struct SlintVideoPresenter::Impl { gl_pending_size = {}; gl_pending_dirty = false; gl_pending_frame.clear(); + gl_last_outcome = {}; + gl_rendered_stream.clear(); } }); @@ -239,6 +272,7 @@ void SlintVideoPresenter::PrepareWindow( impl_->attachment_attempts = kMetalAttachmentAttemptLimit; impl_->skip_attachment_once = true; #elif !defined(__OHOS__) + stream->set_native_video_enabled(false); impl_->ConfigureOpenGlNotifier(); #endif } @@ -313,7 +347,12 @@ bool SlintVideoPresenter::SelectStream(std::string remote_id) { } SlintVideoPresenter::PresentResult SlintVideoPresenter::Present( - const Frame& frame) { + const Frame& incoming_frame) { +#if !defined(__APPLE__) && !defined(__OHOS__) + Frame frame = incoming_frame; +#else + const Frame& frame = incoming_frame; +#endif PresentResult result; if (!impl_->stream || frame.remote_id.empty()) { return result; @@ -355,25 +394,54 @@ SlintVideoPresenter::PresentResult SlintVideoPresenter::Present( auto* opengl_renderer = dynamic_cast(&impl_->renderer); if (opengl_renderer && opengl_renderer->IsReady()) { - opengl_renderer->SetSelectedStream(frame.remote_id); + bool needs_redraw = opengl_renderer->SetSelectedStream(frame.remote_id); if (!(*impl_->stream)->get_native_video_enabled()) { (*impl_->stream)->set_native_video_enabled(true); + needs_redraw = true; } + VideoRenderer::RenderOutcome outcome; + { + std::lock_guard lock(impl_->gl_mutex); + if (impl_->gl_rendered_stream == frame.remote_id) { + outcome = impl_->gl_last_outcome; + } + } const bool has_frame = - frame.width > 0 && frame.height > 0 && frame.sequence > 0; + outcome.result == VideoRenderer::RenderResult::rendered; (*impl_->stream)->set_has_frame(has_frame); if (has_frame) { (*impl_->stream)->set_receiving_text(""); - result.width = frame.width; - result.height = frame.height; - result.new_frame = impl_->MarkPresented(frame.remote_id, frame.sequence); + result.width = outcome.width; + result.height = outcome.height; + result.new_frame = impl_->MarkPresented(frame.remote_id, outcome.sequence); + } + // A rendering acknowledgement also schedules Present(). Do not draw the + // same video again solely to acknowledge it, especially for low-FPS peers. + if (needs_redraw || impl_->gl_requested_stream != frame.remote_id || + impl_->gl_requested_sequence != frame.sequence) { + impl_->gl_requested_stream = frame.remote_id; + impl_->gl_requested_sequence = frame.sequence; + (*impl_->stream)->window().request_redraw(); } - (*impl_->stream)->window().request_redraw(); return result; } if ((*impl_->stream)->get_native_video_enabled()) { (*impl_->stream)->set_native_video_enabled(false); + (*impl_->stream)->set_has_frame(false); + (*impl_->stream)->set_frame(slint::Image{}); + impl_->displayed_frame_sequence.clear(); + impl_->seeded_frame_sequence.clear(); + } + if (!frame.nv12 && impl_->gl_use_pixel_buffers) { + // Native submissions intentionally release the session's CPU buffers. + // Recover the retained frame so even a static remote desktop can switch + // immediately; subsequent decode callbacks will supply CPU frames. + auto pixels = std::make_shared>(); + if (impl_->renderer.CopyLatestNv12(frame.remote_id, pixels.get(), + &frame.width, &frame.height)) { + frame.nv12 = std::move(pixels); + } } #endif @@ -423,7 +491,7 @@ SlintVideoPresenter::PresentResult SlintVideoPresenter::Present( uint32_t texture_id = 0; { std::lock_guard lock(impl_->gl_mutex); - texture_id = impl_->gl_texture; + texture_id = impl_->gl_use_pixel_buffers ? 0 : impl_->gl_texture; } if (texture_id != 0) { impl_->gl_conversion_frame.resize(static_cast(output_width) * diff --git a/apps/desktop/src/gui/runtime/peer_media_callbacks.cpp b/apps/desktop/src/gui/runtime/peer_media_callbacks.cpp index 4b51cd79..fb10acb3 100644 --- a/apps/desktop/src/gui/runtime/peer_media_callbacks.cpp +++ b/apps/desktop/src/gui/runtime/peer_media_callbacks.cpp @@ -183,9 +183,11 @@ void PeerEventHandler::OnReceiveVideoBuffer( return; } if (submit_result == VideoRenderer::SubmitResult::dropped || - submit_result == VideoRenderer::SubmitResult::failed) { + (submit_result == VideoRenderer::SubmitResult::failed && + native_renderer->IsActive())) { // Keep presenting the last native frame. A later decoded frame can - // reuse the renderer without changing ownership mid-window. + // reuse an active renderer. If rendering disabled it concurrently, + // continue below to materialize this frame for the CPU fallback. props->streaming_ = true; return; } diff --git a/apps/desktop/src/platform/common/gui/opengl_video_renderer.cpp b/apps/desktop/src/platform/common/gui/opengl_video_renderer.cpp index 860e06df..13fdb059 100644 --- a/apps/desktop/src/platform/common/gui/opengl_video_renderer.cpp +++ b/apps/desktop/src/platform/common/gui/opengl_video_renderer.cpp @@ -123,6 +123,9 @@ struct OpenGlFunctions { PFNGLUNIFORM4FPROC uniform_4f = nullptr; PFNGLUSEPROGRAMPROC use_program = nullptr; PFNGLVERTEXATTRIBPOINTERPROC vertex_attrib_pointer = nullptr; + PFNGLGENVERTEXARRAYSPROC gen_vertex_arrays = nullptr; + PFNGLDELETEVERTEXARRAYSPROC delete_vertex_arrays = nullptr; + PFNGLBINDVERTEXARRAYPROC bind_vertex_array = nullptr; PFNGLGENFRAMEBUFFERSPROC gen_framebuffers = nullptr; PFNGLDELETEFRAMEBUFFERSPROC delete_framebuffers = nullptr; PFNGLBINDFRAMEBUFFERPROC bind_framebuffer = nullptr; @@ -178,6 +181,12 @@ struct OpenGlFunctions { "glCheckFramebufferStatus"); } + bool LoadVertexArrayFunctions() { + return LoadOpenGlFunction(&gen_vertex_arrays, "glGenVertexArrays") && + LoadOpenGlFunction(&delete_vertex_arrays, "glDeleteVertexArrays") && + LoadOpenGlFunction(&bind_vertex_array, "glBindVertexArray"); + } + #if CROSSDESK_OPENGL_CUDA_INTEROP bool LoadSyncFunctions() { return LoadOpenGlFunction(&fence_sync, "glFenceSync") && @@ -292,6 +301,8 @@ struct OpenGlVideoRenderer::Impl { std::atomic ready{false}; GLuint program = 0; GLuint vertex_buffer = 0; + GLuint vertex_array = 0; + bool modern_gl = false; GLuint y_texture = 0; GLuint uv_texture = 0; GLint position_location = -1; @@ -644,6 +655,8 @@ struct OpenGlVideoRenderer::Impl { void ResetGlHandles() { program = 0; vertex_buffer = 0; + vertex_array = 0; + modern_gl = false; y_texture = 0; uv_texture = 0; position_location = -1; @@ -698,6 +711,13 @@ bool OpenGlVideoRenderer::Setup() { std::sscanf(is_gles ? version + std::strlen("OpenGL ES ") : version, "%d.%d", &major, &minor); } + // Slint may fall back from GLES to a desktop core context. In that context + // VAO 0, legacy luminance textures and legacy shader inputs are not usable. + impl_->modern_gl = major >= 3; + if (impl_->modern_gl && !impl_->gl.LoadVertexArrayFunctions()) { + LOG_WARN("OpenGL NV12 underlay unavailable: vertex array functions missing"); + return false; + } const auto *extensions = major < 3 ? reinterpret_cast(glGetString(GL_EXTENSIONS)) : nullptr; impl_->pixel_unpack_buffer_available = major >= 3 || @@ -727,22 +747,30 @@ bool OpenGlVideoRenderer::Setup() { } #endif - const std::string vertex_source = - std::string(is_gles ? "#version 100\n" : "#version 110\n") + R"glsl( -attribute vec2 position; -attribute vec2 texcoord; -varying vec2 video_texcoord; + const std::string shader_version = is_gles + ? (impl_->modern_gl ? "#version 300 es\n" : "#version 100\n") + : (major > 3 || (major == 3 && minor >= 2) ? "#version 150\n" + : impl_->modern_gl ? "#version 130\n" : "#version 110\n"); + const std::string vertex_source = shader_version + + (impl_->modern_gl ? "#define INPUT in\n#define OUTPUT out\n" + : "#define INPUT attribute\n#define OUTPUT varying\n") + R"glsl( +INPUT vec2 position; +INPUT vec2 texcoord; +OUTPUT vec2 video_texcoord; void main() { video_texcoord = texcoord; gl_Position = vec4(position, 0.0, 1.0); } )glsl"; - const std::string fragment_source = - std::string(is_gles ? "#version 100\n" - "#ifdef GL_FRAGMENT_PRECISION_HIGH\n" - "precision highp float;\n" - "#else\nprecision mediump float;\n#endif\n" - : "#version 110\n") + R"glsl( + const std::string fragment_source = shader_version + + (is_gles ? "#ifdef GL_FRAGMENT_PRECISION_HIGH\n" + "precision highp float;\n" + "#else\nprecision mediump float;\n#endif\n" : "") + + (impl_->modern_gl + ? "#define INPUT in\n#define SAMPLE texture\n" + "#define UV_CHANNELS rg\nout vec4 fragment_color;\n" + : "#define INPUT varying\n#define SAMPLE texture2D\n" + "#define UV_CHANNELS ra\n#define fragment_color gl_FragColor\n") + R"glsl( uniform sampler2D y_texture; uniform sampler2D uv_texture; uniform vec2 target_size; @@ -752,7 +780,7 @@ uniform float packed_video; uniform vec4 video_rect; uniform float corner_radius; uniform float video_enabled; -varying vec2 video_texcoord; +INPUT vec2 video_texcoord; // A separable Lanczos-2 low-pass filter. Each pass reduces by at most 4x, // so 16 taps cover its complete support even for non-integer scale ratios. float lanczos2(float x) { @@ -766,7 +794,7 @@ vec4 sample_plane(sampler2D plane, vec2 size, vec2 coord) { // FBO textures have the opposite vertical origin to the decoded NV12 planes. if (packed_video > 0.5) coord.y = 1.0 - coord.y; if (filter_direction.x + filter_direction.y < 0.5) - return texture2D(plane, coord); + return SAMPLE(plane, coord); float scale = max(1.0, dot(size / video_rect.zw, filter_direction)); float center = dot(coord * size - vec2(0.5), filter_direction); float first = floor(center - 2.0 * scale) + 1.0; @@ -776,7 +804,7 @@ vec4 sample_plane(sampler2D plane, vec2 size, vec2 coord) { float offset = first + float(tap) - center; if (offset >= 2.0 * scale) break; float weight = lanczos2(offset / scale); - sum += weight * texture2D(plane, coord + filter_direction * offset / size); + sum += weight * SAMPLE(plane, coord + filter_direction * offset / size); weight_sum += weight; } return sum / weight_sum; @@ -785,7 +813,7 @@ vec3 sample_video(vec2 coord) { vec4 y_sample = sample_plane(y_texture, source_size, coord); if (packed_video > 0.5) return y_sample.rgb; return vec3(y_sample.r, - sample_plane(uv_texture, source_size * 0.5, coord).ra); + sample_plane(uv_texture, source_size * 0.5, coord).UV_CHANNELS); } void main() { vec2 surface_point = video_texcoord * target_size; @@ -800,7 +828,7 @@ void main() { vec3 yuv = sample_video(frame_texcoord); if (filter_direction.x + filter_direction.y > 0.5) { // Intermediate RGBA textures carry Y/U/V until the final conversion. - gl_FragColor = vec4(yuv, 1.0); + fragment_color = vec4(yuv, 1.0); return; } float y = 1.16438356 * (yuv.x - 16.0 / 255.0); @@ -820,7 +848,7 @@ void main() { min(max(corner.x, corner.y), 0.0) - corner_radius; coverage = clamp(0.5 - distance_to_edge, 0.0, 1.0); } - gl_FragColor = vec4(rgb * coverage, coverage); + fragment_color = vec4(rgb * coverage, coverage); } )glsl"; @@ -903,6 +931,11 @@ void main() { }; GLint previous_array_buffer = 0; glGetIntegerv(GL_ARRAY_BUFFER_BINDING, &previous_array_buffer); + while (glGetError() != GL_NO_ERROR) { + } + if (impl_->modern_gl) { + impl_->gl.gen_vertex_arrays(1, &impl_->vertex_array); + } impl_->gl.gen_buffers(1, &impl_->vertex_buffer); impl_->gl.bind_buffer(GL_ARRAY_BUFFER, impl_->vertex_buffer); impl_->gl.buffer_data(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices, @@ -918,8 +951,6 @@ void main() { impl_->gl.active_texture(GL_TEXTURE1); GLint previous_texture_1 = 0; glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture_1); - while (glGetError() != GL_NO_ERROR) { - } glGenTextures(1, &impl_->y_texture); glGenTextures(1, &impl_->uv_texture); for (const auto [unit, texture] : @@ -940,6 +971,7 @@ void main() { const GLenum setup_error = glGetError(); if (impl_->program == 0 || impl_->vertex_buffer == 0 || + (impl_->modern_gl && impl_->vertex_array == 0) || impl_->y_texture == 0 || impl_->uv_texture == 0 || setup_error != GL_NO_ERROR) { LOG_ERROR("OpenGL NV12 resource setup failed, error={}", setup_error); @@ -950,9 +982,16 @@ void main() { impl_->ready.store(true, std::memory_order_release); const auto *vendor = reinterpret_cast(glGetString(GL_VENDOR)); const auto *renderer = reinterpret_cast(glGetString(GL_RENDERER)); - LOG_INFO("OpenGL NV12 underlay initialized ({}), vendor={}, renderer={}", + GLint profile = 0; + if (!is_gles && (major > 3 || (major == 3 && minor >= 2))) { + glGetIntegerv(GL_CONTEXT_PROFILE_MASK, &profile); + } + LOG_INFO("OpenGL NV12 underlay initialized ({}), vendor={}, renderer={}, " + "profile={}, textures={}, vao={}", version ? version : "unknown OpenGL version", - vendor ? vendor : "unknown", renderer ? renderer : "unknown"); + vendor ? vendor : "unknown", renderer ? renderer : "unknown", + profile, impl_->modern_gl ? "R/RG" : "luminance", + impl_->vertex_array); return true; } @@ -973,6 +1012,9 @@ void OpenGlVideoRenderer::Teardown() { if (impl_->vertex_buffer != 0 && impl_->gl.delete_buffers) { impl_->gl.delete_buffers(1, &impl_->vertex_buffer); } + if (impl_->vertex_array != 0 && impl_->gl.delete_vertex_arrays) { + impl_->gl.delete_vertex_arrays(1, &impl_->vertex_array); + } if (impl_->program != 0 && impl_->gl.delete_program) { impl_->gl.delete_program(impl_->program); } @@ -1056,6 +1098,9 @@ OpenGlVideoRenderer::SubmitResult OpenGlVideoRenderer::SubmitNv12Internal( auto &frames = *impl_->frames; std::lock_guard lock(frames.mutex); + if (!IsReady()) { + return SubmitResult::failed; + } if (frames.selected_stream != remote_id) { return SubmitResult::not_selected; } @@ -1194,9 +1239,12 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, int target_width, int target_height, int top_inset_pixels, int corner_radius_pixels) { - if (!IsReady() || target_width <= 0 || target_height <= 0) { + if (!IsReady()) { return {RenderResult::failed}; } + if (target_width <= 0 || target_height <= 0) { + return {RenderResult::idle}; + } size_t slot_index = kFrameSlotCount; uint64_t sequence = 0; @@ -1286,10 +1334,18 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, GLint previous_texture_1 = 0; glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture_1); - const auto position_state = CaptureVertexAttrib( - impl_->gl, static_cast(impl_->position_location)); - const auto texcoord_state = CaptureVertexAttrib( - impl_->gl, static_cast(impl_->texcoord_location)); + GLint previous_vertex_array = 0; + VertexAttribState position_state; + VertexAttribState texcoord_state; + if (impl_->vertex_array != 0) { + glGetIntegerv(GL_VERTEX_ARRAY_BINDING, &previous_vertex_array); + impl_->gl.bind_vertex_array(impl_->vertex_array); + } else { + position_state = CaptureVertexAttrib( + impl_->gl, static_cast(impl_->position_location)); + texcoord_state = CaptureVertexAttrib( + impl_->gl, static_cast(impl_->texcoord_location)); + } glDisable(GL_BLEND); glDisable(GL_CULL_FACE); @@ -1401,20 +1457,26 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, if (uploaded_texture_width != source_width || uploaded_texture_height != source_height) { impl_->gl.active_texture(GL_TEXTURE0); - glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, source_width, - source_height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, + glTexImage2D(GL_TEXTURE_2D, 0, + impl_->modern_gl ? GL_R8 : GL_LUMINANCE, source_width, + source_height, 0, + impl_->modern_gl ? GL_RED : GL_LUMINANCE, GL_UNSIGNED_BYTE, y_pixels); impl_->gl.active_texture(GL_TEXTURE1); - glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE_ALPHA, source_width / 2, - source_height / 2, 0, GL_LUMINANCE_ALPHA, + glTexImage2D(GL_TEXTURE_2D, 0, + impl_->modern_gl ? GL_RG8 : GL_LUMINANCE_ALPHA, + source_width / 2, source_height / 2, 0, + impl_->modern_gl ? GL_RG : GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, uv_pixels); } else { impl_->gl.active_texture(GL_TEXTURE0); glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, source_width, source_height, - GL_LUMINANCE, GL_UNSIGNED_BYTE, y_pixels); + impl_->modern_gl ? GL_RED : GL_LUMINANCE, + GL_UNSIGNED_BYTE, y_pixels); impl_->gl.active_texture(GL_TEXTURE1); glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, source_width / 2, - source_height / 2, GL_LUMINANCE_ALPHA, + source_height / 2, + impl_->modern_gl ? GL_RG : GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, uv_pixels); } const GLenum upload_error = glGetError(); @@ -1562,10 +1624,23 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, has_video ? source_width : 0, has_video ? source_height : 0, has_video ? sequence : 0}; - RestoreVertexAttrib(impl_->gl, static_cast(impl_->position_location), - position_state); - RestoreVertexAttrib(impl_->gl, static_cast(impl_->texcoord_location), - texcoord_state); + if (outcome.result == RenderResult::failed) { + LOG_WARN("OpenGL NV12 presentation failed: upload={}, draw_error={}, " + "vao={}; disabling native video until the context is recreated", + upload_succeeded, draw_error, impl_->vertex_array); + // Do not expose the desktop during the UI tick that switches to pixels. + glClearColor(0.0f, 0.0f, 0.0f, 1.0f); + glClear(GL_COLOR_BUFFER_BIT); + } + + if (impl_->vertex_array != 0) { + impl_->gl.bind_vertex_array(static_cast(previous_vertex_array)); + } else { + RestoreVertexAttrib(impl_->gl, static_cast(impl_->position_location), + position_state); + RestoreVertexAttrib(impl_->gl, static_cast(impl_->texcoord_location), + texcoord_state); + } impl_->gl.bind_buffer(GL_ARRAY_BUFFER, static_cast(previous_array_buffer)); if (impl_->pixel_unpack_buffer_available) { @@ -1600,9 +1675,13 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, const GLenum restore_error = glGetError(); if (restore_error != GL_NO_ERROR) { LOG_WARN("OpenGL NV12 state restoration failed, error={}", restore_error); - if (outcome.result == RenderResult::rendered) { - outcome.result = RenderResult::failed; - } + outcome.result = RenderResult::failed; + } + if (outcome.result == RenderResult::failed) { + // Keep the last decoded frame available for a CPU snapshot. GL resources + // are released by RenderingTeardown while the owning context is current. + std::lock_guard lock(impl_->frames->mutex); + impl_->ready.store(false, std::memory_order_release); } return outcome; }