[fix] fix OpenGL core rendering and video fallback, refs #138

This commit is contained in:
dijunkun
2026-11-04 23:07:42 +08:00
parent 4d8a650afc
commit 6f7c5a29ab
3 changed files with 199 additions and 50 deletions
@@ -88,6 +88,11 @@ struct SlintVideoPresenter::Impl {
VideoRenderSize gl_pending_size;
bool gl_pending_dirty = false;
VideoLatencyFrame gl_pending_timing;
bool gl_use_pixel_buffers = false;
VideoRenderer::RenderOutcome gl_last_outcome;
std::string gl_rendered_stream;
std::string gl_requested_stream;
uint64_t gl_requested_sequence = 0;
#endif
SurfaceState CurrentState() const {
@@ -125,6 +130,13 @@ struct SlintVideoPresenter::Impl {
auto* opengl_renderer =
dynamic_cast<OpenGlVideoRenderer*>(&renderer);
if (state == slint::RenderingState::RenderingSetup) {
gl_use_pixel_buffers = false;
gl_last_outcome = {};
gl_rendered_stream.clear();
gl_requested_stream.clear();
gl_requested_sequence = 0;
displayed_frame_sequence.clear();
seeded_frame_sequence.clear();
GLuint texture = 0;
glGenTextures(1, &texture);
if (texture == 0) {
@@ -165,9 +177,28 @@ struct SlintVideoPresenter::Impl {
rounded_window ? static_cast<int>(std::lround(
kStreamWindowCornerRadius * scale_factor))
: 0;
opengl_renderer->RenderLatest(
const auto outcome = opengl_renderer->RenderLatest(
surface.selected_stream, static_cast<int>(window_size.width),
static_cast<int>(window_size.height), top_inset, corner_radius);
if (outcome.result == VideoRenderer::RenderResult::idle) {
return;
}
if (outcome.result == VideoRenderer::RenderResult::failed) {
// RenderLatest stops native submissions. Apply the UI fallback
// on the next UI tick, outside Slint's rendering callback.
gl_use_pixel_buffers = true;
LOG_WARN("Switching Slint video presentation to pixel buffers");
}
const bool changed = gl_rendered_stream != surface.selected_stream ||
gl_last_outcome.result != outcome.result ||
gl_last_outcome.sequence != outcome.sequence ||
gl_last_outcome.width != outcome.width ||
gl_last_outcome.height != outcome.height;
gl_last_outcome = outcome;
gl_rendered_stream = surface.selected_stream;
if (changed) {
MarkDirty();
}
return;
}
@@ -211,6 +242,8 @@ struct SlintVideoPresenter::Impl {
gl_pending_size = {};
gl_pending_dirty = false;
gl_pending_frame.clear();
gl_last_outcome = {};
gl_rendered_stream.clear();
}
});
@@ -239,6 +272,7 @@ void SlintVideoPresenter::PrepareWindow(
impl_->attachment_attempts = kMetalAttachmentAttemptLimit;
impl_->skip_attachment_once = true;
#elif !defined(__OHOS__)
stream->set_native_video_enabled(false);
impl_->ConfigureOpenGlNotifier();
#endif
}
@@ -313,7 +347,12 @@ bool SlintVideoPresenter::SelectStream(std::string remote_id) {
}
SlintVideoPresenter::PresentResult SlintVideoPresenter::Present(
const Frame& frame) {
const Frame& incoming_frame) {
#if !defined(__APPLE__) && !defined(__OHOS__)
Frame frame = incoming_frame;
#else
const Frame& frame = incoming_frame;
#endif
PresentResult result;
if (!impl_->stream || frame.remote_id.empty()) {
return result;
@@ -355,25 +394,54 @@ SlintVideoPresenter::PresentResult SlintVideoPresenter::Present(
auto* opengl_renderer =
dynamic_cast<OpenGlVideoRenderer*>(&impl_->renderer);
if (opengl_renderer && opengl_renderer->IsReady()) {
opengl_renderer->SetSelectedStream(frame.remote_id);
bool needs_redraw = opengl_renderer->SetSelectedStream(frame.remote_id);
if (!(*impl_->stream)->get_native_video_enabled()) {
(*impl_->stream)->set_native_video_enabled(true);
needs_redraw = true;
}
VideoRenderer::RenderOutcome outcome;
{
std::lock_guard lock(impl_->gl_mutex);
if (impl_->gl_rendered_stream == frame.remote_id) {
outcome = impl_->gl_last_outcome;
}
}
const bool has_frame =
frame.width > 0 && frame.height > 0 && frame.sequence > 0;
outcome.result == VideoRenderer::RenderResult::rendered;
(*impl_->stream)->set_has_frame(has_frame);
if (has_frame) {
(*impl_->stream)->set_receiving_text("");
result.width = frame.width;
result.height = frame.height;
result.new_frame = impl_->MarkPresented(frame.remote_id, frame.sequence);
result.width = outcome.width;
result.height = outcome.height;
result.new_frame = impl_->MarkPresented(frame.remote_id, outcome.sequence);
}
// A rendering acknowledgement also schedules Present(). Do not draw the
// same video again solely to acknowledge it, especially for low-FPS peers.
if (needs_redraw || impl_->gl_requested_stream != frame.remote_id ||
impl_->gl_requested_sequence != frame.sequence) {
impl_->gl_requested_stream = frame.remote_id;
impl_->gl_requested_sequence = frame.sequence;
(*impl_->stream)->window().request_redraw();
}
(*impl_->stream)->window().request_redraw();
return result;
}
if ((*impl_->stream)->get_native_video_enabled()) {
(*impl_->stream)->set_native_video_enabled(false);
(*impl_->stream)->set_has_frame(false);
(*impl_->stream)->set_frame(slint::Image{});
impl_->displayed_frame_sequence.clear();
impl_->seeded_frame_sequence.clear();
}
if (!frame.nv12 && impl_->gl_use_pixel_buffers) {
// Native submissions intentionally release the session's CPU buffers.
// Recover the retained frame so even a static remote desktop can switch
// immediately; subsequent decode callbacks will supply CPU frames.
auto pixels = std::make_shared<std::vector<unsigned char>>();
if (impl_->renderer.CopyLatestNv12(frame.remote_id, pixels.get(),
&frame.width, &frame.height)) {
frame.nv12 = std::move(pixels);
}
}
#endif
@@ -423,7 +491,7 @@ SlintVideoPresenter::PresentResult SlintVideoPresenter::Present(
uint32_t texture_id = 0;
{
std::lock_guard lock(impl_->gl_mutex);
texture_id = impl_->gl_texture;
texture_id = impl_->gl_use_pixel_buffers ? 0 : impl_->gl_texture;
}
if (texture_id != 0) {
impl_->gl_conversion_frame.resize(static_cast<size_t>(output_width) *
@@ -183,9 +183,11 @@ void PeerEventHandler::OnReceiveVideoBuffer(
return;
}
if (submit_result == VideoRenderer::SubmitResult::dropped ||
submit_result == VideoRenderer::SubmitResult::failed) {
(submit_result == VideoRenderer::SubmitResult::failed &&
native_renderer->IsActive())) {
// Keep presenting the last native frame. A later decoded frame can
// reuse the renderer without changing ownership mid-window.
// reuse an active renderer. If rendering disabled it concurrently,
// continue below to materialize this frame for the CPU fallback.
props->streaming_ = true;
return;
}
@@ -123,6 +123,9 @@ struct OpenGlFunctions {
PFNGLUNIFORM4FPROC uniform_4f = nullptr;
PFNGLUSEPROGRAMPROC use_program = nullptr;
PFNGLVERTEXATTRIBPOINTERPROC vertex_attrib_pointer = nullptr;
PFNGLGENVERTEXARRAYSPROC gen_vertex_arrays = nullptr;
PFNGLDELETEVERTEXARRAYSPROC delete_vertex_arrays = nullptr;
PFNGLBINDVERTEXARRAYPROC bind_vertex_array = nullptr;
PFNGLGENFRAMEBUFFERSPROC gen_framebuffers = nullptr;
PFNGLDELETEFRAMEBUFFERSPROC delete_framebuffers = nullptr;
PFNGLBINDFRAMEBUFFERPROC bind_framebuffer = nullptr;
@@ -178,6 +181,12 @@ struct OpenGlFunctions {
"glCheckFramebufferStatus");
}
bool LoadVertexArrayFunctions() {
return LoadOpenGlFunction(&gen_vertex_arrays, "glGenVertexArrays") &&
LoadOpenGlFunction(&delete_vertex_arrays, "glDeleteVertexArrays") &&
LoadOpenGlFunction(&bind_vertex_array, "glBindVertexArray");
}
#if CROSSDESK_OPENGL_CUDA_INTEROP
bool LoadSyncFunctions() {
return LoadOpenGlFunction(&fence_sync, "glFenceSync") &&
@@ -292,6 +301,8 @@ struct OpenGlVideoRenderer::Impl {
std::atomic<bool> ready{false};
GLuint program = 0;
GLuint vertex_buffer = 0;
GLuint vertex_array = 0;
bool modern_gl = false;
GLuint y_texture = 0;
GLuint uv_texture = 0;
GLint position_location = -1;
@@ -644,6 +655,8 @@ struct OpenGlVideoRenderer::Impl {
void ResetGlHandles() {
program = 0;
vertex_buffer = 0;
vertex_array = 0;
modern_gl = false;
y_texture = 0;
uv_texture = 0;
position_location = -1;
@@ -698,6 +711,13 @@ bool OpenGlVideoRenderer::Setup() {
std::sscanf(is_gles ? version + std::strlen("OpenGL ES ") : version,
"%d.%d", &major, &minor);
}
// Slint may fall back from GLES to a desktop core context. In that context
// VAO 0, legacy luminance textures and legacy shader inputs are not usable.
impl_->modern_gl = major >= 3;
if (impl_->modern_gl && !impl_->gl.LoadVertexArrayFunctions()) {
LOG_WARN("OpenGL NV12 underlay unavailable: vertex array functions missing");
return false;
}
const auto *extensions = major < 3
? reinterpret_cast<const char *>(glGetString(GL_EXTENSIONS)) : nullptr;
impl_->pixel_unpack_buffer_available = major >= 3 ||
@@ -727,22 +747,30 @@ bool OpenGlVideoRenderer::Setup() {
}
#endif
const std::string vertex_source =
std::string(is_gles ? "#version 100\n" : "#version 110\n") + R"glsl(
attribute vec2 position;
attribute vec2 texcoord;
varying vec2 video_texcoord;
const std::string shader_version = is_gles
? (impl_->modern_gl ? "#version 300 es\n" : "#version 100\n")
: (major > 3 || (major == 3 && minor >= 2) ? "#version 150\n"
: impl_->modern_gl ? "#version 130\n" : "#version 110\n");
const std::string vertex_source = shader_version +
(impl_->modern_gl ? "#define INPUT in\n#define OUTPUT out\n"
: "#define INPUT attribute\n#define OUTPUT varying\n") + R"glsl(
INPUT vec2 position;
INPUT vec2 texcoord;
OUTPUT vec2 video_texcoord;
void main() {
video_texcoord = texcoord;
gl_Position = vec4(position, 0.0, 1.0);
}
)glsl";
const std::string fragment_source =
std::string(is_gles ? "#version 100\n"
"#ifdef GL_FRAGMENT_PRECISION_HIGH\n"
"precision highp float;\n"
"#else\nprecision mediump float;\n#endif\n"
: "#version 110\n") + R"glsl(
const std::string fragment_source = shader_version +
(is_gles ? "#ifdef GL_FRAGMENT_PRECISION_HIGH\n"
"precision highp float;\n"
"#else\nprecision mediump float;\n#endif\n" : "") +
(impl_->modern_gl
? "#define INPUT in\n#define SAMPLE texture\n"
"#define UV_CHANNELS rg\nout vec4 fragment_color;\n"
: "#define INPUT varying\n#define SAMPLE texture2D\n"
"#define UV_CHANNELS ra\n#define fragment_color gl_FragColor\n") + R"glsl(
uniform sampler2D y_texture;
uniform sampler2D uv_texture;
uniform vec2 target_size;
@@ -752,7 +780,7 @@ uniform float packed_video;
uniform vec4 video_rect;
uniform float corner_radius;
uniform float video_enabled;
varying vec2 video_texcoord;
INPUT vec2 video_texcoord;
// A separable Lanczos-2 low-pass filter. Each pass reduces by at most 4x,
// so 16 taps cover its complete support even for non-integer scale ratios.
float lanczos2(float x) {
@@ -766,7 +794,7 @@ vec4 sample_plane(sampler2D plane, vec2 size, vec2 coord) {
// FBO textures have the opposite vertical origin to the decoded NV12 planes.
if (packed_video > 0.5) coord.y = 1.0 - coord.y;
if (filter_direction.x + filter_direction.y < 0.5)
return texture2D(plane, coord);
return SAMPLE(plane, coord);
float scale = max(1.0, dot(size / video_rect.zw, filter_direction));
float center = dot(coord * size - vec2(0.5), filter_direction);
float first = floor(center - 2.0 * scale) + 1.0;
@@ -776,7 +804,7 @@ vec4 sample_plane(sampler2D plane, vec2 size, vec2 coord) {
float offset = first + float(tap) - center;
if (offset >= 2.0 * scale) break;
float weight = lanczos2(offset / scale);
sum += weight * texture2D(plane, coord + filter_direction * offset / size);
sum += weight * SAMPLE(plane, coord + filter_direction * offset / size);
weight_sum += weight;
}
return sum / weight_sum;
@@ -785,7 +813,7 @@ vec3 sample_video(vec2 coord) {
vec4 y_sample = sample_plane(y_texture, source_size, coord);
if (packed_video > 0.5) return y_sample.rgb;
return vec3(y_sample.r,
sample_plane(uv_texture, source_size * 0.5, coord).ra);
sample_plane(uv_texture, source_size * 0.5, coord).UV_CHANNELS);
}
void main() {
vec2 surface_point = video_texcoord * target_size;
@@ -800,7 +828,7 @@ void main() {
vec3 yuv = sample_video(frame_texcoord);
if (filter_direction.x + filter_direction.y > 0.5) {
// Intermediate RGBA textures carry Y/U/V until the final conversion.
gl_FragColor = vec4(yuv, 1.0);
fragment_color = vec4(yuv, 1.0);
return;
}
float y = 1.16438356 * (yuv.x - 16.0 / 255.0);
@@ -820,7 +848,7 @@ void main() {
min(max(corner.x, corner.y), 0.0) - corner_radius;
coverage = clamp(0.5 - distance_to_edge, 0.0, 1.0);
}
gl_FragColor = vec4(rgb * coverage, coverage);
fragment_color = vec4(rgb * coverage, coverage);
}
)glsl";
@@ -903,6 +931,11 @@ void main() {
};
GLint previous_array_buffer = 0;
glGetIntegerv(GL_ARRAY_BUFFER_BINDING, &previous_array_buffer);
while (glGetError() != GL_NO_ERROR) {
}
if (impl_->modern_gl) {
impl_->gl.gen_vertex_arrays(1, &impl_->vertex_array);
}
impl_->gl.gen_buffers(1, &impl_->vertex_buffer);
impl_->gl.bind_buffer(GL_ARRAY_BUFFER, impl_->vertex_buffer);
impl_->gl.buffer_data(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices,
@@ -918,8 +951,6 @@ void main() {
impl_->gl.active_texture(GL_TEXTURE1);
GLint previous_texture_1 = 0;
glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture_1);
while (glGetError() != GL_NO_ERROR) {
}
glGenTextures(1, &impl_->y_texture);
glGenTextures(1, &impl_->uv_texture);
for (const auto [unit, texture] :
@@ -940,6 +971,7 @@ void main() {
const GLenum setup_error = glGetError();
if (impl_->program == 0 || impl_->vertex_buffer == 0 ||
(impl_->modern_gl && impl_->vertex_array == 0) ||
impl_->y_texture == 0 || impl_->uv_texture == 0 ||
setup_error != GL_NO_ERROR) {
LOG_ERROR("OpenGL NV12 resource setup failed, error={}", setup_error);
@@ -950,9 +982,16 @@ void main() {
impl_->ready.store(true, std::memory_order_release);
const auto *vendor = reinterpret_cast<const char *>(glGetString(GL_VENDOR));
const auto *renderer = reinterpret_cast<const char *>(glGetString(GL_RENDERER));
LOG_INFO("OpenGL NV12 underlay initialized ({}), vendor={}, renderer={}",
GLint profile = 0;
if (!is_gles && (major > 3 || (major == 3 && minor >= 2))) {
glGetIntegerv(GL_CONTEXT_PROFILE_MASK, &profile);
}
LOG_INFO("OpenGL NV12 underlay initialized ({}), vendor={}, renderer={}, "
"profile={}, textures={}, vao={}",
version ? version : "unknown OpenGL version",
vendor ? vendor : "unknown", renderer ? renderer : "unknown");
vendor ? vendor : "unknown", renderer ? renderer : "unknown",
profile, impl_->modern_gl ? "R/RG" : "luminance",
impl_->vertex_array);
return true;
}
@@ -973,6 +1012,9 @@ void OpenGlVideoRenderer::Teardown() {
if (impl_->vertex_buffer != 0 && impl_->gl.delete_buffers) {
impl_->gl.delete_buffers(1, &impl_->vertex_buffer);
}
if (impl_->vertex_array != 0 && impl_->gl.delete_vertex_arrays) {
impl_->gl.delete_vertex_arrays(1, &impl_->vertex_array);
}
if (impl_->program != 0 && impl_->gl.delete_program) {
impl_->gl.delete_program(impl_->program);
}
@@ -1056,6 +1098,9 @@ OpenGlVideoRenderer::SubmitResult OpenGlVideoRenderer::SubmitNv12Internal(
auto &frames = *impl_->frames;
std::lock_guard lock(frames.mutex);
if (!IsReady()) {
return SubmitResult::failed;
}
if (frames.selected_stream != remote_id) {
return SubmitResult::not_selected;
}
@@ -1194,9 +1239,12 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id,
int target_width, int target_height,
int top_inset_pixels,
int corner_radius_pixels) {
if (!IsReady() || target_width <= 0 || target_height <= 0) {
if (!IsReady()) {
return {RenderResult::failed};
}
if (target_width <= 0 || target_height <= 0) {
return {RenderResult::idle};
}
size_t slot_index = kFrameSlotCount;
uint64_t sequence = 0;
@@ -1286,10 +1334,18 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id,
GLint previous_texture_1 = 0;
glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture_1);
const auto position_state = CaptureVertexAttrib(
impl_->gl, static_cast<GLuint>(impl_->position_location));
const auto texcoord_state = CaptureVertexAttrib(
impl_->gl, static_cast<GLuint>(impl_->texcoord_location));
GLint previous_vertex_array = 0;
VertexAttribState position_state;
VertexAttribState texcoord_state;
if (impl_->vertex_array != 0) {
glGetIntegerv(GL_VERTEX_ARRAY_BINDING, &previous_vertex_array);
impl_->gl.bind_vertex_array(impl_->vertex_array);
} else {
position_state = CaptureVertexAttrib(
impl_->gl, static_cast<GLuint>(impl_->position_location));
texcoord_state = CaptureVertexAttrib(
impl_->gl, static_cast<GLuint>(impl_->texcoord_location));
}
glDisable(GL_BLEND);
glDisable(GL_CULL_FACE);
@@ -1401,20 +1457,26 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id,
if (uploaded_texture_width != source_width ||
uploaded_texture_height != source_height) {
impl_->gl.active_texture(GL_TEXTURE0);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, source_width,
source_height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE,
glTexImage2D(GL_TEXTURE_2D, 0,
impl_->modern_gl ? GL_R8 : GL_LUMINANCE, source_width,
source_height, 0,
impl_->modern_gl ? GL_RED : GL_LUMINANCE, GL_UNSIGNED_BYTE,
y_pixels);
impl_->gl.active_texture(GL_TEXTURE1);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE_ALPHA, source_width / 2,
source_height / 2, 0, GL_LUMINANCE_ALPHA,
glTexImage2D(GL_TEXTURE_2D, 0,
impl_->modern_gl ? GL_RG8 : GL_LUMINANCE_ALPHA,
source_width / 2, source_height / 2, 0,
impl_->modern_gl ? GL_RG : GL_LUMINANCE_ALPHA,
GL_UNSIGNED_BYTE, uv_pixels);
} else {
impl_->gl.active_texture(GL_TEXTURE0);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, source_width, source_height,
GL_LUMINANCE, GL_UNSIGNED_BYTE, y_pixels);
impl_->modern_gl ? GL_RED : GL_LUMINANCE,
GL_UNSIGNED_BYTE, y_pixels);
impl_->gl.active_texture(GL_TEXTURE1);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, source_width / 2,
source_height / 2, GL_LUMINANCE_ALPHA,
source_height / 2,
impl_->modern_gl ? GL_RG : GL_LUMINANCE_ALPHA,
GL_UNSIGNED_BYTE, uv_pixels);
}
const GLenum upload_error = glGetError();
@@ -1562,10 +1624,23 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id,
has_video ? source_width : 0, has_video ? source_height : 0,
has_video ? sequence : 0};
RestoreVertexAttrib(impl_->gl, static_cast<GLuint>(impl_->position_location),
position_state);
RestoreVertexAttrib(impl_->gl, static_cast<GLuint>(impl_->texcoord_location),
texcoord_state);
if (outcome.result == RenderResult::failed) {
LOG_WARN("OpenGL NV12 presentation failed: upload={}, draw_error={}, "
"vao={}; disabling native video until the context is recreated",
upload_succeeded, draw_error, impl_->vertex_array);
// Do not expose the desktop during the UI tick that switches to pixels.
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
}
if (impl_->vertex_array != 0) {
impl_->gl.bind_vertex_array(static_cast<GLuint>(previous_vertex_array));
} else {
RestoreVertexAttrib(impl_->gl, static_cast<GLuint>(impl_->position_location),
position_state);
RestoreVertexAttrib(impl_->gl, static_cast<GLuint>(impl_->texcoord_location),
texcoord_state);
}
impl_->gl.bind_buffer(GL_ARRAY_BUFFER,
static_cast<GLuint>(previous_array_buffer));
if (impl_->pixel_unpack_buffer_available) {
@@ -1600,9 +1675,13 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id,
const GLenum restore_error = glGetError();
if (restore_error != GL_NO_ERROR) {
LOG_WARN("OpenGL NV12 state restoration failed, error={}", restore_error);
if (outcome.result == RenderResult::rendered) {
outcome.result = RenderResult::failed;
}
outcome.result = RenderResult::failed;
}
if (outcome.result == RenderResult::failed) {
// Keep the last decoded frame available for a CPU snapshot. GL resources
// are released by RenderingTeardown while the owning context is current.
std::lock_guard lock(impl_->frames->mutex);
impl_->ready.store(false, std::memory_order_release);
}
return outcome;
}