mirror of
https://github.com/kunkundi/crossdesk.git
synced 2026-08-26 13:29:31 +08:00
[feat] add OpenGL NV12 underlay rendering for Slint on Windows
This commit is contained in:
@@ -35,6 +35,7 @@
|
|||||||
#include <GL/gl.h>
|
#include <GL/gl.h>
|
||||||
|
|
||||||
#include "platform/tray/win_tray.h"
|
#include "platform/tray/win_tray.h"
|
||||||
|
#include "platform/windows_opengl_video_renderer.h"
|
||||||
#elif defined(__APPLE__)
|
#elif defined(__APPLE__)
|
||||||
#include "platform/metal_video_renderer.h"
|
#include "platform/metal_video_renderer.h"
|
||||||
#include "platform/tray/mac_tray.h"
|
#include "platform/tray/mac_tray.h"
|
||||||
@@ -1043,7 +1044,10 @@ void GuiApplication::InitializeModules() {
|
|||||||
if (modules_inited_) {
|
if (modules_inited_) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
#if defined(__APPLE__)
|
#if defined(_WIN32)
|
||||||
|
windows_opengl_video_renderer_ =
|
||||||
|
std::make_unique<WindowsOpenGlVideoRenderer>();
|
||||||
|
#elif defined(__APPLE__)
|
||||||
// Metal setup can log device, shader, or pipeline failures. Initialize it
|
// Metal setup can log device, shader, or pipeline failures. Initialize it
|
||||||
// here, after Run() has selected the application's log directory, rather
|
// here, after Run() has selected the application's log directory, rather
|
||||||
// than from GuiRuntime's constructor before InitializeLogger().
|
// than from GuiRuntime's constructor before InitializeLogger().
|
||||||
@@ -2277,6 +2281,11 @@ void GuiApplication::SyncStreamWindow() {
|
|||||||
}
|
}
|
||||||
if (!has_sessions) {
|
if (!has_sessions) {
|
||||||
SetStreamKeyboardFocus(false);
|
SetStreamKeyboardFocus(false);
|
||||||
|
#if defined(_WIN32)
|
||||||
|
if (windows_opengl_video_renderer_) {
|
||||||
|
windows_opengl_video_renderer_->SetSelectedStream({});
|
||||||
|
}
|
||||||
|
#endif
|
||||||
#if defined(__APPLE__)
|
#if defined(__APPLE__)
|
||||||
void* stream_view = (*ui_->stream)->window().appkit_view();
|
void* stream_view = (*ui_->stream)->window().appkit_view();
|
||||||
SetStreamWindowFullscreen(false);
|
SetStreamWindowFullscreen(false);
|
||||||
@@ -2564,6 +2573,44 @@ void GuiApplication::SyncStreamVideoFrame() {
|
|||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#if defined(_WIN32)
|
||||||
|
if (windows_opengl_video_renderer_ &&
|
||||||
|
windows_opengl_video_renderer_->IsReady()) {
|
||||||
|
windows_opengl_video_renderer_->SetSelectedStream(props->remote_id_);
|
||||||
|
SubmitCachedFrameToOpenGl(props);
|
||||||
|
if (!(*ui_->stream)->get_native_video_enabled()) {
|
||||||
|
(*ui_->stream)->set_native_video_enabled(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
int width = 0;
|
||||||
|
int height = 0;
|
||||||
|
uint64_t sequence = 0;
|
||||||
|
{
|
||||||
|
std::lock_guard lock(props->video_frame_mutex_);
|
||||||
|
width = props->video_width_;
|
||||||
|
height = props->video_height_;
|
||||||
|
sequence = props->video_frame_sequence_;
|
||||||
|
}
|
||||||
|
const bool has_frame = width > 0 && height > 0 && sequence > 0;
|
||||||
|
(*ui_->stream)->set_has_frame(has_frame);
|
||||||
|
if (has_frame) {
|
||||||
|
(*ui_->stream)->set_receiving_text("");
|
||||||
|
(*ui_->stream)
|
||||||
|
->set_stats_resolution(UiText(std::to_string(width) + "x" +
|
||||||
|
std::to_string(height)));
|
||||||
|
if (ui_->displayed_frame_sequence[props->remote_id_] != sequence) {
|
||||||
|
ui_->displayed_frame_sequence[props->remote_id_] = sequence;
|
||||||
|
++props->frame_count_;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(*ui_->stream)->window().request_redraw();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ((*ui_->stream)->get_native_video_enabled()) {
|
||||||
|
(*ui_->stream)->set_native_video_enabled(false);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
std::shared_ptr<std::vector<unsigned char>> frame;
|
std::shared_ptr<std::vector<unsigned char>> frame;
|
||||||
int width = 0;
|
int width = 0;
|
||||||
int height = 0;
|
int height = 0;
|
||||||
@@ -2736,9 +2783,36 @@ void GuiApplication::ConfigureStreamVideoRenderer() {
|
|||||||
glBindTexture(GL_TEXTURE_2D,
|
glBindTexture(GL_TEXTURE_2D,
|
||||||
static_cast<GLuint>(previous_texture));
|
static_cast<GLuint>(previous_texture));
|
||||||
ui_->video_gl_texture = texture;
|
ui_->video_gl_texture = texture;
|
||||||
|
#if defined(_WIN32)
|
||||||
|
if (windows_opengl_video_renderer_ &&
|
||||||
|
windows_opengl_video_renderer_->Setup()) {
|
||||||
|
windows_opengl_video_renderer_->SetSelectedStream(
|
||||||
|
focused_remote_id_);
|
||||||
|
video_frame_dirty_.store(true, std::memory_order_release);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if defined(_WIN32)
|
||||||
|
if (state == slint::RenderingState::BeforeRendering &&
|
||||||
|
windows_opengl_video_renderer_ &&
|
||||||
|
windows_opengl_video_renderer_->IsReady() && ui_->stream &&
|
||||||
|
(*ui_->stream)->get_native_video_enabled()) {
|
||||||
|
const auto window_size = (*ui_->stream)->window().size();
|
||||||
|
const float scale_factor =
|
||||||
|
std::max(1.0f, (*ui_->stream)->window().scale_factor());
|
||||||
|
const int top_inset =
|
||||||
|
!fullscreen_button_pressed_ && ui_->tab_ids.size() > 1
|
||||||
|
? static_cast<int>(std::lround(30.0f * scale_factor))
|
||||||
|
: 0;
|
||||||
|
windows_opengl_video_renderer_->RenderLatest(
|
||||||
|
focused_remote_id_, static_cast<int>(window_size.width),
|
||||||
|
static_cast<int>(window_size.height), top_inset);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
if (state == slint::RenderingState::BeforeRendering &&
|
if (state == slint::RenderingState::BeforeRendering &&
|
||||||
ui_->video_gl_texture != 0 && ui_->video_gl_pending_dirty &&
|
ui_->video_gl_texture != 0 && ui_->video_gl_pending_dirty &&
|
||||||
!ui_->video_gl_pending_frame.empty()) {
|
!ui_->video_gl_pending_frame.empty()) {
|
||||||
@@ -2769,6 +2843,11 @@ void GuiApplication::ConfigureStreamVideoRenderer() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (state == slint::RenderingState::RenderingTeardown) {
|
if (state == slint::RenderingState::RenderingTeardown) {
|
||||||
|
#if defined(_WIN32)
|
||||||
|
if (windows_opengl_video_renderer_) {
|
||||||
|
windows_opengl_video_renderer_->Teardown();
|
||||||
|
}
|
||||||
|
#endif
|
||||||
if (ui_->video_gl_texture != 0) {
|
if (ui_->video_gl_texture != 0) {
|
||||||
const GLuint texture = ui_->video_gl_texture;
|
const GLuint texture = ui_->video_gl_texture;
|
||||||
glDeleteTextures(1, &texture);
|
glDeleteTextures(1, &texture);
|
||||||
@@ -2783,10 +2862,50 @@ void GuiApplication::ConfigureStreamVideoRenderer() {
|
|||||||
});
|
});
|
||||||
if (error.has_value()) {
|
if (error.has_value()) {
|
||||||
LOG_WARN("Slint OpenGL video renderer unavailable, using pixel buffers");
|
LOG_WARN("Slint OpenGL video renderer unavailable, using pixel buffers");
|
||||||
|
#if defined(_WIN32)
|
||||||
|
} else if (windows_opengl_video_renderer_) {
|
||||||
|
windows_opengl_video_renderer_->SetSelectedStream(focused_remote_id_);
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if defined(_WIN32)
|
||||||
|
void GuiApplication::SubmitCachedFrameToOpenGl(
|
||||||
|
const std::shared_ptr<RemoteSession>& session) {
|
||||||
|
if (!session || !windows_opengl_video_renderer_ ||
|
||||||
|
!windows_opengl_video_renderer_->IsReady() ||
|
||||||
|
session->connection_status_.load() != ConnectionStatus::Connected) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::shared_ptr<std::vector<unsigned char>> cached_frame;
|
||||||
|
int cached_width = 0;
|
||||||
|
int cached_height = 0;
|
||||||
|
{
|
||||||
|
std::lock_guard lock(session->video_frame_mutex_);
|
||||||
|
if (session->front_frame_ && !session->front_frame_->empty()) {
|
||||||
|
cached_frame = session->front_frame_;
|
||||||
|
cached_width = session->video_width_;
|
||||||
|
cached_height = session->video_height_;
|
||||||
|
} else if (session->thumbnail_frame_ &&
|
||||||
|
!session->thumbnail_frame_->empty()) {
|
||||||
|
cached_frame = session->thumbnail_frame_;
|
||||||
|
cached_width = session->thumbnail_width_;
|
||||||
|
cached_height = session->thumbnail_height_;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cached_frame && cached_width > 0 && cached_height > 0 &&
|
||||||
|
windows_opengl_video_renderer_->SubmitCachedNv12(
|
||||||
|
session->remote_id_, cached_frame->data(), cached_frame->size(),
|
||||||
|
cached_width, cached_height) ==
|
||||||
|
WindowsOpenGlVideoRenderer::SubmitResult::submitted) {
|
||||||
|
video_frame_dirty_.store(true, std::memory_order_release);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
#if defined(__APPLE__)
|
#if defined(__APPLE__)
|
||||||
void GuiApplication::SubmitCachedFrameToMetal(
|
void GuiApplication::SubmitCachedFrameToMetal(
|
||||||
const std::shared_ptr<RemoteSession>& session) {
|
const std::shared_ptr<RemoteSession>& session) {
|
||||||
@@ -3124,7 +3243,7 @@ void GuiApplication::SelectStreamTab(int index) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const std::string selected_remote_id = ui_->tab_ids[index];
|
const std::string selected_remote_id = ui_->tab_ids[index];
|
||||||
#if defined(__APPLE__)
|
#if defined(__APPLE__) || defined(_WIN32)
|
||||||
const bool selection_changed = focused_remote_id_ != selected_remote_id;
|
const bool selection_changed = focused_remote_id_ != selected_remote_id;
|
||||||
bool renderer_selection_changed = false;
|
bool renderer_selection_changed = false;
|
||||||
#endif
|
#endif
|
||||||
@@ -3134,7 +3253,18 @@ void GuiApplication::SelectStreamTab(int index) {
|
|||||||
}
|
}
|
||||||
focused_remote_id_ = selected_remote_id;
|
focused_remote_id_ = selected_remote_id;
|
||||||
controlled_remote_id_ = selected_remote_id;
|
controlled_remote_id_ = selected_remote_id;
|
||||||
#if defined(__APPLE__)
|
#if defined(_WIN32)
|
||||||
|
if (windows_opengl_video_renderer_) {
|
||||||
|
renderer_selection_changed =
|
||||||
|
windows_opengl_video_renderer_->SetSelectedStream(selected_remote_id);
|
||||||
|
}
|
||||||
|
if (renderer_selection_changed) {
|
||||||
|
video_frame_dirty_.store(true, std::memory_order_release);
|
||||||
|
}
|
||||||
|
if (selection_changed && ui_->stream) {
|
||||||
|
(*ui_->stream)->set_has_frame(false);
|
||||||
|
}
|
||||||
|
#elif defined(__APPLE__)
|
||||||
if (mac_metal_video_renderer_) {
|
if (mac_metal_video_renderer_) {
|
||||||
renderer_selection_changed =
|
renderer_selection_changed =
|
||||||
mac_metal_video_renderer_->SetSelectedStream(selected_remote_id);
|
mac_metal_video_renderer_->SetSelectedStream(selected_remote_id);
|
||||||
@@ -3163,7 +3293,11 @@ void GuiApplication::SelectStreamTab(int index) {
|
|||||||
selected_session->connection_status_.load() ==
|
selected_session->connection_status_.load() ==
|
||||||
ConnectionStatus::Connected;
|
ConnectionStatus::Connected;
|
||||||
}
|
}
|
||||||
#if defined(__APPLE__)
|
#if defined(_WIN32)
|
||||||
|
if (renderer_selection_changed) {
|
||||||
|
SubmitCachedFrameToOpenGl(selected_session);
|
||||||
|
}
|
||||||
|
#elif defined(__APPLE__)
|
||||||
if (renderer_selection_changed) {
|
if (renderer_selection_changed) {
|
||||||
SubmitCachedFrameToMetal(selected_session);
|
SubmitCachedFrameToMetal(selected_session);
|
||||||
}
|
}
|
||||||
@@ -3215,7 +3349,12 @@ void GuiApplication::CloseStreamTab(const std::string& remote_id) {
|
|||||||
if (focused_remote_id_ == remote_id) {
|
if (focused_remote_id_ == remote_id) {
|
||||||
focused_remote_id_.clear();
|
focused_remote_id_.clear();
|
||||||
controlled_remote_id_.clear();
|
controlled_remote_id_.clear();
|
||||||
#if defined(__APPLE__)
|
#if defined(_WIN32)
|
||||||
|
if (windows_opengl_video_renderer_ &&
|
||||||
|
windows_opengl_video_renderer_->SetSelectedStream({})) {
|
||||||
|
video_frame_dirty_.store(true, std::memory_order_release);
|
||||||
|
}
|
||||||
|
#elif defined(__APPLE__)
|
||||||
if (mac_metal_video_renderer_ &&
|
if (mac_metal_video_renderer_ &&
|
||||||
mac_metal_video_renderer_->SetSelectedStream({})) {
|
mac_metal_video_renderer_->SetSelectedStream({})) {
|
||||||
video_frame_dirty_.store(true, std::memory_order_release);
|
video_frame_dirty_.store(true, std::memory_order_release);
|
||||||
@@ -3414,6 +3553,11 @@ void GuiApplication::Cleanup() {
|
|||||||
WaitForThumbnailSaveTasks();
|
WaitForThumbnailSaveTasks();
|
||||||
devices_.DestroyAudioOutput();
|
devices_.DestroyAudioOutput();
|
||||||
if (ui_->stream) {
|
if (ui_->stream) {
|
||||||
|
#if defined(_WIN32)
|
||||||
|
if (windows_opengl_video_renderer_) {
|
||||||
|
windows_opengl_video_renderer_->SetSelectedStream({});
|
||||||
|
}
|
||||||
|
#endif
|
||||||
#if defined(__APPLE__)
|
#if defined(__APPLE__)
|
||||||
void* stream_view = (*ui_->stream)->window().appkit_view();
|
void* stream_view = (*ui_->stream)->window().appkit_view();
|
||||||
SetStreamWindowFullscreen(false);
|
SetStreamWindowFullscreen(false);
|
||||||
|
|||||||
@@ -41,6 +41,10 @@ private:
|
|||||||
void SyncStreamVideoFrame();
|
void SyncStreamVideoFrame();
|
||||||
void ScheduleNextVideoFrame();
|
void ScheduleNextVideoFrame();
|
||||||
void ConfigureStreamVideoRenderer();
|
void ConfigureStreamVideoRenderer();
|
||||||
|
#if defined(_WIN32)
|
||||||
|
void SubmitCachedFrameToOpenGl(
|
||||||
|
const std::shared_ptr<RemoteSession>& session);
|
||||||
|
#endif
|
||||||
#if defined(__APPLE__)
|
#if defined(__APPLE__)
|
||||||
void SubmitCachedFrameToMetal(
|
void SubmitCachedFrameToMetal(
|
||||||
const std::shared_ptr<RemoteSession>& session);
|
const std::shared_ptr<RemoteSession>& session);
|
||||||
|
|||||||
@@ -0,0 +1,812 @@
|
|||||||
|
#include "platform/windows_opengl_video_renderer.h"
|
||||||
|
|
||||||
|
#if !defined(_WIN32)
|
||||||
|
#error "WindowsOpenGlVideoRenderer is only available on Windows"
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#include <windows.h>
|
||||||
|
|
||||||
|
#include <GL/gl.h>
|
||||||
|
#include <GL/glext.h>
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <array>
|
||||||
|
#include <atomic>
|
||||||
|
#include <cstring>
|
||||||
|
#include <limits>
|
||||||
|
#include <mutex>
|
||||||
|
#include <string>
|
||||||
|
#include <utility>
|
||||||
|
|
||||||
|
#include "rd_log.h"
|
||||||
|
|
||||||
|
namespace crossdesk {
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
constexpr size_t kFrameSlotCount = 3;
|
||||||
|
|
||||||
|
enum class SlotUse {
|
||||||
|
available,
|
||||||
|
pending,
|
||||||
|
uploading,
|
||||||
|
};
|
||||||
|
|
||||||
|
struct FrameSlot {
|
||||||
|
std::vector<unsigned char> bytes;
|
||||||
|
int width = 0;
|
||||||
|
int height = 0;
|
||||||
|
std::string remote_id;
|
||||||
|
uint64_t sequence = 0;
|
||||||
|
SlotUse use = SlotUse::available;
|
||||||
|
bool valid = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct SharedFrameState {
|
||||||
|
mutable std::mutex mutex;
|
||||||
|
std::array<FrameSlot, kFrameSlotCount> slots;
|
||||||
|
std::string selected_stream;
|
||||||
|
uint64_t next_sequence = 1;
|
||||||
|
};
|
||||||
|
|
||||||
|
template <typename T> bool LoadOpenGlFunction(T *function, const char *name) {
|
||||||
|
PROC address = wglGetProcAddress(name);
|
||||||
|
if (address == nullptr || address == reinterpret_cast<PROC>(1) ||
|
||||||
|
address == reinterpret_cast<PROC>(2) ||
|
||||||
|
address == reinterpret_cast<PROC>(3) ||
|
||||||
|
address == reinterpret_cast<PROC>(-1)) {
|
||||||
|
HMODULE module = GetModuleHandleW(L"opengl32.dll");
|
||||||
|
address = module ? GetProcAddress(module, name) : nullptr;
|
||||||
|
}
|
||||||
|
*function = reinterpret_cast<T>(address);
|
||||||
|
return *function != nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct OpenGlFunctions {
|
||||||
|
PFNGLACTIVETEXTUREPROC active_texture = nullptr;
|
||||||
|
PFNGLATTACHSHADERPROC attach_shader = nullptr;
|
||||||
|
PFNGLBINDBUFFERPROC bind_buffer = nullptr;
|
||||||
|
PFNGLBUFFERDATAPROC buffer_data = nullptr;
|
||||||
|
PFNGLCOMPILESHADERPROC compile_shader = nullptr;
|
||||||
|
PFNGLCREATEPROGRAMPROC create_program = nullptr;
|
||||||
|
PFNGLCREATESHADERPROC create_shader = nullptr;
|
||||||
|
PFNGLDELETEBUFFERSPROC delete_buffers = nullptr;
|
||||||
|
PFNGLDELETEPROGRAMPROC delete_program = nullptr;
|
||||||
|
PFNGLDELETESHADERPROC delete_shader = nullptr;
|
||||||
|
PFNGLDISABLEVERTEXATTRIBARRAYPROC disable_vertex_attrib_array = nullptr;
|
||||||
|
PFNGLENABLEVERTEXATTRIBARRAYPROC enable_vertex_attrib_array = nullptr;
|
||||||
|
PFNGLGENBUFFERSPROC gen_buffers = nullptr;
|
||||||
|
PFNGLGETATTRIBLOCATIONPROC get_attrib_location = nullptr;
|
||||||
|
PFNGLGETPROGRAMINFOLOGPROC get_program_info_log = nullptr;
|
||||||
|
PFNGLGETPROGRAMIVPROC get_program_iv = nullptr;
|
||||||
|
PFNGLGETSHADERINFOLOGPROC get_shader_info_log = nullptr;
|
||||||
|
PFNGLGETSHADERIVPROC get_shader_iv = nullptr;
|
||||||
|
PFNGLGETUNIFORMLOCATIONPROC get_uniform_location = nullptr;
|
||||||
|
PFNGLGETVERTEXATTRIBIVPROC get_vertex_attrib_iv = nullptr;
|
||||||
|
PFNGLGETVERTEXATTRIBPOINTERVPROC get_vertex_attrib_pointer_v = nullptr;
|
||||||
|
PFNGLLINKPROGRAMPROC link_program = nullptr;
|
||||||
|
PFNGLSHADERSOURCEPROC shader_source = nullptr;
|
||||||
|
PFNGLUNIFORM1IPROC uniform_1i = nullptr;
|
||||||
|
PFNGLUSEPROGRAMPROC use_program = nullptr;
|
||||||
|
PFNGLVERTEXATTRIBPOINTERPROC vertex_attrib_pointer = nullptr;
|
||||||
|
|
||||||
|
bool Load() {
|
||||||
|
return LoadOpenGlFunction(&active_texture, "glActiveTexture") &&
|
||||||
|
LoadOpenGlFunction(&attach_shader, "glAttachShader") &&
|
||||||
|
LoadOpenGlFunction(&bind_buffer, "glBindBuffer") &&
|
||||||
|
LoadOpenGlFunction(&buffer_data, "glBufferData") &&
|
||||||
|
LoadOpenGlFunction(&compile_shader, "glCompileShader") &&
|
||||||
|
LoadOpenGlFunction(&create_program, "glCreateProgram") &&
|
||||||
|
LoadOpenGlFunction(&create_shader, "glCreateShader") &&
|
||||||
|
LoadOpenGlFunction(&delete_buffers, "glDeleteBuffers") &&
|
||||||
|
LoadOpenGlFunction(&delete_program, "glDeleteProgram") &&
|
||||||
|
LoadOpenGlFunction(&delete_shader, "glDeleteShader") &&
|
||||||
|
LoadOpenGlFunction(&disable_vertex_attrib_array,
|
||||||
|
"glDisableVertexAttribArray") &&
|
||||||
|
LoadOpenGlFunction(&enable_vertex_attrib_array,
|
||||||
|
"glEnableVertexAttribArray") &&
|
||||||
|
LoadOpenGlFunction(&gen_buffers, "glGenBuffers") &&
|
||||||
|
LoadOpenGlFunction(&get_attrib_location, "glGetAttribLocation") &&
|
||||||
|
LoadOpenGlFunction(&get_program_info_log, "glGetProgramInfoLog") &&
|
||||||
|
LoadOpenGlFunction(&get_program_iv, "glGetProgramiv") &&
|
||||||
|
LoadOpenGlFunction(&get_shader_info_log, "glGetShaderInfoLog") &&
|
||||||
|
LoadOpenGlFunction(&get_shader_iv, "glGetShaderiv") &&
|
||||||
|
LoadOpenGlFunction(&get_uniform_location, "glGetUniformLocation") &&
|
||||||
|
LoadOpenGlFunction(&get_vertex_attrib_iv, "glGetVertexAttribiv") &&
|
||||||
|
LoadOpenGlFunction(&get_vertex_attrib_pointer_v,
|
||||||
|
"glGetVertexAttribPointerv") &&
|
||||||
|
LoadOpenGlFunction(&link_program, "glLinkProgram") &&
|
||||||
|
LoadOpenGlFunction(&shader_source, "glShaderSource") &&
|
||||||
|
LoadOpenGlFunction(&uniform_1i, "glUniform1i") &&
|
||||||
|
LoadOpenGlFunction(&use_program, "glUseProgram") &&
|
||||||
|
LoadOpenGlFunction(&vertex_attrib_pointer, "glVertexAttribPointer");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
GLuint CompileShader(const OpenGlFunctions &gl, GLenum type,
|
||||||
|
const char *source) {
|
||||||
|
const GLuint shader = gl.create_shader(type);
|
||||||
|
if (shader == 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
gl.shader_source(shader, 1, &source, nullptr);
|
||||||
|
gl.compile_shader(shader);
|
||||||
|
|
||||||
|
GLint compiled = GL_FALSE;
|
||||||
|
gl.get_shader_iv(shader, GL_COMPILE_STATUS, &compiled);
|
||||||
|
if (compiled == GL_TRUE) {
|
||||||
|
return shader;
|
||||||
|
}
|
||||||
|
|
||||||
|
GLint log_length = 0;
|
||||||
|
gl.get_shader_iv(shader, GL_INFO_LOG_LENGTH, &log_length);
|
||||||
|
std::string log(static_cast<size_t>(std::max(log_length, 1)), '\0');
|
||||||
|
GLsizei written = 0;
|
||||||
|
gl.get_shader_info_log(shader, static_cast<GLsizei>(log.size()), &written,
|
||||||
|
log.data());
|
||||||
|
if (written >= 0 && static_cast<size_t>(written) < log.size()) {
|
||||||
|
log.resize(static_cast<size_t>(written));
|
||||||
|
}
|
||||||
|
LOG_ERROR("OpenGL NV12 {} shader compilation failed: {}",
|
||||||
|
type == GL_VERTEX_SHADER ? "vertex" : "fragment", log);
|
||||||
|
gl.delete_shader(shader);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool IsEnabled(GLenum capability) { return glIsEnabled(capability) == GL_TRUE; }
|
||||||
|
|
||||||
|
void RestoreCapability(GLenum capability, bool enabled) {
|
||||||
|
if (enabled) {
|
||||||
|
glEnable(capability);
|
||||||
|
} else {
|
||||||
|
glDisable(capability);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct VertexAttribState {
|
||||||
|
GLint enabled = GL_FALSE;
|
||||||
|
GLint size = 4;
|
||||||
|
GLint stride = 0;
|
||||||
|
GLint type = GL_FLOAT;
|
||||||
|
GLint normalized = GL_FALSE;
|
||||||
|
GLint buffer = 0;
|
||||||
|
void *pointer = nullptr;
|
||||||
|
};
|
||||||
|
|
||||||
|
VertexAttribState CaptureVertexAttrib(const OpenGlFunctions &gl, GLuint index) {
|
||||||
|
VertexAttribState state;
|
||||||
|
gl.get_vertex_attrib_iv(index, GL_VERTEX_ATTRIB_ARRAY_ENABLED,
|
||||||
|
&state.enabled);
|
||||||
|
gl.get_vertex_attrib_iv(index, GL_VERTEX_ATTRIB_ARRAY_SIZE, &state.size);
|
||||||
|
gl.get_vertex_attrib_iv(index, GL_VERTEX_ATTRIB_ARRAY_STRIDE, &state.stride);
|
||||||
|
gl.get_vertex_attrib_iv(index, GL_VERTEX_ATTRIB_ARRAY_TYPE, &state.type);
|
||||||
|
gl.get_vertex_attrib_iv(index, GL_VERTEX_ATTRIB_ARRAY_NORMALIZED,
|
||||||
|
&state.normalized);
|
||||||
|
gl.get_vertex_attrib_iv(index, GL_VERTEX_ATTRIB_ARRAY_BUFFER_BINDING,
|
||||||
|
&state.buffer);
|
||||||
|
gl.get_vertex_attrib_pointer_v(index, GL_VERTEX_ATTRIB_ARRAY_POINTER,
|
||||||
|
&state.pointer);
|
||||||
|
return state;
|
||||||
|
}
|
||||||
|
|
||||||
|
void RestoreVertexAttrib(const OpenGlFunctions &gl, GLuint index,
|
||||||
|
const VertexAttribState &state) {
|
||||||
|
if (state.buffer != 0 || state.pointer != nullptr) {
|
||||||
|
gl.bind_buffer(GL_ARRAY_BUFFER, static_cast<GLuint>(state.buffer));
|
||||||
|
gl.vertex_attrib_pointer(index, state.size, static_cast<GLenum>(state.type),
|
||||||
|
static_cast<GLboolean>(state.normalized),
|
||||||
|
state.stride, state.pointer);
|
||||||
|
}
|
||||||
|
if (state.enabled == GL_TRUE) {
|
||||||
|
gl.enable_vertex_attrib_array(index);
|
||||||
|
} else {
|
||||||
|
gl.disable_vertex_attrib_array(index);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
struct WindowsOpenGlVideoRenderer::Impl {
|
||||||
|
OpenGlFunctions gl;
|
||||||
|
std::shared_ptr<SharedFrameState> frames =
|
||||||
|
std::make_shared<SharedFrameState>();
|
||||||
|
std::atomic<bool> ready{false};
|
||||||
|
GLuint program = 0;
|
||||||
|
GLuint vertex_buffer = 0;
|
||||||
|
GLuint y_texture = 0;
|
||||||
|
GLuint uv_texture = 0;
|
||||||
|
GLint position_location = -1;
|
||||||
|
GLint texcoord_location = -1;
|
||||||
|
GLint y_texture_location = -1;
|
||||||
|
GLint uv_texture_location = -1;
|
||||||
|
int texture_width = 0;
|
||||||
|
int texture_height = 0;
|
||||||
|
std::string uploaded_stream;
|
||||||
|
uint64_t uploaded_sequence = 0;
|
||||||
|
|
||||||
|
void ResetGlHandles() {
|
||||||
|
program = 0;
|
||||||
|
vertex_buffer = 0;
|
||||||
|
y_texture = 0;
|
||||||
|
uv_texture = 0;
|
||||||
|
position_location = -1;
|
||||||
|
texcoord_location = -1;
|
||||||
|
y_texture_location = -1;
|
||||||
|
uv_texture_location = -1;
|
||||||
|
texture_width = 0;
|
||||||
|
texture_height = 0;
|
||||||
|
uploaded_stream.clear();
|
||||||
|
uploaded_sequence = 0;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
WindowsOpenGlVideoRenderer::WindowsOpenGlVideoRenderer()
|
||||||
|
: impl_(std::make_unique<Impl>()) {}
|
||||||
|
|
||||||
|
WindowsOpenGlVideoRenderer::~WindowsOpenGlVideoRenderer() = default;
|
||||||
|
|
||||||
|
bool WindowsOpenGlVideoRenderer::Setup() {
|
||||||
|
if (IsReady()) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
impl_->ready.store(false, std::memory_order_release);
|
||||||
|
impl_->ResetGlHandles();
|
||||||
|
if (!impl_->gl.Load()) {
|
||||||
|
LOG_WARN("OpenGL NV12 underlay unavailable: required functions missing");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
static constexpr char kGlesVertexShader[] = R"glsl(
|
||||||
|
#version 100
|
||||||
|
attribute vec2 position;
|
||||||
|
attribute vec2 texcoord;
|
||||||
|
varying vec2 video_texcoord;
|
||||||
|
void main() {
|
||||||
|
video_texcoord = texcoord;
|
||||||
|
gl_Position = vec4(position, 0.0, 1.0);
|
||||||
|
}
|
||||||
|
)glsl";
|
||||||
|
static constexpr char kGlesFragmentShader[] = R"glsl(
|
||||||
|
#version 100
|
||||||
|
#ifdef GL_FRAGMENT_PRECISION_HIGH
|
||||||
|
precision highp float;
|
||||||
|
#else
|
||||||
|
precision mediump float;
|
||||||
|
#endif
|
||||||
|
uniform sampler2D y_texture;
|
||||||
|
uniform sampler2D uv_texture;
|
||||||
|
varying vec2 video_texcoord;
|
||||||
|
void main() {
|
||||||
|
float y = 1.16438356 *
|
||||||
|
(texture2D(y_texture, video_texcoord).r - 16.0 / 255.0);
|
||||||
|
vec2 uv = texture2D(uv_texture, video_texcoord).ra - vec2(0.5, 0.5);
|
||||||
|
vec3 rgb = vec3(y + 1.59602678 * uv.y,
|
||||||
|
y - 0.39176229 * uv.x - 0.81296764 * uv.y,
|
||||||
|
y + 2.01723214 * uv.x);
|
||||||
|
gl_FragColor = vec4(clamp(rgb, 0.0, 1.0), 1.0);
|
||||||
|
}
|
||||||
|
)glsl";
|
||||||
|
static constexpr char kDesktopVertexShader[] = R"glsl(
|
||||||
|
#version 110
|
||||||
|
attribute vec2 position;
|
||||||
|
attribute vec2 texcoord;
|
||||||
|
varying vec2 video_texcoord;
|
||||||
|
void main() {
|
||||||
|
video_texcoord = texcoord;
|
||||||
|
gl_Position = vec4(position, 0.0, 1.0);
|
||||||
|
}
|
||||||
|
)glsl";
|
||||||
|
static constexpr char kDesktopFragmentShader[] = R"glsl(
|
||||||
|
#version 110
|
||||||
|
uniform sampler2D y_texture;
|
||||||
|
uniform sampler2D uv_texture;
|
||||||
|
varying vec2 video_texcoord;
|
||||||
|
void main() {
|
||||||
|
float y = 1.16438356 *
|
||||||
|
(texture2D(y_texture, video_texcoord).r - 16.0 / 255.0);
|
||||||
|
vec2 uv = texture2D(uv_texture, video_texcoord).ra - vec2(0.5, 0.5);
|
||||||
|
vec3 rgb = vec3(y + 1.59602678 * uv.y,
|
||||||
|
y - 0.39176229 * uv.x - 0.81296764 * uv.y,
|
||||||
|
y + 2.01723214 * uv.x);
|
||||||
|
gl_FragColor = vec4(clamp(rgb, 0.0, 1.0), 1.0);
|
||||||
|
}
|
||||||
|
)glsl";
|
||||||
|
|
||||||
|
const auto *version = reinterpret_cast<const char *>(glGetString(GL_VERSION));
|
||||||
|
const bool is_gles = version && std::strstr(version, "OpenGL ES") != nullptr;
|
||||||
|
const char *vertex_source =
|
||||||
|
is_gles ? kGlesVertexShader : kDesktopVertexShader;
|
||||||
|
const char *fragment_source =
|
||||||
|
is_gles ? kGlesFragmentShader : kDesktopFragmentShader;
|
||||||
|
|
||||||
|
const GLuint vertex_shader =
|
||||||
|
CompileShader(impl_->gl, GL_VERTEX_SHADER, vertex_source);
|
||||||
|
const GLuint fragment_shader =
|
||||||
|
CompileShader(impl_->gl, GL_FRAGMENT_SHADER, fragment_source);
|
||||||
|
if (vertex_shader == 0 || fragment_shader == 0) {
|
||||||
|
if (vertex_shader != 0)
|
||||||
|
impl_->gl.delete_shader(vertex_shader);
|
||||||
|
if (fragment_shader != 0)
|
||||||
|
impl_->gl.delete_shader(fragment_shader);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl_->program = impl_->gl.create_program();
|
||||||
|
impl_->gl.attach_shader(impl_->program, vertex_shader);
|
||||||
|
impl_->gl.attach_shader(impl_->program, fragment_shader);
|
||||||
|
impl_->gl.link_program(impl_->program);
|
||||||
|
impl_->gl.delete_shader(vertex_shader);
|
||||||
|
impl_->gl.delete_shader(fragment_shader);
|
||||||
|
|
||||||
|
GLint linked = GL_FALSE;
|
||||||
|
impl_->gl.get_program_iv(impl_->program, GL_LINK_STATUS, &linked);
|
||||||
|
if (linked != GL_TRUE) {
|
||||||
|
GLint log_length = 0;
|
||||||
|
impl_->gl.get_program_iv(impl_->program, GL_INFO_LOG_LENGTH, &log_length);
|
||||||
|
std::string log(static_cast<size_t>(std::max(log_length, 1)), '\0');
|
||||||
|
GLsizei written = 0;
|
||||||
|
impl_->gl.get_program_info_log(
|
||||||
|
impl_->program, static_cast<GLsizei>(log.size()), &written, log.data());
|
||||||
|
if (written >= 0 && static_cast<size_t>(written) < log.size()) {
|
||||||
|
log.resize(static_cast<size_t>(written));
|
||||||
|
}
|
||||||
|
LOG_ERROR("OpenGL NV12 program link failed: {}", log);
|
||||||
|
impl_->gl.delete_program(impl_->program);
|
||||||
|
impl_->ResetGlHandles();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl_->position_location =
|
||||||
|
impl_->gl.get_attrib_location(impl_->program, "position");
|
||||||
|
impl_->texcoord_location =
|
||||||
|
impl_->gl.get_attrib_location(impl_->program, "texcoord");
|
||||||
|
impl_->y_texture_location =
|
||||||
|
impl_->gl.get_uniform_location(impl_->program, "y_texture");
|
||||||
|
impl_->uv_texture_location =
|
||||||
|
impl_->gl.get_uniform_location(impl_->program, "uv_texture");
|
||||||
|
if (impl_->position_location < 0 || impl_->texcoord_location < 0 ||
|
||||||
|
impl_->y_texture_location < 0 || impl_->uv_texture_location < 0) {
|
||||||
|
LOG_ERROR("OpenGL NV12 program is missing required shader bindings");
|
||||||
|
impl_->gl.delete_program(impl_->program);
|
||||||
|
impl_->ResetGlHandles();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
static constexpr GLfloat kVertices[] = {
|
||||||
|
-1.0f, 1.0f, 0.0f, 0.0f, // top-left
|
||||||
|
-1.0f, -1.0f, 0.0f, 1.0f, // bottom-left
|
||||||
|
1.0f, 1.0f, 1.0f, 0.0f, // top-right
|
||||||
|
1.0f, -1.0f, 1.0f, 1.0f, // bottom-right
|
||||||
|
};
|
||||||
|
GLint previous_array_buffer = 0;
|
||||||
|
glGetIntegerv(GL_ARRAY_BUFFER_BINDING, &previous_array_buffer);
|
||||||
|
impl_->gl.gen_buffers(1, &impl_->vertex_buffer);
|
||||||
|
impl_->gl.bind_buffer(GL_ARRAY_BUFFER, impl_->vertex_buffer);
|
||||||
|
impl_->gl.buffer_data(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices,
|
||||||
|
GL_STATIC_DRAW);
|
||||||
|
impl_->gl.bind_buffer(GL_ARRAY_BUFFER,
|
||||||
|
static_cast<GLuint>(previous_array_buffer));
|
||||||
|
|
||||||
|
GLint previous_active_texture = GL_TEXTURE0;
|
||||||
|
glGetIntegerv(GL_ACTIVE_TEXTURE, &previous_active_texture);
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE0);
|
||||||
|
GLint previous_texture_0 = 0;
|
||||||
|
glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture_0);
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE1);
|
||||||
|
GLint previous_texture_1 = 0;
|
||||||
|
glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture_1);
|
||||||
|
while (glGetError() != GL_NO_ERROR) {
|
||||||
|
}
|
||||||
|
glGenTextures(1, &impl_->y_texture);
|
||||||
|
glGenTextures(1, &impl_->uv_texture);
|
||||||
|
for (const auto [unit, texture] :
|
||||||
|
{std::pair{GL_TEXTURE0, impl_->y_texture},
|
||||||
|
std::pair{GL_TEXTURE1, impl_->uv_texture}}) {
|
||||||
|
impl_->gl.active_texture(unit);
|
||||||
|
glBindTexture(GL_TEXTURE_2D, texture);
|
||||||
|
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
|
||||||
|
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
|
||||||
|
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
|
||||||
|
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
|
||||||
|
}
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE0);
|
||||||
|
glBindTexture(GL_TEXTURE_2D, static_cast<GLuint>(previous_texture_0));
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE1);
|
||||||
|
glBindTexture(GL_TEXTURE_2D, static_cast<GLuint>(previous_texture_1));
|
||||||
|
impl_->gl.active_texture(static_cast<GLenum>(previous_active_texture));
|
||||||
|
|
||||||
|
const GLenum setup_error = glGetError();
|
||||||
|
if (impl_->program == 0 || impl_->vertex_buffer == 0 ||
|
||||||
|
impl_->y_texture == 0 || impl_->uv_texture == 0 ||
|
||||||
|
setup_error != GL_NO_ERROR) {
|
||||||
|
LOG_ERROR("OpenGL NV12 resource setup failed, error={}", setup_error);
|
||||||
|
Teardown();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl_->ready.store(true, std::memory_order_release);
|
||||||
|
LOG_INFO("OpenGL NV12 underlay initialized ({})",
|
||||||
|
version ? version : "unknown OpenGL version");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void WindowsOpenGlVideoRenderer::Teardown() {
|
||||||
|
impl_->ready.store(false, std::memory_order_release);
|
||||||
|
if (impl_->y_texture != 0)
|
||||||
|
glDeleteTextures(1, &impl_->y_texture);
|
||||||
|
if (impl_->uv_texture != 0)
|
||||||
|
glDeleteTextures(1, &impl_->uv_texture);
|
||||||
|
if (impl_->vertex_buffer != 0 && impl_->gl.delete_buffers) {
|
||||||
|
impl_->gl.delete_buffers(1, &impl_->vertex_buffer);
|
||||||
|
}
|
||||||
|
if (impl_->program != 0 && impl_->gl.delete_program) {
|
||||||
|
impl_->gl.delete_program(impl_->program);
|
||||||
|
}
|
||||||
|
impl_->ResetGlHandles();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool WindowsOpenGlVideoRenderer::IsReady() const {
|
||||||
|
return impl_->ready.load(std::memory_order_acquire);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool WindowsOpenGlVideoRenderer::SetSelectedStream(std::string remote_id) {
|
||||||
|
std::lock_guard lock(impl_->frames->mutex);
|
||||||
|
if (impl_->frames->selected_stream == remote_id) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
impl_->frames->selected_stream = std::move(remote_id);
|
||||||
|
for (auto &slot : impl_->frames->slots) {
|
||||||
|
if (slot.use == SlotUse::pending &&
|
||||||
|
slot.remote_id != impl_->frames->selected_stream) {
|
||||||
|
slot.use = SlotUse::available;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void WindowsOpenGlVideoRenderer::DiscardStream(std::string_view remote_id) {
|
||||||
|
std::lock_guard lock(impl_->frames->mutex);
|
||||||
|
for (auto &slot : impl_->frames->slots) {
|
||||||
|
if (slot.remote_id == remote_id && slot.use != SlotUse::uploading) {
|
||||||
|
slot.valid = false;
|
||||||
|
slot.use = SlotUse::available;
|
||||||
|
slot.remote_id.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (impl_->frames->selected_stream == remote_id) {
|
||||||
|
impl_->frames->selected_stream.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
WindowsOpenGlVideoRenderer::SubmitResult
|
||||||
|
WindowsOpenGlVideoRenderer::SubmitNv12(std::string_view remote_id,
|
||||||
|
const uint8_t *data, size_t size,
|
||||||
|
int width, int height) {
|
||||||
|
return SubmitNv12Internal(remote_id, data, size, width, height, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
WindowsOpenGlVideoRenderer::SubmitResult
|
||||||
|
WindowsOpenGlVideoRenderer::SubmitCachedNv12(std::string_view remote_id,
|
||||||
|
const uint8_t *data, size_t size,
|
||||||
|
int width, int height) {
|
||||||
|
return SubmitNv12Internal(remote_id, data, size, width, height, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
WindowsOpenGlVideoRenderer::SubmitResult
|
||||||
|
WindowsOpenGlVideoRenderer::SubmitNv12Internal(std::string_view remote_id,
|
||||||
|
const uint8_t *data, size_t size,
|
||||||
|
int width, int height,
|
||||||
|
bool replace_pending) {
|
||||||
|
if (!IsReady()) {
|
||||||
|
return SubmitResult::failed;
|
||||||
|
}
|
||||||
|
if (data == nullptr || width <= 0 || height <= 0 || (width & 1) != 0 ||
|
||||||
|
(height & 1) != 0) {
|
||||||
|
return SubmitResult::failed;
|
||||||
|
}
|
||||||
|
const size_t y_size = static_cast<size_t>(width) * height;
|
||||||
|
const size_t required_size = y_size + y_size / 2;
|
||||||
|
if (size < required_size) {
|
||||||
|
return SubmitResult::failed;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto &frames = *impl_->frames;
|
||||||
|
std::lock_guard lock(frames.mutex);
|
||||||
|
if (frames.selected_stream != remote_id) {
|
||||||
|
return SubmitResult::not_selected;
|
||||||
|
}
|
||||||
|
|
||||||
|
FrameSlot *target = nullptr;
|
||||||
|
if (replace_pending) {
|
||||||
|
for (auto &slot : frames.slots) {
|
||||||
|
if (slot.use == SlotUse::pending) {
|
||||||
|
target = &slot;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for (const auto &slot : frames.slots) {
|
||||||
|
if (slot.valid && slot.remote_id == remote_id &&
|
||||||
|
(slot.use == SlotUse::pending || slot.use == SlotUse::uploading)) {
|
||||||
|
return SubmitResult::dropped;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (target == nullptr) {
|
||||||
|
uint64_t oldest_sequence = std::numeric_limits<uint64_t>::max();
|
||||||
|
for (auto &slot : frames.slots) {
|
||||||
|
if (slot.use != SlotUse::available) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!slot.valid) {
|
||||||
|
target = &slot;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (slot.sequence < oldest_sequence) {
|
||||||
|
oldest_sequence = slot.sequence;
|
||||||
|
target = &slot;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (target == nullptr) {
|
||||||
|
return SubmitResult::dropped;
|
||||||
|
}
|
||||||
|
|
||||||
|
target->bytes.resize(required_size);
|
||||||
|
std::memcpy(target->bytes.data(), data, required_size);
|
||||||
|
target->width = width;
|
||||||
|
target->height = height;
|
||||||
|
target->remote_id.assign(remote_id);
|
||||||
|
target->sequence = frames.next_sequence++;
|
||||||
|
target->use = SlotUse::pending;
|
||||||
|
target->valid = true;
|
||||||
|
return SubmitResult::submitted;
|
||||||
|
}
|
||||||
|
|
||||||
|
WindowsOpenGlVideoRenderer::RenderOutcome
|
||||||
|
WindowsOpenGlVideoRenderer::RenderLatest(std::string_view remote_id,
|
||||||
|
int target_width, int target_height,
|
||||||
|
int top_inset_pixels) {
|
||||||
|
if (!IsReady() || target_width <= 0 || target_height <= 0) {
|
||||||
|
return {RenderResult::failed};
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t slot_index = kFrameSlotCount;
|
||||||
|
uint64_t sequence = 0;
|
||||||
|
int source_width = 0;
|
||||||
|
int source_height = 0;
|
||||||
|
const unsigned char *frame_data = nullptr;
|
||||||
|
{
|
||||||
|
std::lock_guard lock(impl_->frames->mutex);
|
||||||
|
for (size_t index = 0; index < impl_->frames->slots.size(); ++index) {
|
||||||
|
const auto &slot = impl_->frames->slots[index];
|
||||||
|
if (slot.valid && slot.use == SlotUse::pending &&
|
||||||
|
slot.remote_id == remote_id &&
|
||||||
|
(slot_index == kFrameSlotCount || slot.sequence > sequence)) {
|
||||||
|
slot_index = index;
|
||||||
|
sequence = slot.sequence;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (slot_index != kFrameSlotCount) {
|
||||||
|
auto &selected = impl_->frames->slots[slot_index];
|
||||||
|
selected.use = SlotUse::uploading;
|
||||||
|
source_width = selected.width;
|
||||||
|
source_height = selected.height;
|
||||||
|
frame_data = selected.bytes.data();
|
||||||
|
for (size_t index = 0; index < impl_->frames->slots.size(); ++index) {
|
||||||
|
auto &slot = impl_->frames->slots[index];
|
||||||
|
if (index != slot_index && slot.use == SlotUse::pending &&
|
||||||
|
slot.remote_id == remote_id) {
|
||||||
|
slot.use = SlotUse::available;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
GLint previous_program = 0;
|
||||||
|
GLint previous_active_texture = GL_TEXTURE0;
|
||||||
|
GLint previous_array_buffer = 0;
|
||||||
|
GLint previous_unpack_alignment = 4;
|
||||||
|
GLint previous_viewport[4] = {};
|
||||||
|
GLint previous_scissor_box[4] = {};
|
||||||
|
GLfloat previous_clear_color[4] = {};
|
||||||
|
GLboolean previous_color_mask[4] = {};
|
||||||
|
glGetIntegerv(GL_CURRENT_PROGRAM, &previous_program);
|
||||||
|
glGetIntegerv(GL_ACTIVE_TEXTURE, &previous_active_texture);
|
||||||
|
glGetIntegerv(GL_ARRAY_BUFFER_BINDING, &previous_array_buffer);
|
||||||
|
glGetIntegerv(GL_UNPACK_ALIGNMENT, &previous_unpack_alignment);
|
||||||
|
glGetIntegerv(GL_VIEWPORT, previous_viewport);
|
||||||
|
glGetIntegerv(GL_SCISSOR_BOX, previous_scissor_box);
|
||||||
|
glGetFloatv(GL_COLOR_CLEAR_VALUE, previous_clear_color);
|
||||||
|
glGetBooleanv(GL_COLOR_WRITEMASK, previous_color_mask);
|
||||||
|
const bool blend_enabled = IsEnabled(GL_BLEND);
|
||||||
|
const bool cull_enabled = IsEnabled(GL_CULL_FACE);
|
||||||
|
const bool depth_enabled = IsEnabled(GL_DEPTH_TEST);
|
||||||
|
const bool scissor_enabled = IsEnabled(GL_SCISSOR_TEST);
|
||||||
|
const bool stencil_enabled = IsEnabled(GL_STENCIL_TEST);
|
||||||
|
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE0);
|
||||||
|
GLint previous_texture_0 = 0;
|
||||||
|
glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture_0);
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE1);
|
||||||
|
GLint previous_texture_1 = 0;
|
||||||
|
glGetIntegerv(GL_TEXTURE_BINDING_2D, &previous_texture_1);
|
||||||
|
|
||||||
|
const auto position_state = CaptureVertexAttrib(
|
||||||
|
impl_->gl, static_cast<GLuint>(impl_->position_location));
|
||||||
|
const auto texcoord_state = CaptureVertexAttrib(
|
||||||
|
impl_->gl, static_cast<GLuint>(impl_->texcoord_location));
|
||||||
|
|
||||||
|
glDisable(GL_BLEND);
|
||||||
|
glDisable(GL_CULL_FACE);
|
||||||
|
glDisable(GL_DEPTH_TEST);
|
||||||
|
glDisable(GL_SCISSOR_TEST);
|
||||||
|
glDisable(GL_STENCIL_TEST);
|
||||||
|
glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
|
||||||
|
glViewport(0, 0, target_width, target_height);
|
||||||
|
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
|
||||||
|
glClear(GL_COLOR_BUFFER_BIT);
|
||||||
|
|
||||||
|
bool upload_succeeded = true;
|
||||||
|
if (slot_index != kFrameSlotCount) {
|
||||||
|
while (glGetError() != GL_NO_ERROR) {
|
||||||
|
}
|
||||||
|
const size_t y_size = static_cast<size_t>(source_width) * source_height;
|
||||||
|
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE0);
|
||||||
|
glBindTexture(GL_TEXTURE_2D, impl_->y_texture);
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE1);
|
||||||
|
glBindTexture(GL_TEXTURE_2D, impl_->uv_texture);
|
||||||
|
if (impl_->texture_width != source_width ||
|
||||||
|
impl_->texture_height != source_height) {
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE0);
|
||||||
|
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, source_width, source_height,
|
||||||
|
0, GL_LUMINANCE, GL_UNSIGNED_BYTE, frame_data);
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE1);
|
||||||
|
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE_ALPHA, source_width / 2,
|
||||||
|
source_height / 2, 0, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE,
|
||||||
|
frame_data + y_size);
|
||||||
|
} else {
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE0);
|
||||||
|
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, source_width, source_height,
|
||||||
|
GL_LUMINANCE, GL_UNSIGNED_BYTE, frame_data);
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE1);
|
||||||
|
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, source_width / 2,
|
||||||
|
source_height / 2, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE,
|
||||||
|
frame_data + y_size);
|
||||||
|
}
|
||||||
|
const GLenum upload_error = glGetError();
|
||||||
|
upload_succeeded = upload_error == GL_NO_ERROR;
|
||||||
|
if (upload_succeeded) {
|
||||||
|
impl_->texture_width = source_width;
|
||||||
|
impl_->texture_height = source_height;
|
||||||
|
impl_->uploaded_stream.assign(remote_id);
|
||||||
|
impl_->uploaded_sequence = sequence;
|
||||||
|
} else {
|
||||||
|
LOG_WARN("OpenGL NV12 texture upload failed, error={}", upload_error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
std::lock_guard lock(impl_->frames->mutex);
|
||||||
|
if (slot_index != kFrameSlotCount) {
|
||||||
|
auto &slot = impl_->frames->slots[slot_index];
|
||||||
|
if (slot.sequence == sequence && slot.use == SlotUse::uploading) {
|
||||||
|
slot.use = SlotUse::available;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
RenderOutcome outcome{RenderResult::empty};
|
||||||
|
if (upload_succeeded && impl_->uploaded_stream == remote_id &&
|
||||||
|
impl_->texture_width > 0 && impl_->texture_height > 0) {
|
||||||
|
source_width = impl_->texture_width;
|
||||||
|
source_height = impl_->texture_height;
|
||||||
|
sequence = impl_->uploaded_sequence;
|
||||||
|
const int video_height = std::max(
|
||||||
|
0, target_height - std::clamp(top_inset_pixels, 0, target_height));
|
||||||
|
if (video_height > 0) {
|
||||||
|
const double source_aspect =
|
||||||
|
static_cast<double>(source_width) / source_height;
|
||||||
|
const double target_aspect =
|
||||||
|
static_cast<double>(target_width) / video_height;
|
||||||
|
int viewport_width = target_width;
|
||||||
|
int viewport_height = video_height;
|
||||||
|
int viewport_x = 0;
|
||||||
|
int viewport_y = 0;
|
||||||
|
if (source_aspect > target_aspect) {
|
||||||
|
viewport_height =
|
||||||
|
std::max(1, static_cast<int>(target_width / source_aspect + 0.5));
|
||||||
|
viewport_y = (video_height - viewport_height) / 2;
|
||||||
|
} else {
|
||||||
|
viewport_width =
|
||||||
|
std::max(1, static_cast<int>(video_height * source_aspect + 0.5));
|
||||||
|
viewport_x = (target_width - viewport_width) / 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
glViewport(viewport_x, viewport_y, viewport_width, viewport_height);
|
||||||
|
impl_->gl.use_program(impl_->program);
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE0);
|
||||||
|
glBindTexture(GL_TEXTURE_2D, impl_->y_texture);
|
||||||
|
impl_->gl.uniform_1i(impl_->y_texture_location, 0);
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE1);
|
||||||
|
glBindTexture(GL_TEXTURE_2D, impl_->uv_texture);
|
||||||
|
impl_->gl.uniform_1i(impl_->uv_texture_location, 1);
|
||||||
|
impl_->gl.bind_buffer(GL_ARRAY_BUFFER, impl_->vertex_buffer);
|
||||||
|
impl_->gl.vertex_attrib_pointer(
|
||||||
|
static_cast<GLuint>(impl_->position_location), 2, GL_FLOAT, GL_FALSE,
|
||||||
|
4 * static_cast<GLsizei>(sizeof(GLfloat)), nullptr);
|
||||||
|
impl_->gl.vertex_attrib_pointer(
|
||||||
|
static_cast<GLuint>(impl_->texcoord_location), 2, GL_FLOAT, GL_FALSE,
|
||||||
|
4 * static_cast<GLsizei>(sizeof(GLfloat)),
|
||||||
|
reinterpret_cast<const void *>(2 * sizeof(GLfloat)));
|
||||||
|
impl_->gl.enable_vertex_attrib_array(
|
||||||
|
static_cast<GLuint>(impl_->position_location));
|
||||||
|
impl_->gl.enable_vertex_attrib_array(
|
||||||
|
static_cast<GLuint>(impl_->texcoord_location));
|
||||||
|
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
|
||||||
|
outcome = {glGetError() == GL_NO_ERROR ? RenderResult::rendered
|
||||||
|
: RenderResult::failed,
|
||||||
|
source_width, source_height, sequence};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
RestoreVertexAttrib(impl_->gl, static_cast<GLuint>(impl_->position_location),
|
||||||
|
position_state);
|
||||||
|
RestoreVertexAttrib(impl_->gl, static_cast<GLuint>(impl_->texcoord_location),
|
||||||
|
texcoord_state);
|
||||||
|
impl_->gl.bind_buffer(GL_ARRAY_BUFFER,
|
||||||
|
static_cast<GLuint>(previous_array_buffer));
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE0);
|
||||||
|
glBindTexture(GL_TEXTURE_2D, static_cast<GLuint>(previous_texture_0));
|
||||||
|
impl_->gl.active_texture(GL_TEXTURE1);
|
||||||
|
glBindTexture(GL_TEXTURE_2D, static_cast<GLuint>(previous_texture_1));
|
||||||
|
impl_->gl.active_texture(static_cast<GLenum>(previous_active_texture));
|
||||||
|
impl_->gl.use_program(static_cast<GLuint>(previous_program));
|
||||||
|
glPixelStorei(GL_UNPACK_ALIGNMENT, previous_unpack_alignment);
|
||||||
|
glViewport(previous_viewport[0], previous_viewport[1], previous_viewport[2],
|
||||||
|
previous_viewport[3]);
|
||||||
|
glScissor(previous_scissor_box[0], previous_scissor_box[1],
|
||||||
|
previous_scissor_box[2], previous_scissor_box[3]);
|
||||||
|
glClearColor(previous_clear_color[0], previous_clear_color[1],
|
||||||
|
previous_clear_color[2], previous_clear_color[3]);
|
||||||
|
glColorMask(previous_color_mask[0], previous_color_mask[1],
|
||||||
|
previous_color_mask[2], previous_color_mask[3]);
|
||||||
|
RestoreCapability(GL_BLEND, blend_enabled);
|
||||||
|
RestoreCapability(GL_CULL_FACE, cull_enabled);
|
||||||
|
RestoreCapability(GL_DEPTH_TEST, depth_enabled);
|
||||||
|
RestoreCapability(GL_SCISSOR_TEST, scissor_enabled);
|
||||||
|
RestoreCapability(GL_STENCIL_TEST, stencil_enabled);
|
||||||
|
const GLenum restore_error = glGetError();
|
||||||
|
if (restore_error != GL_NO_ERROR) {
|
||||||
|
LOG_WARN("OpenGL NV12 state restoration failed, error={}", restore_error);
|
||||||
|
if (outcome.result == RenderResult::rendered) {
|
||||||
|
outcome.result = RenderResult::failed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return outcome;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool WindowsOpenGlVideoRenderer::CopyLatestNv12(
|
||||||
|
std::string_view remote_id, std::vector<unsigned char> *output, int *width,
|
||||||
|
int *height) const {
|
||||||
|
if (!output || !width || !height) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
std::lock_guard lock(impl_->frames->mutex);
|
||||||
|
const FrameSlot *newest = nullptr;
|
||||||
|
for (const auto &slot : impl_->frames->slots) {
|
||||||
|
if (slot.valid && slot.remote_id == remote_id &&
|
||||||
|
(!newest || slot.sequence > newest->sequence)) {
|
||||||
|
newest = &slot;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!newest || newest->bytes.empty()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
*output = newest->bytes;
|
||||||
|
*width = newest->width;
|
||||||
|
*height = newest->height;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace crossdesk
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
#ifndef CROSSDESK_GUI_PLATFORM_WINDOWS_OPENGL_VIDEO_RENDERER_H_
|
||||||
|
#define CROSSDESK_GUI_PLATFORM_WINDOWS_OPENGL_VIDEO_RENDERER_H_
|
||||||
|
|
||||||
|
#include <cstddef>
|
||||||
|
#include <cstdint>
|
||||||
|
#include <memory>
|
||||||
|
#include <string>
|
||||||
|
#include <string_view>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
namespace crossdesk {
|
||||||
|
|
||||||
|
// Windows NV12 underlay for Slint's FemtoVG OpenGL renderer. MiniRTC submits
|
||||||
|
// tightly packed CPU NV12 frames from its decode callback thread. The Slint UI
|
||||||
|
// thread uploads the newest frame as Y and UV textures and performs color
|
||||||
|
// conversion and scaling in a fragment shader before Slint draws its overlay.
|
||||||
|
class WindowsOpenGlVideoRenderer {
|
||||||
|
public:
|
||||||
|
enum class SubmitResult {
|
||||||
|
submitted,
|
||||||
|
not_selected,
|
||||||
|
dropped,
|
||||||
|
failed,
|
||||||
|
};
|
||||||
|
|
||||||
|
enum class RenderResult {
|
||||||
|
rendered,
|
||||||
|
idle,
|
||||||
|
empty,
|
||||||
|
failed,
|
||||||
|
};
|
||||||
|
|
||||||
|
struct RenderOutcome {
|
||||||
|
RenderResult result = RenderResult::failed;
|
||||||
|
int width = 0;
|
||||||
|
int height = 0;
|
||||||
|
uint64_t sequence = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
WindowsOpenGlVideoRenderer();
|
||||||
|
~WindowsOpenGlVideoRenderer();
|
||||||
|
|
||||||
|
WindowsOpenGlVideoRenderer(const WindowsOpenGlVideoRenderer &) = delete;
|
||||||
|
WindowsOpenGlVideoRenderer &
|
||||||
|
operator=(const WindowsOpenGlVideoRenderer &) = delete;
|
||||||
|
|
||||||
|
// These methods must run while Slint's OpenGL context is current, from the
|
||||||
|
// RenderingSetup and RenderingTeardown notifier states respectively.
|
||||||
|
bool Setup();
|
||||||
|
void Teardown();
|
||||||
|
bool IsReady() const;
|
||||||
|
|
||||||
|
// Only the selected stream is queued for upload. Frames for other tabs are
|
||||||
|
// retained by GuiRuntime at a throttled rate for thumbnail/tab restoration.
|
||||||
|
bool SetSelectedStream(std::string remote_id);
|
||||||
|
void DiscardStream(std::string_view remote_id);
|
||||||
|
|
||||||
|
// Thread-safe; called from MiniRTC's decode callback thread.
|
||||||
|
SubmitResult SubmitNv12(std::string_view remote_id, const uint8_t *data,
|
||||||
|
size_t size, int width, int height);
|
||||||
|
SubmitResult SubmitCachedNv12(std::string_view remote_id, const uint8_t *data,
|
||||||
|
size_t size, int width, int height);
|
||||||
|
|
||||||
|
// Draws below Slint while its OpenGL context is current. target dimensions
|
||||||
|
// and top_inset_pixels are physical pixels in the window client area.
|
||||||
|
RenderOutcome RenderLatest(std::string_view remote_id, int target_width,
|
||||||
|
int target_height, int top_inset_pixels);
|
||||||
|
|
||||||
|
// Used during disconnect cleanup so thumbnail generation does not require a
|
||||||
|
// second per-frame CPU copy during normal playback.
|
||||||
|
bool CopyLatestNv12(std::string_view remote_id,
|
||||||
|
std::vector<unsigned char> *output, int *width,
|
||||||
|
int *height) const;
|
||||||
|
|
||||||
|
private:
|
||||||
|
SubmitResult SubmitNv12Internal(std::string_view remote_id,
|
||||||
|
const uint8_t *data, size_t size, int width,
|
||||||
|
int height, bool replace_pending);
|
||||||
|
|
||||||
|
struct Impl;
|
||||||
|
std::unique_ptr<Impl> impl_;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace crossdesk
|
||||||
|
|
||||||
|
#endif // CROSSDESK_GUI_PLATFORM_WINDOWS_OPENGL_VIDEO_RENDERER_H_
|
||||||
@@ -11,6 +11,9 @@
|
|||||||
#include "display_stream_id.h"
|
#include "display_stream_id.h"
|
||||||
#include "localization.h"
|
#include "localization.h"
|
||||||
#include "platform.h"
|
#include "platform.h"
|
||||||
|
#if defined(_WIN32)
|
||||||
|
#include "platform/windows_opengl_video_renderer.h"
|
||||||
|
#endif
|
||||||
#ifdef __APPLE__
|
#ifdef __APPLE__
|
||||||
#include "platform/metal_video_renderer.h"
|
#include "platform/metal_video_renderer.h"
|
||||||
#endif
|
#endif
|
||||||
@@ -226,7 +229,7 @@ void GuiRuntime::CloseRemoteSession(std::shared_ptr<RemoteSession> props) {
|
|||||||
frame_snapshot = props->front_frame_;
|
frame_snapshot = props->front_frame_;
|
||||||
video_width = props->video_width_;
|
video_width = props->video_width_;
|
||||||
video_height = props->video_height_;
|
video_height = props->video_height_;
|
||||||
#ifdef __APPLE__
|
#if defined(__APPLE__) || defined(_WIN32)
|
||||||
if ((!frame_snapshot || frame_snapshot->empty()) &&
|
if ((!frame_snapshot || frame_snapshot->empty()) &&
|
||||||
props->thumbnail_frame_ && !props->thumbnail_frame_->empty()) {
|
props->thumbnail_frame_ && !props->thumbnail_frame_->empty()) {
|
||||||
frame_snapshot = props->thumbnail_frame_;
|
frame_snapshot = props->thumbnail_frame_;
|
||||||
@@ -235,15 +238,18 @@ void GuiRuntime::CloseRemoteSession(std::shared_ptr<RemoteSession> props) {
|
|||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
#ifdef __APPLE__
|
#if defined(__APPLE__) || defined(_WIN32)
|
||||||
if ((!frame_snapshot || frame_snapshot->empty()) &&
|
#if defined(_WIN32)
|
||||||
mac_metal_video_renderer_) {
|
auto* native_renderer = windows_opengl_video_renderer_.get();
|
||||||
auto metal_snapshot =
|
#else
|
||||||
std::make_shared<std::vector<unsigned char>>();
|
auto* native_renderer = mac_metal_video_renderer_.get();
|
||||||
if (mac_metal_video_renderer_->CopyLatestNv12(
|
#endif
|
||||||
props->remote_id_, metal_snapshot.get(), &video_width,
|
if ((!frame_snapshot || frame_snapshot->empty()) && native_renderer) {
|
||||||
&video_height)) {
|
auto native_snapshot = std::make_shared<std::vector<unsigned char>>();
|
||||||
frame_snapshot = std::move(metal_snapshot);
|
if (native_renderer->CopyLatestNv12(props->remote_id_,
|
||||||
|
native_snapshot.get(), &video_width,
|
||||||
|
&video_height)) {
|
||||||
|
frame_snapshot = std::move(native_snapshot);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
@@ -270,9 +276,9 @@ void GuiRuntime::CloseRemoteSession(std::shared_ptr<RemoteSession> props) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef __APPLE__
|
#if defined(__APPLE__) || defined(_WIN32)
|
||||||
if (mac_metal_video_renderer_) {
|
if (native_renderer) {
|
||||||
mac_metal_video_renderer_->DiscardStream(props->remote_id_);
|
native_renderer->DiscardStream(props->remote_id_);
|
||||||
video_frame_dirty_.store(true, std::memory_order_release);
|
video_frame_dirty_.store(true, std::memory_order_release);
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
@@ -337,7 +343,7 @@ void GuiRuntime::ResetRemoteSessionResources(
|
|||||||
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
|
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
|
||||||
props->front_frame_.reset();
|
props->front_frame_.reset();
|
||||||
props->back_frame_.reset();
|
props->back_frame_.reset();
|
||||||
#ifdef __APPLE__
|
#if defined(__APPLE__) || defined(_WIN32)
|
||||||
props->thumbnail_frame_.reset();
|
props->thumbnail_frame_.reset();
|
||||||
props->thumbnail_width_ = 0;
|
props->thumbnail_width_ = 0;
|
||||||
props->thumbnail_height_ = 0;
|
props->thumbnail_height_ = 0;
|
||||||
|
|||||||
@@ -12,6 +12,9 @@
|
|||||||
|
|
||||||
#include "display_stream_id.h"
|
#include "display_stream_id.h"
|
||||||
#include "localization.h"
|
#include "localization.h"
|
||||||
|
#if defined(_WIN32)
|
||||||
|
#include "platform/windows_opengl_video_renderer.h"
|
||||||
|
#endif
|
||||||
#ifdef __APPLE__
|
#ifdef __APPLE__
|
||||||
#include "platform/metal_video_renderer.h"
|
#include "platform/metal_video_renderer.h"
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -15,6 +15,9 @@
|
|||||||
|
|
||||||
namespace crossdesk {
|
namespace crossdesk {
|
||||||
|
|
||||||
|
#if defined(_WIN32)
|
||||||
|
class WindowsOpenGlVideoRenderer;
|
||||||
|
#endif
|
||||||
#ifdef __APPLE__
|
#ifdef __APPLE__
|
||||||
class MacMetalVideoRenderer;
|
class MacMetalVideoRenderer;
|
||||||
#endif
|
#endif
|
||||||
@@ -87,6 +90,9 @@ class GuiRuntime : protected gui_detail::GuiState {
|
|||||||
KeyboardController keyboard_;
|
KeyboardController keyboard_;
|
||||||
PeerEventHandler peer_events_;
|
PeerEventHandler peer_events_;
|
||||||
std::atomic<bool> video_frame_dirty_{false};
|
std::atomic<bool> video_frame_dirty_{false};
|
||||||
|
#if defined(_WIN32)
|
||||||
|
std::unique_ptr<WindowsOpenGlVideoRenderer> windows_opengl_video_renderer_;
|
||||||
|
#endif
|
||||||
#ifdef __APPLE__
|
#ifdef __APPLE__
|
||||||
std::unique_ptr<MacMetalVideoRenderer> mac_metal_video_renderer_;
|
std::unique_ptr<MacMetalVideoRenderer> mac_metal_video_renderer_;
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -16,6 +16,9 @@
|
|||||||
#include "file_transfer.h"
|
#include "file_transfer.h"
|
||||||
#include "localization.h"
|
#include "localization.h"
|
||||||
#include "platform.h"
|
#include "platform.h"
|
||||||
|
#if defined(_WIN32)
|
||||||
|
#include "platform/windows_opengl_video_renderer.h"
|
||||||
|
#endif
|
||||||
#ifdef __APPLE__
|
#ifdef __APPLE__
|
||||||
#include "platform/metal_video_renderer.h"
|
#include "platform/metal_video_renderer.h"
|
||||||
#endif
|
#endif
|
||||||
@@ -316,22 +319,29 @@ void PeerEventHandler::OnConnectionStatus(ConnectionStatus status,
|
|||||||
props->enable_mouse_control_ = false;
|
props->enable_mouse_control_ = false;
|
||||||
runtime->ResetRemoteServiceStatus(*props);
|
runtime->ResetRemoteServiceStatus(*props);
|
||||||
|
|
||||||
#ifdef __APPLE__
|
#if defined(__APPLE__) || defined(_WIN32)
|
||||||
std::shared_ptr<std::vector<unsigned char>> metal_snapshot;
|
std::shared_ptr<std::vector<unsigned char>> native_snapshot;
|
||||||
int metal_snapshot_width = 0;
|
int native_snapshot_width = 0;
|
||||||
int metal_snapshot_height = 0;
|
int native_snapshot_height = 0;
|
||||||
bool needs_metal_snapshot = false;
|
bool needs_native_snapshot = false;
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
|
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
|
||||||
needs_metal_snapshot = !props->thumbnail_frame_ ||
|
needs_native_snapshot = !props->thumbnail_frame_ ||
|
||||||
props->thumbnail_frame_->empty();
|
props->thumbnail_frame_->empty();
|
||||||
}
|
}
|
||||||
if (needs_metal_snapshot && runtime->mac_metal_video_renderer_) {
|
if (needs_native_snapshot) {
|
||||||
|
#if defined(_WIN32)
|
||||||
|
auto* native_renderer =
|
||||||
|
runtime->windows_opengl_video_renderer_.get();
|
||||||
|
#else
|
||||||
|
auto* native_renderer = runtime->mac_metal_video_renderer_.get();
|
||||||
|
#endif
|
||||||
auto snapshot = std::make_shared<std::vector<unsigned char>>();
|
auto snapshot = std::make_shared<std::vector<unsigned char>>();
|
||||||
if (runtime->mac_metal_video_renderer_->CopyLatestNv12(
|
if (native_renderer && native_renderer->CopyLatestNv12(
|
||||||
remote_id, snapshot.get(), &metal_snapshot_width,
|
remote_id, snapshot.get(),
|
||||||
&metal_snapshot_height)) {
|
&native_snapshot_width,
|
||||||
metal_snapshot = std::move(snapshot);
|
&native_snapshot_height)) {
|
||||||
|
native_snapshot = std::move(snapshot);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
@@ -339,12 +349,12 @@ void PeerEventHandler::OnConnectionStatus(ConnectionStatus status,
|
|||||||
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
|
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
|
||||||
props->front_frame_.reset();
|
props->front_frame_.reset();
|
||||||
props->back_frame_.reset();
|
props->back_frame_.reset();
|
||||||
#ifdef __APPLE__
|
#if defined(__APPLE__) || defined(_WIN32)
|
||||||
if (metal_snapshot &&
|
if (native_snapshot &&
|
||||||
(!props->thumbnail_frame_ || props->thumbnail_frame_->empty())) {
|
(!props->thumbnail_frame_ || props->thumbnail_frame_->empty())) {
|
||||||
props->thumbnail_frame_ = std::move(metal_snapshot);
|
props->thumbnail_frame_ = std::move(native_snapshot);
|
||||||
props->thumbnail_width_ = metal_snapshot_width;
|
props->thumbnail_width_ = native_snapshot_width;
|
||||||
props->thumbnail_height_ = metal_snapshot_height;
|
props->thumbnail_height_ = native_snapshot_height;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
props->video_width_ = 0;
|
props->video_width_ = 0;
|
||||||
@@ -353,9 +363,14 @@ void PeerEventHandler::OnConnectionStatus(ConnectionStatus status,
|
|||||||
props->render_rect_dirty_ = true;
|
props->render_rect_dirty_ = true;
|
||||||
props->stream_cleanup_pending_ = true;
|
props->stream_cleanup_pending_ = true;
|
||||||
}
|
}
|
||||||
#ifdef __APPLE__
|
#if defined(__APPLE__) || defined(_WIN32)
|
||||||
if (runtime->mac_metal_video_renderer_) {
|
#if defined(_WIN32)
|
||||||
runtime->mac_metal_video_renderer_->DiscardStream(remote_id);
|
auto* native_renderer = runtime->windows_opengl_video_renderer_.get();
|
||||||
|
#else
|
||||||
|
auto* native_renderer = runtime->mac_metal_video_renderer_.get();
|
||||||
|
#endif
|
||||||
|
if (native_renderer) {
|
||||||
|
native_renderer->DiscardStream(remote_id);
|
||||||
runtime->video_frame_dirty_.store(true,
|
runtime->video_frame_dirty_.store(true,
|
||||||
std::memory_order_release);
|
std::memory_order_release);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,14 +7,18 @@
|
|||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "runtime/gui_runtime.h"
|
#include "runtime/gui_runtime.h"
|
||||||
#ifdef __APPLE__
|
#if defined(__APPLE__) || defined(_WIN32)
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
|
|
||||||
|
#if defined(_WIN32)
|
||||||
|
#include "platform/windows_opengl_video_renderer.h"
|
||||||
|
#else
|
||||||
#include "platform/metal_video_renderer.h"
|
#include "platform/metal_video_renderer.h"
|
||||||
#endif
|
#endif
|
||||||
|
#endif
|
||||||
|
|
||||||
namespace crossdesk {
|
namespace crossdesk {
|
||||||
#ifdef __APPLE__
|
#if defined(__APPLE__) || defined(_WIN32)
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
constexpr auto kBackgroundSnapshotInterval = std::chrono::seconds(1);
|
constexpr auto kBackgroundSnapshotInterval = std::chrono::seconds(1);
|
||||||
@@ -41,17 +45,24 @@ void PeerEventHandler::OnReceiveVideoBuffer(
|
|||||||
runtime->remote_sessions_.find(remote_id)->second.get();
|
runtime->remote_sessions_.find(remote_id)->second.get();
|
||||||
|
|
||||||
if (props->connection_established_) {
|
if (props->connection_established_) {
|
||||||
#ifdef __APPLE__
|
#if defined(__APPLE__) || defined(_WIN32)
|
||||||
bool background_snapshot_only = false;
|
bool background_snapshot_only = false;
|
||||||
if (runtime->mac_metal_video_renderer_ &&
|
#if defined(_WIN32)
|
||||||
runtime->mac_metal_video_renderer_->IsReady() &&
|
auto* native_renderer = runtime->windows_opengl_video_renderer_.get();
|
||||||
runtime->mac_metal_video_renderer_->IsAttached()) {
|
using NativeVideoRenderer = WindowsOpenGlVideoRenderer;
|
||||||
const auto submit_result =
|
#else
|
||||||
runtime->mac_metal_video_renderer_->SubmitNv12(
|
auto* native_renderer = runtime->mac_metal_video_renderer_.get();
|
||||||
remote_id,
|
using NativeVideoRenderer = MacMetalVideoRenderer;
|
||||||
reinterpret_cast<const uint8_t*>(video_frame->data),
|
#endif
|
||||||
video_frame->size, video_frame->width, video_frame->height);
|
if (native_renderer && native_renderer->IsReady()
|
||||||
if (submit_result == MacMetalVideoRenderer::SubmitResult::submitted) {
|
#if defined(__APPLE__)
|
||||||
|
&& native_renderer->IsAttached()
|
||||||
|
#endif
|
||||||
|
) {
|
||||||
|
const auto submit_result = native_renderer->SubmitNv12(
|
||||||
|
remote_id, reinterpret_cast<const uint8_t*>(video_frame->data),
|
||||||
|
video_frame->size, video_frame->width, video_frame->height);
|
||||||
|
if (submit_result == NativeVideoRenderer::SubmitResult::submitted) {
|
||||||
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
|
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
|
||||||
const bool size_changed =
|
const bool size_changed =
|
||||||
(props->video_width_ != video_frame->width) ||
|
(props->video_width_ != video_frame->width) ||
|
||||||
@@ -63,7 +74,7 @@ void PeerEventHandler::OnReceiveVideoBuffer(
|
|||||||
props->video_height_ = video_frame->height;
|
props->video_height_ = video_frame->height;
|
||||||
props->video_size_ = video_frame->size;
|
props->video_size_ = video_frame->size;
|
||||||
// Once the native renderer owns the current stream, do not leave an
|
// Once the native renderer owns the current stream, do not leave an
|
||||||
// older CPU frame ahead of the Metal close snapshot.
|
// older CPU frame ahead of the renderer's close snapshot.
|
||||||
props->front_frame_.reset();
|
props->front_frame_.reset();
|
||||||
props->back_frame_.reset();
|
props->back_frame_.reset();
|
||||||
props->thumbnail_frame_.reset();
|
props->thumbnail_frame_.reset();
|
||||||
@@ -75,21 +86,22 @@ void PeerEventHandler::OnReceiveVideoBuffer(
|
|||||||
runtime->video_frame_dirty_.store(true, std::memory_order_release);
|
runtime->video_frame_dirty_.store(true, std::memory_order_release);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (submit_result == MacMetalVideoRenderer::SubmitResult::dropped ||
|
if (submit_result == NativeVideoRenderer::SubmitResult::dropped ||
|
||||||
submit_result == MacMetalVideoRenderer::SubmitResult::failed) {
|
submit_result == NativeVideoRenderer::SubmitResult::failed) {
|
||||||
// Keep presenting the last Metal frame. A later decoded frame can
|
// Keep presenting the last native frame. A later decoded frame can
|
||||||
// reuse the renderer without changing ownership mid-window.
|
// reuse the renderer without changing ownership mid-window.
|
||||||
props->streaming_ = true;
|
props->streaming_ = true;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (submit_result == MacMetalVideoRenderer::SubmitResult::not_selected) {
|
if (submit_result ==
|
||||||
|
NativeVideoRenderer::SubmitResult::not_selected) {
|
||||||
background_snapshot_only = true;
|
background_snapshot_only = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
|
std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
|
||||||
#ifdef __APPLE__
|
#if defined(__APPLE__) || defined(_WIN32)
|
||||||
const auto now = std::chrono::steady_clock::now();
|
const auto now = std::chrono::steady_clock::now();
|
||||||
if (background_snapshot_only && props->thumbnail_frame_ &&
|
if (background_snapshot_only && props->thumbnail_frame_ &&
|
||||||
!props->thumbnail_frame_->empty() &&
|
!props->thumbnail_frame_->empty() &&
|
||||||
@@ -125,7 +137,7 @@ void PeerEventHandler::OnReceiveVideoBuffer(
|
|||||||
props->video_size_ = video_frame->size;
|
props->video_size_ = video_frame->size;
|
||||||
|
|
||||||
props->front_frame_.swap(props->back_frame_);
|
props->front_frame_.swap(props->back_frame_);
|
||||||
#ifdef __APPLE__
|
#if defined(__APPLE__) || defined(_WIN32)
|
||||||
props->thumbnail_frame_ = props->front_frame_;
|
props->thumbnail_frame_ = props->front_frame_;
|
||||||
props->thumbnail_width_ = video_frame->width;
|
props->thumbnail_width_ = video_frame->width;
|
||||||
props->thumbnail_height_ = video_frame->height;
|
props->thumbnail_height_ = video_frame->height;
|
||||||
@@ -137,7 +149,7 @@ void PeerEventHandler::OnReceiveVideoBuffer(
|
|||||||
}
|
}
|
||||||
|
|
||||||
props->streaming_ = true;
|
props->streaming_ = true;
|
||||||
#ifdef __APPLE__
|
#if defined(__APPLE__) || defined(_WIN32)
|
||||||
if (background_snapshot_only) {
|
if (background_snapshot_only) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -110,7 +110,7 @@ struct RemoteSession {
|
|||||||
std::mutex video_frame_mutex_;
|
std::mutex video_frame_mutex_;
|
||||||
std::shared_ptr<std::vector<unsigned char>> front_frame_;
|
std::shared_ptr<std::vector<unsigned char>> front_frame_;
|
||||||
std::shared_ptr<std::vector<unsigned char>> back_frame_;
|
std::shared_ptr<std::vector<unsigned char>> back_frame_;
|
||||||
#ifdef __APPLE__
|
#if defined(__APPLE__) || defined(_WIN32)
|
||||||
// Retains the most recent background CPU frame across disconnect cleanup
|
// Retains the most recent background CPU frame across disconnect cleanup
|
||||||
// so closing an unselected tab can still update its recent-item thumbnail.
|
// so closing an unselected tab can still update its recent-item thumbnail.
|
||||||
std::shared_ptr<std::vector<unsigned char>> thumbnail_frame_;
|
std::shared_ptr<std::vector<unsigned char>> thumbnail_frame_;
|
||||||
|
|||||||
+2
-1
@@ -277,7 +277,8 @@ function setup_targets()
|
|||||||
if is_os("windows") then
|
if is_os("windows") then
|
||||||
add_cxxflags("/bigobj")
|
add_cxxflags("/bigobj")
|
||||||
add_links("opengl32")
|
add_links("opengl32")
|
||||||
add_files("src/gui/platform/tray/win_tray.cpp")
|
add_files("src/gui/platform/tray/win_tray.cpp",
|
||||||
|
"src/gui/platform/windows_opengl_video_renderer.cpp")
|
||||||
add_includedirs("src/service/windows", {public = true})
|
add_includedirs("src/service/windows", {public = true})
|
||||||
elseif is_os("macosx") then
|
elseif is_os("macosx") then
|
||||||
add_files("src/gui/runtime/*.mm", "src/gui/platform/tray/*.mm",
|
add_files("src/gui/runtime/*.mm", "src/gui/platform/tray/*.mm",
|
||||||
|
|||||||
Reference in New Issue
Block a user