diff --git a/apps/desktop/src/gui/runtime/gui_runtime.cpp b/apps/desktop/src/gui/runtime/gui_runtime.cpp index 9a50fa50..3ee85277 100644 --- a/apps/desktop/src/gui/runtime/gui_runtime.cpp +++ b/apps/desktop/src/gui/runtime/gui_runtime.cpp @@ -141,9 +141,10 @@ int GuiRuntime::CreateConnectionPeer() { sizeof(params_.log_path) - 1); params_.log_path[sizeof(params_.log_path) - 1] = '\0'; params_.hardware_acceleration = config_center_->IsHardwareVideoCodec(); -#if defined(_WIN32) - // Windows renderers retain pooled CPU NV12 frames or CUDA device frames and - // fall back to a packed CPU copy when native upload is unavailable. +#if defined(_WIN32) || defined(__APPLE__) + // Windows renderers retain pooled CPU NV12 frames or CUDA device frames; + // macOS retains VideoToolbox CVPixelBuffers for direct Metal sampling. Both + // platforms fall back to a packed CPU copy when native upload is unavailable. params_.native_video_output = true; #else params_.native_video_output = false; diff --git a/apps/desktop/src/platform/macos/gui/metal_video_renderer.h b/apps/desktop/src/platform/macos/gui/metal_video_renderer.h index f1fdd65d..7abbf0e4 100644 --- a/apps/desktop/src/platform/macos/gui/metal_video_renderer.h +++ b/apps/desktop/src/platform/macos/gui/metal_video_renderer.h @@ -12,11 +12,11 @@ namespace crossdesk { -// macOS video compositor used by CrossDesk only. MiniRTC continues to expose -// tightly packed CPU NV12 frames; this renderer performs the single required -// copy into shared Metal buffers and converts NV12 to RGB in the fragment -// shader. AppKit attachment and rendering are main-thread operations, while -// SubmitNv12() is safe to call from MiniRTC's decode callback thread. +// macOS video compositor used by CrossDesk only. VideoToolbox CVPixelBuffers +// are sampled directly through CVMetalTextureCache; packed CPU NV12 remains a +// fallback for software decoding and cached snapshots. AppKit attachment and +// rendering are main-thread operations, while frame submission is safe from +// MiniRTC's decode callback thread. class MacMetalVideoRenderer final : public VideoRenderer { public: MacMetalVideoRenderer(); @@ -37,6 +37,11 @@ class MacMetalVideoRenderer final : public VideoRenderer { SubmitResult SubmitNv12(std::string_view remote_id, const uint8_t* data, size_t size, int width, int height) override; + // Retains an IOSurface-backed VideoToolbox CVPixelBuffer until the Metal + // command buffer using it has completed. + SubmitResult SubmitNativeFrame(std::string_view remote_id, + const XNativeVideoFrame& frame) override; + // Seeds a newly selected stream from its retained CPU snapshot without // replacing a newer decoded frame that is already queued or rendering. SubmitResult SubmitCachedNv12(std::string_view remote_id, diff --git a/apps/desktop/src/platform/macos/gui/metal_video_renderer.mm b/apps/desktop/src/platform/macos/gui/metal_video_renderer.mm index b789a884..c6c9b3ef 100644 --- a/apps/desktop/src/platform/macos/gui/metal_video_renderer.mm +++ b/apps/desktop/src/platform/macos/gui/metal_video_renderer.mm @@ -1,6 +1,7 @@ #include "platform/macos/gui/metal_video_renderer.h" #import +#import #import #import @@ -194,6 +195,74 @@ namespace { constexpr size_t kFrameSlotCount = 3; +class NativeVideoFrameRef { + public: + NativeVideoFrameRef() = default; + + explicit NativeVideoFrameRef(const XNativeVideoFrame* frame) { + if (frame && frame->owner && frame->retain && frame->release) { + frame_ = *frame; + frame_.struct_size = sizeof(frame_); + frame_.retain(frame_.owner); + valid_ = true; + } + } + + NativeVideoFrameRef(const NativeVideoFrameRef& other) + : NativeVideoFrameRef(other.Get()) {} + + NativeVideoFrameRef(NativeVideoFrameRef&& other) noexcept + : frame_(other.frame_), valid_(std::exchange(other.valid_, false)) { + other.frame_ = {}; + } + + NativeVideoFrameRef& operator=(NativeVideoFrameRef other) noexcept { + Swap(other); + return *this; + } + + ~NativeVideoFrameRef() { Reset(); } + + void Reset() { + if (valid_ && frame_.owner && frame_.release) { + frame_.release(frame_.owner); + } + frame_ = {}; + valid_ = false; + } + + void Swap(NativeVideoFrameRef& other) noexcept { + std::swap(frame_, other.frame_); + std::swap(valid_, other.valid_); + } + + const XNativeVideoFrame* Get() const { + return valid_ ? &frame_ : nullptr; + } + explicit operator bool() const { return valid_; } + + private: + XNativeVideoFrame frame_{}; + bool valid_ = false; +}; + +const XNativeVideoFrame* GetMacNativeFrame( + const XNativeVideoFrame& frame) { + const XNativeVideoFrame* native = &frame; + if (native->struct_size < static_cast(sizeof(*native)) || + native->type != XNativeVideoFrameCVPixelBuffer || + !native->payload.cv_pixel_buffer || !native->owner || !native->retain || + !native->release || !native->copy_to_nv12 || native->width == 0 || + native->height == 0 || (native->width & 1U) != 0 || + (native->height & 1U) != 0 || + native->width > static_cast(std::numeric_limits::max()) || + native->height > + static_cast(std::numeric_limits::max())) { + return nullptr; + } + return native; +} + // Slint's winit backend creates AppKit windows with transparent backing so // transparent Slint items can be composited. Keep the content out of the // titlebar and choose whether AppKit's material or the semantic window color @@ -262,6 +331,9 @@ struct FrameSlot { id uv_buffer = nil; id y_texture = nil; id uv_texture = nil; + CVMetalTextureRef y_cv_texture = nullptr; + CVMetalTextureRef uv_cv_texture = nullptr; + NativeVideoFrameRef native_frame; size_t y_stride = 0; size_t uv_stride = 0; int width = 0; @@ -274,6 +346,15 @@ struct FrameSlot { void ReleaseResources() { y_texture = nil; uv_texture = nil; + if (y_cv_texture) { + CFRelease(y_cv_texture); + y_cv_texture = nullptr; + } + if (uv_cv_texture) { + CFRelease(uv_cv_texture); + uv_cv_texture = nullptr; + } + native_frame.Reset(); y_buffer = nil; uv_buffer = nil; y_stride = 0; @@ -406,6 +487,7 @@ struct MacMetalVideoRenderer::Impl { id device = nil; id command_queue = nil; id pipeline = nil; + CVMetalTextureCacheRef texture_cache = nullptr; std::shared_ptr frames = std::make_shared(); NSView* slint_view = nil; @@ -417,6 +499,8 @@ struct MacMetalVideoRenderer::Impl { uint64_t rendered_content_generation = 0; std::shared_ptr> native_surface_needs_redraw = std::make_shared>(true); + std::atomic native_path_logged{false}; + std::atomic native_fallback_logged{false}; Impl() { @autoreleasepool { @@ -430,6 +514,12 @@ struct MacMetalVideoRenderer::Impl { LOG_ERROR("Metal video renderer unavailable: command queue creation failed"); return; } + const CVReturn cache_status = CVMetalTextureCacheCreate( + kCFAllocatorDefault, nullptr, device, nullptr, &texture_cache); + if (cache_status != kCVReturnSuccess || !texture_cache) { + LOG_WARN("Metal video renderer could not create CV texture cache: {}", + cache_status); + } NSError* library_error = nil; id library = @@ -466,6 +556,11 @@ struct MacMetalVideoRenderer::Impl { ~Impl() { DetachViews(); frames.reset(); + if (texture_cache) { + CVMetalTextureCacheFlush(texture_cache, 0); + CFRelease(texture_cache); + texture_cache = nullptr; + } } bool Ready() const { @@ -655,6 +750,142 @@ MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNv12( return SubmitNv12Internal(remote_id, data, size, width, height, true); } +MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNativeFrame( + std::string_view remote_id, const XNativeVideoFrame& frame) { + if (!IsReady()) { + return SubmitResult::failed; + } + const XNativeVideoFrame* native = GetMacNativeFrame(frame); + if (!native) { + return SubmitResult::failed; + } + + { + std::lock_guard lock(impl_->frames->mutex); + if (impl_->frames->selected_stream != remote_id) { + return SubmitResult::not_selected; + } + } + + const int width = static_cast(native->width); + const int height = static_cast(native->height); + const auto submit_cpu_fallback = [&]() { + if (!impl_->native_fallback_logged.exchange(true, + std::memory_order_acq_rel)) { + LOG_WARN("macOS native CVPixelBuffer upload unavailable; using CPU NV12 " + "fallback"); + } + const size_t y_size = static_cast(width) * height; + std::vector packed_nv12(y_size + y_size / 2); + if (native->copy_to_nv12(native->owner, packed_nv12.data(), + packed_nv12.size()) != 0) { + return SubmitResult::failed; + } + return SubmitNv12(remote_id, packed_nv12.data(), packed_nv12.size(), width, + height); + }; + + CVPixelBufferRef pixel_buffer = + static_cast(native->payload.cv_pixel_buffer); + const OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer); + if (!CVPixelBufferIsPlanar(pixel_buffer) || + CVPixelBufferGetPlaneCount(pixel_buffer) < 2 || + CVPixelBufferGetWidth(pixel_buffer) != native->width || + CVPixelBufferGetHeight(pixel_buffer) != native->height || + (pixel_format != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange && + pixel_format != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)) { + return submit_cpu_fallback(); + } + + NativeVideoFrameRef retained(native); + if (!retained || !impl_->texture_cache) { + return submit_cpu_fallback(); + } + + CVMetalTextureRef y_cv_texture = nullptr; + CVReturn y_status = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, impl_->texture_cache, pixel_buffer, nullptr, + MTLPixelFormatR8Unorm, native->width, native->height, 0, + &y_cv_texture); + if (y_status != kCVReturnSuccess || !y_cv_texture) { + return submit_cpu_fallback(); + } + + CVMetalTextureRef uv_cv_texture = nullptr; + CVReturn uv_status = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, impl_->texture_cache, pixel_buffer, nullptr, + MTLPixelFormatRG8Unorm, native->width / 2U, native->height / 2U, 1, + &uv_cv_texture); + if (uv_status != kCVReturnSuccess || !uv_cv_texture) { + CFRelease(y_cv_texture); + return submit_cpu_fallback(); + } + + id y_texture = CVMetalTextureGetTexture(y_cv_texture); + id uv_texture = CVMetalTextureGetTexture(uv_cv_texture); + if (y_texture == nil || uv_texture == nil) { + CFRelease(y_cv_texture); + CFRelease(uv_cv_texture); + return submit_cpu_fallback(); + } + + auto& frames = *impl_->frames; + std::lock_guard lock(frames.mutex); + if (frames.selected_stream != remote_id) { + CFRelease(y_cv_texture); + CFRelease(uv_cv_texture); + return SubmitResult::not_selected; + } + + // Latest-frame-only queue: replace an unconsumed decoded frame to bound + // latency, while never recycling a frame still referenced by the GPU. + FrameSlot* target = nullptr; + for (auto& slot : frames.slots) { + if (slot.use == SlotUse::pending) { + target = &slot; + break; + } + } + if (!target) { + uint64_t oldest_sequence = std::numeric_limits::max(); + for (auto& slot : frames.slots) { + if (slot.use != SlotUse::available) { + continue; + } + if (!slot.valid) { + target = &slot; + break; + } + if (slot.sequence < oldest_sequence) { + oldest_sequence = slot.sequence; + target = &slot; + } + } + } + if (!target) { + CFRelease(y_cv_texture); + CFRelease(uv_cv_texture); + return SubmitResult::dropped; + } + + target->ReleaseResources(); + target->y_cv_texture = y_cv_texture; + target->uv_cv_texture = uv_cv_texture; + target->y_texture = y_texture; + target->uv_texture = uv_texture; + target->native_frame = std::move(retained); + target->width = width; + target->height = height; + target->remote_id.assign(remote_id); + target->sequence = frames.next_sequence++; + target->use = SlotUse::pending; + target->valid = true; + if (!impl_->native_path_logged.exchange(true, std::memory_order_acq_rel)) { + LOG_INFO("macOS VideoToolbox-to-Metal native NV12 path enabled"); + } + return SubmitResult::submitted; +} + MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitCachedNv12( std::string_view remote_id, const uint8_t* data, size_t size, int width, int height) { @@ -1045,40 +1276,57 @@ bool MacMetalVideoRenderer::CopyLatestNv12( return false; } - std::lock_guard lock(impl_->frames->mutex); - const FrameSlot* newest = nullptr; - for (const auto& slot : impl_->frames->slots) { - if (!slot.valid || slot.remote_id != remote_id || slot.y_buffer == nil || - slot.uv_buffer == nil || slot.sequence == 0) { - continue; + NativeVideoFrameRef native_frame; + { + std::lock_guard lock(impl_->frames->mutex); + const FrameSlot* newest = nullptr; + for (const auto& slot : impl_->frames->slots) { + if (!slot.valid || slot.remote_id != remote_id || slot.sequence == 0 || + (!slot.native_frame && + (slot.y_buffer == nil || slot.uv_buffer == nil))) { + continue; + } + if (newest == nullptr || slot.sequence > newest->sequence) { + newest = &slot; + } } - if (newest == nullptr || slot.sequence > newest->sequence) { - newest = &slot; + if (newest == nullptr || newest->width <= 0 || newest->height <= 0) { + return false; + } + + *width = newest->width; + *height = newest->height; + native_frame = newest->native_frame; + if (!native_frame) { + const size_t y_size = + static_cast(newest->width) * newest->height; + output->resize(y_size + y_size / 2); + auto* y_source = + static_cast(newest->y_buffer.contents); + auto* uv_source = + static_cast(newest->uv_buffer.contents); + for (int row = 0; row < newest->height; ++row) { + std::memcpy(output->data() + + static_cast(row) * newest->width, + y_source + static_cast(row) * newest->y_stride, + newest->width); + } + uint8_t* uv_destination = output->data() + y_size; + for (int row = 0; row < newest->height / 2; ++row) { + std::memcpy(uv_destination + + static_cast(row) * newest->width, + uv_source + static_cast(row) * newest->uv_stride, + newest->width); + } + return true; } - } - if (newest == nullptr || newest->width <= 0 || newest->height <= 0) { - return false; } - const size_t y_size = - static_cast(newest->width) * newest->height; + const XNativeVideoFrame* native = native_frame.Get(); + const size_t y_size = static_cast(native->width) * native->height; output->resize(y_size + y_size / 2); - auto* y_source = static_cast(newest->y_buffer.contents); - auto* uv_source = static_cast(newest->uv_buffer.contents); - for (int row = 0; row < newest->height; ++row) { - std::memcpy(output->data() + static_cast(row) * newest->width, - y_source + static_cast(row) * newest->y_stride, - newest->width); - } - uint8_t* uv_destination = output->data() + y_size; - for (int row = 0; row < newest->height / 2; ++row) { - std::memcpy(uv_destination + static_cast(row) * newest->width, - uv_source + static_cast(row) * newest->uv_stride, - newest->width); - } - *width = newest->width; - *height = newest->height; - return true; + return native->copy_to_nv12(native->owner, output->data(), output->size()) == + 0; } } // namespace crossdesk diff --git a/deps/submodules/minirtc b/deps/submodules/minirtc index 7c16988b..38b116d2 160000 --- a/deps/submodules/minirtc +++ b/deps/submodules/minirtc @@ -1 +1 @@ -Subproject commit 7c16988b762fc8fa6f5605bf0ec534126803d1a8 +Subproject commit 38b116d29eeb00026d3283663eb194da919d5d78