[feat] add native Metal video rendering for Slint on macOS

src/gui/runtime/connection_runtime.cpp

@@ -11,6 +11,9 @@
 #include "display_stream_id.h"
 #include "localization.h"
 #include "platform.h"
+#ifdef __APPLE__
+#include "platform/metal_video_renderer.h"
+#endif
 #include "rd_log.h"
 #include "runtime/gui_runtime.h"
 
@@ -223,7 +226,27 @@ void GuiRuntime::CloseRemoteSession(std::shared_ptr<RemoteSession> props) {
     frame_snapshot = props->front_frame_;
     video_width = props->video_width_;
     video_height = props->video_height_;
+#ifdef __APPLE__
+    if ((!frame_snapshot || frame_snapshot->empty()) &&
+        props->thumbnail_frame_ && !props->thumbnail_frame_->empty()) {
+      frame_snapshot = props->thumbnail_frame_;
+      video_width = props->thumbnail_width_;
+      video_height = props->thumbnail_height_;
+    }
+#endif
   }
+#ifdef __APPLE__
+  if ((!frame_snapshot || frame_snapshot->empty()) &&
+      mac_metal_video_renderer_) {
+    auto metal_snapshot =
+        std::make_shared<std::vector<unsigned char>>();
+    if (mac_metal_video_renderer_->CopyLatestNv12(
+            props->remote_id_, metal_snapshot.get(), &video_width,
+            &video_height)) {
+      frame_snapshot = std::move(metal_snapshot);
+    }
+  }
+#endif
 
   if (frame_snapshot && !frame_snapshot->empty() && video_width > 0 &&
       video_height > 0) {
@@ -247,6 +270,13 @@ void GuiRuntime::CloseRemoteSession(std::shared_ptr<RemoteSession> props) {
     }
   }
 
+#ifdef __APPLE__
+  if (mac_metal_video_renderer_) {
+    mac_metal_video_renderer_->DiscardStream(props->remote_id_);
+    video_frame_dirty_.store(true, std::memory_order_release);
+  }
+#endif
+
   if (props->peer_) {
     LOG_INFO("[{}] Leave connection [{}]", props->local_id_, props->remote_id_);
     LeaveConnection(props->peer_, props->remote_id_.c_str());
@@ -307,6 +337,12 @@ void GuiRuntime::ResetRemoteSessionResources(
     std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
     props->front_frame_.reset();
     props->back_frame_.reset();
+#ifdef __APPLE__
+    props->thumbnail_frame_.reset();
+    props->thumbnail_width_ = 0;
+    props->thumbnail_height_ = 0;
+    props->background_snapshot_time_ = {};
+#endif
     props->video_width_ = 0;
     props->video_height_ = 0;
     props->video_size_ = 0;

src/gui/runtime/gui_runtime.cpp

@@ -12,6 +12,9 @@
 
 #include "display_stream_id.h"
 #include "localization.h"
+#ifdef __APPLE__
+#include "platform/metal_video_renderer.h"
+#endif
 #include "rd_log.h"
 
 namespace crossdesk {

src/gui/runtime/gui_runtime.h

@@ -15,6 +15,10 @@
 
 namespace crossdesk {
 
+#ifdef __APPLE__
+class MacMetalVideoRenderer;
+#endif
+
 // Shared GUI runtime. It owns subsystem controllers and cross-cutting session
 // state, but no window lifecycle, ImGui view, or transport callback methods.
 class GuiRuntime : protected gui_detail::GuiState {
@@ -83,6 +87,9 @@ class GuiRuntime : protected gui_detail::GuiState {
   KeyboardController keyboard_;
   PeerEventHandler peer_events_;
   std::atomic<bool> video_frame_dirty_{false};
+#ifdef __APPLE__
+  std::unique_ptr<MacMetalVideoRenderer> mac_metal_video_renderer_;
+#endif
 
  private:
   friend class ClipboardController;

src/gui/runtime/peer_event_handler.cpp

@@ -10,11 +10,15 @@
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
+#include <vector>
 
 #include "device_controller.h"
 #include "file_transfer.h"
 #include "localization.h"
 #include "platform.h"
+#ifdef __APPLE__
+#include "platform/metal_video_renderer.h"
+#endif
 #include "rd_log.h"
 #include "runtime/gui_runtime.h"
 #include "runtime/remote_action_codec.h"
@@ -312,16 +316,50 @@ void PeerEventHandler::OnConnectionStatus(ConnectionStatus status,
         props->enable_mouse_control_ = false;
         runtime->ResetRemoteServiceStatus(*props);
 
+#ifdef __APPLE__
+        std::shared_ptr<std::vector<unsigned char>> metal_snapshot;
+        int metal_snapshot_width = 0;
+        int metal_snapshot_height = 0;
+        bool needs_metal_snapshot = false;
+        {
+          std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
+          needs_metal_snapshot = !props->thumbnail_frame_ ||
+                                 props->thumbnail_frame_->empty();
+        }
+        if (needs_metal_snapshot && runtime->mac_metal_video_renderer_) {
+          auto snapshot = std::make_shared<std::vector<unsigned char>>();
+          if (runtime->mac_metal_video_renderer_->CopyLatestNv12(
+                  remote_id, snapshot.get(), &metal_snapshot_width,
+                  &metal_snapshot_height)) {
+            metal_snapshot = std::move(snapshot);
+          }
+        }
+#endif
         {
           std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
           props->front_frame_.reset();
           props->back_frame_.reset();
+#ifdef __APPLE__
+          if (metal_snapshot &&
+              (!props->thumbnail_frame_ || props->thumbnail_frame_->empty())) {
+            props->thumbnail_frame_ = std::move(metal_snapshot);
+            props->thumbnail_width_ = metal_snapshot_width;
+            props->thumbnail_height_ = metal_snapshot_height;
+          }
+#endif
           props->video_width_ = 0;
           props->video_height_ = 0;
           props->video_size_ = 0;
           props->render_rect_dirty_ = true;
           props->stream_cleanup_pending_ = true;
         }
+#ifdef __APPLE__
+        if (runtime->mac_metal_video_renderer_) {
+          runtime->mac_metal_video_renderer_->DiscardStream(remote_id);
+          runtime->video_frame_dirty_.store(true,
+                                            std::memory_order_release);
+        }
+#endif
 
         runtime->focus_on_stream_window_ = false;
 

src/gui/runtime/peer_media_callbacks.cpp

@@ -7,8 +7,20 @@
 #include <vector>
 
 #include "runtime/gui_runtime.h"
+#ifdef __APPLE__
+#include <chrono>
+
+#include "platform/metal_video_renderer.h"
+#endif
 
 namespace crossdesk {
+#ifdef __APPLE__
+namespace {
+
+constexpr auto kBackgroundSnapshotInterval = std::chrono::seconds(1);
+
+}  // namespace
+#endif
 
 void PeerEventHandler::OnReceiveVideoBuffer(
     const XVideoFrame *video_frame, const char *user_id, size_t user_id_size,
@@ -29,8 +41,66 @@ void PeerEventHandler::OnReceiveVideoBuffer(
       runtime->remote_sessions_.find(remote_id)->second.get();
 
   if (props->connection_established_) {
+#ifdef __APPLE__
+    bool background_snapshot_only = false;
+    if (runtime->mac_metal_video_renderer_ &&
+        runtime->mac_metal_video_renderer_->IsReady() &&
+        runtime->mac_metal_video_renderer_->IsAttached()) {
+      const auto submit_result =
+          runtime->mac_metal_video_renderer_->SubmitNv12(
+              remote_id,
+              reinterpret_cast<const uint8_t*>(video_frame->data),
+              video_frame->size, video_frame->width, video_frame->height);
+      if (submit_result == MacMetalVideoRenderer::SubmitResult::submitted) {
+        std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
+        const bool size_changed =
+            (props->video_width_ != video_frame->width) ||
+            (props->video_height_ != video_frame->height);
+        if (size_changed) {
+          props->render_rect_dirty_ = true;
+        }
+        props->video_width_ = video_frame->width;
+        props->video_height_ = video_frame->height;
+        props->video_size_ = video_frame->size;
+        // Once the native renderer owns the current stream, do not leave an
+        // older CPU frame ahead of the Metal close snapshot.
+        props->front_frame_.reset();
+        props->back_frame_.reset();
+        props->thumbnail_frame_.reset();
+        props->thumbnail_width_ = 0;
+        props->thumbnail_height_ = 0;
+        props->background_snapshot_time_ = {};
+        ++props->video_frame_sequence_;
+        props->streaming_ = true;
+        runtime->video_frame_dirty_.store(true, std::memory_order_release);
+        return;
+      }
+      if (submit_result == MacMetalVideoRenderer::SubmitResult::dropped ||
+          submit_result == MacMetalVideoRenderer::SubmitResult::failed) {
+        // Keep presenting the last Metal frame. A later decoded frame can
+        // reuse the renderer without changing ownership mid-window.
+        props->streaming_ = true;
+        return;
+      }
+      if (submit_result == MacMetalVideoRenderer::SubmitResult::not_selected) {
+        background_snapshot_only = true;
+      }
+    }
+#endif
     {
       std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
+#ifdef __APPLE__
+      const auto now = std::chrono::steady_clock::now();
+      if (background_snapshot_only && props->thumbnail_frame_ &&
+          !props->thumbnail_frame_->empty() &&
+          props->background_snapshot_time_ !=
+              std::chrono::steady_clock::time_point{} &&
+          now - props->background_snapshot_time_ <
+              kBackgroundSnapshotInterval) {
+        props->streaming_ = true;
+        return;
+      }
+#endif
 
       // Allocate a third buffer only while the UI still owns the old snapshot.
       if (!props->back_frame_ || props->back_frame_.use_count() != 1) {
@@ -55,10 +125,23 @@ void PeerEventHandler::OnReceiveVideoBuffer(
       props->video_size_ = video_frame->size;
 
       props->front_frame_.swap(props->back_frame_);
+#ifdef __APPLE__
+      props->thumbnail_frame_ = props->front_frame_;
+      props->thumbnail_width_ = video_frame->width;
+      props->thumbnail_height_ = video_frame->height;
+      if (background_snapshot_only) {
+        props->background_snapshot_time_ = now;
+      }
+#endif
       ++props->video_frame_sequence_;
     }
 
     props->streaming_ = true;
+#ifdef __APPLE__
+    if (background_snapshot_only) {
+      return;
+    }
+#endif
     runtime->video_frame_dirty_.store(true, std::memory_order_release);
   }
 }

src/gui/runtime/remote_session.h

@@ -110,6 +110,14 @@ struct RemoteSession {
   std::mutex video_frame_mutex_;
   std::shared_ptr<std::vector<unsigned char>> front_frame_;
   std::shared_ptr<std::vector<unsigned char>> back_frame_;
+#ifdef __APPLE__
+  // Retains the most recent background CPU frame across disconnect cleanup
+  // so closing an unselected tab can still update its recent-item thumbnail.
+  std::shared_ptr<std::vector<unsigned char>> thumbnail_frame_;
+  int thumbnail_width_ = 0;
+  int thumbnail_height_ = 0;
+  std::chrono::steady_clock::time_point background_snapshot_time_;
+#endif
   bool render_rect_dirty_ = false;
   bool stream_cleanup_pending_ = false;
   float mouse_pos_x_ = 0;