[feat] add Windows native video rendering pipeline

apps/desktop/src/gui/rendering/slint_video_presenter.cpp

@@ -119,9 +119,21 @@ struct SlintVideoPresenter::Impl {
             return;
           }
 
-          std::lock_guard lock(gl_mutex);
           auto* opengl_renderer =
               dynamic_cast<OpenGlVideoRenderer*>(&renderer);
+          if (state == slint::RenderingState::AfterRendering && stream &&
+              opengl_renderer &&
+              (*stream)->get_native_video_enabled() &&
+              opengl_renderer->ShouldContinueRendering()) {
+            // Queue the next frame directly from the display callback. A
+            // 16/17 ms polling timer can drift across vsync and make Windows
+            // coalesce every second request, effectively capping video near
+            // 30 fps even when decode and submission stay near 60 fps.
+            (*stream)->window().request_redraw();
+            return;
+          }
+
+          std::lock_guard lock(gl_mutex);
           if (state == slint::RenderingState::RenderingSetup) {
             GLuint texture = 0;
             glGenTextures(1, &texture);

apps/desktop/src/gui/runtime/gui_runtime.cpp

@@ -141,9 +141,13 @@ int GuiRuntime::CreateConnectionPeer() {
           sizeof(params_.log_path) - 1);
   params_.log_path[sizeof(params_.log_path) - 1] = '\0';
   params_.hardware_acceleration = config_center_->IsHardwareVideoCodec();
-  // The Slint desktop renderer currently consumes packed CPU frames. Native
-  // renderers can opt into platform-native frame output independently.
+#if defined(_WIN32)
+  // Windows accepts ref-counted CPU NV12 and CUDA device NV12. The OpenGL
+  // renderer retains the decoded frame until the UI thread has uploaded it.
+  params_.native_video_output = true;
+#else
   params_.native_video_output = false;
+#endif
   params_.av1_encoding = config_center_->GetVideoEncodeFormat() ==
                                  ConfigCenter::VIDEO_ENCODE_FORMAT::AV1
                              ? true

apps/desktop/src/gui/runtime/peer_media_callbacks.cpp

@@ -14,6 +14,23 @@ namespace {
 
 constexpr auto kBackgroundSnapshotInterval = std::chrono::seconds(1);
 
+#if defined(_WIN32)
+const XWindowsVideoFrame* GetWindowsNativeFrame(const XVideoFrame* frame) {
+  if (!frame ||
+      frame->native_handle_type != XVideoFrameNativeHandleWindowsNv12 ||
+      !frame->native_handle) {
+    return nullptr;
+  }
+  const auto* native =
+      static_cast<const XWindowsVideoFrame*>(frame->native_handle);
+  return native->struct_size >=
+                     static_cast<uint32_t>(sizeof(XWindowsVideoFrame)) &&
+                 native->owner && native->copy_to_cpu
+             ? native
+             : nullptr;
+}
+#endif
+
 }  // namespace
 
 void PeerEventHandler::OnReceiveVideoBuffer(
@@ -38,9 +55,19 @@ void PeerEventHandler::OnReceiveVideoBuffer(
     bool background_snapshot_only = false;
     auto* native_renderer = runtime->video_renderer_.get();
     if (native_renderer && native_renderer->IsActive()) {
-      const auto submit_result = native_renderer->SubmitNv12(
-          remote_id, reinterpret_cast<const uint8_t*>(video_frame->data),
-          video_frame->size, video_frame->width, video_frame->height);
+      VideoRenderer::SubmitResult submit_result =
+          VideoRenderer::SubmitResult::failed;
+#if defined(_WIN32)
+      if (GetWindowsNativeFrame(video_frame)) {
+        submit_result =
+            native_renderer->SubmitNativeFrame(remote_id, *video_frame);
+      } else
+#endif
+      {
+        submit_result = native_renderer->SubmitNv12(
+            remote_id, reinterpret_cast<const uint8_t*>(video_frame->data),
+            video_frame->size, video_frame->width, video_frame->height);
+      }
       if (submit_result == VideoRenderer::SubmitResult::submitted) {
         std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
         const bool size_changed = (props->video_width_ != video_frame->width) ||
@@ -88,17 +115,43 @@ void PeerEventHandler::OnReceiveVideoBuffer(
         return;
       }
 
+      size_t frame_size = video_frame->size;
+#if defined(_WIN32)
+      const XWindowsVideoFrame* windows_native_frame =
+          GetWindowsNativeFrame(video_frame);
+      if (windows_native_frame) {
+        frame_size = static_cast<size_t>(windows_native_frame->width) *
+                     windows_native_frame->height * 3U / 2U;
+      }
+#endif
+      if (frame_size == 0) {
+        return;
+      }
+
       // Allocate a third buffer only while the UI still owns the old snapshot.
       if (!props->back_frame_ || props->back_frame_.use_count() != 1) {
         props->back_frame_ =
-            std::make_shared<std::vector<unsigned char>>(video_frame->size);
+            std::make_shared<std::vector<unsigned char>>(frame_size);
       }
-      if (props->back_frame_->size() != video_frame->size) {
-        props->back_frame_->resize(video_frame->size);
+      if (props->back_frame_->size() != frame_size) {
+        props->back_frame_->resize(frame_size);
       }
 
-      std::memcpy(props->back_frame_->data(), video_frame->data,
-                  video_frame->size);
+#if defined(_WIN32)
+      if (windows_native_frame) {
+        if (windows_native_frame->copy_to_cpu(
+                windows_native_frame->owner, props->back_frame_->data(),
+                props->back_frame_->size()) != 0) {
+          return;
+        }
+      } else
+#endif
+      {
+        if (!video_frame->data || video_frame->size < frame_size) {
+          return;
+        }
+        std::memcpy(props->back_frame_->data(), video_frame->data, frame_size);
+      }
 
       const bool size_changed = (props->video_width_ != video_frame->width) ||
                                 (props->video_height_ != video_frame->height);
@@ -108,7 +161,7 @@ void PeerEventHandler::OnReceiveVideoBuffer(
 
       props->video_width_ = video_frame->width;
       props->video_height_ = video_frame->height;
-      props->video_size_ = video_frame->size;
+      props->video_size_ = frame_size;
 
       props->front_frame_.swap(props->back_frame_);
       props->thumbnail_frame_ = props->front_frame_;

apps/desktop/src/platform/common/gui/opengl_video_renderer.h

@@ -13,10 +13,11 @@
 namespace crossdesk {
 
 // NV12 underlay for Slint's FemtoVG OpenGL renderer on Windows and Linux.
-// MiniRTC submits tightly packed CPU NV12 frames from its decode callback
-// thread. The Slint UI thread uploads the newest frame as Y and UV textures and
-// performs color conversion and scaling in a fragment shader before Slint
-// draws its overlay.
+// MiniRTC submits either ref-counted Windows native NV12 frames or packed CPU
+// NV12 fallback frames from its decode callback thread. CUDA-backed frames use
+// CUDA/OpenGL interop; CPU-backed frames are retained without an intermediate
+// queue copy. The Slint UI thread updates Y and UV textures and performs color
+// conversion and scaling before Slint draws its overlay.
 class OpenGlVideoRenderer final : public VideoRenderer {
 public:
   OpenGlVideoRenderer();
@@ -32,6 +33,11 @@ public:
   bool IsReady() const override;
   bool IsActive() const override;
 
+  // Keeps Slint's native surface on a display-driven redraw loop while decoded
+  // frames are arriving. This avoids timer/vsync phase drift that otherwise
+  // causes every second redraw request to be coalesced on Windows.
+  bool ShouldContinueRendering() const;
+
   // Only the selected stream is queued for upload. Frames for other tabs are
   // retained by GuiRuntime at a throttled rate for thumbnail/tab restoration.
   bool SetSelectedStream(std::string remote_id) override;
@@ -42,6 +48,8 @@ public:
                           size_t size, int width, int height) override;
   SubmitResult SubmitCachedNv12(std::string_view remote_id, const uint8_t *data,
                                 size_t size, int width, int height) override;
+  SubmitResult SubmitNativeFrame(std::string_view remote_id,
+                                 const XVideoFrame &frame) override;
 
   // Draws below Slint while its OpenGL context is current. All dimensions are
   // physical pixels in the window client area. corner_radius_pixels clips the

apps/desktop/src/platform/video_renderer.h

@@ -8,6 +8,8 @@
 #include <string_view>
 #include <vector>
 
+#include "minirtc.h"
+
 namespace crossdesk {
 
 // Platform-independent contract for the native stream video renderer.
@@ -56,6 +58,10 @@ public:
   virtual SubmitResult SubmitCachedNv12(std::string_view remote_id,
                                         const uint8_t* data, size_t size,
                                         int width, int height) = 0;
+  virtual SubmitResult SubmitNativeFrame(std::string_view,
+                                         const XVideoFrame&) {
+    return SubmitResult::failed;
+  }
 
   virtual bool CopyLatestNv12(std::string_view remote_id,
                               std::vector<unsigned char>* output, int* width,

apps/desktop/tests/video_receive_scheduler_test.cpp

@@ -0,0 +1,200 @@
+#include <cstdint>
+#include <iostream>
+#include <memory>
+
+#include "video_receive_scheduler.h"
+
+namespace {
+
+std::unique_ptr<minirtc::ReceivedFrame>
+MakeFrame(uint32_t rtp_timestamp, int64_t received_us, bool key_frame = false) {
+  const uint8_t payload = key_frame ? 0x65 : 0x41;
+  auto frame =
+      std::make_unique<minirtc::ReceivedFrame>(&payload, sizeof(payload));
+  frame->SetRtpTimestamp(rtp_timestamp);
+  frame->SetReceivedTimestamp(received_us);
+  frame->SetCapturedTimestamp(received_us - 5000);
+  frame->SetFrameType(key_frame ? minirtc::kVideoFrameKey
+                                : minirtc::kVideoFrameDelta);
+  return frame;
+}
+
+bool Expect(bool condition, const char *message) {
+  if (!condition) {
+    std::cerr << message << '\n';
+  }
+  return condition;
+}
+
+bool TestOneFramePerPlaybackCycle() {
+  minirtc::VideoReceiveScheduler scheduler(60);
+  scheduler.Enqueue(MakeFrame(90000, 1000000), 1000000);
+  scheduler.Enqueue(MakeFrame(91500, 1016667), 1016667);
+
+  auto first = scheduler.Poll(1016667, false);
+  bool ok = Expect(first.frame != nullptr, "first frame was not released");
+  ok &= Expect(first.frame && first.frame->RtpTimestamp() == 90000,
+               "scheduler did not preserve RTP order");
+
+  auto same_cycle = scheduler.Poll(1016667, false);
+  ok &= Expect(!same_cycle.frame,
+               "scheduler released more than one frame in a cycle");
+  ok &= Expect(same_cycle.next_delay_us > 0,
+               "scheduler did not schedule the next playback cycle");
+
+  auto second = scheduler.Poll(1033334, false);
+  ok &= Expect(second.frame != nullptr,
+               "second frame was not released on the next cycle");
+  ok &= Expect(second.frame && second.frame->RtpTimestamp() == 91500,
+               "second RTP timestamp was incorrect");
+  return ok;
+}
+
+bool TestSevereLagJumpsToNewestKeyFrame() {
+  minirtc::VideoReceiveScheduler scheduler(60);
+  for (uint32_t i = 0; i < 10; ++i) {
+    scheduler.Enqueue(MakeFrame(90000 + i * 1500, 1000000 + i * 16667, i == 8),
+                      1000000 + i * 16667);
+  }
+
+  auto result = scheduler.Poll(1200000, false);
+  const auto metrics = scheduler.GetMetrics(1200000);
+  bool ok = Expect(result.frame && result.frame->IsKeyFrame(),
+                   "catch-up did not jump to the newest key frame");
+  ok &= Expect(result.dropped_frames == 8,
+               "catch-up dropped an unexpected number of frames");
+  ok &= Expect(metrics.catch_up_count == 1,
+               "catch-up metric was not incremented");
+  return ok;
+}
+
+bool TestSevereLagRequestsKeyFrameWhenNoSyncPointExists() {
+  minirtc::VideoReceiveScheduler scheduler(60);
+  for (uint32_t i = 0; i < 10; ++i) {
+    scheduler.Enqueue(MakeFrame(180000 + i * 1500, 2000000 + i * 16667),
+                      2000000 + i * 16667);
+  }
+
+  auto result = scheduler.Poll(2200000, false);
+  const auto metrics = scheduler.GetMetrics(2200000);
+  bool ok = Expect(!result.frame,
+                   "undecodable delta frame was released during catch-up");
+  ok &=
+      Expect(result.request_key_frame, "catch-up did not request a key frame");
+  ok &= Expect(metrics.buffered_frames == 0 && metrics.awaiting_key_frame,
+               "scheduler did not enter key-frame wait state");
+  return ok;
+}
+
+bool TestAdaptiveBufferStaysWithinOneToThreeFrames() {
+  minirtc::VideoReceiveScheduler scheduler(60);
+  int64_t arrival_us = 3000000;
+  for (uint32_t i = 0; i < 40; ++i) {
+    arrival_us += (i % 2 == 0) ? 5000 : 30000;
+    scheduler.Enqueue(MakeFrame(270000 + i * 1500, arrival_us), arrival_us);
+  }
+
+  const auto metrics = scheduler.GetMetrics(arrival_us);
+  bool ok = Expect(metrics.target_buffer_frames >= 1 &&
+                       metrics.target_buffer_frames <= 3,
+                   "adaptive target left the 1-3 frame range");
+  ok &= Expect(metrics.target_buffer_frames > 1,
+               "arrival jitter did not increase the target buffer");
+  return ok;
+}
+
+bool TestLegacyMicrosecondTimestampFallsBackToArrivalCadence() {
+  minirtc::VideoReceiveScheduler scheduler(60);
+  int64_t arrival_us = 4000000;
+  for (uint32_t i = 0; i < 20; ++i) {
+    scheduler.Enqueue(MakeFrame(500000 + i * 16667, arrival_us), arrival_us);
+    scheduler.Poll(arrival_us, false);
+    arrival_us += 16667;
+  }
+  scheduler.Poll(arrival_us, false);
+
+  const auto metrics = scheduler.GetMetrics(arrival_us);
+  bool ok = Expect(metrics.estimated_frame_interval_us >= 15000 &&
+                       metrics.estimated_frame_interval_us <= 20000,
+                   "legacy RTP units slowed the playback interval");
+  ok &= Expect(metrics.timestamp_fallbacks > 0,
+               "legacy RTP units did not activate timestamp fallback");
+  ok &= Expect(metrics.released_frames == 20 && metrics.dropped_frames == 0,
+               "legacy RTP units caused scheduler frame loss");
+  return ok;
+}
+
+bool TestSixtyFpsCadenceDoesNotTriggerCatchUp() {
+  minirtc::VideoReceiveScheduler scheduler(60);
+  constexpr int64_t kStartUs = 6000000;
+  constexpr int64_t kFrameIntervalUs = 16667;
+  constexpr uint32_t kStartRtpTimestamp = 700000;
+  constexpr uint32_t kRtpFrameInterval = 1500;
+
+  for (uint32_t i = 0; i < 60; ++i) {
+    const int64_t now_us = kStartUs + static_cast<int64_t>(i) *
+                                          kFrameIntervalUs;
+    scheduler.Enqueue(
+        MakeFrame(kStartRtpTimestamp + i * kRtpFrameInterval, now_us),
+        now_us);
+    scheduler.Poll(now_us, false);
+  }
+  scheduler.Poll(kStartUs + 60 * kFrameIntervalUs, false);
+
+  const auto metrics =
+      scheduler.GetMetrics(kStartUs + 60 * kFrameIntervalUs);
+  bool ok = Expect(metrics.released_frames == 60,
+                   "60 fps input was not released at 60 fps cadence");
+  ok &= Expect(metrics.dropped_frames == 0 && metrics.catch_up_count == 0,
+               "steady 60 fps input incorrectly triggered catch-up");
+  ok &= Expect(metrics.estimated_frame_interval_us >= 15000 &&
+                   metrics.estimated_frame_interval_us <= 18000,
+               "steady 60 fps interval estimate drifted");
+  return ok;
+}
+
+bool TestKeyFrameRequestCooldownKeepsDecodableProgress() {
+  minirtc::VideoReceiveScheduler scheduler(60);
+  for (uint32_t i = 0; i < 10; ++i) {
+    scheduler.Enqueue(MakeFrame(600000 + i * 1500,
+                                5000000 + static_cast<int64_t>(i) * 16667),
+                      5000000 + static_cast<int64_t>(i) * 16667);
+  }
+  auto first_catch_up = scheduler.Poll(5200000, false);
+  bool ok = Expect(first_catch_up.request_key_frame,
+                   "initial severe lag did not request a key frame");
+
+  scheduler.Enqueue(MakeFrame(615000, 5220000, true), 5220000);
+  auto key_frame = scheduler.Poll(5240000, false);
+  ok &= Expect(key_frame.frame && key_frame.frame->IsKeyFrame(),
+               "requested key frame was not released");
+
+  for (uint32_t i = 0; i < 10; ++i) {
+    scheduler.Enqueue(MakeFrame(616500 + i * 1500,
+                                5250000 + static_cast<int64_t>(i) * 16667),
+                      5250000 + static_cast<int64_t>(i) * 16667);
+  }
+  auto cooldown_catch_up = scheduler.Poll(5450000, false);
+  const auto metrics = scheduler.GetMetrics(5450000);
+  ok &= Expect(!cooldown_catch_up.request_key_frame,
+               "key frame request cooldown was ignored");
+  ok &= Expect(cooldown_catch_up.frame != nullptr,
+               "cooldown prevented forward playback progress");
+  ok &= Expect(metrics.key_frame_requests == 1,
+               "key frame request metric ignored cooldown");
+  return ok;
+}
+
+} // namespace
+
+int main() {
+  bool ok = true;
+  ok &= TestOneFramePerPlaybackCycle();
+  ok &= TestSevereLagJumpsToNewestKeyFrame();
+  ok &= TestSevereLagRequestsKeyFrameWhenNoSyncPointExists();
+  ok &= TestAdaptiveBufferStaysWithinOneToThreeFrames();
+  ok &= TestLegacyMicrosecondTimestampFallsBackToArrivalCadence();
+  ok &= TestSixtyFpsCadenceDoesNotTriggerCatchUp();
+  ok &= TestKeyFrameRequestCooldownKeepsDecodableProgress();
+  return ok ? 0 : 1;
+}

apps/desktop/tests/video_rtp_timestamp_test.cpp

@@ -0,0 +1,84 @@
+#include <cstdint>
+#include <iostream>
+#include <memory>
+#include <vector>
+
+#include "rtp_defines.h"
+#include "rtp_packetizer_av1.h"
+#include "rtp_packetizer_h264.h"
+
+namespace {
+
+bool Expect(bool condition, const char *message) {
+  if (!condition) {
+    std::cerr << message << '\n';
+  }
+  return condition;
+}
+
+bool AllPacketsUseTimestamp(
+    const std::vector<std::unique_ptr<minirtc::RtpPacket>> &packets,
+    uint32_t expected_timestamp) {
+  if (packets.empty()) {
+    return false;
+  }
+  for (const auto &packet : packets) {
+    if (!packet || packet->Timestamp() != expected_timestamp) {
+      return false;
+    }
+  }
+  return true;
+}
+
+bool TestTimestampConversion() {
+  bool ok = Expect(minirtc::rtp::VideoTimestampFromMicroseconds(1000000) ==
+                       90000,
+                   "one second did not convert to 90 kHz");
+  ok &= Expect(minirtc::rtp::VideoTimestampFromMicroseconds(16667) == 1500,
+               "60 fps interval did not convert to 1500 RTP ticks");
+  ok &= Expect(minirtc::rtp::VideoTimestampToMicroseconds(90000) == 1000000,
+               "90 kHz timestamp did not convert to one second");
+  return ok;
+}
+
+bool TestH264PacketizerPreservesTimestamp() {
+  constexpr uint32_t kTimestamp = 0x89abcdef;
+  minirtc::RtpPacketizerH264 packetizer(1234);
+  std::vector<uint8_t> small_nalu = {0x65, 0x01, 0x02};
+  auto single_packet =
+      packetizer.Build(small_nalu.data(), small_nalu.size(), kTimestamp, true);
+  bool ok = Expect(AllPacketsUseTimestamp(single_packet, kTimestamp),
+                   "H.264 single-NAL packetizer changed RTP timestamp");
+
+  std::vector<uint8_t> large_nalu(MAX_NALU_LEN + 100, 0x55);
+  large_nalu.front() = 0x65;
+  auto fragmented =
+      packetizer.Build(large_nalu.data(), large_nalu.size(), kTimestamp, true);
+  ok &= Expect(AllPacketsUseTimestamp(fragmented, kTimestamp),
+               "H.264 FU-A packetizer changed RTP timestamp");
+
+  auto padding = packetizer.BuildPadding(2000, kTimestamp, true);
+  ok &= Expect(AllPacketsUseTimestamp(padding, kTimestamp),
+               "H.264 padding packetizer changed RTP timestamp");
+  return ok;
+}
+
+bool TestAv1PacketizerPreservesTimestamp() {
+  constexpr uint32_t kTimestamp = 0x12345678;
+  minirtc::RtpPacketizerAv1 packetizer(5678);
+  // One frame OBU with a one-byte payload.
+  std::vector<uint8_t> obu = {0x32, 0x01, 0x00};
+  auto packets = packetizer.Build(obu.data(), obu.size(), kTimestamp, true);
+  return Expect(AllPacketsUseTimestamp(packets, kTimestamp),
+                "AV1 packetizer changed RTP timestamp");
+}
+
+} // namespace
+
+int main() {
+  bool ok = true;
+  ok &= TestTimestampConversion();
+  ok &= TestH264PacketizerPreservesTimestamp();
+  ok &= TestAv1PacketizerPreservesTimestamp();
+  return ok ? 0 : 1;
+}

apps/desktop/xmake/targets.lua

@@ -106,6 +106,18 @@ function setup_targets()
         set_default(false)
         add_files("apps/desktop/tests/video_callback_lifetime_test.cpp")
 
+    target("video_receive_scheduler_test")
+        set_kind("binary")
+        set_default(false)
+        add_deps("minirtc")
+        add_files("apps/desktop/tests/video_receive_scheduler_test.cpp")
+
+    target("video_rtp_timestamp_test")
+        set_kind("binary")
+        set_default(false)
+        add_deps("minirtc")
+        add_files("apps/desktop/tests/video_rtp_timestamp_test.cpp")
+
     target("windows_manifest_resource_test")
         set_kind("binary")
         set_default(false)