[feat] add video latency to network statistics

apps/desktop/src/platform/common/gui/opengl_video_renderer.cpp

@@ -59,6 +59,7 @@ struct FrameSlot {
   int height = 0;
   std::string remote_id;
   uint64_t sequence = 0;
+  VideoLatencyFrame timing;
   SlotUse use = SlotUse::available;
   bool valid = false;
 };
@@ -320,6 +321,7 @@ struct OpenGlVideoRenderer::Impl {
   bool unpack_subimage_available = false;
   std::string uploaded_stream;
   uint64_t uploaded_sequence = 0;
+  VideoLatencyFrame uploaded_timing;
 
 #if CROSSDESK_OPENGL_CUDA_INTEROP
   enum class CudaUploadResult {
@@ -1023,25 +1025,22 @@ void OpenGlVideoRenderer::DiscardStream(std::string_view remote_id) {
   }
 }
 
-OpenGlVideoRenderer::SubmitResult
-OpenGlVideoRenderer::SubmitNv12(std::string_view remote_id,
-                                       const uint8_t *data, size_t size,
-                                       int width, int height) {
-  return SubmitNv12Internal(remote_id, data, size, width, height, true);
+OpenGlVideoRenderer::SubmitResult OpenGlVideoRenderer::SubmitNv12(
+    std::string_view remote_id, const uint8_t *data, size_t size, int width,
+    int height, VideoLatencyFrame timing) {
+  return SubmitNv12Internal(remote_id, data, size, width, height, true, timing);
 }
 
-OpenGlVideoRenderer::SubmitResult
-OpenGlVideoRenderer::SubmitCachedNv12(std::string_view remote_id,
-                                             const uint8_t *data, size_t size,
-                                             int width, int height) {
-  return SubmitNv12Internal(remote_id, data, size, width, height, false);
+OpenGlVideoRenderer::SubmitResult OpenGlVideoRenderer::SubmitCachedNv12(
+    std::string_view remote_id, const uint8_t *data, size_t size, int width,
+    int height, VideoLatencyFrame timing) {
+  return SubmitNv12Internal(remote_id, data, size, width, height, false,
+                            timing);
 }
 
-OpenGlVideoRenderer::SubmitResult
-OpenGlVideoRenderer::SubmitNv12Internal(std::string_view remote_id,
-                                               const uint8_t *data, size_t size,
-                                               int width, int height,
-                                               bool replace_pending) {
+OpenGlVideoRenderer::SubmitResult OpenGlVideoRenderer::SubmitNv12Internal(
+    std::string_view remote_id, const uint8_t *data, size_t size, int width,
+    int height, bool replace_pending, VideoLatencyFrame timing) {
   if (!IsReady()) {
     return SubmitResult::failed;
   }
@@ -1104,14 +1103,15 @@ OpenGlVideoRenderer::SubmitNv12Internal(std::string_view remote_id,
   target->height = height;
   target->remote_id.assign(remote_id);
   target->sequence = frames.next_sequence++;
+  target->timing = timing;
   target->use = SlotUse::pending;
   target->valid = true;
   return SubmitResult::submitted;
 }
 
-OpenGlVideoRenderer::SubmitResult
-OpenGlVideoRenderer::SubmitNativeFrame(std::string_view remote_id,
-                                       const MiniRtcNativeVideoFrame &frame) {
+OpenGlVideoRenderer::SubmitResult OpenGlVideoRenderer::SubmitNativeFrame(
+    std::string_view remote_id, const MiniRtcNativeVideoFrame &frame,
+    VideoLatencyFrame timing) {
   if (!IsReady()) {
     return SubmitResult::failed;
   }
@@ -1183,6 +1183,7 @@ OpenGlVideoRenderer::SubmitNativeFrame(std::string_view remote_id,
   target->height = static_cast<int>(native->height);
   target->remote_id.assign(remote_id);
   target->sequence = frames.next_sequence++;
+  target->timing = timing;
   target->use = SlotUse::pending;
   target->valid = true;
   return SubmitResult::submitted;
@@ -1204,6 +1205,7 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id,
   const unsigned char *y_data = nullptr;
   const unsigned char *uv_data = nullptr;
   NativeVideoFrameRef active_native_frame;
+  VideoLatencyFrame timing;
   {
     std::lock_guard lock(impl_->frames->mutex);
     for (size_t index = 0; index < impl_->frames->slots.size(); ++index) {
@@ -1221,6 +1223,7 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id,
       source_width = selected.width;
       source_height = selected.height;
       active_native_frame = selected.native_frame;
+      timing = selected.timing;
       if (!selected.bytes.empty()) {
         const size_t y_size = static_cast<size_t>(source_width) * source_height;
         y_data = selected.bytes.data();
@@ -1421,6 +1424,7 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id,
         impl_->texture_height = source_height;
         impl_->uploaded_stream.assign(remote_id);
         impl_->uploaded_sequence = sequence;
+        impl_->uploaded_timing = timing;
         if (!use_pixel_unpack_buffer) {
           impl_->cpu_texture_width = source_width;
           impl_->cpu_texture_height = source_height;
@@ -1549,6 +1553,8 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id,
   }
 #endif
   const GLenum draw_error = glGetError();
+  if (has_video && draw_error == GL_NO_ERROR)
+    impl_->uploaded_timing.MarkSubmitted();
   RenderOutcome outcome{
       !upload_succeeded || draw_error != GL_NO_ERROR
           ? RenderResult::failed

apps/desktop/src/platform/common/gui/opengl_video_renderer.h

@@ -39,11 +39,14 @@ public:
 
   // Thread-safe; called from MiniRTC's decode callback thread.
   SubmitResult SubmitNv12(std::string_view remote_id, const uint8_t *data,
-                          size_t size, int width, int height) override;
+                          size_t size, int width, int height,
+                          VideoLatencyFrame timing = {}) override;
   SubmitResult SubmitCachedNv12(std::string_view remote_id, const uint8_t *data,
-                                size_t size, int width, int height) override;
+                                size_t size, int width, int height,
+                                VideoLatencyFrame timing = {}) override;
   SubmitResult SubmitNativeFrame(std::string_view remote_id,
-                                 const MiniRtcNativeVideoFrame &frame) override;
+                                 const MiniRtcNativeVideoFrame &frame,
+                                 VideoLatencyFrame timing = {}) override;
 
   // Draws below Slint while its OpenGL context is current. All dimensions are
   // physical pixels in the window client area. corner_radius_pixels clips the
@@ -61,7 +64,8 @@ public:
 private:
   SubmitResult SubmitNv12Internal(std::string_view remote_id,
                                   const uint8_t *data, size_t size, int width,
-                                  int height, bool replace_pending);
+                                  int height, bool replace_pending,
+                                  VideoLatencyFrame timing);
 
   struct Impl;
   std::unique_ptr<Impl> impl_;

apps/desktop/src/platform/macos/gui/metal_video_renderer.h

@@ -35,18 +35,20 @@ class MacMetalVideoRenderer final : public VideoRenderer {
 
   // Copies one tightly packed NV12 frame into a free shared Metal slot.
   SubmitResult SubmitNv12(std::string_view remote_id, const uint8_t* data,
-                          size_t size, int width, int height) override;
+                          size_t size, int width, int height,
+                          VideoLatencyFrame timing = {}) override;
 
   // Retains an IOSurface-backed VideoToolbox CVPixelBuffer until the Metal
   // command buffer using it has completed.
   SubmitResult SubmitNativeFrame(std::string_view remote_id,
-                                 const MiniRtcNativeVideoFrame& frame) override;
+                                 const MiniRtcNativeVideoFrame& frame,
+                                 VideoLatencyFrame timing = {}) override;
 
   // Seeds a newly selected stream from its retained CPU snapshot without
   // replacing a newer decoded frame that is already queued or rendering.
-  SubmitResult SubmitCachedNv12(std::string_view remote_id,
-                                const uint8_t* data, size_t size, int width,
-                                int height) override;
+  SubmitResult SubmitCachedNv12(std::string_view remote_id, const uint8_t* data,
+                                size_t size, int width, int height,
+                                VideoLatencyFrame timing = {}) override;
 
   // Attaches a native CAMetalLayer-backed sibling below Slint's NSView and
   // restores the containing AppKit window's ordinary opaque titlebar.
@@ -76,7 +78,8 @@ class MacMetalVideoRenderer final : public VideoRenderer {
  private:
   SubmitResult SubmitNv12Internal(std::string_view remote_id,
                                   const uint8_t* data, size_t size, int width,
-                                  int height, bool replace_pending);
+                                  int height, bool replace_pending,
+                                  VideoLatencyFrame timing);
 
   struct Impl;
   std::unique_ptr<Impl> impl_;

apps/desktop/src/platform/macos/gui/metal_video_renderer.mm

@@ -340,6 +340,7 @@ struct FrameSlot {
   int height = 0;
   std::string remote_id;
   uint64_t sequence = 0;
+  VideoLatencyFrame timing;
   SlotUse use = SlotUse::available;
   bool valid = false;
 
@@ -744,14 +745,16 @@ void MacMetalVideoRenderer::DiscardStream(std::string_view remote_id) {
   }
 }
 
-MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNv12(
-    std::string_view remote_id, const uint8_t* data, size_t size, int width,
-    int height) {
-  return SubmitNv12Internal(remote_id, data, size, width, height, true);
+MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNv12(std::string_view remote_id,
+                                                                      const uint8_t* data,
+                                                                      size_t size, int width,
+                                                                      int height,
+                                                                      VideoLatencyFrame timing) {
+  return SubmitNv12Internal(remote_id, data, size, width, height, true, timing);
 }
 
 MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNativeFrame(
-    std::string_view remote_id, const MiniRtcNativeVideoFrame& frame) {
+    std::string_view remote_id, const MiniRtcNativeVideoFrame& frame, VideoLatencyFrame timing) {
   if (!IsReady()) {
     return SubmitResult::failed;
   }
@@ -781,8 +784,7 @@ MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNativeFrame(
                              packed_nv12.size()) != 0) {
       return SubmitResult::failed;
     }
-    return SubmitNv12(remote_id, packed_nv12.data(), packed_nv12.size(), width,
-                      height);
+    return SubmitNv12(remote_id, packed_nv12.data(), packed_nv12.size(), width, height, timing);
   };
 
   CVPixelBufferRef pixel_buffer =
@@ -878,6 +880,7 @@ MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNativeFrame(
   target->height = height;
   target->remote_id.assign(remote_id);
   target->sequence = frames.next_sequence++;
+  target->timing = timing;
   target->use = SlotUse::pending;
   target->valid = true;
   if (!impl_->native_path_logged.exchange(true, std::memory_order_acq_rel)) {
@@ -887,14 +890,14 @@ MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNativeFrame(
 }
 
 MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitCachedNv12(
-    std::string_view remote_id, const uint8_t* data, size_t size, int width,
-    int height) {
-  return SubmitNv12Internal(remote_id, data, size, width, height, false);
+    std::string_view remote_id, const uint8_t* data, size_t size, int width, int height,
+    VideoLatencyFrame timing) {
+  return SubmitNv12Internal(remote_id, data, size, width, height, false, timing);
 }
 
 MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNv12Internal(
-    std::string_view remote_id, const uint8_t* data, size_t size, int width,
-    int height, bool replace_pending) {
+    std::string_view remote_id, const uint8_t* data, size_t size, int width, int height,
+    bool replace_pending, VideoLatencyFrame timing) {
   if (!IsReady()) {
     return SubmitResult::failed;
   }
@@ -968,6 +971,7 @@ MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNv12Internal(
 
   target->remote_id.assign(remote_id);
   target->sequence = frames.next_sequence++;
+  target->timing = timing;
   target->use = SlotUse::pending;
   target->valid = true;
   return SubmitResult::submitted;
@@ -1189,6 +1193,7 @@ MacMetalVideoRenderer::RenderOutcome MacMetalVideoRenderer::RenderLatest(
 
     id<MTLTexture> y_texture = nil;
     id<MTLTexture> uv_texture = nil;
+    VideoLatencyFrame timing;
     {
       std::lock_guard lock(impl_->frames->mutex);
       auto& slot = impl_->frames->slots[slot_index];
@@ -1202,6 +1207,7 @@ MacMetalVideoRenderer::RenderOutcome MacMetalVideoRenderer::RenderLatest(
       slot.use = SlotUse::in_flight;
       y_texture = slot.y_texture;
       uv_texture = slot.uv_texture;
+      timing = slot.timing;
     }
 
     MTLRenderPassDescriptor* pass = [MTLRenderPassDescriptor renderPassDescriptor];
@@ -1261,6 +1267,7 @@ MacMetalVideoRenderer::RenderOutcome MacMetalVideoRenderer::RenderLatest(
     }];
     [command_buffer presentDrawable:drawable];
     [command_buffer commit];
+    timing.MarkSubmitted();
     impl_->rendered_stream.assign(remote_id);
     impl_->rendered_content_generation = content_generation;
     impl_->native_surface_needs_redraw->store(false,

apps/desktop/src/platform/video_renderer.h

@@ -9,6 +9,7 @@
 #include <vector>
 
 #include "minirtc.h"
+#include "rendering/video_latency.h"
 
 namespace crossdesk {
 
@@ -54,14 +55,17 @@ public:
 
   virtual SubmitResult SubmitNv12(std::string_view remote_id,
                                   const uint8_t* data, size_t size, int width,
-                                  int height) = 0;
+                                  int height,
+                                  VideoLatencyFrame timing = {}) = 0;
   virtual SubmitResult SubmitCachedNv12(std::string_view remote_id,
                                         const uint8_t* data, size_t size,
-                                        int width, int height) = 0;
+                                        int width, int height,
+                                        VideoLatencyFrame timing = {}) = 0;
   // Retains a platform-native decoded frame when the backend supports its
   // storage and handle types. Unsupported backends keep the default fallback.
   virtual SubmitResult SubmitNativeFrame(std::string_view,
-                                         const MiniRtcNativeVideoFrame&) {
+                                         const MiniRtcNativeVideoFrame&,
+                                         VideoLatencyFrame = {}) {
     return SubmitResult::failed;
   }