[feat] add video latency to network statistics

apps/desktop/src/gui/runtime/connection_runtime.cpp

@@ -419,6 +419,8 @@ void GuiRuntime::ResetRemoteSessionResources(
     std::shared_ptr<RemoteSession> props) {
   {
     std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
+    props->video_latency_->Reset();
+    props->video_frame_timing_ = {};
     props->front_frame_.reset();
     props->back_frame_.reset();
     props->thumbnail_frame_.reset();

apps/desktop/src/gui/runtime/peer_event_handler.cpp

@@ -336,6 +336,8 @@ void PeerEventHandler::OnConnectionStatus(ConnectionStatus status,
         }
         {
           std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
+          props->video_latency_->Reset();
+          props->video_frame_timing_ = {};
           props->front_frame_.reset();
           props->back_frame_.reset();
           if (native_snapshot &&

apps/desktop/src/gui/runtime/peer_media_callbacks.cpp

@@ -36,7 +36,7 @@ void PeerEventHandler::OnReceiveVideoBuffer(
   auto callback = handler->EnterCallback();
   if (!callback) return;
   GuiRuntime* runtime = handler ? &handler->owner_ : nullptr;
-  if (!runtime) {
+  if (!runtime || !video_frame) {
     return;
   }
 
@@ -45,13 +45,18 @@ void PeerEventHandler::OnReceiveVideoBuffer(
   if (!props) return;
 
   if (props->connection_established_) {
+    const auto timing_now = VideoLatencyFrame::Clock::now();
+    const auto timing = props->video_latency_->Frame(
+        video_frame->captured_timestamp, GetSystemTimeMicros(props->peer_),
+        timing_now);
     bool background_snapshot_only = false;
     auto* native_renderer = runtime->video_renderer_.get();
     if (native_renderer && native_renderer->IsActive()) {
       VideoRenderer::SubmitResult submit_result =
           VideoRenderer::SubmitResult::failed;
       if (const auto* native = GetNativeVideoFrame(video_frame)) {
-        submit_result = native_renderer->SubmitNativeFrame(remote_id, *native);
+        submit_result =
+            native_renderer->SubmitNativeFrame(remote_id, *native, timing);
         // A CPU NV12 native frame is valid MiniRTC input, but the macOS Metal
         // fast path only accepts CVPixelBuffer. Materialize it when that path
         // rejects a software decoder's descriptor. This also supports peers
@@ -69,14 +74,14 @@ void PeerEventHandler::OnReceiveVideoBuffer(
             submit_result = native_renderer->SubmitNv12(
                 remote_id, fallback.data(), fallback.size(),
                 static_cast<int>(native->width),
-                static_cast<int>(native->height));
+                static_cast<int>(native->height), timing);
           }
         }
       } else
       {
         submit_result = native_renderer->SubmitNv12(
             remote_id, reinterpret_cast<const uint8_t*>(video_frame->data),
-            video_frame->size, video_frame->width, video_frame->height);
+            video_frame->size, video_frame->width, video_frame->height, timing);
       }
       if (submit_result == VideoRenderer::SubmitResult::submitted) {
         std::lock_guard<std::mutex> lock(props->video_frame_mutex_);
@@ -170,6 +175,7 @@ void PeerEventHandler::OnReceiveVideoBuffer(
       props->video_size_ = frame_size;
 
       props->front_frame_.swap(props->back_frame_);
+      props->video_frame_timing_ = timing;
       props->thumbnail_frame_ = props->front_frame_;
       props->thumbnail_width_ = video_frame->width;
       props->thumbnail_height_ = video_frame->height;

apps/desktop/src/gui/runtime/remote_session.h

@@ -14,6 +14,7 @@
 #include <filesystem>
 #include <memory>
 #include <mutex>
+#include <optional>
 #include <queue>
 #include <string>
 #include <vector>
@@ -23,6 +24,7 @@
 #include "display_info.h"
 #include "cursor_presentation.h"
 #include "minirtc.h"
+#include "rendering/video_latency.h"
 
 namespace crossdesk { class PeerEventHandler; }
 
@@ -115,6 +117,9 @@ struct RemoteSession {
   int selected_display_ = 0;
   size_t video_size_ = 0;
   uint64_t video_frame_sequence_ = 0;
+  VideoLatencyFrame video_frame_timing_;
+  std::shared_ptr<VideoLatencyStats> video_latency_ =
+      std::make_shared<VideoLatencyStats>();
   bool tab_selected_ = false;
   bool tab_opened_ = true;
   std::string remote_host_name_;
@@ -150,6 +155,8 @@ struct RemoteSession {
   int fps_ = 0;
   int frame_count_ = 0;
   std::chrono::steady_clock::time_point last_time_;
+  // UI-thread snapshot refreshed with FPS once per second.
+  std::optional<VideoLatencyStats::Snapshot> video_latency_snapshot_;
   MiniRtcNetTrafficStats net_traffic_stats_{};
 
   using QueuedFile = FileTransferState::QueuedFile;