From bbbcfb9d6f48ce247d680a3a0e934224d0c576c9 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Sat, 26 Sep 2026 03:15:47 +0800 Subject: [PATCH] [fix] correct video latency measurement and smooth connection RTT --- .../src/gui/application/gui_application.cpp | 14 ++++- .../desktop/src/gui/rendering/video_latency.h | 6 ++ .../src/gui/runtime/connection_latency.h | 59 +++++++++++++++++++ .../src/gui/runtime/net_traffic_stats_cache.h | 16 ++++- .../src/gui/runtime/peer_event_handler.cpp | 1 + .../macos/gui/metal_video_renderer.mm | 16 ++++- deps/submodules/minirtc | 2 +- 7 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 apps/desktop/src/gui/runtime/connection_latency.h diff --git a/apps/desktop/src/gui/application/gui_application.cpp b/apps/desktop/src/gui/application/gui_application.cpp index 9ce13e58..b3fb5d8e 100644 --- a/apps/desktop/src/gui/application/gui_application.cpp +++ b/apps/desktop/src/gui/application/gui_application.cpp @@ -2925,17 +2925,25 @@ void GuiApplication::SyncStreamWindow() { static_cast(std::lround(latency->average_ms))) + " ms" : "—")); + const auto display_rtt = props->net_traffic_stats_.SmoothedRttMs(fps_now); const bool rtt_available = props->net_traffic_stats_button_pressed_ && status == ConnectionStatus::Connected && - net.rtt_ms >= 0; + display_rtt.has_value(); std::string connection_latency = "—"; if (rtt_available) { connection_latency = - net.rtt_ms < 1 + *display_rtt < 1 ? "<1 ms" - : std::to_string(static_cast(std::lround(net.rtt_ms))) + " ms"; + : std::to_string(static_cast(std::lround(*display_rtt))) + " ms"; } (*ui_->stream)->set_stats_connection_latency(UiText(connection_latency)); + if (refresh_stats && status == ConnectionStatus::Connected) { + LOG_DEBUG("Latency stats: remote={} video_ms={} video_samples={} " + "rtt_raw_ms={} rtt_display_ms={}", + props->remote_id_, latency ? latency->average_ms : -1, + latency ? latency->samples : 0, net.rtt_ms, + display_rtt.value_or(-1)); + } } void GuiApplication::SyncStreamVideoFrame() { diff --git a/apps/desktop/src/gui/rendering/video_latency.h b/apps/desktop/src/gui/rendering/video_latency.h index 26fa7a11..65558289 100644 --- a/apps/desktop/src/gui/rendering/video_latency.h +++ b/apps/desktop/src/gui/rendering/video_latency.h @@ -26,6 +26,8 @@ struct VideoLatencyFrame { std::shared_ptr stats; uint64_t id = 0; Clock::time_point capture_time{}; + void MarkPresented(Clock::time_point now = Clock::now()) const; + // Fallback for renderers which expose submission but no presentation time. void MarkSubmitted(Clock::time_point now = Clock::now()) const; }; @@ -97,6 +99,10 @@ class VideoLatencyStats }; inline void VideoLatencyFrame::MarkSubmitted(Clock::time_point now) const { + MarkPresented(now); +} + +inline void VideoLatencyFrame::MarkPresented(Clock::time_point now) const { if (stats) stats->Record(*this, now); } } // namespace crossdesk diff --git a/apps/desktop/src/gui/runtime/connection_latency.h b/apps/desktop/src/gui/runtime/connection_latency.h new file mode 100644 index 00000000..b34ea9f5 --- /dev/null +++ b/apps/desktop/src/gui/runtime/connection_latency.h @@ -0,0 +1,59 @@ +/* + * @Author: DI JUNKUN + * @Date: 2026-09-26 + * Copyright (c) 2026 by DI JUNKUN, All Rights Reserved. + */ + +#ifndef _CONNECTION_LATENCY_H_ +#define _CONNECTION_LATENCY_H_ + +#include +#include +#include +#include + +namespace crossdesk::gui_detail { + +// Presentation-only smoothing. The transport's raw RTT remains available for +// recovery, congestion control and diagnostics. A missing one-second report +// does not flicker, but a disconnected/stalled peer cannot leave a stale value. +class ConnectionLatencyStats { + public: + using Clock = std::chrono::steady_clock; + + void Update(double rtt_ms, Clock::time_point now = Clock::now()) { + if (!std::isfinite(rtt_ms) || rtt_ms < 0 || rtt_ms > 2000) return; + std::lock_guard lock(mutex_); + if (average_ms_ && now < updated_) return; + if (!Fresh(now)) { + average_ms_ = rtt_ms; + } else { + *average_ms_ += 0.25 * (rtt_ms - *average_ms_); + } + updated_ = now; + } + + std::optional Get(Clock::time_point now = Clock::now()) const { + std::lock_guard lock(mutex_); + return Fresh(now) ? average_ms_ : std::nullopt; + } + + void Reset() { + std::lock_guard lock(mutex_); + average_ms_.reset(); + } + + private: + bool Fresh(Clock::time_point now) const { + // The UI may snapshot `now` just before the report thread updates us. + // A newer valid report must not cause a one-frame unavailable flicker. + return average_ms_ && now - updated_ < std::chrono::seconds(3); + } + mutable std::mutex mutex_; + std::optional average_ms_; + Clock::time_point updated_{}; +}; + +} // namespace crossdesk::gui_detail + +#endif \ No newline at end of file diff --git a/apps/desktop/src/gui/runtime/net_traffic_stats_cache.h b/apps/desktop/src/gui/runtime/net_traffic_stats_cache.h index 616f230d..3028c105 100644 --- a/apps/desktop/src/gui/runtime/net_traffic_stats_cache.h +++ b/apps/desktop/src/gui/runtime/net_traffic_stats_cache.h @@ -10,6 +10,7 @@ #include #include "minirtc.h" +#include "runtime/connection_latency.h" namespace crossdesk::gui_detail { @@ -34,6 +35,7 @@ class NetTrafficStatsCache { total_outbound_stats_.Store(stats.total_outbound_stats); srtp_active_.store(stats.srtp_active, std::memory_order_relaxed); rtt_ms_.store(stats.rtt_ms, std::memory_order_relaxed); + connection_latency_.Update(stats.rtt_ms); } MiniRtcNetTrafficStats Load() const { @@ -55,7 +57,18 @@ class NetTrafficStatsCache { return data_outbound_stats_.bitrate.load(std::memory_order_relaxed); } - void Reset() { Store(MiniRtcNetTrafficStats{}); } + std::optional SmoothedRttMs( + ConnectionLatencyStats::Clock::time_point now = + ConnectionLatencyStats::Clock::now()) const { + return connection_latency_.Get(now); + } + + void ResetLatency() { connection_latency_.Reset(); } + + void Reset() { + Store(MiniRtcNetTrafficStats{}); + ResetLatency(); + } private: struct InboundStats { @@ -101,6 +114,7 @@ class NetTrafficStatsCache { OutboundStats total_outbound_stats_; std::atomic srtp_active_{false}; std::atomic rtt_ms_{-1}; + ConnectionLatencyStats connection_latency_; }; } // namespace crossdesk::gui_detail diff --git a/apps/desktop/src/gui/runtime/peer_event_handler.cpp b/apps/desktop/src/gui/runtime/peer_event_handler.cpp index 8079380e..a1701436 100644 --- a/apps/desktop/src/gui/runtime/peer_event_handler.cpp +++ b/apps/desktop/src/gui/runtime/peer_event_handler.cpp @@ -511,6 +511,7 @@ void PeerEventHandler::OnNetStatusReport( auto props = runtime->FindRemoteSession(remote_id); if (!props) return; if (props->traversal_mode_ != mode) { + props->net_traffic_stats_.ResetLatency(); props->traversal_mode_ = mode; LOG_INFO("Net mode: [{}]", int(props->traversal_mode_)); } diff --git a/apps/desktop/src/platform/macos/gui/metal_video_renderer.mm b/apps/desktop/src/platform/macos/gui/metal_video_renderer.mm index 45b5b739..69445c53 100644 --- a/apps/desktop/src/platform/macos/gui/metal_video_renderer.mm +++ b/apps/desktop/src/platform/macos/gui/metal_video_renderer.mm @@ -3,11 +3,13 @@ #import #import #import +#import #import #include #include #include +#include #include #include #include @@ -1265,9 +1267,21 @@ MacMetalVideoRenderer::RenderOutcome MacMetalVideoRenderer::RenderLatest( slot.use = SlotUse::available; } }]; + [drawable addPresentedHandler:^(id presented) { + // presentedTime uses the Core Animation host clock. Translate its age + // into our steady clock, excluding callback scheduling delay. Capturing + // timing by value keeps this safe after a session/renderer is destroyed. + const double presented_time = presented.presentedTime; + const auto now = VideoLatencyFrame::Clock::now(); + const double age = CACurrentMediaTime() - presented_time; + if (presented_time > 0 && std::isfinite(age) && age >= 0 && age <= 5) { + timing.MarkPresented( + now - std::chrono::duration_cast( + std::chrono::duration(age))); + } + }]; [command_buffer presentDrawable:drawable]; [command_buffer commit]; - timing.MarkSubmitted(); impl_->rendered_stream.assign(remote_id); impl_->rendered_content_generation = content_generation; impl_->native_surface_needs_redraw->store(false, diff --git a/deps/submodules/minirtc b/deps/submodules/minirtc index 2ef58ae5..e4dcd826 160000 --- a/deps/submodules/minirtc +++ b/deps/submodules/minirtc @@ -1 +1 @@ -Subproject commit 2ef58ae5af9c68cbce5644ff0884fb8a0586fdd9 +Subproject commit e4dcd8260969e5ebfd3f8c6ccd2329502277931c