diff --git a/apps/desktop/src/gui/application/gui_application.cpp b/apps/desktop/src/gui/application/gui_application.cpp index 2a2b06f1..e92f05d9 100644 --- a/apps/desktop/src/gui/application/gui_application.cpp +++ b/apps/desktop/src/gui/application/gui_application.cpp @@ -2892,12 +2892,14 @@ void GuiApplication::SyncStreamWindow() { props->file_transfer_.file_transfer_window_visible_); const auto fps_now = std::chrono::steady_clock::now(); + bool refresh_stats = false; if (!props->net_traffic_stats_button_pressed_) { props->fps_ = 0; props->frame_count_ = 0; props->last_time_ = {}; } else if (props->last_time_.time_since_epoch().count() == 0) { props->last_time_ = fps_now; + refresh_stats = true; } else { const auto elapsed = std::chrono::duration_cast( fps_now - props->last_time_) @@ -2906,9 +2908,23 @@ void GuiApplication::SyncStreamWindow() { props->fps_ = static_cast(props->frame_count_ * 1000 / elapsed); props->frame_count_ = 0; props->last_time_ = fps_now; + refresh_stats = true; } } (*ui_->stream)->set_stats_fps(UiText(std::to_string(props->fps_))); + if (!props->net_traffic_stats_button_pressed_ || + status != ConnectionStatus::Connected) { + props->video_latency_snapshot_.reset(); + } else if (refresh_stats) { + props->video_latency_snapshot_ = props->video_latency_->Get(fps_now); + } + const auto& latency = props->video_latency_snapshot_; + (*ui_->stream) + ->set_stats_video_latency(UiText( + latency ? std::to_string( + static_cast(std::lround(latency->average_ms))) + + " ms" + : "—")); } void GuiApplication::SyncStreamVideoFrame() { @@ -2928,6 +2944,7 @@ void GuiApplication::SyncStreamVideoFrame() { frame.width = props->video_width_; frame.height = props->video_height_; frame.sequence = props->video_frame_sequence_; + frame.timing = props->video_frame_timing_; } const auto result = ui_->video_presenter->Present(frame); diff --git a/apps/desktop/src/gui/assets/localization/localization_data.h b/apps/desktop/src/gui/assets/localization/localization_data.h index 9c5a946d..a14bdf2b 100644 --- a/apps/desktop/src/gui/assets/localization/localization_data.h +++ b/apps/desktop/src/gui/assets/localization/localization_data.h @@ -305,6 +305,7 @@ struct TranslationRow { "File Transfer Save Path:", u8"Путь сохранения файлов:") \ X(default_desktop, u8"桌面", "Desktop", u8"Рабочий стол") \ X(resolution, u8"分辨率", "Res", u8"Разрешение") \ + X(video_latency, u8"视频延时", "Video delay", u8"Задержка") \ X(connection_mode, u8"连接模式", "Mode", u8"Режим") \ X(connection_mode_direct, u8"直连", "Direct", u8"Прямой") \ X(connection_mode_relay, u8"中继", "Relay", u8"Релейный") \ diff --git a/apps/desktop/src/gui/rendering/slint_video_presenter.cpp b/apps/desktop/src/gui/rendering/slint_video_presenter.cpp index c36f380c..4ae9168a 100644 --- a/apps/desktop/src/gui/rendering/slint_video_presenter.cpp +++ b/apps/desktop/src/gui/rendering/slint_video_presenter.cpp @@ -87,6 +87,7 @@ struct SlintVideoPresenter::Impl { VideoRenderSize gl_image_size; VideoRenderSize gl_pending_size; bool gl_pending_dirty = false; + VideoLatencyFrame gl_pending_timing; #endif SurfaceState CurrentState() const { @@ -192,6 +193,7 @@ struct SlintVideoPresenter::Impl { glPixelStorei(GL_UNPACK_ALIGNMENT, previous_unpack_alignment); glBindTexture(GL_TEXTURE_2D, static_cast(previous_texture)); gl_pending_dirty = false; + gl_pending_timing.MarkSubmitted(); return; } @@ -322,7 +324,7 @@ SlintVideoPresenter::PresentResult SlintVideoPresenter::Present( impl_->seeded_frame_sequence[frame.remote_id] != frame.sequence) { const auto submit_result = impl_->renderer.SubmitCachedNv12( frame.remote_id, frame.nv12->data(), frame.nv12->size(), frame.width, - frame.height); + frame.height, frame.timing); if (submit_result == VideoRenderer::SubmitResult::submitted || submit_result == VideoRenderer::SubmitResult::dropped) { impl_->seeded_frame_sequence[frame.remote_id] = frame.sequence; @@ -440,6 +442,7 @@ SlintVideoPresenter::PresentResult SlintVideoPresenter::Present( impl_->gl_pending_frame.swap(impl_->gl_conversion_frame); impl_->gl_pending_size = {output_width, output_height}; impl_->gl_pending_dirty = true; + impl_->gl_pending_timing = frame.timing; } const VideoRenderSize output_size{output_width, output_height}; if (impl_->gl_image_size != output_size) { @@ -469,6 +472,7 @@ SlintVideoPresenter::PresentResult SlintVideoPresenter::Present( } (*impl_->stream)->set_frame(slint::Image(std::move(pixels))); + frame.timing.MarkSubmitted(); (*impl_->stream)->set_has_frame(true); (*impl_->stream)->set_receiving_text(""); result.width = frame.width; diff --git a/apps/desktop/src/gui/rendering/slint_video_presenter.h b/apps/desktop/src/gui/rendering/slint_video_presenter.h index 2fe2df21..4d6232ed 100644 --- a/apps/desktop/src/gui/rendering/slint_video_presenter.h +++ b/apps/desktop/src/gui/rendering/slint_video_presenter.h @@ -14,6 +14,8 @@ #include #include +#include "rendering/video_latency.h" + namespace slint { template class ComponentHandle; } @@ -42,6 +44,7 @@ public: int width = 0; int height = 0; uint64_t sequence = 0; + VideoLatencyFrame timing; }; struct PresentResult { diff --git a/apps/desktop/src/gui/rendering/video_latency.h b/apps/desktop/src/gui/rendering/video_latency.h new file mode 100644 index 00000000..26fa7a11 --- /dev/null +++ b/apps/desktop/src/gui/rendering/video_latency.h @@ -0,0 +1,104 @@ +/* + * @Author: DI JUNKUN + * @Date: 2026-09-26 + * Copyright (c) 2026 by DI JUNKUN, All Rights Reserved. + */ + +#ifndef _VIDEO_LATENCY_H_ +#define _VIDEO_LATENCY_H_ + +#include +#include +#include +#include +#include +#include +#include + +namespace crossdesk { + +class VideoLatencyStats; + +// Travels with the exact queued pixels, including native frames. Its clock is +// translated once at the receive callback; rendering needs no live PeerPtr. +struct VideoLatencyFrame { + using Clock = std::chrono::steady_clock; + std::shared_ptr stats; + uint64_t id = 0; + Clock::time_point capture_time{}; + void MarkSubmitted(Clock::time_point now = Clock::now()) const; +}; + +class VideoLatencyStats + : public std::enable_shared_from_this { + public: + using Clock = VideoLatencyFrame::Clock; + struct Snapshot { + double average_ms; + size_t samples; + }; + + VideoLatencyFrame Frame(uint64_t captured_us, int64_t local_now_us, + Clock::time_point now = Clock::now()) { + if (captured_us == 0 || local_now_us <= 0 || + captured_us > uint64_t(local_now_us) || + uint64_t(local_now_us) - captured_us > 5'000'000) + return {}; + std::lock_guard lock(mutex_); + return {shared_from_this(), ++issued_id_, + now - std::chrono::microseconds(local_now_us - captured_us)}; + } + + void Record(const VideoLatencyFrame& frame, Clock::time_point now) { + std::lock_guard lock(mutex_); + // Redraws, cached snapshots and late completions must not count twice. + if (frame.id <= last_id_) return; + last_id_ = frame.id; + const double ms = + std::chrono::duration(now - frame.capture_time) + .count(); + if (ms < 0 || ms > 5000) return; + samples_.push_back({now, ms}); + Prune(now); + } + + std::optional Get(Clock::time_point now = Clock::now()) { + std::lock_guard lock(mutex_); + Prune(now); + if (samples_.empty()) return std::nullopt; + double sum = 0; + for (const auto& sample : samples_) { + sum += sample.ms; + } + return Snapshot{sum / samples_.size(), samples_.size()}; + } + + void Reset() { + std::lock_guard lock(mutex_); + samples_.clear(); + last_id_ = issued_id_; // Invalidate frames still held by a renderer. + } + + private: + void Prune(Clock::time_point now) { + while (!samples_.empty() && + (now - samples_.front().time > std::chrono::seconds(1) || + samples_.size() > 240)) + samples_.pop_front(); + } + struct Sample { + Clock::time_point time; + double ms; + }; + std::mutex mutex_; + uint64_t issued_id_ = 0; + uint64_t last_id_ = 0; + std::deque samples_; +}; + +inline void VideoLatencyFrame::MarkSubmitted(Clock::time_point now) const { + if (stats) stats->Record(*this, now); +} +} // namespace crossdesk + +#endif \ No newline at end of file diff --git a/apps/desktop/src/gui/runtime/connection_runtime.cpp b/apps/desktop/src/gui/runtime/connection_runtime.cpp index 46aaaba1..6a493fc2 100644 --- a/apps/desktop/src/gui/runtime/connection_runtime.cpp +++ b/apps/desktop/src/gui/runtime/connection_runtime.cpp @@ -419,6 +419,8 @@ void GuiRuntime::ResetRemoteSessionResources( std::shared_ptr props) { { std::lock_guard lock(props->video_frame_mutex_); + props->video_latency_->Reset(); + props->video_frame_timing_ = {}; props->front_frame_.reset(); props->back_frame_.reset(); props->thumbnail_frame_.reset(); diff --git a/apps/desktop/src/gui/runtime/peer_event_handler.cpp b/apps/desktop/src/gui/runtime/peer_event_handler.cpp index ccf3f6e2..297ceaa3 100644 --- a/apps/desktop/src/gui/runtime/peer_event_handler.cpp +++ b/apps/desktop/src/gui/runtime/peer_event_handler.cpp @@ -336,6 +336,8 @@ void PeerEventHandler::OnConnectionStatus(ConnectionStatus status, } { std::lock_guard lock(props->video_frame_mutex_); + props->video_latency_->Reset(); + props->video_frame_timing_ = {}; props->front_frame_.reset(); props->back_frame_.reset(); if (native_snapshot && diff --git a/apps/desktop/src/gui/runtime/peer_media_callbacks.cpp b/apps/desktop/src/gui/runtime/peer_media_callbacks.cpp index 18426a57..05a0cc7d 100644 --- a/apps/desktop/src/gui/runtime/peer_media_callbacks.cpp +++ b/apps/desktop/src/gui/runtime/peer_media_callbacks.cpp @@ -36,7 +36,7 @@ void PeerEventHandler::OnReceiveVideoBuffer( auto callback = handler->EnterCallback(); if (!callback) return; GuiRuntime* runtime = handler ? &handler->owner_ : nullptr; - if (!runtime) { + if (!runtime || !video_frame) { return; } @@ -45,13 +45,18 @@ void PeerEventHandler::OnReceiveVideoBuffer( if (!props) return; if (props->connection_established_) { + const auto timing_now = VideoLatencyFrame::Clock::now(); + const auto timing = props->video_latency_->Frame( + video_frame->captured_timestamp, GetSystemTimeMicros(props->peer_), + timing_now); bool background_snapshot_only = false; auto* native_renderer = runtime->video_renderer_.get(); if (native_renderer && native_renderer->IsActive()) { VideoRenderer::SubmitResult submit_result = VideoRenderer::SubmitResult::failed; if (const auto* native = GetNativeVideoFrame(video_frame)) { - submit_result = native_renderer->SubmitNativeFrame(remote_id, *native); + submit_result = + native_renderer->SubmitNativeFrame(remote_id, *native, timing); // A CPU NV12 native frame is valid MiniRTC input, but the macOS Metal // fast path only accepts CVPixelBuffer. Materialize it when that path // rejects a software decoder's descriptor. This also supports peers @@ -69,14 +74,14 @@ void PeerEventHandler::OnReceiveVideoBuffer( submit_result = native_renderer->SubmitNv12( remote_id, fallback.data(), fallback.size(), static_cast(native->width), - static_cast(native->height)); + static_cast(native->height), timing); } } } else { submit_result = native_renderer->SubmitNv12( remote_id, reinterpret_cast(video_frame->data), - video_frame->size, video_frame->width, video_frame->height); + video_frame->size, video_frame->width, video_frame->height, timing); } if (submit_result == VideoRenderer::SubmitResult::submitted) { std::lock_guard lock(props->video_frame_mutex_); @@ -170,6 +175,7 @@ void PeerEventHandler::OnReceiveVideoBuffer( props->video_size_ = frame_size; props->front_frame_.swap(props->back_frame_); + props->video_frame_timing_ = timing; props->thumbnail_frame_ = props->front_frame_; props->thumbnail_width_ = video_frame->width; props->thumbnail_height_ = video_frame->height; diff --git a/apps/desktop/src/gui/runtime/remote_session.h b/apps/desktop/src/gui/runtime/remote_session.h index 3b8fdc5e..90651f6b 100644 --- a/apps/desktop/src/gui/runtime/remote_session.h +++ b/apps/desktop/src/gui/runtime/remote_session.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,7 @@ #include "display_info.h" #include "cursor_presentation.h" #include "minirtc.h" +#include "rendering/video_latency.h" namespace crossdesk { class PeerEventHandler; } @@ -115,6 +117,9 @@ struct RemoteSession { int selected_display_ = 0; size_t video_size_ = 0; uint64_t video_frame_sequence_ = 0; + VideoLatencyFrame video_frame_timing_; + std::shared_ptr video_latency_ = + std::make_shared(); bool tab_selected_ = false; bool tab_opened_ = true; std::string remote_host_name_; @@ -150,6 +155,8 @@ struct RemoteSession { int fps_ = 0; int frame_count_ = 0; std::chrono::steady_clock::time_point last_time_; + // UI-thread snapshot refreshed with FPS once per second. + std::optional video_latency_snapshot_; MiniRtcNetTrafficStats net_traffic_stats_{}; using QueuedFile = FileTransferState::QueuedFile; diff --git a/apps/desktop/src/gui/ui/stream_window.slint b/apps/desktop/src/gui/ui/stream_window.slint index d7642578..0d9b2d8b 100644 --- a/apps/desktop/src/gui/ui/stream_window.slint +++ b/apps/desktop/src/gui/ui/stream_window.slint @@ -56,6 +56,7 @@ export global StreamStrings { in-out property stats-out: "Out"; in-out property stats-loss-rate: "Loss Rate"; in-out property stats-resolution: "Res"; + in-out property stats-video-latency: "Video delay"; in-out property stats-connection-mode: "Mode"; in-out property stats-encryption: "Encryption"; in-out property stats-encryption-enabled: "Enabled"; @@ -218,6 +219,7 @@ export component StreamWindow inherits Window { in property <[NetworkStatsRow]> stats-rows; in property stats-fps: "0"; in property stats-resolution: ""; + in property stats-video-latency: "—"; in property stats-connection-mode: ""; in property stats-encryption-enabled: false; in property file-transfer-visible: false; @@ -253,7 +255,7 @@ export component StreamWindow inherits Window { private property stats-row-height: 18px; private property stats-header-height: 18px; private property stats-content-height: - root.stats-header-height + root.stats-rows.length * root.stats-row-height + 44px; + root.stats-header-height + root.stats-rows.length * root.stats-row-height + 65px; private property display-menu-open: false; private property display-menu-animation-enabled: false; private property display-menu-progress: 0.0; @@ -1026,6 +1028,29 @@ export component StreamWindow inherits Window { } } + HorizontalLayout { + x: 6px; + y: root.stats-header-height + root.stats-rows.length * stats-table.row-height + 44px; + width: parent.width - 12px; + height: 21px; + spacing: 6px; + Text { + text: StreamStrings.stats-video-latency; + color: #667085; + font-size: ImGuiFontStyle.small; + vertical-alignment: center; + } + Text { + text: root.stats-video-latency; + color: #253247; + font-size: ImGuiFontStyle.body; + font-weight: 600; + vertical-alignment: center; + horizontal-stretch: 1; + overflow: elide; + } + } + for metric[index] in [ { label: "FPS", value: root.stats-fps }, { label: StreamStrings.stats-resolution, value: root.stats-resolution }, diff --git a/apps/desktop/src/gui/ui/ui_localization.h b/apps/desktop/src/gui/ui/ui_localization.h index a61a52d4..bf24b407 100644 --- a/apps/desktop/src/gui/ui/ui_localization.h +++ b/apps/desktop/src/gui/ui/ui_localization.h @@ -175,6 +175,7 @@ inline int ApplyStreamWindowStrings( strings.set_stats_out(Text(localization::out[language])); strings.set_stats_loss_rate(Text(localization::loss_rate[language])); strings.set_stats_resolution(Text(localization::resolution[language])); + strings.set_stats_video_latency(Text(localization::video_latency[language])); strings.set_stats_connection_mode( Text(localization::connection_mode[language])); strings.set_stats_encryption( diff --git a/apps/desktop/src/platform/common/gui/opengl_video_renderer.cpp b/apps/desktop/src/platform/common/gui/opengl_video_renderer.cpp index b9284b3a..860e06df 100644 --- a/apps/desktop/src/platform/common/gui/opengl_video_renderer.cpp +++ b/apps/desktop/src/platform/common/gui/opengl_video_renderer.cpp @@ -59,6 +59,7 @@ struct FrameSlot { int height = 0; std::string remote_id; uint64_t sequence = 0; + VideoLatencyFrame timing; SlotUse use = SlotUse::available; bool valid = false; }; @@ -320,6 +321,7 @@ struct OpenGlVideoRenderer::Impl { bool unpack_subimage_available = false; std::string uploaded_stream; uint64_t uploaded_sequence = 0; + VideoLatencyFrame uploaded_timing; #if CROSSDESK_OPENGL_CUDA_INTEROP enum class CudaUploadResult { @@ -1023,25 +1025,22 @@ void OpenGlVideoRenderer::DiscardStream(std::string_view remote_id) { } } -OpenGlVideoRenderer::SubmitResult -OpenGlVideoRenderer::SubmitNv12(std::string_view remote_id, - const uint8_t *data, size_t size, - int width, int height) { - return SubmitNv12Internal(remote_id, data, size, width, height, true); +OpenGlVideoRenderer::SubmitResult OpenGlVideoRenderer::SubmitNv12( + std::string_view remote_id, const uint8_t *data, size_t size, int width, + int height, VideoLatencyFrame timing) { + return SubmitNv12Internal(remote_id, data, size, width, height, true, timing); } -OpenGlVideoRenderer::SubmitResult -OpenGlVideoRenderer::SubmitCachedNv12(std::string_view remote_id, - const uint8_t *data, size_t size, - int width, int height) { - return SubmitNv12Internal(remote_id, data, size, width, height, false); +OpenGlVideoRenderer::SubmitResult OpenGlVideoRenderer::SubmitCachedNv12( + std::string_view remote_id, const uint8_t *data, size_t size, int width, + int height, VideoLatencyFrame timing) { + return SubmitNv12Internal(remote_id, data, size, width, height, false, + timing); } -OpenGlVideoRenderer::SubmitResult -OpenGlVideoRenderer::SubmitNv12Internal(std::string_view remote_id, - const uint8_t *data, size_t size, - int width, int height, - bool replace_pending) { +OpenGlVideoRenderer::SubmitResult OpenGlVideoRenderer::SubmitNv12Internal( + std::string_view remote_id, const uint8_t *data, size_t size, int width, + int height, bool replace_pending, VideoLatencyFrame timing) { if (!IsReady()) { return SubmitResult::failed; } @@ -1104,14 +1103,15 @@ OpenGlVideoRenderer::SubmitNv12Internal(std::string_view remote_id, target->height = height; target->remote_id.assign(remote_id); target->sequence = frames.next_sequence++; + target->timing = timing; target->use = SlotUse::pending; target->valid = true; return SubmitResult::submitted; } -OpenGlVideoRenderer::SubmitResult -OpenGlVideoRenderer::SubmitNativeFrame(std::string_view remote_id, - const MiniRtcNativeVideoFrame &frame) { +OpenGlVideoRenderer::SubmitResult OpenGlVideoRenderer::SubmitNativeFrame( + std::string_view remote_id, const MiniRtcNativeVideoFrame &frame, + VideoLatencyFrame timing) { if (!IsReady()) { return SubmitResult::failed; } @@ -1183,6 +1183,7 @@ OpenGlVideoRenderer::SubmitNativeFrame(std::string_view remote_id, target->height = static_cast(native->height); target->remote_id.assign(remote_id); target->sequence = frames.next_sequence++; + target->timing = timing; target->use = SlotUse::pending; target->valid = true; return SubmitResult::submitted; @@ -1204,6 +1205,7 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, const unsigned char *y_data = nullptr; const unsigned char *uv_data = nullptr; NativeVideoFrameRef active_native_frame; + VideoLatencyFrame timing; { std::lock_guard lock(impl_->frames->mutex); for (size_t index = 0; index < impl_->frames->slots.size(); ++index) { @@ -1221,6 +1223,7 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, source_width = selected.width; source_height = selected.height; active_native_frame = selected.native_frame; + timing = selected.timing; if (!selected.bytes.empty()) { const size_t y_size = static_cast(source_width) * source_height; y_data = selected.bytes.data(); @@ -1421,6 +1424,7 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, impl_->texture_height = source_height; impl_->uploaded_stream.assign(remote_id); impl_->uploaded_sequence = sequence; + impl_->uploaded_timing = timing; if (!use_pixel_unpack_buffer) { impl_->cpu_texture_width = source_width; impl_->cpu_texture_height = source_height; @@ -1549,6 +1553,8 @@ OpenGlVideoRenderer::RenderLatest(std::string_view remote_id, } #endif const GLenum draw_error = glGetError(); + if (has_video && draw_error == GL_NO_ERROR) + impl_->uploaded_timing.MarkSubmitted(); RenderOutcome outcome{ !upload_succeeded || draw_error != GL_NO_ERROR ? RenderResult::failed diff --git a/apps/desktop/src/platform/common/gui/opengl_video_renderer.h b/apps/desktop/src/platform/common/gui/opengl_video_renderer.h index 04b7f4dc..242d5893 100644 --- a/apps/desktop/src/platform/common/gui/opengl_video_renderer.h +++ b/apps/desktop/src/platform/common/gui/opengl_video_renderer.h @@ -39,11 +39,14 @@ public: // Thread-safe; called from MiniRTC's decode callback thread. SubmitResult SubmitNv12(std::string_view remote_id, const uint8_t *data, - size_t size, int width, int height) override; + size_t size, int width, int height, + VideoLatencyFrame timing = {}) override; SubmitResult SubmitCachedNv12(std::string_view remote_id, const uint8_t *data, - size_t size, int width, int height) override; + size_t size, int width, int height, + VideoLatencyFrame timing = {}) override; SubmitResult SubmitNativeFrame(std::string_view remote_id, - const MiniRtcNativeVideoFrame &frame) override; + const MiniRtcNativeVideoFrame &frame, + VideoLatencyFrame timing = {}) override; // Draws below Slint while its OpenGL context is current. All dimensions are // physical pixels in the window client area. corner_radius_pixels clips the @@ -61,7 +64,8 @@ public: private: SubmitResult SubmitNv12Internal(std::string_view remote_id, const uint8_t *data, size_t size, int width, - int height, bool replace_pending); + int height, bool replace_pending, + VideoLatencyFrame timing); struct Impl; std::unique_ptr impl_; diff --git a/apps/desktop/src/platform/macos/gui/metal_video_renderer.h b/apps/desktop/src/platform/macos/gui/metal_video_renderer.h index d34a2e25..2549a5f7 100644 --- a/apps/desktop/src/platform/macos/gui/metal_video_renderer.h +++ b/apps/desktop/src/platform/macos/gui/metal_video_renderer.h @@ -35,18 +35,20 @@ class MacMetalVideoRenderer final : public VideoRenderer { // Copies one tightly packed NV12 frame into a free shared Metal slot. SubmitResult SubmitNv12(std::string_view remote_id, const uint8_t* data, - size_t size, int width, int height) override; + size_t size, int width, int height, + VideoLatencyFrame timing = {}) override; // Retains an IOSurface-backed VideoToolbox CVPixelBuffer until the Metal // command buffer using it has completed. SubmitResult SubmitNativeFrame(std::string_view remote_id, - const MiniRtcNativeVideoFrame& frame) override; + const MiniRtcNativeVideoFrame& frame, + VideoLatencyFrame timing = {}) override; // Seeds a newly selected stream from its retained CPU snapshot without // replacing a newer decoded frame that is already queued or rendering. - SubmitResult SubmitCachedNv12(std::string_view remote_id, - const uint8_t* data, size_t size, int width, - int height) override; + SubmitResult SubmitCachedNv12(std::string_view remote_id, const uint8_t* data, + size_t size, int width, int height, + VideoLatencyFrame timing = {}) override; // Attaches a native CAMetalLayer-backed sibling below Slint's NSView and // restores the containing AppKit window's ordinary opaque titlebar. @@ -76,7 +78,8 @@ class MacMetalVideoRenderer final : public VideoRenderer { private: SubmitResult SubmitNv12Internal(std::string_view remote_id, const uint8_t* data, size_t size, int width, - int height, bool replace_pending); + int height, bool replace_pending, + VideoLatencyFrame timing); struct Impl; std::unique_ptr impl_; diff --git a/apps/desktop/src/platform/macos/gui/metal_video_renderer.mm b/apps/desktop/src/platform/macos/gui/metal_video_renderer.mm index edf3be5b..45b5b739 100644 --- a/apps/desktop/src/platform/macos/gui/metal_video_renderer.mm +++ b/apps/desktop/src/platform/macos/gui/metal_video_renderer.mm @@ -340,6 +340,7 @@ struct FrameSlot { int height = 0; std::string remote_id; uint64_t sequence = 0; + VideoLatencyFrame timing; SlotUse use = SlotUse::available; bool valid = false; @@ -744,14 +745,16 @@ void MacMetalVideoRenderer::DiscardStream(std::string_view remote_id) { } } -MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNv12( - std::string_view remote_id, const uint8_t* data, size_t size, int width, - int height) { - return SubmitNv12Internal(remote_id, data, size, width, height, true); +MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNv12(std::string_view remote_id, + const uint8_t* data, + size_t size, int width, + int height, + VideoLatencyFrame timing) { + return SubmitNv12Internal(remote_id, data, size, width, height, true, timing); } MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNativeFrame( - std::string_view remote_id, const MiniRtcNativeVideoFrame& frame) { + std::string_view remote_id, const MiniRtcNativeVideoFrame& frame, VideoLatencyFrame timing) { if (!IsReady()) { return SubmitResult::failed; } @@ -781,8 +784,7 @@ MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNativeFrame( packed_nv12.size()) != 0) { return SubmitResult::failed; } - return SubmitNv12(remote_id, packed_nv12.data(), packed_nv12.size(), width, - height); + return SubmitNv12(remote_id, packed_nv12.data(), packed_nv12.size(), width, height, timing); }; CVPixelBufferRef pixel_buffer = @@ -878,6 +880,7 @@ MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNativeFrame( target->height = height; target->remote_id.assign(remote_id); target->sequence = frames.next_sequence++; + target->timing = timing; target->use = SlotUse::pending; target->valid = true; if (!impl_->native_path_logged.exchange(true, std::memory_order_acq_rel)) { @@ -887,14 +890,14 @@ MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNativeFrame( } MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitCachedNv12( - std::string_view remote_id, const uint8_t* data, size_t size, int width, - int height) { - return SubmitNv12Internal(remote_id, data, size, width, height, false); + std::string_view remote_id, const uint8_t* data, size_t size, int width, int height, + VideoLatencyFrame timing) { + return SubmitNv12Internal(remote_id, data, size, width, height, false, timing); } MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNv12Internal( - std::string_view remote_id, const uint8_t* data, size_t size, int width, - int height, bool replace_pending) { + std::string_view remote_id, const uint8_t* data, size_t size, int width, int height, + bool replace_pending, VideoLatencyFrame timing) { if (!IsReady()) { return SubmitResult::failed; } @@ -968,6 +971,7 @@ MacMetalVideoRenderer::SubmitResult MacMetalVideoRenderer::SubmitNv12Internal( target->remote_id.assign(remote_id); target->sequence = frames.next_sequence++; + target->timing = timing; target->use = SlotUse::pending; target->valid = true; return SubmitResult::submitted; @@ -1189,6 +1193,7 @@ MacMetalVideoRenderer::RenderOutcome MacMetalVideoRenderer::RenderLatest( id y_texture = nil; id uv_texture = nil; + VideoLatencyFrame timing; { std::lock_guard lock(impl_->frames->mutex); auto& slot = impl_->frames->slots[slot_index]; @@ -1202,6 +1207,7 @@ MacMetalVideoRenderer::RenderOutcome MacMetalVideoRenderer::RenderLatest( slot.use = SlotUse::in_flight; y_texture = slot.y_texture; uv_texture = slot.uv_texture; + timing = slot.timing; } MTLRenderPassDescriptor* pass = [MTLRenderPassDescriptor renderPassDescriptor]; @@ -1261,6 +1267,7 @@ MacMetalVideoRenderer::RenderOutcome MacMetalVideoRenderer::RenderLatest( }]; [command_buffer presentDrawable:drawable]; [command_buffer commit]; + timing.MarkSubmitted(); impl_->rendered_stream.assign(remote_id); impl_->rendered_content_generation = content_generation; impl_->native_surface_needs_redraw->store(false, diff --git a/apps/desktop/src/platform/video_renderer.h b/apps/desktop/src/platform/video_renderer.h index 70b3d088..7b5c27c9 100644 --- a/apps/desktop/src/platform/video_renderer.h +++ b/apps/desktop/src/platform/video_renderer.h @@ -9,6 +9,7 @@ #include #include "minirtc.h" +#include "rendering/video_latency.h" namespace crossdesk { @@ -54,14 +55,17 @@ public: virtual SubmitResult SubmitNv12(std::string_view remote_id, const uint8_t* data, size_t size, int width, - int height) = 0; + int height, + VideoLatencyFrame timing = {}) = 0; virtual SubmitResult SubmitCachedNv12(std::string_view remote_id, const uint8_t* data, size_t size, - int width, int height) = 0; + int width, int height, + VideoLatencyFrame timing = {}) = 0; // Retains a platform-native decoded frame when the backend supports its // storage and handle types. Unsupported backends keep the default fallback. virtual SubmitResult SubmitNativeFrame(std::string_view, - const MiniRtcNativeVideoFrame&) { + const MiniRtcNativeVideoFrame&, + VideoLatencyFrame = {}) { return SubmitResult::failed; } diff --git a/deps/submodules/minirtc b/deps/submodules/minirtc index 19c31f2c..a25eff65 160000 --- a/deps/submodules/minirtc +++ b/deps/submodules/minirtc @@ -1 +1 @@ -Subproject commit 19c31f2c7b0b72918147ff9029e7869583da2eac +Subproject commit a25eff65158c56c7ca5be5ef1df2b1e5d77e7dbf