diff --git a/apps/ios/CrossDeskMobile/Bridge/CrossDeskRTCBridge.h b/apps/ios/CrossDeskMobile/Bridge/CrossDeskRTCBridge.h index b0a2ffd8..4539180a 100644 --- a/apps/ios/CrossDeskMobile/Bridge/CrossDeskRTCBridge.h +++ b/apps/ios/CrossDeskMobile/Bridge/CrossDeskRTCBridge.h @@ -111,6 +111,7 @@ typedef NS_ENUM(NSInteger, CrossDeskVideoAdaptationPolicy) { - (void)connectToRemoteID:(NSString *)remoteID password:(NSString *)password; - (void)disconnect; +- (void)requestKeyFrame; - (void)sendPointerAtX:(float)x y:(float)y diff --git a/apps/ios/CrossDeskMobile/Bridge/CrossDeskRTCBridge.mm b/apps/ios/CrossDeskMobile/Bridge/CrossDeskRTCBridge.mm index 12c86546..0d080531 100644 --- a/apps/ios/CrossDeskMobile/Bridge/CrossDeskRTCBridge.mm +++ b/apps/ios/CrossDeskMobile/Bridge/CrossDeskRTCBridge.mm @@ -233,7 +233,6 @@ void DispatchMain(dispatch_block_t block) { - (void)createControllerPeer; - (void)destroyIdentityPeer; - (void)destroyControllerPeer; -- (void)requestKeyFrame; @end namespace { diff --git a/apps/ios/CrossDeskMobile/ContentView.swift b/apps/ios/CrossDeskMobile/ContentView.swift index 87328e30..a91a7415 100644 --- a/apps/ios/CrossDeskMobile/ContentView.swift +++ b/apps/ios/CrossDeskMobile/ContentView.swift @@ -23,12 +23,15 @@ struct ContentView: View { } if scenePhase == .active { session.refreshRecentConnectionPresenceAfterForeground() + session.resumeVideoAfterForeground() } } .onChange(of: scenePhase) { phase in if phase == .active { session.refreshRecentConnectionPresenceAfterForeground() + session.resumeVideoAfterForeground() } else if phase == .background { + session.suspendVideoForBackground() session.suspendPresenceMonitoring() } } diff --git a/apps/ios/CrossDeskMobile/RemoteSessionModel.swift b/apps/ios/CrossDeskMobile/RemoteSessionModel.swift index d3af8546..2ef08dfe 100644 --- a/apps/ios/CrossDeskMobile/RemoteSessionModel.swift +++ b/apps/ios/CrossDeskMobile/RemoteSessionModel.swift @@ -408,6 +408,8 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe private var shouldCaptureThumbnail = false private var activeDisplayName = "" private var remoteCursorSequence: UInt32? + private var videoWasBackgrounded = false + private var videoRecoveryTask: Task? private var pendingPresenceRemoteID: String? private var presenceProbeGeneration: UInt64 = 0 private var signalConnected = false @@ -503,6 +505,7 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe } private func beginRemoteConnection(_ identifier: String) { + cancelVideoRecovery() cancelPresenceProbe() activeRemoteID = identifier activeDisplayName = "" @@ -577,6 +580,7 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe } func disconnect() { + cancelVideoRecovery() let wasCheckingPresence = pendingPresenceRemoteID != nil cancelPresenceProbe() if wasCheckingPresence { @@ -599,6 +603,7 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe } func retry() { + cancelVideoRecovery() bridge.disconnect() connect() } @@ -690,6 +695,41 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe cancelPendingPresenceConnection() } + func suspendVideoForBackground() { + videoWasBackgrounded = true + cancelVideoRecovery() + } + + func resumeVideoAfterForeground() { + guard videoWasBackgrounded else { return } + videoWasBackgrounded = false + guard isConnected, sessionVisible else { return } + + cancelVideoRecovery() + NSLog("CrossDesk resuming foreground video, requesting key frame") + bridge.requestKeyFrame() + // The first request can race transport resumption. Retry briefly until + // a decoded frame arrives, and never carry retries into another session. + videoRecoveryTask = Task { @MainActor [weak self] in + for delay in [500_000_000, 1_000_000_000, 2_000_000_000] as [UInt64] { + do { + try await Task.sleep(nanoseconds: delay) + } catch { + return + } + guard !Task.isCancelled, let self, + !self.videoWasBackgrounded, + self.isConnected, self.sessionVisible else { return } + self.bridge.requestKeyFrame() + } + } + } + + private func cancelVideoRecovery() { + videoRecoveryTask?.cancel() + videoRecoveryTask = nil + } + func refreshRecentConnectionPresenceAfterForeground() { presenceIsForeground = true invalidatePresence() @@ -846,6 +886,9 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe didChange state: CrossDeskConnectionState, remoteID: String) { isConnected = state.rawValue == 1 + if !isConnected { + cancelVideoRecovery() + } isConnecting = [0, 2].contains(state.rawValue) switch state.rawValue { case 0: connectionStatus = "正在建立远程连接" @@ -934,6 +977,8 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe didReceive pixelBuffer: CVPixelBuffer, width: Int, height: Int) { + guard !videoWasBackgrounded else { return } + cancelVideoRecovery() // Buffer and encoded dimensions must be one observable value. Adaptive // resolution changes must never expose a new frame with the previous // frame's geometry to SwiftUI. diff --git a/apps/ios/CrossDeskMobile/Rendering/NativeMetalVideoView.swift b/apps/ios/CrossDeskMobile/Rendering/NativeMetalVideoView.swift index 9fcfdabf..d5e5b51d 100644 --- a/apps/ios/CrossDeskMobile/Rendering/NativeMetalVideoView.swift +++ b/apps/ios/CrossDeskMobile/Rendering/NativeMetalVideoView.swift @@ -10,6 +10,7 @@ import UIKit /// itself. This avoids the Core Image -> MTKView path, which can remain black /// on a physical device even though VideoToolbox is producing valid frames. struct NativeVideoView: UIViewRepresentable { + @Environment(\.scenePhase) private var scenePhase let pixelBuffer: CVPixelBuffer? func makeUIView(context: Context) -> SampleBufferVideoView { @@ -17,7 +18,7 @@ struct NativeVideoView: UIViewRepresentable { } func updateUIView(_ uiView: SampleBufferVideoView, context: Context) { - uiView.display(pixelBuffer) + uiView.display(pixelBuffer, isActive: scenePhase == .active) } } @@ -31,6 +32,7 @@ final class SampleBufferVideoView: UIView { } private var lastPixelBuffer: CVPixelBuffer? + private var renderingActive = false private var submittedFrames: UInt64 = 0 private var droppedFrames: UInt64 = 0 @@ -55,7 +57,16 @@ final class SampleBufferVideoView: UIView { videoLayer.videoGravity = .resize } - func display(_ pixelBuffer: CVPixelBuffer?) { + func display(_ pixelBuffer: CVPixelBuffer?, isActive: Bool) { + if renderingActive != isActive { + renderingActive = isActive + // Backgrounding can invalidate the display layer without changing + // the retained frame. Clear the queue and allow that frame to be + // submitted again when the scene becomes active. + lastPixelBuffer = nil + videoLayer.flushAndRemoveImage() + } + guard let pixelBuffer else { lastPixelBuffer = nil submittedFrames = 0 @@ -63,18 +74,21 @@ final class SampleBufferVideoView: UIView { videoLayer.flushAndRemoveImage() return } + guard renderingActive else { return } - // SwiftUI can refresh for unrelated status fields. Do not enqueue the - // same retained CVPixelBuffer more than once. - guard lastPixelBuffer !== pixelBuffer else { return } - lastPixelBuffer = pixelBuffer - - if videoLayer.status == .failed { + // Check recovery before deduplicating frames: a failed layer may need + // to display the same buffer again even if no new decoded frame arrives. + if videoLayer.status == .failed || videoLayer.requiresFlushToResumeDecoding { NSLog("CrossDesk video layer failed: %@", videoLayer.error?.localizedDescription ?? "unknown error") videoLayer.flush() + lastPixelBuffer = nil } + // SwiftUI can refresh for unrelated status fields. Only deduplicate a + // frame after it was actually submitted to a healthy, active layer. + guard lastPixelBuffer !== pixelBuffer else { return } + // Never let AVSampleBufferDisplayLayer turn temporary rendering // pressure into seconds of latency. Its queued buffers are already // stale by the time it stops accepting more data, so discard them and @@ -132,6 +146,7 @@ final class SampleBufferVideoView: UIView { } videoLayer.enqueue(sampleBuffer) + lastPixelBuffer = pixelBuffer submittedFrames &+= 1 if submittedFrames == 1 || submittedFrames.isMultiple(of: 300) { NSLog("CrossDesk presented video frame %llu (%zu x %zu)", diff --git a/deps/submodules/minirtc b/deps/submodules/minirtc index 6c395894..ababf526 160000 --- a/deps/submodules/minirtc +++ b/deps/submodules/minirtc @@ -1 +1 @@ -Subproject commit 6c39589408fbf6cecc1f563a1f3c0b7d94068a3e +Subproject commit ababf526fb86116925c0e35480cb89e23c6bff00