[feat] keep mobile sessions connected in background

apps/ios/CrossDeskMobile/Info.plist

@@ -22,6 +22,10 @@
 	<string>$(CURRENT_PROJECT_VERSION)</string>
     <key>LSRequiresIPhoneOS</key>
     <true/>
+    <key>UIBackgroundModes</key>
+    <array>
+        <string>audio</string>
+    </array>
     <key>NSLocalNetworkUsageDescription</key>
     <string>CrossDesk 使用局域网发现并建立原生 P2P 远程控制连接。</string>
 	<key>UIApplicationSupportsIndirectInputEvents</key>

apps/ios/CrossDeskMobile/RemoteSessionModel.swift

@@ -239,6 +239,16 @@ private final class RemoteAudioPlayer {
         }
     }
 
+    func endSession() {
+        queue.async {
+            self.enabled = false
+            self.player.stop()
+            self.engine.stop()
+            self.queuedBuffers = 0
+            try? AVAudioSession.sharedInstance().setActive(false, options: [.notifyOthersOnDeactivation])
+        }
+    }
+
     func enqueue(_ data: Data) {
         guard !data.isEmpty, data.count.isMultiple(of: MemoryLayout<Int16>.size) else { return }
         queue.async {
@@ -339,6 +349,7 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe
     @Published private(set) var remotePlatform: RemoteHostPlatform = .unknown
 
     let bridge = CrossDeskRTCBridge()
+    let pictureInPicture = RemotePictureInPicture()
     let announcements = AnnouncementInbox()
     let appUpdates = AppUpdateChecker()
     private var appIsActive = false
@@ -392,6 +403,22 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe
     override init() {
         super.init()
         bridge.delegate = self
+        pictureInPicture.requestFrame = { [weak self] in
+            guard let self, self.isConnected else { return }
+            self.bridge.requestKeyFrame()
+        }
+        pictureInPicture.restoreInterface = { [weak self] in
+            guard let self, self.isConnected else { return false }
+            self.sessionVisible = true
+            return true
+        }
+        pictureInPicture.playbackChanged = { [weak self] playing in
+            guard let self, self.isConnected else { return }
+            self.audioPlayer.setEnabled(playing && self.audioEnabled)
+        }
+        pictureInPicture.failed = { [weak self] in
+            self?.controlStatus = "画中画未能开启，请保持应用在前台后重试。"
+        }
         announcements.send = { [weak bridge] data, requestID in
             bridge?.sendAnnouncementRequest(data, requestID: requestID)
         }
@@ -408,6 +435,7 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe
 
     func applicationDidBecomeActive() {
         appIsActive = true
+        pictureInPicture.didBecomeActive()
         appUpdates.setEnabled(hasNetworkConsent)
         privacyNotice.didBecomeActive(hasNetworkConsent: hasNetworkConsent)
         refreshRecentConnectionPresenceAfterForeground()
@@ -656,6 +684,7 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe
     }
 
     private func resetConnection() {
+        pictureInPicture.endSession()
         clipboardStatus = ""
         controlStatus = ""
         transferStatus = ""
@@ -682,7 +711,7 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe
         displaySizes = []
         selectedDisplay = 0
         connectionStatus = "未连接"
-        audioPlayer.setEnabled(false)
+        audioPlayer.endSession()
         AppOrientation.update(to: .portrait)
         appUpdates.checkNow()
     }
@@ -1124,6 +1153,7 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe
         case 1:
             connectionStatus = "已连接"
             sessionVisible = true
+            pictureInPicture.beginSession()
             if videoSettings.requestID == 0 { updateVideoSettings() }
             checkRemoteVersionIfNeeded()
             recordSuccessfulConnectionIfNeeded()
@@ -1226,7 +1256,16 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe
                    width: Int,
                    height: Int,
                    captureUptime: TimeInterval) {
-        guard hasNetworkConsent, isConnected, !videoWasBackgrounded else { return }
+        guard hasNetworkConsent, isConnected else { return }
+        if videoWasBackgrounded {
+            // Feed PiP directly, without publishing background SwiftUI updates.
+            if pictureInPicture.display(pixelBuffer) {
+                videoFrameID &+= 1
+                networkStatistics.recordSubmittedFrame(id: videoFrameID, captureUptime: captureUptime,
+                                                       at: ProcessInfo.processInfo.systemUptime)
+            }
+            return
+        }
         cancelVideoRecovery()
         // Buffer and encoded dimensions must be one observable value. Adaptive
         // resolution changes must never expose a new frame with the previous

apps/ios/CrossDeskMobile/RemoteSessionView.swift

@@ -60,7 +60,8 @@ struct RemoteSessionView: View {
                                 pixelBuffer: session.pixelBuffer,
                                 frameID: session.currentVideoFrameID,
                                 captureUptime: session.frameCaptureUptime,
-                                onFrameSubmitted: session.recordVideoFrameSubmission
+                                onFrameSubmitted: session.recordVideoFrameSubmission,
+                                pictureInPicture: session.pictureInPicture
                             )
 
                             if let cursorPosition,

apps/ios/CrossDeskMobile/Rendering/NativeMetalVideoView.swift

@@ -1,4 +1,5 @@
 import AVFoundation
+import AVKit
 import CoreMedia
 import CoreVideo
 import SwiftUI
@@ -15,16 +16,158 @@ struct NativeVideoView: UIViewRepresentable {
     let frameID: UInt64
     let captureUptime: TimeInterval
     let onFrameSubmitted: (UInt64, TimeInterval) -> Void
+    let pictureInPicture: RemotePictureInPicture
 
     func makeUIView(context: Context) -> SampleBufferVideoView {
-        SampleBufferVideoView(frame: .zero)
+        let view = SampleBufferVideoView(frame: .zero)
+        pictureInPicture.attach(view)
+        return view
     }
 
     func updateUIView(_ uiView: SampleBufferVideoView, context: Context) {
-        if uiView.display(pixelBuffer, isActive: scenePhase == .active) {
+        // Background frames go straight from the RTC delegate to the display
+        // layer: SwiftUI does not promise view updates while backgrounded.
+        guard scenePhase != .background, !pictureInPicture.isPaused else { return }
+        if uiView.display(pixelBuffer, isActive: true) {
             onFrameSubmitted(frameID, captureUptime)
         }
     }
+
+    static func dismantleUIView(_ uiView: SampleBufferVideoView, coordinator: ()) {
+        _ = uiView.display(nil, isActive: false)
+    }
+}
+
+/// Uses actual live remote video for background playback, including when muted.
+/// Closing/pausing PiP removes that background execution opportunity; it does
+/// not fabricate silent audio or repeatedly request background task time.
+final class RemotePictureInPicture: NSObject, AVPictureInPictureControllerDelegate,
+                                    AVPictureInPictureSampleBufferPlaybackDelegate {
+    private var controller: AVPictureInPictureController?
+    private var videoView: SampleBufferVideoView?
+    private var sessionActive = false
+    private var starting = false
+    private var paused = false
+    var requestFrame: (() -> Void)?
+    var restoreInterface: (() -> Bool)?
+    var playbackChanged: ((Bool) -> Void)?
+    var failed: (() -> Void)?
+
+    var isRendering: Bool {
+        sessionActive && !paused && (starting || controller?.isPictureInPictureActive == true)
+    }
+    var isPaused: Bool { paused && controller?.isPictureInPictureActive == true }
+
+    func attach(_ view: SampleBufferVideoView) {
+        guard videoView !== view else { return }
+        controller?.stopPictureInPicture()
+        controller?.delegate = nil
+        videoView = view
+        guard AVPictureInPictureController.isPictureInPictureSupported() else { return }
+        let source = AVPictureInPictureController.ContentSource(
+            sampleBufferDisplayLayer: view.videoLayer, playbackDelegate: self)
+        let controller = AVPictureInPictureController(contentSource: source)
+        controller.delegate = self
+        controller.requiresLinearPlayback = true
+        controller.canStartPictureInPictureAutomaticallyFromInline = sessionActive
+        self.controller = controller
+    }
+
+    func beginSession() {
+        guard !sessionActive else { return }
+        sessionActive = true
+        paused = false
+        // PiP needs an active playback session even when the remote is muted.
+        // There is no generated audio: only received desktop audio is played.
+        do {
+            let audio = AVAudioSession.sharedInstance()
+            try audio.setCategory(.playback, mode: .moviePlayback, options: [.mixWithOthers])
+            try audio.setActive(true)
+        } catch {
+            NSLog("CrossDesk could not activate Picture in Picture audio session: %@", error.localizedDescription)
+        }
+        controller?.canStartPictureInPictureAutomaticallyFromInline = true
+        controller?.invalidatePlaybackState()
+    }
+
+    func endSession() {
+        sessionActive = false
+        starting = false
+        paused = false
+        controller?.canStartPictureInPictureAutomaticallyFromInline = false
+        controller?.stopPictureInPicture()
+        controller?.invalidatePlaybackState()
+        _ = videoView?.display(nil, isActive: false)
+    }
+
+    func didBecomeActive() {
+        paused = false
+        if controller?.isPictureInPictureActive == true { controller?.stopPictureInPicture() }
+        controller?.invalidatePlaybackState()
+        playbackChanged?(true)
+    }
+
+    @discardableResult
+    func display(_ buffer: CVPixelBuffer) -> Bool {
+        guard isRendering else { return false }
+        return videoView?.display(buffer, isActive: true) ?? false
+    }
+
+    func pictureInPictureControllerWillStartPictureInPicture(_ controller: AVPictureInPictureController) {
+        guard self.controller === controller, sessionActive else { return }
+        starting = true
+        requestFrame?()
+    }
+
+    func pictureInPictureControllerDidStartPictureInPicture(_ controller: AVPictureInPictureController) {
+        guard self.controller === controller else { return }
+        starting = false
+        if !sessionActive { controller.stopPictureInPicture() }
+        else { requestFrame?() }
+    }
+
+    func pictureInPictureController(_ controller: AVPictureInPictureController,
+                                   failedToStartPictureInPictureWithError error: Error) {
+        guard self.controller === controller else { return }
+        starting = false
+        NSLog("CrossDesk Picture in Picture could not start: %@", error.localizedDescription)
+        failed?()
+    }
+
+    func pictureInPictureControllerDidStopPictureInPicture(_ controller: AVPictureInPictureController) {
+        guard self.controller === controller else { return }
+        starting = false
+        paused = false
+        playbackChanged?(true)
+    }
+
+    func pictureInPictureController(_ controller: AVPictureInPictureController,
+                                   restoreUserInterfaceForPictureInPictureStopWithCompletionHandler completion: @escaping (Bool) -> Void) {
+        completion(sessionActive && (restoreInterface?() ?? false))
+    }
+
+    func pictureInPictureController(_ controller: AVPictureInPictureController, setPlaying playing: Bool) {
+        paused = !playing
+        playbackChanged?(playing)
+        if playing { requestFrame?() }
+        controller.invalidatePlaybackState()
+    }
+
+    func pictureInPictureControllerTimeRangeForPlayback(_ controller: AVPictureInPictureController) -> CMTimeRange {
+        sessionActive ? CMTimeRange(start: .zero, duration: .positiveInfinity) : .invalid
+    }
+
+    func pictureInPictureControllerIsPlaybackPaused(_ controller: AVPictureInPictureController) -> Bool {
+        !sessionActive || paused
+    }
+
+    func pictureInPictureController(_ controller: AVPictureInPictureController,
+                                   didTransitionToRenderSize newRenderSize: CMVideoDimensions) { }
+
+    func pictureInPictureController(_ controller: AVPictureInPictureController,
+                                   skipByInterval skipInterval: CMTime, completion: @escaping () -> Void) {
+        completion() // Live desktop video has no seekable history.
+    }
 }
 
 final class SampleBufferVideoView: UIView {
@@ -32,7 +175,7 @@ final class SampleBufferVideoView: UIView {
         AVSampleBufferDisplayLayer.self
     }
 
-    private var videoLayer: AVSampleBufferDisplayLayer {
+    var videoLayer: AVSampleBufferDisplayLayer {
         layer as! AVSampleBufferDisplayLayer
     }
 
@@ -60,6 +203,14 @@ final class SampleBufferVideoView: UIView {
         // calculation inside AVSampleBufferDisplayLayer. That rounding becomes
         // visibly amplified when the remote desktop is zoomed up to 10x.
         videoLayer.videoGravity = .resize
+        var timebase: CMTimebase?
+        if CMTimebaseCreateWithSourceClock(allocator: kCFAllocatorDefault,
+                                          sourceClock: CMClockGetHostTimeClock(),
+                                          timebaseOut: &timebase) == noErr, let timebase {
+            CMTimebaseSetTime(timebase, time: CMClockGetTime(CMClockGetHostTimeClock()))
+            CMTimebaseSetRate(timebase, rate: 1)
+            videoLayer.controlTimebase = timebase
+        }
     }
 
     /// True only when a new buffer was submitted; no presentation timestamp is
@@ -123,7 +274,7 @@ final class SampleBufferVideoView: UIView {
 
         var timing = CMSampleTimingInfo(
             duration: .invalid,
-            presentationTimeStamp: .invalid,
+            presentationTimeStamp: CMClockGetTime(CMClockGetHostTimeClock()),
             decodeTimeStamp: .invalid
         )
         var sampleBuffer: CMSampleBuffer?

apps/ios/README.md

@@ -4,6 +4,34 @@ This target is a native MiniRTC controller. It uses the same WebSocket
 signaling, libnice ICE, SRTP/RTP and data-stream protocol as the desktop app.
 There is no `WKWebView` or browser runtime.
 
+## Background connection and Picture in Picture
+
+An established remote desktop session supports native Picture in Picture (PiP).
+After the first video frame appears, switching to another app automatically
+starts PiP when the device and system settings allow it. The PiP window shows
+the live remote desktop; restoring the app returns to the existing session and
+requests a fresh key frame. Remote mouse buttons are released when the app loses focus.
+
+The app declares the `audio` background mode required by AVKit for PiP and
+activates a playback audio session, including for muted remote video. Background
+frames are submitted directly from the RTC delegate to the sample-buffer display
+layer, without relying on SwiftUI background redraws. Only real remote audio is
+played; no synthetic silent audio or recurring background-task assertions are
+used. Disconnecting or withdrawing consent stops PiP and releases playback.
+
+Background execution depends on active PiP or actual remote audio playback.
+Closing or pausing PiP, disabling automatic PiP in system settings, locking the
+device, losing the network, or force-quitting can suspend or interrupt the
+session. iOS does not provide an unrestricted background socket keepalive.
+The connection is not deliberately closed on background entry, but continuous
+connectivity cannot be promised outside the supported playback modes.
+
+On a physical iPhone/iPad, verify automatic PiP when switching apps with remote
+audio both enabled and muted, continuously changing remote frames while another
+app is in front, restoration, pause/resume, close, lock/unlock, and disconnection while PiP
+is active. Also verify the system setting that disables automatic PiP. A build
+or simulator check alone does not establish long-duration background behavior.
+
 ## Requirements
 
 - Xcode 16 or newer