[feat] keep mobile sessions connected in background

2026-11-04 13:02:39 +08:00 · 2026-10-10 00:29:07 +08:00
parent be639eaf97
commit 1dccbe42a3
17 changed files with 693 additions and 37 deletions
@@ -22,6 +22,10 @@
 	<string>$(CURRENT_PROJECT_VERSION)</string>
    <key>LSRequiresIPhoneOS</key>
    <true/>
+    <key>UIBackgroundModes</key>
+    <array>
+        <string>audio</string>
+    </array>
    <key>NSLocalNetworkUsageDescription</key>
    <string>CrossDesk 使用局域网发现并建立原生 P2P 远程控制连接。</string>
 	<key>UIApplicationSupportsIndirectInputEvents</key>
@@ -239,6 +239,16 @@ private final class RemoteAudioPlayer {
        }
    }

+    func endSession() {
+        queue.async {
+            self.enabled = false
+            self.player.stop()
+            self.engine.stop()
+            self.queuedBuffers = 0
+            try? AVAudioSession.sharedInstance().setActive(false, options: [.notifyOthersOnDeactivation])
+        }
+    }
+
    func enqueue(_ data: Data) {
        guard !data.isEmpty, data.count.isMultiple(of: MemoryLayout<Int16>.size) else { return }
        queue.async {
@@ -339,6 +349,7 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe
    @Published private(set) var remotePlatform: RemoteHostPlatform = .unknown

    let bridge = CrossDeskRTCBridge()
+    let pictureInPicture = RemotePictureInPicture()
    let announcements = AnnouncementInbox()
    let appUpdates = AppUpdateChecker()
    private var appIsActive = false
@@ -392,6 +403,22 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe
    override init() {
        super.init()
        bridge.delegate = self
+        pictureInPicture.requestFrame = { [weak self] in
+            guard let self, self.isConnected else { return }
+            self.bridge.requestKeyFrame()
+        }
+        pictureInPicture.restoreInterface = { [weak self] in
+            guard let self, self.isConnected else { return false }
+            self.sessionVisible = true
+            return true
+        }
+        pictureInPicture.playbackChanged = { [weak self] playing in
+            guard let self, self.isConnected else { return }
+            self.audioPlayer.setEnabled(playing && self.audioEnabled)
+        }
+        pictureInPicture.failed = { [weak self] in
+            self?.controlStatus = "画中画未能开启，请保持应用在前台后重试。"
+        }
        announcements.send = { [weak bridge] data, requestID in
            bridge?.sendAnnouncementRequest(data, requestID: requestID)
        }
@@ -408,6 +435,7 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe

    func applicationDidBecomeActive() {
        appIsActive = true
+        pictureInPicture.didBecomeActive()
        appUpdates.setEnabled(hasNetworkConsent)
        privacyNotice.didBecomeActive(hasNetworkConsent: hasNetworkConsent)
        refreshRecentConnectionPresenceAfterForeground()
@@ -656,6 +684,7 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe
    }

    private func resetConnection() {
+        pictureInPicture.endSession()
        clipboardStatus = ""
        controlStatus = ""
        transferStatus = ""
@@ -682,7 +711,7 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe
        displaySizes = []
        selectedDisplay = 0
        connectionStatus = "未连接"
-        audioPlayer.setEnabled(false)
+        audioPlayer.endSession()
        AppOrientation.update(to: .portrait)
        appUpdates.checkNow()
    }
@@ -1124,6 +1153,7 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe
        case 1:
            connectionStatus = "已连接"
            sessionVisible = true
+            pictureInPicture.beginSession()
            if videoSettings.requestID == 0 { updateVideoSettings() }
            checkRemoteVersionIfNeeded()
            recordSuccessfulConnectionIfNeeded()
@@ -1226,7 +1256,16 @@ final class RemoteSessionModel: NSObject, ObservableObject, CrossDeskRTCBridgeDe
                   width: Int,
                   height: Int,
                   captureUptime: TimeInterval) {
-        guard hasNetworkConsent, isConnected, !videoWasBackgrounded else { return }
+        guard hasNetworkConsent, isConnected else { return }
+        if videoWasBackgrounded {
+            // Feed PiP directly, without publishing background SwiftUI updates.
+            if pictureInPicture.display(pixelBuffer) {
+                videoFrameID &+= 1
+                networkStatistics.recordSubmittedFrame(id: videoFrameID, captureUptime: captureUptime,
+                                                       at: ProcessInfo.processInfo.systemUptime)
+            }
+            return
+        }
        cancelVideoRecovery()
        // Buffer and encoded dimensions must be one observable value. Adaptive
        // resolution changes must never expose a new frame with the previous
@@ -60,7 +60,8 @@ struct RemoteSessionView: View {
                                pixelBuffer: session.pixelBuffer,
                                frameID: session.currentVideoFrameID,
                                captureUptime: session.frameCaptureUptime,
-                                onFrameSubmitted: session.recordVideoFrameSubmission
+                                onFrameSubmitted: session.recordVideoFrameSubmission,
+                                pictureInPicture: session.pictureInPicture
                            )

                            if let cursorPosition,
@@ -1,4 +1,5 @@
 import AVFoundation
+import AVKit
 import CoreMedia
 import CoreVideo
 import SwiftUI
@@ -15,16 +16,158 @@ struct NativeVideoView: UIViewRepresentable {
    let frameID: UInt64
    let captureUptime: TimeInterval
    let onFrameSubmitted: (UInt64, TimeInterval) -> Void
+    let pictureInPicture: RemotePictureInPicture

    func makeUIView(context: Context) -> SampleBufferVideoView {
-        SampleBufferVideoView(frame: .zero)
+        let view = SampleBufferVideoView(frame: .zero)
+        pictureInPicture.attach(view)
+        return view
    }

    func updateUIView(_ uiView: SampleBufferVideoView, context: Context) {
-        if uiView.display(pixelBuffer, isActive: scenePhase == .active) {
+        // Background frames go straight from the RTC delegate to the display
+        // layer: SwiftUI does not promise view updates while backgrounded.
+        guard scenePhase != .background, !pictureInPicture.isPaused else { return }
+        if uiView.display(pixelBuffer, isActive: true) {
            onFrameSubmitted(frameID, captureUptime)
        }
    }
+
+    static func dismantleUIView(_ uiView: SampleBufferVideoView, coordinator: ()) {
+        _ = uiView.display(nil, isActive: false)
+    }
+}
+
+/// Uses actual live remote video for background playback, including when muted.
+/// Closing/pausing PiP removes that background execution opportunity; it does
+/// not fabricate silent audio or repeatedly request background task time.
+final class RemotePictureInPicture: NSObject, AVPictureInPictureControllerDelegate,
+                                    AVPictureInPictureSampleBufferPlaybackDelegate {
+    private var controller: AVPictureInPictureController?
+    private var videoView: SampleBufferVideoView?
+    private var sessionActive = false
+    private var starting = false
+    private var paused = false
+    var requestFrame: (() -> Void)?
+    var restoreInterface: (() -> Bool)?
+    var playbackChanged: ((Bool) -> Void)?
+    var failed: (() -> Void)?
+
+    var isRendering: Bool {
+        sessionActive && !paused && (starting || controller?.isPictureInPictureActive == true)
+    }
+    var isPaused: Bool { paused && controller?.isPictureInPictureActive == true }
+
+    func attach(_ view: SampleBufferVideoView) {
+        guard videoView !== view else { return }
+        controller?.stopPictureInPicture()
+        controller?.delegate = nil
+        videoView = view
+        guard AVPictureInPictureController.isPictureInPictureSupported() else { return }
+        let source = AVPictureInPictureController.ContentSource(
+            sampleBufferDisplayLayer: view.videoLayer, playbackDelegate: self)
+        let controller = AVPictureInPictureController(contentSource: source)
+        controller.delegate = self
+        controller.requiresLinearPlayback = true
+        controller.canStartPictureInPictureAutomaticallyFromInline = sessionActive
+        self.controller = controller
+    }
+
+    func beginSession() {
+        guard !sessionActive else { return }
+        sessionActive = true
+        paused = false
+        // PiP needs an active playback session even when the remote is muted.
+        // There is no generated audio: only received desktop audio is played.
+        do {
+            let audio = AVAudioSession.sharedInstance()
+            try audio.setCategory(.playback, mode: .moviePlayback, options: [.mixWithOthers])
+            try audio.setActive(true)
+        } catch {
+            NSLog("CrossDesk could not activate Picture in Picture audio session: %@", error.localizedDescription)
+        }
+        controller?.canStartPictureInPictureAutomaticallyFromInline = true
+        controller?.invalidatePlaybackState()
+    }
+
+    func endSession() {
+        sessionActive = false
+        starting = false
+        paused = false
+        controller?.canStartPictureInPictureAutomaticallyFromInline = false
+        controller?.stopPictureInPicture()
+        controller?.invalidatePlaybackState()
+        _ = videoView?.display(nil, isActive: false)
+    }
+
+    func didBecomeActive() {
+        paused = false
+        if controller?.isPictureInPictureActive == true { controller?.stopPictureInPicture() }
+        controller?.invalidatePlaybackState()
+        playbackChanged?(true)
+    }
+
+    @discardableResult
+    func display(_ buffer: CVPixelBuffer) -> Bool {
+        guard isRendering else { return false }
+        return videoView?.display(buffer, isActive: true) ?? false
+    }
+
+    func pictureInPictureControllerWillStartPictureInPicture(_ controller: AVPictureInPictureController) {
+        guard self.controller === controller, sessionActive else { return }
+        starting = true
+        requestFrame?()
+    }
+
+    func pictureInPictureControllerDidStartPictureInPicture(_ controller: AVPictureInPictureController) {
+        guard self.controller === controller else { return }
+        starting = false
+        if !sessionActive { controller.stopPictureInPicture() }
+        else { requestFrame?() }
+    }
+
+    func pictureInPictureController(_ controller: AVPictureInPictureController,
+                                   failedToStartPictureInPictureWithError error: Error) {
+        guard self.controller === controller else { return }
+        starting = false
+        NSLog("CrossDesk Picture in Picture could not start: %@", error.localizedDescription)
+        failed?()
+    }
+
+    func pictureInPictureControllerDidStopPictureInPicture(_ controller: AVPictureInPictureController) {
+        guard self.controller === controller else { return }
+        starting = false
+        paused = false
+        playbackChanged?(true)
+    }
+
+    func pictureInPictureController(_ controller: AVPictureInPictureController,
+                                   restoreUserInterfaceForPictureInPictureStopWithCompletionHandler completion: @escaping (Bool) -> Void) {
+        completion(sessionActive && (restoreInterface?() ?? false))
+    }
+
+    func pictureInPictureController(_ controller: AVPictureInPictureController, setPlaying playing: Bool) {
+        paused = !playing
+        playbackChanged?(playing)
+        if playing { requestFrame?() }
+        controller.invalidatePlaybackState()
+    }
+
+    func pictureInPictureControllerTimeRangeForPlayback(_ controller: AVPictureInPictureController) -> CMTimeRange {
+        sessionActive ? CMTimeRange(start: .zero, duration: .positiveInfinity) : .invalid
+    }
+
+    func pictureInPictureControllerIsPlaybackPaused(_ controller: AVPictureInPictureController) -> Bool {
+        !sessionActive || paused
+    }
+
+    func pictureInPictureController(_ controller: AVPictureInPictureController,
+                                   didTransitionToRenderSize newRenderSize: CMVideoDimensions) { }
+
+    func pictureInPictureController(_ controller: AVPictureInPictureController,
+                                   skipByInterval skipInterval: CMTime, completion: @escaping () -> Void) {
+        completion() // Live desktop video has no seekable history.
+    }
 }

 final class SampleBufferVideoView: UIView {
@@ -32,7 +175,7 @@ final class SampleBufferVideoView: UIView {
        AVSampleBufferDisplayLayer.self
    }

-    private var videoLayer: AVSampleBufferDisplayLayer {
+    var videoLayer: AVSampleBufferDisplayLayer {
        layer as! AVSampleBufferDisplayLayer
    }

@@ -60,6 +203,14 @@ final class SampleBufferVideoView: UIView {
        // calculation inside AVSampleBufferDisplayLayer. That rounding becomes
        // visibly amplified when the remote desktop is zoomed up to 10x.
        videoLayer.videoGravity = .resize
+        var timebase: CMTimebase?
+        if CMTimebaseCreateWithSourceClock(allocator: kCFAllocatorDefault,
+                                          sourceClock: CMClockGetHostTimeClock(),
+                                          timebaseOut: &timebase) == noErr, let timebase {
+            CMTimebaseSetTime(timebase, time: CMClockGetTime(CMClockGetHostTimeClock()))
+            CMTimebaseSetRate(timebase, rate: 1)
+            videoLayer.controlTimebase = timebase
+        }
    }

    /// True only when a new buffer was submitted; no presentation timestamp is
@@ -123,7 +274,7 @@ final class SampleBufferVideoView: UIView {

        var timing = CMSampleTimingInfo(
            duration: .invalid,
-            presentationTimeStamp: .invalid,
+            presentationTimeStamp: CMClockGetTime(CMClockGetHostTimeClock()),
            decodeTimeStamp: .invalid
        )
        var sampleBuffer: CMSampleBuffer?