[feat] add native iOS client support

2026-08-31 13:32:10 +08:00 · 2026-08-31 02:14:27 +08:00
parent 126f455681
commit 7595ab8bd8
50 changed files with 7391 additions and 432 deletions
@@ -0,0 +1,151 @@
+import AVFoundation
+import CoreMedia
+import CoreVideo
+import SwiftUI
+import UIKit
+
+/// Presents decoded NV12 frames through iOS's native video display path.
+///
+/// `AVSampleBufferDisplayLayer` handles YUV conversion and drawable scheduling
+/// itself. This avoids the Core Image -> MTKView path, which can remain black
+/// on a physical device even though VideoToolbox is producing valid frames.
+struct NativeVideoView: UIViewRepresentable {
+    let pixelBuffer: CVPixelBuffer?
+
+    func makeUIView(context: Context) -> SampleBufferVideoView {
+        SampleBufferVideoView(frame: .zero)
+    }
+
+    func updateUIView(_ uiView: SampleBufferVideoView, context: Context) {
+        uiView.display(pixelBuffer)
+    }
+}
+
+final class SampleBufferVideoView: UIView {
+    override class var layerClass: AnyClass {
+        AVSampleBufferDisplayLayer.self
+    }
+
+    private var videoLayer: AVSampleBufferDisplayLayer {
+        layer as! AVSampleBufferDisplayLayer
+    }
+
+    private var lastPixelBuffer: CVPixelBuffer?
+    private var submittedFrames: UInt64 = 0
+    private var droppedFrames: UInt64 = 0
+
+    override init(frame: CGRect) {
+        super.init(frame: frame)
+        configureLayer()
+    }
+
+    required init?(coder: NSCoder) {
+        super.init(coder: coder)
+        configureLayer()
+    }
+
+    private func configureLayer() {
+        backgroundColor = .black
+        isOpaque = true
+        videoLayer.backgroundColor = UIColor.black.cgColor
+        // The SwiftUI host gives this view the exact aspect-fit video rect.
+        // Filling that rect avoids a second, independently rounded aspect-fit
+        // calculation inside AVSampleBufferDisplayLayer. That rounding becomes
+        // visibly amplified when the remote desktop is zoomed up to 10x.
+        videoLayer.videoGravity = .resize
+    }
+
+    func display(_ pixelBuffer: CVPixelBuffer?) {
+        guard let pixelBuffer else {
+            lastPixelBuffer = nil
+            submittedFrames = 0
+            droppedFrames = 0
+            videoLayer.flushAndRemoveImage()
+            return
+        }
+
+        // SwiftUI can refresh for unrelated status fields. Do not enqueue the
+        // same retained CVPixelBuffer more than once.
+        guard lastPixelBuffer !== pixelBuffer else { return }
+        lastPixelBuffer = pixelBuffer
+
+        if videoLayer.status == .failed {
+            NSLog("CrossDesk video layer failed: %@",
+                  videoLayer.error?.localizedDescription ?? "unknown error")
+            videoLayer.flush()
+        }
+
+        // Never let AVSampleBufferDisplayLayer turn temporary rendering
+        // pressure into seconds of latency. Its queued buffers are already
+        // stale by the time it stops accepting more data, so discard them and
+        // present the newest decoded IOSurface instead.
+        if !videoLayer.isReadyForMoreMediaData {
+            droppedFrames &+= 1
+            videoLayer.flush()
+            if droppedFrames == 1 || droppedFrames.isMultiple(of: 300) {
+                NSLog("CrossDesk dropped stale display frames: %llu",
+                      droppedFrames)
+            }
+        }
+        guard videoLayer.isReadyForMoreMediaData else { return }
+
+        var formatDescription: CMVideoFormatDescription?
+        guard CMVideoFormatDescriptionCreateForImageBuffer(
+            allocator: kCFAllocatorDefault,
+            imageBuffer: pixelBuffer,
+            formatDescriptionOut: &formatDescription
+        ) == noErr, let formatDescription else {
+            NSLog("CrossDesk could not create a video format description")
+            return
+        }
+
+        var timing = CMSampleTimingInfo(
+            duration: .invalid,
+            presentationTimeStamp: .invalid,
+            decodeTimeStamp: .invalid
+        )
+        var sampleBuffer: CMSampleBuffer?
+        guard CMSampleBufferCreateReadyWithImageBuffer(
+            allocator: kCFAllocatorDefault,
+            imageBuffer: pixelBuffer,
+            formatDescription: formatDescription,
+            sampleTiming: &timing,
+            sampleBufferOut: &sampleBuffer
+        ) == noErr, let sampleBuffer else {
+            NSLog("CrossDesk could not create a video sample buffer")
+            return
+        }
+
+        if let attachments = CMSampleBufferGetSampleAttachmentsArray(
+            sampleBuffer,
+            createIfNecessary: true
+        ), CFArrayGetCount(attachments) > 0 {
+            let attachment = unsafeBitCast(
+                CFArrayGetValueAtIndex(attachments, 0),
+                to: CFMutableDictionary.self
+            )
+            CFDictionarySetValue(
+                attachment,
+                Unmanaged.passUnretained(kCMSampleAttachmentKey_DisplayImmediately).toOpaque(),
+                Unmanaged.passUnretained(kCFBooleanTrue).toOpaque()
+            )
+        }
+
+        videoLayer.enqueue(sampleBuffer)
+        submittedFrames &+= 1
+        if submittedFrames == 1 || submittedFrames.isMultiple(of: 300) {
+            NSLog("CrossDesk presented video frame %llu (%zu x %zu)",
+                  submittedFrames,
+                  CVPixelBufferGetWidth(pixelBuffer),
+                  CVPixelBufferGetHeight(pixelBuffer))
+        }
+        if submittedFrames == 1 {
+            DispatchQueue.main.asyncAfter(deadline: .now() + 0.25) { [weak self] in
+                guard let self else { return }
+                NSLog("CrossDesk video layer status=%ld error=%@",
+                      self.videoLayer.status.rawValue,
+                      self.videoLayer.error?.localizedDescription ?? "none")
+            }
+        }
+    }
+}