[feat] add compact mobile virtual mouse controls

apps/ios/CrossDeskMobile/Input/RemoteMouseInput.swift

@@ -0,0 +1,59 @@
+import Foundation
+
+/// Shared by the video surface and the virtual buttons. Coordinates always
+/// refer to the remote cursor, never to the floating controls' screen location.
+final class RemoteMouseInput {
+    var send: ((Float, Float, Int, Int) -> Void)?
+    var didCancel: (() -> Void)?
+    var viewportSize = CGSize(width: 0, height: 0)
+    private(set) var x: Float = 0.5
+    private(set) var y: Float = 0.5
+    private(set) var revision: UInt64 = 0
+    private var gestureLeft = false
+    private var virtualLeft = false
+    private var virtualRight = false
+    var hasVirtualButton: Bool { virtualLeft || virtualRight }
+
+    func position(x: Float, y: Float) {
+        self.x = min(max(x, 0), 1)
+        self.y = min(max(y, 0), 1)
+    }
+
+    func drag(dx: Float, dy: Float) {
+        guard hasVirtualButton, viewportSize.width > 0, viewportSize.height > 0 else { return }
+        position(x: x + dx / Float(viewportSize.width) * 1.35,
+                 y: y + dy / Float(viewportSize.height) * 1.35)
+        emit(0)
+    }
+
+    func setGestureLeft(_ down: Bool) {
+        guard !(down && hasVirtualButton), gestureLeft != down else { return }
+        gestureLeft = down
+        emit(down ? 1 : 2)
+    }
+
+    func setVirtualButton(left: Bool, down: Bool) {
+        guard (left ? virtualLeft : virtualRight) != down else { return }
+        if down {
+            setGestureLeft(false)
+            revision &+= 1
+        }
+        if left { virtualLeft = down } else { virtualRight = down }
+        emit(left ? (down ? 1 : 2) : (down ? 3 : 4))
+    }
+
+    func scroll(_ direction: Int) {
+        guard direction != 0 else { return }
+        emit(7, wheel: direction > 0 ? 1 : -1)
+    }
+
+    func releaseAll() {
+        setGestureLeft(false)
+        setVirtualButton(left: true, down: false)
+        setVirtualButton(left: false, down: false)
+        revision &+= 1
+        didCancel?()
+    }
+
+    private func emit(_ flag: Int, wheel: Int = 0) { send?(x, y, flag, wheel) }
+}

apps/ios/CrossDeskMobile/Input/RemoteTouchInputView.swift

@@ -55,6 +55,7 @@ enum RemoteVideoGeometry {
 }
 
 struct RemoteTouchInputView: UIViewRepresentable {
+    let mouseInput: RemoteMouseInput
     let videoSize: CGSize
     let controlMode: MouseControlMode
     let remoteCursorPosition: CGPoint?
@@ -73,6 +74,7 @@ struct RemoteTouchInputView: UIViewRepresentable {
     }
 
     func updateUIView(_ view: RemoteTouchSurface, context: Context) {
+        view.mouseInput = mouseInput
         view.videoSize = videoSize
         view.controlMode = controlMode
         view.setViewport(scale: viewportScale, offset: viewportOffset)
@@ -88,6 +90,8 @@ struct RemoteTouchInputView: UIViewRepresentable {
 }
 
 final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
+    var mouseInput: RemoteMouseInput?
+    private var gestureRevisions: [ObjectIdentifier: UInt64] = [:]
     var videoSize = CGSize.zero {
         didSet { setNeedsLayout() }
     }
@@ -108,7 +112,14 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
     var onScroll: ((Float, Float, Int, Int) -> Void)?
     var onViewportChanged: ((CGFloat, CGSize) -> Void)?
 
-    private var relativeCursorPoint: (Float, Float) = (0.5, 0.5)
+    private var fallbackCursorPoint: (Float, Float) = (0.5, 0.5)
+    private var relativeCursorPoint: (Float, Float) {
+        get { mouseInput.map { ($0.x, $0.y) } ?? fallbackCursorPoint }
+        set {
+            fallbackCursorPoint = newValue
+            mouseInput?.position(x: newValue.0, y: newValue.1)
+        }
+    }
     private var lastRemoteCursorPoint: (Float, Float)?
     private var lastRemoteCursorRevision: UInt64?
     private var relativePanActive = false
@@ -128,10 +139,12 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
         isMultipleTouchEnabled = true
 
         let tap = UITapGestureRecognizer(target: self, action: #selector(handleTap(_:)))
+        tap.delegate = self
         tap.numberOfTouchesRequired = 1
         addGestureRecognizer(tap)
 
         let rightTap = UITapGestureRecognizer(target: self, action: #selector(handleRightTap(_:)))
+        rightTap.delegate = self
         rightTap.numberOfTouchesRequired = 2
         tap.require(toFail: rightTap)
         addGestureRecognizer(rightTap)
@@ -174,6 +187,7 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
 
     override func layoutSubviews() {
         super.layoutSubviews()
+        mouseInput?.viewportSize = renderRect()?.size ?? .zero
         let constrained = constrainedViewportOffset(viewportOffset,
                                                      scale: viewportScale)
         guard constrained != viewportOffset else { return }
@@ -185,6 +199,7 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
         let nextScale = min(max(scale, 1), maximumViewportScale)
         viewportScale = nextScale
         viewportOffset = constrainedViewportOffset(offset, scale: nextScale)
+        mouseInput?.viewportSize = renderRect()?.size ?? .zero
     }
 
     func synchronizeRemoteCursor(_ position: CGPoint?, revision: UInt64) {
@@ -203,12 +218,39 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
         )
         lastRemoteCursorPoint = point
         guard controlMode == .relative,
+              mouseInput?.hasVirtualButton != true,
               !relativePanActive,
               heldDragPoint == nil,
               hoverLastLocation == nil else { return }
         relativeCursorPoint = point
     }
 
+    func gestureRecognizer(_ gestureRecognizer: UIGestureRecognizer,
+                           shouldReceive touch: UITouch) -> Bool {
+        if gestureRecognizer is UITapGestureRecognizer || gestureRecognizer is UILongPressGestureRecognizer {
+            guard mouseInput?.hasVirtualButton != true else { return false }
+            gestureRevisions[ObjectIdentifier(gestureRecognizer)] = mouseInput?.revision ?? 0
+        }
+        return true
+    }
+
+    private func allowsPointerGesture(_ recognizer: UIGestureRecognizer) -> Bool {
+        mouseInput?.hasVirtualButton != true &&
+            gestureRevisions[ObjectIdentifier(recognizer)] == (mouseInput?.revision ?? 0)
+    }
+
+    override func gestureRecognizerShouldBegin(_ gestureRecognizer: UIGestureRecognizer) -> Bool {
+        if gestureRecognizer is UITapGestureRecognizer || gestureRecognizer is UILongPressGestureRecognizer {
+            // Decline recognition before a stale long press can cancel the pan
+            // used by the other finger to move a held virtual mouse button.
+            return allowsPointerGesture(gestureRecognizer)
+        }
+        if gestureRecognizer is UIPinchGestureRecognizer {
+            return mouseInput?.hasVirtualButton != true
+        }
+        return true
+    }
+
     func gestureRecognizer(_ gestureRecognizer: UIGestureRecognizer,
                            shouldRecognizeSimultaneouslyWith otherGestureRecognizer: UIGestureRecognizer) -> Bool {
         let recognizers = [gestureRecognizer, otherGestureRecognizer]
@@ -223,6 +265,7 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
     }
 
     @objc private func handleTap(_ recognizer: UITapGestureRecognizer) {
+        guard allowsPointerGesture(recognizer) else { return }
         let location = recognizer.location(in: self)
         guard recognizer.state == .ended,
               let point = pointerPoint(for: location) else { return }
@@ -240,6 +283,7 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
     }
 
     @objc private func handleRightTap(_ recognizer: UITapGestureRecognizer) {
+        guard allowsPointerGesture(recognizer) else { return }
         guard recognizer.state == .ended,
               let point = pointerPoint(for: recognizer.location(in: self)) else { return }
         onMove?(point.0, point.1)
@@ -262,6 +306,12 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
     }
 
     @objc private func handleHeldDrag(_ recognizer: UILongPressGestureRecognizer) {
+        guard allowsPointerGesture(recognizer) else {
+            // A virtual button took ownership while the surface gesture was in flight.
+            heldDragPoint = nil
+            heldDragLastLocation = nil
+            return
+        }
         if controlMode == .relative {
             handleRelativeHeldDrag(recognizer)
             return
@@ -289,6 +339,7 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
     }
 
     @objc private func handlePinch(_ recognizer: UIPinchGestureRecognizer) {
+        guard mouseInput?.hasVirtualButton != true else { pinchActive = false; return }
         let focus = recognizer.location(in: self)
         switch recognizer.state {
         case .began:
@@ -327,6 +378,7 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
     }
 
     @objc private func handleWheel(_ recognizer: UIPanGestureRecognizer) {
+        guard mouseInput?.hasVirtualButton != true else { viewportPanActive = false; return }
         if viewportScale > 1.001 || pinchActive {
             switch recognizer.state {
             case .began:

apps/ios/CrossDeskMobile/Input/VirtualMouseBar.swift

@@ -0,0 +1,209 @@
+import SwiftUI
+import UIKit
+
+struct VirtualMouseBar: UIViewRepresentable {
+    let input: RemoteMouseInput
+
+    func makeUIView(context: Context) -> VirtualMouseBarView {
+        VirtualMouseBarView(input: input)
+    }
+
+    func updateUIView(_ view: VirtualMouseBarView, context: Context) {}
+
+    static func dismantleUIView(_ view: VirtualMouseBarView, coordinator: ()) {
+        view.input.releaseAll()
+    }
+}
+
+final class VirtualMouseBarView: UIView {
+    let input: RemoteMouseInput
+    private let keys: [VirtualMouseKey]
+
+    init(input: RemoteMouseInput) {
+        self.input = input
+        keys = [VirtualMouseKey(kind: .left, input: input),
+                VirtualMouseKey(kind: .wheel, input: input),
+                VirtualMouseKey(kind: .right, input: input)]
+        super.init(frame: .zero)
+        backgroundColor = .white
+        layer.cornerRadius = 18
+        layer.borderWidth = 0.5
+        layer.borderColor = UIColor(red: 212 / 255, green: 218 / 255, blue: 226 / 255, alpha: 1).cgColor
+        isMultipleTouchEnabled = true
+        keys.forEach(addSubview)
+        input.didCancel = { [weak self] in self?.keys.forEach { $0.cancelTracking() } }
+    }
+
+    required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") }
+
+    override func layoutSubviews() {
+        super.layoutSubviews()
+        let width = max(0, bounds.width - 8) / 3
+        for (index, key) in keys.enumerated() {
+            key.frame = CGRect(x: 4 + CGFloat(index) * width, y: 2,
+                               width: width, height: max(0, bounds.height - 4))
+        }
+    }
+}
+
+private final class VirtualMouseKey: UIView {
+    enum Kind { case left, wheel, right }
+    private let kind: Kind
+    private let input: RemoteMouseInput
+    private let face = UIView()
+    private let faceMask = CAShapeLayer()
+    private let wheelIcon = VirtualMouseWheelView()
+    private var trackedTouch: UITouch?
+    private var lastPoint = CGPoint.zero
+    private var remainder: CGFloat = 0
+
+    init(kind: Kind, input: RemoteMouseInput) {
+        self.kind = kind
+        self.input = input
+        super.init(frame: .zero)
+        isMultipleTouchEnabled = false
+        isAccessibilityElement = true
+        accessibilityTraits = kind == .wheel ? [.adjustable] : [.button]
+        accessibilityLabel = kind == .left ? "鼠标左键" : kind == .right ? "鼠标右键" : "鼠标滚轮"
+        accessibilityHint = kind == .wheel ? "上下滑动，可滑出按键范围" : "按住并拖动，可滑出按键范围，松开结束拖拽"
+        face.isUserInteractionEnabled = false
+        face.layer.mask = faceMask
+        addSubview(face)
+        wheelIcon.isHidden = kind != .wheel
+        wheelIcon.isUserInteractionEnabled = false
+        addSubview(wheelIcon)
+        highlight(false)
+    }
+
+    required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") }
+    override func layoutSubviews() {
+        super.layoutSubviews()
+        // All three controls keep a 44 × 44 pt hit area; only the wheel's face
+        // narrows to 18 pt. Empty gutters separate it from the two paddles.
+        face.frame = kind == .wheel
+            ? CGRect(x: (bounds.width - 18) / 2, y: (bounds.height - 30) / 2, width: 18, height: 30)
+            : bounds.insetBy(dx: 1, dy: 2)
+        let leftRadius: CGFloat = kind == .left ? 14 : 6
+        let rightRadius: CGFloat = kind == .right ? 14 : 6
+        faceMask.frame = face.bounds
+        faceMask.path = facePath(in: face.bounds, leftRadius: leftRadius, rightRadius: rightRadius).cgPath
+        wheelIcon.frame = face.frame.insetBy(dx: 3, dy: 3)
+    }
+
+    private func facePath(in rect: CGRect, leftRadius: CGFloat, rightRadius: CGFloat) -> UIBezierPath {
+        let path = UIBezierPath()
+        let left = min(leftRadius, rect.height / 2, rect.width / 2)
+        let right = min(rightRadius, rect.height / 2, rect.width / 2)
+        path.move(to: CGPoint(x: rect.minX + left, y: rect.minY))
+        path.addLine(to: CGPoint(x: rect.maxX - right, y: rect.minY))
+        path.addArc(withCenter: CGPoint(x: rect.maxX - right, y: rect.minY + right),
+                    radius: right, startAngle: -.pi / 2, endAngle: 0, clockwise: true)
+        path.addLine(to: CGPoint(x: rect.maxX, y: rect.maxY - right))
+        path.addArc(withCenter: CGPoint(x: rect.maxX - right, y: rect.maxY - right),
+                    radius: right, startAngle: 0, endAngle: .pi / 2, clockwise: true)
+        path.addLine(to: CGPoint(x: rect.minX + left, y: rect.maxY))
+        path.addArc(withCenter: CGPoint(x: rect.minX + left, y: rect.maxY - left),
+                    radius: left, startAngle: .pi / 2, endAngle: .pi, clockwise: true)
+        path.addLine(to: CGPoint(x: rect.minX, y: rect.minY + left))
+        path.addArc(withCenter: CGPoint(x: rect.minX + left, y: rect.minY + left),
+                    radius: left, startAngle: .pi, endAngle: .pi * 1.5, clockwise: true)
+        path.close()
+        return path
+    }
+
+    override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) {
+        guard trackedTouch == nil, let touch = touches.first else { return }
+        trackedTouch = touch
+        lastPoint = touch.location(in: window)
+        remainder = 0
+        highlight(true)
+        if kind != .wheel { input.setVirtualButton(left: kind == .left, down: true) }
+    }
+
+    override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) {
+        guard let touch = trackedTouch, touches.contains(touch) else { return }
+        let point = touch.location(in: window)
+        let delta = CGPoint(x: point.x - lastPoint.x, y: point.y - lastPoint.y)
+        lastPoint = point
+        if kind != .wheel {
+            input.drag(dx: Float(delta.x), dy: Float(delta.y))
+            return
+        }
+        wheelIcon.advance(by: delta.y)
+        remainder += delta.y
+        let ticks = Int(remainder / 10)
+        if ticks != 0 {
+            // UIKit keeps delivering this touch outside the 44 pt key. Finger
+            // down maps to negative wheel units (scroll down) on the desktop.
+            for _ in 0..<min(abs(ticks), 32) { input.scroll(ticks > 0 ? -1 : 1) }
+            remainder -= CGFloat(ticks) * 10
+        }
+    }
+
+    override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) {
+        guard let touch = trackedTouch, touches.contains(touch) else { return }
+        cancelTracking()
+    }
+
+    override func touchesCancelled(_ touches: Set<UITouch>, with event: UIEvent?) { cancelTracking() }
+
+    func cancelTracking() {
+        if trackedTouch != nil, kind != .wheel {
+            input.setVirtualButton(left: kind == .left, down: false)
+        }
+        trackedTouch = nil
+        remainder = 0
+        highlight(false)
+    }
+
+    override func accessibilityActivate() -> Bool {
+        if kind == .wheel { accessibilityIncrement() }
+        else {
+            input.setVirtualButton(left: kind == .left, down: true)
+            input.setVirtualButton(left: kind == .left, down: false)
+        }
+        return true
+    }
+    override func accessibilityIncrement() {
+        if kind == .wheel { wheelIcon.advance(by: 10); input.scroll(-1) }
+    }
+    override func accessibilityDecrement() {
+        if kind == .wheel { wheelIcon.advance(by: -10); input.scroll(1) }
+    }
+
+    private func highlight(_ pressed: Bool) {
+        face.backgroundColor = pressed ? UIColor(red: 46 / 255, green: 122 / 255, blue: 219 / 255, alpha: 1)
+            : UIColor(red: 237 / 255, green: 240 / 255, blue: 244 / 255, alpha: 1)
+        wheelIcon.strokeColor = pressed ? .white
+            : UIColor(red: 48 / 255, green: 54 / 255, blue: 64 / 255, alpha: 0.72)
+    }
+}
+
+/// A clipped conveyor of wheel treads: follows touch movement and stops on lift.
+private final class VirtualMouseWheelView: UIView {
+    var strokeColor = UIColor.black { didSet { setNeedsDisplay() } }
+    private var phase: CGFloat = 2
+    private let pitch: CGFloat = 5
+
+    init() {
+        super.init(frame: .zero)
+        isOpaque = false
+        backgroundColor = .clear
+    }
+    required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") }
+
+    func advance(by distance: CGFloat) {
+        phase = (phase + distance * 0.65).truncatingRemainder(dividingBy: pitch)
+        setNeedsDisplay()
+    }
+
+    override func draw(_ rect: CGRect) {
+        guard let context = UIGraphicsGetCurrentContext() else { return }
+        UIBezierPath(roundedRect: bounds, cornerRadius: 2).addClip()
+        context.setFillColor(strokeColor.cgColor)
+        for index in -1...6 {
+            let y = CGFloat(index) * pitch + phase
+            context.fill(CGRect(x: 0, y: y, width: bounds.width, height: 1.5))
+        }
+    }
+}