diff --git a/apps/android/README.md b/apps/android/README.md index 28653993..404e3b45 100644 --- a/apps/android/README.md +++ b/apps/android/README.md @@ -7,7 +7,7 @@ - 官方或自托管 WSS 信令服务器；授权后进入前台即自动注册或登录本机身份，首页实时显示连接、重连及失败状态。发起桌面连接时复用该身份；切换服务器会重新登录，撤回授权会关闭连接。已有远程会话优先通过系统画中画保持连接，画中画不可用时由带连接通知的前台服务兜底；首页空闲信令在后台关闭。 - 优先使用 MediaCodec 硬件解码 H.264 / AV1，硬件缓冲区直接提交 Surface；设备无对应硬件、分辨率超限或运行失败时，按原编码格式回退到 OpenH264 / dav1d。软件输出由独立显示线程将 NV12 转 RGBA；只保留最新待显示帧。MiniRTC 原有的通用压缩帧解码队列保持不变。 - 48 kHz 单声道 PCM 播放，支持静音，音频队列有容量限制。 -- 相对鼠标（触控板）和绝对位置模式；单指点击、绝对模式按住拖动、长按右键，以及外接键盘、鼠标。画面手势与 iOS 一致：双指捏合以触点为中心缩放（1–10 倍），放大后双指拖动画面，原始比例下双指上下滑动控制滚轮。 +- 相对鼠标（触控板）和绝对位置模式；单指点击、绝对模式按住拖动、长按右键，以及外接键盘、鼠标。画面手势与 iOS 一致：双指捏合以触点为中心缩放（1–10 倍），放大后双指拖动画面，原始比例下双指上下滑动控制滚轮。相对鼠标移到可见画面的上、下边缘后继续向外移动，会反向上下平移画面，不触发左右平移；未放大时和按住虚拟鼠标键拖动时同样生效，停止移动即停止平移，抬手保留位置。移动到极限时，画面可见部分至少占当前显示区域高度的 20%，与缩放倍数无关；整幅画面高度不足该比例时完整保留，范围计入上下黑边；缩放结束及缩回 1 倍时保留平移位置，仅在超过可见边界时限制位置；切换显示器恢复居中。 - 横竖屏等比例显示、远端多显示器切换。与 iOS 一致的画面质量（低/中/高）、采集帧率（30/60 fps）、画面偏好；支持能力检测、电脑端确认、失败/5 秒超时回退及连续操作的应答排序。 - 根据常见移动设备解码能力和屏幕尺寸设置起始参数：显示长边不超过 1280 像素时默认低画质（最高 720p）/30 fps，其余默认中画质（最高 1080p）/30 fps。用户保存的画面偏好不变，仍可手动选择高画质或 60 fps。 - 网络状态按视频、音频、数据、合计展示收发速率和丢包率，并显示实际提交帧率、分辨率、画面延时、平滑 RTT、P2P/TURN 和实际 SRTP 状态。每秒刷新，过期或无效测量显示“—”。 diff --git a/apps/android/app/src/main/java/cn/crossdesk/mobile/RemoteMouseInput.java b/apps/android/app/src/main/java/cn/crossdesk/mobile/RemoteMouseInput.java index 42de343a..0031d83f 100644 --- a/apps/android/app/src/main/java/cn/crossdesk/mobile/RemoteMouseInput.java +++ b/apps/android/app/src/main/java/cn/crossdesk/mobile/RemoteMouseInput.java @@ -3,6 +3,8 @@ package cn.crossdesk.mobile; /** Shared button ownership and cursor position for the video surface and mouse bar. */ final class RemoteMouseInput { interface Sender { void send(float x, float y, int flag, int wheel); } + interface RelativeMotion { void move(float dx, float dy); } + RelativeMotion relativeMotion; private final Sender sender; private float x = .5f, y = .5f; private boolean gestureLeft, virtualLeft, virtualRight; @@ -19,7 +21,8 @@ final class RemoteMouseInput { } void drag(float dx, float dy) { if (!hasVirtualButton() || viewportWidth <= 0 || viewportHeight <= 0) return; - move(x + dx / viewportWidth, y + dy / viewportHeight); + if (relativeMotion != null) relativeMotion.move(dx, dy); + else move(x + dx / viewportWidth, y + dy / viewportHeight); } void gestureLeft(boolean down) { if (down && hasVirtualButton()) return; diff --git a/apps/android/app/src/main/java/cn/crossdesk/mobile/RemoteVideoView.java b/apps/android/app/src/main/java/cn/crossdesk/mobile/RemoteVideoView.java index 2a9c7e36..5fad0b57 100644 --- a/apps/android/app/src/main/java/cn/crossdesk/mobile/RemoteVideoView.java +++ b/apps/android/app/src/main/java/cn/crossdesk/mobile/RemoteVideoView.java @@ -51,6 +51,7 @@ final class RemoteVideoView extends FrameLayout implements SurfaceHolder.Callbac RemoteVideoView(Context context, NativeSession session) { super(context); this.session = session; mouseInput = new RemoteMouseInput(this::sendMouse); + mouseInput.relativeMotion = this::moveRelative; setBackgroundColor(Color.BLACK); video = new SurfaceView(context); video.getHolder().addCallback(this); addView(video, new LayoutParams(1, 1, android.view.Gravity.CENTER)); @@ -147,14 +148,21 @@ final class RemoteVideoView extends FrameLayout implements SurfaceHolder.Callbac cursor.invalidate(); } @Override protected void onSizeChanged(int w,int h,int oldw,int oldh) { post(this::fit); } + private void moveRelative(float dx, float dy) { + if (pictureInPicture || viewport.width() <= 0 || viewport.height() <= 0) return; + viewport.panForPointer(y, dy); + applyViewport(); + mouseInput.move(x + dx / viewport.width(), y + dy / viewport.height()); + } private boolean point(MotionEvent e) { boolean touchpad = relative && e.getToolType(0) != MotionEvent.TOOL_TYPE_MOUSE; float px=e.getX(),py=e.getY(); if (!touchpad && !viewport.contains(px,py)) return false; if(touchpad){ if(e.getActionMasked()==MotionEvent.ACTION_MOVE){ - x=Math.max(0,Math.min(1,x+(px-lastX)/Math.max(1,viewport.width()))); - y=Math.max(0,Math.min(1,y+(py-lastY)/Math.max(1,viewport.height()))); + moveRelative(px-lastX,py-lastY); + lastX=px;lastY=py; + return true; } }else{ x=viewport.normalizedX(px); diff --git a/apps/android/app/src/main/java/cn/crossdesk/mobile/RemoteViewport.java b/apps/android/app/src/main/java/cn/crossdesk/mobile/RemoteViewport.java index c848a05d..2ee04c09 100644 --- a/apps/android/app/src/main/java/cn/crossdesk/mobile/RemoteViewport.java +++ b/apps/android/app/src/main/java/cn/crossdesk/mobile/RemoteViewport.java @@ -3,11 +3,14 @@ package cn.crossdesk.mobile; /** Geometry shared by the video Surface, cursor and remote pointer input. */ final class RemoteViewport { private static final float MAXIMUM_SCALE = 10; - private int baseWidth, baseHeight, baseLeft, baseTop; + private static final float MINIMUM_VISIBLE_HEIGHT_FRACTION = .2f; + private int baseWidth, baseHeight, baseLeft, baseTop, containerHeight; + private boolean edgePanned; private float scale = 1, offsetX, offsetY; void fit(int width, int height, int videoWidth, int videoHeight) { if (width <= 0 || height <= 0 || videoWidth <= 0 || videoHeight <= 0) return; + containerHeight = height; double fitScale = Math.min((double) width / videoWidth, (double) height / videoHeight); baseWidth = Math.max(1, (int) (videoWidth * fitScale)); baseHeight = Math.max(1, (int) (videoHeight * fitScale)); @@ -54,18 +57,46 @@ final class RemoteViewport { constrainOffset(); } - void finishGesture() { - if (!isZoomed()) reset(); + void panForPointer(float y, float dy) { + if (baseHeight == 0 || !Float.isFinite(dy)) return; + float overflowY = pointerOverflow(top() + y * height(), dy, + Math.max(0, top()), Math.min(containerHeight, top() + height())); + if (overflowY == 0) return; + edgePanned = true; + panBy(0, -overflowY); } - void reset() { scale = 1; offsetX = offsetY = 0; } + // Only the outward portion of this movement pans; an offscreen cursor must + // not make a tiny movement jump the viewport by its entire hidden distance. + private static float pointerOverflow(float position, float delta, float low, float high) { + if (delta > 0) return Math.min(delta, Math.max(0, position + delta - high)); + if (delta < 0) return Math.max(delta, Math.min(0, position + delta - low)); + return 0; + } + + void finishGesture() { + if (!isZoomed()) { + scale = 1; + constrainOffset(); + } + } + + void reset() { scale = 1; offsetX = offsetY = 0; edgePanned = false; } private void constrainOffset() { - // Keep the original fitted area covered, as on iOS, even with letterboxing. float maximumX = baseWidth * (scale - 1) / 2; - float maximumY = baseHeight * (scale - 1) / 2; + float minimumY = -baseHeight * (scale - 1) / 2; + float maximumY = -minimumY; + if (edgePanned) { + // Reserve 20% of the viewport height regardless of zoom. If the + // whole image is shorter than that, keep its full height visible. + float visibleHeight = Math.min(height(), containerHeight * MINIMUM_VISIBLE_HEIGHT_FRACTION); + float centerY = baseTop + baseHeight / 2f; + minimumY = visibleHeight - centerY - height() / 2; + maximumY = containerHeight - visibleHeight - centerY + height() / 2; + } offsetX = clamp(offsetX, -maximumX, maximumX); - offsetY = clamp(offsetY, -maximumY, maximumY); + offsetY = clamp(offsetY, minimumY, maximumY); } private static float clamp(float value, float minimum, float maximum) { diff --git a/apps/ios/CrossDeskMobile/Input/RemoteMouseInput.swift b/apps/ios/CrossDeskMobile/Input/RemoteMouseInput.swift index f5cbbb5e..affd7853 100644 --- a/apps/ios/CrossDeskMobile/Input/RemoteMouseInput.swift +++ b/apps/ios/CrossDeskMobile/Input/RemoteMouseInput.swift @@ -5,6 +5,7 @@ import Foundation final class RemoteMouseInput { var send: ((Float, Float, Int, Int) -> Void)? var didCancel: (() -> Void)? + var relativeMotion: ((Float, Float) -> Void)? var viewportSize = CGSize(width: 0, height: 0) private(set) var x: Float = 0.5 private(set) var y: Float = 0.5 @@ -21,8 +22,12 @@ final class RemoteMouseInput { func drag(dx: Float, dy: Float) { guard hasVirtualButton, viewportSize.width > 0, viewportSize.height > 0 else { return } - position(x: x + dx / Float(viewportSize.width) * 1.35, - y: y + dy / Float(viewportSize.height) * 1.35) + if let relativeMotion { + relativeMotion(dx, dy) + } else { + position(x: x + dx / Float(viewportSize.width) * 1.35, + y: y + dy / Float(viewportSize.height) * 1.35) + } emit(0) } diff --git a/apps/ios/CrossDeskMobile/Input/RemoteTouchInputView.swift b/apps/ios/CrossDeskMobile/Input/RemoteTouchInputView.swift index 867a1046..54b944e7 100644 --- a/apps/ios/CrossDeskMobile/Input/RemoteTouchInputView.swift +++ b/apps/ios/CrossDeskMobile/Input/RemoteTouchInputView.swift @@ -52,6 +52,14 @@ enum RemoteVideoGeometry { height: baseRect.height * scale ) } + + static func verticalPointerOverflow(y: CGFloat, dy: CGFloat, visibleRect: CGRect) -> CGFloat { + guard !visibleRect.isNull, !visibleRect.isEmpty else { return 0 } + if dy > 0 { return min(dy, max(0, y + dy - visibleRect.maxY)) } + if dy < 0 { return max(dy, min(0, y + dy - visibleRect.minY)) } + return 0 + } + } struct RemoteTouchInputView: UIViewRepresentable { @@ -90,7 +98,15 @@ struct RemoteTouchInputView: UIViewRepresentable { } final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate { - var mouseInput: RemoteMouseInput? + var mouseInput: RemoteMouseInput? { + didSet { + guard oldValue !== mouseInput else { return } + oldValue?.relativeMotion = nil + mouseInput?.relativeMotion = { [weak self] dx, dy in + _ = self?.updateRelativeCursor(by: CGPoint(x: CGFloat(dx), y: CGFloat(dy))) + } + } + } private var gestureRevisions: [ObjectIdentifier: UInt64] = [:] var videoSize = CGSize.zero { didSet { setNeedsLayout() } @@ -128,10 +144,13 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate { private var hoverLastLocation: CGPoint? private var viewportScale: CGFloat = 1 private var viewportOffset = CGSize.zero + private var edgePanned = false + private var needsViewportRestore = true private var pinchActive = false private var viewportPanActive = false private let relativePointerSpeed: CGFloat = 1.35 private let maximumViewportScale: CGFloat = 10 + private let minimumVisibleHeightFraction: CGFloat = 0.2 override init(frame: CGRect) { super.init(frame: frame) @@ -187,6 +206,7 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate { override func layoutSubviews() { super.layoutSubviews() + if needsViewportRestore { restoreEdgePanAllowance() } mouseInput?.viewportSize = renderRect()?.size ?? .zero let constrained = constrainedViewportOffset(viewportOffset, scale: viewportScale) @@ -198,10 +218,22 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate { func setViewport(scale: CGFloat, offset: CGSize) { let nextScale = min(max(scale, 1), maximumViewportScale) viewportScale = nextScale + if nextScale == 1 && offset == .zero { edgePanned = false } + viewportOffset = offset + if needsViewportRestore { restoreEdgePanAllowance() } viewportOffset = constrainedViewportOffset(offset, scale: nextScale) mouseInput?.viewportSize = renderRect()?.size ?? .zero } + private func restoreEdgePanAllowance() { + // SwiftUI can restore state before assigning the new surface's bounds. + guard let baseRect = baseRenderRect() else { return } + needsViewportRestore = false + if abs(viewportOffset.height) > baseRect.height * (viewportScale - 1) / 2 { + edgePanned = true + } + } + func synchronizeRemoteCursor(_ position: CGPoint?, revision: UInt64) { // SwiftUI refreshes this view for video frames and shape-only feedback // too. Consume each position update once, even when it arrives during @@ -369,7 +401,7 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate { pinchActive = false if viewportScale <= 1.001 { viewportScale = 1 - viewportOffset = .zero + viewportOffset = constrainedViewportOffset(viewportOffset, scale: 1) publishViewport() } default: @@ -514,10 +546,24 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate { private func updateRelativeCursor(by translation: CGPoint) -> (Float, Float)? { guard let renderRect = renderRect(), renderRect.width > 0, renderRect.height > 0 else { return nil } - let nextX = CGFloat(relativeCursorPoint.0) + - translation.x / renderRect.width * relativePointerSpeed - let nextY = CGFloat(relativeCursorPoint.1) + - translation.y / renderRect.height * relativePointerSpeed + let delta = CGPoint(x: translation.x * relativePointerSpeed, + y: translation.y * relativePointerSpeed) + guard delta.x.isFinite, delta.y.isFinite else { return nil } + let cursorY = renderRect.minY + CGFloat(relativeCursorPoint.1) * renderRect.height + let overflowY = RemoteVideoGeometry.verticalPointerOverflow( + y: cursorY, dy: delta.y, visibleRect: renderRect.intersection(bounds)) + if overflowY != 0 { + edgePanned = true + let proposedOffset = CGSize(width: viewportOffset.width, + height: viewportOffset.height - overflowY) + let nextOffset = constrainedViewportOffset(proposedOffset, scale: viewportScale) + if nextOffset != viewportOffset { + viewportOffset = nextOffset + publishViewport() + } + } + let nextX = CGFloat(relativeCursorPoint.0) + delta.x / renderRect.width + let nextY = CGFloat(relativeCursorPoint.1) + delta.y / renderRect.height relativeCursorPoint = ( Float(min(max(nextX, 0), 1)), Float(min(max(nextY, 0), 1)) @@ -549,10 +595,20 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate { scale: CGFloat) -> CGSize { guard let baseRect = baseRenderRect() else { return offset } let maximumX = max(0, baseRect.width * (scale - 1) / 2) - let maximumY = max(0, baseRect.height * (scale - 1) / 2) + var minimumY = -max(0, baseRect.height * (scale - 1) / 2) + var maximumY = -minimumY + if edgePanned { + // Reserve 20% of the viewport height at any zoom; keep the entire + // image visible when its height is below that minimum. + let visibleHeight = min(baseRect.height * scale, + bounds.height * minimumVisibleHeightFraction) + let halfHeight = baseRect.height * scale / 2 + minimumY = visibleHeight - baseRect.midY - halfHeight + maximumY = bounds.height - visibleHeight - baseRect.midY + halfHeight + } return CGSize( width: min(max(offset.width, -maximumX), maximumX), - height: min(max(offset.height, -maximumY), maximumY) + height: min(max(offset.height, minimumY), maximumY) ) } diff --git a/apps/ios/README.md b/apps/ios/README.md index 5725e898..b416c2eb 100644 --- a/apps/ios/README.md +++ b/apps/ios/README.md @@ -429,6 +429,20 @@ and are excluded from device backups. Disabling saving or withdrawing consent clears them; clearing is serialized after pending writes to prevent resurrection. Old previews saved without this choice are removed on upgrade. +## Pointer edge panning + +In relative mouse mode, continuing to move outward at the top or bottom of the +visible image pans the video vertically in the opposite direction. Moving +down-left at the bottom edge pans upward without shifting the video horizontally; +left and right edges do not trigger panning. +This also works at 1× scale and while dragging a held virtual mouse button. +Panning stops with the pointer movement and retains its position after release. +At the vertical limit, the visible image occupies at least 20% of the viewport +height regardless of zoom. If the entire image is shorter than that, its full +height remains visible. The bounds account for letterboxing. Ending a pinch, +including returning to 1×, preserves the translated position within those bounds +instead of recentering it. Switching displays restores the centered view. + ## Physical-device test checklist The floating control menu includes **虚拟鼠标** (virtual mouse), a 140 × 48 pt