mirror of
https://github.com/kunkundi/crossdesk.git
synced 2026-10-19 02:13:03 +08:00
[feat] add vertical pointer edge panning on mobile
This commit is contained in:
dijunkun
@@ -7,7 +7,7 @@
|
||||
- 官方或自托管 WSS 信令服务器；授权后进入前台即自动注册或登录本机身份，首页实时显示连接、重连及失败状态。发起桌面连接时复用该身份；切换服务器会重新登录，撤回授权会关闭连接。已有远程会话优先通过系统画中画保持连接，画中画不可用时由带连接通知的前台服务兜底；首页空闲信令在后台关闭。
|
||||
- 优先使用 MediaCodec 硬件解码 H.264 / AV1，硬件缓冲区直接提交 Surface；设备无对应硬件、分辨率超限或运行失败时，按原编码格式回退到 OpenH264 / dav1d。软件输出由独立显示线程将 NV12 转 RGBA；只保留最新待显示帧。MiniRTC 原有的通用压缩帧解码队列保持不变。
|
||||
- 48 kHz 单声道 PCM 播放，支持静音，音频队列有容量限制。
|
||||
- 相对鼠标（触控板）和绝对位置模式；单指点击、绝对模式按住拖动、长按右键，以及外接键盘、鼠标。画面手势与 iOS 一致：双指捏合以触点为中心缩放（1–10 倍），放大后双指拖动画面，原始比例下双指上下滑动控制滚轮。
|
||||
- 相对鼠标（触控板）和绝对位置模式；单指点击、绝对模式按住拖动、长按右键，以及外接键盘、鼠标。画面手势与 iOS 一致：双指捏合以触点为中心缩放（1–10 倍），放大后双指拖动画面，原始比例下双指上下滑动控制滚轮。相对鼠标移到可见画面的上、下边缘后继续向外移动，会反向上下平移画面，不触发左右平移；未放大时和按住虚拟鼠标键拖动时同样生效，停止移动即停止平移，抬手保留位置。移动到极限时，画面可见部分至少占当前显示区域高度的 20%，与缩放倍数无关；整幅画面高度不足该比例时完整保留，范围计入上下黑边；缩放结束及缩回 1 倍时保留平移位置，仅在超过可见边界时限制位置；切换显示器恢复居中。
|
||||
- 横竖屏等比例显示、远端多显示器切换。与 iOS 一致的画面质量（低/中/高）、采集帧率（30/60 fps）、画面偏好；支持能力检测、电脑端确认、失败/5 秒超时回退及连续操作的应答排序。
|
||||
- 根据常见移动设备解码能力和屏幕尺寸设置起始参数：显示长边不超过 1280 像素时默认低画质（最高 720p）/30 fps，其余默认中画质（最高 1080p）/30 fps。用户保存的画面偏好不变，仍可手动选择高画质或 60 fps。
|
||||
- 网络状态按视频、音频、数据、合计展示收发速率和丢包率，并显示实际提交帧率、分辨率、画面延时、平滑 RTT、P2P/TURN 和实际 SRTP 状态。每秒刷新，过期或无效测量显示“—”。
|
||||
|
||||
@@ -3,6 +3,8 @@ package cn.crossdesk.mobile;
|
||||
/** Shared button ownership and cursor position for the video surface and mouse bar. */
|
||||
final class RemoteMouseInput {
|
||||
interface Sender { void send(float x, float y, int flag, int wheel); }
|
||||
interface RelativeMotion { void move(float dx, float dy); }
|
||||
RelativeMotion relativeMotion;
|
||||
private final Sender sender;
|
||||
private float x = .5f, y = .5f;
|
||||
private boolean gestureLeft, virtualLeft, virtualRight;
|
||||
@@ -19,7 +21,8 @@ final class RemoteMouseInput {
|
||||
}
|
||||
void drag(float dx, float dy) {
|
||||
if (!hasVirtualButton() || viewportWidth <= 0 || viewportHeight <= 0) return;
|
||||
move(x + dx / viewportWidth, y + dy / viewportHeight);
|
||||
if (relativeMotion != null) relativeMotion.move(dx, dy);
|
||||
else move(x + dx / viewportWidth, y + dy / viewportHeight);
|
||||
}
|
||||
void gestureLeft(boolean down) {
|
||||
if (down && hasVirtualButton()) return;
|
||||
|
||||
@@ -51,6 +51,7 @@ final class RemoteVideoView extends FrameLayout implements SurfaceHolder.Callbac
|
||||
RemoteVideoView(Context context, NativeSession session) {
|
||||
super(context); this.session = session;
|
||||
mouseInput = new RemoteMouseInput(this::sendMouse);
|
||||
mouseInput.relativeMotion = this::moveRelative;
|
||||
setBackgroundColor(Color.BLACK);
|
||||
video = new SurfaceView(context); video.getHolder().addCallback(this);
|
||||
addView(video, new LayoutParams(1, 1, android.view.Gravity.CENTER));
|
||||
@@ -147,14 +148,21 @@ final class RemoteVideoView extends FrameLayout implements SurfaceHolder.Callbac
|
||||
cursor.invalidate();
|
||||
}
|
||||
@Override protected void onSizeChanged(int w,int h,int oldw,int oldh) { post(this::fit); }
|
||||
private void moveRelative(float dx, float dy) {
|
||||
if (pictureInPicture || viewport.width() <= 0 || viewport.height() <= 0) return;
|
||||
viewport.panForPointer(y, dy);
|
||||
applyViewport();
|
||||
mouseInput.move(x + dx / viewport.width(), y + dy / viewport.height());
|
||||
}
|
||||
private boolean point(MotionEvent e) {
|
||||
boolean touchpad = relative && e.getToolType(0) != MotionEvent.TOOL_TYPE_MOUSE;
|
||||
float px=e.getX(),py=e.getY();
|
||||
if (!touchpad && !viewport.contains(px,py)) return false;
|
||||
if(touchpad){
|
||||
if(e.getActionMasked()==MotionEvent.ACTION_MOVE){
|
||||
x=Math.max(0,Math.min(1,x+(px-lastX)/Math.max(1,viewport.width())));
|
||||
y=Math.max(0,Math.min(1,y+(py-lastY)/Math.max(1,viewport.height())));
|
||||
moveRelative(px-lastX,py-lastY);
|
||||
lastX=px;lastY=py;
|
||||
return true;
|
||||
}
|
||||
}else{
|
||||
x=viewport.normalizedX(px);
|
||||
|
||||
@@ -3,11 +3,14 @@ package cn.crossdesk.mobile;
|
||||
/** Geometry shared by the video Surface, cursor and remote pointer input. */
|
||||
final class RemoteViewport {
|
||||
private static final float MAXIMUM_SCALE = 10;
|
||||
private int baseWidth, baseHeight, baseLeft, baseTop;
|
||||
private static final float MINIMUM_VISIBLE_HEIGHT_FRACTION = .2f;
|
||||
private int baseWidth, baseHeight, baseLeft, baseTop, containerHeight;
|
||||
private boolean edgePanned;
|
||||
private float scale = 1, offsetX, offsetY;
|
||||
|
||||
void fit(int width, int height, int videoWidth, int videoHeight) {
|
||||
if (width <= 0 || height <= 0 || videoWidth <= 0 || videoHeight <= 0) return;
|
||||
containerHeight = height;
|
||||
double fitScale = Math.min((double) width / videoWidth, (double) height / videoHeight);
|
||||
baseWidth = Math.max(1, (int) (videoWidth * fitScale));
|
||||
baseHeight = Math.max(1, (int) (videoHeight * fitScale));
|
||||
@@ -54,18 +57,46 @@ final class RemoteViewport {
|
||||
constrainOffset();
|
||||
}
|
||||
|
||||
void finishGesture() {
|
||||
if (!isZoomed()) reset();
|
||||
void panForPointer(float y, float dy) {
|
||||
if (baseHeight == 0 || !Float.isFinite(dy)) return;
|
||||
float overflowY = pointerOverflow(top() + y * height(), dy,
|
||||
Math.max(0, top()), Math.min(containerHeight, top() + height()));
|
||||
if (overflowY == 0) return;
|
||||
edgePanned = true;
|
||||
panBy(0, -overflowY);
|
||||
}
|
||||
|
||||
void reset() { scale = 1; offsetX = offsetY = 0; }
|
||||
// Only the outward portion of this movement pans; an offscreen cursor must
|
||||
// not make a tiny movement jump the viewport by its entire hidden distance.
|
||||
private static float pointerOverflow(float position, float delta, float low, float high) {
|
||||
if (delta > 0) return Math.min(delta, Math.max(0, position + delta - high));
|
||||
if (delta < 0) return Math.max(delta, Math.min(0, position + delta - low));
|
||||
return 0;
|
||||
}
|
||||
|
||||
void finishGesture() {
|
||||
if (!isZoomed()) {
|
||||
scale = 1;
|
||||
constrainOffset();
|
||||
}
|
||||
}
|
||||
|
||||
void reset() { scale = 1; offsetX = offsetY = 0; edgePanned = false; }
|
||||
|
||||
private void constrainOffset() {
|
||||
// Keep the original fitted area covered, as on iOS, even with letterboxing.
|
||||
float maximumX = baseWidth * (scale - 1) / 2;
|
||||
float maximumY = baseHeight * (scale - 1) / 2;
|
||||
float minimumY = -baseHeight * (scale - 1) / 2;
|
||||
float maximumY = -minimumY;
|
||||
if (edgePanned) {
|
||||
// Reserve 20% of the viewport height regardless of zoom. If the
|
||||
// whole image is shorter than that, keep its full height visible.
|
||||
float visibleHeight = Math.min(height(), containerHeight * MINIMUM_VISIBLE_HEIGHT_FRACTION);
|
||||
float centerY = baseTop + baseHeight / 2f;
|
||||
minimumY = visibleHeight - centerY - height() / 2;
|
||||
maximumY = containerHeight - visibleHeight - centerY + height() / 2;
|
||||
}
|
||||
offsetX = clamp(offsetX, -maximumX, maximumX);
|
||||
offsetY = clamp(offsetY, -maximumY, maximumY);
|
||||
offsetY = clamp(offsetY, minimumY, maximumY);
|
||||
}
|
||||
|
||||
private static float clamp(float value, float minimum, float maximum) {
|
||||
|
||||
@@ -5,6 +5,7 @@ import Foundation
|
||||
final class RemoteMouseInput {
|
||||
var send: ((Float, Float, Int, Int) -> Void)?
|
||||
var didCancel: (() -> Void)?
|
||||
var relativeMotion: ((Float, Float) -> Void)?
|
||||
var viewportSize = CGSize(width: 0, height: 0)
|
||||
private(set) var x: Float = 0.5
|
||||
private(set) var y: Float = 0.5
|
||||
@@ -21,8 +22,12 @@ final class RemoteMouseInput {
|
||||
|
||||
func drag(dx: Float, dy: Float) {
|
||||
guard hasVirtualButton, viewportSize.width > 0, viewportSize.height > 0 else { return }
|
||||
position(x: x + dx / Float(viewportSize.width) * 1.35,
|
||||
y: y + dy / Float(viewportSize.height) * 1.35)
|
||||
if let relativeMotion {
|
||||
relativeMotion(dx, dy)
|
||||
} else {
|
||||
position(x: x + dx / Float(viewportSize.width) * 1.35,
|
||||
y: y + dy / Float(viewportSize.height) * 1.35)
|
||||
}
|
||||
emit(0)
|
||||
}
|
||||
|
||||
|
||||
@@ -52,6 +52,14 @@ enum RemoteVideoGeometry {
|
||||
height: baseRect.height * scale
|
||||
)
|
||||
}
|
||||
|
||||
static func verticalPointerOverflow(y: CGFloat, dy: CGFloat, visibleRect: CGRect) -> CGFloat {
|
||||
guard !visibleRect.isNull, !visibleRect.isEmpty else { return 0 }
|
||||
if dy > 0 { return min(dy, max(0, y + dy - visibleRect.maxY)) }
|
||||
if dy < 0 { return max(dy, min(0, y + dy - visibleRect.minY)) }
|
||||
return 0
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
struct RemoteTouchInputView: UIViewRepresentable {
|
||||
@@ -90,7 +98,15 @@ struct RemoteTouchInputView: UIViewRepresentable {
|
||||
}
|
||||
|
||||
final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
|
||||
var mouseInput: RemoteMouseInput?
|
||||
var mouseInput: RemoteMouseInput? {
|
||||
didSet {
|
||||
guard oldValue !== mouseInput else { return }
|
||||
oldValue?.relativeMotion = nil
|
||||
mouseInput?.relativeMotion = { [weak self] dx, dy in
|
||||
_ = self?.updateRelativeCursor(by: CGPoint(x: CGFloat(dx), y: CGFloat(dy)))
|
||||
}
|
||||
}
|
||||
}
|
||||
private var gestureRevisions: [ObjectIdentifier: UInt64] = [:]
|
||||
var videoSize = CGSize.zero {
|
||||
didSet { setNeedsLayout() }
|
||||
@@ -128,10 +144,13 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
|
||||
private var hoverLastLocation: CGPoint?
|
||||
private var viewportScale: CGFloat = 1
|
||||
private var viewportOffset = CGSize.zero
|
||||
private var edgePanned = false
|
||||
private var needsViewportRestore = true
|
||||
private var pinchActive = false
|
||||
private var viewportPanActive = false
|
||||
private let relativePointerSpeed: CGFloat = 1.35
|
||||
private let maximumViewportScale: CGFloat = 10
|
||||
private let minimumVisibleHeightFraction: CGFloat = 0.2
|
||||
|
||||
override init(frame: CGRect) {
|
||||
super.init(frame: frame)
|
||||
@@ -187,6 +206,7 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
|
||||
|
||||
override func layoutSubviews() {
|
||||
super.layoutSubviews()
|
||||
if needsViewportRestore { restoreEdgePanAllowance() }
|
||||
mouseInput?.viewportSize = renderRect()?.size ?? .zero
|
||||
let constrained = constrainedViewportOffset(viewportOffset,
|
||||
scale: viewportScale)
|
||||
@@ -198,10 +218,22 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
|
||||
func setViewport(scale: CGFloat, offset: CGSize) {
|
||||
let nextScale = min(max(scale, 1), maximumViewportScale)
|
||||
viewportScale = nextScale
|
||||
if nextScale == 1 && offset == .zero { edgePanned = false }
|
||||
viewportOffset = offset
|
||||
if needsViewportRestore { restoreEdgePanAllowance() }
|
||||
viewportOffset = constrainedViewportOffset(offset, scale: nextScale)
|
||||
mouseInput?.viewportSize = renderRect()?.size ?? .zero
|
||||
}
|
||||
|
||||
private func restoreEdgePanAllowance() {
|
||||
// SwiftUI can restore state before assigning the new surface's bounds.
|
||||
guard let baseRect = baseRenderRect() else { return }
|
||||
needsViewportRestore = false
|
||||
if abs(viewportOffset.height) > baseRect.height * (viewportScale - 1) / 2 {
|
||||
edgePanned = true
|
||||
}
|
||||
}
|
||||
|
||||
func synchronizeRemoteCursor(_ position: CGPoint?, revision: UInt64) {
|
||||
// SwiftUI refreshes this view for video frames and shape-only feedback
|
||||
// too. Consume each position update once, even when it arrives during
|
||||
@@ -369,7 +401,7 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
|
||||
pinchActive = false
|
||||
if viewportScale <= 1.001 {
|
||||
viewportScale = 1
|
||||
viewportOffset = .zero
|
||||
viewportOffset = constrainedViewportOffset(viewportOffset, scale: 1)
|
||||
publishViewport()
|
||||
}
|
||||
default:
|
||||
@@ -514,10 +546,24 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
|
||||
private func updateRelativeCursor(by translation: CGPoint) -> (Float, Float)? {
|
||||
guard let renderRect = renderRect(),
|
||||
renderRect.width > 0, renderRect.height > 0 else { return nil }
|
||||
let nextX = CGFloat(relativeCursorPoint.0) +
|
||||
translation.x / renderRect.width * relativePointerSpeed
|
||||
let nextY = CGFloat(relativeCursorPoint.1) +
|
||||
translation.y / renderRect.height * relativePointerSpeed
|
||||
let delta = CGPoint(x: translation.x * relativePointerSpeed,
|
||||
y: translation.y * relativePointerSpeed)
|
||||
guard delta.x.isFinite, delta.y.isFinite else { return nil }
|
||||
let cursorY = renderRect.minY + CGFloat(relativeCursorPoint.1) * renderRect.height
|
||||
let overflowY = RemoteVideoGeometry.verticalPointerOverflow(
|
||||
y: cursorY, dy: delta.y, visibleRect: renderRect.intersection(bounds))
|
||||
if overflowY != 0 {
|
||||
edgePanned = true
|
||||
let proposedOffset = CGSize(width: viewportOffset.width,
|
||||
height: viewportOffset.height - overflowY)
|
||||
let nextOffset = constrainedViewportOffset(proposedOffset, scale: viewportScale)
|
||||
if nextOffset != viewportOffset {
|
||||
viewportOffset = nextOffset
|
||||
publishViewport()
|
||||
}
|
||||
}
|
||||
let nextX = CGFloat(relativeCursorPoint.0) + delta.x / renderRect.width
|
||||
let nextY = CGFloat(relativeCursorPoint.1) + delta.y / renderRect.height
|
||||
relativeCursorPoint = (
|
||||
Float(min(max(nextX, 0), 1)),
|
||||
Float(min(max(nextY, 0), 1))
|
||||
@@ -549,10 +595,20 @@ final class RemoteTouchSurface: UIView, UIGestureRecognizerDelegate {
|
||||
scale: CGFloat) -> CGSize {
|
||||
guard let baseRect = baseRenderRect() else { return offset }
|
||||
let maximumX = max(0, baseRect.width * (scale - 1) / 2)
|
||||
let maximumY = max(0, baseRect.height * (scale - 1) / 2)
|
||||
var minimumY = -max(0, baseRect.height * (scale - 1) / 2)
|
||||
var maximumY = -minimumY
|
||||
if edgePanned {
|
||||
// Reserve 20% of the viewport height at any zoom; keep the entire
|
||||
// image visible when its height is below that minimum.
|
||||
let visibleHeight = min(baseRect.height * scale,
|
||||
bounds.height * minimumVisibleHeightFraction)
|
||||
let halfHeight = baseRect.height * scale / 2
|
||||
minimumY = visibleHeight - baseRect.midY - halfHeight
|
||||
maximumY = bounds.height - visibleHeight - baseRect.midY + halfHeight
|
||||
}
|
||||
return CGSize(
|
||||
width: min(max(offset.width, -maximumX), maximumX),
|
||||
height: min(max(offset.height, -maximumY), maximumY)
|
||||
height: min(max(offset.height, minimumY), maximumY)
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -429,6 +429,20 @@ and are excluded from device backups. Disabling saving or withdrawing consent
|
||||
clears them; clearing is serialized after pending writes to prevent resurrection.
|
||||
Old previews saved without this choice are removed on upgrade.
|
||||
|
||||
## Pointer edge panning
|
||||
|
||||
In relative mouse mode, continuing to move outward at the top or bottom of the
|
||||
visible image pans the video vertically in the opposite direction. Moving
|
||||
down-left at the bottom edge pans upward without shifting the video horizontally;
|
||||
left and right edges do not trigger panning.
|
||||
This also works at 1× scale and while dragging a held virtual mouse button.
|
||||
Panning stops with the pointer movement and retains its position after release.
|
||||
At the vertical limit, the visible image occupies at least 20% of the viewport
|
||||
height regardless of zoom. If the entire image is shorter than that, its full
|
||||
height remains visible. The bounds account for letterboxing. Ending a pinch,
|
||||
including returning to 1×, preserves the translated position within those bounds
|
||||
instead of recentering it. Switching displays restores the centered view.
|
||||
|
||||
## Physical-device test checklist
|
||||
|
||||
The floating control menu includes **虚拟鼠标** (virtual mouse), a 140 × 48 pt
|
||||
|
||||
Reference in New Issue
Block a user