diff --git a/apps/cli/agent.cpp b/apps/cli/agent.cpp index 26dbb2e0..b10ea3d9 100644 --- a/apps/cli/agent.cpp +++ b/apps/cli/agent.cpp @@ -201,6 +201,10 @@ private: result["geometry_epoch"] = epoch_; result["cursor"] = Cursor(); result["human_paused"] = paused_; + result["control_owner"] = human_control_ ? "human" : !paused_ && session_.ControlEnabled() ? "agent" : "none"; + result["human_view"] = human_owner_; + result["human_input_epoch"] = human_epoch_; + result["viewer_input_protocol"] = "events/1"; result["control_allowed"] = allowed_; result["frame_age_ms"] = frame_ ? json(Age(frame_->queued)) : json(nullptr); result["source_size"] = @@ -771,6 +775,7 @@ private: {"video_transport", "loopback-websocket/1"}, {"terminal_protocol", "ansi/1"}, {"unlock_protocol", "password/1"}, + {"viewer_input_protocol", "events/1"}, {"action_completion", "submitted"}, {"max_request_bytes", 65536}, {"max_response_bytes", kMaxResponse}, @@ -818,6 +823,7 @@ private: return false; } else if (method == "release_inputs") { Fields(j, {"id", "method"}); + if (human_control_) throw std::invalid_argument("HUMAN_CONTROL_ACTIVE"); FinishAction("CANCELLED"); session_.ReleaseInput(); } else if (method == "system") { @@ -918,6 +924,7 @@ private: return false; } else if (method == "control") { Fields(j, {"id", "method", "enabled"}); + if (human_control_) throw std::invalid_argument("HUMAN_CONTROL_ACTIVE"); const bool enable = Flag(j, "enabled"); if (enable && (!allowed_ || paused_)) throw std::invalid_argument("CONTROL_DENIED"); @@ -926,26 +933,80 @@ private: if (!enable) CancelRemoteJobs("CONTROL_DENIED"); session_.SetControl(enable); } else if (method == "viewer_control") { - Fields(j, {"id", "method", "operation"}); + Fields(j, {"id", "method", "operation", "owner"}); const auto operation = String(j, "operation", 16); - if (operation == "pause") { + if (operation == "pause" || operation == "take_over") { + const auto owner = operation == "take_over" ? String(j, "owner") : std::string{}; + if (operation == "take_over" && !allowed_) throw std::invalid_argument("CONTROL_DENIED"); paused_ = true; FinishAction("USER_PAUSED"); CancelRemoteJobs("USER_PAUSED"); - session_.SetControl(false); + session_.ReleaseInput(); + human_control_ = operation == "take_over"; + human_owner_ = owner; + ++human_epoch_; + session_.BeginInput(false); + session_.SetControl(human_control_); } else if (operation == "resume") { if (!allowed_) throw std::invalid_argument("CONTROL_DENIED"); paused_ = false; + human_control_ = false; + human_owner_.clear(); + ++human_epoch_; + session_.ReleaseInput(); session_.SetControl(true); } else if (operation == "disconnect") { Stop("CANCELLED"); session_.Disconnect(); } else throw std::invalid_argument("INVALID_REQUEST"); + } else if (method == "viewer_input") { + Fields(j, {"id", "method", "owner", "input_epoch", "generation", "width", "height", "action"}); + if (!allowed_ || !human_control_ || String(j, "owner") != human_owner_) + throw std::invalid_argument("HUMAN_CONTROL_REQUIRED"); + if (Sequence(j, "input_epoch") != human_epoch_) throw std::invalid_argument("STALE_CONTROL"); + auto action = j.at("action"); + const auto type = String(action, "type", 16); + if (type == "release") { + Fields(action, {"type"}); + session_.ReleaseInput(); + } else { + if (!frame_ || !session_.Connected() || Age(frame_->queued) > 2000 || + Sequence(j, "generation") != session_.Generation() || + Number(j, "width", 1, 8192, 0) != int(frame_->width) || + Number(j, "height", 1, 8192, 0) != int(frame_->height)) { + session_.ReleaseInput(); + throw std::invalid_argument("STALE_OBSERVATION"); + } + if (type != "move" && type != "mouse_down" && type != "mouse_up" && type != "scroll" && + type != "key_down" && type != "key_up" && type != "type_text") + throw std::invalid_argument("INVALID_ARGUMENT"); + if (action.contains("x") || action.contains("y")) { + const double x = action.at("x").get(), y = action.at("y").get(); + if (!std::isfinite(x) || !std::isfinite(y) || x < 0 || x > 1 || y < 0 || y > 1) + throw std::invalid_argument("INVALID_COORDINATE"); + action["x"] = std::min(x * frame_->width, double(frame_->width) - 1); + action["y"] = std::min(y * frame_->height, double(frame_->height) - 1); + } + const auto steps = BuildInputSteps(action, {int(frame_->width), int(frame_->height), + int(frame_->width), int(frame_->height), {0, 0, int(frame_->width), int(frame_->height)}}); + if (type == "key_down" && session_.Status()["held_key_codes"].size() >= 32) + throw std::invalid_argument("INPUT_LIMIT"); + if (type == "type_text") session_.ReleaseInput(); + for (const auto &step : steps) { + const bool sent = step.kind == StepKind::Pointer ? session_.Pointer(step.x, step.y, step.flag, step.wheel) + : step.kind == StepKind::Key ? session_.Key(step.key, step.down) : session_.Text(step.text); + if (!sent) { session_.ReleaseInput(); Error(id, "INPUT_FAILED", true); return false; } + } + input_lease_ = Clock::now() + std::chrono::seconds(15); + } + Reply(id, {{"submitted", true}}); + return false; } else if (method == "video_settings") { Fields(j, {"id", "method", "fps", "width", "height", "quality", "max_bitrate_bps"}); if (action_) throw std::invalid_argument("BUSY"); + if (human_control_) throw std::invalid_argument("HUMAN_CONTROL_ACTIVE"); if (session_.HasPressedInput()) throw std::invalid_argument("INPUT_HELD"); const auto status = session_.Status(); if (!session_.Connected() || !status.value("preview_supported", false)) @@ -961,6 +1022,7 @@ private: Number(j, "max_bitrate_bps", 128000, 100000000, status.value("preview_bitrate_bps", 6000000))); } else if (method == "display") { Fields(j, {"id", "method", "index", "preserve_input"}); + if (human_control_) throw std::invalid_argument("HUMAN_CONTROL_ACTIVE"); const int index = Number(j, "index", 0, 63, -1); if (!session_.Connected() || index < 0 || size_t(index) >= session_.Status()["displays"].size()) @@ -1050,6 +1112,9 @@ private: int password_fd_, output_fd_ = -1; bool hello_ = false, connected_once_ = false, allowed_ = false, paused_ = false; + bool human_control_ = false; + std::string human_owner_; + uint64_t human_epoch_ = 0; uint64_t epoch_ = 0, observation_sequence_ = 0; std::shared_ptr frame_; std::deque observations_; diff --git a/apps/mcp/README.md b/apps/mcp/README.md index f07cffed..3c56bde2 100644 --- a/apps/mcp/README.md +++ b/apps/mcp/README.md @@ -288,3 +288,9 @@ npm run pack:npm -- --out-dir /absolute/path/artifacts `crossdesk_set_recovery` 和面板的“断线自动恢复”开关用于启用恢复。默认关闭,启用要求设备已保存连接凭据及持久授权。后台监测覆盖临时信令中断、连接丢失和原生进程异常,使用退避策略,在六十秒预算内最多执行配置次数的重连。每次重连重新核对设备身份和授权,人工暂停、关闭恢复开关和主动断开会停止恢复。 恢复期间返回 `reconnecting`,可以用 `crossdesk_get_session` 携带 `session_id` 和 `wait_ms` 等待。恢复成功后返回新的 `session_id`,既有监看面板跟随新连接。旧会话标识可用于查询替代连接,后续操作使用新会话及新截图。截图标识包含所属会话,避免不同原生进程的观测编号相互混淆。先前的输入、解锁和文件任务保持原有结果边界,恢复流程只重建连接并恢复观看配置。 + +### 面板人工接管 + +浏览器及 Apps 面板提供“人工接管”和“交还 Agent”。接管后面板可操作鼠标、滚轮、键盘快捷键和中文输入法,连接信息展示当前控制权。输入绑定面板、控制批次和画面尺寸,交接会释放按住状态并使旧批次失效。画面超过两秒未更新时停止发送新输入,窗口失焦或隐藏时释放按键。原生层保留十五秒的输入租约作为断线清理保障。 + +Agent 在人工接管期间仍可观察画面,键鼠、显示设置和其他要求控制权的操作等待用户在面板交还控制。连接需预先授予控制权限,并使用支持 `events/1` 的原生 CLI。 diff --git a/apps/mcp/src/browser.ts b/apps/mcp/src/browser.ts index 5f69014e..4f381c07 100644 --- a/apps/mcp/src/browser.ts +++ b/apps/mcp/src/browser.ts @@ -255,7 +255,7 @@ export class BrowserPanel { if (typeof body.view_id !== "string" || !client.views.has(body.view_id)) throw new Failure("VIEW_UNAUTHORIZED"); const name = req.url?.match( - /^\/api\/(ready|devices|status|connect|video_open|control|unlock|recovery)$/, + /^\/api\/(ready|devices|status|connect|video_open|control|unlock|recovery|input)$/, )?.[1]; if (!name) throw new Failure("UNKNOWN_ACTION"); const { view_id, ...args } = body; diff --git a/apps/mcp/src/common.ts b/apps/mcp/src/common.ts index a091c59d..65472270 100644 --- a/apps/mcp/src/common.ts +++ b/apps/mcp/src/common.ts @@ -120,6 +120,22 @@ export const action = z.discriminatedUnion("type", [ .refine((s) => Buffer.byteLength(s) <= 16384 && !s.includes("\0")), }), ]); +const normalizedPoint = z.strictObject({ x: z.number().finite().min(0).max(1), y: z.number().finite().min(0).max(1) }); +export const viewerInput = z.strictObject({ + input_epoch: z.int().min(0), generation: z.int().min(0), + width: z.int().min(1).max(8192), height: z.int().min(1).max(8192), + action: z.discriminatedUnion("type", [ + normalizedPoint.extend({ type: z.literal("move") }), + normalizedPoint.extend({ type: z.literal("mouse_down"), button }), + normalizedPoint.extend({ type: z.literal("mouse_up"), button }), + normalizedPoint.extend({ type: z.literal("scroll"), direction: z.enum(["up", "down", "left", "right"]), ticks: z.int().min(1).max(10) }), + z.strictObject({ type: z.literal("key_down"), key }), + z.strictObject({ type: z.literal("key_up"), key }), + z.strictObject({ type: z.literal("type_text"), text: z.string().min(1).max(16384) + .refine((value) => Buffer.byteLength(value) <= 16384 && !value.includes("\0")) }), + z.strictObject({ type: z.literal("release") }), + ]), +}); export const screenshot = z.strictObject({ session_id: id, region: z diff --git a/apps/mcp/src/native.ts b/apps/mcp/src/native.ts index ef63a833..18ff494d 100644 --- a/apps/mcp/src/native.ts +++ b/apps/mcp/src/native.ts @@ -93,6 +93,7 @@ type Pending = { }; export class NativeClient { unlockSupported = false; + viewerInputSupported = false; private child: ChildProcess; private pending = new Map(); private sequence = 0; @@ -150,6 +151,7 @@ export class NativeClient { signal, ); this.unlockSupported = hello.unlock_protocol === "password/1"; + this.viewerInputSupported = hello.viewer_input_protocol === "events/1"; if (hello.protocol !== "crossdesk-automation/1" || hello.terminal_protocol !== "ansi/1" || hello.video_transport !== "loopback-websocket/1" || @@ -173,7 +175,7 @@ export class NativeClient { if (this.pending.size >= 32 || this.child.stdin!.writableLength > 262144) return Promise.reject(new Failure("BUSY", false, true)); const id = ++this.sequence; - const sideEffect = method === "action" || method === "unlock" || method === "release_inputs" || + const sideEffect = method === "action" || method === "unlock" || method === "release_inputs" || method === "viewer_input" || (method === "clipboard" && args.operation === "set") || (method === "system" && args.command !== "status") || (method === "files" && ["upload", "download", "cancel"].includes(args.operation)); diff --git a/apps/mcp/src/server.ts b/apps/mcp/src/server.ts index 0cb9afee..7a2cf173 100644 --- a/apps/mcp/src/server.ts +++ b/apps/mcp/src/server.ts @@ -40,6 +40,7 @@ const uiNames: Record = { control: "crossdesk_viewer_control", unlock: "crossdesk_ui_unlock", recovery: "crossdesk_ui_recovery", + input: "crossdesk_ui_input", }; function output(value: Data, meta?: Data): CallToolResult { const data = { ...value }; diff --git a/apps/mcp/src/sessions.ts b/apps/mcp/src/sessions.ts index 4d9a3b90..6de1c004 100644 --- a/apps/mcp/src/sessions.ts +++ b/apps/mcp/src/sessions.ts @@ -853,7 +853,9 @@ export class Sessions { async viewerControl(v: View, operation: string) { if (!v.session) throw new Failure("NOT_READY"); if (operation === "disconnect") return this.disconnect(v.session); - if (operation === "pause") { + if (operation === "take_over" && !this.session(v.session).native.viewerInputSupported) + throw new Failure("NATIVE_INPUT_UNSUPPORTED"); + if (operation === "pause" || operation === "take_over") { const s = this.session(v.session); this.cancelUnlock(s); s.status.human_paused = true; @@ -862,12 +864,18 @@ export class Sessions { } const s = this.session(v.session); const result = await s.native.request("viewer_control", { - operation, + operation, owner: v.id, }); if (operation === "resume") s.desiredControl = true; s.status = result; return result; } + async viewerInput(view: View, args: Data, signal?: AbortSignal): Promise { + if (!view.session) throw new Failure("NOT_READY"); + const s = this.session(view.session); + if (s.displayBatch) throw new Failure("BUSY"); + return s.native.request("viewer_input", { ...args, owner: view.id }, 5000, signal); + } private unlockPending(request: UnlockRequest) { return ["preparing", "awaiting_password", "submitting", "verifying"].includes(request.state); } diff --git a/apps/mcp/src/ui-actions.ts b/apps/mcp/src/ui-actions.ts index 039364d3..d45e3dbf 100644 --- a/apps/mcp/src/ui-actions.ts +++ b/apps/mcp/src/ui-actions.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { parse, uiConnect, unlockPassword, id, Failure, type Data } from "./common.js"; +import { parse, uiConnect, unlockPassword, viewerInput, id, Failure, type Data } from "./common.js"; import { Sessions, type View } from "./sessions.js"; export const uiSchemas = { @@ -8,6 +8,7 @@ export const uiSchemas = { status: z.strictObject({}), connect: uiConnect, video_open: z.strictObject({}), + input: viewerInput, recovery: z.strictObject({ enabled: z.boolean(), max_attempts: z.int().min(1).max(5).default(3) }), unlock: z.strictObject({ request_id: id, @@ -16,7 +17,7 @@ export const uiSchemas = { cancel: z.boolean().default(false), }), control: z.strictObject({ - operation: z.enum(["pause", "resume", "disconnect"]), + operation: z.enum(["pause", "resume", "disconnect", "take_over"]), }), }; export type UIAction = keyof typeof uiSchemas; @@ -45,6 +46,8 @@ export async function uiAction( return sessions.status(undefined, view.id, signal); case "connect": return sessions.connectUI(view, p, signal); + case "input": + return sessions.viewerInput(view, p, signal); case "control": return sessions.viewerControl(view, p.operation); case "unlock": diff --git a/apps/mcp/ui/human-input.ts b/apps/mcp/ui/human-input.ts new file mode 100644 index 00000000..47dbfd29 --- /dev/null +++ b/apps/mcp/ui/human-input.ts @@ -0,0 +1,196 @@ +type Data = Record; +export type InputFrame = { width: number; height: number; generation: number }; +const keys: Record = { + Backspace: "BACKSPACE", Tab: "TAB", Enter: "ENTER", Escape: "ESC", Space: "SPACE", + ShiftLeft: "SHIFT", ShiftRight: "SHIFT", ControlLeft: "CTRL", ControlRight: "CTRL", + AltLeft: "ALT", AltRight: "ALT", MetaLeft: "META", MetaRight: "META", + ArrowLeft: "LEFT", ArrowRight: "RIGHT", ArrowUp: "UP", ArrowDown: "DOWN", + PageUp: "PAGE_UP", PageDown: "PAGE_DOWN", Home: "HOME", End: "END", + Insert: "INSERT", Delete: "DELETE", CapsLock: "CAPS_LOCK", NumLock: "NUM_LOCK", + ScrollLock: "SCROLL_LOCK", PrintScreen: "PRINT_SCREEN", ContextMenu: "MENU", Pause: "PAUSE", + Semicolon: "SEMICOLON", Equal: "EQUAL", Comma: "COMMA", Minus: "MINUS", Period: "PERIOD", + Slash: "SLASH", Backquote: "BACKQUOTE", BracketLeft: "BRACKET_LEFT", Backslash: "BACKSLASH", + BracketRight: "BRACKET_RIGHT", Quote: "QUOTE", NumpadEnter: "ENTER", + NumpadMultiply: "MULTIPLY", NumpadAdd: "ADD", NumpadSubtract: "SUBTRACT", + NumpadDecimal: "DECIMAL", NumpadDivide: "DIVIDE", +}; +function keyName(event: KeyboardEvent) { + if (/^Key[A-Z]$/.test(event.code)) return event.code.slice(3); + if (/^Digit[0-9]$/.test(event.code)) return event.code.slice(5); + if (/^Numpad[0-9]$/.test(event.code)) return "NUMPAD" + event.code.slice(6); + if (/^F([1-9]|1[0-9]|2[0-4])$/.test(event.code)) return event.code; + return keys[event.code]; +} + +// Native ownership and epochs also fence queued events after a panel handoff. +export class HumanInput { + private frame?: InputFrame; + private frameAt = 0; + private epoch?: number; + private queue: Data[] = []; + private sending = false; + private pressed = new Set(); + private move?: Data; + private timer?: ReturnType; + constructor(private surface: HTMLElement, private editor: HTMLTextAreaElement, + private send: (args: Data) => Promise, private error: (error: unknown) => void) { + surface.addEventListener("pointerdown", (event) => { + if (this.epoch === undefined || event.button > 2) return; + const point = this.point(event, false); + if (!point) return; + event.preventDefault(); + editor.focus({ preventScroll: true }); + surface.setPointerCapture(event.pointerId); + this.flushMove(); + this.enqueue({ type: "mouse_down", ...point, button: this.button(event.button) }); + }); + surface.addEventListener("pointermove", (event) => { + if (this.epoch === undefined) return; + const point = this.point(event, surface.hasPointerCapture(event.pointerId)); + if (!point) return; + this.move = { type: "move", ...point }; + this.timer ??= setTimeout(() => this.flushMove(), 33); + }); + surface.addEventListener("pointerup", (event) => { + if (this.epoch === undefined || event.button > 2) return; + this.flushMove(); + const point = this.point(event, true); + if (point) this.enqueue({ type: "mouse_up", ...point, button: this.button(event.button) }); + if (surface.hasPointerCapture(event.pointerId)) surface.releasePointerCapture(event.pointerId); + }); + surface.addEventListener("pointercancel", () => this.release()); + surface.addEventListener("contextmenu", (event) => { if (this.epoch !== undefined) event.preventDefault(); }); + surface.addEventListener("wheel", (event) => { + if (this.epoch === undefined) return; + const point = this.point(event, false); + if (!point) return; + event.preventDefault(); + const horizontal = Math.abs(event.deltaX) > Math.abs(event.deltaY); + const delta = horizontal ? event.deltaX : event.deltaY; + if (!delta) return; + this.flushMove(); + this.enqueue({ type: "scroll", ...point, + direction: horizontal ? delta > 0 ? "right" : "left" : delta > 0 ? "down" : "up", + ticks: Math.min(10, Math.max(1, Math.round(Math.abs(delta) / (event.deltaMode === 0 ? 100 : 1)))) }); + }, { passive: false }); + editor.addEventListener("keydown", (event) => { + if (this.epoch === undefined || event.isComposing) return; + // Let the local input method produce committed Unicode text. + if (event.key.length === 1 && !event.ctrlKey && !event.metaKey && !event.altKey) return; + const key = keyName(event); + if (!key) return; + event.preventDefault(); + for (const [name, held] of [["CTRL", event.ctrlKey], ["ALT", event.altKey], + ["META", event.metaKey], ["SHIFT", event.shiftKey]] as const) { + if (held && name !== key && !this.pressed.has(name)) { + this.pressed.add(name); + this.enqueue({ type: "key_down", key: name }); + } + } + if (!this.pressed.has(key)) { + this.pressed.add(key); + this.enqueue({ type: "key_down", key }); + } + }); + editor.addEventListener("keyup", (event) => { + const key = keyName(event); + if (key && this.pressed.delete(key)) { + event.preventDefault(); + this.enqueue({ type: "key_up", key }); + } + }); + const commit = () => { + const value = editor.value; + editor.value = ""; + if (!value || this.epoch === undefined) return; + if (new TextEncoder().encode(value).length > 16384) { + this.release(); + this.error(new Error("INVALID_ARGUMENT")); + return; + } + this.pressed.clear(); + this.enqueue({ type: "type_text", text: value }); + }; + editor.addEventListener("input", (event) => { if (!(event as InputEvent).isComposing) commit(); }); + editor.addEventListener("compositionend", commit); + editor.addEventListener("blur", () => this.release()); + window.addEventListener("blur", () => this.release()); + window.addEventListener("pagehide", () => this.release()); + document.addEventListener("visibilitychange", () => { if (document.hidden) this.release(); }); + } + update(epoch: number | undefined) { + if (this.epoch === epoch) return; + this.clear(); + this.epoch = epoch; + this.surface.dataset.human = String(epoch !== undefined); + if (epoch === undefined) this.editor.blur(); + } + setFrame(frame?: InputFrame) { + this.frame = frame ? { width: frame.width, height: frame.height, generation: frame.generation } : undefined; + this.frameAt = Date.now(); + } + private button(index: number) { return ["left", "middle", "right"][index]; } + private point(event: MouseEvent, clamp: boolean) { + if (!this.frame) return; + const rect = this.surface.getBoundingClientRect(); + const scale = Math.min(rect.width / this.frame.width, rect.height / this.frame.height); + if (!scale) return; + const width = this.frame.width * scale, height = this.frame.height * scale; + let x = (event.clientX - rect.left - (rect.width - width) / 2) / width; + let y = (event.clientY - rect.top - (rect.height - height) / 2) / height; + if (!clamp && (x < 0 || x > 1 || y < 0 || y > 1)) return; + return { x: Math.max(0, Math.min(1, x)), y: Math.max(0, Math.min(1, y)) }; + } + private flushMove() { + clearTimeout(this.timer); + this.timer = undefined; + const move = this.move; + this.move = undefined; + if (move) this.enqueue(move); + } + private clear() { + clearTimeout(this.timer); + this.timer = undefined; + this.move = undefined; + this.queue = []; + this.pressed.clear(); + this.editor.value = ""; + } + release() { + this.clear(); + this.enqueue({ type: "release" }); + } + private enqueue(action: Data) { + if (this.epoch === undefined || !this.frame) return; + if (action.type !== "release" && Date.now() - this.frameAt > 2000) { + this.release(); + return; + } + if (this.queue.length >= 64) { + this.clear(); + action = { type: "release" }; + this.error(new Error("INPUT_QUEUE_FULL")); + } + this.queue.push({ ...this.frame, input_epoch: this.epoch, action }); + void this.pump(); + } + private async pump() { + if (this.sending) return; + this.sending = true; + try { + while (this.queue.length) { + const packet = this.queue.shift()!; + if (packet.input_epoch !== this.epoch) continue; + try { await this.send(packet); } + catch (error) { + this.clear(); + // Never replay clicks or text after an uncertain response. + if (packet.input_epoch === this.epoch) { + try { await this.send({ ...packet, action: { type: "release" } }); } catch { /* Lease releases on lost transport. */ } + this.error(error); + } + } + } + } finally { this.sending = false; } + } +} diff --git a/apps/mcp/ui/video-player.ts b/apps/mcp/ui/video-player.ts index ba9466db..20ee666e 100644 --- a/apps/mcp/ui/video-player.ts +++ b/apps/mcp/ui/video-player.ts @@ -1,4 +1,4 @@ -type FrameInfo = { width: number; height: number; fps: number; frames: number }; +type FrameInfo = { width: number; height: number; generation: number; fps: number; frames: number }; type Cursor = { type: "cursor"; visible: boolean; x: number; y: number; generation: number; geometry_epoch: number; width: number; height: number }; function parameters(data: Uint8Array) { @@ -233,7 +233,7 @@ export class VideoPlayer { this.sampledAt = performance.now(); } this.canvas.dataset.frames = String(this.count); - this.onFrame({ width, height, frames: this.count, fps: this.fps }); + this.onFrame({ width, height, generation: this.generation, frames: this.count, fps: this.fps }); } catch { this.fail(new Error("VIDEO_INVALID")); } finally { frame.close(); } } diff --git a/apps/mcp/ui/viewer.css b/apps/mcp/ui/viewer.css index 28a85e5e..577cc31e 100644 --- a/apps/mcp/ui/viewer.css +++ b/apps/mcp/ui/viewer.css @@ -199,3 +199,7 @@ button:disabled { cursor: default; color: #a3a7ad; background: #fafafa; border-c .viewer-toolbar { padding-top: 5px; padding-bottom: 5px; } .telemetry { padding-top: 6px; padding-bottom: 6px; } } + +#remote-input { position: absolute; width: 1px; height: 1px; opacity: 0; padding: 0; border: 0; pointer-events: none; } +.screen[data-human="true"] { touch-action: none; cursor: crosshair; } +.screen[data-human="true"]:focus-within { outline: 2px solid #3976ff; outline-offset: 2px; } diff --git a/apps/mcp/ui/viewer.ts b/apps/mcp/ui/viewer.ts index 614d6a13..0d359f70 100644 --- a/apps/mcp/ui/viewer.ts +++ b/apps/mcp/ui/viewer.ts @@ -1,5 +1,6 @@ import type { App } from "@modelcontextprotocol/ext-apps"; import "./viewer.css"; +import { HumanInput } from "./human-input.js"; import { VideoPlayer } from "./video-player.js"; declare const CROSSDESK_APPS: boolean; @@ -54,11 +55,12 @@ root.innerHTML = `

远程画面

待连接
-
+
+

未连接设备

选择设备并输入访问密码。

@@ -105,7 +107,15 @@ let imageAt = 0; let busy = false, polling = false, started = false; +const humanInput = new HumanInput(element("screen"), element("remote-input"), + (args) => invoke("input", args), showError); +function ownsInput() { return status.control_owner === "human" && status.human_view === context?.view_id; } const messages: Record = { + HUMAN_CONTROL_ACTIVE: "远端正在由人工操作,请先交还 Agent。", + STALE_OBSERVATION: "画面已变化,请等待新画面后继续操作。", + STALE_CONTROL: "控制权已交接。", + INPUT_QUEUE_FULL: "输入过快,已释放按键,请稍后继续。", + NATIVE_INPUT_UNSUPPORTED: "请更新原生 CLI 以启用人工接管。", VIDEO_UNSUPPORTED: "当前浏览器不支持此视频格式,请使用支持 WebCodecs 的浏览器。", VIDEO_UNAVAILABLE: "视频通道暂不可用。可重新加载画面;内嵌面板受限时请在浏览器中观看。", VIDEO_INVALID: "视频格式发生变化,请重新加载画面。", @@ -160,6 +170,7 @@ function platformName(platform?: string) { return ({ macos: "macOS", windows: "Windows", linux: "Linux", android: "Android", ios: "iOS" } as Record)[platform ?? ""] ?? "远程电脑"; } function resetMetrics() { + humanInput.setFrame(undefined); imageAt = 0; frameSize = frameRate = ""; text("resolution", "—"); @@ -184,7 +195,7 @@ function setDetails(open: boolean) { element("details-toggle").onclick = () => setDetails(!root.classList.contains("details-open")); element("focus-view").onclick = () => setFocus(!root.classList.contains("focus-mode")); window.addEventListener("keydown", (event) => { - if (event.key === "Escape" && !element("unlock-dialog").open && root.classList.contains("focus-mode")) setFocus(false); + if (!ownsInput() && event.key === "Escape" && !element("unlock-dialog").open && root.classList.contains("focus-mode")) setFocus(false); }); function hidePassword() { input("password").type = "password"; @@ -375,6 +386,10 @@ function renderStatus(s: Data) { if (lost && !disconnectedByUser && ["ready", "waiting_frame"].includes(status.state)) showError(new Error("CONNECTION_LOST")); const changedSession = s.session_id !== status.session_id; status = s; + humanInput.update(connected && ownsInput() ? s.human_input_epoch : undefined); + text("take-over", ownsInput() ? "交还 Agent" : "人工接管"); + element("take-over").disabled = !connected || !s.control_allowed || + s.viewer_input_protocol !== "events/1" || controlBusy || (!imageAt && !ownsInput()); renderUnlock(s); const connecting = busy || s.state === "waiting_frame"; root.dataset.connected = String(connected); @@ -387,7 +402,7 @@ function renderStatus(s: Data) { text("sidebar-title", connected ? "连接信息" : "连接设备"); text("device-name", connected ? s.device_alias || "远程电脑" : "远程画面"); if (!connected || changedSession) setDetails(false); - text("permission", connected ? s.human_paused ? "控制已暂停" : s.control_allowed ? "允许 Agent 操作" : "仅查看画面" : "等待授权"); + text("permission", connected ? s.control_owner === "human" ? ownsInput() ? "正在人工操作" : "其他面板正在操作" : s.human_paused ? "控制已暂停" : s.control_allowed ? "允许 Agent 操作" : "仅查看画面" : "等待授权"); element("permission").dataset.state = !connected ? "idle" : s.human_paused ? "paused" : "active"; const platform = platformName(s.platform); text("session-info", connected ? `${platform} · 显示器 ${(s.display ?? 0) + 1}` : "等待设备连接"); @@ -398,11 +413,11 @@ function renderStatus(s: Data) { text("session-device-id", device ? String(device.device_id).replace(/(\d{3})(?=\d)/g, "$1 ") : ""); text("session-access", s.control_allowed ? "查看与操作" : "仅查看"); text("session-display", `显示器 ${(s.display ?? 0) + 1}`); - text("session-agent", s.state === "reconnecting" ? "正在恢复连接" : s.human_paused ? "已暂停" : s.control_allowed ? "允许操作" : "仅可观察"); + text("session-agent", s.state === "reconnecting" ? "正在恢复连接" : s.control_owner === "human" ? "人工接管中" : s.human_paused ? "已暂停" : s.control_allowed ? "允许操作" : "仅可观察"); input("recovery").checked = s.reconnect_enabled === true; input("recovery").disabled = !s.session_id; element("session-agent").dataset.paused = String(!!s.human_paused); - element("pause").disabled = !connected || !s.control_allowed || controlBusy; + element("pause").disabled = !connected || !s.control_allowed || controlBusy || s.control_owner === "human"; const pauseLabel = s.human_paused ? "恢复控制" : "暂停控制"; text("pause-label", pauseLabel); element("disconnect").disabled = !s.session_id || controlBusy || busy; @@ -488,6 +503,17 @@ input("recovery").onchange = async () => { renderStatus(await invoke("status")); } catch (error) { showError(error); renderStatus(status); } }; +element("take-over").onclick = async () => { + if (controlBusy) return; + controlBusy = true; + humanInput.release(); + renderStatus(status); + try { + await invoke("control", { operation: ownsInput() ? "resume" : "take_over" }); + renderStatus(await invoke("status")); + } catch (error) { showError(error); } + finally { controlBusy = false; renderStatus(status); } +}; element("pause").onclick = async () => { if (controlBusy) return; controlBusy = true; @@ -558,6 +584,7 @@ async function startPreview() { if (epoch !== videoEpoch || document.hidden) return; player = new VideoPlayer(element("frame"), (info) => { if (epoch !== videoEpoch) return; + humanInput.setFrame(info); imageAt = Date.now(); element("frame").hidden = false; element("empty").hidden = true; @@ -665,6 +692,7 @@ async function start() { control: "crossdesk_viewer_control", unlock: "crossdesk_ui_unlock", recovery: "crossdesk_ui_recovery", + input: "crossdesk_ui_input", }; let connected = false, initial: Data | undefined; diff --git a/libs/controller/include/controller_session.h b/libs/controller/include/controller_session.h index 76850f4b..458670c9 100644 --- a/libs/controller/include/controller_session.h +++ b/libs/controller/include/controller_session.h @@ -86,7 +86,7 @@ public: void ForgetRemoteRequest(uint64_t request) { remote_results_.erase(request); } uint64_t Clipboard(bool write, const std::string &text = {}); nlohmann::json Files(const nlohmann::json &request); - void BeginInput(); + void BeginInput(bool acknowledge = true); std::string InputResult(); bool InputAcknowledged() const { return remote_capabilities_.value("input_ack", nlohmann::json{}) == true; } std::optional TakeFrame(DecodedFrame *recycle = nullptr); @@ -130,6 +130,7 @@ private: std::vector input_requests_; std::string input_result_ = "applied"; bool input_ack_active_ = false; + bool input_ack_transport_ = false; bool clipboard_ready_ = false; web_clipboard::Assembler clipboard_assembler_; std::unique_ptr files_; diff --git a/libs/controller/src/controller_requests.cpp b/libs/controller/src/controller_requests.cpp index e6f2e53c..fdb34efd 100644 --- a/libs/controller/src/controller_requests.cpp +++ b/libs/controller/src/controller_requests.cpp @@ -25,20 +25,22 @@ std::optional ControllerSession::TakeRemoteResult(uint64_t request) { return result; } -void ControllerSession::BeginInput() { +void ControllerSession::BeginInput(bool acknowledge) { action_keys_ = keys_; action_buttons_ = buttons_; for (const auto id : input_requests_) remote_results_.erase(id); input_requests_.clear(); input_result_ = "applied"; - input_ack_active_ = InputAcknowledged(); + input_ack_transport_ = InputAcknowledged(); + input_ack_active_ = acknowledge && input_ack_transport_; } bool ControllerSession::SendInput(const char *stream, const RemoteAction &action) { - if (!input_ack_active_) return Send(stream, action.to_json()); + if (!input_ack_transport_) return Send(stream, action.to_json()); const auto id = RequestRemote("input", json::parse(action.to_json())); if (!id) return false; - input_requests_.push_back(id); + if (input_ack_active_) input_requests_.push_back(id); + else remote_results_.erase(id); return true; }