[feat] add panel desktop takeover with exclusive input ownership

apps/cli/agent.cpp

@@ -201,6 +201,10 @@ private:
     result["geometry_epoch"] = epoch_;
     result["cursor"] = Cursor();
     result["human_paused"] = paused_;
+    result["control_owner"] = human_control_ ? "human" : !paused_ && session_.ControlEnabled() ? "agent" : "none";
+    result["human_view"] = human_owner_;
+    result["human_input_epoch"] = human_epoch_;
+    result["viewer_input_protocol"] = "events/1";
     result["control_allowed"] = allowed_;
     result["frame_age_ms"] = frame_ ? json(Age(frame_->queued)) : json(nullptr);
     result["source_size"] =
@@ -771,6 +775,7 @@ private:
                    {"video_transport", "loopback-websocket/1"},
                    {"terminal_protocol", "ansi/1"},
                    {"unlock_protocol", "password/1"},
+                   {"viewer_input_protocol", "events/1"},
                    {"action_completion", "submitted"},
                    {"max_request_bytes", 65536},
                    {"max_response_bytes", kMaxResponse},
@@ -818,6 +823,7 @@ private:
         return false;
       } else if (method == "release_inputs") {
         Fields(j, {"id", "method"});
+        if (human_control_) throw std::invalid_argument("HUMAN_CONTROL_ACTIVE");
         FinishAction("CANCELLED");
         session_.ReleaseInput();
       } else if (method == "system") {
@@ -918,6 +924,7 @@ private:
         return false;
       } else if (method == "control") {
         Fields(j, {"id", "method", "enabled"});
+        if (human_control_) throw std::invalid_argument("HUMAN_CONTROL_ACTIVE");
         const bool enable = Flag(j, "enabled");
         if (enable && (!allowed_ || paused_))
           throw std::invalid_argument("CONTROL_DENIED");
@@ -926,26 +933,80 @@ private:
         if (!enable) CancelRemoteJobs("CONTROL_DENIED");
         session_.SetControl(enable);
       } else if (method == "viewer_control") {
-        Fields(j, {"id", "method", "operation"});
+        Fields(j, {"id", "method", "operation", "owner"});
         const auto operation = String(j, "operation", 16);
-        if (operation == "pause") {
+        if (operation == "pause" || operation == "take_over") {
+          const auto owner = operation == "take_over" ? String(j, "owner") : std::string{};
+          if (operation == "take_over" && !allowed_) throw std::invalid_argument("CONTROL_DENIED");
           paused_ = true;
           FinishAction("USER_PAUSED");
           CancelRemoteJobs("USER_PAUSED");
-          session_.SetControl(false);
+          session_.ReleaseInput();
+          human_control_ = operation == "take_over";
+          human_owner_ = owner;
+          ++human_epoch_;
+          session_.BeginInput(false);
+          session_.SetControl(human_control_);
         } else if (operation == "resume") {
           if (!allowed_)
             throw std::invalid_argument("CONTROL_DENIED");
           paused_ = false;
+          human_control_ = false;
+          human_owner_.clear();
+          ++human_epoch_;
+          session_.ReleaseInput();
           session_.SetControl(true);
         } else if (operation == "disconnect") {
           Stop("CANCELLED");
           session_.Disconnect();
         } else
           throw std::invalid_argument("INVALID_REQUEST");
+      } else if (method == "viewer_input") {
+        Fields(j, {"id", "method", "owner", "input_epoch", "generation", "width", "height", "action"});
+        if (!allowed_ || !human_control_ || String(j, "owner") != human_owner_)
+          throw std::invalid_argument("HUMAN_CONTROL_REQUIRED");
+        if (Sequence(j, "input_epoch") != human_epoch_) throw std::invalid_argument("STALE_CONTROL");
+        auto action = j.at("action");
+        const auto type = String(action, "type", 16);
+        if (type == "release") {
+          Fields(action, {"type"});
+          session_.ReleaseInput();
+        } else {
+          if (!frame_ || !session_.Connected() || Age(frame_->queued) > 2000 ||
+              Sequence(j, "generation") != session_.Generation() ||
+              Number(j, "width", 1, 8192, 0) != int(frame_->width) ||
+              Number(j, "height", 1, 8192, 0) != int(frame_->height)) {
+            session_.ReleaseInput();
+            throw std::invalid_argument("STALE_OBSERVATION");
+          }
+          if (type != "move" && type != "mouse_down" && type != "mouse_up" && type != "scroll" &&
+              type != "key_down" && type != "key_up" && type != "type_text")
+            throw std::invalid_argument("INVALID_ARGUMENT");
+          if (action.contains("x") || action.contains("y")) {
+            const double x = action.at("x").get<double>(), y = action.at("y").get<double>();
+            if (!std::isfinite(x) || !std::isfinite(y) || x < 0 || x > 1 || y < 0 || y > 1)
+              throw std::invalid_argument("INVALID_COORDINATE");
+            action["x"] = std::min(x * frame_->width, double(frame_->width) - 1);
+            action["y"] = std::min(y * frame_->height, double(frame_->height) - 1);
+          }
+          const auto steps = BuildInputSteps(action, {int(frame_->width), int(frame_->height),
+              int(frame_->width), int(frame_->height), {0, 0, int(frame_->width), int(frame_->height)}});
+          if (type == "key_down" && session_.Status()["held_key_codes"].size() >= 32)
+            throw std::invalid_argument("INPUT_LIMIT");
+          if (type == "type_text") session_.ReleaseInput();
+          for (const auto &step : steps) {
+            const bool sent = step.kind == StepKind::Pointer ? session_.Pointer(step.x, step.y, step.flag, step.wheel)
+                : step.kind == StepKind::Key ? session_.Key(step.key, step.down) : session_.Text(step.text);
+            if (!sent) { session_.ReleaseInput(); Error(id, "INPUT_FAILED", true); return false; }
+          }
+          input_lease_ = Clock::now() + std::chrono::seconds(15);
+        }
+        Reply(id, {{"submitted", true}});
+        return false;
       } else if (method == "video_settings") {
         Fields(j, {"id", "method", "fps", "width", "height", "quality", "max_bitrate_bps"});
         if (action_) throw std::invalid_argument("BUSY");
+        if (human_control_) throw std::invalid_argument("HUMAN_CONTROL_ACTIVE");
         if (session_.HasPressedInput()) throw std::invalid_argument("INPUT_HELD");
         const auto status = session_.Status();
         if (!session_.Connected() || !status.value("preview_supported", false))
@@ -961,6 +1022,7 @@ private:
             Number(j, "max_bitrate_bps", 128000, 100000000, status.value("preview_bitrate_bps", 6000000)));
       } else if (method == "display") {
         Fields(j, {"id", "method", "index", "preserve_input"});
+        if (human_control_) throw std::invalid_argument("HUMAN_CONTROL_ACTIVE");
         const int index = Number(j, "index", 0, 63, -1);
         if (!session_.Connected() || index < 0 ||
             size_t(index) >= session_.Status()["displays"].size())
@@ -1050,6 +1112,9 @@ private:
   int password_fd_, output_fd_ = -1;
   bool hello_ = false, connected_once_ = false, allowed_ = false,
        paused_ = false;
+  bool human_control_ = false;
+  std::string human_owner_;
+  uint64_t human_epoch_ = 0;
   uint64_t epoch_ = 0, observation_sequence_ = 0;
   std::shared_ptr<DecodedFrame> frame_;
   std::deque<Observation> observations_;

apps/mcp/README.md

@@ -288,3 +288,9 @@ npm run pack:npm -- --out-dir /absolute/path/artifacts
 `crossdesk_set_recovery` 和面板的“断线自动恢复”开关用于启用恢复。默认关闭，启用要求设备已保存连接凭据及持久授权。后台监测覆盖临时信令中断、连接丢失和原生进程异常，使用退避策略，在六十秒预算内最多执行配置次数的重连。每次重连重新核对设备身份和授权，人工暂停、关闭恢复开关和主动断开会停止恢复。
 
 恢复期间返回 `reconnecting`，可以用 `crossdesk_get_session` 携带 `session_id` 和 `wait_ms` 等待。恢复成功后返回新的 `session_id`，既有监看面板跟随新连接。旧会话标识可用于查询替代连接，后续操作使用新会话及新截图。截图标识包含所属会话，避免不同原生进程的观测编号相互混淆。先前的输入、解锁和文件任务保持原有结果边界，恢复流程只重建连接并恢复观看配置。
+
+### 面板人工接管
+
+浏览器及 Apps 面板提供“人工接管”和“交还 Agent”。接管后面板可操作鼠标、滚轮、键盘快捷键和中文输入法，连接信息展示当前控制权。输入绑定面板、控制批次和画面尺寸，交接会释放按住状态并使旧批次失效。画面超过两秒未更新时停止发送新输入，窗口失焦或隐藏时释放按键。原生层保留十五秒的输入租约作为断线清理保障。
+
+Agent 在人工接管期间仍可观察画面，键鼠、显示设置和其他要求控制权的操作等待用户在面板交还控制。连接需预先授予控制权限，并使用支持 `events/1` 的原生 CLI。

apps/mcp/src/browser.ts

@@ -255,7 +255,7 @@ export class BrowserPanel {
       if (typeof body.view_id !== "string" || !client.views.has(body.view_id))
         throw new Failure("VIEW_UNAUTHORIZED");
       const name = req.url?.match(
-        /^\/api\/(ready|devices|status|connect|video_open|control|unlock|recovery)$/,
+        /^\/api\/(ready|devices|status|connect|video_open|control|unlock|recovery|input)$/,
       )?.[1];
       if (!name) throw new Failure("UNKNOWN_ACTION");
       const { view_id, ...args } = body;

apps/mcp/src/common.ts

@@ -120,6 +120,22 @@ export const action = z.discriminatedUnion("type", [
       .refine((s) => Buffer.byteLength(s) <= 16384 && !s.includes("\0")),
   }),
 ]);
+const normalizedPoint = z.strictObject({ x: z.number().finite().min(0).max(1), y: z.number().finite().min(0).max(1) });
+export const viewerInput = z.strictObject({
+  input_epoch: z.int().min(0), generation: z.int().min(0),
+  width: z.int().min(1).max(8192), height: z.int().min(1).max(8192),
+  action: z.discriminatedUnion("type", [
+    normalizedPoint.extend({ type: z.literal("move") }),
+    normalizedPoint.extend({ type: z.literal("mouse_down"), button }),
+    normalizedPoint.extend({ type: z.literal("mouse_up"), button }),
+    normalizedPoint.extend({ type: z.literal("scroll"), direction: z.enum(["up", "down", "left", "right"]), ticks: z.int().min(1).max(10) }),
+    z.strictObject({ type: z.literal("key_down"), key }),
+    z.strictObject({ type: z.literal("key_up"), key }),
+    z.strictObject({ type: z.literal("type_text"), text: z.string().min(1).max(16384)
+      .refine((value) => Buffer.byteLength(value) <= 16384 && !value.includes("\0")) }),
+    z.strictObject({ type: z.literal("release") }),
+  ]),
+});
 export const screenshot = z.strictObject({
   session_id: id,
   region: z

apps/mcp/src/native.ts

@@ -93,6 +93,7 @@ type Pending = {
 };
 export class NativeClient {
   unlockSupported = false;
+  viewerInputSupported = false;
   private child: ChildProcess;
   private pending = new Map<number, Pending>();
   private sequence = 0;
@@ -150,6 +151,7 @@ export class NativeClient {
       signal,
     );
     this.unlockSupported = hello.unlock_protocol === "password/1";
+    this.viewerInputSupported = hello.viewer_input_protocol === "events/1";
     if (hello.protocol !== "crossdesk-automation/1" ||
         hello.terminal_protocol !== "ansi/1" ||
         hello.video_transport !== "loopback-websocket/1" ||
@@ -173,7 +175,7 @@ export class NativeClient {
     if (this.pending.size >= 32 || this.child.stdin!.writableLength > 262144)
       return Promise.reject(new Failure("BUSY", false, true));
     const id = ++this.sequence;
-    const sideEffect = method === "action" || method === "unlock" || method === "release_inputs" ||
+    const sideEffect = method === "action" || method === "unlock" || method === "release_inputs" || method === "viewer_input" ||
       (method === "clipboard" && args.operation === "set") ||
       (method === "system" && args.command !== "status") ||
       (method === "files" && ["upload", "download", "cancel"].includes(args.operation));

apps/mcp/src/server.ts

@@ -40,6 +40,7 @@ const uiNames: Record<UIAction, string> = {
   control: "crossdesk_viewer_control",
   unlock: "crossdesk_ui_unlock",
   recovery: "crossdesk_ui_recovery",
+  input: "crossdesk_ui_input",
 };
 function output(value: Data, meta?: Data): CallToolResult {
   const data = { ...value };

apps/mcp/src/sessions.ts

@@ -853,7 +853,9 @@ export class Sessions {
   async viewerControl(v: View, operation: string) {
     if (!v.session) throw new Failure("NOT_READY");
     if (operation === "disconnect") return this.disconnect(v.session);
-    if (operation === "pause") {
+    if (operation === "take_over" && !this.session(v.session).native.viewerInputSupported)
+      throw new Failure("NATIVE_INPUT_UNSUPPORTED");
+    if (operation === "pause" || operation === "take_over") {
       const s = this.session(v.session);
       this.cancelUnlock(s);
       s.status.human_paused = true;
@@ -862,12 +864,18 @@ export class Sessions {
     }
     const s = this.session(v.session);
     const result = await s.native.request("viewer_control", {
-      operation,
+      operation, owner: v.id,
     });
     if (operation === "resume") s.desiredControl = true;
     s.status = result;
     return result;
   }
+  async viewerInput(view: View, args: Data, signal?: AbortSignal): Promise<Data> {
+    if (!view.session) throw new Failure("NOT_READY");
+    const s = this.session(view.session);
+    if (s.displayBatch) throw new Failure("BUSY");
+    return s.native.request("viewer_input", { ...args, owner: view.id }, 5000, signal);
+  }
   private unlockPending(request: UnlockRequest) {
     return ["preparing", "awaiting_password", "submitting", "verifying"].includes(request.state);
   }

apps/mcp/src/ui-actions.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { parse, uiConnect, unlockPassword, id, Failure, type Data } from "./common.js";
+import { parse, uiConnect, unlockPassword, viewerInput, id, Failure, type Data } from "./common.js";
 import { Sessions, type View } from "./sessions.js";
 
 export const uiSchemas = {
@@ -8,6 +8,7 @@ export const uiSchemas = {
   status: z.strictObject({}),
   connect: uiConnect,
   video_open: z.strictObject({}),
+  input: viewerInput,
   recovery: z.strictObject({ enabled: z.boolean(), max_attempts: z.int().min(1).max(5).default(3) }),
   unlock: z.strictObject({
     request_id: id,
@@ -16,7 +17,7 @@ export const uiSchemas = {
     cancel: z.boolean().default(false),
   }),
   control: z.strictObject({
-    operation: z.enum(["pause", "resume", "disconnect"]),
+    operation: z.enum(["pause", "resume", "disconnect", "take_over"]),
   }),
 };
 export type UIAction = keyof typeof uiSchemas;
@@ -45,6 +46,8 @@ export async function uiAction(
       return sessions.status(undefined, view.id, signal);
     case "connect":
       return sessions.connectUI(view, p, signal);
+    case "input":
+      return sessions.viewerInput(view, p, signal);
     case "control":
       return sessions.viewerControl(view, p.operation);
     case "unlock":

apps/mcp/ui/human-input.ts

@@ -0,0 +1,196 @@
+type Data = Record<string, any>;
+export type InputFrame = { width: number; height: number; generation: number };
+const keys: Record<string, string> = {
+  Backspace: "BACKSPACE", Tab: "TAB", Enter: "ENTER", Escape: "ESC", Space: "SPACE",
+  ShiftLeft: "SHIFT", ShiftRight: "SHIFT", ControlLeft: "CTRL", ControlRight: "CTRL",
+  AltLeft: "ALT", AltRight: "ALT", MetaLeft: "META", MetaRight: "META",
+  ArrowLeft: "LEFT", ArrowRight: "RIGHT", ArrowUp: "UP", ArrowDown: "DOWN",
+  PageUp: "PAGE_UP", PageDown: "PAGE_DOWN", Home: "HOME", End: "END",
+  Insert: "INSERT", Delete: "DELETE", CapsLock: "CAPS_LOCK", NumLock: "NUM_LOCK",
+  ScrollLock: "SCROLL_LOCK", PrintScreen: "PRINT_SCREEN", ContextMenu: "MENU", Pause: "PAUSE",
+  Semicolon: "SEMICOLON", Equal: "EQUAL", Comma: "COMMA", Minus: "MINUS", Period: "PERIOD",
+  Slash: "SLASH", Backquote: "BACKQUOTE", BracketLeft: "BRACKET_LEFT", Backslash: "BACKSLASH",
+  BracketRight: "BRACKET_RIGHT", Quote: "QUOTE", NumpadEnter: "ENTER",
+  NumpadMultiply: "MULTIPLY", NumpadAdd: "ADD", NumpadSubtract: "SUBTRACT",
+  NumpadDecimal: "DECIMAL", NumpadDivide: "DIVIDE",
+};
+function keyName(event: KeyboardEvent) {
+  if (/^Key[A-Z]$/.test(event.code)) return event.code.slice(3);
+  if (/^Digit[0-9]$/.test(event.code)) return event.code.slice(5);
+  if (/^Numpad[0-9]$/.test(event.code)) return "NUMPAD" + event.code.slice(6);
+  if (/^F([1-9]|1[0-9]|2[0-4])$/.test(event.code)) return event.code;
+  return keys[event.code];
+}
+
+// Native ownership and epochs also fence queued events after a panel handoff.
+export class HumanInput {
+  private frame?: InputFrame;
+  private frameAt = 0;
+  private epoch?: number;
+  private queue: Data[] = [];
+  private sending = false;
+  private pressed = new Set<string>();
+  private move?: Data;
+  private timer?: ReturnType<typeof setTimeout>;
+  constructor(private surface: HTMLElement, private editor: HTMLTextAreaElement,
+    private send: (args: Data) => Promise<unknown>, private error: (error: unknown) => void) {
+    surface.addEventListener("pointerdown", (event) => {
+      if (this.epoch === undefined || event.button > 2) return;
+      const point = this.point(event, false);
+      if (!point) return;
+      event.preventDefault();
+      editor.focus({ preventScroll: true });
+      surface.setPointerCapture(event.pointerId);
+      this.flushMove();
+      this.enqueue({ type: "mouse_down", ...point, button: this.button(event.button) });
+    });
+    surface.addEventListener("pointermove", (event) => {
+      if (this.epoch === undefined) return;
+      const point = this.point(event, surface.hasPointerCapture(event.pointerId));
+      if (!point) return;
+      this.move = { type: "move", ...point };
+      this.timer ??= setTimeout(() => this.flushMove(), 33);
+    });
+    surface.addEventListener("pointerup", (event) => {
+      if (this.epoch === undefined || event.button > 2) return;
+      this.flushMove();
+      const point = this.point(event, true);
+      if (point) this.enqueue({ type: "mouse_up", ...point, button: this.button(event.button) });
+      if (surface.hasPointerCapture(event.pointerId)) surface.releasePointerCapture(event.pointerId);
+    });
+    surface.addEventListener("pointercancel", () => this.release());
+    surface.addEventListener("contextmenu", (event) => { if (this.epoch !== undefined) event.preventDefault(); });
+    surface.addEventListener("wheel", (event) => {
+      if (this.epoch === undefined) return;
+      const point = this.point(event, false);
+      if (!point) return;
+      event.preventDefault();
+      const horizontal = Math.abs(event.deltaX) > Math.abs(event.deltaY);
+      const delta = horizontal ? event.deltaX : event.deltaY;
+      if (!delta) return;
+      this.flushMove();
+      this.enqueue({ type: "scroll", ...point,
+        direction: horizontal ? delta > 0 ? "right" : "left" : delta > 0 ? "down" : "up",
+        ticks: Math.min(10, Math.max(1, Math.round(Math.abs(delta) / (event.deltaMode === 0 ? 100 : 1)))) });
+    }, { passive: false });
+    editor.addEventListener("keydown", (event) => {
+      if (this.epoch === undefined || event.isComposing) return;
+      // Let the local input method produce committed Unicode text.
+      if (event.key.length === 1 && !event.ctrlKey && !event.metaKey && !event.altKey) return;
+      const key = keyName(event);
+      if (!key) return;
+      event.preventDefault();
+      for (const [name, held] of [["CTRL", event.ctrlKey], ["ALT", event.altKey],
+        ["META", event.metaKey], ["SHIFT", event.shiftKey]] as const) {
+        if (held && name !== key && !this.pressed.has(name)) {
+          this.pressed.add(name);
+          this.enqueue({ type: "key_down", key: name });
+        }
+      }
+      if (!this.pressed.has(key)) {
+        this.pressed.add(key);
+        this.enqueue({ type: "key_down", key });
+      }
+    });
+    editor.addEventListener("keyup", (event) => {
+      const key = keyName(event);
+      if (key && this.pressed.delete(key)) {
+        event.preventDefault();
+        this.enqueue({ type: "key_up", key });
+      }
+    });
+    const commit = () => {
+      const value = editor.value;
+      editor.value = "";
+      if (!value || this.epoch === undefined) return;
+      if (new TextEncoder().encode(value).length > 16384) {
+        this.release();
+        this.error(new Error("INVALID_ARGUMENT"));
+        return;
+      }
+      this.pressed.clear();
+      this.enqueue({ type: "type_text", text: value });
+    };
+    editor.addEventListener("input", (event) => { if (!(event as InputEvent).isComposing) commit(); });
+    editor.addEventListener("compositionend", commit);
+    editor.addEventListener("blur", () => this.release());
+    window.addEventListener("blur", () => this.release());
+    window.addEventListener("pagehide", () => this.release());
+    document.addEventListener("visibilitychange", () => { if (document.hidden) this.release(); });
+  }
+  update(epoch: number | undefined) {
+    if (this.epoch === epoch) return;
+    this.clear();
+    this.epoch = epoch;
+    this.surface.dataset.human = String(epoch !== undefined);
+    if (epoch === undefined) this.editor.blur();
+  }
+  setFrame(frame?: InputFrame) {
+    this.frame = frame ? { width: frame.width, height: frame.height, generation: frame.generation } : undefined;
+    this.frameAt = Date.now();
+  }
+  private button(index: number) { return ["left", "middle", "right"][index]; }
+  private point(event: MouseEvent, clamp: boolean) {
+    if (!this.frame) return;
+    const rect = this.surface.getBoundingClientRect();
+    const scale = Math.min(rect.width / this.frame.width, rect.height / this.frame.height);
+    if (!scale) return;
+    const width = this.frame.width * scale, height = this.frame.height * scale;
+    let x = (event.clientX - rect.left - (rect.width - width) / 2) / width;
+    let y = (event.clientY - rect.top - (rect.height - height) / 2) / height;
+    if (!clamp && (x < 0 || x > 1 || y < 0 || y > 1)) return;
+    return { x: Math.max(0, Math.min(1, x)), y: Math.max(0, Math.min(1, y)) };
+  }
+  private flushMove() {
+    clearTimeout(this.timer);
+    this.timer = undefined;
+    const move = this.move;
+    this.move = undefined;
+    if (move) this.enqueue(move);
+  }
+  private clear() {
+    clearTimeout(this.timer);
+    this.timer = undefined;
+    this.move = undefined;
+    this.queue = [];
+    this.pressed.clear();
+    this.editor.value = "";
+  }
+  release() {
+    this.clear();
+    this.enqueue({ type: "release" });
+  }
+  private enqueue(action: Data) {
+    if (this.epoch === undefined || !this.frame) return;
+    if (action.type !== "release" && Date.now() - this.frameAt > 2000) {
+      this.release();
+      return;
+    }
+    if (this.queue.length >= 64) {
+      this.clear();
+      action = { type: "release" };
+      this.error(new Error("INPUT_QUEUE_FULL"));
+    }
+    this.queue.push({ ...this.frame, input_epoch: this.epoch, action });
+    void this.pump();
+  }
+  private async pump() {
+    if (this.sending) return;
+    this.sending = true;
+    try {
+      while (this.queue.length) {
+        const packet = this.queue.shift()!;
+        if (packet.input_epoch !== this.epoch) continue;
+        try { await this.send(packet); }
+        catch (error) {
+          this.clear();
+          // Never replay clicks or text after an uncertain response.
+          if (packet.input_epoch === this.epoch) {
+            try { await this.send({ ...packet, action: { type: "release" } }); } catch { /* Lease releases on lost transport. */ }
+            this.error(error);
+          }
+        }
+      }
+    } finally { this.sending = false; }
+  }
+}

apps/mcp/ui/video-player.ts

@@ -1,4 +1,4 @@
-type FrameInfo = { width: number; height: number; fps: number; frames: number };
+type FrameInfo = { width: number; height: number; generation: number; fps: number; frames: number };
 type Cursor = { type: "cursor"; visible: boolean; x: number; y: number;
   generation: number; geometry_epoch: number; width: number; height: number };
 function parameters(data: Uint8Array) {
@@ -233,7 +233,7 @@ export class VideoPlayer {
         this.sampledAt = performance.now();
       }
       this.canvas.dataset.frames = String(this.count);
-      this.onFrame({ width, height, frames: this.count, fps: this.fps });
+      this.onFrame({ width, height, generation: this.generation, frames: this.count, fps: this.fps });
     } catch { this.fail(new Error("VIDEO_INVALID")); }
     finally { frame.close(); }
   }

apps/mcp/ui/viewer.css

@@ -199,3 +199,7 @@ button:disabled { cursor: default; color: #a3a7ad; background: #fafafa; border-c
   .viewer-toolbar { padding-top: 5px; padding-bottom: 5px; }
   .telemetry { padding-top: 6px; padding-bottom: 6px; }
 }
+
+#remote-input { position: absolute; width: 1px; height: 1px; opacity: 0; padding: 0; border: 0; pointer-events: none; }
+.screen[data-human="true"] { touch-action: none; cursor: crosshair; }
+.screen[data-human="true"]:focus-within { outline: 2px solid #3976ff; outline-offset: 2px; }

apps/mcp/ui/viewer.ts

@@ -1,5 +1,6 @@
 import type { App } from "@modelcontextprotocol/ext-apps";
 import "./viewer.css";
+import { HumanInput } from "./human-input.js";
 import { VideoPlayer } from "./video-player.js";
 
 declare const CROSSDESK_APPS: boolean;
@@ -54,11 +55,12 @@ root.innerHTML = `
     <div class="viewer-card">
       <div class="viewer-toolbar">
         <div class="viewer-title"><h2 id="device-name">远程画面</h2><span id="video-state">待连接</span></div>
-        <div class="viewer-controls"><button id="pause" disabled><span id="pause-label">暂停控制</span></button><button id="disconnect" class="danger" disabled>断开连接</button><button id="focus-view" aria-label="放大画面" aria-pressed="false" disabled>放大画面</button></div>
+        <div class="viewer-controls"><button id="take-over" disabled>人工接管</button><button id="pause" disabled><span id="pause-label">暂停控制</span></button><button id="disconnect" class="danger" disabled>断开连接</button><button id="focus-view" aria-label="放大画面" aria-pressed="false" disabled>放大画面</button></div>
       </div>
       <div class="screen" id="screen">
         <canvas id="frame" role="img" aria-label="远程桌面实时画面" hidden></canvas>
         <canvas id="agent-cursor" aria-hidden="true"></canvas>
+        <textarea id="remote-input" aria-label="远程键盘输入" autocomplete="off" autocapitalize="off" spellcheck="false"></textarea>
         <div class="control-halo" aria-hidden="true"></div>
         <div id="empty"><h3 id="empty-title">未连接设备</h3><p id="empty-description">选择设备并输入访问密码。</p></div>
         <div class="stale" id="stale" role="status" hidden>画面暂未更新</div>
@@ -105,7 +107,15 @@ let imageAt = 0;
 let busy = false,
   polling = false,
   started = false;
+const humanInput = new HumanInput(element("screen"), element<HTMLTextAreaElement>("remote-input"),
+  (args) => invoke("input", args), showError);
+function ownsInput() { return status.control_owner === "human" && status.human_view === context?.view_id; }
 const messages: Record<string, string> = {
+  HUMAN_CONTROL_ACTIVE: "远端正在由人工操作，请先交还 Agent。",
+  STALE_OBSERVATION: "画面已变化，请等待新画面后继续操作。",
+  STALE_CONTROL: "控制权已交接。",
+  INPUT_QUEUE_FULL: "输入过快，已释放按键，请稍后继续。",
+  NATIVE_INPUT_UNSUPPORTED: "请更新原生 CLI 以启用人工接管。",
   VIDEO_UNSUPPORTED: "当前浏览器不支持此视频格式，请使用支持 WebCodecs 的浏览器。",
   VIDEO_UNAVAILABLE: "视频通道暂不可用。可重新加载画面；内嵌面板受限时请在浏览器中观看。",
   VIDEO_INVALID: "视频格式发生变化，请重新加载画面。",
@@ -160,6 +170,7 @@ function platformName(platform?: string) {
   return ({ macos: "macOS", windows: "Windows", linux: "Linux", android: "Android", ios: "iOS" } as Record<string, string>)[platform ?? ""] ?? "远程电脑";
 }
 function resetMetrics() {
+  humanInput.setFrame(undefined);
   imageAt = 0;
   frameSize = frameRate = "";
   text("resolution", "—");
@@ -184,7 +195,7 @@ function setDetails(open: boolean) {
 element("details-toggle").onclick = () => setDetails(!root.classList.contains("details-open"));
 element("focus-view").onclick = () => setFocus(!root.classList.contains("focus-mode"));
 window.addEventListener("keydown", (event) => {
-  if (event.key === "Escape" && !element<HTMLDialogElement>("unlock-dialog").open && root.classList.contains("focus-mode")) setFocus(false);
+  if (!ownsInput() && event.key === "Escape" && !element<HTMLDialogElement>("unlock-dialog").open && root.classList.contains("focus-mode")) setFocus(false);
 });
 function hidePassword() {
   input("password").type = "password";
@@ -375,6 +386,10 @@ function renderStatus(s: Data) {
   if (lost && !disconnectedByUser && ["ready", "waiting_frame"].includes(status.state)) showError(new Error("CONNECTION_LOST"));
   const changedSession = s.session_id !== status.session_id;
   status = s;
+  humanInput.update(connected && ownsInput() ? s.human_input_epoch : undefined);
+  text("take-over", ownsInput() ? "交还 Agent" : "人工接管");
+  element<HTMLButtonElement>("take-over").disabled = !connected || !s.control_allowed ||
+    s.viewer_input_protocol !== "events/1" || controlBusy || (!imageAt && !ownsInput());
   renderUnlock(s);
   const connecting = busy || s.state === "waiting_frame";
   root.dataset.connected = String(connected);
@@ -387,7 +402,7 @@ function renderStatus(s: Data) {
   text("sidebar-title", connected ? "连接信息" : "连接设备");
   text("device-name", connected ? s.device_alias || "远程电脑" : "远程画面");
   if (!connected || changedSession) setDetails(false);
-  text("permission", connected ? s.human_paused ? "控制已暂停" : s.control_allowed ? "允许 Agent 操作" : "仅查看画面" : "等待授权");
+  text("permission", connected ? s.control_owner === "human" ? ownsInput() ? "正在人工操作" : "其他面板正在操作" : s.human_paused ? "控制已暂停" : s.control_allowed ? "允许 Agent 操作" : "仅查看画面" : "等待授权");
   element("permission").dataset.state = !connected ? "idle" : s.human_paused ? "paused" : "active";
   const platform = platformName(s.platform);
   text("session-info", connected ? `${platform} · 显示器 ${(s.display ?? 0) + 1}` : "等待设备连接");
@@ -398,11 +413,11 @@ function renderStatus(s: Data) {
   text("session-device-id", device ? String(device.device_id).replace(/(\d{3})(?=\d)/g, "$1 ") : "");
   text("session-access", s.control_allowed ? "查看与操作" : "仅查看");
   text("session-display", `显示器 ${(s.display ?? 0) + 1}`);
-  text("session-agent", s.state === "reconnecting" ? "正在恢复连接" : s.human_paused ? "已暂停" : s.control_allowed ? "允许操作" : "仅可观察");
+  text("session-agent", s.state === "reconnecting" ? "正在恢复连接" : s.control_owner === "human" ? "人工接管中" : s.human_paused ? "已暂停" : s.control_allowed ? "允许操作" : "仅可观察");
   input("recovery").checked = s.reconnect_enabled === true;
   input("recovery").disabled = !s.session_id;
   element("session-agent").dataset.paused = String(!!s.human_paused);
-  element<HTMLButtonElement>("pause").disabled = !connected || !s.control_allowed || controlBusy;
+  element<HTMLButtonElement>("pause").disabled = !connected || !s.control_allowed || controlBusy || s.control_owner === "human";
   const pauseLabel = s.human_paused ? "恢复控制" : "暂停控制";
   text("pause-label", pauseLabel);
   element<HTMLButtonElement>("disconnect").disabled = !s.session_id || controlBusy || busy;
@@ -488,6 +503,17 @@ input("recovery").onchange = async () => {
     renderStatus(await invoke("status"));
   } catch (error) { showError(error); renderStatus(status); }
 };
+element("take-over").onclick = async () => {
+  if (controlBusy) return;
+  controlBusy = true;
+  humanInput.release();
+  renderStatus(status);
+  try {
+    await invoke("control", { operation: ownsInput() ? "resume" : "take_over" });
+    renderStatus(await invoke("status"));
+  } catch (error) { showError(error); }
+  finally { controlBusy = false; renderStatus(status); }
+};
 element("pause").onclick = async () => {
   if (controlBusy) return;
   controlBusy = true;
@@ -558,6 +584,7 @@ async function startPreview() {
     if (epoch !== videoEpoch || document.hidden) return;
     player = new VideoPlayer(element<HTMLCanvasElement>("frame"), (info) => {
       if (epoch !== videoEpoch) return;
+      humanInput.setFrame(info);
       imageAt = Date.now();
       element("frame").hidden = false;
       element("empty").hidden = true;
@@ -665,6 +692,7 @@ async function start() {
       control: "crossdesk_viewer_control",
       unlock: "crossdesk_ui_unlock",
       recovery: "crossdesk_ui_recovery",
+      input: "crossdesk_ui_input",
     };
     let connected = false,
       initial: Data | undefined;

libs/controller/include/controller_session.h

@@ -86,7 +86,7 @@ public:
   void ForgetRemoteRequest(uint64_t request) { remote_results_.erase(request); }
   uint64_t Clipboard(bool write, const std::string &text = {});
   nlohmann::json Files(const nlohmann::json &request);
-  void BeginInput();
+  void BeginInput(bool acknowledge = true);
   std::string InputResult();
   bool InputAcknowledged() const { return remote_capabilities_.value("input_ack", nlohmann::json{}) == true; }
   std::optional<DecodedFrame> TakeFrame(DecodedFrame *recycle = nullptr);
@@ -130,6 +130,7 @@ private:
   std::vector<uint64_t> input_requests_;
   std::string input_result_ = "applied";
   bool input_ack_active_ = false;
+  bool input_ack_transport_ = false;
   bool clipboard_ready_ = false;
   web_clipboard::Assembler clipboard_assembler_;
   std::unique_ptr<ControllerFiles> files_;

libs/controller/src/controller_requests.cpp

@@ -25,20 +25,22 @@ std::optional<json> ControllerSession::TakeRemoteResult(uint64_t request) {
   return result;
 }
 
-void ControllerSession::BeginInput() {
+void ControllerSession::BeginInput(bool acknowledge) {
   action_keys_ = keys_;
   action_buttons_ = buttons_;
   for (const auto id : input_requests_) remote_results_.erase(id);
   input_requests_.clear();
   input_result_ = "applied";
-  input_ack_active_ = InputAcknowledged();
+  input_ack_transport_ = InputAcknowledged();
+  input_ack_active_ = acknowledge && input_ack_transport_;
 }
 
 bool ControllerSession::SendInput(const char *stream, const RemoteAction &action) {
-  if (!input_ack_active_) return Send(stream, action.to_json());
+  if (!input_ack_transport_) return Send(stream, action.to_json());
   const auto id = RequestRemote("input", json::parse(action.to_json()));
   if (!id) return false;
-  input_requests_.push_back(id);
+  if (input_ack_active_) input_requests_.push_back(id);
+  else remote_results_.erase(id);
   return true;
 }