[feat] add MCP terminal views and agent cursor overlays

apps/cli/agent.cpp

@@ -10,6 +10,7 @@
 #include <map>
 #include <memory>
 #include <stdexcept>
+#include <terminal_renderer.h>
 
 namespace crossdesk::cli {
 namespace {
@@ -120,6 +121,12 @@ struct Encoding {
   bool cancelled = false;
   std::future<FrameImage> result;
 };
+struct TerminalEncoding {
+  json id;
+  uint64_t generation, epoch;
+  Clock::time_point queued;
+  std::future<terminal::Grid> result;
+};
 struct Step {
   int delay = 0, kind = 0, key = 0, flag = 0, wheel = 0;
   float x = 0, y = 0;
@@ -232,6 +239,7 @@ private:
     }
     result["transport_state"] = result["state"];
     result["geometry_epoch"] = epoch_;
+    result["cursor"] = Cursor();
     result["human_paused"] = paused_;
     result["control_allowed"] = allowed_;
     result["frame_age_ms"] = frame_ ? json(Age(frame_->queued)) : json(nullptr);
@@ -244,6 +252,16 @@ private:
                             : "waiting_frame";
     return result;
   }
+  json Cursor() const {
+    const bool visible = cursor_visible_ && frame_ && session_.Connected() &&
+        frame_->generation == session_.Generation() && allowed_ && !paused_ &&
+        session_.ControlEnabled();
+    return {{"type", "cursor"}, {"visible", visible},
+            {"x", cursor_x_}, {"y", cursor_y_},
+            {"generation", session_.Generation()}, {"geometry_epoch", epoch_},
+            {"width", frame_ ? frame_->width : 0},
+            {"height", frame_ ? frame_->height : 0}};
+  }
   void PumpVideo() {
     if (!video_) return;
     const auto generation = session_.Generation();
@@ -255,6 +273,11 @@ private:
       video_connected_ = connected;
       last_video_frame_ = 0;
     }
+    const auto cursor = Cursor();
+    if (cursor != video_cursor_) {
+      video_->Cursor(cursor);
+      video_cursor_ = cursor;
+    }
     const bool enabled = video_->Enabled();
     const bool key = video_->TakeKeyframeRequest();
     if (enabled != video_enabled_ || key) {
@@ -337,6 +360,7 @@ private:
       if (!frame_ || f->generation != frame_->generation ||
           f->width != frame_->width || f->height != frame_->height) {
         ++epoch_;
+        cursor_visible_ = false;
         observations_.clear();
         preview_ = json();
       }
@@ -345,6 +369,7 @@ private:
     if (frame_ && frame_->generation != session_.Generation()) {
       frame_.reset();
       ++epoch_;
+      cursor_visible_ = false;
       observations_.clear();
       preview_ = json();
     }
@@ -367,11 +392,41 @@ private:
           FinishAction("ACTION_OUTCOME_UNKNOWN");
           break;
         }
+        if (step.kind == 0) {
+          cursor_x_ = step.x;
+          cursor_y_ = step.y;
+          cursor_visible_ = true;
+        }
         action_->submitted = true;
       }
       if (action_ && action_->next == action_->steps.size())
         FinishAction();
     }
+    if (terminal_encoding_ &&
+        terminal_encoding_->result.wait_for(Ms(0)) == std::future_status::ready) {
+      auto job = std::move(terminal_encoding_);
+      try {
+        auto grid = job->result.get();
+        if (!session_.Connected() || job->generation != session_.Generation() ||
+            job->epoch != epoch_)
+          Error(job->id, "STALE_OBSERVATION");
+        else if (Age(job->queued) > 2000)
+          Error(job->id, "FRAME_TIMEOUT");
+        else {
+          json cursor = nullptr;
+          if (Cursor()["visible"].get<bool>()) {
+            const int x = grid.content.x + std::clamp(
+                int(cursor_x_ * grid.content.width), 0, grid.content.width - 1);
+            const int y = grid.content.y + std::clamp(
+                int(cursor_y_ * grid.content.height), 0, grid.content.height - 1);
+            cursor = {{"column", x / grid.samples_x}, {"row", y / grid.samples_y}};
+          }
+          Reply(job->id, {{"columns", grid.viewport.columns}, {"rows", grid.viewport.rows},
+                          {"lines", terminal::Lines(grid)}, {"cursor", cursor},
+                          {"generation", job->generation}, {"geometry_epoch", job->epoch}});
+        }
+      } catch (...) { Error(job->id, "FRAME_RENDER_FAILED"); }
+    }
     if (encoding_ &&
         encoding_->result.wait_for(Ms(0)) == std::future_status::ready) {
       auto job = std::move(encoding_);
@@ -730,6 +785,7 @@ private:
                    {"video_protocols", {"h264-annexb/1", "nv12/1"}},
                    {"video_decoders", {"auto", "software"}},
                    {"video_transport", "loopback-websocket/1"},
+                   {"terminal_protocol", "ansi/1"},
                    {"action_completion", "submitted"},
                    {"max_request_bytes", 65536},
                    {"max_response_bytes", kMaxResponse},
@@ -775,6 +831,32 @@ private:
       } else if (method == "action") {
         StartAction(j, id);
         return false;
+      } else if (method == "terminal_frame") {
+        Fields(j, {"id", "method", "columns", "rows", "colors", "cell_aspect"});
+        if (terminal_encoding_) throw std::invalid_argument("BUSY");
+        if (!session_.Connected() || !frame_ ||
+            frame_->generation != session_.Generation() || Age(frame_->queued) > 2000)
+          throw std::invalid_argument("NOT_READY");
+        terminal::Viewport viewport;
+        viewport.columns = Number(j, "columns", 1, 160, 80);
+        viewport.rows = Number(j, "rows", 1, 60, 22);
+        viewport.colors = Number(j, "colors", 8, 256, 24);
+        if (viewport.colors != 8 && viewport.colors != 24 && viewport.colors != 256)
+          throw std::invalid_argument("INVALID_REQUEST");
+        viewport.cell_aspect = j.value("cell_aspect", .5);
+        if (!std::isfinite(viewport.cell_aspect) || viewport.cell_aspect < .2 || viewport.cell_aspect > 2)
+          throw std::invalid_argument("INVALID_REQUEST");
+        viewport.detail = terminal::AnsiDetail::Quadrant;
+        auto job = std::make_unique<TerminalEncoding>();
+        job->id = id;
+        job->generation = frame_->generation;
+        job->epoch = epoch_;
+        job->queued = frame_->queued;
+        job->result = std::async(std::launch::async, [frame = frame_, viewport] {
+          return terminal::Convert(*frame, viewport);
+        });
+        terminal_encoding_ = std::move(job);
+        return false;
       } else if (method == "control") {
         Fields(j, {"id", "method", "enabled"});
         const bool enable = Flag(j, "enabled");
@@ -807,6 +889,7 @@ private:
           throw std::invalid_argument("DISPLAY_UNAVAILABLE");
         frame_.reset();
         ++epoch_;
+        cursor_visible_ = false;
       } else if (method == "cancel") {
         Fields(j, {"id", "method", "request_id"});
         if (!j.contains("request_id") || !j["request_id"].is_number_integer())
@@ -875,6 +958,7 @@ private:
   std::deque<Observation> observations_;
   std::deque<Capture> captures_;
   std::unique_ptr<Encoding> encoding_;
+  std::unique_ptr<TerminalEncoding> terminal_encoding_;
   std::unique_ptr<RunningAction> action_;
   std::deque<Record> records_;
   json preview_;
@@ -883,6 +967,9 @@ private:
   std::unique_ptr<VideoServer> video_;
   uint64_t video_generation_ = 0, last_video_frame_ = 0;
   bool video_connected_ = false, video_enabled_ = false;
+  bool cursor_visible_ = false;
+  float cursor_x_ = 0, cursor_y_ = 0;
+  json video_cursor_;
 };
 } // namespace
 int Agent(int output_fd, SessionOptions options, int password_fd,

apps/cli/docs/automation-protocol.md

@@ -35,6 +35,7 @@ The output queue is bounded to 64 messages and 8 MiB; a blocked output pipe term
 | `action` | `operation_id`, `observation_id`, `action` |
 | `control` | `enabled` boolean; cannot override observe-only permission or human pause |
 | `viewer_control` | `operation: "pause" | "resume" | "disconnect"`; restricted to trusted UI by the MCP adapter |
+| `terminal_frame` | `columns` (1–160), `rows` (1–60), `colors` (8/24/256), `cell_aspect` (0.2–2); private terminal viewer only |
 | `display` | `index` (0–63) |
 | `cancel` | `request_id`; cancels queued/in-flight captures or input sequences |
 | `disconnect`, `exit` | none; `exit` also ends the process |
@@ -45,6 +46,12 @@ The output queue is bounded to 64 messages and 8 MiB; a blocked output pipe term
 
 `doctor --json` advertises `automation_video_transport: "loopback-websocket/1"`, `automation_video_protocols: ["h264-annexb/1", "nv12/1"]`, and `automation_video_decoders: ["auto", "software"]`. `hello` repeats these without the `automation_` prefix. The MCP adapter requires the transport capability, so an old pipe-only CLI is rejected. Node spawns only stdin, stdout, stderr, and the credential pipe; live video never enters Node.
 
+The terminal viewer additionally requires `automation_terminal_protocol: "ansi/1"` in `doctor` and `terminal_protocol: "ansi/1"` in `hello`. `terminal_frame` asynchronously converts the current decoded frame using the existing ANSI quadrant renderer and returns `columns`, `rows`, `lines`, `generation`, `geometry_epoch`, and a nullable `cursor` with zero-based terminal `column`/`row`. Only one render may be in flight; obsolete geometry or frames older than two seconds are rejected. These rendered text rows travel through the private authenticated terminal IPC, never through public MCP tools or model content. The viewer requests at most 8 FPS and waits for terminal writes before requesting another frame. It shares the automation connection; no second media session is opened.
+
+Status includes `cursor` with normalized full-frame `x`/`y`, `visible`, `generation`, `geometry_epoch`, `width`, and `height`. The native layer updates it after each successfully sent pointer step, including drag steps, using the same ROI-to-source mapping as remote input. It hides on pause/control disable and clears on geometry changes. It is an Agent position indicator, not remote cursor telemetry or execution confirmation; observation PNGs and source video remain unchanged.
+
+Authenticated video sockets also receive small JSON text messages with `type: "cursor"` and these fields, independently of binary video frames. Pointer updates are coalesced to at most 30 Hz, retained for newly authorized viewers, and deferred under socket backpressure; they require no video ACK. Browser/Apps render a separate transparent overlay with the same aspect-ratio fitting as the video, including when no new video frame arrives.
+
 `connect.decode_path` defaults to `browser` (compressed H.264), or selects `minirtc` (decoded NV12). `connect.decoder` defaults to `auto` (hardware preferred, software when unavailable); `software` selects the native software decoder. Both paths maintain native screenshot frames with the selected decoder. These choices are fixed for the connection. Status exposes `video` and the reported `native_video_decoder` backend name. PNG encoding runs only when requested. The legacy `preview` image method remains available to private protocol callers but is not used by MCP panels.
 
 ### Authorization and transport

apps/cli/main.cpp

@@ -1091,6 +1091,7 @@ int main(int argc, char **argv) {
                 {"sixel", "probe_on_view"},
                 {"audio_playback", false},
                 {"bridge_version", 1},
+                {"automation_terminal_protocol", "ansi/1"},
                 {"automation_protocol", "crossdesk-automation/1"},
                 {"automation_video", "h264-annexb/1"},
                 {"automation_video_transport", "loopback-websocket/1"},

apps/cli/video_server.cpp

@@ -57,6 +57,7 @@ struct VideoServer::State {
   struct Client {
     std::string origin, view, session;
     uint64_t view_generation = 0;
+    uint64_t cursor_revision = 0;
     bool authorized = false, waiting_key = true;
     size_t bytes = 0;
     Clock::time_point opened = Clock::now(), pinged = Clock::now();
@@ -93,6 +94,8 @@ struct VideoServer::State {
   std::deque<Packet> queue;
   size_t queued_bytes = 0;
   bool scheduled = false, wait_key = true;
+  json cursor;
+  uint64_t cursor_revision = 0;
 
   State() {
     server.clear_access_channels(websocketpp::log::alevel::all);
@@ -250,10 +253,17 @@ struct VideoServer::State {
   }
   void Maintain() {
     timer = server.set_timer(
-        250, [this](const websocketpp::lib::error_code &error) {
+        33, [this](const websocketpp::lib::error_code &error) {
           if (error)
             return;
           const auto now = Clock::now();
+          std::string pointer;
+          uint64_t revision;
+          {
+            std::lock_guard lock(mutex);
+            revision = cursor_revision;
+            if (!cursor.is_null()) pointer = cursor.dump();
+          }
           for (auto it = tickets.begin(); it != tickets.end();)
             if (it->second.expires <= now)
               it = tickets.erase(it);
@@ -261,6 +271,16 @@ struct VideoServer::State {
               ++it;
           std::vector<Handle> close;
           for (auto &[h, c] : clients) {
+            if (c.authorized && c.cursor_revision != revision && !pointer.empty()) {
+              websocketpp::lib::error_code ec;
+              auto connection = server.get_con_from_hdl(h, ec);
+              // Coalesce pointer changes while a viewer is backpressured.
+              if (!ec && connection->get_buffered_amount() < 256 * 1024) {
+                server.send(h, pointer, websocketpp::frame::opcode::TEXT, ec);
+                if (!ec) c.cursor_revision = revision;
+              }
+              if (ec) close.push_back(h);
+            }
             if ((!c.authorized && now - c.opened >= 3s) ||
                 (!c.receipts.empty() && now - c.receipts.front().sent >= 10s)) {
               close.push_back(h);
@@ -468,6 +488,12 @@ void VideoServer::Submit(EncodedVideoFrame frame) {
   if (Enabled())
     state_->Enqueue({{}, std::move(frame)});
 }
+void VideoServer::Cursor(json cursor) {
+  auto &s = *state_;
+  std::lock_guard lock(s.mutex);
+  s.cursor = std::move(cursor);
+  ++s.cursor_revision;
+}
 json VideoServer::Status() const {
   const auto &s = *state_;
   std::lock_guard lock(s.mutex);

apps/cli/video_server.h

@@ -36,6 +36,7 @@ public:
   bool TakeKeyframeRequest();
   void Submit(std::shared_ptr<const DecodedFrame> frame);
   void Submit(EncodedVideoFrame frame);
+  void Cursor(nlohmann::json cursor);
   nlohmann::json Status() const;
 
 private:

apps/mcp/README.md

@@ -1,6 +1,6 @@
 # CrossDesk MCP
 
-让本机 Agent 通过 MCP 观察并操作远端 CrossDesk，同时为用户提供 MCP Apps 或浏览器监看面板。控制端不需要安装 CrossDesk 桌面客户端；被控端仍需运行兼容的 CrossDesk，并连接支持 `cli` 注册的信令服务器。
+让本机 Agent 通过 MCP 观察并操作远端 CrossDesk，同时为用户提供浏览器、终端或 MCP Apps 监看面板。控制端不需要安装 CrossDesk 桌面客户端；被控端仍需运行兼容的 CrossDesk，并连接支持 `cli` 注册的信令服务器。
 
 此目录实现 MCP 包 `@chunkdash/crossdesk-mcp` 0.1.0，绑定原生 CLI 包 0.2.0。**源码实现和本地打包不代表这些版本已经发布到 npm。** 发布后的安装示例需要以实际可用版本为准。
 
@@ -73,12 +73,27 @@ Windows 和 Linux 使用各自构建产物路径；`--native-binary` 必须指
 
 | 模式 | 行为 |
 | --- | --- |
-| `auto` | 当前请求声明 MCP Apps MIME 能力时提供 Apps；其他情况打开本机浏览器 |
+| `auto` | 有本机 GUI 时打开浏览器；无 GUI 且检测到终端环境时提供 CLI 画面入口；两者均无则使用 `off` |
 | `apps` | 只使用 MCP Apps；能力不匹配返回 `PANEL_UNSUPPORTED` |
 | `browser` | 只使用本地浏览器，不附加 Apps 工具资源元数据 |
+| `terminal` | 提供独立终端 `view` 命令，共用当前 MCP 会话，不启动网页服务 |
 | `off` | 不启动网页服务，保留截图和操作工具，凭据通过独立 `setup` 配置 |
 
-使用 `extensions["io.modelcontextprotocol/ui"].mimeTypes` 中的 `text/html;profile=mcp-app` 判断能力。旧协议读取 initialize 能力；2026-07-28 协议读取当前请求的能力，不按宿主产品名猜测支持情况。Apps 的卡片、侧栏或窗口形式由宿主决定。
+`auto` 不再优先选择 MCP Apps；需要宿主内嵌面板时显式使用 `--panel=apps`。Apps 使用 `extensions["io.modelcontextprotocol/ui"].mimeTypes` 中的 `text/html;profile=mcp-app` 判断能力。旧协议读取 initialize 能力；2026-07-28 协议读取当前请求的能力，不按宿主产品名猜测支持情况。Apps 的卡片、侧栏或窗口形式由宿主决定。
+
+Linux 根据 `DISPLAY` / `WAYLAND_DISPLAY` 判断 GUI；macOS 检查当前用户的桌面登录会话，SSH 会话不自动打开远端 Mac 的浏览器；Windows 排除 SSH 和 Services 会话。终端判断包含 TTY、`SSH_TTY` 和非 `dumb` 的 `TERM`，因为 MCP 的 stdin/stdout 通常是管道。环境变量被宿主清除或识别不适用时，可显式设置 `--panel=terminal` 或 `--panel=browser`。
+
+终端模式的 `crossdesk_open` / `crossdesk_connect` 返回 `terminal_command`。用户在同一主机的另一个交互终端执行（源码运行时将 `crossdesk-mcp` 替换为 `node /absolute/path/apps/mcp/dist/main.js`）：
+
+```sh
+crossdesk-mcp view --view <view_id> --config-dir <相同的配置目录>
+```
+
+该命令提供设备选择、隐藏密码输入和本次控制授权；已建立会话则直接显示画面。终端通过受限配置目录中的运行描述符和认证本地 IPC 复用 MCP 的原生连接，不启动第二个远程连接，也不占用 Agent 的终端或 MCP stdio。Agent 应展示命令供用户执行，再通过 `crossdesk_get_session` 等待；不能在工具输出捕获通道中运行 `view`。无交互终端时 `view` 返回 `VIEW_REQUIRES_TERMINAL`。
+
+终端画面采用原生 CLI 的 ANSI 四分块渲染器，目标 8 FPS，最多 160 列 × 60 行，根据终端环境使用 8/256/真彩色。默认字符宽高比 0.5，可用 `--cell-aspect`（0.2–2）调整。此入口暂不使用 Kitty/Sixel。`p` 暂停 Agent、`r` 恢复已授权控制、`d` 断开、`q` / Ctrl+C 仅关闭画面并恢复终端；关闭画面后会话保留。暂停仍由原生层锁定，Agent 不能自行恢复。
+
+浏览器/Apps 与终端均绘制 Agent 最后成功发送的鼠标位置，支持移动、点击、滚动和拖拽；坐标来自原生层对截图缩放及裁剪的映射。暂停或关闭控制时隐藏，切换显示器或图像尺寸变化时清除。该指针是 Agent 的操作位置提示，不代表远端输入执行 ACK，也不跟踪远端用户自己的鼠标。它不写入模型截图或源视频。
 
 Apps 资源为 `ui://crossdesk/viewer.html`，工具只在允许使用 Apps 且当前请求支持时暴露 UI 元数据和面板专用接口。页面回报 `crossdesk_ui_ready` 后才标记就绪；10 秒未回报显示 `not_confirmed`，不自动弹第二个浏览器。用户明确要求回退时调用 `crossdesk_open_browser`；该工具不附带 Apps 元数据，`apps`/`off` 策略拒绝覆盖。
 
@@ -193,6 +208,6 @@ npm run build
 npm run pack:npm -- --out-dir /absolute/path/artifacts
 ```
 
-脚本生成可发布 tarball，包含面板、运行代码、源代码、许可和固定版本的五个平台可选依赖。CI 的 `MCP package` 工作流仅构建并上传 `mcp-package`，不发布 npm。原生包继续由 CLI 工作流构建，`doctor` 必须同时声明自动化协议和 `automation_video_protocols: ["h264-annexb/1", "nv12/1"]` 和 `automation_video_decoders: ["auto", "software"]`，以及 `automation_video_transport: "loopback-websocket/1"`；先发布对应 0.2.0 平台包，再发布 MCP tarball。尚未进行真实跨平台发布/远端验收时，不应把构建成功等同于各宿主和设备上的联调完成。
+脚本生成可发布 tarball，包含面板、运行代码、源代码、许可和固定版本的五个平台可选依赖。CI 的 `MCP package` 工作流仅构建并上传 `mcp-package`，不发布 npm。原生包继续由 CLI 工作流构建，`doctor` 必须同时声明自动化协议和 `automation_video_protocols: ["h264-annexb/1", "nv12/1"]` 和 `automation_video_decoders: ["auto", "software"]`，以及 `automation_video_transport: "loopback-websocket/1"`、`automation_terminal_protocol: "ansi/1"`；先发布对应 0.2.0 平台包，再发布 MCP tarball。尚未进行真实跨平台发布/远端验收时，不应把构建成功等同于各宿主和设备上的联调完成。
 
 从发布 tarball 重建 TypeScript/面板时，将包中的 `build-package.json` 复制为 `package.json`，运行 `npm ci && npm run build`。原生源码和依赖来源随平台包提供，详见 [源码来源](../cli/docs/source-provenance.md)。

apps/mcp/src/browser.ts

@@ -17,6 +17,7 @@ import { fileURLToPath } from "node:url";
 import { Failure, token, equal, failure, type Data } from "./common.js";
 import { Sessions, type View } from "./sessions.js";
 import { uiAction } from "./ui-actions.js";
+import { displayEnvironment } from "./environment.js";
 
 const uiFile = fileURLToPath(new URL("./viewer-browser.html", import.meta.url));
 interface BrowserSession {
@@ -25,12 +26,7 @@ interface BrowserSession {
   expires: number;
 }
 export async function openURL(url: string): Promise<boolean> {
-  if (
-    process.platform === "linux" &&
-    !process.env.DISPLAY &&
-    !process.env.WAYLAND_DISPLAY
-  )
-    return false;
+  if (!displayEnvironment().gui) return false;
   const command =
     process.platform === "darwin"
       ? "open"

apps/mcp/src/common.ts

@@ -44,7 +44,7 @@ export const alias = z
   .max(80)
   .regex(/^[\p{L}\p{N} _.-]+$/u);
 export const mode = z.enum(["observe", "control"]);
-export const panel = z.enum(["auto", "apps", "browser", "off"]);
+export const panel = z.enum(["auto", "apps", "browser", "terminal", "off"]);
 export type Panel = z.infer<typeof panel>;
 export const videoOptions = z.strictObject({
   decode_path: z.enum(["browser", "minirtc"]).default("browser"),

apps/mcp/src/environment.ts

@@ -0,0 +1,25 @@
+import { statSync } from "node:fs";
+
+// MCP stdio is normally piped even when its host is an interactive terminal.
+// Detect the desktop independently, without opening or taking over /dev/tty.
+export function displayEnvironment(
+  platform = process.platform,
+  env = process.env,
+  tty = !!(process.stdin.isTTY || process.stdout.isTTY || process.stderr.isTTY),
+): { gui: boolean; terminal: boolean } {
+  const ssh = !!(env.SSH_CONNECTION || env.SSH_TTY || env.SSH_CLIENT);
+  let gui = false;
+  if (platform === "linux") gui = !!(env.DISPLAY || env.WAYLAND_DISPLAY);
+  else if (platform === "darwin" && !ssh) {
+    try {
+      const uid = statSync("/dev/console").uid;
+      gui = uid !== 0 && uid === process.getuid?.();
+    } catch { /* No logged-in desktop session. */ }
+  } else if (platform === "win32") {
+    gui = !ssh && env.SESSIONNAME?.toLowerCase() !== "services";
+  }
+  return {
+    gui,
+    terminal: tty || !!env.SSH_TTY || (!!env.TERM && env.TERM !== "dumb"),
+  };
+}

apps/mcp/src/main.ts

@@ -10,6 +10,8 @@ import { DeviceStore, type Device } from "./store.js";
 import { Sessions } from "./sessions.js";
 import { BrowserPanel, pairBrowser, openURL } from "./browser.js";
 import { createServer } from "./server.js";
+import { TerminalPanel } from "./terminal.js";
+import { terminalView } from "./terminal-view.js";
 import {
   panel,
   parse,
@@ -120,6 +122,7 @@ async function main() {
       "native-binary": { type: "string" },
       "open-panel": { type: "boolean" },
       view: { type: "string" },
+      "cell-aspect": { type: "string" },
       help: { type: "boolean" },
       version: { type: "boolean" },
     },
@@ -130,16 +133,21 @@ async function main() {
   }
   if (values.help || !positionals.length) {
     process.stdout.write(
-      "crossdesk-mcp serve --stdio [--panel auto|apps|browser|off] [--video-path browser|minirtc] [--video-decoder auto|software] [--config-dir directory]\ncrossdesk-mcp setup [--config-dir directory]\ncrossdesk-mcp setup --open-panel --view VIEW_ID [--config-dir directory]\nDevelopment: --native-binary /absolute/path/to/crossdesk-cli\n",
+      "crossdesk-mcp serve --stdio [--panel auto|apps|browser|terminal|off] [--video-path browser|minirtc] [--video-decoder auto|software] [--config-dir directory]\ncrossdesk-mcp view --view VIEW_ID [--config-dir directory] [--cell-aspect 0.5]\ncrossdesk-mcp setup [--config-dir directory]\ncrossdesk-mcp setup --open-panel --view VIEW_ID [--config-dir directory]\nDevelopment: --native-binary /absolute/path/to/crossdesk-cli\n",
     );
     return;
   }
-  if (positionals.length !== 1 || !["serve", "setup"].includes(positionals[0]!))
+  if (positionals.length !== 1 || !["serve", "setup", "view"].includes(positionals[0]!))
     throw new Failure("INVALID_COMMAND");
   const store = new DeviceStore(
     values["config-dir"] ?? process.env.CROSSDESK_MCP_CONFIG_DIR,
   );
   await store.initialize();
+  if (positionals[0] === "view") {
+    if (!values.view) throw new Failure("VIEW_REQUIRED");
+    await terminalView(store.directory, values.view, Number(values["cell-aspect"] ?? 0.5));
+    return;
+  }
   if (positionals[0] === "setup") {
     if (values["open-panel"] && !values.view)
       throw new Failure("VIEW_REQUIRED");
@@ -162,7 +170,8 @@ async function main() {
     }),
   );
   const browser = new BrowserPanel(sessions);
-  const handle = serveStdio(() => createServer(sessions, browser), {
+  const terminal = new TerminalPanel(sessions);
+  const handle = serveStdio(() => createServer(sessions, browser, terminal), {
     transport: new StdioServerTransport(),
     onerror: () => process.stderr.write("crossdesk-mcp: protocol error\n"),
   });
@@ -170,6 +179,7 @@ async function main() {
   const close = async () => {
     if (closing) return;
     closing = true;
+    await terminal.close();
     await browser.close();
     await sessions.close();
     await handle.close();

apps/mcp/src/native.ts

@@ -74,6 +74,7 @@ export async function nativeBinary(override?: string): Promise<string> {
     });
     const doctor = JSON.parse(stdout);
     if (doctor.automation_protocol !== "crossdesk-automation/1" ||
+        doctor.automation_terminal_protocol !== "ansi/1" ||
         doctor.automation_video_transport !== "loopback-websocket/1" ||
         !["h264-annexb/1", "nv12/1"].every((p) => doctor.automation_video_protocols?.includes(p)) ||
         !["auto", "software"].every((p) => doctor.automation_video_decoders?.includes(p)))
@@ -144,6 +145,7 @@ export class NativeClient {
       signal,
     );
     if (hello.protocol !== "crossdesk-automation/1" ||
+        hello.terminal_protocol !== "ansi/1" ||
         hello.video_transport !== "loopback-websocket/1" ||
         !hello.video_protocols?.includes(this.videoOptions.decode_path === "minirtc" ? "nv12/1" : "h264-annexb/1") ||
         !hello.video_decoders?.includes(this.videoOptions.decoder))

apps/mcp/src/server.ts

@@ -15,6 +15,7 @@ import {
 } from "@modelcontextprotocol/ext-apps/server";
 import { Sessions, type View } from "./sessions.js";
 import { BrowserPanel } from "./browser.js";
+import { TerminalPanel } from "./terminal.js";
 import { uiAction, uiSchemas, type UIAction } from "./ui-actions.js";
 import {
   id,
@@ -60,6 +61,7 @@ function output(value: Data, meta?: Data): CallToolResult {
 export function createServer(
   sessions: Sessions,
   browser: BrowserPanel,
+  terminal: TerminalPanel,
 ): McpServer {
   const server = new McpServer(
     { name: "crossdesk-mcp", version: "0.1.0" },
@@ -91,7 +93,9 @@ export function createServer(
     );
   };
   const apps = (ctx: ServerContext) =>
-    ["auto", "apps"].includes(sessions.policy) && supported(ctx);
+    sessions.policy === "apps" && supported(ctx);
+  const openPanel = async (v: View) => v.panel === "browser" ? browser.open(v)
+    : v.panel === "terminal" ? terminal.open(v) : sessions.describe(v);
   const viewResult = (v: View, result: Data) =>
     output(
       result,
@@ -147,7 +151,7 @@ export function createServer(
   add(
     "crossdesk_open",
     z.strictObject({ device_alias: alias.optional() }),
-    "Open CrossDesk so the user can choose a device, enter its password, and authorize this session. Returns immediately so the panel can render. To continue a pending remote task, call crossdesk_get_session with the returned view_id and wait_ms: 25000; no chat confirmation is needed.",
+    "Open CrossDesk so the user can choose a device, enter its password, and authorize this session. GUI environments use a browser; headless terminal environments return terminal_command, which you must show to the user to run in a separate terminal. Never run the terminal viewer in a captured tool-output channel. Returns immediately. To continue a pending remote task, call crossdesk_get_session with the returned view_id and wait_ms: 25000; no chat confirmation is needed.",
     async (p, ctx) => {
       if (p.device_alias) await sessions.store.device(p.device_alias);
       const v = sessions.create(
@@ -156,7 +160,7 @@ export function createServer(
       );
       return viewResult(
         v,
-        v.panel === "browser" ? await browser.open(v) : sessions.describe(v),
+        await openPanel(v),
       );
     },
     { ui: true },
@@ -209,9 +213,7 @@ export function createServer(
       v.session = result.session_id;
       return viewResult(v, {
         ...result,
-        ...(selected === "browser"
-          ? await browser.open(v)
-          : sessions.describe(v)),
+        ...await openPanel(v),
         state: result.state,
       });
     },

apps/mcp/src/sessions.ts

@@ -12,6 +12,7 @@ import {
 } from "./common.js";
 import { DeviceStore, type Device } from "./store.js";
 import { nativeBinary, NativeClient } from "./native.js";
+import { displayEnvironment } from "./environment.js";
 
 export interface View {
   id: string;
@@ -46,7 +47,9 @@ export class Sessions {
   ) {}
   choose(apps: boolean): View["panel"] {
     if (this.policy === "apps" && !apps) throw new Failure("PANEL_UNSUPPORTED");
-    return this.policy === "auto" ? (apps ? "apps" : "browser") : this.policy;
+    if (this.policy !== "auto") return this.policy;
+    const environment = displayEnvironment();
+    return environment.gui ? "browser" : environment.terminal ? "terminal" : "off";
   }
   create(panel: View["panel"], alias?: string): View {
     for (const [id, view] of this.views)
@@ -383,6 +386,12 @@ export class Sessions {
       operation,
     });
   }
+  async terminalFrame(v: View, args: Data, signal?: AbortSignal) {
+    if (v.panel !== "terminal" || !v.session || this.connecting)
+      throw new Failure("NOT_READY");
+    const s = this.session(v.session);
+    return s.native.request("terminal_frame", args, 5000, signal);
+  }
   async openVideo(v: View, origin: string): Promise<Data> {
     if (!v.session || this.views.get(v.id) !== v) throw new Failure("NOT_READY");
     const s = this.session(v.session);

apps/mcp/src/terminal-view.ts

@@ -0,0 +1,125 @@
+import { createInterface } from "node:readline/promises";
+import { emitKeypressEvents } from "node:readline";
+import { Writable } from "node:stream";
+import { setTimeout as delay } from "node:timers/promises";
+import { Failure, failure, type Data } from "./common.js";
+import { TerminalClient } from "./terminal.js";
+
+async function connect(client: TerminalClient) {
+  let hidden = false;
+  const output = new Writable({ write(chunk, _encoding, done) {
+    if (!hidden) process.stdout.write(chunk);
+    done();
+  } });
+  const rl = createInterface({ input: process.stdin, output, terminal: true, historySize: 0 });
+  const ask = async (label: string) => (await rl.question(label)).trim();
+  try {
+    const { devices, defaults } = await client.call("devices");
+    const status = await client.call("status");
+    process.stdout.write("CrossDesk MCP · 终端画面\n");
+    for (const d of devices)
+      process.stdout.write(`${d.device_alias} (${d.device_id})\n`);
+    const selected = (await ask(`设备别名或 9 位 ID${status.device_alias ? ` [${status.device_alias}]` : ""}：`)) || status.device_alias;
+    const device = devices.find((d: Data) => d.device_alias === selected);
+    const args: Data = device ? { device_alias: selected } : {
+      device_id: selected,
+      server: (await ask(`服务器 [${defaults.server}]：`)) || defaults.server,
+      port: Number((await ask(`端口 [${defaults.port}]：`)) || defaults.port),
+    };
+    args.mode = (await ask("允许 Agent 操作？[y/N]：")).toLowerCase() === "y" ? "control" : "observe";
+    args.display_index = Number((await ask("显示器索引 [0]：")) || "0");
+    const saved = device?.credential_provider !== "none" && !!device;
+    if (!saved || (await ask("使用已保存凭据？[Y/n]：")).toLowerCase() === "n") {
+      process.stdout.write("访问密码（隐藏输入，可为空）：");
+      hidden = true;
+      try { args.password = await rl.question(""); }
+      finally { hidden = false; process.stdout.write("\n"); }
+    }
+    process.stdout.write("正在连接…\n");
+    try { await client.call("connect", args); }
+    finally { args.password = undefined; }
+  } finally { rl.close(); }
+}
+
+export async function terminalView(directory: string, view: string, cellAspect = 0.5) {
+  if (!process.stdin.isTTY || !process.stdout.isTTY || process.env.TERM === "dumb")
+    throw new Failure("VIEW_REQUIRES_TERMINAL");
+  if (!Number.isFinite(cellAspect) || cellAspect < 0.2 || cellAspect > 2)
+    throw new Failure("INVALID_ARGUMENT");
+  const client = await TerminalClient.attach(directory, view);
+  let raw = false, stopped = false, controls = false, message = "";
+  const previousRaw = process.stdin.isRaw;
+  const stop = () => { stopped = true; };
+  const keypress = (_text: string, key: { name?: string; ctrl?: boolean; meta?: boolean; sequence?: string }) => {
+    if (key.name === "q" || (key.ctrl && key.name === "c")) { stop(); return; }
+    if (controls || key.ctrl || key.meta || !["p", "r", "d"].includes(key.sequence ?? "")) return;
+    controls = true;
+    void client.call("control", { operation: key.sequence === "p" ? "pause" : key.sequence === "r" ? "resume" : "disconnect" })
+      .then(() => { message = ""; if (key.sequence === "d") stopped = true; })
+      .catch((error) => { message = failure(error).code; })
+      .finally(() => { controls = false; });
+  };
+  const write = (value: string) => new Promise<void>((resolve, reject) => {
+    process.stdout.write(value, (error) => error ? reject(error) : resolve());
+  });
+  try {
+    let status = await client.call("status");
+    if (!status.session_id) await connect(client);
+    process.stdin.setRawMode(true);
+    raw = true;
+    emitKeypressEvents(process.stdin);
+    process.stdin.on("keypress", keypress);
+    process.stdin.resume();
+    for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) process.on(signal, stop);
+    await write("\x1b[?1049h\x1b[?25l\x1b[?7l\x1b[2J");
+    let statusAt = 0, size = "";
+    const colors = /truecolor|24bit/i.test(process.env.COLORTERM ?? "") ? 24
+      : /256color/i.test(process.env.TERM ?? "") ? 256 : 8;
+    while (!stopped) {
+      const started = Date.now();
+      if (started - statusAt >= 500) {
+        status = await client.call("status", {}, 7000);
+        statusAt = Date.now();
+        if (["closed", "failed", "disconnected"].includes(status.state)) break;
+      }
+      // Leave a spare column so terminals with unusual wrap behavior stay safe.
+      const columns = Math.min(160, Math.max(1, (process.stdout.columns || 80) - 1));
+      const rows = Math.min(60, Math.max(1, (process.stdout.rows || 24) - 2));
+      const nextSize = `${columns}:${rows}`;
+      let screen = nextSize !== size ? "\x1b[2J" : "";
+      size = nextSize;
+      try {
+        const frame = await client.call("frame", { columns, rows, colors, cell_aspect: cellAspect }, 7000);
+        if (frame.columns !== columns || frame.rows !== rows || !Array.isArray(frame.lines) || frame.lines.length !== rows)
+          throw new Failure("INVALID_FRAME");
+        // Only renderer-generated SGR sequences and single-width block glyphs.
+        for (const [index, line] of frame.lines.entries()) {
+          if (typeof line !== "string" || !/^(?:\x1b\[[0-9;]*m|[ \u2580-\u259f])*$/u.test(line))
+            throw new Failure("INVALID_FRAME");
+          screen += `\x1b[${index + 1};1H${line}\x1b[0m\x1b[K`;
+        }
+        const cursor = frame.cursor;
+        if (cursor && Number.isInteger(cursor.column) && Number.isInteger(cursor.row) &&
+            cursor.column >= 0 && cursor.column < columns && cursor.row >= 0 && cursor.row < rows)
+          screen += `\x1b[${cursor.row + 1};${cursor.column + 1}H\x1b[97;44m↖\x1b[0m`;
+      } catch (error) {
+        if (!["NOT_READY", "BUSY", "FRAME_TIMEOUT", "STALE_OBSERVATION"].includes(failure(error).code)) throw error;
+      }
+      const label = status.human_paused ? "PAUSED" : status.control ? "AGENT CONTROL" : "OBSERVE";
+      const bar = `${label} | p pause / r resume / d disconnect / q close view${message ? ` | ${message}` : ""}`;
+      // No remote strings or credentials are interpolated into terminal control output.
+      screen += `\x1b[${rows + 1};1H\x1b[0m${bar.replace(/[^\x20-\x7e]/g, "").slice(0, columns)}\x1b[K`;
+      await write(screen);
+      await delay(Math.max(0, 125 - (Date.now() - started)));
+    }
+  } finally {
+    process.stdin.off("keypress", keypress);
+    for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) process.off(signal, stop);
+    if (raw) {
+      process.stdin.setRawMode(previousRaw);
+      process.stdin.pause();
+      process.stdout.write("\x1b[0m\x1b[?7h\x1b[?25h\x1b[?1049l");
+    }
+    await client.call("detach", {}, 1000).catch(() => {});
+  }
+}

apps/mcp/src/terminal.ts

@@ -0,0 +1,171 @@
+import { createServer, createConnection, type Server, type Socket } from "node:net";
+import { chmod, readFile, readdir, unlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { z } from "zod";
+import { Failure, equal, failure, id, parse, token, type Data } from "./common.js";
+import { Sessions, type View } from "./sessions.js";
+import { uiAction } from "./ui-actions.js";
+
+const frameSchema = z.strictObject({
+  columns: z.int().min(1).max(160),
+  rows: z.int().min(1).max(60),
+  colors: z.union([z.literal(8), z.literal(24), z.literal(256)]),
+  cell_aspect: z.number().finite().min(0.2).max(2),
+});
+
+// A private local channel, separate from MCP stdio and browser HTTP. Only the
+// user's terminal receives rendered frames and connection credentials.
+export class TerminalPanel {
+  private server?: Server;
+  private starting?: Promise<void>;
+  private endpoint?: string;
+  private descriptor?: string;
+  private closed = false;
+  private sockets = new Set<Socket>();
+  private frames = new Set<string>();
+  private frameTimes = new Map<string, number>();
+  constructor(private sessions: Sessions) {}
+
+  async open(view: View): Promise<Data> {
+    await this.start();
+    for (const id of this.frameTimes.keys())
+      if (!this.sessions.views.has(id)) this.frameTimes.delete(id);
+    return {
+      ...this.sessions.describe(view),
+      terminal_command: {
+        command: "crossdesk-mcp",
+        args: ["view", "--view", view.id, "--config-dir", this.sessions.store.directory],
+      },
+      next_step: "Show terminal_command to the user to run in a separate interactive terminal on this host. It displays the existing MCP session and accepts credentials locally. Never run it inside MCP stdio or capture its video into chat. Then wait with crossdesk_get_session(view_id, wait_ms: 25000).",
+    };
+  }
+
+  private async start() {
+    if (this.closed) throw new Failure("PANEL_CLOSED");
+    this.starting ??= (async () => {
+      const secret = token(), suffix = token().slice(0, 20);
+      this.endpoint = process.platform === "win32"
+        ? `\\\\.\\pipe\\crossdesk-terminal-${suffix}`
+        : join(tmpdir(), `cd-term-${suffix}.sock`);
+      this.server = createServer((socket) => {
+        this.sockets.add(socket);
+        const controller = new AbortController();
+        socket.on("error", () => {});
+        socket.once("close", () => {
+          this.sockets.delete(socket);
+          controller.abort();
+        });
+        socket.setTimeout(45000, () => socket.destroy());
+        let buffer = Buffer.alloc(0), consumed = false;
+        socket.on("data", (chunk: Buffer) => {
+          if (consumed) return socket.destroy();
+          buffer = Buffer.concat([buffer, chunk]);
+          if (buffer.length > 65536) return socket.destroy();
+          if (!buffer.includes(10)) return;
+          consumed = true;
+          void (async () => {
+            try {
+              const p = JSON.parse(buffer.toString("utf8"));
+              buffer = Buffer.alloc(0);
+              if (typeof p.secret !== "string" || !equal(secret, p.secret))
+                throw new Failure("PANEL_UNAUTHORIZED");
+              const view = this.sessions.view(parse(id, p.view_id));
+              if (view.panel !== "terminal" || !["auto", "terminal"].includes(this.sessions.policy))
+                throw new Failure("PANEL_POLICY_DENIED");
+              view.lastSeen = Date.now();
+              let result: Data;
+              if (p.method === "frame") {
+                const args = parse(frameSchema, p.args);
+                if (this.frames.has(view.id) || Date.now() - (this.frameTimes.get(view.id) ?? 0) < 100)
+                  throw new Failure("BUSY", false, true);
+                this.frames.add(view.id);
+                this.frameTimes.set(view.id, Date.now());
+                try { result = await this.sessions.terminalFrame(view, args, controller.signal); }
+                finally { this.frames.delete(view.id); }
+              } else if (p.method === "detach") {
+                view.ready = false;
+                result = this.sessions.describe(view);
+              } else {
+                result = await uiAction(this.sessions, view, p.method, p.args, controller.signal);
+              }
+              socket.end(JSON.stringify({ result }) + "\n");
+            } catch (error) {
+              socket.end(JSON.stringify({ error: failure(error) }) + "\n");
+            }
+          })();
+        });
+      });
+      await new Promise<void>((resolve, reject) => {
+        this.server!.once("error", reject);
+        this.server!.listen(this.endpoint!, resolve);
+      });
+      if (process.platform !== "win32") await chmod(this.endpoint, 0o600);
+      this.descriptor = join(this.sessions.store.directory, `terminal-${process.pid}-${suffix}.json`);
+      await writeFile(this.descriptor, JSON.stringify({ endpoint: this.endpoint, secret }), { mode: 0o600, flag: "wx" });
+      if (this.closed) throw new Failure("PANEL_CLOSED");
+    })().catch(async (error) => {
+      await this.cleanup();
+      this.starting = undefined;
+      throw error;
+    });
+    return this.starting;
+  }
+
+  private async cleanup() {
+    for (const socket of this.sockets) socket.destroy();
+    if (this.server?.listening) this.server.close();
+    if (this.descriptor) await unlink(this.descriptor).catch(() => {});
+    if (this.endpoint && process.platform !== "win32") await unlink(this.endpoint).catch(() => {});
+  }
+  async close() {
+    this.closed = true;
+    await this.starting?.catch(() => {});
+    await this.cleanup();
+  }
+}
+
+export class TerminalClient {
+  private constructor(private endpoint: string, private secret: string, private view: string) {}
+  static async attach(directory: string, view: string) {
+    parse(id, view);
+    for (const file of await readdir(directory)) {
+      if (!/^terminal-\d+-[a-zA-Z0-9_-]+\.json$/.test(file)) continue;
+      try {
+        const d = JSON.parse(await readFile(join(directory, file), "utf8"));
+        const client = new TerminalClient(d.endpoint, d.secret, view);
+        await client.call("ready", {}, 2000);
+        return client;
+      } catch { /* An unrelated or exited MCP instance. */ }
+    }
+    throw new Failure("PANEL_NOT_FOUND");
+  }
+  call(method: string, args: Data = {}, timeout = 45000): Promise<Data> {
+    return new Promise((resolve, reject) => {
+      const socket = createConnection(this.endpoint);
+      let buffer = Buffer.alloc(0), settled = false;
+      const finish = (error?: Failure, result?: Data) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        socket.destroy();
+        if (error) reject(error); else resolve(result!);
+      };
+      const timer = setTimeout(() => finish(new Failure("PANEL_TIMEOUT")), timeout);
+      socket.on("error", () => finish(new Failure("PANEL_CLOSED")));
+      socket.once("close", () => finish(new Failure("PANEL_CLOSED")));
+      socket.once("connect", () => socket.write(JSON.stringify({ secret: this.secret, view_id: this.view, method, args }) + "\n"));
+      socket.on("data", (chunk: Buffer) => {
+        buffer = Buffer.concat([buffer, chunk]);
+        if (buffer.length > 2 * 1024 * 1024) return finish(new Failure("MESSAGE_TOO_LARGE"));
+        if (!buffer.includes(10)) return;
+        try {
+          const reply = JSON.parse(buffer.toString("utf8"));
+          if (reply.error) finish(new Failure(reply.error.code));
+          else if (reply.result && typeof reply.result === "object") finish(undefined, reply.result);
+          else finish(new Failure("INVALID_RESPONSE"));
+        } catch { finish(new Failure("INVALID_RESPONSE")); }
+      });
+    });
+  }
+}

apps/mcp/ui/video-player.ts

@@ -1,4 +1,6 @@
 type FrameInfo = { width: number; height: number; fps: number; frames: number };
+type Cursor = { type: "cursor"; visible: boolean; x: number; y: number;
+  generation: number; geometry_epoch: number; width: number; height: number };
 function parameters(data: Uint8Array) {
   let codec = "", signature = "";
   for (let i = 0; i + 4 < data.length; i++) {
@@ -43,9 +45,18 @@ export class VideoPlayer {
   private raw = false;
   private software = false;
   private automatic = true;
+  private cursor?: Cursor;
+  private cursorDrawn = "";
+  private resize?: ResizeObserver;
   constructor(private canvas: HTMLCanvasElement,
     private onFrame: (info: FrameInfo) => void,
-    private onError: (error: Error) => void) {}
+    private onError: (error: Error) => void,
+    private overlay?: HTMLCanvasElement) {
+    if (overlay) {
+      this.resize = new ResizeObserver(() => this.drawCursor());
+      this.resize.observe(overlay);
+    }
+  }
 
   start(url: string, ticket: string, protocol = "h264-annexb/1", decoder = "auto") {
     if (!["h264-annexb/1", "nv12/1"].includes(protocol) ||
@@ -63,7 +74,24 @@ export class VideoPlayer {
     socket.onclose = () => { if (!this.stopped) this.fail(new Error("VIDEO_UNAVAILABLE")); };
     this.firstFrame = setTimeout(() => this.fail(new Error("VIDEO_UNAVAILABLE")), 10000);
     socket.onmessage = (event) => {
-      if (this.stopped || !(event.data instanceof ArrayBuffer)) return;
+      if (this.stopped) return;
+      if (typeof event.data === "string") {
+        try {
+          if (event.data.length > 1024) throw new Error();
+          const p = JSON.parse(event.data) as Cursor;
+          if (p.type !== "cursor" || typeof p.visible !== "boolean" ||
+              !Number.isFinite(p.x) || !Number.isFinite(p.y) || p.x < 0 || p.x > 1 || p.y < 0 || p.y > 1 ||
+              !Number.isSafeInteger(p.generation) || p.generation < 0 ||
+              !Number.isSafeInteger(p.geometry_epoch) || p.geometry_epoch < 0 ||
+              !Number.isInteger(p.width) || !Number.isInteger(p.height) ||
+              p.width < 0 || p.width > 8192 || p.height < 0 || p.height > 8192)
+            throw new Error();
+          this.cursor = p;
+          this.drawCursor();
+        } catch { this.fail(new Error("VIDEO_INVALID")); }
+        return;
+      }
+      if (!(event.data instanceof ArrayBuffer)) return;
       const data = event.data;
       if (data.byteLength < (this.raw ? 38 : 25) ||
           data.byteLength > (this.raw ? 16777216 * 3 / 2 + 32 : 4 * 1024 * 1024 + 24))
@@ -196,6 +224,7 @@ export class VideoPlayer {
       this.canvas.getContext("2d", { alpha: false })!.drawImage(frame, 0, 0);
       clearTimeout(this.firstFrame);
       this.count++;
+      this.drawCursor();
       this.samples++;
       const elapsed = performance.now() - this.sampledAt;
       if (elapsed >= 1000) {
@@ -208,6 +237,45 @@ export class VideoPlayer {
     } catch { this.fail(new Error("VIDEO_INVALID")); }
     finally { frame.close(); }
   }
+  private drawCursor() {
+    const overlay = this.overlay;
+    if (!overlay) return;
+    if (overlay.width !== this.canvas.width) overlay.width = this.canvas.width;
+    if (overlay.height !== this.canvas.height) overlay.height = this.canvas.height;
+    const p = this.cursor;
+    const state = JSON.stringify([this.stopped, !!this.count, this.generation,
+      overlay.width, overlay.height, overlay.clientWidth, overlay.clientHeight, p]);
+    if (state === this.cursorDrawn) return;
+    this.cursorDrawn = state;
+    const context = overlay.getContext("2d")!;
+    context.clearRect(0, 0, overlay.width, overlay.height);
+    if (this.stopped || !this.count || !p?.visible || p.generation !== this.generation ||
+        p.width !== overlay.width || p.height !== overlay.height) return;
+    // Match the video canvas's object-fit: contain, including letterboxing.
+    const scale = Math.min(overlay.clientWidth / overlay.width, overlay.clientHeight / overlay.height);
+    if (!scale) return;
+    context.save();
+    context.translate(Math.min(overlay.width - 1, p.x * overlay.width), Math.min(overlay.height - 1, p.y * overlay.height));
+    context.scale(1 / scale, 1 / scale);
+    context.beginPath();
+    context.moveTo(0, 0);
+    context.lineTo(0, 22);
+    context.lineTo(6, 17);
+    context.lineTo(10, 26);
+    context.lineTo(14, 24);
+    context.lineTo(10, 15);
+    context.lineTo(18, 15);
+    context.closePath();
+    context.fillStyle = "#3269e8";
+    context.strokeStyle = "#fff";
+    context.lineWidth = 2;
+    context.lineJoin = "round";
+    context.shadowColor = "#0009";
+    context.shadowBlur = 3;
+    context.fill();
+    context.stroke();
+    context.restore();
+  }
   private fail(error: Error) {
     if (this.failed || this.stopped) return;
     this.failed = true;
@@ -216,6 +284,9 @@ export class VideoPlayer {
   }
   stop() {
     this.stopped = true;
+    this.cursor = undefined;
+    this.resize?.disconnect();
+    this.drawCursor();
     clearTimeout(this.firstFrame);
     this.socket?.close();
     if (this.decoder && this.decoder.state !== "closed") this.decoder.close();

apps/mcp/ui/viewer.css

@@ -82,6 +82,7 @@ button:disabled { cursor: default; color: #a3a7ad; background: #fafafa; border-c
 /* The screen takes the remaining panel height; object-fit preserves the remote aspect ratio. */
 .screen { position: relative; flex: 1; min-height: 60px; width: 100%; display: flex; align-items: center; justify-content: center; overflow: hidden; isolation: isolate; border-radius: 8px; background: #17191c; }
 .screen canvas { position: absolute; inset: 0; width: 100%; height: 100%; object-fit: contain; }
+#agent-cursor { pointer-events: none; z-index: 4; }
 /* Overlay the control indicator so toggling control never resizes the video. */
 .control-halo { display: none; }
 #app[data-controlling="true"] .screen::before,

apps/mcp/ui/viewer.ts

@@ -55,6 +55,7 @@ root.innerHTML = `
       </div>
       <div class="screen" id="screen">
         <canvas id="frame" role="img" aria-label="远程桌面实时画面" hidden></canvas>
+        <canvas id="agent-cursor" aria-hidden="true"></canvas>
         <div class="control-halo" aria-hidden="true"></div>
         <div id="empty"><h3 id="empty-title">未连接设备</h3><p id="empty-description">选择设备并输入访问密码。</p></div>
         <div class="stale" id="stale" role="status" hidden>画面暂未更新</div>
@@ -393,7 +394,7 @@ async function startPreview() {
       element<HTMLButtonElement>("focus-view").disabled = false;
       text("video-state", "实时");
       element("video-state").dataset.state = "live";
-    }, failed);
+    }, failed, element<HTMLCanvasElement>("agent-cursor"));
     player.start(grant.url, grant.ticket, grant.protocol, grant.decoder);
   } catch (error) {
     failed(error instanceof Error ? error : new Error("VIDEO_UNAVAILABLE"));

libs/controller/include/controller_session.h

@@ -76,6 +76,7 @@ public:
   std::optional<DecodedFrame> TakeFrame(DecodedFrame *recycle = nullptr);
   nlohmann::json Status() const;
   bool Connected() const { return state_ == "connected"; }
+  bool ControlEnabled() const { return control_; }
   bool PreviewAccepted() const { return preview_accepted_; }
   bool SupportsPreviewFps15() const { return fps15_; }
   bool Finished() const { return state_ == "failed" || state_ == "closed"; }