From 504759908424fb5a459c8565e8ee34441bb7d337 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Thu, 12 Nov 2026 01:43:04 +0800 Subject: [PATCH] [feat] add batched actions and agent monitoring for MCP --- apps/cli/agent.cpp | 61 +++++- apps/cli/automation_protocol.cpp | 24 +++ apps/cli/automation_protocol.h | 2 + apps/cli/docs/automation-protocol.md | 6 + apps/mcp/README.md | 35 +++- apps/mcp/src/common.ts | 27 ++- apps/mcp/src/native.ts | 4 + apps/mcp/src/server.ts | 8 +- apps/mcp/src/sessions.ts | 56 +++++- apps/mcp/ui/viewer.css | 285 ++++++++++++++++----------- apps/mcp/ui/viewer.ts | 149 +++++++++++--- 11 files changed, 496 insertions(+), 161 deletions(-) diff --git a/apps/cli/agent.cpp b/apps/cli/agent.cpp index 25c49115..d1d776d1 100644 --- a/apps/cli/agent.cpp +++ b/apps/cli/agent.cpp @@ -77,6 +77,7 @@ struct FrameWait { bool stable; int stable_ms; uint64_t signature, frame; + uint64_t after_frame = 0; Clock::time_point deadline, changed; }; struct RemoteJob { @@ -117,10 +118,14 @@ struct RunningAction { std::string operation; uint64_t generation = 0, epoch = 0; std::vector steps; + std::deque> remaining; + size_t action_index = 0, action_count = 1; + int interval_ms = 0; + bool sequence = false, unacknowledged = false; size_t next = 0; bool submitted = false; bool retain_input = false; - Clock::time_point started, awaiting_ack{}; + Clock::time_point started, total_started, awaiting_ack{}; }; struct Record { std::string operation, arguments; @@ -213,6 +218,8 @@ private: result["human_input_epoch"] = human_epoch_; result["viewer_input_protocol"] = "events/1"; result["control_allowed"] = allowed_; + result["action_sequence_protocol"] = "sequence/1"; + result["wait_after_frame_supported"] = true; result["frame_age_ms"] = frame_ ? json(Age(frame_->queued)) : json(nullptr); result["source_size"] = frame_ ? json{{"width", frame_->width}, {"height", frame_->height}} @@ -297,18 +304,37 @@ private: } if (!error && remote == "failed") error = "REMOTE_INPUT_FAILED"; if (error) session_.ReleaseInput(); + if (remote == "unsupported") action_->unacknowledged = true; + if (!error && !action_->remaining.empty()) { + // The full sequence was validated before the first event. Only advance + // after this action's input receipts; never replay an uncertain action. + action_->steps = std::move(action_->remaining.front()); + action_->remaining.pop_front(); + ++action_->action_index; + action_->next = 0; + action_->awaiting_ack = {}; + session_.BeginInput(); + action_->started = Clock::now() + Ms(action_->interval_ms); + return; + } if (session_.HasPressedInput()) input_lease_ = Clock::now() + std::chrono::seconds(15); json result = { {"status", error ? (action_->submitted ? "outcome_unknown" : "failed_before_submit") : "submitted"}, {"operation_id", action_->operation}, - {"remote_execution", remote}, + {"remote_execution", !error && action_->unacknowledged ? "unsupported" : remote}, {"input_lease_ms", session_.HasPressedInput() ? 15000 : 0}, {"generation", action_->generation}, {"after_frame_id", frame_ ? frame_->frame_id : 0}, {"side_effect_possible", action_->submitted}, {"retryable", false}}; + result["timing_ms"] = {{"input", Age(action_->total_started)}}; + if (action_->sequence) { + result["sequence"] = {{"action_count", action_->action_count}, + {"finished_actions", action_->action_index + (error ? 0 : 1)}}; + if (error) result["sequence"]["stopped_at"] = action_->action_index; + } if (error) result["error"] = error; for (auto &record : records_) @@ -319,6 +345,8 @@ private: Reply(action_->id, result); for (auto &step : action_->steps) std::fill(step.text.begin(), step.text.end(), '\0'); + for (auto &steps : action_->remaining) + for (auto &step : steps) std::fill(step.text.begin(), step.text.end(), '\0'); action_.reset(); } void FinishAudio(const char *error = nullptr) { @@ -411,10 +439,13 @@ private: preview_ = json(); } if (action_) { - if (!session_.Connected() || !frame_ || paused_ || !allowed_ || + if (!session_.Connected() || !frame_ || paused_ || !allowed_ || human_control_ || + !session_.ControlEnabled() || action_->generation != session_.Generation() || action_->epoch != epoch_) FinishAction("SESSION_CHANGED"); + if (action_ && action_->sequence && Age(action_->total_started) >= 25000) + FinishAction("SEQUENCE_TIMEOUT"); while (action_ && action_->next < action_->steps.size() && Age(action_->started) >= action_->steps[action_->next].delay) { auto &step = action_->steps[action_->next++]; @@ -448,7 +479,7 @@ private: continue; } bool matched = false; - if (frame_->frame_id != it->frame && Age(frame_->queued) <= 2000) { + if (frame_->frame_id > it->after_frame && frame_->frame_id != it->frame && Age(frame_->queued) <= 2000) { it->frame = frame_->frame_id; const auto signature = FrameSignature(*frame_, it->observation.region); if (signature != it->signature) { @@ -741,7 +772,21 @@ private: running->epoch = o.epoch; if (type == "type_text" && !session_.Status()["text_input"].get()) throw std::invalid_argument("TEXT_INPUT_UNSUPPORTED"); - running->steps = unlock + if (type == "sequence") { + if (session_.HasPressedInput()) throw std::invalid_argument("INPUT_HELD"); + auto groups = BuildInputSequence(j.at("action"), + {o.width, o.height, o.source_width, o.source_height, o.region}); + for (const auto &steps : groups) + for (const auto &step : steps) + if (step.kind == StepKind::Text && !session_.Status()["text_input"].get()) + throw std::invalid_argument("TEXT_INPUT_UNSUPPORTED"); + running->sequence = true; + running->action_count = groups.size(); + running->interval_ms = Number(j.at("action"), "interval_ms", 0, 500, 100); + running->steps = std::move(groups.front()); + for (size_t index = 1; index < groups.size(); ++index) + running->remaining.push_back(std::move(groups[index])); + } else running->steps = unlock ? BuildUnlockSteps(String(j, "password", 256), session_.Status().value("platform", "") == "macos") : BuildInputSteps(j.at("action"), {o.width, o.height, o.source_width, o.source_height, o.region}); if (unlock) session_.ReleaseInput(); @@ -775,6 +820,7 @@ private: Clock::now()}); session_.BeginInput(); running->started = Clock::now(); + running->total_started = running->started; action_ = std::move(running); } std::string Password() { @@ -823,6 +869,8 @@ private: {"unlock_protocol", "password/1"}, {"viewer_input_protocol", "events/1"}, {"action_completion", "submitted"}, + {"action_sequence_protocol", "sequence/1"}, + {"wait_after_frame_supported", true}, {"max_request_bytes", 65536}, {"max_response_bytes", kMaxResponse}, {"operation_cache_entries", 128}, @@ -940,7 +988,7 @@ private: TrackRemote(j, id, session_.Clipboard(write, text), write, operation_id); return false; } else if (method == "wait_frame") { - Fields(j, {"id", "method", "observation_id", "condition", "stable_ms", "timeout_ms"}); + Fields(j, {"id", "method", "observation_id", "condition", "stable_ms", "timeout_ms", "after_frame_id"}); const auto &observation = Observe(j); const auto condition = String(j, "condition", 16); if (condition != "changed" && condition != "stable") @@ -949,6 +997,7 @@ private: const auto now = Clock::now(); frame_waits_.push_back({id, observation, condition == "stable", Number(j, "stable_ms", 100, 5000, 500), observation.signature, 0, + Sequence(j, "after_frame_id"), now + Ms(Number(j, "timeout_ms", 0, 25000, 5000)), now}); return false; } else if (method == "unlock_prepare") { diff --git a/apps/cli/automation_protocol.cpp b/apps/cli/automation_protocol.cpp index 6fdc783b..0da00462 100644 --- a/apps/cli/automation_protocol.cpp +++ b/apps/cli/automation_protocol.cpp @@ -141,6 +141,30 @@ int KeyCode(const std::string &key) { } } // namespace +std::vector> BuildInputSequence(const json &a, const InputGeometry &o) { + Fields(a, {"type", "actions", "interval_ms"}); + if (String(a, "type", 32) != "sequence" || !a.contains("actions") || + !a["actions"].is_array() || a["actions"].empty() || a["actions"].size() > 8) + throw std::invalid_argument("INVALID_SEQUENCE"); + const int interval = Number(a, "interval_ms", 0, 500, 100); + int duration = int(a["actions"].size() - 1) * interval; + size_t text_bytes = 0; + std::vector> result; + for (const auto &item : a["actions"]) { + const auto type = String(item, "type", 32); + if (type == "sequence" || type == "key_down" || type == "key_up" || + type == "mouse_down" || type == "mouse_up") + throw std::invalid_argument("INVALID_SEQUENCE"); + auto steps = BuildInputSteps(item, o); + duration += steps.back().delay; + for (const auto &step : steps) text_bytes += step.text.size(); + if (duration > 8000 || text_bytes > 16384) + throw std::invalid_argument("SEQUENCE_LIMIT"); + result.push_back(std::move(steps)); + } + return result; +} + std::vector BuildInputSteps(const json &a, const InputGeometry &o) { const auto type = String(a, "type", 32); std::vector steps; diff --git a/apps/cli/automation_protocol.h b/apps/cli/automation_protocol.h index 92041167..09b227f3 100644 --- a/apps/cli/automation_protocol.h +++ b/apps/cli/automation_protocol.h @@ -36,6 +36,8 @@ struct Step { // Pure validation and translation: no session access or input side effects. std::vector BuildInputSteps(const json &action, const InputGeometry &geometry); +std::vector> BuildInputSequence(const json &action, + const InputGeometry &geometry); std::vector BuildUnlockSteps(const std::string &password, bool macos); } // namespace crossdesk::cli::automation diff --git a/apps/cli/docs/automation-protocol.md b/apps/cli/docs/automation-protocol.md index dc9ebe0b..3bf561db 100644 --- a/apps/cli/docs/automation-protocol.md +++ b/apps/cli/docs/automation-protocol.md @@ -120,6 +120,10 @@ All pointer actions and `key_press` accept optional `modifiers`, an array of at Only one input sequence runs at once. Timed steps are advanced by the owner loop; cancellation, disconnect, display changes and pause release held buttons/keys. A failed send is conservatively considered potentially submitted. +`hello` and status advertise `action_sequence_protocol: "sequence/1"`. An action may be `{type:"sequence", actions:[...], interval_ms:100}` with 1–8 ordinary actions. Nested sequences and `key_down`, `key_up`, `mouse_down`, `mouse_up` are rejected. Sequences require no pre-held inputs, at most 16 KiB of total text and at most 8000 ms of planned input duration including the 0–500 ms inter-action interval. The complete sequence is validated against the initial observation before any input is sent. Only use fixed sequences that require no intermediate visual decision. + +Each sub-action finishes its bounded receipt wait before the next starts; errors, acknowledgement timeouts, cancellation, pause and geometry changes stop the remainder. Legacy hosts retain `remote_execution: "unsupported"`; an interval is not an application-readiness guarantee. A sequence has a 25-second overall deadline and one operation-cache entry. Results include `sequence.action_count`, `finished_actions` (finished input stages, not application success) and a zero-based `stopped_at` on failure. `timing_ms.input` measures native input processing including receipt waits and intervals. + New actions require a current frame no older than two seconds as well as a valid observation. Cached operation lookups remain available with a stalled feed or paused control. Failed key/button releases remain tracked for subsequent cleanup, including after control is disabled; a successful local send still does not prove remote execution. Invalid display indices leave existing observations and input sequences intact. Results use `status: submitted | in_progress | failed_before_submit | outcome_unknown`, with `operation_id`, `retryable`, `side_effect_possible` and, for completed actions, `generation`/`after_frame_id`. The same operation ID and arguments retrieve the cached result; different arguments conflict. Retention is bounded to 128 entries, with entries older than five minutes eligible for removal on a new action. Never retry an uncertain effect with a new ID. `submitted` does not claim remote execution. @@ -142,6 +146,8 @@ Updated desktop hosts answer the versioned `automation` capability handshake. Wh `wait_frame` accepts an `observation_id`, `condition` (`changed` or `stable`), `stable_ms` (100–5000, default 500), and `timeout_ms` (0–25000, default 5000). It watches the observation's source region, checks both luma and chroma, and returns `matched`, `wait_timed_out`, `generation`, and `frame_id`. Display changes, disconnection and explicit cancellation terminate pending waits. `crossdesk_wait` exposes this operation through MCP and includes a fresh screenshot when its condition matches. +With `wait_after_frame_supported: true` in `hello`/status, `wait_frame` also accepts `after_frame_id`. Only frames strictly beyond that boundary can match. The MCP action wrapper uses the completed action's frame boundary, runs a bounded visual or UI-condition wait, and returns one final screenshot within the same tool call. A visual match is not a causal acknowledgement; use a relevant cropped observation or a native UI condition for a more specific check. Wait timeouts preserve the input result and never trigger a replay. + ## Clipboard `clipboard` accepts `operation: get | set`. Writes additionally require `operation_id` and `text` (up to 16384 UTF-8 bytes, including an empty string to clear). Reads return up to 128 KiB of text. The existing chunked `web-clipboard` stream carries values and applied/rejected replies. Additive `read`, `empty`, and `clear` frames are enabled by the remote `clipboard` capability. Clipboard access requires control permission and respects human pause. Cancellation of an issued write reports an uncertain effect. Repeated write operation IDs retrieve the cached result. diff --git a/apps/mcp/README.md b/apps/mcp/README.md index 5fcb6f8a..4268d784 100644 --- a/apps/mcp/README.md +++ b/apps/mcp/README.md @@ -105,7 +105,9 @@ Apps 资源为 `ui://crossdesk/viewer.html`，工具只在允许使用 Apps 且 浏览器面板只监听随机端口的 `127.0.0.1`，与 MCP 服务同一进程。它不是 MCP Streamable HTTP，也不向局域网提供 MCP 接入。MCP、CLI、用户浏览器应在同一控制端电脑上运行；SSH 容器/远程宿主的浏览器转发不属于此本地接入模式。 -页面标题为 CrossDesk MCP，仅保留正式应用标志，使用文字按钮和简洁状态提示。布局填满浏览器或宿主面板的可用宽高，远程画面保持原比例自动缩放；连接设置在高度不足时独立滚动。连接后侧栏显示设备与授权信息，声音播放与音量、人工接管/交还 Agent、暂停/恢复和断开按钮位于画面工具栏。「放大画面」收起侧栏，按 Esc 还原；窄屏默认显示连接表单，连接后显示画面，可用「连接信息」展开设备详情。界面资源均可离线使用，保存设备与持久授权仍由用户分别选择。 +页面以 Agent 会话监看为中心：首次连接显示授权卡片，连接后远程画面占满可用区域，设备、权限和恢复设置收进「会话详情」抽屉。工具栏显示设备与实际控制状态，仅在具有控制授权时保留「暂停 Agent / 恢复 Agent」；本面板人工接管时改为「交还 Agent」。画面放大、人工接管、声音与音量、断开连接收进「会话选项」，仅查看会话不显示人工接管入口。等待系统解锁时显示独立提示及密码入口，关闭解锁弹窗后仍可再次打开。状态区分仅查看、控制已启用、已暂停、人工接管和恢复连接，不将控制权限解释为 Agent 正在执行任务。 + +页面标题为 CrossDesk MCP，页内使用 CrossDesk 正式标志和名称。配色与桌面客户端的 WorkbenchStyle / ButtonStyle 一致，采用浅蓝灰底色、蓝色主操作、6 像素控件圆角及 12 像素卡片圆角；工具栏按钮统一为 32 像素高，图标按钮保留可访问名称与悬停提示。布局填满浏览器或宿主面板的可用宽高，远程画面保持原比例缩放，底部集中显示视频状态、分辨率和帧率；授权卡片在高度不足时独立滚动，窄侧栏沿用相同的监看布局。「放大画面」隐藏页眉并缩小外边距；退出菜单、详情和非人工接管状态下的放大模式支持 Esc。界面资源均可离线使用，保存设备与持久授权仍由用户分别选择。 首次打开使用 URL fragment 中的短期一次性 bootstrap，页面立即清除 fragment，并换取 HttpOnly/SameSite cookie 和 CSRF 令牌。模型只收到无特权的网页地址。复制无凭据地址到另一浏览器不能获得会话控制权。 @@ -264,13 +266,42 @@ Apps 依赖宿主遵守 app-only 工具可见性和 `_meta` 的模型不可见 `key_press`、`hotkey` 支持 `duration_ms`，最长三十秒；`drag` 时长上限同样为三十秒。状态返回 `held_key_codes` 和 `held_mouse_flags`。文本输入前应释放持续按住的键鼠状态。 +### 减少 Agent 操作往返 + +`crossdesk_action` 支持 `action.type: "sequence"`，在一次调用中依次执行 1–8 个固定动作，最后只返回一次截图。例如已聚焦远程浏览器时，可以将聚焦地址栏、输入网址、回车合并： + +```json +{ + "session_id": "current_session", + "operation_id": "open_bilibili", + "observation_id": "current_observation", + "action": { + "type": "sequence", + "actions": [ + { "type": "hotkey", "keys": ["META", "L"] }, + { "type": "type_text", "text": "https://www.bilibili.com" }, + { "type": "key_press", "key": "ENTER" } + ] + }, + "wait": { "condition": "changed", "timeout_ms": 1500 } +} +``` + +该例适用于 macOS；其他系统按实际快捷键选择修饰键。只有无需中途观察判断的动作才能合并，所有坐标仍对应起始观测。序列禁止嵌套和跨动作保持按下状态，累计文本不超过 16 KiB，计划输入时长（含间隔）不超过 8 秒，执行总期限为 25 秒。`interval_ms` 为动作间最小间隔，默认 100 毫秒，可设为 0–500；它不代表应用完成加载。原生层在首次输入前验证整组动作，每步等待支持的输入回执；回执失败、超时、人工暂停、取消或几何变化都会停止余下动作。旧被控端没有回执时保留 `remote_execution: "unsupported"`，最终画面仍需检查。`sequence.finished_actions` 表示已结束输入阶段的动作数，错误时的 `stopped_at` 是从 0 开始的停止位置，不代表应用业务结果。 + +连接和状态结果的 `automation` 汇总本机序列支持、远端输入回执及桌面自动化能力。Agent 应复用当前会话和有效观测，支持桌面自动化时优先用 `crossdesk_desktop_action` 直接打开网址、应用或操作可访问性元素；未声明支持时直接使用截图输入路径。旧 CLI 不支持序列时在任何输入前返回 `NATIVE_SEQUENCE_UNSUPPORTED`，需更新本机 CLI。远端能力未声明时会给出兼容性提示，不自动断开连接或尝试不支持的操作。 + +动作结果的 `timing_ms` 包含原生输入阶段、MCP 到 CLI 的输入往返、等待及截图阶段和工具内总耗时，可用来区分工具执行与工具调用之外的宿主、模型或语音延迟。 + ### 截图与条件等待 截图默认最长边 1280，最大 2048；允许指定源帧像素区域。返回 `source_size`、`source_rect`、`image_size`、`generation`、`geometry_epoch`、`frame_id`、`frame_age_ms` 和 `observation_id`。动作坐标使用返回图像坐标，不使用浏览器 CSS 像素。观测保留最多 32 个，30 秒过期；切换显示器、连接代际或图像几何变化立即失效。截图使用不超过 2 秒的帧，PNG 不超过 2 MiB，必要时继续缩小输出并返回真实尺寸。 `timeout_ms` 限制等待可用帧的时间，PNG 编码独立执行；`0` 表示只使用当前已就绪的新鲜帧，无帧返回 `FRAME_TIMEOUT`，编码器繁忙返回 `BUSY`。截图按到达顺序处理，并优先于预览。视频超过 2 秒未更新时拒绝新的键鼠输入，恢复新鲜画面后可继续；已提交操作的去重查询仍可读取原结果。 -键鼠动作的 `operation_id` 在当前原生会话中缓存最近 128 个操作，至少在未被容量淘汰的 5 分钟窗口内支持同参数查询；不同参数复用 ID 返回冲突。`submitted` 表示输入已发送，远端原生输入处理状态见 `remote_execution`，应用界面的实际结果仍需后续观察确认。超时、断网、取消或发送失败可能返回 `outcome_unknown`/`side_effect_possible`，禁止自动重放。`observe_after` 默认开启，后续图像只证明取得更新帧，不证明应用已完成预期操作。 +键鼠动作的 `operation_id` 在当前原生会话中缓存最近 128 个操作，至少在未被容量淘汰的 5 分钟窗口内支持同参数查询；不同参数复用 ID 返回冲突，整组序列同样受此规则约束。`submitted` 表示输入已发送，远端原生输入处理状态见 `remote_execution`，应用界面的实际结果仍需后续观察确认。超时、断网、取消或发送失败可能返回 `outcome_unknown`/`side_effect_possible`，禁止自动重放。 + +`observe_after` 默认开启：在同一次调用内最多等待 1000 毫秒的画面变化，再返回最终截图及 `wait.matched` / `wait.wait_timed_out`。纯移动和单独按下／释放默认只取新帧，避免等待本来就不会变化的像素；仍可显式指定 `wait`。显式 `wait` 可选择 `changed`、`stable`，或指定 `condition: "ui"` 与 `text` / `role` 等待远端可访问性元素出现；最长等待 5000 毫秒。视觉等待可以提供同一会话的预先裁剪观测 `wait.observation_id`，避免视频、动画或时钟等无关区域干扰。新版 CLI 的视觉等待只接受动作结束帧之后的帧，旧 CLI 返回 `wait.frame_bound: false`。等待超时或截图错误不会覆盖已经提交的输入结果，也不会重放动作；变化或稳定不等于应用任务成功。`observe_after: false` 只返回输入结果，不能同时指定 `wait`。 更新后的被控端通过 `capabilities.input_ack` 声明输入处理回执。动作结果的 `remote_execution` 表示远端原生输入处理状态，应用界面的实际结果由后续观察确认。旧被控端返回 `unsupported`。回执等待有明确期限，超时动作保留结果不确定状态。 diff --git a/apps/mcp/src/common.ts b/apps/mcp/src/common.ts index 4644d2be..3e3bc9a3 100644 --- a/apps/mcp/src/common.ts +++ b/apps/mcp/src/common.ts @@ -84,7 +84,7 @@ const key = z.enum([ "MEDIA_STOP", "MEDIA_PLAY_PAUSE", "SEMICOLON", "EQUAL", "COMMA", "MINUS", "PERIOD", "SLASH", "BACKQUOTE", "BRACKET_LEFT", "BACKSLASH", "BRACKET_RIGHT", "QUOTE", ]); -export const action = z.discriminatedUnion("type", [ +export const singleAction = z.discriminatedUnion("type", [ point.extend({ type: z.literal("move"), modifiers }), point.extend({ type: z.literal("mouse_down"), button }), point.extend({ type: z.literal("mouse_up"), button }), @@ -122,6 +122,31 @@ export const action = z.discriminatedUnion("type", [ .refine((s) => Buffer.byteLength(s) <= 16384 && !s.includes("\0")), }), ]); +const sequenceAction = singleAction.refine((value) => + !["key_down", "key_up", "mouse_down", "mouse_up"].includes(value.type), + "Sequences cannot hold inputs across actions."); +export const action = z.union([singleAction, z.strictObject({ + type: z.literal("sequence"), + actions: z.array(sequenceAction).min(1).max(8), + interval_ms: z.int().min(0).max(500).default(100), +}).refine((value) => value.actions.reduce((bytes, item) => + bytes + (item.type === "type_text" ? Buffer.byteLength(item.text) : 0), 0) <= 16384, + "Sequence text is limited to 16 KiB.")]); +export const actionWait = z.union([ + z.strictObject({ + condition: z.enum(["changed", "stable"]), + observation_id: id.optional().describe("Optional pre-action observation of a relevant cropped region."), + stable_ms: z.int().min(100).max(5000).default(200), + timeout_ms: z.int().min(0).max(5000).default(1000), + }), + z.strictObject({ + condition: z.literal("ui"), + text: z.string().min(1).max(256).optional(), + role: z.string().min(1).max(128).optional(), + pid: z.int().min(0).max(2147483647).optional(), + timeout_ms: z.int().min(0).max(5000).default(2000), + }).refine((value) => !!value.text || !!value.role), +]); const normalizedPoint = z.strictObject({ x: z.number().finite().min(0).max(1), y: z.number().finite().min(0).max(1) }); export const viewerInput = z.strictObject({ input_epoch: z.int().min(0), generation: z.int().min(0), diff --git a/apps/mcp/src/native.ts b/apps/mcp/src/native.ts index 3e8569da..07cad84f 100644 --- a/apps/mcp/src/native.ts +++ b/apps/mcp/src/native.ts @@ -94,6 +94,8 @@ type Pending = { export class NativeClient { unlockSupported = false; viewerInputSupported = false; + sequenceSupported = false; + waitAfterFrameSupported = false; private child: ChildProcess; private pending = new Map(); private sequence = 0; @@ -152,6 +154,8 @@ export class NativeClient { ); this.unlockSupported = hello.unlock_protocol === "password/1"; this.viewerInputSupported = hello.viewer_input_protocol === "events/1"; + this.sequenceSupported = hello.action_sequence_protocol === "sequence/1"; + this.waitAfterFrameSupported = hello.wait_after_frame_supported === true; if (hello.protocol !== "crossdesk-automation/1" || hello.terminal_protocol !== "ansi/1" || hello.video_transport !== "loopback-websocket/1" || diff --git a/apps/mcp/src/server.ts b/apps/mcp/src/server.ts index 268d67bc..b8561f78 100644 --- a/apps/mcp/src/server.ts +++ b/apps/mcp/src/server.ts @@ -22,6 +22,7 @@ import { alias, mode, action, + actionWait, screenshot, Failure, failure, @@ -82,7 +83,7 @@ export function createServer( { capabilities: { tools: {}, resources: {} }, instructions: - "Remote computer control. Open a panel for the user to select and authorize a device. Never request passwords in chat. In Codex, the default panel is the host built-in browser/sidebar for every user; no personal agent instructions are required. Other clients keep their local environment and system browser defaults. Explicit panel and launch settings override these defaults. Whenever browser_launch_url is returned, immediately open that local launcher using the host browser-open tool before waiting for authorization or continuing the remote task, even if the connection is already ready. Do not wait for the user to ask to open the sidebar. Do not read or quote its contents; it contains a one-time pairing credential. In Codex, use open_in_codex with target.type=browser, target.url=browser_launch_url, placement=right. If the host has no browser-open tool, explain the limitation and show pairing_command for the user to run locally; do not claim the sidebar is open. Reissue expired launchers with crossdesk_open_browser using the existing view_id, without reconnecting. When a remote task is pending, immediately follow crossdesk_open with crossdesk_get_session(view_id, wait_ms: 25000) in the same turn. Repeat bounded waits on wait_timed_out until ready or the user cancels; do not ask the user to type a connection confirmation or click Continue. On ready, continue the requested task within current authorization. When state is reconnecting, continue bounded waits for the explicitly enabled recovery policy. Use the returned session_id after recovery and inspect a fresh screenshot. On closed/failed, stop waiting and do not automatically reconnect. Obtain a screenshot before actions and pass its observation_id; coordinates refer to that returned image. If the remote system is locked, focus its visible password field, take a fresh screenshot and call crossdesk_unlock. A saved system password is used automatically; otherwise the local panel asks the user. Wait with crossdesk_get_session(unlock_request_id, wait_ms: 25000), then verify a fresh screenshot before continuing the original task. Submitted actions are not remote execution acknowledgements. Never replay actions after uncertain outcomes. Human pause can only be resumed in the user panel.", + "Remote computer control. Open a panel for the user to select and authorize a device. Never request passwords in chat. In Codex, the default panel is the host built-in browser/sidebar for every user; no personal agent instructions are required. Other clients keep their local environment and system browser defaults. Explicit panel and launch settings override these defaults. Whenever browser_launch_url is returned, immediately open that local launcher using the host browser-open tool before waiting for authorization or continuing the remote task, even if the connection is already ready. Do not wait for the user to ask to open the sidebar. Do not read or quote its contents; it contains a one-time pairing credential. In Codex, use open_in_codex with target.type=browser, target.url=browser_launch_url, placement=right. If the host has no browser-open tool, explain the limitation and show pairing_command for the user to run locally; do not claim the sidebar is open. Reissue expired launchers with crossdesk_open_browser using the existing view_id, without reconnecting. When a remote task is pending, immediately follow crossdesk_open with crossdesk_get_session(view_id, wait_ms: 25000) in the same turn. Repeat bounded waits on wait_timed_out until ready or the user cancels; do not ask the user to type a connection confirmation or click Continue. On ready, continue the requested task within current authorization. When state is reconnecting, continue bounded waits for the explicitly enabled recovery policy. Use the returned session_id after recovery and inspect a fresh screenshot. On closed/failed, stop waiting and do not automatically reconnect. Reuse the active session and its advertised automation capabilities. Prefer desktop_action for supported URL/app/accessibility operations. Use a recent screenshot observation for coordinate input; reuse action results instead of taking duplicate screenshots. Group fixed inputs that need no intermediate visual decision into a short action sequence, and wait for a relevant visual region or UI condition in that same call. Coordinates refer to the observation image. If the remote system is locked, focus its visible password field, take a fresh screenshot and call crossdesk_unlock. A saved system password is used automatically; otherwise the local panel asks the user. Wait with crossdesk_get_session(unlock_request_id, wait_ms: 25000), then verify a fresh screenshot before continuing the original task. Submitted actions are not remote execution acknowledgements. Never replay actions after uncertain outcomes. Human pause can only be resumed in the user panel.", }, ); type Definition = { @@ -354,8 +355,9 @@ export function createServer( observation_id: id, action, observe_after: z.boolean().default(true), - }), - "Submit one structured action against a recent observation. Reuse operation_id only to retrieve that same operation, never replay uncertain effects with a new ID. Human pause prevents control. A post-action screenshot is not an execution acknowledgement.", + wait: actionWait.optional(), + }).refine((p) => !p.wait || p.observe_after), + 'Submit an action or a short fixed sequence against a recent observation. Example action: {"type":"sequence","actions":[{"type":"hotkey","keys":["META","L"]},{"type":"type_text","text":"https://www.bilibili.com"},{"type":"key_press","key":"ENTER"}]}. Sequences allow 1–8 actions, no nested sequences or held inputs; all coordinates use the initial observation. Use only when no intermediate visual decision is needed. Defaults to waiting up to 1 second for visual change and returning one final screenshot (move/down/up only capture a newer frame); pass wait for a cropped-region condition or native UI text, or observe_after:false for input only. An unmatched wait does not mean input failed; never replay it automatically. Reuse operation_id only to query the same operation. Human pause interrupts sequences. Receipts confirm input processing, not application success.', async (p, ctx) => output(await sessions.action(p, ctx.mcpReq.signal)), ); add( diff --git a/apps/mcp/src/sessions.ts b/apps/mcp/src/sessions.ts index dc2e6f23..0eb20a7c 100644 --- a/apps/mcp/src/sessions.ts +++ b/apps/mcp/src/sessions.ts @@ -179,6 +179,17 @@ export class Sessions { if (!session) throw new Failure("SESSION_NOT_FOUND"); return session; } + private automation(s: Session): Data { + const desktop = s.status.capabilities?.desktop === true; + return { + sequence_supported: s.native.sequenceSupported, + input_ack_supported: s.status.capabilities?.input_ack === true, + desktop_supported: desktop, + frame_bound_wait_supported: s.native.waitAfterFrameSupported, + preferred_input: desktop ? "desktop_action" : "action", + ...(!desktop ? { compatibility_note: "Remote desktop automation is not advertised. Use screenshot-based input; update the remote client to enable direct URL/app and accessibility operations." } : {}), + }; + } async status(sessionId?: string, viewId?: string, signal?: AbortSignal): Promise { const view = viewId ? this.view(viewId) : undefined; const id = sessionId ?? view?.session; @@ -226,6 +237,7 @@ export class Sessions { this.checkUnlock(s); return { ...s.status, + automation: this.automation(s), session_id: s.id, device_alias: s.device.alias, reconnect_enabled: s.recoveryEnabled, @@ -235,7 +247,7 @@ export class Sessions { ...(view ? this.describe(view) : {}), state: s.status.state, ...(s.status.state === "ready" ? { - next_step: "Connection ready. Continue the user's pending task within the current authorization. Obtain a screenshot before input; respect human_paused and control. No user confirmation message is needed.", + next_step: "Reuse this session and its advertised automation capabilities. Prefer desktop_action for supported URL/app/UI operations; otherwise use action. Group fixed inputs into short sequences and use the returned observation for the next decision. Do not duplicate screenshots unless stale or insufficient. Respect human_paused and control; continue the pending task without chat confirmation.", } : ["closed", "failed", "disconnected"].includes(s.status.state) ? { next_step: "The connection ended. Stop waiting and do not automatically reconnect or replay actions.", } : {}), @@ -573,7 +585,7 @@ export class Sessions { if (this.active.get(s.id) !== s || signal.aborted) throw new Failure("CANCELLED"); if (s.status.state === "ready" && typeof s.status.frame_age_ms === "number" && s.status.frame_age_ms <= 2000) { s.connecting = false; - return { ...s.status, session_id: s.id, device_alias: s.device.alias }; + return { ...s.status, automation: this.automation(s), session_id: s.id, device_alias: s.device.alias }; } if (["failed", "closed", "disconnected"].includes(s.status.state)) { const code = connectionErrors.get(s.status.error); @@ -627,20 +639,51 @@ export class Sessions { return observation.slice(prefix.length); } async action(p: Data, signal?: AbortSignal): Promise { - const { session_id, observe_after = true, ...args } = p; + const { session_id, observe_after = true, wait, ...args } = p; const s = this.session(session_id); if (s.displayBatch) throw new Failure("BUSY"); this.checkUnlock(s); if (s.unlock && this.unlockPending(s.unlock)) throw new Failure("UNLOCK_PENDING"); + if (args.action.type === "sequence" && !s.native.sequenceSupported) + throw new Failure("NATIVE_SEQUENCE_UNSUPPORTED"); + if (wait?.condition === "ui" && s.status.capabilities?.desktop !== true) + throw new Failure("UI_AUTOMATION_UNSUPPORTED"); args.observation_id = this.nativeObservation(s, args.observation_id); - const result = await s.native.request("action", args, (args.action.duration_ms ?? 0) + 5000, signal); + // Reject cross-session wait references before sending any input. + const waitObservation = wait?.observation_id + ? this.nativeObservation(s, wait.observation_id) : args.observation_id; + const started = performance.now(); + const result = await s.native.request("action", args, + args.action.type === "sequence" ? 30000 : (args.action.duration_ms ?? 0) + 5000, signal); + const inputMs = Math.round(performance.now() - started); if (observe_after && result.status === "submitted") { + try { + // Pointer motion/held-input primitives need not change captured pixels. + const condition = wait ?? (["move", "key_down", "key_up", "mouse_down", "mouse_up"].includes(args.action.type) + ? undefined : { condition: "changed", timeout_ms: 1000 }); + if (condition?.condition === "ui") { + const { condition: _, ...query } = condition; + result.wait = await this.waitUI({ ...query, session_id, condition: "present" }, signal); + } else if (condition) { + result.wait = await s.native.request("wait_frame", { + observation_id: waitObservation, condition: condition.condition, + stable_ms: condition.stable_ms ?? 200, timeout_ms: condition.timeout_ms ?? 1000, + ...(s.native.waitAfterFrameSupported ? { after_frame_id: result.after_frame_id } : {}), + }, (condition.timeout_ms ?? 1000) + 3000, signal); + result.wait.frame_bound = s.native.waitAfterFrameSupported; + } + } catch (error) { + // Observation errors never erase a successful or uncertain input result. + result.wait = { matched: false, error: failure(error) }; + } try { result.observation = await this.screenshot( { session_id, generation: result.generation, - after_frame_id: result.after_frame_id, + // A matched frame can be encoded directly; avoid another frame wait. + after_frame_id: result.wait?.matched && result.wait.frame_id + ? Math.max(result.after_frame_id, result.wait.frame_id - 1) : result.after_frame_id, timeout_ms: 2000, }, signal, @@ -650,6 +693,9 @@ export class Sessions { error instanceof Failure ? error.code : "OBSERVATION_FAILED"; } } + result.timing_ms = { ...result.timing_ms, input_round_trip: inputMs, + observation: Math.round(performance.now() - started) - inputMs, + total: Math.round(performance.now() - started) }; return result; } async waitFor(p: Data, signal?: AbortSignal): Promise { diff --git a/apps/mcp/ui/viewer.css b/apps/mcp/ui/viewer.css index 0b676c74..19fabd7a 100644 --- a/apps/mcp/ui/viewer.css +++ b/apps/mcp/ui/viewer.css @@ -1,12 +1,24 @@ +/* Shared with desktop WorkbenchStyle, ButtonStyle, and UiRadius. */ :root { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "PingFang SC", "Microsoft YaHei", sans-serif; - color: #292d33; - background: #fff; font-size: 13px; color-scheme: light; - --accent: #3269b7; - --muted: #777d85; - --line: #e4e6e9; + color: var(--text); + background: var(--canvas); + --canvas: #edf2f8; + --surface: #fff; + --field: #dee7f2; + --accent: #3564e3; + --text: #1c2940; + --muted: #69788e; + --line: #e1e7ef; + --edge: #d7dfe9; + --button-edge: #b8c7db; + --hover: #f0f5ff; + --pressed: #dfe9fa; + --success: #18835a; + --radius-control: 6px; + --radius-surface: 12px; } * { box-sizing: border-box; } html, body { width: 100%; height: 100%; margin: 0; } @@ -14,89 +26,135 @@ body { overflow: hidden; } [hidden] { display: none !important; } button, input, select { font: inherit; } button, summary, input, select { -webkit-tap-highlight-color: transparent; } -:focus-visible { outline: 2px solid #7194c6; outline-offset: 2px; } +:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; } +.icon { width: 16px; height: 16px; flex: none; pointer-events: none; } +.sr-only { position: absolute; width: 1px; height: 1px; padding: 0; margin: -1px; overflow: hidden; clip-path: inset(50%); white-space: nowrap; border: 0; } #app { width: 100%; height: 100vh; height: 100dvh; display: flex; flex-direction: column; } -.app-header { display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 8px 16px; padding: 12px 20px; border-bottom: 1px solid var(--line); flex: none; } -.brand, .header-right { display: flex; align-items: center; gap: 10px; min-width: 0; } -.brand-logo { display: block; width: 28px; height: 28px; } -h1 { margin: 0; font-size: 16px; font-weight: 600; line-height: 1.5; white-space: nowrap; } +.app-header { display: flex; align-items: center; justify-content: space-between; gap: 12px; min-height: 44px; padding: 6px 16px; border-bottom: 1px solid var(--line); background: var(--surface); flex: none; } +.brand, .header-right { display: flex; align-items: center; gap: 8px; min-width: 0; } +.brand-logo { display: block; width: 24px; height: 24px; } +h1 { margin: 0; font-size: 15px; font-weight: 650; line-height: 1.5; white-space: nowrap; } h2 { margin: 0; font-size: 13px; font-weight: 600; } -.connection { color: var(--muted); font-size: 12px; white-space: nowrap; } -.connection[data-state="online"] { color: #397557; } -.connection[data-state="connecting"] { color: #8a682d; } -.connection[data-state="offline"] { color: #a15149; } -#details-toggle { display: none; } -main { display: grid; grid-template-columns: 270px minmax(0, 1fr); flex: 1; min-height: 0; } -aside { min-width: 0; min-height: 0; overflow: auto; overscroll-behavior: contain; padding: 22px 20px; border-right: 1px solid var(--line); } -#sidebar-title { margin-bottom: 20px; } +.brand-context { margin-left: 4px; padding-left: 12px; border-left: 1px solid var(--line); color: var(--muted); font-size: 12px; white-space: nowrap; } +.connection { display: inline-flex; align-items: center; gap: 6px; color: var(--muted); font-size: 11px; white-space: nowrap; } +.connection::before { content: ""; width: 6px; height: 6px; border-radius: 50%; background: currentColor; } +.connection[data-state="online"] { color: var(--success); } +.connection[data-state="connecting"] { color: #a46b1b; } +.connection[data-state="offline"] { color: #b42318; } +#details-toggle, #details-close { display: none; } +main { position: relative; display: flex; flex: 1; min-height: 0; overflow: hidden; } +aside { width: min(400px, calc(100% - 32px)); max-height: calc(100% - 48px); min-width: 0; overflow: auto; overscroll-behavior: contain; align-self: center; margin: 24px auto; padding: 24px; border: 1px solid var(--edge); border-radius: var(--radius-surface); background: var(--surface); } +.sidebar-heading { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin-bottom: 18px; } +#sidebar-title { font-size: 15px; } +.authorization-note { margin: 0 0 20px; color: var(--muted); font-size: 12px; line-height: 1.7; } +#app:not([data-session="true"]) .workspace { display: none; } +#app[data-session="true"] #details-toggle, #app[data-session="true"] #details-close { display: inline-flex; } +#app[data-session="true"] aside { display: none; position: absolute; z-index: 31; inset: 12px 12px 12px auto; width: 296px; max-width: calc(100% - 24px); max-height: none; margin: 0; padding: 18px; align-self: auto; box-shadow: 0 12px 40px #1c294024; } +#app[data-session="true"].details-open aside { display: block; } +#details-close { font-size: 20px; } +.details-backdrop, .details-backdrop:not(:disabled):hover, .details-backdrop:not(:disabled):active { position: absolute; inset: 0; z-index: 30; width: 100%; height: 100%; padding: 0; border: 0; border-radius: 0; background: #1c294026; cursor: default; } fieldset { border: 0; margin: 0; padding: 0; min-width: 0; } -label { display: block; margin: 16px 0 7px; font-size: 12px; } +label { display: block; margin: 16px 0 6px; color: #52657f; font-size: 12px; font-weight: 500; } fieldset > label:first-child { margin-top: 0; } -.optional { color: var(--muted); margin-left: 4px; font-size: 11px; } -input:not([type="checkbox"]), select { display: block; width: 100%; height: 36px; min-width: 0; border: 1px solid #d7dade; border-radius: 4px; padding: 7px 9px; background: #fff; color: inherit; } -input:not([type="checkbox"]):focus, select:focus { border-color: var(--accent); outline: 1px solid var(--accent); } -input:disabled, select:disabled { background: #f5f5f5; color: #999; } -input::placeholder { color: #92969c; } +.optional { color: var(--muted); margin-left: 4px; font-size: 11px; font-weight: 400; } +input:not([type="checkbox"]):not([type="range"]), select { display: block; width: 100%; height: 36px; min-width: 0; border: 1px solid transparent; border-radius: var(--radius-control); padding: 7px 10px; background: var(--field); color: inherit; transition: border-color 120ms, background 120ms; } +input:not([type="checkbox"]):not([type="range"]):focus, select:focus { border-color: var(--accent); outline: 1px solid var(--accent); background: #e5ecf5; } +input:disabled, select:disabled { background: #f0f2f5; color: #98a2b3; } +input::placeholder { color: #566a84; } #device-id, #session-device-id { font-variant-numeric: tabular-nums; } +#device-id { font-size: 15px; letter-spacing: .6px; } .password-field { position: relative; } .password-field input { padding-right: 48px; } -.password-field button { position: absolute; right: 2px; top: 2px; height: 32px; min-height: 32px; padding: 0 8px; border: 0; background: transparent; color: var(--muted); } -.field-hint, .privacy, .session-note { color: var(--muted); font-size: 11px; line-height: 1.7; margin: 7px 0 0; } -.preferences { border-top: 1px solid var(--line); margin-top: 20px; padding-top: 7px; } +.password-field button { position: absolute; right: 3px; top: 3px; height: 30px; min-height: 30px; padding: 0 7px; border: 0; background: transparent; color: #52657f; } +.field-hint, .privacy, .session-note { color: var(--muted); font-size: 11px; line-height: 1.65; margin: 7px 0 0; } +.preferences { border-top: 1px solid var(--line); margin-top: 18px; padding-top: 6px; } details { font-size: 12px; } -summary { padding: 9px 0; cursor: pointer; color: #666d76; } +summary { padding: 9px 0; cursor: pointer; color: #52657f; } +summary:hover { color: var(--accent); } summary::marker { font-size: 10px; } .checks { padding-bottom: 10px; } -.checks label { display: flex; align-items: flex-start; gap: 8px; margin-top: 13px; font-size: 12px; line-height: 1.5; cursor: pointer; } -.checks label:has(input:disabled) { color: #999; cursor: default; } +.checks label, .check { display: flex; align-items: flex-start; gap: 8px; margin-top: 12px; font-size: 12px; font-weight: 400; line-height: 1.5; cursor: pointer; } +.checks label:has(input:disabled), .check:has(input:disabled) { color: #98a2b3; cursor: default; } .checks small { display: block; margin-top: 2px; color: var(--muted); font-size: 11px; } input[type="checkbox"] { width: 14px; height: 14px; flex: none; margin: 2px 0 0; accent-color: var(--accent); } -.row { display: flex; gap: 12px; } +.row { display: flex; gap: 10px; } .row > div { flex: 1; min-width: 0; } -button { display: inline-flex; align-items: center; justify-content: center; min-height: 32px; padding: 6px 10px; border: 1px solid #d9dce0; border-radius: 4px; background: #fff; color: #4d535c; font-size: 12px; cursor: pointer; white-space: nowrap; } -button:not(:disabled):hover { background: #f3f4f5; border-color: #bfc4cb; } -button:not(:disabled):active { background: #e9ebee; } -button:disabled { cursor: default; color: #a3a7ad; background: #fafafa; border-color: #e7e9eb; } -.primary { width: 100%; min-height: 36px; background: var(--accent); color: #fff; border-color: var(--accent); margin-top: 20px; } -.primary:not(:disabled):hover { background: #285b9f; border-color: #285b9f; } -.privacy { margin-top: 12px; } -#error { flex: none; max-height: 25%; overflow: auto; padding: 10px 20px; background: #fff5f2; border-bottom: 1px solid #edddd6; color: #964a37; font-size: 12px; line-height: 1.6; overflow-wrap: anywhere; } -.device-card strong { display: block; font-size: 14px; font-weight: 500; overflow-wrap: anywhere; } -.device-card > span { display: block; margin-top: 5px; color: var(--muted); font-size: 12px; } -.session-details { margin: 20px 0 0; font-size: 12px; } -.session-details > div { display: flex; justify-content: space-between; gap: 12px; padding: 11px 0; border-bottom: 1px solid #eee; } +button { display: inline-flex; align-items: center; justify-content: center; gap: 6px; min-height: 32px; padding: 6px 10px; border: 1px solid var(--button-edge); border-radius: var(--radius-control); background: var(--surface); color: var(--text); font-size: 12px; font-weight: 500; line-height: 18px; cursor: pointer; white-space: nowrap; transition: background 120ms, border-color 120ms, color 120ms; } +button:not(:disabled):hover { background: var(--hover); border-color: #7e9fd1; } +button:not(:disabled):active { background: var(--pressed); border-color: #6488bf; } +button:disabled { cursor: default; color: #98a2b3; background: #f5f7fb; border-color: #dce2eb; } +button:disabled .icon { opacity: .65; } +button.quiet { border-color: transparent; background: transparent; color: #52657f; } +button.quiet:not(:disabled):hover { background: var(--hover); border-color: transparent; color: var(--accent); } +button.quiet:disabled { color: #98a2b3; } +button.soft:not(:disabled) { background: #dce7ff; color: var(--accent); border-color: transparent; } +button.soft:not(:disabled):hover, button.soft[aria-pressed="true"]:not(:disabled) { background: var(--accent); color: #fff; } +.icon-button { width: 32px; height: 32px; padding: 0; flex: none; } +.primary { width: 100%; min-height: 36px; background: var(--accent); color: #fff; border-color: var(--accent); margin-top: 18px; font-weight: 600; } +.primary:not(:disabled):hover { background: #4574f0; border-color: #4574f0; } +.primary:not(:disabled):active { background: #254cc0; border-color: #254cc0; } +.privacy { margin-top: 10px; text-align: center; } +#error { flex: none; max-height: 25%; overflow: auto; padding: 10px 16px; background: #fff4f2; border-bottom: 1px solid #f5d6d1; color: #b42318; font-size: 12px; line-height: 1.6; overflow-wrap: anywhere; } +.device-card { display: flex; align-items: center; gap: 10px; } +.device-icon { display: grid; place-items: center; width: 38px; height: 38px; flex: none; background: #edf2ff; border: 1px solid #dce7ff; border-radius: 8px; color: var(--accent); } +.device-icon .icon { width: 22px; height: 22px; } +.device-identity { min-width: 0; } +.device-card strong { display: block; font-size: 14px; font-weight: 600; overflow-wrap: anywhere; } +.device-identity > span { display: block; margin-top: 3px; color: var(--muted); font-size: 11px; } +.session-details { margin: 18px 0 0; font-size: 12px; } +.session-details > div { display: flex; align-items: baseline; justify-content: space-between; gap: 12px; padding: 10px 0; border-bottom: 1px solid var(--line); } .session-details dt { color: var(--muted); flex: none; } .session-details dd { margin: 0; text-align: right; overflow-wrap: anywhere; } -#session-agent[data-paused="true"] { color: #956e2d; } -.session-note { margin-top: 18px; } -.workspace { display: flex; flex-direction: column; gap: 10px; min-width: 0; min-height: 0; overflow: auto; padding: 16px; background: #f6f7f8; } -.session-unlock-details { display: flex; flex-direction: column; align-items: flex-end; gap: 7px; } -#session-unlock-state[data-state="awaiting_password"] { color: #956e2d; } -#session-unlock-state[data-state="failed"] { color: #9a4e47; } -.unlock-dialog { width: min(420px, calc(100% - 32px)); max-height: calc(100% - 32px); margin: auto; padding: 22px; color: inherit; background: #fff; border: 1px solid var(--line); border-radius: 10px; box-shadow: 0 18px 64px #17243733; } -.unlock-dialog::backdrop { background: #19212e4d; } -.unlock-dialog-header { display: flex; align-items: center; justify-content: space-between; gap: 16px; } -.unlock-dialog-header h2 { font-size: 16px; } -.unlock-dialog-header button, .unlock-toast > button { width: 28px; min-height: 28px; padding: 0; flex: none; border: 0; color: var(--muted); font-size: 20px; line-height: 1; } -.unlock-dialog p { margin: 12px 0 0; color: var(--muted); font-size: 12px; line-height: 1.7; } -.unlock-buttons { display: flex; gap: 8px; } +#session-agent[data-paused="true"] { color: #a46b1b; } +#session-card > .check { margin: 16px 0 0; padding: 10px; border-radius: var(--radius-control); background: #f5f7fb; } +.session-note { margin-top: 10px; } +.workspace { display: flex; flex: 1; flex-direction: column; gap: 8px; min-width: 0; min-height: 0; overflow: auto; padding: 12px; background: var(--canvas); } +.session-unlock-details { display: flex; flex-direction: column; align-items: flex-end; gap: 6px; } +#session-unlock-state[data-state="awaiting_password"] { color: #a46b1b; } +#session-unlock-state[data-state="failed"] { color: #b42318; } +.unlock-dialog { width: min(400px, calc(100% - 32px)); max-height: calc(100% - 32px); margin: auto; padding: 20px; color: inherit; background: var(--surface); border: 1px solid var(--edge); border-radius: var(--radius-surface); box-shadow: 0 16px 48px #1c294029; } +.unlock-dialog::backdrop { background: #1c29404d; } +.unlock-dialog-header { display: flex; align-items: center; justify-content: space-between; gap: 12px; } +.unlock-dialog-header h2 { font-size: 15px; } +.unlock-dialog-header button, .unlock-toast > button { width: 28px; height: 28px; min-height: 28px; padding: 0; flex: none; border: 0; color: var(--muted); font-size: 20px; line-height: 1; } +.unlock-dialog p { margin: 10px 0 0; color: var(--muted); font-size: 12px; line-height: 1.65; } +.unlock-buttons { display: flex; justify-content: flex-end; gap: 8px; margin-top: 8px; } .unlock-buttons .primary { margin: 0; width: auto; min-width: 80px; } -.unlock-toast { position: fixed; top: 70px; right: 20px; z-index: 10; display: flex; align-items: flex-start; gap: 12px; width: min(360px, calc(100% - 32px)); padding: 14px 14px 14px 16px; border: 1px solid var(--line); border-radius: 8px; background: #fff; box-shadow: 0 6px 24px #17243726; } +.unlock-toast { position: fixed; top: 60px; right: 16px; z-index: 10; display: flex; align-items: flex-start; gap: 10px; width: min(360px, calc(100% - 32px)); padding: 12px 12px 12px 16px; border: 1px solid var(--edge); border-radius: 8px; background: var(--surface); box-shadow: 0 6px 24px #1c29401f; } .unlock-toast > div { min-width: 0; flex: 1; } .unlock-toast strong { display: block; padding-top: 4px; font-size: 13px; font-weight: 600; } -.unlock-toast p { margin: 6px 0 0; color: var(--muted); font-size: 12px; line-height: 1.7; } -.unlock-toast[data-state="failed"] { border-color: #d6b4ae; } -.viewer-card { display: flex; flex-direction: column; flex: 1; min-height: 160px; min-width: 0; background: #fff; border: 1px solid var(--line); border-radius: 5px; overflow: hidden; } -.viewer-toolbar { display: flex; justify-content: space-between; align-items: center; flex-wrap: wrap; gap: 8px 12px; padding: 9px 12px; flex: none; } -.viewer-title { display: flex; align-items: center; gap: 10px; min-width: 0; flex: 1; } -.viewer-title h2 { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.unlock-toast p { margin: 6px 0 0; color: var(--muted); font-size: 12px; line-height: 1.65; } +.unlock-toast[data-state="failed"] { border-color: #f0b4ae; } +.viewer-card { display: flex; flex-direction: column; flex: 1; min-height: 160px; min-width: 0; background: var(--surface); border: 1px solid var(--edge); border-radius: var(--radius-surface); overflow: hidden; } +.viewer-toolbar { display: flex; align-items: center; gap: 12px; padding: 10px 14px; flex: none; border-bottom: 1px solid var(--line); } +.viewer-title { display: flex; flex-direction: column; align-items: flex-start; gap: 4px; min-width: 0; flex: 1; } +.viewer-title h2 { max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +#agent-state { display: inline-flex; align-items: baseline; gap: 6px; color: var(--muted); font-size: 11px; line-height: 16px; } +#agent-state::before { content: ""; width: 5px; height: 5px; flex: none; border-radius: 50%; background: currentColor; align-self: center; } +#agent-state[data-state="enabled"] { color: var(--accent); } +#agent-state:is([data-state="paused"], [data-state="human"], [data-state="unlock"], [data-state="recovering"]) { color: #a46b1b; } #video-state { color: var(--muted); font-size: 11px; white-space: nowrap; flex: none; } -#video-state[data-state="stale"] { color: #956e2d; } -.viewer-controls { display: flex; gap: 6px; flex-wrap: wrap; } -.danger:not(:disabled) { color: #9a4e47; } -.danger:not(:disabled):hover { border-color: #d6b4ae; background: #fff6f4; } -/* The screen takes the remaining panel height; object-fit preserves the remote aspect ratio. */ -.screen { position: relative; flex: 1; min-height: 60px; width: 100%; display: flex; align-items: center; justify-content: center; overflow: hidden; isolation: isolate; border-radius: 8px; background: #17191c; } +#video-state[data-state="live"] { color: var(--success); } +#video-state[data-state="stale"] { color: #a46b1b; } +.viewer-controls { display: flex; align-items: center; gap: 6px; flex: none; } +.viewer-controls button { min-height: 32px; flex-shrink: 0; } +#more-toggle[aria-expanded="true"] { background: var(--hover); color: var(--accent); } +.session-menu { position: fixed; z-index: 40; width: 264px; max-width: calc(100vw - 16px); max-height: calc(100dvh - 16px); overflow: auto; padding: 6px; background: var(--surface); border: 1px solid var(--edge); border-radius: 8px; box-shadow: 0 8px 32px #1c294026; } +.session-menu > button, .menu-audio > button { width: 100%; min-height: 36px; justify-content: flex-start; gap: 10px; } +.menu-caption, .menu-note { margin: 6px 10px; color: var(--muted); font-size: 11px; line-height: 1.6; } +.menu-audio { margin: 4px 0; padding: 4px 0 8px; border-block: 1px solid var(--line); } +.audio-volume { display: flex; align-items: center; justify-content: space-between; gap: 12px; margin: 4px 10px 0 36px; font-size: 11px; font-weight: 400; } +input[type="range"] { display: block; width: 132px; max-width: 100%; height: 24px; padding: 0; margin: 0; accent-color: var(--accent); cursor: pointer; } +input[type="range"]:disabled { opacity: .45; cursor: default; } +#audio-play[aria-pressed="true"] { color: var(--accent); background: #edf2ff; } +.attention { display: flex; align-items: center; justify-content: space-between; gap: 12px; padding: 8px 14px; background: #fff8e9; border-bottom: 1px solid #f1dfb9; color: #886023; font-size: 12px; } +.attention button { min-height: 28px; padding: 4px 8px; color: #886023; border-color: #dec997; } +.danger:not(:disabled) { color: #b42318; border-color: #edc5c0; } +.danger:not(:disabled):hover { border-color: #d92d20; background: #fff4f2; } +.session-menu .danger:not(:disabled) { border-color: transparent; } +/* Fit the full remote desktop without cropping or changing input coordinates. */ +.screen { position: relative; flex: 1; min-height: 60px; width: 100%; display: flex; align-items: center; justify-content: center; overflow: hidden; isolation: isolate; background: #f5f7fb; } +#app[data-connected="true"] .screen { background: #141b26; } .screen canvas { position: absolute; inset: 0; width: 100%; height: 100%; object-fit: contain; } #agent-cursor { pointer-events: none; z-index: 4; } /* Overlay the control indicator so toggling control never resizes the video. */ @@ -149,60 +207,53 @@ button:disabled { cursor: default; color: #a3a7ad; background: #fafafa; border-c @media (prefers-reduced-motion: reduce) { .control-halo::before { animation: none; will-change: auto; } } -#empty { text-align: center; padding: 16px; max-width: 100%; color: #b4b7bd; } -#empty h3 { font-size: 14px; font-weight: 400; margin: 0 0 8px; } -#empty p { color: #7e848e; font-size: 12px; line-height: 1.7; margin: 0; } -.stale { position: absolute; left: 12px; bottom: 12px; padding: 7px 10px; background: #25272cdd; color: #e2c495; border-radius: 3px; font-size: 11px; } -.telemetry { display: flex; align-items: center; flex-wrap: wrap; gap: 7px 18px; padding: 9px 12px; flex: none; color: var(--muted); font-size: 11px; } -.stream-metrics { display: flex; gap: 14px; font-variant-numeric: tabular-nums; } -#permission { margin-left: auto; } -#permission[data-state="paused"] { color: #956e2d; } -.panel-status { flex: none; font-size: 12px; line-height: 1.6; color: var(--muted); margin: 0; } +#empty { text-align: center; padding: 24px; max-width: 100%; color: var(--text); } +.empty-icon { display: grid; place-items: center; width: 64px; height: 64px; margin: 0 auto 16px; color: #8fa7c5; background: #e8eef7; border-radius: 16px; } +.empty-icon .icon { width: 32px; height: 32px; stroke-width: 1.4; } +#empty h3 { font-size: 15px; font-weight: 600; margin: 0 0 8px; } +#empty p { color: var(--muted); font-size: 12px; line-height: 1.7; margin: 0; } +#app[data-connected="true"] #empty { color: #d7dfe9; } +#app[data-connected="true"] .empty-icon { background: #202d40; color: #8fa7c5; } +.stale { position: absolute; left: 12px; bottom: 12px; padding: 6px 10px; background: #1c2940e8; color: #e2c495; border-radius: var(--radius-control); font-size: 11px; } +.telemetry { display: flex; align-items: center; flex-wrap: wrap; gap: 6px 14px; min-height: 32px; padding: 7px 10px; flex: none; color: var(--muted); font-size: 11px; border-top: 1px solid var(--line); } +.stream-metrics { display: flex; gap: 12px; font-variant-numeric: tabular-nums; } +#session-info { margin-left: auto; } +.panel-status { flex: none; font-size: 11px; line-height: 1.6; color: var(--muted); margin: 0; padding: 0 2px; } .video-options { display: flex; flex-wrap: wrap; gap: 8px; flex: none; } .video-options:not(:has(button:not([hidden]))) { display: none; } -.focus-mode main { grid-template-columns: minmax(0, 1fr); } .focus-mode .app-header { display: none; } .focus-mode aside, .focus-mode #details-toggle { display: none !important; } -@media (max-width: 760px) { - .app-header { padding: 10px 14px; } - main { display: flex; flex-direction: column; } - aside { flex: 1; border-right: 0; padding: 20px; } - #app:not([data-connected="true"]) .workspace { display: none; } - #app[data-connected="true"] aside { display: none; flex: none; max-height: 40%; border-bottom: 1px solid var(--line); } - #app[data-connected="true"].details-open aside { display: block; } - #app[data-connected="true"] #details-toggle { display: inline-flex; } - #app[data-connected="true"] .workspace { flex: 1; } - .workspace { padding: 10px; } - #sidebar-title { margin-bottom: 16px; } - .session-details { margin-top: 12px; } -} -@media (max-width: 480px) { - #app[data-controlling="true"] .control-halo { --halo-spread: 10px; } - .app-header { gap: 8px; padding: 10px 12px; } - .brand, .header-right { gap: 7px; } - .brand-logo { width: 24px; height: 24px; } - h1 { font-size: 15px; } - .connection { font-size: 11px; } - #details-toggle { padding: 5px 7px; min-height: 28px; font-size: 11px; } - .workspace { padding: 8px; gap: 8px; } - .viewer-toolbar { padding: 9px; } - .viewer-title, .viewer-controls { width: 100%; flex: auto; } - .viewer-controls button { flex: 1; min-height: 36px; } - .telemetry { padding: 8px 9px; gap: 6px 10px; } +.focus-mode .workspace { padding: 6px; } +#focus-view[aria-pressed="true"] { background: #edf2ff; color: var(--accent); } +@media (max-width: 600px) { + .app-header { padding: 6px 12px; } + .brand-context { display: none; } + .workspace { padding: 8px; } + .viewer-toolbar { padding: 10px; } #session-info { display: none; } - .stream-metrics { gap: 10px; } - #permission { font-size: 10px; } } -@media (max-height: 460px) and (min-width: 481px) { - .app-header { padding-top: 6px; padding-bottom: 6px; } - .workspace { padding: 8px; gap: 6px; } - .viewer-toolbar { padding-top: 5px; padding-bottom: 5px; } - .telemetry { padding-top: 6px; padding-bottom: 6px; } +@media (max-width: 400px) { + .app-header { gap: 8px; padding: 6px 10px; } + .brand, .header-right { gap: 6px; } + h1 { font-size: 14px; } + #details-toggle { padding: 4px 6px; font-size: 11px; } + #details-toggle .icon { display: none; } + .connection { font-size: 10px; gap: 4px; } + .workspace { padding: 6px; } + .viewer-toolbar { gap: 8px; } + .viewer-controls { gap: 4px; } + .viewer-controls button { padding: 6px 8px; } + .viewer-controls .icon-button { padding: 0; } + #pause .icon, #return-agent .icon { display: none; } + aside { padding: 20px; } + .telemetry { padding: 7px 10px; gap: 6px 12px; } +} +@media (max-height: 460px) { + aside { margin-block: 12px; max-height: calc(100% - 24px); padding: 16px; } + .workspace { padding: 6px; gap: 6px; } + .viewer-toolbar { padding-top: 6px; padding-bottom: 6px; } + .telemetry { min-height: 28px; padding-top: 5px; padding-bottom: 5px; } } - #remote-input { position: absolute; width: 1px; height: 1px; opacity: 0; padding: 0; border: 0; pointer-events: none; } .screen[data-human="true"] { touch-action: none; cursor: crosshair; } -.screen[data-human="true"]:focus-within { outline: 2px solid #3976ff; outline-offset: 2px; } - -.audio-volume { display: inline-flex; align-items: center; gap: 4px; font-size: 12px; } -.audio-volume input { width: 60px; padding: 0; } +.screen[data-human="true"]:focus-within { outline: 2px solid var(--accent); outline-offset: -2px; } diff --git a/apps/mcp/ui/viewer.ts b/apps/mcp/ui/viewer.ts index 8331c9f0..7f3f9f82 100644 --- a/apps/mcp/ui/viewer.ts +++ b/apps/mcp/ui/viewer.ts @@ -9,17 +9,31 @@ declare const CROSSDESK_APPS: boolean; declare const CROSSDESK_LOGO: string; type Data = Record; type Invoke = (name: string, args?: Data) => Promise; +const icons = { + monitor: '', + info: '', + volume: '', + mouse: '', + pause: '', + power: '', + expand: '', + arrow: '', + more: '', +}; +const icon = (name: keyof typeof icons) => ``; const root = document.querySelector("#app")!; root.innerHTML = `
-

CrossDesk MCP

-
等待连接
+

CrossDesk

Agent 会话
+
等待连接
+
-

远程画面

待连接
-
+

远程画面

等待授权
+
+ + + +
+
-

未连接设备

选择设备并输入访问密码。

+
${icon("monitor")}

未连接设备

选择设备并输入访问密码。

-
——
等待设备连接等待授权
+
待连接
——
等待设备连接
+

解锁远程电脑

@@ -188,21 +215,62 @@ function setFocus(focused: boolean) { root.classList.toggle("focus-mode", focused); const button = element("focus-view"); const label = focused ? "还原布局" : "放大画面"; - button.textContent = label; + text("focus-label", label); if (focused) setDetails(false); button.setAttribute("aria-pressed", String(focused)); button.setAttribute("aria-label", label); button.title = label; } function setDetails(open: boolean) { + const restoreFocus = !open && root.classList.contains("details-open") && + element("sidebar").contains(document.activeElement); root.classList.toggle("details-open", open); + element("details-backdrop").hidden = !open; element("details-toggle").setAttribute("aria-expanded", String(open)); - text("details-toggle", open ? "收起信息" : "连接信息"); + text("details-label", open ? "收起详情" : "会话详情"); + if (open) { setSessionMenu(false, false); element("details-close").focus(); } + else if (restoreFocus && !root.classList.contains("focus-mode")) element("details-toggle").focus(); +} +function positionSessionMenu() { + const menu = element("session-menu"); + const rect = element("more-toggle").getBoundingClientRect(); + menu.style.top = `${Math.max(8, Math.min(rect.bottom + 6, window.innerHeight - menu.offsetHeight - 8))}px`; + menu.style.right = `${Math.max(8, window.innerWidth - rect.right)}px`; +} +function setSessionMenu(open: boolean, restoreFocus = true) { + const menu = element("session-menu"); + const hadFocus = menu.contains(document.activeElement); + menu.hidden = !open; + element("more-toggle").setAttribute("aria-expanded", String(open)); + if (open) { + setDetails(false); + positionSessionMenu(); + menu.querySelector("button:not(:disabled):not([hidden])")?.focus(); + } else if (restoreFocus && hadFocus) element("more-toggle").focus(); } element("details-toggle").onclick = () => setDetails(!root.classList.contains("details-open")); -element("focus-view").onclick = () => setFocus(!root.classList.contains("focus-mode")); +element("details-close").onclick = element("details-backdrop").onclick = () => setDetails(false); +element("more-toggle").onclick = () => setSessionMenu(element("session-menu").hidden); +element("focus-view").onclick = () => { + setSessionMenu(false); + setFocus(!root.classList.contains("focus-mode")); +}; +document.addEventListener("pointerdown", (event) => { + if (!element("session-menu").hidden && event.target instanceof Node && + !element("session-menu").contains(event.target) && !element("more-toggle").contains(event.target)) + setSessionMenu(false, false); +}); +document.addEventListener("focusin", (event) => { + if (!element("session-menu").hidden && event.target instanceof Node && + !element("session-menu").contains(event.target) && !element("more-toggle").contains(event.target)) + setSessionMenu(false, false); +}); +window.addEventListener("resize", () => { if (!element("session-menu").hidden) positionSessionMenu(); }); window.addEventListener("keydown", (event) => { - if (!ownsInput() && event.key === "Escape" && !element("unlock-dialog").open && root.classList.contains("focus-mode")) setFocus(false); + if (event.key !== "Escape" || element("unlock-dialog").open) return; + if (!element("session-menu").hidden) { event.preventDefault(); setSessionMenu(false); } + else if (root.classList.contains("details-open")) { event.preventDefault(); setDetails(false); } + else if (!ownsInput() && root.classList.contains("focus-mode")) setFocus(false); }); function hidePassword() { input("password").type = "password"; @@ -331,6 +399,7 @@ function renderUnlock(s: Data) { text("session-unlock-state", states[request?.state] ?? "—"); element("session-unlock-state").dataset.state = request?.state ?? ""; element("unlock-open").hidden = !visible || !pending; + element("attention").hidden = !visible || !pending; element("unlock-open").disabled = unlockBusy; element("unlock-form").hidden = !pending; input("unlock-password").disabled = unlockBusy; @@ -389,19 +458,30 @@ element("unlock-cancel").onclick = async () => { }; function renderStatus(s: Data) { const connected = s.state === "ready" || s.state === "waiting_frame"; + const hasSession = connected || s.state === "reconnecting"; const lost = ["closed", "failed"].includes(s.state); if (lost && !disconnectedByUser && ["ready", "waiting_frame"].includes(status.state)) showError(new Error("CONNECTION_LOST")); const changedSession = s.session_id !== status.session_id; status = s; element("audio-play").disabled = !connected || usePreview; - text("audio-play", audioPlayer.enabled ? "静音" : "播放声音"); + input("audio-volume").disabled = !connected || usePreview; + const audioLabel = audioPlayer.enabled ? "静音" : "播放声音"; + text("audio-label", audioLabel); + element("audio-play").setAttribute("aria-label", audioLabel); + element("audio-play").setAttribute("aria-pressed", String(audioPlayer.enabled)); + element("audio-play").title = audioLabel; if (!connected || changedSession) audioPlayer.flush(); humanInput.update(connected && ownsInput() ? s.human_input_epoch : undefined); - text("take-over", ownsInput() ? "交还 Agent" : "人工接管"); + text("take-over-label", ownsInput() ? "交还 Agent" : "人工接管"); + element("take-over").setAttribute("aria-pressed", String(ownsInput())); element("take-over").disabled = !connected || !s.control_allowed || s.viewer_input_protocol !== "events/1" || controlBusy || (!imageAt && !ownsInput()); + element("take-over").hidden = !s.control_allowed || ownsInput(); + element("return-agent").hidden = !connected || !ownsInput(); + element("return-agent").disabled = element("take-over").disabled; renderUnlock(s); - const connecting = busy || s.state === "waiting_frame"; + const connecting = busy || s.state === "waiting_frame" || s.state === "reconnecting"; + root.dataset.session = String(hasSession); root.dataset.connected = String(connected); root.dataset.connecting = String(connecting); root.dataset.controlling = String(connected && s.control === true && s.control_allowed === true && !s.human_paused); @@ -409,11 +489,20 @@ function renderStatus(s: Data) { text("connection-label", connected ? "已连接" : connecting ? "正在连接" : lost ? "未连接" : "等待连接"); element("connect-form").hidden = connected || s.state === "reconnecting"; element("session-card").hidden = !connected && s.state !== "reconnecting"; - text("sidebar-title", connected ? "连接信息" : "连接设备"); - text("device-name", connected ? s.device_alias || "远程电脑" : "远程画面"); - if (!connected || changedSession) setDetails(false); - text("permission", connected ? s.control_owner === "human" ? ownsInput() ? "正在人工操作" : "其他面板正在操作" : s.human_paused ? "控制已暂停" : s.control_allowed ? "允许 Agent 操作" : "仅查看画面" : "等待授权"); - element("permission").dataset.state = !connected ? "idle" : s.human_paused ? "paused" : "active"; + text("sidebar-title", hasSession ? "会话详情" : "授权 Agent 连接"); + text("device-name", hasSession ? s.device_alias || "远程电脑" : "远程画面"); + if (!hasSession || changedSession) { setDetails(false); setSessionMenu(false, false); } + // Native control state describes permissions/ownership, not task execution. + const agentState = s.state === "reconnecting" ? "recovering" : !connected ? "idle" + : s.control_owner === "human" ? "human" : s.human_paused ? "paused" + : s.unlock?.state === "awaiting_password" ? "unlock" + : !s.control_allowed ? "observe" : s.control ? "enabled" : "idle"; + const agentLabel = s.state === "reconnecting" ? "正在恢复连接" : !connected ? "等待授权" + : agentState === "human" ? ownsInput() ? "人工接管中 · Agent 已让出控制" : "其他面板正在人工操作" + : agentState === "paused" ? "Agent 控制已暂停" : agentState === "unlock" ? "Agent 等待系统解锁" + : agentState === "observe" ? "Agent 仅查看" : agentState === "enabled" ? "Agent 控制已启用" : "Agent 控制未启用"; + text("agent-state", agentLabel); + element("agent-state").dataset.state = agentState; const platform = platformName(s.platform); text("session-info", connected ? `${platform} · 显示器 ${(s.display ?? 0) + 1}` : "等待设备连接"); text("session-device", s.device_alias || "远程电脑"); @@ -423,18 +512,21 @@ function renderStatus(s: Data) { text("session-device-id", device ? String(device.device_id).replace(/(\d{3})(?=\d)/g, "$1 ") : ""); text("session-access", s.control_allowed ? "查看与操作" : "仅查看"); text("session-display", `显示器 ${(s.display ?? 0) + 1}`); - text("session-agent", s.state === "reconnecting" ? "正在恢复连接" : s.control_owner === "human" ? "人工接管中" : s.human_paused ? "已暂停" : s.control_allowed ? "允许操作" : "仅可观察"); + text("session-agent", agentLabel); input("recovery").checked = s.reconnect_enabled === true; input("recovery").disabled = !s.session_id; element("session-agent").dataset.paused = String(!!s.human_paused); element("pause").disabled = !connected || !s.control_allowed || controlBusy || s.control_owner === "human"; - const pauseLabel = s.human_paused ? "恢复控制" : "暂停控制"; + element("pause").hidden = !connected || !s.control_allowed || s.control_owner === "human"; + element("pause").classList.toggle("soft", !!s.human_paused); + const pauseLabel = s.human_paused ? "恢复 Agent" : "暂停 Agent"; text("pause-label", pauseLabel); + element("pause").title = s.human_paused ? "恢复 Agent 控制" : "暂停 Agent 控制"; element("disconnect").disabled = !s.session_id || controlBusy || busy; element("connect").disabled = busy || connected || !invoke; element("connection-fields").disabled = busy; element("connect").setAttribute("aria-busy", String(busy)); - text("connect-label", busy ? "正在连接…" : "连接"); + text("connect-label", busy ? "正在连接…" : "授权并连接"); if (!connected) { element("frame").hidden = true; element("empty").hidden = false; @@ -524,8 +616,9 @@ element("audio-play").onclick = async () => { finally { renderStatus(status); } }; input("audio-volume").oninput = () => audioPlayer.volume(Number(input("audio-volume").value)); -element("take-over").onclick = async () => { +element("take-over").onclick = element("return-agent").onclick = async () => { if (controlBusy) return; + setSessionMenu(false); controlBusy = true; humanInput.release(); renderStatus(status); @@ -553,6 +646,7 @@ element("pause").onclick = async () => { }; element("disconnect").onclick = async () => { if (controlBusy) return; + setSessionMenu(false); controlBusy = true; disconnectedByUser = true; renderStatus(status); @@ -630,6 +724,7 @@ async function startPreview() { if (usePreview) { audioPlayer.flush(); element("audio-play").disabled = true; + input("audio-volume").disabled = true; element("error").hidden = true; text("panel-status", "实时视频不可用，已切换为内嵌预览：最高每秒 1 帧，不含声音。需要流畅画面时可使用浏览器面板。"); element("panel-status").hidden = false;