diff --git a/apps/cli/agent.cpp b/apps/cli/agent.cpp index 5f968853..3983e14b 100644 --- a/apps/cli/agent.cpp +++ b/apps/cli/agent.cpp @@ -112,6 +112,7 @@ struct RunningAction { std::vector steps; size_t next = 0; bool submitted = false; + bool retain_input = false; Clock::time_point started, awaiting_ack{}; }; struct Record { @@ -275,7 +276,8 @@ private: void FinishAction(const char *error = nullptr) { if (!action_) return; - session_.ReleaseInput(); + if (error) session_.ReleaseInput(); + else if (!action_->retain_input) session_.ReleaseActionInput(); const auto remote = session_.InputResult(); if (!error && remote == "pending") { if (action_->awaiting_ack == Clock::time_point{}) action_->awaiting_ack = Clock::now(); @@ -283,12 +285,15 @@ private: error = "REMOTE_ACK_TIMEOUT"; } if (!error && remote == "failed") error = "REMOTE_INPUT_FAILED"; + if (error) session_.ReleaseInput(); + if (session_.HasPressedInput()) input_lease_ = Clock::now() + std::chrono::seconds(15); json result = { {"status", error ? (action_->submitted ? "outcome_unknown" : "failed_before_submit") : "submitted"}, {"operation_id", action_->operation}, {"remote_execution", remote}, + {"input_lease_ms", session_.HasPressedInput() ? 15000 : 0}, {"generation", action_->generation}, {"after_frame_id", frame_ ? frame_->frame_id : 0}, {"side_effect_possible", action_->submitted}, @@ -330,6 +335,8 @@ private: } void Tick() { session_.Tick(); + if (!action_ && session_.HasPressedInput() && Clock::now() >= input_lease_) + session_.ReleaseInput(); for (auto it = remote_jobs_.begin(); it != remote_jobs_.end();) { auto result = session_.TakeRemoteResult(it->request); const char *error = !session_.Connected() ? "SESSION_CLOSED" @@ -687,6 +694,27 @@ private: running->steps = unlock ? BuildUnlockSteps(String(j, "password", 256), session_.Status().value("platform", "") == "macos") : BuildInputSteps(j.at("action"), {o.width, o.height, o.source_width, o.source_height, o.region}); + if (unlock) session_.ReleaseInput(); + if (type == "type_text" && session_.HasPressedInput()) throw std::invalid_argument("INPUT_HELD"); + running->retain_input = type == "key_down" || type == "mouse_down"; + for (auto it = running->steps.begin(); it != running->steps.end();) { + if (it->kind == StepKind::Key && session_.KeyPressed(it->key) && type != "key_up") { + const bool modifier = it->key == 16 || it->key == 17 || it->key == 18 || it->key == 91; + if (type != "key_down" && !modifier) throw std::invalid_argument("KEY_ALREADY_HELD"); + it = running->steps.erase(it); + } else { + if (it->kind == StepKind::Pointer && session_.ButtonPressed(it->flag) && type == "mouse_down") { + it = running->steps.erase(it); + continue; + } + if (it->kind == StepKind::Pointer && (it->flag == 1 || it->flag == 3 || it->flag == 5) && + session_.ButtonPressed(it->flag) && type != "mouse_down") + throw std::invalid_argument("BUTTON_ALREADY_HELD"); + ++it; + } + } + if (type == "key_down" && !running->steps.empty() && session_.Status()["held_key_codes"].size() >= 32) + throw std::invalid_argument("INPUT_LIMIT"); while (records_.size() >= 128 || (!records_.empty() && Age(records_.front().created) > 300000)) records_.pop_front(); @@ -788,6 +816,10 @@ private: } else if (method == "action") { StartAction(j, id); return false; + } else if (method == "release_inputs") { + Fields(j, {"id", "method"}); + FinishAction("CANCELLED"); + session_.ReleaseInput(); } else if (method == "system") { Fields(j, {"id", "method", "command", "operation_id"}); const auto command = String(j, "command", 16); @@ -1015,6 +1047,7 @@ private: json preview_; int preview_edge_ = 960; Clock::time_point preview_time_; + Clock::time_point input_lease_{}; std::unique_ptr video_; uint64_t video_generation_ = 0, last_video_frame_ = 0; bool video_connected_ = false, video_enabled_ = false; diff --git a/apps/cli/automation_protocol.cpp b/apps/cli/automation_protocol.cpp index 8fa2375e..6fdc783b 100644 --- a/apps/cli/automation_protocol.cpp +++ b/apps/cli/automation_protocol.cpp @@ -85,6 +85,44 @@ int KeyCode(const std::string &key) { {"F10", 121}, {"F11", 122}, {"F12", 123}, + {"F13", 124}, + {"F14", 125}, + {"F15", 126}, + {"F16", 127}, + {"F17", 128}, + {"F18", 129}, + {"F19", 130}, + {"F20", 131}, + {"F21", 132}, + {"F22", 133}, + {"F23", 134}, + {"F24", 135}, + {"NUMPAD0", 96}, + {"NUMPAD1", 97}, + {"NUMPAD2", 98}, + {"NUMPAD3", 99}, + {"NUMPAD4", 100}, + {"NUMPAD5", 101}, + {"NUMPAD6", 102}, + {"NUMPAD7", 103}, + {"NUMPAD8", 104}, + {"NUMPAD9", 105}, + {"NUM_LOCK", 144}, + {"SCROLL_LOCK", 145}, + {"PRINT_SCREEN", 44}, + {"MENU", 93}, + {"MULTIPLY", 106}, + {"ADD", 107}, + {"SUBTRACT", 109}, + {"DECIMAL", 110}, + {"DIVIDE", 111}, + {"VOLUME_MUTE", 173}, + {"VOLUME_DOWN", 174}, + {"VOLUME_UP", 175}, + {"MEDIA_NEXT", 176}, + {"MEDIA_PREVIOUS", 177}, + {"MEDIA_STOP", 178}, + {"MEDIA_PLAY_PAUSE", 179}, {"SEMICOLON", 186}, {"EQUAL", 187}, {"COMMA", 188}, @@ -135,7 +173,17 @@ std::vector BuildInputSteps(const json &a, const InputGeometry &o) { step.delay = delay; steps.push_back(std::move(step)); }; - if (type == "move" || type == "click" || type == "double_click") { + if (type == "mouse_down" || type == "mouse_up") { + Fields(a, {"type", "x", "y", "button"}); + const auto button = a.contains("button") ? String(a, "button", 8) : "left"; + const int flag = button == "left" ? 1 : button == "right" ? 3 : button == "middle" ? 5 : -1; + if (flag < 0) throw std::invalid_argument("INVALID_BUTTON"); + point(a, 0); + point(a, 10, flag + (type == "mouse_up" ? 1 : 0)); + } else if (type == "key_down" || type == "key_up") { + Fields(a, {"type", "key"}); + key(KeyCode(String(a, "key", 16)), type == "key_down", 0); + } else if (type == "move" || type == "click" || type == "double_click") { Fields(a, type == "move" ? std::initializer_list{"type", "x", "y", "modifiers"} @@ -180,7 +228,7 @@ std::vector BuildInputSteps(const json &a, const InputGeometry &o) { const auto &path = a.at("path"); if (!path.is_array() || path.size() < 2 || path.size() > 128) throw std::invalid_argument("INVALID_PATH"); - const int duration = Number(a, "duration_ms", 100, 2000, 500); + const int duration = Number(a, "duration_ms", 100, 30000, 500); for (const auto &p : path) Fields(p, {"x", "y"}); point(path.front(), 0); @@ -189,12 +237,12 @@ std::vector BuildInputSteps(const json &a, const InputGeometry &o) { point(path[i], 10 + int(i * duration / (path.size() - 1))); point(path.back(), duration + 30, flag + 1); } else if (type == "key_press") { - Fields(a, {"type", "key", "modifiers"}); + Fields(a, {"type", "key", "modifiers", "duration_ms"}); const auto code = KeyCode(String(a, "key", 16)); key(code, true, 0); - key(code, false, 40); + key(code, false, Number(a, "duration_ms", 10, 30000, 40)); } else if (type == "hotkey") { - Fields(a, {"type", "keys"}); + Fields(a, {"type", "keys", "duration_ms"}); const auto &keys = a.at("keys"); if (!keys.is_array() || keys.size() < 2 || keys.size() > 6) throw std::invalid_argument("INVALID_KEYS"); @@ -213,7 +261,7 @@ std::vector BuildInputSteps(const json &a, const InputGeometry &o) { key(code, true, delay); delay += 15; } - delay += 30; + delay += Number(a, "duration_ms", 10, 30000, 30); for (auto it = codes.rbegin(); it != codes.rend(); ++it) { key(*it, false, delay); delay += 15; diff --git a/apps/cli/docs/automation-protocol.md b/apps/cli/docs/automation-protocol.md index 05ea794e..005e6032 100644 --- a/apps/cli/docs/automation-protocol.md +++ b/apps/cli/docs/automation-protocol.md @@ -157,3 +157,9 @@ The desktop and CLI share `FileBrowser` and the existing file transfer engine. P `system` accepts `command: status | lock | send_sas`. Commands that change system state require an `operation_id` and control permission. Status queries are available during human pause. The host executes system operations through its application-thread queue, checks connection authorization again, and expires queued operations after two seconds. Windows secure attention uses the existing service integration. `lock_state` is `locked`, `unlocked`, or `unknown`. macOS queries the current console session, Windows uses its session lock state, and Linux queries the current logind session's `LockedHint`. The MCP unlock workflow verifies this state on compatible hosts and returns `verified` when the system reports an unlocked session. + +## Held input + +`key_down`, `key_up`, `mouse_down`, and `mouse_up` extend the action union. Each successful action renews a fifteen-second lease for held input. Existing held modifiers survive subsequent clicks, movement and hotkeys. `release_inputs` releases the controller's tracked keys/buttons and cancels current input without requiring an observation. Errors, human pause, display changes and disconnection release held input. + +`key_press` and `hotkey` accept `duration_ms` from 10 to 30000. Drags accept durations up to 30000 milliseconds. Extended key names include F13–F24, NUMPAD0–NUMPAD9, arithmetic keypad keys, lock/menu/print keys and media controls. Target platform key mapping determines availability. Text input requires held keys/buttons to be released first. diff --git a/apps/mcp/README.md b/apps/mcp/README.md index 58f388ed..c2563f1a 100644 --- a/apps/mcp/README.md +++ b/apps/mcp/README.md @@ -268,3 +268,9 @@ npm run pack:npm -- --out-dir /absolute/path/artifacts `crossdesk_system_status` 查询系统锁定状态。Windows 同时报告服务可用性和交互桌面阶段。`crossdesk_system_command` 支持 `lock`，Windows 可通过已运行的服务执行 `send_sas`。命令需要控制授权和独立的 `operation_id`，相同请求查询原结果。系统接受命令后，通过状态及画面观察确认实际界面。 支持系统状态的被控端会在密码提交后接受最长八秒的解锁验证。确认解锁后返回 `unlock.state: verified`，并保存用户选择记住的密码。系统状态无法确认时保留 `submitted` 和验证信息。已经解锁的桌面返回 `already_unlocked`。旧被控端沿用截图确认流程。 + +### 持续键鼠输入 + +`crossdesk_action` 增加 `key_down`、`key_up`、`mouse_down` 和 `mouse_up`。按下状态可跨动作保持，例如按住 Ctrl 后移动或拖动鼠标。最后一次动作完成后，按住状态的有效期为十五秒，超时自动释放。`crossdesk_release_inputs` 可立即释放本会话的全部按键和按钮，并取消当前输入动作。人工暂停、断开及输入错误同样释放按住状态。 + +`key_press`、`hotkey` 支持 `duration_ms`，拖拽时长上限提高到三十秒。扩展键名覆盖 F13–F24、数字键盘、菜单、锁定键和媒体键，实际效果取决于远端系统支持。状态返回 `held_key_codes` 和 `held_mouse_flags`。文本输入前应释放持续按住的键鼠状态。 diff --git a/apps/mcp/src/common.ts b/apps/mcp/src/common.ts index 9727100d..a091c59d 100644 --- a/apps/mcp/src/common.ts +++ b/apps/mcp/src/common.ts @@ -73,10 +73,21 @@ const key = z.enum([ "BACKSPACE", "TAB", "ENTER", "SHIFT", "CTRL", "ALT", "PAUSE", "CAPS_LOCK", "ESC", "SPACE", "PAGE_UP", "PAGE_DOWN", "END", "HOME", "LEFT", "UP", "RIGHT", "DOWN", "INSERT", "DELETE", "META", "F1", "F2", "F3", "F4", "F5", "F6", "F7", "F8", "F9", "F10", "F11", "F12", + "F13", "F14", "F15", "F16", "F17", "F18", + "F19", "F20", "F21", "F22", "F23", "F24", + "NUMPAD0", "NUMPAD1", "NUMPAD2", "NUMPAD3", "NUMPAD4", "NUMPAD5", + "NUMPAD6", "NUMPAD7", "NUMPAD8", "NUMPAD9", "NUM_LOCK", "SCROLL_LOCK", + "PRINT_SCREEN", "MENU", "MULTIPLY", "ADD", "SUBTRACT", "DECIMAL", + "DIVIDE", "VOLUME_MUTE", "VOLUME_DOWN", "VOLUME_UP", "MEDIA_NEXT", "MEDIA_PREVIOUS", + "MEDIA_STOP", "MEDIA_PLAY_PAUSE", "SEMICOLON", "EQUAL", "COMMA", "MINUS", "PERIOD", "SLASH", "BACKQUOTE", "BRACKET_LEFT", "BACKSLASH", "BRACKET_RIGHT", "QUOTE", ]); export const action = z.discriminatedUnion("type", [ point.extend({ type: z.literal("move"), modifiers }), + point.extend({ type: z.literal("mouse_down"), button }), + point.extend({ type: z.literal("mouse_up"), button }), + z.strictObject({ type: z.literal("key_down"), key }), + z.strictObject({ type: z.literal("key_up"), key }), point.extend({ type: z.literal("click"), button, modifiers }), point.extend({ type: z.literal("double_click"), button, modifiers }), point.extend({ @@ -88,14 +99,16 @@ export const action = z.discriminatedUnion("type", [ z.strictObject({ type: z.literal("drag"), path: z.array(point).min(2).max(128), - duration_ms: z.int().min(100).max(2000).optional(), + duration_ms: z.int().min(100).max(30000).optional(), button, modifiers, }), - z.strictObject({ type: z.literal("key_press"), key, modifiers }) + z.strictObject({ type: z.literal("key_press"), key, modifiers, + duration_ms: z.int().min(10).max(30000).optional() }) .refine((p) => !p.modifiers?.some((modifier) => modifier === p.key)), z.strictObject({ type: z.literal("hotkey"), + duration_ms: z.int().min(10).max(30000).optional(), keys: z.array(key).min(2).max(6).refine(unique), }), z.strictObject({ diff --git a/apps/mcp/src/native.ts b/apps/mcp/src/native.ts index c47e20f8..ef63a833 100644 --- a/apps/mcp/src/native.ts +++ b/apps/mcp/src/native.ts @@ -173,7 +173,7 @@ export class NativeClient { if (this.pending.size >= 32 || this.child.stdin!.writableLength > 262144) return Promise.reject(new Failure("BUSY", false, true)); const id = ++this.sequence; - const sideEffect = method === "action" || method === "unlock" || + const sideEffect = method === "action" || method === "unlock" || method === "release_inputs" || (method === "clipboard" && args.operation === "set") || (method === "system" && args.command !== "status") || (method === "files" && ["upload", "download", "cancel"].includes(args.operation)); diff --git a/apps/mcp/src/server.ts b/apps/mcp/src/server.ts index ed3d0411..faa9c2b0 100644 --- a/apps/mcp/src/server.ts +++ b/apps/mcp/src/server.ts @@ -343,6 +343,9 @@ export function createServer( "Query the remote operating system's lock state and available secure-desktop service. unknown means the platform could not determine the state.", async (p, ctx) => output(await sessions.system({ ...p, command: "status" }, ctx.mcpReq.signal)), { readOnly: true }); + add("crossdesk_release_inputs", z.strictObject({ session_id: id }), + "Release all keys and mouse buttons held by this session and cancel its current input action. Use when finishing a sequence of key_down/mouse_down actions or recovering from interruption. Held input expires after 15 seconds without another action.", + async (p) => output(await sessions.releaseInputs(p.session_id))); add("crossdesk_system_command", z.strictObject({ session_id: id, operation_id: id, command: z.enum(["lock", "send_sas"]) }), "Lock the remote session or request Windows secure attention through its service. Requires control authorization. Reuse operation_id to query the same command. Check system status and a fresh screenshot after acceptance.", diff --git a/apps/mcp/src/sessions.ts b/apps/mcp/src/sessions.ts index 5d9324bb..6a5b43bb 100644 --- a/apps/mcp/src/sessions.ts +++ b/apps/mcp/src/sessions.ts @@ -441,7 +441,7 @@ export class Sessions { const s = this.session(session_id); this.checkUnlock(s); if (s.unlock && this.unlockPending(s.unlock)) throw new Failure("UNLOCK_PENDING"); - const result = await s.native.request("action", args, 5000, signal); + const result = await s.native.request("action", args, (args.action.duration_ms ?? 0) + 5000, signal); if (observe_after && result.status === "submitted") { try { result.observation = await this.screenshot( @@ -509,6 +509,11 @@ export class Sessions { throw new Failure("UNLOCK_PENDING"); return s.native.request("system", args, 7000, signal); } + async releaseInputs(sessionId: string) { + const s = this.session(sessionId); + this.cancelUnlock(s); + return s.native.request("release_inputs"); + } async control(id: string, enabled: boolean) { if (!enabled) this.cancelUnlock(this.session(id)); return this.session(id).native.request("control", { enabled }); diff --git a/libs/controller/include/controller_session.h b/libs/controller/include/controller_session.h index 22325533..c1788597 100644 --- a/libs/controller/include/controller_session.h +++ b/libs/controller/include/controller_session.h @@ -74,6 +74,10 @@ public: int max_bitrate_bps = -1); void SetControl(bool enabled); void ReleaseInput(); + void ReleaseActionInput(); + bool KeyPressed(int key) const { return keys_.count(key) != 0; } + bool ButtonPressed(int flag) const { return buttons_.count(flag) != 0; } + bool HasPressedInput() const { return !keys_.empty() || !buttons_.empty(); } bool Text(const std::string &utf8); bool Key(int key, bool down); bool Pointer(float x, float y, int flag, int wheel = 0); @@ -137,6 +141,7 @@ private: HostPlatform host_platform_ = HostPlatform::Unknown; std::vector displays_; std::set keys_, buttons_; + std::set action_keys_, action_buttons_; float pointer_x_ = .5f, pointer_y_ = .5f; std::chrono::steady_clock::time_point deadline_; mutable std::mutex mutex_; diff --git a/libs/controller/src/controller_requests.cpp b/libs/controller/src/controller_requests.cpp index b123c664..e6f2e53c 100644 --- a/libs/controller/src/controller_requests.cpp +++ b/libs/controller/src/controller_requests.cpp @@ -26,6 +26,8 @@ std::optional ControllerSession::TakeRemoteResult(uint64_t request) { } void ControllerSession::BeginInput() { + action_keys_ = keys_; + action_buttons_ = buttons_; for (const auto id : input_requests_) remote_results_.erase(id); input_requests_.clear(); input_result_ = "applied"; diff --git a/libs/controller/src/controller_session.cpp b/libs/controller/src/controller_session.cpp index f9c32ceb..94c25e0b 100644 --- a/libs/controller/src/controller_session.cpp +++ b/libs/controller/src/controller_session.cpp @@ -476,6 +476,13 @@ bool ControllerSession::Key(int key, bool down) { keys_.erase(key); return sent; } +void ControllerSession::ReleaseActionInput() { + const auto buttons = buttons_, keys = keys_; + for (int button : buttons) + if (!action_buttons_.count(button)) Pointer(pointer_x_, pointer_y_, button + 1); + for (auto it = keys.rbegin(); it != keys.rend(); ++it) + if (!action_keys_.count(*it)) Key(*it, false); +} bool ControllerSession::Pointer(float x, float y, int flag, int wheel) { const bool releasing = flag >= 2 && flag <= 6 && !(flag & 1) && buttons_.count(flag - 1); @@ -523,6 +530,8 @@ json ControllerSession::Status() const { {"error", error_}, {"generation", generation_}, {"control", control_}, + {"held_key_codes", keys_}, + {"held_mouse_flags", buttons_}, {"displays", ds}, {"platform", HostPlatformName(host_platform_)}, {"display", options_.display},