YangWX/crossdesk
1
0
Fork 0
mirror of https://github.com/kunkundi/crossdesk.git synced 2026-11-07 06:39:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feat] extend keyboard and pointer control with bounded held input

Browse Source
This commit is contained in:
dijunkun
2026-11-06 20:54:28 +08:00
parent dd4baf2fc4
commit 12afcce690
11 changed files with 141 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/cli/agent.cpp
+34 -1
View File
@@ -112,6 +112,7 @@ struct RunningAction {
std::vector<Step> steps;
size_t next = 0;
bool submitted = false;
bool retain_input = false;
Clock::time_point started, awaiting_ack{};
};
struct Record {
@@ -275,7 +276,8 @@ private:
void FinishAction(const char *error = nullptr) {
if (!action_)
return;
session_.ReleaseInput();
if (error) session_.ReleaseInput();
else if (!action_->retain_input) session_.ReleaseActionInput();
const auto remote = session_.InputResult();
if (!error && remote == "pending") {
if (action_->awaiting_ack == Clock::time_point{}) action_->awaiting_ack = Clock::now();
@@ -283,12 +285,15 @@ private:
error = "REMOTE_ACK_TIMEOUT";
}
if (!error && remote == "failed") error = "REMOTE_INPUT_FAILED";
if (error) session_.ReleaseInput();
if (session_.HasPressedInput()) input_lease_ = Clock::now() + std::chrono::seconds(15);
json result = {
{"status", error ? (action_->submitted ? "outcome_unknown"
: "failed_before_submit")
: "submitted"},
{"operation_id", action_->operation},
{"remote_execution", remote},
{"input_lease_ms", session_.HasPressedInput() ? 15000 : 0},
{"generation", action_->generation},
{"after_frame_id", frame_ ? frame_->frame_id : 0},
{"side_effect_possible", action_->submitted},
@@ -330,6 +335,8 @@ private:
}
void Tick() {
session_.Tick();
if (!action_ && session_.HasPressedInput() && Clock::now() >= input_lease_)
session_.ReleaseInput();
for (auto it = remote_jobs_.begin(); it != remote_jobs_.end();) {
auto result = session_.TakeRemoteResult(it->request);
const char *error = !session_.Connected() ? "SESSION_CLOSED"
@@ -687,6 +694,27 @@ private:
running->steps = unlock
? BuildUnlockSteps(String(j, "password", 256), session_.Status().value("platform", "") == "macos")
: BuildInputSteps(j.at("action"), {o.width, o.height, o.source_width, o.source_height, o.region});
if (unlock) session_.ReleaseInput();
if (type == "type_text" && session_.HasPressedInput()) throw std::invalid_argument("INPUT_HELD");
running->retain_input = type == "key_down" || type == "mouse_down";
for (auto it = running->steps.begin(); it != running->steps.end();) {
if (it->kind == StepKind::Key && session_.KeyPressed(it->key) && type != "key_up") {
const bool modifier = it->key == 16 || it->key == 17 || it->key == 18 || it->key == 91;
if (type != "key_down" && !modifier) throw std::invalid_argument("KEY_ALREADY_HELD");
it = running->steps.erase(it);
} else {
if (it->kind == StepKind::Pointer && session_.ButtonPressed(it->flag) && type == "mouse_down") {
it = running->steps.erase(it);
continue;
}
if (it->kind == StepKind::Pointer && (it->flag == 1 || it->flag == 3 || it->flag == 5) &&
session_.ButtonPressed(it->flag) && type != "mouse_down")
throw std::invalid_argument("BUTTON_ALREADY_HELD");
++it;
}
}
if (type == "key_down" && !running->steps.empty() && session_.Status()["held_key_codes"].size() >= 32)
throw std::invalid_argument("INPUT_LIMIT");
while (records_.size() >= 128 ||
(!records_.empty() && Age(records_.front().created) > 300000))
records_.pop_front();
@@ -788,6 +816,10 @@ private:
} else if (method == "action") {
StartAction(j, id);
return false;
} else if (method == "release_inputs") {
Fields(j, {"id", "method"});
FinishAction("CANCELLED");
session_.ReleaseInput();
} else if (method == "system") {
Fields(j, {"id", "method", "command", "operation_id"});
const auto command = String(j, "command", 16);
@@ -1015,6 +1047,7 @@ private:
json preview_;
int preview_edge_ = 960;
Clock::time_point preview_time_;
Clock::time_point input_lease_{};
std::unique_ptr<VideoServer> video_;
uint64_t video_generation_ = 0, last_video_frame_ = 0;
bool video_connected_ = false, video_enabled_ = false;
apps/cli/automation_protocol.cpp
+54 -6
View File
@@ -85,6 +85,44 @@ int KeyCode(const std::string &key) {
{"F10", 121},
{"F11", 122},
{"F12", 123},
{"F13", 124},
{"F14", 125},
{"F15", 126},
{"F16", 127},
{"F17", 128},
{"F18", 129},
{"F19", 130},
{"F20", 131},
{"F21", 132},
{"F22", 133},
{"F23", 134},
{"F24", 135},
{"NUMPAD0", 96},
{"NUMPAD1", 97},
{"NUMPAD2", 98},
{"NUMPAD3", 99},
{"NUMPAD4", 100},
{"NUMPAD5", 101},
{"NUMPAD6", 102},
{"NUMPAD7", 103},
{"NUMPAD8", 104},
{"NUMPAD9", 105},
{"NUM_LOCK", 144},
{"SCROLL_LOCK", 145},
{"PRINT_SCREEN", 44},
{"MENU", 93},
{"MULTIPLY", 106},
{"ADD", 107},
{"SUBTRACT", 109},
{"DECIMAL", 110},
{"DIVIDE", 111},
{"VOLUME_MUTE", 173},
{"VOLUME_DOWN", 174},
{"VOLUME_UP", 175},
{"MEDIA_NEXT", 176},
{"MEDIA_PREVIOUS", 177},
{"MEDIA_STOP", 178},
{"MEDIA_PLAY_PAUSE", 179},
{"SEMICOLON", 186},
{"EQUAL", 187},
{"COMMA", 188},
@@ -135,7 +173,17 @@ std::vector<Step> BuildInputSteps(const json &a, const InputGeometry &o) {
step.delay = delay;
steps.push_back(std::move(step));
};
if (type == "move" || type == "click" || type == "double_click") {
if (type == "mouse_down" || type == "mouse_up") {
Fields(a, {"type", "x", "y", "button"});
const auto button = a.contains("button") ? String(a, "button", 8) : "left";
const int flag = button == "left" ? 1 : button == "right" ? 3 : button == "middle" ? 5 : -1;
if (flag < 0) throw std::invalid_argument("INVALID_BUTTON");
point(a, 0);
point(a, 10, flag + (type == "mouse_up" ? 1 : 0));
} else if (type == "key_down" || type == "key_up") {
Fields(a, {"type", "key"});
key(KeyCode(String(a, "key", 16)), type == "key_down", 0);
} else if (type == "move" || type == "click" || type == "double_click") {
Fields(a, type == "move"
? std::initializer_list<const char *>{"type", "x", "y",
"modifiers"}
@@ -180,7 +228,7 @@ std::vector<Step> BuildInputSteps(const json &a, const InputGeometry &o) {
const auto &path = a.at("path");
if (!path.is_array() || path.size() < 2 || path.size() > 128)
throw std::invalid_argument("INVALID_PATH");
const int duration = Number(a, "duration_ms", 100, 2000, 500);
const int duration = Number(a, "duration_ms", 100, 30000, 500);
for (const auto &p : path)
Fields(p, {"x", "y"});
point(path.front(), 0);
@@ -189,12 +237,12 @@ std::vector<Step> BuildInputSteps(const json &a, const InputGeometry &o) {
point(path[i], 10 + int(i * duration / (path.size() - 1)));
point(path.back(), duration + 30, flag + 1);
} else if (type == "key_press") {
Fields(a, {"type", "key", "modifiers"});
Fields(a, {"type", "key", "modifiers", "duration_ms"});
const auto code = KeyCode(String(a, "key", 16));
key(code, true, 0);
key(code, false, 40);
key(code, false, Number(a, "duration_ms", 10, 30000, 40));
} else if (type == "hotkey") {
Fields(a, {"type", "keys"});
Fields(a, {"type", "keys", "duration_ms"});
const auto &keys = a.at("keys");
if (!keys.is_array() || keys.size() < 2 || keys.size() > 6)
throw std::invalid_argument("INVALID_KEYS");
@@ -213,7 +261,7 @@ std::vector<Step> BuildInputSteps(const json &a, const InputGeometry &o) {
key(code, true, delay);
delay += 15;
}
delay += 30;
delay += Number(a, "duration_ms", 10, 30000, 30);
for (auto it = codes.rbegin(); it != codes.rend(); ++it) {
key(*it, false, delay);
delay += 15;
apps/cli/docs/automation-protocol.md
+6
View File
@@ -157,3 +157,9 @@ The desktop and CLI share `FileBrowser` and the existing file transfer engine. P
`system` accepts `command: status | lock | send_sas`. Commands that change system state require an `operation_id` and control permission. Status queries are available during human pause. The host executes system operations through its application-thread queue, checks connection authorization again, and expires queued operations after two seconds. Windows secure attention uses the existing service integration.
`lock_state` is `locked`, `unlocked`, or `unknown`. macOS queries the current console session, Windows uses its session lock state, and Linux queries the current logind session's `LockedHint`. The MCP unlock workflow verifies this state on compatible hosts and returns `verified` when the system reports an unlocked session.
## Held input
`key_down`, `key_up`, `mouse_down`, and `mouse_up` extend the action union. Each successful action renews a fifteen-second lease for held input. Existing held modifiers survive subsequent clicks, movement and hotkeys. `release_inputs` releases the controller's tracked keys/buttons and cancels current input without requiring an observation. Errors, human pause, display changes and disconnection release held input.
`key_press` and `hotkey` accept `duration_ms` from 10 to 30000. Drags accept durations up to 30000 milliseconds. Extended key names include F13–F24, NUMPAD0–NUMPAD9, arithmetic keypad keys, lock/menu/print keys and media controls. Target platform key mapping determines availability. Text input requires held keys/buttons to be released first.
apps/mcp/README.md
+6
View File
@@ -268,3 +268,9 @@ npm run pack:npm -- --out-dir /absolute/path/artifacts
`crossdesk_system_status` 查询系统锁定状态。Windows 同时报告服务可用性和交互桌面阶段。`crossdesk_system_command` 支持 `lock`，Windows 可通过已运行的服务执行 `send_sas`。命令需要控制授权和独立的 `operation_id`，相同请求查询原结果。系统接受命令后，通过状态及画面观察确认实际界面。
支持系统状态的被控端会在密码提交后接受最长八秒的解锁验证。确认解锁后返回 `unlock.state: verified`，并保存用户选择记住的密码。系统状态无法确认时保留 `submitted` 和验证信息。已经解锁的桌面返回 `already_unlocked`。旧被控端沿用截图确认流程。
### 持续键鼠输入
`crossdesk_action` 增加 `key_down`、`key_up`、`mouse_down` 和 `mouse_up`。按下状态可跨动作保持，例如按住 Ctrl 后移动或拖动鼠标。最后一次动作完成后，按住状态的有效期为十五秒，超时自动释放。`crossdesk_release_inputs` 可立即释放本会话的全部按键和按钮，并取消当前输入动作。人工暂停、断开及输入错误同样释放按住状态。
`key_press`、`hotkey` 支持 `duration_ms`，拖拽时长上限提高到三十秒。扩展键名覆盖 F13–F24、数字键盘、菜单、锁定键和媒体键，实际效果取决于远端系统支持。状态返回 `held_key_codes` 和 `held_mouse_flags`。文本输入前应释放持续按住的键鼠状态。
apps/mcp/src/common.ts
+15 -2
View File
@@ -73,10 +73,21 @@ const key = z.enum([
"BACKSPACE", "TAB", "ENTER", "SHIFT", "CTRL", "ALT", "PAUSE", "CAPS_LOCK",
"ESC", "SPACE", "PAGE_UP", "PAGE_DOWN", "END", "HOME", "LEFT", "UP", "RIGHT", "DOWN",
"INSERT", "DELETE", "META", "F1", "F2", "F3", "F4", "F5", "F6", "F7", "F8", "F9", "F10", "F11", "F12",
"F13", "F14", "F15", "F16", "F17", "F18",
"F19", "F20", "F21", "F22", "F23", "F24",
"NUMPAD0", "NUMPAD1", "NUMPAD2", "NUMPAD3", "NUMPAD4", "NUMPAD5",
"NUMPAD6", "NUMPAD7", "NUMPAD8", "NUMPAD9", "NUM_LOCK", "SCROLL_LOCK",
"PRINT_SCREEN", "MENU", "MULTIPLY", "ADD", "SUBTRACT", "DECIMAL",
"DIVIDE", "VOLUME_MUTE", "VOLUME_DOWN", "VOLUME_UP", "MEDIA_NEXT", "MEDIA_PREVIOUS",
"MEDIA_STOP", "MEDIA_PLAY_PAUSE",
"SEMICOLON", "EQUAL", "COMMA", "MINUS", "PERIOD", "SLASH", "BACKQUOTE", "BRACKET_LEFT", "BACKSLASH", "BRACKET_RIGHT", "QUOTE",
]);
export const action = z.discriminatedUnion("type", [
point.extend({ type: z.literal("move"), modifiers }),
point.extend({ type: z.literal("mouse_down"), button }),
point.extend({ type: z.literal("mouse_up"), button }),
z.strictObject({ type: z.literal("key_down"), key }),
z.strictObject({ type: z.literal("key_up"), key }),
point.extend({ type: z.literal("click"), button, modifiers }),
point.extend({ type: z.literal("double_click"), button, modifiers }),
point.extend({
@@ -88,14 +99,16 @@ export const action = z.discriminatedUnion("type", [
z.strictObject({
type: z.literal("drag"),
path: z.array(point).min(2).max(128),
duration_ms: z.int().min(100).max(2000).optional(),
duration_ms: z.int().min(100).max(30000).optional(),
button,
modifiers,
}),
z.strictObject({ type: z.literal("key_press"), key, modifiers })
z.strictObject({ type: z.literal("key_press"), key, modifiers,
duration_ms: z.int().min(10).max(30000).optional() })
.refine((p) => !p.modifiers?.some((modifier) => modifier === p.key)),
z.strictObject({
type: z.literal("hotkey"),
duration_ms: z.int().min(10).max(30000).optional(),
keys: z.array(key).min(2).max(6).refine(unique),
}),
z.strictObject({
apps/mcp/src/native.ts
+1 -1
View File
@@ -173,7 +173,7 @@ export class NativeClient {
if (this.pending.size >= 32 || this.child.stdin!.writableLength > 262144)
return Promise.reject(new Failure("BUSY", false, true));
const id = ++this.sequence;
const sideEffect = method === "action" || method === "unlock" ||
const sideEffect = method === "action" || method === "unlock" || method === "release_inputs" ||
(method === "clipboard" && args.operation === "set") ||
(method === "system" && args.command !== "status") ||
(method === "files" && ["upload", "download", "cancel"].includes(args.operation));
apps/mcp/src/server.ts
+3
View File
@@ -343,6 +343,9 @@ export function createServer(
"Query the remote operating system's lock state and available secure-desktop service. unknown means the platform could not determine the state.",
async (p, ctx) => output(await sessions.system({ ...p, command: "status" }, ctx.mcpReq.signal)),
{ readOnly: true });
add("crossdesk_release_inputs", z.strictObject({ session_id: id }),
"Release all keys and mouse buttons held by this session and cancel its current input action. Use when finishing a sequence of key_down/mouse_down actions or recovering from interruption. Held input expires after 15 seconds without another action.",
async (p) => output(await sessions.releaseInputs(p.session_id)));
add("crossdesk_system_command", z.strictObject({ session_id: id, operation_id: id,
command: z.enum(["lock", "send_sas"]) }),
"Lock the remote session or request Windows secure attention through its service. Requires control authorization. Reuse operation_id to query the same command. Check system status and a fresh screenshot after acceptance.",
apps/mcp/src/sessions.ts
+6 -1
View File
@@ -441,7 +441,7 @@ export class Sessions {
const s = this.session(session_id);
this.checkUnlock(s);
if (s.unlock && this.unlockPending(s.unlock)) throw new Failure("UNLOCK_PENDING");
const result = await s.native.request("action", args, 5000, signal);
const result = await s.native.request("action", args, (args.action.duration_ms ?? 0) + 5000, signal);
if (observe_after && result.status === "submitted") {
try {
result.observation = await this.screenshot(
@@ -509,6 +509,11 @@ export class Sessions {
throw new Failure("UNLOCK_PENDING");
return s.native.request("system", args, 7000, signal);
}
async releaseInputs(sessionId: string) {
const s = this.session(sessionId);
this.cancelUnlock(s);
return s.native.request("release_inputs");
}
async control(id: string, enabled: boolean) {
if (!enabled) this.cancelUnlock(this.session(id));
return this.session(id).native.request("control", { enabled });
libs/controller/include/controller_session.h
+5
View File
@@ -74,6 +74,10 @@ public:
int max_bitrate_bps = -1);
void SetControl(bool enabled);
void ReleaseInput();
void ReleaseActionInput();
bool KeyPressed(int key) const { return keys_.count(key) != 0; }
bool ButtonPressed(int flag) const { return buttons_.count(flag) != 0; }
bool HasPressedInput() const { return !keys_.empty() || !buttons_.empty(); }
bool Text(const std::string &utf8);
bool Key(int key, bool down);
bool Pointer(float x, float y, int flag, int wheel = 0);
@@ -137,6 +141,7 @@ private:
HostPlatform host_platform_ = HostPlatform::Unknown;
std::vector<HostDisplay> displays_;
std::set<int> keys_, buttons_;
std::set<int> action_keys_, action_buttons_;
float pointer_x_ = .5f, pointer_y_ = .5f;
std::chrono::steady_clock::time_point deadline_;
mutable std::mutex mutex_;
libs/controller/src/controller_requests.cpp
+2
View File
@@ -26,6 +26,8 @@ std::optional<json> ControllerSession::TakeRemoteResult(uint64_t request) {
}
void ControllerSession::BeginInput() {
action_keys_ = keys_;
action_buttons_ = buttons_;
for (const auto id : input_requests_) remote_results_.erase(id);
input_requests_.clear();
input_result_ = "applied";
libs/controller/src/controller_session.cpp
+9
View File
@@ -476,6 +476,13 @@ bool ControllerSession::Key(int key, bool down) {
keys_.erase(key);
return sent;
}
void ControllerSession::ReleaseActionInput() {
const auto buttons = buttons_, keys = keys_;
for (int button : buttons)
if (!action_buttons_.count(button)) Pointer(pointer_x_, pointer_y_, button + 1);
for (auto it = keys.rbegin(); it != keys.rend(); ++it)
if (!action_keys_.count(*it)) Key(*it, false);
}
bool ControllerSession::Pointer(float x, float y, int flag, int wheel) {
const bool releasing =
flag >= 2 && flag <= 6 && !(flag & 1) && buttons_.count(flag - 1);
@@ -523,6 +530,8 @@ json ControllerSession::Status() const {
{"error", error_},
{"generation", generation_},
{"control", control_},
{"held_key_codes", keys_},
{"held_mouse_flags", buttons_},
{"displays", ds},
{"platform", HostPlatformName(host_platform_)},
{"display", options_.display},