[feat] extend keyboard and pointer control with bounded held input

2026-11-07 23:09:21 +08:00 · 2026-11-06 20:54:28 +08:00
parent dd4baf2fc4
commit 12afcce690
11 changed files with 141 additions and 11 deletions
@@ -112,6 +112,7 @@ struct RunningAction {
  std::vector<Step> steps;
  size_t next = 0;
  bool submitted = false;
+  bool retain_input = false;
  Clock::time_point started, awaiting_ack{};
 };
 struct Record {
@@ -275,7 +276,8 @@ private:
  void FinishAction(const char *error = nullptr) {
    if (!action_)
      return;
-    session_.ReleaseInput();
+    if (error) session_.ReleaseInput();
+    else if (!action_->retain_input) session_.ReleaseActionInput();
    const auto remote = session_.InputResult();
    if (!error && remote == "pending") {
      if (action_->awaiting_ack == Clock::time_point{}) action_->awaiting_ack = Clock::now();
@@ -283,12 +285,15 @@ private:
      error = "REMOTE_ACK_TIMEOUT";
    }
    if (!error && remote == "failed") error = "REMOTE_INPUT_FAILED";
+    if (error) session_.ReleaseInput();
+    if (session_.HasPressedInput()) input_lease_ = Clock::now() + std::chrono::seconds(15);
    json result = {
        {"status", error ? (action_->submitted ? "outcome_unknown"
                                               : "failed_before_submit")
                         : "submitted"},
        {"operation_id", action_->operation},
        {"remote_execution", remote},
+        {"input_lease_ms", session_.HasPressedInput() ? 15000 : 0},
        {"generation", action_->generation},
        {"after_frame_id", frame_ ? frame_->frame_id : 0},
        {"side_effect_possible", action_->submitted},
@@ -330,6 +335,8 @@ private:
  }
  void Tick() {
    session_.Tick();
+    if (!action_ && session_.HasPressedInput() && Clock::now() >= input_lease_)
+      session_.ReleaseInput();
    for (auto it = remote_jobs_.begin(); it != remote_jobs_.end();) {
      auto result = session_.TakeRemoteResult(it->request);
      const char *error = !session_.Connected() ? "SESSION_CLOSED"
@@ -687,6 +694,27 @@ private:
    running->steps = unlock
        ? BuildUnlockSteps(String(j, "password", 256), session_.Status().value("platform", "") == "macos")
        : BuildInputSteps(j.at("action"), {o.width, o.height, o.source_width, o.source_height, o.region});
+    if (unlock) session_.ReleaseInput();
+    if (type == "type_text" && session_.HasPressedInput()) throw std::invalid_argument("INPUT_HELD");
+    running->retain_input = type == "key_down" || type == "mouse_down";
+    for (auto it = running->steps.begin(); it != running->steps.end();) {
+      if (it->kind == StepKind::Key && session_.KeyPressed(it->key) && type != "key_up") {
+        const bool modifier = it->key == 16 || it->key == 17 || it->key == 18 || it->key == 91;
+        if (type != "key_down" && !modifier) throw std::invalid_argument("KEY_ALREADY_HELD");
+        it = running->steps.erase(it);
+      } else {
+        if (it->kind == StepKind::Pointer && session_.ButtonPressed(it->flag) && type == "mouse_down") {
+          it = running->steps.erase(it);
+          continue;
+        }
+        if (it->kind == StepKind::Pointer && (it->flag == 1 || it->flag == 3 || it->flag == 5) &&
+            session_.ButtonPressed(it->flag) && type != "mouse_down")
+          throw std::invalid_argument("BUTTON_ALREADY_HELD");
+        ++it;
+      }
+    }
+    if (type == "key_down" && !running->steps.empty() && session_.Status()["held_key_codes"].size() >= 32)
+      throw std::invalid_argument("INPUT_LIMIT");
    while (records_.size() >= 128 ||
           (!records_.empty() && Age(records_.front().created) > 300000))
      records_.pop_front();
@@ -788,6 +816,10 @@ private:
      } else if (method == "action") {
        StartAction(j, id);
        return false;
+      } else if (method == "release_inputs") {
+        Fields(j, {"id", "method"});
+        FinishAction("CANCELLED");
+        session_.ReleaseInput();
      } else if (method == "system") {
        Fields(j, {"id", "method", "command", "operation_id"});
        const auto command = String(j, "command", 16);
@@ -1015,6 +1047,7 @@ private:
  json preview_;
  int preview_edge_ = 960;
  Clock::time_point preview_time_;
+  Clock::time_point input_lease_{};
  std::unique_ptr<VideoServer> video_;
  uint64_t video_generation_ = 0, last_video_frame_ = 0;
  bool video_connected_ = false, video_enabled_ = false;
@@ -85,6 +85,44 @@ int KeyCode(const std::string &key) {
                                                  {"F10", 121},
                                                  {"F11", 122},
                                                  {"F12", 123},
+                                                  {"F13", 124},
+                                                  {"F14", 125},
+                                                  {"F15", 126},
+                                                  {"F16", 127},
+                                                  {"F17", 128},
+                                                  {"F18", 129},
+                                                  {"F19", 130},
+                                                  {"F20", 131},
+                                                  {"F21", 132},
+                                                  {"F22", 133},
+                                                  {"F23", 134},
+                                                  {"F24", 135},
+                                                  {"NUMPAD0", 96},
+                                                  {"NUMPAD1", 97},
+                                                  {"NUMPAD2", 98},
+                                                  {"NUMPAD3", 99},
+                                                  {"NUMPAD4", 100},
+                                                  {"NUMPAD5", 101},
+                                                  {"NUMPAD6", 102},
+                                                  {"NUMPAD7", 103},
+                                                  {"NUMPAD8", 104},
+                                                  {"NUMPAD9", 105},
+                                                  {"NUM_LOCK", 144},
+                                                  {"SCROLL_LOCK", 145},
+                                                  {"PRINT_SCREEN", 44},
+                                                  {"MENU", 93},
+                                                  {"MULTIPLY", 106},
+                                                  {"ADD", 107},
+                                                  {"SUBTRACT", 109},
+                                                  {"DECIMAL", 110},
+                                                  {"DIVIDE", 111},
+                                                  {"VOLUME_MUTE", 173},
+                                                  {"VOLUME_DOWN", 174},
+                                                  {"VOLUME_UP", 175},
+                                                  {"MEDIA_NEXT", 176},
+                                                  {"MEDIA_PREVIOUS", 177},
+                                                  {"MEDIA_STOP", 178},
+                                                  {"MEDIA_PLAY_PAUSE", 179},
                                                  {"SEMICOLON", 186},
                                                  {"EQUAL", 187},
                                                  {"COMMA", 188},
@@ -135,7 +173,17 @@ std::vector<Step> BuildInputSteps(const json &a, const InputGeometry &o) {
    step.delay = delay;
    steps.push_back(std::move(step));
  };
-  if (type == "move" || type == "click" || type == "double_click") {
+  if (type == "mouse_down" || type == "mouse_up") {
+    Fields(a, {"type", "x", "y", "button"});
+    const auto button = a.contains("button") ? String(a, "button", 8) : "left";
+    const int flag = button == "left" ? 1 : button == "right" ? 3 : button == "middle" ? 5 : -1;
+    if (flag < 0) throw std::invalid_argument("INVALID_BUTTON");
+    point(a, 0);
+    point(a, 10, flag + (type == "mouse_up" ? 1 : 0));
+  } else if (type == "key_down" || type == "key_up") {
+    Fields(a, {"type", "key"});
+    key(KeyCode(String(a, "key", 16)), type == "key_down", 0);
+  } else if (type == "move" || type == "click" || type == "double_click") {
    Fields(a, type == "move"
                  ? std::initializer_list<const char *>{"type", "x", "y",
                                                        "modifiers"}
@@ -180,7 +228,7 @@ std::vector<Step> BuildInputSteps(const json &a, const InputGeometry &o) {
    const auto &path = a.at("path");
    if (!path.is_array() || path.size() < 2 || path.size() > 128)
      throw std::invalid_argument("INVALID_PATH");
-    const int duration = Number(a, "duration_ms", 100, 2000, 500);
+    const int duration = Number(a, "duration_ms", 100, 30000, 500);
    for (const auto &p : path)
      Fields(p, {"x", "y"});
    point(path.front(), 0);
@@ -189,12 +237,12 @@ std::vector<Step> BuildInputSteps(const json &a, const InputGeometry &o) {
      point(path[i], 10 + int(i * duration / (path.size() - 1)));
    point(path.back(), duration + 30, flag + 1);
  } else if (type == "key_press") {
-    Fields(a, {"type", "key", "modifiers"});
+    Fields(a, {"type", "key", "modifiers", "duration_ms"});
    const auto code = KeyCode(String(a, "key", 16));
    key(code, true, 0);
-    key(code, false, 40);
+    key(code, false, Number(a, "duration_ms", 10, 30000, 40));
  } else if (type == "hotkey") {
-    Fields(a, {"type", "keys"});
+    Fields(a, {"type", "keys", "duration_ms"});
    const auto &keys = a.at("keys");
    if (!keys.is_array() || keys.size() < 2 || keys.size() > 6)
      throw std::invalid_argument("INVALID_KEYS");
@@ -213,7 +261,7 @@ std::vector<Step> BuildInputSteps(const json &a, const InputGeometry &o) {
      key(code, true, delay);
      delay += 15;
    }
-    delay += 30;
+    delay += Number(a, "duration_ms", 10, 30000, 30);
    for (auto it = codes.rbegin(); it != codes.rend(); ++it) {
      key(*it, false, delay);
      delay += 15;
@@ -157,3 +157,9 @@ The desktop and CLI share `FileBrowser` and the existing file transfer engine. P
 `system` accepts `command: status | lock | send_sas`. Commands that change system state require an `operation_id` and control permission. Status queries are available during human pause. The host executes system operations through its application-thread queue, checks connection authorization again, and expires queued operations after two seconds. Windows secure attention uses the existing service integration.

 `lock_state` is `locked`, `unlocked`, or `unknown`. macOS queries the current console session, Windows uses its session lock state, and Linux queries the current logind session's `LockedHint`. The MCP unlock workflow verifies this state on compatible hosts and returns `verified` when the system reports an unlocked session.
+
+## Held input
+
+`key_down`, `key_up`, `mouse_down`, and `mouse_up` extend the action union. Each successful action renews a fifteen-second lease for held input. Existing held modifiers survive subsequent clicks, movement and hotkeys. `release_inputs` releases the controller's tracked keys/buttons and cancels current input without requiring an observation. Errors, human pause, display changes and disconnection release held input.
+
+`key_press` and `hotkey` accept `duration_ms` from 10 to 30000. Drags accept durations up to 30000 milliseconds. Extended key names include F13–F24, NUMPAD0–NUMPAD9, arithmetic keypad keys, lock/menu/print keys and media controls. Target platform key mapping determines availability. Text input requires held keys/buttons to be released first.