YangWX/crossdesk
1
0
Fork 0
mirror of https://github.com/kunkundi/crossdesk.git synced 2026-11-05 13:42:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

[fix] stabilize MCP sessions and extend input controls

Browse Source
This commit is contained in:
dijunkun
2026-11-03 14:19:39 +08:00
parent e1ae449d36
commit fc1529ee2b
15 changed files with 596 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/cli/agent.cpp
+61 -207
View File
@@ -1,4 +1,5 @@
#include "agent.h"
#include "automation_protocol.h"
#include "terminal_io.h"
#include "video_server.h"
#include <algorithm>
@@ -7,7 +8,6 @@
#include <deque>
#include <frame_image.h>
#include <future>
#include <map>
#include <memory>
#include <stdexcept>
#include <terminal_renderer.h>
@@ -15,49 +15,10 @@
namespace crossdesk::cli {
namespace {
using nlohmann::json;
using namespace automation;
using Clock = std::chrono::steady_clock;
using Ms = std::chrono::milliseconds;
constexpr size_t kMaxResponse = 4 * 1024 * 1024;
void Fields(const json &j, std::initializer_list<const char *> names) {
if (!j.is_object())
throw std::invalid_argument("INVALID_REQUEST");
for (auto it = j.begin(); it != j.end(); ++it)
if (std::none_of(names.begin(), names.end(),
[&](const char *s) { return it.key() == s; }))
throw std::invalid_argument("UNKNOWN_FIELD");
}
int Number(const json &j, const char *key, int low, int high, int fallback) {
if (!j.contains(key))
return fallback;
if (!j[key].is_number_integer())
throw std::invalid_argument("INVALID_REQUEST");
const auto n = j[key].get<int64_t>();
if (n < low || n > high)
throw std::invalid_argument("INVALID_REQUEST");
return int(n);
}
uint64_t Sequence(const json &j, const char *key, uint64_t fallback = 0) {
if (!j.contains(key))
return fallback;
if (!j[key].is_number_unsigned())
throw std::invalid_argument("INVALID_REQUEST");
return j[key].get<uint64_t>();
}
bool Flag(const json &j, const char *key, bool fallback = false) {
if (!j.contains(key))
return fallback;
if (!j[key].is_boolean())
throw std::invalid_argument("INVALID_REQUEST");
return j[key].get<bool>();
}
std::string String(const json &j, const char *key, size_t limit = 128) {
if (!j.contains(key) || !j[key].is_string())
throw std::invalid_argument("INVALID_REQUEST");
auto s = j[key].get<std::string>();
if (s.empty() || s.size() > limit || s.find('\0') != std::string::npos)
throw std::invalid_argument("INVALID_REQUEST");
return s;
}
int64_t Age(Clock::time_point queued) {
return std::chrono::duration_cast<Ms>(Clock::now() - queued).count();
}
@@ -125,14 +86,9 @@ struct TerminalEncoding {
json id;
uint64_t generation, epoch;
Clock::time_point queued;
bool cancelled = false;
std::future<terminal::Grid> result;
};
struct Step {
int delay = 0, kind = 0, key = 0, flag = 0, wheel = 0;
float x = 0, y = 0;
bool down = false;
std::string text;
};
struct RunningAction {
json id;
std::string operation;
@@ -147,25 +103,6 @@ struct Record {
json result;
Clock::time_point created;
};
int KeyCode(const std::string &key) {
if (key.size() == 1 &&
((key[0] >= 'A' && key[0] <= 'Z') || (key[0] >= '0' && key[0] <= '9')))
return key[0];
static const std::map<std::string, int> keys = {
{"BACKSPACE", 8}, {"TAB", 9}, {"ENTER", 13}, {"SHIFT", 16},
{"CTRL", 17}, {"ALT", 18}, {"PAUSE", 19}, {"CAPS_LOCK", 20},
{"ESC", 27}, {"SPACE", 32}, {"PAGE_UP", 33}, {"PAGE_DOWN", 34},
{"END", 35}, {"HOME", 36}, {"LEFT", 37}, {"UP", 38},
{"RIGHT", 39}, {"DOWN", 40}, {"INSERT", 45}, {"DELETE", 46},
{"META", 91}, {"F1", 112}, {"F2", 113}, {"F3", 114},
{"F4", 115}, {"F5", 116}, {"F6", 117}, {"F7", 118},
{"F8", 119}, {"F9", 120}, {"F10", 121}, {"F11", 122},
{"F12", 123}};
auto it = keys.find(key);
if (it == keys.end())
throw std::invalid_argument("INVALID_KEY");
return it->second;
}
class Automation {
public:
Automation(int output, SessionOptions options, int password, bool video)
@@ -190,18 +127,24 @@ public:
eof = true;
else if (errno != EAGAIN && errno != EINTR)
eof = true;
if (buffer.size() > 65536) {
Error(nullptr, "MESSAGE_TOO_LARGE");
quit = true;
}
for (int count = 0; count < 16 && !quit; ++count) {
const auto end = buffer.find('\n');
if ((end == std::string::npos && buffer.size() > 65536) ||
(end != std::string::npos && end > 65536)) {
Error(nullptr, "MESSAGE_TOO_LARGE");
quit = true;
break;
}
if (end == std::string::npos)
break;
auto request = ParseBoundedJson(buffer.substr(0, end));
buffer.erase(0, end + 1);
quit = Handle(request);
}
if (!quit && buffer.size() > 128 * 1024) {
Error(nullptr, "INPUT_BACKPRESSURE");
quit = true;
}
if (eof && buffer.find('\n') == std::string::npos) {
if (!buffer.empty())
Error(nullptr, "INCOMPLETE_MESSAGE");
@@ -313,16 +256,10 @@ private:
code == "BUSY")},
{"side_effect_possible", side_effect}}}});
}
void Release() {
const bool enabled = session_.Status()["control"];
session_.SetControl(false);
if (enabled && allowed_ && !paused_ && session_.Connected())
session_.SetControl(true);
}
void FinishAction(const char *error = nullptr) {
if (!action_)
return;
Release();
session_.ReleaseInput();
json result = {
{"status", error ? (action_->submitted ? "outcome_unknown"
: "failed_before_submit")
@@ -351,12 +288,18 @@ private:
Error(encoding_->capture.id, code);
encoding_->cancelled = true;
}
if (terminal_encoding_ && !terminal_encoding_->cancelled) {
Error(terminal_encoding_->id, code);
terminal_encoding_->cancelled = true;
}
observations_.clear();
preview_ = json();
}
void Tick() {
session_.Tick();
if (auto f = session_.TakeFrame()) {
// Recycle only uniquely owned frames; an encoder/video worker may still be
// reading the old pixels. Save geometry before Take can swap its storage.
if (auto f = session_.TakeFrame(frame_ && frame_.use_count() == 1 ? frame_.get() : nullptr)) {
if (!frame_ || f->generation != frame_->generation ||
f->width != frame_->width || f->height != frame_->height) {
++epoch_;
@@ -384,15 +327,15 @@ private:
action_->submitted =
true; // A failed send may still have reached the peer.
const bool sent =
step.kind == 0
step.kind == StepKind::Pointer
? session_.Pointer(step.x, step.y, step.flag, step.wheel)
: step.kind == 1 ? session_.Key(step.key, step.down)
: step.kind == StepKind::Key ? session_.Key(step.key, step.down)
: session_.Text(step.text);
if (!sent) {
FinishAction("ACTION_OUTCOME_UNKNOWN");
break;
}
if (step.kind == 0) {
if (step.kind == StepKind::Pointer) {
cursor_x_ = step.x;
cursor_y_ = step.y;
cursor_visible_ = true;
@@ -407,7 +350,9 @@ private:
auto job = std::move(terminal_encoding_);
try {
auto grid = job->result.get();
if (!session_.Connected() || job->generation != session_.Generation() ||
if (job->cancelled) {
// Consume the worker result without replying twice.
} else if (!session_.Connected() || job->generation != session_.Generation() ||
job->epoch != epoch_)
Error(job->id, "STALE_OBSERVATION");
else if (Age(job->queued) > 2000)
@@ -425,7 +370,7 @@ private:
{"lines", terminal::Lines(grid)}, {"cursor", cursor},
{"generation", job->generation}, {"geometry_epoch", job->epoch}});
}
} catch (...) { Error(job->id, "FRAME_RENDER_FAILED"); }
} catch (...) { if (!job->cancelled) Error(job->id, "FRAME_RENDER_FAILED"); }
}
if (encoding_ &&
encoding_->result.wait_for(Ms(0)) == std::future_status::ready) {
@@ -436,8 +381,7 @@ private:
if (!frame_ || job->epoch != epoch_ ||
job->frame->generation != session_.Generation())
Error(job->capture.id, "STALE_OBSERVATION");
else if (Age(job->frame->queued) > 2000 ||
Clock::now() > job->capture.deadline)
else if (Age(job->frame->queued) > 2000)
Error(job->capture.id, "FRAME_TIMEOUT");
else {
const auto &f = *job->frame;
@@ -492,7 +436,7 @@ private:
} else
++it;
}
if (!encoding_ && !captures_.empty() && frame_ &&
if (!encoding_ && !captures_.empty() && frame_ && session_.Connected() &&
Age(frame_->queued) <= 2000) {
auto it = std::find_if(
captures_.begin(), captures_.end(), [&](const Capture &c) {
@@ -574,12 +518,24 @@ private:
}
if (captures_.size() >= 16)
throw std::invalid_argument("BUSY");
Capture c{id, j, preview,
Clock::now() + Ms(Number(j, "timeout_ms", 0, 5000, 2000))};
const int timeout = Number(j, "timeout_ms", 0, 5000, 2000);
Capture c{id, j, preview, Clock::now() + Ms(timeout)};
// timeout_ms bounds waiting for a suitable frame, not PNG encoding.
// A zero timeout can capture an already available frame immediately.
const bool available = frame_ && frame_->generation == session_.Generation() &&
Age(frame_->queued) <= 2000 &&
(!j.contains("after_frame_id") || frame_->frame_id > Sequence(j, "after_frame_id"));
if (!encoding_ && captures_.empty() && available) {
Encode(std::move(c));
return;
}
if (timeout == 0)
throw std::invalid_argument(encoding_ || !captures_.empty() ? "BUSY" : "FRAME_TIMEOUT");
if (preview)
captures_.push_back(c);
else
captures_.push_front(c);
captures_.insert(std::find_if(captures_.begin(), captures_.end(),
[](const Capture &queued) { return queued.preview; }), c);
}
const Observation &Observe(const json &j) {
auto name = String(j, "observation_id");
@@ -604,9 +560,9 @@ private:
Reply(id, r.result);
return;
}
if (!allowed_ || paused_ || !session_.Status()["control"].get<bool>())
if (!allowed_ || paused_ || !session_.ControlEnabled())
throw std::invalid_argument("CONTROL_DENIED");
if (!session_.Connected() || !frame_)
if (!session_.Connected() || !frame_ || Age(frame_->queued) > 2000)
throw std::invalid_argument("NOT_READY");
if (action_)
throw std::invalid_argument("BUSY");
@@ -618,120 +574,10 @@ private:
running->operation = operation;
running->generation = o.generation;
running->epoch = o.epoch;
auto point = [&](const json &p, int delay, int flag = 0, int wheel = 0) {
auto coordinate = [&](const char *key, int size, int offset, int span,
int source) {
if (!p.contains(key) || !p[key].is_number())
throw std::invalid_argument("INVALID_COORDINATE");
const auto value = p[key].get<double>();
if (!std::isfinite(value) || value < 0 || value >= size)
throw std::invalid_argument("INVALID_COORDINATE");
return float(std::clamp((offset + (value + .5) * span / size) / source,
0.0, 1.0));
};
Step step;
step.delay = delay;
step.flag = flag;
step.wheel = wheel;
step.x =
coordinate("x", o.width, o.region.x, o.region.width, o.source_width);
step.y = coordinate("y", o.height, o.region.y, o.region.height,
o.source_height);
running->steps.push_back(std::move(step));
};
auto key = [&](int code, bool down, int delay) {
Step step;
step.kind = 1;
step.key = code;
step.down = down;
step.delay = delay;
running->steps.push_back(std::move(step));
};
if (type == "move" || type == "click" || type == "double_click") {
Fields(a, type == "move"
? std::initializer_list<const char *>{"type", "x", "y"}
: std::initializer_list<const char *>{"type", "x", "y",
"button"});
point(a, 0);
if (type != "move") {
const auto button =
a.contains("button") ? String(a, "button", 8) : "left";
const int flag = button == "left" ? 1
: button == "right" ? 3
: button == "middle" ? 5
: -1;
if (flag < 0)
throw std::invalid_argument("INVALID_BUTTON");
point(a, 10, flag);
point(a, 40, flag + 1);
if (type == "double_click") {
point(a, 110, flag);
point(a, 140, flag + 1);
}
}
} else if (type == "scroll") {
Fields(a, {"type", "x", "y", "direction", "ticks"});
const auto direction = String(a, "direction", 8);
if (direction != "up" && direction != "down" && direction != "left" &&
direction != "right")
throw std::invalid_argument("INVALID_DIRECTION");
const int ticks = Number(a, "ticks", 1, 10, 1);
point(a, 0);
point(a, 10, direction == "left" || direction == "right" ? 8 : 7,
(direction == "up" || direction == "right" ? 120 : -120) * ticks);
} else if (type == "drag") {
Fields(a, {"type", "path", "duration_ms"});
const auto &path = a.at("path");
if (!path.is_array() || path.size() < 2 || path.size() > 128)
throw std::invalid_argument("INVALID_PATH");
const int duration = Number(a, "duration_ms", 100, 2000, 500);
for (const auto &p : path)
Fields(p, {"x", "y"});
point(path.front(), 0);
point(path.front(), 10, 1);
for (size_t i = 1; i < path.size(); ++i)
point(path[i], 10 + int(i * duration / (path.size() - 1)));
point(path.back(), duration + 30, 2);
} else if (type == "key_press") {
Fields(a, {"type", "key"});
const auto code = KeyCode(String(a, "key", 16));
key(code, true, 0);
key(code, false, 40);
} else if (type == "hotkey") {
Fields(a, {"type", "keys"});
const auto &keys = a.at("keys");
if (!keys.is_array() || keys.size() < 2 || keys.size() > 6)
throw std::invalid_argument("INVALID_KEYS");
std::set<int> seen;
std::vector<int> codes;
for (const auto &item : keys) {
if (!item.is_string())
throw std::invalid_argument("INVALID_KEY");
int code = KeyCode(item.get<std::string>());
if (!seen.insert(code).second)
throw std::invalid_argument("DUPLICATE_KEY");
codes.push_back(code);
}
int delay = 0;
for (int code : codes) {
key(code, true, delay);
delay += 15;
}
delay += 30;
for (auto it = codes.rbegin(); it != codes.rend(); ++it) {
key(*it, false, delay);
delay += 15;
}
} else if (type == "type_text") {
Fields(a, {"type", "text"});
if (!session_.Status()["text_input"].get<bool>())
throw std::invalid_argument("TEXT_INPUT_UNSUPPORTED");
Step step;
step.kind = 2;
step.text = String(a, "text", 16384);
running->steps.push_back(std::move(step));
} else
throw std::invalid_argument("UNSUPPORTED_ACTION");
if (type == "type_text" && !session_.Status()["text_input"].get<bool>())
throw std::invalid_argument("TEXT_INPUT_UNSUPPORTED");
running->steps = automation::BuildInputSteps(a,
{o.width, o.height, o.source_width, o.source_height, o.region});
while (records_.size() >= 128 ||
(!records_.empty() && Age(records_.front().created) > 300000))
records_.pop_front();
@@ -884,8 +730,12 @@ private:
throw std::invalid_argument("INVALID_REQUEST");
} else if (method == "display") {
Fields(j, {"id", "method", "index"});
const int index = Number(j, "index", 0, 63, -1);
if (!session_.Connected() || index < 0 ||
size_t(index) >= session_.Status()["displays"].size())
throw std::invalid_argument("DISPLAY_UNAVAILABLE");
Stop("SESSION_CHANGED");
if (!session_.SelectDisplay(Number(j, "index", 0, 63, -1)))
if (!session_.SelectDisplay(index))
throw std::invalid_argument("DISPLAY_UNAVAILABLE");
frame_.reset();
++epoch_;
@@ -902,6 +752,10 @@ private:
encoding_->cancelled = true;
Error(target, "CANCELLED");
}
if (terminal_encoding_ && !terminal_encoding_->cancelled && terminal_encoding_->id == target) {
terminal_encoding_->cancelled = true;
Error(target, "CANCELLED");
}
for (auto it = captures_.begin(); it != captures_.end();) {
if (it->id == target) {
Error(target, "CANCELLED");
@@ -954,7 +808,7 @@ private:
bool hello_ = false, connected_once_ = false, allowed_ = false,
paused_ = false;
uint64_t epoch_ = 0, observation_sequence_ = 0;
std::shared_ptr<const DecodedFrame> frame_;
std::shared_ptr<DecodedFrame> frame_;
std::deque<Observation> observations_;
std::deque<Capture> captures_;
std::unique_ptr<Encoding> encoding_;
apps/cli/automation_protocol.cpp
+262
View File
@@ -0,0 +1,262 @@
#include "automation_protocol.h"
#include <algorithm>
#include <cmath>
#include <map>
#include <set>
#include <stdexcept>
namespace crossdesk::cli::automation {
void Fields(const json &j, std::initializer_list<const char *> names) {
if (!j.is_object())
throw std::invalid_argument("INVALID_REQUEST");
for (auto it = j.begin(); it != j.end(); ++it)
if (std::none_of(names.begin(), names.end(),
[&](const char *s) { return it.key() == s; }))
throw std::invalid_argument("UNKNOWN_FIELD");
}
int Number(const json &j, const char *key, int low, int high, int fallback) {
if (!j.contains(key))
return fallback;
if (!j[key].is_number_integer())
throw std::invalid_argument("INVALID_REQUEST");
const auto n = j[key].get<int64_t>();
if (n < low || n > high)
throw std::invalid_argument("INVALID_REQUEST");
return int(n);
}
uint64_t Sequence(const json &j, const char *key, uint64_t fallback) {
if (!j.contains(key))
return fallback;
if (!j[key].is_number_unsigned())
throw std::invalid_argument("INVALID_REQUEST");
return j[key].get<uint64_t>();
}
bool Flag(const json &j, const char *key, bool fallback) {
if (!j.contains(key))
return fallback;
if (!j[key].is_boolean())
throw std::invalid_argument("INVALID_REQUEST");
return j[key].get<bool>();
}
std::string String(const json &j, const char *key, size_t limit) {
if (!j.contains(key) || !j[key].is_string())
throw std::invalid_argument("INVALID_REQUEST");
auto s = j[key].get<std::string>();
if (s.empty() || s.size() > limit || s.find('\0') != std::string::npos)
throw std::invalid_argument("INVALID_REQUEST");
return s;
}
namespace {
int KeyCode(const std::string &key) {
if (key.size() == 1 &&
((key[0] >= 'A' && key[0] <= 'Z') || (key[0] >= '0' && key[0] <= '9')))
return key[0];
static const std::map<std::string, int> keys = {{"BACKSPACE", 8},
{"TAB", 9},
{"ENTER", 13},
{"SHIFT", 16},
{"CTRL", 17},
{"ALT", 18},
{"PAUSE", 19},
{"CAPS_LOCK", 20},
{"ESC", 27},
{"SPACE", 32},
{"PAGE_UP", 33},
{"PAGE_DOWN", 34},
{"END", 35},
{"HOME", 36},
{"LEFT", 37},
{"UP", 38},
{"RIGHT", 39},
{"DOWN", 40},
{"INSERT", 45},
{"DELETE", 46},
{"META", 91},
{"F1", 112},
{"F2", 113},
{"F3", 114},
{"F4", 115},
{"F5", 116},
{"F6", 117},
{"F7", 118},
{"F8", 119},
{"F9", 120},
{"F10", 121},
{"F11", 122},
{"F12", 123},
{"SEMICOLON", 186},
{"EQUAL", 187},
{"COMMA", 188},
{"MINUS", 189},
{"PERIOD", 190},
{"SLASH", 191},
{"BACKQUOTE", 192},
{"BRACKET_LEFT", 219},
{"BACKSLASH", 220},
{"BRACKET_RIGHT", 221},
{"QUOTE", 222}};
auto it = keys.find(key);
if (it == keys.end())
throw std::invalid_argument("INVALID_KEY");
return it->second;
}
} // namespace
std::vector<Step> BuildInputSteps(const json &a, const InputGeometry &o) {
const auto type = String(a, "type", 32);
std::vector<Step> steps;
auto point = [&](const json &p, int delay, int flag = 0, int wheel = 0) {
auto coordinate = [&](const char *key, int size, int offset, int span,
int source) {
if (!p.contains(key) || !p[key].is_number())
throw std::invalid_argument("INVALID_COORDINATE");
const auto value = p[key].get<double>();
if (!std::isfinite(value) || value < 0 || value >= size)
throw std::invalid_argument("INVALID_COORDINATE");
const auto center = std::min(value + .5, double(size) - .5);
return float(std::clamp((offset + center * span / size) / source, 0.0, 1.0));
};
Step step;
step.delay = delay;
step.flag = flag;
step.wheel = wheel;
step.x =
coordinate("x", o.width, o.region.x, o.region.width, o.source_width);
step.y =
coordinate("y", o.height, o.region.y, o.region.height, o.source_height);
steps.push_back(std::move(step));
};
auto key = [&](int code, bool down, int delay) {
Step step;
step.kind = StepKind::Key;
step.key = code;
step.down = down;
step.delay = delay;
steps.push_back(std::move(step));
};
if (type == "move" || type == "click" || type == "double_click") {
Fields(a, type == "move"
? std::initializer_list<const char *>{"type", "x", "y",
"modifiers"}
: std::initializer_list<const char *>{"type", "x", "y",
"button", "modifiers"});
point(a, 0);
if (type != "move") {
const auto button =
a.contains("button") ? String(a, "button", 8) : "left";
const int flag = button == "left" ? 1
: button == "right" ? 3
: button == "middle" ? 5
: -1;
if (flag < 0)
throw std::invalid_argument("INVALID_BUTTON");
point(a, 10, flag);
point(a, 40, flag + 1);
if (type == "double_click") {
point(a, 110, flag);
point(a, 140, flag + 1);
}
}
} else if (type == "scroll") {
Fields(a, {"type", "x", "y", "direction", "ticks", "modifiers"});
const auto direction = String(a, "direction", 8);
if (direction != "up" && direction != "down" && direction != "left" &&
direction != "right")
throw std::invalid_argument("INVALID_DIRECTION");
const int ticks = Number(a, "ticks", 1, 10, 1);
point(a, 0);
point(a, 10, direction == "left" || direction == "right" ? 8 : 7,
(direction == "up" || direction == "right" ? 120 : -120) * ticks);
} else if (type == "drag") {
Fields(a, {"type", "path", "duration_ms", "button", "modifiers"});
const auto button = a.contains("button") ? String(a, "button", 8) : "left";
const int flag = button == "left" ? 1
: button == "right" ? 3
: button == "middle" ? 5
: -1;
if (flag < 0)
throw std::invalid_argument("INVALID_BUTTON");
const auto &path = a.at("path");
if (!path.is_array() || path.size() < 2 || path.size() > 128)
throw std::invalid_argument("INVALID_PATH");
const int duration = Number(a, "duration_ms", 100, 2000, 500);
for (const auto &p : path)
Fields(p, {"x", "y"});
point(path.front(), 0);
point(path.front(), 10, flag);
for (size_t i = 1; i < path.size(); ++i)
point(path[i], 10 + int(i * duration / (path.size() - 1)));
point(path.back(), duration + 30, flag + 1);
} else if (type == "key_press") {
Fields(a, {"type", "key", "modifiers"});
const auto code = KeyCode(String(a, "key", 16));
key(code, true, 0);
key(code, false, 40);
} else if (type == "hotkey") {
Fields(a, {"type", "keys"});
const auto &keys = a.at("keys");
if (!keys.is_array() || keys.size() < 2 || keys.size() > 6)
throw std::invalid_argument("INVALID_KEYS");
std::set<int> seen;
std::vector<int> codes;
for (const auto &item : keys) {
if (!item.is_string())
throw std::invalid_argument("INVALID_KEY");
int code = KeyCode(item.get<std::string>());
if (!seen.insert(code).second)
throw std::invalid_argument("DUPLICATE_KEY");
codes.push_back(code);
}
int delay = 0;
for (int code : codes) {
key(code, true, delay);
delay += 15;
}
delay += 30;
for (auto it = codes.rbegin(); it != codes.rend(); ++it) {
key(*it, false, delay);
delay += 15;
}
} else if (type == "type_text") {
Fields(a, {"type", "text"});
Step step;
step.kind = StepKind::Text;
step.text = String(a, "text", 16384);
steps.push_back(std::move(step));
} else
throw std::invalid_argument("UNSUPPORTED_ACTION");
if (a.contains("modifiers")) {
const auto &modifiers = a["modifiers"];
if (!modifiers.is_array() || modifiers.size() > 4)
throw std::invalid_argument("INVALID_KEYS");
std::set<int> seen;
std::vector<Step> prefix;
for (const auto &item : modifiers) {
if (!item.is_string())
throw std::invalid_argument("INVALID_KEY");
const auto name = item.get<std::string>();
if (name != "CTRL" && name != "ALT" && name != "SHIFT" && name != "META")
throw std::invalid_argument("INVALID_KEY");
const int code = KeyCode(name);
if (!seen.insert(code).second ||
(type == "key_press" && String(a, "key", 16) == name))
throw std::invalid_argument("DUPLICATE_KEY");
Step step;
step.kind = StepKind::Key;
step.key = code;
step.down = true;
step.delay = int(prefix.size()) * 15;
prefix.push_back(step);
}
const int offset = int(prefix.size()) * 15;
for (auto &step : steps)
step.delay += offset;
steps.insert(steps.begin(), prefix.begin(), prefix.end());
int delay = steps.back().delay + 15;
for (auto it = prefix.rbegin(); it != prefix.rend(); ++it, delay += 15)
key(it->key, false, delay);
}
return steps;
}
} // namespace crossdesk::cli::automation
apps/cli/automation_protocol.h
+41
View File
@@ -0,0 +1,41 @@
/*
* @Author: DI JUNKUN
* @Date: 2026-11-03
* Copyright (c) 2026 by DI JUNKUN, All Rights Reserved.
*/
#ifndef _AUTOMATION_PROTOCOL_H_
#define _AUTOMATION_PROTOCOL_H_
#include <frame_image.h>
#include <initializer_list>
#include <string>
#include <vector>
namespace crossdesk::cli::automation {
using nlohmann::json;
void Fields(const json &, std::initializer_list<const char *> names);
int Number(const json &, const char *key, int low, int high, int fallback);
uint64_t Sequence(const json &, const char *key, uint64_t fallback = 0);
bool Flag(const json &, const char *key, bool fallback = false);
std::string String(const json &, const char *key, size_t limit = 128);
struct InputGeometry {
int width, height, source_width, source_height;
ImageRegion region;
};
enum class StepKind { Pointer, Key, Text };
struct Step {
StepKind kind = StepKind::Pointer;
int delay = 0, key = 0, flag = 0, wheel = 0;
float x = 0, y = 0;
bool down = false;
std::string text;
};
// Pure validation and translation: no session access or input side effects.
std::vector<Step> BuildInputSteps(const json &action,
const InputGeometry &geometry);
} // namespace crossdesk::cli::automation
#endif
apps/cli/docs/automation-protocol.md
+7 -1
View File
@@ -93,6 +93,8 @@ NV12 frames must be tightly packed, even dimensions, at most 8192 per axis and 1
`after_frame_id` requires `generation`. A screenshot waits up to `timeout_ms` for a newer frame; a preview can return `{unchanged:true, frame_age_ms, stale, generation, frame_id}`. Neither path treats an image as an input acknowledgement.
`timeout_ms` bounds frame/worker availability, not the subsequent PNG encoding. With `timeout_ms: 0`, a fresh eligible frame is encoded immediately if the worker and queue are idle; otherwise return `FRAME_TIMEOUT` or `BUSY`. Captures of the same priority retain arrival order. Encoding still rejects obsolete geometry and frames that become older than two seconds. Terminal rendering also honors cancellation without sending a second result when the worker finishes.
## Actions
All coordinates refer to the immutable returned observation image. The native layer maps them through its source rectangle to normalized remote coordinates, without terminal layout padding. Coordinates must satisfy `0 <= x < image_width` and `0 <= y < image_height`. Observations expire after 30 seconds, retain at most 32 entries, and are invalidated by geometry/display generation changes.
@@ -106,12 +108,16 @@ Supported payloads:
- `move`: `x`, `y`.
- `click`, `double_click`: `x`, `y`, optional `button` (`left`, `right`, `middle`).
- `scroll`: `x`, `y`, `direction` (`up`, `down`, `left`, `right`), optional `ticks` (1–10).
- `drag`: `path` of 2–128 `{x,y}` points, optional `duration_ms` (100–2000).
- `drag`: `path` of 2–128 `{x,y}` points, optional `duration_ms` (100–2000), optional `button` (`left`, `right`, `middle`; default `left`).
- `key_press`: `key`; `hotkey`: 2–6 unique `keys`.
- `type_text`: `text`, at most 16 KiB UTF-8; requires negotiated remote text-input capability.
All pointer actions and `key_press` accept optional `modifiers`, an array of at most four unique names from `CTRL`, `ALT`, `SHIFT`, `META`. They are pressed before the action and released afterwards in reverse order; a `key_press` cannot also list its own key as a modifier. `hotkey` already describes the complete chord and does not accept `modifiers`. Punctuation VK names are `SEMICOLON`, `EQUAL`, `COMMA`, `MINUS`, `PERIOD`, `SLASH`, `BACKQUOTE`, `BRACKET_LEFT`, `BACKSLASH`, `BRACKET_RIGHT`, `QUOTE`; these name keys, not layout-independent text.
Only one input sequence runs at once. Timed steps are advanced by the owner loop; cancellation, disconnect, display changes and pause release held buttons/keys. A failed send is conservatively considered potentially submitted.
New actions require a current frame no older than two seconds as well as a valid observation. Cached operation lookups remain available with a stalled feed or paused control. Failed key/button releases remain tracked for subsequent cleanup, including after control is disabled; a successful local send still does not prove remote execution. Invalid display indices leave existing observations and input sequences intact.
Results use `status: submitted | in_progress | failed_before_submit | outcome_unknown`, with `operation_id`, `retryable`, `side_effect_possible` and, for completed actions, `generation`/`after_frame_id`. The same operation ID and arguments retrieve the cached result; different arguments conflict. Retention is bounded to 128 entries, with entries older than five minutes eligible for removal on a new action. Never retry an uncertain effect with a new ID. `submitted` does not claim remote execution.
The native process trusts its parent. The parent must authenticate panel-only `viewer_control` calls and protect descriptor 3; this private pipe does not isolate arbitrary programs running as the same OS user.
apps/cli/xmake.lua
+1 -1
View File
@@ -37,7 +37,7 @@ target("crossdesk-cli")
set_version("0.2.0")
add_deps("crossdesk_terminal")
add_files("main.cpp")
add_files("agent.cpp", "video_server.cpp")
add_files("agent.cpp", "automation_protocol.cpp", "video_server.cpp")
add_packages("asio", "websocketpp")
add_files(is_plat("windows") and "terminal_io_windows.cpp" or "terminal_io_posix.cpp")
if is_plat("windows") then