YangWX/crossdesk
1
0
Fork 0
mirror of https://github.com/kunkundi/crossdesk.git synced 2026-11-04 21:12:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

[fix] stabilize MCP sessions and extend input controls

Browse Source
This commit is contained in:
dijunkun
2026-11-03 14:19:39 +08:00
parent e1ae449d36
commit fc1529ee2b
15 changed files with 596 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/cli/agent.cpp
+61 -207
View File
@@ -1,4 +1,5 @@
#include "agent.h"
#include "automation_protocol.h"
#include "terminal_io.h"
#include "video_server.h"
#include <algorithm>
@@ -7,7 +8,6 @@
#include <deque>
#include <frame_image.h>
#include <future>
#include <map>
#include <memory>
#include <stdexcept>
#include <terminal_renderer.h>
@@ -15,49 +15,10 @@
namespace crossdesk::cli {
namespace {
using nlohmann::json;
using namespace automation;
using Clock = std::chrono::steady_clock;
using Ms = std::chrono::milliseconds;
constexpr size_t kMaxResponse = 4 * 1024 * 1024;
void Fields(const json &j, std::initializer_list<const char *> names) {
if (!j.is_object())
throw std::invalid_argument("INVALID_REQUEST");
for (auto it = j.begin(); it != j.end(); ++it)
if (std::none_of(names.begin(), names.end(),
[&](const char *s) { return it.key() == s; }))
throw std::invalid_argument("UNKNOWN_FIELD");
}
int Number(const json &j, const char *key, int low, int high, int fallback) {
if (!j.contains(key))
return fallback;
if (!j[key].is_number_integer())
throw std::invalid_argument("INVALID_REQUEST");
const auto n = j[key].get<int64_t>();
if (n < low || n > high)
throw std::invalid_argument("INVALID_REQUEST");
return int(n);
}
uint64_t Sequence(const json &j, const char *key, uint64_t fallback = 0) {
if (!j.contains(key))
return fallback;
if (!j[key].is_number_unsigned())
throw std::invalid_argument("INVALID_REQUEST");
return j[key].get<uint64_t>();
}
bool Flag(const json &j, const char *key, bool fallback = false) {
if (!j.contains(key))
return fallback;
if (!j[key].is_boolean())
throw std::invalid_argument("INVALID_REQUEST");
return j[key].get<bool>();
}
std::string String(const json &j, const char *key, size_t limit = 128) {
if (!j.contains(key) || !j[key].is_string())
throw std::invalid_argument("INVALID_REQUEST");
auto s = j[key].get<std::string>();
if (s.empty() || s.size() > limit || s.find('\0') != std::string::npos)
throw std::invalid_argument("INVALID_REQUEST");
return s;
}
int64_t Age(Clock::time_point queued) {
return std::chrono::duration_cast<Ms>(Clock::now() - queued).count();
}
@@ -125,14 +86,9 @@ struct TerminalEncoding {
json id;
uint64_t generation, epoch;
Clock::time_point queued;
bool cancelled = false;
std::future<terminal::Grid> result;
};
struct Step {
int delay = 0, kind = 0, key = 0, flag = 0, wheel = 0;
float x = 0, y = 0;
bool down = false;
std::string text;
};
struct RunningAction {
json id;
std::string operation;
@@ -147,25 +103,6 @@ struct Record {
json result;
Clock::time_point created;
};
int KeyCode(const std::string &key) {
if (key.size() == 1 &&
((key[0] >= 'A' && key[0] <= 'Z') || (key[0] >= '0' && key[0] <= '9')))
return key[0];
static const std::map<std::string, int> keys = {
{"BACKSPACE", 8}, {"TAB", 9}, {"ENTER", 13}, {"SHIFT", 16},
{"CTRL", 17}, {"ALT", 18}, {"PAUSE", 19}, {"CAPS_LOCK", 20},
{"ESC", 27}, {"SPACE", 32}, {"PAGE_UP", 33}, {"PAGE_DOWN", 34},
{"END", 35}, {"HOME", 36}, {"LEFT", 37}, {"UP", 38},
{"RIGHT", 39}, {"DOWN", 40}, {"INSERT", 45}, {"DELETE", 46},
{"META", 91}, {"F1", 112}, {"F2", 113}, {"F3", 114},
{"F4", 115}, {"F5", 116}, {"F6", 117}, {"F7", 118},
{"F8", 119}, {"F9", 120}, {"F10", 121}, {"F11", 122},
{"F12", 123}};
auto it = keys.find(key);
if (it == keys.end())
throw std::invalid_argument("INVALID_KEY");
return it->second;
}
class Automation {
public:
Automation(int output, SessionOptions options, int password, bool video)
@@ -190,18 +127,24 @@ public:
eof = true;
else if (errno != EAGAIN && errno != EINTR)
eof = true;
if (buffer.size() > 65536) {
Error(nullptr, "MESSAGE_TOO_LARGE");
quit = true;
}
for (int count = 0; count < 16 && !quit; ++count) {
const auto end = buffer.find('\n');
if ((end == std::string::npos && buffer.size() > 65536) ||
(end != std::string::npos && end > 65536)) {
Error(nullptr, "MESSAGE_TOO_LARGE");
quit = true;
break;
}
if (end == std::string::npos)
break;
auto request = ParseBoundedJson(buffer.substr(0, end));
buffer.erase(0, end + 1);
quit = Handle(request);
}
if (!quit && buffer.size() > 128 * 1024) {
Error(nullptr, "INPUT_BACKPRESSURE");
quit = true;
}
if (eof && buffer.find('\n') == std::string::npos) {
if (!buffer.empty())
Error(nullptr, "INCOMPLETE_MESSAGE");
@@ -313,16 +256,10 @@ private:
code == "BUSY")},
{"side_effect_possible", side_effect}}}});
}
void Release() {
const bool enabled = session_.Status()["control"];
session_.SetControl(false);
if (enabled && allowed_ && !paused_ && session_.Connected())
session_.SetControl(true);
}
void FinishAction(const char *error = nullptr) {
if (!action_)
return;
Release();
session_.ReleaseInput();
json result = {
{"status", error ? (action_->submitted ? "outcome_unknown"
: "failed_before_submit")
@@ -351,12 +288,18 @@ private:
Error(encoding_->capture.id, code);
encoding_->cancelled = true;
}
if (terminal_encoding_ && !terminal_encoding_->cancelled) {
Error(terminal_encoding_->id, code);
terminal_encoding_->cancelled = true;
}
observations_.clear();
preview_ = json();
}
void Tick() {
session_.Tick();
if (auto f = session_.TakeFrame()) {
// Recycle only uniquely owned frames; an encoder/video worker may still be
// reading the old pixels. Save geometry before Take can swap its storage.
if (auto f = session_.TakeFrame(frame_ && frame_.use_count() == 1 ? frame_.get() : nullptr)) {
if (!frame_ || f->generation != frame_->generation ||
f->width != frame_->width || f->height != frame_->height) {
++epoch_;
@@ -384,15 +327,15 @@ private:
action_->submitted =
true; // A failed send may still have reached the peer.
const bool sent =
step.kind == 0
step.kind == StepKind::Pointer
? session_.Pointer(step.x, step.y, step.flag, step.wheel)
: step.kind == 1 ? session_.Key(step.key, step.down)
: step.kind == StepKind::Key ? session_.Key(step.key, step.down)
: session_.Text(step.text);
if (!sent) {
FinishAction("ACTION_OUTCOME_UNKNOWN");
break;
}
if (step.kind == 0) {
if (step.kind == StepKind::Pointer) {
cursor_x_ = step.x;
cursor_y_ = step.y;
cursor_visible_ = true;
@@ -407,7 +350,9 @@ private:
auto job = std::move(terminal_encoding_);
try {
auto grid = job->result.get();
if (!session_.Connected() || job->generation != session_.Generation() ||
if (job->cancelled) {
// Consume the worker result without replying twice.
} else if (!session_.Connected() || job->generation != session_.Generation() ||
job->epoch != epoch_)
Error(job->id, "STALE_OBSERVATION");
else if (Age(job->queued) > 2000)
@@ -425,7 +370,7 @@ private:
{"lines", terminal::Lines(grid)}, {"cursor", cursor},
{"generation", job->generation}, {"geometry_epoch", job->epoch}});
}
} catch (...) { Error(job->id, "FRAME_RENDER_FAILED"); }
} catch (...) { if (!job->cancelled) Error(job->id, "FRAME_RENDER_FAILED"); }
}
if (encoding_ &&
encoding_->result.wait_for(Ms(0)) == std::future_status::ready) {
@@ -436,8 +381,7 @@ private:
if (!frame_ || job->epoch != epoch_ ||
job->frame->generation != session_.Generation())
Error(job->capture.id, "STALE_OBSERVATION");
else if (Age(job->frame->queued) > 2000 ||
Clock::now() > job->capture.deadline)
else if (Age(job->frame->queued) > 2000)
Error(job->capture.id, "FRAME_TIMEOUT");
else {
const auto &f = *job->frame;
@@ -492,7 +436,7 @@ private:
} else
++it;
}
if (!encoding_ && !captures_.empty() && frame_ &&
if (!encoding_ && !captures_.empty() && frame_ && session_.Connected() &&
Age(frame_->queued) <= 2000) {
auto it = std::find_if(
captures_.begin(), captures_.end(), [&](const Capture &c) {
@@ -574,12 +518,24 @@ private:
}
if (captures_.size() >= 16)
throw std::invalid_argument("BUSY");
Capture c{id, j, preview,
Clock::now() + Ms(Number(j, "timeout_ms", 0, 5000, 2000))};
const int timeout = Number(j, "timeout_ms", 0, 5000, 2000);
Capture c{id, j, preview, Clock::now() + Ms(timeout)};
// timeout_ms bounds waiting for a suitable frame, not PNG encoding.
// A zero timeout can capture an already available frame immediately.
const bool available = frame_ && frame_->generation == session_.Generation() &&
Age(frame_->queued) <= 2000 &&
(!j.contains("after_frame_id") || frame_->frame_id > Sequence(j, "after_frame_id"));
if (!encoding_ && captures_.empty() && available) {
Encode(std::move(c));
return;
}
if (timeout == 0)
throw std::invalid_argument(encoding_ || !captures_.empty() ? "BUSY" : "FRAME_TIMEOUT");
if (preview)
captures_.push_back(c);
else
captures_.push_front(c);
captures_.insert(std::find_if(captures_.begin(), captures_.end(),
[](const Capture &queued) { return queued.preview; }), c);
}
const Observation &Observe(const json &j) {
auto name = String(j, "observation_id");
@@ -604,9 +560,9 @@ private:
Reply(id, r.result);
return;
}
if (!allowed_ || paused_ || !session_.Status()["control"].get<bool>())
if (!allowed_ || paused_ || !session_.ControlEnabled())
throw std::invalid_argument("CONTROL_DENIED");
if (!session_.Connected() || !frame_)
if (!session_.Connected() || !frame_ || Age(frame_->queued) > 2000)
throw std::invalid_argument("NOT_READY");
if (action_)
throw std::invalid_argument("BUSY");
@@ -618,120 +574,10 @@ private:
running->operation = operation;
running->generation = o.generation;
running->epoch = o.epoch;
auto point = [&](const json &p, int delay, int flag = 0, int wheel = 0) {
auto coordinate = [&](const char *key, int size, int offset, int span,
int source) {
if (!p.contains(key) || !p[key].is_number())
throw std::invalid_argument("INVALID_COORDINATE");
const auto value = p[key].get<double>();
if (!std::isfinite(value) || value < 0 || value >= size)
throw std::invalid_argument("INVALID_COORDINATE");
return float(std::clamp((offset + (value + .5) * span / size) / source,
0.0, 1.0));
};
Step step;
step.delay = delay;
step.flag = flag;
step.wheel = wheel;
step.x =
coordinate("x", o.width, o.region.x, o.region.width, o.source_width);
step.y = coordinate("y", o.height, o.region.y, o.region.height,
o.source_height);
running->steps.push_back(std::move(step));
};
auto key = [&](int code, bool down, int delay) {
Step step;
step.kind = 1;
step.key = code;
step.down = down;
step.delay = delay;
running->steps.push_back(std::move(step));
};
if (type == "move" || type == "click" || type == "double_click") {
Fields(a, type == "move"
? std::initializer_list<const char *>{"type", "x", "y"}
: std::initializer_list<const char *>{"type", "x", "y",
"button"});
point(a, 0);
if (type != "move") {
const auto button =
a.contains("button") ? String(a, "button", 8) : "left";
const int flag = button == "left" ? 1
: button == "right" ? 3
: button == "middle" ? 5
: -1;
if (flag < 0)
throw std::invalid_argument("INVALID_BUTTON");
point(a, 10, flag);
point(a, 40, flag + 1);
if (type == "double_click") {
point(a, 110, flag);
point(a, 140, flag + 1);
}
}
} else if (type == "scroll") {
Fields(a, {"type", "x", "y", "direction", "ticks"});
const auto direction = String(a, "direction", 8);
if (direction != "up" && direction != "down" && direction != "left" &&
direction != "right")
throw std::invalid_argument("INVALID_DIRECTION");
const int ticks = Number(a, "ticks", 1, 10, 1);
point(a, 0);
point(a, 10, direction == "left" || direction == "right" ? 8 : 7,
(direction == "up" || direction == "right" ? 120 : -120) * ticks);
} else if (type == "drag") {
Fields(a, {"type", "path", "duration_ms"});
const auto &path = a.at("path");
if (!path.is_array() || path.size() < 2 || path.size() > 128)
throw std::invalid_argument("INVALID_PATH");
const int duration = Number(a, "duration_ms", 100, 2000, 500);
for (const auto &p : path)
Fields(p, {"x", "y"});
point(path.front(), 0);
point(path.front(), 10, 1);
for (size_t i = 1; i < path.size(); ++i)
point(path[i], 10 + int(i * duration / (path.size() - 1)));
point(path.back(), duration + 30, 2);
} else if (type == "key_press") {
Fields(a, {"type", "key"});
const auto code = KeyCode(String(a, "key", 16));
key(code, true, 0);
key(code, false, 40);
} else if (type == "hotkey") {
Fields(a, {"type", "keys"});
const auto &keys = a.at("keys");
if (!keys.is_array() || keys.size() < 2 || keys.size() > 6)
throw std::invalid_argument("INVALID_KEYS");
std::set<int> seen;
std::vector<int> codes;
for (const auto &item : keys) {
if (!item.is_string())
throw std::invalid_argument("INVALID_KEY");
int code = KeyCode(item.get<std::string>());
if (!seen.insert(code).second)
throw std::invalid_argument("DUPLICATE_KEY");
codes.push_back(code);
}
int delay = 0;
for (int code : codes) {
key(code, true, delay);
delay += 15;
}
delay += 30;
for (auto it = codes.rbegin(); it != codes.rend(); ++it) {
key(*it, false, delay);
delay += 15;
}
} else if (type == "type_text") {
Fields(a, {"type", "text"});
if (!session_.Status()["text_input"].get<bool>())
throw std::invalid_argument("TEXT_INPUT_UNSUPPORTED");
Step step;
step.kind = 2;
step.text = String(a, "text", 16384);
running->steps.push_back(std::move(step));
} else
throw std::invalid_argument("UNSUPPORTED_ACTION");
if (type == "type_text" && !session_.Status()["text_input"].get<bool>())
throw std::invalid_argument("TEXT_INPUT_UNSUPPORTED");
running->steps = automation::BuildInputSteps(a,
{o.width, o.height, o.source_width, o.source_height, o.region});
while (records_.size() >= 128 ||
(!records_.empty() && Age(records_.front().created) > 300000))
records_.pop_front();
@@ -884,8 +730,12 @@ private:
throw std::invalid_argument("INVALID_REQUEST");
} else if (method == "display") {
Fields(j, {"id", "method", "index"});
const int index = Number(j, "index", 0, 63, -1);
if (!session_.Connected() || index < 0 ||
size_t(index) >= session_.Status()["displays"].size())
throw std::invalid_argument("DISPLAY_UNAVAILABLE");
Stop("SESSION_CHANGED");
if (!session_.SelectDisplay(Number(j, "index", 0, 63, -1)))
if (!session_.SelectDisplay(index))
throw std::invalid_argument("DISPLAY_UNAVAILABLE");
frame_.reset();
++epoch_;
@@ -902,6 +752,10 @@ private:
encoding_->cancelled = true;
Error(target, "CANCELLED");
}
if (terminal_encoding_ && !terminal_encoding_->cancelled && terminal_encoding_->id == target) {
terminal_encoding_->cancelled = true;
Error(target, "CANCELLED");
}
for (auto it = captures_.begin(); it != captures_.end();) {
if (it->id == target) {
Error(target, "CANCELLED");
@@ -954,7 +808,7 @@ private:
bool hello_ = false, connected_once_ = false, allowed_ = false,
paused_ = false;
uint64_t epoch_ = 0, observation_sequence_ = 0;
std::shared_ptr<const DecodedFrame> frame_;
std::shared_ptr<DecodedFrame> frame_;
std::deque<Observation> observations_;
std::deque<Capture> captures_;
std::unique_ptr<Encoding> encoding_;