YangWX/crossdesk
1
0
Fork 0
mirror of https://github.com/kunkundi/crossdesk.git synced 2026-11-05 21:52:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feat] add MCP terminal views and agent cursor overlays

Browse Source
This commit is contained in:
dijunkun
2026-11-03 03:00:57 +08:00
parent f57b4cc4e3
commit e1ae449d36
19 changed files with 576 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/cli/agent.cpp
+87
View File
@@ -10,6 +10,7 @@
#include <map>
#include <memory>
#include <stdexcept>
#include <terminal_renderer.h>
namespace crossdesk::cli {
namespace {
@@ -120,6 +121,12 @@ struct Encoding {
bool cancelled = false;
std::future<FrameImage> result;
};
struct TerminalEncoding {
json id;
uint64_t generation, epoch;
Clock::time_point queued;
std::future<terminal::Grid> result;
};
struct Step {
int delay = 0, kind = 0, key = 0, flag = 0, wheel = 0;
float x = 0, y = 0;
@@ -232,6 +239,7 @@ private:
}
result["transport_state"] = result["state"];
result["geometry_epoch"] = epoch_;
result["cursor"] = Cursor();
result["human_paused"] = paused_;
result["control_allowed"] = allowed_;
result["frame_age_ms"] = frame_ ? json(Age(frame_->queued)) : json(nullptr);
@@ -244,6 +252,16 @@ private:
: "waiting_frame";
return result;
}
json Cursor() const {
const bool visible = cursor_visible_ && frame_ && session_.Connected() &&
frame_->generation == session_.Generation() && allowed_ && !paused_ &&
session_.ControlEnabled();
return {{"type", "cursor"}, {"visible", visible},
{"x", cursor_x_}, {"y", cursor_y_},
{"generation", session_.Generation()}, {"geometry_epoch", epoch_},
{"width", frame_ ? frame_->width : 0},
{"height", frame_ ? frame_->height : 0}};
}
void PumpVideo() {
if (!video_) return;
const auto generation = session_.Generation();
@@ -255,6 +273,11 @@ private:
video_connected_ = connected;
last_video_frame_ = 0;
}
const auto cursor = Cursor();
if (cursor != video_cursor_) {
video_->Cursor(cursor);
video_cursor_ = cursor;
}
const bool enabled = video_->Enabled();
const bool key = video_->TakeKeyframeRequest();
if (enabled != video_enabled_ || key) {
@@ -337,6 +360,7 @@ private:
if (!frame_ || f->generation != frame_->generation ||
f->width != frame_->width || f->height != frame_->height) {
++epoch_;
cursor_visible_ = false;
observations_.clear();
preview_ = json();
}
@@ -345,6 +369,7 @@ private:
if (frame_ && frame_->generation != session_.Generation()) {
frame_.reset();
++epoch_;
cursor_visible_ = false;
observations_.clear();
preview_ = json();
}
@@ -367,11 +392,41 @@ private:
FinishAction("ACTION_OUTCOME_UNKNOWN");
break;
}
if (step.kind == 0) {
cursor_x_ = step.x;
cursor_y_ = step.y;
cursor_visible_ = true;
}
action_->submitted = true;
}
if (action_ && action_->next == action_->steps.size())
FinishAction();
}
if (terminal_encoding_ &&
terminal_encoding_->result.wait_for(Ms(0)) == std::future_status::ready) {
auto job = std::move(terminal_encoding_);
try {
auto grid = job->result.get();
if (!session_.Connected() || job->generation != session_.Generation() ||
job->epoch != epoch_)
Error(job->id, "STALE_OBSERVATION");
else if (Age(job->queued) > 2000)
Error(job->id, "FRAME_TIMEOUT");
else {
json cursor = nullptr;
if (Cursor()["visible"].get<bool>()) {
const int x = grid.content.x + std::clamp(
int(cursor_x_ * grid.content.width), 0, grid.content.width - 1);
const int y = grid.content.y + std::clamp(
int(cursor_y_ * grid.content.height), 0, grid.content.height - 1);
cursor = {{"column", x / grid.samples_x}, {"row", y / grid.samples_y}};
}
Reply(job->id, {{"columns", grid.viewport.columns}, {"rows", grid.viewport.rows},
{"lines", terminal::Lines(grid)}, {"cursor", cursor},
{"generation", job->generation}, {"geometry_epoch", job->epoch}});
}
} catch (...) { Error(job->id, "FRAME_RENDER_FAILED"); }
}
if (encoding_ &&
encoding_->result.wait_for(Ms(0)) == std::future_status::ready) {
auto job = std::move(encoding_);
@@ -730,6 +785,7 @@ private:
{"video_protocols", {"h264-annexb/1", "nv12/1"}},
{"video_decoders", {"auto", "software"}},
{"video_transport", "loopback-websocket/1"},
{"terminal_protocol", "ansi/1"},
{"action_completion", "submitted"},
{"max_request_bytes", 65536},
{"max_response_bytes", kMaxResponse},
@@ -775,6 +831,32 @@ private:
} else if (method == "action") {
StartAction(j, id);
return false;
} else if (method == "terminal_frame") {
Fields(j, {"id", "method", "columns", "rows", "colors", "cell_aspect"});
if (terminal_encoding_) throw std::invalid_argument("BUSY");
if (!session_.Connected() || !frame_ ||
frame_->generation != session_.Generation() || Age(frame_->queued) > 2000)
throw std::invalid_argument("NOT_READY");
terminal::Viewport viewport;
viewport.columns = Number(j, "columns", 1, 160, 80);
viewport.rows = Number(j, "rows", 1, 60, 22);
viewport.colors = Number(j, "colors", 8, 256, 24);
if (viewport.colors != 8 && viewport.colors != 24 && viewport.colors != 256)
throw std::invalid_argument("INVALID_REQUEST");
viewport.cell_aspect = j.value("cell_aspect", .5);
if (!std::isfinite(viewport.cell_aspect) || viewport.cell_aspect < .2 || viewport.cell_aspect > 2)
throw std::invalid_argument("INVALID_REQUEST");
viewport.detail = terminal::AnsiDetail::Quadrant;
auto job = std::make_unique<TerminalEncoding>();
job->id = id;
job->generation = frame_->generation;
job->epoch = epoch_;
job->queued = frame_->queued;
job->result = std::async(std::launch::async, [frame = frame_, viewport] {
return terminal::Convert(*frame, viewport);
});
terminal_encoding_ = std::move(job);
return false;
} else if (method == "control") {
Fields(j, {"id", "method", "enabled"});
const bool enable = Flag(j, "enabled");
@@ -807,6 +889,7 @@ private:
throw std::invalid_argument("DISPLAY_UNAVAILABLE");
frame_.reset();
++epoch_;
cursor_visible_ = false;
} else if (method == "cancel") {
Fields(j, {"id", "method", "request_id"});
if (!j.contains("request_id") || !j["request_id"].is_number_integer())
@@ -875,6 +958,7 @@ private:
std::deque<Observation> observations_;
std::deque<Capture> captures_;
std::unique_ptr<Encoding> encoding_;
std::unique_ptr<TerminalEncoding> terminal_encoding_;
std::unique_ptr<RunningAction> action_;
std::deque<Record> records_;
json preview_;
@@ -883,6 +967,9 @@ private:
std::unique_ptr<VideoServer> video_;
uint64_t video_generation_ = 0, last_video_frame_ = 0;
bool video_connected_ = false, video_enabled_ = false;
bool cursor_visible_ = false;
float cursor_x_ = 0, cursor_y_ = 0;
json video_cursor_;
};
} // namespace
int Agent(int output_fd, SessionOptions options, int password_fd,
apps/cli/docs/automation-protocol.md
+7
View File
@@ -35,6 +35,7 @@ The output queue is bounded to 64 messages and 8 MiB; a blocked output pipe term
| `action` | `operation_id`, `observation_id`, `action` |
| `control` | `enabled` boolean; cannot override observe-only permission or human pause |
| `viewer_control` | `operation: "pause" | "resume" | "disconnect"`; restricted to trusted UI by the MCP adapter |
| `terminal_frame` | `columns` (1–160), `rows` (1–60), `colors` (8/24/256), `cell_aspect` (0.2–2); private terminal viewer only |
| `display` | `index` (0–63) |
| `cancel` | `request_id`; cancels queued/in-flight captures or input sequences |
| `disconnect`, `exit` | none; `exit` also ends the process |
@@ -45,6 +46,12 @@ The output queue is bounded to 64 messages and 8 MiB; a blocked output pipe term
`doctor --json` advertises `automation_video_transport: "loopback-websocket/1"`, `automation_video_protocols: ["h264-annexb/1", "nv12/1"]`, and `automation_video_decoders: ["auto", "software"]`. `hello` repeats these without the `automation_` prefix. The MCP adapter requires the transport capability, so an old pipe-only CLI is rejected. Node spawns only stdin, stdout, stderr, and the credential pipe; live video never enters Node.
The terminal viewer additionally requires `automation_terminal_protocol: "ansi/1"` in `doctor` and `terminal_protocol: "ansi/1"` in `hello`. `terminal_frame` asynchronously converts the current decoded frame using the existing ANSI quadrant renderer and returns `columns`, `rows`, `lines`, `generation`, `geometry_epoch`, and a nullable `cursor` with zero-based terminal `column`/`row`. Only one render may be in flight; obsolete geometry or frames older than two seconds are rejected. These rendered text rows travel through the private authenticated terminal IPC, never through public MCP tools or model content. The viewer requests at most 8 FPS and waits for terminal writes before requesting another frame. It shares the automation connection; no second media session is opened.
Status includes `cursor` with normalized full-frame `x`/`y`, `visible`, `generation`, `geometry_epoch`, `width`, and `height`. The native layer updates it after each successfully sent pointer step, including drag steps, using the same ROI-to-source mapping as remote input. It hides on pause/control disable and clears on geometry changes. It is an Agent position indicator, not remote cursor telemetry or execution confirmation; observation PNGs and source video remain unchanged.
Authenticated video sockets also receive small JSON text messages with `type: "cursor"` and these fields, independently of binary video frames. Pointer updates are coalesced to at most 30 Hz, retained for newly authorized viewers, and deferred under socket backpressure; they require no video ACK. Browser/Apps render a separate transparent overlay with the same aspect-ratio fitting as the video, including when no new video frame arrives.
`connect.decode_path` defaults to `browser` (compressed H.264), or selects `minirtc` (decoded NV12). `connect.decoder` defaults to `auto` (hardware preferred, software when unavailable); `software` selects the native software decoder. Both paths maintain native screenshot frames with the selected decoder. These choices are fixed for the connection. Status exposes `video` and the reported `native_video_decoder` backend name. PNG encoding runs only when requested. The legacy `preview` image method remains available to private protocol callers but is not used by MCP panels.
### Authorization and transport
apps/cli/main.cpp
+1
View File
@@ -1091,6 +1091,7 @@ int main(int argc, char **argv) {
{"sixel", "probe_on_view"},
{"audio_playback", false},
{"bridge_version", 1},
{"automation_terminal_protocol", "ansi/1"},
{"automation_protocol", "crossdesk-automation/1"},
{"automation_video", "h264-annexb/1"},
{"automation_video_transport", "loopback-websocket/1"},
apps/cli/video_server.cpp
+27 -1
View File
@@ -57,6 +57,7 @@ struct VideoServer::State {
struct Client {
std::string origin, view, session;
uint64_t view_generation = 0;
uint64_t cursor_revision = 0;
bool authorized = false, waiting_key = true;
size_t bytes = 0;
Clock::time_point opened = Clock::now(), pinged = Clock::now();
@@ -93,6 +94,8 @@ struct VideoServer::State {
std::deque<Packet> queue;
size_t queued_bytes = 0;
bool scheduled = false, wait_key = true;
json cursor;
uint64_t cursor_revision = 0;
State() {
server.clear_access_channels(websocketpp::log::alevel::all);
@@ -250,10 +253,17 @@ struct VideoServer::State {
}
void Maintain() {
timer = server.set_timer(
250, [this](const websocketpp::lib::error_code &error) {
33, [this](const websocketpp::lib::error_code &error) {
if (error)
return;
const auto now = Clock::now();
std::string pointer;
uint64_t revision;
{
std::lock_guard lock(mutex);
revision = cursor_revision;
if (!cursor.is_null()) pointer = cursor.dump();
}
for (auto it = tickets.begin(); it != tickets.end();)
if (it->second.expires <= now)
it = tickets.erase(it);
@@ -261,6 +271,16 @@ struct VideoServer::State {
++it;
std::vector<Handle> close;
for (auto &[h, c] : clients) {
if (c.authorized && c.cursor_revision != revision && !pointer.empty()) {
websocketpp::lib::error_code ec;
auto connection = server.get_con_from_hdl(h, ec);
// Coalesce pointer changes while a viewer is backpressured.
if (!ec && connection->get_buffered_amount() < 256 * 1024) {
server.send(h, pointer, websocketpp::frame::opcode::TEXT, ec);
if (!ec) c.cursor_revision = revision;
}
if (ec) close.push_back(h);
}
if ((!c.authorized && now - c.opened >= 3s) ||
(!c.receipts.empty() && now - c.receipts.front().sent >= 10s)) {
close.push_back(h);
@@ -468,6 +488,12 @@ void VideoServer::Submit(EncodedVideoFrame frame) {
if (Enabled())
state_->Enqueue({{}, std::move(frame)});
}
void VideoServer::Cursor(json cursor) {
auto &s = *state_;
std::lock_guard lock(s.mutex);
s.cursor = std::move(cursor);
++s.cursor_revision;
}
json VideoServer::Status() const {
const auto &s = *state_;
std::lock_guard lock(s.mutex);
apps/cli/video_server.h
+1
View File
@@ -36,6 +36,7 @@ public:
bool TakeKeyframeRequest();
void Submit(std::shared_ptr<const DecodedFrame> frame);
void Submit(EncodedVideoFrame frame);
void Cursor(nlohmann::json cursor);
nlohmann::json Status() const;
private: