[feat] add acknowledged remote clipboard read and write

This commit is contained in:
dijunkun
2026-11-06 20:20:25 +08:00
parent 0eab020c38
commit 0aed57ed4d
13 changed files with 217 additions and 5 deletions
+85
View File
@@ -79,6 +79,13 @@ struct FrameWait {
uint64_t signature, frame;
Clock::time_point deadline, changed;
};
struct RemoteJob {
json id;
uint64_t request;
std::string operation;
bool write;
Clock::time_point deadline;
};
struct Capture {
json id, request;
bool preview = false;
@@ -298,8 +305,13 @@ private:
std::fill(step.text.begin(), step.text.end(), '\0');
action_.reset();
}
void CancelRemoteJobs(const char *code) {
for (const auto &job : remote_jobs_) FinishRemote(job, {}, code);
remote_jobs_.clear();
}
void Stop(const char *code) {
FinishAction(code);
CancelRemoteJobs(code);
for (const auto &capture : captures_)
Error(capture.id, code);
captures_.clear();
@@ -318,6 +330,15 @@ private:
}
void Tick() {
session_.Tick();
for (auto it = remote_jobs_.begin(); it != remote_jobs_.end();) {
auto result = session_.TakeRemoteResult(it->request);
const char *error = !session_.Connected() ? "SESSION_CLOSED"
: Clock::now() >= it->deadline ? "REMOTE_REQUEST_TIMEOUT" : nullptr;
if (result || error) {
FinishRemote(*it, result ? *result : json{}, error);
it = remote_jobs_.erase(it);
} else ++it;
}
// Recycle only uniquely owned frames; an encoder/video worker may still be
// reading the old pixels. Save geometry before Take can swap its storage.
if (auto f = session_.TakeFrame(frame_ && frame_.use_count() == 1 ? frame_.get() : nullptr)) {
@@ -593,6 +614,40 @@ private:
throw std::invalid_argument("STALE_OBSERVATION");
return *it;
}
void FinishRemote(const RemoteJob &job, const json &reply, const char *error = nullptr) {
session_.ForgetRemoteRequest(job.request);
if (!error && reply.value("ok", json{}) != true) error = "REMOTE_REQUEST_FAILED";
json result = error ? json{{"status", job.write ? "outcome_unknown" : "failed"},
{"error", error}, {"side_effect_possible", job.write}} : reply.value("result", json::object());
if (!job.operation.empty()) {
result["operation_id"] = job.operation;
for (auto &record : records_) if (record.operation == job.operation) record.result = result;
}
Reply(job.id, result);
}
bool PreviousRemote(const json &j, const json &id, const std::string &operation) {
for (const auto &record : records_) {
if (record.operation != operation) continue;
auto arguments = j;
arguments.erase("id");
if (record.arguments != arguments.dump()) throw std::invalid_argument("OPERATION_ID_CONFLICT");
Reply(id, record.result);
return true;
}
return false;
}
void TrackRemote(const json &j, const json &id, uint64_t request, bool write,
const std::string &operation = {}) {
if (!request) throw std::invalid_argument("NOT_READY");
if (!operation.empty()) {
auto arguments = j;
arguments.erase("id");
while (records_.size() >= 128) records_.pop_front();
records_.push_back({operation, arguments.dump(),
{{"status", "in_progress"}, {"operation_id", operation}}, Clock::now()});
}
remote_jobs_.push_back({id, request, operation, write, Clock::now() + Ms(5000)});
}
const Observation &InputObservation(const json &j) {
if (!allowed_ || paused_ || !session_.ControlEnabled())
throw std::invalid_argument("CONTROL_DENIED");
@@ -733,6 +788,27 @@ private:
} else if (method == "action") {
StartAction(j, id);
return false;
} else if (method == "clipboard") {
Fields(j, {"id", "method", "operation", "operation_id", "text"});
if (!allowed_ || paused_ || !session_.ControlEnabled())
throw std::invalid_argument("CONTROL_DENIED");
if (session_.Status()["capabilities"].value("clipboard", json{}) != true)
throw std::invalid_argument("CLIPBOARD_UNSUPPORTED");
const auto operation = String(j, "operation", 8);
if (operation != "get" && operation != "set") throw std::invalid_argument("INVALID_ARGUMENT");
const bool write = operation == "set";
const auto operation_id = write ? String(j, "operation_id") : std::string{};
if (write && PreviousRemote(j, id, operation_id)) return false;
if (remote_jobs_.size() >= 16) throw std::invalid_argument("BUSY");
std::string text;
if (write) {
if (!j.contains("text") || !j["text"].is_string()) throw std::invalid_argument("INVALID_ARGUMENT");
text = j["text"].get<std::string>();
if (text.size() > 16384 || text.find('\0') != std::string::npos)
throw std::invalid_argument("INVALID_ARGUMENT");
}
TrackRemote(j, id, session_.Clipboard(write, text), write, operation_id);
return false;
} else if (method == "wait_frame") {
Fields(j, {"id", "method", "observation_id", "condition", "stable_ms", "timeout_ms"});
const auto &observation = Observe(j);
@@ -788,6 +864,7 @@ private:
throw std::invalid_argument("CONTROL_DENIED");
if (!enable)
FinishAction("CONTROL_DISABLED");
if (!enable) CancelRemoteJobs("CONTROL_DENIED");
session_.SetControl(enable);
} else if (method == "viewer_control") {
Fields(j, {"id", "method", "operation"});
@@ -795,6 +872,7 @@ private:
if (operation == "pause") {
paused_ = true;
FinishAction("USER_PAUSED");
CancelRemoteJobs("USER_PAUSED");
session_.SetControl(false);
} else if (operation == "resume") {
if (!allowed_)
@@ -823,6 +901,12 @@ private:
if (!j.contains("request_id") || !j["request_id"].is_number_integer())
throw std::invalid_argument("INVALID_REQUEST");
const auto target = j["request_id"];
for (auto it = remote_jobs_.begin(); it != remote_jobs_.end();) {
if (it->id == target) {
FinishRemote(*it, {}, "CANCELLED");
it = remote_jobs_.erase(it);
} else ++it;
}
for (auto it = frame_waits_.begin(); it != frame_waits_.end();) {
if (it->id == target) {
Error(it->id, "CANCELLED");
@@ -896,6 +980,7 @@ private:
std::deque<Observation> observations_;
std::deque<Capture> captures_;
std::deque<FrameWait> frame_waits_;
std::deque<RemoteJob> remote_jobs_;
std::unique_ptr<Encoding> encoding_;
std::unique_ptr<TerminalEncoding> terminal_encoding_;
std::unique_ptr<RunningAction> action_;
+4
View File
@@ -141,3 +141,7 @@ The result follows the existing action result format. Repeated operation IDs ret
Updated desktop hosts answer the versioned `automation` capability handshake. When `capabilities.input_ack` is true, input travels through the host input dispatcher and the action result includes `remote_execution` with `applied`, `failed` or `pending`. A receipt confirms native input processing. The caller checks the target application's resulting state separately. Legacy hosts return `remote_execution` as `unsupported`. Missing receipts terminate the bounded wait with `REMOTE_ACK_TIMEOUT` and an uncertain outcome.
`wait_frame` accepts an `observation_id`, `condition` (`changed` or `stable`), `stable_ms` (100–5000, default 500), and `timeout_ms` (0–25000, default 5000). It watches the observation's source region, checks both luma and chroma, and returns `matched`, `wait_timed_out`, `generation`, and `frame_id`. Display changes, disconnection and explicit cancellation terminate pending waits. `crossdesk_wait` exposes this operation through MCP and includes a fresh screenshot when its condition matches.
## Clipboard
`clipboard` accepts `operation: get | set`. Writes additionally require `operation_id` and `text` (up to 16384 UTF-8 bytes, including an empty string to clear). Reads return up to 128 KiB of text. The existing chunked `web-clipboard` stream carries values and applied/rejected replies. Additive `read`, `empty`, and `clear` frames are enabled by the remote `clipboard` capability. Clipboard access requires control permission and respects human pause. Cancellation of an issued write reports an uncertain effect. Repeated write operation IDs retrieve the cached result.
@@ -59,6 +59,7 @@ void ClipboardController::Shutdown() {
pending_remote_text_.reset();
pending_complete_ = {};
pending_allowed_ = {};
pending_reads_.clear();
}
bool ClipboardController::WebSessionActive(
@@ -107,7 +108,7 @@ void ClipboardController::ReceiveWeb(const char* data, size_t size,
if (found == web_sessions_.end() || found->second->events.lock() != events) return;
session = found->second;
}
if (!hello) {
if (!hello && kind != clip::kRead && kind != clip::kClear) {
if (!session->ready.load(std::memory_order_acquire) ||
kind != clip::kText || !request) return;
message = session->assembler.Receive(std::string_view(data, size));
@@ -121,6 +122,23 @@ void ClipboardController::ReceiveWeb(const char* data, size_t size,
session->ready.store(true, std::memory_order_release);
return;
}
if ((kind == clip::kRead || kind == clip::kClear) &&
(!session->ready.load(std::memory_order_acquire) || !request ||
size != clip::kHeaderBytes)) return;
if (kind == clip::kRead) {
const auto send = [this, remote_id, session](const std::string& frame) {
return WebSessionActive(remote_id, session) &&
owner_.SendControllerData(remote_id, frame.data(), frame.size(), clip::kStream) == 0;
};
if (!QueueRead([send, request](std::optional<std::string> text) {
if (!text) send(clip::Frame(clip::kRejected, request));
else if (text->empty()) send(clip::Frame(clip::kEmpty, request));
else if (!clip::SendText(*text, request, 16 * 1024, send))
send(clip::Frame(clip::kRejected, request));
})) send(clip::Frame(clip::kRejected, request));
return;
}
if (kind == clip::kClear) message = clip::Frame(clip::kText, request);
if (message.empty()) return;
const std::weak_ptr<WebSession> weak = session;
auto allowed = [this, remote_id, weak] {
@@ -163,7 +181,7 @@ void ClipboardController::QueueRemoteText(const char* data, size_t size,
if (!events_enabled_.load(std::memory_order_acquire)) {
return;
}
if (!data || size == 0) {
if (!data || (size == 0 && !complete)) {
return;
}
if (size > kMaxClipboardTextBytes) {
@@ -207,11 +225,37 @@ void ClipboardController::QueueRemoteText(const char* data, size_t size,
}
}
bool ClipboardController::QueueRead(std::function<void(std::optional<std::string>)> complete) {
{
std::lock_guard lock(pending_mutex_);
if (!events_enabled_.load() || pending_reads_.size() >= 32) return false;
pending_reads_.push_back(std::move(complete));
}
SDL_Event event{};
event.type = event_type_;
if (event_type_) SDL_PushEvent(&event);
return true;
}
void ClipboardController::ApplyPendingRemoteText() {
if (!events_enabled_.load(std::memory_order_acquire)) {
return;
}
ApplyPendingWrite();
std::deque<std::function<void(std::optional<std::string>)>> reads;
{
std::lock_guard lock(pending_mutex_);
reads.swap(pending_reads_);
}
if (!reads.empty()) {
char* value = SDL_GetClipboardText();
const std::optional<std::string> text = value ? std::optional<std::string>{value} : std::nullopt;
SDL_free(value);
for (auto& complete : reads) complete(text);
}
}
void ClipboardController::ApplyPendingWrite() {
std::optional<std::string> pending_text;
std::string source_id;
std::weak_ptr<gui_detail::RemoteSession> source_session;
@@ -11,6 +11,7 @@
#include <cstddef>
#include <cstdint>
#include <functional>
#include <deque>
#include <memory>
#include <mutex>
#include <optional>
@@ -42,6 +43,7 @@ public:
std::function<void(bool)> complete = {},
std::function<bool()> allowed = {});
void ApplyPendingRemoteText();
bool QueueRead(std::function<void(std::optional<std::string>)> complete);
void HandleLocalUpdate();
void ReceiveWeb(const char* data, size_t size, const std::string& remote_id,
PeerEventHandler* source);
@@ -49,6 +51,7 @@ public:
private:
struct WebSession;
void ApplyPendingWrite();
bool WebSessionActive(const std::string& remote_id,
const std::shared_ptr<WebSession>& session);
int SendToController(const std::string& remote_id, const std::string& text);
@@ -59,6 +62,7 @@ private:
uint32_t event_type_ = 0;
std::atomic<bool> events_enabled_{false};
std::mutex pending_mutex_;
std::deque<std::function<void(std::optional<std::string>)>> pending_reads_;
std::optional<std::string> pending_remote_text_;
std::string pending_remote_id_;
std::weak_ptr<gui_detail::RemoteSession> pending_session_;
@@ -21,7 +21,7 @@ void GuiRuntime::HandleAutomation(const std::string& remote_id,
try {
const auto method = request.value("method", std::string{});
if (method == "capabilities") {
reply(true, {{"input_ack", true}});
reply(true, {{"input_ack", true}, {"clipboard", true}});
} else if (method == "input") {
RemoteAction action{};
if (!request.contains("arguments") ||
+4
View File
@@ -250,3 +250,7 @@ npm run pack:npm -- --out-dir /absolute/path/artifacts
更新后的被控端通过 `capabilities.input_ack` 声明输入处理回执。动作结果的 `remote_execution` 表示远端原生输入处理状态,应用界面的实际结果由后续观察确认。旧被控端返回 `unsupported`。回执等待有明确期限,超时动作保留结果不确定状态。
`crossdesk_wait` 使用已有截图的 `observation_id`,等待对应区域变化(`changed`)或持续稳定(`stable`)。可设置 `stable_ms` 和最长 25000 毫秒的 `timeout_ms`。满足条件后返回新截图,切换显示器、断开和取消会结束等待。
### 远端剪贴板
`crossdesk_clipboard_get` 读取远端文本剪贴板,`crossdesk_clipboard_set` 设置文本,空字符串清空。读取上限为 128 KiB,单次工具写入上限为 16 KiB UTF-8。写入需要 `operation_id`,相同请求查询原结果。工具使用已授权控制会话,并遵守人工暂停及解锁等待状态。更新后的被控端通过 `capabilities.clipboard` 声明支持,写入成功由系统剪贴板更新回执确认。
+4 -2
View File
@@ -173,6 +173,8 @@ export class NativeClient {
if (this.pending.size >= 32 || this.child.stdin!.writableLength > 262144)
return Promise.reject(new Failure("BUSY", false, true));
const id = ++this.sequence;
const sideEffect = method === "action" || method === "unlock" ||
(method === "clipboard" && args.operation === "set");
const line = JSON.stringify({ ...args, id, method }) + "\n";
if (Buffer.byteLength(line) > 65536)
return Promise.reject(new Failure("MESSAGE_TOO_LARGE"));
@@ -190,7 +192,7 @@ export class NativeClient {
request_id: id,
}) + "\n",
);
reject(new Failure(code, method === "action" || method === "unlock"));
reject(new Failure(code, sideEffect));
};
const abort = () => cancel("CANCELLED");
const timer = setTimeout(() => cancel("NATIVE_TIMEOUT"), timeout);
@@ -198,7 +200,7 @@ export class NativeClient {
this.pending.set(id, {
resolve,
reject,
action: method === "action" || method === "unlock",
action: sideEffect,
clean: () => {
clearTimeout(timer);
signal?.removeEventListener("abort", abort);
+8
View File
@@ -312,6 +312,14 @@ export function createServer(
"Enable or disable control within the user-authorized scope. Cannot override human pause or upgrade an observe-only connection.",
async (p) => output(await sessions.control(p.session_id, p.enabled)),
);
add("crossdesk_clipboard_get", z.strictObject({ session_id: id }),
"Read the remote text clipboard within the authorized control session.",
async (p, ctx) => output(await sessions.clipboard({ ...p, operation: "get" }, ctx.mcpReq.signal)),
{ readOnly: true });
add("crossdesk_clipboard_set", z.strictObject({ session_id: id, operation_id: id,
text: z.string().max(16384).refine((text) => Buffer.byteLength(text) <= 16384 && !text.includes("\0")) }),
"Set or clear the remote text clipboard. Reuse the same operation_id to query the original result. Returns applied after the remote clipboard update.",
async (p, ctx) => output(await sessions.clipboard({ ...p, operation: "set" }, ctx.mcpReq.signal)));
add(
"crossdesk_select_display",
z.strictObject({ session_id: id, display_index: z.int().min(0).max(63) }),
+7
View File
@@ -466,6 +466,13 @@ export class Sessions {
if (result.matched) result.observation = await this.screenshot({ session_id }, signal);
return result;
}
async clipboard(p: Data, signal?: AbortSignal): Promise<Data> {
const { session_id, ...args } = p;
const s = this.session(session_id);
this.checkUnlock(s);
if (s.unlock && this.unlockPending(s.unlock)) throw new Failure("UNLOCK_PENDING");
return s.native.request("clipboard", args, 7000, signal);
}
async control(id: string, enabled: boolean) {
if (!enabled) this.cancelUnlock(this.session(id));
return this.session(id).native.request("control", { enabled });
@@ -15,6 +15,7 @@
#include <nlohmann/json.hpp>
#include <optional>
#include <remote_action.h>
#include <web_clipboard_protocol.h>
#include <set>
#include <string>
#include <vector>
@@ -77,6 +78,8 @@ public:
bool Pointer(float x, float y, int flag, int wheel = 0);
uint64_t RequestRemote(const std::string &method, nlohmann::json arguments = {});
std::optional<nlohmann::json> TakeRemoteResult(uint64_t request);
void ForgetRemoteRequest(uint64_t request) { remote_results_.erase(request); }
uint64_t Clipboard(bool write, const std::string &text = {});
void BeginInput();
std::string InputResult();
bool InputAcknowledged() const { return remote_capabilities_.value("input_ack", nlohmann::json{}) == true; }
@@ -121,6 +124,8 @@ private:
std::vector<uint64_t> input_requests_;
std::string input_result_ = "applied";
bool input_ack_active_ = false;
bool clipboard_ready_ = false;
web_clipboard::Assembler clipboard_assembler_;
uint64_t generation_ = 0;
uint32_t preview_request_ = 0, preview_ack_ = 0;
bool control_ = false, preview_supported_ = false, resolution_ = false;
@@ -54,3 +54,19 @@ std::string ControllerSession::InputResult() {
return input_requests_.empty() ? input_result_ : "pending";
}
} // namespace crossdesk
namespace crossdesk {
uint64_t ControllerSession::Clipboard(bool write, const std::string &text) {
namespace clip = web_clipboard;
if (!Connected() || !control_ || !clipboard_ready_ || remote_results_.size() >= 512 ||
remote_sequence_ >= UINT32_MAX || (write && !text.empty() && !clip::ValidText(text)))
return 0;
const auto id = uint32_t(++remote_sequence_);
remote_results_[id] = nullptr;
const auto send = [this](const std::string &frame) { return Send(clip::kStream, frame); };
const bool sent = write && !text.empty() ? clip::SendText(text, id, 16 * 1024, send)
: send(clip::Frame(write ? clip::kClear : clip::kRead, id));
if (!sent) { remote_results_.erase(id); return 0; }
return id;
}
} // namespace crossdesk
@@ -3,6 +3,7 @@
#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstring>
#include <display_stream_id.h>
#include <log.h>
#include <preview_subscription.h>
@@ -42,6 +43,12 @@ void ControllerSession::Data(const char *data, size_t n, const char *, size_t,
const char *stream, size_t sn, void *p) {
auto &session = *static_cast<ControllerSession *>(p);
try {
if (data && n && n <= web_clipboard::kHeaderBytes + web_clipboard::kChunkBytes &&
stream && sn == std::strlen(web_clipboard::kStream) &&
std::string_view(stream, sn) == web_clipboard::kStream) {
session.Enqueue({3, 0, std::string(data, n)});
return;
}
if (data && n && n <= 65536 && stream && sn < 64 &&
(std::string(stream, sn) == kControlStream ||
std::string(stream, sn) == kWebControlStream))
@@ -122,6 +129,8 @@ void ControllerSession::Connect(SessionOptions options, std::string device,
remote_results_.clear();
input_requests_.clear();
capability_request_ = 0;
clipboard_ready_ = false;
clipboard_assembler_ = {};
displays_.clear();
host_platform_ = HostPlatform::Unknown;
preview_supported_ = resolution_ = fps15_ = bandwidth_ = text_supported_ =
@@ -155,6 +164,7 @@ void ControllerSession::Connect(SessionOptions options, std::string device,
AddDataStream(peer_, kDataStream, false);
for (auto stream : {kMouseStream, kKeyboardStream, kControlStream})
AddDataStream(peer_, stream, true);
AddDataStream(peer_, web_clipboard::kStream, true);
if (Init(peer_) != 0)
Fail("media_initialization_failed");
}
@@ -240,6 +250,24 @@ void ControllerSession::Tick() {
Fail(Connected() ? "connection_lost" : "connection_failed");
else if (e.code == Closed || e.code == Disconnected)
Fail("remote_disconnected");
} else if (e.kind == 3 && Connected()) {
namespace clip = web_clipboard;
if (e.data.size() < clip::kHeaderBytes) continue;
const auto kind = clip::Read32(e.data.data());
const auto id = clip::Read32(e.data.data() + 4);
if (kind == clip::kReady && !id) { clipboard_ready_ = true; continue; }
const auto found = remote_results_.find(id);
if (!id || found == remote_results_.end()) continue;
if (kind == clip::kApplied || kind == clip::kRejected) {
found->second = {{"ok", kind == clip::kApplied},
{"result", {{"applied", kind == clip::kApplied}}}, {"error", "CLIPBOARD_REJECTED"}};
} else if (kind == clip::kEmpty) {
found->second = {{"ok", true}, {"result", {{"text", ""}}}};
} else if (kind == clip::kText) {
const auto text = clipboard_assembler_.Receive(e.data);
if (!text.empty()) found->second = {{"ok", true},
{"result", {{"text", text.substr(clip::kHeaderBytes)}}}};
}
} else if (e.kind == 2 && Connected()) {
auto j = ParseBoundedJson(e.data);
if (!j.is_object())
@@ -254,6 +282,9 @@ void ControllerSession::Tick() {
if (id == capability_request_) {
if (j.value("ok", json{}) == true && j.value("result", json{}).is_object())
remote_capabilities_ = j["result"];
if (remote_capabilities_.value("clipboard", json{}) == true)
Send(web_clipboard::kStream, web_clipboard::Frame(
web_clipboard::kHello, 0, web_clipboard::kVersion));
remote_results_.erase(found);
}
}
@@ -475,6 +506,7 @@ json ControllerSession::Status() const {
{"display", options_.display},
{"text_input", text_supported_},
{"capabilities", remote_capabilities_},
{"clipboard_ready", clipboard_ready_},
{"preview_supported", preview_supported_},
{"preview_request", preview_request_},
{"preview_ack", preview_ack_},
@@ -23,6 +23,7 @@ inline constexpr size_t kHeaderBytes = 16;
inline constexpr size_t kChunkBytes = 16 * 1024;
inline constexpr uint32_t kText = 1, kApplied = 2, kRejected = 3;
inline constexpr uint32_t kHello = 4, kReady = 5;
inline constexpr uint32_t kRead = 6, kEmpty = 7, kClear = 8;
inline uint32_t Read32(const char* data) {
const auto* p = reinterpret_cast<const unsigned char*>(data);