[feat] add native desktop inspection and accessibility actions

apps/cli/agent.cpp

@@ -671,6 +671,11 @@ private:
     if (!error && reply.value("ok", json{}) != true) error = "REMOTE_REQUEST_FAILED";
     json result = error ? json{{"status", job.write ? "outcome_unknown" : "failed"},
         {"error", error}, {"side_effect_possible", job.write}} : reply.value("result", json::object());
+    if (error && reply.contains("error") && reply["error"].is_string()) {
+      const auto reason = reply["error"].get<std::string>();
+      if (!reason.empty() && reason.size() <= 64 && reason.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ_") == std::string::npos)
+        result["remote_error"] = reason;
+    }
     if (!job.operation.empty()) {
       result["operation_id"] = job.operation;
       for (auto &record : records_) if (record.operation == job.operation) record.result = result;
@@ -867,6 +872,25 @@ private:
         if (human_control_) throw std::invalid_argument("HUMAN_CONTROL_ACTIVE");
         FinishAction("CANCELLED");
         session_.ReleaseInput();
+      } else if (method == "desktop") {
+        Fields(j, {"id", "method", "operation", "operation_id", "pid", "max_nodes", "max_depth",
+                   "snapshot_id", "element_id", "application", "url", "value"});
+        const auto command = String(j, "operation", 16);
+        const bool write = command != "capabilities" && command != "windows" && command != "tree";
+        if (write && command != "activate" && command != "press" && command != "focus" &&
+            command != "set_value" && command != "launch" && command != "open_url")
+          throw std::invalid_argument("INVALID_ARGUMENT");
+        const auto operation = write ? String(j, "operation_id") : std::string{};
+        if (write && PreviousRemote(j, id, operation)) return false;
+        if (write && (!allowed_ || paused_ || !session_.ControlEnabled())) throw std::invalid_argument("CONTROL_DENIED");
+        if (write && (action_ || session_.HasPressedInput())) throw std::invalid_argument("INPUT_HELD");
+        if (session_.Status()["capabilities"].value("desktop", json{}) != true)
+          throw std::invalid_argument("UI_AUTOMATION_UNSUPPORTED");
+        if (remote_jobs_.size() >= 16) throw std::invalid_argument("BUSY");
+        auto arguments = j;
+        arguments.erase("id"); arguments.erase("method"); arguments.erase("operation_id");
+        TrackRemote(j, id, session_.RequestRemote("desktop", arguments), write, operation);
+        return false;
       } else if (method == "system") {
         Fields(j, {"id", "method", "command", "operation_id"});
         const auto command = String(j, "command", 16);

apps/desktop/src/common/desktop_automation.cpp

@@ -0,0 +1,87 @@
+#include "desktop_automation.h"
+
+#include <algorithm>
+#include <stdexcept>
+
+namespace crossdesk {
+using nlohmann::json;
+std::string DesktopText(std::string text, size_t limit) {
+  if (text.size() > limit) {
+    while (limit && (static_cast<unsigned char>(text[limit]) & 0xc0) == 0x80) --limit;
+    text.resize(limit);
+  }
+  for (auto& character : text) if (static_cast<unsigned char>(character) < 32 && character != '\n') character = ' ';
+  return json::parse(json(text).dump(-1, ' ', false, json::error_handler_t::replace)).get<std::string>();
+}
+json DesktopAutomation::Execute(const std::string& owner, const json& request) {
+  const auto now = std::chrono::steady_clock::now();
+  while (!snapshots_.empty() && snapshots_.front().expires <= now) snapshots_.pop_front();
+  const auto operation = request.at("operation").get<std::string>();
+  if (operation == "capabilities") return DesktopCapabilities();
+  if (operation == "launch" || operation == "open_url") {
+    const auto value = request.at(operation == "launch" ? "application" : "url").get<std::string>();
+    if (value.empty() || value.size() > 4096 || value.find_first_of("\r\n") != std::string::npos ||
+        value.find('\0') != std::string::npos) throw std::invalid_argument("INVALID_ARGUMENT");
+    if (operation == "launch") return DesktopLaunch(value);
+    if (value.rfind("https://", 0) != 0 && value.rfind("http://", 0) != 0)
+      throw std::invalid_argument("INVALID_URL");
+    return DesktopOpenUrl(value);
+  }
+  DesktopItem item;
+  auto found = snapshots_.end();
+  if (request.contains("snapshot_id")) {
+    const auto id = request.at("snapshot_id").get<std::string>();
+    found = std::find_if(snapshots_.begin(), snapshots_.end(), [&](const auto& snapshot) {
+      return snapshot.id == id && snapshot.owner == owner;
+    });
+    if (found == snapshots_.end()) throw std::invalid_argument("STALE_UI_SNAPSHOT");
+    const int index = request.at("element_id").get<int>();
+    if (index < 0 || size_t(index) >= found->content.items.size()) throw std::invalid_argument("INVALID_ELEMENT");
+    item = found->content.items[size_t(index)];
+  }
+  if (operation == "windows" || operation == "tree") {
+    const int limit = request.value("max_nodes", 128), depth = request.value("max_depth", 8);
+    const int pid = request.value("pid", 0);
+    if (limit < 1 || limit > 128 || depth < 0 || depth > 16 || pid < 0)
+      throw std::invalid_argument("INVALID_ARGUMENT");
+    DesktopSnapshot content;
+    if (operation == "windows") content = DesktopWindows();
+    else if (found != snapshots_.end()) {
+      if (!item.inspect) throw std::invalid_argument("UNSUPPORTED_OPERATION");
+      content = item.inspect(limit, depth);
+    } else content = DesktopTree(pid, limit, depth);
+    json elements = json::array();
+    size_t bytes = 0;
+    for (size_t index = 0; index < content.items.size(); ++index) {
+      auto data = content.items[index].data;
+      data["element_id"] = index;
+      bytes += data.dump(-1, ' ', false, json::error_handler_t::replace).size();
+      if (bytes > 48000 || index >= 128) { content.truncated = true; content.items.resize(index); break; }
+      elements.push_back(std::move(data));
+    }
+    const auto id = "ui_" + std::to_string(++sequence_);
+    if (snapshots_.size() >= 8) snapshots_.pop_front();
+    const bool truncated = content.truncated;
+    snapshots_.push_back({id, owner, now + std::chrono::seconds(30), std::move(content)});
+    return {{"snapshot_id", id}, {"expires_in_ms", 30000}, {"elements", elements},
+      {"truncated", truncated}, {"coordinate_space", "desktop_native"}};
+  }
+  if (operation != "activate" && operation != "press" && operation != "focus" && operation != "set_value")
+    throw std::invalid_argument("UNSUPPORTED_OPERATION");
+  if (found == snapshots_.end() || !item.perform) throw std::invalid_argument("INVALID_ELEMENT");
+  const auto value = request.value("value", std::string{});
+  if (value.size() > 16384 || value.find('\0') != std::string::npos) throw std::invalid_argument("INVALID_ARGUMENT");
+  if (item.data.value("secure", false)) throw std::invalid_argument("SECURE_ELEMENT");
+  // The next operation must inspect the resulting UI again, even if an OS
+  // provider returns an uncertain result after accepting a mutation.
+  snapshots_.erase(found);
+  return item.perform(operation, value);
+}
+#if defined(__OHOS__)
+json DesktopCapabilities() { return {{"available", false}}; }
+DesktopSnapshot DesktopWindows() { throw std::runtime_error("UI_AUTOMATION_UNSUPPORTED"); }
+DesktopSnapshot DesktopTree(int, int, int) { throw std::runtime_error("UI_AUTOMATION_UNSUPPORTED"); }
+json DesktopLaunch(const std::string&) { throw std::runtime_error("UI_AUTOMATION_UNSUPPORTED"); }
+json DesktopOpenUrl(const std::string&) { throw std::runtime_error("UI_AUTOMATION_UNSUPPORTED"); }
+#endif
+} // namespace crossdesk

apps/desktop/src/common/desktop_automation.h

@@ -0,0 +1,50 @@
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2026-11-06
+ * Copyright (c) 2026 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _DESKTOP_AUTOMATION_H_
+#define _DESKTOP_AUTOMATION_H_
+
+#include <chrono>
+#include <deque>
+#include <functional>
+#include <nlohmann/json.hpp>
+#include <string>
+#include <vector>
+
+namespace crossdesk {
+struct DesktopSnapshot;
+struct DesktopItem {
+  nlohmann::json data;
+  std::function<nlohmann::json(const std::string&, const std::string&)> perform;
+  std::function<DesktopSnapshot(int, int)> inspect;
+};
+struct DesktopSnapshot {
+  std::vector<DesktopItem> items;
+  bool truncated = false;
+};
+// Platform functions execute on the desktop UI thread. References remain owned
+// by a bounded, connection-scoped snapshot until expiry or a mutating action.
+nlohmann::json DesktopCapabilities();
+DesktopSnapshot DesktopWindows();
+DesktopSnapshot DesktopTree(int pid, int limit, int depth);
+nlohmann::json DesktopLaunch(const std::string& application);
+nlohmann::json DesktopOpenUrl(const std::string& url);
+std::string DesktopText(std::string text, size_t limit = 256);
+class DesktopAutomation {
+ public:
+  nlohmann::json Execute(const std::string& owner, const nlohmann::json& request);
+ private:
+  struct Snapshot {
+    std::string id, owner;
+    std::chrono::steady_clock::time_point expires;
+    DesktopSnapshot content;
+  };
+  std::deque<Snapshot> snapshots_;
+  uint64_t sequence_ = 0;
+};
+} // namespace crossdesk
+
+#endif

apps/desktop/src/gui/runtime/automation_runtime.cpp

@@ -25,6 +25,7 @@ void GuiRuntime::HandleAutomation(const std::string& remote_id,
       json capabilities{{"input_ack", true}, {"clipboard", true}, {"files", true}};
 #if defined(_WIN32) || defined(__APPLE__) || (defined(__linux__) && !defined(__OHOS__))
       capabilities["system"] = true;
+      capabilities["desktop"] = true;
 #endif
       reply(true, std::move(capabilities));
     } else if (method == "input") {
@@ -42,6 +43,22 @@ void GuiRuntime::HandleAutomation(const std::string& remote_id,
       if (result == RemoteInputDispatcher::Result::rejected ||
           result == RemoteInputDispatcher::Result::overflow)
         reply(false, "INPUT_REJECTED");
+    } else if (method == "desktop") {
+      const auto arguments = request.at("arguments");
+      if (!arguments.is_object()) { reply(false, "INVALID_ARGUMENT"); return; }
+      std::lock_guard lock(automation_mutex_);
+      if (automation_closing_ || automation_tasks_.size() >= 32) { reply(false, "BUSY"); return; }
+      const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2);
+      automation_tasks_.push_back([this, source, remote_id, arguments, reply, deadline] {
+        if (std::chrono::steady_clock::now() > deadline) { reply(false, "REQUEST_EXPIRED"); return; }
+        if (ControllerEvents(remote_id) != source || !IsAuthorizedController(remote_id)) return;
+        try { reply(true, desktop_automation_.Execute(remote_id, arguments)); }
+        catch (const std::exception& error) {
+          const std::string code = error.what();
+          reply(false, !code.empty() && code.size() <= 64 && code.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ_") == std::string::npos
+              ? code : "UI_AUTOMATION_FAILED");
+        }
+      });
     } else if (method == "system") {
       const auto operation = request.at("arguments").at("operation").get<std::string>();
       if (operation != "status" && operation != "lock" && operation != "send_sas") {

apps/desktop/src/gui/runtime/gui_runtime.h

@@ -13,6 +13,7 @@
 #include <memory>
 #include <string>
 
+#include "desktop_automation.h"
 #include "features/clipboard/clipboard_controller.h"
 #include "features/devices/session_device_manager.h"
 #include "features/file_transfer/file_transfer_manager.h"
@@ -154,6 +155,7 @@ class GuiRuntime : protected gui_detail::GuiState {
   void SendRemoteInput(const RemoteAction& action);
   InputOutcome ApplyRemoteInput(const std::string& remote_id, const RemoteAction& action);
   void HandleAutomation(const std::string& remote_id, const nlohmann::json& request);
+  DesktopAutomation desktop_automation_;
   std::mutex automation_mutex_;
   std::deque<std::function<void()>> automation_tasks_;
   bool automation_closing_ = false;

apps/desktop/src/platform/linux/desktop_automation.cpp

@@ -0,0 +1,273 @@
+#include "desktop_automation.h"
+
+#include <xcb/xcb.h>
+#include <cstring>
+#include <algorithm>
+#include <cstdlib>
+#include <cerrno>
+#include <deque>
+#include <condition_variable>
+#include <mutex>
+#include <memory>
+#include <spawn.h>
+#include <stdexcept>
+#include <sys/wait.h>
+#include <thread>
+#include <unistd.h>
+#ifdef CROSSDESK_HAS_ATSPI
+#include <atspi/atspi.h>
+#endif
+extern char** environ;
+
+namespace crossdesk {
+namespace {
+using nlohmann::json;
+class LaunchedProcesses {
+ public:
+  LaunchedProcesses() {
+    children_.reserve(128);
+    worker_ = std::thread([this] {
+      std::unique_lock lock(mutex_);
+      while (!stopping_) {
+        wake_.wait_for(lock, std::chrono::milliseconds(500));
+        children_.erase(std::remove_if(children_.begin(), children_.end(), [](pid_t child) {
+          const auto result = waitpid(child, nullptr, WNOHANG);
+          return result > 0 || (result < 0 && errno == ECHILD);
+        }), children_.end());
+      }
+    });
+  }
+  ~LaunchedProcesses() {
+    { std::lock_guard lock(mutex_); stopping_ = true; }
+    wake_.notify_all();
+    worker_.join();
+  }
+  pid_t Spawn(const std::string& executable, const std::string& argument) {
+    std::lock_guard lock(mutex_);
+    if (children_.size() >= 128) throw std::runtime_error("BUSY");
+    char* args[] = {const_cast<char*>(executable.c_str()), argument.empty() ? nullptr : const_cast<char*>(argument.c_str()), nullptr};
+    pid_t pid = 0;
+    if (posix_spawn(&pid, executable.c_str(), nullptr, nullptr, args, environ) != 0)
+      throw std::runtime_error("APPLICATION_UNAVAILABLE");
+    children_.push_back(pid);
+    return pid;
+  }
+ private:
+  std::mutex mutex_;
+  std::condition_variable wake_;
+  bool stopping_ = false;
+  std::vector<pid_t> children_;
+  std::thread worker_;
+};
+json Spawn(const std::string& executable, const std::string& argument = {}) {
+  static LaunchedProcesses processes;
+  return {{"accepted", true}, {"pid", processes.Spawn(executable, argument)}};
+}
+struct XConnection {
+  std::shared_ptr<xcb_connection_t> connection;
+  xcb_window_t root = 0;
+  XConnection() {
+    int screen_index = 0;
+    connection = {xcb_connect(nullptr, &screen_index), xcb_disconnect};
+    if (!connection || xcb_connection_has_error(connection.get())) throw std::runtime_error("X11_UNAVAILABLE");
+    auto screen = xcb_setup_roots_iterator(xcb_get_setup(connection.get()));
+    for (int index = 0; index < screen_index && screen.rem; ++index) xcb_screen_next(&screen);
+    if (!screen.rem) throw std::runtime_error("X11_UNAVAILABLE");
+    root = screen.data->root;
+  }
+  xcb_atom_t Atom(const char* name) const {
+    auto reply = xcb_intern_atom_reply(connection.get(), xcb_intern_atom(connection.get(), 0, uint16_t(std::strlen(name)), name), nullptr);
+    const auto atom = reply ? reply->atom : XCB_ATOM_NONE;
+    std::free(reply);
+    return atom;
+  }
+  std::vector<uint32_t> Property(xcb_window_t window, const char* name, xcb_atom_t type) const {
+    auto reply = xcb_get_property_reply(connection.get(),
+        xcb_get_property(connection.get(), 0, window, Atom(name), type, 0, 256), nullptr);
+    std::vector<uint32_t> result;
+    if (reply && reply->type == type && reply->format == 32) {
+      const auto data = static_cast<const uint32_t*>(xcb_get_property_value(reply));
+      result.assign(data, data + xcb_get_property_value_length(reply) / 4);
+    }
+    std::free(reply);
+    return result;
+  }
+  std::string Title(xcb_window_t window) const {
+    auto reply = xcb_get_property_reply(connection.get(),
+        xcb_get_property(connection.get(), 0, window, Atom("_NET_WM_NAME"), Atom("UTF8_STRING"), 0, 256), nullptr);
+    std::string text;
+    if (reply && reply->format == 8) text.assign(static_cast<const char*>(xcb_get_property_value(reply)), xcb_get_property_value_length(reply));
+    std::free(reply);
+    return DesktopText(std::move(text));
+  }
+};
+#ifdef CROSSDESK_HAS_ATSPI
+template <class T> std::shared_ptr<T> Own(T* object) { return {object, [](T* value) { if (value) g_object_unref(value); }}; }
+using Accessible = std::shared_ptr<AtspiAccessible>;
+std::string Text(char* value) {
+  auto text = DesktopText(value ? value : "");
+  g_free(value);
+  return text;
+}
+Accessible Desktop() {
+  static const int initialized = atspi_init();
+  if (initialized > 1) throw std::runtime_error("ACCESSIBILITY_UNAVAILABLE");
+  atspi_set_timeout(120, 120);
+  for (int count = 0; count < 32 && g_main_context_pending(nullptr); ++count) g_main_context_iteration(nullptr, false);
+  auto desktop = Own(atspi_get_desktop(0));
+  if (!desktop) throw std::runtime_error("ACCESSIBILITY_UNAVAILABLE");
+  return desktop;
+}
+json Perform(Accessible element, const std::string& operation, const std::string& value) {
+  atspi_accessible_clear_cache(element.get());
+  auto state = Own(atspi_accessible_get_state_set(element.get()));
+  if (!state || atspi_state_set_contains(state.get(), ATSPI_STATE_DEFUNCT)) throw std::runtime_error("STALE_ELEMENT");
+  const auto role = atspi_accessible_get_role(element.get(), nullptr);
+  if (role == ATSPI_ROLE_INVALID) throw std::runtime_error("STALE_ELEMENT");
+  if (role == ATSPI_ROLE_PASSWORD_TEXT) throw std::runtime_error("SECURE_ELEMENT");
+  bool accepted = false;
+  if (operation == "focus" || operation == "activate") {
+    auto component = Own(atspi_accessible_get_component_iface(element.get()));
+    if (component) accepted = atspi_component_grab_focus(component.get(), nullptr);
+  } else if (operation == "set_value") {
+    auto editable = Own(atspi_accessible_get_editable_text_iface(element.get()));
+    if (editable) accepted = atspi_editable_text_set_text_contents(editable.get(), value.c_str(), nullptr);
+  } else if (operation == "press") {
+    auto action = Own(atspi_accessible_get_action_iface(element.get()));
+    if (action) {
+      const int count = std::min(16, atspi_action_get_n_actions(action.get(), nullptr));
+      for (int index = 0; index < count; ++index) {
+        const auto name = Text(atspi_action_get_action_name(action.get(), index, nullptr));
+        if (name == "click" || name == "press" || name == "activate" || name == "toggle") {
+          accepted = atspi_action_do_action(action.get(), index, nullptr);
+          break;
+        }
+      }
+    }
+  }
+  if (!accepted) throw std::runtime_error("UI_ACTION_FAILED");
+  return {{"accepted", true}};
+}
+DesktopSnapshot Tree(Accessible root, int limit, int depth) {
+  struct Entry { Accessible element; int depth, parent; };
+  std::deque<Entry> queue{{root, 0, -1}};
+  DesktopSnapshot result;
+  const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(1200);
+  while (!queue.empty() && int(result.items.size()) < limit && std::chrono::steady_clock::now() < deadline) {
+    auto entry = std::move(queue.front()); queue.pop_front();
+    auto element = entry.element;
+    atspi_accessible_clear_cache(element.get());
+    const auto role = atspi_accessible_get_role(element.get(), nullptr);
+    if (role == ATSPI_ROLE_INVALID) continue;
+    const bool secure = role == ATSPI_ROLE_PASSWORD_TEXT;
+    json data{{"role", Text(atspi_accessible_get_role_name(element.get(), nullptr))},
+      {"name", secure ? std::string{} : Text(atspi_accessible_get_name(element.get(), nullptr))},
+      {"pid", atspi_accessible_get_process_id(element.get(), nullptr)}, {"parent", entry.parent}, {"secure", secure}};
+    auto state = Own(atspi_accessible_get_state_set(element.get()));
+    if (state) {
+      data["foreground"] = bool(atspi_state_set_contains(state.get(), ATSPI_STATE_ACTIVE));
+      data["focused"] = bool(atspi_state_set_contains(state.get(), ATSPI_STATE_FOCUSED));
+    }
+    auto component = Own(atspi_accessible_get_component_iface(element.get()));
+    if (component && state && atspi_state_set_contains(state.get(), ATSPI_STATE_SHOWING)) {
+      const auto rect = atspi_component_get_extents(component.get(), ATSPI_COORD_TYPE_SCREEN, nullptr);
+      if (rect) {
+        data["bounds"] = {{"x", rect->x}, {"y", rect->y}, {"width", rect->width}, {"height", rect->height}};
+        g_free(rect);
+      }
+    }
+    if (!secure) {
+      auto text = Own(atspi_accessible_get_text_iface(element.get()));
+      if (text) {
+        const int count = std::clamp(atspi_text_get_character_count(text.get(), nullptr), 0, 256);
+        if (count) data["value"] = Text(atspi_text_get_text(text.get(), 0, count, nullptr));
+      }
+    }
+    const int parent = int(result.items.size());
+    result.items.push_back({data,
+      [element](const auto& operation, const auto& value) { return Perform(element, operation, value); },
+      [element](int max, int levels) { return Tree(element, max, levels); }});
+    const int count = std::max(0, atspi_accessible_get_child_count(element.get(), nullptr));
+    if (entry.depth >= depth) { result.truncated |= count > 0; continue; }
+    const int available = std::max(0, limit - int(result.items.size() + queue.size()));
+    result.truncated |= count > available;
+    for (int index = 0; index < std::min(count, available) && std::chrono::steady_clock::now() < deadline; ++index) {
+      auto child = Own(atspi_accessible_get_child_at_index(element.get(), index, nullptr));
+      if (child) queue.push_back({child, entry.depth + 1, parent});
+    }
+  }
+  result.truncated |= !queue.empty();
+  return result;
+}
+#endif
+} // namespace
+nlohmann::json DesktopCapabilities() {
+  bool accessibility = false;
+#ifdef CROSSDESK_HAS_ATSPI
+  try { accessibility = bool(Desktop()); } catch (...) {}
+#endif
+  return {{"available", true}, {"backend", accessibility ? "linux-atspi" : "linux-x11"},
+    {"tree", accessibility}, {"windows", accessibility || std::getenv("DISPLAY") != nullptr},
+    {"launch", true}, {"open_url", access("/usr/bin/xdg-open", X_OK) == 0}};
+}
+DesktopSnapshot DesktopWindows() {
+#ifdef CROSSDESK_HAS_ATSPI
+  try { return Tree(Desktop(), 128, 2); }
+  catch (const std::runtime_error&) { /* X11 remains usable when the accessibility bus is unavailable. */ }
+#endif
+  XConnection display;
+  const auto windows = display.Property(display.root, "_NET_CLIENT_LIST_STACKING", XCB_ATOM_WINDOW);
+  const auto active = display.Property(display.root, "_NET_ACTIVE_WINDOW", XCB_ATOM_WINDOW);
+  DesktopSnapshot result;
+  for (const auto window : windows) {
+    const auto pid = display.Property(window, "_NET_WM_PID", XCB_ATOM_CARDINAL);
+    json data{{"role", "window"}, {"name", display.Title(window)},
+      {"pid", pid.empty() ? 0 : pid.front()}, {"foreground", !active.empty() && active.front() == window}};
+    result.items.push_back({data, [window, pid](const std::string& operation, const std::string&) -> json {
+      if (operation != "activate" && operation != "focus") throw std::runtime_error("UNSUPPORTED_OPERATION");
+      XConnection display;
+      const auto windows = display.Property(display.root, "_NET_CLIENT_LIST_STACKING", XCB_ATOM_WINDOW);
+      if (std::find(windows.begin(), windows.end(), window) == windows.end() ||
+          display.Property(window, "_NET_WM_PID", XCB_ATOM_CARDINAL) != pid) throw std::runtime_error("STALE_ELEMENT");
+      xcb_client_message_event_t event{};
+      event.response_type = XCB_CLIENT_MESSAGE;
+      event.window = window;
+      event.type = display.Atom("_NET_ACTIVE_WINDOW");
+      event.format = 32;
+      event.data.data32[0] = 2;
+      event.data.data32[1] = XCB_CURRENT_TIME;
+      const auto cookie = xcb_send_event_checked(display.connection.get(), 0, display.root,
+          XCB_EVENT_MASK_SUBSTRUCTURE_REDIRECT | XCB_EVENT_MASK_SUBSTRUCTURE_NOTIFY, reinterpret_cast<const char*>(&event));
+      const auto error = xcb_request_check(display.connection.get(), cookie);
+      const bool accepted = !error;
+      std::free(error);
+      return {{"accepted", accepted}, {"state", "requested"}};
+    }, {}});
+    if (result.items.size() >= 128) { result.truncated = true; break; }
+  }
+  return result;
+}
+DesktopSnapshot DesktopTree(int pid, int limit, int depth) {
+#ifdef CROSSDESK_HAS_ATSPI
+  auto desktop = Desktop();
+  if (!pid) return Tree(desktop, limit, depth);
+  const int count = std::min(256, atspi_accessible_get_child_count(desktop.get(), nullptr));
+  const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(1200);
+  for (int index = 0; index < count; ++index) {
+    if (std::chrono::steady_clock::now() >= deadline) throw std::runtime_error("UI_QUERY_TIMEOUT");
+    auto app = Own(atspi_accessible_get_child_at_index(desktop.get(), index, nullptr));
+    if (app && atspi_accessible_get_process_id(app.get(), nullptr) == unsigned(pid)) return Tree(app, limit, depth);
+  }
+  throw std::runtime_error("APPLICATION_UNAVAILABLE");
+#else
+  (void)pid; (void)limit; (void)depth;
+  throw std::runtime_error("ATSPI_NOT_BUILT");
+#endif
+}
+nlohmann::json DesktopLaunch(const std::string& application) {
+  if (application.empty() || application.front() != '/' || access(application.c_str(), X_OK) != 0)
+    throw std::invalid_argument("APPLICATION_UNAVAILABLE");
+  return Spawn(application);
+}
+nlohmann::json DesktopOpenUrl(const std::string& url) { return Spawn("/usr/bin/xdg-open", url); }
+} // namespace crossdesk

apps/desktop/src/platform/macos/desktop_automation.mm

@@ -0,0 +1,168 @@
+#include "desktop_automation.h"
+
+#import <AppKit/AppKit.h>
+#include <ApplicationServices/ApplicationServices.h>
+#include <deque>
+#include <memory>
+#include <stdexcept>
+
+namespace crossdesk {
+namespace {
+using nlohmann::json;
+using Element = std::shared_ptr<const __AXUIElement>;
+Element Own(AXUIElementRef element) { return Element(element, [](auto value) { if (value) CFRelease(value); }); }
+CFTypeRef Attribute(AXUIElementRef element, CFStringRef name) {
+  CFTypeRef value = nullptr;
+  AXUIElementCopyAttributeValue(element, name, &value);
+  return value;
+}
+std::string String(CFTypeRef value) {
+  if (!value || CFGetTypeID(value) != CFStringGetTypeID()) return {};
+  char bytes[2048]{};
+  CFIndex used = 0;
+  const auto count = CFStringGetBytes(static_cast<CFStringRef>(value),
+      CFRangeMake(0, std::min<CFIndex>(CFStringGetLength(static_cast<CFStringRef>(value)), 256)),
+      kCFStringEncodingUTF8, 0, false, reinterpret_cast<UInt8*>(bytes), sizeof(bytes) - 1, &used);
+  (void)count;
+  return DesktopText(std::string(bytes, size_t(used)));
+}
+std::string Text(AXUIElementRef element, CFStringRef name) {
+  const auto value = Attribute(element, name);
+  auto result = String(value);
+  if (value) CFRelease(value);
+  return result;
+}
+void Accessible() { if (!AXIsProcessTrusted()) throw std::runtime_error("ACCESSIBILITY_PERMISSION_REQUIRED"); }
+json Perform(Element element, const std::string& operation, const std::string& value) {
+  Accessible();
+  if (Text(element.get(), kAXSubroleAttribute) == "AXSecureTextField") throw std::runtime_error("SECURE_ELEMENT");
+  AXError error = kAXErrorActionUnsupported;
+  if (operation == "press") error = AXUIElementPerformAction(element.get(), kAXPressAction);
+  else if (operation == "activate") {
+    pid_t pid = 0;
+    AXUIElementGetPid(element.get(), &pid);
+    [[NSRunningApplication runningApplicationWithProcessIdentifier:pid] activateWithOptions:0];
+    error = AXUIElementPerformAction(element.get(), kAXRaiseAction);
+  } else if (operation == "focus") error = AXUIElementSetAttributeValue(element.get(), kAXFocusedAttribute, kCFBooleanTrue);
+  else if (operation == "set_value") {
+    const auto text = CFStringCreateWithBytes(nullptr, reinterpret_cast<const UInt8*>(value.data()), value.size(), kCFStringEncodingUTF8, false);
+    if (!text) throw std::invalid_argument("INVALID_ARGUMENT");
+    error = AXUIElementSetAttributeValue(element.get(), kAXValueAttribute, text);
+    CFRelease(text);
+  }
+  if (error != kAXErrorSuccess) throw std::runtime_error(error == kAXErrorInvalidUIElement ? "STALE_ELEMENT" : "UI_ACTION_FAILED");
+  return {{"accepted", true}};
+}
+DesktopSnapshot Tree(Element root, int limit, int depth) {
+  Accessible();
+  struct Entry { Element element; int depth, parent; };
+  std::deque<Entry> queue{{root, 0, -1}};
+  DesktopSnapshot result;
+  const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(1200);
+  while (!queue.empty() && int(result.items.size()) < limit && std::chrono::steady_clock::now() < deadline) {
+    auto entry = std::move(queue.front()); queue.pop_front();
+    const auto element = entry.element.get();
+    AXUIElementSetMessagingTimeout(element, .1f);
+    const auto role = Text(element, kAXRoleAttribute);
+    if (role.empty()) continue;
+    const bool secure = Text(element, kAXSubroleAttribute) == "AXSecureTextField";
+    json data{{"role", role}, {"name", Text(element, kAXTitleAttribute)}, {"parent", entry.parent}, {"secure", secure}};
+    if (!secure) data["value"] = Text(element, kAXValueAttribute);
+    CGPoint point{}; CGSize size{};
+    auto position = Attribute(element, kAXPositionAttribute), extent = Attribute(element, kAXSizeAttribute);
+    if (position && extent && CFGetTypeID(position) == AXValueGetTypeID() && CFGetTypeID(extent) == AXValueGetTypeID() &&
+        AXValueGetValue(static_cast<AXValueRef>(position), kAXValueTypeCGPoint, &point) &&
+        AXValueGetValue(static_cast<AXValueRef>(extent), kAXValueTypeCGSize, &size))
+      data["bounds"] = {{"x", point.x}, {"y", point.y}, {"width", size.width}, {"height", size.height}};
+    if (position) CFRelease(position);
+    if (extent) CFRelease(extent);
+    const int index = int(result.items.size());
+    result.items.push_back({data,
+      [element = entry.element](const auto& operation, const auto& value) { return Perform(element, operation, value); },
+      [element = entry.element](int max, int levels) { return Tree(element, max, levels); }});
+    CFIndex count = 0;
+    if (AXUIElementGetAttributeValueCount(element, kAXChildrenAttribute, &count) != kAXErrorSuccess || count <= 0) continue;
+    if (entry.depth >= depth) { result.truncated = true; continue; }
+    const auto available = std::max(0, limit - int(result.items.size()) - int(queue.size()));
+    if (count > available) result.truncated = true;
+    CFArrayRef children = nullptr;
+    if (available && AXUIElementCopyAttributeValues(element, kAXChildrenAttribute, 0, std::min<CFIndex>(count, available), &children) == kAXErrorSuccess) {
+      for (CFIndex index = 0; index < CFArrayGetCount(children); ++index) {
+        const auto child = static_cast<AXUIElementRef>(CFArrayGetValueAtIndex(children, index));
+        if (CFGetTypeID(child) == AXUIElementGetTypeID()) {
+          CFRetain(child); queue.push_back({Own(child), entry.depth + 1, int(result.items.size()) - 1});
+        }
+      }
+      CFRelease(children);
+    }
+  }
+  result.truncated |= !queue.empty();
+  return result;
+}
+} // namespace
+nlohmann::json DesktopCapabilities() {
+  return {{"available", true}, {"backend", "macos-ax"}, {"accessibility_permission", bool(AXIsProcessTrusted())},
+    {"windows", bool(AXIsProcessTrusted())}, {"tree", bool(AXIsProcessTrusted())}, {"launch", true}, {"open_url", true}};
+}
+DesktopSnapshot DesktopTree(int pid, int limit, int depth) {
+  @autoreleasepool {
+    if (!pid) pid = NSWorkspace.sharedWorkspace.frontmostApplication.processIdentifier;
+    if (pid <= 0) throw std::runtime_error("APPLICATION_UNAVAILABLE");
+    return Tree(Own(AXUIElementCreateApplication(pid)), limit, depth);
+  }
+}
+DesktopSnapshot DesktopWindows() {
+  @autoreleasepool {
+    DesktopSnapshot result;
+    const auto front = NSWorkspace.sharedWorkspace.frontmostApplication.processIdentifier;
+    // AX window references provide exact activation targets when permission is present.
+    const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(1200);
+    Accessible();
+    for (NSRunningApplication* app in NSWorkspace.sharedWorkspace.runningApplications) {
+      if (std::chrono::steady_clock::now() >= deadline) { result.truncated = true; break; }
+      if (app.activationPolicy != NSApplicationActivationPolicyRegular) continue;
+      auto application = Own(AXUIElementCreateApplication(app.processIdentifier));
+      AXUIElementSetMessagingTimeout(application.get(), .1f);
+      const auto raw = Attribute(application.get(), kAXWindowsAttribute);
+      if (raw && CFGetTypeID(raw) == CFArrayGetTypeID()) {
+        const auto windows = static_cast<CFArrayRef>(raw);
+        for (CFIndex index = 0; index < CFArrayGetCount(windows) && result.items.size() < 128; ++index) {
+          if (std::chrono::steady_clock::now() >= deadline) { result.truncated = true; break; }
+          const auto window = static_cast<AXUIElementRef>(CFArrayGetValueAtIndex(windows, index));
+          if (CFGetTypeID(window) != AXUIElementGetTypeID()) continue;
+          CFRetain(window); const auto element = Own(window);
+          const auto main = Attribute(window, kAXMainAttribute);
+          const bool foreground = app.processIdentifier == front && main == kCFBooleanTrue;
+          if (main) CFRelease(main);
+          result.items.push_back({{{"role", "window"}, {"name", Text(window, kAXTitleAttribute)},
+              {"application", DesktopText(app.localizedName.UTF8String ?: "")}, {"pid", app.processIdentifier}, {"foreground", foreground}, {"foreground_application", app.processIdentifier == front}},
+            [element](const auto& operation, const auto& value) { return Perform(element, operation, value); },
+            [element](int max, int levels) { return Tree(element, max, levels); }});
+        }
+      }
+      if (raw) CFRelease(raw);
+      if (result.items.size() >= 128) { result.truncated = true; break; }
+    }
+    if (!AXIsProcessTrusted()) throw std::runtime_error("ACCESSIBILITY_PERMISSION_REQUIRED");
+    return result;
+  }
+}
+nlohmann::json DesktopLaunch(const std::string& application) {
+  @autoreleasepool {
+    NSString* value = [NSString stringWithUTF8String:application.c_str()];
+    NSURL* url = [value hasPrefix:@"/"] ? [NSURL fileURLWithPath:value] :
+        [NSWorkspace.sharedWorkspace URLForApplicationWithBundleIdentifier:value];
+    if (!url || ![url.path.pathExtension.lowercaseString isEqualToString:@"app"])
+      throw std::runtime_error("APPLICATION_UNAVAILABLE");
+    [NSWorkspace.sharedWorkspace openApplicationAtURL:url configuration:NSWorkspaceOpenConfiguration.configuration completionHandler:nil];
+    return {{"accepted", true}, {"state", "requested"}};
+  }
+}
+nlohmann::json DesktopOpenUrl(const std::string& value) {
+  @autoreleasepool {
+    NSURL* url = [NSURL URLWithString:[NSString stringWithUTF8String:value.c_str()]];
+    if (!url || ![NSWorkspace.sharedWorkspace openURL:url]) throw std::runtime_error("UI_ACTION_FAILED");
+    return {{"accepted", true}};
+  }
+}
+} // namespace crossdesk

apps/desktop/src/platform/windows/desktop_automation.cpp

@@ -0,0 +1,190 @@
+#include "desktop_automation.h"
+
+#include <Windows.h>
+#include <UIAutomation.h>
+#include <shellapi.h>
+#include <wrl/client.h>
+#include <algorithm>
+#include <deque>
+#include <stdexcept>
+
+namespace crossdesk {
+namespace {
+using nlohmann::json;
+using Microsoft::WRL::ComPtr;
+std::string Utf8(BSTR value) {
+  if (!value) return {};
+  const int length = std::min<int>(int(SysStringLen(value)), 256);
+  const int size = WideCharToMultiByte(CP_UTF8, 0, value, length, nullptr, 0, nullptr, nullptr);
+  std::string text(size > 0 ? size : 0, '\0');
+  if (size) WideCharToMultiByte(CP_UTF8, 0, value, length, text.data(), size, nullptr, nullptr);
+  SysFreeString(value);
+  return DesktopText(std::move(text));
+}
+std::wstring Wide(const std::string& value) {
+  const int size = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, value.data(), int(value.size()), nullptr, 0);
+  if (!size && !value.empty()) throw std::invalid_argument("INVALID_ARGUMENT");
+  std::wstring text(size, L'\0');
+  if (size) MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, value.data(), int(value.size()), text.data(), size);
+  return text;
+}
+ComPtr<IUIAutomation> Automation() {
+  struct Apartment {
+    HRESULT result = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED);
+    ~Apartment() { if (SUCCEEDED(result)) CoUninitialize(); }
+  };
+  static thread_local Apartment apartment;
+  ComPtr<IUIAutomation> automation;
+  if (FAILED(CoCreateInstance(__uuidof(CUIAutomation8), nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&automation))) &&
+      FAILED(CoCreateInstance(__uuidof(CUIAutomation), nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&automation))))
+    throw std::runtime_error("UI_AUTOMATION_UNAVAILABLE");
+  ComPtr<IUIAutomation2> timed;
+  if (SUCCEEDED(automation.As(&timed))) {
+    timed->put_ConnectionTimeout(200);
+    timed->put_TransactionTimeout(200);
+  }
+  return automation;
+}
+json Perform(ComPtr<IUIAutomationElement> element, const std::string& operation, const std::string& value) {
+  BOOL secure = TRUE;
+  if (FAILED(element->get_CurrentIsPassword(&secure))) throw std::runtime_error("STALE_ELEMENT");
+  if (secure) throw std::runtime_error("SECURE_ELEMENT");
+  HRESULT result = E_NOTIMPL;
+  if (operation == "focus" || operation == "activate") result = element->SetFocus();
+  else if (operation == "press") {
+    ComPtr<IUIAutomationInvokePattern> invoke;
+    if (SUCCEEDED(element->GetCurrentPatternAs(UIA_InvokePatternId, IID_PPV_ARGS(&invoke)))) result = invoke->Invoke();
+    else {
+      ComPtr<IUIAutomationSelectionItemPattern> select;
+      if (SUCCEEDED(element->GetCurrentPatternAs(UIA_SelectionItemPatternId, IID_PPV_ARGS(&select)))) result = select->Select();
+      else {
+        ComPtr<IUIAutomationTogglePattern> toggle;
+        if (SUCCEEDED(element->GetCurrentPatternAs(UIA_TogglePatternId, IID_PPV_ARGS(&toggle)))) result = toggle->Toggle();
+      }
+    }
+  } else if (operation == "set_value") {
+    ComPtr<IUIAutomationValuePattern> pattern;
+    if (SUCCEEDED(element->GetCurrentPatternAs(UIA_ValuePatternId, IID_PPV_ARGS(&pattern)))) result = pattern->SetValue(Wide(value).c_str());
+  }
+  if (FAILED(result)) throw std::runtime_error(result == UIA_E_ELEMENTNOTAVAILABLE ? "STALE_ELEMENT" : "UI_ACTION_FAILED");
+  return {{"accepted", true}};
+}
+DesktopSnapshot Tree(ComPtr<IUIAutomationElement> root, int limit, int depth) {
+  auto automation = Automation();
+  ComPtr<IUIAutomationTreeWalker> walker;
+  if (FAILED(automation->get_ControlViewWalker(&walker))) throw std::runtime_error("UI_AUTOMATION_UNAVAILABLE");
+  struct Entry { ComPtr<IUIAutomationElement> element; int depth, parent; };
+  std::deque<Entry> queue{{root, 0, -1}};
+  DesktopSnapshot result;
+  const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(1200);
+  while (!queue.empty() && int(result.items.size()) < limit && std::chrono::steady_clock::now() < deadline) {
+    auto entry = std::move(queue.front()); queue.pop_front();
+    auto element = entry.element;
+    BOOL secure = TRUE, enabled = FALSE;
+    if (FAILED(element->get_CurrentIsPassword(&secure))) continue;
+    element->get_CurrentIsEnabled(&enabled);
+    BSTR name = nullptr, role = nullptr;
+    element->get_CurrentName(&name);
+    element->get_CurrentLocalizedControlType(&role);
+    RECT bounds{};
+    element->get_CurrentBoundingRectangle(&bounds);
+    if (secure) { SysFreeString(name); name = nullptr; }
+    json data{{"name", Utf8(name)}, {"role", Utf8(role)},
+      {"secure", bool(secure)}, {"enabled", bool(enabled)}, {"parent", entry.parent},
+      {"bounds", {{"x", bounds.left}, {"y", bounds.top}, {"width", bounds.right - bounds.left}, {"height", bounds.bottom - bounds.top}}}};
+    if (!secure) {
+      ComPtr<IUIAutomationValuePattern> pattern;
+      if (SUCCEEDED(element->GetCurrentPatternAs(UIA_ValuePatternId, IID_PPV_ARGS(&pattern)))) {
+        BSTR value = nullptr;
+        if (SUCCEEDED(pattern->get_CurrentValue(&value))) data["value"] = Utf8(value);
+      }
+    }
+    const int parent = int(result.items.size());
+    result.items.push_back({data,
+      [element](const auto& operation, const auto& value) { return Perform(element, operation, value); },
+      [element](int max, int levels) { return Tree(element, max, levels); }});
+    ComPtr<IUIAutomationElement> child;
+    walker->GetFirstChildElement(element.Get(), &child);
+    if (entry.depth >= depth) { result.truncated |= bool(child); continue; }
+    while (child && int(queue.size() + result.items.size()) < limit && std::chrono::steady_clock::now() < deadline) {
+      queue.push_back({child, entry.depth + 1, parent});
+      ComPtr<IUIAutomationElement> next;
+      walker->GetNextSiblingElement(child.Get(), &next);
+      child = std::move(next);
+    }
+    result.truncated |= bool(child);
+  }
+  result.truncated |= !queue.empty();
+  return result;
+}
+DesktopSnapshot WindowTree(HWND window, int limit, int depth) {
+  ComPtr<IUIAutomationElement> root;
+  if (!IsWindow(window) || FAILED(Automation()->ElementFromHandle(window, &root)) || !root)
+    throw std::runtime_error("STALE_ELEMENT");
+  return Tree(root, limit, depth);
+}
+} // namespace
+nlohmann::json DesktopCapabilities() {
+  return {{"available", true}, {"backend", "windows-uia"}, {"windows", true}, {"tree", true}, {"launch", true}, {"open_url", true}};
+}
+DesktopSnapshot DesktopWindows() {
+  DesktopSnapshot result;
+  EnumWindows([](HWND window, LPARAM context) -> BOOL {
+    auto& result = *reinterpret_cast<DesktopSnapshot*>(context);
+    if (!IsWindowVisible(window) || GetWindow(window, GW_OWNER)) return TRUE;
+    wchar_t title[257]{};
+    GetWindowTextW(window, title, 257);
+    DWORD pid = 0;
+    GetWindowThreadProcessId(window, &pid);
+    const auto valid = [window, pid] {
+      DWORD current = 0;
+      GetWindowThreadProcessId(window, &current);
+      if (!IsWindow(window) || current != pid) throw std::runtime_error("STALE_ELEMENT");
+    };
+    RECT bounds{}; GetWindowRect(window, &bounds);
+    result.items.push_back({{{"name", Utf8(SysAllocString(title))}, {"role", "window"}, {"pid", pid},
+        {"foreground", window == GetForegroundWindow()},
+        {"bounds", {{"x", bounds.left}, {"y", bounds.top}, {"width", bounds.right - bounds.left}, {"height", bounds.bottom - bounds.top}}}},
+      [window, valid](const std::string& operation, const std::string&) -> json {
+        valid();
+        if (operation != "activate" && operation != "focus") throw std::runtime_error("UNSUPPORTED_OPERATION");
+        if (IsIconic(window)) ShowWindowAsync(window, SW_RESTORE);
+        if (!SetForegroundWindow(window)) throw std::runtime_error("UI_ACTION_FAILED");
+        return {{"accepted", true}};
+      }, [window, valid](int limit, int depth) { valid(); return WindowTree(window, limit, depth); }});
+    result.truncated = result.items.size() >= 128;
+    return result.truncated ? FALSE : TRUE;
+  }, reinterpret_cast<LPARAM>(&result));
+  return result;
+}
+DesktopSnapshot DesktopTree(int pid, int limit, int depth) {
+  HWND window = GetForegroundWindow();
+  if (pid) {
+    struct Search { DWORD pid; HWND window = nullptr; } search{DWORD(pid)};
+    EnumWindows([](HWND candidate, LPARAM context) -> BOOL {
+      auto& search = *reinterpret_cast<Search*>(context);
+      DWORD current = 0; GetWindowThreadProcessId(candidate, &current);
+      if (current == search.pid && IsWindowVisible(candidate) && !GetWindow(candidate, GW_OWNER)) {
+        search.window = candidate; return FALSE;
+      }
+      return TRUE;
+    }, reinterpret_cast<LPARAM>(&search));
+    window = search.window;
+  }
+  return WindowTree(window, limit, depth);
+}
+nlohmann::json DesktopLaunch(const std::string& application) {
+  const auto path = Wide(application);
+  if (path.size() < 7 || path[1] != L':' || (path[2] != L'\\' && path[2] != L'/') ||
+      _wcsicmp(path.c_str() + path.size() - 4, L".exe") != 0 || GetFileAttributesW(path.c_str()) == INVALID_FILE_ATTRIBUTES)
+    throw std::invalid_argument("APPLICATION_UNAVAILABLE");
+  if (reinterpret_cast<INT_PTR>(ShellExecuteW(nullptr, L"open", path.c_str(), nullptr, nullptr, SW_SHOWNORMAL)) <= 32)
+    throw std::runtime_error("UI_ACTION_FAILED");
+  return {{"accepted", true}};
+}
+nlohmann::json DesktopOpenUrl(const std::string& url) {
+  if (reinterpret_cast<INT_PTR>(ShellExecuteW(nullptr, L"open", Wide(url).c_str(), nullptr, nullptr, SW_SHOWNORMAL)) <= 32)
+    throw std::runtime_error("UI_ACTION_FAILED");
+  return {{"accepted", true}};
+}
+} // namespace crossdesk

apps/desktop/xmake/platform.lua

@@ -47,6 +47,7 @@ function setup_platform_settings()
         add_cxflags("/WX")
         set_runtimes("MT")
     elseif is_os("linux") then
+        add_requires("pkgconfig::atspi-2", {alias = "atspi", system = true, optional = true})
         add_links("pulse-simple", "pulse")
         add_requires("libyuv")
         add_syslinks("pthread", "dl")

apps/desktop/xmake/targets.lua

@@ -87,17 +87,25 @@ function setup_targets()
         add_files("apps/desktop/src/common/*.cpp")
         if is_os("windows") then
             add_files("apps/desktop/src/platform/windows/system_info.cpp",
-                "apps/desktop/src/platform/windows/screen_lock.cpp")
-            add_syslinks("Advapi32", "Wtsapi32", {public = true})
+                "apps/desktop/src/platform/windows/screen_lock.cpp",
+                "apps/desktop/src/platform/windows/desktop_automation.cpp")
+            add_syslinks("Advapi32", "Wtsapi32", "Ole32", "OleAut32", "Uiautomationcore", {public = true})
         elseif is_os("macosx") then
             add_files("apps/desktop/src/platform/macos/system_info.cpp",
-                "apps/desktop/src/platform/macos/screen_lock.cpp")
+                "apps/desktop/src/platform/macos/screen_lock.cpp",
+                "apps/desktop/src/platform/macos/desktop_automation.mm")
+            add_frameworks("AppKit", "ApplicationServices", {public = true})
         elseif is_os("linux") then
             add_files("apps/desktop/src/platform/linux/system_info.cpp",
                 "apps/desktop/src/platform/linux/screen_lock.cpp",
+                "apps/desktop/src/platform/linux/desktop_automation.cpp",
                 "apps/desktop/src/platform/linux/common/*.cpp")
             add_includedirs("apps/desktop/src/platform/linux/common")
-            add_syslinks("Xi", {public = true})
+            add_syslinks("Xi", "xcb", {public = true})
+            add_packages("atspi", {public = true})
+            on_load(function (target)
+                if target:pkg("atspi") then target:add("defines", "CROSSDESK_HAS_ATSPI=1") end
+            end)
         end
         add_includedirs("apps/desktop/src/common", {public = true})
 

apps/mcp/README.md

@@ -300,3 +300,13 @@ Agent 在人工接管期间仍可观察画面，键鼠、显示设置和其他
 `crossdesk_audio` 开关远端桌面音频接收，默认关闭。面板的“播放声音”按钮同时启用接收和本地播放，音量在本面板调节。播放使用已授权的视频 WebSocket 通道，PCM 队列与浏览器播放时延均设有上限，拥塞时丢弃过时音频。浏览器首次播放需要用户点击按钮。
 
 `crossdesk_audio_capture` 采集接下来 100 至 10000 毫秒的音频，通过 MCP 音频内容块返回单声道 48 kHz WAV。采集期间临时开启接收，结束或取消后恢复之前的设置，未收到足够音频时返回 `AUDIO_UNAVAILABLE`。系统需允许被控端捕获桌面声音。自动恢复连接会恢复用户选择的音频接收设置。
+
+### 窗口、应用与可访问性元素
+
+`crossdesk_desktop` 查询平台能力、窗口清单及可访问性树。可按进程或先前快照中的元素继续读取子树，节点包含角色、名称、可读取值和系统原生桌面坐标。每份快照最多 128 个节点，响应限制为 48 KiB 左右，超出范围返回 `truncated`。元素引用绑定连接，有效期三十秒，最多保留八份快照。
+
+`crossdesk_desktop_action` 支持激活窗口、按下或聚焦元素、设置文本值、启动应用以及打开 HTTP(S) 地址。元素操作使用快照和 `element_id`，执行后该快照失效。所有修改操作需要控制授权及 `operation_id`，相同请求查询已有结果。安全密码字段隐藏值并拒绝通过结构化接口修改，系统解锁使用本地密码流程。操作结果表示系统接口接受情况，随后使用画面或新树确认界面变化。
+
+`crossdesk_wait_ui` 根据文本及角色等待元素出现或消失。读取范围不完整时无法确认消失，会在超时结果中返回 `search_incomplete`。
+
+macOS 使用 AX，需要系统辅助功能权限。Windows 使用 UI Automation，窗口激活遵守系统前台窗口及权限约束。Linux 构建时若检测到 `atspi-2` 开发包，会启用 AT-SPI 元素访问，否则提供 X11 窗口查询和激活。Wayland 的元素访问依赖桌面和应用提供 AT-SPI。应用启动参数接受 macOS bundle ID 或绝对 `.app` 路径、Windows 绝对 `.exe` 路径、Linux 绝对可执行文件路径。Linux 接口的所有权和超时处理依据官方文档。[AT-SPI 接口文档](https://docs.gtk.org/atspi2/)

apps/mcp/src/native.ts

@@ -178,6 +178,7 @@ export class NativeClient {
     const sideEffect = method === "action" || method === "unlock" || method === "release_inputs" || method === "viewer_input" ||
       (method === "clipboard" && args.operation === "set") ||
       (method === "system" && args.command !== "status") ||
+      (method === "desktop" && !["capabilities", "windows", "tree"].includes(args.operation)) ||
       (method === "files" && ["upload", "download", "cancel"].includes(args.operation));
     const line = JSON.stringify({ ...args, id, method }) + "\n";
     if (Buffer.byteLength(line) > 65536)

apps/mcp/src/server.ts

@@ -266,6 +266,37 @@ export function createServer(
     },
     { readOnly: true },
   );
+  add("crossdesk_desktop", z.strictObject({ session_id: id,
+    operation: z.enum(["capabilities", "windows", "tree"]), pid: z.int().min(0).max(2147483647).optional(),
+    snapshot_id: id.optional(), element_id: z.int().min(0).max(127).optional(),
+    max_nodes: z.int().min(1).max(128).default(128), max_depth: z.int().min(0).max(16).default(8),
+  }).refine((p) => (p.snapshot_id === undefined) === (p.element_id === undefined) &&
+    !(p.snapshot_id && p.pid !== undefined)),
+    "Inspect native desktop capabilities, windows/applications or an accessibility tree. Use pid or a prior snapshot's element_id to inspect a subtree. Returns at most 128 nodes with role, name, available value and desktop-native bounds. Snapshot references expire after 30 seconds and are scoped to this session. Secure text values are omitted. Check truncated and drill into subtrees. Backend support and accessibility permissions vary by platform.",
+    async (p, ctx) => output(await sessions.desktop(p, ctx.mcpReq.signal)), { readOnly: true });
+  const target = { snapshot_id: id, element_id: z.int().min(0).max(127) };
+  add("crossdesk_desktop_action", z.strictObject({ session_id: id, operation_id: id,
+    action: z.discriminatedUnion("operation", [
+      z.strictObject({ operation: z.literal("activate"), ...target }),
+      z.strictObject({ operation: z.literal("press"), ...target }),
+      z.strictObject({ operation: z.literal("focus"), ...target }),
+      z.strictObject({ operation: z.literal("set_value"), ...target, value: z.string().max(16384)
+        .refine((value) => Buffer.byteLength(value) <= 16384 && !value.includes("\0")) }),
+      z.strictObject({ operation: z.literal("launch"), application: z.string().min(1).max(4096) }),
+      z.strictObject({ operation: z.literal("open_url"), url: z.string().url().max(4096)
+        .refine((value) => /^https?:\/\//.test(value)) }),
+    ]),
+  }),
+    "Activate a window, press/focus an accessibility element, set its value, launch an application or open an HTTP(S) URL in the default browser. Requires active control authorization. Element operations consume their 30-second snapshot, so inspect again afterward. Use operation_id for deduplication. Secure fields use the local unlock workflow. launch accepts a macOS bundle ID or absolute .app path, an absolute Windows .exe path, or an absolute Linux executable path. OS acceptance requires subsequent UI verification.",
+    async (p, ctx) => output(await sessions.desktop(p, ctx.mcpReq.signal)));
+  add("crossdesk_wait_ui", z.strictObject({ session_id: id,
+    scope: z.enum(["tree", "windows"]).default("tree"), pid: z.int().min(0).max(2147483647).optional(),
+    text: z.string().min(1).max(256).optional(), role: z.string().min(1).max(128).optional(),
+    condition: z.enum(["present", "absent"]).default("present"), case_sensitive: z.boolean().default(false),
+    timeout_ms: z.int().min(0).max(25000).default(5000),
+  }).refine((p) => !!p.text || !!p.role),
+    "Wait for native UI text (substring) and/or role (exact) to appear or disappear, optionally within a process. Returns a fresh snapshot and matching elements. An incomplete tree cannot confirm absence and returns search_incomplete on timeout. Use shorter waits or a narrower process for large applications.",
+    async (p, ctx) => output(await sessions.waitUI(p, ctx.mcpReq.signal)), { readOnly: true });
   add("crossdesk_audio", z.strictObject({ session_id: id, enabled: z.boolean() }),
     "Enable or disable remote desktop audio reception. Audio starts disabled. Panel playback also requires the user's Play audio button. Status reports PCM reception counters.",
     async (p, ctx) => output(await sessions.audio(p, false, ctx.mcpReq.signal)));

apps/mcp/src/sessions.ts

@@ -818,6 +818,41 @@ export class Sessions {
       throw error;
     }
   }
+  async desktop(p: Data, signal?: AbortSignal): Promise<Data> {
+    const { session_id, ...args } = p;
+    const s = this.session(session_id);
+    if (args.action) { Object.assign(args, args.action); delete args.action; }
+    const write = !["capabilities", "windows", "tree"].includes(args.operation);
+    if (write && s.displayBatch) throw new Failure("BUSY");
+    if (write && s.unlock && this.unlockPending(s.unlock)) throw new Failure("UNLOCK_PENDING");
+    if (args.snapshot_id) {
+      const prefix = `${s.id}_`;
+      if (!args.snapshot_id.startsWith(prefix)) throw new Failure("STALE_UI_SNAPSHOT");
+      args.snapshot_id = args.snapshot_id.slice(prefix.length);
+    }
+    const result = await s.native.request("desktop", args, 7000, signal);
+    if (result.snapshot_id) result.snapshot_id = `${s.id}_${result.snapshot_id}`;
+    return result;
+  }
+  async waitUI(p: Data, signal?: AbortSignal): Promise<Data> {
+    const deadline = Date.now() + (p.timeout_ms ?? 5000);
+    const normalized = (value: unknown) => p.case_sensitive ? String(value ?? "") : String(value ?? "").toLocaleLowerCase();
+    let incomplete = false;
+    do {
+      const result = await this.desktop({ session_id: p.session_id, operation: p.scope ?? "tree",
+        ...(p.pid !== undefined ? { pid: p.pid } : {}), max_nodes: 128, max_depth: 16 }, signal);
+      if (result.error) return result;
+      const matches = (result.elements ?? []).filter((element: Data) =>
+        (!p.text || normalized(`${element.name ?? ""} ${element.value ?? ""}`).includes(normalized(p.text))) &&
+        (!p.role || normalized(element.role) === normalized(p.role)));
+      incomplete = result.truncated === true;
+      if (p.condition === "absent" ? !matches.length && !incomplete : matches.length > 0)
+        return { ...result, matched: true, matches };
+      if (Date.now() >= deadline) break;
+      await delay(Math.min(250, deadline - Date.now()), undefined, { signal });
+    } while (Date.now() <= deadline);
+    return { matched: false, wait_timed_out: true, search_incomplete: incomplete };
+  }
   async system(p: Data, signal?: AbortSignal): Promise<Data> {
     const { session_id, ...args } = p;
     const s = this.session(session_id);