[feat] add native desktop inspection and accessibility actions

apps/desktop/src/common/desktop_automation.cpp

@@ -0,0 +1,87 @@
+#include "desktop_automation.h"
+
+#include <algorithm>
+#include <stdexcept>
+
+namespace crossdesk {
+using nlohmann::json;
+std::string DesktopText(std::string text, size_t limit) {
+  if (text.size() > limit) {
+    while (limit && (static_cast<unsigned char>(text[limit]) & 0xc0) == 0x80) --limit;
+    text.resize(limit);
+  }
+  for (auto& character : text) if (static_cast<unsigned char>(character) < 32 && character != '\n') character = ' ';
+  return json::parse(json(text).dump(-1, ' ', false, json::error_handler_t::replace)).get<std::string>();
+}
+json DesktopAutomation::Execute(const std::string& owner, const json& request) {
+  const auto now = std::chrono::steady_clock::now();
+  while (!snapshots_.empty() && snapshots_.front().expires <= now) snapshots_.pop_front();
+  const auto operation = request.at("operation").get<std::string>();
+  if (operation == "capabilities") return DesktopCapabilities();
+  if (operation == "launch" || operation == "open_url") {
+    const auto value = request.at(operation == "launch" ? "application" : "url").get<std::string>();
+    if (value.empty() || value.size() > 4096 || value.find_first_of("\r\n") != std::string::npos ||
+        value.find('\0') != std::string::npos) throw std::invalid_argument("INVALID_ARGUMENT");
+    if (operation == "launch") return DesktopLaunch(value);
+    if (value.rfind("https://", 0) != 0 && value.rfind("http://", 0) != 0)
+      throw std::invalid_argument("INVALID_URL");
+    return DesktopOpenUrl(value);
+  }
+  DesktopItem item;
+  auto found = snapshots_.end();
+  if (request.contains("snapshot_id")) {
+    const auto id = request.at("snapshot_id").get<std::string>();
+    found = std::find_if(snapshots_.begin(), snapshots_.end(), [&](const auto& snapshot) {
+      return snapshot.id == id && snapshot.owner == owner;
+    });
+    if (found == snapshots_.end()) throw std::invalid_argument("STALE_UI_SNAPSHOT");
+    const int index = request.at("element_id").get<int>();
+    if (index < 0 || size_t(index) >= found->content.items.size()) throw std::invalid_argument("INVALID_ELEMENT");
+    item = found->content.items[size_t(index)];
+  }
+  if (operation == "windows" || operation == "tree") {
+    const int limit = request.value("max_nodes", 128), depth = request.value("max_depth", 8);
+    const int pid = request.value("pid", 0);
+    if (limit < 1 || limit > 128 || depth < 0 || depth > 16 || pid < 0)
+      throw std::invalid_argument("INVALID_ARGUMENT");
+    DesktopSnapshot content;
+    if (operation == "windows") content = DesktopWindows();
+    else if (found != snapshots_.end()) {
+      if (!item.inspect) throw std::invalid_argument("UNSUPPORTED_OPERATION");
+      content = item.inspect(limit, depth);
+    } else content = DesktopTree(pid, limit, depth);
+    json elements = json::array();
+    size_t bytes = 0;
+    for (size_t index = 0; index < content.items.size(); ++index) {
+      auto data = content.items[index].data;
+      data["element_id"] = index;
+      bytes += data.dump(-1, ' ', false, json::error_handler_t::replace).size();
+      if (bytes > 48000 || index >= 128) { content.truncated = true; content.items.resize(index); break; }
+      elements.push_back(std::move(data));
+    }
+    const auto id = "ui_" + std::to_string(++sequence_);
+    if (snapshots_.size() >= 8) snapshots_.pop_front();
+    const bool truncated = content.truncated;
+    snapshots_.push_back({id, owner, now + std::chrono::seconds(30), std::move(content)});
+    return {{"snapshot_id", id}, {"expires_in_ms", 30000}, {"elements", elements},
+      {"truncated", truncated}, {"coordinate_space", "desktop_native"}};
+  }
+  if (operation != "activate" && operation != "press" && operation != "focus" && operation != "set_value")
+    throw std::invalid_argument("UNSUPPORTED_OPERATION");
+  if (found == snapshots_.end() || !item.perform) throw std::invalid_argument("INVALID_ELEMENT");
+  const auto value = request.value("value", std::string{});
+  if (value.size() > 16384 || value.find('\0') != std::string::npos) throw std::invalid_argument("INVALID_ARGUMENT");
+  if (item.data.value("secure", false)) throw std::invalid_argument("SECURE_ELEMENT");
+  // The next operation must inspect the resulting UI again, even if an OS
+  // provider returns an uncertain result after accepting a mutation.
+  snapshots_.erase(found);
+  return item.perform(operation, value);
+}
+#if defined(__OHOS__)
+json DesktopCapabilities() { return {{"available", false}}; }
+DesktopSnapshot DesktopWindows() { throw std::runtime_error("UI_AUTOMATION_UNSUPPORTED"); }
+DesktopSnapshot DesktopTree(int, int, int) { throw std::runtime_error("UI_AUTOMATION_UNSUPPORTED"); }
+json DesktopLaunch(const std::string&) { throw std::runtime_error("UI_AUTOMATION_UNSUPPORTED"); }
+json DesktopOpenUrl(const std::string&) { throw std::runtime_error("UI_AUTOMATION_UNSUPPORTED"); }
+#endif
+} // namespace crossdesk

apps/desktop/src/common/desktop_automation.h

@@ -0,0 +1,50 @@
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2026-11-06
+ * Copyright (c) 2026 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _DESKTOP_AUTOMATION_H_
+#define _DESKTOP_AUTOMATION_H_
+
+#include <chrono>
+#include <deque>
+#include <functional>
+#include <nlohmann/json.hpp>
+#include <string>
+#include <vector>
+
+namespace crossdesk {
+struct DesktopSnapshot;
+struct DesktopItem {
+  nlohmann::json data;
+  std::function<nlohmann::json(const std::string&, const std::string&)> perform;
+  std::function<DesktopSnapshot(int, int)> inspect;
+};
+struct DesktopSnapshot {
+  std::vector<DesktopItem> items;
+  bool truncated = false;
+};
+// Platform functions execute on the desktop UI thread. References remain owned
+// by a bounded, connection-scoped snapshot until expiry or a mutating action.
+nlohmann::json DesktopCapabilities();
+DesktopSnapshot DesktopWindows();
+DesktopSnapshot DesktopTree(int pid, int limit, int depth);
+nlohmann::json DesktopLaunch(const std::string& application);
+nlohmann::json DesktopOpenUrl(const std::string& url);
+std::string DesktopText(std::string text, size_t limit = 256);
+class DesktopAutomation {
+ public:
+  nlohmann::json Execute(const std::string& owner, const nlohmann::json& request);
+ private:
+  struct Snapshot {
+    std::string id, owner;
+    std::chrono::steady_clock::time_point expires;
+    DesktopSnapshot content;
+  };
+  std::deque<Snapshot> snapshots_;
+  uint64_t sequence_ = 0;
+};
+} // namespace crossdesk
+
+#endif

apps/desktop/src/gui/runtime/automation_runtime.cpp

@@ -25,6 +25,7 @@ void GuiRuntime::HandleAutomation(const std::string& remote_id,
       json capabilities{{"input_ack", true}, {"clipboard", true}, {"files", true}};
 #if defined(_WIN32) || defined(__APPLE__) || (defined(__linux__) && !defined(__OHOS__))
       capabilities["system"] = true;
+      capabilities["desktop"] = true;
 #endif
       reply(true, std::move(capabilities));
     } else if (method == "input") {
@@ -42,6 +43,22 @@ void GuiRuntime::HandleAutomation(const std::string& remote_id,
       if (result == RemoteInputDispatcher::Result::rejected ||
           result == RemoteInputDispatcher::Result::overflow)
         reply(false, "INPUT_REJECTED");
+    } else if (method == "desktop") {
+      const auto arguments = request.at("arguments");
+      if (!arguments.is_object()) { reply(false, "INVALID_ARGUMENT"); return; }
+      std::lock_guard lock(automation_mutex_);
+      if (automation_closing_ || automation_tasks_.size() >= 32) { reply(false, "BUSY"); return; }
+      const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2);
+      automation_tasks_.push_back([this, source, remote_id, arguments, reply, deadline] {
+        if (std::chrono::steady_clock::now() > deadline) { reply(false, "REQUEST_EXPIRED"); return; }
+        if (ControllerEvents(remote_id) != source || !IsAuthorizedController(remote_id)) return;
+        try { reply(true, desktop_automation_.Execute(remote_id, arguments)); }
+        catch (const std::exception& error) {
+          const std::string code = error.what();
+          reply(false, !code.empty() && code.size() <= 64 && code.find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZ_") == std::string::npos
+              ? code : "UI_AUTOMATION_FAILED");
+        }
+      });
     } else if (method == "system") {
       const auto operation = request.at("arguments").at("operation").get<std::string>();
       if (operation != "status" && operation != "lock" && operation != "send_sas") {

apps/desktop/src/gui/runtime/gui_runtime.h

@@ -13,6 +13,7 @@
 #include <memory>
 #include <string>
 
+#include "desktop_automation.h"
 #include "features/clipboard/clipboard_controller.h"
 #include "features/devices/session_device_manager.h"
 #include "features/file_transfer/file_transfer_manager.h"
@@ -154,6 +155,7 @@ class GuiRuntime : protected gui_detail::GuiState {
   void SendRemoteInput(const RemoteAction& action);
   InputOutcome ApplyRemoteInput(const std::string& remote_id, const RemoteAction& action);
   void HandleAutomation(const std::string& remote_id, const nlohmann::json& request);
+  DesktopAutomation desktop_automation_;
   std::mutex automation_mutex_;
   std::deque<std::function<void()>> automation_tasks_;
   bool automation_closing_ = false;

apps/desktop/src/platform/linux/desktop_automation.cpp

@@ -0,0 +1,273 @@
+#include "desktop_automation.h"
+
+#include <xcb/xcb.h>
+#include <cstring>
+#include <algorithm>
+#include <cstdlib>
+#include <cerrno>
+#include <deque>
+#include <condition_variable>
+#include <mutex>
+#include <memory>
+#include <spawn.h>
+#include <stdexcept>
+#include <sys/wait.h>
+#include <thread>
+#include <unistd.h>
+#ifdef CROSSDESK_HAS_ATSPI
+#include <atspi/atspi.h>
+#endif
+extern char** environ;
+
+namespace crossdesk {
+namespace {
+using nlohmann::json;
+class LaunchedProcesses {
+ public:
+  LaunchedProcesses() {
+    children_.reserve(128);
+    worker_ = std::thread([this] {
+      std::unique_lock lock(mutex_);
+      while (!stopping_) {
+        wake_.wait_for(lock, std::chrono::milliseconds(500));
+        children_.erase(std::remove_if(children_.begin(), children_.end(), [](pid_t child) {
+          const auto result = waitpid(child, nullptr, WNOHANG);
+          return result > 0 || (result < 0 && errno == ECHILD);
+        }), children_.end());
+      }
+    });
+  }
+  ~LaunchedProcesses() {
+    { std::lock_guard lock(mutex_); stopping_ = true; }
+    wake_.notify_all();
+    worker_.join();
+  }
+  pid_t Spawn(const std::string& executable, const std::string& argument) {
+    std::lock_guard lock(mutex_);
+    if (children_.size() >= 128) throw std::runtime_error("BUSY");
+    char* args[] = {const_cast<char*>(executable.c_str()), argument.empty() ? nullptr : const_cast<char*>(argument.c_str()), nullptr};
+    pid_t pid = 0;
+    if (posix_spawn(&pid, executable.c_str(), nullptr, nullptr, args, environ) != 0)
+      throw std::runtime_error("APPLICATION_UNAVAILABLE");
+    children_.push_back(pid);
+    return pid;
+  }
+ private:
+  std::mutex mutex_;
+  std::condition_variable wake_;
+  bool stopping_ = false;
+  std::vector<pid_t> children_;
+  std::thread worker_;
+};
+json Spawn(const std::string& executable, const std::string& argument = {}) {
+  static LaunchedProcesses processes;
+  return {{"accepted", true}, {"pid", processes.Spawn(executable, argument)}};
+}
+struct XConnection {
+  std::shared_ptr<xcb_connection_t> connection;
+  xcb_window_t root = 0;
+  XConnection() {
+    int screen_index = 0;
+    connection = {xcb_connect(nullptr, &screen_index), xcb_disconnect};
+    if (!connection || xcb_connection_has_error(connection.get())) throw std::runtime_error("X11_UNAVAILABLE");
+    auto screen = xcb_setup_roots_iterator(xcb_get_setup(connection.get()));
+    for (int index = 0; index < screen_index && screen.rem; ++index) xcb_screen_next(&screen);
+    if (!screen.rem) throw std::runtime_error("X11_UNAVAILABLE");
+    root = screen.data->root;
+  }
+  xcb_atom_t Atom(const char* name) const {
+    auto reply = xcb_intern_atom_reply(connection.get(), xcb_intern_atom(connection.get(), 0, uint16_t(std::strlen(name)), name), nullptr);
+    const auto atom = reply ? reply->atom : XCB_ATOM_NONE;
+    std::free(reply);
+    return atom;
+  }
+  std::vector<uint32_t> Property(xcb_window_t window, const char* name, xcb_atom_t type) const {
+    auto reply = xcb_get_property_reply(connection.get(),
+        xcb_get_property(connection.get(), 0, window, Atom(name), type, 0, 256), nullptr);
+    std::vector<uint32_t> result;
+    if (reply && reply->type == type && reply->format == 32) {
+      const auto data = static_cast<const uint32_t*>(xcb_get_property_value(reply));
+      result.assign(data, data + xcb_get_property_value_length(reply) / 4);
+    }
+    std::free(reply);
+    return result;
+  }
+  std::string Title(xcb_window_t window) const {
+    auto reply = xcb_get_property_reply(connection.get(),
+        xcb_get_property(connection.get(), 0, window, Atom("_NET_WM_NAME"), Atom("UTF8_STRING"), 0, 256), nullptr);
+    std::string text;
+    if (reply && reply->format == 8) text.assign(static_cast<const char*>(xcb_get_property_value(reply)), xcb_get_property_value_length(reply));
+    std::free(reply);
+    return DesktopText(std::move(text));
+  }
+};
+#ifdef CROSSDESK_HAS_ATSPI
+template <class T> std::shared_ptr<T> Own(T* object) { return {object, [](T* value) { if (value) g_object_unref(value); }}; }
+using Accessible = std::shared_ptr<AtspiAccessible>;
+std::string Text(char* value) {
+  auto text = DesktopText(value ? value : "");
+  g_free(value);
+  return text;
+}
+Accessible Desktop() {
+  static const int initialized = atspi_init();
+  if (initialized > 1) throw std::runtime_error("ACCESSIBILITY_UNAVAILABLE");
+  atspi_set_timeout(120, 120);
+  for (int count = 0; count < 32 && g_main_context_pending(nullptr); ++count) g_main_context_iteration(nullptr, false);
+  auto desktop = Own(atspi_get_desktop(0));
+  if (!desktop) throw std::runtime_error("ACCESSIBILITY_UNAVAILABLE");
+  return desktop;
+}
+json Perform(Accessible element, const std::string& operation, const std::string& value) {
+  atspi_accessible_clear_cache(element.get());
+  auto state = Own(atspi_accessible_get_state_set(element.get()));
+  if (!state || atspi_state_set_contains(state.get(), ATSPI_STATE_DEFUNCT)) throw std::runtime_error("STALE_ELEMENT");
+  const auto role = atspi_accessible_get_role(element.get(), nullptr);
+  if (role == ATSPI_ROLE_INVALID) throw std::runtime_error("STALE_ELEMENT");
+  if (role == ATSPI_ROLE_PASSWORD_TEXT) throw std::runtime_error("SECURE_ELEMENT");
+  bool accepted = false;
+  if (operation == "focus" || operation == "activate") {
+    auto component = Own(atspi_accessible_get_component_iface(element.get()));
+    if (component) accepted = atspi_component_grab_focus(component.get(), nullptr);
+  } else if (operation == "set_value") {
+    auto editable = Own(atspi_accessible_get_editable_text_iface(element.get()));
+    if (editable) accepted = atspi_editable_text_set_text_contents(editable.get(), value.c_str(), nullptr);
+  } else if (operation == "press") {
+    auto action = Own(atspi_accessible_get_action_iface(element.get()));
+    if (action) {
+      const int count = std::min(16, atspi_action_get_n_actions(action.get(), nullptr));
+      for (int index = 0; index < count; ++index) {
+        const auto name = Text(atspi_action_get_action_name(action.get(), index, nullptr));
+        if (name == "click" || name == "press" || name == "activate" || name == "toggle") {
+          accepted = atspi_action_do_action(action.get(), index, nullptr);
+          break;
+        }
+      }
+    }
+  }
+  if (!accepted) throw std::runtime_error("UI_ACTION_FAILED");
+  return {{"accepted", true}};
+}
+DesktopSnapshot Tree(Accessible root, int limit, int depth) {
+  struct Entry { Accessible element; int depth, parent; };
+  std::deque<Entry> queue{{root, 0, -1}};
+  DesktopSnapshot result;
+  const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(1200);
+  while (!queue.empty() && int(result.items.size()) < limit && std::chrono::steady_clock::now() < deadline) {
+    auto entry = std::move(queue.front()); queue.pop_front();
+    auto element = entry.element;
+    atspi_accessible_clear_cache(element.get());
+    const auto role = atspi_accessible_get_role(element.get(), nullptr);
+    if (role == ATSPI_ROLE_INVALID) continue;
+    const bool secure = role == ATSPI_ROLE_PASSWORD_TEXT;
+    json data{{"role", Text(atspi_accessible_get_role_name(element.get(), nullptr))},
+      {"name", secure ? std::string{} : Text(atspi_accessible_get_name(element.get(), nullptr))},
+      {"pid", atspi_accessible_get_process_id(element.get(), nullptr)}, {"parent", entry.parent}, {"secure", secure}};
+    auto state = Own(atspi_accessible_get_state_set(element.get()));
+    if (state) {
+      data["foreground"] = bool(atspi_state_set_contains(state.get(), ATSPI_STATE_ACTIVE));
+      data["focused"] = bool(atspi_state_set_contains(state.get(), ATSPI_STATE_FOCUSED));
+    }
+    auto component = Own(atspi_accessible_get_component_iface(element.get()));
+    if (component && state && atspi_state_set_contains(state.get(), ATSPI_STATE_SHOWING)) {
+      const auto rect = atspi_component_get_extents(component.get(), ATSPI_COORD_TYPE_SCREEN, nullptr);
+      if (rect) {
+        data["bounds"] = {{"x", rect->x}, {"y", rect->y}, {"width", rect->width}, {"height", rect->height}};
+        g_free(rect);
+      }
+    }
+    if (!secure) {
+      auto text = Own(atspi_accessible_get_text_iface(element.get()));
+      if (text) {
+        const int count = std::clamp(atspi_text_get_character_count(text.get(), nullptr), 0, 256);
+        if (count) data["value"] = Text(atspi_text_get_text(text.get(), 0, count, nullptr));
+      }
+    }
+    const int parent = int(result.items.size());
+    result.items.push_back({data,
+      [element](const auto& operation, const auto& value) { return Perform(element, operation, value); },
+      [element](int max, int levels) { return Tree(element, max, levels); }});
+    const int count = std::max(0, atspi_accessible_get_child_count(element.get(), nullptr));
+    if (entry.depth >= depth) { result.truncated |= count > 0; continue; }
+    const int available = std::max(0, limit - int(result.items.size() + queue.size()));
+    result.truncated |= count > available;
+    for (int index = 0; index < std::min(count, available) && std::chrono::steady_clock::now() < deadline; ++index) {
+      auto child = Own(atspi_accessible_get_child_at_index(element.get(), index, nullptr));
+      if (child) queue.push_back({child, entry.depth + 1, parent});
+    }
+  }
+  result.truncated |= !queue.empty();
+  return result;
+}
+#endif
+} // namespace
+nlohmann::json DesktopCapabilities() {
+  bool accessibility = false;
+#ifdef CROSSDESK_HAS_ATSPI
+  try { accessibility = bool(Desktop()); } catch (...) {}
+#endif
+  return {{"available", true}, {"backend", accessibility ? "linux-atspi" : "linux-x11"},
+    {"tree", accessibility}, {"windows", accessibility || std::getenv("DISPLAY") != nullptr},
+    {"launch", true}, {"open_url", access("/usr/bin/xdg-open", X_OK) == 0}};
+}
+DesktopSnapshot DesktopWindows() {
+#ifdef CROSSDESK_HAS_ATSPI
+  try { return Tree(Desktop(), 128, 2); }
+  catch (const std::runtime_error&) { /* X11 remains usable when the accessibility bus is unavailable. */ }
+#endif
+  XConnection display;
+  const auto windows = display.Property(display.root, "_NET_CLIENT_LIST_STACKING", XCB_ATOM_WINDOW);
+  const auto active = display.Property(display.root, "_NET_ACTIVE_WINDOW", XCB_ATOM_WINDOW);
+  DesktopSnapshot result;
+  for (const auto window : windows) {
+    const auto pid = display.Property(window, "_NET_WM_PID", XCB_ATOM_CARDINAL);
+    json data{{"role", "window"}, {"name", display.Title(window)},
+      {"pid", pid.empty() ? 0 : pid.front()}, {"foreground", !active.empty() && active.front() == window}};
+    result.items.push_back({data, [window, pid](const std::string& operation, const std::string&) -> json {
+      if (operation != "activate" && operation != "focus") throw std::runtime_error("UNSUPPORTED_OPERATION");
+      XConnection display;
+      const auto windows = display.Property(display.root, "_NET_CLIENT_LIST_STACKING", XCB_ATOM_WINDOW);
+      if (std::find(windows.begin(), windows.end(), window) == windows.end() ||
+          display.Property(window, "_NET_WM_PID", XCB_ATOM_CARDINAL) != pid) throw std::runtime_error("STALE_ELEMENT");
+      xcb_client_message_event_t event{};
+      event.response_type = XCB_CLIENT_MESSAGE;
+      event.window = window;
+      event.type = display.Atom("_NET_ACTIVE_WINDOW");
+      event.format = 32;
+      event.data.data32[0] = 2;
+      event.data.data32[1] = XCB_CURRENT_TIME;
+      const auto cookie = xcb_send_event_checked(display.connection.get(), 0, display.root,
+          XCB_EVENT_MASK_SUBSTRUCTURE_REDIRECT | XCB_EVENT_MASK_SUBSTRUCTURE_NOTIFY, reinterpret_cast<const char*>(&event));
+      const auto error = xcb_request_check(display.connection.get(), cookie);
+      const bool accepted = !error;
+      std::free(error);
+      return {{"accepted", accepted}, {"state", "requested"}};
+    }, {}});
+    if (result.items.size() >= 128) { result.truncated = true; break; }
+  }
+  return result;
+}
+DesktopSnapshot DesktopTree(int pid, int limit, int depth) {
+#ifdef CROSSDESK_HAS_ATSPI
+  auto desktop = Desktop();
+  if (!pid) return Tree(desktop, limit, depth);
+  const int count = std::min(256, atspi_accessible_get_child_count(desktop.get(), nullptr));
+  const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(1200);
+  for (int index = 0; index < count; ++index) {
+    if (std::chrono::steady_clock::now() >= deadline) throw std::runtime_error("UI_QUERY_TIMEOUT");
+    auto app = Own(atspi_accessible_get_child_at_index(desktop.get(), index, nullptr));
+    if (app && atspi_accessible_get_process_id(app.get(), nullptr) == unsigned(pid)) return Tree(app, limit, depth);
+  }
+  throw std::runtime_error("APPLICATION_UNAVAILABLE");
+#else
+  (void)pid; (void)limit; (void)depth;
+  throw std::runtime_error("ATSPI_NOT_BUILT");
+#endif
+}
+nlohmann::json DesktopLaunch(const std::string& application) {
+  if (application.empty() || application.front() != '/' || access(application.c_str(), X_OK) != 0)
+    throw std::invalid_argument("APPLICATION_UNAVAILABLE");
+  return Spawn(application);
+}
+nlohmann::json DesktopOpenUrl(const std::string& url) { return Spawn("/usr/bin/xdg-open", url); }
+} // namespace crossdesk

apps/desktop/src/platform/macos/desktop_automation.mm

@@ -0,0 +1,168 @@
+#include "desktop_automation.h"
+
+#import <AppKit/AppKit.h>
+#include <ApplicationServices/ApplicationServices.h>
+#include <deque>
+#include <memory>
+#include <stdexcept>
+
+namespace crossdesk {
+namespace {
+using nlohmann::json;
+using Element = std::shared_ptr<const __AXUIElement>;
+Element Own(AXUIElementRef element) { return Element(element, [](auto value) { if (value) CFRelease(value); }); }
+CFTypeRef Attribute(AXUIElementRef element, CFStringRef name) {
+  CFTypeRef value = nullptr;
+  AXUIElementCopyAttributeValue(element, name, &value);
+  return value;
+}
+std::string String(CFTypeRef value) {
+  if (!value || CFGetTypeID(value) != CFStringGetTypeID()) return {};
+  char bytes[2048]{};
+  CFIndex used = 0;
+  const auto count = CFStringGetBytes(static_cast<CFStringRef>(value),
+      CFRangeMake(0, std::min<CFIndex>(CFStringGetLength(static_cast<CFStringRef>(value)), 256)),
+      kCFStringEncodingUTF8, 0, false, reinterpret_cast<UInt8*>(bytes), sizeof(bytes) - 1, &used);
+  (void)count;
+  return DesktopText(std::string(bytes, size_t(used)));
+}
+std::string Text(AXUIElementRef element, CFStringRef name) {
+  const auto value = Attribute(element, name);
+  auto result = String(value);
+  if (value) CFRelease(value);
+  return result;
+}
+void Accessible() { if (!AXIsProcessTrusted()) throw std::runtime_error("ACCESSIBILITY_PERMISSION_REQUIRED"); }
+json Perform(Element element, const std::string& operation, const std::string& value) {
+  Accessible();
+  if (Text(element.get(), kAXSubroleAttribute) == "AXSecureTextField") throw std::runtime_error("SECURE_ELEMENT");
+  AXError error = kAXErrorActionUnsupported;
+  if (operation == "press") error = AXUIElementPerformAction(element.get(), kAXPressAction);
+  else if (operation == "activate") {
+    pid_t pid = 0;
+    AXUIElementGetPid(element.get(), &pid);
+    [[NSRunningApplication runningApplicationWithProcessIdentifier:pid] activateWithOptions:0];
+    error = AXUIElementPerformAction(element.get(), kAXRaiseAction);
+  } else if (operation == "focus") error = AXUIElementSetAttributeValue(element.get(), kAXFocusedAttribute, kCFBooleanTrue);
+  else if (operation == "set_value") {
+    const auto text = CFStringCreateWithBytes(nullptr, reinterpret_cast<const UInt8*>(value.data()), value.size(), kCFStringEncodingUTF8, false);
+    if (!text) throw std::invalid_argument("INVALID_ARGUMENT");
+    error = AXUIElementSetAttributeValue(element.get(), kAXValueAttribute, text);
+    CFRelease(text);
+  }
+  if (error != kAXErrorSuccess) throw std::runtime_error(error == kAXErrorInvalidUIElement ? "STALE_ELEMENT" : "UI_ACTION_FAILED");
+  return {{"accepted", true}};
+}
+DesktopSnapshot Tree(Element root, int limit, int depth) {
+  Accessible();
+  struct Entry { Element element; int depth, parent; };
+  std::deque<Entry> queue{{root, 0, -1}};
+  DesktopSnapshot result;
+  const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(1200);
+  while (!queue.empty() && int(result.items.size()) < limit && std::chrono::steady_clock::now() < deadline) {
+    auto entry = std::move(queue.front()); queue.pop_front();
+    const auto element = entry.element.get();
+    AXUIElementSetMessagingTimeout(element, .1f);
+    const auto role = Text(element, kAXRoleAttribute);
+    if (role.empty()) continue;
+    const bool secure = Text(element, kAXSubroleAttribute) == "AXSecureTextField";
+    json data{{"role", role}, {"name", Text(element, kAXTitleAttribute)}, {"parent", entry.parent}, {"secure", secure}};
+    if (!secure) data["value"] = Text(element, kAXValueAttribute);
+    CGPoint point{}; CGSize size{};
+    auto position = Attribute(element, kAXPositionAttribute), extent = Attribute(element, kAXSizeAttribute);
+    if (position && extent && CFGetTypeID(position) == AXValueGetTypeID() && CFGetTypeID(extent) == AXValueGetTypeID() &&
+        AXValueGetValue(static_cast<AXValueRef>(position), kAXValueTypeCGPoint, &point) &&
+        AXValueGetValue(static_cast<AXValueRef>(extent), kAXValueTypeCGSize, &size))
+      data["bounds"] = {{"x", point.x}, {"y", point.y}, {"width", size.width}, {"height", size.height}};
+    if (position) CFRelease(position);
+    if (extent) CFRelease(extent);
+    const int index = int(result.items.size());
+    result.items.push_back({data,
+      [element = entry.element](const auto& operation, const auto& value) { return Perform(element, operation, value); },
+      [element = entry.element](int max, int levels) { return Tree(element, max, levels); }});
+    CFIndex count = 0;
+    if (AXUIElementGetAttributeValueCount(element, kAXChildrenAttribute, &count) != kAXErrorSuccess || count <= 0) continue;
+    if (entry.depth >= depth) { result.truncated = true; continue; }
+    const auto available = std::max(0, limit - int(result.items.size()) - int(queue.size()));
+    if (count > available) result.truncated = true;
+    CFArrayRef children = nullptr;
+    if (available && AXUIElementCopyAttributeValues(element, kAXChildrenAttribute, 0, std::min<CFIndex>(count, available), &children) == kAXErrorSuccess) {
+      for (CFIndex index = 0; index < CFArrayGetCount(children); ++index) {
+        const auto child = static_cast<AXUIElementRef>(CFArrayGetValueAtIndex(children, index));
+        if (CFGetTypeID(child) == AXUIElementGetTypeID()) {
+          CFRetain(child); queue.push_back({Own(child), entry.depth + 1, int(result.items.size()) - 1});
+        }
+      }
+      CFRelease(children);
+    }
+  }
+  result.truncated |= !queue.empty();
+  return result;
+}
+} // namespace
+nlohmann::json DesktopCapabilities() {
+  return {{"available", true}, {"backend", "macos-ax"}, {"accessibility_permission", bool(AXIsProcessTrusted())},
+    {"windows", bool(AXIsProcessTrusted())}, {"tree", bool(AXIsProcessTrusted())}, {"launch", true}, {"open_url", true}};
+}
+DesktopSnapshot DesktopTree(int pid, int limit, int depth) {
+  @autoreleasepool {
+    if (!pid) pid = NSWorkspace.sharedWorkspace.frontmostApplication.processIdentifier;
+    if (pid <= 0) throw std::runtime_error("APPLICATION_UNAVAILABLE");
+    return Tree(Own(AXUIElementCreateApplication(pid)), limit, depth);
+  }
+}
+DesktopSnapshot DesktopWindows() {
+  @autoreleasepool {
+    DesktopSnapshot result;
+    const auto front = NSWorkspace.sharedWorkspace.frontmostApplication.processIdentifier;
+    // AX window references provide exact activation targets when permission is present.
+    const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(1200);
+    Accessible();
+    for (NSRunningApplication* app in NSWorkspace.sharedWorkspace.runningApplications) {
+      if (std::chrono::steady_clock::now() >= deadline) { result.truncated = true; break; }
+      if (app.activationPolicy != NSApplicationActivationPolicyRegular) continue;
+      auto application = Own(AXUIElementCreateApplication(app.processIdentifier));
+      AXUIElementSetMessagingTimeout(application.get(), .1f);
+      const auto raw = Attribute(application.get(), kAXWindowsAttribute);
+      if (raw && CFGetTypeID(raw) == CFArrayGetTypeID()) {
+        const auto windows = static_cast<CFArrayRef>(raw);
+        for (CFIndex index = 0; index < CFArrayGetCount(windows) && result.items.size() < 128; ++index) {
+          if (std::chrono::steady_clock::now() >= deadline) { result.truncated = true; break; }
+          const auto window = static_cast<AXUIElementRef>(CFArrayGetValueAtIndex(windows, index));
+          if (CFGetTypeID(window) != AXUIElementGetTypeID()) continue;
+          CFRetain(window); const auto element = Own(window);
+          const auto main = Attribute(window, kAXMainAttribute);
+          const bool foreground = app.processIdentifier == front && main == kCFBooleanTrue;
+          if (main) CFRelease(main);
+          result.items.push_back({{{"role", "window"}, {"name", Text(window, kAXTitleAttribute)},
+              {"application", DesktopText(app.localizedName.UTF8String ?: "")}, {"pid", app.processIdentifier}, {"foreground", foreground}, {"foreground_application", app.processIdentifier == front}},
+            [element](const auto& operation, const auto& value) { return Perform(element, operation, value); },
+            [element](int max, int levels) { return Tree(element, max, levels); }});
+        }
+      }
+      if (raw) CFRelease(raw);
+      if (result.items.size() >= 128) { result.truncated = true; break; }
+    }
+    if (!AXIsProcessTrusted()) throw std::runtime_error("ACCESSIBILITY_PERMISSION_REQUIRED");
+    return result;
+  }
+}
+nlohmann::json DesktopLaunch(const std::string& application) {
+  @autoreleasepool {
+    NSString* value = [NSString stringWithUTF8String:application.c_str()];
+    NSURL* url = [value hasPrefix:@"/"] ? [NSURL fileURLWithPath:value] :
+        [NSWorkspace.sharedWorkspace URLForApplicationWithBundleIdentifier:value];
+    if (!url || ![url.path.pathExtension.lowercaseString isEqualToString:@"app"])
+      throw std::runtime_error("APPLICATION_UNAVAILABLE");
+    [NSWorkspace.sharedWorkspace openApplicationAtURL:url configuration:NSWorkspaceOpenConfiguration.configuration completionHandler:nil];
+    return {{"accepted", true}, {"state", "requested"}};
+  }
+}
+nlohmann::json DesktopOpenUrl(const std::string& value) {
+  @autoreleasepool {
+    NSURL* url = [NSURL URLWithString:[NSString stringWithUTF8String:value.c_str()]];
+    if (!url || ![NSWorkspace.sharedWorkspace openURL:url]) throw std::runtime_error("UI_ACTION_FAILED");
+    return {{"accepted", true}};
+  }
+}
+} // namespace crossdesk

apps/desktop/src/platform/windows/desktop_automation.cpp

@@ -0,0 +1,190 @@
+#include "desktop_automation.h"
+
+#include <Windows.h>
+#include <UIAutomation.h>
+#include <shellapi.h>
+#include <wrl/client.h>
+#include <algorithm>
+#include <deque>
+#include <stdexcept>
+
+namespace crossdesk {
+namespace {
+using nlohmann::json;
+using Microsoft::WRL::ComPtr;
+std::string Utf8(BSTR value) {
+  if (!value) return {};
+  const int length = std::min<int>(int(SysStringLen(value)), 256);
+  const int size = WideCharToMultiByte(CP_UTF8, 0, value, length, nullptr, 0, nullptr, nullptr);
+  std::string text(size > 0 ? size : 0, '\0');
+  if (size) WideCharToMultiByte(CP_UTF8, 0, value, length, text.data(), size, nullptr, nullptr);
+  SysFreeString(value);
+  return DesktopText(std::move(text));
+}
+std::wstring Wide(const std::string& value) {
+  const int size = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, value.data(), int(value.size()), nullptr, 0);
+  if (!size && !value.empty()) throw std::invalid_argument("INVALID_ARGUMENT");
+  std::wstring text(size, L'\0');
+  if (size) MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, value.data(), int(value.size()), text.data(), size);
+  return text;
+}
+ComPtr<IUIAutomation> Automation() {
+  struct Apartment {
+    HRESULT result = CoInitializeEx(nullptr, COINIT_APARTMENTTHREADED);
+    ~Apartment() { if (SUCCEEDED(result)) CoUninitialize(); }
+  };
+  static thread_local Apartment apartment;
+  ComPtr<IUIAutomation> automation;
+  if (FAILED(CoCreateInstance(__uuidof(CUIAutomation8), nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&automation))) &&
+      FAILED(CoCreateInstance(__uuidof(CUIAutomation), nullptr, CLSCTX_INPROC_SERVER, IID_PPV_ARGS(&automation))))
+    throw std::runtime_error("UI_AUTOMATION_UNAVAILABLE");
+  ComPtr<IUIAutomation2> timed;
+  if (SUCCEEDED(automation.As(&timed))) {
+    timed->put_ConnectionTimeout(200);
+    timed->put_TransactionTimeout(200);
+  }
+  return automation;
+}
+json Perform(ComPtr<IUIAutomationElement> element, const std::string& operation, const std::string& value) {
+  BOOL secure = TRUE;
+  if (FAILED(element->get_CurrentIsPassword(&secure))) throw std::runtime_error("STALE_ELEMENT");
+  if (secure) throw std::runtime_error("SECURE_ELEMENT");
+  HRESULT result = E_NOTIMPL;
+  if (operation == "focus" || operation == "activate") result = element->SetFocus();
+  else if (operation == "press") {
+    ComPtr<IUIAutomationInvokePattern> invoke;
+    if (SUCCEEDED(element->GetCurrentPatternAs(UIA_InvokePatternId, IID_PPV_ARGS(&invoke)))) result = invoke->Invoke();
+    else {
+      ComPtr<IUIAutomationSelectionItemPattern> select;
+      if (SUCCEEDED(element->GetCurrentPatternAs(UIA_SelectionItemPatternId, IID_PPV_ARGS(&select)))) result = select->Select();
+      else {
+        ComPtr<IUIAutomationTogglePattern> toggle;
+        if (SUCCEEDED(element->GetCurrentPatternAs(UIA_TogglePatternId, IID_PPV_ARGS(&toggle)))) result = toggle->Toggle();
+      }
+    }
+  } else if (operation == "set_value") {
+    ComPtr<IUIAutomationValuePattern> pattern;
+    if (SUCCEEDED(element->GetCurrentPatternAs(UIA_ValuePatternId, IID_PPV_ARGS(&pattern)))) result = pattern->SetValue(Wide(value).c_str());
+  }
+  if (FAILED(result)) throw std::runtime_error(result == UIA_E_ELEMENTNOTAVAILABLE ? "STALE_ELEMENT" : "UI_ACTION_FAILED");
+  return {{"accepted", true}};
+}
+DesktopSnapshot Tree(ComPtr<IUIAutomationElement> root, int limit, int depth) {
+  auto automation = Automation();
+  ComPtr<IUIAutomationTreeWalker> walker;
+  if (FAILED(automation->get_ControlViewWalker(&walker))) throw std::runtime_error("UI_AUTOMATION_UNAVAILABLE");
+  struct Entry { ComPtr<IUIAutomationElement> element; int depth, parent; };
+  std::deque<Entry> queue{{root, 0, -1}};
+  DesktopSnapshot result;
+  const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(1200);
+  while (!queue.empty() && int(result.items.size()) < limit && std::chrono::steady_clock::now() < deadline) {
+    auto entry = std::move(queue.front()); queue.pop_front();
+    auto element = entry.element;
+    BOOL secure = TRUE, enabled = FALSE;
+    if (FAILED(element->get_CurrentIsPassword(&secure))) continue;
+    element->get_CurrentIsEnabled(&enabled);
+    BSTR name = nullptr, role = nullptr;
+    element->get_CurrentName(&name);
+    element->get_CurrentLocalizedControlType(&role);
+    RECT bounds{};
+    element->get_CurrentBoundingRectangle(&bounds);
+    if (secure) { SysFreeString(name); name = nullptr; }
+    json data{{"name", Utf8(name)}, {"role", Utf8(role)},
+      {"secure", bool(secure)}, {"enabled", bool(enabled)}, {"parent", entry.parent},
+      {"bounds", {{"x", bounds.left}, {"y", bounds.top}, {"width", bounds.right - bounds.left}, {"height", bounds.bottom - bounds.top}}}};
+    if (!secure) {
+      ComPtr<IUIAutomationValuePattern> pattern;
+      if (SUCCEEDED(element->GetCurrentPatternAs(UIA_ValuePatternId, IID_PPV_ARGS(&pattern)))) {
+        BSTR value = nullptr;
+        if (SUCCEEDED(pattern->get_CurrentValue(&value))) data["value"] = Utf8(value);
+      }
+    }
+    const int parent = int(result.items.size());
+    result.items.push_back({data,
+      [element](const auto& operation, const auto& value) { return Perform(element, operation, value); },
+      [element](int max, int levels) { return Tree(element, max, levels); }});
+    ComPtr<IUIAutomationElement> child;
+    walker->GetFirstChildElement(element.Get(), &child);
+    if (entry.depth >= depth) { result.truncated |= bool(child); continue; }
+    while (child && int(queue.size() + result.items.size()) < limit && std::chrono::steady_clock::now() < deadline) {
+      queue.push_back({child, entry.depth + 1, parent});
+      ComPtr<IUIAutomationElement> next;
+      walker->GetNextSiblingElement(child.Get(), &next);
+      child = std::move(next);
+    }
+    result.truncated |= bool(child);
+  }
+  result.truncated |= !queue.empty();
+  return result;
+}
+DesktopSnapshot WindowTree(HWND window, int limit, int depth) {
+  ComPtr<IUIAutomationElement> root;
+  if (!IsWindow(window) || FAILED(Automation()->ElementFromHandle(window, &root)) || !root)
+    throw std::runtime_error("STALE_ELEMENT");
+  return Tree(root, limit, depth);
+}
+} // namespace
+nlohmann::json DesktopCapabilities() {
+  return {{"available", true}, {"backend", "windows-uia"}, {"windows", true}, {"tree", true}, {"launch", true}, {"open_url", true}};
+}
+DesktopSnapshot DesktopWindows() {
+  DesktopSnapshot result;
+  EnumWindows([](HWND window, LPARAM context) -> BOOL {
+    auto& result = *reinterpret_cast<DesktopSnapshot*>(context);
+    if (!IsWindowVisible(window) || GetWindow(window, GW_OWNER)) return TRUE;
+    wchar_t title[257]{};
+    GetWindowTextW(window, title, 257);
+    DWORD pid = 0;
+    GetWindowThreadProcessId(window, &pid);
+    const auto valid = [window, pid] {
+      DWORD current = 0;
+      GetWindowThreadProcessId(window, &current);
+      if (!IsWindow(window) || current != pid) throw std::runtime_error("STALE_ELEMENT");
+    };
+    RECT bounds{}; GetWindowRect(window, &bounds);
+    result.items.push_back({{{"name", Utf8(SysAllocString(title))}, {"role", "window"}, {"pid", pid},
+        {"foreground", window == GetForegroundWindow()},
+        {"bounds", {{"x", bounds.left}, {"y", bounds.top}, {"width", bounds.right - bounds.left}, {"height", bounds.bottom - bounds.top}}}},
+      [window, valid](const std::string& operation, const std::string&) -> json {
+        valid();
+        if (operation != "activate" && operation != "focus") throw std::runtime_error("UNSUPPORTED_OPERATION");
+        if (IsIconic(window)) ShowWindowAsync(window, SW_RESTORE);
+        if (!SetForegroundWindow(window)) throw std::runtime_error("UI_ACTION_FAILED");
+        return {{"accepted", true}};
+      }, [window, valid](int limit, int depth) { valid(); return WindowTree(window, limit, depth); }});
+    result.truncated = result.items.size() >= 128;
+    return result.truncated ? FALSE : TRUE;
+  }, reinterpret_cast<LPARAM>(&result));
+  return result;
+}
+DesktopSnapshot DesktopTree(int pid, int limit, int depth) {
+  HWND window = GetForegroundWindow();
+  if (pid) {
+    struct Search { DWORD pid; HWND window = nullptr; } search{DWORD(pid)};
+    EnumWindows([](HWND candidate, LPARAM context) -> BOOL {
+      auto& search = *reinterpret_cast<Search*>(context);
+      DWORD current = 0; GetWindowThreadProcessId(candidate, &current);
+      if (current == search.pid && IsWindowVisible(candidate) && !GetWindow(candidate, GW_OWNER)) {
+        search.window = candidate; return FALSE;
+      }
+      return TRUE;
+    }, reinterpret_cast<LPARAM>(&search));
+    window = search.window;
+  }
+  return WindowTree(window, limit, depth);
+}
+nlohmann::json DesktopLaunch(const std::string& application) {
+  const auto path = Wide(application);
+  if (path.size() < 7 || path[1] != L':' || (path[2] != L'\\' && path[2] != L'/') ||
+      _wcsicmp(path.c_str() + path.size() - 4, L".exe") != 0 || GetFileAttributesW(path.c_str()) == INVALID_FILE_ATTRIBUTES)
+    throw std::invalid_argument("APPLICATION_UNAVAILABLE");
+  if (reinterpret_cast<INT_PTR>(ShellExecuteW(nullptr, L"open", path.c_str(), nullptr, nullptr, SW_SHOWNORMAL)) <= 32)
+    throw std::runtime_error("UI_ACTION_FAILED");
+  return {{"accepted", true}};
+}
+nlohmann::json DesktopOpenUrl(const std::string& url) {
+  if (reinterpret_cast<INT_PTR>(ShellExecuteW(nullptr, L"open", Wide(url).c_str(), nullptr, nullptr, SW_SHOWNORMAL)) <= 32)
+    throw std::runtime_error("UI_ACTION_FAILED");
+  return {{"accepted", true}};
+}
+} // namespace crossdesk