[feat] acknowledge remote input and wait for visual changes

apps/cli/agent.cpp

@@ -69,6 +69,15 @@ struct Observation {
   int width, height, source_width, source_height;
   ImageRegion region;
   Clock::time_point created;
+  uint64_t signature;
+};
+struct FrameWait {
+  json id;
+  Observation observation;
+  bool stable;
+  int stable_ms;
+  uint64_t signature, frame;
+  Clock::time_point deadline, changed;
 };
 struct Capture {
   json id, request;
@@ -96,7 +105,7 @@ struct RunningAction {
   std::vector<Step> steps;
   size_t next = 0;
   bool submitted = false;
-  Clock::time_point started;
+  Clock::time_point started, awaiting_ack{};
 };
 struct Record {
   std::string operation, arguments;
@@ -260,11 +269,19 @@ private:
     if (!action_)
       return;
     session_.ReleaseInput();
+    const auto remote = session_.InputResult();
+    if (!error && remote == "pending") {
+      if (action_->awaiting_ack == Clock::time_point{}) action_->awaiting_ack = Clock::now();
+      if (Age(action_->awaiting_ack) < 2000) return;
+      error = "REMOTE_ACK_TIMEOUT";
+    }
+    if (!error && remote == "failed") error = "REMOTE_INPUT_FAILED";
     json result = {
         {"status", error ? (action_->submitted ? "outcome_unknown"
                                                : "failed_before_submit")
                          : "submitted"},
         {"operation_id", action_->operation},
+        {"remote_execution", remote},
         {"generation", action_->generation},
         {"after_frame_id", frame_ ? frame_->frame_id : 0},
         {"side_effect_possible", action_->submitted},
@@ -295,6 +312,8 @@ private:
       terminal_encoding_->cancelled = true;
     }
     observations_.clear();
+    for (const auto &wait : frame_waits_) Error(wait.id, code);
+    frame_waits_.clear();
     preview_ = json();
   }
   void Tick() {
@@ -348,6 +367,30 @@ private:
       if (action_ && action_->next == action_->steps.size())
         FinishAction();
     }
+    for (auto it = frame_waits_.begin(); it != frame_waits_.end();) {
+      if (!session_.Connected() || !frame_ ||
+          it->observation.generation != session_.Generation() || it->observation.epoch != epoch_) {
+        Error(it->id, "SESSION_CHANGED");
+        it = frame_waits_.erase(it);
+        continue;
+      }
+      bool matched = false;
+      if (frame_->frame_id != it->frame && Age(frame_->queued) <= 2000) {
+        it->frame = frame_->frame_id;
+        const auto signature = FrameSignature(*frame_, it->observation.region);
+        if (signature != it->signature) {
+          it->signature = signature;
+          it->changed = Clock::now();
+        }
+        matched = it->stable ? Age(it->changed) >= it->stable_ms
+                             : signature != it->observation.signature;
+      }
+      if (matched || Clock::now() >= it->deadline) {
+        Reply(it->id, {{"matched", matched}, {"wait_timed_out", !matched},
+            {"generation", session_.Generation()}, {"frame_id", frame_->frame_id}});
+        it = frame_waits_.erase(it);
+      } else ++it;
+    }
     if (terminal_encoding_ &&
         terminal_encoding_->result.wait_for(Ms(0)) == std::future_status::ready) {
       auto job = std::move(terminal_encoding_);
@@ -418,7 +461,8 @@ private:
                 observations_.pop_front();
               observations_.push_back(
                   {id, f.generation, job->epoch, image.width, image.height,
-                   int(f.width), int(f.height), image.region, Clock::now()});
+                   int(f.width), int(f.height), image.region, Clock::now(),
+                   FrameSignature(f, image.region)});
             }
             Reply(job->capture.id, std::move(result));
           }
@@ -596,6 +640,7 @@ private:
          arguments,
          {{"status", "in_progress"}, {"operation_id", operation}},
          Clock::now()});
+    session_.BeginInput();
     running->started = Clock::now();
     action_ = std::move(running);
   }
@@ -688,6 +733,18 @@ private:
       } else if (method == "action") {
         StartAction(j, id);
         return false;
+      } else if (method == "wait_frame") {
+        Fields(j, {"id", "method", "observation_id", "condition", "stable_ms", "timeout_ms"});
+        const auto &observation = Observe(j);
+        const auto condition = String(j, "condition", 16);
+        if (condition != "changed" && condition != "stable")
+          throw std::invalid_argument("INVALID_ARGUMENT");
+        if (frame_waits_.size() >= 16) throw std::invalid_argument("BUSY");
+        const auto now = Clock::now();
+        frame_waits_.push_back({id, observation, condition == "stable",
+            Number(j, "stable_ms", 100, 5000, 500), observation.signature, 0,
+            now + Ms(Number(j, "timeout_ms", 0, 25000, 5000)), now});
+        return false;
       } else if (method == "unlock_prepare") {
         Fields(j, {"id", "method", "observation_id"});
         const auto &observation = InputObservation(j);
@@ -766,6 +823,12 @@ private:
         if (!j.contains("request_id") || !j["request_id"].is_number_integer())
           throw std::invalid_argument("INVALID_REQUEST");
         const auto target = j["request_id"];
+        for (auto it = frame_waits_.begin(); it != frame_waits_.end();) {
+          if (it->id == target) {
+            Error(it->id, "CANCELLED");
+            it = frame_waits_.erase(it);
+          } else ++it;
+        }
         if (action_ && action_->id == target)
           FinishAction("CANCELLED");
         if (encoding_ && !encoding_->cancelled &&
@@ -832,6 +895,7 @@ private:
   std::shared_ptr<DecodedFrame> frame_;
   std::deque<Observation> observations_;
   std::deque<Capture> captures_;
+  std::deque<FrameWait> frame_waits_;
   std::unique_ptr<Encoding> encoding_;
   std::unique_ptr<TerminalEncoding> terminal_encoding_;
   std::unique_ptr<RunningAction> action_;

apps/cli/docs/automation-protocol.md

@@ -135,3 +135,9 @@ After the Agent identifies the system lock screen and focuses its password field
 `unlock` accepts a password of 1–256 UTF-8 bytes, rejecting NUL, newline, carriage return and tab. It uses the existing input scheduler to select the focused field contents, type the password and press Enter. Selection uses Command+A on macOS and Ctrl+A on other platforms. Control permission, human pause, observation freshness and cancellation follow the action rules. The operation cache stores the observation and unlock operation kind, with secret text confined to the transient input request and scheduled text step. Completed or cancelled steps clear their text buffers.
 
 The result follows the existing action result format. Repeated operation IDs retrieve the first result, and the caller verifies the visible desktop using a subsequent screenshot. Password persistence and the five-minute user-input request lifetime belong to the TypeScript adapter.
+
+## Input receipts and visual conditions
+
+Updated desktop hosts answer the versioned `automation` capability handshake. When `capabilities.input_ack` is true, input travels through the host input dispatcher and the action result includes `remote_execution` with `applied`, `failed` or `pending`. A receipt confirms native input processing. The caller checks the target application's resulting state separately. Legacy hosts return `remote_execution` as `unsupported`. Missing receipts terminate the bounded wait with `REMOTE_ACK_TIMEOUT` and an uncertain outcome.
+
+`wait_frame` accepts an `observation_id`, `condition` (`changed` or `stable`), `stable_ms` (100–5000, default 500), and `timeout_ms` (0–25000, default 5000). It watches the observation's source region, checks both luma and chroma, and returns `matched`, `wait_timed_out`, `generation`, and `frame_id`. Display changes, disconnection and explicit cancellation terminate pending waits. `crossdesk_wait` exposes this operation through MCP and includes a fresh screenshot when its condition matches.

apps/desktop/src/gui/features/input/remote_input_dispatcher.cpp

@@ -39,14 +39,14 @@ void RemoteInputDispatcher::Connect(const std::string& id) {
 }
 
 RemoteInputDispatcher::Result RemoteInputDispatcher::Push(
-    const std::string& id, const RemoteAction& action) {
+    const std::string& id, const RemoteAction& action, Completion completion) {
   // HostInfo owns raw arrays; it must never enter this value-copy queue.
   if (!Accepts(action)) return Result::rejected;
   std::lock_guard lock(mutex_);
   if (stopping_ || !connected_.count(id)) return Result::rejected;
-  if (IsMove(action) && !pending_.empty()) {
+  if (!completion && IsMove(action) && !pending_.empty()) {
     auto& last = pending_.back();
-    if (last.id == id && last.IsInput() && IsMove(*last.action)) {
+    if (!last.completion && last.id == id && last.IsInput() && IsMove(*last.action)) {
       last.action = action;
       last.queued_at = Clock::now();
       return Result::coalesced;
@@ -56,7 +56,7 @@ RemoteInputDispatcher::Result RemoteInputDispatcher::Push(
     QueueReleaseLocked(id);
     return Result::overflow;
   }
-  pending_.push_back({id, action, Clock::now(), ItemKind::input});
+  pending_.push_back({id, action, Clock::now(), ItemKind::input, 0, std::move(completion)});
   ++pending_counts_[id];
   ready_.notify_one();
   return Result::queued;
@@ -328,6 +328,9 @@ void RemoteInputDispatcher::Run() {
         !pending_counts_.count(item.id))
       logs_.erase(item.id);
     lock.unlock();
+    if (item.completion) {
+      try { item.completion(outcome); } catch (...) {}
+    }
     if (report) {
       try {
         report_(item.id, queued_ms, execution_ms, outcome);

apps/desktop/src/gui/features/input/remote_input_dispatcher.h

@@ -38,6 +38,7 @@ class RemoteInputDispatcher {
   // Optional targeted adapter; legacy callers fall back to full Release.
   using ReleaseMouse =
       std::function<InputOutcome(const std::string&, unsigned)>;
+  using Completion = std::function<void(InputOutcome)>;
   enum class Result { queued, coalesced, rejected, overflow };
   static constexpr size_t kMaxPendingPerController = 128;
 
@@ -47,7 +48,8 @@ class RemoteInputDispatcher {
   ~RemoteInputDispatcher();
   static bool Accepts(const RemoteAction& action);
   void Connect(const std::string& id);
-  Result Push(const std::string& id, const RemoteAction& action);
+  Result Push(const std::string& id, const RemoteAction& action,
+              Completion completion = {});
   void Disconnect(const std::string& id);
 
   // Scheduled recovery is not idle work. Check HasPendingReleases() as well,
@@ -68,6 +70,7 @@ class RemoteInputDispatcher {
     // Zero requests full controller cleanup; otherwise only these mouse
     // buttons.
     unsigned mouse_mask = 0;
+    Completion completion;
     bool IsReset() const { return kind == ItemKind::reset_display; }
     bool IsInput() const { return kind == ItemKind::input; }
     bool IsCleanup() const { return kind == ItemKind::cleanup; }
@@ -120,4 +123,4 @@ class RemoteInputDispatcher {
 
 }  // namespace crossdesk
 
-#endif
+#endif

apps/desktop/src/gui/runtime/automation_runtime.cpp

@@ -0,0 +1,47 @@
+#include "runtime/gui_runtime.h"
+#include <stream_names.h>
+
+namespace crossdesk {
+void GuiRuntime::HandleAutomation(const std::string& remote_id,
+                                  const nlohmann::json& request) {
+  using nlohmann::json;
+  if (request.value("version", json{}) != 1 || !request.contains("id") ||
+      !request["id"].is_number_unsigned() || request["id"].get<uint64_t>() == 0)
+    return;
+  const auto source = ControllerEvents(remote_id);
+  if (!source || !source->DataAllowed()) return;
+  const auto id = request["id"].get<uint64_t>();
+  const auto reply = [this, remote_id, id, source](bool ok, json result) {
+    const auto current = ControllerEvents(remote_id);
+    if (current != source || !current->DataAllowed()) return;
+    const auto message = json{{"type", "automation_result"}, {"version", 1},
+        {"id", id}, {"ok", ok}, {ok ? "result" : "error", std::move(result)}}.dump();
+    SendControllerData(remote_id, message.data(), message.size(), kControlStream);
+  };
+  try {
+    const auto method = request.value("method", std::string{});
+    if (method == "capabilities") {
+      reply(true, {{"input_ack", true}});
+    } else if (method == "input") {
+      RemoteAction action{};
+      if (!request.contains("arguments") ||
+          !action.from_json(request["arguments"].dump())) {
+        reply(false, "INVALID_ARGUMENT");
+        return;
+      }
+      const auto result = remote_input_.Push(remote_id, action,
+          [reply](InputOutcome outcome) {
+            reply(true, {{"state", InputOutcomeName(outcome)}});
+          });
+      FreeRemoteAction(action);
+      if (result == RemoteInputDispatcher::Result::rejected ||
+          result == RemoteInputDispatcher::Result::overflow)
+        reply(false, "INPUT_REJECTED");
+    } else {
+      reply(false, "UNSUPPORTED_OPERATION");
+    }
+  } catch (...) {
+    reply(false, "INVALID_ARGUMENT");
+  }
+}
+} // namespace crossdesk

apps/desktop/src/gui/runtime/gui_runtime.h

@@ -152,6 +152,7 @@ class GuiRuntime : protected gui_detail::GuiState {
   // or every connected session while that target leads DeskHub group control.
   void SendRemoteInput(const RemoteAction& action);
   InputOutcome ApplyRemoteInput(const std::string& remote_id, const RemoteAction& action);
+  void HandleAutomation(const std::string& remote_id, const nlohmann::json& request);
   InputOutcome InjectRemoteMouse(const RemoteAction& action, int display);
   std::string gallery_group_control_id_;
 

apps/desktop/src/gui/runtime/peer_data_callbacks.cpp

@@ -179,6 +179,14 @@ void PeerEventHandler::OnReceiveDataBuffer(
   PeerPtr* bound_peer =
       handler->IsDirect() ? handler->DirectPeer() : runtime->peer_;
   std::string json_str(data, size);
+  if (json_str.find("\"automation\"") != std::string::npos) {
+    const auto request = nlohmann::json::parse(json_str, nullptr, false);
+    if (request.is_object() && request.value("type", nlohmann::json{}) == "automation") {
+      if (source_id == runtime->control_data_label_ && runtime->IsAuthorizedController(remote_id))
+        runtime->HandleAutomation(remote_id, request);
+      return;
+    }
+  }
   const auto preview_json =
       json_str.find("\"deskhub_preview") != std::string::npos
           ? nlohmann::json::parse(json_str, nullptr, false)

apps/mcp/README.md

@@ -244,3 +244,9 @@ npm run pack:npm -- --out-dir /absolute/path/artifacts
 本机 macOS arm64 完成 TypeScript/面板构建、原生 CLI 构建、现有 wire/keyboard-state 测试，以及工程外的 MCP、会话和原生动作回归检查。ASan/UBSan 检查覆盖动作调度和输入清理；140 组随机图像覆盖完整帧、奇偶裁剪边界、缩放及容量回退，优化前后 PNG 字节一致。
 
 使用 1920 × 1080 合成 NV12、最长边 1280、每项 6 次的 `-O2` 本地基准：完整画面编码约 22.5 → 11.5 ms，321 × 241 裁剪约 2.1 → 0.9 ms。数字仅衡量图像转换与编码，不包含网络、解码、远端执行或宿主渲染；未连接真实远端或完成 Windows/Linux 验收。额外测试代码、脚本、数据和产物均存放于工程外系统临时目录。
+
+### 远端输入回执与条件等待
+
+更新后的被控端通过 `capabilities.input_ack` 声明输入处理回执。动作结果的 `remote_execution` 表示远端原生输入处理状态，应用界面的实际结果由后续观察确认。旧被控端返回 `unsupported`。回执等待有明确期限，超时动作保留结果不确定状态。
+
+`crossdesk_wait` 使用已有截图的 `observation_id`，等待对应区域变化（`changed`）或持续稳定（`stable`）。可设置 `stable_ms` 和最长 25000 毫秒的 `timeout_ms`。满足条件后返回新截图，切换显示器、断开和取消会结束等待。

apps/mcp/src/server.ts

@@ -296,6 +296,16 @@ export function createServer(
     },
     { ui: true },
   );
+  add(
+    "crossdesk_wait",
+    z.strictObject({ session_id: id, observation_id: id,
+      condition: z.enum(["changed", "stable"]),
+      stable_ms: z.int().min(100).max(5000).default(500),
+      timeout_ms: z.int().min(0).max(25000).default(5000) }),
+    "Wait for the observed screenshot region to change or remain stable. Returns matched, wait_timed_out and a fresh screenshot on success. A visual condition can be checked after an action with a separate screenshot observation. Remote input receipts report native input acceptance; verify the application's intended result visually.",
+    async (p, ctx) => output(await sessions.waitFor(p, ctx.mcpReq.signal)),
+    { readOnly: true },
+  );
   add(
     "crossdesk_set_control",
     z.strictObject({ session_id: id, enabled: z.boolean() }),

apps/mcp/src/sessions.ts

@@ -459,6 +459,13 @@ export class Sessions {
     }
     return result;
   }
+  async waitFor(p: Data, signal?: AbortSignal): Promise<Data> {
+    const { session_id, ...args } = p;
+    const result = await this.session(session_id).native.request(
+      "wait_frame", args, (args.timeout_ms ?? 5000) + 3000, signal);
+    if (result.matched) result.observation = await this.screenshot({ session_id }, signal);
+    return result;
+  }
   async control(id: string, enabled: boolean) {
     if (!enabled) this.cancelUnlock(this.session(id));
     return this.session(id).native.request("control", { enabled });

libs/controller/include/controller_session.h

@@ -11,6 +11,7 @@
 #include <deque>
 #include <minirtc.h>
 #include <mutex>
+#include <map>
 #include <nlohmann/json.hpp>
 #include <optional>
 #include <remote_action.h>
@@ -74,6 +75,11 @@ public:
   bool Text(const std::string &utf8);
   bool Key(int key, bool down);
   bool Pointer(float x, float y, int flag, int wheel = 0);
+  uint64_t RequestRemote(const std::string &method, nlohmann::json arguments = {});
+  std::optional<nlohmann::json> TakeRemoteResult(uint64_t request);
+  void BeginInput();
+  std::string InputResult();
+  bool InputAcknowledged() const { return remote_capabilities_.value("input_ack", nlohmann::json{}) == true; }
   std::optional<DecodedFrame> TakeFrame(DecodedFrame *recycle = nullptr);
   nlohmann::json Status() const;
   bool Connected() const { return state_ == "connected"; }
@@ -93,6 +99,7 @@ private:
   void ClosePeers();
   Params Parameters();
   bool Send(const char *stream, const std::string &payload);
+  bool SendInput(const char *stream, const RemoteAction &action);
   void Preview();
   static void Signal(SignalStatus, const char *, size_t, void *);
   static void Connection(ConnectionStatus, const char *, size_t, void *);
@@ -108,6 +115,12 @@ private:
   SessionOptions options_;
   PeerPtr *peer_ = nullptr;
   std::string state_ = "idle", error_, device_, password_, client_id_;
+  nlohmann::json remote_capabilities_ = nlohmann::json::object();
+  uint64_t remote_sequence_ = 0, capability_request_ = 0;
+  std::map<uint64_t, nlohmann::json> remote_results_;
+  std::vector<uint64_t> input_requests_;
+  std::string input_result_ = "applied";
+  bool input_ack_active_ = false;
   uint64_t generation_ = 0;
   uint32_t preview_request_ = 0, preview_ack_ = 0;
   bool control_ = false, preview_supported_ = false, resolution_ = false;

libs/controller/include/frame_image.h

@@ -22,6 +22,7 @@ struct FrameImage {
 FrameImage EncodeFrameImage(const DecodedFrame &frame, ImageRegion region,
                             int max_edge, size_t max_bytes);
 std::string ImageBase64(const std::vector<uint8_t> &bytes);
+uint64_t FrameSignature(const DecodedFrame &frame, ImageRegion region);
 } // namespace crossdesk
 
 #endif

libs/controller/src/controller_requests.cpp

@@ -0,0 +1,56 @@
+#include "controller_session.h"
+#include <stream_names.h>
+
+namespace crossdesk {
+using nlohmann::json;
+
+uint64_t ControllerSession::RequestRemote(const std::string &method, json arguments) {
+  if (!Connected() || remote_results_.size() >= 512) return 0;
+  const auto id = ++remote_sequence_;
+  remote_results_[id] = nullptr;
+  const auto message = json{{"type", "automation"}, {"version", 1},
+      {"id", id}, {"method", method}, {"arguments", std::move(arguments)}}.dump();
+  if (!Send(kControlStream, message)) {
+    remote_results_.erase(id);
+    return 0;
+  }
+  return id;
+}
+
+std::optional<json> ControllerSession::TakeRemoteResult(uint64_t request) {
+  const auto found = remote_results_.find(request);
+  if (found == remote_results_.end() || found->second.is_null()) return {};
+  auto result = std::move(found->second);
+  remote_results_.erase(found);
+  return result;
+}
+
+void ControllerSession::BeginInput() {
+  for (const auto id : input_requests_) remote_results_.erase(id);
+  input_requests_.clear();
+  input_result_ = "applied";
+  input_ack_active_ = InputAcknowledged();
+}
+
+bool ControllerSession::SendInput(const char *stream, const RemoteAction &action) {
+  if (!input_ack_active_) return Send(stream, action.to_json());
+  const auto id = RequestRemote("input", json::parse(action.to_json()));
+  if (!id) return false;
+  input_requests_.push_back(id);
+  return true;
+}
+
+std::string ControllerSession::InputResult() {
+  if (!input_ack_active_) return "unsupported";
+  for (auto it = input_requests_.begin(); it != input_requests_.end();) {
+    auto reply = TakeRemoteResult(*it);
+    if (!reply) { ++it; continue; }
+    const auto result = reply->value("result", json{});
+    if (reply->value("ok", json{}) != true || !result.is_object() ||
+        result.value("state", json{}) != "applied")
+      input_result_ = "failed";
+    it = input_requests_.erase(it);
+  }
+  return input_requests_.empty() ? input_result_ : "pending";
+}
+} // namespace crossdesk

libs/controller/src/controller_session.cpp

@@ -118,6 +118,10 @@ void ControllerSession::Connect(SessionOptions options, std::string device,
   password_ = std::move(password);
   state_ = "registering";
   error_.clear();
+  remote_capabilities_ = json::object();
+  remote_results_.clear();
+  input_requests_.clear();
+  capability_request_ = 0;
   displays_.clear();
   host_platform_ = HostPlatform::Unknown;
   preview_supported_ = resolution_ = fps15_ = bandwidth_ = text_supported_ =
@@ -225,6 +229,7 @@ void ControllerSession::Tick() {
         mute.type = audio_capture;
         mute.a = false;
         Send(kControlStream, mute.to_json());
+        capability_request_ = RequestRemote("capabilities");
       } else if (e.code == IncorrectPassword)
         Fail("incorrect_password");
       else if (e.code == NoSuchTransmissionId)
@@ -239,6 +244,21 @@ void ControllerSession::Tick() {
       auto j = ParseBoundedJson(e.data);
       if (!j.is_object())
         continue;
+      if (j.value("type", json{}) == "automation_result" &&
+          j.value("version", json{}) == 1 && j.contains("id") &&
+          j["id"].is_number_unsigned()) {
+        const auto id = j["id"].get<uint64_t>();
+        const auto found = remote_results_.find(id);
+        if (found != remote_results_.end()) {
+          found->second = j;
+          if (id == capability_request_) {
+            if (j.value("ok", json{}) == true && j.value("result", json{}).is_object())
+              remote_capabilities_ = j["result"];
+            remote_results_.erase(found);
+          }
+        }
+        continue;
+      }
       if (j.value("type", json{}) == "deskhub_preview_status") {
         if (j.value("request_id", json{}) == preview_request_ &&
             j.contains("accepted") && j["accepted"].is_boolean()) {
@@ -398,7 +418,7 @@ bool ControllerSession::Key(int key, bool down) {
   a.k = {size_t(key), 0, false, down ? key_down : key_up};
   if (down)
     keys_.insert(key);
-  const bool sent = Send(kKeyboardStream, a.to_json());
+  const bool sent = SendInput(kKeyboardStream, a);
   if (sent && !down)
     keys_.erase(key);
   return sent;
@@ -417,7 +437,7 @@ bool ControllerSession::Pointer(float x, float y, int flag, int wheel) {
   pointer_y_ = y;
   if (flag >= 1 && flag <= 6 && (flag & 1))
     buttons_.insert(flag);
-  const bool sent = Send(kMouseStream, a.to_json());
+  const bool sent = SendInput(kMouseStream, a);
   if (sent && flag >= 2 && flag <= 6 && !(flag & 1))
     buttons_.erase(flag - 1);
   return sent;
@@ -430,7 +450,7 @@ bool ControllerSession::Text(const std::string &text) {
   a.type = text_input;
   a.text = text;
   try {
-    return Send(kKeyboardStream, a.to_json());
+    return SendInput(kKeyboardStream, a);
   } catch (...) {
     return false;
   }
@@ -454,6 +474,7 @@ json ControllerSession::Status() const {
           {"platform", HostPlatformName(host_platform_)},
           {"display", options_.display},
           {"text_input", text_supported_},
+          {"capabilities", remote_capabilities_},
           {"preview_supported", preview_supported_},
           {"preview_request", preview_request_},
           {"preview_ack", preview_ack_},

libs/controller/src/frame_observation.cpp

@@ -0,0 +1,28 @@
+#include "frame_image.h"
+#include <stdexcept>
+
+namespace crossdesk {
+uint64_t FrameSignature(const DecodedFrame &frame, ImageRegion region) {
+  if (!frame.width || !frame.height || (frame.width & 1) || (frame.height & 1) ||
+      region.x < 0 || region.y < 0 || region.width <= 0 || region.height <= 0 ||
+      uint64_t(region.x) + region.width > frame.width ||
+      uint64_t(region.y) + region.height > frame.height ||
+      frame.nv12.size() < size_t(frame.width) * frame.height * 3 / 2)
+    throw std::invalid_argument("INVALID_REGION");
+  uint64_t hash = 14695981039346656037ULL;
+  const auto row = [&](size_t offset, int count) {
+    for (int x = 0; x < count; ++x) {
+      hash ^= frame.nv12[offset + x];
+      hash *= 1099511628211ULL;
+    }
+  };
+  for (int y = region.y; y < region.y + region.height; ++y)
+    row(size_t(y) * frame.width + region.x, region.width);
+  const int chroma_x = region.x & ~1;
+  const int chroma_end = (region.x + region.width + 1) & ~1;
+  for (int y = region.y / 2; y < (region.y + region.height + 1) / 2; ++y)
+    row(size_t(frame.width) * frame.height + size_t(y) * frame.width + chroma_x,
+        chroma_end - chroma_x);
+  return hash;
+}
+} // namespace crossdesk