[feat] acknowledge remote input and wait for visual changes

apps/desktop/src/gui/features/input/remote_input_dispatcher.cpp

@@ -39,14 +39,14 @@ void RemoteInputDispatcher::Connect(const std::string& id) {
 }
 
 RemoteInputDispatcher::Result RemoteInputDispatcher::Push(
-    const std::string& id, const RemoteAction& action) {
+    const std::string& id, const RemoteAction& action, Completion completion) {
   // HostInfo owns raw arrays; it must never enter this value-copy queue.
   if (!Accepts(action)) return Result::rejected;
   std::lock_guard lock(mutex_);
   if (stopping_ || !connected_.count(id)) return Result::rejected;
-  if (IsMove(action) && !pending_.empty()) {
+  if (!completion && IsMove(action) && !pending_.empty()) {
     auto& last = pending_.back();
-    if (last.id == id && last.IsInput() && IsMove(*last.action)) {
+    if (!last.completion && last.id == id && last.IsInput() && IsMove(*last.action)) {
       last.action = action;
       last.queued_at = Clock::now();
       return Result::coalesced;
@@ -56,7 +56,7 @@ RemoteInputDispatcher::Result RemoteInputDispatcher::Push(
     QueueReleaseLocked(id);
     return Result::overflow;
   }
-  pending_.push_back({id, action, Clock::now(), ItemKind::input});
+  pending_.push_back({id, action, Clock::now(), ItemKind::input, 0, std::move(completion)});
   ++pending_counts_[id];
   ready_.notify_one();
   return Result::queued;
@@ -328,6 +328,9 @@ void RemoteInputDispatcher::Run() {
         !pending_counts_.count(item.id))
       logs_.erase(item.id);
     lock.unlock();
+    if (item.completion) {
+      try { item.completion(outcome); } catch (...) {}
+    }
     if (report) {
       try {
         report_(item.id, queued_ms, execution_ms, outcome);

apps/desktop/src/gui/features/input/remote_input_dispatcher.h

@@ -38,6 +38,7 @@ class RemoteInputDispatcher {
   // Optional targeted adapter; legacy callers fall back to full Release.
   using ReleaseMouse =
       std::function<InputOutcome(const std::string&, unsigned)>;
+  using Completion = std::function<void(InputOutcome)>;
   enum class Result { queued, coalesced, rejected, overflow };
   static constexpr size_t kMaxPendingPerController = 128;
 
@@ -47,7 +48,8 @@ class RemoteInputDispatcher {
   ~RemoteInputDispatcher();
   static bool Accepts(const RemoteAction& action);
   void Connect(const std::string& id);
-  Result Push(const std::string& id, const RemoteAction& action);
+  Result Push(const std::string& id, const RemoteAction& action,
+              Completion completion = {});
   void Disconnect(const std::string& id);
 
   // Scheduled recovery is not idle work. Check HasPendingReleases() as well,
@@ -68,6 +70,7 @@ class RemoteInputDispatcher {
     // Zero requests full controller cleanup; otherwise only these mouse
     // buttons.
     unsigned mouse_mask = 0;
+    Completion completion;
     bool IsReset() const { return kind == ItemKind::reset_display; }
     bool IsInput() const { return kind == ItemKind::input; }
     bool IsCleanup() const { return kind == ItemKind::cleanup; }
@@ -120,4 +123,4 @@ class RemoteInputDispatcher {
 
 }  // namespace crossdesk
 
-#endif
+#endif

apps/desktop/src/gui/runtime/automation_runtime.cpp

@@ -0,0 +1,47 @@
+#include "runtime/gui_runtime.h"
+#include <stream_names.h>
+
+namespace crossdesk {
+void GuiRuntime::HandleAutomation(const std::string& remote_id,
+                                  const nlohmann::json& request) {
+  using nlohmann::json;
+  if (request.value("version", json{}) != 1 || !request.contains("id") ||
+      !request["id"].is_number_unsigned() || request["id"].get<uint64_t>() == 0)
+    return;
+  const auto source = ControllerEvents(remote_id);
+  if (!source || !source->DataAllowed()) return;
+  const auto id = request["id"].get<uint64_t>();
+  const auto reply = [this, remote_id, id, source](bool ok, json result) {
+    const auto current = ControllerEvents(remote_id);
+    if (current != source || !current->DataAllowed()) return;
+    const auto message = json{{"type", "automation_result"}, {"version", 1},
+        {"id", id}, {"ok", ok}, {ok ? "result" : "error", std::move(result)}}.dump();
+    SendControllerData(remote_id, message.data(), message.size(), kControlStream);
+  };
+  try {
+    const auto method = request.value("method", std::string{});
+    if (method == "capabilities") {
+      reply(true, {{"input_ack", true}});
+    } else if (method == "input") {
+      RemoteAction action{};
+      if (!request.contains("arguments") ||
+          !action.from_json(request["arguments"].dump())) {
+        reply(false, "INVALID_ARGUMENT");
+        return;
+      }
+      const auto result = remote_input_.Push(remote_id, action,
+          [reply](InputOutcome outcome) {
+            reply(true, {{"state", InputOutcomeName(outcome)}});
+          });
+      FreeRemoteAction(action);
+      if (result == RemoteInputDispatcher::Result::rejected ||
+          result == RemoteInputDispatcher::Result::overflow)
+        reply(false, "INPUT_REJECTED");
+    } else {
+      reply(false, "UNSUPPORTED_OPERATION");
+    }
+  } catch (...) {
+    reply(false, "INVALID_ARGUMENT");
+  }
+}
+} // namespace crossdesk

apps/desktop/src/gui/runtime/gui_runtime.h

@@ -152,6 +152,7 @@ class GuiRuntime : protected gui_detail::GuiState {
   // or every connected session while that target leads DeskHub group control.
   void SendRemoteInput(const RemoteAction& action);
   InputOutcome ApplyRemoteInput(const std::string& remote_id, const RemoteAction& action);
+  void HandleAutomation(const std::string& remote_id, const nlohmann::json& request);
   InputOutcome InjectRemoteMouse(const RemoteAction& action, int display);
   std::string gallery_group_control_id_;
 

apps/desktop/src/gui/runtime/peer_data_callbacks.cpp

@@ -179,6 +179,14 @@ void PeerEventHandler::OnReceiveDataBuffer(
   PeerPtr* bound_peer =
       handler->IsDirect() ? handler->DirectPeer() : runtime->peer_;
   std::string json_str(data, size);
+  if (json_str.find("\"automation\"") != std::string::npos) {
+    const auto request = nlohmann::json::parse(json_str, nullptr, false);
+    if (request.is_object() && request.value("type", nlohmann::json{}) == "automation") {
+      if (source_id == runtime->control_data_label_ && runtime->IsAuthorizedController(remote_id))
+        runtime->HandleAutomation(remote_id, request);
+      return;
+    }
+  }
   const auto preview_json =
       json_str.find("\"deskhub_preview") != std::string::npos
           ? nlohmann::json::parse(json_str, nullptr, false)