[feat] acknowledge remote input and wait for visual changes

libs/controller/src/controller_requests.cpp

@@ -0,0 +1,56 @@
+#include "controller_session.h"
+#include <stream_names.h>
+
+namespace crossdesk {
+using nlohmann::json;
+
+uint64_t ControllerSession::RequestRemote(const std::string &method, json arguments) {
+  if (!Connected() || remote_results_.size() >= 512) return 0;
+  const auto id = ++remote_sequence_;
+  remote_results_[id] = nullptr;
+  const auto message = json{{"type", "automation"}, {"version", 1},
+      {"id", id}, {"method", method}, {"arguments", std::move(arguments)}}.dump();
+  if (!Send(kControlStream, message)) {
+    remote_results_.erase(id);
+    return 0;
+  }
+  return id;
+}
+
+std::optional<json> ControllerSession::TakeRemoteResult(uint64_t request) {
+  const auto found = remote_results_.find(request);
+  if (found == remote_results_.end() || found->second.is_null()) return {};
+  auto result = std::move(found->second);
+  remote_results_.erase(found);
+  return result;
+}
+
+void ControllerSession::BeginInput() {
+  for (const auto id : input_requests_) remote_results_.erase(id);
+  input_requests_.clear();
+  input_result_ = "applied";
+  input_ack_active_ = InputAcknowledged();
+}
+
+bool ControllerSession::SendInput(const char *stream, const RemoteAction &action) {
+  if (!input_ack_active_) return Send(stream, action.to_json());
+  const auto id = RequestRemote("input", json::parse(action.to_json()));
+  if (!id) return false;
+  input_requests_.push_back(id);
+  return true;
+}
+
+std::string ControllerSession::InputResult() {
+  if (!input_ack_active_) return "unsupported";
+  for (auto it = input_requests_.begin(); it != input_requests_.end();) {
+    auto reply = TakeRemoteResult(*it);
+    if (!reply) { ++it; continue; }
+    const auto result = reply->value("result", json{});
+    if (reply->value("ok", json{}) != true || !result.is_object() ||
+        result.value("state", json{}) != "applied")
+      input_result_ = "failed";
+    it = input_requests_.erase(it);
+  }
+  return input_requests_.empty() ? input_result_ : "pending";
+}
+} // namespace crossdesk

libs/controller/src/controller_session.cpp

@@ -118,6 +118,10 @@ void ControllerSession::Connect(SessionOptions options, std::string device,
   password_ = std::move(password);
   state_ = "registering";
   error_.clear();
+  remote_capabilities_ = json::object();
+  remote_results_.clear();
+  input_requests_.clear();
+  capability_request_ = 0;
   displays_.clear();
   host_platform_ = HostPlatform::Unknown;
   preview_supported_ = resolution_ = fps15_ = bandwidth_ = text_supported_ =
@@ -225,6 +229,7 @@ void ControllerSession::Tick() {
         mute.type = audio_capture;
         mute.a = false;
         Send(kControlStream, mute.to_json());
+        capability_request_ = RequestRemote("capabilities");
       } else if (e.code == IncorrectPassword)
         Fail("incorrect_password");
       else if (e.code == NoSuchTransmissionId)
@@ -239,6 +244,21 @@ void ControllerSession::Tick() {
       auto j = ParseBoundedJson(e.data);
       if (!j.is_object())
         continue;
+      if (j.value("type", json{}) == "automation_result" &&
+          j.value("version", json{}) == 1 && j.contains("id") &&
+          j["id"].is_number_unsigned()) {
+        const auto id = j["id"].get<uint64_t>();
+        const auto found = remote_results_.find(id);
+        if (found != remote_results_.end()) {
+          found->second = j;
+          if (id == capability_request_) {
+            if (j.value("ok", json{}) == true && j.value("result", json{}).is_object())
+              remote_capabilities_ = j["result"];
+            remote_results_.erase(found);
+          }
+        }
+        continue;
+      }
       if (j.value("type", json{}) == "deskhub_preview_status") {
         if (j.value("request_id", json{}) == preview_request_ &&
             j.contains("accepted") && j["accepted"].is_boolean()) {
@@ -398,7 +418,7 @@ bool ControllerSession::Key(int key, bool down) {
   a.k = {size_t(key), 0, false, down ? key_down : key_up};
   if (down)
     keys_.insert(key);
-  const bool sent = Send(kKeyboardStream, a.to_json());
+  const bool sent = SendInput(kKeyboardStream, a);
   if (sent && !down)
     keys_.erase(key);
   return sent;
@@ -417,7 +437,7 @@ bool ControllerSession::Pointer(float x, float y, int flag, int wheel) {
   pointer_y_ = y;
   if (flag >= 1 && flag <= 6 && (flag & 1))
     buttons_.insert(flag);
-  const bool sent = Send(kMouseStream, a.to_json());
+  const bool sent = SendInput(kMouseStream, a);
   if (sent && flag >= 2 && flag <= 6 && !(flag & 1))
     buttons_.erase(flag - 1);
   return sent;
@@ -430,7 +450,7 @@ bool ControllerSession::Text(const std::string &text) {
   a.type = text_input;
   a.text = text;
   try {
-    return Send(kKeyboardStream, a.to_json());
+    return SendInput(kKeyboardStream, a);
   } catch (...) {
     return false;
   }
@@ -454,6 +474,7 @@ json ControllerSession::Status() const {
           {"platform", HostPlatformName(host_platform_)},
           {"display", options_.display},
           {"text_input", text_supported_},
+          {"capabilities", remote_capabilities_},
           {"preview_supported", preview_supported_},
           {"preview_request", preview_request_},
           {"preview_ack", preview_ack_},

libs/controller/src/frame_observation.cpp

@@ -0,0 +1,28 @@
+#include "frame_image.h"
+#include <stdexcept>
+
+namespace crossdesk {
+uint64_t FrameSignature(const DecodedFrame &frame, ImageRegion region) {
+  if (!frame.width || !frame.height || (frame.width & 1) || (frame.height & 1) ||
+      region.x < 0 || region.y < 0 || region.width <= 0 || region.height <= 0 ||
+      uint64_t(region.x) + region.width > frame.width ||
+      uint64_t(region.y) + region.height > frame.height ||
+      frame.nv12.size() < size_t(frame.width) * frame.height * 3 / 2)
+    throw std::invalid_argument("INVALID_REGION");
+  uint64_t hash = 14695981039346656037ULL;
+  const auto row = [&](size_t offset, int count) {
+    for (int x = 0; x < count; ++x) {
+      hash ^= frame.nv12[offset + x];
+      hash *= 1099511628211ULL;
+    }
+  };
+  for (int y = region.y; y < region.y + region.height; ++y)
+    row(size_t(y) * frame.width + region.x, region.width);
+  const int chroma_x = region.x & ~1;
+  const int chroma_end = (region.x + region.width + 1) & ~1;
+  for (int y = region.y / 2; y < (region.y + region.height + 1) / 2; ++y)
+    row(size_t(frame.width) * frame.height + size_t(y) * frame.width + chroma_x,
+        chroma_end - chroma_x);
+  return hash;
+}
+} // namespace crossdesk