[fix] stabilize MCP sessions and extend input controls

libs/controller/include/controller_session.h

@@ -70,6 +70,7 @@ public:
   void SetPreview(int fps, int width, int height, int quality = -1,
                   int max_bitrate_bps = -1);
   void SetControl(bool enabled);
+  void ReleaseInput();
   bool Text(const std::string &utf8);
   bool Key(int key, bool down);
   bool Pointer(float x, float y, int flag, int wheel = 0);
@@ -93,7 +94,6 @@ private:
   Params Parameters();
   bool Send(const char *stream, const std::string &payload);
   void Preview();
-  void ReleaseInput();
   static void Signal(SignalStatus, const char *, size_t, void *);
   static void Connection(ConnectionStatus, const char *, size_t, void *);
   static void Data(const char *, size_t, const char *, size_t, const char *,

libs/controller/src/controller_session.cpp

@@ -288,6 +288,8 @@ void ControllerSession::Fail(const char *reason) {
   state_ = "failed";
   control_ = false;
   ClosePeers();
+  keys_.clear();
+  buttons_.clear();
 }
 void ControllerSession::Disconnect() {
   if (state_ == "closed" && !control_)
@@ -295,6 +297,8 @@ void ControllerSession::Disconnect() {
   ReleaseInput();
   control_ = false;
   ClosePeers();
+  keys_.clear();
+  buttons_.clear();
   state_ = "closed";
 }
 bool ControllerSession::SelectDisplay(int index) {
@@ -367,30 +371,34 @@ void ControllerSession::SetControl(bool enabled) {
   control_ = enabled;
 }
 void ControllerSession::ReleaseInput() {
-  auto keys = keys_;
-  for (int k : keys)
-    Key(k, false);
+  // Release buttons before modifiers, so interrupted modified drags do not
+  // turn into an unmodified drop. Failed releases remain tracked for cleanup.
   auto buttons = buttons_;
   for (int b : buttons)
     Pointer(pointer_x_, pointer_y_, b + 1);
-  keys_.clear();
-  buttons_.clear();
+  auto keys = keys_;
+  for (auto it = keys.rbegin(); it != keys.rend(); ++it)
+    Key(*it, false);
 }
 bool ControllerSession::Key(int key, bool down) {
-  if (!control_ || !Connected() || key < 0 || key > 255)
+  if ((!control_ && (down || !keys_.count(key))) || !Connected() || key < 0 ||
+      key > 255)
     return false;
   RemoteAction a{};
   a.type = keyboard;
   a.k = {size_t(key), 0, false, down ? key_down : key_up};
   if (down)
     keys_.insert(key);
-  else
+  const bool sent = Send(kKeyboardStream, a.to_json());
+  if (sent && !down)
     keys_.erase(key);
-  return Send(kKeyboardStream, a.to_json());
+  return sent;
 }
 bool ControllerSession::Pointer(float x, float y, int flag, int wheel) {
-  if (!control_ || !Connected() || !std::isfinite(x) || !std::isfinite(y) ||
-      x < 0 || y < 0 || x > 1 || y > 1 || flag < 0 || flag > 8 ||
+  const bool releasing =
+      flag >= 2 && flag <= 6 && !(flag & 1) && buttons_.count(flag - 1);
+  if ((!control_ && !releasing) || !Connected() || !std::isfinite(x) ||
+      !std::isfinite(y) || x < 0 || y < 0 || x > 1 || y > 1 || flag < 0 || flag > 8 ||
       std::abs(int64_t(wheel)) > 1200)
     return false;
   RemoteAction a{};
@@ -398,13 +406,12 @@ bool ControllerSession::Pointer(float x, float y, int flag, int wheel) {
   a.m = {x, y, wheel, static_cast<MouseFlag>(flag)};
   pointer_x_ = x;
   pointer_y_ = y;
-  if (flag >= 1 && flag <= 6) {
-    if (flag & 1)
-      buttons_.insert(flag);
-    else
-      buttons_.erase(flag - 1);
-  }
-  return Send(kMouseStream, a.to_json());
+  if (flag >= 1 && flag <= 6 && (flag & 1))
+    buttons_.insert(flag);
+  const bool sent = Send(kMouseStream, a.to_json());
+  if (sent && flag >= 2 && flag <= 6 && !(flag & 1))
+    buttons_.erase(flag - 1);
+  return sent;
 }
 bool ControllerSession::Text(const std::string &text) {
   if (!control_ || !Connected() || !text_supported_ || text.empty() ||

libs/controller/src/frame_image.cpp

@@ -19,10 +19,15 @@ FrameImage EncodeFrameImage(const DecodedFrame &f, ImageRegion r, int max_edge,
       r.width > int(f.width) || r.height > int(f.height) ||
       r.x > int(f.width) - r.width || r.y > int(f.height) - r.height)
     throw std::invalid_argument("INVALID_REGION");
-  std::vector<uint8_t> argb(size_t(f.width) * f.height * 4);
-  if (libyuv::NV12ToARGB(f.nv12.data(), f.width,
-                         f.nv12.data() + size_t(f.width) * f.height, f.width,
-                         argb.data(), f.width * 4, f.width, f.height))
+  // Align the NV12 crop to its chroma grid, then offset the RGB crop by at
+  // most one pixel. Small observations do not need a full-frame RGB buffer.
+  const int left = r.x & ~1, top = r.y & ~1;
+  const int width = r.width + r.x - left, height = r.height + r.y - top;
+  std::vector<uint8_t> argb(size_t(width) * height * 4);
+  if (libyuv::NV12ToARGB(f.nv12.data() + size_t(top) * f.width + left, f.width,
+                        f.nv12.data() + size_t(f.width) * f.height +
+                            size_t(top / 2) * f.width + left,
+                        f.width, argb.data(), width * 4, width, height))
     throw std::runtime_error("IMAGE_CONVERSION_FAILED");
   FrameImage result;
   result.region = r;
@@ -33,8 +38,8 @@ FrameImage EncodeFrameImage(const DecodedFrame &f, ImageRegion r, int max_edge,
     result.height = std::max(1, int(r.height * scale));
     std::vector<uint8_t> scaled(size_t(result.width) * result.height * 4);
     std::vector<uint8_t> rgb(size_t(result.width) * result.height * 3);
-    if (libyuv::ARGBScale(argb.data() + (size_t(r.y) * f.width + r.x) * 4,
-                          f.width * 4, r.width, r.height, scaled.data(),
+    if (libyuv::ARGBScale(argb.data() + (size_t(r.y - top) * width + r.x - left) * 4,
+                          width * 4, r.width, r.height, scaled.data(),
                           result.width * 4, result.width, result.height,
                           libyuv::kFilterBox) ||
         libyuv::ARGBToRAW(scaled.data(), result.width * 4, rgb.data(),
@@ -45,23 +50,22 @@ FrameImage EncodeFrameImage(const DecodedFrame &f, ImageRegion r, int max_edge,
     image.width = result.width;
     image.height = result.height;
     image.format = PNG_FORMAT_RGB;
-    png_alloc_size_t size = 0;
-    if (!png_image_write_to_memory(&image, nullptr, &size, 0, rgb.data(), 0,
-                                   nullptr)) {
-      png_image_free(&image);
-      throw std::runtime_error("IMAGE_ENCODING_FAILED");
-    }
-    if (size <= max_bytes) {
-      result.png.resize(size);
-      const auto ok = png_image_write_to_memory(
-          &image, result.png.data(), &size, 0, rgb.data(), 0, nullptr);
-      png_image_free(&image);
-      if (!ok)
-        throw std::runtime_error("IMAGE_ENCODING_FAILED");
+    // libpng reports the required size on buffer exhaustion. Compress once
+    // into the bounded output, instead of compressing once to measure and
+    // again to write every successful screenshot.
+    const auto capacity =
+        std::min(png_alloc_size_t(max_bytes), PNG_IMAGE_PNG_SIZE_MAX(image));
+    result.png.resize(capacity);
+    png_alloc_size_t size = capacity;
+    const auto ok = png_image_write_to_memory(
+        &image, result.png.data(), &size, 0, rgb.data(), 0, nullptr);
+    png_image_free(&image);
+    if (ok) {
       result.png.resize(size);
       return result;
     }
-    png_image_free(&image);
+    if (size <= max_bytes)
+      throw std::runtime_error("IMAGE_ENCODING_FAILED");
     if (edge == 32)
       throw std::runtime_error("IMAGE_TOO_LARGE");
   }