[feat] add CrossDesk MCP with remote desktop panels

libs/controller/include/controller_session.h

@@ -19,6 +19,11 @@
 #include <vector>
 
 namespace crossdesk {
+struct EncodedVideoFrame {
+  std::vector<uint8_t> data;
+  uint64_t generation = 0, sequence = 0, timestamp_us = 0;
+  bool key = false;
+};
 struct DecodedFrame {
   unsigned width = 0, height = 0;
   uint64_t generation = 0, frame_id = 0;
@@ -46,6 +51,7 @@ struct SessionOptions {
   int port = 9099, display = 0, fps = 8;
   int preview_width = 1920, preview_height = 1080;
   int quality = 2, max_bitrate_bps = 6000000;
+  bool hardware_acceleration = false;
   std::chrono::seconds timeout{30};
 };
 // All public methods except the destructor run on one owner thread. Call Tick
@@ -59,6 +65,8 @@ public:
   void Tick();
   void Disconnect();
   bool SelectDisplay(int index);
+  void SetVideoOutput(bool enabled, bool request_keyframe = false);
+  std::optional<EncodedVideoFrame> TakeEncodedVideo();
   void SetPreview(int fps, int width, int height, int quality = -1,
                   int max_bitrate_bps = -1);
   void SetControl(bool enabled);
@@ -91,6 +99,8 @@ private:
                    size_t, void *);
   static void Video(const MiniRtcVideoFrame *, const char *, size_t,
                     const char *, size_t, void *);
+  static int EncodedVideo(const uint8_t *, size_t, int64_t, const char *,
+                          size_t, const char *, size_t, void *);
   static void Network(const char *, size_t, TraversalMode,
                       const MiniRtcNetTrafficStats *, const char *, size_t,
                       void *);
@@ -102,6 +112,7 @@ private:
   bool control_ = false, preview_supported_ = false, resolution_ = false;
   bool fps15_ = false, bandwidth_ = false, text_supported_ = false;
   bool preview_accepted_ = false;
+  HostPlatform host_platform_ = HostPlatform::Unknown;
   std::vector<HostDisplay> displays_;
   std::set<int> keys_, buttons_;
   float pointer_x_ = .5f, pointer_y_ = .5f;
@@ -114,6 +125,12 @@ private:
   MiniRtcNetTrafficStats network_{};
   TraversalMode traversal_ = UnknownMode;
   LatestFrame frames_;
+  bool video_output_ = false, video_wait_key_ = true, video_request_key_ = false;
+  size_t video_bytes_ = 0;
+  uint64_t video_generation_ = 0, video_sequence_ = 0;
+  std::deque<EncodedVideoFrame> video_frames_;
+  std::vector<uint8_t> video_sps_, video_pps_;
+  std::chrono::steady_clock::time_point video_key_requested_{};
 };
 nlohmann::json ParseBoundedJson(const std::string &text);
 std::string SafeText(const std::string &text, size_t limit = 160);

libs/controller/include/frame_image.h

@@ -0,0 +1,27 @@
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2026-11-02
+ * Copyright (c) 2026 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _FRAME_IMAGE_H_
+#define _FRAME_IMAGE_H_
+
+#include <controller_session.h>
+
+namespace crossdesk {
+struct ImageRegion {
+  int x = 0, y = 0, width = 0, height = 0;
+};
+struct FrameImage {
+  int width = 0, height = 0;
+  ImageRegion region;
+  std::vector<uint8_t> png;
+};
+// Pure bounded conversion; safe on an encoding worker, never calls the session.
+FrameImage EncodeFrameImage(const DecodedFrame &frame, ImageRegion region,
+                            int max_edge, size_t max_bytes);
+std::string ImageBase64(const std::vector<uint8_t> &bytes);
+} // namespace crossdesk
+
+#endif

libs/controller/src/controller_session.cpp

@@ -77,7 +77,7 @@ Params ControllerSession::Parameters() {
   std::snprintf(p.log_path, sizeof(p.log_path), "%s",
                 options_.log_directory.c_str());
   p.signal_server_port = options_.port;
-  p.hardware_acceleration = false;
+  p.hardware_acceleration = options_.hardware_acceleration;
   p.native_video_output = true;
   p.av1_encoding = false;
   p.turn_mode = TurnAutoUdpTcp;
@@ -92,6 +92,7 @@ Params ControllerSession::Parameters() {
   p.on_net_status_report = Network;
   p.on_receive_data_buffer = Data;
   p.on_receive_video_frame = Video;
+  p.on_receive_encoded_video = EncodedVideo;
   // No audio callback, clipboard receiver, file writer or incoming input
   // handler.
   p.user_id = "cli";
@@ -118,6 +119,7 @@ void ControllerSession::Connect(SessionOptions options, std::string device,
   state_ = "registering";
   error_.clear();
   displays_.clear();
+  host_platform_ = HostPlatform::Unknown;
   preview_supported_ = resolution_ = fps15_ = bandwidth_ = text_supported_ =
       false;
   preview_request_ = preview_ack_ = 0;
@@ -238,6 +240,7 @@ void ControllerSession::Tick() {
       if (!a.from_json(e.data))
         continue;
       if (a.type == host_infomation) {
+        host_platform_ = a.i.platform;
         displays_.clear();
         for (size_t i = 0; i < std::min(a.i.display_num, size_t(64)); ++i)
           displays_.push_back({SafeText(a.i.display_list[i]), a.i.left[i],
@@ -422,7 +425,9 @@ std::optional<DecodedFrame> ControllerSession::TakeFrame(DecodedFrame *recycle)
 json ControllerSession::Status() const {
   json ds = json::array();
   for (size_t i = 0; i < displays_.size(); ++i)
-    ds.push_back({{"index", i}, {"name", displays_[i].name}});
+    ds.push_back({{"index", i}, {"name", displays_[i].name},
+                  {"left", displays_[i].left}, {"top", displays_[i].top},
+                  {"right", displays_[i].right}, {"bottom", displays_[i].bottom}});
   std::lock_guard lock(mutex_);
   return {{"state", state_},
           {"client_id", client_id_},
@@ -430,6 +435,7 @@ json ControllerSession::Status() const {
           {"generation", generation_},
           {"control", control_},
           {"displays", ds},
+          {"platform", HostPlatformName(host_platform_)},
           {"display", options_.display},
           {"text_input", text_supported_},
           {"preview_supported", preview_supported_},
@@ -440,7 +446,19 @@ json ControllerSession::Status() const {
           {"preview_height", options_.preview_height},
           {"preview_quality", options_.quality},
           {"preview_bitrate_bps", options_.max_bitrate_bps},
+          {"native_video_decoder", SafeText(std::string(
+              network_.video_decoder,
+              std::find(network_.video_decoder,
+                        network_.video_decoder + sizeof(network_.video_decoder), '\0')))},
           {"frames", frames_.Stats()},
+          {"video_output",
+           {{"enabled", video_output_},
+            {"received", video_sequence_},
+            {"queued", video_frames_.size()},
+            {"queued_bytes", video_bytes_},
+            {"waiting_keyframe", video_wait_key_},
+            {"has_sps", !video_sps_.empty()},
+            {"has_pps", !video_pps_.empty()}}},
           {"network",
            {{"video_bps", network_.video_inbound_stats.bitrate},
             {"rtt_ms", network_.rtt_ms},

libs/controller/src/encoded_video.cpp

@@ -0,0 +1,129 @@
+#include "controller_session.h"
+#include <algorithm>
+#include <display_stream_id.h>
+
+namespace crossdesk {
+namespace {
+constexpr size_t kMaxAccessUnit = 4 * 1024 * 1024;
+// NAL spans include their Annex B start code, so cached parameter sets can be
+// prepended to an IDR without converting or re-encoding the original stream.
+size_t StartCode(const uint8_t *data, size_t size, size_t at) {
+  if (at + 3 <= size && !data[at] && !data[at + 1]) {
+    if (data[at + 2] == 1)
+      return 3;
+    if (at + 4 <= size && !data[at + 2] && data[at + 3] == 1)
+      return 4;
+  }
+  return 0;
+}
+} // namespace
+void ControllerSession::SetVideoOutput(bool enabled, bool request_keyframe) {
+  std::lock_guard lock(mutex_);
+  if (video_output_ != enabled || request_keyframe) {
+    video_frames_.clear();
+    video_bytes_ = 0;
+    video_wait_key_ = true;
+    video_request_key_ = enabled;
+  }
+  video_output_ = enabled;
+}
+std::optional<EncodedVideoFrame> ControllerSession::TakeEncodedVideo() {
+  std::lock_guard lock(mutex_);
+  if (video_frames_.empty())
+    return {};
+  auto frame = std::move(video_frames_.front());
+  video_frames_.pop_front();
+  video_bytes_ -= frame.data.size();
+  return frame;
+}
+int ControllerSession::EncodedVideo(const uint8_t *data, size_t size, int64_t,
+                                    const char *, size_t, const char *stream,
+                                    size_t stream_size, void *context) {
+  auto &s = *static_cast<ControllerSession *>(context);
+  try {
+    std::lock_guard lock(s.mutex_);
+    if (s.stopping_ || !s.video_output_ || !data || !size ||
+        size > kMaxAccessUnit || !stream || stream_size >= 64 ||
+        std::string(stream, stream_size) !=
+            MakeDisplayStreamId(s.callback_display_))
+      return 0;
+    const auto now = std::chrono::steady_clock::now();
+    if (s.video_generation_ != s.callback_generation_) {
+      s.video_generation_ = s.callback_generation_;
+      s.video_frames_.clear();
+      s.video_bytes_ = 0;
+      s.video_sps_.clear();
+      s.video_pps_.clear();
+      s.video_wait_key_ = s.video_request_key_ = true;
+    }
+    bool key = false;
+    for (size_t at = 0; at + 3 < size;) {
+      const auto prefix = StartCode(data, size, at);
+      if (!prefix) {
+        ++at;
+        continue;
+      }
+      const auto body = at + prefix;
+      if (body >= size)
+        break;
+      size_t end = body + 1;
+      while (end < size && !StartCode(data, size, end))
+        ++end;
+      const auto type = data[body] & 31;
+      key |= type == 5;
+      if ((type == 7 || type == 8) && end - at <= 65536) {
+        auto &parameter = type == 7 ? s.video_sps_ : s.video_pps_;
+        const std::vector<uint8_t> next(data + at, data + end);
+        if (parameter != next) {
+          parameter = next;
+          s.video_wait_key_ = s.video_request_key_ = true;
+          s.video_frames_.clear();
+          s.video_bytes_ = 0;
+        }
+      }
+      at = end;
+    }
+    ++s.video_sequence_;
+    const auto output_size =
+        size + (key ? s.video_sps_.size() + s.video_pps_.size() : 0);
+    if (s.video_frames_.size() >= 60 ||
+        s.video_bytes_ + output_size > 8 * 1024 * 1024) {
+      s.video_frames_.clear();
+      s.video_bytes_ = 0;
+      s.video_wait_key_ = s.video_request_key_ = true;
+    }
+    if (key && !s.video_sps_.empty() && !s.video_pps_.empty()) {
+      s.video_wait_key_ = s.video_request_key_ = false;
+    }
+    if (!s.video_wait_key_) {
+      EncodedVideoFrame frame;
+      frame.key = key;
+      frame.generation = s.callback_generation_;
+      frame.sequence = s.video_sequence_;
+      frame.timestamp_us =
+          std::chrono::duration_cast<std::chrono::microseconds>(
+              now.time_since_epoch())
+              .count();
+      if (key) {
+        frame.data = s.video_sps_;
+        frame.data.insert(frame.data.end(), s.video_pps_.begin(),
+                          s.video_pps_.end());
+      }
+      frame.data.insert(frame.data.end(), data, data + size);
+      if (frame.data.size() <= kMaxAccessUnit) {
+        s.video_bytes_ += frame.data.size();
+        s.video_frames_.push_back(std::move(frame));
+      } else {
+        s.video_wait_key_ = s.video_request_key_ = true;
+      }
+    }
+    if (s.video_request_key_ &&
+        now - s.video_key_requested_ >= std::chrono::seconds(1)) {
+      s.video_key_requested_ = now;
+      return 1;
+    }
+  } catch (...) { /* The receive callback must not unwind into MiniRTC. */
+  }
+  return 0;
+}
+} // namespace crossdesk

libs/controller/src/frame_image.cpp

@@ -0,0 +1,85 @@
+#include "frame_image.h"
+#include <algorithm>
+#include <libyuv.h>
+#include <png.h>
+#include <stdexcept>
+
+namespace crossdesk {
+FrameImage EncodeFrameImage(const DecodedFrame &f, ImageRegion r, int max_edge,
+                            size_t max_bytes) {
+  if (!f.width || !f.height || f.width > 8192 || f.height > 8192 ||
+      (f.width & 1) || (f.height & 1) ||
+      uint64_t(f.width) * f.height > 16777216 ||
+      f.nv12.size() != size_t(f.width) * f.height * 3 / 2 || max_edge < 64 ||
+      max_edge > 2048 || max_bytes < 4096 || max_bytes > 2 * 1024 * 1024)
+    throw std::invalid_argument("INVALID_IMAGE_REQUEST");
+  if (!r.width && !r.height)
+    r = {0, 0, int(f.width), int(f.height)};
+  if (r.x < 0 || r.y < 0 || r.width < 1 || r.height < 1 ||
+      r.width > int(f.width) || r.height > int(f.height) ||
+      r.x > int(f.width) - r.width || r.y > int(f.height) - r.height)
+    throw std::invalid_argument("INVALID_REGION");
+  std::vector<uint8_t> argb(size_t(f.width) * f.height * 4);
+  if (libyuv::NV12ToARGB(f.nv12.data(), f.width,
+                         f.nv12.data() + size_t(f.width) * f.height, f.width,
+                         argb.data(), f.width * 4, f.width, f.height))
+    throw std::runtime_error("IMAGE_CONVERSION_FAILED");
+  FrameImage result;
+  result.region = r;
+  for (int edge = max_edge;; edge = std::max(32, edge * 3 / 4)) {
+    const double scale =
+        std::min(1.0, double(edge) / std::max(r.width, r.height));
+    result.width = std::max(1, int(r.width * scale));
+    result.height = std::max(1, int(r.height * scale));
+    std::vector<uint8_t> scaled(size_t(result.width) * result.height * 4);
+    std::vector<uint8_t> rgb(size_t(result.width) * result.height * 3);
+    if (libyuv::ARGBScale(argb.data() + (size_t(r.y) * f.width + r.x) * 4,
+                          f.width * 4, r.width, r.height, scaled.data(),
+                          result.width * 4, result.width, result.height,
+                          libyuv::kFilterBox) ||
+        libyuv::ARGBToRAW(scaled.data(), result.width * 4, rgb.data(),
+                          result.width * 3, result.width, result.height))
+      throw std::runtime_error("IMAGE_CONVERSION_FAILED");
+    png_image image{};
+    image.version = PNG_IMAGE_VERSION;
+    image.width = result.width;
+    image.height = result.height;
+    image.format = PNG_FORMAT_RGB;
+    png_alloc_size_t size = 0;
+    if (!png_image_write_to_memory(&image, nullptr, &size, 0, rgb.data(), 0,
+                                   nullptr)) {
+      png_image_free(&image);
+      throw std::runtime_error("IMAGE_ENCODING_FAILED");
+    }
+    if (size <= max_bytes) {
+      result.png.resize(size);
+      const auto ok = png_image_write_to_memory(
+          &image, result.png.data(), &size, 0, rgb.data(), 0, nullptr);
+      png_image_free(&image);
+      if (!ok)
+        throw std::runtime_error("IMAGE_ENCODING_FAILED");
+      result.png.resize(size);
+      return result;
+    }
+    png_image_free(&image);
+    if (edge == 32)
+      throw std::runtime_error("IMAGE_TOO_LARGE");
+  }
+}
+std::string ImageBase64(const std::vector<uint8_t> &data) {
+  constexpr char alphabet[] =
+      "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+  std::string out;
+  out.reserve((data.size() + 2) / 3 * 4);
+  for (size_t i = 0; i < data.size(); i += 3) {
+    const uint32_t n = (uint32_t(data[i]) << 16) |
+                       (i + 1 < data.size() ? uint32_t(data[i + 1]) << 8 : 0) |
+                       (i + 2 < data.size() ? data[i + 2] : 0);
+    out += alphabet[n >> 18];
+    out += alphabet[(n >> 12) & 63];
+    out += i + 1 < data.size() ? alphabet[(n >> 6) & 63] : '=';
+    out += i + 2 < data.size() ? alphabet[n & 63] : '=';
+  }
+  return out;
+}
+} // namespace crossdesk