[feat] enable mouse control for multi-display

src/device_controller/device_controller.h

@@ -45,6 +45,10 @@ typedef struct {
   size_t host_name_size;
   char** display_list;
   size_t display_num;
+  int* left;
+  int* top;
+  int* right;
+  int* bottom;
 } HostInfo;
 
 typedef struct {

src/screen_capturer/screen_capturer.h

@@ -16,6 +16,19 @@ class ScreenCapturer {
 
   class DisplayInfo {
    public:
+    DisplayInfo(std::string name, int left, int top, int right, int bottom)
+        : name(name), left(left), top(top), right(right), bottom(bottom) {}
+    DisplayInfo(void* handle, std::string name, bool is_primary, int left,
+                int top, int right, int bottom)
+        : handle(handle),
+          name(name),
+          is_primary(is_primary),
+          left(left),
+          top(top),
+          right(right),
+          bottom(bottom) {}
+    ~DisplayInfo() {}
+
     void* handle = nullptr;
     std::string name = "";
     bool is_primary = false;

src/single_window/control_bar.cpp

@@ -58,9 +58,9 @@ int Render::ControlBar(std::shared_ptr<SubStreamWindowProperties>& props) {
 
     if (ImGui::BeginPopup("display")) {
       ImGui::SetWindowFontScale(0.5f);
-      for (int i = 0; i < props->display_names_.size(); i++) {
+      for (int i = 0; i < props->display_info_list_.size(); i++) {
-        if (ImGui::Selectable(props->display_names_[i].c_str())) {
+        if (ImGui::Selectable(props->display_info_list_[i].name.c_str())) {
-          selected_display_ = i + 1;
+          props->selected_display_ = i + 1;
 
           RemoteAction remote_action;
           remote_action.type = ControlType::display_id;
@@ -77,13 +77,13 @@ int Render::ControlBar(std::shared_ptr<SubStreamWindowProperties>& props) {
 
     ImGui::SetWindowFontScale(0.6f);
     ImVec2 text_size =
-        ImGui::CalcTextSize(std::to_string(selected_display_).c_str());
+        ImGui::CalcTextSize(std::to_string(props->selected_display_).c_str());
     ImVec2 text_pos =
         ImVec2(btn_min.x + (btn_size_actual.x - text_size.x) * 0.5f,
                btn_min.y + (btn_size_actual.y - text_size.y) * 0.5f - 2.0f);
     ImGui::GetWindowDrawList()->AddText(
         text_pos, IM_COL32(0, 0, 0, 255),
-        std::to_string(selected_display_).c_str());
+        std::to_string(props->selected_display_).c_str());
     ImGui::SetWindowFontScale(1.0f);
 
     ImGui::SameLine();

src/single_window/render.cpp

@@ -24,60 +24,78 @@
 std::vector<char> Render::SerializeRemoteAction(const RemoteAction& action) {
   std::vector<char> buffer;
   buffer.push_back(static_cast<char>(action.type));
+
+  auto insert_bytes = [&](const void* ptr, size_t len) {
+    buffer.insert(buffer.end(), (const char*)ptr, (const char*)ptr + len);
+  };
+
   if (action.type == ControlType::host_infomation) {
-    size_t name_len = action.i.host_name_size;
+    insert_bytes(&action.i.host_name_size, sizeof(size_t));
-    buffer.insert(buffer.end(), reinterpret_cast<char*>(&name_len),
+    insert_bytes(action.i.host_name, action.i.host_name_size);
-                  reinterpret_cast<char*>(&name_len) + sizeof(size_t));
+
-    buffer.insert(buffer.end(), action.i.host_name,
+    size_t num = action.i.display_num;
-                  action.i.host_name + name_len);
+    insert_bytes(&num, sizeof(size_t));
-    size_t display_num = action.i.display_num;
+
-    buffer.insert(buffer.end(), reinterpret_cast<char*>(&display_num),
+    for (size_t i = 0; i < num; ++i) {
-                  reinterpret_cast<char*>(&display_num) + sizeof(size_t));
+      size_t len = strlen(action.i.display_list[i]);
-    for (size_t i = 0; i < display_num; ++i) {
+      insert_bytes(&len, sizeof(size_t));
-      const char* name = action.i.display_list[i];
+      insert_bytes(action.i.display_list[i], len);
-      size_t len = strlen(name);
-      buffer.insert(buffer.end(), reinterpret_cast<char*>(&len),
-                    reinterpret_cast<char*>(&len) + sizeof(size_t));
-      buffer.insert(buffer.end(), reinterpret_cast<const char*>(name),
-                    reinterpret_cast<const char*>(name) + len);
     }
+
+    insert_bytes(action.i.left, sizeof(int) * num);
+    insert_bytes(action.i.top, sizeof(int) * num);
+    insert_bytes(action.i.right, sizeof(int) * num);
+    insert_bytes(action.i.bottom, sizeof(int) * num);
   }
+
   return buffer;
 }
 
 bool Render::DeserializeRemoteAction(const char* data, size_t size,
                                      RemoteAction& out) {
   size_t offset = 0;
+  auto read = [&](void* dst, size_t len) -> bool {
+    if (offset + len > size) return false;
+    memcpy(dst, data + offset, len);
+    offset += len;
+    return true;
+  };
+
   if (size < 1) return false;
-  out.type = static_cast<ControlType>(data[offset]);
+  out.type = static_cast<ControlType>(data[offset++]);
-  offset += 1;
+
   if (out.type == ControlType::host_infomation) {
-    if (offset + sizeof(size_t) > size) return false;
+    size_t name_len;
-    size_t host_name_len = *reinterpret_cast<const size_t*>(data + offset);
+    if (!read(&name_len, sizeof(size_t)) || name_len >= sizeof(out.i.host_name))
-    offset += sizeof(size_t);
-    if (offset + host_name_len > size ||
-        host_name_len >= sizeof(out.i.host_name))
       return false;
-    memcpy(out.i.host_name, data + offset, host_name_len);
+    if (!read(out.i.host_name, name_len)) return false;
-    out.i.host_name[host_name_len] = '\0';
+    out.i.host_name[name_len] = '\0';
-    out.i.host_name_size = host_name_len;
+    out.i.host_name_size = name_len;
-    offset += host_name_len;
+
-    if (offset + sizeof(size_t) > size) return false;
+    size_t num;
-    size_t display_num = *reinterpret_cast<const size_t*>(data + offset);
+    if (!read(&num, sizeof(size_t))) return false;
-    out.i.display_num = display_num;
+    out.i.display_num = num;
-    offset += sizeof(size_t);
+
-    out.i.display_list = (char**)malloc(display_num * sizeof(char*));
+    out.i.display_list = (char**)malloc(num * sizeof(char*));
-    for (size_t i = 0; i < display_num; ++i) {
+    for (size_t i = 0; i < num; ++i) {
-      if (offset + sizeof(size_t) > size) return false;
+      size_t len;
-      size_t len = *reinterpret_cast<const size_t*>(data + offset);
+      if (!read(&len, sizeof(size_t))) return false;
-      offset += sizeof(size_t);
       if (offset + len > size) return false;
       out.i.display_list[i] = (char*)malloc(len + 1);
       memcpy(out.i.display_list[i], data + offset, len);
       out.i.display_list[i][len] = '\0';
       offset += len;
     }
+
+    auto alloc_int_array = [&](int*& arr) {
+      arr = (int*)malloc(num * sizeof(int));
+      return read(arr, num * sizeof(int));
+    };
+
+    return alloc_int_array(out.i.left) && alloc_int_array(out.i.top) &&
+           alloc_int_array(out.i.right) && alloc_int_array(out.i.bottom);
   }
+
   return true;
 }
 
@@ -87,7 +105,13 @@ void Render::FreeRemoteAction(RemoteAction& action) {
       free(action.i.display_list[i]);
     }
     free(action.i.display_list);
+    free(action.i.left);
+    free(action.i.top);
+    free(action.i.right);
+    free(action.i.bottom);
+
     action.i.display_list = nullptr;
+    action.i.left = action.i.top = action.i.right = action.i.bottom = nullptr;
     action.i.display_num = 0;
   }
 }
@@ -921,6 +945,14 @@ void Render::MainLoop() {
       remote_action.i.display_num = display_info_list_.size();
       remote_action.i.display_list =
           (char**)malloc(remote_action.i.display_num * sizeof(char*));
+      remote_action.i.left =
+          (int*)malloc(remote_action.i.display_num * sizeof(int));
+      remote_action.i.top =
+          (int*)malloc(remote_action.i.display_num * sizeof(int));
+      remote_action.i.right =
+          (int*)malloc(remote_action.i.display_num * sizeof(int));
+      remote_action.i.bottom =
+          (int*)malloc(remote_action.i.display_num * sizeof(int));
       for (int i = 0; i < remote_action.i.display_num; i++) {
         LOG_INFO("Local display [{}:{}]", i + 1, display_info_list_[i].name);
         remote_action.i.display_list[i] =
@@ -930,6 +962,10 @@ void Render::MainLoop() {
                 display_info_list_[i].name.length());
         remote_action.i.display_list[i][display_info_list_[i].name.length()] =
             '\0';
+        remote_action.i.left[i] = display_info_list_[i].left;
+        remote_action.i.top[i] = display_info_list_[i].top;
+        remote_action.i.right[i] = display_info_list_[i].right;
+        remote_action.i.bottom[i] = display_info_list_[i].bottom;
       }
 
       std::string host_name = GetHostName();

src/single_window/render.h

@@ -83,6 +83,7 @@ class Render {
     int video_height_ = 0;
     int video_width_last_ = 0;
     int video_height_last_ = 0;
+    int selected_display_ = 1;
     size_t video_size_ = 0;
     bool tab_selected_ = false;
     bool tab_opened_ = true;
@@ -97,7 +98,7 @@ class Render {
     std::string mouse_control_button_label_ = "Mouse Control";
     std::string audio_capture_button_label_ = "Audio Capture";
     std::string remote_host_name_ = "";
-    std::vector<std::string> display_names_;
+    std::vector<ScreenCapturer::DisplayInfo> display_info_list_;
     SDL_Texture *stream_texture_ = nullptr;
     SDL_Rect stream_render_rect_;
     SDL_Rect stream_render_rect_last_;
@@ -413,7 +414,6 @@ class Render {
   bool enable_hardware_video_codec_last_ = false;
   bool enable_turn_last_ = false;
   bool settings_window_pos_reset_ = true;
-  int selected_display_ = 1;
   /* ------ main window property end ------ */
 
   /* ------ sub stream window property start ------ */

src/single_window/render_callback_func.cpp

@@ -63,9 +63,13 @@ int Render::ProcessMouseEvent(SDL_Event &event) {
       last_mouse_event.button.y = event.button.y;
 
       remote_action.m.x =
-          (float)(event.button.x - props->stream_render_rect_.x) / render_width;
+          (float)(event.button.x - props->stream_render_rect_.x +
+                  props->display_info_list_[props->selected_display_ - 1]
+                      .left) /
+          render_width;
       remote_action.m.y =
-          (float)(event.button.y - props->stream_render_rect_.y) /
+          (float)(event.button.y - props->stream_render_rect_.y +
+                  props->display_info_list_[props->selected_display_ - 1].top) /
           render_height;
 
       if (SDL_MOUSEBUTTONDOWN == event.type) {
@@ -293,9 +297,16 @@ void Render::OnReceiveDataBufferCb(const char *data, size_t size,
         LOG_INFO("Remote hostname: [{}]", props->remote_host_name_);
 
         for (int i = 0; i < host_info.i.display_num; i++) {
-          props->display_names_.push_back(
+          props->display_info_list_.push_back(
-              std::string(host_info.i.display_list[i]));
+              {std::string(host_info.i.display_list[i]), host_info.i.left[i],
-          LOG_INFO("Remote display [{}:{}]", i + 1, props->display_names_[i]);
+               host_info.i.top[i], host_info.i.right[i],
+               host_info.i.bottom[i]});
+          LOG_INFO("Remote display [{}:{}], bound [({}, {}) ({}, {})]", i + 1,
+                   props->display_info_list_[i].name,
+                   props->display_info_list_[i].left,
+                   props->display_info_list_[i].top,
+                   props->display_info_list_[i].right,
+                   props->display_info_list_[i].bottom);
         }
       }
     } else {