[feat] cursor mapping in different displays supported

src/common/display_info.h

@@ -0,0 +1,44 @@
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2025-05-15
+ * Copyright (c) 2025 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _DISPLAY_INFO_H_
+#define _DISPLAY_INFO_H_
+
+#include <string>
+
+class DisplayInfo {
+ public:
+  DisplayInfo(std::string name, int left, int top, int right, int bottom)
+      : name(name), left(left), top(top), right(right), bottom(bottom) {
+    width = right - left;
+    height = bottom - top;
+  }
+  DisplayInfo(void* handle, std::string name, bool is_primary, int left,
+              int top, int right, int bottom)
+      : handle(handle),
+        name(name),
+        is_primary(is_primary),
+        left(left),
+        top(top),
+        right(right),
+        bottom(bottom) {
+    width = right - left;
+    height = bottom - top;
+  }
+  ~DisplayInfo() {}
+
+  void* handle = nullptr;
+  std::string name = "";
+  bool is_primary = false;
+  int left = 0;
+  int top = 0;
+  int right = 0;
+  int bottom = 0;
+  int width = 0;
+  int height = 0;
+};
+
+#endif

src/device_controller/device_controller.h

@@ -9,6 +9,8 @@
 
 #include <stdio.h>
 
+#include "display_info.h"
+
 typedef enum {
   mouse = 0,
   keyboard,

src/device_controller/mouse/windows/mouse_controller.cpp

@@ -6,22 +6,26 @@ MouseController::MouseController() {}
 
 MouseController::~MouseController() {}
 
-int MouseController::Init(int screen_width, int screen_height) {
-  screen_width_ = screen_width;
-  screen_height_ = screen_height;
+int MouseController::Init(std::vector<DisplayInfo> display_info_list) {
+  display_info_list_ = display_info_list;
 
   return 0;
 }
 
 int MouseController::Destroy() { return 0; }
 
-int MouseController::SendMouseCommand(RemoteAction remote_action) {
+int MouseController::SendMouseCommand(RemoteAction remote_action,
+                                      int display_index) {
   INPUT ip;
 
   if (remote_action.type == ControlType::mouse) {
     ip.type = INPUT_MOUSE;
-    ip.mi.dx = (LONG)(remote_action.m.x * screen_width_);
-    ip.mi.dy = (LONG)(remote_action.m.y * screen_height_);
+    ip.mi.dx =
+        (LONG)(remote_action.m.x * display_info_list_[display_index].width) +
+        display_info_list_[display_index].left;
+    ip.mi.dy =
+        (LONG)(remote_action.m.y * display_info_list_[display_index].height) +
+        display_info_list_[display_index].top;
 
     switch (remote_action.m.flag) {
       case MouseFlag::left_down:
@@ -58,6 +62,7 @@ int MouseController::SendMouseCommand(RemoteAction remote_action) {
     ip.mi.time = 0;
 
     SetCursorPos(ip.mi.dx, ip.mi.dy);
+    LOG_ERROR("mouse {}x{}", ip.mi.dx, ip.mi.dy);
 
     if (ip.mi.dwFlags != MOUSEEVENTF_MOVE) {
       SendInput(1, &ip, sizeof(INPUT));

src/device_controller/mouse/windows/mouse_controller.h

@@ -7,6 +7,8 @@
 #ifndef _MOUSE_CONTROLLER_H_
 #define _MOUSE_CONTROLLER_H_
 
+#include <vector>
+
 #include "device_controller.h"
 
 class MouseController : public DeviceController {
@@ -15,13 +17,12 @@ class MouseController : public DeviceController {
   virtual ~MouseController();
 
  public:
-  virtual int Init(int screen_width, int screen_height);
+  virtual int Init(std::vector<DisplayInfo> display_info_list);
   virtual int Destroy();
-  virtual int SendMouseCommand(RemoteAction remote_action);
+  virtual int SendMouseCommand(RemoteAction remote_action, int display_index);
 
  private:
-  int screen_width_ = 0;
-  int screen_height_ = 0;
+  std::vector<DisplayInfo> display_info_list_;
 };
 
 #endif

src/screen_capturer/screen_capturer.h

@@ -9,35 +9,13 @@
 
 #include <functional>
 
+#include "display_info.h"
+
 class ScreenCapturer {
  public:
   typedef std::function<void(unsigned char*, int, int, int, int)>
       cb_desktop_data;
 
-  class DisplayInfo {
-   public:
-    DisplayInfo(std::string name, int left, int top, int right, int bottom)
-        : name(name), left(left), top(top), right(right), bottom(bottom) {}
-    DisplayInfo(void* handle, std::string name, bool is_primary, int left,
-                int top, int right, int bottom)
-        : handle(handle),
-          name(name),
-          is_primary(is_primary),
-          left(left),
-          top(top),
-          right(right),
-          bottom(bottom) {}
-    ~DisplayInfo() {}
-
-    void* handle = nullptr;
-    std::string name = "";
-    bool is_primary = false;
-    int left = 0;
-    int top = 0;
-    int right = 0;
-    int bottom = 0;
-  };
-
  public:
   virtual ~ScreenCapturer() {}
 

src/screen_capturer/windows/screen_capturer_wgc.cpp

@@ -10,7 +10,7 @@
 #include "libyuv.h"
 #include "rd_log.h"
 
-static std::vector<ScreenCapturer::DisplayInfo> gs_display_list;
+static std::vector<DisplayInfo> gs_display_list;
 
 std::string WideToUtf8(const wchar_t *wideStr) {
   int size_needed = WideCharToMultiByte(CP_UTF8, 0, wideStr, -1, nullptr, 0,
@@ -37,11 +37,11 @@ BOOL WINAPI EnumMonitorProc(HMONITOR hmonitor, [[maybe_unused]] HDC hdc,
            monitor_info_.rcMonitor.right, monitor_info_.rcMonitor.bottom});
       *(HMONITOR *)data = hmonitor;
     } else {
-      gs_display_list.push_back(
-          {(void *)hmonitor, WideToUtf8(monitor_info_.szDevice),
-           (monitor_info_.dwFlags & MONITORINFOF_PRIMARY) ? true : false,
-           monitor_info_.rcMonitor.left, monitor_info_.rcMonitor.top,
-           monitor_info_.rcMonitor.right, monitor_info_.rcMonitor.bottom});
+      gs_display_list.push_back(DisplayInfo(
+          (void *)hmonitor, WideToUtf8(monitor_info_.szDevice),
+          (monitor_info_.dwFlags & MONITORINFOF_PRIMARY) ? true : false,
+          monitor_info_.rcMonitor.left, monitor_info_.rcMonitor.top,
+          monitor_info_.rcMonitor.right, monitor_info_.rcMonitor.bottom));
     }
   }
 

src/single_window/render.cpp

@@ -369,8 +369,11 @@ int Render::StartMouseController() {
   }
   mouse_controller_ = (MouseController*)device_controller_factory_->Create(
       DeviceControllerFactory::Device::Mouse);
-  int mouse_controller_init_ret =
-      mouse_controller_->Init(screen_width_, screen_height_);
+
+  if (screen_capturer_ && display_info_list_.empty()) {
+    display_info_list_ = screen_capturer_->GetDisplayInfoList();
+  }
+  int mouse_controller_init_ret = mouse_controller_->Init(display_info_list_);
   if (0 != mouse_controller_init_ret) {
     LOG_INFO("Destroy mouse controller")
     mouse_controller_->Destroy();
@@ -937,11 +940,11 @@ void Render::MainLoop() {
     }
 
     if (need_to_send_host_info_) {
-      RemoteAction remote_action;
-      if (screen_capturer_) {
+      if (screen_capturer_ && display_info_list_.empty()) {
         display_info_list_ = screen_capturer_->GetDisplayInfoList();
       }
 
+      RemoteAction remote_action;
       remote_action.i.display_num = display_info_list_.size();
       remote_action.i.display_list =
           (char**)malloc(remote_action.i.display_num * sizeof(char*));

src/single_window/render.h

@@ -83,7 +83,7 @@ class Render {
     int video_height_ = 0;
     int video_width_last_ = 0;
     int video_height_last_ = 0;
-    int selected_display_ = 1;
+    int selected_display_ = 0;
     size_t video_size_ = 0;
     bool tab_selected_ = false;
     bool tab_opened_ = true;
@@ -98,7 +98,7 @@ class Render {
     std::string mouse_control_button_label_ = "Mouse Control";
     std::string audio_capture_button_label_ = "Audio Capture";
     std::string remote_host_name_ = "";
-    std::vector<ScreenCapturer::DisplayInfo> display_info_list_;
+    std::vector<DisplayInfo> display_info_list_;
     SDL_Texture *stream_texture_ = nullptr;
     SDL_Rect stream_render_rect_;
     SDL_Rect stream_render_rect_last_;
@@ -316,6 +316,7 @@ class Render {
   float about_window_height_ = 150;
   int screen_width_ = 1280;
   int screen_height_ = 720;
+  int selected_display_ = 0;
   std::string connect_button_label_ = "Connect";
   char input_password_tmp_[7] = "";
   char input_password_[7] = "";
@@ -398,7 +399,7 @@ class Render {
   DeviceControllerFactory *device_controller_factory_ = nullptr;
   MouseController *mouse_controller_ = nullptr;
   KeyboardCapturer *keyboard_capturer_ = nullptr;
-  std::vector<ScreenCapturer::DisplayInfo> display_info_list_;
+  std::vector<DisplayInfo> display_info_list_;
   uint64_t last_frame_time_;
   char client_id_[10] = "";
   char client_id_display_[12] = "";

src/single_window/render_callback_func.cpp

@@ -63,13 +63,9 @@ int Render::ProcessMouseEvent(SDL_Event &event) {
       last_mouse_event.button.y = event.button.y;
 
       remote_action.m.x =
-          (float)(event.button.x - props->stream_render_rect_.x +
-                  props->display_info_list_[props->selected_display_ - 1]
-                      .left) /
-          render_width;
+          (float)(event.button.x - props->stream_render_rect_.x) / render_width;
       remote_action.m.y =
-          (float)(event.button.y - props->stream_render_rect_.y +
-                  props->display_info_list_[props->selected_display_ - 1].top) /
+          (float)(event.button.y - props->stream_render_rect_.y) /
           render_height;
 
       if (SDL_MOUSEBUTTONDOWN == event.type) {
@@ -297,10 +293,9 @@ void Render::OnReceiveDataBufferCb(const char *data, size_t size,
         LOG_INFO("Remote hostname: [{}]", props->remote_host_name_);
 
         for (int i = 0; i < host_info.i.display_num; i++) {
-          props->display_info_list_.push_back(
-              {std::string(host_info.i.display_list[i]), host_info.i.left[i],
-               host_info.i.top[i], host_info.i.right[i],
-               host_info.i.bottom[i]});
+          props->display_info_list_.push_back(DisplayInfo(
+              std::string(host_info.i.display_list[i]), host_info.i.left[i],
+              host_info.i.top[i], host_info.i.right[i], host_info.i.bottom[i]));
           LOG_INFO("Remote display [{}:{}], bound [({}, {}) ({}, {})]", i + 1,
                    props->display_info_list_[i].name,
                    props->display_info_list_[i].left,
@@ -319,7 +314,8 @@ void Render::OnReceiveDataBufferCb(const char *data, size_t size,
   } else {
     // remote
     if (ControlType::mouse == remote_action.type && render->mouse_controller_) {
-      render->mouse_controller_->SendMouseCommand(remote_action);
+      render->mouse_controller_->SendMouseCommand(remote_action,
+                                                  render->selected_display_);
     } else if (ControlType::audio_capture == remote_action.type) {
       if (remote_action.a) {
         render->StartSpeakerCapturer();
@@ -335,6 +331,7 @@ void Render::OnReceiveDataBufferCb(const char *data, size_t size,
           remote_action.k.flag == KeyFlag::key_down);
     } else if (ControlType::display_id == remote_action.type) {
       if (render->screen_capturer_) {
+        render->selected_display_ = remote_action.d;
         render->screen_capturer_->SwitchTo(remote_action.d);
       }
     }