[fix] fix Wayland reconnect black screen by keeping capturer warm and also fix Wayland mouse control

src/gui/render.cpp

@@ -8,6 +8,7 @@
 #endif
 
 #include <cstdlib>
+#include <cmath>
 #include <cstring>
 #include <filesystem>
 #include <fstream>
@@ -580,8 +581,9 @@ int Render::ScreenCapturerInit() {
 
   if (0 == screen_capturer_init_ret) {
     LOG_INFO("Init screen capturer success");
-    if (display_info_list_.empty()) {
-      display_info_list_ = screen_capturer_->GetDisplayInfoList();
+    const auto latest_display_info = screen_capturer_->GetDisplayInfoList();
+    if (!latest_display_info.empty()) {
+      display_info_list_ = latest_display_info;
     }
     return 0;
   } else {
@@ -594,10 +596,22 @@ int Render::ScreenCapturerInit() {
 }
 
 int Render::StartScreenCapturer() {
+  if (!screen_capturer_) {
+    LOG_INFO("Screen capturer instance missing, recreating before start");
+    if (0 != ScreenCapturerInit()) {
+      LOG_ERROR("Recreate screen capturer failed");
+      return -1;
+    }
+  }
+
   if (screen_capturer_) {
     LOG_INFO("Start screen capturer, show cursor: {}", show_cursor_);
 
-    screen_capturer_->Start(show_cursor_);
+    const int ret = screen_capturer_->Start(show_cursor_);
+    if (ret != 0) {
+      LOG_ERROR("Start screen capturer failed: {}", ret);
+      return ret;
+    }
   }
 
   return 0;
@@ -650,14 +664,42 @@ int Render::StartMouseController() {
     LOG_INFO("Device controller factory is nullptr");
     return -1;
   }
+
+#if defined(__linux__) && !defined(__APPLE__)
+  if (IsWaylandSession()) {
+    if (!screen_capturer_) {
+      return 1;
+    }
+
+    const auto latest_display_info = screen_capturer_->GetDisplayInfoList();
+    if (latest_display_info.empty() ||
+        latest_display_info[0].handle == nullptr) {
+      return 1;
+    }
+  }
+
+  if (screen_capturer_) {
+    const auto latest_display_info = screen_capturer_->GetDisplayInfoList();
+    if (!latest_display_info.empty()) {
+      display_info_list_ = latest_display_info;
+    }
+  }
+#endif
+
   mouse_controller_ = (MouseController*)device_controller_factory_->Create(
       DeviceControllerFactory::Device::Mouse);
+  if (!mouse_controller_) {
+    LOG_ERROR("Create mouse controller failed");
+    return -1;
+  }
 
   int mouse_controller_init_ret = mouse_controller_->Init(display_info_list_);
   if (0 != mouse_controller_init_ret) {
     LOG_INFO("Destroy mouse controller");
     mouse_controller_->Destroy();
+    delete mouse_controller_;
     mouse_controller_ = nullptr;
+    return mouse_controller_init_ret;
   }
 
   return 0;
@@ -924,9 +966,24 @@ int Render::AudioDeviceDestroy() {
 }
 
 void Render::UpdateInteractions() {
+#if defined(__linux__) && !defined(__APPLE__)
+  const bool is_wayland_session = IsWaylandSession();
+  const bool stop_wayland_mouse_before_screen =
+      is_wayland_session && !start_screen_capturer_ &&
+      screen_capturer_is_started_ && !start_mouse_controller_ &&
+      mouse_controller_is_started_;
+  if (stop_wayland_mouse_before_screen) {
+    LOG_INFO("Stopping Wayland mouse controller before screen capturer to "
+             "cleanly release the shared portal session");
+    StopMouseController();
+    mouse_controller_is_started_ = false;
+  }
+#endif
+
   if (start_screen_capturer_ && !screen_capturer_is_started_) {
-    StartScreenCapturer();
-    screen_capturer_is_started_ = true;
+    if (0 == StartScreenCapturer()) {
+      screen_capturer_is_started_ = true;
+    }
   } else if (!start_screen_capturer_ && screen_capturer_is_started_) {
     StopScreenCapturer();
     screen_capturer_is_started_ = false;
@@ -941,13 +998,24 @@ void Render::UpdateInteractions() {
   }
 
   if (start_mouse_controller_ && !mouse_controller_is_started_) {
-    StartMouseController();
-    mouse_controller_is_started_ = true;
+    if (0 == StartMouseController()) {
+      mouse_controller_is_started_ = true;
+    }
   } else if (!start_mouse_controller_ && mouse_controller_is_started_) {
     StopMouseController();
     mouse_controller_is_started_ = false;
   }
 
+#if defined(__linux__) && !defined(__APPLE__)
+  if (screen_capturer_is_started_ && screen_capturer_ && mouse_controller_) {
+    const auto latest_display_info = screen_capturer_->GetDisplayInfoList();
+    if (!latest_display_info.empty()) {
+      display_info_list_ = latest_display_info;
+      mouse_controller_->UpdateDisplayInfoList(display_info_list_);
+    }
+  }
+#endif
+
   if (start_keyboard_capturer_ && focus_on_stream_window_) {
     if (!keyboard_capturer_is_started_) {
       StartKeyboardCapturer();
@@ -1439,10 +1507,8 @@ int Render::DrawStreamWindow() {
     auto props = it.second;
     if (props->tab_selected_) {
       SDL_FRect render_rect_f = {
-          static_cast<float>(props->stream_render_rect_.x),
-          static_cast<float>(props->stream_render_rect_.y),
-          static_cast<float>(props->stream_render_rect_.w),
-          static_cast<float>(props->stream_render_rect_.h)};
+          props->stream_render_rect_f_.x, props->stream_render_rect_f_.y,
+          props->stream_render_rect_f_.w, props->stream_render_rect_f_.h};
       SDL_RenderTexture(stream_renderer_, props->stream_texture_, NULL,
                         &render_rect_f);
     }
@@ -1850,6 +1916,12 @@ void Render::HandleServerWindow() {
 void Render::Cleanup() {
   Clipboard::StopMonitoring();
 
+  if (mouse_controller_) {
+    mouse_controller_->Destroy();
+    delete mouse_controller_;
+    mouse_controller_ = nullptr;
+  }
+
   if (screen_capturer_) {
     screen_capturer_->Destroy();
     delete screen_capturer_;
@@ -1862,12 +1934,6 @@ void Render::Cleanup() {
     speaker_capturer_ = nullptr;
   }
 
-  if (mouse_controller_) {
-    mouse_controller_->Destroy();
-    delete mouse_controller_;
-    mouse_controller_ = nullptr;
-  }
-
   if (keyboard_capturer_) {
     delete keyboard_capturer_;
     keyboard_capturer_ = nullptr;
@@ -1949,9 +2015,9 @@ void Render::CleanupPeers() {
     LOG_INFO("[{}] Leave connection [{}]", client_id_, client_id_);
     LeaveConnection(peer_, client_id_);
     is_client_mode_ = false;
+    StopMouseController();
     StopScreenCapturer();
     StopSpeakerCapturer();
-    StopMouseController();
     StopKeyboardCapturer();
     LOG_INFO("Destroy peer [{}]", client_id_);
     DestroyPeer(&peer_);
@@ -2229,26 +2295,36 @@ void Render::UpdateRenderRect() {
     float render_area_height = props->render_window_height_;
 
     props->stream_render_rect_last_ = props->stream_render_rect_;
+
+    SDL_FRect rect_f{props->render_window_x_, props->render_window_y_,
+                     render_area_width, render_area_height};
     if (render_area_width < render_area_height * video_ratio) {
-      props->stream_render_rect_ = {
-          (int)props->render_window_x_,
-          (int)(abs(render_area_height -
-                    render_area_width * video_ratio_reverse) /
-                    2 +
-                (int)props->render_window_y_),
-          (int)render_area_width,
-          (int)(render_area_width * video_ratio_reverse)};
+      rect_f.x = props->render_window_x_;
+      rect_f.y = std::abs(render_area_height -
+                          render_area_width * video_ratio_reverse) /
+                     2.0f +
+                 props->render_window_y_;
+      rect_f.w = render_area_width;
+      rect_f.h = render_area_width * video_ratio_reverse;
     } else if (render_area_width > render_area_height * video_ratio) {
-      props->stream_render_rect_ = {
-          (int)abs(render_area_width - render_area_height * video_ratio) / 2 +
-              (int)props->render_window_x_,
-          (int)props->render_window_y_, (int)(render_area_height * video_ratio),
-          (int)render_area_height};
+      rect_f.x =
+          std::abs(render_area_width - render_area_height * video_ratio) / 2.0f +
+          props->render_window_x_;
+      rect_f.y = props->render_window_y_;
+      rect_f.w = render_area_height * video_ratio;
+      rect_f.h = render_area_height;
     } else {
-      props->stream_render_rect_ = {
-          (int)props->render_window_x_, (int)props->render_window_y_,
-          (int)render_area_width, (int)render_area_height};
+      rect_f.x = props->render_window_x_;
+      rect_f.y = props->render_window_y_;
+      rect_f.w = render_area_width;
+      rect_f.h = render_area_height;
     }
+
+    props->stream_render_rect_f_ = rect_f;
+    props->stream_render_rect_ = {static_cast<int>(std::lround(rect_f.x)),
+                                  static_cast<int>(std::lround(rect_f.y)),
+                                  static_cast<int>(std::lround(rect_f.w)),
+                                  static_cast<int>(std::lround(rect_f.h))};
   }
 }
 
@@ -2389,12 +2465,23 @@ void Render::ProcessSdlEvent(const SDL_Event& event) {
     case SDL_EVENT_MOUSE_MOTION:
     case SDL_EVENT_MOUSE_BUTTON_DOWN:
     case SDL_EVENT_MOUSE_BUTTON_UP:
-    case SDL_EVENT_MOUSE_WHEEL:
+    case SDL_EVENT_MOUSE_WHEEL: {
+      Uint32 mouse_window_id = 0;
+      if (event.type == SDL_EVENT_MOUSE_MOTION) {
+        mouse_window_id = event.motion.windowID;
+      } else if (event.type == SDL_EVENT_MOUSE_BUTTON_DOWN ||
+                 event.type == SDL_EVENT_MOUSE_BUTTON_UP) {
+        mouse_window_id = event.button.windowID;
+      } else if (event.type == SDL_EVENT_MOUSE_WHEEL) {
+        mouse_window_id = event.wheel.windowID;
+      }
+
       if (focus_on_stream_window_ && stream_window_ &&
-          SDL_GetWindowID(stream_window_) == event.motion.windowID) {
+          SDL_GetWindowID(stream_window_) == mouse_window_id) {
         ProcessMouseEvent(event);
       }
       break;
+    }
 
     default:
       if (event.type == STREAM_REFRESH_EVENT) {

src/gui/render.h

@@ -160,6 +160,7 @@ class Render {
     SDL_Texture* stream_texture_ = nullptr;
     uint8_t* argb_buffer_ = nullptr;
     int argb_buffer_size_ = 0;
+    SDL_FRect stream_render_rect_f_ = {0.0f, 0.0f, 0.0f, 0.0f};
     SDL_Rect stream_render_rect_;
     SDL_Rect stream_render_rect_last_;
     ImVec2 control_window_pos_;

src/gui/render_callback.cpp

@@ -1,3 +1,4 @@
+#include <algorithm>
 #include <chrono>
 #include <cmath>
 #include <cstdlib>
@@ -109,10 +110,67 @@ int Render::SendKeyCommand(int key_code, bool is_down) {
 
 int Render::ProcessMouseEvent(const SDL_Event& event) {
   controlled_remote_id_ = "";
-  int video_width, video_height = 0;
-  int render_width, render_height = 0;
-  float ratio_x, ratio_y = 0;
   RemoteAction remote_action;
+  float cursor_x = last_mouse_event.motion.x;
+  float cursor_y = last_mouse_event.motion.y;
+
+  auto normalize_cursor_to_window_space = [&](float* x, float* y) {
+    if (!x || !y || !stream_window_) {
+      return;
+    }
+
+    int window_width = 0;
+    int window_height = 0;
+    int pixel_width = 0;
+    int pixel_height = 0;
+    SDL_GetWindowSize(stream_window_, &window_width, &window_height);
+    SDL_GetWindowSizeInPixels(stream_window_, &pixel_width, &pixel_height);
+
+    if (window_width <= 0 || window_height <= 0 || pixel_width <= 0 ||
+        pixel_height <= 0) {
+      return;
+    }
+
+    if ((window_width != pixel_width || window_height != pixel_height) &&
+        (*x > static_cast<float>(window_width) + 1.0f ||
+         *y > static_cast<float>(window_height) + 1.0f)) {
+      const float scale_x =
+          static_cast<float>(window_width) / static_cast<float>(pixel_width);
+      const float scale_y =
+          static_cast<float>(window_height) / static_cast<float>(pixel_height);
+      *x *= scale_x;
+      *y *= scale_y;
+
+      static bool logged_pixel_to_window_conversion = false;
+      if (!logged_pixel_to_window_conversion) {
+        LOG_INFO(
+            "Mouse coordinate space converted from pixels to window units: "
+            "window={}x{}, pixels={}x{}, scale=({:.4f},{:.4f})",
+            window_width, window_height, pixel_width, pixel_height, scale_x,
+            scale_y);
+        logged_pixel_to_window_conversion = true;
+      }
+    }
+  };
+
+  if (event.type == SDL_EVENT_MOUSE_MOTION) {
+    cursor_x = event.motion.x;
+    cursor_y = event.motion.y;
+    normalize_cursor_to_window_space(&cursor_x, &cursor_y);
+  } else if (event.type == SDL_EVENT_MOUSE_BUTTON_DOWN ||
+             event.type == SDL_EVENT_MOUSE_BUTTON_UP) {
+    cursor_x = event.button.x;
+    cursor_y = event.button.y;
+    normalize_cursor_to_window_space(&cursor_x, &cursor_y);
+  } else if (event.type == SDL_EVENT_MOUSE_WHEEL) {
+    cursor_x = last_mouse_event.motion.x;
+    cursor_y = last_mouse_event.motion.y;
+  }
+
+  const bool is_pointer_position_event =
+      (event.type == SDL_EVENT_MOUSE_MOTION ||
+       event.type == SDL_EVENT_MOUSE_BUTTON_DOWN ||
+       event.type == SDL_EVENT_MOUSE_BUTTON_UP);
 
   // std::shared_lock lock(client_properties_mutex_);
   for (auto& it : client_properties_) {
@@ -121,23 +179,24 @@ int Render::ProcessMouseEvent(const SDL_Event& event) {
       continue;
     }
 
-    if (event.button.x >= props->stream_render_rect_.x &&
-        event.button.x <=
-            props->stream_render_rect_.x + props->stream_render_rect_.w &&
-        event.button.y >= props->stream_render_rect_.y &&
-        event.button.y <=
-            props->stream_render_rect_.y + props->stream_render_rect_.h) {
-      controlled_remote_id_ = it.first;
-      render_width = props->stream_render_rect_.w;
-      render_height = props->stream_render_rect_.h;
-      last_mouse_event.button.x = event.button.x;
-      last_mouse_event.button.y = event.button.y;
+    const SDL_FRect render_rect = props->stream_render_rect_f_;
+    if (render_rect.w <= 1.0f || render_rect.h <= 1.0f) {
+      continue;
+    }
 
-      remote_action.m.x =
-          (float)(event.button.x - props->stream_render_rect_.x) / render_width;
-      remote_action.m.y =
-          (float)(event.button.y - props->stream_render_rect_.y) /
-          render_height;
+    if (is_pointer_position_event && cursor_x >= render_rect.x &&
+        cursor_x <= render_rect.x + render_rect.w && cursor_y >= render_rect.y &&
+        cursor_y <= render_rect.y + render_rect.h) {
+      controlled_remote_id_ = it.first;
+      last_mouse_event.motion.x = cursor_x;
+      last_mouse_event.motion.y = cursor_y;
+      last_mouse_event.button.x = cursor_x;
+      last_mouse_event.button.y = cursor_y;
+
+      remote_action.m.x = (cursor_x - render_rect.x) / render_rect.w;
+      remote_action.m.y = (cursor_y - render_rect.y) / render_rect.h;
+      remote_action.m.x = std::clamp(remote_action.m.x, 0.0f, 1.0f);
+      remote_action.m.y = std::clamp(remote_action.m.y, 0.0f, 1.0f);
 
       if (SDL_EVENT_MOUSE_BUTTON_DOWN == event.type) {
         remote_action.type = ControlType::mouse;
@@ -171,12 +230,10 @@ int Render::ProcessMouseEvent(const SDL_Event& event) {
                       props->data_label_.c_str());
       }
     } else if (SDL_EVENT_MOUSE_WHEEL == event.type &&
-               last_mouse_event.button.x >= props->stream_render_rect_.x &&
-               last_mouse_event.button.x <= props->stream_render_rect_.x +
-                                                props->stream_render_rect_.w &&
-               last_mouse_event.button.y >= props->stream_render_rect_.y &&
-               last_mouse_event.button.y <= props->stream_render_rect_.y +
-                                                props->stream_render_rect_.h) {
+               last_mouse_event.button.x >= render_rect.x &&
+               last_mouse_event.button.x <= render_rect.x + render_rect.w &&
+               last_mouse_event.button.y >= render_rect.y &&
+               last_mouse_event.button.y <= render_rect.y + render_rect.h) {
       float scroll_x = event.wheel.x;
       float scroll_y = event.wheel.y;
       if (event.wheel.direction == SDL_MOUSEWHEEL_FLIPPED) {
@@ -203,14 +260,12 @@ int Render::ProcessMouseEvent(const SDL_Event& event) {
         remote_action.m.s = roundUp(scroll_x);
       }
 
-      render_width = props->stream_render_rect_.w;
-      render_height = props->stream_render_rect_.h;
-      remote_action.m.x =
-          (float)(last_mouse_event.button.x - props->stream_render_rect_.x) /
-          render_width;
-      remote_action.m.y =
-          (float)(last_mouse_event.button.y - props->stream_render_rect_.y) /
-          render_height;
+      remote_action.m.x = (last_mouse_event.button.x - render_rect.x) /
+                          (std::max)(render_rect.w, 1.0f);
+      remote_action.m.y = (last_mouse_event.button.y - render_rect.y) /
+                          (std::max)(render_rect.h, 1.0f);
+      remote_action.m.x = std::clamp(remote_action.m.x, 0.0f, 1.0f);
+      remote_action.m.y = std::clamp(remote_action.m.y, 0.0f, 1.0f);
 
       if (props->control_bar_hovered_) {
         continue;
@@ -783,6 +838,9 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char* user_id,
             0, (int)render->title_bar_height_,
             (int)render->stream_window_width_,
             (int)(render->stream_window_height_ - render->title_bar_height_)};
+        props->stream_render_rect_f_ = {
+            0.0f, render->title_bar_height_, render->stream_window_width_,
+            render->stream_window_height_ - render->title_bar_height_};
         render->start_keyboard_capturer_ = true;
         break;
       }
@@ -910,7 +968,19 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char* user_id,
                         })) {
           render->need_to_destroy_server_window_ = true;
           render->is_server_mode_ = false;
+#if defined(__linux__) && !defined(__APPLE__)
+          if (IsWaylandSession()) {
+            // Keep Wayland capture session warm to avoid black screen on
+            // subsequent reconnects.
+            render->start_screen_capturer_ = true;
+            LOG_INFO("Keeping Wayland screen capturer running after "
+                     "disconnect to preserve reconnect stability");
+          } else {
+            render->start_screen_capturer_ = false;
+          }
+#else
           render->start_screen_capturer_ = false;
+#endif
           render->start_speaker_capturer_ = false;
           render->start_mouse_controller_ = false;
           render->start_keyboard_capturer_ = false;
@@ -1074,4 +1144,4 @@ void Render::OnNetStatusReport(const char* client_id, size_t client_id_size,
     props->net_traffic_stats_ = *net_traffic_stats;
   }
 }
-}  // namespace crossdesk
+}  // namespace crossdesk