[feat] disable AV1 encoding when connected to a Web client

src/config_center/config_center.h

@@ -72,7 +72,7 @@ class ConfigCenter {
 
   LANGUAGE language_ = LANGUAGE::CHINESE;
   VIDEO_QUALITY video_quality_ = VIDEO_QUALITY::MEDIUM;
-  VIDEO_FRAME_RATE video_frame_rate_ = VIDEO_FRAME_RATE::FPS_30;
+  VIDEO_FRAME_RATE video_frame_rate_ = VIDEO_FRAME_RATE::FPS_60;
   VIDEO_ENCODE_FORMAT video_encode_format_ = VIDEO_ENCODE_FORMAT::H264;
   bool hardware_video_codec_ = false;
   bool enable_turn_ = false;

src/device_controller/device_controller.h

@@ -9,7 +9,11 @@
 
 #include <stdio.h>
 
+#include <nlohmann/json.hpp>
+#include <string>
+
 #include "display_info.h"
+using json = nlohmann::json;
 
 namespace crossdesk {
 
@@ -55,7 +59,7 @@ typedef struct {
   int* bottom;
 } HostInfo;
 
-typedef struct {
+struct RemoteAction {
   ControlType type;
   union {
     Mouse m;
@@ -64,7 +68,111 @@ typedef struct {
     bool a;
     int d;
   };
-} RemoteAction;
+
+  // parse
+  std::string to_json() const { return ToJson(*this); }
+
+  bool from_json(const std::string& json_str) {
+    RemoteAction temp;
+    if (!FromJson(json_str, temp)) return false;
+    *this = temp;
+    return true;
+  }
+
+  static std::string ToJson(const RemoteAction& a) {
+    json j;
+    j["type"] = a.type;
+    switch (a.type) {
+      case ControlType::mouse:
+        j["mouse"] = {
+            {"x", a.m.x}, {"y", a.m.y}, {"s", a.m.s}, {"flag", a.m.flag}};
+        break;
+      case ControlType::keyboard:
+        j["keyboard"] = {{"key_value", a.k.key_value}, {"flag", a.k.flag}};
+        break;
+      case ControlType::audio_capture:
+        j["audio_capture"] = a.a;
+        break;
+      case ControlType::display_id:
+        j["display_id"] = a.d;
+        break;
+      case ControlType::host_infomation: {
+        json displays = json::array();
+        for (size_t idx = 0; idx < a.i.display_num; idx++) {
+          displays.push_back(
+              {{"name", a.i.display_list ? a.i.display_list[idx] : ""},
+               {"left", a.i.left ? a.i.left[idx] : 0},
+               {"top", a.i.top ? a.i.top[idx] : 0},
+               {"right", a.i.right ? a.i.right[idx] : 0},
+               {"bottom", a.i.bottom ? a.i.bottom[idx] : 0}});
+        }
+
+        j["host_info"] = {{"host_name", a.i.host_name},
+                          {"display_num", a.i.display_num},
+                          {"displays", displays}};
+        break;
+      }
+    }
+    return j.dump();
+  }
+
+  static bool FromJson(const std::string& json_str, RemoteAction& out) {
+    try {
+      json j = json::parse(json_str);
+      out.type = (ControlType)j.at("type").get<int>();
+      switch (out.type) {
+        case ControlType::mouse:
+          out.m.x = j.at("mouse").at("x").get<float>();
+          out.m.y = j.at("mouse").at("y").get<float>();
+          out.m.s = j.at("mouse").at("s").get<int>();
+          out.m.flag = (MouseFlag)j.at("mouse").at("flag").get<int>();
+          break;
+        case ControlType::keyboard:
+          out.k.key_value = j.at("keyboard").at("key_value").get<size_t>();
+          out.k.flag = (KeyFlag)j.at("keyboard").at("flag").get<int>();
+          break;
+        case ControlType::audio_capture:
+          out.a = j.at("audio_capture").get<bool>();
+          break;
+        case ControlType::display_id:
+          out.d = j.at("display_id").get<int>();
+          break;
+        case ControlType::host_infomation: {
+          std::string host_name =
+              j.at("host_info").at("host_name").get<std::string>();
+          strncpy(out.i.host_name, host_name.c_str(), sizeof(out.i.host_name));
+          out.i.host_name[sizeof(out.i.host_name) - 1] = '\0';
+          out.i.host_name_size = host_name.size();
+
+          out.i.display_num = j.at("host_info").at("display_num").get<size_t>();
+          auto displays = j.at("host_info").at("displays");
+
+          out.i.display_list =
+              (char**)malloc(out.i.display_num * sizeof(char*));
+          out.i.left = (int*)malloc(out.i.display_num * sizeof(int));
+          out.i.top = (int*)malloc(out.i.display_num * sizeof(int));
+          out.i.right = (int*)malloc(out.i.display_num * sizeof(int));
+          out.i.bottom = (int*)malloc(out.i.display_num * sizeof(int));
+
+          for (size_t idx = 0; idx < out.i.display_num; idx++) {
+            std::string name = displays[idx].at("name").get<std::string>();
+            out.i.display_list[idx] = (char*)malloc(name.size() + 1);
+            strcpy(out.i.display_list[idx], name.c_str());
+            out.i.left[idx] = displays[idx].at("left").get<int>();
+            out.i.top[idx] = displays[idx].at("top").get<int>();
+            out.i.right[idx] = displays[idx].at("right").get<int>();
+            out.i.bottom[idx] = displays[idx].at("bottom").get<int>();
+          }
+          break;
+        }
+      }
+      return true;
+    } catch (const std::exception& e) {
+      printf("Failed to parse RemoteAction JSON: %s\n", e.what());
+      return false;
+    }
+  }
+};
 
 // int key_code, bool is_down
 typedef void (*OnKeyAction)(int, bool, void*);

src/gui/render.cpp

@@ -495,6 +495,8 @@ int Render::CreateConnectionPeer() {
                              : false;
   params_.enable_turn = config_center_->IsEnableTurn();
   params_.enable_srtp = config_center_->IsEnableSrtp();
+  params_.video_quality =
+      static_cast<VideoQuality>(config_center_->GetVideoQuality());
   params_.on_receive_video_buffer = nullptr;
   params_.on_receive_audio_buffer = OnReceiveAudioBufferCb;
   params_.on_receive_data_buffer = OnReceiveDataBufferCb;
@@ -566,6 +568,14 @@ void Render::UpdateInteractions() {
     screen_capturer_is_started_ = false;
   }
 
+  if (start_speaker_capturer_ && !speaker_capturer_is_started_) {
+    StartSpeakerCapturer();
+    speaker_capturer_is_started_ = true;
+  } else if (!start_speaker_capturer_ && speaker_capturer_is_started_) {
+    StopSpeakerCapturer();
+    speaker_capturer_is_started_ = false;
+  }
+
   if (start_mouse_controller_ && !mouse_controller_is_started_) {
     StartMouseController();
     mouse_controller_is_started_ = true;
@@ -1055,9 +1065,9 @@ void Render::MainLoop() {
       remote_action.i.host_name[host_name.size()] = '\0';
       remote_action.i.host_name_size = host_name.size();
 
-      std::vector<char> serialized = SerializeRemoteAction(remote_action);
-      int ret = SendDataFrame(peer_, serialized.data(), serialized.size(),
-                              data_label_.c_str());
+      std::string msg = remote_action.to_json();
+      int ret =
+          SendDataFrame(peer_, msg.data(), msg.size(), data_label_.c_str());
       FreeRemoteAction(remote_action);
       if (0 == ret) {
         need_to_send_host_info_ = false;
@@ -1183,6 +1193,10 @@ void Render::CleanupPeers() {
     LOG_INFO("[{}] Leave connection [{}]", client_id_, client_id_);
     LeaveConnection(peer_, client_id_);
     is_client_mode_ = false;
+    StopScreenCapturer();
+    StopSpeakerCapturer();
+    StopMouseController();
+    StopKeyboardCapturer();
     LOG_INFO("Destroy peer [{}]", client_id_);
     DestroyPeer(&peer_);
   }

src/gui/render.h

@@ -49,10 +49,10 @@ class Render {
     bool connection_established_ = false;
     bool rejoin_ = false;
     bool net_traffic_stats_button_pressed_ = false;
-    bool mouse_control_button_pressed_ = false;
+    bool mouse_control_button_pressed_ = true;
     bool mouse_controller_is_started_ = false;
-    bool audio_capture_button_pressed_ = false;
-    bool control_mouse_ = false;
+    bool audio_capture_button_pressed_ = true;
+    bool control_mouse_ = true;
     bool streaming_ = false;
     bool is_control_bar_in_left_ = true;
     bool control_bar_hovered_ = false;
@@ -311,7 +311,9 @@ class Render {
   bool mouse_controller_is_started_ = false;
   bool start_screen_capturer_ = false;
   bool screen_capturer_is_started_ = false;
-  bool start_keyboard_capturer_ = false;
+  bool start_speaker_capturer_ = false;
+  bool speaker_capturer_is_started_ = false;
+  bool start_keyboard_capturer_ = true;
   bool keyboard_capturer_is_started_ = false;
   bool foucs_on_main_window_ = false;
   bool foucs_on_stream_window_ = false;
@@ -436,7 +438,7 @@ class Render {
   char password_saved_[7] = "";
   int language_button_value_ = 0;
   int video_quality_button_value_ = 0;
-  int video_frame_rate_button_value_ = 0;
+  int video_frame_rate_button_value_ = 1;
   int video_encode_format_button_value_ = 0;
   bool enable_hardware_video_codec_ = false;
   bool enable_turn_ = false;
@@ -471,6 +473,9 @@ class Render {
       client_properties_;
   void CloseTab(decltype(client_properties_)::iterator& it);
   /* ------ stream window property end ------ */
+
+  /* ------ server mode ------ */
+  std::unordered_map<std::string, ConnectionStatus> connection_status_;
 };
 }  // namespace crossdesk
 #endif

src/gui/render_callback.cpp

@@ -6,11 +6,6 @@
 
 #define NV12_BUFFER_SIZE 1280 * 720 * 3 / 2
 
-#ifdef CROSSDESK_DEBUG
-#else
-#define MOUSE_CONTROL 1
-#endif
-
 namespace crossdesk {
 
 int Render::SendKeyCommand(int key_code, bool is_down) {
@@ -28,8 +23,9 @@ int Render::SendKeyCommand(int key_code, bool is_down) {
         client_properties_.end()) {
       auto props = client_properties_[controlled_remote_id_];
       if (props->connection_status_ == ConnectionStatus::Connected) {
-        SendDataFrame(props->peer_, (const char*)&remote_action,
-                      sizeof(remote_action), props->data_label_.c_str());
+        std::string msg = remote_action.to_json();
+        SendDataFrame(props->peer_, msg.c_str(), msg.size(),
+                      props->data_label_.c_str());
       }
     }
   }
@@ -94,8 +90,10 @@ int Render::ProcessMouseEvent(const SDL_Event& event) {
       if (props->control_bar_hovered_ || props->display_selectable_hovered_) {
         remote_action.m.flag = MouseFlag::move;
       }
-      SendDataFrame(props->peer_, (const char*)&remote_action,
-                    sizeof(remote_action), props->data_label_.c_str());
+
+      std::string msg = remote_action.to_json();
+      SendDataFrame(props->peer_, msg.c_str(), msg.size(),
+                    props->data_label_.c_str());
     } else if (SDL_EVENT_MOUSE_WHEEL == event.type &&
                last_mouse_event.button.x >= props->stream_render_rect_.x &&
                last_mouse_event.button.x <= props->stream_render_rect_.x +
@@ -127,8 +125,9 @@ int Render::ProcessMouseEvent(const SDL_Event& event) {
           (float)(event.button.y - props->stream_render_rect_.y) /
           render_height;
 
-      SendDataFrame(props->peer_, (const char*)&remote_action,
-                    sizeof(remote_action), props->data_label_.c_str());
+      std::string msg = remote_action.to_json();
+      SendDataFrame(props->peer_, msg.c_str(), msg.size(),
+                    props->data_label_.c_str());
     }
   }
 
@@ -226,6 +225,8 @@ void Render::OnReceiveVideoBufferCb(const XVideoFrame* video_frame,
     props->video_height_ = video_frame->height;
     props->video_size_ = video_frame->size;
 
+    LOG_ERROR("receive: {}x{}", props->video_width_, props->video_height_);
+
     if (need_to_update_render_rect) {
       render->UpdateRenderRect();
     }
@@ -279,64 +280,54 @@ void Render::OnReceiveDataBufferCb(const char* data, size_t size,
     return;
   }
 
+  std::string json_str(data, size);
   RemoteAction remote_action;
-  memcpy(&remote_action, data, size);
+
+  try {
+    remote_action.from_json(json_str);
+  } catch (const std::exception& e) {
+    LOG_ERROR("Failed to parse RemoteAction JSON: {}", e.what());
+    return;
+  }
 
   std::string remote_id(user_id, user_id_size);
   if (render->client_properties_.find(remote_id) !=
       render->client_properties_.end()) {
     // local
     auto props = render->client_properties_.find(remote_id)->second;
-    RemoteAction host_info;
-    if (DeserializeRemoteAction(data, size, host_info)) {
-      if (ControlType::host_infomation == host_info.type &&
-          props->remote_host_name_.empty()) {
-        props->remote_host_name_ =
-            std::string(host_info.i.host_name, host_info.i.host_name_size);
-        LOG_INFO("Remote hostname: [{}]", props->remote_host_name_);
-
-        for (int i = 0; i < host_info.i.display_num; i++) {
-          props->display_info_list_.push_back(DisplayInfo(
-              std::string(host_info.i.display_list[i]), host_info.i.left[i],
-              host_info.i.top[i], host_info.i.right[i], host_info.i.bottom[i]));
-          LOG_INFO("Remote display [{}:{}], bound [({}, {}) ({}, {})]", i + 1,
-                   props->display_info_list_[i].name,
-                   props->display_info_list_[i].left,
-                   props->display_info_list_[i].top,
-                   props->display_info_list_[i].right,
-                   props->display_info_list_[i].bottom);
-        }
-      }
-    } else {
+    if (remote_action.type == ControlType::host_infomation &&
+        props->remote_host_name_.empty()) {
       props->remote_host_name_ = std::string(remote_action.i.host_name,
                                              remote_action.i.host_name_size);
       LOG_INFO("Remote hostname: [{}]", props->remote_host_name_);
-      LOG_ERROR("No remote display detected");
+
+      for (int i = 0; i < remote_action.i.display_num; i++) {
+        props->display_info_list_.push_back(
+            DisplayInfo(remote_action.i.display_list[i],
+                        remote_action.i.left[i], remote_action.i.top[i],
+                        remote_action.i.right[i], remote_action.i.bottom[i]));
+      }
     }
-    FreeRemoteAction(host_info);
+    FreeRemoteAction(remote_action);
   } else {
     // remote
-    if (ControlType::mouse == remote_action.type && render->mouse_controller_) {
+    if (remote_action.type == ControlType::mouse && render->mouse_controller_) {
       render->mouse_controller_->SendMouseCommand(remote_action,
                                                   render->selected_display_);
-    } else if (ControlType::audio_capture == remote_action.type) {
-      if (remote_action.a) {
+    } else if (remote_action.type == ControlType::audio_capture) {
+      if (remote_action.a && !render->start_speaker_capturer_)
         render->StartSpeakerCapturer();
-        render->audio_capture_ = true;
-      } else {
+      else if (!remote_action.a && render->start_speaker_capturer_)
         render->StopSpeakerCapturer();
-        render->audio_capture_ = false;
-      }
-    } else if (ControlType::keyboard == remote_action.type &&
+    } else if (remote_action.type == ControlType::keyboard &&
                render->keyboard_capturer_) {
       render->keyboard_capturer_->SendKeyboardCommand(
           (int)remote_action.k.key_value,
           remote_action.k.flag == KeyFlag::key_down);
-    } else if (ControlType::display_id == remote_action.type) {
-      if (render->screen_capturer_) {
-        render->selected_display_ = remote_action.d;
-        render->screen_capturer_->SwitchTo(remote_action.d);
-      }
+    } else if (remote_action.type == ControlType::display_id &&
+               render->screen_capturer_) {
+      render->selected_display_ = remote_action.d;
+      render->screen_capturer_->SwitchTo(remote_action.d);
     }
   }
 }
@@ -409,7 +400,7 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char* user_id,
     props->connection_status_ = status;
 
     switch (status) {
-      case ConnectionStatus::Connected:
+      case ConnectionStatus::Connected: {
         if (!render->need_to_create_stream_window_ &&
             !render->client_properties_.empty()) {
           render->need_to_create_stream_window_ = true;
@@ -420,14 +411,10 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char* user_id,
             (int)render->stream_window_width_,
             (int)(render->stream_window_height_ - render->title_bar_height_)};
         break;
+      }
       case ConnectionStatus::Disconnected:
       case ConnectionStatus::Failed:
-      case ConnectionStatus::Closed:
-        render->password_validating_time_ = 0;
-        render->start_screen_capturer_ = false;
-        render->start_mouse_controller_ = false;
-        render->start_keyboard_capturer_ = false;
-        render->control_mouse_ = false;
+      case ConnectionStatus::Closed: {
         props->connection_established_ = false;
         props->mouse_control_button_pressed_ = false;
         if (props->dst_buffer_) {
@@ -436,8 +423,10 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char* user_id,
                             props->texture_width_);
         }
         render->CleanSubStreamWindowProperties(props);
+
         break;
-      case ConnectionStatus::IncorrectPassword:
+      }
+      case ConnectionStatus::IncorrectPassword: {
         render->password_validating_ = false;
         render->password_validating_time_++;
         if (render->connect_button_pressed_) {
@@ -447,37 +436,58 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char* user_id,
               localization::connect[render->localization_language_index_];
         }
         break;
-      case ConnectionStatus::NoSuchTransmissionId:
+      }
+      case ConnectionStatus::NoSuchTransmissionId: {
         if (render->connect_button_pressed_) {
           props->connection_established_ = false;
           render->connect_button_label_ =
               localization::connect[render->localization_language_index_];
         }
         break;
+      }
       default:
         break;
     }
   } else {
     render->is_client_mode_ = false;
     render->show_connection_status_window_ = true;
+    render->connection_status_[remote_id] = status;
 
     switch (status) {
-      case ConnectionStatus::Connected:
+      case ConnectionStatus::Connected: {
         render->need_to_send_host_info_ = true;
         render->start_screen_capturer_ = true;
-        render->start_mouse_controller_ = true;
-        break;
-      case ConnectionStatus::Closed:
-        render->start_screen_capturer_ = false;
+        render->start_speaker_capturer_ = true;
+#ifdef CROSSDESK_DEBUG
         render->start_mouse_controller_ = false;
         render->start_keyboard_capturer_ = false;
-        render->need_to_send_host_info_ = false;
-        if (props) props->connection_established_ = false;
-        if (render->audio_capture_) {
-          render->StopSpeakerCapturer();
-          render->audio_capture_ = false;
+#else
+        render->start_mouse_controller_ = true;
+#endif
+        break;
+      }
+      case ConnectionStatus::Closed: {
+        if (std::all_of(render->connection_status_.begin(),
+                        render->connection_status_.end(), [](const auto& kv) {
+                          return kv.second == ConnectionStatus::Closed ||
+                                 kv.second == ConnectionStatus::Failed ||
+                                 kv.second == ConnectionStatus::Disconnected;
+                        })) {
+          render->start_screen_capturer_ = false;
+          render->start_speaker_capturer_ = false;
+          render->start_mouse_controller_ = false;
+          render->start_keyboard_capturer_ = false;
+          render->need_to_send_host_info_ = false;
+          if (props) props->connection_established_ = false;
+          if (render->audio_capture_) {
+            render->StopSpeakerCapturer();
+            render->audio_capture_ = false;
+          }
+
+          render->connection_status_.erase(remote_id);
         }
         break;
+      }
       default:
         break;
     }

src/gui/toolbars/control_bar.cpp

@@ -68,8 +68,9 @@ int Render::ControlBar(std::shared_ptr<SubStreamWindowProperties>& props) {
           remote_action.type = ControlType::display_id;
           remote_action.d = i;
           if (props->connection_status_ == ConnectionStatus::Connected) {
-            SendDataFrame(props->peer_, (const char*)&remote_action,
-                          sizeof(remote_action), props->data_label_.c_str());
+            std::string msg = remote_action.to_json();
+            SendDataFrame(props->peer_, msg.c_str(), msg.size(),
+                          props->data_label_.c_str());
           }
         }
         props->display_selectable_hovered_ = ImGui::IsWindowHovered();
@@ -142,8 +143,9 @@ int Render::ControlBar(std::shared_ptr<SubStreamWindowProperties>& props) {
         RemoteAction remote_action;
         remote_action.type = ControlType::audio_capture;
         remote_action.a = props->audio_capture_button_pressed_;
-        SendDataFrame(props->peer_, (const char*)&remote_action,
-                      sizeof(remote_action), props->data_label_.c_str());
+        std::string msg = remote_action.to_json();
+        SendDataFrame(props->peer_, msg.c_str(), msg.size(),
+                      props->data_label_.c_str());
       }
     }
     if (!props->audio_capture_button_pressed_) {

src/gui/windows/main_settings_window.cpp

@@ -82,11 +82,11 @@ int Render::SettingWindow() {
 
       {
         const char* video_quality_items[] = {
-            localization::video_quality_high[localization_language_index_]
+            localization::video_quality_low[localization_language_index_]
                 .c_str(),
             localization::video_quality_medium[localization_language_index_]
                 .c_str(),
-            localization::video_quality_low[localization_language_index_]
+            localization::video_quality_high[localization_language_index_]
                 .c_str()};
 
         settings_items_offset += settings_items_padding;

xmake.lua

@@ -23,6 +23,7 @@ end
 add_requires("spdlog 1.14.1", {system = false})
 add_requires("imgui v1.91.5-docking", {configs = {sdl3 = true, sdl3_renderer = true}})
 add_requires("openssl3 3.3.2", {system = false})
+add_requires("nlohmann_json 3.11.3")
 
 if is_os("windows") then
     add_requires("libyuv", "miniaudio 0.11.21")
@@ -45,7 +46,7 @@ elseif is_os("macosx") then
         "CoreMedia", "CoreVideo", "CoreAudio", "AudioToolbox")
 end
 
-add_packages("spdlog", "imgui")
+add_packages("spdlog", "imgui", "nlohmann_json")
 
 includes("submodules")