[feat] disable AV1 encoding when connected to a Web client

src/config_center/config_center.h

@@ -72,7 +72,7 @@ class ConfigCenter {
 
   LANGUAGE language_ = LANGUAGE::CHINESE;
   VIDEO_QUALITY video_quality_ = VIDEO_QUALITY::MEDIUM;
-  VIDEO_FRAME_RATE video_frame_rate_ = VIDEO_FRAME_RATE::FPS_30;
+  VIDEO_FRAME_RATE video_frame_rate_ = VIDEO_FRAME_RATE::FPS_60;
   VIDEO_ENCODE_FORMAT video_encode_format_ = VIDEO_ENCODE_FORMAT::H264;
   bool hardware_video_codec_ = false;
   bool enable_turn_ = false;

src/gui/render.cpp

@@ -495,6 +495,8 @@ int Render::CreateConnectionPeer() {
                              : false;
   params_.enable_turn = config_center_->IsEnableTurn();
   params_.enable_srtp = config_center_->IsEnableSrtp();
+  params_.video_quality =
+      static_cast<VideoQuality>(config_center_->GetVideoQuality());
   params_.on_receive_video_buffer = nullptr;
   params_.on_receive_audio_buffer = OnReceiveAudioBufferCb;
   params_.on_receive_data_buffer = OnReceiveDataBufferCb;
@@ -566,6 +568,14 @@ void Render::UpdateInteractions() {
     screen_capturer_is_started_ = false;
   }
 
+  if (start_speaker_capturer_ && !speaker_capturer_is_started_) {
+    StartSpeakerCapturer();
+    speaker_capturer_is_started_ = true;
+  } else if (!start_speaker_capturer_ && speaker_capturer_is_started_) {
+    StopSpeakerCapturer();
+    speaker_capturer_is_started_ = false;
+  }
+
   if (start_mouse_controller_ && !mouse_controller_is_started_) {
     StartMouseController();
     mouse_controller_is_started_ = true;
@@ -1055,9 +1065,9 @@ void Render::MainLoop() {
       remote_action.i.host_name[host_name.size()] = '\0';
       remote_action.i.host_name_size = host_name.size();
 
-      std::vector<char> serialized = SerializeRemoteAction(remote_action);
+      std::string msg = remote_action.to_json();
-      int ret = SendDataFrame(peer_, serialized.data(), serialized.size(),
+      int ret =
-                              data_label_.c_str());
+          SendDataFrame(peer_, msg.data(), msg.size(), data_label_.c_str());
       FreeRemoteAction(remote_action);
       if (0 == ret) {
         need_to_send_host_info_ = false;
@@ -1183,6 +1193,10 @@ void Render::CleanupPeers() {
     LOG_INFO("[{}] Leave connection [{}]", client_id_, client_id_);
     LeaveConnection(peer_, client_id_);
     is_client_mode_ = false;
+    StopScreenCapturer();
+    StopSpeakerCapturer();
+    StopMouseController();
+    StopKeyboardCapturer();
     LOG_INFO("Destroy peer [{}]", client_id_);
     DestroyPeer(&peer_);
   }

src/gui/render.h

@@ -49,10 +49,10 @@ class Render {
     bool connection_established_ = false;
     bool rejoin_ = false;
     bool net_traffic_stats_button_pressed_ = false;
-    bool mouse_control_button_pressed_ = false;
+    bool mouse_control_button_pressed_ = true;
     bool mouse_controller_is_started_ = false;
-    bool audio_capture_button_pressed_ = false;
+    bool audio_capture_button_pressed_ = true;
-    bool control_mouse_ = false;
+    bool control_mouse_ = true;
     bool streaming_ = false;
     bool is_control_bar_in_left_ = true;
     bool control_bar_hovered_ = false;
@@ -311,7 +311,9 @@ class Render {
   bool mouse_controller_is_started_ = false;
   bool start_screen_capturer_ = false;
   bool screen_capturer_is_started_ = false;
-  bool start_keyboard_capturer_ = false;
+  bool start_speaker_capturer_ = false;
+  bool speaker_capturer_is_started_ = false;
+  bool start_keyboard_capturer_ = true;
   bool keyboard_capturer_is_started_ = false;
   bool foucs_on_main_window_ = false;
   bool foucs_on_stream_window_ = false;
@@ -436,7 +438,7 @@ class Render {
   char password_saved_[7] = "";
   int language_button_value_ = 0;
   int video_quality_button_value_ = 0;
-  int video_frame_rate_button_value_ = 0;
+  int video_frame_rate_button_value_ = 1;
   int video_encode_format_button_value_ = 0;
   bool enable_hardware_video_codec_ = false;
   bool enable_turn_ = false;
@@ -448,6 +450,7 @@ class Render {
   bool enable_self_hosted_ = false;
   int language_button_value_last_ = 0;
   int video_quality_button_value_last_ = 0;
+  int video_frame_rate_button_value_last_ = 0;
   int video_encode_format_button_value_last_ = 0;
   bool enable_hardware_video_codec_last_ = false;
   bool enable_turn_last_ = false;
@@ -470,6 +473,9 @@ class Render {
       client_properties_;
   void CloseTab(decltype(client_properties_)::iterator& it);
   /* ------ stream window property end ------ */
+
+  /* ------ server mode ------ */
+  std::unordered_map<std::string, ConnectionStatus> connection_status_;
 };
 }  // namespace crossdesk
 #endif

src/gui/render_callback.cpp

@@ -6,11 +6,6 @@
 
 #define NV12_BUFFER_SIZE 1280 * 720 * 3 / 2
 
-#ifdef CROSSDESK_DEBUG
-#else
-#define MOUSE_CONTROL 1
-#endif
-
 namespace crossdesk {
 
 int Render::SendKeyCommand(int key_code, bool is_down) {
@@ -28,8 +23,7 @@ int Render::SendKeyCommand(int key_code, bool is_down) {
         client_properties_.end()) {
       auto props = client_properties_[controlled_remote_id_];
       if (props->connection_status_ == ConnectionStatus::Connected) {
-        json j = remote_action.to_json();
+        std::string msg = remote_action.to_json();
-        std::string msg = j.dump();
         SendDataFrame(props->peer_, msg.c_str(), msg.size(),
                       props->data_label_.c_str());
       }
@@ -97,8 +91,7 @@ int Render::ProcessMouseEvent(const SDL_Event& event) {
         remote_action.m.flag = MouseFlag::move;
       }
 
-      json j = remote_action.to_json();
+      std::string msg = remote_action.to_json();
-      std::string msg = j.dump();
       SendDataFrame(props->peer_, msg.c_str(), msg.size(),
                     props->data_label_.c_str());
     } else if (SDL_EVENT_MOUSE_WHEEL == event.type &&
@@ -132,8 +125,7 @@ int Render::ProcessMouseEvent(const SDL_Event& event) {
           (float)(event.button.y - props->stream_render_rect_.y) /
           render_height;
 
-      json j = remote_action.to_json();
+      std::string msg = remote_action.to_json();
-      std::string msg = j.dump();
       SendDataFrame(props->peer_, msg.c_str(), msg.size(),
                     props->data_label_.c_str());
     }
@@ -233,6 +225,8 @@ void Render::OnReceiveVideoBufferCb(const XVideoFrame* video_frame,
     props->video_height_ = video_frame->height;
     props->video_size_ = video_frame->size;
 
+    LOG_ERROR("receive: {}x{}", props->video_width_, props->video_height_);
+
     if (need_to_update_render_rect) {
       render->UpdateRenderRect();
     }
@@ -321,9 +315,9 @@ void Render::OnReceiveDataBufferCb(const char* data, size_t size,
       render->mouse_controller_->SendMouseCommand(remote_action,
                                                   render->selected_display_);
     } else if (remote_action.type == ControlType::audio_capture) {
-      if (remote_action.a)
+      if (remote_action.a && !render->start_speaker_capturer_)
         render->StartSpeakerCapturer();
-      else
+      else if (!remote_action.a && render->start_speaker_capturer_)
         render->StopSpeakerCapturer();
     } else if (remote_action.type == ControlType::keyboard &&
                render->keyboard_capturer_) {
@@ -406,7 +400,7 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char* user_id,
     props->connection_status_ = status;
 
     switch (status) {
-      case ConnectionStatus::Connected:
+      case ConnectionStatus::Connected: {
         if (!render->need_to_create_stream_window_ &&
             !render->client_properties_.empty()) {
           render->need_to_create_stream_window_ = true;
@@ -417,14 +411,10 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char* user_id,
             (int)render->stream_window_width_,
             (int)(render->stream_window_height_ - render->title_bar_height_)};
         break;
+      }
       case ConnectionStatus::Disconnected:
       case ConnectionStatus::Failed:
-      case ConnectionStatus::Closed:
+      case ConnectionStatus::Closed: {
-        render->password_validating_time_ = 0;
-        render->start_screen_capturer_ = false;
-        render->start_mouse_controller_ = false;
-        render->start_keyboard_capturer_ = false;
-        render->control_mouse_ = false;
         props->connection_established_ = false;
         props->mouse_control_button_pressed_ = false;
         if (props->dst_buffer_) {
@@ -433,8 +423,10 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char* user_id,
                             props->texture_width_);
         }
         render->CleanSubStreamWindowProperties(props);
+
         break;
-      case ConnectionStatus::IncorrectPassword:
+      }
+      case ConnectionStatus::IncorrectPassword: {
         render->password_validating_ = false;
         render->password_validating_time_++;
         if (render->connect_button_pressed_) {
@@ -444,28 +436,45 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char* user_id,
               localization::connect[render->localization_language_index_];
         }
         break;
-      case ConnectionStatus::NoSuchTransmissionId:
+      }
+      case ConnectionStatus::NoSuchTransmissionId: {
         if (render->connect_button_pressed_) {
           props->connection_established_ = false;
           render->connect_button_label_ =
               localization::connect[render->localization_language_index_];
         }
         break;
+      }
       default:
         break;
     }
   } else {
     render->is_client_mode_ = false;
     render->show_connection_status_window_ = true;
+    render->connection_status_[remote_id] = status;
 
     switch (status) {
-      case ConnectionStatus::Connected:
+      case ConnectionStatus::Connected: {
         render->need_to_send_host_info_ = true;
         render->start_screen_capturer_ = true;
+        render->start_speaker_capturer_ = true;
+#ifdef CROSSDESK_DEBUG
+        render->start_mouse_controller_ = false;
+        render->start_keyboard_capturer_ = false;
+#else
         render->start_mouse_controller_ = true;
+#endif
         break;
-      case ConnectionStatus::Closed:
+      }
+      case ConnectionStatus::Closed: {
+        if (std::all_of(render->connection_status_.begin(),
+                        render->connection_status_.end(), [](const auto& kv) {
+                          return kv.second == ConnectionStatus::Closed ||
+                                 kv.second == ConnectionStatus::Failed ||
+                                 kv.second == ConnectionStatus::Disconnected;
+                        })) {
           render->start_screen_capturer_ = false;
+          render->start_speaker_capturer_ = false;
           render->start_mouse_controller_ = false;
           render->start_keyboard_capturer_ = false;
           render->need_to_send_host_info_ = false;
@@ -474,7 +483,11 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char* user_id,
             render->StopSpeakerCapturer();
             render->audio_capture_ = false;
           }
+
+          render->connection_status_.erase(remote_id);
+        }
         break;
+      }
       default:
         break;
     }

src/gui/toolbars/control_bar.cpp

@@ -68,8 +68,9 @@ int Render::ControlBar(std::shared_ptr<SubStreamWindowProperties>& props) {
           remote_action.type = ControlType::display_id;
           remote_action.d = i;
           if (props->connection_status_ == ConnectionStatus::Connected) {
-            SendDataFrame(props->peer_, (const char*)&remote_action,
+            std::string msg = remote_action.to_json();
-                          sizeof(remote_action), props->data_label_.c_str());
+            SendDataFrame(props->peer_, msg.c_str(), msg.size(),
+                          props->data_label_.c_str());
           }
         }
         props->display_selectable_hovered_ = ImGui::IsWindowHovered();
@@ -142,8 +143,9 @@ int Render::ControlBar(std::shared_ptr<SubStreamWindowProperties>& props) {
         RemoteAction remote_action;
         remote_action.type = ControlType::audio_capture;
         remote_action.a = props->audio_capture_button_pressed_;
-        SendDataFrame(props->peer_, (const char*)&remote_action,
+        std::string msg = remote_action.to_json();
-                      sizeof(remote_action), props->data_label_.c_str());
+        SendDataFrame(props->peer_, msg.c_str(), msg.size(),
+                      props->data_label_.c_str());
       }
     }
     if (!props->audio_capture_button_pressed_) {

src/gui/windows/main_settings_window.cpp

@@ -82,11 +82,11 @@ int Render::SettingWindow() {
 
       {
         const char* video_quality_items[] = {
-            localization::video_quality_high[localization_language_index_]
+            localization::video_quality_low[localization_language_index_]
                 .c_str(),
             localization::video_quality_medium[localization_language_index_]
                 .c_str(),
-            localization::video_quality_low[localization_language_index_]
+            localization::video_quality_high[localization_language_index_]
                 .c_str()};
 
         settings_items_offset += settings_items_padding;
@@ -288,14 +288,23 @@ int Render::SettingWindow() {
 
         // Video quality
         if (video_quality_button_value_ == 0) {
-          config_center_->SetVideoQuality(ConfigCenter::VIDEO_QUALITY::HIGH);
+          config_center_->SetVideoQuality(ConfigCenter::VIDEO_QUALITY::LOW);
         } else if (video_quality_button_value_ == 1) {
           config_center_->SetVideoQuality(ConfigCenter::VIDEO_QUALITY::MEDIUM);
         } else {
-          config_center_->SetVideoQuality(ConfigCenter::VIDEO_QUALITY::LOW);
+          config_center_->SetVideoQuality(ConfigCenter::VIDEO_QUALITY::HIGH);
         }
         video_quality_button_value_last_ = video_quality_button_value_;
 
+        if (video_frame_rate_button_value_ == 0) {
+          config_center_->SetVideoFrameRate(
+              ConfigCenter::VIDEO_FRAME_RATE::FPS_30);
+        } else if (video_frame_rate_button_value_ == 1) {
+          config_center_->SetVideoFrameRate(
+              ConfigCenter::VIDEO_FRAME_RATE::FPS_60);
+        }
+        video_frame_rate_button_value_last_ = video_frame_rate_button_value_;
+
         // Video encode format
         if (video_encode_format_button_value_ == 0) {
           config_center_->SetVideoEncodeFormat(
@@ -366,6 +375,11 @@ int Render::SettingWindow() {
           video_quality_button_value_ = video_quality_button_value_last_;
         }
 
+        if (video_frame_rate_button_value_ !=
+            video_frame_rate_button_value_last_) {
+          video_frame_rate_button_value_ = video_frame_rate_button_value_last_;
+        }
+
         if (video_encode_format_button_value_ !=
             video_encode_format_button_value_last_) {
           video_encode_format_button_value_ =