From 018231eee4d835be29fed700c2662d9485e3391f Mon Sep 17 00:00:00 2001 From: dijunkun Date: Thu, 15 Aug 2024 14:50:47 +0800 Subject: [PATCH] [feat] add protocol to control audio capture --- src/device_controller/device_controller.h | 3 +- src/single_window/control_bar.cpp | 6 + src/single_window/render.cpp | 164 ++++++++++-------- src/single_window/render.h | 3 - src/single_window/render_callback_func.cpp | 10 +- .../windows/speaker_capturer_wasapi.cpp | 7 +- 6 files changed, 107 insertions(+), 86 deletions(-) diff --git a/src/device_controller/device_controller.h b/src/device_controller/device_controller.h index ff9c41f..53c861f 100644 --- a/src/device_controller/device_controller.h +++ b/src/device_controller/device_controller.h @@ -9,7 +9,7 @@ #include -typedef enum { mouse = 0, keyboard } ControlType; +typedef enum { mouse = 0, keyboard, audio_capture } ControlType; typedef enum { move = 0, left_down, left_up, right_down, right_up } MouseFlag; typedef enum { key_down = 0, key_up } KeyFlag; typedef struct { @@ -28,6 +28,7 @@ typedef struct { union { Mouse m; Key k; + bool a; }; } RemoteAction; diff --git a/src/single_window/control_bar.cpp b/src/single_window/control_bar.cpp index eb19cd5..1fcd3ad 100644 --- a/src/single_window/control_bar.cpp +++ b/src/single_window/control_bar.cpp @@ -48,6 +48,12 @@ int Render::ControlBar() { localization::audio_capture[localization_language_index_]; } audio_capture_button_pressed_ = !audio_capture_button_pressed_; + + RemoteAction remote_action; + remote_action.type = ControlType::audio_capture; + remote_action.a = audio_capture_button_pressed_; + SendData(peer_, DATA_TYPE::DATA, (const char *)&remote_action, + sizeof(remote_action)); } ImGui::SameLine(); diff --git a/src/single_window/render.cpp b/src/single_window/render.cpp index 155da50..a4c2e66 100644 --- a/src/single_window/render.cpp +++ b/src/single_window/render.cpp @@ -152,79 +152,16 @@ int Render::LoadSettingsFromCacheFile() { return 0; } -int Render::StartScreenCapture() { - screen_capturer_ = (ScreenCapturer *)screen_capturer_factory_->Create(); - ScreenCapturer::RECORD_DESKTOP_RECT rect; - rect.left = 0; - rect.top = 0; - rect.right = screen_width_; - rect.bottom = screen_height_; - last_frame_time_ = std::chrono::steady_clock::now(); - - int screen_capturer_init_ret = screen_capturer_->Init( - rect, 60, - [this](unsigned char *data, int size, int width, int height) -> void { - auto now_time = std::chrono::steady_clock::now(); - std::chrono::duration duration = now_time - last_frame_time_; - auto tc = duration.count() * 1000; - - if (tc >= 0 && connection_established_) { - SendData(peer_, DATA_TYPE::VIDEO, (const char *)data, - NV12_BUFFER_SIZE); - last_frame_time_ = now_time; - } - }); - - if (0 == screen_capturer_init_ret) { - screen_capturer_->Start(); - } else { - screen_capturer_->Destroy(); - delete screen_capturer_; - screen_capturer_ = nullptr; - } - - return 0; -} - -int Render::StopScreenCapture() { - if (screen_capturer_) { - LOG_INFO("Stop screen capturer") - screen_capturer_->Stop(); - screen_capturer_->Destroy(); - delete screen_capturer_; - screen_capturer_ = nullptr; - } - - return 0; -} - int Render::StartSpeakerCapture() { - speaker_capturer_ = (SpeakerCapturer *)speaker_capturer_factory_->Create(); - - int speaker_capturer_init_ret = - speaker_capturer_->Init([this](unsigned char *data, size_t size) -> void { - if (connection_established_) { - SendData(peer_, DATA_TYPE::AUDIO, (const char *)data, size); - } - }); - - if (0 == speaker_capturer_init_ret) { + if (speaker_capturer_) { speaker_capturer_->Start(); - } else { - speaker_capturer_->Destroy(); - delete speaker_capturer_; - speaker_capturer_ = nullptr; } - return 0; } int Render::StopSpeakerCapture() { if (speaker_capturer_) { - LOG_INFO("Destroy speaker capturer") - speaker_capturer_->Destroy(); - delete speaker_capturer_; - speaker_capturer_ = nullptr; + speaker_capturer_->Stop(); } return 0; @@ -419,22 +356,62 @@ int Render::Run() { { nv12_buffer_ = new char[NV12_BUFFER_SIZE]; - // Screen capture + // Screen capture init screen_capturer_factory_ = new ScreenCapturerFactory(); + screen_capturer_ = (ScreenCapturer *)screen_capturer_factory_->Create(); + ScreenCapturer::RECORD_DESKTOP_RECT rect; + rect.left = 0; + rect.top = 0; + rect.right = screen_width_; + rect.bottom = screen_height_; + last_frame_time_ = std::chrono::steady_clock::now(); - // Speaker capture + int screen_capturer_init_ret = screen_capturer_->Init( + rect, 60, + [this](unsigned char *data, int size, int width, int height) -> void { + auto now_time = std::chrono::steady_clock::now(); + std::chrono::duration duration = now_time - last_frame_time_; + auto tc = duration.count() * 1000; + + if (tc >= 0 && connection_established_) { + SendData(peer_, DATA_TYPE::VIDEO, (const char *)data, + NV12_BUFFER_SIZE); + last_frame_time_ = now_time; + } + }); + + if (0 == screen_capturer_init_ret) { + screen_capturer_->Start(); + } else { + screen_capturer_->Destroy(); + delete screen_capturer_; + screen_capturer_ = nullptr; + } + + // Speaker capture init speaker_capturer_factory_ = new SpeakerCapturerFactory(); + speaker_capturer_ = (SpeakerCapturer *)speaker_capturer_factory_->Create(); + int speaker_capturer_init_ret = speaker_capturer_->Init( + [this](unsigned char *data, size_t size) -> void { + if (connection_established_) { + SendData(peer_, DATA_TYPE::AUDIO, (const char *)data, size); + } + }); + + if (0 != speaker_capturer_init_ret) { + speaker_capturer_->Destroy(); + delete speaker_capturer_; + speaker_capturer_ = nullptr; + } // Mouse control device_controller_factory_ = new DeviceControllerFactory(); } - StartSpeakerCapture(); - // Main loop while (!exit_) { if (SignalStatus::SignalConnected == signal_status_ && - !is_create_connection_) { + !is_create_connection_ && password_inited_) { LOG_INFO("Connected with signal server, create p2p connection"); is_create_connection_ = CreateConnection(peer_, client_id_, password_saved_.c_str()) ? false @@ -463,13 +440,13 @@ int Render::Run() { localization_language_index_last_ = localization_language_index_; } - if (start_screen_capture_ && !screen_capture_is_started_) { - StartScreenCapture(); - screen_capture_is_started_ = true; - } else if (!start_screen_capture_ && screen_capture_is_started_) { - StopScreenCapture(); - screen_capture_is_started_ = false; - } + // if (start_screen_capture_ && !screen_capture_is_started_) { + // StartScreenCapture(); + // screen_capture_is_started_ = true; + // } else if (!start_screen_capture_ && screen_capture_is_started_) { + // StopScreenCapture(); + // screen_capture_is_started_ = false; + // } if (start_mouse_control_ && !mouse_control_is_started_) { StartMouseControl(); @@ -621,6 +598,39 @@ int Render::Run() { } // Cleanup + if (screen_capturer_) { + screen_capturer_->Destroy(); + delete screen_capturer_; + screen_capturer_ = nullptr; + } + + if (speaker_capturer_) { + speaker_capturer_->Destroy(); + delete speaker_capturer_; + speaker_capturer_ = nullptr; + } + + if (mouse_controller_) { + mouse_controller_->Destroy(); + delete mouse_controller_; + mouse_controller_ = nullptr; + } + + if (screen_capturer_factory_) { + delete screen_capturer_factory_; + screen_capturer_factory_ = nullptr; + } + + if (speaker_capturer_factory_) { + delete speaker_capturer_factory_; + speaker_capturer_factory_ = nullptr; + } + + if (device_controller_factory_) { + delete device_controller_factory_; + device_controller_factory_ = nullptr; + } + if (is_create_connection_) { LOG_INFO("[{}] Leave connection [{}]", client_id_, client_id_); LeaveConnection(peer_, client_id_); diff --git a/src/single_window/render.h b/src/single_window/render.h index bbc57d4..1a15374 100644 --- a/src/single_window/render.h +++ b/src/single_window/render.h @@ -77,9 +77,6 @@ class Render { int SaveSettingsIntoCacheFile(); int LoadSettingsFromCacheFile(); - int StartScreenCapture(); - int StopScreenCapture(); - int StartSpeakerCapture(); int StopSpeakerCapture(); diff --git a/src/single_window/render_callback_func.cpp b/src/single_window/render_callback_func.cpp index 725e471..e394469 100644 --- a/src/single_window/render_callback_func.cpp +++ b/src/single_window/render_callback_func.cpp @@ -145,11 +145,15 @@ void Render::OnReceiveDataBufferCb(const char *data, size_t size, RemoteAction remote_action; memcpy(&remote_action, data, sizeof(remote_action)); -#if MOUSE_CONTROL - if (render->mouse_controller_) { + if (ControlType::mouse == remote_action.type && render->mouse_controller_) { render->mouse_controller_->SendCommand(remote_action); + } else if (ControlType::audio_capture == remote_action.type) { + if (remote_action.a) { + render->StartSpeakerCapture(); + } else { + render->StopSpeakerCapture(); + } } -#endif } void Render::OnSignalStatusCb(SignalStatus status, void *user_data) { diff --git a/src/speaker_capturer/windows/speaker_capturer_wasapi.cpp b/src/speaker_capturer/windows/speaker_capturer_wasapi.cpp index 61e5b9e..46ca733 100644 --- a/src/speaker_capturer/windows/speaker_capturer_wasapi.cpp +++ b/src/speaker_capturer/windows/speaker_capturer_wasapi.cpp @@ -88,10 +88,13 @@ int SpeakerCapturerWasapi::Start() { } int SpeakerCapturerWasapi::Stop() { + ma_device_stop(&device_); + return 0; +} + +int SpeakerCapturerWasapi::Destroy() { ma_device_uninit(&device_); return 0; } -int SpeakerCapturerWasapi::Destroy() { return 0; } - int SpeakerCapturerWasapi::Pause() { return 0; }