From 52828183a182b6136512dc4e1d2cde27e15c7012 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Fri, 22 Nov 2024 16:39:01 +0800 Subject: [PATCH] [feat] keyboard capture supported on Windows --- src/device_controller/device_controller.h | 12 ++- .../device_controller_factory.h | 3 +- .../keyboard/linux/keyboard_capturer.cpp | 9 ++ .../keyboard/linux/keyboard_capturer.h | 24 ++++++ .../keyboard/mac/keyboard_capturer.cpp | 9 ++ .../keyboard/mac/keyboard_capturer.h | 24 ++++++ .../keyboard/windows/keyboard_capturer.cpp | 55 ++++++++++++ .../keyboard/windows/keyboard_capturer.h | 27 ++++++ src/single_window/control_bar.cpp | 1 + src/single_window/render.cpp | 86 ++++++++++++++----- src/single_window/render.h | 30 ++++--- src/single_window/render_callback_func.cpp | 42 ++++++--- xmake.lua | 18 ++-- 13 files changed, 285 insertions(+), 55 deletions(-) create mode 100644 src/device_controller/keyboard/linux/keyboard_capturer.cpp create mode 100644 src/device_controller/keyboard/linux/keyboard_capturer.h create mode 100644 src/device_controller/keyboard/mac/keyboard_capturer.cpp create mode 100644 src/device_controller/keyboard/mac/keyboard_capturer.h create mode 100644 src/device_controller/keyboard/windows/keyboard_capturer.cpp create mode 100644 src/device_controller/keyboard/windows/keyboard_capturer.h diff --git a/src/device_controller/device_controller.h b/src/device_controller/device_controller.h index d68a6ff..93be044 100644 --- a/src/device_controller/device_controller.h +++ b/src/device_controller/device_controller.h @@ -43,14 +43,20 @@ typedef struct { }; } RemoteAction; +// int key_code, bool is_down +typedef void (*OnKeyAction)(int, bool, void*); + class DeviceController { public: virtual ~DeviceController() {} public: - virtual int Init(int screen_width, int screen_height) = 0; - virtual int Destroy() = 0; - virtual int SendCommand(RemoteAction remote_action) = 0; + // virtual int Init(int screen_width, int screen_height); + // virtual int Destroy(); + // virtual int SendCommand(RemoteAction remote_action); + + // virtual int Hook(); + // virtual int Unhook(); }; #endif \ No newline at end of file diff --git a/src/device_controller/device_controller_factory.h b/src/device_controller/device_controller_factory.h index eb48d9e..4b9bb98 100644 --- a/src/device_controller/device_controller_factory.h +++ b/src/device_controller/device_controller_factory.h @@ -8,6 +8,7 @@ #define _DEVICE_CONTROLLER_FACTORY_H_ #include "device_controller.h" +#include "keyboard_capturer.h" #include "mouse_controller.h" class DeviceControllerFactory { @@ -23,7 +24,7 @@ class DeviceControllerFactory { case Mouse: return new MouseController(); case Keyboard: - return nullptr; + return new KeyboardCapturer(); default: return nullptr; } diff --git a/src/device_controller/keyboard/linux/keyboard_capturer.cpp b/src/device_controller/keyboard/linux/keyboard_capturer.cpp new file mode 100644 index 0000000..fbc75dd --- /dev/null +++ b/src/device_controller/keyboard/linux/keyboard_capturer.cpp @@ -0,0 +1,9 @@ +#include "keyboard_capturer.h" + +KeyboardCapturer::KeyboardCapturer() {} + +KeyboardCapturer::~KeyboardCapturer() {} + +int KeyboardCapturer::Hook() { return 0; } + +int KeyboardCapturer::Unhook() { return 0; } \ No newline at end of file diff --git a/src/device_controller/keyboard/linux/keyboard_capturer.h b/src/device_controller/keyboard/linux/keyboard_capturer.h new file mode 100644 index 0000000..3a45c84 --- /dev/null +++ b/src/device_controller/keyboard/linux/keyboard_capturer.h @@ -0,0 +1,24 @@ +/* + * @Author: DI JUNKUN + * @Date: 2024-11-22 + * Copyright (c) 2024 by DI JUNKUN, All Rights Reserved. + */ + +#ifndef _KEYBOARD_CAPTURER_H_ +#define _KEYBOARD_CAPTURER_H_ + +#include "device_controller.h" + +class KeyboardCapturer : public DeviceController { + public: + KeyboardCapturer(); + virtual ~KeyboardCapturer(); + + public: + virtual int Hook(); + virtual int Unhook(); + + private: +}; + +#endif \ No newline at end of file diff --git a/src/device_controller/keyboard/mac/keyboard_capturer.cpp b/src/device_controller/keyboard/mac/keyboard_capturer.cpp new file mode 100644 index 0000000..fbc75dd --- /dev/null +++ b/src/device_controller/keyboard/mac/keyboard_capturer.cpp @@ -0,0 +1,9 @@ +#include "keyboard_capturer.h" + +KeyboardCapturer::KeyboardCapturer() {} + +KeyboardCapturer::~KeyboardCapturer() {} + +int KeyboardCapturer::Hook() { return 0; } + +int KeyboardCapturer::Unhook() { return 0; } \ No newline at end of file diff --git a/src/device_controller/keyboard/mac/keyboard_capturer.h b/src/device_controller/keyboard/mac/keyboard_capturer.h new file mode 100644 index 0000000..3a45c84 --- /dev/null +++ b/src/device_controller/keyboard/mac/keyboard_capturer.h @@ -0,0 +1,24 @@ +/* + * @Author: DI JUNKUN + * @Date: 2024-11-22 + * Copyright (c) 2024 by DI JUNKUN, All Rights Reserved. + */ + +#ifndef _KEYBOARD_CAPTURER_H_ +#define _KEYBOARD_CAPTURER_H_ + +#include "device_controller.h" + +class KeyboardCapturer : public DeviceController { + public: + KeyboardCapturer(); + virtual ~KeyboardCapturer(); + + public: + virtual int Hook(); + virtual int Unhook(); + + private: +}; + +#endif \ No newline at end of file diff --git a/src/device_controller/keyboard/windows/keyboard_capturer.cpp b/src/device_controller/keyboard/windows/keyboard_capturer.cpp new file mode 100644 index 0000000..bfa993b --- /dev/null +++ b/src/device_controller/keyboard/windows/keyboard_capturer.cpp @@ -0,0 +1,55 @@ +#include "keyboard_capturer.h" + +#include "rd_log.h" + +static OnKeyAction g_on_key_action = nullptr; +static void* g_user_ptr = nullptr; + +LRESULT CALLBACK KeyboardProc(int nCode, WPARAM wParam, LPARAM lParam) { + if (nCode == HC_ACTION && g_on_key_action) { + KBDLLHOOKSTRUCT* kbData = reinterpret_cast(lParam); + + // 处理键盘事件 + if (wParam == WM_KEYDOWN) { + // LOG_ERROR("Keydown: [{}]", kbData->vkCode); + g_on_key_action(kbData->vkCode, true, g_user_ptr); + return 1; + } else if (wParam == WM_KEYUP) { + // LOG_ERROR("Keyup: [{}]", kbData->vkCode); + g_on_key_action(kbData->vkCode, false, g_user_ptr); + return 1; + } else if (wParam == WM_SYSKEYDOWN) { + // LOG_ERROR("System keydown: [{}]", kbData->vkCode); + g_on_key_action(kbData->vkCode, true, g_user_ptr); + return 1; + } else if (wParam == WM_SYSKEYUP) { + // LOG_ERROR("System keyup: [{}]", kbData->vkCode); + g_on_key_action(kbData->vkCode, false, g_user_ptr); + return 1; + } + } + + // 调用下一个钩子 + return CallNextHookEx(NULL, nCode, wParam, lParam); +} + +KeyboardCapturer::KeyboardCapturer() {} + +KeyboardCapturer::~KeyboardCapturer() {} + +int KeyboardCapturer::Hook(OnKeyAction on_key_action, void* user_ptr) { + g_on_key_action = on_key_action; + g_user_ptr = user_ptr; + + keyboard_hook_ = SetWindowsHookEx(WH_KEYBOARD_LL, KeyboardProc, NULL, 0); + if (!keyboard_hook_) { + LOG_ERROR("Failed to install keyboard hook!") + return -1; + } + return 0; +} + +int KeyboardCapturer::Unhook() { + UnhookWindowsHookEx(keyboard_hook_); + return 0; +} \ No newline at end of file diff --git a/src/device_controller/keyboard/windows/keyboard_capturer.h b/src/device_controller/keyboard/windows/keyboard_capturer.h new file mode 100644 index 0000000..b1cf2aa --- /dev/null +++ b/src/device_controller/keyboard/windows/keyboard_capturer.h @@ -0,0 +1,27 @@ +/* + * @Author: DI JUNKUN + * @Date: 2024-11-22 + * Copyright (c) 2024 by DI JUNKUN, All Rights Reserved. + */ + +#ifndef _KEYBOARD_CAPTURER_H_ +#define _KEYBOARD_CAPTURER_H_ + +#include + +#include "device_controller.h" + +class KeyboardCapturer : public DeviceController { + public: + KeyboardCapturer(); + virtual ~KeyboardCapturer(); + + public: + virtual int Hook(OnKeyAction on_key_action, void *user_ptr); + virtual int Unhook(); + + private: + HHOOK keyboard_hook_ = nullptr; +}; + +#endif \ No newline at end of file diff --git a/src/single_window/control_bar.cpp b/src/single_window/control_bar.cpp index afe3c4e..66cbca6 100644 --- a/src/single_window/control_bar.cpp +++ b/src/single_window/control_bar.cpp @@ -47,6 +47,7 @@ int Render::ControlBar() { if (ImGui::Button(mouse.c_str(), ImVec2(25, 25))) { if (connection_established_) { control_mouse_ = !control_mouse_; + start_keyboard_capturer_ = !start_keyboard_capturer_; mouse_control_button_pressed_ = !mouse_control_button_pressed_; mouse_control_button_label_ = mouse_control_button_pressed_ diff --git a/src/single_window/render.cpp b/src/single_window/render.cpp index f8148e1..a499b70 100644 --- a/src/single_window/render.cpp +++ b/src/single_window/render.cpp @@ -186,7 +186,7 @@ int Render::LoadSettingsFromCacheFile() { return 0; } -int Render::StartScreenCapture() { +int Render::StartScreenCapturer() { screen_capturer_ = (ScreenCapturer*)screen_capturer_factory_->Create(); last_frame_time_ = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) @@ -224,7 +224,7 @@ int Render::StartScreenCapture() { return 0; } -int Render::StopScreenCapture() { +int Render::StopScreenCapturer() { if (screen_capturer_) { LOG_INFO("Stop screen capturer") screen_capturer_->Stop(); @@ -236,7 +236,7 @@ int Render::StopScreenCapture() { return 0; } -int Render::StartSpeakerCapture() { +int Render::StartSpeakerCapturer() { if (!speaker_capturer_) { speaker_capturer_ = (SpeakerCapturer*)speaker_capturer_factory_->Create(); int speaker_capturer_init_ret = speaker_capturer_->Init( @@ -259,7 +259,7 @@ int Render::StartSpeakerCapture() { return 0; } -int Render::StopSpeakerCapture() { +int Render::StopSpeakerCapturer() { if (speaker_capturer_) { speaker_capturer_->Stop(); } @@ -267,8 +267,11 @@ int Render::StopSpeakerCapture() { return 0; } -int Render::StartMouseControl() { - device_controller_factory_ = new DeviceControllerFactory(); +int Render::StartMouseController() { + if (!device_controller_factory_) { + LOG_INFO("Device controller factory is nullptr"); + return -1; + } mouse_controller_ = (MouseController*)device_controller_factory_->Create( DeviceControllerFactory::Device::Mouse); int mouse_controller_init_ret = @@ -282,7 +285,7 @@ int Render::StartMouseControl() { return 0; } -int Render::StopMouseControl() { +int Render::StopMouseController() { if (mouse_controller_) { mouse_controller_->Destroy(); delete mouse_controller_; @@ -291,6 +294,41 @@ int Render::StopMouseControl() { return 0; } +int Render::StartKeyboardCapturer() { + if (!device_controller_factory_) { + LOG_INFO("Device controller factory is nullptr"); + return -1; + } + keyboard_capturer_ = (KeyboardCapturer*)device_controller_factory_->Create( + DeviceControllerFactory::Device::Keyboard); + int keyboard_capturer_init_ret = keyboard_capturer_->Hook( + [](int key_code, bool is_down, void* user_ptr) { + if (user_ptr) { + Render* render = (Render*)user_ptr; + render->SendKeyEvent(key_code, is_down); + } + }, + this); + if (0 != keyboard_capturer_init_ret) { + LOG_INFO("Destroy keyboard capturer") + keyboard_capturer_->Unhook(); + keyboard_capturer_ = nullptr; + } else { + LOG_INFO("Start keyboard capturer"); + } + + return 0; +} + +int Render::StopKeyboardCapturer() { + if (keyboard_capturer_) { + keyboard_capturer_->Unhook(); + delete keyboard_capturer_; + keyboard_capturer_ = nullptr; + } + return 0; +} + int Render::CreateConnectionPeer() { mac_addr_str_ = GetMac(); @@ -386,20 +424,28 @@ int Render::CreateRtcConnection() { CreateConnection(peer_, client_id_, password_saved_) ? false : true; } - if (start_screen_capture_ && !screen_capture_is_started_) { - StartScreenCapture(); - screen_capture_is_started_ = true; - } else if (!start_screen_capture_ && screen_capture_is_started_) { - StopScreenCapture(); - screen_capture_is_started_ = false; + if (start_screen_capturer_ && !screen_capturer_is_started_) { + StartScreenCapturer(); + screen_capturer_is_started_ = true; + } else if (!start_screen_capturer_ && screen_capturer_is_started_) { + StopScreenCapturer(); + screen_capturer_is_started_ = false; } - if (start_mouse_control_ && !mouse_control_is_started_) { - StartMouseControl(); - mouse_control_is_started_ = true; - } else if (!start_mouse_control_ && mouse_control_is_started_) { - StopMouseControl(); - mouse_control_is_started_ = false; + if (start_mouse_controller_ && !mouse_controller_is_started_) { + StartMouseController(); + mouse_controller_is_started_ = true; + } else if (!start_mouse_controller_ && mouse_controller_is_started_) { + StopMouseController(); + mouse_controller_is_started_ = false; + } + + if (start_keyboard_capturer_ && !keyboard_capturer_is_started_) { + StartKeyboardCapturer(); + keyboard_capturer_is_started_ = true; + } else if (!start_keyboard_capturer_ && keyboard_capturer_is_started_) { + StopKeyboardCapturer(); + keyboard_capturer_is_started_ = false; } return 0; @@ -741,7 +787,7 @@ int Render::Run() { // speaker capture init speaker_capturer_factory_ = new SpeakerCapturerFactory(); - // mouse control + // mouse control/keyboard capturer device_controller_factory_ = new DeviceControllerFactory(); // RTC diff --git a/src/single_window/render.h b/src/single_window/render.h index e70d091..9decef4 100644 --- a/src/single_window/render.h +++ b/src/single_window/render.h @@ -95,9 +95,11 @@ class Render { private: int ProcessMouseKeyEvent(SDL_Event &event); - int ProcessKeyEvent(SDL_Event &event); int ProcessMouseEvent(SDL_Event &event); + int SendKeyEvent(int key_code, bool is_down); + int ProcessKeyEvent(int key_code, bool is_down); + static void SdlCaptureAudioIn(void *userdata, Uint8 *stream, int len); static void SdlCaptureAudioOut(void *userdata, Uint8 *stream, int len); @@ -105,14 +107,17 @@ class Render { int SaveSettingsIntoCacheFile(); int LoadSettingsFromCacheFile(); - int StartScreenCapture(); - int StopScreenCapture(); + int StartScreenCapturer(); + int StopScreenCapturer(); - int StartSpeakerCapture(); - int StopSpeakerCapture(); + int StartSpeakerCapturer(); + int StopSpeakerCapturer(); - int StartMouseControl(); - int StopMouseControl(); + int StartMouseController(); + int StopMouseController(); + + int StartKeyboardCapturer(); + int StopKeyboardCapturer(); int CreateConnectionPeer(); @@ -350,6 +355,7 @@ class Render { SpeakerCapturer *speaker_capturer_ = nullptr; DeviceControllerFactory *device_controller_factory_ = nullptr; MouseController *mouse_controller_ = nullptr; + KeyboardCapturer *keyboard_capturer_ = nullptr; uint32_t last_frame_time_; private: @@ -369,10 +375,12 @@ class Render { bool enable_turn_last_ = false; private: - std::atomic start_screen_capture_{false}; - std::atomic start_mouse_control_{false}; - std::atomic screen_capture_is_started_{false}; - std::atomic mouse_control_is_started_{false}; + std::atomic start_screen_capturer_{false}; + std::atomic start_mouse_controller_{false}; + std::atomic start_keyboard_capturer_{false}; + std::atomic screen_capturer_is_started_{false}; + std::atomic mouse_controller_is_started_{false}; + std::atomic keyboard_capturer_is_started_{false}; private: bool settings_window_pos_reset_ = true; diff --git a/src/single_window/render_callback_func.cpp b/src/single_window/render_callback_func.cpp index 7664ed8..9469986 100644 --- a/src/single_window/render_callback_func.cpp +++ b/src/single_window/render_callback_func.cpp @@ -19,7 +19,6 @@ int Render::ProcessMouseKeyEvent(SDL_Event &event) { } if (SDL_KEYDOWN == event.type || SDL_KEYUP == event.type) { - ProcessKeyEvent(event); } else { ProcessMouseEvent(event); } @@ -87,14 +86,24 @@ int Render::ProcessMouseEvent(SDL_Event &event) { return 0; } -int Render::ProcessKeyEvent(SDL_Event &event) { +int Render::SendKeyEvent(int key_code, bool is_down) { RemoteAction remote_action; - SDL_Keycode key = event.key.keysym.sym; - if (SDL_KEYDOWN == event.type) { - std::cout << "Key pressed: " << SDL_GetKeyName(key) << std::endl; - } else if (SDL_KEYUP == event.type) { - std::cout << "Key released: " << SDL_GetKeyName(key) << std::endl; + remote_action.type = ControlType::keyboard; + if (is_down) { + remote_action.k.flag = KeyFlag::key_down; + } else { + remote_action.k.flag = KeyFlag::key_up; } + remote_action.k.key_value = key_code; + + SendData(peer_, DATA_TYPE::DATA, (const char *)&remote_action, + sizeof(remote_action)); + + return 0; +} + +int Render::ProcessKeyEvent(int key_code, bool is_down) { + LOG_ERROR("key code [{}], is down [{}]", key_code, is_down); return 0; } @@ -206,11 +215,12 @@ void Render::OnReceiveDataBufferCb(const char *data, size_t size, render->mouse_controller_->SendCommand(remote_action); } else if (ControlType::audio_capture == remote_action.type) { if (remote_action.a) { - render->StartSpeakerCapture(); + render->StartSpeakerCapturer(); } else { - render->StopSpeakerCapture(); + render->StopSpeakerCapturer(); } } else if (ControlType::keyboard == remote_action.type) { + render->ProcessKeyEvent(remote_action.k.key_value, remote_action.k.flag); } else if (ControlType::host_infomation == remote_action.type) { render->host_name_ = std::string(remote_action.i.host_name, remote_action.i.host_name_size); @@ -264,8 +274,8 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char *user_id, render->connection_status_str_ = "Connected"; render->connection_established_ = true; if (render->peer_reserved_ || !render->is_client_mode_) { - render->start_screen_capture_ = true; - render->start_mouse_control_ = true; + render->start_screen_capturer_ = true; + render->start_mouse_controller_ = true; } if (!render->hostname_sent_) { // TODO: self and remote hostname @@ -289,13 +299,17 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char *user_id, } else if (ConnectionStatus::Closed == status) { render->connection_status_str_ = "Closed"; render->password_validating_time_ = 0; - render->start_screen_capture_ = false; - render->start_mouse_control_ = false; + render->start_screen_capturer_ = false; + render->mouse_controller_is_started_ = false; + render->start_mouse_controller_ = false; + render->mouse_controller_is_started_ = false; render->connection_established_ = false; render->control_mouse_ = false; + render->start_keyboard_capturer_ = false; + render->keyboard_capturer_is_started_ = false; render->hostname_sent_ = false; if (render->audio_capture_) { - render->StopSpeakerCapture(); + render->StopSpeakerCapturer(); render->audio_capture_ = false; render->audio_capture_button_pressed_ = false; } diff --git a/xmake.lua b/xmake.lua index cf8bee3..21a681e 100644 --- a/xmake.lua +++ b/xmake.lua @@ -103,14 +103,20 @@ target("device_controller") add_deps("rd_log") add_includedirs("src/device_controller", {public = true}) if is_os("windows") then - add_files("src/device_controller/mouse/windows/*.cpp") - add_includedirs("src/device_controller/mouse/windows", {public = true}) + add_files("src/device_controller/mouse/windows/*.cpp", + "src/device_controller/keyboard/windows/*.cpp") + add_includedirs("src/device_controller/mouse/windows", + "src/device_controller/keyboard/windows", {public = true}) elseif is_os("macosx") then - add_files("src/device_controller/mouse/mac/*.cpp") - add_includedirs("src/device_controller/mouse/mac", {public = true}) + add_files("src/device_controller/mouse/mac/*.cpp", + "src/device_controller/mouse/keyboard/*.cpp") + add_includedirs("src/device_controller/mouse/mac", + "src/device_controller/keyboard/mac", {public = true}) elseif is_os("linux") then - add_files("src/device_controller/mouse/linux/*.cpp") - add_includedirs("src/device_controller/mouse/linux", {public = true}) + add_files("src/device_controller/mouse/linux/*.cpp", + "src/device_controller/keyboard/linux/*.cpp") + add_includedirs("src/device_controller/mouse/linux", + "src/device_controller/keyboard/linux", {public = true}) end target("config_center")