[feat] keyboard capture supported on Windows

src/device_controller/device_controller.h

@@ -43,14 +43,20 @@ typedef struct {
   };
 } RemoteAction;
 
+// int key_code, bool is_down
+typedef void (*OnKeyAction)(int, bool, void*);
+
 class DeviceController {
  public:
   virtual ~DeviceController() {}
 
  public:
-  virtual int Init(int screen_width, int screen_height) = 0;
-  virtual int Destroy() = 0;
-  virtual int SendCommand(RemoteAction remote_action) = 0;
+  // virtual int Init(int screen_width, int screen_height);
+  // virtual int Destroy();
+  // virtual int SendCommand(RemoteAction remote_action);
+
+  // virtual int Hook();
+  // virtual int Unhook();
 };
 
 #endif

src/device_controller/device_controller_factory.h

@@ -8,6 +8,7 @@
 #define _DEVICE_CONTROLLER_FACTORY_H_
 
 #include "device_controller.h"
+#include "keyboard_capturer.h"
 #include "mouse_controller.h"
 
 class DeviceControllerFactory {
@@ -23,7 +24,7 @@ class DeviceControllerFactory {
       case Mouse:
         return new MouseController();
       case Keyboard:
-        return nullptr;
+        return new KeyboardCapturer();
       default:
         return nullptr;
     }

src/device_controller/keyboard/linux/keyboard_capturer.cpp

@@ -0,0 +1,9 @@
+#include "keyboard_capturer.h"
+
+KeyboardCapturer::KeyboardCapturer() {}
+
+KeyboardCapturer::~KeyboardCapturer() {}
+
+int KeyboardCapturer::Hook() { return 0; }
+
+int KeyboardCapturer::Unhook() { return 0; }

src/device_controller/keyboard/linux/keyboard_capturer.h

@@ -0,0 +1,24 @@
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2024-11-22
+ * Copyright (c) 2024 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _KEYBOARD_CAPTURER_H_
+#define _KEYBOARD_CAPTURER_H_
+
+#include "device_controller.h"
+
+class KeyboardCapturer : public DeviceController {
+ public:
+  KeyboardCapturer();
+  virtual ~KeyboardCapturer();
+
+ public:
+  virtual int Hook();
+  virtual int Unhook();
+
+ private:
+};
+
+#endif

src/device_controller/keyboard/mac/keyboard_capturer.cpp

@@ -0,0 +1,9 @@
+#include "keyboard_capturer.h"
+
+KeyboardCapturer::KeyboardCapturer() {}
+
+KeyboardCapturer::~KeyboardCapturer() {}
+
+int KeyboardCapturer::Hook() { return 0; }
+
+int KeyboardCapturer::Unhook() { return 0; }

src/device_controller/keyboard/mac/keyboard_capturer.h

@@ -0,0 +1,24 @@
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2024-11-22
+ * Copyright (c) 2024 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _KEYBOARD_CAPTURER_H_
+#define _KEYBOARD_CAPTURER_H_
+
+#include "device_controller.h"
+
+class KeyboardCapturer : public DeviceController {
+ public:
+  KeyboardCapturer();
+  virtual ~KeyboardCapturer();
+
+ public:
+  virtual int Hook();
+  virtual int Unhook();
+
+ private:
+};
+
+#endif

src/device_controller/keyboard/windows/keyboard_capturer.cpp

@@ -0,0 +1,55 @@
+#include "keyboard_capturer.h"
+
+#include "rd_log.h"
+
+static OnKeyAction g_on_key_action = nullptr;
+static void* g_user_ptr = nullptr;
+
+LRESULT CALLBACK KeyboardProc(int nCode, WPARAM wParam, LPARAM lParam) {
+  if (nCode == HC_ACTION && g_on_key_action) {
+    KBDLLHOOKSTRUCT* kbData = reinterpret_cast<KBDLLHOOKSTRUCT*>(lParam);
+
+    // 处理键盘事件
+    if (wParam == WM_KEYDOWN) {
+      // LOG_ERROR("Keydown: [{}]", kbData->vkCode);
+      g_on_key_action(kbData->vkCode, true, g_user_ptr);
+      return 1;
+    } else if (wParam == WM_KEYUP) {
+      // LOG_ERROR("Keyup: [{}]", kbData->vkCode);
+      g_on_key_action(kbData->vkCode, false, g_user_ptr);
+      return 1;
+    } else if (wParam == WM_SYSKEYDOWN) {
+      // LOG_ERROR("System keydown: [{}]", kbData->vkCode);
+      g_on_key_action(kbData->vkCode, true, g_user_ptr);
+      return 1;
+    } else if (wParam == WM_SYSKEYUP) {
+      // LOG_ERROR("System keyup: [{}]", kbData->vkCode);
+      g_on_key_action(kbData->vkCode, false, g_user_ptr);
+      return 1;
+    }
+  }
+
+  // 调用下一个钩子
+  return CallNextHookEx(NULL, nCode, wParam, lParam);
+}
+
+KeyboardCapturer::KeyboardCapturer() {}
+
+KeyboardCapturer::~KeyboardCapturer() {}
+
+int KeyboardCapturer::Hook(OnKeyAction on_key_action, void* user_ptr) {
+  g_on_key_action = on_key_action;
+  g_user_ptr = user_ptr;
+
+  keyboard_hook_ = SetWindowsHookEx(WH_KEYBOARD_LL, KeyboardProc, NULL, 0);
+  if (!keyboard_hook_) {
+    LOG_ERROR("Failed to install keyboard hook!")
+    return -1;
+  }
+  return 0;
+}
+
+int KeyboardCapturer::Unhook() {
+  UnhookWindowsHookEx(keyboard_hook_);
+  return 0;
+}

src/device_controller/keyboard/windows/keyboard_capturer.h

@@ -0,0 +1,27 @@
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2024-11-22
+ * Copyright (c) 2024 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _KEYBOARD_CAPTURER_H_
+#define _KEYBOARD_CAPTURER_H_
+
+#include <Windows.h>
+
+#include "device_controller.h"
+
+class KeyboardCapturer : public DeviceController {
+ public:
+  KeyboardCapturer();
+  virtual ~KeyboardCapturer();
+
+ public:
+  virtual int Hook(OnKeyAction on_key_action, void *user_ptr);
+  virtual int Unhook();
+
+ private:
+  HHOOK keyboard_hook_ = nullptr;
+};
+
+#endif

src/single_window/control_bar.cpp

@@ -47,6 +47,7 @@ int Render::ControlBar() {
     if (ImGui::Button(mouse.c_str(), ImVec2(25, 25))) {
       if (connection_established_) {
         control_mouse_ = !control_mouse_;
+        start_keyboard_capturer_ = !start_keyboard_capturer_;
         mouse_control_button_pressed_ = !mouse_control_button_pressed_;
         mouse_control_button_label_ =
             mouse_control_button_pressed_

src/single_window/render.cpp

@@ -186,7 +186,7 @@ int Render::LoadSettingsFromCacheFile() {
   return 0;
 }
 
-int Render::StartScreenCapture() {
+int Render::StartScreenCapturer() {
   screen_capturer_ = (ScreenCapturer*)screen_capturer_factory_->Create();
   last_frame_time_ = std::chrono::duration_cast<std::chrono::milliseconds>(
                          std::chrono::steady_clock::now().time_since_epoch())
@@ -224,7 +224,7 @@ int Render::StartScreenCapture() {
   return 0;
 }
 
-int Render::StopScreenCapture() {
+int Render::StopScreenCapturer() {
   if (screen_capturer_) {
     LOG_INFO("Stop screen capturer")
     screen_capturer_->Stop();
@@ -236,7 +236,7 @@ int Render::StopScreenCapture() {
   return 0;
 }
 
-int Render::StartSpeakerCapture() {
+int Render::StartSpeakerCapturer() {
   if (!speaker_capturer_) {
     speaker_capturer_ = (SpeakerCapturer*)speaker_capturer_factory_->Create();
     int speaker_capturer_init_ret = speaker_capturer_->Init(
@@ -259,7 +259,7 @@ int Render::StartSpeakerCapture() {
   return 0;
 }
 
-int Render::StopSpeakerCapture() {
+int Render::StopSpeakerCapturer() {
   if (speaker_capturer_) {
     speaker_capturer_->Stop();
   }
@@ -267,8 +267,11 @@ int Render::StopSpeakerCapture() {
   return 0;
 }
 
-int Render::StartMouseControl() {
-  device_controller_factory_ = new DeviceControllerFactory();
+int Render::StartMouseController() {
+  if (!device_controller_factory_) {
+    LOG_INFO("Device controller factory is nullptr");
+    return -1;
+  }
   mouse_controller_ = (MouseController*)device_controller_factory_->Create(
       DeviceControllerFactory::Device::Mouse);
   int mouse_controller_init_ret =
@@ -282,7 +285,7 @@ int Render::StartMouseControl() {
   return 0;
 }
 
-int Render::StopMouseControl() {
+int Render::StopMouseController() {
   if (mouse_controller_) {
     mouse_controller_->Destroy();
     delete mouse_controller_;
@@ -291,6 +294,41 @@ int Render::StopMouseControl() {
   return 0;
 }
 
+int Render::StartKeyboardCapturer() {
+  if (!device_controller_factory_) {
+    LOG_INFO("Device controller factory is nullptr");
+    return -1;
+  }
+  keyboard_capturer_ = (KeyboardCapturer*)device_controller_factory_->Create(
+      DeviceControllerFactory::Device::Keyboard);
+  int keyboard_capturer_init_ret = keyboard_capturer_->Hook(
+      [](int key_code, bool is_down, void* user_ptr) {
+        if (user_ptr) {
+          Render* render = (Render*)user_ptr;
+          render->SendKeyEvent(key_code, is_down);
+        }
+      },
+      this);
+  if (0 != keyboard_capturer_init_ret) {
+    LOG_INFO("Destroy keyboard capturer")
+    keyboard_capturer_->Unhook();
+    keyboard_capturer_ = nullptr;
+  } else {
+    LOG_INFO("Start keyboard capturer");
+  }
+
+  return 0;
+}
+
+int Render::StopKeyboardCapturer() {
+  if (keyboard_capturer_) {
+    keyboard_capturer_->Unhook();
+    delete keyboard_capturer_;
+    keyboard_capturer_ = nullptr;
+  }
+  return 0;
+}
+
 int Render::CreateConnectionPeer() {
   mac_addr_str_ = GetMac();
 
@@ -386,20 +424,28 @@ int Render::CreateRtcConnection() {
         CreateConnection(peer_, client_id_, password_saved_) ? false : true;
   }
 
-  if (start_screen_capture_ && !screen_capture_is_started_) {
-    StartScreenCapture();
-    screen_capture_is_started_ = true;
-  } else if (!start_screen_capture_ && screen_capture_is_started_) {
-    StopScreenCapture();
-    screen_capture_is_started_ = false;
+  if (start_screen_capturer_ && !screen_capturer_is_started_) {
+    StartScreenCapturer();
+    screen_capturer_is_started_ = true;
+  } else if (!start_screen_capturer_ && screen_capturer_is_started_) {
+    StopScreenCapturer();
+    screen_capturer_is_started_ = false;
   }
 
-  if (start_mouse_control_ && !mouse_control_is_started_) {
-    StartMouseControl();
-    mouse_control_is_started_ = true;
-  } else if (!start_mouse_control_ && mouse_control_is_started_) {
-    StopMouseControl();
-    mouse_control_is_started_ = false;
+  if (start_mouse_controller_ && !mouse_controller_is_started_) {
+    StartMouseController();
+    mouse_controller_is_started_ = true;
+  } else if (!start_mouse_controller_ && mouse_controller_is_started_) {
+    StopMouseController();
+    mouse_controller_is_started_ = false;
+  }
+
+  if (start_keyboard_capturer_ && !keyboard_capturer_is_started_) {
+    StartKeyboardCapturer();
+    keyboard_capturer_is_started_ = true;
+  } else if (!start_keyboard_capturer_ && keyboard_capturer_is_started_) {
+    StopKeyboardCapturer();
+    keyboard_capturer_is_started_ = false;
   }
 
   return 0;
@@ -741,7 +787,7 @@ int Render::Run() {
     // speaker capture init
     speaker_capturer_factory_ = new SpeakerCapturerFactory();
 
-    // mouse control
+    // mouse control/keyboard capturer
     device_controller_factory_ = new DeviceControllerFactory();
 
     // RTC

src/single_window/render.h

@@ -95,9 +95,11 @@ class Render {
 
  private:
   int ProcessMouseKeyEvent(SDL_Event &event);
-  int ProcessKeyEvent(SDL_Event &event);
   int ProcessMouseEvent(SDL_Event &event);
 
+  int SendKeyEvent(int key_code, bool is_down);
+  int ProcessKeyEvent(int key_code, bool is_down);
+
   static void SdlCaptureAudioIn(void *userdata, Uint8 *stream, int len);
   static void SdlCaptureAudioOut(void *userdata, Uint8 *stream, int len);
 
@@ -105,14 +107,17 @@ class Render {
   int SaveSettingsIntoCacheFile();
   int LoadSettingsFromCacheFile();
 
-  int StartScreenCapture();
-  int StopScreenCapture();
+  int StartScreenCapturer();
+  int StopScreenCapturer();
 
-  int StartSpeakerCapture();
-  int StopSpeakerCapture();
+  int StartSpeakerCapturer();
+  int StopSpeakerCapturer();
 
-  int StartMouseControl();
-  int StopMouseControl();
+  int StartMouseController();
+  int StopMouseController();
+
+  int StartKeyboardCapturer();
+  int StopKeyboardCapturer();
 
   int CreateConnectionPeer();
 
@@ -350,6 +355,7 @@ class Render {
   SpeakerCapturer *speaker_capturer_ = nullptr;
   DeviceControllerFactory *device_controller_factory_ = nullptr;
   MouseController *mouse_controller_ = nullptr;
+  KeyboardCapturer *keyboard_capturer_ = nullptr;
   uint32_t last_frame_time_;
 
  private:
@@ -369,10 +375,12 @@ class Render {
   bool enable_turn_last_ = false;
 
  private:
-  std::atomic<bool> start_screen_capture_{false};
-  std::atomic<bool> start_mouse_control_{false};
-  std::atomic<bool> screen_capture_is_started_{false};
-  std::atomic<bool> mouse_control_is_started_{false};
+  std::atomic<bool> start_screen_capturer_{false};
+  std::atomic<bool> start_mouse_controller_{false};
+  std::atomic<bool> start_keyboard_capturer_{false};
+  std::atomic<bool> screen_capturer_is_started_{false};
+  std::atomic<bool> mouse_controller_is_started_{false};
+  std::atomic<bool> keyboard_capturer_is_started_{false};
 
  private:
   bool settings_window_pos_reset_ = true;

src/single_window/render_callback_func.cpp

@@ -19,7 +19,6 @@ int Render::ProcessMouseKeyEvent(SDL_Event &event) {
   }
 
   if (SDL_KEYDOWN == event.type || SDL_KEYUP == event.type) {
-    ProcessKeyEvent(event);
   } else {
     ProcessMouseEvent(event);
   }
@@ -87,14 +86,24 @@ int Render::ProcessMouseEvent(SDL_Event &event) {
   return 0;
 }
 
-int Render::ProcessKeyEvent(SDL_Event &event) {
+int Render::SendKeyEvent(int key_code, bool is_down) {
   RemoteAction remote_action;
-  SDL_Keycode key = event.key.keysym.sym;
-  if (SDL_KEYDOWN == event.type) {
-    std::cout << "Key pressed: " << SDL_GetKeyName(key) << std::endl;
-  } else if (SDL_KEYUP == event.type) {
-    std::cout << "Key released: " << SDL_GetKeyName(key) << std::endl;
+  remote_action.type = ControlType::keyboard;
+  if (is_down) {
+    remote_action.k.flag = KeyFlag::key_down;
+  } else {
+    remote_action.k.flag = KeyFlag::key_up;
   }
+  remote_action.k.key_value = key_code;
+
+  SendData(peer_, DATA_TYPE::DATA, (const char *)&remote_action,
+           sizeof(remote_action));
+
+  return 0;
+}
+
+int Render::ProcessKeyEvent(int key_code, bool is_down) {
+  LOG_ERROR("key code [{}], is down [{}]", key_code, is_down);
 
   return 0;
 }
@@ -206,11 +215,12 @@ void Render::OnReceiveDataBufferCb(const char *data, size_t size,
     render->mouse_controller_->SendCommand(remote_action);
   } else if (ControlType::audio_capture == remote_action.type) {
     if (remote_action.a) {
-      render->StartSpeakerCapture();
+      render->StartSpeakerCapturer();
     } else {
-      render->StopSpeakerCapture();
+      render->StopSpeakerCapturer();
     }
   } else if (ControlType::keyboard == remote_action.type) {
+    render->ProcessKeyEvent(remote_action.k.key_value, remote_action.k.flag);
   } else if (ControlType::host_infomation == remote_action.type) {
     render->host_name_ =
         std::string(remote_action.i.host_name, remote_action.i.host_name_size);
@@ -264,8 +274,8 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char *user_id,
     render->connection_status_str_ = "Connected";
     render->connection_established_ = true;
     if (render->peer_reserved_ || !render->is_client_mode_) {
-      render->start_screen_capture_ = true;
-      render->start_mouse_control_ = true;
+      render->start_screen_capturer_ = true;
+      render->start_mouse_controller_ = true;
     }
     if (!render->hostname_sent_) {
       // TODO: self and remote hostname
@@ -289,13 +299,17 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char *user_id,
   } else if (ConnectionStatus::Closed == status) {
     render->connection_status_str_ = "Closed";
     render->password_validating_time_ = 0;
-    render->start_screen_capture_ = false;
-    render->start_mouse_control_ = false;
+    render->start_screen_capturer_ = false;
+    render->mouse_controller_is_started_ = false;
+    render->start_mouse_controller_ = false;
+    render->mouse_controller_is_started_ = false;
     render->connection_established_ = false;
     render->control_mouse_ = false;
+    render->start_keyboard_capturer_ = false;
+    render->keyboard_capturer_is_started_ = false;
     render->hostname_sent_ = false;
     if (render->audio_capture_) {
-      render->StopSpeakerCapture();
+      render->StopSpeakerCapturer();
       render->audio_capture_ = false;
       render->audio_capture_button_pressed_ = false;
     }

xmake.lua

@@ -103,14 +103,20 @@ target("device_controller")
     add_deps("rd_log")
     add_includedirs("src/device_controller", {public = true})
     if is_os("windows") then
-        add_files("src/device_controller/mouse/windows/*.cpp")
-        add_includedirs("src/device_controller/mouse/windows", {public = true})
+        add_files("src/device_controller/mouse/windows/*.cpp",
+        "src/device_controller/keyboard/windows/*.cpp")
+        add_includedirs("src/device_controller/mouse/windows",
+        "src/device_controller/keyboard/windows", {public = true})
     elseif is_os("macosx") then
-        add_files("src/device_controller/mouse/mac/*.cpp")
-        add_includedirs("src/device_controller/mouse/mac", {public = true})
+        add_files("src/device_controller/mouse/mac/*.cpp",
+        "src/device_controller/mouse/keyboard/*.cpp")
+        add_includedirs("src/device_controller/mouse/mac",
+        "src/device_controller/keyboard/mac", {public = true})
     elseif is_os("linux") then
-         add_files("src/device_controller/mouse/linux/*.cpp")
-         add_includedirs("src/device_controller/mouse/linux", {public = true})
+         add_files("src/device_controller/mouse/linux/*.cpp",
+         "src/device_controller/keyboard/linux/*.cpp")
+         add_includedirs("src/device_controller/mouse/linux",
+         "src/device_controller/keyboard/linux", {public = true})
     end
 
 target("config_center")