[feat] mouse/keyboard control and screen capture supported by using X11 on Linux platform

src/device_controller/keyboard/linux/keyboard_capturer.cpp

@@ -1,15 +1,69 @@
 #include "keyboard_capturer.h"
 
-KeyboardCapturer::KeyboardCapturer() {}
+#include "keyboard_converter.h"
+#include "rd_log.h"
 
-KeyboardCapturer::~KeyboardCapturer() {}
+static OnKeyAction g_on_key_action = nullptr;
+static void* g_user_ptr = nullptr;
 
-int KeyboardCapturer::Hook(OnKeyAction on_key_action, void *user_ptr) {
+static int KeyboardEventHandler(Display* display, XEvent* event) {
+  if (event->xkey.type == KeyPress || event->xkey.type == KeyRelease) {
+    KeySym keySym = XKeycodeToKeysym(display, event->xkey.keycode, 0);
+    int key_code = XKeysymToKeycode(display, keySym);
+    bool is_key_down = (event->xkey.type == KeyPress);
+
+    if (g_on_key_action) {
+      g_on_key_action(key_code, is_key_down, g_user_ptr);
+    }
+  }
   return 0;
 }
 
-int KeyboardCapturer::Unhook() { return 0; }
+KeyboardCapturer::KeyboardCapturer() : display_(nullptr), running_(true) {
+  display_ = XOpenDisplay(nullptr);
+  if (!display_) {
+    LOG_ERROR("Failed to open X display.");
+  }
+}
+
+KeyboardCapturer::~KeyboardCapturer() {
+  if (display_) {
+    XCloseDisplay(display_);
+  }
+}
+
+int KeyboardCapturer::Hook(OnKeyAction on_key_action, void* user_ptr) {
+  g_on_key_action = on_key_action;
+  g_user_ptr = user_ptr;
+
+  XSelectInput(display_, DefaultRootWindow(display_),
+               KeyPressMask | KeyReleaseMask);
+
+  while (running_) {
+    XEvent event;
+    XNextEvent(display_, &event);
+    KeyboardEventHandler(display_, &event);
+  }
+
+  return 0;
+}
+
+int KeyboardCapturer::Unhook() {
+  running_ = false;
+  return 0;
+}
 
 int KeyboardCapturer::SendKeyboardCommand(int key_code, bool is_down) {
+  if (!display_) {
+    LOG_ERROR("Display not initialized.");
+    return -1;
+  }
+
+  if (vkCodeToX11KeySym.find(key_code) != vkCodeToX11KeySym.end()) {
+    int x11_key_code = vkCodeToX11KeySym[key_code];
+    KeyCode keycode = XKeysymToKeycode(display_, x11_key_code);
+    XTestFakeKeyEvent(display_, keycode, is_down, CurrentTime);
+    XFlush(display_);
+  }
   return 0;
 }

src/device_controller/keyboard/linux/keyboard_capturer.h

@@ -7,6 +7,10 @@
 #ifndef _KEYBOARD_CAPTURER_H_
 #define _KEYBOARD_CAPTURER_H_
 
+#include <X11/Xlib.h>
+#include <X11/extensions/XTest.h>
+#include <X11/keysym.h>
+
 #include "device_controller.h"
 
 class KeyboardCapturer : public DeviceController {
@@ -20,6 +24,9 @@ class KeyboardCapturer : public DeviceController {
   virtual int SendKeyboardCommand(int key_code, bool is_down);
 
  private:
+  Display *display_;
+  Window root_;
+  bool running_;
 };
 
 #endif

src/device_controller/keyboard_converter.h

@@ -245,4 +245,68 @@ std::map<int, int> CGKeyCodeToVkCode = {
     {0x36, 0x5C},  // Right Command
 };
 
+// Windows vkCode to X11 KeySym
+std::map<int, int> vkCodeToX11KeySym = {
+    {0x41, XK_A},         {0x42, XK_B},      {0x43, XK_C},
+    {0x44, XK_D},         {0x45, XK_E},      {0x46, XK_F},
+    {0x47, XK_G},         {0x48, XK_H},      {0x49, XK_I},
+    {0x4A, XK_J},         {0x4B, XK_K},      {0x4C, XK_L},
+    {0x4D, XK_M},         {0x4E, XK_N},      {0x4F, XK_O},
+    {0x50, XK_P},         {0x51, XK_Q},      {0x52, XK_R},
+    {0x53, XK_S},         {0x54, XK_T},      {0x55, XK_U},
+    {0x56, XK_V},         {0x57, XK_W},      {0x58, XK_X},
+    {0x59, XK_Y},         {0x5A, XK_Z},      {0x30, XK_0},
+    {0x31, XK_1},         {0x32, XK_2},      {0x33, XK_3},
+    {0x34, XK_4},         {0x35, XK_5},      {0x36, XK_6},
+    {0x37, XK_7},         {0x38, XK_8},      {0x39, XK_9},
+    {0x1B, XK_Escape},    {0x0D, XK_Return}, {0x20, XK_space},
+    {0x08, XK_BackSpace}, {0x09, XK_Tab},    {0x25, XK_Left},
+    {0x27, XK_Right},     {0x26, XK_Up},     {0x28, XK_Down},
+    {0x70, XK_F1},        {0x71, XK_F2},     {0x72, XK_F3},
+    {0x73, XK_F4},        {0x74, XK_F5},     {0x75, XK_F6},
+    {0x76, XK_F7},        {0x77, XK_F8},     {0x78, XK_F9},
+    {0x79, XK_F10},       {0x7A, XK_F11},    {0x7B, XK_F12},
+};
+
+// X11 KeySym to Windows vkCode
+std::map<int, int> x11KeySymToVkCode = []() {
+  std::map<int, int> result;
+  for (const auto& pair : vkCodeToX11KeySym) {
+    result[pair.second] = pair.first;
+  }
+  return result;
+}();
+
+// macOS CGKeyCode to X11 KeySym
+std::map<int, int> cgKeyCodeToX11KeySym = {
+    {0x00, XK_A},         {0x0B, XK_B},      {0x08, XK_C},
+    {0x02, XK_D},         {0x0E, XK_E},      {0x03, XK_F},
+    {0x05, XK_G},         {0x04, XK_H},      {0x22, XK_I},
+    {0x26, XK_J},         {0x28, XK_K},      {0x25, XK_L},
+    {0x2E, XK_M},         {0x2D, XK_N},      {0x1F, XK_O},
+    {0x23, XK_P},         {0x0C, XK_Q},      {0x0F, XK_R},
+    {0x01, XK_S},         {0x11, XK_T},      {0x20, XK_U},
+    {0x09, XK_V},         {0x0D, XK_W},      {0x07, XK_X},
+    {0x10, XK_Y},         {0x06, XK_Z},      {0x12, XK_1},
+    {0x13, XK_2},         {0x14, XK_3},      {0x15, XK_4},
+    {0x17, XK_5},         {0x16, XK_6},      {0x1A, XK_7},
+    {0x1C, XK_8},         {0x19, XK_9},      {0x1D, XK_0},
+    {0x35, XK_Escape},    {0x24, XK_Return}, {0x31, XK_space},
+    {0x33, XK_BackSpace}, {0x30, XK_Tab},    {0x7B, XK_Left},
+    {0x7C, XK_Right},     {0x7E, XK_Up},     {0x7D, XK_Down},
+    {0x7A, XK_F1},        {0x78, XK_F2},     {0x63, XK_F3},
+    {0x76, XK_F4},        {0x60, XK_F5},     {0x61, XK_F6},
+    {0x62, XK_F7},        {0x64, XK_F8},     {0x65, XK_F9},
+    {0x6D, XK_F10},       {0x67, XK_F11},    {0x6F, XK_F12},
+};
+
+// X11 KeySym to macOS CGKeyCode
+std::map<int, int> x11KeySymToCgKeyCode = []() {
+  std::map<int, int> result;
+  for (const auto& pair : cgKeyCodeToX11KeySym) {
+    result[pair.second] = pair.first;
+  }
+  return result;
+}();
+
 #endif

src/device_controller/mouse/linux/mouse_controller.cpp

@@ -1,151 +1,120 @@
 #include "mouse_controller.h"
 
+#include <X11/extensions/XTest.h>
+
 #include "rd_log.h"
 
 MouseController::MouseController() {}
 
-MouseController::~MouseController() {
+MouseController::~MouseController() { Destroy(); }
-  if (uinput_fd_) {
-    ioctl(uinput_fd_, UI_DEV_DESTROY);
-    close(uinput_fd_);
-  }
-}
 
 int MouseController::Init(int screen_width, int screen_height) {
-  screen_width_ = screen_width;
+  display_ = XOpenDisplay(NULL);
-  screen_height_ = screen_height;
+  if (!display_) {
-
+    LOG_ERROR("Cannot connect to X server");
-  uinput_fd_ = open("/dev/uinput", O_WRONLY | O_NONBLOCK);
-  if (uinput_fd_ < 0) {
-    LOG_ERROR("Cannot open device: /dev/uinput");
     return -1;
   }
 
-  ioctl(uinput_fd_, UI_SET_EVBIT, EV_KEY);
+  root_ = DefaultRootWindow(display_);
-  ioctl(uinput_fd_, UI_SET_KEYBIT, BTN_RIGHT);
+  screen_width_ = screen_width;
-  ioctl(uinput_fd_, UI_SET_KEYBIT, BTN_LEFT);
+  screen_height_ = screen_height;
-  ioctl(uinput_fd_, UI_SET_EVBIT, EV_ABS);
-  ioctl(uinput_fd_, UI_SET_ABSBIT, ABS_X);
-  ioctl(uinput_fd_, UI_SET_ABSBIT, ABS_Y);
-  ioctl(uinput_fd_, UI_SET_EVBIT, EV_REL);
 
-  struct uinput_user_dev uidev;
+  int event_base, error_base, major_version, minor_version;
-  memset(&uidev, 0, sizeof(uidev));
+  if (!XTestQueryExtension(display_, &event_base, &error_base, &major_version,
-  snprintf(uidev.name, UINPUT_MAX_NAME_SIZE, "VirtualMouse");
+                           &minor_version)) {
-  uidev.id.bustype = BUS_USB;
+    LOG_ERROR("XTest extension not available");
-  uidev.id.version = 1;
+    XCloseDisplay(display_);
-  uidev.id.vendor = 0x1;
+    return -2;
-  uidev.id.product = 0x1;
+  }
-  uidev.absmin[ABS_X] = 0;
-  uidev.absmax[ABS_X] = screen_width_;
-  uidev.absmin[ABS_Y] = 0;
-  uidev.absmax[ABS_Y] = screen_height_;
 
-  int res_uidev = write(uinput_fd_, &uidev, sizeof(uidev));
-  ioctl(uinput_fd_, UI_DEV_CREATE);
   return 0;
 }
 
-int MouseController::Destroy() { return 0; }
+int MouseController::Destroy() {
+  if (display_) {
+    XCloseDisplay(display_);
+    display_ = nullptr;
+  }
+  return 0;
+}
 
 int MouseController::SendMouseCommand(RemoteAction remote_action) {
-  int mouse_pos_x = remote_action.m.x * screen_width_;
+  switch (remote_action.type) {
-  int mouse_pos_y = remote_action.m.y * screen_height_;
+    case mouse:
-
-  if (remote_action.type == ControlType::mouse) {
-    struct input_event event;
-    memset(&event, 0, sizeof(event));
-    gettimeofday(&event.time, NULL);
-
       switch (remote_action.m.flag) {
+        case MouseFlag::move:
+          SetMousePosition(
+              static_cast<int>(remote_action.m.x * screen_width_),
+              static_cast<int>(remote_action.m.y * screen_height_));
+          break;
         case MouseFlag::left_down:
-        SimulateKeyDown(uinput_fd_, BTN_LEFT);
+          XTestFakeButtonEvent(display_, 1, True, CurrentTime);
+          XFlush(display_);
           break;
         case MouseFlag::left_up:
-        SimulateKeyUp(uinput_fd_, BTN_LEFT);
+          XTestFakeButtonEvent(display_, 1, False, CurrentTime);
+          XFlush(display_);
           break;
         case MouseFlag::right_down:
-        SimulateKeyDown(uinput_fd_, BTN_RIGHT);
+          XTestFakeButtonEvent(display_, 3, True, CurrentTime);
+          XFlush(display_);
           break;
         case MouseFlag::right_up:
-        SimulateKeyUp(uinput_fd_, BTN_RIGHT);
+          XTestFakeButtonEvent(display_, 3, False, CurrentTime);
+          XFlush(display_);
           break;
         case MouseFlag::middle_down:
-        SimulateKeyDown(uinput_fd_, BTN_MIDDLE);
+          XTestFakeButtonEvent(display_, 2, True, CurrentTime);
+          XFlush(display_);
           break;
         case MouseFlag::middle_up:
-        SimulateKeyUp(uinput_fd_, BTN_MIDDLE);
+          XTestFakeButtonEvent(display_, 2, False, CurrentTime);
+          XFlush(display_);
           break;
-      case MouseFlag::wheel_vertical:
+        case MouseFlag::wheel_vertical: {
-        event.type = EV_REL;
+          if (remote_action.m.s > 0) {
-        event.code = REL_WHEEL;
+            SimulateMouseWheel(4, remote_action.m.s);
-        event.value = remote_action.m.s;
+          } else if (remote_action.m.s < 0) {
-        (void)write(uinput_fd_, &event, sizeof(event));
+            SimulateMouseWheel(5, -remote_action.m.s);
-        break;
+          }
-      case MouseFlag::wheel_horizontal:
-        event.type = EV_REL;
-        event.code = REL_HWHEEL;
-        event.value = remote_action.m.s;
-        (void)write(uinput_fd_, &event, sizeof(event));
-        break;
-      default:
-        SetMousePosition(uinput_fd_, mouse_pos_x, mouse_pos_y);
           break;
         }
+        case MouseFlag::wheel_horizontal: {
+          if (remote_action.m.s > 0) {
+            SimulateMouseWheel(6, remote_action.m.s);
+          } else if (remote_action.m.s < 0) {
+            SimulateMouseWheel(7, -remote_action.m.s);
+          }
+          break;
+        }
+      }
+      break;
+    default:
+      break;
   }
 
   return 0;
 }
 
-void MouseController::SimulateKeyDown(int fd, int kval) {
+void MouseController::SetMousePosition(int x, int y) {
-  int res_ev = 0;
+  XWarpPointer(display_, None, root_, 0, 0, 0, 0, x, y);
-  struct input_event event;
+  XFlush(display_);
-  memset(&event, 0, sizeof(event));
-  gettimeofday(&event.time, 0);
-
-  event.type = EV_KEY;
-  event.value = 1;
-  event.code = kval;
-  res_ev = write(fd, &event, sizeof(event));
-
-  event.type = EV_SYN;
-  event.value = 0;
-  event.code = SYN_REPORT;
-  res_ev = write(fd, &event, sizeof(event));
 }
 
-void MouseController::SimulateKeyUp(int fd, int kval) {
+void MouseController::SimulateKeyDown(int kval) {
-  int res_ev = 0;
+  XTestFakeKeyEvent(display_, kval, True, CurrentTime);
-  struct input_event event;
+  XFlush(display_);
-  memset(&event, 0, sizeof(event));
-  gettimeofday(&event.time, 0);
-
-  event.type = EV_KEY;
-  event.value = 0;
-  event.code = kval;
-  res_ev = write(fd, &event, sizeof(event));
-
-  event.type = EV_SYN;
-  event.value = 0;
-  event.code = SYN_REPORT;
-  res_ev = write(fd, &event, sizeof(event));
 }
 
-void MouseController::SetMousePosition(int fd, int x, int y) {
+void MouseController::SimulateKeyUp(int kval) {
-  struct input_event ev[2], ev_sync;
+  XTestFakeKeyEvent(display_, kval, False, CurrentTime);
-  memset(ev, 0, sizeof(ev));
+  XFlush(display_);
-  memset(&ev_sync, 0, sizeof(ev_sync));
+}
-
+
-  ev[0].type = EV_ABS;
+void MouseController::SimulateMouseWheel(int direction_button, int count) {
-  ev[0].code = ABS_X;
+  for (int i = 0; i < count; ++i) {
-  ev[0].value = x;
+    XTestFakeButtonEvent(display_, direction_button, True, CurrentTime);
-  ev[1].type = EV_ABS;
+    XTestFakeButtonEvent(display_, direction_button, False, CurrentTime);
-  ev[1].code = ABS_Y;
+  }
-  ev[1].value = y;
+  XFlush(display_);
-  int res_w = write(fd, ev, sizeof(ev));
-
-  ev_sync.type = EV_SYN;
-  ev_sync.value = 0;
-  ev_sync.code = 0;
-  int res_ev_sync = write(fd, &ev_sync, sizeof(ev_sync));
 }

src/device_controller/mouse/linux/mouse_controller.h

@@ -1,17 +1,14 @@
 /*
  * @Author: DI JUNKUN
- * @Date: 2023-12-14
+ * @Date: 2025-05-07
- * Copyright (c) 2023 by DI JUNKUN, All Rights Reserved.
+ * Copyright (c) 2025 by DI JUNKUN, All Rights Reserved.
  */
 
 #ifndef _MOUSE_CONTROLLER_H_
 #define _MOUSE_CONTROLLER_H_
 
-#include <fcntl.h>
+#include <X11/Xlib.h>
-#include <linux/uinput.h>
+#include <X11/Xutil.h>
-#include <net/if.h>
-#include <sys/ioctl.h>
-#include <sys/socket.h>
 #include <unistd.h>
 
 #include "device_controller.h"
@@ -27,13 +24,13 @@ class MouseController : public DeviceController {
   virtual int SendMouseCommand(RemoteAction remote_action);
 
  private:
-  void SimulateKeyDown(int fd, int kval);
+  void SimulateKeyDown(int kval);
-  void SimulateKeyUp(int fd, int kval);
+  void SimulateKeyUp(int kval);
-  void SetMousePosition(int fd, int x, int y);
+  void SetMousePosition(int x, int y);
+  void SimulateMouseWheel(int direction_button, int count);
 
- private:
+  Display* display_ = nullptr;
-  int uinput_fd_;
+  Window root_ = 0;
-  struct uinput_user_dev uinput_dev_;
   int screen_width_ = 0;
   int screen_height_ = 0;
 };

src/screen_capturer/linux/screen_capturer_x11.cpp

@@ -1,160 +1,118 @@
 #include "screen_capturer_x11.h"
 
-#include <iostream>
+#include <chrono>
+#include <thread>
 
+#include "libyuv.h"
 #include "rd_log.h"
 
-#define NV12_BUFFER_SIZE 1280 * 720 * 3 / 2
-unsigned char nv12_buffer_[NV12_BUFFER_SIZE];
-
 ScreenCapturerX11::ScreenCapturerX11() {}
 
-ScreenCapturerX11::~ScreenCapturerX11() {
+ScreenCapturerX11::~ScreenCapturerX11() { Destroy(); }
-  if (inited_ && capture_thread_.joinable()) {
-    capture_thread_.join();
-    inited_ = false;
-  }
-}
 
 int ScreenCapturerX11::Init(const int fps, cb_desktop_data cb) {
-  if (cb) {
+  display_ = XOpenDisplay(nullptr);
-    _on_data = cb;
+  if (!display_) {
+    LOG_ERROR("Cannot connect to X server");
+    return -1;
+  }
+
+  root_ = DefaultRootWindow(display_);
+  XWindowAttributes attr;
+  XGetWindowAttributes(display_, root_, &attr);
+
+  width_ = attr.width;
+  height_ = attr.height;
+
+  if (width_ % 2 != 0 || height_ % 2 != 0) {
+    LOG_ERROR("Width and height must be even numbers");
+    return -2;
   }
 
   fps_ = fps;
+  callback_ = cb;
 
-  av_log_set_level(AV_LOG_QUIET);
+  y_plane_.resize(width_ * height_);
-
+  uv_plane_.resize((width_ / 2) * (height_ / 2) * 2);
-  pFormatCtx_ = avformat_alloc_context();
-
-  avdevice_register_all();
-
-  // grabbing frame rate
-  av_dict_set(&options_, "framerate", "30", 0);
-  // show remote cursor
-  av_dict_set(&options_, "capture_cursor", "0", 0);
-  // Make the grabbed area follow the mouse
-  // av_dict_set(&options_, "follow_mouse", "centered", 0);
-  // Video frame size. The default is to capture the full screen
-  // av_dict_set(&options_, "video_size", "1280x720", 0);
-  std::string capture_method = "x11grab";
-  ifmt_ = (AVInputFormat *)av_find_input_format(capture_method.c_str());
-  if (!ifmt_) {
-    LOG_ERROR("Couldn't find_input_format [{}]", capture_method.c_str());
-  }
-
-  const char *display = std::getenv("DISPLAY");
-  // Grab at position 10,20
-  if (display) {
-    if (avformat_open_input(&pFormatCtx_, display, ifmt_, &options_) != 0) {
-      LOG_ERROR("Couldn't open input stream {}", display);
-      return -1;
-    } else {
-      LOG_INFO("Open input stream [{}]", display);
-    }
-  } else {
-    LOG_ERROR("DISPLAY environment variable not set");
-    return -1;
-  }
-
-  if (avformat_find_stream_info(pFormatCtx_, NULL) < 0) {
-    LOG_ERROR("Couldn't find stream information");
-    return -1;
-  }
-
-  videoindex_ = -1;
-  for (i_ = 0; i_ < pFormatCtx_->nb_streams; i_++)
-    if (pFormatCtx_->streams[i_]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
-      videoindex_ = i_;
-      break;
-    }
-  if (videoindex_ == -1) {
-    LOG_ERROR("Didn't find a video stream");
-    return -1;
-  }
-
-  pCodecParam_ = pFormatCtx_->streams[videoindex_]->codecpar;
-
-  pCodecCtx_ = avcodec_alloc_context3(NULL);
-  avcodec_parameters_to_context(pCodecCtx_, pCodecParam_);
-
-  pCodec_ = const_cast<AVCodec *>(avcodec_find_decoder(pCodecCtx_->codec_id));
-  if (pCodec_ == NULL) {
-    LOG_ERROR("Codec not found");
-    return -1;
-  }
-  if (avcodec_open2(pCodecCtx_, pCodec_, NULL) < 0) {
-    LOG_ERROR("Could not open codec");
-    return -1;
-  }
-
-  const int screen_w = pFormatCtx_->streams[videoindex_]->codecpar->width;
-  const int screen_h = pFormatCtx_->streams[videoindex_]->codecpar->height;
-
-  pFrame_ = av_frame_alloc();
-  pFrameNv12_ = av_frame_alloc();
-
-  pFrame_->width = screen_w;
-  pFrame_->height = screen_h;
-  pFrameNv12_->width = 1280;
-  pFrameNv12_->height = 720;
-
-  packet_ = (AVPacket *)av_malloc(sizeof(AVPacket));
-
-  img_convert_ctx_ = sws_getContext(
-      pFrame_->width, pFrame_->height, pCodecCtx_->pix_fmt, pFrameNv12_->width,
-      pFrameNv12_->height, AV_PIX_FMT_NV12, SWS_BICUBIC, NULL, NULL, NULL);
-
-  inited_ = true;
 
   return 0;
 }
 
 int ScreenCapturerX11::Destroy() {
-  running_ = false;
+  Stop();
+  CleanUp();
   return 0;
 }
 
 int ScreenCapturerX11::Start() {
+  if (running_) return 0;
   running_ = true;
-  capture_thread_ = std::thread([this]() {
+  paused_ = false;
+  thread_ = std::thread([this]() {
     while (running_) {
-      if (av_read_frame(pFormatCtx_, packet_) >= 0) {
+      if (!paused_) OnFrame();
-        if (packet_->stream_index == videoindex_) {
-          avcodec_send_packet(pCodecCtx_, packet_);
-          av_packet_unref(packet_);
-          got_picture_ = avcodec_receive_frame(pCodecCtx_, pFrame_);
-
-          if (!got_picture_) {
-            av_image_fill_arrays(pFrameNv12_->data, pFrameNv12_->linesize,
-                                 nv12_buffer_, AV_PIX_FMT_NV12,
-                                 pFrameNv12_->width, pFrameNv12_->height, 1);
-
-            sws_scale(img_convert_ctx_, pFrame_->data, pFrame_->linesize, 0,
-                      pFrame_->height, pFrameNv12_->data,
-                      pFrameNv12_->linesize);
-
-            _on_data((unsigned char *)nv12_buffer_,
-                     pFrameNv12_->width * pFrameNv12_->height * 3 / 2,
-                     pFrameNv12_->width, pFrameNv12_->height);
-          }
-        }
-      }
     }
   });
-
   return 0;
 }
 
 int ScreenCapturerX11::Stop() {
+  if (!running_) return 0;
   running_ = false;
+  if (thread_.joinable()) thread_.join();
   return 0;
 }
 
-int ScreenCapturerX11::Pause() { return 0; }
+int ScreenCapturerX11::Pause() {
+  paused_ = true;
+  return 0;
+}
 
-int ScreenCapturerX11::Resume() { return 0; }
+int ScreenCapturerX11::Resume() {
+  paused_ = false;
+  return 0;
+}
 
-void ScreenCapturerX11::OnFrame() {}
+void ScreenCapturerX11::OnFrame() {
+  if (!display_) return;
 
-void ScreenCapturerX11::CleanUp() {}
+  XImage* image =
+      XGetImage(display_, root_, 0, 0, width_, height_, AllPlanes, ZPixmap);
+  if (!image) return;
+
+  bool needs_copy = image->bytes_per_line != width_ * 4;
+  std::vector<uint8_t> argb_buf;
+  uint8_t* src_argb = nullptr;
+
+  if (needs_copy) {
+    argb_buf.resize(width_ * height_ * 4);
+    for (int y = 0; y < height_; ++y) {
+      memcpy(&argb_buf[y * width_ * 4], image->data + y * image->bytes_per_line,
+             width_ * 4);
+    }
+    src_argb = argb_buf.data();
+  } else {
+    src_argb = reinterpret_cast<uint8_t*>(image->data);
+  }
+
+  libyuv::ARGBToNV12(src_argb, width_ * 4, y_plane_.data(), width_,
+                     uv_plane_.data(), width_, width_, height_);
+
+  std::vector<uint8_t> nv12;
+  nv12.reserve(y_plane_.size() + uv_plane_.size());
+  nv12.insert(nv12.end(), y_plane_.begin(), y_plane_.end());
+  nv12.insert(nv12.end(), uv_plane_.begin(), uv_plane_.end());
+
+  if (callback_) {
+    callback_(nv12.data(), width_ * height_ * 3 / 2, width_, height_);
+  }
+
+  XDestroyImage(image);
+}
+
+void ScreenCapturerX11::CleanUp() {
+  if (display_) {
+    XCloseDisplay(display_);
+    display_ = nullptr;
+  }
+}

src/screen_capturer/linux/screen_capturer_x11.h

@@ -1,23 +1,33 @@
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2025-05-07
+ * Copyright (c) 2025 by DI JUNKUN, All Rights Reserved.
+ */
+
 #ifndef _SCREEN_CAPTURER_X11_H_
 #define _SCREEN_CAPTURER_X11_H_
 
-#include <atomic>
+#include <X11/Xlib.h>
-#include <functional>
+#include <X11/Xutil.h>
-#include <string>
-#include <thread>
 
-#include "screen_capturer.h"
+#include <atomic>
-#ifdef __cplusplus
+#include <cstring>
-extern "C" {
+#include <functional>
-#endif
+#include <iostream>
-#include <libavcodec/avcodec.h>
+#include <thread>
-#include <libavdevice/avdevice.h>
+#include <vector>
-#include <libavformat/avformat.h>
+
-#include <libavutil/imgutils.h>
+class ScreenCapturer {
-#include <libswscale/swscale.h>
+ public:
-#ifdef __cplusplus
+  typedef std::function<void(unsigned char*, int width, int height, int stride)>
+      cb_desktop_data;
+
+  virtual ~ScreenCapturer() {}
+  virtual int Init(const int fps, cb_desktop_data cb) = 0;
+  virtual int Destroy() = 0;
+  virtual int Start() = 0;
+  virtual int Stop() = 0;
 };
-#endif
 
 class ScreenCapturerX11 : public ScreenCapturer {
  public:
@@ -25,10 +35,10 @@ class ScreenCapturerX11 : public ScreenCapturer {
   ~ScreenCapturerX11();
 
  public:
-  virtual int Init(const int fps, cb_desktop_data cb) override;
+  int Init(const int fps, cb_desktop_data cb) override;
-  virtual int Destroy() override;
+  int Destroy() override;
-  virtual int Start() override;
+  int Start() override;
-  virtual int Stop() override;
+  int Stop() override;
 
   int Pause();
   int Resume();
@@ -39,40 +49,19 @@ class ScreenCapturerX11 : public ScreenCapturer {
   void CleanUp();
 
  private:
-  std::atomic_bool _running;
+  Display* display_ = nullptr;
-  std::atomic_bool _paused;
+  Window root_ = 0;
-  std::atomic_bool _inited;
+  int width_ = 0;
+  int height_ = 0;
+  std::thread thread_;
+  std::atomic<bool> running_{false};
+  std::atomic<bool> paused_{false};
+  int fps_ = 30;
+  cb_desktop_data callback_;
 
-  std::thread _thread;
+  // 缓冲区
-
+  std::vector<uint8_t> y_plane_;
-  std::string _device_name;
+  std::vector<uint8_t> uv_plane_;
-
-  int _fps;
-
-  cb_desktop_data _on_data;
-
- private:
-  int i_ = 0;
-  int videoindex_ = 0;
-  int got_picture_ = 0;
-  int fps_ = 0;
-  bool inited_ = false;
-
-  // ffmpeg
-  AVFormatContext *pFormatCtx_ = nullptr;
-  AVCodecContext *pCodecCtx_ = nullptr;
-  AVCodec *pCodec_ = nullptr;
-  AVCodecParameters *pCodecParam_ = nullptr;
-  AVDictionary *options_ = nullptr;
-  AVInputFormat *ifmt_ = nullptr;
-  AVFrame *pFrame_ = nullptr;
-  AVFrame *pFrameNv12_ = nullptr;
-  AVPacket *packet_ = nullptr;
-  struct SwsContext *img_convert_ctx_ = nullptr;
-
-  // thread
-  std::thread capture_thread_;
-  std::atomic_bool running_;
 };
 
 #endif

src/screen_capturer/linux/x11_session.h

@@ -1,37 +0,0 @@
-#ifndef _X11_SESSION_H_
-#define _X11_SESSION_H_
-
-class X11Session {
- public:
-  struct x11_session_frame {
-    unsigned int width;
-    unsigned int height;
-    unsigned int row_pitch;
-
-    const unsigned char *data;
-  };
-
-  class x11_session_observer {
-   public:
-    virtual ~x11_session_observer() {}
-    virtual void OnFrame(const x11_session_frame &frame) = 0;
-  };
-
- public:
-  virtual void Release() = 0;
-
-  virtual int Initialize() = 0;
-
-  virtual void RegisterObserver(x11_session_observer *observer) = 0;
-
-  virtual int Start() = 0;
-  virtual int Stop() = 0;
-
-  virtual int Pause() = 0;
-  virtual int Resume() = 0;
-
- protected:
-  virtual ~X11Session(){};
-};
-
-#endif

src/screen_capturer/linux/x11_session_impl.cpp

@@ -1,49 +0,0 @@
-#include "x11_session_impl.h"
-
-#include <atomic>
-#include <functional>
-#include <iostream>
-#include <memory>
-
-#define CHECK_INIT                            \
-  if (!is_initialized_) {                     \
-    std::cout << "AE_NEED_INIT" << std::endl; \
-    return 4;                                 \
-  }
-
-X11SessionImpl::X11SessionImpl() {}
-
-X11SessionImpl::~X11SessionImpl() {
-  Stop();
-  CleanUp();
-}
-
-void X11SessionImpl::Release() { delete this; }
-
-int X11SessionImpl::Initialize() { return 0; }
-
-void X11SessionImpl::RegisterObserver(x11_session_observer *observer) {
-  observer_ = observer;
-}
-
-int X11SessionImpl::Start() {
-  if (is_running_) return 0;
-
-  int error = 1;
-
-  CHECK_INIT;
-
-  return error;
-}
-
-int X11SessionImpl::Stop() { return 0; }
-
-int X11SessionImpl::Pause() { return 0; }
-
-int X11SessionImpl::Resume() { return 0; }
-
-void X11SessionImpl::OnFrame() {}
-
-void X11SessionImpl::OnClosed() {}
-
-void X11SessionImpl::CleanUp() {}

src/screen_capturer/linux/x11_session_impl.h

@@ -1,44 +0,0 @@
-#ifndef _WGC_SESSION_IMPL_H_
-#define _WGC_SESSION_IMPL_H_
-
-#include <mutex>
-#include <thread>
-
-#include "x11_session.h"
-
-class X11SessionImpl : public X11Session {
- public:
-  X11SessionImpl();
-  ~X11SessionImpl() override;
-
- public:
-  void Release() override;
-
-  int Initialize() override;
-
-  void RegisterObserver(x11_session_observer *observer) override;
-
-  int Start() override;
-  int Stop() override;
-
-  int Pause() override;
-  int Resume() override;
-
- private:
-  void OnFrame();
-  void OnClosed();
-
-  void CleanUp();
-
-  // void message_func();
-
- private:
-  std::mutex lock_;
-  bool is_initialized_ = false;
-  bool is_running_ = false;
-  bool is_paused_ = false;
-
-  x11_session_observer *observer_ = nullptr;
-};
-
-#endif

xmake.lua

@@ -27,6 +27,7 @@ if is_os("windows") then
         "Imm32", "iphlpapi")
     add_cxflags("/WX")
 elseif is_os("linux") then
+    add_requires("libyuv") 
     add_requires("ffmpeg 5.1.2", {system = false})
     add_syslinks("pthread", "dl")
     add_linkdirs("thirdparty/projectx/thirdparty/nvcodec/lib/x64")
@@ -34,7 +35,7 @@ elseif is_os("linux") then
     add_ldflags("-lavformat", "-lavdevice", "-lavfilter", "-lavcodec",
         "-lswscale", "-lavutil", "-lswresample",
         "-lasound", "-lxcb-shape", "-lxcb-xfixes", "-lsndio", "-lxcb", 
-        "-lxcb-shm", "-lXext", "-lX11", "-lXv", "-ldl", "-lpthread",
+        "-lxcb-shm", "-lXext", "-lX11", "-lXv", "-lXtst", "-ldl", "-lpthread",
         {force = true})
     add_cxflags("-Wno-unused-variable")   
 elseif is_os("macosx") then
@@ -79,7 +80,7 @@ target("screen_capturer")
         add_includedirs("src/screen_capturer/macosx/avfoundation",
         "src/screen_capturer/macosx/screen_capturer_kit", {public = true})
     elseif is_os("linux") then
-        add_packages("ffmpeg")
+        add_packages("libyuv", "ffmpeg")
         add_files("src/screen_capturer/linux/*.cpp")
         add_includedirs("src/screen_capturer/linux", {public = true})
     end