From 09c42dd7ed1b145feee4bd37fa9c6127d61216be Mon Sep 17 00:00:00 2001 From: dijunkun Date: Wed, 13 Dec 2023 17:26:08 +0800 Subject: [PATCH] 1.Add linux mouse control test;2.Fix unused variables warnning --- src/gui/main.cpp | 204 +++++++++++------- .../linux/screen_capture_x11.cpp | 7 +- src/screen_capture/linux/screen_capture_x11.h | 1 + test/linux_mouse_control/mouse_control.cpp | 150 +++++++++++++ thirdparty/ffmpeg/xmake.lua | 1 + thirdparty/projectx | 2 +- thirdparty/sdl2/xmake.lua | 1 - xmake.lua | 13 +- 8 files changed, 292 insertions(+), 87 deletions(-) create mode 100644 test/linux_mouse_control/mouse_control.cpp diff --git a/src/gui/main.cpp b/src/gui/main.cpp index 9e39be4..553d9a4 100644 --- a/src/gui/main.cpp +++ b/src/gui/main.cpp @@ -78,8 +78,13 @@ SDL_Window *window; static SDL_AudioDeviceID input_dev; static SDL_AudioDeviceID output_dev; static int uinput_fd; +static struct uinput_user_dev uinput_dev; +#define KEY_CUSTOM_UP 0x20 +#define KEY_CUSTOM_DOWN 0x30 static int fd_mouse = -1; static int fd_kbd = -1; +static std::atomic mouse_pos_x_last = -65535; +static std::atomic mouse_pos_y_last = -65535; uint32_t start_time, end_time, elapsed_time; uint32_t frame_count = 0; @@ -92,9 +97,6 @@ static int out_pos = 0; static std::atomic audio_buffer_fresh = false; static uint32_t last_ts = 0; -char *out = "audio_old.pcm"; -FILE *outfile = fopen(out, "wb+"); - int64_t src_ch_layout = AV_CH_LAYOUT_MONO; int src_rate = 48000; enum AVSampleFormat src_sample_fmt = AV_SAMPLE_FMT_S16; @@ -228,10 +230,6 @@ inline int ProcessMouseKeyEven(SDL_Event &ev) { } } else if (SDL_MOUSEBUTTONDOWN == ev.type) { if (SDL_BUTTON_LEFT == ev.button.button) { - int px = ev.button.x; - int py = ev.button.y; - // printf("SDL_MOUSEBUTTONDOWN x, y %d %d \n", px, py); - remote_action.type = ControlType::mouse; remote_action.m.flag = MouseFlag::left_down; remote_action.m.x = ev.button.x * ratio; @@ -241,10 +239,6 @@ inline int ProcessMouseKeyEven(SDL_Event &ev) { sizeof(remote_action)); } else if (SDL_BUTTON_RIGHT == ev.button.button) { - int px = ev.button.x; - int py = ev.button.y; - // printf("SDL_BUTTON_RIGHT x, y %d %d \n", px, py); - remote_action.type = ControlType::mouse; remote_action.m.flag = MouseFlag::right_down; remote_action.m.x = ev.button.x * ratio; @@ -255,10 +249,6 @@ inline int ProcessMouseKeyEven(SDL_Event &ev) { } } else if (SDL_MOUSEBUTTONUP == ev.type) { if (SDL_BUTTON_LEFT == ev.button.button) { - int px = ev.button.x; - int py = ev.button.y; - // printf("SDL_MOUSEBUTTONUP x, y %d %d \n", px, py); - remote_action.type = ControlType::mouse; remote_action.m.flag = MouseFlag::left_up; remote_action.m.x = ev.button.x * ratio; @@ -268,10 +258,6 @@ inline int ProcessMouseKeyEven(SDL_Event &ev) { sizeof(remote_action)); } else if (SDL_BUTTON_RIGHT == ev.button.button) { - int px = ev.button.x; - int py = ev.button.y; - // printf("SDL_MOUSEBUTTONUP x, y %d %d \n", px, py); - remote_action.type = ControlType::mouse; remote_action.m.flag = MouseFlag::right_up; remote_action.m.x = ev.button.x * ratio; @@ -281,11 +267,6 @@ inline int ProcessMouseKeyEven(SDL_Event &ev) { sizeof(remote_action)); } } else if (SDL_MOUSEMOTION == ev.type) { - int px = ev.motion.x; - int py = ev.motion.y; - - // printf("SDL_MOUSEMOTION x, y %d %d \n", px, py); - remote_action.type = ControlType::mouse; remote_action.m.flag = MouseFlag::move; remote_action.m.x = ev.button.x * ratio; @@ -403,8 +384,9 @@ void ClientReceiveAudioBuffer(const char *data, size_t size, SDL_QueueAudio(output_dev, data, size); } -void simulate_key(int fd, int kval) { +void simulate_key_down(int fd, int kval) { struct input_event event; + memset(&event, 0, sizeof(event)); gettimeofday(&event.time, 0); // 按下kval键 event.type = EV_KEY; @@ -416,7 +398,10 @@ void simulate_key(int fd, int kval) { event.value = 0; event.code = SYN_REPORT; write(fd, &event, sizeof(event)); +} +void simulate_key_up(int fd, int kval) { + struct input_event event; memset(&event, 0, sizeof(event)); gettimeofday(&event.time, 0); // 松开kval键 @@ -431,25 +416,93 @@ void simulate_key(int fd, int kval) { write(fd, &event, sizeof(event)); } +void mouseSetPosition(int fd, int x, int y) { + struct input_event ev[2], ev_sync; + memset(ev, 0, sizeof(ev)); + memset(&ev_sync, 0, sizeof(ev_sync)); + + ev[0].type = EV_ABS; + ev[0].code = ABS_X; + ev[0].value = x; + ev[1].type = EV_ABS; + ev[1].code = ABS_Y; + ev[1].value = y; + + int res_w = write(fd, ev, sizeof(ev)); + + std::cout << "res w : " << res_w << "\n"; + + ev_sync.type = EV_SYN; + ev_sync.value = 0; + ev_sync.code = 0; + int res_ev_sync = write(fd, &ev_sync, sizeof(ev_sync)); + + std::cout << "res syn : " << res_ev_sync << "\n"; +} + // 鼠标移动模拟 -void simulate_mouse(int fd, int rel_x, int rel_y) { - struct input_event event; - gettimeofday(&event.time, 0); - // x轴坐标的相对位移 - event.type = EV_REL; - event.value = rel_x; - event.code = REL_X; - write(fd, &event, sizeof(event)); - // y轴坐标的相对位移 - event.type = EV_REL; - event.value = rel_y; - event.code = REL_Y; - write(fd, &event, sizeof(event)); +void simulate_mouse(int fd, int x, int y) { + struct input_event ev; + memset(&ev, 0, sizeof(struct input_event)); + gettimeofday(&ev.time, NULL); + ev.type = EV_REL; + ev.code = REL_X; + ev.value = x; + if (write(fd, &ev, sizeof(struct input_event)) < 0) + LOG_ERROR("error: write1"); + memset(&ev, 0, sizeof(struct input_event)); + ev.type = EV_SYN; + if (write(fd, &ev, sizeof(struct input_event)) < 0) + LOG_ERROR("error: write4"); + + memset(&ev, 0, sizeof(struct input_event)); + ev.type = EV_REL; + ev.code = REL_Y; + ev.value = y; + if (write(fd, &ev, sizeof(struct input_event)) < 0) + LOG_ERROR("error: write2"); + memset(&ev, 0, sizeof(struct input_event)); + ev.type = EV_SYN; + if (write(fd, &ev, sizeof(struct input_event)) < 0) + LOG_ERROR("error: write3"); + // 同步 - event.type = EV_SYN; - event.value = 0; - event.code = SYN_REPORT; - write(fd, &event, sizeof(event)); + // ev.type = EV_SYN; + // ev.value = 0; + // ev.code = SYN_REPORT; + // write(fd, &ev, sizeof(ev)); +} + +void simulate_mouse_abs(int fd, int x, int y) { + struct input_event ev; + memset(&ev, 0, sizeof(struct input_event)); + gettimeofday(&ev.time, NULL); + ev.type = EV_ABS; + ev.code = ABS_X; + ev.value = x; + if (write(fd, &ev, sizeof(struct input_event)) < 0) + LOG_ERROR("error: write1"); + memset(&ev, 0, sizeof(struct input_event)); + ev.type = EV_SYN; + if (write(fd, &ev, sizeof(struct input_event)) < 0) + LOG_ERROR("error: write4"); + + memset(&ev, 0, sizeof(struct input_event)); + ev.type = EV_ABS; + ev.code = ABS_Y; + ev.value = y; + if (write(fd, &ev, sizeof(struct input_event)) < 0) + LOG_ERROR("error: write2"); + memset(&ev, 0, sizeof(struct input_event)); + ev.type = EV_SYN; + if (write(fd, &ev, sizeof(struct input_event)) < 0) + LOG_ERROR("error: write3"); + + // 同步 + ev.type = EV_SYN; + ev.value = 0; + ev.code = SYN_REPORT; + write(fd, &ev, sizeof(ev)); } void ServerReceiveDataBuffer(const char *data, size_t size, const char *user_id, @@ -465,6 +518,9 @@ void ServerReceiveDataBuffer(const char *data, size_t size, const char *user_id, int mouse_pos_x = remote_action.m.x * screen_w / 1280; int mouse_pos_y = remote_action.m.y * screen_h / 720; + + LOG_ERROR("[{} {}] [{} {}]", screen_w, screen_h, mouse_pos_x, mouse_pos_y); + #if 1 #ifdef _WIN32 INPUT ip; @@ -530,16 +586,23 @@ void ServerReceiveDataBuffer(const char *data, size_t size, const char *user_id, gettimeofday(&event.time, NULL); if (remote_action.m.flag == MouseFlag::left_down) { - simulate_key(fd_mouse, BTN_LEFT); + simulate_key_down(uinput_fd, BTN_LEFT); } else if (remote_action.m.flag == MouseFlag::left_up) { - simulate_key(fd_mouse, BTN_LEFT); + simulate_key_up(uinput_fd, BTN_LEFT); } else if (remote_action.m.flag == MouseFlag::right_down) { - simulate_key(fd_mouse, BTN_RIGHT); + simulate_key_down(uinput_fd, BTN_RIGHT); } else if (remote_action.m.flag == MouseFlag::right_up) { - simulate_key(fd_mouse, BTN_RIGHT); + simulate_key_up(uinput_fd, BTN_RIGHT); } else { - simulate_mouse(fd_mouse, mouse_pos_x, mouse_pos_y); + mouseSetPosition(uinput_fd, mouse_pos_x, mouse_pos_y); + // simulate_mouse(uinput_fd, rel_x, rel_y); + // simulate_mouse_abs(uinput_fd, 65535, 65535); + mouse_pos_x_last = mouse_pos_x; + mouse_pos_y_last = mouse_pos_y; } + + // report_key(EV_KEY, KEY_A, 1); // Report BUTTON A CLICK - PRESS event + // report_key(EV_KEY, KEY_A, 0); } #endif #endif @@ -954,53 +1017,32 @@ int main() { #elif __linux__ { - uinput_fd = open("/dev/uinput", O_WRONLY | O_NDELAY); + uinput_fd = open("/dev/uinput", O_WRONLY | O_NONBLOCK); if (uinput_fd < 0) { - perror("无法打开 /dev/uinput"); - return; + LOG_ERROR("Cannot open device: /dev/uinput"); } - fd_kbd = open("/dev/input/event3", O_RDWR); - if (fd_kbd <= 0) { - printf("Can not open keyboard input file\n"); - return; - } - - fd_mouse = open("/dev/input/event2", O_RDWR); - if (fd_mouse <= 0) { - printf("Can not open mouse input file\n"); - return; - } - - // 设置uinput设备的属性 - struct uinput_user_dev uidev; + ioctl(uinput_fd, UI_SET_EVBIT, EV_KEY); + ioctl(uinput_fd, UI_SET_KEYBIT, BTN_RIGHT); + ioctl(uinput_fd, UI_SET_KEYBIT, BTN_LEFT); ioctl(uinput_fd, UI_SET_EVBIT, EV_ABS); ioctl(uinput_fd, UI_SET_ABSBIT, ABS_X); ioctl(uinput_fd, UI_SET_ABSBIT, ABS_Y); - ioctl(uinput_fd, UI_SET_EVBIT, EV_KEY); - ioctl(uinput_fd, UI_SET_KEYBIT, BTN_LEFT); - ioctl(uinput_fd, UI_SET_KEYBIT, BTN_RIGHT); - ioctl(uinput_fd, UI_SET_KEYBIT, BTN_MIDDLE); ioctl(uinput_fd, UI_SET_EVBIT, EV_REL); - ioctl(uinput_fd, UI_SET_RELBIT, REL_X); - ioctl(uinput_fd, UI_SET_RELBIT, REL_Y); + struct uinput_user_dev uidev; memset(&uidev, 0, sizeof(uidev)); - strncpy(uidev.name, "Virtual Mouse", UINPUT_MAX_NAME_SIZE); + snprintf(uidev.name, UINPUT_MAX_NAME_SIZE, "VirtualMouse"); uidev.id.bustype = BUS_USB; - uidev.id.vendor = 0x1234; // 自定义厂商ID - uidev.id.product = 0x5678; // 自定义产品ID + uidev.id.version = 1; + uidev.id.vendor = 0x1; + uidev.id.product = 0x1; uidev.absmin[ABS_X] = 0; - uidev.absmax[ABS_X] = 1280; - uidev.absfuzz[ABS_X] = 0; - uidev.absflat[ABS_X] = 0; + uidev.absmax[ABS_X] = screen_w; uidev.absmin[ABS_Y] = 0; - uidev.absmax[ABS_Y] = 720; - uidev.absfuzz[ABS_Y] = 0; - uidev.absflat[ABS_Y] = 0; - - ioctl(uinput_fd, UI_DEV_SETUP, &uidev); + uidev.absmax[ABS_Y] = screen_h; + write(uinput_fd, &uidev, sizeof(uidev)); ioctl(uinput_fd, UI_DEV_CREATE); } diff --git a/src/screen_capture/linux/screen_capture_x11.cpp b/src/screen_capture/linux/screen_capture_x11.cpp index 1ff4ba9..fcc80f7 100644 --- a/src/screen_capture/linux/screen_capture_x11.cpp +++ b/src/screen_capture/linux/screen_capture_x11.cpp @@ -21,6 +21,8 @@ int ScreenCaptureX11::Init(const RECORD_DESKTOP_RECT &rect, const int fps, _on_data = cb; } + fps_ = fps; + av_log_set_level(AV_LOG_QUIET); pFormatCtx_ = avformat_alloc_context(); @@ -33,9 +35,10 @@ int ScreenCaptureX11::Init(const RECORD_DESKTOP_RECT &rect, const int fps, // av_dict_set(&options_, "follow_mouse", "centered", 0); // Video frame size. The default is to capture the full screen // av_dict_set(&options_, "video_size", "1280x720", 0); - ifmt_ = (AVInputFormat *)av_find_input_format("x11grab"); + std::string capture_method = "x11grab"; + ifmt_ = (AVInputFormat *)av_find_input_format(capture_method.c_str()); if (!ifmt_) { - printf("Couldn't find_input_format\n"); + LOG_ERROR("Couldn't find_input_format [{}]", capture_method.c_str()); } // Grab at position 10,20 diff --git a/src/screen_capture/linux/screen_capture_x11.h b/src/screen_capture/linux/screen_capture_x11.h index 93b3719..8229236 100644 --- a/src/screen_capture/linux/screen_capture_x11.h +++ b/src/screen_capture/linux/screen_capture_x11.h @@ -66,6 +66,7 @@ class ScreenCaptureX11 { int i_ = 0; int videoindex_ = 0; int got_picture_ = 0; + int fps_ = 0; // ffmpeg AVFormatContext *pFormatCtx_ = nullptr; AVCodecContext *pCodecCtx_ = nullptr; diff --git a/test/linux_mouse_control/mouse_control.cpp b/test/linux_mouse_control/mouse_control.cpp new file mode 100644 index 0000000..165f920 --- /dev/null +++ b/test/linux_mouse_control/mouse_control.cpp @@ -0,0 +1,150 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace std; + +int fd; +Display *dpy; + +void initMouse(); +void destroyMouse(); +void mouseLeftClick(); +void mouseRightClick(); +void mouseGetPosition(int &x, int &y); +void mouseMove(int xdelta, int ydelta); + +void initMouse() { + fd = open("/dev/uinput", O_WRONLY | O_NONBLOCK); + ioctl(fd, UI_SET_EVBIT, EV_KEY); + ioctl(fd, UI_SET_KEYBIT, BTN_RIGHT); + ioctl(fd, UI_SET_KEYBIT, BTN_LEFT); + ioctl(fd, UI_SET_EVBIT, EV_ABS); + ioctl(fd, UI_SET_ABSBIT, ABS_X); + ioctl(fd, UI_SET_ABSBIT, ABS_Y); + ioctl(fd, UI_SET_EVBIT, EV_REL); + + struct uinput_user_dev uidev; + memset(&uidev, 0, sizeof(uidev)); + snprintf(uidev.name, UINPUT_MAX_NAME_SIZE, "VirtualMouse"); + uidev.id.bustype = BUS_USB; + uidev.id.version = 1; + uidev.id.vendor = 0x1; + uidev.id.product = 0x1; + uidev.absmin[ABS_X] = 0; + uidev.absmax[ABS_X] = 3200; + uidev.absmin[ABS_Y] = 0; + uidev.absmax[ABS_Y] = 900; + + write(fd, &uidev, sizeof(uidev)); + ioctl(fd, UI_DEV_CREATE); + + sleep(2); +} + +void mouseLeftClick() { + struct input_event ev_click, ev_sync; + memset(&ev_click, 0, sizeof(ev_click)); + memset(&ev_sync, 0, sizeof(ev_sync)); + + ev_click.type = EV_KEY; + ev_click.code = BTN_LEFT; + ev_click.value = 1; + + // write left click event + write(fd, &ev_click, sizeof(ev_click)); + + // sync left click event + ev_sync.type = EV_SYN; + write(fd, &ev_sync, sizeof(ev_sync)); +} + +void mouseRightClick() { + struct input_event ev_click, ev_sync; + memset(&ev_click, 0, sizeof(ev_click)); + memset(&ev_sync, 0, sizeof(ev_sync)); + + ev_click.type = EV_KEY; + ev_click.code = BTN_RIGHT; + ev_click.value = 1; + + // write right click event + write(fd, &ev_click, sizeof(ev_click)); + + // sync right click event + ev_sync.type = EV_SYN; + write(fd, &ev_sync, sizeof(ev_sync)); +} + +void mouseSetPosition(int x, int y) { + struct input_event ev[2], ev_sync; + memset(ev, 0, sizeof(ev)); + memset(&ev_sync, 0, sizeof(ev_sync)); + + ev[0].type = EV_ABS; + ev[0].code = ABS_X; + ev[0].value = x; + ev[1].type = EV_ABS; + ev[1].code = ABS_Y; + ev[1].value = y; + + int res_w = write(fd, ev, sizeof(ev)); + + std::cout << "res w : " << res_w << "\n"; + + ev_sync.type = EV_SYN; + ev_sync.value = 0; + ev_sync.code = 0; + int res_ev_sync = write(fd, &ev_sync, sizeof(ev_sync)); + + std::cout << "res syn : " << res_ev_sync << "\n"; +} + +void initDisplay() { dpy = XOpenDisplay(NULL); } + +void destroyMouse() { ioctl(fd, UI_DEV_DESTROY); } + +void mouseMove(int xdelta, int ydelta) { + int xx, yy; + mouseGetPosition(xx, yy); + + mouseSetPosition(xx + xdelta, yy + ydelta); +} + +void mouseGetPosition(int &x, int &y) { + Window root, child; + int rootX, rootY, winX, winY; + unsigned int mask; + + XQueryPointer(dpy, DefaultRootWindow(dpy), &root, &child, &rootX, &rootY, + &winX, &winY, &mask); + + std::cout << "root x : " << rootX << "\n"; + std::cout << "root y : " << rootY << "\n"; + + x = rootX; + y = rootY; +} + +int main() { + initMouse(); + initDisplay(); + + int tempx, tempy; + + for (int i = 0; i < 5; ++i) { + mouseMove(100, 100); + sleep(1); + std::cout << "i : " << i << "\n"; + } + + destroyMouse(); + return 0; +} \ No newline at end of file diff --git a/thirdparty/ffmpeg/xmake.lua b/thirdparty/ffmpeg/xmake.lua index bf6a493..9dd5f04 100644 --- a/thirdparty/ffmpeg/xmake.lua +++ b/thirdparty/ffmpeg/xmake.lua @@ -43,6 +43,7 @@ package("ffmpeg") add_configs("hardcoded-tables", {description = "Enable hardcoded tables.", default = true, type = "boolean"}) add_configs("asm", {description = "Enable asm", default = false, type = "boolean"}) add_configs("libopenh264", {description = "Enable libopenh264", default = false, type = "boolean"}) + add_configs("libxcb", {description = "Enable libxcb", default = true, type = "boolean"}) end add_links("avfilter", "avdevice", "avformat", "avcodec", "swscale", "swresample", "avutil", "postproc") diff --git a/thirdparty/projectx b/thirdparty/projectx index d4abc31..0b80124 160000 --- a/thirdparty/projectx +++ b/thirdparty/projectx @@ -1 +1 @@ -Subproject commit d4abc318a4fe28ea23d62d416512415f716fe136 +Subproject commit 0b80124b3c9751c72382adc838a53ac5a2f47db9 diff --git a/thirdparty/sdl2/xmake.lua b/thirdparty/sdl2/xmake.lua index 09158cb..1378cb7 100644 --- a/thirdparty/sdl2/xmake.lua +++ b/thirdparty/sdl2/xmake.lua @@ -2,7 +2,6 @@ package("sdl2") add_urls("https://github.com/libsdl-org/SDL/archive/refs/tags/release-2.28.3.tar.gz", {alias = "github"}) add_versions("github:2.28.3", "c17455d6e0c484bfe634b8de6af4c608e86ee449c28e40af04064aa6643fe382") - add_deps("cmake") on_install(function (package) local configs = {} table.insert(configs, "-DSDL_SHARED_ENABLED_BY_DEFAULT=OFF -DSDL_TEST_ENABLED_BY_DEFAULT=OFF") diff --git a/xmake.lua b/xmake.lua index de51bde..7b83d5e 100644 --- a/xmake.lua +++ b/xmake.lua @@ -5,6 +5,9 @@ set_license("LGPL-3.0") add_rules("mode.release", "mode.debug") set_languages("c++17") +-- set_policy("build.warning", true) +-- set_warnings("all", "extra") + add_defines("UNICODE") if is_mode("debug") then add_defines("REMOTE_DESK_DEBUG") @@ -24,7 +27,8 @@ elseif is_os("linux") then add_requires("ffmpeg 5.1.2", {system = false}) add_packages("ffmpeg") add_syslinks("pthread", "dl") - add_links("SDL2") + add_linkdirs("thirdparty/projectx/thirdparty/nvcodec/Lib/x64") + add_links("SDL2", "cuda", "nvidia-encode", "nvcuvid") add_ldflags("-lavformat", "-lavdevice", "-lavfilter", "-lavcodec", "-lswscale", "-lavutil", "-lswresample", "-lasound", "-lxcb-shape", "-lxcb-xfixes", "-lsndio", "-lxcb", @@ -129,4 +133,9 @@ target("remote_desk") -- "-lswscale", "-lavutil", "-lswresample", -- "-lasound", "-lxcb-shape", "-lxcb-xfixes", "-lsndio", "-lxcb", -- "-lxcb-shm", "-lXext", "-lX11", "-lXv", "-lpthread", "-lSDL2", "-lopenh264", --- "-ldl", {force = true}) \ No newline at end of file +-- "-ldl", {force = true}) + +target("mouse_control") + set_kind("binary") + add_files("test/linux_mouse_control/mouse_control.cpp") + add_includedirs("test/linux_mouse_control") \ No newline at end of file