diff --git a/apps/desktop/src/device_controller/device_controller.h b/apps/desktop/src/device_controller/device_controller.h index d562f213..7089cddf 100644 --- a/apps/desktop/src/device_controller/device_controller.h +++ b/apps/desktop/src/device_controller/device_controller.h @@ -42,6 +42,9 @@ class KeyboardCapturer : public DeviceController { virtual int Hook(OnKeyAction on_key_action, void* user_ptr) = 0; virtual int Unhook() = 0; + // Backends with an asynchronous capture loop override this so the UI can + // fall back to window events if that loop stops unexpectedly. + virtual bool IsHookActive() const { return true; } virtual int SendKeyboardCommand(int key_code, bool is_down, uint32_t scan_code = 0, bool extended = false) = 0; diff --git a/apps/desktop/src/gui/application/gui_application.cpp b/apps/desktop/src/gui/application/gui_application.cpp index 6dc25667..ed116e9b 100644 --- a/apps/desktop/src/gui/application/gui_application.cpp +++ b/apps/desktop/src/gui/application/gui_application.cpp @@ -1644,6 +1644,7 @@ void GuiApplication::BindStreamCallbacks() { props->enable_mouse_control_ = props->control_mouse_; start_keyboard_capturer_ = props->control_mouse_; if (!props->control_mouse_) { + devices_.StopKeyboardCapturer(); keyboard_.ForceReleasePressedKeys(); } }); @@ -2363,6 +2364,7 @@ void GuiApplication::SetStreamKeyboardFocus(bool focused) { focus_on_stream_window_ = focused; if (!focused) { + devices_.StopKeyboardCapturer(); keyboard_.ForceReleasePressedKeys(); } } @@ -3095,6 +3097,7 @@ void GuiApplication::SelectStreamTab(int index) { bool renderer_selection_changed = false; if (!controlled_remote_id_.empty() && controlled_remote_id_ != selected_remote_id) { + devices_.StopKeyboardCapturer(); keyboard_.ForceReleasePressedKeys(); } focused_remote_id_ = selected_remote_id; diff --git a/apps/desktop/src/gui/features/devices/session_device_manager.cpp b/apps/desktop/src/gui/features/devices/session_device_manager.cpp index 2b9ad77a..ecca12c0 100644 --- a/apps/desktop/src/gui/features/devices/session_device_manager.cpp +++ b/apps/desktop/src/gui/features/devices/session_device_manager.cpp @@ -301,6 +301,7 @@ int SessionDeviceManager::StopMouseController() { } int SessionDeviceManager::StartKeyboardCapturer() { + ClearCapturedKeyboardInput(); owner_.keyboard_capturer_uses_window_events_ = false; #ifdef __APPLE__ @@ -336,6 +337,7 @@ int SessionDeviceManager::StartKeyboardCapturer() { }, this); if (hook_ret != 0) { + keyboard_capturer_->Unhook(); owner_.keyboard_capturer_uses_window_events_ = true; LOG_WARN( "Start keyboard capturer failed, falling back to Slint keyboard events"); @@ -346,6 +348,9 @@ int SessionDeviceManager::StartKeyboardCapturer() { } int SessionDeviceManager::StopKeyboardCapturer() { + owner_.keyboard_capturer_is_started_ = false; + if (keyboard_capturer_) keyboard_capturer_->Unhook(); + ClearCapturedKeyboardInput(); if (owner_.keyboard_capturer_uses_window_events_) { owner_.keyboard_capturer_uses_window_events_ = false; LOG_INFO("Stop keyboard capturer with Slint keyboard backend"); @@ -353,8 +358,6 @@ int SessionDeviceManager::StopKeyboardCapturer() { } if (keyboard_capturer_) { - keyboard_capturer_->Unhook(); - ClearCapturedKeyboardInput(); LOG_INFO("Stop keyboard capturer"); } return 0; @@ -474,6 +477,14 @@ void SessionDeviceManager::UpdateInteractions() { owner_.keyboard_capturer_is_started_ = true; } if (owner_.keyboard_capturer_is_started_) { + if (!owner_.keyboard_capturer_uses_window_events_ && + (!keyboard_capturer_ || !keyboard_capturer_->IsHookActive())) { + StopKeyboardCapturer(); + owner_.keyboard_.ForceReleasePressedKeys(); + owner_.keyboard_capturer_uses_window_events_ = true; + owner_.keyboard_capturer_is_started_ = true; + LOG_WARN("Native keyboard capture stopped; using Slint keyboard events"); + } DrainCapturedKeyboardInput(); owner_.keyboard_.SendHeartbeat(false); } diff --git a/apps/desktop/src/platform/linux/input/keyboard/keyboard_capturer.cpp b/apps/desktop/src/platform/linux/input/keyboard/keyboard_capturer.cpp index 28a3738d..2a814831 100644 --- a/apps/desktop/src/platform/linux/input/keyboard/keyboard_capturer.cpp +++ b/apps/desktop/src/platform/linux/input/keyboard/keyboard_capturer.cpp @@ -1,9 +1,15 @@ #include "keyboard_capturer.h" +#include +#include #include +#include #include #include +#include +#include + #include #include @@ -15,9 +21,6 @@ namespace crossdesk { -static OnKeyAction g_on_key_action = nullptr; -static void* g_user_ptr = nullptr; - static KeyCode ResolveX11Keycode(Display* display, int key_code, uint32_t scan_code, bool extended) { if (!display) { @@ -59,36 +62,8 @@ static KeySym NormalizeKeySym(KeySym key_sym) { return key_sym; } -static int KeyboardEventHandler(Display* display, XEvent* event) { - (void)display; - if (event->xkey.type == KeyPress || event->xkey.type == KeyRelease) { - KeySym key_sym = NormalizeKeySym(XLookupKeysym(&event->xkey, 0)); - auto key_it = x11KeySymToVkCode.find(static_cast(key_sym)); - if (key_it == x11KeySymToVkCode.end()) { - key_sym = NormalizeKeySym(XLookupKeysym(&event->xkey, 1)); - key_it = x11KeySymToVkCode.find(static_cast(key_sym)); - } - - if (key_it == x11KeySymToVkCode.end()) { - return 0; - } - - int key_code = key_it->second; - bool is_key_down = (event->xkey.type == KeyPress); - uint32_t scan_code = 0; - bool extended = false; - LookupWindowsKeyMetadataFromVk(key_code, &scan_code, &extended); - - if (g_on_key_action) { - g_on_key_action(key_code, is_key_down, scan_code, extended, g_user_ptr); - } - } - return 0; -} - PlatformKeyboardCapturer::PlatformKeyboardCapturer() : display_(nullptr), - root_(0), running_(false), use_wayland_portal_(false), wayland_init_attempted_(false), @@ -121,82 +96,253 @@ PlatformKeyboardCapturer::~PlatformKeyboardCapturer() { } } +bool PlatformKeyboardCapturer::RefreshXTestDevices() { + // XTEST identifies injected input. Do not filter by names such as "virtual": + // an xrdp keyboard is a legitimate controller input device. + const Atom property = XInternAtom(capture_display_, "XTEST Device", True); + if (property == None) return false; + int count = 0; + XIDeviceInfo* devices = XIQueryDevice(capture_display_, XIAllDevices, &count); + if (!devices) return false; + xtest_devices_.clear(); + for (int i = 0; i < count; ++i) { + if (devices[i].use != XISlaveKeyboard) continue; + Atom type = None; + int format = 0; + unsigned long items = 0, remaining = 0; + unsigned char* value = nullptr; + const int result = XIGetProperty( + capture_display_, devices[i].deviceid, property, 0, 1, False, + XA_INTEGER, &type, &format, &items, &remaining, &value); + if (result == Success && type == XA_INTEGER && format == 8 && + items == 1 && value && value[0] == 1) { + xtest_devices_.insert(devices[i].deviceid); + } + if (value) XFree(value); + } + XIFreeDeviceInfo(devices); + return true; +} + +void PlatformKeyboardCapturer::CaptureKey(int source_id, int x11_keycode, + bool is_down) { + if (!capture_has_focus_ || xtest_devices_.count(source_id) || + x11_keycode < 8 || x11_keycode > 255) + return; + + // The GUI stops capture on focus loss, but its event loop may lag behind + // the native input thread. Never forward keys typed into another window. + Window focus = None; + int revert = 0; + XGetInputFocus(capture_display_, &focus, &revert); + if (focus != capture_focus_) return; + + const uint64_t identity = (static_cast(source_id) << 32) | + static_cast(x11_keycode); + auto found = captured_keys_.find(identity); + if (found == captured_keys_.end()) { + // A press can have reached the Slint fallback just before native capture + // started. Forward its release too, so that handoff cannot leave a key down. + CapturedKey key; + // Use unshifted key identity, independently of the local input method. + // Preserve it until release, even if the keyboard mapping changes. + for (int level = 0; level < 2 && key.key_code == 0; ++level) { + const KeySym symbol = NormalizeKeySym(XkbKeycodeToKeysym( + capture_display_, static_cast(x11_keycode), 0, level)); + if (symbol == XK_KP_Enter) { + key = {0x0D, 0x1C, true}; + } else if (symbol == XK_ISO_Level3_Shift) { + key.key_code = 0xA5; + } else if (symbol == XK_grave) { + key.key_code = 0xC0; + } else if (const auto vk = x11KeySymToVkCode.find(symbol); + vk != x11KeySymToVkCode.end()) { + key.key_code = vk->second; + } + } + if (!key.key_code) return; + if (!key.scan_code) { + LookupWindowsKeyMetadataFromVk(key.key_code, &key.scan_code, &key.extended); + if (!key.scan_code) { + const int evdev = LinuxEvdevKeycodeFromWindowsVk(key.key_code); + for (uint32_t scan = 1; scan <= 0x58; ++scan) { + if (evdev >= 0 && + LinuxEvdevKeycodeFromWindowsScanCode(scan, false) == evdev) { + key.scan_code = scan; + break; + } + } + } + } + found = captured_keys_.emplace(identity, key).first; + } + const CapturedKey key = found->second; + if (!is_down) captured_keys_.erase(found); + if (on_key_action_) { + on_key_action_(key.key_code, is_down, key.scan_code, key.extended, user_ptr_); + } +} + +void PlatformKeyboardCapturer::RunCapture() { + const int fd = ConnectionNumber(capture_display_); + while (running_) { + while (running_ && XPending(capture_display_) > 0) { + XEvent event{}; + XNextEvent(capture_display_, &event); + if (event.type == xkb_event_base_) { + auto* xkb = reinterpret_cast(&event); + if (xkb->any.xkb_type == XkbMapNotify) + XkbRefreshKeyboardMapping(&xkb->map); + } else if (event.type == MappingNotify) { + XRefreshKeyboardMapping(&event.xmapping); + } else if (event.type == FocusOut) { + // Keep focus changes in event order too: a fast switch away and back + // may finish before either the GUI or XGetInputFocus observes it. + capture_has_focus_ = false; + for (const auto& [identity, key] : captured_keys_) { + if (on_key_action_) + on_key_action_(key.key_code, false, key.scan_code, key.extended, + user_ptr_); + } + captured_keys_.clear(); + raw_presses_.fill({}); + } else if (event.type == FocusIn) { + capture_has_focus_ = true; + } else if (event.type == KeyPress && !event.xkey.send_event && + event.xkey.keycode < raw_presses_.size() && + event.xkey.time != raw_presses_[event.xkey.keycode].time) { + // Server-generated repeats have no matching raw press. Subscribe to + // core events on the focused window so Slint's core/XIM delivery is + // preserved (selecting XI_KeyPress here would take precedence over it). + const int source = raw_presses_[event.xkey.keycode].source_id; + const uint64_t identity = (static_cast(source) << 32) | + event.xkey.keycode; + if (captured_keys_.count(identity)) + CaptureKey(source, event.xkey.keycode, true); + } else if (event.type == GenericEvent && + event.xcookie.extension == xi_opcode_ && + XGetEventData(capture_display_, &event.xcookie)) { + if (event.xcookie.evtype == XI_RawKeyPress || + event.xcookie.evtype == XI_RawKeyRelease) { + const auto* key = static_cast(event.xcookie.data); + const bool is_down = event.xcookie.evtype == XI_RawKeyPress; + if (is_down && key->detail >= 0 && + static_cast(key->detail) < raw_presses_.size()) { + // Include filtered XTEST presses, so their core events cannot be + // mistaken for repeats of a physically held key. + raw_presses_[key->detail] = {key->time, key->sourceid}; + } + CaptureKey(key->sourceid, key->detail, is_down); + } else if (event.xcookie.evtype == XI_HierarchyChanged && + !RefreshXTestDevices()) { + LOG_ERROR("Failed to refresh XInput2 keyboard devices"); + running_ = false; + } + XFreeEventData(capture_display_, &event.xcookie); + } + } + if (!running_) break; + struct pollfd pfd = {fd, POLLIN, 0}; + const int result = poll(&pfd, 1, 50); + if ((result < 0 && errno != EINTR) || + (result > 0 && (pfd.revents & (POLLERR | POLLHUP | POLLNVAL)))) { + LOG_ERROR("XInput2 keyboard capture event connection failed"); + break; + } + } + running_ = false; +} + int PlatformKeyboardCapturer::Hook(OnKeyAction on_key_action, void* user_ptr) { - if (!display_) { - LOG_ERROR("Display not initialized."); + if (running_) return 0; + Unhook(); + capture_display_ = XOpenDisplay(nullptr); + if (!capture_display_ || !on_key_action) { + LOG_ERROR("Cannot open X11 keyboard capture display or callback is empty"); + Unhook(); return -1; } - g_on_key_action = on_key_action; - g_user_ptr = user_ptr; - - if (running_) { - return 0; + int event = 0, error = 0; + // XI 2.1 supplies raw-event source IDs, needed to exclude XTEST devices. + int major = 2, minor = 1; + if (!XQueryExtension(capture_display_, "XInputExtension", &xi_opcode_, + &event, &error) || + XIQueryVersion(capture_display_, &major, &minor) != Success || major < 2 || + (major == 2 && minor < 1)) { + LOG_WARN("XInput2 keyboard capture is unavailable"); + Unhook(); + return -1; + } + major = XkbMajorVersion; + minor = XkbMinorVersion; + int xkb_opcode = 0; + if (!XkbQueryExtension(capture_display_, &xkb_opcode, &xkb_event_base_, + &error, &major, &minor) || !RefreshXTestDevices()) { + LOG_WARN("Cannot initialize XKB or enumerate XInput2 keyboard devices"); + Unhook(); + return -1; } - root_ = DefaultRootWindow(display_); - XSelectInput(display_, root_, KeyPressMask | KeyReleaseMask); - XFlush(display_); + int revert = 0; + XGetInputFocus(capture_display_, &capture_focus_, &revert); + if (capture_focus_ == None || capture_focus_ == PointerRoot) { + LOG_WARN("XInput2 keyboard capture requires a focused application window"); + Unhook(); + return -1; + } + capture_has_focus_ = true; - running_ = true; - const int x11_fd = ConnectionNumber(display_); - event_thread_ = std::thread([this, x11_fd]() { - while (running_) { - while (running_ && XPending(display_) > 0) { - XEvent event; - XNextEvent(display_, &event); - KeyboardEventHandler(display_, &event); - } - - if (!running_) { - break; - } - - struct pollfd pfd = {x11_fd, POLLIN, 0}; - int poll_ret = poll(&pfd, 1, 50); - if (poll_ret < 0) { - if (errno == EINTR) { - continue; - } - LOG_ERROR("poll for X11 events failed."); - running_ = false; - break; - } - - if (poll_ret == 0) { - continue; - } - - if ((pfd.revents & (POLLERR | POLLHUP | POLLNVAL)) != 0) { - LOG_ERROR("poll got invalid X11 event fd state."); - running_ = false; - break; - } - - if ((pfd.revents & POLLIN) == 0) { - continue; - } + std::array raw{}; + std::array hierarchy{}; + XISetMask(raw.data(), XI_RawKeyPress); + XISetMask(raw.data(), XI_RawKeyRelease); + XISetMask(hierarchy.data(), XI_HierarchyChanged); + XIEventMask masks[] = { + {XIAllMasterDevices, static_cast(raw.size()), raw.data()}, + {XIAllDevices, static_cast(hierarchy.size()), hierarchy.data()}}; + for (int screen = 0; screen < ScreenCount(capture_display_); ++screen) { + if (XISelectEvents(capture_display_, RootWindow(capture_display_, screen), + masks, 2) != Success) { + LOG_WARN("Cannot subscribe to XInput2 keyboard events"); + Unhook(); + return -1; } - }); - + } + XSelectInput(capture_display_, capture_focus_, KeyPressMask | FocusChangeMask); + XkbSelectEvents(capture_display_, XkbUseCoreKbd, XkbMapNotifyMask, + XkbMapNotifyMask); + XSync(capture_display_, False); + on_key_action_ = on_key_action; + user_ptr_ = user_ptr; + running_ = true; + try { + event_thread_ = std::thread([this] { RunCapture(); }); + } catch (const std::exception& error) { + LOG_ERROR("Cannot start XInput2 keyboard capture thread: {}", error.what()); + Unhook(); + return -1; + } + LOG_INFO("XInput2 keyboard capture started"); return 0; } int PlatformKeyboardCapturer::Unhook() { running_ = false; - - if (event_thread_.joinable()) { - event_thread_.join(); + if (event_thread_.joinable()) event_thread_.join(); + // Closing the private connection removes every subscription and queued + // event, so restarting capture cannot forward keys from the previous focus. + if (capture_display_) { + XCloseDisplay(capture_display_); + capture_display_ = nullptr; } - - g_on_key_action = nullptr; - g_user_ptr = nullptr; - - if (display_ && root_ != 0) { - XSelectInput(display_, root_, 0); - XFlush(display_); - } - + on_key_action_ = nullptr; + user_ptr_ = nullptr; + captured_keys_.clear(); + raw_presses_.fill({}); + xtest_devices_.clear(); + capture_focus_ = None; + capture_has_focus_ = false; return 0; } diff --git a/apps/desktop/src/platform/linux/input/keyboard/keyboard_capturer.h b/apps/desktop/src/platform/linux/input/keyboard/keyboard_capturer.h index 79812f49..2c131bbf 100644 --- a/apps/desktop/src/platform/linux/input/keyboard/keyboard_capturer.h +++ b/apps/desktop/src/platform/linux/input/keyboard/keyboard_capturer.h @@ -7,12 +7,15 @@ #ifndef _KEYBOARD_CAPTURER_H_ #define _KEYBOARD_CAPTURER_H_ +#include #include #include #include #include #include #include +#include +#include #include "device_controller.h" @@ -30,11 +33,25 @@ class PlatformKeyboardCapturer final : public KeyboardCapturer { public: virtual int Hook(OnKeyAction on_key_action, void* user_ptr); virtual int Unhook(); + bool IsHookActive() const override { return running_.load(); } virtual int SendKeyboardCommand(int key_code, bool is_down, uint32_t scan_code = 0, bool extended = false); private: + struct CapturedKey { + int key_code = 0; + uint32_t scan_code = 0; + bool extended = false; + }; + struct RawKeyPress { + unsigned long time = 0; + int source_id = 0; + }; + bool RefreshXTestDevices(); + void CaptureKey(int source_id, int x11_keycode, bool is_down); + void RunCapture(); + bool InitWaylandPortal(); void CleanupWaylandPortal(); int SendWaylandKeyboardCommand(int key_code, bool is_down, uint32_t scan_code, @@ -47,7 +64,17 @@ class PlatformKeyboardCapturer final : public KeyboardCapturer { private: _XDisplay* display_; - unsigned long root_; + // Capture owns a separate connection, including its XKB event queue. + _XDisplay* capture_display_ = nullptr; + unsigned long capture_focus_ = 0; + bool capture_has_focus_ = false; + int xi_opcode_ = 0; + int xkb_event_base_ = 0; + OnKeyAction on_key_action_ = nullptr; + void* user_ptr_ = nullptr; + std::unordered_set xtest_devices_; + std::unordered_map captured_keys_; + std::array raw_presses_{}; std::atomic running_; std::thread event_thread_; std::mutex x11_injection_mutex_; diff --git a/apps/desktop/tests/linux_keyboard_x11_integration_test.cpp b/apps/desktop/tests/linux_keyboard_x11_integration_test.cpp deleted file mode 100644 index f43fb8eb..00000000 --- a/apps/desktop/tests/linux_keyboard_x11_integration_test.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include - -#include -#include -#include - -#include "keyboard_capturer.h" - -int main() { - Display* receiver = XOpenDisplay(nullptr); - if (!receiver) { - std::fprintf(stderr, "open display failed\n"); - return 1; - } - - const int screen = DefaultScreen(receiver); - Window window = XCreateSimpleWindow(receiver, RootWindow(receiver, screen), - 0, 0, 100, 100, 0, 0, 0); - XSelectInput(receiver, window, KeyPressMask | KeyReleaseMask); - XMapWindow(receiver, window); - XSync(receiver, False); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - XSetInputFocus(receiver, window, RevertToParent, CurrentTime); - XSync(receiver, False); - - crossdesk::PlatformKeyboardCapturer keyboard; - // Exercise the scan-code fallback: the controller supplied no usable VK, - // but did preserve the set-1 scan code for the A key. - const int down = keyboard.SendKeyboardCommand(0, true, 0x1E, false); - const int up = keyboard.SendKeyboardCommand(0, false, 0x1E, false); - - bool saw_down = false; - bool saw_up = false; - const auto deadline = std::chrono::steady_clock::now() + - std::chrono::seconds(2); - while (std::chrono::steady_clock::now() < deadline && - (!saw_down || !saw_up)) { - while (XPending(receiver) > 0) { - XEvent event{}; - XNextEvent(receiver, &event); - saw_down = saw_down || event.type == KeyPress; - saw_up = saw_up || event.type == KeyRelease; - } - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } - - XDestroyWindow(receiver, window); - XCloseDisplay(receiver); - std::printf("down_ret=%d up_ret=%d saw_down=%d saw_up=%d\n", down, up, - saw_down, saw_up); - return down == 0 && up == 0 && saw_down && saw_up ? 0 : 2; -} diff --git a/apps/desktop/xmake/targets.lua b/apps/desktop/xmake/targets.lua index 0c5d1f91..ad95fbf0 100644 --- a/apps/desktop/xmake/targets.lua +++ b/apps/desktop/xmake/targets.lua @@ -297,20 +297,13 @@ function setup_targets() add_files("apps/desktop/src/platform/linux/input/mouse/*.cpp", "apps/desktop/src/platform/linux/input/keyboard/*.cpp", "apps/desktop/src/platform/linux/input/device_controller_factory.cpp") + add_syslinks("Xi", {public = true}) add_includedirs("apps/desktop/src/platform/linux/input/mouse", "apps/desktop/src/platform/linux/input/keyboard", "apps/desktop/src/platform/linux/input", "apps/desktop/src/platform/linux/common") end - if is_os("linux") then - target("linux_keyboard_x11_integration_test") - set_kind("binary") - set_default(false) - add_deps("device_controller") - add_files("apps/desktop/tests/linux_keyboard_x11_integration_test.cpp") - end - target("thumbnail") set_kind("object") add_packages("libyuv", "openssl3")