[fix] restore Windows remote keyboard shortcuts and input recovery

This commit is contained in:
dijunkun
2026-10-09 03:43:14 +08:00
parent 3cfe7cbb80
commit 06e23007ed
10 changed files with 626 additions and 312 deletions
@@ -27,8 +27,10 @@ namespace crossdesk {
namespace {
constexpr uint64_t kCaptureResumeKeyFrameGapMs = 500;
constexpr size_t kMaxCapturedKeyboardInputs = 512;
constexpr auto kFrameDeadlineTolerance = std::chrono::milliseconds(1);
#ifdef _WIN32
constexpr auto kKeyboardCaptureRetryDelay = std::chrono::seconds(1);
#endif
} // namespace
@@ -419,8 +421,12 @@ int SessionDeviceManager::StopMouseController() {
}
int SessionDeviceManager::StartKeyboardCapturer() {
ClearCapturedKeyboardInput();
captured_keyboard_inputs_.Clear();
owner_.keyboard_capturer_uses_window_events_ = false;
#ifdef _WIN32
next_keyboard_capture_retry_ =
std::chrono::steady_clock::now() + kKeyboardCaptureRetryDelay;
#endif
#ifdef __APPLE__
if (!owner_.EnsureMacAccessibilityPermission()) {
@@ -449,8 +455,10 @@ int SessionDeviceManager::StartKeyboardCapturer() {
void *user_ptr) {
if (user_ptr) {
auto *devices = static_cast<SessionDeviceManager *>(user_ptr);
devices->QueueCapturedKeyboardInput(key_code, is_down, scan_code,
extended);
// Native hooks only enqueue; forwarding and logging stay off the
// hook thread so they cannot trigger LowLevelHooksTimeout.
devices->captured_keyboard_inputs_.Push(
{key_code, is_down, scan_code, extended});
}
},
this);
@@ -468,7 +476,7 @@ int SessionDeviceManager::StartKeyboardCapturer() {
int SessionDeviceManager::StopKeyboardCapturer() {
owner_.keyboard_capturer_is_started_ = false;
if (keyboard_capturer_) keyboard_capturer_->Unhook();
ClearCapturedKeyboardInput();
captured_keyboard_inputs_.Clear();
if (owner_.keyboard_capturer_uses_window_events_) {
owner_.keyboard_capturer_uses_window_events_ = false;
LOG_INFO("Stop keyboard capturer with Slint keyboard backend");
@@ -602,7 +610,18 @@ void SessionDeviceManager::UpdateInteractions() {
#endif
}
if (owner_.start_keyboard_capturer_ && owner_.focus_on_stream_window_) {
if (owner_.start_keyboard_capturer_ && owner_.focus_on_stream_window_ &&
!owner_.privacy_.Snapshot().input_blocked) {
#ifdef _WIN32
if (owner_.keyboard_capturer_is_started_ &&
owner_.keyboard_capturer_uses_window_events_ && keyboard_capturer_ &&
std::chrono::steady_clock::now() >= next_keyboard_capture_retry_) {
// A transient installation failure or stalled hook must not leave
// system shortcuts in the window-event fallback until focus changes.
StopKeyboardCapturer();
owner_.keyboard_.ForceReleasePressedKeys();
}
#endif
if (!owner_.keyboard_capturer_is_started_ && StartKeyboardCapturer() == 0) {
owner_.keyboard_capturer_is_started_ = true;
}
@@ -613,6 +632,10 @@ void SessionDeviceManager::UpdateInteractions() {
owner_.keyboard_.ForceReleasePressedKeys();
owner_.keyboard_capturer_uses_window_events_ = true;
owner_.keyboard_capturer_is_started_ = true;
#ifdef _WIN32
next_keyboard_capture_retry_ =
std::chrono::steady_clock::now() + kKeyboardCaptureRetryDelay;
#endif
LOG_WARN("Native keyboard capture stopped; using Slint keyboard events");
}
DrainCapturedKeyboardInput();
@@ -621,43 +644,25 @@ void SessionDeviceManager::UpdateInteractions() {
} else if (owner_.keyboard_capturer_is_started_) {
owner_.keyboard_.ForceReleasePressedKeys();
StopKeyboardCapturer();
owner_.keyboard_capturer_is_started_ = false;
}
owner_.keyboard_.CheckRemoteTimeouts();
}
void SessionDeviceManager::QueueCapturedKeyboardInput(int key_code,
bool is_down,
uint32_t scan_code,
bool extended) {
std::lock_guard<std::mutex> lock(captured_keyboard_inputs_mutex_);
if (captured_keyboard_inputs_.size() >= kMaxCapturedKeyboardInputs) {
captured_keyboard_inputs_.pop_front();
LOG_WARN("Captured keyboard input queue overflow, dropping oldest event");
}
captured_keyboard_inputs_.push_back(
CapturedKeyboardInput{key_code, is_down, scan_code, extended});
}
void SessionDeviceManager::DrainCapturedKeyboardInput() {
std::deque<CapturedKeyboardInput> inputs;
{
std::lock_guard<std::mutex> lock(captured_keyboard_inputs_mutex_);
inputs.swap(captured_keyboard_inputs_);
}
const auto inputs = captured_keyboard_inputs_.Drain();
if (owner_.privacy_.Snapshot().input_blocked) return;
for (const CapturedKeyboardInput &input : inputs) {
if (inputs.resync) {
LOG_WARN("Captured keyboard queue overflow; reconciling pressed keys");
owner_.keyboard_.RecoverCapturedState(inputs.pressed);
return;
}
for (const KeyboardInput &input : inputs.events) {
owner_.keyboard_.SendKeyCommand(input.key_code, input.is_down,
input.scan_code, input.extended);
}
}
void SessionDeviceManager::ClearCapturedKeyboardInput() {
std::lock_guard<std::mutex> lock(captured_keyboard_inputs_mutex_);
captured_keyboard_inputs_.clear();
}
bool SessionDeviceManager::SendKeyboardCommand(int key_code, bool is_down,
uint32_t scan_code,
bool extended) {
@@ -7,12 +7,10 @@
#ifndef _SESSION_DEVICE_MANAGER_H_
#define _SESSION_DEVICE_MANAGER_H_
#include <atomic>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <deque>
#include <mutex>
#include <string>
#include <vector>
@@ -21,6 +19,7 @@
#include "device_controller.h"
#include "display_info.h"
#include "features/input/captured_keyboard_queue.h"
#include "speaker_capture_controller.h"
struct SDL_AudioStream;
@@ -83,17 +82,7 @@ public:
std::vector<HostDisplay> host_display_list() const;
private:
struct CapturedKeyboardInput {
int key_code = 0;
bool is_down = false;
uint32_t scan_code = 0;
bool extended = false;
};
void QueueCapturedKeyboardInput(int key_code, bool is_down,
uint32_t scan_code, bool extended);
void DrainCapturedKeyboardInput();
void ClearCapturedKeyboardInput();
bool ShouldSendCapturedFrame(std::chrono::steady_clock::time_point now,
int fps);
bool ShouldCaptureCursor() const;
@@ -115,8 +104,10 @@ private:
mutable std::mutex display_info_mutex_;
std::vector<DisplayInfo> display_info_list_;
size_t registered_display_stream_count_ = 0;
std::deque<CapturedKeyboardInput> captured_keyboard_inputs_;
std::mutex captured_keyboard_inputs_mutex_;
CapturedKeyboardQueue captured_keyboard_inputs_;
#ifdef _WIN32
std::chrono::steady_clock::time_point next_keyboard_capture_retry_{};
#endif
std::chrono::steady_clock::time_point last_frame_time_{};
std::chrono::steady_clock::time_point next_frame_deadline_{};
std::mutex capture_metrics_mutex_;
@@ -0,0 +1,75 @@
/*
* @Author: DI JUNKUN
* @Date: 2026-10-09
* Copyright (c) 2026 by DI JUNKUN, All Rights Reserved.
*/
#ifndef _CAPTURED_KEYBOARD_QUEUE_H_
#define _CAPTURED_KEYBOARD_QUEUE_H_
#include <array>
#include <cstddef>
#include <mutex>
#include <vector>
#include "keyboard_state.h"
namespace crossdesk {
class CapturedKeyboardQueue {
public:
struct Batch {
std::vector<KeyboardInput> events;
bool resync = false;
PressedKeys pressed;
};
void Push(const KeyboardInput& input) {
std::lock_guard lock(mutex_);
if (input.is_down)
pressed_[input.key_code] =
{input.key_code, input.scan_code, input.extended};
else
pressed_.erase(input.key_code);
if (resync_) return;
if (count_ == kCapacity) {
// The event history is incomplete. Retain the authoritative state,
// including releases, rather than dropping arbitrary key edges.
count_ = 0;
resync_ = true;
return;
}
events_[count_++] = input;
}
Batch Drain() {
std::lock_guard lock(mutex_);
Batch batch;
batch.resync = resync_;
if (resync_) batch.pressed = pressed_;
else batch.events.assign(events_.begin(), events_.begin() + count_);
count_ = 0;
resync_ = false;
return batch;
}
void Clear() {
std::lock_guard lock(mutex_);
count_ = 0;
resync_ = false;
pressed_.clear();
}
private:
static constexpr size_t kCapacity = 512;
std::mutex mutex_;
std::array<KeyboardInput, kCapacity> events_{};
size_t count_ = 0;
bool resync_ = false;
PressedKeys pressed_;
};
} // namespace crossdesk
#endif
@@ -8,7 +8,7 @@
#include <mutex>
#include <nlohmann/json.hpp>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include "minirtc.h"
@@ -26,6 +26,7 @@ namespace {
constexpr uint32_t kHeartbeatIntervalMs = 500;
constexpr uint32_t kRemoteReleaseTimeoutMs = 2500;
constexpr uint32_t kReleaseRetryIntervalMs = 250;
int NormalizeWindowsModifierVk(int key_code, uint32_t scan_code,
bool extended) {
@@ -99,29 +100,38 @@ void KeyboardController::TrackPressedKey(int key_code, bool is_down,
uint32_t scan_code, bool extended) {
std::lock_guard<std::mutex> lock(pressed_keys_mutex_);
if (is_down) {
pressed_keys_[key_code] = PressedKey{key_code, scan_code, extended};
pressed_keys_[key_code] = KeyboardKey{key_code, scan_code, extended};
} else {
pressed_keys_.erase(key_code);
}
}
void KeyboardController::ForceReleasePressedKeys() {
std::vector<PressedKey> pressed_keys;
PressedKeys pressed_keys;
{
std::lock_guard<std::mutex> lock(pressed_keys_mutex_);
pressed_keys.reserve(pressed_keys_.size());
for (const auto& [_, key] : pressed_keys_) {
pressed_keys.push_back(key);
}
pressed_keys_.clear();
pressed_keys.swap(pressed_keys_);
}
for (const PressedKey& key : pressed_keys) {
for (const auto& [_, key] : pressed_keys) {
SendKeyCommand(key.key_code, false, key.scan_code, key.extended);
}
SendHeartbeat(true);
}
void KeyboardController::RecoverCapturedState(const PressedKeys& pressed) {
std::vector<KeyboardInput> changes;
{
std::lock_guard lock(pressed_keys_mutex_);
changes = ReconcileKeyboardState(pressed_keys_, pressed);
}
for (const auto& event : changes) {
SendKeyCommand(event.key_code, event.is_down, event.scan_code,
event.extended);
}
SendHeartbeat(true);
}
void KeyboardController::SendHeartbeat(bool force) {
const uint32_t now = static_cast<uint32_t>(SDL_GetTicks());
if (!force && now - last_heartbeat_tick_ < kHeartbeatIntervalMs) {
@@ -246,7 +256,9 @@ bool KeyboardController::InjectRemoteKey(int key_code, bool is_down,
"Secure desktop keyboard injection transient failure, "
"key_code={}, is_down={}, response={}",
key_code, is_down, response);
return {true, false};
// The release was not injected. Keep it tracked so cleanup can
// retry on the ordinary desktop after the transition.
return {false, false};
}
LogSecureDesktopInputBlocked(
@@ -269,17 +281,18 @@ bool KeyboardController::InjectRemoteKey(int key_code, bool is_down,
void KeyboardController::ApplyRemoteEvent(const std::string& remote_id,
const RemoteAction& action) {
std::lock_guard input_lock(remote_input_mutex_);
const int key_code = static_cast<int>(action.k.key_value);
const bool is_down = action.k.flag == KeyFlag::key_down;
const bool injected =
InjectRemoteKey(key_code, is_down, action.k.scan_code, action.k.extended);
if (injected) pending_key_releases_.erase(key_code);
std::lock_guard<std::mutex> lock(remote_states_mutex_);
auto& state = remote_states_[remote_id];
state.last_seen_tick = static_cast<uint32_t>(SDL_GetTicks());
if (is_down && injected) {
state.pressed_keys[key_code] =
PressedKey{key_code, action.k.scan_code, action.k.extended};
KeyboardKey{key_code, action.k.scan_code, action.k.extended};
} else if (!is_down && injected) {
state.pressed_keys.erase(key_code);
}
@@ -287,136 +300,117 @@ void KeyboardController::ApplyRemoteEvent(const std::string& remote_id,
void KeyboardController::ApplyRemoteState(const std::string& remote_id,
const RemoteAction& action) {
std::vector<PressedKey> keys_to_release;
std::vector<PressedKey> keys_to_press;
{
std::lock_guard<std::mutex> lock(remote_states_mutex_);
auto& state = remote_states_[remote_id];
if (action.ks.seq != 0 && state.last_seq != 0 &&
static_cast<int32_t>(action.ks.seq - state.last_seq) <= 0) {
return;
}
state.last_seq = action.ks.seq;
state.last_seen_tick = static_cast<uint32_t>(SDL_GetTicks());
state.keyboard_state_seen = true;
std::unordered_map<int, PressedKey> desired_keys;
const size_t count =
(std::min)(action.ks.pressed_count, kMaxKeyboardStateKeys);
for (size_t index = 0; index < count; ++index) {
const auto& key = action.ks.pressed_keys[index];
const int key_code = static_cast<int>(key.key_value);
desired_keys[key_code] =
PressedKey{key_code, key.scan_code, key.extended};
}
for (const auto& [key_code, key] : state.pressed_keys) {
if (desired_keys.find(key_code) == desired_keys.end()) {
keys_to_release.push_back(key);
}
}
for (const auto& [key_code, key] : desired_keys) {
if (state.pressed_keys.find(key_code) == state.pressed_keys.end()) {
keys_to_press.push_back(key);
}
}
std::lock_guard input_lock(remote_input_mutex_);
auto& state = remote_states_[remote_id];
if (action.ks.seq != 0 && state.last_seq != 0 &&
static_cast<int32_t>(action.ks.seq - state.last_seq) <= 0) {
return;
}
for (const PressedKey& key : keys_to_release) {
if (InjectRemoteKey(key.key_code, false, key.scan_code, key.extended)) {
std::lock_guard<std::mutex> lock(remote_states_mutex_);
const auto state_it = remote_states_.find(remote_id);
if (state_it != remote_states_.end()) {
state_it->second.pressed_keys.erase(key.key_code);
}
}
state.last_seq = action.ks.seq;
state.last_seen_tick = static_cast<uint32_t>(SDL_GetTicks());
state.keyboard_state_seen = true;
PressedKeys desired_keys;
const size_t count =
(std::min)(action.ks.pressed_count, kMaxKeyboardStateKeys);
for (size_t index = 0; index < count; ++index) {
const auto& key = action.ks.pressed_keys[index];
const int key_code = static_cast<int>(key.key_value);
desired_keys[key_code] = {key_code, key.scan_code, key.extended};
}
for (const PressedKey& key : keys_to_press) {
if (InjectRemoteKey(key.key_code, true, key.scan_code, key.extended)) {
std::lock_guard<std::mutex> lock(remote_states_mutex_);
remote_states_[remote_id].pressed_keys[key.key_code] = key;
const auto changes = ReconcileKeyboardState(state.pressed_keys, desired_keys);
bool can_press = true;
for (const auto& key : changes) {
if (key.is_down && !can_press) continue;
if (!InjectRemoteKey(key.key_code, key.is_down, key.scan_code, key.extended)) {
// Do not replay a shortcut with a stale/missing modifier. The next
// heartbeat will retry from the state that was actually injected.
can_press = false;
continue;
}
pending_key_releases_.erase(key.key_code);
if (key.is_down)
state.pressed_keys[key.key_code] =
{key.key_code, key.scan_code, key.extended};
else
state.pressed_keys.erase(key.key_code);
}
}
void KeyboardController::ReleaseRemotePressedKeys(const std::string& remote_id,
const char* reason) {
std::vector<PressedKey> keys_to_release;
{
std::lock_guard<std::mutex> lock(remote_states_mutex_);
const auto state_it = remote_states_.find(remote_id);
if (state_it == remote_states_.end()) {
return;
}
for (const auto& [_, key] : state_it->second.pressed_keys) {
keys_to_release.push_back(key);
}
remote_states_.erase(state_it);
}
std::lock_guard input_lock(remote_input_mutex_);
ReleaseRemotePressedKeysLocked(remote_id, reason);
}
void KeyboardController::ReleaseRemotePressedKeysLocked(
std::string remote_id, const char* reason) {
const auto state_it = remote_states_.find(remote_id);
if (state_it == remote_states_.end()) return;
auto keys_to_release = std::move(state_it->second.pressed_keys);
remote_states_.erase(state_it);
if (!keys_to_release.empty()) {
LOG_WARN("Releasing {} remote keyboard keys for remote_id={}, reason={}",
keys_to_release.size(), remote_id, reason ? reason : "unknown");
}
for (const PressedKey& key : keys_to_release) {
for (const auto& [code, key] : keys_to_release) {
if (!InjectRemoteKey(key.key_code, false, key.scan_code, key.extended)) {
#ifdef _WIN32
if (owner_.privacy_.Engaged()) {
std::lock_guard lock(remote_states_mutex_);
pending_privacy_releases_[key.key_code] = key;
}
#endif
pending_key_releases_[code] = key;
} else {
pending_key_releases_.erase(code);
}
}
}
void KeyboardController::ReleaseAllRemotePressedKeys(const char* reason) {
std::vector<std::string> remotes;
{
std::lock_guard lock(remote_states_mutex_);
for (const auto& [id, state] : remote_states_) remotes.push_back(id);
std::lock_guard input_lock(remote_input_mutex_);
while (!remote_states_.empty())
ReleaseRemotePressedKeysLocked(remote_states_.begin()->first, reason);
}
for (const auto& id : remotes) ReleaseRemotePressedKeys(id, reason);
CheckRemoteTimeouts();
}
void KeyboardController::CheckRemoteTimeouts() {
#ifdef _WIN32
bool pending = false;
{
std::lock_guard lock(remote_states_mutex_);
pending = !pending_privacy_releases_.empty();
}
// A key-up rejected on the secure desktop is retained until the ordinary
// desktop returns. Never route cleanup through the security UI helper.
if (pending && IsWindowsPrivacyDesktopAvailable()) {
std::unordered_map<int, PressedKey> releases;
{
std::lock_guard lock(remote_states_mutex_);
releases.swap(pending_privacy_releases_);
}
for (const auto& [code, key] : releases) {
if (!owner_.devices_.SendKeyboardCommand(code, false, key.scan_code, key.extended)) {
std::lock_guard lock(remote_states_mutex_);
pending_privacy_releases_[code] = key;
}
}
}
#endif
// Do not stall the UI while a transport thread waits on desktop IPC.
std::unique_lock input_lock(remote_input_mutex_, std::try_to_lock);
if (!input_lock.owns_lock()) return;
const uint32_t now = static_cast<uint32_t>(SDL_GetTicks());
std::vector<std::string> timed_out_remotes;
{
std::lock_guard<std::mutex> lock(remote_states_mutex_);
for (const auto& [remote_id, state] : remote_states_) {
if (state.keyboard_state_seen && !state.pressed_keys.empty() &&
state.last_seen_tick != 0 &&
now - state.last_seen_tick > kRemoteReleaseTimeoutMs) {
timed_out_remotes.push_back(remote_id);
}
// Retain failed releases in ordinary mode as well as privacy mode. Retry
// only on the ordinary desktop, never through the security UI helper.
if (!pending_key_releases_.empty() &&
(last_release_retry_tick_ == 0 ||
now - last_release_retry_tick_ >= kReleaseRetryIntervalMs)
#ifdef _WIN32
&& IsCurrentSessionUserDesktopActive()
#endif
) {
last_release_retry_tick_ = now;
for (auto it = pending_key_releases_.begin();
it != pending_key_releases_.end();) {
const auto& key = it->second;
if (owner_.devices_.SendKeyboardCommand(
key.key_code, false, key.scan_code, key.extended))
it = pending_key_releases_.erase(it);
else
++it;
}
}
for (const std::string& remote_id : timed_out_remotes) {
ReleaseRemotePressedKeys(remote_id, "keyboard_heartbeat_timeout");
// Keep the timeout decision and release under the same lock, so a fresh
// heartbeat cannot arrive between selecting a stale peer and clearing it.
for (auto it = remote_states_.begin(); it != remote_states_.end();) {
const auto& state = it->second;
if (state.keyboard_state_seen && !state.pressed_keys.empty() &&
state.last_seen_tick != 0 &&
now - state.last_seen_tick > kRemoteReleaseTimeoutMs) {
const auto expired = it++;
ReleaseRemotePressedKeysLocked(expired->first, "keyboard_heartbeat_timeout");
} else {
++it;
}
}
}
@@ -8,7 +8,7 @@
#include <remote_action.h>
#include "device_controller.h"
#include "keyboard_state.h"
namespace crossdesk {
@@ -23,6 +23,7 @@ public:
int SendKeyCommand(int key_code, bool is_down, uint32_t scan_code = 0,
bool extended = false);
void ForceReleasePressedKeys();
void RecoverCapturedState(const PressedKeys& pressed);
void SendHeartbeat(bool force);
void ApplyRemoteEvent(const std::string &remote_id,
const RemoteAction &remote_action);
@@ -34,14 +35,8 @@ public:
void CheckRemoteTimeouts();
private:
struct PressedKey {
int key_code = 0;
uint32_t scan_code = 0;
bool extended = false;
};
struct RemoteState {
std::unordered_map<int, PressedKey> pressed_keys;
PressedKeys pressed_keys;
uint32_t last_seq = 0;
uint32_t last_seen_tick = 0;
bool keyboard_state_seen = false;
@@ -51,17 +46,20 @@ private:
bool extended);
bool InjectRemoteKey(int key_code, bool is_down, uint32_t scan_code,
bool extended);
// Caller holds remote_input_mutex_. Copy the id because this erases state.
void ReleaseRemotePressedKeysLocked(std::string remote_id, const char* reason);
GuiRuntime &owner_;
std::unordered_map<int, PressedKey> pressed_keys_;
PressedKeys pressed_keys_;
std::mutex pressed_keys_mutex_;
uint32_t state_sequence_ = 0;
uint32_t last_heartbeat_tick_ = 0;
std::unordered_map<std::string, RemoteState> remote_states_;
std::mutex remote_states_mutex_;
#ifdef _WIN32
std::unordered_map<int, PressedKey> pending_privacy_releases_;
#endif
// Guards all remote key state and serializes injection, cleanup and retries.
std::mutex remote_input_mutex_;
// Successful new input supersedes cleanup of an older press of that key.
PressedKeys pending_key_releases_;
uint32_t last_release_retry_tick_ = 0;
};
} // namespace crossdesk
+13 -1
View File
@@ -282,9 +282,21 @@ export component StreamWindow inherits Window {
root.display-menu-open = false;
root.shortcut-menu-open = false;
}
input-focus.focus();
// Changed handlers may run while Winit/AccessKit is building the
// accessibility tree. Changing focus there re-enters its borrowed
// adapter and aborts the process; defer focus to the event loop.
input-focus-timer.running = true;
input-focus-timer.restart();
root.keyboard-focus-changed(!root.remote-update-visible);
}
input-focus-timer := Timer {
interval: 1ms;
running: false;
triggered => {
self.running = false;
input-focus.focus();
}
}
private property <bool> control-tooltip-visible: false;
private property <ControlBarTooltip> hovered-control: {
if root.window-active && !root.control-dragging