[fix] prevent remote keyboard keys from getting stuck

This commit is contained in:
dijunkun
2026-06-15 17:28:34 +08:00
parent 1c1a33fdce
commit fbde3f6a47
6 changed files with 434 additions and 56 deletions
+63 -6
View File
@@ -21,12 +21,13 @@ namespace crossdesk {
typedef enum {
mouse = 0,
keyboard,
audio_capture,
host_infomation,
display_id,
service_status,
service_command,
keyboard = 1,
audio_capture = 2,
host_infomation = 3,
display_id = 4,
service_status = 5,
service_command = 6,
keyboard_state = 7,
} ControlType;
typedef enum {
move = 0,
@@ -55,6 +56,20 @@ typedef struct {
KeyFlag flag;
} Key;
inline constexpr size_t kMaxKeyboardStateKeys = 32;
typedef struct {
size_t key_value;
uint32_t scan_code;
bool extended;
} KeyboardStateKey;
typedef struct {
uint32_t seq;
size_t pressed_count;
KeyboardStateKey pressed_keys[kMaxKeyboardStateKeys];
} KeyboardState;
typedef struct {
char host_name[64];
size_t host_name_size;
@@ -80,6 +95,7 @@ struct RemoteAction {
union {
Mouse m;
Key k;
KeyboardState ks;
HostInfo i;
bool a;
int d;
@@ -111,6 +127,20 @@ struct RemoteAction {
{"extended", a.k.extended},
{"flag", a.k.flag}};
break;
case ControlType::keyboard_state: {
json keys = json::array();
const size_t pressed_count =
a.ks.pressed_count < kMaxKeyboardStateKeys
? a.ks.pressed_count
: kMaxKeyboardStateKeys;
for (size_t idx = 0; idx < pressed_count; ++idx) {
keys.push_back({{"key_value", a.ks.pressed_keys[idx].key_value},
{"scan_code", a.ks.pressed_keys[idx].scan_code},
{"extended", a.ks.pressed_keys[idx].extended}});
}
j["keyboard_state"] = {{"seq", a.ks.seq}, {"pressed_keys", keys}};
break;
}
case ControlType::audio_capture:
j["audio_capture"] = a.a;
break;
@@ -162,6 +192,33 @@ struct RemoteAction {
out.k.extended = j.at("keyboard").value("extended", false);
out.k.flag = (KeyFlag)j.at("keyboard").at("flag").get<int>();
break;
case ControlType::keyboard_state: {
const auto& keyboard_state_json = j.at("keyboard_state");
out.ks.seq = keyboard_state_json.value("seq", 0u);
out.ks.pressed_count = 0;
const auto keys_json =
keyboard_state_json.value("pressed_keys", json::array());
if (!keys_json.is_array()) {
break;
}
const size_t count =
keys_json.size() < kMaxKeyboardStateKeys
? keys_json.size()
: kMaxKeyboardStateKeys;
for (size_t idx = 0; idx < count; ++idx) {
const auto& key_json = keys_json[idx];
out.ks.pressed_keys[idx].key_value =
key_json.at("key_value").get<size_t>();
out.ks.pressed_keys[idx].scan_code =
key_json.value("scan_code", static_cast<uint32_t>(0));
out.ks.pressed_keys[idx].extended =
key_json.value("extended", false);
}
out.ks.pressed_count = count;
break;
}
case ControlType::audio_capture:
out.a = j.at("audio_capture").get<bool>();
break;
+6
View File
@@ -1192,10 +1192,16 @@ void Render::UpdateInteractions() {
keyboard_capturer_is_started_ = true;
}
}
if (keyboard_capturer_is_started_) {
SendKeyboardHeartbeat(false);
}
} else if (keyboard_capturer_is_started_) {
ForceReleasePressedKeys();
StopKeyboardCapturer();
keyboard_capturer_is_started_ = false;
}
CheckRemoteKeyboardTimeouts();
}
int Render::CreateMainWindow() {
+30 -2
View File
@@ -343,11 +343,35 @@ class Render {
static void FreeRemoteAction(RemoteAction& action);
private:
struct PressedKeyboardKey {
int key_code = 0;
uint32_t scan_code = 0;
bool extended = false;
};
struct RemoteKeyboardState {
std::unordered_map<int, PressedKeyboardKey> pressed_keys;
uint32_t last_seq = 0;
uint32_t last_seen_tick = 0;
bool keyboard_state_seen = false;
};
int SendKeyCommand(int key_code, bool is_down, uint32_t scan_code = 0,
bool extended = false);
static bool IsModifierVkKey(int key_code);
void TrackPressedKeyState(int key_code, bool is_down);
void TrackPressedKeyState(int key_code, bool is_down, uint32_t scan_code,
bool extended);
void ForceReleasePressedKeys();
void SendKeyboardHeartbeat(bool force);
void ApplyRemoteKeyboardEvent(const std::string& remote_id,
const RemoteAction& remote_action);
void ApplyRemoteKeyboardState(const std::string& remote_id,
const RemoteAction& remote_action);
bool InjectRemoteKeyboardKey(int key_code, bool is_down, uint32_t scan_code,
bool extended);
void ReleaseRemotePressedKeys(const std::string& remote_id,
const char* reason);
void CheckRemoteKeyboardTimeouts();
int ProcessKeyboardEvent(const SDL_Event& event);
int ProcessMouseEvent(const SDL_Event& event);
@@ -551,8 +575,12 @@ class Render {
std::string controlled_remote_id_ = "";
std::string focused_remote_id_ = "";
std::string remote_client_id_ = "";
std::unordered_set<int> pressed_keyboard_keys_;
std::unordered_map<int, PressedKeyboardKey> pressed_keyboard_keys_;
std::mutex pressed_keyboard_keys_mutex_;
uint32_t keyboard_state_seq_ = 0;
uint32_t last_keyboard_heartbeat_tick_ = 0;
std::unordered_map<std::string, RemoteKeyboardState> remote_keyboard_states_;
std::mutex remote_keyboard_states_mutex_;
SDL_Event last_mouse_event{};
SDL_AudioStream* output_stream_ = nullptr;
uint32_t STREAM_REFRESH_EVENT = 0;
+258 -48
View File
@@ -28,6 +28,8 @@
namespace crossdesk {
namespace {
constexpr uint32_t kKeyboardHeartbeatIntervalMs = 500;
constexpr uint32_t kRemoteKeyboardReleaseTimeoutMs = 2500;
int TranslateSdlKeypadScancodeToVk(const SDL_KeyboardEvent& event) {
const bool numlock_enabled = (event.mod & SDL_KMOD_NUM) != 0;
@@ -415,34 +417,92 @@ bool Render::IsModifierVkKey(int key_code) {
}
}
void Render::TrackPressedKeyState(int key_code, bool is_down) {
if (!IsWaylandSession() && !IsModifierVkKey(key_code)) {
return;
}
void Render::TrackPressedKeyState(int key_code, bool is_down,
uint32_t scan_code, bool extended) {
std::lock_guard<std::mutex> lock(pressed_keyboard_keys_mutex_);
if (is_down) {
pressed_keyboard_keys_.insert(key_code);
pressed_keyboard_keys_[key_code] =
PressedKeyboardKey{key_code, scan_code, extended};
} else {
pressed_keyboard_keys_.erase(key_code);
}
}
void Render::ForceReleasePressedKeys() {
std::vector<int> pressed_keys;
std::vector<PressedKeyboardKey> pressed_keys;
{
std::lock_guard<std::mutex> lock(pressed_keyboard_keys_mutex_);
if (pressed_keyboard_keys_.empty()) {
return;
pressed_keys.reserve(pressed_keyboard_keys_.size());
for (const auto& [_, key] : pressed_keyboard_keys_) {
pressed_keys.push_back(key);
}
pressed_keys.assign(pressed_keyboard_keys_.begin(),
pressed_keyboard_keys_.end());
pressed_keyboard_keys_.clear();
}
for (int key_code : pressed_keys) {
SendKeyCommand(key_code, false);
for (const PressedKeyboardKey& key : pressed_keys) {
SendKeyCommand(key.key_code, false, key.scan_code, key.extended);
}
SendKeyboardHeartbeat(true);
}
void Render::SendKeyboardHeartbeat(bool force) {
const uint32_t now = static_cast<uint32_t>(SDL_GetTicks());
if (!force && now - last_keyboard_heartbeat_tick_ <
kKeyboardHeartbeatIntervalMs) {
return;
}
RemoteAction remote_action{};
remote_action.type = ControlType::keyboard_state;
remote_action.ks.seq = ++keyboard_state_seq_;
{
std::lock_guard<std::mutex> lock(pressed_keyboard_keys_mutex_);
size_t idx = 0;
for (const auto& [_, key] : pressed_keyboard_keys_) {
if (idx >= kMaxKeyboardStateKeys) {
LOG_WARN("Keyboard heartbeat truncated, pressed_keys={}",
pressed_keyboard_keys_.size());
break;
}
remote_action.ks.pressed_keys[idx].key_value =
static_cast<size_t>(key.key_code);
remote_action.ks.pressed_keys[idx].scan_code = key.scan_code;
remote_action.ks.pressed_keys[idx].extended = key.extended;
++idx;
}
remote_action.ks.pressed_count = idx;
}
const std::string target_id = controlled_remote_id_.empty()
? focused_remote_id_
: controlled_remote_id_;
if (target_id.empty()) {
last_keyboard_heartbeat_tick_ = now;
return;
}
auto props_it = client_properties_.find(target_id);
if (props_it == client_properties_.end()) {
last_keyboard_heartbeat_tick_ = now;
return;
}
const auto props = props_it->second;
if (props->connection_status_ != ConnectionStatus::Connected ||
!props->peer_) {
last_keyboard_heartbeat_tick_ = now;
return;
}
const std::string msg = remote_action.to_json();
const int ret = SendReliableDataFrame(props->peer_, msg.c_str(), msg.size(),
props->keyboard_label_.c_str());
if (ret != 0) {
LOG_WARN("Send keyboard heartbeat failed, remote_id={}, ret={}", target_id,
ret);
}
last_keyboard_heartbeat_tick_ = now;
}
int Render::SendKeyCommand(int key_code, bool is_down, uint32_t scan_code,
@@ -484,7 +544,7 @@ int Render::SendKeyCommand(int key_code, bool is_down, uint32_t scan_code,
}
}
TrackPressedKeyState(key_code, is_down);
TrackPressedKeyState(key_code, is_down, scan_code, extended);
return 0;
}
@@ -506,6 +566,181 @@ int Render::ProcessKeyboardEvent(const SDL_Event& event) {
return SendKeyCommand(key_code, event.type == SDL_EVENT_KEY_DOWN);
}
bool Render::InjectRemoteKeyboardKey(int key_code, bool is_down,
uint32_t scan_code, bool extended) {
#if _WIN32
if (local_service_status_received_ &&
IsSecureDesktopInteractionRequired(local_interactive_stage_)) {
const std::string response = SendCrossDeskSecureDesktopKeyInput(
key_code, is_down, scan_code, extended, 1000);
auto json = nlohmann::json::parse(response, nullptr, false);
if (json.is_discarded() || !json.value("ok", false)) {
RemoteAction action{};
action.type = ControlType::keyboard;
action.k.key_value = static_cast<size_t>(key_code);
action.k.scan_code = scan_code;
action.k.extended = extended;
action.k.flag = is_down ? KeyFlag::key_down : KeyFlag::key_up;
if (!json.is_discarded() &&
IsTransientSecureDesktopInputFailure(json, action)) {
LOG_INFO(
"Secure desktop keyboard injection transient failure, "
"key_code={}, is_down={}, response={}",
key_code, is_down, response);
return true;
}
LogSecureDesktopInputBlocked(&last_local_secure_input_block_log_tick_,
"local",
local_interactive_stage_.c_str());
LOG_WARN(
"Secure desktop keyboard injection failed, key_code={}, is_down={}, "
"response={}",
key_code, is_down, response);
return false;
}
return true;
}
#endif
if (!keyboard_capturer_) {
return false;
}
return keyboard_capturer_->SendKeyboardCommand(key_code, is_down, scan_code,
extended) == 0;
}
void Render::ApplyRemoteKeyboardEvent(const std::string& remote_id,
const RemoteAction& remote_action) {
const int key_code = static_cast<int>(remote_action.k.key_value);
const bool is_down = remote_action.k.flag == KeyFlag::key_down;
const uint32_t scan_code = remote_action.k.scan_code;
const bool extended = remote_action.k.extended;
const bool injected =
InjectRemoteKeyboardKey(key_code, is_down, scan_code, extended);
std::lock_guard<std::mutex> lock(remote_keyboard_states_mutex_);
auto& state = remote_keyboard_states_[remote_id];
state.last_seen_tick = static_cast<uint32_t>(SDL_GetTicks());
if (is_down) {
if (injected) {
state.pressed_keys[key_code] =
PressedKeyboardKey{key_code, scan_code, extended};
}
} else if (injected) {
state.pressed_keys.erase(key_code);
}
}
void Render::ApplyRemoteKeyboardState(const std::string& remote_id,
const RemoteAction& remote_action) {
std::vector<PressedKeyboardKey> keys_to_release;
std::vector<PressedKeyboardKey> keys_to_press;
{
std::lock_guard<std::mutex> lock(remote_keyboard_states_mutex_);
auto& state = remote_keyboard_states_[remote_id];
if (remote_action.ks.seq != 0 && state.last_seq != 0 &&
static_cast<int32_t>(remote_action.ks.seq - state.last_seq) <= 0) {
return;
}
state.last_seq = remote_action.ks.seq;
state.last_seen_tick = static_cast<uint32_t>(SDL_GetTicks());
state.keyboard_state_seen = true;
std::unordered_map<int, PressedKeyboardKey> desired_keys;
const size_t pressed_count =
remote_action.ks.pressed_count < kMaxKeyboardStateKeys
? remote_action.ks.pressed_count
: kMaxKeyboardStateKeys;
for (size_t idx = 0; idx < pressed_count; ++idx) {
const auto& key = remote_action.ks.pressed_keys[idx];
const int key_code = static_cast<int>(key.key_value);
desired_keys[key_code] =
PressedKeyboardKey{key_code, key.scan_code, key.extended};
}
for (const auto& [key_code, key] : state.pressed_keys) {
if (desired_keys.find(key_code) == desired_keys.end()) {
keys_to_release.push_back(key);
}
}
for (const auto& [key_code, key] : desired_keys) {
if (state.pressed_keys.find(key_code) == state.pressed_keys.end()) {
keys_to_press.push_back(key);
}
}
}
for (const PressedKeyboardKey& key : keys_to_release) {
if (InjectRemoteKeyboardKey(key.key_code, false, key.scan_code,
key.extended)) {
std::lock_guard<std::mutex> lock(remote_keyboard_states_mutex_);
auto state_it = remote_keyboard_states_.find(remote_id);
if (state_it != remote_keyboard_states_.end()) {
state_it->second.pressed_keys.erase(key.key_code);
}
}
}
for (const PressedKeyboardKey& key : keys_to_press) {
if (InjectRemoteKeyboardKey(key.key_code, true, key.scan_code,
key.extended)) {
std::lock_guard<std::mutex> lock(remote_keyboard_states_mutex_);
auto& state = remote_keyboard_states_[remote_id];
state.pressed_keys[key.key_code] = key;
}
}
}
void Render::ReleaseRemotePressedKeys(const std::string& remote_id,
const char* reason) {
std::vector<PressedKeyboardKey> keys_to_release;
{
std::lock_guard<std::mutex> lock(remote_keyboard_states_mutex_);
auto state_it = remote_keyboard_states_.find(remote_id);
if (state_it == remote_keyboard_states_.end()) {
return;
}
keys_to_release.reserve(state_it->second.pressed_keys.size());
for (const auto& [_, key] : state_it->second.pressed_keys) {
keys_to_release.push_back(key);
}
remote_keyboard_states_.erase(state_it);
}
if (!keys_to_release.empty()) {
LOG_WARN("Releasing {} remote keyboard keys for remote_id={}, reason={}",
keys_to_release.size(), remote_id, reason ? reason : "unknown");
}
for (const PressedKeyboardKey& key : keys_to_release) {
InjectRemoteKeyboardKey(key.key_code, false, key.scan_code, key.extended);
}
}
void Render::CheckRemoteKeyboardTimeouts() {
const uint32_t now = static_cast<uint32_t>(SDL_GetTicks());
std::vector<std::string> timed_out_remotes;
{
std::lock_guard<std::mutex> lock(remote_keyboard_states_mutex_);
for (const auto& [remote_id, state] : remote_keyboard_states_) {
if (state.keyboard_state_seen && !state.pressed_keys.empty() &&
state.last_seen_tick != 0 &&
now - state.last_seen_tick > kRemoteKeyboardReleaseTimeoutMs) {
timed_out_remotes.push_back(remote_id);
}
}
}
for (const std::string& remote_id : timed_out_remotes) {
ReleaseRemotePressedKeys(remote_id, "keyboard_heartbeat_timeout");
}
}
int Render::ProcessMouseEvent(const SDL_Event& event) {
controlled_remote_id_ = "";
RemoteAction remote_action{};
@@ -1120,7 +1355,9 @@ void Render::OnReceiveDataBufferCb(const char* data, size_t size,
// remote
#if _WIN32
if (render->local_service_status_received_ &&
IsSecureDesktopInteractionRequired(render->local_interactive_stage_)) {
IsSecureDesktopInteractionRequired(render->local_interactive_stage_) &&
remote_action.type != ControlType::keyboard &&
remote_action.type != ControlType::keyboard_state) {
if (remote_action.type == ControlType::mouse) {
int absolute_x = 0;
int absolute_y = 0;
@@ -1151,33 +1388,6 @@ void Render::OnReceiveDataBufferCb(const char* data, size_t size,
}
return;
}
if (remote_action.type == ControlType::keyboard) {
const int key_code = static_cast<int>(remote_action.k.key_value);
const bool is_down = remote_action.k.flag == KeyFlag::key_down;
const std::string response = SendCrossDeskSecureDesktopKeyInput(
key_code, is_down, remote_action.k.scan_code,
remote_action.k.extended, 1000);
auto json = nlohmann::json::parse(response, nullptr, false);
if (json.is_discarded() || !json.value("ok", false)) {
if (!json.is_discarded() &&
IsTransientSecureDesktopInputFailure(json, remote_action)) {
LOG_INFO(
"Secure desktop keyboard injection transient failure, "
"key_code={}, is_down={}, response={}",
key_code, is_down, response);
return;
}
LogSecureDesktopInputBlocked(
&render->last_local_secure_input_block_log_tick_, "local",
render->local_interactive_stage_.c_str());
LOG_WARN(
"Secure desktop keyboard injection failed, key_code={}, "
"is_down={}, response={}",
key_code, is_down, response);
}
return;
}
}
#endif
if (remote_action.type == ControlType::mouse && render->mouse_controller_) {
@@ -1188,12 +1398,10 @@ void Render::OnReceiveDataBufferCb(const char* data, size_t size,
render->StartSpeakerCapturer();
else if (!remote_action.a && render->start_speaker_capturer_)
render->StopSpeakerCapturer();
} else if (remote_action.type == ControlType::keyboard &&
render->keyboard_capturer_) {
render->keyboard_capturer_->SendKeyboardCommand(
(int)remote_action.k.key_value,
remote_action.k.flag == KeyFlag::key_down, remote_action.k.scan_code,
remote_action.k.extended);
} else if (remote_action.type == ControlType::keyboard) {
render->ApplyRemoteKeyboardEvent(remote_id, remote_action);
} else if (remote_action.type == ControlType::keyboard_state) {
render->ApplyRemoteKeyboardState(remote_id, remote_action);
} else if (remote_action.type == ControlType::display_id &&
render->screen_capturer_) {
const int ret = render->screen_capturer_->SwitchTo(remote_action.d);
@@ -1345,6 +1553,7 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char* user_id,
case ConnectionStatus::Disconnected:
case ConnectionStatus::Failed:
case ConnectionStatus::Closed: {
render->ReleaseRemotePressedKeys(remote_id, "connection_closed");
props->connection_established_ = false;
props->enable_mouse_control_ = false;
render->ResetRemoteServiceStatus(*props);
@@ -1462,6 +1671,7 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char* user_id,
case ConnectionStatus::Disconnected:
case ConnectionStatus::Failed:
case ConnectionStatus::Closed: {
render->ReleaseRemotePressedKeys(remote_id, "connection_closed");
if (std::all_of(render->connection_status_.begin(),
render->connection_status_.end(), [](const auto& kv) {
return kv.second == ConnectionStatus::Closed ||
+71
View File
@@ -0,0 +1,71 @@
#include "device_controller.h"
#include <iostream>
#include <string>
namespace {
bool ExpectEqual(const char* name, size_t actual, size_t expected) {
if (actual == expected) {
return true;
}
std::cerr << name << " mismatch\n"
<< " expected: " << expected << "\n"
<< " actual: " << actual << "\n";
return false;
}
bool ExpectTrue(const char* name, bool value) {
if (value) {
return true;
}
std::cerr << name << " expected true\n";
return false;
}
} // namespace
int main() {
bool ok = true;
ok &= ExpectEqual("mouse type", crossdesk::ControlType::mouse, 0);
ok &= ExpectEqual("keyboard type", crossdesk::ControlType::keyboard, 1);
ok &= ExpectEqual("audio_capture type", crossdesk::ControlType::audio_capture,
2);
ok &= ExpectEqual("host_infomation type",
crossdesk::ControlType::host_infomation, 3);
ok &= ExpectEqual("display_id type", crossdesk::ControlType::display_id, 4);
ok &= ExpectEqual("service_status type",
crossdesk::ControlType::service_status, 5);
ok &= ExpectEqual("service_command type",
crossdesk::ControlType::service_command, 6);
ok &= ExpectEqual("keyboard_state type",
crossdesk::ControlType::keyboard_state, 7);
crossdesk::RemoteAction action{};
action.type = crossdesk::ControlType::keyboard_state;
action.ks.seq = 42;
action.ks.pressed_count = 2;
action.ks.pressed_keys[0] = {65, 30, false};
action.ks.pressed_keys[1] = {0xA3, 29, true};
const std::string json = action.to_json();
crossdesk::RemoteAction parsed{};
ok &= ExpectTrue("parse keyboard_state", parsed.from_json(json));
ok &= ExpectEqual("parsed type", parsed.type,
crossdesk::ControlType::keyboard_state);
ok &= ExpectEqual("parsed seq", parsed.ks.seq, 42);
ok &= ExpectEqual("parsed pressed_count", parsed.ks.pressed_count, 2);
ok &= ExpectEqual("parsed key 0", parsed.ks.pressed_keys[0].key_value, 65);
ok &= ExpectEqual("parsed scan 0", parsed.ks.pressed_keys[0].scan_code, 30);
ok &= ExpectTrue("parsed extended 0",
!parsed.ks.pressed_keys[0].extended);
ok &= ExpectEqual("parsed key 1", parsed.ks.pressed_keys[1].key_value,
0xA3);
ok &= ExpectEqual("parsed scan 1", parsed.ks.pressed_keys[1].scan_code, 29);
ok &= ExpectTrue("parsed extended 1", parsed.ks.pressed_keys[1].extended);
return ok ? 0 : 1;
}
+6
View File
@@ -44,6 +44,12 @@ function setup_targets()
add_includedirs("src/device_controller")
add_files("tests/macos_keyboard_modifier_state_test.cpp")
target("keyboard_state_protocol_test")
set_kind("binary")
set_default(false)
add_includedirs("src/device_controller", "src/common")
add_files("tests/keyboard_state_protocol_test.cpp")
target("windows_manifest_resource_test")
set_kind("binary")
set_default(false)