diff --git a/src/device_controller/device_controller.h b/src/device_controller/device_controller.h index c3f006f..ee41fc7 100644 --- a/src/device_controller/device_controller.h +++ b/src/device_controller/device_controller.h @@ -21,12 +21,13 @@ namespace crossdesk { typedef enum { mouse = 0, - keyboard, - audio_capture, - host_infomation, - display_id, - service_status, - service_command, + keyboard = 1, + audio_capture = 2, + host_infomation = 3, + display_id = 4, + service_status = 5, + service_command = 6, + keyboard_state = 7, } ControlType; typedef enum { move = 0, @@ -55,6 +56,20 @@ typedef struct { KeyFlag flag; } Key; +inline constexpr size_t kMaxKeyboardStateKeys = 32; + +typedef struct { + size_t key_value; + uint32_t scan_code; + bool extended; +} KeyboardStateKey; + +typedef struct { + uint32_t seq; + size_t pressed_count; + KeyboardStateKey pressed_keys[kMaxKeyboardStateKeys]; +} KeyboardState; + typedef struct { char host_name[64]; size_t host_name_size; @@ -80,6 +95,7 @@ struct RemoteAction { union { Mouse m; Key k; + KeyboardState ks; HostInfo i; bool a; int d; @@ -111,6 +127,20 @@ struct RemoteAction { {"extended", a.k.extended}, {"flag", a.k.flag}}; break; + case ControlType::keyboard_state: { + json keys = json::array(); + const size_t pressed_count = + a.ks.pressed_count < kMaxKeyboardStateKeys + ? a.ks.pressed_count + : kMaxKeyboardStateKeys; + for (size_t idx = 0; idx < pressed_count; ++idx) { + keys.push_back({{"key_value", a.ks.pressed_keys[idx].key_value}, + {"scan_code", a.ks.pressed_keys[idx].scan_code}, + {"extended", a.ks.pressed_keys[idx].extended}}); + } + j["keyboard_state"] = {{"seq", a.ks.seq}, {"pressed_keys", keys}}; + break; + } case ControlType::audio_capture: j["audio_capture"] = a.a; break; @@ -162,6 +192,33 @@ struct RemoteAction { out.k.extended = j.at("keyboard").value("extended", false); out.k.flag = (KeyFlag)j.at("keyboard").at("flag").get(); break; + case ControlType::keyboard_state: { + const auto& keyboard_state_json = j.at("keyboard_state"); + out.ks.seq = keyboard_state_json.value("seq", 0u); + out.ks.pressed_count = 0; + + const auto keys_json = + keyboard_state_json.value("pressed_keys", json::array()); + if (!keys_json.is_array()) { + break; + } + + const size_t count = + keys_json.size() < kMaxKeyboardStateKeys + ? keys_json.size() + : kMaxKeyboardStateKeys; + for (size_t idx = 0; idx < count; ++idx) { + const auto& key_json = keys_json[idx]; + out.ks.pressed_keys[idx].key_value = + key_json.at("key_value").get(); + out.ks.pressed_keys[idx].scan_code = + key_json.value("scan_code", static_cast(0)); + out.ks.pressed_keys[idx].extended = + key_json.value("extended", false); + } + out.ks.pressed_count = count; + break; + } case ControlType::audio_capture: out.a = j.at("audio_capture").get(); break; diff --git a/src/gui/render.cpp b/src/gui/render.cpp index 81a172e..efb2deb 100644 --- a/src/gui/render.cpp +++ b/src/gui/render.cpp @@ -1192,10 +1192,16 @@ void Render::UpdateInteractions() { keyboard_capturer_is_started_ = true; } } + if (keyboard_capturer_is_started_) { + SendKeyboardHeartbeat(false); + } } else if (keyboard_capturer_is_started_) { + ForceReleasePressedKeys(); StopKeyboardCapturer(); keyboard_capturer_is_started_ = false; } + + CheckRemoteKeyboardTimeouts(); } int Render::CreateMainWindow() { diff --git a/src/gui/render.h b/src/gui/render.h index 674c58c..cfae904 100644 --- a/src/gui/render.h +++ b/src/gui/render.h @@ -343,11 +343,35 @@ class Render { static void FreeRemoteAction(RemoteAction& action); private: + struct PressedKeyboardKey { + int key_code = 0; + uint32_t scan_code = 0; + bool extended = false; + }; + + struct RemoteKeyboardState { + std::unordered_map pressed_keys; + uint32_t last_seq = 0; + uint32_t last_seen_tick = 0; + bool keyboard_state_seen = false; + }; + int SendKeyCommand(int key_code, bool is_down, uint32_t scan_code = 0, bool extended = false); static bool IsModifierVkKey(int key_code); - void TrackPressedKeyState(int key_code, bool is_down); + void TrackPressedKeyState(int key_code, bool is_down, uint32_t scan_code, + bool extended); void ForceReleasePressedKeys(); + void SendKeyboardHeartbeat(bool force); + void ApplyRemoteKeyboardEvent(const std::string& remote_id, + const RemoteAction& remote_action); + void ApplyRemoteKeyboardState(const std::string& remote_id, + const RemoteAction& remote_action); + bool InjectRemoteKeyboardKey(int key_code, bool is_down, uint32_t scan_code, + bool extended); + void ReleaseRemotePressedKeys(const std::string& remote_id, + const char* reason); + void CheckRemoteKeyboardTimeouts(); int ProcessKeyboardEvent(const SDL_Event& event); int ProcessMouseEvent(const SDL_Event& event); @@ -551,8 +575,12 @@ class Render { std::string controlled_remote_id_ = ""; std::string focused_remote_id_ = ""; std::string remote_client_id_ = ""; - std::unordered_set pressed_keyboard_keys_; + std::unordered_map pressed_keyboard_keys_; std::mutex pressed_keyboard_keys_mutex_; + uint32_t keyboard_state_seq_ = 0; + uint32_t last_keyboard_heartbeat_tick_ = 0; + std::unordered_map remote_keyboard_states_; + std::mutex remote_keyboard_states_mutex_; SDL_Event last_mouse_event{}; SDL_AudioStream* output_stream_ = nullptr; uint32_t STREAM_REFRESH_EVENT = 0; diff --git a/src/gui/render_callback.cpp b/src/gui/render_callback.cpp index faf1dc7..d430134 100644 --- a/src/gui/render_callback.cpp +++ b/src/gui/render_callback.cpp @@ -28,6 +28,8 @@ namespace crossdesk { namespace { +constexpr uint32_t kKeyboardHeartbeatIntervalMs = 500; +constexpr uint32_t kRemoteKeyboardReleaseTimeoutMs = 2500; int TranslateSdlKeypadScancodeToVk(const SDL_KeyboardEvent& event) { const bool numlock_enabled = (event.mod & SDL_KMOD_NUM) != 0; @@ -415,34 +417,92 @@ bool Render::IsModifierVkKey(int key_code) { } } -void Render::TrackPressedKeyState(int key_code, bool is_down) { - if (!IsWaylandSession() && !IsModifierVkKey(key_code)) { - return; - } - +void Render::TrackPressedKeyState(int key_code, bool is_down, + uint32_t scan_code, bool extended) { std::lock_guard lock(pressed_keyboard_keys_mutex_); if (is_down) { - pressed_keyboard_keys_.insert(key_code); + pressed_keyboard_keys_[key_code] = + PressedKeyboardKey{key_code, scan_code, extended}; } else { pressed_keyboard_keys_.erase(key_code); } } void Render::ForceReleasePressedKeys() { - std::vector pressed_keys; + std::vector pressed_keys; { std::lock_guard lock(pressed_keyboard_keys_mutex_); - if (pressed_keyboard_keys_.empty()) { - return; + pressed_keys.reserve(pressed_keyboard_keys_.size()); + for (const auto& [_, key] : pressed_keyboard_keys_) { + pressed_keys.push_back(key); } - pressed_keys.assign(pressed_keyboard_keys_.begin(), - pressed_keyboard_keys_.end()); pressed_keyboard_keys_.clear(); } - for (int key_code : pressed_keys) { - SendKeyCommand(key_code, false); + for (const PressedKeyboardKey& key : pressed_keys) { + SendKeyCommand(key.key_code, false, key.scan_code, key.extended); } + SendKeyboardHeartbeat(true); +} + +void Render::SendKeyboardHeartbeat(bool force) { + const uint32_t now = static_cast(SDL_GetTicks()); + if (!force && now - last_keyboard_heartbeat_tick_ < + kKeyboardHeartbeatIntervalMs) { + return; + } + + RemoteAction remote_action{}; + remote_action.type = ControlType::keyboard_state; + remote_action.ks.seq = ++keyboard_state_seq_; + + { + std::lock_guard lock(pressed_keyboard_keys_mutex_); + size_t idx = 0; + for (const auto& [_, key] : pressed_keyboard_keys_) { + if (idx >= kMaxKeyboardStateKeys) { + LOG_WARN("Keyboard heartbeat truncated, pressed_keys={}", + pressed_keyboard_keys_.size()); + break; + } + remote_action.ks.pressed_keys[idx].key_value = + static_cast(key.key_code); + remote_action.ks.pressed_keys[idx].scan_code = key.scan_code; + remote_action.ks.pressed_keys[idx].extended = key.extended; + ++idx; + } + remote_action.ks.pressed_count = idx; + } + + const std::string target_id = controlled_remote_id_.empty() + ? focused_remote_id_ + : controlled_remote_id_; + if (target_id.empty()) { + last_keyboard_heartbeat_tick_ = now; + return; + } + + auto props_it = client_properties_.find(target_id); + if (props_it == client_properties_.end()) { + last_keyboard_heartbeat_tick_ = now; + return; + } + + const auto props = props_it->second; + if (props->connection_status_ != ConnectionStatus::Connected || + !props->peer_) { + last_keyboard_heartbeat_tick_ = now; + return; + } + + const std::string msg = remote_action.to_json(); + const int ret = SendReliableDataFrame(props->peer_, msg.c_str(), msg.size(), + props->keyboard_label_.c_str()); + if (ret != 0) { + LOG_WARN("Send keyboard heartbeat failed, remote_id={}, ret={}", target_id, + ret); + } + last_keyboard_heartbeat_tick_ = now; } int Render::SendKeyCommand(int key_code, bool is_down, uint32_t scan_code, @@ -484,7 +544,7 @@ int Render::SendKeyCommand(int key_code, bool is_down, uint32_t scan_code, } } - TrackPressedKeyState(key_code, is_down); + TrackPressedKeyState(key_code, is_down, scan_code, extended); return 0; } @@ -506,6 +566,181 @@ int Render::ProcessKeyboardEvent(const SDL_Event& event) { return SendKeyCommand(key_code, event.type == SDL_EVENT_KEY_DOWN); } +bool Render::InjectRemoteKeyboardKey(int key_code, bool is_down, + uint32_t scan_code, bool extended) { +#if _WIN32 + if (local_service_status_received_ && + IsSecureDesktopInteractionRequired(local_interactive_stage_)) { + const std::string response = SendCrossDeskSecureDesktopKeyInput( + key_code, is_down, scan_code, extended, 1000); + auto json = nlohmann::json::parse(response, nullptr, false); + if (json.is_discarded() || !json.value("ok", false)) { + RemoteAction action{}; + action.type = ControlType::keyboard; + action.k.key_value = static_cast(key_code); + action.k.scan_code = scan_code; + action.k.extended = extended; + action.k.flag = is_down ? KeyFlag::key_down : KeyFlag::key_up; + if (!json.is_discarded() && + IsTransientSecureDesktopInputFailure(json, action)) { + LOG_INFO( + "Secure desktop keyboard injection transient failure, " + "key_code={}, is_down={}, response={}", + key_code, is_down, response); + return true; + } + + LogSecureDesktopInputBlocked(&last_local_secure_input_block_log_tick_, + "local", + local_interactive_stage_.c_str()); + LOG_WARN( + "Secure desktop keyboard injection failed, key_code={}, is_down={}, " + "response={}", + key_code, is_down, response); + return false; + } + return true; + } +#endif + + if (!keyboard_capturer_) { + return false; + } + + return keyboard_capturer_->SendKeyboardCommand(key_code, is_down, scan_code, + extended) == 0; +} + +void Render::ApplyRemoteKeyboardEvent(const std::string& remote_id, + const RemoteAction& remote_action) { + const int key_code = static_cast(remote_action.k.key_value); + const bool is_down = remote_action.k.flag == KeyFlag::key_down; + const uint32_t scan_code = remote_action.k.scan_code; + const bool extended = remote_action.k.extended; + const bool injected = + InjectRemoteKeyboardKey(key_code, is_down, scan_code, extended); + + std::lock_guard lock(remote_keyboard_states_mutex_); + auto& state = remote_keyboard_states_[remote_id]; + state.last_seen_tick = static_cast(SDL_GetTicks()); + if (is_down) { + if (injected) { + state.pressed_keys[key_code] = + PressedKeyboardKey{key_code, scan_code, extended}; + } + } else if (injected) { + state.pressed_keys.erase(key_code); + } +} + +void Render::ApplyRemoteKeyboardState(const std::string& remote_id, + const RemoteAction& remote_action) { + std::vector keys_to_release; + std::vector keys_to_press; + + { + std::lock_guard lock(remote_keyboard_states_mutex_); + auto& state = remote_keyboard_states_[remote_id]; + if (remote_action.ks.seq != 0 && state.last_seq != 0 && + static_cast(remote_action.ks.seq - state.last_seq) <= 0) { + return; + } + + state.last_seq = remote_action.ks.seq; + state.last_seen_tick = static_cast(SDL_GetTicks()); + state.keyboard_state_seen = true; + + std::unordered_map desired_keys; + const size_t pressed_count = + remote_action.ks.pressed_count < kMaxKeyboardStateKeys + ? remote_action.ks.pressed_count + : kMaxKeyboardStateKeys; + for (size_t idx = 0; idx < pressed_count; ++idx) { + const auto& key = remote_action.ks.pressed_keys[idx]; + const int key_code = static_cast(key.key_value); + desired_keys[key_code] = + PressedKeyboardKey{key_code, key.scan_code, key.extended}; + } + + for (const auto& [key_code, key] : state.pressed_keys) { + if (desired_keys.find(key_code) == desired_keys.end()) { + keys_to_release.push_back(key); + } + } + + for (const auto& [key_code, key] : desired_keys) { + if (state.pressed_keys.find(key_code) == state.pressed_keys.end()) { + keys_to_press.push_back(key); + } + } + } + + for (const PressedKeyboardKey& key : keys_to_release) { + if (InjectRemoteKeyboardKey(key.key_code, false, key.scan_code, + key.extended)) { + std::lock_guard lock(remote_keyboard_states_mutex_); + auto state_it = remote_keyboard_states_.find(remote_id); + if (state_it != remote_keyboard_states_.end()) { + state_it->second.pressed_keys.erase(key.key_code); + } + } + } + + for (const PressedKeyboardKey& key : keys_to_press) { + if (InjectRemoteKeyboardKey(key.key_code, true, key.scan_code, + key.extended)) { + std::lock_guard lock(remote_keyboard_states_mutex_); + auto& state = remote_keyboard_states_[remote_id]; + state.pressed_keys[key.key_code] = key; + } + } +} + +void Render::ReleaseRemotePressedKeys(const std::string& remote_id, + const char* reason) { + std::vector keys_to_release; + { + std::lock_guard lock(remote_keyboard_states_mutex_); + auto state_it = remote_keyboard_states_.find(remote_id); + if (state_it == remote_keyboard_states_.end()) { + return; + } + + keys_to_release.reserve(state_it->second.pressed_keys.size()); + for (const auto& [_, key] : state_it->second.pressed_keys) { + keys_to_release.push_back(key); + } + remote_keyboard_states_.erase(state_it); + } + + if (!keys_to_release.empty()) { + LOG_WARN("Releasing {} remote keyboard keys for remote_id={}, reason={}", + keys_to_release.size(), remote_id, reason ? reason : "unknown"); + } + for (const PressedKeyboardKey& key : keys_to_release) { + InjectRemoteKeyboardKey(key.key_code, false, key.scan_code, key.extended); + } +} + +void Render::CheckRemoteKeyboardTimeouts() { + const uint32_t now = static_cast(SDL_GetTicks()); + std::vector timed_out_remotes; + { + std::lock_guard lock(remote_keyboard_states_mutex_); + for (const auto& [remote_id, state] : remote_keyboard_states_) { + if (state.keyboard_state_seen && !state.pressed_keys.empty() && + state.last_seen_tick != 0 && + now - state.last_seen_tick > kRemoteKeyboardReleaseTimeoutMs) { + timed_out_remotes.push_back(remote_id); + } + } + } + + for (const std::string& remote_id : timed_out_remotes) { + ReleaseRemotePressedKeys(remote_id, "keyboard_heartbeat_timeout"); + } +} + int Render::ProcessMouseEvent(const SDL_Event& event) { controlled_remote_id_ = ""; RemoteAction remote_action{}; @@ -1120,7 +1355,9 @@ void Render::OnReceiveDataBufferCb(const char* data, size_t size, // remote #if _WIN32 if (render->local_service_status_received_ && - IsSecureDesktopInteractionRequired(render->local_interactive_stage_)) { + IsSecureDesktopInteractionRequired(render->local_interactive_stage_) && + remote_action.type != ControlType::keyboard && + remote_action.type != ControlType::keyboard_state) { if (remote_action.type == ControlType::mouse) { int absolute_x = 0; int absolute_y = 0; @@ -1151,33 +1388,6 @@ void Render::OnReceiveDataBufferCb(const char* data, size_t size, } return; } - - if (remote_action.type == ControlType::keyboard) { - const int key_code = static_cast(remote_action.k.key_value); - const bool is_down = remote_action.k.flag == KeyFlag::key_down; - const std::string response = SendCrossDeskSecureDesktopKeyInput( - key_code, is_down, remote_action.k.scan_code, - remote_action.k.extended, 1000); - auto json = nlohmann::json::parse(response, nullptr, false); - if (json.is_discarded() || !json.value("ok", false)) { - if (!json.is_discarded() && - IsTransientSecureDesktopInputFailure(json, remote_action)) { - LOG_INFO( - "Secure desktop keyboard injection transient failure, " - "key_code={}, is_down={}, response={}", - key_code, is_down, response); - return; - } - LogSecureDesktopInputBlocked( - &render->last_local_secure_input_block_log_tick_, "local", - render->local_interactive_stage_.c_str()); - LOG_WARN( - "Secure desktop keyboard injection failed, key_code={}, " - "is_down={}, response={}", - key_code, is_down, response); - } - return; - } } #endif if (remote_action.type == ControlType::mouse && render->mouse_controller_) { @@ -1188,12 +1398,10 @@ void Render::OnReceiveDataBufferCb(const char* data, size_t size, render->StartSpeakerCapturer(); else if (!remote_action.a && render->start_speaker_capturer_) render->StopSpeakerCapturer(); - } else if (remote_action.type == ControlType::keyboard && - render->keyboard_capturer_) { - render->keyboard_capturer_->SendKeyboardCommand( - (int)remote_action.k.key_value, - remote_action.k.flag == KeyFlag::key_down, remote_action.k.scan_code, - remote_action.k.extended); + } else if (remote_action.type == ControlType::keyboard) { + render->ApplyRemoteKeyboardEvent(remote_id, remote_action); + } else if (remote_action.type == ControlType::keyboard_state) { + render->ApplyRemoteKeyboardState(remote_id, remote_action); } else if (remote_action.type == ControlType::display_id && render->screen_capturer_) { const int ret = render->screen_capturer_->SwitchTo(remote_action.d); @@ -1345,6 +1553,7 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char* user_id, case ConnectionStatus::Disconnected: case ConnectionStatus::Failed: case ConnectionStatus::Closed: { + render->ReleaseRemotePressedKeys(remote_id, "connection_closed"); props->connection_established_ = false; props->enable_mouse_control_ = false; render->ResetRemoteServiceStatus(*props); @@ -1462,6 +1671,7 @@ void Render::OnConnectionStatusCb(ConnectionStatus status, const char* user_id, case ConnectionStatus::Disconnected: case ConnectionStatus::Failed: case ConnectionStatus::Closed: { + render->ReleaseRemotePressedKeys(remote_id, "connection_closed"); if (std::all_of(render->connection_status_.begin(), render->connection_status_.end(), [](const auto& kv) { return kv.second == ConnectionStatus::Closed || diff --git a/tests/keyboard_state_protocol_test.cpp b/tests/keyboard_state_protocol_test.cpp new file mode 100644 index 0000000..8c7efd5 --- /dev/null +++ b/tests/keyboard_state_protocol_test.cpp @@ -0,0 +1,71 @@ +#include "device_controller.h" + +#include +#include + +namespace { + +bool ExpectEqual(const char* name, size_t actual, size_t expected) { + if (actual == expected) { + return true; + } + + std::cerr << name << " mismatch\n" + << " expected: " << expected << "\n" + << " actual: " << actual << "\n"; + return false; +} + +bool ExpectTrue(const char* name, bool value) { + if (value) { + return true; + } + + std::cerr << name << " expected true\n"; + return false; +} + +} // namespace + +int main() { + bool ok = true; + ok &= ExpectEqual("mouse type", crossdesk::ControlType::mouse, 0); + ok &= ExpectEqual("keyboard type", crossdesk::ControlType::keyboard, 1); + ok &= ExpectEqual("audio_capture type", crossdesk::ControlType::audio_capture, + 2); + ok &= ExpectEqual("host_infomation type", + crossdesk::ControlType::host_infomation, 3); + ok &= ExpectEqual("display_id type", crossdesk::ControlType::display_id, 4); + ok &= ExpectEqual("service_status type", + crossdesk::ControlType::service_status, 5); + ok &= ExpectEqual("service_command type", + crossdesk::ControlType::service_command, 6); + ok &= ExpectEqual("keyboard_state type", + crossdesk::ControlType::keyboard_state, 7); + + crossdesk::RemoteAction action{}; + action.type = crossdesk::ControlType::keyboard_state; + action.ks.seq = 42; + action.ks.pressed_count = 2; + action.ks.pressed_keys[0] = {65, 30, false}; + action.ks.pressed_keys[1] = {0xA3, 29, true}; + + const std::string json = action.to_json(); + + crossdesk::RemoteAction parsed{}; + ok &= ExpectTrue("parse keyboard_state", parsed.from_json(json)); + ok &= ExpectEqual("parsed type", parsed.type, + crossdesk::ControlType::keyboard_state); + ok &= ExpectEqual("parsed seq", parsed.ks.seq, 42); + ok &= ExpectEqual("parsed pressed_count", parsed.ks.pressed_count, 2); + ok &= ExpectEqual("parsed key 0", parsed.ks.pressed_keys[0].key_value, 65); + ok &= ExpectEqual("parsed scan 0", parsed.ks.pressed_keys[0].scan_code, 30); + ok &= ExpectTrue("parsed extended 0", + !parsed.ks.pressed_keys[0].extended); + ok &= ExpectEqual("parsed key 1", parsed.ks.pressed_keys[1].key_value, + 0xA3); + ok &= ExpectEqual("parsed scan 1", parsed.ks.pressed_keys[1].scan_code, 29); + ok &= ExpectTrue("parsed extended 1", parsed.ks.pressed_keys[1].extended); + + return ok ? 0 : 1; +} diff --git a/xmake/targets.lua b/xmake/targets.lua index 6b8d0ed..ee122e0 100644 --- a/xmake/targets.lua +++ b/xmake/targets.lua @@ -44,6 +44,12 @@ function setup_targets() add_includedirs("src/device_controller") add_files("tests/macos_keyboard_modifier_state_test.cpp") + target("keyboard_state_protocol_test") + set_kind("binary") + set_default(false) + add_includedirs("src/device_controller", "src/common") + add_files("tests/keyboard_state_protocol_test.cpp") + target("windows_manifest_resource_test") set_kind("binary") set_default(false)