diff --git a/apps/cli/agent.cpp b/apps/cli/agent.cpp index 3983e14b..26dbb2e0 100644 --- a/apps/cli/agent.cpp +++ b/apps/cli/agent.cpp @@ -943,14 +943,30 @@ private: session_.Disconnect(); } else throw std::invalid_argument("INVALID_REQUEST"); + } else if (method == "video_settings") { + Fields(j, {"id", "method", "fps", "width", "height", "quality", "max_bitrate_bps"}); + if (action_) throw std::invalid_argument("BUSY"); + if (session_.HasPressedInput()) throw std::invalid_argument("INPUT_HELD"); + const auto status = session_.Status(); + if (!session_.Connected() || !status.value("preview_supported", false)) + throw std::invalid_argument("VIDEO_SETTINGS_UNSUPPORTED"); + if (((j.contains("width") || j.contains("height")) && !status.value("preview_resolution_supported", false)) || + (j.contains("max_bitrate_bps") && !status.value("preview_bitrate_supported", false)) || + (j.value("fps", json{}) == 15 && !status.value("preview_fps15_supported", false))) + throw std::invalid_argument("VIDEO_SETTINGS_UNSUPPORTED"); + session_.SetPreview(Number(j, "fps", 1, 60, status.value("preview_fps", 30)), + Number(j, "width", 2, 3840, status.value("preview_width", 1920)), + Number(j, "height", 2, 2160, status.value("preview_height", 1080)), + Number(j, "quality", 0, 2, status.value("preview_quality", 2)), + Number(j, "max_bitrate_bps", 128000, 100000000, status.value("preview_bitrate_bps", 6000000))); } else if (method == "display") { - Fields(j, {"id", "method", "index"}); + Fields(j, {"id", "method", "index", "preserve_input"}); const int index = Number(j, "index", 0, 63, -1); if (!session_.Connected() || index < 0 || size_t(index) >= session_.Status()["displays"].size()) throw std::invalid_argument("DISPLAY_UNAVAILABLE"); Stop("SESSION_CHANGED"); - if (!session_.SelectDisplay(index)) + if (!session_.SelectDisplay(index, Flag(j, "preserve_input"))) throw std::invalid_argument("DISPLAY_UNAVAILABLE"); frame_.reset(); ++epoch_; diff --git a/apps/cli/docs/automation-protocol.md b/apps/cli/docs/automation-protocol.md index 005e6032..5cd75d1e 100644 --- a/apps/cli/docs/automation-protocol.md +++ b/apps/cli/docs/automation-protocol.md @@ -163,3 +163,9 @@ The desktop and CLI share `FileBrowser` and the existing file transfer engine. P `key_down`, `key_up`, `mouse_down`, and `mouse_up` extend the action union. Each successful action renews a fifteen-second lease for held input. Existing held modifiers survive subsequent clicks, movement and hotkeys. `release_inputs` releases the controller's tracked keys/buttons and cancels current input without requiring an observation. Errors, human pause, display changes and disconnection release held input. `key_press` and `hotkey` accept `duration_ms` from 10 to 30000. Drags accept durations up to 30000 milliseconds. Extended key names include F13–F24, NUMPAD0–NUMPAD9, arithmetic keypad keys, lock/menu/print keys and media controls. Target platform key mapping determines availability. Text input requires held keys/buttons to be released first. + +## Display management + +`video_settings` accepts optional `fps`, even `width`/`height`, `quality`, and `max_bitrate_bps`. The controller checks negotiated preview capabilities and publishes the requested cadence and acceptance state. Supported MCP frame-rate tiers are 1, 8, 15, 30 and 60. + +The trusted adapter may use `display` with `preserve_input: true` during a cross-display drag. It captures a fresh observation after each switch, maps the requested point from its source display image, and releases input in a final cleanup step. Ordinary display selection releases input. Multi-display capture returns sequential observations and restores the previous display. diff --git a/apps/mcp/README.md b/apps/mcp/README.md index c2563f1a..3cb5dbcb 100644 --- a/apps/mcp/README.md +++ b/apps/mcp/README.md @@ -63,7 +63,7 @@ Windows 和 Linux 使用各自构建产物路径；`--native-binary` 必须指 面板发起的连接失败后，按 `view_id` 等待也会返回 `failed` 和错误码，避免 Agent 一直停留在等待用户状态。断开正在建立的连接会取消整个连接尝试。多个面板和 Agent 同时查询状态时共用一次原生查询，单次查询取消或暂时繁忙不会断开会话。`crossdesk_connect` 在连接成功但面板初始化失败时仍返回有效的 `session_id`，并附加 `panel_error`。 -初次信令注册使用原生连接的整体 30 秒期限，期间允许 MiniRTC 按退避策略恢复临时传输异常。期限覆盖所有初始重试。证书校验失败、登录被拒绝，以及设备连接请求发出后的信令中断会结束当前尝试。连接失败结果使用 `error.code` 表示错误类别，并以 `error.reason` 提供已识别的原生原因，例如 `signaling_timeout`、`signaling_disconnected` 或 `device_offline_or_unavailable`。面板显示对应说明，终端错误包含原因标识。`logs/transport.log` 记录连接阶段、受控的传输错误分类与数字码、连接关闭码和重试退避信息。 +初次信令注册使用原生连接的整体 30 秒期限，期间允许 MiniRTC 按退避策略恢复临时传输异常。期限覆盖所有初始重试。证书校验失败、登录被拒绝，以及设备连接请求发出后的信令中断会结束当前尝试。连接失败结果使用 `error.code` 表示错误类别，并以 `error.reason` 提供已识别的原生原因，例如 `signaling_timeout`、`signaling_disconnected` 或 `device_offline_or_unavailable`。面板显示对应说明，终端错误包含原因标识。`logs//transport.log` 记录连接阶段、受控的传输错误分类与数字码、连接关闭码和重试退避信息。 自动继续依赖宿主保持当前 Agent 任务及工具调用流程。它不通过网页伪造用户消息，也不承诺从已经结束的对话中主动唤醒 Agent；此类唤醒由宿主负责。 @@ -274,3 +274,11 @@ npm run pack:npm -- --out-dir /absolute/path/artifacts `crossdesk_action` 增加 `key_down`、`key_up`、`mouse_down` 和 `mouse_up`。按下状态可跨动作保持，例如按住 Ctrl 后移动或拖动鼠标。最后一次动作完成后，按住状态的有效期为十五秒，超时自动释放。`crossdesk_release_inputs` 可立即释放本会话的全部按键和按钮，并取消当前输入动作。人工暂停、断开及输入错误同样释放按住状态。 `key_press`、`hotkey` 支持 `duration_ms`，拖拽时长上限提高到三十秒。扩展键名覆盖 F13–F24、数字键盘、菜单、锁定键和媒体键，实际效果取决于远端系统支持。状态返回 `held_key_codes` 和 `held_mouse_flags`。文本输入前应释放持续按住的键鼠状态。 + +### 多会话与显示管理 + +同一 MCP 实例支持最多八个并行设备连接。`crossdesk_list_sessions` 返回各连接的设备、显示器及控制状态。面板按设备绑定会话，断开一个连接会释放该连接的资源。每个连接的诊断日志位于 `logs//transport.log`。 + +`crossdesk_capture_displays` 按顺序截取最多八个显示器并恢复原显示器，通过一次工具结果展示多屏画面。`capture_id` 有效期三十秒，供 `crossdesk_drag_across` 使用各屏截图坐标完成跨屏拖拽。拖拽保持按下状态跨越显示器切换，在完成、取消或异常时释放按钮。普通输入操作使用重新选择显示器后取得的新截图。 + +`crossdesk_video_settings` 在运行时调整预览帧率、分辨率、质量和码率，返回远端接受状态。设置受对端能力限制。多屏采集及跨屏拖拽期间，同一会话的其他输入请求返回忙碌状态。 diff --git a/apps/mcp/src/server.ts b/apps/mcp/src/server.ts index faa9c2b0..fc94a3d3 100644 --- a/apps/mcp/src/server.ts +++ b/apps/mcp/src/server.ts @@ -56,6 +56,11 @@ function output(value: Data, meta?: Data): CallToolResult { data.observation = { ...data.observation }; delete data.observation.data; } + if (Array.isArray(data.observations)) data.observations = data.observations.map((observation: Data) => { + const { data: bytes, ...metadata } = observation; + if (bytes) content.push({ type: "image", mimeType: observation.mime_type, data: bytes }); + return metadata; + }); content.unshift({ type: "text", text: JSON.stringify(data) }); return { content, structuredContent: data, ...(meta ? { _meta: meta } : {}) }; } @@ -125,7 +130,7 @@ export function createServer( annotations: { readOnlyHint: !!d.readOnly, destructiveHint: ["crossdesk_action", "crossdesk_unlock", "crossdesk_clipboard_set", - "crossdesk_files_transfer", "crossdesk_files_cancel", "crossdesk_system_command"].includes(d.name), + "crossdesk_files_transfer", "crossdesk_files_cancel", "crossdesk_system_command", "crossdesk_drag_across"].includes(d.name), openWorldHint: true, }, ...(d.ui || d.app @@ -254,6 +259,29 @@ export function createServer( }, { readOnly: true }, ); + add("crossdesk_list_sessions", z.strictObject({}), + "List the active remote sessions with their device aliases, displays and control state. Each session_id addresses one connection. Up to eight connections can be active concurrently.", + async () => output(await sessions.listSessions()), { readOnly: true }); + add("crossdesk_capture_displays", z.strictObject({ session_id: id, + display_indices: z.array(z.int().min(0).max(63)).min(1).max(8) + .refine((values) => new Set(values).size === values.length).optional(), + max_edge: z.int().min(64).max(2048).default(1024) }), + "Capture up to eight remote displays in sequence and restore the selected display. Returns all images and a capture_id valid for 30 seconds. Use capture_id for cross-display dragging. For ordinary actions select a display and capture a fresh observation.", + async (p, ctx) => output(await sessions.captureDisplays(p, ctx.mcpReq.signal))); + add("crossdesk_drag_across", z.strictObject({ session_id: id, operation_id: id, capture_id: id, + path: z.array(z.strictObject({ display_index: z.int().min(0).max(63), + x: z.number().finite().min(0).max(8192), y: z.number().finite().min(0).max(8192) })).min(2).max(32), + button: z.enum(["left", "right", "middle"]).default("left"), + duration_ms: z.int().min(100).max(10000).default(500) }), + "Drag across displays using coordinates in the matching images from crossdesk_capture_displays. Keeps the mouse button held while switching displays. Always releases on interruption. Reuse the same operation_id to query an uncertain result. Display switching and network latency add to duration_ms.", + async (p, ctx) => output(await sessions.dragAcross(p, ctx.mcpReq.signal))); + add("crossdesk_video_settings", z.strictObject({ session_id: id, + fps: z.union([z.literal(1), z.literal(8), z.literal(15), z.literal(30), z.literal(60)]).optional(), + width: z.int().min(2).max(3840).refine((value) => value % 2 === 0).optional(), + height: z.int().min(2).max(2160).refine((value) => value % 2 === 0).optional(), + quality: z.int().min(0).max(2).optional(), max_bitrate_bps: z.int().min(128000).max(100000000).optional() }), + "Adjust this connection's remote preview frame rate, resolution, quality and bitrate at runtime. Returns the remote preview acceptance status or wait_timed_out. Release held input before changing these settings.", + async (p, ctx) => output(await sessions.videoSettings(p, ctx.mcpReq.signal))); add( "crossdesk_screenshot", screenshot, @@ -449,7 +477,7 @@ export function createServer( annotations: { readOnlyHint: !!d.readOnly, destructiveHint: ["crossdesk_action", "crossdesk_unlock", "crossdesk_clipboard_set", - "crossdesk_files_transfer", "crossdesk_files_cancel", "crossdesk_system_command"].includes(d.name), + "crossdesk_files_transfer", "crossdesk_files_cancel", "crossdesk_system_command", "crossdesk_drag_across"].includes(d.name), openWorldHint: true, }, ...((d.ui || d.app) && apps(ctx) diff --git a/apps/mcp/src/sessions.ts b/apps/mcp/src/sessions.ts index 6a5b43bb..51ebed0d 100644 --- a/apps/mcp/src/sessions.ts +++ b/apps/mcp/src/sessions.ts @@ -1,4 +1,5 @@ import { join } from "node:path"; +import { mkdir } from "node:fs/promises"; import { setTimeout as delay } from "node:timers/promises"; import { Failure, @@ -56,6 +57,8 @@ interface Session { status: Data; release: () => Promise; connecting: boolean; + connection: AbortController; + displayBatch?: boolean; statusRequest?: Promise; closing?: Promise; unlock?: UnlockRequest; @@ -80,8 +83,11 @@ interface UnlockRequest { export class Sessions { readonly views = new Map(); private current?: Session; - private connecting?: AbortController; - private connectionDone?: Promise; + private active = new Map(); + private captures = new Map(); + private drags = new Map(); + private connecting = new Set(); + private connectionDone = new Set>(); private closing = false; private binary?: Promise; constructor( @@ -103,6 +109,8 @@ export class Sessions { this.views.delete(id); } if (this.views.size >= 32) throw new Failure("VIEW_LIMIT"); + const selected = alias ? [...this.active.values()].find((s) => s.device.alias === alias && !s.closing) + : this.current && !this.current.closing ? this.current : [...this.active.values()].find((s) => !s.closing); const view: View = { id: token(), generation: 1, @@ -111,8 +119,8 @@ export class Sessions { created: Date.now(), lastSeen: Date.now(), ready: false, - alias: alias ?? this.current?.device.alias, - session: this.current?.id, + alias: alias ?? selected?.device.alias, + session: selected?.id, }; this.views.set(view.id, view); return view; @@ -152,9 +160,9 @@ export class Sessions { }; } private session(id: string): Session { - if (!this.current || this.current.id !== id) - throw new Failure("SESSION_NOT_FOUND"); - return this.current; + const session = this.active.get(id); + if (!session) throw new Failure("SESSION_NOT_FOUND"); + return session; } async status(sessionId?: string, viewId?: string, signal?: AbortSignal): Promise { const view = viewId ? this.view(viewId) : undefined; @@ -184,7 +192,7 @@ export class Sessions { await this.waitStatus(s.statusRequest, signal); } // The user may have disconnected or replaced the session during the read. - if (this.current !== s || (view && view.session !== id)) + if (this.active.get(s.id) !== s || (view && view.session !== id)) return view ? this.describe(view) : { session_id: id, state: "closed", control: false }; if (["closed", "failed"].includes(s.status.state)) { await this.dispose(s); @@ -331,21 +339,29 @@ export class Sessions { signal?: AbortSignal, ): Promise { if (this.closing) throw new Failure("SESSION_CLOSED"); - if (this.connecting) throw new Failure("BUSY"); - // Reserve the single connection slot before the first await. - const controller = this.connecting = new AbortController(); + const established = [...this.active.values()].filter((s) => !s.connecting && !s.closing).length; + if (established + this.connecting.size >= 8) throw new Failure("SESSION_LIMIT"); + const controller = new AbortController(); + this.connecting.add(controller); signal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal; let complete!: () => void; - this.connectionDone = new Promise((resolve) => { complete = resolve; }); + const done = new Promise((resolve) => { complete = resolve; }); + this.connectionDone.add(done); let release: (() => Promise) | undefined; let native: NativeClient | undefined; let session: Session | undefined; try { - if (this.current && !["closed", "failed", "disconnected"].includes( - (await this.status(this.current.id, undefined, signal)).state, - )) throw new Failure("SESSION_LIMIT"); - if (this.current) await this.disconnect(this.current.id); + for (const s of this.active.values()) { + if (s.closing) { + this.active.delete(s.id); + for (const v of this.views.values()) if (v.session === s.id) { + v.session = undefined; + v.disconnected = true; + } + } + } if (view) { + await this.revokeVideo(view); view.disconnected = false; view.connectionError = undefined; } @@ -358,23 +374,28 @@ export class Sessions { }); const binary = await this.binary; if (this.closing || signal?.aborted) throw new Failure("CANCELLED"); + const sessionId = token(); + const logs = join(this.store.directory, "logs", sessionId); + await mkdir(logs, { recursive: true, mode: 0o700 }); native = new NativeClient( binary, d.server, d.port, - join(this.store.directory, "logs"), + logs, this.videoOptions ?? (await this.store.config()).video, ); const s: Session = session = { - id: token(), + id: sessionId, device: d, native, status: {}, release, connecting: true, + connection: controller, unlocks: new Map(), }; this.current = s; + this.active.set(s.id, s); if (view) { view.session = s.id; view.alias = d.alias; @@ -392,7 +413,7 @@ export class Sessions { while (Date.now() < deadline) { if (this.closing || signal?.aborted) throw new Failure("CANCELLED"); s.status = await native.request("status", {}, 5000, signal); - if (this.current !== s || signal.aborted) throw new Failure("CANCELLED"); + if (this.active.get(s.id) !== s || signal.aborted) throw new Failure("CANCELLED"); if (s.status.state === "ready" && typeof s.status.frame_age_ms === "number" && s.status.frame_age_ms <= 2000) { s.connecting = false; return { ...s.status, session_id: s.id, device_alias: s.device.alias }; @@ -406,12 +427,12 @@ export class Sessions { } throw new Failure("CONNECTION_TIMEOUT"); } catch (error) { - if (session) await this.dispose(session); + if (session) { await this.dispose(session); this.active.delete(session.id); } else { await native?.close(); await release?.(); } - if (this.current?.native === native) this.current = undefined; + if (this.current?.native === native) this.current = [...this.active.values()].find((s) => !s.closing); const reason = signal.aborted ? new Failure("CANCELLED") : error; if (view && (!view.session || view.session === session?.id)) { view.session = undefined; @@ -420,13 +441,14 @@ export class Sessions { throw reason; } finally { password = ""; - this.connecting = undefined; - this.connectionDone = undefined; + this.connecting.delete(controller); + this.connectionDone.delete(done); complete(); } } async screenshot(p: Data, signal?: AbortSignal): Promise { const { session_id, ...args } = p; + if (this.session(session_id).displayBatch) throw new Failure("BUSY"); if (args.after_frame_id !== undefined && args.generation === undefined) throw new Failure("GENERATION_REQUIRED"); return this.session(session_id).native.request( @@ -439,6 +461,7 @@ export class Sessions { async action(p: Data, signal?: AbortSignal): Promise { const { session_id, observe_after = true, ...args } = p; const s = this.session(session_id); + if (s.displayBatch) throw new Failure("BUSY"); this.checkUnlock(s); if (s.unlock && this.unlockPending(s.unlock)) throw new Failure("UNLOCK_PENDING"); const result = await s.native.request("action", args, (args.action.duration_ms ?? 0) + 5000, signal); @@ -467,6 +490,123 @@ export class Sessions { if (result.matched) result.observation = await this.screenshot({ session_id }, signal); return result; } + async captureDisplays(p: Data, signal?: AbortSignal): Promise { + const s = this.session(p.session_id); + if (s.displayBatch) throw new Failure("BUSY"); + const status = await this.status(s.id, undefined, signal); + if (s.displayBatch) throw new Failure("BUSY"); + if (s.unlock && this.unlockPending(s.unlock)) throw new Failure("UNLOCK_PENDING"); + if (status.held_key_codes?.length || status.held_mouse_flags?.length) throw new Failure("INPUT_HELD"); + const indices: number[] = p.display_indices ?? status.displays.map((display: Data) => display.index); + if (!indices.length || indices.length > 8) throw new Failure("DISPLAY_LIMIT"); + if (indices.some((index) => !status.displays.some((display: Data) => display.index === index))) + throw new Failure("DISPLAY_UNAVAILABLE"); + const native = s.native; + const created = Date.now(); + if (!["ready", "waiting_frame"].includes(status.state)) throw new Failure("NOT_READY"); + s.displayBatch = true; + const observations: Data[] = []; + let error: unknown; + try { + for (const index of indices) { + await native.request("display", { index }, 5000, signal); + const observation = await native.request("screenshot", { max_edge: p.max_edge ?? 1024 }, 5000, signal); + observations.push({ ...observation, actionable: false }); + } + } catch (failure) { error = failure; } + finally { + try { + await native.request("display", { index: status.display }); + await native.request("screenshot", { max_edge: 64 }); + } catch (restoreError) { error ??= restoreError; } + s.displayBatch = false; + } + if (error) throw error; + for (const [key, capture] of this.captures) if (Date.now() - capture.created > 30000) this.captures.delete(key); + if (this.captures.size >= 32) this.captures.delete(this.captures.keys().next().value!); + const capture_id = token(); + this.captures.set(capture_id, { session: s.id, native, created, displays: observations.map((o) => o.metadata) }); + return { session_id: s.id, capture_id, observations, restored_display: status.display }; + } + async dragAcross(p: Data, signal?: AbortSignal): Promise { + const s = this.session(p.session_id); + const key = `${s.id}:${p.operation_id}`; + const args = JSON.stringify({ capture_id: p.capture_id, path: p.path, button: p.button ?? "left", duration_ms: p.duration_ms ?? 500 }); + const previous = this.drags.get(key); + if (previous) { + if (previous.arguments !== args) throw new Failure("OPERATION_ID_CONFLICT"); + return previous.result; + } + const capture = this.captures.get(p.capture_id); + if (!capture || capture.session !== s.id || capture.native !== s.native || Date.now() - capture.created > 30000) + throw new Failure("STALE_OBSERVATION"); + const sources = new Map(capture.displays.map((display) => [display.display_index, display])); + for (const point of p.path) { + const source = sources.get(point.display_index); + if (!source || point.x < 0 || point.y < 0 || point.x >= source.image_size.width || point.y >= source.image_size.height) + throw new Failure("INVALID_COORDINATE"); + } + if (s.displayBatch) throw new Failure("BUSY"); + if (s.unlock && this.unlockPending(s.unlock)) throw new Failure("UNLOCK_PENDING"); + s.displayBatch = true; + const native = s.native; + const record = { arguments: args, result: { operation_id: p.operation_id, status: "in_progress" } as Data }; + if (this.drags.size >= 128) this.drags.delete(this.drags.keys().next().value!); + this.drags.set(key, record); + let submitted = false; + const deadline = Date.now() + 30000; + try { + const status = await native.request("status", {}, 5000, signal); + if (!status.control || status.human_paused) throw new Failure("CONTROL_DENIED"); + if (status.held_key_codes?.length || status.held_mouse_flags?.length) throw new Failure("INPUT_HELD"); + let display = -1, observation: Data = {}; + const submit = async (type: string, point: Data) => { + if (Date.now() > deadline) throw new Failure("ACTION_TIMEOUT", submitted); + if (point.display_index !== display) { + await native.request("display", { index: point.display_index, preserve_input: submitted }, 5000, signal); + observation = await native.request("screenshot", { max_edge: 1024 }, 5000, signal); + display = point.display_index; + } + const source = sources.get(display)!; + const image = observation.metadata.image_size; + const action = { type, x: point.x * image.width / source.image_size.width, + y: point.y * image.height / source.image_size.height, ...(type === "move" ? {} : { button: p.button ?? "left" }) }; + submitted = true; + const result = await native.request("action", { operation_id: token(), + observation_id: observation.metadata.observation_id, action }, 5000, signal); + if (result.status !== "submitted") throw new Failure("ACTION_OUTCOME_UNKNOWN", true); + }; + await submit("mouse_down", p.path[0]); + for (const point of p.path.slice(1)) { + await delay((p.duration_ms ?? 500) / (p.path.length - 1), undefined, { signal }); + await submit("move", point); + } + await submit("mouse_up", p.path.at(-1)); + record.result = { operation_id: p.operation_id, status: "submitted", side_effect_possible: true, + observation: await native.request("screenshot", {}, 5000, signal) }; + } catch (error) { + record.result = { operation_id: p.operation_id, status: submitted ? "outcome_unknown" : "failed_before_submit", + error: failure(error), side_effect_possible: submitted }; + } finally { + await native.request("release_inputs").catch(() => {}); + s.displayBatch = false; + } + return record.result; + } + async videoSettings(p: Data, signal?: AbortSignal): Promise { + const { session_id, ...args } = p; + const s = this.session(session_id); + if (s.displayBatch) throw new Failure("BUSY"); + if (s.unlock && this.unlockPending(s.unlock)) throw new Failure("UNLOCK_PENDING"); + const requested = await s.native.request("video_settings", args, 5000, signal); + const deadline = Date.now() + 2500; + do { + const status = await this.status(s.id, undefined, signal); + if (status.preview_ack === requested.preview_request) return status; + await delay(100, undefined, { signal }); + } while (Date.now() < deadline); + return { ...requested, wait_timed_out: true }; + } async clipboard(p: Data, signal?: AbortSignal): Promise { const { session_id, ...args } = p; const s = this.session(session_id); @@ -520,13 +660,14 @@ export class Sessions { } async display(id: string, index: number) { const s = this.session(id); + if (s.displayBatch) throw new Failure("BUSY"); this.checkUnlock(s); if (s.unlock && this.unlockPending(s.unlock)) throw new Failure("UNLOCK_PENDING"); return s.native.request("display", { index }); } async disconnect(id: string) { const s = this.session(id); - if (s.connecting) this.connecting?.abort(); + if (s.connecting) s.connection.abort(); for (const v of this.views.values()) if (v.session === id) { v.session = undefined; @@ -534,7 +675,8 @@ export class Sessions { v.connectionError = undefined; } await this.dispose(s); - if (this.current === s) this.current = undefined; + this.active.delete(s.id); + if (this.current === s) this.current = [...this.active.values()].find((other) => !other.closing); return { session_id: id, state: "closed" }; } async viewerControl(v: View, operation: string) { @@ -584,6 +726,7 @@ export class Sessions { } async requestUnlock(p: Data, signal?: AbortSignal): Promise { const s = this.session(p.session_id); + if (s.displayBatch) throw new Failure("BUSY"); if (!s.native.unlockSupported) throw new Failure("NATIVE_UNLOCK_UNSUPPORTED"); const previous = s.unlocks.get(p.operation_id); if (previous) { @@ -705,7 +848,7 @@ export class Sessions { const deadline = Date.now() + waitMs; while (true) { const result = await this.status(sessionId, viewId, signal); - const s = result.session_id && this.current?.id === result.session_id ? this.current : undefined; + const s = this.active.get(result.session_id); if (!s || ["closed", "failed"].includes(result.state)) return result; const request = s.unlocks.get(requestId); if (!request) throw new Failure("UNLOCK_NOT_FOUND"); @@ -716,7 +859,7 @@ export class Sessions { } } async terminalFrame(v: View, args: Data, signal?: AbortSignal) { - if (v.panel !== "terminal" || !v.session || this.connecting) + if (v.panel !== "terminal" || !v.session) throw new Failure("NOT_READY"); const s = this.session(v.session); return s.native.request("terminal_frame", args, 5000, signal); @@ -724,14 +867,14 @@ export class Sessions { async openVideo(v: View, origin: string): Promise { if (!v.session || this.views.get(v.id) !== v) throw new Failure("NOT_READY"); const s = this.session(v.session); - if (this.closing || this.connecting || ["closed", "failed"].includes(s.status.state)) + if (this.closing || s.connecting || ["closed", "failed"].includes(s.status.state)) throw new Failure("NOT_READY"); const generation = v.generation, session = v.session; const ticket = token(); const result = await s.native.request("video_open", { ticket, view: v.id, view_generation: generation, session, origin, apps: v.panel === "apps", }); - if (this.current !== s || this.views.get(v.id) !== v || + if (this.active.get(s.id) !== s || this.views.get(v.id) !== v || v.generation !== generation || v.session !== session) { await s.native.request("video_revoke", { view: v.id }).catch(() => {}); throw new Failure("SESSION_CHANGED"); @@ -743,24 +886,26 @@ export class Sessions { throw new Failure("NATIVE_PROTOCOL_MISMATCH"); return { ...result, ticket }; } + async listSessions(): Promise { + const sessions = await Promise.all([...this.active.keys()].map((id) => this.status(id))); + return { sessions }; + } async revokeVideo(view?: View) { - const s = this.current; - if (!s || (view && view.session !== s.id)) return; - try { - await s.native.request("video_revoke", view ? { view: view.id } : {}); - } catch { - // Losing the control channel must not leave an authorized video socket - // alive after its view was removed or replaced. - if (this.current === s) await this.disconnect(s.id); - else await s.native.close(); - } + const targets = view ? [this.active.get(view.session ?? "")].filter((s): s is Session => !!s) + : [...this.active.values()]; + await Promise.all(targets.map(async (s) => { + try { await s.native.request("video_revoke", view ? { view: view.id } : {}); } + catch { if (this.active.get(s.id) === s) await this.disconnect(s.id); } + })); } async close() { this.closing = true; - this.connecting?.abort(); - const pending = this.connectionDone; - if (this.current) await this.disconnect(this.current.id); - await pending; + for (const controller of this.connecting) controller.abort(); + const pending = [...this.connectionDone]; + const results = await Promise.allSettled([...this.active.keys()].map((id) => this.disconnect(id))); + await Promise.allSettled(pending); this.views.clear(); + const error = results.find((result) => result.status === "rejected"); + if (error?.status === "rejected") throw error.reason; } } diff --git a/apps/mcp/ui/viewer.ts b/apps/mcp/ui/viewer.ts index 8c0dc384..dd728ec5 100644 --- a/apps/mcp/ui/viewer.ts +++ b/apps/mcp/ui/viewer.ts @@ -130,8 +130,8 @@ const messages: Record = { display_not_available: "所选显示器当前不可用，请检查显示器索引。", media_initialization_failed: "本机媒体组件初始化失败，请检查原生 CLI 运行环境。", control_event_overflow: "连接事件队列已满，请稍后重新连接。", - SESSION_LIMIT: "已有活动连接，请先断开当前电脑。", - DEVICE_BUSY: "另一实例正在使用此设备。", + SESSION_LIMIT: "活动连接已达到八个，请先断开一个连接。", + DEVICE_BUSY: "此设备已有活动连接。", CREDENTIAL_STORE_UNAVAILABLE: "系统凭据库不可用，密码未保存。", CREDENTIAL_ROLLBACK_FAILED: "凭据保存中断，请通过本机 setup 重新配置密码。", PAIRING_REQUIRED: @@ -710,7 +710,10 @@ async function start() { csrf = result.csrf; invoke = (name, args = {}) => request(name, { ...args, view_id: context!.view_id }); - await ready(bootstrap ? result : result.views.at(-1)); + const previous = sessionStorage.getItem("crossdesk-view"); + const selected = bootstrap ? result : result.views.find((view: Data) => view.view_id === previous) ?? result.views.at(-1); + await ready(selected); + sessionStorage.setItem("crossdesk-view", selected.view_id); } } void start().catch(showError); diff --git a/libs/controller/include/controller_session.h b/libs/controller/include/controller_session.h index c1788597..76850f4b 100644 --- a/libs/controller/include/controller_session.h +++ b/libs/controller/include/controller_session.h @@ -67,7 +67,7 @@ public: std::string password); void Tick(); void Disconnect(); - bool SelectDisplay(int index); + bool SelectDisplay(int index, bool preserve_input = false); void SetVideoOutput(bool enabled, bool request_keyframe = false); std::optional TakeEncodedVideo(); void SetPreview(int fps, int width, int height, int quality = -1, diff --git a/libs/controller/src/controller_session.cpp b/libs/controller/src/controller_session.cpp index 94c25e0b..2ca9af7d 100644 --- a/libs/controller/src/controller_session.cpp +++ b/libs/controller/src/controller_session.cpp @@ -383,10 +383,10 @@ void ControllerSession::Disconnect() { buttons_.clear(); state_ = "closed"; } -bool ControllerSession::SelectDisplay(int index) { +bool ControllerSession::SelectDisplay(int index, bool preserve_input) { if (!Connected() || index < 0 || size_t(index) >= displays_.size()) return false; - ReleaseInput(); + if (!preserve_input) ReleaseInput(); options_.display = index; { std::lock_guard lock(mutex_); @@ -401,7 +401,7 @@ bool ControllerSession::SelectDisplay(int index) { } void ControllerSession::SetPreview(int fps, int w, int h, int quality, int bitrate) { - if (fps < 0 || fps > 30 || w < 2 || h < 2 || w > 3840 || h > 2160 || + if (fps < 0 || fps > 60 || w < 2 || h < 2 || (w & 1) || (h & 1) || w > 3840 || h > 2160 || quality < -1 || quality > 2 || bitrate < -1 || bitrate > 100000000) throw std::invalid_argument("preview out of range"); quality = quality < 0 ? options_.quality : quality; @@ -430,7 +430,7 @@ void ControllerSession::Preview() { : 15; // 60 is the existing interactive-mode sentinel, not a 60 FPS request. // The encoder cadence is independently specified by frame_rate. - s.frame_rate = 30; + s.frame_rate = options_.fps > 30 ? 60 : 30; // Preserve desktop text detail at every cadence, including 30 FPS previews. s.preference = static_cast(VideoDegradationPreference::MaintainResolution); @@ -539,10 +539,14 @@ json ControllerSession::Status() const { {"capabilities", remote_capabilities_}, {"clipboard_ready", clipboard_ready_}, {"preview_supported", preview_supported_}, + {"preview_resolution_supported", resolution_}, + {"preview_bitrate_supported", bandwidth_}, + {"preview_fps15_supported", fps15_}, {"preview_request", preview_request_}, {"preview_ack", preview_ack_}, {"preview_accepted", preview_accepted_}, {"preview_width", options_.preview_width}, + {"preview_fps", options_.fps}, {"preview_height", options_.preview_height}, {"preview_quality", options_.quality}, {"preview_bitrate_bps", options_.max_bitrate_bps},