[feat] add CrossDesk MCP with remote desktop panels

2026-11-04 21:12:42 +08:00 · 2026-11-03 02:44:55 +08:00
parent 2fab61871e
commit f57b4cc4e3
40 changed files with 6325 additions and 28 deletions
@@ -0,0 +1,399 @@
+import { readFile } from "node:fs/promises";
+import { z } from "zod";
+import { McpServer } from "@modelcontextprotocol/server";
+import type {
+  CallToolResult,
+  ClientCapabilities,
+  Tool,
+  ServerContext,
+} from "@modelcontextprotocol/server";
+import {
+  getUiCapability,
+  registerAppResource,
+  registerAppTool,
+  RESOURCE_MIME_TYPE,
+} from "@modelcontextprotocol/ext-apps/server";
+import { Sessions, type View } from "./sessions.js";
+import { BrowserPanel } from "./browser.js";
+import { uiAction, uiSchemas, type UIAction } from "./ui-actions.js";
+import {
+  id,
+  alias,
+  mode,
+  action,
+  screenshot,
+  Failure,
+  failure,
+  parse,
+  token,
+  type Data,
+} from "./common.js";
+
+const resource = "ui://crossdesk/viewer.html";
+const uiNames: Record<UIAction, string> = {
+  ready: "crossdesk_ui_ready",
+  devices: "crossdesk_ui_list_devices",
+  status: "crossdesk_ui_get_session",
+  connect: "crossdesk_ui_connect",
+  video_open: "crossdesk_video_open",
+  control: "crossdesk_viewer_control",
+};
+function output(value: Data, meta?: Data): CallToolResult {
+  const data = { ...value };
+  const content: any[] = [];
+  if (data.mime_type === "image/png" && typeof data.data === "string") {
+    content.push({ type: "image", mimeType: data.mime_type, data: data.data });
+    delete data.data;
+  }
+  if (data.observation?.data) {
+    content.push({
+      type: "image",
+      mimeType: data.observation.mime_type,
+      data: data.observation.data,
+    });
+    data.observation = { ...data.observation };
+    delete data.observation.data;
+  }
+  content.unshift({ type: "text", text: JSON.stringify(data) });
+  return { content, structuredContent: data, ...(meta ? { _meta: meta } : {}) };
+}
+export function createServer(
+  sessions: Sessions,
+  browser: BrowserPanel,
+): McpServer {
+  const server = new McpServer(
+    { name: "crossdesk-mcp", version: "0.1.0" },
+    {
+      capabilities: { tools: {}, resources: {} },
+      instructions:
+        "Remote computer control. Open a panel for the user to select and authorize a device. Never request passwords in chat. When a remote task is pending, immediately follow crossdesk_open with crossdesk_get_session(view_id, wait_ms: 25000) in the same turn. Repeat bounded waits on wait_timed_out until ready or the user cancels; do not ask the user to type a connection confirmation or click Continue. On ready, continue the requested task within current authorization. On closed/failed, stop waiting and do not automatically reconnect. Obtain a screenshot before actions and pass its observation_id; coordinates refer to that returned image. Submitted actions are not remote execution acknowledgements. Never replay actions after uncertain outcomes. Human pause can only be resumed in the user panel.",
+    },
+  );
+  type Definition = {
+    name: string;
+    schema: z.ZodType;
+    description: string;
+    ui?: boolean;
+    app?: boolean;
+    readOnly?: boolean;
+    run: (p: Data, ctx: ServerContext) => Promise<CallToolResult>;
+  };
+  const definitions = new Map<string, Definition>();
+  const supported = (ctx: ServerContext) => {
+    const envelope = ctx.mcpReq.envelope as Data | undefined;
+    const caps = envelope
+      ? envelope["io.modelcontextprotocol/clientCapabilities"]
+      : server.server.getClientCapabilities();
+    return (
+      getUiCapability((caps ?? {}) as ClientCapabilities)?.mimeTypes?.includes(
+        RESOURCE_MIME_TYPE,
+      ) ?? false
+    );
+  };
+  const apps = (ctx: ServerContext) =>
+    ["auto", "apps"].includes(sessions.policy) && supported(ctx);
+  const viewResult = (v: View, result: Data) =>
+    output(
+      result,
+      v.panel === "apps"
+        ? {
+            "crossdesk/ui": {
+              view_id: v.id,
+              view_generation: v.generation,
+              ui_token: v.secret,
+            },
+          }
+        : undefined,
+    );
+  const add = (
+    name: string,
+    schema: z.ZodType,
+    description: string,
+    run: Definition["run"],
+    options: Partial<Definition> = {},
+  ) => {
+    const d = { name, schema, description, run, ...options };
+    definitions.set(name, d);
+    const config = {
+      description,
+      inputSchema: schema,
+      annotations: {
+        readOnlyHint: !!d.readOnly,
+        destructiveHint: d.name === "crossdesk_action",
+        openWorldHint: true,
+      },
+      ...(d.ui || d.app
+        ? {
+            _meta: {
+              ui: {
+                ...(d.ui ? { resourceUri: resource } : {}),
+                visibility: d.app
+                  ? ["app" as const]
+                  : ["model" as const, "app" as const],
+              },
+            },
+          }
+        : {}),
+    };
+    if (d.ui || d.app)
+      registerAppTool(
+        server,
+        name,
+        { ...config, _meta: config._meta! },
+        (p, ctx) => run(p as Data, ctx),
+      );
+    else server.registerTool(name, config, (p, ctx) => run(p as Data, ctx));
+  };
+  add(
+    "crossdesk_open",
+    z.strictObject({ device_alias: alias.optional() }),
+    "Open CrossDesk so the user can choose a device, enter its password, and authorize this session. Returns immediately so the panel can render. To continue a pending remote task, call crossdesk_get_session with the returned view_id and wait_ms: 25000; no chat confirmation is needed.",
+    async (p, ctx) => {
+      if (p.device_alias) await sessions.store.device(p.device_alias);
+      const v = sessions.create(
+        sessions.choose(supported(ctx)),
+        p.device_alias,
+      );
+      return viewResult(
+        v,
+        v.panel === "browser" ? await browser.open(v) : sessions.describe(v),
+      );
+    },
+    { ui: true },
+  );
+  add(
+    "crossdesk_open_browser",
+    z.strictObject({ view_id: id.optional() }),
+    "Explicitly open the local browser panel after the user requests browser fallback. Does not create another remote connection.",
+    async (p) => {
+      if (!["auto", "browser"].includes(sessions.policy))
+        throw new Failure("PANEL_POLICY_DENIED");
+      const v = p.view_id
+        ? sessions.view(p.view_id)
+        : sessions.create("browser");
+      if (v.panel !== "browser") {
+        await sessions.revokeVideo(v);
+        v.panel = "browser";
+        v.generation++;
+        v.secret = token();
+        v.ready = false;
+        v.created = Date.now();
+      }
+      return output(await browser.open(v));
+    },
+  );
+  add(
+    "crossdesk_list_devices",
+    z.strictObject({}),
+    "List the local saved device book and persistent authorization, without credentials. This is not online discovery.",
+    async () => output({ devices: await sessions.store.list() }),
+    { readOnly: true },
+  );
+  add(
+    "crossdesk_connect",
+    z.strictObject({
+      device_alias: alias,
+      mode: mode.default("observe"),
+      display_index: z.int().min(0).max(63).default(0),
+    }),
+    "Connect only a saved, previously authorized device with available credentials. AUTH_REQUIRED means the user must authorize through the panel or setup.",
+    async (p, ctx) => {
+      const selected = sessions.choose(supported(ctx));
+      const result = await sessions.connectSaved(
+        p.device_alias,
+        p.mode,
+        p.display_index,
+        ctx.mcpReq.signal,
+      );
+      const v = sessions.create(selected, p.device_alias);
+      v.session = result.session_id;
+      return viewResult(v, {
+        ...result,
+        ...(selected === "browser"
+          ? await browser.open(v)
+          : sessions.describe(v)),
+        state: result.state,
+      });
+    },
+    { ui: true },
+  );
+  add(
+    "crossdesk_get_session",
+    z.strictObject({
+      session_id: id.optional(),
+      view_id: id.optional(),
+      wait_ms: z.int().min(0).max(25000).optional(),
+    }),
+    "Get connection, display geometry, frame age and control state. Supply exactly one of session_id or view_id. With view_id, wait for connection (default 25000 ms; 0 returns immediately). On wait_timed_out, repeat while the remote task is pending. On ready, continue without asking the user to send a message. Waiting never connects or grants control. session_id returns immediately and requires wait_ms to be 0 or omitted.",
+    async (p, ctx) => {
+      if (!!p.session_id === !!p.view_id)
+        throw new Failure("SELECT_ONE_SESSION");
+      if (p.session_id && p.wait_ms) throw new Failure("INVALID_ARGUMENT");
+      return output(p.view_id
+        ? await sessions.waitForConnection(p.view_id, p.wait_ms ?? 25000, ctx.mcpReq.signal)
+        : await sessions.status(p.session_id, undefined, ctx.mcpReq.signal));
+    },
+    { readOnly: true },
+  );
+  add(
+    "crossdesk_screenshot",
+    screenshot,
+    "Capture the remote display as a PNG image and immutable observation metadata. Coordinates in subsequent actions use this returned image. after_frame_id requires generation.",
+    async (p, ctx) => output(await sessions.screenshot(p, ctx.mcpReq.signal)),
+    { readOnly: true },
+  );
+  add(
+    "crossdesk_action",
+    z.strictObject({
+      session_id: id,
+      operation_id: id,
+      observation_id: id,
+      action,
+      observe_after: z.boolean().default(true),
+    }),
+    "Submit one structured action against a recent observation. Reuse operation_id only to retrieve that same operation, never replay uncertain effects with a new ID. Human pause prevents control. A post-action screenshot is not an execution acknowledgement.",
+    async (p, ctx) => output(await sessions.action(p, ctx.mcpReq.signal)),
+  );
+  add(
+    "crossdesk_set_control",
+    z.strictObject({ session_id: id, enabled: z.boolean() }),
+    "Enable or disable control within the user-authorized scope. Cannot override human pause or upgrade an observe-only connection.",
+    async (p) => output(await sessions.control(p.session_id, p.enabled)),
+  );
+  add(
+    "crossdesk_select_display",
+    z.strictObject({ session_id: id, display_index: z.int().min(0).max(63) }),
+    "Select a remote display. Invalidates previous observations; capture a new screenshot before any further input.",
+    async (p) => output(await sessions.display(p.session_id, p.display_index)),
+  );
+  add(
+    "crossdesk_disconnect",
+    z.strictObject({ session_id: id }),
+    "Disconnect the remote session, release input state and stop associated previews.",
+    async (p) => output(await sessions.disconnect(p.session_id)),
+  );
+  for (const [name, schema] of Object.entries(uiSchemas)) {
+    add(
+      uiNames[name as UIAction],
+      schema.extend({
+        view_id: id,
+        view_generation: z.int().min(1),
+        ui_token: id,
+      }),
+      `CrossDesk panel only: ${name}.`,
+      async (p, ctx) => {
+        if (!apps(ctx)) throw new Failure("PANEL_UNSUPPORTED");
+        const { view_id, view_generation, ui_token, ...args } = p;
+        const v = sessions.authenticate(view_id, view_generation, ui_token);
+        if (v.panel !== "apps") throw new Failure("VIEW_UNAUTHORIZED");
+        // Video grants stay in the authenticated app-only response.
+        const result = name === "video_open" ? await browser.video(v) : await uiAction(
+          sessions,
+          v,
+          name,
+          args,
+          ctx.mcpReq.signal,
+        );
+        return {
+          content: [
+            {
+              type: "text",
+              text: name === "video_open" ? "Video viewer connection" : JSON.stringify(result),
+            },
+          ],
+          structuredContent: result,
+        };
+      },
+      { app: true },
+    );
+  }
+  registerAppResource(
+    server,
+    "CrossDesk viewer",
+    resource,
+    { mimeType: RESOURCE_MIME_TYPE },
+    async (uri, ctx) => {
+      if (!apps(ctx)) throw new Failure("PANEL_UNSUPPORTED");
+      return {
+        contents: [
+          {
+            uri: uri.href,
+            mimeType: RESOURCE_MIME_TYPE,
+            text: await readFile(
+              new URL("./viewer-apps.html", import.meta.url),
+              "utf8",
+            ),
+            _meta: {
+              ui: {
+                prefersBorder: true,
+                csp: { connectDomains: await browser.connectDomains(), resourceDomains: [] },
+              },
+            },
+          },
+        ],
+      };
+    },
+  );
+  // Request-scoped projection avoids global tool mutations when modern clients
+  // send concurrent calls with different capability envelopes.
+  server.server.setRequestHandler("tools/list", async (_req, ctx) => ({
+    tools: [...definitions.values()]
+      .filter((d) => !d.app || apps(ctx))
+      .map((d) => {
+        const inputSchema = z.toJSONSchema(d.schema, {
+          io: "input",
+        }) as Tool["inputSchema"];
+        if (d.name === "crossdesk_get_session")
+          inputSchema.oneOf = [
+            { required: ["session_id"] },
+            { required: ["view_id"] },
+          ];
+        return {
+          name: d.name,
+          description: d.description,
+          inputSchema,
+          annotations: {
+            readOnlyHint: !!d.readOnly,
+            destructiveHint: d.name === "crossdesk_action",
+            openWorldHint: true,
+          },
+          ...((d.ui || d.app) && apps(ctx)
+            ? {
+                _meta: {
+                  ui: {
+                    ...(d.ui ? { resourceUri: resource } : {}),
+                    visibility: d.app ? ["app"] : ["model", "app"],
+                  },
+                  ...(d.ui ? { "ui/resourceUri": resource } : {}),
+                },
+              }
+            : {}),
+        };
+      }),
+  }));
+  server.server.setRequestHandler("tools/call", async (req, ctx) => {
+    try {
+      const d = definitions.get(req.params.name);
+      if (!d || (d.app && !apps(ctx))) throw new Failure("UNKNOWN_TOOL");
+      if (Buffer.byteLength(JSON.stringify(req.params.arguments ?? {})) > 65536)
+        throw new Failure("MESSAGE_TOO_LARGE");
+      const p = parse(d.schema, req.params.arguments ?? {}) as Data;
+      return await d.run(p, ctx);
+    } catch (error) {
+      const value = { error: failure(error) };
+      return { ...output(value), isError: true };
+    }
+  });
+  server.server.setRequestHandler("resources/list", async (_req, ctx) => ({
+    resources: apps(ctx)
+      ? [
+          {
+            name: "CrossDesk viewer",
+            uri: resource,
+            mimeType: RESOURCE_MIME_TYPE,
+          },
+        ]
+      : [],
+  }));
+  return server;
+}