[feat] add Codex sidebar video and MCP Apps preview fallback

apps/mcp/src/browser.ts

@@ -10,11 +10,11 @@ import {
   type Server as IPCServer,
 } from "node:net";
 import { spawn } from "node:child_process";
-import { readFile, writeFile, unlink, chmod, readdir } from "node:fs/promises";
+import { readFile, writeFile, unlink, chmod, readdir, mkdtemp, rm } from "node:fs/promises";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
-import { fileURLToPath } from "node:url";
-import { Failure, token, equal, failure, type Data } from "./common.js";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { Failure, token, equal, failure, type BrowserLaunch, type Data } from "./common.js";
 import { Sessions, type View } from "./sessions.js";
 import { uiAction } from "./ui-actions.js";
 import { displayEnvironment } from "./environment.js";
@@ -25,6 +25,13 @@ interface BrowserSession {
   views: Set<string>;
   expires: number;
 }
+interface BrowserGrant {
+  view: string;
+  generation: number;
+  expires: number;
+  directory?: string;
+  timer?: ReturnType<typeof setTimeout>;
+}
 export async function openURL(url: string): Promise<boolean> {
   if (!displayEnvironment().gui) return false;
   const command =
@@ -64,10 +71,10 @@ export class BrowserPanel {
   private descriptor?: string;
   private origin = "";
   private cookie = "crossdesk_" + token();
-  private grants = new Map<string, { view: string; expires: number }>();
+  private grants = new Map<string, BrowserGrant>();
   private clients = new Map<string, BrowserSession>();
   private html = "";
-  constructor(private sessions: Sessions) {}
+  constructor(private sessions: Sessions, private launch: BrowserLaunch = "auto") {}
   async connectDomains() {
     await this.start();
     // Native processes own ephemeral video ports, including after reconnect.
@@ -79,15 +86,40 @@ export class BrowserPanel {
     if (this.closed) throw new Failure("PANEL_CLOSED");
     return this.sessions.openVideo(view, this.origin);
   }
-  async open(view: View): Promise<Data> {
+  async open(view: View, preferHost = false): Promise<Data> {
     if (this.closed) throw new Failure("PANEL_CLOSED");
     await this.start();
     if (this.closed) throw new Failure("PANEL_CLOSED");
-    const opened = await openURL(this.bootstrapURL(view));
+    const { key, grant, url } = this.bootstrap(view);
+    const launch = this.launch === "auto" ? (preferHost ? "host" : "system") : this.launch;
+    let launcher: string | undefined;
+    let opened = false;
+    try {
+      if (launch === "host") {
+        // The host opens a local document, never a model-visible bearer URL.
+        // Its browser gets its own cookie through the normal bootstrap flow.
+        grant.directory = await mkdtemp(join(tmpdir(), "crossdesk-mcp-launch-"));
+        const path = join(grant.directory, "open.html");
+        await writeFile(path,
+          `<!doctype html><html lang="zh-CN"><head><meta charset="utf-8"><meta name="referrer" content="no-referrer"><meta http-equiv="Content-Security-Policy" content="default-src 'none'; base-uri 'none'"><meta http-equiv="refresh" content="0;url=${url}"><title>CrossDesk MCP</title></head><body><p>正在打开 CrossDesk…</p><a href="${url}" rel="noreferrer">打开画面</a></body></html>`,
+          { mode: 0o600, flag: "wx" });
+        if (this.closed || this.grants.get(key) !== grant) throw new Failure("PANEL_CLOSED");
+        launcher = pathToFileURL(path).href;
+      } else opened = await openURL(url);
+    } catch (error) {
+      await this.removeGrant(key, grant);
+      throw error;
+    }
     return {
       ...this.sessions.describe(view),
       browser_opened: opened,
       browser_url: this.origin,
+      browser_launch: launch,
+      ...(launcher ? {
+        browser_launch_url: launcher,
+        browser_launch_expires_in_ms: Math.max(0, grant.expires - Date.now()),
+        next_step: "Open browser_launch_url in the host's built-in browser/sidebar using its browser-open tool (Codex: open_in_codex, target.type=browser, target.url=browser_launch_url, placement=right). Do not read or quote the launcher file: it contains a one-time pairing credential. If no host browser is available, show the pairing_command for the user to run locally. Then call crossdesk_get_session with view_id and wait_ms=25000. If the launcher expired, call crossdesk_open_browser with this view_id to issue another; do not reconnect the remote device.",
+      } : {}),
       pairing_command: {
         command: "crossdesk-mcp",
         args: [
@@ -101,14 +133,22 @@ export class BrowserPanel {
       },
     };
   }
-  private bootstrapURL(view: View): string {
+  private async removeGrant(key: string, grant: BrowserGrant) {
+    if (this.grants.get(key) === grant) this.grants.delete(key);
+    clearTimeout(grant.timer);
+    if (grant.directory) await rm(grant.directory, { recursive: true, force: true });
+  }
+  private bootstrap(view: View) {
     if (this.closed) throw new Failure("PANEL_CLOSED");
-    for (const [key, value] of this.grants)
-      if (value.expires < Date.now()) this.grants.delete(key);
+    if (view.panel !== "browser" || !["auto", "browser"].includes(this.sessions.policy))
+      throw new Failure("PANEL_POLICY_DENIED");
     if (this.grants.size >= 32) throw new Failure("VIEW_LIMIT");
-    const grant = token();
-    this.grants.set(grant, { view: view.id, expires: Date.now() + 60000 });
-    return `${this.origin}/#bootstrap=${grant}`;
+    const key = token();
+    const grant: BrowserGrant = { view: view.id, generation: view.generation, expires: Date.now() + 60000 };
+    this.grants.set(key, grant);
+    grant.timer = setTimeout(() => void this.removeGrant(key, grant).catch(() => {}), 60000);
+    grant.timer.unref();
+    return { key, grant, url: `${this.origin}/#bootstrap=${key}` };
   }
   private async start() {
     this.starting ??= (async () => {
@@ -217,9 +257,14 @@ export class BrowserPanel {
           typeof body.token === "string"
             ? this.grants.get(body.token)
             : undefined;
-        if (!grant || grant.expires < Date.now())
+        const view = grant ? this.sessions.views.get(grant.view) : undefined;
+        if (!grant || grant.expires < Date.now() || !view ||
+            view.panel !== "browser" || view.generation !== grant.generation)
+          throw new Failure("PAIRING_REQUIRED");
+        await this.removeGrant(body.token, grant);
+        if (this.closed || this.sessions.views.get(grant.view) !== view ||
+            view.panel !== "browser" || view.generation !== grant.generation)
           throw new Failure("PAIRING_REQUIRED");
-        this.grants.delete(body.token);
         for (const [id, client] of this.clients)
           if (client.expires < Date.now()) this.clients.delete(id);
         const existing = this.client(req);
@@ -242,7 +287,7 @@ export class BrowserPanel {
         );
         return this.respond(res, 200, {
           csrf: client.csrf,
-          ...this.sessions.describe(this.sessions.view(grant.view)),
+          ...this.sessions.describe(view),
         });
       }
       const client = this.client(req);
@@ -313,7 +358,7 @@ export class BrowserPanel {
             // Only the authenticated local setup process receives this URL.
             // It uses its own desktop environment to open the browser.
             socket.end(
-              JSON.stringify({ ok: true, url: this.bootstrapURL(v) }) + "\n",
+              JSON.stringify({ ok: true, url: this.bootstrap(v).url }) + "\n",
             );
           } catch {
             socket.end('{"ok":false}\n');
@@ -339,7 +384,7 @@ export class BrowserPanel {
   async close(final = true) {
     if (final) this.closed = true;
     await this.sessions.revokeVideo();
-    this.grants.clear();
+    await Promise.all([...this.grants].map(([key, grant]) => this.removeGrant(key, grant)));
     this.clients.clear();
     this.server?.closeAllConnections();
     if (this.server?.listening) this.server.close();

apps/mcp/src/common.ts

@@ -50,6 +50,8 @@ export const unlockPassword = z.string().min(1).max(256)
   .refine((value) => Buffer.byteLength(value) <= 256 && !/[\0\r\n\t]/.test(value));
 export const panel = z.enum(["auto", "apps", "browser", "terminal", "off"]);
 export type Panel = z.infer<typeof panel>;
+export const browserLaunch = z.enum(["auto", "system", "host"]);
+export type BrowserLaunch = z.infer<typeof browserLaunch>;
 export const videoOptions = z.strictObject({
   decode_path: z.enum(["browser", "minirtc"]).default("browser"),
   decoder: z.enum(["auto", "software"]).default("auto"),

apps/mcp/src/main.ts

@@ -14,6 +14,7 @@ import { TerminalPanel } from "./terminal.js";
 import { terminalView } from "./terminal-view.js";
 import {
   panel,
+  browserLaunch,
   parse,
   Failure,
   failure,
@@ -135,6 +136,7 @@ async function main() {
     options: {
       stdio: { type: "boolean" },
       panel: { type: "string" },
+      "browser-launch": { type: "string" },
       "video-path": { type: "string" },
       "video-decoder": { type: "string" },
       "config-dir": { type: "string" },
@@ -152,7 +154,7 @@ async function main() {
   }
   if (values.help || !positionals.length) {
     process.stdout.write(
-      "crossdesk-mcp serve --stdio [--panel auto|apps|browser|terminal|off] [--video-path browser|minirtc] [--video-decoder auto|software] [--config-dir directory]\ncrossdesk-mcp view --view VIEW_ID [--config-dir directory] [--cell-aspect 0.5]\ncrossdesk-mcp setup [--config-dir directory]\ncrossdesk-mcp setup --open-panel --view VIEW_ID [--config-dir directory]\nDevelopment: --native-binary /absolute/path/to/crossdesk-cli\n",
+      "crossdesk-mcp serve --stdio [--panel auto|apps|browser|terminal|off] [--browser-launch auto|host|system] [--video-path browser|minirtc] [--video-decoder auto|software] [--config-dir directory]\nDefaults: --panel=auto --browser-launch=auto (Codex: host sidebar browser; other clients: local environment/system browser)\nExplicit host sidebar: --panel=browser --browser-launch=host\nSystem browser: --panel=browser --browser-launch=system\ncrossdesk-mcp view --view VIEW_ID [--config-dir directory] [--cell-aspect 0.5]\ncrossdesk-mcp setup [--config-dir directory]\ncrossdesk-mcp setup --open-panel --view VIEW_ID [--config-dir directory]\nDevelopment: --native-binary /absolute/path/to/crossdesk-cli\n",
     );
     return;
   }
@@ -179,6 +181,11 @@ async function main() {
     panel,
     values.panel ?? config.panel,
   );
+  const launch = parse(browserLaunch, values["browser-launch"] ?? "auto");
+  // Other explicit panel policies do not use the browser launch default.
+  // auto resolves per client when a browser panel is opened.
+  if (values["browser-launch"] === "host" && !["auto", "browser"].includes(policy))
+    throw new Failure("BROWSER_LAUNCH_REQUIRES_BROWSER");
   const sessions = new Sessions(
     store,
     policy,
@@ -188,7 +195,7 @@ async function main() {
       decoder: values["video-decoder"] ?? config.video.decoder,
     }),
   );
-  const browser = new BrowserPanel(sessions);
+  const browser = new BrowserPanel(sessions, launch);
   const terminal = new TerminalPanel(sessions);
   const handle = serveStdio(() => createServer(sessions, browser, terminal), {
     transport: new StdioServerTransport(),

apps/mcp/src/server.ts

@@ -37,6 +37,7 @@ const uiNames: Record<UIAction, string> = {
   status: "crossdesk_ui_get_session",
   connect: "crossdesk_ui_connect",
   video_open: "crossdesk_video_open",
+  preview: "crossdesk_ui_preview",
   control: "crossdesk_viewer_control",
   unlock: "crossdesk_ui_unlock",
   recovery: "crossdesk_ui_recovery",
@@ -81,7 +82,7 @@ export function createServer(
     {
       capabilities: { tools: {}, resources: {} },
       instructions:
-        "Remote computer control. Open a panel for the user to select and authorize a device. Never request passwords in chat. When a remote task is pending, immediately follow crossdesk_open with crossdesk_get_session(view_id, wait_ms: 25000) in the same turn. Repeat bounded waits on wait_timed_out until ready or the user cancels; do not ask the user to type a connection confirmation or click Continue. On ready, continue the requested task within current authorization. When state is reconnecting, continue bounded waits for the explicitly enabled recovery policy. Use the returned session_id after recovery and inspect a fresh screenshot. On closed/failed, stop waiting and do not automatically reconnect. Obtain a screenshot before actions and pass its observation_id; coordinates refer to that returned image. If the remote system is locked, focus its visible password field, take a fresh screenshot and call crossdesk_unlock. A saved system password is used automatically; otherwise the local panel asks the user. Wait with crossdesk_get_session(unlock_request_id, wait_ms: 25000), then verify a fresh screenshot before continuing the original task. Submitted actions are not remote execution acknowledgements. Never replay actions after uncertain outcomes. Human pause can only be resumed in the user panel.",
+        "Remote computer control. Open a panel for the user to select and authorize a device. Never request passwords in chat. In Codex, the default panel is the host built-in browser/sidebar for every user; no personal agent instructions are required. Other clients keep their local environment and system browser defaults. Explicit panel and launch settings override these defaults. Whenever browser_launch_url is returned, immediately open that local launcher using the host browser-open tool before waiting for authorization or continuing the remote task, even if the connection is already ready. Do not wait for the user to ask to open the sidebar. Do not read or quote its contents; it contains a one-time pairing credential. In Codex, use open_in_codex with target.type=browser, target.url=browser_launch_url, placement=right. If the host has no browser-open tool, explain the limitation and show pairing_command for the user to run locally; do not claim the sidebar is open. Reissue expired launchers with crossdesk_open_browser using the existing view_id, without reconnecting. When a remote task is pending, immediately follow crossdesk_open with crossdesk_get_session(view_id, wait_ms: 25000) in the same turn. Repeat bounded waits on wait_timed_out until ready or the user cancels; do not ask the user to type a connection confirmation or click Continue. On ready, continue the requested task within current authorization. When state is reconnecting, continue bounded waits for the explicitly enabled recovery policy. Use the returned session_id after recovery and inspect a fresh screenshot. On closed/failed, stop waiting and do not automatically reconnect. Obtain a screenshot before actions and pass its observation_id; coordinates refer to that returned image. If the remote system is locked, focus its visible password field, take a fresh screenshot and call crossdesk_unlock. A saved system password is used automatically; otherwise the local panel asks the user. Wait with crossdesk_get_session(unlock_request_id, wait_ms: 25000), then verify a fresh screenshot before continuing the original task. Submitted actions are not remote execution acknowledgements. Never replay actions after uncertain outcomes. Human pause can only be resumed in the user panel.",
     },
   );
   type Definition = {
@@ -94,6 +95,14 @@ export function createServer(
     run: (p: Data, ctx: ServerContext) => Promise<CallToolResult>;
   };
   const definitions = new Map<string, Definition>();
+  const codex = (ctx: ServerContext) => {
+    const envelope = ctx.mcpReq.envelope as Data | undefined;
+    const info = envelope
+      ? envelope["io.modelcontextprotocol/clientInfo"]
+      : server.server.getClientVersion();
+    // Client identity selects a presentation default, never authorization.
+    return info?.name === "codex-mcp-client";
+  };
   const supported = (ctx: ServerContext) => {
     const envelope = ctx.mcpReq.envelope as Data | undefined;
     const caps = envelope
@@ -107,7 +116,7 @@ export function createServer(
   };
   const apps = (ctx: ServerContext) =>
     sessions.policy === "apps" && supported(ctx);
-  const openPanel = async (v: View) => v.panel === "browser" ? browser.open(v)
+  const openPanel = async (v: View, ctx: ServerContext) => v.panel === "browser" ? browser.open(v, codex(ctx))
     : v.panel === "terminal" ? terminal.open(v) : sessions.describe(v);
   const viewResult = (v: View, result: Data) =>
     output(
@@ -165,16 +174,16 @@ export function createServer(
   add(
     "crossdesk_open",
     z.strictObject({ device_alias: alias.optional() }),
-    "Open CrossDesk so the user can choose a device, enter its password, and authorize this session. GUI environments use a browser; headless terminal environments return terminal_command, which you must show to the user to run in a separate terminal. Never run the terminal viewer in a captured tool-output channel. Returns immediately. To continue a pending remote task, call crossdesk_get_session with the returned view_id and wait_ms: 25000; no chat confirmation is needed.",
+    "Open CrossDesk so the user can choose a device, enter its password, and authorize this session. In Codex, defaults to browser_launch_url: immediately open it in the host sidebar without reading the launcher file or asking the user to request the sidebar. Other clients default to the local environment and system browser; explicit settings take precedence. Headless terminal environments return terminal_command, which you must show to the user to run in a separate terminal. Never run the terminal viewer in a captured tool-output channel. Returns immediately. To continue a pending remote task, call crossdesk_get_session with the returned view_id and wait_ms: 25000; no chat confirmation is needed.",
     async (p, ctx) => {
       if (p.device_alias) await sessions.store.device(p.device_alias);
       const v = sessions.create(
-        sessions.choose(supported(ctx)),
+        sessions.choose(supported(ctx), codex(ctx)),
         p.device_alias,
       );
       return viewResult(
         v,
-        await openPanel(v),
+        await openPanel(v, ctx),
       );
     },
     { ui: true },
@@ -182,8 +191,8 @@ export function createServer(
   add(
     "crossdesk_open_browser",
     z.strictObject({ view_id: id.optional() }),
-    "Explicitly open the local browser panel after the user requests browser fallback. Does not create another remote connection.",
-    async (p) => {
+    "Open or reissue the local browser panel using the configured launch mode. In host mode (the Codex default), immediately open browser_launch_url in the host sidebar without reading the launcher file. Use the existing view_id to renew an expired launcher without creating another remote connection. Strict apps/off policies deny browser fallback.",
+    async (p, ctx) => {
       if (!["auto", "browser"].includes(sessions.policy))
         throw new Failure("PANEL_POLICY_DENIED");
       const v = p.view_id
@@ -197,7 +206,7 @@ export function createServer(
         v.ready = false;
         v.created = Date.now();
       }
-      return output(await browser.open(v));
+      return output(await browser.open(v, codex(ctx)));
     },
   );
   add(
@@ -214,9 +223,9 @@ export function createServer(
       mode: mode.default("observe"),
       display_index: z.int().min(0).max(63).default(0),
     }),
-    "Connect only a saved, previously authorized device with available credentials. AUTH_REQUIRED means the user must authorize through the panel or setup.",
+    "Connect only a saved, previously authorized device with available credentials. In Codex, defaults to browser_launch_url: immediately open it in the host sidebar even when the remote connection is already ready. Do not read the launcher file. AUTH_REQUIRED means the user must authorize through the panel or setup.",
     async (p, ctx) => {
-      const selected = sessions.choose(supported(ctx));
+      const selected = sessions.choose(supported(ctx), codex(ctx));
       // Reserve panel capacity before starting a connection. Once connected,
       // panel setup errors must not hide the live session from the caller.
       const v = sessions.create(selected, p.device_alias);
@@ -234,7 +243,7 @@ export function createServer(
       }
       v.session = result.session_id;
       let panelResult: Data;
-      try { panelResult = await openPanel(v); }
+      try { panelResult = await openPanel(v, ctx); }
       catch (error) { panelResult = { ...sessions.describe(v), panel_error: failure(error) }; }
       return viewResult(v, {
         ...result,
@@ -361,13 +370,13 @@ export function createServer(
     async (p, ctx) => {
       let v = [...sessions.views.values()].find((view) => view.session === p.session_id);
       const created = !v;
-      if (!v) v = sessions.create(sessions.choose(supported(ctx)));
+      if (!v) v = sessions.create(sessions.choose(supported(ctx), codex(ctx)));
       let result: Data;
       try { result = await sessions.requestUnlock(p, ctx.mcpReq.signal); }
       catch (error) { if (created) sessions.views.delete(v.id); throw error; }
       let panel = sessions.describe(v);
       try {
-        if (result.unlock.state === "awaiting_password" && !v.ready) panel = await openPanel(v);
+        if (result.unlock.state === "awaiting_password" && !v.ready) panel = await openPanel(v, ctx);
       } catch (error) { panel.panel_error = failure(error); }
       return viewResult(v, { ...panel, ...result, next_step: result.unlock.next_step });
     },
@@ -464,6 +473,11 @@ export function createServer(
           args,
           ctx.mcpReq.signal,
         );
+        if (name === "preview") return {
+          content: [{ type: "text", text: "Panel preview frame" }],
+          structuredContent: { transport: "mcp-preview/1" },
+          _meta: { "crossdesk/preview": result },
+        };
         return {
           content: [
             {
@@ -474,7 +488,7 @@ export function createServer(
           structuredContent: result,
         };
       },
-      { app: true },
+      { app: true, readOnly: name === "preview" },
     );
   }
   registerAppResource(

apps/mcp/src/sessions.ts

@@ -49,6 +49,8 @@ export interface View {
   session?: string;
   disconnected?: boolean;
   connectionError?: Data;
+  previewPending?: boolean;
+  previewAt?: number;
 }
 interface Session {
   id: string;
@@ -107,9 +109,10 @@ export class Sessions {
     private nativePath?: string,
     private videoOptions?: VideoOptions,
   ) {}
-  choose(apps: boolean): View["panel"] {
+  choose(apps: boolean, preferHostBrowser = false): View["panel"] {
     if (this.policy === "apps" && !apps) throw new Failure("PANEL_UNSUPPORTED");
     if (this.policy !== "auto") return this.policy;
+    if (preferHostBrowser) return "browser";
     const environment = displayEnvironment();
     return environment.gui ? "browser" : environment.terminal ? "terminal" : "off";
   }
@@ -1123,6 +1126,29 @@ export class Sessions {
       throw new Failure("NATIVE_PROTOCOL_MISMATCH");
     return { ...result, ticket };
   }
+  async preview(v: View, args: Data, signal?: AbortSignal): Promise<Data> {
+    if (v.panel !== "apps" || !v.session || this.views.get(v.id) !== v)
+      throw new Failure("NOT_READY");
+    const s = this.session(v.session);
+    if (this.closing || s.connecting || s.displayBatch || s.status.state !== "ready")
+      throw new Failure("NOT_READY");
+    if (args.after_frame_id !== undefined && args.generation === undefined)
+      throw new Failure("GENERATION_REQUIRED");
+    if (v.previewPending || Date.now() - (v.previewAt ?? 0) < 1000)
+      throw new Failure("BUSY", false, true);
+    const generation = v.generation, session = v.session;
+    v.previewPending = true;
+    v.previewAt = Date.now();
+    try {
+      // Native preview is capped at 960px / 256 KiB and does not consume the
+      // agent's observation history. Never send these frames through tools visible to the model.
+      const result = await s.native.request("preview", { ...args, max_edge: 960, timeout_ms: 1000 }, 4000, signal);
+      if (this.active.get(s.id) !== s || this.views.get(v.id) !== v ||
+          v.generation !== generation || v.session !== session || s.closing)
+        throw new Failure("SESSION_CHANGED");
+      return result;
+    } finally { v.previewPending = false; }
+  }
   async listSessions(): Promise<Data> {
     const sessions = await Promise.all([...this.active.keys()].map((id) => this.status(id)));
     return { sessions };

apps/mcp/src/ui-actions.ts

@@ -8,6 +8,10 @@ export const uiSchemas = {
   status: z.strictObject({}),
   connect: uiConnect,
   video_open: z.strictObject({}),
+  preview: z.strictObject({
+    generation: z.int().nonnegative().optional(),
+    after_frame_id: z.int().nonnegative().optional(),
+  }),
   input: viewerInput,
   audio: z.strictObject({ enabled: z.boolean() }),
   recovery: z.strictObject({ enabled: z.boolean(), max_attempts: z.int().min(1).max(5).default(3) }),
@@ -45,6 +49,8 @@ export async function uiAction(
     }
     case "status":
       return sessions.status(undefined, view.id, signal);
+    case "preview":
+      return sessions.preview(view, p, signal);
     case "connect":
       return sessions.connectUI(view, p, signal);
     case "audio":