[feat] add batched actions and agent monitoring for MCP

apps/mcp/src/server.ts

@@ -22,6 +22,7 @@ import {
   alias,
   mode,
   action,
+  actionWait,
   screenshot,
   Failure,
   failure,
@@ -82,7 +83,7 @@ export function createServer(
     {
       capabilities: { tools: {}, resources: {} },
       instructions:
-        "Remote computer control. Open a panel for the user to select and authorize a device. Never request passwords in chat. In Codex, the default panel is the host built-in browser/sidebar for every user; no personal agent instructions are required. Other clients keep their local environment and system browser defaults. Explicit panel and launch settings override these defaults. Whenever browser_launch_url is returned, immediately open that local launcher using the host browser-open tool before waiting for authorization or continuing the remote task, even if the connection is already ready. Do not wait for the user to ask to open the sidebar. Do not read or quote its contents; it contains a one-time pairing credential. In Codex, use open_in_codex with target.type=browser, target.url=browser_launch_url, placement=right. If the host has no browser-open tool, explain the limitation and show pairing_command for the user to run locally; do not claim the sidebar is open. Reissue expired launchers with crossdesk_open_browser using the existing view_id, without reconnecting. When a remote task is pending, immediately follow crossdesk_open with crossdesk_get_session(view_id, wait_ms: 25000) in the same turn. Repeat bounded waits on wait_timed_out until ready or the user cancels; do not ask the user to type a connection confirmation or click Continue. On ready, continue the requested task within current authorization. When state is reconnecting, continue bounded waits for the explicitly enabled recovery policy. Use the returned session_id after recovery and inspect a fresh screenshot. On closed/failed, stop waiting and do not automatically reconnect. Obtain a screenshot before actions and pass its observation_id; coordinates refer to that returned image. If the remote system is locked, focus its visible password field, take a fresh screenshot and call crossdesk_unlock. A saved system password is used automatically; otherwise the local panel asks the user. Wait with crossdesk_get_session(unlock_request_id, wait_ms: 25000), then verify a fresh screenshot before continuing the original task. Submitted actions are not remote execution acknowledgements. Never replay actions after uncertain outcomes. Human pause can only be resumed in the user panel.",
+        "Remote computer control. Open a panel for the user to select and authorize a device. Never request passwords in chat. In Codex, the default panel is the host built-in browser/sidebar for every user; no personal agent instructions are required. Other clients keep their local environment and system browser defaults. Explicit panel and launch settings override these defaults. Whenever browser_launch_url is returned, immediately open that local launcher using the host browser-open tool before waiting for authorization or continuing the remote task, even if the connection is already ready. Do not wait for the user to ask to open the sidebar. Do not read or quote its contents; it contains a one-time pairing credential. In Codex, use open_in_codex with target.type=browser, target.url=browser_launch_url, placement=right. If the host has no browser-open tool, explain the limitation and show pairing_command for the user to run locally; do not claim the sidebar is open. Reissue expired launchers with crossdesk_open_browser using the existing view_id, without reconnecting. When a remote task is pending, immediately follow crossdesk_open with crossdesk_get_session(view_id, wait_ms: 25000) in the same turn. Repeat bounded waits on wait_timed_out until ready or the user cancels; do not ask the user to type a connection confirmation or click Continue. On ready, continue the requested task within current authorization. When state is reconnecting, continue bounded waits for the explicitly enabled recovery policy. Use the returned session_id after recovery and inspect a fresh screenshot. On closed/failed, stop waiting and do not automatically reconnect. Reuse the active session and its advertised automation capabilities. Prefer desktop_action for supported URL/app/accessibility operations. Use a recent screenshot observation for coordinate input; reuse action results instead of taking duplicate screenshots. Group fixed inputs that need no intermediate visual decision into a short action sequence, and wait for a relevant visual region or UI condition in that same call. Coordinates refer to the observation image. If the remote system is locked, focus its visible password field, take a fresh screenshot and call crossdesk_unlock. A saved system password is used automatically; otherwise the local panel asks the user. Wait with crossdesk_get_session(unlock_request_id, wait_ms: 25000), then verify a fresh screenshot before continuing the original task. Submitted actions are not remote execution acknowledgements. Never replay actions after uncertain outcomes. Human pause can only be resumed in the user panel.",
     },
   );
   type Definition = {
@@ -354,8 +355,9 @@ export function createServer(
       observation_id: id,
       action,
       observe_after: z.boolean().default(true),
-    }),
-    "Submit one structured action against a recent observation. Reuse operation_id only to retrieve that same operation, never replay uncertain effects with a new ID. Human pause prevents control. A post-action screenshot is not an execution acknowledgement.",
+      wait: actionWait.optional(),
+    }).refine((p) => !p.wait || p.observe_after),
+    'Submit an action or a short fixed sequence against a recent observation. Example action: {"type":"sequence","actions":[{"type":"hotkey","keys":["META","L"]},{"type":"type_text","text":"https://www.bilibili.com"},{"type":"key_press","key":"ENTER"}]}. Sequences allow 1–8 actions, no nested sequences or held inputs; all coordinates use the initial observation. Use only when no intermediate visual decision is needed. Defaults to waiting up to 1 second for visual change and returning one final screenshot (move/down/up only capture a newer frame); pass wait for a cropped-region condition or native UI text, or observe_after:false for input only. An unmatched wait does not mean input failed; never replay it automatically. Reuse operation_id only to query the same operation. Human pause interrupts sequences. Receipts confirm input processing, not application success.',
     async (p, ctx) => output(await sessions.action(p, ctx.mcpReq.signal)),
   );
   add(