[feat] receive remote audio with panel playback and bounded WAV capture

apps/cli/agent.cpp

@@ -86,6 +86,13 @@ struct RemoteJob {
   bool write;
   Clock::time_point deadline;
 };
+struct AudioCapture {
+  json id;
+  size_t target;
+  bool restore_enabled;
+  Clock::time_point deadline;
+  std::vector<uint8_t> pcm;
+};
 struct Capture {
   json id, request;
   bool preview = false;
@@ -314,11 +321,44 @@ private:
       std::fill(step.text.begin(), step.text.end(), '\0');
     action_.reset();
   }
+  void FinishAudio(const char *error = nullptr) {
+    if (!audio_capture_) return;
+    const auto capture = std::move(audio_capture_);
+    if (session_.Connected() && session_.AudioEnabled() != capture->restore_enabled)
+      session_.SetAudio(capture->restore_enabled);
+    if (error) { Error(capture->id, error); return; }
+    std::vector<uint8_t> wave;
+    const auto number = [&](uint32_t value, int count) {
+      for (int index = 0; index < count; ++index) wave.push_back(uint8_t(value >> (8 * index)));
+    };
+    const auto tag = [&](const char *value) { wave.insert(wave.end(), value, value + 4); };
+    tag("RIFF"); number(uint32_t(capture->pcm.size()) + 36, 4); tag("WAVE");
+    tag("fmt "); number(16, 4); number(1, 2); number(1, 2);
+    number(48000, 4); number(96000, 4); number(2, 2); number(16, 2);
+    tag("data"); number(uint32_t(capture->pcm.size()), 4);
+    wave.insert(wave.end(), capture->pcm.begin(), capture->pcm.end());
+    Reply(capture->id, {{"mime_type", "audio/wav"}, {"data", ImageBase64(wave)},
+      {"duration_ms", capture->pcm.size() / 96}, {"sample_rate", 48000}, {"channels", 1}});
+  }
+  void PumpAudio() {
+    while (auto frame = session_.TakeAudio()) {
+      if (audio_capture_) {
+        const auto size = std::min(frame->pcm.size(), audio_capture_->target - audio_capture_->pcm.size());
+        audio_capture_->pcm.insert(audio_capture_->pcm.end(), frame->pcm.begin(), frame->pcm.begin() + size);
+      }
+      if (video_ && video_->Enabled()) video_->Submit(std::move(*frame));
+    }
+    if (!audio_capture_) return;
+    if (!session_.Connected()) FinishAudio("SESSION_CLOSED");
+    else if (audio_capture_->pcm.size() >= audio_capture_->target) FinishAudio();
+    else if (Clock::now() >= audio_capture_->deadline) FinishAudio("AUDIO_UNAVAILABLE");
+  }
   void CancelRemoteJobs(const char *code) {
     for (const auto &job : remote_jobs_) FinishRemote(job, {}, code);
     remote_jobs_.clear();
   }
   void Stop(const char *code) {
+    FinishAudio(code);
     FinishAction(code);
     CancelRemoteJobs(code);
     for (const auto &capture : captures_)
@@ -339,6 +379,7 @@ private:
   }
   void Tick() {
     session_.Tick();
+    PumpAudio();
     if (!action_ && session_.HasPressedInput() && Clock::now() >= input_lease_)
       session_.ReleaseInput();
     for (auto it = remote_jobs_.begin(); it != remote_jobs_.end();) {
@@ -922,6 +963,20 @@ private:
         });
         terminal_encoding_ = std::move(job);
         return false;
+      } else if (method == "audio") {
+        Fields(j, {"id", "method", "enabled"});
+        FinishAudio("CANCELLED");
+        session_.SetAudio(Flag(j, "enabled"));
+      } else if (method == "audio_capture") {
+        Fields(j, {"id", "method", "duration_ms"});
+        if (!session_.Connected()) throw std::invalid_argument("NOT_CONNECTED");
+        if (audio_capture_) throw std::invalid_argument("BUSY");
+        const int duration = Number(j, "duration_ms", 100, 10000, 3000);
+        const bool restore = session_.AudioEnabled();
+        session_.SetAudio(true);
+        audio_capture_ = std::make_unique<AudioCapture>(AudioCapture{
+            id, size_t(duration) * 96, restore, Clock::now() + Ms(duration + 3000), {}});
+        return false;
       } else if (method == "control") {
         Fields(j, {"id", "method", "enabled"});
         if (human_control_) throw std::invalid_argument("HUMAN_CONTROL_ACTIVE");
@@ -1038,6 +1093,7 @@ private:
         if (!j.contains("request_id") || !j["request_id"].is_number_integer())
           throw std::invalid_argument("INVALID_REQUEST");
         const auto target = j["request_id"];
+        if (audio_capture_ && audio_capture_->id == target) FinishAudio("CANCELLED");
         for (auto it = remote_jobs_.begin(); it != remote_jobs_.end();) {
           if (it->id == target) {
             FinishRemote(*it, {}, "CANCELLED");
@@ -1121,6 +1177,7 @@ private:
   std::deque<Capture> captures_;
   std::deque<FrameWait> frame_waits_;
   std::deque<RemoteJob> remote_jobs_;
+  std::unique_ptr<AudioCapture> audio_capture_;
   std::unique_ptr<Encoding> encoding_;
   std::unique_ptr<TerminalEncoding> terminal_encoding_;
   std::unique_ptr<RunningAction> action_;

apps/cli/video_server.cpp

@@ -92,6 +92,8 @@ struct VideoServer::State {
   // each other; the compressed handoff is bounded and recovers at an IDR.
   mutable std::mutex mutex;
   std::deque<Packet> queue;
+  std::deque<AudioFrame> audio_queue;
+  size_t audio_bytes = 0;
   size_t queued_bytes = 0;
   bool scheduled = false, wait_key = true;
   json cursor;
@@ -257,6 +259,33 @@ struct VideoServer::State {
           if (error)
             return;
           const auto now = Clock::now();
+          std::deque<AudioFrame> audio;
+          {
+            std::lock_guard lock(mutex);
+            audio.swap(audio_queue);
+            audio_bytes = 0;
+          }
+          for (const auto &frame : audio) {
+            if (!connected || frame.generation != generation) continue;
+            std::string bytes;
+            Number(bytes, 0x43444131, 4); // CDA1, network-order header, s16le payload.
+            Number(bytes, frame.generation, 4);
+            Number(bytes, frame.sequence, 4);
+            Number(bytes, 48000, 4);
+            Number(bytes, 1, 2);
+            Number(bytes, 16, 2);
+            bytes.append(reinterpret_cast<const char *>(frame.pcm.data()), frame.pcm.size());
+            std::vector<Handle> failed;
+            for (auto &[handle, client] : clients) {
+              if (!client.authorized) continue;
+              websocketpp::lib::error_code error;
+              const auto connection = server.get_con_from_hdl(handle, error);
+              if (!error && connection->get_buffered_amount() < 256 * 1024)
+                server.send(handle, bytes, websocketpp::frame::opcode::BINARY, error);
+              if (error) failed.push_back(handle);
+            }
+            for (const auto handle : failed) Drop(handle);
+          }
           std::string pointer;
           uint64_t revision;
           {
@@ -429,6 +458,8 @@ void VideoServer::Bind(uint64_t generation, bool decoded, bool connected) {
     s.key_requested = false;
     std::lock_guard lock(s.mutex);
     s.queue.clear();
+    s.audio_queue.clear();
+    s.audio_bytes = 0;
     s.queued_bytes = 0;
     s.wait_key = true;
   });
@@ -488,6 +519,17 @@ void VideoServer::Submit(EncodedVideoFrame frame) {
   if (Enabled())
     state_->Enqueue({{}, std::move(frame)});
 }
+void VideoServer::Submit(AudioFrame frame) {
+  if (!Enabled() || frame.pcm.empty() || frame.pcm.size() > 11520 || (frame.pcm.size() & 1)) return;
+  auto &s = *state_;
+  std::lock_guard lock(s.mutex);
+  while (!s.audio_queue.empty() && s.audio_bytes + frame.pcm.size() > 48000) {
+    s.audio_bytes -= s.audio_queue.front().pcm.size();
+    s.audio_queue.pop_front();
+  }
+  s.audio_bytes += frame.pcm.size();
+  s.audio_queue.push_back(std::move(frame));
+}
 void VideoServer::Cursor(json cursor) {
   auto &s = *state_;
   std::lock_guard lock(s.mutex);

apps/cli/video_server.h

@@ -36,6 +36,7 @@ public:
   bool TakeKeyframeRequest();
   void Submit(std::shared_ptr<const DecodedFrame> frame);
   void Submit(EncodedVideoFrame frame);
+  void Submit(AudioFrame frame);
   void Cursor(nlohmann::json cursor);
   nlohmann::json Status() const;
 

apps/mcp/README.md

@@ -294,3 +294,9 @@ npm run pack:npm -- --out-dir /absolute/path/artifacts
 浏览器及 Apps 面板提供“人工接管”和“交还 Agent”。接管后面板可操作鼠标、滚轮、键盘快捷键和中文输入法，连接信息展示当前控制权。输入绑定面板、控制批次和画面尺寸，交接会释放按住状态并使旧批次失效。画面超过两秒未更新时停止发送新输入，窗口失焦或隐藏时释放按键。原生层保留十五秒的输入租约作为断线清理保障。
 
 Agent 在人工接管期间仍可观察画面，键鼠、显示设置和其他要求控制权的操作等待用户在面板交还控制。连接需预先授予控制权限，并使用支持 `events/1` 的原生 CLI。
+
+### 远端声音
+
+`crossdesk_audio` 开关远端桌面音频接收，默认关闭。面板的“播放声音”按钮同时启用接收和本地播放，音量在本面板调节。播放使用已授权的视频 WebSocket 通道，PCM 队列与浏览器播放时延均设有上限，拥塞时丢弃过时音频。浏览器首次播放需要用户点击按钮。
+
+`crossdesk_audio_capture` 采集接下来 100 至 10000 毫秒的音频，通过 MCP 音频内容块返回单声道 48 kHz WAV。采集期间临时开启接收，结束或取消后恢复之前的设置，未收到足够音频时返回 `AUDIO_UNAVAILABLE`。系统需允许被控端捕获桌面声音。自动恢复连接会恢复用户选择的音频接收设置。

apps/mcp/src/browser.ts

@@ -255,7 +255,7 @@ export class BrowserPanel {
       if (typeof body.view_id !== "string" || !client.views.has(body.view_id))
         throw new Failure("VIEW_UNAUTHORIZED");
       const name = req.url?.match(
-        /^\/api\/(ready|devices|status|connect|video_open|control|unlock|recovery|input)$/,
+        /^\/api\/(ready|devices|status|connect|video_open|control|unlock|recovery|input|audio)$/,
       )?.[1];
       if (!name) throw new Failure("UNKNOWN_ACTION");
       const { view_id, ...args } = body;

apps/mcp/src/server.ts

@@ -41,10 +41,15 @@ const uiNames: Record<UIAction, string> = {
   unlock: "crossdesk_ui_unlock",
   recovery: "crossdesk_ui_recovery",
   input: "crossdesk_ui_input",
+  audio: "crossdesk_ui_audio",
 };
 function output(value: Data, meta?: Data): CallToolResult {
   const data = { ...value };
   const content: any[] = [];
+  if (data.mime_type === "audio/wav" && typeof data.data === "string") {
+    content.push({ type: "audio", mimeType: data.mime_type, data: data.data });
+    delete data.data;
+  }
   if (data.mime_type === "image/png" && typeof data.data === "string") {
     content.push({ type: "image", mimeType: data.mime_type, data: data.data });
     delete data.data;
@@ -261,6 +266,12 @@ export function createServer(
     },
     { readOnly: true },
   );
+  add("crossdesk_audio", z.strictObject({ session_id: id, enabled: z.boolean() }),
+    "Enable or disable remote desktop audio reception. Audio starts disabled. Panel playback also requires the user's Play audio button. Status reports PCM reception counters.",
+    async (p, ctx) => output(await sessions.audio(p, false, ctx.mcpReq.signal)));
+  add("crossdesk_audio_capture", z.strictObject({ session_id: id, duration_ms: z.int().min(100).max(10000).default(3000) }),
+    "Capture the next 100–10000 ms of remote desktop audio as a bounded mono 48 kHz WAV audio content block. Temporarily enables reception, then restores the previous setting. AUDIO_UNAVAILABLE means insufficient PCM arrived. Cancellation discards the capture.",
+    async (p, ctx) => output(await sessions.audio(p, true, ctx.mcpReq.signal)), { readOnly: true });
   add("crossdesk_list_sessions", z.strictObject({}),
     "List the active remote sessions with their device aliases, displays and control state. Each session_id addresses one connection. Up to eight connections can be active concurrently.",
     async () => output(await sessions.listSessions()), { readOnly: true });

apps/mcp/src/sessions.ts

@@ -61,6 +61,7 @@ interface Session {
   displayBatch?: boolean;
   mode: string;
   desiredControl: boolean;
+  audioEnabled?: boolean;
   recoveryEnabled: boolean;
   recoveryAttempts: number;
   recoveryAttempt: number;
@@ -317,6 +318,10 @@ export class Sessions {
           if (next.status.preview_bitrate_supported) settings.max_bitrate_bps = s.status.preview_bitrate_bps;
           await next.native.request("video_settings", settings).catch(() => {});
         }
+        if (s.audioEnabled) {
+          await next.native.request("audio", { enabled: true });
+          next.audioEnabled = true;
+        }
         next.recoveryEnabled = true;
         next.recoveryAttempts = s.recoveryAttempts;
         next.recoveryAttempt = attempt;
@@ -756,6 +761,14 @@ export class Sessions {
     }
     return record.result;
   }
+  async audio(p: Data, capture: boolean, signal?: AbortSignal): Promise<Data> {
+    const { session_id, ...args } = p;
+    const s = this.session(session_id);
+    const result = await s.native.request(capture ? "audio_capture" : "audio", args,
+      capture ? (args.duration_ms ?? 3000) + 5000 : 5000, signal);
+    if (!capture) { s.status = result; s.audioEnabled = args.enabled; }
+    return result;
+  }
   async videoSettings(p: Data, signal?: AbortSignal): Promise<Data> {
     const { session_id, ...args } = p;
     const s = this.session(session_id);

apps/mcp/src/ui-actions.ts

@@ -9,6 +9,7 @@ export const uiSchemas = {
   connect: uiConnect,
   video_open: z.strictObject({}),
   input: viewerInput,
+  audio: z.strictObject({ enabled: z.boolean() }),
   recovery: z.strictObject({ enabled: z.boolean(), max_attempts: z.int().min(1).max(5).default(3) }),
   unlock: z.strictObject({
     request_id: id,
@@ -46,6 +47,9 @@ export async function uiAction(
       return sessions.status(undefined, view.id, signal);
     case "connect":
       return sessions.connectUI(view, p, signal);
+    case "audio":
+      if (!view.session) throw new Failure("NOT_READY");
+      return sessions.audio({ session_id: view.session, ...p }, false, signal);
     case "input":
       return sessions.viewerInput(view, p, signal);
     case "control":

apps/mcp/ui/audio-player.ts

@@ -0,0 +1,52 @@
+export class AudioPlayer {
+  private context?: AudioContext;
+  private gain?: GainNode;
+  private next = 0;
+  private sequence = -1;
+  private generation = -1;
+  private sources = new Set<AudioBufferSourceNode>();
+  enabled = false;
+  async enable(enabled: boolean) {
+    if (enabled) {
+      this.context ??= new AudioContext();
+      if (!this.gain) {
+        this.gain = this.context.createGain();
+        this.gain.connect(this.context.destination);
+      }
+      await this.context.resume();
+    }
+    this.enabled = enabled;
+    if (!enabled) this.flush();
+  }
+  volume(value: number) { if (this.gain) this.gain.gain.value = Math.max(0, Math.min(1, value)); }
+  flush() {
+    for (const source of this.sources) { source.stop(); source.disconnect(); }
+    this.sources.clear();
+    this.next = 0;
+    this.sequence = this.generation = -1;
+  }
+  accept(bytes: ArrayBuffer) {
+    if (bytes.byteLength < 22 || bytes.byteLength > 11540 || (bytes.byteLength & 1)) throw new Error("AUDIO_INVALID");
+    const data = new DataView(bytes);
+    if (data.getUint32(0) !== 0x43444131 || data.getUint32(12) !== 48000 ||
+        data.getUint16(16) !== 1 || data.getUint16(18) !== 16) throw new Error("AUDIO_INVALID");
+    if (!this.enabled || !this.context || !this.gain || this.context.state !== "running") return;
+    const generation = data.getUint32(4), sequence = data.getUint32(8);
+    if (generation < this.generation || (generation === this.generation && sequence <= this.sequence)) return;
+    const now = this.context.currentTime;
+    if (generation !== this.generation || this.next > now + .25 || sequence !== this.sequence + 1) this.flush();
+    this.generation = generation;
+    this.sequence = sequence;
+    const buffer = this.context.createBuffer(1, (bytes.byteLength - 20) / 2, 48000);
+    const samples = buffer.getChannelData(0);
+    for (let index = 0; index < samples.length; index++) samples[index] = data.getInt16(20 + index * 2, true) / 32768;
+    const source = this.context.createBufferSource();
+    source.buffer = buffer;
+    source.connect(this.gain);
+    source.onended = () => { source.disconnect(); this.sources.delete(source); };
+    this.sources.add(source);
+    this.next = Math.max(now + .03, this.next);
+    source.start(this.next);
+    this.next += buffer.duration;
+  }
+}

apps/mcp/ui/video-player.ts

@@ -51,7 +51,8 @@ export class VideoPlayer {
   constructor(private canvas: HTMLCanvasElement,
     private onFrame: (info: FrameInfo) => void,
     private onError: (error: Error) => void,
-    private overlay?: HTMLCanvasElement) {
+    private overlay?: HTMLCanvasElement,
+    private onAudio?: (bytes: ArrayBuffer) => void) {
     if (overlay) {
       this.resize = new ResizeObserver(() => this.drawCursor());
       this.resize.observe(overlay);
@@ -93,6 +94,10 @@ export class VideoPlayer {
       }
       if (!(event.data instanceof ArrayBuffer)) return;
       const data = event.data;
+      if (data.byteLength >= 4 && new DataView(data).getUint32(0) === 0x43444131) {
+        try { this.onAudio?.(data); } catch { this.onError(new Error("AUDIO_INVALID")); }
+        return;
+      }
       if (data.byteLength < (this.raw ? 38 : 25) ||
           data.byteLength > (this.raw ? 16777216 * 3 / 2 + 32 : 4 * 1024 * 1024 + 24))
         return this.fail(new Error("VIDEO_INVALID"));

apps/mcp/ui/viewer.css

@@ -203,3 +203,6 @@ button:disabled { cursor: default; color: #a3a7ad; background: #fafafa; border-c
 #remote-input { position: absolute; width: 1px; height: 1px; opacity: 0; padding: 0; border: 0; pointer-events: none; }
 .screen[data-human="true"] { touch-action: none; cursor: crosshair; }
 .screen[data-human="true"]:focus-within { outline: 2px solid #3976ff; outline-offset: 2px; }
+
+.audio-volume { display: inline-flex; align-items: center; gap: 4px; font-size: 12px; }
+.audio-volume input { width: 60px; padding: 0; }

apps/mcp/ui/viewer.ts

@@ -1,5 +1,6 @@
 import type { App } from "@modelcontextprotocol/ext-apps";
 import "./viewer.css";
+import { AudioPlayer } from "./audio-player.js";
 import { HumanInput } from "./human-input.js";
 import { VideoPlayer } from "./video-player.js";
 
@@ -55,7 +56,7 @@ root.innerHTML = `
     <div class="viewer-card">
       <div class="viewer-toolbar">
         <div class="viewer-title"><h2 id="device-name">远程画面</h2><span id="video-state">待连接</span></div>
-        <div class="viewer-controls"><button id="take-over" disabled>人工接管</button><button id="pause" disabled><span id="pause-label">暂停控制</span></button><button id="disconnect" class="danger" disabled>断开连接</button><button id="focus-view" aria-label="放大画面" aria-pressed="false" disabled>放大画面</button></div>
+        <div class="viewer-controls"><button id="audio-play" disabled>播放声音</button><label class="audio-volume">音量<input id="audio-volume" type="range" min="0" max="1" step="0.05" value="1"></label><button id="take-over" disabled>人工接管</button><button id="pause" disabled><span id="pause-label">暂停控制</span></button><button id="disconnect" class="danger" disabled>断开连接</button><button id="focus-view" aria-label="放大画面" aria-pressed="false" disabled>放大画面</button></div>
       </div>
       <div class="screen" id="screen">
         <canvas id="frame" role="img" aria-label="远程桌面实时画面" hidden></canvas>
@@ -107,10 +108,13 @@ let imageAt = 0;
 let busy = false,
   polling = false,
   started = false;
+const audioPlayer = new AudioPlayer();
 const humanInput = new HumanInput(element("screen"), element<HTMLTextAreaElement>("remote-input"),
   (args) => invoke("input", args), showError);
 function ownsInput() { return status.control_owner === "human" && status.human_view === context?.view_id; }
 const messages: Record<string, string> = {
+  AUDIO_INVALID: "远端音频格式无效。",
+  AUDIO_UNAVAILABLE: "尚未收到远端声音，请检查远端音频捕获权限。",
   HUMAN_CONTROL_ACTIVE: "远端正在由人工操作，请先交还 Agent。",
   STALE_OBSERVATION: "画面已变化，请等待新画面后继续操作。",
   STALE_CONTROL: "控制权已交接。",
@@ -386,6 +390,9 @@ function renderStatus(s: Data) {
   if (lost && !disconnectedByUser && ["ready", "waiting_frame"].includes(status.state)) showError(new Error("CONNECTION_LOST"));
   const changedSession = s.session_id !== status.session_id;
   status = s;
+  element<HTMLButtonElement>("audio-play").disabled = !connected;
+  text("audio-play", audioPlayer.enabled ? "静音" : "播放声音");
+  if (!connected || changedSession) audioPlayer.flush();
   humanInput.update(connected && ownsInput() ? s.human_input_epoch : undefined);
   text("take-over", ownsInput() ? "交还 Agent" : "人工接管");
   element<HTMLButtonElement>("take-over").disabled = !connected || !s.control_allowed ||
@@ -503,6 +510,17 @@ input("recovery").onchange = async () => {
     renderStatus(await invoke("status"));
   } catch (error) { showError(error); renderStatus(status); }
 };
+element("audio-play").onclick = async () => {
+  const enabled = !audioPlayer.enabled;
+  element<HTMLButtonElement>("audio-play").disabled = true;
+  try {
+    await audioPlayer.enable(enabled);
+    audioPlayer.volume(Number(input("audio-volume").value));
+    await invoke("audio", { enabled });
+  } catch (error) { await audioPlayer.enable(false); showError(error); }
+  finally { renderStatus(status); }
+};
+input("audio-volume").oninput = () => audioPlayer.volume(Number(input("audio-volume").value));
 element("take-over").onclick = async () => {
   if (controlBusy) return;
   controlBusy = true;
@@ -550,6 +568,7 @@ element("disconnect").onclick = async () => {
   }
 };
 async function closePreview() {
+  audioPlayer.flush();
   ++videoEpoch;
   player?.stop();
   player = undefined;
@@ -596,7 +615,7 @@ async function startPreview() {
       element<HTMLButtonElement>("focus-view").disabled = false;
       text("video-state", "实时");
       element("video-state").dataset.state = "live";
-    }, failed, element<HTMLCanvasElement>("agent-cursor"));
+    }, failed, element<HTMLCanvasElement>("agent-cursor"), (bytes) => audioPlayer.accept(bytes));
     player.start(grant.url, grant.ticket, grant.protocol, grant.decoder);
   } catch (error) {
     failed(error instanceof Error ? error : new Error("VIDEO_UNAVAILABLE"));
@@ -693,6 +712,7 @@ async function start() {
       unlock: "crossdesk_ui_unlock",
       recovery: "crossdesk_ui_recovery",
       input: "crossdesk_ui_input",
+      audio: "crossdesk_ui_audio",
     };
     let connected = false,
       initial: Data | undefined;

libs/controller/include/controller_session.h

@@ -22,6 +22,10 @@
 #include <vector>
 
 namespace crossdesk {
+struct AudioFrame {
+  uint64_t generation = 0, sequence = 0;
+  std::vector<uint8_t> pcm;
+};
 struct EncodedVideoFrame {
   std::vector<uint8_t> data;
   uint64_t generation = 0, sequence = 0, timestamp_us = 0;
@@ -72,6 +76,9 @@ public:
   std::optional<EncodedVideoFrame> TakeEncodedVideo();
   void SetPreview(int fps, int width, int height, int quality = -1,
                   int max_bitrate_bps = -1);
+  void SetAudio(bool enabled);
+  bool AudioEnabled() const { return audio_enabled_; }
+  std::optional<AudioFrame> TakeAudio();
   void SetControl(bool enabled);
   void ReleaseInput();
   void ReleaseActionInput();
@@ -114,6 +121,8 @@ private:
   static void Connection(ConnectionStatus, const char *, size_t, void *);
   static void Data(const char *, size_t, const char *, size_t, const char *,
                    size_t, void *);
+  static void Audio(const char *, size_t, const char *, size_t,
+                    const char *, size_t, void *);
   static void Video(const MiniRtcVideoFrame *, const char *, size_t,
                     const char *, size_t, void *);
   static int EncodedVideo(const uint8_t *, size_t, int64_t, const char *,
@@ -153,6 +162,10 @@ private:
   MiniRtcNetTrafficStats network_{};
   TraversalMode traversal_ = UnknownMode;
   LatestFrame frames_;
+  bool audio_enabled_ = false;
+  size_t audio_bytes_ = 0;
+  uint64_t audio_sequence_ = 0, audio_dropped_ = 0;
+  std::deque<AudioFrame> audio_frames_;
   bool video_output_ = false, video_wait_key_ = true, video_request_key_ = false;
   size_t video_bytes_ = 0;
   uint64_t video_generation_ = 0, video_sequence_ = 0;

libs/controller/src/controller_audio.cpp

@@ -0,0 +1,49 @@
+#include "controller_session.h"
+
+#include <stream_names.h>
+#include <string_view>
+
+namespace crossdesk {
+void ControllerSession::Audio(const char *data, size_t size, const char *, size_t,
+                              const char *stream, size_t stream_size, void *opaque) {
+  // MiniRTC's receive decoder produces 48 kHz, mono, signed 16-bit PCM.
+  auto &session = *static_cast<ControllerSession *>(opaque);
+  if (!data || !size || size > 11520 || (size & 1) || !stream ||
+      std::string_view(stream, stream_size) != kAudioStream) return;
+  std::lock_guard lock(session.mutex_);
+  if (session.stopping_ || !session.audio_enabled_) return;
+  while (!session.audio_frames_.empty() && session.audio_bytes_ + size > 96000) {
+    session.audio_bytes_ -= session.audio_frames_.front().pcm.size();
+    session.audio_frames_.pop_front();
+    ++session.audio_dropped_;
+  }
+  AudioFrame frame;
+  frame.generation = session.callback_generation_;
+  frame.sequence = ++session.audio_sequence_;
+  frame.pcm.assign(data, data + size);
+  session.audio_bytes_ += size;
+  session.audio_frames_.push_back(std::move(frame));
+}
+void ControllerSession::SetAudio(bool enabled) {
+  if (!Connected()) throw std::invalid_argument("NOT_CONNECTED");
+  {
+    std::lock_guard lock(mutex_);
+    audio_enabled_ = enabled;
+    audio_frames_.clear();
+    audio_bytes_ = 0;
+  }
+  RemoteAction action{};
+  action.type = audio_capture;
+  action.a = enabled;
+  Send(kControlStream, action.to_json());
+  Preview();
+}
+std::optional<AudioFrame> ControllerSession::TakeAudio() {
+  std::lock_guard lock(mutex_);
+  if (audio_frames_.empty()) return std::nullopt;
+  auto frame = std::move(audio_frames_.front());
+  audio_frames_.pop_front();
+  audio_bytes_ -= frame.pcm.size();
+  return frame;
+}
+} // namespace crossdesk

libs/controller/src/controller_session.cpp

@@ -107,8 +107,7 @@ Params ControllerSession::Parameters() {
   p.on_receive_data_buffer = Data;
   p.on_receive_video_frame = Video;
   p.on_receive_encoded_video = EncodedVideo;
-  // No audio callback, clipboard receiver, file writer or incoming input
-  // handler.
+  p.on_receive_audio_buffer = Audio;
   p.user_id = "cli";
   p.user_data = this;
   return p;
@@ -352,6 +351,9 @@ void ControllerSession::ClosePeers() {
   {
     std::lock_guard lock(mutex_);
     stopping_ = true;
+    audio_enabled_ = false;
+    audio_frames_.clear();
+    audio_bytes_ = 0;
     callback_generation_ = ++generation_;
     frames_.Reset(generation_);
   }
@@ -434,7 +436,7 @@ void ControllerSession::Preview() {
   // Preserve desktop text detail at every cadence, including 30 FPS previews.
   s.preference =
       static_cast<int>(VideoDegradationPreference::MaintainResolution);
-  s.audio = false;
+  s.audio = audio_enabled_;
   s.quality = options_.quality;
   if (resolution_) {
     s.version = 2;
@@ -536,6 +538,8 @@ json ControllerSession::Status() const {
           {"platform", HostPlatformName(host_platform_)},
           {"display", options_.display},
           {"text_input", text_supported_},
+          {"audio", {{"enabled", audio_enabled_}, {"sample_rate", 48000}, {"channels", 1},
+                     {"format", "s16le"}, {"received", audio_sequence_}, {"dropped", audio_dropped_}}},
           {"capabilities", remote_capabilities_},
           {"clipboard_ready", clipboard_ready_},
           {"preview_supported", preview_supported_},