diff --git a/apps/cli/agent.cpp b/apps/cli/agent.cpp index b10ea3d9..de42e8ac 100644 --- a/apps/cli/agent.cpp +++ b/apps/cli/agent.cpp @@ -86,6 +86,13 @@ struct RemoteJob { bool write; Clock::time_point deadline; }; +struct AudioCapture { + json id; + size_t target; + bool restore_enabled; + Clock::time_point deadline; + std::vector pcm; +}; struct Capture { json id, request; bool preview = false; @@ -314,11 +321,44 @@ private: std::fill(step.text.begin(), step.text.end(), '\0'); action_.reset(); } + void FinishAudio(const char *error = nullptr) { + if (!audio_capture_) return; + const auto capture = std::move(audio_capture_); + if (session_.Connected() && session_.AudioEnabled() != capture->restore_enabled) + session_.SetAudio(capture->restore_enabled); + if (error) { Error(capture->id, error); return; } + std::vector wave; + const auto number = [&](uint32_t value, int count) { + for (int index = 0; index < count; ++index) wave.push_back(uint8_t(value >> (8 * index))); + }; + const auto tag = [&](const char *value) { wave.insert(wave.end(), value, value + 4); }; + tag("RIFF"); number(uint32_t(capture->pcm.size()) + 36, 4); tag("WAVE"); + tag("fmt "); number(16, 4); number(1, 2); number(1, 2); + number(48000, 4); number(96000, 4); number(2, 2); number(16, 2); + tag("data"); number(uint32_t(capture->pcm.size()), 4); + wave.insert(wave.end(), capture->pcm.begin(), capture->pcm.end()); + Reply(capture->id, {{"mime_type", "audio/wav"}, {"data", ImageBase64(wave)}, + {"duration_ms", capture->pcm.size() / 96}, {"sample_rate", 48000}, {"channels", 1}}); + } + void PumpAudio() { + while (auto frame = session_.TakeAudio()) { + if (audio_capture_) { + const auto size = std::min(frame->pcm.size(), audio_capture_->target - audio_capture_->pcm.size()); + audio_capture_->pcm.insert(audio_capture_->pcm.end(), frame->pcm.begin(), frame->pcm.begin() + size); + } + if (video_ && video_->Enabled()) video_->Submit(std::move(*frame)); + } + if (!audio_capture_) return; + if (!session_.Connected()) FinishAudio("SESSION_CLOSED"); + else if (audio_capture_->pcm.size() >= audio_capture_->target) FinishAudio(); + else if (Clock::now() >= audio_capture_->deadline) FinishAudio("AUDIO_UNAVAILABLE"); + } void CancelRemoteJobs(const char *code) { for (const auto &job : remote_jobs_) FinishRemote(job, {}, code); remote_jobs_.clear(); } void Stop(const char *code) { + FinishAudio(code); FinishAction(code); CancelRemoteJobs(code); for (const auto &capture : captures_) @@ -339,6 +379,7 @@ private: } void Tick() { session_.Tick(); + PumpAudio(); if (!action_ && session_.HasPressedInput() && Clock::now() >= input_lease_) session_.ReleaseInput(); for (auto it = remote_jobs_.begin(); it != remote_jobs_.end();) { @@ -922,6 +963,20 @@ private: }); terminal_encoding_ = std::move(job); return false; + } else if (method == "audio") { + Fields(j, {"id", "method", "enabled"}); + FinishAudio("CANCELLED"); + session_.SetAudio(Flag(j, "enabled")); + } else if (method == "audio_capture") { + Fields(j, {"id", "method", "duration_ms"}); + if (!session_.Connected()) throw std::invalid_argument("NOT_CONNECTED"); + if (audio_capture_) throw std::invalid_argument("BUSY"); + const int duration = Number(j, "duration_ms", 100, 10000, 3000); + const bool restore = session_.AudioEnabled(); + session_.SetAudio(true); + audio_capture_ = std::make_unique(AudioCapture{ + id, size_t(duration) * 96, restore, Clock::now() + Ms(duration + 3000), {}}); + return false; } else if (method == "control") { Fields(j, {"id", "method", "enabled"}); if (human_control_) throw std::invalid_argument("HUMAN_CONTROL_ACTIVE"); @@ -1038,6 +1093,7 @@ private: if (!j.contains("request_id") || !j["request_id"].is_number_integer()) throw std::invalid_argument("INVALID_REQUEST"); const auto target = j["request_id"]; + if (audio_capture_ && audio_capture_->id == target) FinishAudio("CANCELLED"); for (auto it = remote_jobs_.begin(); it != remote_jobs_.end();) { if (it->id == target) { FinishRemote(*it, {}, "CANCELLED"); @@ -1121,6 +1177,7 @@ private: std::deque captures_; std::deque frame_waits_; std::deque remote_jobs_; + std::unique_ptr audio_capture_; std::unique_ptr encoding_; std::unique_ptr terminal_encoding_; std::unique_ptr action_; diff --git a/apps/cli/video_server.cpp b/apps/cli/video_server.cpp index 5dae62c2..e97b3cce 100644 --- a/apps/cli/video_server.cpp +++ b/apps/cli/video_server.cpp @@ -92,6 +92,8 @@ struct VideoServer::State { // each other; the compressed handoff is bounded and recovers at an IDR. mutable std::mutex mutex; std::deque queue; + std::deque audio_queue; + size_t audio_bytes = 0; size_t queued_bytes = 0; bool scheduled = false, wait_key = true; json cursor; @@ -257,6 +259,33 @@ struct VideoServer::State { if (error) return; const auto now = Clock::now(); + std::deque audio; + { + std::lock_guard lock(mutex); + audio.swap(audio_queue); + audio_bytes = 0; + } + for (const auto &frame : audio) { + if (!connected || frame.generation != generation) continue; + std::string bytes; + Number(bytes, 0x43444131, 4); // CDA1, network-order header, s16le payload. + Number(bytes, frame.generation, 4); + Number(bytes, frame.sequence, 4); + Number(bytes, 48000, 4); + Number(bytes, 1, 2); + Number(bytes, 16, 2); + bytes.append(reinterpret_cast(frame.pcm.data()), frame.pcm.size()); + std::vector failed; + for (auto &[handle, client] : clients) { + if (!client.authorized) continue; + websocketpp::lib::error_code error; + const auto connection = server.get_con_from_hdl(handle, error); + if (!error && connection->get_buffered_amount() < 256 * 1024) + server.send(handle, bytes, websocketpp::frame::opcode::BINARY, error); + if (error) failed.push_back(handle); + } + for (const auto handle : failed) Drop(handle); + } std::string pointer; uint64_t revision; { @@ -429,6 +458,8 @@ void VideoServer::Bind(uint64_t generation, bool decoded, bool connected) { s.key_requested = false; std::lock_guard lock(s.mutex); s.queue.clear(); + s.audio_queue.clear(); + s.audio_bytes = 0; s.queued_bytes = 0; s.wait_key = true; }); @@ -488,6 +519,17 @@ void VideoServer::Submit(EncodedVideoFrame frame) { if (Enabled()) state_->Enqueue({{}, std::move(frame)}); } +void VideoServer::Submit(AudioFrame frame) { + if (!Enabled() || frame.pcm.empty() || frame.pcm.size() > 11520 || (frame.pcm.size() & 1)) return; + auto &s = *state_; + std::lock_guard lock(s.mutex); + while (!s.audio_queue.empty() && s.audio_bytes + frame.pcm.size() > 48000) { + s.audio_bytes -= s.audio_queue.front().pcm.size(); + s.audio_queue.pop_front(); + } + s.audio_bytes += frame.pcm.size(); + s.audio_queue.push_back(std::move(frame)); +} void VideoServer::Cursor(json cursor) { auto &s = *state_; std::lock_guard lock(s.mutex); diff --git a/apps/cli/video_server.h b/apps/cli/video_server.h index 3b9bbece..39032a62 100644 --- a/apps/cli/video_server.h +++ b/apps/cli/video_server.h @@ -36,6 +36,7 @@ public: bool TakeKeyframeRequest(); void Submit(std::shared_ptr frame); void Submit(EncodedVideoFrame frame); + void Submit(AudioFrame frame); void Cursor(nlohmann::json cursor); nlohmann::json Status() const; diff --git a/apps/mcp/README.md b/apps/mcp/README.md index 3c56bde2..8a52ff10 100644 --- a/apps/mcp/README.md +++ b/apps/mcp/README.md @@ -294,3 +294,9 @@ npm run pack:npm -- --out-dir /absolute/path/artifacts 浏览器及 Apps 面板提供“人工接管”和“交还 Agent”。接管后面板可操作鼠标、滚轮、键盘快捷键和中文输入法,连接信息展示当前控制权。输入绑定面板、控制批次和画面尺寸,交接会释放按住状态并使旧批次失效。画面超过两秒未更新时停止发送新输入,窗口失焦或隐藏时释放按键。原生层保留十五秒的输入租约作为断线清理保障。 Agent 在人工接管期间仍可观察画面,键鼠、显示设置和其他要求控制权的操作等待用户在面板交还控制。连接需预先授予控制权限,并使用支持 `events/1` 的原生 CLI。 + +### 远端声音 + +`crossdesk_audio` 开关远端桌面音频接收,默认关闭。面板的“播放声音”按钮同时启用接收和本地播放,音量在本面板调节。播放使用已授权的视频 WebSocket 通道,PCM 队列与浏览器播放时延均设有上限,拥塞时丢弃过时音频。浏览器首次播放需要用户点击按钮。 + +`crossdesk_audio_capture` 采集接下来 100 至 10000 毫秒的音频,通过 MCP 音频内容块返回单声道 48 kHz WAV。采集期间临时开启接收,结束或取消后恢复之前的设置,未收到足够音频时返回 `AUDIO_UNAVAILABLE`。系统需允许被控端捕获桌面声音。自动恢复连接会恢复用户选择的音频接收设置。 diff --git a/apps/mcp/src/browser.ts b/apps/mcp/src/browser.ts index 4f381c07..f5e54ed2 100644 --- a/apps/mcp/src/browser.ts +++ b/apps/mcp/src/browser.ts @@ -255,7 +255,7 @@ export class BrowserPanel { if (typeof body.view_id !== "string" || !client.views.has(body.view_id)) throw new Failure("VIEW_UNAUTHORIZED"); const name = req.url?.match( - /^\/api\/(ready|devices|status|connect|video_open|control|unlock|recovery|input)$/, + /^\/api\/(ready|devices|status|connect|video_open|control|unlock|recovery|input|audio)$/, )?.[1]; if (!name) throw new Failure("UNKNOWN_ACTION"); const { view_id, ...args } = body; diff --git a/apps/mcp/src/server.ts b/apps/mcp/src/server.ts index 7a2cf173..203b4915 100644 --- a/apps/mcp/src/server.ts +++ b/apps/mcp/src/server.ts @@ -41,10 +41,15 @@ const uiNames: Record = { unlock: "crossdesk_ui_unlock", recovery: "crossdesk_ui_recovery", input: "crossdesk_ui_input", + audio: "crossdesk_ui_audio", }; function output(value: Data, meta?: Data): CallToolResult { const data = { ...value }; const content: any[] = []; + if (data.mime_type === "audio/wav" && typeof data.data === "string") { + content.push({ type: "audio", mimeType: data.mime_type, data: data.data }); + delete data.data; + } if (data.mime_type === "image/png" && typeof data.data === "string") { content.push({ type: "image", mimeType: data.mime_type, data: data.data }); delete data.data; @@ -261,6 +266,12 @@ export function createServer( }, { readOnly: true }, ); + add("crossdesk_audio", z.strictObject({ session_id: id, enabled: z.boolean() }), + "Enable or disable remote desktop audio reception. Audio starts disabled. Panel playback also requires the user's Play audio button. Status reports PCM reception counters.", + async (p, ctx) => output(await sessions.audio(p, false, ctx.mcpReq.signal))); + add("crossdesk_audio_capture", z.strictObject({ session_id: id, duration_ms: z.int().min(100).max(10000).default(3000) }), + "Capture the next 100–10000 ms of remote desktop audio as a bounded mono 48 kHz WAV audio content block. Temporarily enables reception, then restores the previous setting. AUDIO_UNAVAILABLE means insufficient PCM arrived. Cancellation discards the capture.", + async (p, ctx) => output(await sessions.audio(p, true, ctx.mcpReq.signal)), { readOnly: true }); add("crossdesk_list_sessions", z.strictObject({}), "List the active remote sessions with their device aliases, displays and control state. Each session_id addresses one connection. Up to eight connections can be active concurrently.", async () => output(await sessions.listSessions()), { readOnly: true }); diff --git a/apps/mcp/src/sessions.ts b/apps/mcp/src/sessions.ts index 6de1c004..cd7a8976 100644 --- a/apps/mcp/src/sessions.ts +++ b/apps/mcp/src/sessions.ts @@ -61,6 +61,7 @@ interface Session { displayBatch?: boolean; mode: string; desiredControl: boolean; + audioEnabled?: boolean; recoveryEnabled: boolean; recoveryAttempts: number; recoveryAttempt: number; @@ -317,6 +318,10 @@ export class Sessions { if (next.status.preview_bitrate_supported) settings.max_bitrate_bps = s.status.preview_bitrate_bps; await next.native.request("video_settings", settings).catch(() => {}); } + if (s.audioEnabled) { + await next.native.request("audio", { enabled: true }); + next.audioEnabled = true; + } next.recoveryEnabled = true; next.recoveryAttempts = s.recoveryAttempts; next.recoveryAttempt = attempt; @@ -756,6 +761,14 @@ export class Sessions { } return record.result; } + async audio(p: Data, capture: boolean, signal?: AbortSignal): Promise { + const { session_id, ...args } = p; + const s = this.session(session_id); + const result = await s.native.request(capture ? "audio_capture" : "audio", args, + capture ? (args.duration_ms ?? 3000) + 5000 : 5000, signal); + if (!capture) { s.status = result; s.audioEnabled = args.enabled; } + return result; + } async videoSettings(p: Data, signal?: AbortSignal): Promise { const { session_id, ...args } = p; const s = this.session(session_id); diff --git a/apps/mcp/src/ui-actions.ts b/apps/mcp/src/ui-actions.ts index d45e3dbf..33158617 100644 --- a/apps/mcp/src/ui-actions.ts +++ b/apps/mcp/src/ui-actions.ts @@ -9,6 +9,7 @@ export const uiSchemas = { connect: uiConnect, video_open: z.strictObject({}), input: viewerInput, + audio: z.strictObject({ enabled: z.boolean() }), recovery: z.strictObject({ enabled: z.boolean(), max_attempts: z.int().min(1).max(5).default(3) }), unlock: z.strictObject({ request_id: id, @@ -46,6 +47,9 @@ export async function uiAction( return sessions.status(undefined, view.id, signal); case "connect": return sessions.connectUI(view, p, signal); + case "audio": + if (!view.session) throw new Failure("NOT_READY"); + return sessions.audio({ session_id: view.session, ...p }, false, signal); case "input": return sessions.viewerInput(view, p, signal); case "control": diff --git a/apps/mcp/ui/audio-player.ts b/apps/mcp/ui/audio-player.ts new file mode 100644 index 00000000..97df3521 --- /dev/null +++ b/apps/mcp/ui/audio-player.ts @@ -0,0 +1,52 @@ +export class AudioPlayer { + private context?: AudioContext; + private gain?: GainNode; + private next = 0; + private sequence = -1; + private generation = -1; + private sources = new Set(); + enabled = false; + async enable(enabled: boolean) { + if (enabled) { + this.context ??= new AudioContext(); + if (!this.gain) { + this.gain = this.context.createGain(); + this.gain.connect(this.context.destination); + } + await this.context.resume(); + } + this.enabled = enabled; + if (!enabled) this.flush(); + } + volume(value: number) { if (this.gain) this.gain.gain.value = Math.max(0, Math.min(1, value)); } + flush() { + for (const source of this.sources) { source.stop(); source.disconnect(); } + this.sources.clear(); + this.next = 0; + this.sequence = this.generation = -1; + } + accept(bytes: ArrayBuffer) { + if (bytes.byteLength < 22 || bytes.byteLength > 11540 || (bytes.byteLength & 1)) throw new Error("AUDIO_INVALID"); + const data = new DataView(bytes); + if (data.getUint32(0) !== 0x43444131 || data.getUint32(12) !== 48000 || + data.getUint16(16) !== 1 || data.getUint16(18) !== 16) throw new Error("AUDIO_INVALID"); + if (!this.enabled || !this.context || !this.gain || this.context.state !== "running") return; + const generation = data.getUint32(4), sequence = data.getUint32(8); + if (generation < this.generation || (generation === this.generation && sequence <= this.sequence)) return; + const now = this.context.currentTime; + if (generation !== this.generation || this.next > now + .25 || sequence !== this.sequence + 1) this.flush(); + this.generation = generation; + this.sequence = sequence; + const buffer = this.context.createBuffer(1, (bytes.byteLength - 20) / 2, 48000); + const samples = buffer.getChannelData(0); + for (let index = 0; index < samples.length; index++) samples[index] = data.getInt16(20 + index * 2, true) / 32768; + const source = this.context.createBufferSource(); + source.buffer = buffer; + source.connect(this.gain); + source.onended = () => { source.disconnect(); this.sources.delete(source); }; + this.sources.add(source); + this.next = Math.max(now + .03, this.next); + source.start(this.next); + this.next += buffer.duration; + } +} diff --git a/apps/mcp/ui/video-player.ts b/apps/mcp/ui/video-player.ts index 20ee666e..5414910c 100644 --- a/apps/mcp/ui/video-player.ts +++ b/apps/mcp/ui/video-player.ts @@ -51,7 +51,8 @@ export class VideoPlayer { constructor(private canvas: HTMLCanvasElement, private onFrame: (info: FrameInfo) => void, private onError: (error: Error) => void, - private overlay?: HTMLCanvasElement) { + private overlay?: HTMLCanvasElement, + private onAudio?: (bytes: ArrayBuffer) => void) { if (overlay) { this.resize = new ResizeObserver(() => this.drawCursor()); this.resize.observe(overlay); @@ -93,6 +94,10 @@ export class VideoPlayer { } if (!(event.data instanceof ArrayBuffer)) return; const data = event.data; + if (data.byteLength >= 4 && new DataView(data).getUint32(0) === 0x43444131) { + try { this.onAudio?.(data); } catch { this.onError(new Error("AUDIO_INVALID")); } + return; + } if (data.byteLength < (this.raw ? 38 : 25) || data.byteLength > (this.raw ? 16777216 * 3 / 2 + 32 : 4 * 1024 * 1024 + 24)) return this.fail(new Error("VIDEO_INVALID")); diff --git a/apps/mcp/ui/viewer.css b/apps/mcp/ui/viewer.css index 577cc31e..0b676c74 100644 --- a/apps/mcp/ui/viewer.css +++ b/apps/mcp/ui/viewer.css @@ -203,3 +203,6 @@ button:disabled { cursor: default; color: #a3a7ad; background: #fafafa; border-c #remote-input { position: absolute; width: 1px; height: 1px; opacity: 0; padding: 0; border: 0; pointer-events: none; } .screen[data-human="true"] { touch-action: none; cursor: crosshair; } .screen[data-human="true"]:focus-within { outline: 2px solid #3976ff; outline-offset: 2px; } + +.audio-volume { display: inline-flex; align-items: center; gap: 4px; font-size: 12px; } +.audio-volume input { width: 60px; padding: 0; } diff --git a/apps/mcp/ui/viewer.ts b/apps/mcp/ui/viewer.ts index 0d359f70..77cbb8e1 100644 --- a/apps/mcp/ui/viewer.ts +++ b/apps/mcp/ui/viewer.ts @@ -1,5 +1,6 @@ import type { App } from "@modelcontextprotocol/ext-apps"; import "./viewer.css"; +import { AudioPlayer } from "./audio-player.js"; import { HumanInput } from "./human-input.js"; import { VideoPlayer } from "./video-player.js"; @@ -55,7 +56,7 @@ root.innerHTML = `

远程画面

待连接
-
+
@@ -107,10 +108,13 @@ let imageAt = 0; let busy = false, polling = false, started = false; +const audioPlayer = new AudioPlayer(); const humanInput = new HumanInput(element("screen"), element("remote-input"), (args) => invoke("input", args), showError); function ownsInput() { return status.control_owner === "human" && status.human_view === context?.view_id; } const messages: Record = { + AUDIO_INVALID: "远端音频格式无效。", + AUDIO_UNAVAILABLE: "尚未收到远端声音,请检查远端音频捕获权限。", HUMAN_CONTROL_ACTIVE: "远端正在由人工操作,请先交还 Agent。", STALE_OBSERVATION: "画面已变化,请等待新画面后继续操作。", STALE_CONTROL: "控制权已交接。", @@ -386,6 +390,9 @@ function renderStatus(s: Data) { if (lost && !disconnectedByUser && ["ready", "waiting_frame"].includes(status.state)) showError(new Error("CONNECTION_LOST")); const changedSession = s.session_id !== status.session_id; status = s; + element("audio-play").disabled = !connected; + text("audio-play", audioPlayer.enabled ? "静音" : "播放声音"); + if (!connected || changedSession) audioPlayer.flush(); humanInput.update(connected && ownsInput() ? s.human_input_epoch : undefined); text("take-over", ownsInput() ? "交还 Agent" : "人工接管"); element("take-over").disabled = !connected || !s.control_allowed || @@ -503,6 +510,17 @@ input("recovery").onchange = async () => { renderStatus(await invoke("status")); } catch (error) { showError(error); renderStatus(status); } }; +element("audio-play").onclick = async () => { + const enabled = !audioPlayer.enabled; + element("audio-play").disabled = true; + try { + await audioPlayer.enable(enabled); + audioPlayer.volume(Number(input("audio-volume").value)); + await invoke("audio", { enabled }); + } catch (error) { await audioPlayer.enable(false); showError(error); } + finally { renderStatus(status); } +}; +input("audio-volume").oninput = () => audioPlayer.volume(Number(input("audio-volume").value)); element("take-over").onclick = async () => { if (controlBusy) return; controlBusy = true; @@ -550,6 +568,7 @@ element("disconnect").onclick = async () => { } }; async function closePreview() { + audioPlayer.flush(); ++videoEpoch; player?.stop(); player = undefined; @@ -596,7 +615,7 @@ async function startPreview() { element("focus-view").disabled = false; text("video-state", "实时"); element("video-state").dataset.state = "live"; - }, failed, element("agent-cursor")); + }, failed, element("agent-cursor"), (bytes) => audioPlayer.accept(bytes)); player.start(grant.url, grant.ticket, grant.protocol, grant.decoder); } catch (error) { failed(error instanceof Error ? error : new Error("VIDEO_UNAVAILABLE")); @@ -693,6 +712,7 @@ async function start() { unlock: "crossdesk_ui_unlock", recovery: "crossdesk_ui_recovery", input: "crossdesk_ui_input", + audio: "crossdesk_ui_audio", }; let connected = false, initial: Data | undefined; diff --git a/libs/controller/include/controller_session.h b/libs/controller/include/controller_session.h index 458670c9..e35d5050 100644 --- a/libs/controller/include/controller_session.h +++ b/libs/controller/include/controller_session.h @@ -22,6 +22,10 @@ #include namespace crossdesk { +struct AudioFrame { + uint64_t generation = 0, sequence = 0; + std::vector pcm; +}; struct EncodedVideoFrame { std::vector data; uint64_t generation = 0, sequence = 0, timestamp_us = 0; @@ -72,6 +76,9 @@ public: std::optional TakeEncodedVideo(); void SetPreview(int fps, int width, int height, int quality = -1, int max_bitrate_bps = -1); + void SetAudio(bool enabled); + bool AudioEnabled() const { return audio_enabled_; } + std::optional TakeAudio(); void SetControl(bool enabled); void ReleaseInput(); void ReleaseActionInput(); @@ -114,6 +121,8 @@ private: static void Connection(ConnectionStatus, const char *, size_t, void *); static void Data(const char *, size_t, const char *, size_t, const char *, size_t, void *); + static void Audio(const char *, size_t, const char *, size_t, + const char *, size_t, void *); static void Video(const MiniRtcVideoFrame *, const char *, size_t, const char *, size_t, void *); static int EncodedVideo(const uint8_t *, size_t, int64_t, const char *, @@ -153,6 +162,10 @@ private: MiniRtcNetTrafficStats network_{}; TraversalMode traversal_ = UnknownMode; LatestFrame frames_; + bool audio_enabled_ = false; + size_t audio_bytes_ = 0; + uint64_t audio_sequence_ = 0, audio_dropped_ = 0; + std::deque audio_frames_; bool video_output_ = false, video_wait_key_ = true, video_request_key_ = false; size_t video_bytes_ = 0; uint64_t video_generation_ = 0, video_sequence_ = 0; diff --git a/libs/controller/src/controller_audio.cpp b/libs/controller/src/controller_audio.cpp new file mode 100644 index 00000000..46bafd98 --- /dev/null +++ b/libs/controller/src/controller_audio.cpp @@ -0,0 +1,49 @@ +#include "controller_session.h" + +#include +#include + +namespace crossdesk { +void ControllerSession::Audio(const char *data, size_t size, const char *, size_t, + const char *stream, size_t stream_size, void *opaque) { + // MiniRTC's receive decoder produces 48 kHz, mono, signed 16-bit PCM. + auto &session = *static_cast(opaque); + if (!data || !size || size > 11520 || (size & 1) || !stream || + std::string_view(stream, stream_size) != kAudioStream) return; + std::lock_guard lock(session.mutex_); + if (session.stopping_ || !session.audio_enabled_) return; + while (!session.audio_frames_.empty() && session.audio_bytes_ + size > 96000) { + session.audio_bytes_ -= session.audio_frames_.front().pcm.size(); + session.audio_frames_.pop_front(); + ++session.audio_dropped_; + } + AudioFrame frame; + frame.generation = session.callback_generation_; + frame.sequence = ++session.audio_sequence_; + frame.pcm.assign(data, data + size); + session.audio_bytes_ += size; + session.audio_frames_.push_back(std::move(frame)); +} +void ControllerSession::SetAudio(bool enabled) { + if (!Connected()) throw std::invalid_argument("NOT_CONNECTED"); + { + std::lock_guard lock(mutex_); + audio_enabled_ = enabled; + audio_frames_.clear(); + audio_bytes_ = 0; + } + RemoteAction action{}; + action.type = audio_capture; + action.a = enabled; + Send(kControlStream, action.to_json()); + Preview(); +} +std::optional ControllerSession::TakeAudio() { + std::lock_guard lock(mutex_); + if (audio_frames_.empty()) return std::nullopt; + auto frame = std::move(audio_frames_.front()); + audio_frames_.pop_front(); + audio_bytes_ -= frame.pcm.size(); + return frame; +} +} // namespace crossdesk diff --git a/libs/controller/src/controller_session.cpp b/libs/controller/src/controller_session.cpp index 2ca9af7d..0e7e20e4 100644 --- a/libs/controller/src/controller_session.cpp +++ b/libs/controller/src/controller_session.cpp @@ -107,8 +107,7 @@ Params ControllerSession::Parameters() { p.on_receive_data_buffer = Data; p.on_receive_video_frame = Video; p.on_receive_encoded_video = EncodedVideo; - // No audio callback, clipboard receiver, file writer or incoming input - // handler. + p.on_receive_audio_buffer = Audio; p.user_id = "cli"; p.user_data = this; return p; @@ -352,6 +351,9 @@ void ControllerSession::ClosePeers() { { std::lock_guard lock(mutex_); stopping_ = true; + audio_enabled_ = false; + audio_frames_.clear(); + audio_bytes_ = 0; callback_generation_ = ++generation_; frames_.Reset(generation_); } @@ -434,7 +436,7 @@ void ControllerSession::Preview() { // Preserve desktop text detail at every cadence, including 30 FPS previews. s.preference = static_cast(VideoDegradationPreference::MaintainResolution); - s.audio = false; + s.audio = audio_enabled_; s.quality = options_.quality; if (resolution_) { s.version = 2; @@ -536,6 +538,8 @@ json ControllerSession::Status() const { {"platform", HostPlatformName(host_platform_)}, {"display", options_.display}, {"text_input", text_supported_}, + {"audio", {{"enabled", audio_enabled_}, {"sample_rate", 48000}, {"channels", 1}, + {"format", "s16le"}, {"received", audio_sequence_}, {"dropped", audio_dropped_}}}, {"capabilities", remote_capabilities_}, {"clipboard_ready", clipboard_ready_}, {"preview_supported", preview_supported_},