[feat] receive remote audio with panel playback and bounded WAV capture

libs/controller/include/controller_session.h

@@ -22,6 +22,10 @@
 #include <vector>
 
 namespace crossdesk {
+struct AudioFrame {
+  uint64_t generation = 0, sequence = 0;
+  std::vector<uint8_t> pcm;
+};
 struct EncodedVideoFrame {
   std::vector<uint8_t> data;
   uint64_t generation = 0, sequence = 0, timestamp_us = 0;
@@ -72,6 +76,9 @@ public:
   std::optional<EncodedVideoFrame> TakeEncodedVideo();
   void SetPreview(int fps, int width, int height, int quality = -1,
                   int max_bitrate_bps = -1);
+  void SetAudio(bool enabled);
+  bool AudioEnabled() const { return audio_enabled_; }
+  std::optional<AudioFrame> TakeAudio();
   void SetControl(bool enabled);
   void ReleaseInput();
   void ReleaseActionInput();
@@ -114,6 +121,8 @@ private:
   static void Connection(ConnectionStatus, const char *, size_t, void *);
   static void Data(const char *, size_t, const char *, size_t, const char *,
                    size_t, void *);
+  static void Audio(const char *, size_t, const char *, size_t,
+                    const char *, size_t, void *);
   static void Video(const MiniRtcVideoFrame *, const char *, size_t,
                     const char *, size_t, void *);
   static int EncodedVideo(const uint8_t *, size_t, int64_t, const char *,
@@ -153,6 +162,10 @@ private:
   MiniRtcNetTrafficStats network_{};
   TraversalMode traversal_ = UnknownMode;
   LatestFrame frames_;
+  bool audio_enabled_ = false;
+  size_t audio_bytes_ = 0;
+  uint64_t audio_sequence_ = 0, audio_dropped_ = 0;
+  std::deque<AudioFrame> audio_frames_;
   bool video_output_ = false, video_wait_key_ = true, video_request_key_ = false;
   size_t video_bytes_ = 0;
   uint64_t video_generation_ = 0, video_sequence_ = 0;

libs/controller/src/controller_audio.cpp

@@ -0,0 +1,49 @@
+#include "controller_session.h"
+
+#include <stream_names.h>
+#include <string_view>
+
+namespace crossdesk {
+void ControllerSession::Audio(const char *data, size_t size, const char *, size_t,
+                              const char *stream, size_t stream_size, void *opaque) {
+  // MiniRTC's receive decoder produces 48 kHz, mono, signed 16-bit PCM.
+  auto &session = *static_cast<ControllerSession *>(opaque);
+  if (!data || !size || size > 11520 || (size & 1) || !stream ||
+      std::string_view(stream, stream_size) != kAudioStream) return;
+  std::lock_guard lock(session.mutex_);
+  if (session.stopping_ || !session.audio_enabled_) return;
+  while (!session.audio_frames_.empty() && session.audio_bytes_ + size > 96000) {
+    session.audio_bytes_ -= session.audio_frames_.front().pcm.size();
+    session.audio_frames_.pop_front();
+    ++session.audio_dropped_;
+  }
+  AudioFrame frame;
+  frame.generation = session.callback_generation_;
+  frame.sequence = ++session.audio_sequence_;
+  frame.pcm.assign(data, data + size);
+  session.audio_bytes_ += size;
+  session.audio_frames_.push_back(std::move(frame));
+}
+void ControllerSession::SetAudio(bool enabled) {
+  if (!Connected()) throw std::invalid_argument("NOT_CONNECTED");
+  {
+    std::lock_guard lock(mutex_);
+    audio_enabled_ = enabled;
+    audio_frames_.clear();
+    audio_bytes_ = 0;
+  }
+  RemoteAction action{};
+  action.type = audio_capture;
+  action.a = enabled;
+  Send(kControlStream, action.to_json());
+  Preview();
+}
+std::optional<AudioFrame> ControllerSession::TakeAudio() {
+  std::lock_guard lock(mutex_);
+  if (audio_frames_.empty()) return std::nullopt;
+  auto frame = std::move(audio_frames_.front());
+  audio_frames_.pop_front();
+  audio_bytes_ -= frame.pcm.size();
+  return frame;
+}
+} // namespace crossdesk

libs/controller/src/controller_session.cpp

@@ -107,8 +107,7 @@ Params ControllerSession::Parameters() {
   p.on_receive_data_buffer = Data;
   p.on_receive_video_frame = Video;
   p.on_receive_encoded_video = EncodedVideo;
-  // No audio callback, clipboard receiver, file writer or incoming input
-  // handler.
+  p.on_receive_audio_buffer = Audio;
   p.user_id = "cli";
   p.user_data = this;
   return p;
@@ -352,6 +351,9 @@ void ControllerSession::ClosePeers() {
   {
     std::lock_guard lock(mutex_);
     stopping_ = true;
+    audio_enabled_ = false;
+    audio_frames_.clear();
+    audio_bytes_ = 0;
     callback_generation_ = ++generation_;
     frames_.Reset(generation_);
   }
@@ -434,7 +436,7 @@ void ControllerSession::Preview() {
   // Preserve desktop text detail at every cadence, including 30 FPS previews.
   s.preference =
       static_cast<int>(VideoDegradationPreference::MaintainResolution);
-  s.audio = false;
+  s.audio = audio_enabled_;
   s.quality = options_.quality;
   if (resolution_) {
     s.version = 2;
@@ -536,6 +538,8 @@ json ControllerSession::Status() const {
           {"platform", HostPlatformName(host_platform_)},
           {"display", options_.display},
           {"text_input", text_supported_},
+          {"audio", {{"enabled", audio_enabled_}, {"sample_rate", 48000}, {"channels", 1},
+                     {"format", "s16le"}, {"received", audio_sequence_}, {"dropped", audio_dropped_}}},
           {"capabilities", remote_capabilities_},
           {"clipboard_ready", clipboard_ready_},
           {"preview_supported", preview_supported_},