[feat] enable Android MediaCodec decoding and simplify tests

apps/android/native/controller_protocol.h

@@ -1,4 +1,11 @@
-#pragma once
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2026-10-02
+ * Copyright (c) 2026 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _CONTROLLER_PROTOCOL_H_
+#define _CONTROLLER_PROTOCOL_H_
 
 #include <minirtc.h>
 #include <nlohmann/json.hpp>
@@ -24,3 +31,5 @@ inline nlohmann::json NetworkReport(const MiniRtcNetTrafficStats& stats, Travers
     {"total", traffic(stats.total_inbound_stats, stats.total_outbound_stats)}};
 }
 }  // namespace crossdesk::android_controller
+
+#endif

apps/android/native/latest_frame_worker.h

@@ -1,4 +1,11 @@
-#pragma once
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2026-10-02
+ * Copyright (c) 2026 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _LATEST_FRAME_WORKER_H_
+#define _LATEST_FRAME_WORKER_H_
 
 #include <condition_variable>
 #include <functional>
@@ -61,3 +68,5 @@ class LatestFrameWorker {
   std::thread thread_;
 };
 }  // namespace crossdesk::android_controller
+
+#endif

apps/android/native/rtc_bridge.cpp

@@ -18,6 +18,7 @@
 #include <stdexcept>
 #include <vector>
 #include "controller_protocol.h"
+#include "android_codec_support.h"
 #include "latest_frame_worker.h"
 #include "video_picture.h"
 #include "video_timing.h"
@@ -63,6 +64,11 @@ struct Session {
   ANativeWindow* window = nullptr;
   uint32_t width = 0, height = 0;
   uint64_t frame_id = 0;
+  std::optional<VideoPicture> surface_picture;
+  void DetachDecoderSurface() {
+    if (surface_picture) surface_picture->RenderSurface(nullptr);
+    surface_picture.reset();
+  }
   std::set<int> pressed_keys;
   unsigned pressed_buttons = 0;
   float pointer_x = .5f, pointer_y = .5f;
@@ -91,6 +97,7 @@ struct Session {
     std::lock_guard<std::mutex> lock(render_mutex);
     ++video_generation;
     renderer.Clear();
+    DetachDecoderSurface();
     if (window) ANativeWindow_release(window);
     window = surface ? ANativeWindow_fromSurface(env, surface) : nullptr;
     surface_ready = window != nullptr;
@@ -193,17 +200,28 @@ void RenderVideo(Session* s, VideoPicture picture) {
   const auto w = picture.width, h = picture.height;
   std::lock_guard<std::mutex> lock(s->render_mutex);
   if (s->stopping || !s->window || picture.generation != s->video_generation) return;
-  if (s->width != w || s->height != h) {
-    if (ANativeWindow_setBuffersGeometry(s->window, w, h, WINDOW_FORMAT_RGBA_8888) != 0) return;
+  const bool size_changed = s->width != w || s->height != h;
+  if (picture.IsSurface()) {
+    if (picture.RenderSurface(s->window) != 0) return;
+    // Keep a consumed token to detach the codec before Surface destruction,
+    // monitor switching, or CPU rendering after a software fallback.
+    s->surface_picture = picture;
+  } else {
+    const bool was_surface = s->surface_picture.has_value();
+    s->DetachDecoderSurface();
+    if ((size_changed || was_surface) &&
+        ANativeWindow_setBuffersGeometry(s->window, w, h, WINDOW_FORMAT_RGBA_8888) != 0) return;
+    ANativeWindow_Buffer buffer{};
+    if (ANativeWindow_lock(s->window, &buffer, nullptr) != 0) return;
+    const bool rendered = buffer.width == static_cast<int>(w) && buffer.height == static_cast<int>(h) &&
+      libyuv::NV12ToABGR(picture.Y(), picture.YStride(), picture.UV(), picture.UVStride(),
+                       static_cast<uint8_t*>(buffer.bits), buffer.stride * 4, w, h) == 0;
+    if (ANativeWindow_unlockAndPost(s->window) != 0 || !rendered) return;
+  }
+  if (size_changed) {
     s->width = w; s->height = h;
     s->Event(5, {{"width", w}, {"height", h}});
   }
-  ANativeWindow_Buffer buffer{};
-  if (ANativeWindow_lock(s->window, &buffer, nullptr) != 0) return;
-  const bool rendered = buffer.width == static_cast<int>(w) && buffer.height == static_cast<int>(h) &&
-    libyuv::NV12ToABGR(picture.Y(), picture.YStride(), picture.UV(), picture.UVStride(),
-                     static_cast<uint8_t*>(buffer.bits), buffer.stride * 4, w, h) == 0;
-  if (ANativeWindow_unlockAndPost(s->window) != 0 || !rendered) return;
   // Measure through display submission, using MiniRTC's calibrated capture clock.
   const auto now = GetSystemTimeMicros(s->controller);
   const double latency = android_controller::VideoLatencyMilliseconds(
@@ -219,9 +237,9 @@ Params Parameters(Session* s, bool controller) {
   std::snprintf(p.signal_server_ip, sizeof(p.signal_server_ip), "%s", s->host.c_str());
   std::snprintf(p.log_path, sizeof(p.log_path), "%s", s->log_path.c_str());
   p.signal_server_port = s->port;
-  p.hardware_acceleration = false;
-  // OpenH264/dav1d expose pooled CPU NV12 buffers on Android. Retain them
-  // across display submission instead of copying full-resolution pixels.
+  p.hardware_acceleration = true;
+  // MediaCodec returns retained Surface buffers; software fallback returns
+  // pooled CPU NV12 with the same ownership contract.
   p.native_video_output = controller;
   p.turn_mode = TurnAutoUdpTcp;
   p.enable_srtp = true;
@@ -241,7 +259,9 @@ Session* From(jlong handle) { return reinterpret_cast<Session*>(handle); }
 } // namespace
 
 extern "C" JNIEXPORT jint JNI_OnLoad(JavaVM* java_vm, void*) {
-  vm = java_vm; return JNI_VERSION_1_6;
+  vm = java_vm;
+  minirtc::android::SetJavaVm(java_vm);
+  return JNI_VERSION_1_6;
 }
 #define JNI_METHOD(name) Java_cn_crossdesk_mobile_NativeSession_##name
 extern "C" JNIEXPORT jboolean JNICALL JNI_METHOD(nValidAppVersion)(
@@ -274,7 +294,7 @@ extern "C" JNIEXPORT jlong JNICALL JNI_METHOD(nCreate)(
     s->host = String(env, host); s->port = port;
     s->identity_login = String(env, identity); s->log_path = String(env, log_path);
     // The bundle is exported from Android's default TrustManager, never the
-    // build machine's certificate store. The Android MiniRTC patch explicitly
+    // build machine's certificate store. MiniRTC on Android explicitly
     // loads this path: OpenSSL's default loader ignores SSL_CERT_FILE in an
     // Android app process (AT_SECURE=1). Chain and hostname checks stay enabled.
     const auto bundle = String(env, certificates);
@@ -338,7 +358,7 @@ extern "C" JNIEXPORT void JNICALL JNI_METHOD(nControl)(
   auto* s = From(handle); if (!s) return;
   RemoteAction action{};
   if (type == display_id && value >= 0 && value < 8) {
-    { std::lock_guard<std::mutex> lock(s->render_mutex); s->display = value; ++s->video_generation; s->renderer.Clear(); s->width = s->height = 0; }
+    { std::lock_guard<std::mutex> lock(s->render_mutex); s->display = value; ++s->video_generation; s->renderer.Clear(); s->DetachDecoderSurface(); s->width = s->height = 0; }
     action.type = display_id; action.d = value;
   } else if (type == audio_capture) {
     action.type = audio_capture; action.a = value != 0;

apps/android/native/tests/controller_protocol_test.cpp

@@ -1,39 +0,0 @@
-#include "controller_protocol.h"
-#include <cassert>
-#include <cmath>
-#include <limits>
-
-int main() {
-  using namespace crossdesk;
-  for (int quality = -1; quality <= 3; ++quality)
-    for (int rate : {0, 24, 30, 60, 120})
-      for (int preference = -1; preference <= 3; ++preference) {
-        const auto message = android_controller::VideoSettingsMessage(quality, rate, preference, UINT32_MAX);
-        const bool valid = quality >= 0 && quality <= 2 && (rate == 30 || rate == 60) && preference >= 0 && preference <= 2;
-        assert(message.empty() != valid);
-        if (!valid) continue;
-        // Decode the actual Android outgoing message with the desktop's wire parser.
-        RemoteAction desktop{};
-        assert(desktop.from_json(message) && desktop.type == video_settings);
-        assert(desktop.vs.quality == quality && desktop.vs.frame_rate == rate && desktop.vs.preference == preference);
-        assert(desktop.vs.request_id == UINT32_MAX && !desktop.vs.accepted);
-        desktop.type = video_settings_status; desktop.vs.accepted = true;
-        RemoteAction reply{}; assert(reply.from_json(desktop.to_json()));
-        assert(reply.type == video_settings_status && reply.vs.accepted && reply.vs.request_id == UINT32_MAX);
-      }
-  MiniRtcNetTrafficStats stats{};
-  stats.video_inbound_stats.bitrate = 8'000'000; stats.video_inbound_stats.loss_rate = .02f;
-  stats.audio_inbound_stats.bitrate = 64'000; stats.audio_outbound_stats.bitrate = 128'000;
-  stats.data_inbound_stats.bitrate = 2000; stats.data_outbound_stats.bitrate = 3000;
-  stats.total_inbound_stats.bitrate = UINT32_MAX; stats.total_outbound_stats.bitrate = 131'000;
-  stats.total_inbound_stats.loss_rate = 1.2f; stats.rtt_ms = 40; stats.srtp_active = true;
-  auto json = android_controller::NetworkReport(stats, Relay);
-  assert(json.at("mode") == 1 && json.at("srtp") == true && json.at("rtt") == 40);
-  assert(json.at("video").at("inbound") == 8'000'000 && json.at("audio").at("outbound") == 128'000);
-  assert(json.at("data").at("inbound") == 2000 && json.at("data").at("outbound") == 3000);
-  assert(json.at("total").at("inbound").get<uint32_t>() == UINT32_MAX);
-  assert(std::abs(json.at("total").at("loss").get<double>() - 1.2) < .001);
-  stats.rtt_ms = -1; stats.srtp_active = false;
-  json = android_controller::NetworkReport(stats, UnknownMode);
-  assert(json.at("mode") == 2 && json.at("srtp") == false && json.at("rtt") == -1);
-}

apps/android/native/tests/video_pipeline_test.cpp

@@ -1,84 +0,0 @@
-#include "latest_frame_worker.h"
-#include "video_timing.h"
-#include "video_picture.h"
-#include <cassert>
-#include <chrono>
-#include <future>
-#include <memory>
-#include <vector>
-
-int main() {
-  using namespace crossdesk::android_controller;
-  using namespace std::chrono_literals;
-  assert(VideoLatencyMilliseconds(1'000'000, 1'020'000, 1'050'000) == 50);
-  // Reproduce a valid frame delayed in the local decode/Surface queue.
-  // The old submission-time five-second cutoff hid both of these samples.
-  assert(VideoLatencyMilliseconds(1'000'000, 1'020'000, 7'000'000) == 6000);
-  assert(VideoLatencyMilliseconds(1'000'000, 1'020'000, 31'000'000) == 30000);
-  assert(VideoLatencyMilliseconds(0, 1'020'000, 7'000'000) < 0);
-  assert(VideoLatencyMilliseconds(1'030'000, 1'020'000, 7'000'000) < 0);
-  assert(VideoLatencyMilliseconds(1'000'000, 6'000'001, 7'000'000) < 0);
-  assert(VideoLatencyMilliseconds(1'000'000, 1'020'000, 1'010'000) < 0);
-  assert(VideoLatencyMilliseconds(1'000'000, 1'020'000, -1) < 0);
-
-  struct Owner { int references = 1; uint8_t pixels[12]{}; } owner;
-  MiniRtcNativeVideoFrame native{};
-  native.struct_size = sizeof(native); native.type = MiniRtcNativeVideoFrameCpuNv12;
-  native.width = native.height = 2; native.owner = &owner;
-  native.retain = [](void* p) { ++static_cast<Owner*>(p)->references; };
-  native.release = [](void* p) { --static_cast<Owner*>(p)->references; };
-  native.payload.cpu_nv12 = {owner.pixels, owner.pixels + 8, 4, 4};
-  MiniRtcVideoFrame source{}; source.width = source.height = 2;
-  source.native_frame = &native; source.captured_timestamp = 100; source.received_timestamp = 120;
-  {
-    auto retained = VideoPicture::RetainOrCopy(source, 3);
-    assert(retained && owner.references == 2 && retained->pixels.empty());
-    assert(retained->Y() == owner.pixels && retained->UV() == owner.pixels + 8);
-    assert(retained->YStride() == 4 && retained->UVStride() == 4);
-    assert(retained->captured_us == 100 && retained->received_us == 120 && retained->generation == 3);
-    auto moved = std::move(*retained); retained.reset(); assert(owner.references == 2);
-  }
-  assert(owner.references == 1);
-  uint8_t borrowed[6]{1, 2, 3, 4, 5, 6};
-  source.native_frame = nullptr; source.data = reinterpret_cast<const char*>(borrowed); source.size = 6;
-  auto copied = VideoPicture::RetainOrCopy(source, 4); borrowed[0] = 0;
-  assert(copied && copied->Y()[0] == 1 && copied->UV()[0] == 5 && copied->YStride() == 2);
-  source.size = 5; assert(!VideoPicture::RetainOrCopy(source, 4));
-  source.width = 3; assert(!VideoPicture::RetainOrCopy(source, 4));
-
-  std::promise<void> started, release, finished;
-  auto released = release.get_future();
-  std::vector<int> rendered;
-  LatestFrameWorker<std::unique_ptr<int>> worker([&](std::unique_ptr<int> value) {
-    rendered.push_back(*value);
-    if (*value == 1) { started.set_value(); released.wait(); }
-    if (*value == 4) finished.set_value();
-  });
-  assert(worker.Submit(std::make_unique<int>(1)));
-  assert(started.get_future().wait_for(2s) == std::future_status::ready);
-  // A blocked Surface must not block frame submission; obsolete frames 2/3
-  // must not be displayed after it recovers, even with move-only ownership.
-  assert(worker.Submit(std::make_unique<int>(2)));
-  assert(worker.Submit(std::make_unique<int>(3)));
-  assert(worker.Submit(std::make_unique<int>(4)));
-  release.set_value();
-  assert(finished.get_future().wait_for(2s) == std::future_status::ready);
-  worker.Stop();
-  assert((rendered == std::vector<int>{1, 4}));
-  assert(!worker.Submit(std::make_unique<int>(5)));
-  worker.Stop();
-
-  std::promise<void> first_started, first_release;
-  auto first_released = first_release.get_future();
-  int count = 0;
-  LatestFrameWorker<int> cleared([&](int value) {
-    ++count; assert(value == 1); first_started.set_value(); first_released.wait();
-  });
-  cleared.Submit(1);
-  assert(first_started.get_future().wait_for(2s) == std::future_status::ready);
-  cleared.Submit(2);
-  cleared.Clear(); // Surface replacement/display switch discards pending pixels.
-  first_release.set_value();
-  cleared.Stop();
-  assert(count == 1);
-}

apps/android/native/video_picture.h

@@ -1,4 +1,11 @@
-#pragma once
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2026-10-02
+ * Copyright (c) 2026 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _VIDEO_PICTURE_H_
+#define _VIDEO_PICTURE_H_
 
 #include <minirtc.h>
 #include <climits>
@@ -14,6 +21,12 @@ struct VideoPicture {
   std::vector<uint8_t> pixels;
   std::shared_ptr<void> native_owner;
   MiniRtcCpuNv12Frame planes{};
+  MiniRtcAndroidMediaCodecFrame media_codec{};
+
+  bool IsSurface() const { return media_codec.render && native_owner; }
+  int RenderSurface(void* window) const {
+    return IsSurface() ? media_codec.render(native_owner.get(), window) : -1;
+  }
 
   const uint8_t* Y() const { return native_owner ? planes.y_plane : pixels.data(); }
   const uint8_t* UV() const { return native_owner ? planes.uv_plane : pixels.data() + size_t(width) * height; }
@@ -26,10 +39,17 @@ struct VideoPicture {
     if (!w || !h || w > 8192 || h > 8192 || (w & 1) || (h & 1) ||
         static_cast<uint64_t>(w) * h > 33554432) return std::nullopt;
     VideoPicture result{w, h, frame.captured_timestamp, frame.received_timestamp,
-                        generation, {}, {}, {}};
+                        generation, {}, {}, {}, {}};
     const auto* native = frame.native_frame;
     const bool valid_native = native && native->struct_size >= sizeof(*native) &&
         native->width == w && native->height == h && native->owner;
+    if (valid_native && native->type == MiniRtcNativeVideoFrameAndroidMediaCodec &&
+        native->retain && native->release && native->payload.android_media_codec.render) {
+      native->retain(native->owner);
+      result.native_owner = std::shared_ptr<void>(native->owner, native->release);
+      result.media_codec = native->payload.android_media_codec;
+      return result;
+    }
     if (valid_native && native->type == MiniRtcNativeVideoFrameCpuNv12 &&
         native->retain && native->release) {
       const auto& planes = native->payload.cpu_nv12;
@@ -56,3 +76,5 @@ struct VideoPicture {
   }
 };
 }  // namespace crossdesk::android_controller
+
+#endif

apps/android/native/video_timing.h

@@ -1,4 +1,11 @@
-#pragma once
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2026-10-02
+ * Copyright (c) 2026 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _VIDEO_TIMING_H_
+#define _VIDEO_TIMING_H_
 
 #include <cstdint>
 
@@ -16,3 +23,5 @@ inline double VideoLatencyMilliseconds(uint64_t captured_us,
   return (static_cast<uint64_t>(submitted_us) - captured_us) / 1000.0;
 }
 }  // namespace crossdesk::android_controller
+
+#endif

apps/android/native/xmake.lua

@@ -11,6 +11,8 @@ target("crossdesk_android")
     add_packages("nlohmann_json", "libyuv", "spdlog")
     add_files("rtc_bridge.cpp", "../../../libs/wire/src/remote_action.cpp", "../../../libs/wire/src/app_version.cpp")
     add_includedirs("../../../deps/submodules/minirtc/src/api", "../../../libs/wire/include")
+    -- This source-level JNI adapter is the only application layer using MiniRTC internals.
+    add_includedirs("../../../deps/submodules/minirtc/src/media/mediacodec")
     add_syslinks("android", "log")
     add_shflags("-Wl,-z,max-page-size=16384", "-Wl,-z,common-page-size=16384", {force = true})
     set_targetdir("../app/build/native/arm64-v8a")