mirror of
https://github.com/kunkundi/crossdesk.git
synced 2026-10-18 18:03:08 +08:00
[feat] add native Android controller
This commit is contained in:
dijunkun
@@ -0,0 +1,26 @@
|
||||
#pragma once
|
||||
|
||||
#include <minirtc.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <remote_action.h>
|
||||
|
||||
namespace crossdesk::android_controller {
|
||||
inline std::string VideoSettingsMessage(int quality, int frame_rate, int preference,
|
||||
uint32_t request_id) {
|
||||
RemoteAction action{};
|
||||
action.type = video_settings;
|
||||
action.vs = {quality, frame_rate, preference, request_id, false};
|
||||
return action.to_json();
|
||||
}
|
||||
|
||||
inline nlohmann::json NetworkReport(const MiniRtcNetTrafficStats& stats, TraversalMode mode) {
|
||||
const auto traffic = [](const MiniRtcInboundStats& in, const MiniRtcOutboundStats& out) {
|
||||
return nlohmann::json{{"inbound", in.bitrate}, {"outbound", out.bitrate}, {"loss", in.loss_rate}};
|
||||
};
|
||||
return {{"rtt", stats.rtt_ms}, {"mode", mode}, {"srtp", stats.srtp_active},
|
||||
{"video", traffic(stats.video_inbound_stats, stats.video_outbound_stats)},
|
||||
{"audio", traffic(stats.audio_inbound_stats, stats.audio_outbound_stats)},
|
||||
{"data", traffic(stats.data_inbound_stats, stats.data_outbound_stats)},
|
||||
{"total", traffic(stats.total_inbound_stats, stats.total_outbound_stats)}};
|
||||
}
|
||||
} // namespace crossdesk::android_controller
|
||||
@@ -0,0 +1,63 @@
|
||||
#pragma once
|
||||
|
||||
#include <condition_variable>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
|
||||
namespace crossdesk::android_controller {
|
||||
// Rendering may wait for Surface buffers. Never hold up the decoder or build
|
||||
// a FIFO of obsolete pictures: keep at most one pending decoded frame.
|
||||
template <typename Frame>
|
||||
class LatestFrameWorker {
|
||||
public:
|
||||
explicit LatestFrameWorker(std::function<void(Frame)> render)
|
||||
: render_(std::move(render)), thread_([this] { Run(); }) {}
|
||||
~LatestFrameWorker() { Stop(); }
|
||||
LatestFrameWorker(const LatestFrameWorker&) = delete;
|
||||
LatestFrameWorker& operator=(const LatestFrameWorker&) = delete;
|
||||
|
||||
bool Submit(Frame frame) {
|
||||
std::lock_guard lock(mutex_);
|
||||
if (stopped_) return false;
|
||||
pending_ = std::move(frame);
|
||||
changed_.notify_one();
|
||||
return true;
|
||||
}
|
||||
void Clear() {
|
||||
std::lock_guard lock(mutex_);
|
||||
pending_.reset();
|
||||
}
|
||||
// Called by the session owner, before destroying the peer or Surface.
|
||||
void Stop() {
|
||||
{
|
||||
std::lock_guard lock(mutex_);
|
||||
stopped_ = true;
|
||||
pending_.reset();
|
||||
}
|
||||
changed_.notify_one();
|
||||
if (thread_.joinable()) thread_.join();
|
||||
}
|
||||
|
||||
private:
|
||||
void Run() {
|
||||
for (;;) {
|
||||
std::unique_lock lock(mutex_);
|
||||
changed_.wait(lock, [this] { return stopped_ || pending_.has_value(); });
|
||||
if (stopped_) return;
|
||||
Frame frame = std::move(*pending_);
|
||||
pending_.reset();
|
||||
lock.unlock();
|
||||
render_(std::move(frame));
|
||||
}
|
||||
}
|
||||
const std::function<void(Frame)> render_;
|
||||
std::mutex mutex_;
|
||||
std::condition_variable changed_;
|
||||
std::optional<Frame> pending_;
|
||||
bool stopped_ = false;
|
||||
std::thread thread_;
|
||||
};
|
||||
} // namespace crossdesk::android_controller
|
||||
@@ -0,0 +1,382 @@
|
||||
#include <jni.h>
|
||||
#include <android/native_window_jni.h>
|
||||
#include <libyuv.h>
|
||||
#include <minirtc.h>
|
||||
#include <log.h>
|
||||
#include <remote_action.h>
|
||||
#include <app_version.h>
|
||||
#include <stream_names.h>
|
||||
#include <display_stream_id.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <atomic>
|
||||
#include <algorithm>
|
||||
#include <cstring>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <stdexcept>
|
||||
#include <vector>
|
||||
#include "controller_protocol.h"
|
||||
#include "latest_frame_worker.h"
|
||||
#include "video_picture.h"
|
||||
#include "video_timing.h"
|
||||
|
||||
namespace {
|
||||
using namespace crossdesk;
|
||||
using nlohmann::json;
|
||||
JavaVM* vm = nullptr;
|
||||
struct Environment {
|
||||
JNIEnv* env = nullptr;
|
||||
bool attached = false;
|
||||
Environment() {
|
||||
if (vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6) != JNI_OK) {
|
||||
attached = vm->AttachCurrentThread(&env, nullptr) == JNI_OK;
|
||||
}
|
||||
}
|
||||
~Environment() { if (attached) vm->DetachCurrentThread(); }
|
||||
};
|
||||
std::string String(JNIEnv* env, jstring value) {
|
||||
if (!value) return {};
|
||||
const char* bytes = env->GetStringUTFChars(value, nullptr);
|
||||
std::string result = bytes ? bytes : "";
|
||||
if (bytes) env->ReleaseStringUTFChars(value, bytes);
|
||||
return result;
|
||||
}
|
||||
struct Session;
|
||||
using android_controller::VideoPicture;
|
||||
void RenderVideo(Session* s, VideoPicture picture);
|
||||
struct CallbackContext { Session* owner; bool controller; };
|
||||
struct Session {
|
||||
jobject owner = nullptr;
|
||||
jmethodID event_method, audio_method, frame_method;
|
||||
PeerPtr* identity = nullptr;
|
||||
PeerPtr* controller = nullptr;
|
||||
CallbackContext identity_context{this, false}, controller_context{this, true};
|
||||
std::string host, log_path, identity_login, controller_login, remote;
|
||||
int port;
|
||||
std::atomic<bool> stopping{false};
|
||||
std::atomic<int> display{0};
|
||||
std::atomic<bool> surface_ready{false};
|
||||
std::atomic<uint64_t> video_generation{0};
|
||||
std::mutex render_mutex;
|
||||
ANativeWindow* window = nullptr;
|
||||
uint32_t width = 0, height = 0;
|
||||
uint64_t frame_id = 0;
|
||||
std::set<int> pressed_keys;
|
||||
unsigned pressed_buttons = 0;
|
||||
float pointer_x = .5f, pointer_y = .5f;
|
||||
android_controller::LatestFrameWorker<VideoPicture> renderer{
|
||||
[this](VideoPicture picture) { RenderVideo(this, std::move(picture)); }};
|
||||
|
||||
void Event(int type, const char* data, size_t size) {
|
||||
if (stopping || size > 1024 * 1024) return;
|
||||
Environment scope;
|
||||
if (!scope.env) return;
|
||||
jbyteArray bytes = scope.env->NewByteArray(static_cast<jsize>(size));
|
||||
if (!bytes) return;
|
||||
scope.env->SetByteArrayRegion(bytes, 0, size, reinterpret_cast<const jbyte*>(data));
|
||||
scope.env->CallVoidMethod(owner, event_method, type, bytes);
|
||||
scope.env->DeleteLocalRef(bytes);
|
||||
if (scope.env->ExceptionCheck()) scope.env->ExceptionClear();
|
||||
}
|
||||
void Event(int type, const json& value) {
|
||||
const std::string data = value.dump(); Event(type, data.data(), data.size());
|
||||
}
|
||||
void Send(const char* stream, const std::string& data) {
|
||||
if (controller && !stopping && !data.empty())
|
||||
SendReliableDataFrame(controller, data.data(), data.size(), stream);
|
||||
}
|
||||
void Surface(JNIEnv* env, jobject surface) {
|
||||
std::lock_guard<std::mutex> lock(render_mutex);
|
||||
++video_generation;
|
||||
renderer.Clear();
|
||||
if (window) ANativeWindow_release(window);
|
||||
window = surface ? ANativeWindow_fromSurface(env, surface) : nullptr;
|
||||
surface_ready = window != nullptr;
|
||||
width = height = 0;
|
||||
}
|
||||
~Session() {
|
||||
// API calls and destruction share the Java RTC executor. Release any
|
||||
// delivered input even when queued Java key-up events were cancelled.
|
||||
if (controller) {
|
||||
for (int key : pressed_keys) {
|
||||
RemoteAction action{}; action.type = keyboard;
|
||||
action.k = {static_cast<size_t>(key), 0, false, key_up};
|
||||
Send(kKeyboardStream, action.to_json());
|
||||
}
|
||||
for (int button = 0; button < 3; ++button) {
|
||||
if (!(pressed_buttons & (1u << button))) continue;
|
||||
RemoteAction action{}; action.type = mouse;
|
||||
action.m = {pointer_x, pointer_y, 0, static_cast<MouseFlag>(2 + button * 2)};
|
||||
Send(kMouseStream, action.to_json());
|
||||
}
|
||||
}
|
||||
stopping = true;
|
||||
renderer.Stop();
|
||||
if (controller) {
|
||||
LeaveConnection(controller, remote.c_str());
|
||||
DestroyPeer(&controller);
|
||||
}
|
||||
if (identity) DestroyPeer(&identity);
|
||||
Environment scope;
|
||||
Surface(scope.env, nullptr);
|
||||
if (scope.env) scope.env->DeleteGlobalRef(owner);
|
||||
}
|
||||
};
|
||||
|
||||
void Signal(SignalStatus status, const char*, size_t, void* user) {
|
||||
auto* ctx = static_cast<CallbackContext*>(user);
|
||||
ctx->owner->Event(1, {{"controller", ctx->controller}, {"status", status}});
|
||||
}
|
||||
void SignalMessage(const char* data, size_t size, void* user) {
|
||||
auto* ctx = static_cast<CallbackContext*>(user);
|
||||
if (ctx->controller || !data || !size || size > 1024 * 1024) return;
|
||||
auto message = json::parse(data, data + size, nullptr, false);
|
||||
if (!message.is_object() || !message.contains("type") || !message["type"].is_string()) return;
|
||||
const auto type = message["type"].get<std::string>();
|
||||
if (type == "announcements" || type == "announcements_changed") ctx->owner->Event(8, data, size);
|
||||
}
|
||||
void Connection(ConnectionStatus status, const char*, size_t, void* user) {
|
||||
auto* ctx = static_cast<CallbackContext*>(user);
|
||||
if (ctx->controller) ctx->owner->Event(2, {{"status", status}});
|
||||
}
|
||||
void Network(const char* id, size_t size, TraversalMode mode,
|
||||
const MiniRtcNetTrafficStats* stats, const char*, size_t, void* user) {
|
||||
auto* ctx = static_cast<CallbackContext*>(user);
|
||||
if (!ctx->controller && mode == UnknownMode && id && size) {
|
||||
ctx->owner->Event(3, id, size);
|
||||
} else if (ctx->controller && stats) {
|
||||
ctx->owner->Event(7, android_controller::NetworkReport(*stats, mode));
|
||||
}
|
||||
}
|
||||
void Data(const char* data, size_t size, const char*, size_t,
|
||||
const char* stream, size_t stream_size, void* user) {
|
||||
auto* ctx = static_cast<CallbackContext*>(user);
|
||||
if (!ctx->controller || !data || !stream || size > kMaxClipboardBytes) return;
|
||||
const std::string name(stream, stream_size);
|
||||
if (name == kControlStream) {
|
||||
RemoteAction action{};
|
||||
if (action.from_json(std::string(data, size))) {
|
||||
const auto message = action.to_json();
|
||||
ctx->owner->Event(4, message.data(), message.size());
|
||||
FreeRemoteAction(action);
|
||||
}
|
||||
}
|
||||
if (name == kClipboardStream) ctx->owner->Event(6, data, size);
|
||||
}
|
||||
void Audio(const char* data, size_t size, const char*, size_t,
|
||||
const char*, size_t, void* user) {
|
||||
auto* ctx = static_cast<CallbackContext*>(user);
|
||||
auto* s = ctx->owner;
|
||||
if (!ctx->controller || s->stopping || !data || !size || size > 23040) return;
|
||||
Environment scope;
|
||||
if (!scope.env) return;
|
||||
auto bytes = scope.env->NewByteArray(size);
|
||||
if (!bytes) return;
|
||||
scope.env->SetByteArrayRegion(bytes, 0, size, reinterpret_cast<const jbyte*>(data));
|
||||
scope.env->CallVoidMethod(s->owner, s->audio_method, bytes);
|
||||
scope.env->DeleteLocalRef(bytes);
|
||||
if (scope.env->ExceptionCheck()) scope.env->ExceptionClear();
|
||||
}
|
||||
void Video(const MiniRtcVideoFrame* frame, const char*, size_t,
|
||||
const char* stream, size_t stream_size, void* user) {
|
||||
auto* ctx = static_cast<CallbackContext*>(user);
|
||||
auto* s = ctx->owner;
|
||||
const auto generation = s->video_generation.load();
|
||||
if (!ctx->controller || s->stopping || !s->surface_ready || !frame || !stream ||
|
||||
std::string(stream, stream_size) != MakeDisplayStreamId(s->display.load())) return;
|
||||
auto picture = VideoPicture::RetainOrCopy(*frame, generation);
|
||||
if (picture) s->renderer.Submit(std::move(*picture));
|
||||
}
|
||||
void RenderVideo(Session* s, VideoPicture picture) {
|
||||
const auto w = picture.width, h = picture.height;
|
||||
std::lock_guard<std::mutex> lock(s->render_mutex);
|
||||
if (s->stopping || !s->window || picture.generation != s->video_generation) return;
|
||||
if (s->width != w || s->height != h) {
|
||||
if (ANativeWindow_setBuffersGeometry(s->window, w, h, WINDOW_FORMAT_RGBA_8888) != 0) return;
|
||||
s->width = w; s->height = h;
|
||||
s->Event(5, {{"width", w}, {"height", h}});
|
||||
}
|
||||
ANativeWindow_Buffer buffer{};
|
||||
if (ANativeWindow_lock(s->window, &buffer, nullptr) != 0) return;
|
||||
const bool rendered = buffer.width == static_cast<int>(w) && buffer.height == static_cast<int>(h) &&
|
||||
libyuv::NV12ToABGR(picture.Y(), picture.YStride(), picture.UV(), picture.UVStride(),
|
||||
static_cast<uint8_t*>(buffer.bits), buffer.stride * 4, w, h) == 0;
|
||||
if (ANativeWindow_unlockAndPost(s->window) != 0 || !rendered) return;
|
||||
// Measure through display submission, using MiniRTC's calibrated capture clock.
|
||||
const auto now = GetSystemTimeMicros(s->controller);
|
||||
const double latency = android_controller::VideoLatencyMilliseconds(
|
||||
picture.captured_us, picture.received_us, now);
|
||||
Environment scope;
|
||||
if (scope.env) {
|
||||
scope.env->CallVoidMethod(s->owner, s->frame_method, static_cast<jlong>(++s->frame_id), latency, static_cast<jint>(w), static_cast<jint>(h));
|
||||
if (scope.env->ExceptionCheck()) scope.env->ExceptionClear();
|
||||
}
|
||||
}
|
||||
Params Parameters(Session* s, bool controller) {
|
||||
Params p{};
|
||||
std::snprintf(p.signal_server_ip, sizeof(p.signal_server_ip), "%s", s->host.c_str());
|
||||
std::snprintf(p.log_path, sizeof(p.log_path), "%s", s->log_path.c_str());
|
||||
p.signal_server_port = s->port;
|
||||
p.hardware_acceleration = false;
|
||||
// OpenH264/dav1d expose pooled CPU NV12 buffers on Android. Retain them
|
||||
// across display submission instead of copying full-resolution pixels.
|
||||
p.native_video_output = controller;
|
||||
p.turn_mode = TurnAutoUdpTcp;
|
||||
p.enable_srtp = true;
|
||||
p.video_content_type = VideoContentType::ScreenContent;
|
||||
p.video_quality = QualityMedium;
|
||||
p.video_frame_rate = 30;
|
||||
p.video_degradation_preference = VideoDegradationPreference::Balanced;
|
||||
p.on_signal_status = Signal; p.on_connection_status = Connection;
|
||||
p.on_signal_message = SignalMessage;
|
||||
p.on_net_status_report = Network; p.on_receive_data_buffer = Data;
|
||||
p.on_receive_video_frame = Video; p.on_receive_audio_buffer = Audio;
|
||||
p.user_id = (controller ? s->controller_login : s->identity_login).c_str();
|
||||
p.user_data = controller ? &s->controller_context : &s->identity_context;
|
||||
return p;
|
||||
}
|
||||
Session* From(jlong handle) { return reinterpret_cast<Session*>(handle); }
|
||||
} // namespace
|
||||
|
||||
extern "C" JNIEXPORT jint JNI_OnLoad(JavaVM* java_vm, void*) {
|
||||
vm = java_vm; return JNI_VERSION_1_6;
|
||||
}
|
||||
#define JNI_METHOD(name) Java_cn_crossdesk_mobile_NativeSession_##name
|
||||
extern "C" JNIEXPORT jboolean JNICALL JNI_METHOD(nValidAppVersion)(
|
||||
JNIEnv* env, jclass, jstring version) {
|
||||
return version && env->GetStringUTFLength(version) < 64 &&
|
||||
crossdesk::IsValidAppVersion(String(env, version));
|
||||
}
|
||||
extern "C" JNIEXPORT jboolean JNICALL JNI_METHOD(nHasAppUpdate)(
|
||||
JNIEnv* env, jclass, jstring version, jbyteArray release_json) {
|
||||
if (!release_json) return false;
|
||||
const auto size = env->GetArrayLength(release_json);
|
||||
if (size <= 0 || size > 256 * 1024) return false;
|
||||
std::string data(size, '\0');
|
||||
env->GetByteArrayRegion(release_json, 0, size, reinterpret_cast<jbyte*>(data.data()));
|
||||
if (env->ExceptionCheck()) return false;
|
||||
const auto latest = crossdesk::ParseVersionInfoJSON(data);
|
||||
return latest && crossdesk::AvailableAppUpdate(String(env, version), *latest).has_value();
|
||||
}
|
||||
extern "C" JNIEXPORT jlong JNICALL JNI_METHOD(nCreate)(
|
||||
JNIEnv* env, jclass, jobject owner, jstring host, jint port, jstring identity,
|
||||
jstring log_path, jstring certificates) {
|
||||
try {
|
||||
auto s = std::make_unique<Session>();
|
||||
s->owner = env->NewGlobalRef(owner);
|
||||
auto cls = env->GetObjectClass(owner);
|
||||
s->event_method = env->GetMethodID(cls, "onNativeEvent", "(I[B)V");
|
||||
s->audio_method = env->GetMethodID(cls, "onNativeAudio", "([B)V");
|
||||
s->frame_method = env->GetMethodID(cls, "onNativeVideoFrame", "(JDII)V");
|
||||
env->DeleteLocalRef(cls);
|
||||
s->host = String(env, host); s->port = port;
|
||||
s->identity_login = String(env, identity); s->log_path = String(env, log_path);
|
||||
// The bundle is exported from Android's default TrustManager, never the
|
||||
// build machine's certificate store. The Android MiniRTC patch explicitly
|
||||
// loads this path: OpenSSL's default loader ignores SSL_CERT_FILE in an
|
||||
// Android app process (AT_SECURE=1). Chain and hostname checks stay enabled.
|
||||
const auto bundle = String(env, certificates);
|
||||
if (bundle.empty() || setenv("SSL_CERT_FILE", bundle.c_str(), 1) != 0)
|
||||
throw std::runtime_error("Unable to configure Android trust bundle");
|
||||
// MiniRTC's diagnostic log includes login identities. Keep credentials
|
||||
// solely in Keystore-backed storage; do not persist its session logging.
|
||||
minirtc::InitLogger(s->log_path);
|
||||
minirtc::get_logger()->set_level(spdlog::level::off);
|
||||
auto params = Parameters(s.get(), false);
|
||||
s->identity = CreatePeer(¶ms);
|
||||
if (!s->identity || Init(s->identity) != 0) return 0;
|
||||
return reinterpret_cast<jlong>(s.release());
|
||||
} catch (const std::exception&) { return 0; }
|
||||
}
|
||||
extern "C" JNIEXPORT jboolean JNICALL JNI_METHOD(nConnect)(
|
||||
JNIEnv* env, jclass, jlong handle, jstring identity, jstring remote, jstring password) {
|
||||
auto* s = From(handle);
|
||||
if (!s || s->controller) return false;
|
||||
s->controller_login = "C-" + String(env, identity);
|
||||
s->remote = String(env, remote);
|
||||
auto params = Parameters(s, true);
|
||||
s->controller = CreatePeer(¶ms);
|
||||
if (!s->controller) return false;
|
||||
AddAudioStream(s->controller, kAudioStream);
|
||||
AddDataStream(s->controller, kDataStream, false);
|
||||
for (const char* name : {kMouseStream, kKeyboardStream, kControlStream,
|
||||
kClipboardStream}) AddDataStream(s->controller, name, true);
|
||||
if (Init(s->controller) != 0) return false;
|
||||
const auto join = s->remote + "@" + String(env, password);
|
||||
return JoinConnection(s->controller, join.c_str()) == 0;
|
||||
}
|
||||
extern "C" JNIEXPORT void JNICALL JNI_METHOD(nReady)(JNIEnv*, jclass, jlong handle) {
|
||||
auto* s = From(handle); if (!s || !s->controller) return;
|
||||
const std::string info = R"({"type":"client_info","version":"android-native","platform":"android"})";
|
||||
SendSignalMessage(s->controller, info.data(), info.size());
|
||||
auto action = MakeHostInformation("CrossDesk Android", {}, false, "0.1.0", HostPlatform::Android);
|
||||
s->Send(kControlStream, action.to_json()); FreeRemoteAction(action);
|
||||
}
|
||||
extern "C" JNIEXPORT void JNICALL JNI_METHOD(nPointer)(
|
||||
JNIEnv*, jclass, jlong handle, jfloat x, jfloat y, jint flag, jint wheel) {
|
||||
auto* s = From(handle); if (!s || flag < 0 || flag > 8) return;
|
||||
RemoteAction action{}; action.type = mouse;
|
||||
action.m = {std::clamp(x, 0.f, 1.f), std::clamp(y, 0.f, 1.f), wheel, static_cast<MouseFlag>(flag)};
|
||||
s->pointer_x = action.m.x; s->pointer_y = action.m.y;
|
||||
if (flag >= 1 && flag <= 6) {
|
||||
const unsigned bit = 1u << ((flag - 1) / 2);
|
||||
if (flag & 1) s->pressed_buttons |= bit; else s->pressed_buttons &= ~bit;
|
||||
}
|
||||
s->Send(kMouseStream, action.to_json());
|
||||
}
|
||||
extern "C" JNIEXPORT void JNICALL JNI_METHOD(nKey)(JNIEnv*, jclass, jlong handle, jint key, jboolean down) {
|
||||
auto* s = From(handle); if (!s) return;
|
||||
RemoteAction action{}; action.type = keyboard;
|
||||
action.k = {static_cast<size_t>(key), 0, false, down ? key_down : key_up};
|
||||
if (down) s->pressed_keys.insert(key); else s->pressed_keys.erase(key);
|
||||
s->Send(kKeyboardStream, action.to_json());
|
||||
}
|
||||
extern "C" JNIEXPORT void JNICALL JNI_METHOD(nControl)(
|
||||
JNIEnv*, jclass, jlong handle, jint type, jint value) {
|
||||
auto* s = From(handle); if (!s) return;
|
||||
RemoteAction action{};
|
||||
if (type == display_id && value >= 0 && value < 8) {
|
||||
{ std::lock_guard<std::mutex> lock(s->render_mutex); s->display = value; ++s->video_generation; s->renderer.Clear(); s->width = s->height = 0; }
|
||||
action.type = display_id; action.d = value;
|
||||
} else if (type == audio_capture) {
|
||||
action.type = audio_capture; action.a = value != 0;
|
||||
} else return;
|
||||
s->Send(kControlStream, action.to_json());
|
||||
}
|
||||
extern "C" JNIEXPORT jboolean JNICALL JNI_METHOD(nVideoSettings)(
|
||||
JNIEnv*, jclass, jlong handle, jint quality, jint rate, jint preference, jlong request_id) {
|
||||
auto* s = From(handle);
|
||||
if (!s || !s->controller || s->stopping || request_id < 0 || request_id > UINT32_MAX) return false;
|
||||
const auto data = android_controller::VideoSettingsMessage(quality, rate, preference, static_cast<uint32_t>(request_id));
|
||||
if (data.empty()) return false;
|
||||
return SendReliableDataFrame(s->controller, data.data(), data.size(), kControlStream) == 0;
|
||||
}
|
||||
extern "C" JNIEXPORT jboolean JNICALL JNI_METHOD(nAnnouncementRequest)(
|
||||
JNIEnv* env, jclass, jlong handle, jbyteArray data) {
|
||||
auto* s = From(handle);
|
||||
if (!s || !s->identity || s->stopping || !data) return false;
|
||||
const auto size = env->GetArrayLength(data);
|
||||
if (size <= 0 || size > 4096) return false;
|
||||
std::string bytes(size, '\0');
|
||||
env->GetByteArrayRegion(data, 0, size, reinterpret_cast<jbyte*>(bytes.data()));
|
||||
if (env->ExceptionCheck()) return false;
|
||||
auto message = json::parse(bytes, nullptr, false);
|
||||
if (!message.is_object() || !message.contains("type") ||
|
||||
!message["type"].is_string() || message["type"] != "announcements_list") return false;
|
||||
return SendSignalMessage(s->identity, bytes.data(), bytes.size()) == 0;
|
||||
}
|
||||
extern "C" JNIEXPORT void JNICALL JNI_METHOD(nClipboard)(
|
||||
JNIEnv* env, jclass, jlong handle, jbyteArray data) {
|
||||
auto* s = From(handle); if (!s || !data) return;
|
||||
const auto size = env->GetArrayLength(data);
|
||||
if (!size || size > static_cast<int>(kMaxClipboardBytes)) return;
|
||||
std::string bytes(size, '\0');
|
||||
env->GetByteArrayRegion(data, 0, size, reinterpret_cast<jbyte*>(bytes.data()));
|
||||
s->Send(kClipboardStream, bytes);
|
||||
}
|
||||
extern "C" JNIEXPORT void JNICALL JNI_METHOD(nSurface)(JNIEnv* env, jclass, jlong handle, jobject surface) {
|
||||
if (auto* s = From(handle)) s->Surface(env, surface);
|
||||
}
|
||||
extern "C" JNIEXPORT void JNICALL JNI_METHOD(nDestroy)(JNIEnv*, jclass, jlong handle) { delete From(handle); }
|
||||
@@ -0,0 +1,39 @@
|
||||
#include "controller_protocol.h"
|
||||
#include <cassert>
|
||||
#include <cmath>
|
||||
#include <limits>
|
||||
|
||||
int main() {
|
||||
using namespace crossdesk;
|
||||
for (int quality = -1; quality <= 3; ++quality)
|
||||
for (int rate : {0, 24, 30, 60, 120})
|
||||
for (int preference = -1; preference <= 3; ++preference) {
|
||||
const auto message = android_controller::VideoSettingsMessage(quality, rate, preference, UINT32_MAX);
|
||||
const bool valid = quality >= 0 && quality <= 2 && (rate == 30 || rate == 60) && preference >= 0 && preference <= 2;
|
||||
assert(message.empty() != valid);
|
||||
if (!valid) continue;
|
||||
// Decode the actual Android outgoing message with the desktop's wire parser.
|
||||
RemoteAction desktop{};
|
||||
assert(desktop.from_json(message) && desktop.type == video_settings);
|
||||
assert(desktop.vs.quality == quality && desktop.vs.frame_rate == rate && desktop.vs.preference == preference);
|
||||
assert(desktop.vs.request_id == UINT32_MAX && !desktop.vs.accepted);
|
||||
desktop.type = video_settings_status; desktop.vs.accepted = true;
|
||||
RemoteAction reply{}; assert(reply.from_json(desktop.to_json()));
|
||||
assert(reply.type == video_settings_status && reply.vs.accepted && reply.vs.request_id == UINT32_MAX);
|
||||
}
|
||||
MiniRtcNetTrafficStats stats{};
|
||||
stats.video_inbound_stats.bitrate = 8'000'000; stats.video_inbound_stats.loss_rate = .02f;
|
||||
stats.audio_inbound_stats.bitrate = 64'000; stats.audio_outbound_stats.bitrate = 128'000;
|
||||
stats.data_inbound_stats.bitrate = 2000; stats.data_outbound_stats.bitrate = 3000;
|
||||
stats.total_inbound_stats.bitrate = UINT32_MAX; stats.total_outbound_stats.bitrate = 131'000;
|
||||
stats.total_inbound_stats.loss_rate = 1.2f; stats.rtt_ms = 40; stats.srtp_active = true;
|
||||
auto json = android_controller::NetworkReport(stats, Relay);
|
||||
assert(json.at("mode") == 1 && json.at("srtp") == true && json.at("rtt") == 40);
|
||||
assert(json.at("video").at("inbound") == 8'000'000 && json.at("audio").at("outbound") == 128'000);
|
||||
assert(json.at("data").at("inbound") == 2000 && json.at("data").at("outbound") == 3000);
|
||||
assert(json.at("total").at("inbound").get<uint32_t>() == UINT32_MAX);
|
||||
assert(std::abs(json.at("total").at("loss").get<double>() - 1.2) < .001);
|
||||
stats.rtt_ms = -1; stats.srtp_active = false;
|
||||
json = android_controller::NetworkReport(stats, UnknownMode);
|
||||
assert(json.at("mode") == 2 && json.at("srtp") == false && json.at("rtt") == -1);
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
#include "latest_frame_worker.h"
|
||||
#include "video_timing.h"
|
||||
#include "video_picture.h"
|
||||
#include <cassert>
|
||||
#include <chrono>
|
||||
#include <future>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
int main() {
|
||||
using namespace crossdesk::android_controller;
|
||||
using namespace std::chrono_literals;
|
||||
assert(VideoLatencyMilliseconds(1'000'000, 1'020'000, 1'050'000) == 50);
|
||||
// Reproduce a valid frame delayed in the local decode/Surface queue.
|
||||
// The old submission-time five-second cutoff hid both of these samples.
|
||||
assert(VideoLatencyMilliseconds(1'000'000, 1'020'000, 7'000'000) == 6000);
|
||||
assert(VideoLatencyMilliseconds(1'000'000, 1'020'000, 31'000'000) == 30000);
|
||||
assert(VideoLatencyMilliseconds(0, 1'020'000, 7'000'000) < 0);
|
||||
assert(VideoLatencyMilliseconds(1'030'000, 1'020'000, 7'000'000) < 0);
|
||||
assert(VideoLatencyMilliseconds(1'000'000, 6'000'001, 7'000'000) < 0);
|
||||
assert(VideoLatencyMilliseconds(1'000'000, 1'020'000, 1'010'000) < 0);
|
||||
assert(VideoLatencyMilliseconds(1'000'000, 1'020'000, -1) < 0);
|
||||
|
||||
struct Owner { int references = 1; uint8_t pixels[12]{}; } owner;
|
||||
MiniRtcNativeVideoFrame native{};
|
||||
native.struct_size = sizeof(native); native.type = MiniRtcNativeVideoFrameCpuNv12;
|
||||
native.width = native.height = 2; native.owner = &owner;
|
||||
native.retain = [](void* p) { ++static_cast<Owner*>(p)->references; };
|
||||
native.release = [](void* p) { --static_cast<Owner*>(p)->references; };
|
||||
native.payload.cpu_nv12 = {owner.pixels, owner.pixels + 8, 4, 4};
|
||||
MiniRtcVideoFrame source{}; source.width = source.height = 2;
|
||||
source.native_frame = &native; source.captured_timestamp = 100; source.received_timestamp = 120;
|
||||
{
|
||||
auto retained = VideoPicture::RetainOrCopy(source, 3);
|
||||
assert(retained && owner.references == 2 && retained->pixels.empty());
|
||||
assert(retained->Y() == owner.pixels && retained->UV() == owner.pixels + 8);
|
||||
assert(retained->YStride() == 4 && retained->UVStride() == 4);
|
||||
assert(retained->captured_us == 100 && retained->received_us == 120 && retained->generation == 3);
|
||||
auto moved = std::move(*retained); retained.reset(); assert(owner.references == 2);
|
||||
}
|
||||
assert(owner.references == 1);
|
||||
uint8_t borrowed[6]{1, 2, 3, 4, 5, 6};
|
||||
source.native_frame = nullptr; source.data = reinterpret_cast<const char*>(borrowed); source.size = 6;
|
||||
auto copied = VideoPicture::RetainOrCopy(source, 4); borrowed[0] = 0;
|
||||
assert(copied && copied->Y()[0] == 1 && copied->UV()[0] == 5 && copied->YStride() == 2);
|
||||
source.size = 5; assert(!VideoPicture::RetainOrCopy(source, 4));
|
||||
source.width = 3; assert(!VideoPicture::RetainOrCopy(source, 4));
|
||||
|
||||
std::promise<void> started, release, finished;
|
||||
auto released = release.get_future();
|
||||
std::vector<int> rendered;
|
||||
LatestFrameWorker<std::unique_ptr<int>> worker([&](std::unique_ptr<int> value) {
|
||||
rendered.push_back(*value);
|
||||
if (*value == 1) { started.set_value(); released.wait(); }
|
||||
if (*value == 4) finished.set_value();
|
||||
});
|
||||
assert(worker.Submit(std::make_unique<int>(1)));
|
||||
assert(started.get_future().wait_for(2s) == std::future_status::ready);
|
||||
// A blocked Surface must not block frame submission; obsolete frames 2/3
|
||||
// must not be displayed after it recovers, even with move-only ownership.
|
||||
assert(worker.Submit(std::make_unique<int>(2)));
|
||||
assert(worker.Submit(std::make_unique<int>(3)));
|
||||
assert(worker.Submit(std::make_unique<int>(4)));
|
||||
release.set_value();
|
||||
assert(finished.get_future().wait_for(2s) == std::future_status::ready);
|
||||
worker.Stop();
|
||||
assert((rendered == std::vector<int>{1, 4}));
|
||||
assert(!worker.Submit(std::make_unique<int>(5)));
|
||||
worker.Stop();
|
||||
|
||||
std::promise<void> first_started, first_release;
|
||||
auto first_released = first_release.get_future();
|
||||
int count = 0;
|
||||
LatestFrameWorker<int> cleared([&](int value) {
|
||||
++count; assert(value == 1); first_started.set_value(); first_released.wait();
|
||||
});
|
||||
cleared.Submit(1);
|
||||
assert(first_started.get_future().wait_for(2s) == std::future_status::ready);
|
||||
cleared.Submit(2);
|
||||
cleared.Clear(); // Surface replacement/display switch discards pending pixels.
|
||||
first_release.set_value();
|
||||
cleared.Stop();
|
||||
assert(count == 1);
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
#pragma once
|
||||
|
||||
#include <minirtc.h>
|
||||
#include <climits>
|
||||
#include <cstring>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
|
||||
namespace crossdesk::android_controller {
|
||||
struct VideoPicture {
|
||||
uint32_t width, height;
|
||||
uint64_t captured_us, received_us, generation;
|
||||
std::vector<uint8_t> pixels;
|
||||
std::shared_ptr<void> native_owner;
|
||||
MiniRtcCpuNv12Frame planes{};
|
||||
|
||||
const uint8_t* Y() const { return native_owner ? planes.y_plane : pixels.data(); }
|
||||
const uint8_t* UV() const { return native_owner ? planes.uv_plane : pixels.data() + size_t(width) * height; }
|
||||
int YStride() const { return native_owner ? planes.y_stride : width; }
|
||||
int UVStride() const { return native_owner ? planes.uv_stride : width; }
|
||||
|
||||
static std::optional<VideoPicture> RetainOrCopy(const MiniRtcVideoFrame& frame,
|
||||
uint64_t generation) {
|
||||
const auto w = frame.width, h = frame.height;
|
||||
if (!w || !h || w > 8192 || h > 8192 || (w & 1) || (h & 1) ||
|
||||
static_cast<uint64_t>(w) * h > 33554432) return std::nullopt;
|
||||
VideoPicture result{w, h, frame.captured_timestamp, frame.received_timestamp,
|
||||
generation, {}, {}, {}};
|
||||
const auto* native = frame.native_frame;
|
||||
const bool valid_native = native && native->struct_size >= sizeof(*native) &&
|
||||
native->width == w && native->height == h && native->owner;
|
||||
if (valid_native && native->type == MiniRtcNativeVideoFrameCpuNv12 &&
|
||||
native->retain && native->release) {
|
||||
const auto& planes = native->payload.cpu_nv12;
|
||||
if (planes.y_plane && planes.uv_plane && planes.y_stride >= w &&
|
||||
planes.uv_stride >= w && planes.y_stride <= INT_MAX && planes.uv_stride <= INT_MAX) {
|
||||
native->retain(native->owner);
|
||||
result.native_owner = std::shared_ptr<void>(native->owner, native->release);
|
||||
result.planes = planes;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
const size_t size = size_t(w) * h * 3 / 2;
|
||||
if (frame.data && frame.size >= size) {
|
||||
result.pixels.resize(size);
|
||||
std::memcpy(result.pixels.data(), frame.data, size);
|
||||
} else if (valid_native && native->copy_to_nv12) {
|
||||
result.pixels.resize(size);
|
||||
if (native->copy_to_nv12(native->owner, result.pixels.data(), size) != 0)
|
||||
return std::nullopt;
|
||||
} else {
|
||||
return std::nullopt;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
};
|
||||
} // namespace crossdesk::android_controller
|
||||
@@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace crossdesk::android_controller {
|
||||
inline double VideoLatencyMilliseconds(uint64_t captured_us,
|
||||
uint64_t received_us,
|
||||
int64_t submitted_us) {
|
||||
// MiniRTC calibrates capture time when the frame is reassembled. Validate
|
||||
// that mapping at reception, not after decoding/Surface waits: a real local
|
||||
// backlog over five seconds must remain visible in the latency measurement.
|
||||
if (!captured_us || received_us < captured_us || submitted_us <= 0 ||
|
||||
received_us > static_cast<uint64_t>(submitted_us) ||
|
||||
received_us - captured_us > 5'000'000)
|
||||
return -1;
|
||||
return (static_cast<uint64_t>(submitted_us) - captured_us) / 1000.0;
|
||||
}
|
||||
} // namespace crossdesk::android_controller
|
||||
@@ -0,0 +1,16 @@
|
||||
set_project("crossdesk_android")
|
||||
set_languages("c++17")
|
||||
add_rules("mode.release", "mode.debug")
|
||||
set_policy("package.precompiled", false)
|
||||
add_cxflags("-fPIC")
|
||||
includes("../../../deps/submodules/minirtc")
|
||||
|
||||
target("crossdesk_android")
|
||||
set_kind("shared")
|
||||
add_deps("minirtc")
|
||||
add_packages("nlohmann_json", "libyuv", "spdlog")
|
||||
add_files("rtc_bridge.cpp", "../../../libs/wire/src/remote_action.cpp", "../../../libs/wire/src/app_version.cpp")
|
||||
add_includedirs("../../../deps/submodules/minirtc/src/api", "../../../libs/wire/include")
|
||||
add_syslinks("android", "log")
|
||||
add_shflags("-Wl,-z,max-page-size=16384", "-Wl,-z,common-page-size=16384", {force = true})
|
||||
set_targetdir("../app/build/native/arm64-v8a")
|
||||
Reference in New Issue
Block a user