Remove dependency on FFmpeg

This commit is contained in:
dijunkun
2024-05-17 17:55:57 +08:00
parent d58ae3a6b1
commit 1393615f01
5 changed files with 539 additions and 506 deletions

View File

@@ -8,6 +8,8 @@
#endif
#endif
#include <stdio.h>
#include <atomic>
#include <chrono>
#include <cstring>
@@ -16,22 +18,13 @@
#include <string>
#include <thread>
#include "platform.h"
extern "C" {
#include <libavdevice/avdevice.h>
#include <libswresample/swresample.h>
#include <libswscale/swscale.h>
};
#include <stdio.h>
#include "../../thirdparty/projectx/src/interface/x.h"
#include "device_controller_factory.h"
#include "imgui.h"
#include "imgui_impl_sdl2.h"
#include "imgui_impl_sdlrenderer2.h"
#include "log.h"
#include "platform.h"
#include "screen_capturer_factory.h"
#define NV12_BUFFER_SIZE 1280 * 720 * 3 / 2
@@ -62,22 +55,22 @@ int fps = 0;
static std::atomic<bool> audio_buffer_fresh{false};
static uint32_t last_ts = 0;
int64_t src_ch_layout = AV_CH_LAYOUT_MONO;
int src_rate = 48000;
enum AVSampleFormat src_sample_fmt = AV_SAMPLE_FMT_S16;
int src_nb_channels = 0;
uint8_t **src_data = NULL;
int src_linesize;
int src_nb_samples = 480;
// int64_t src_ch_layout = AV_CH_LAYOUT_MONO;
// int src_rate = 48000;
// enum AVSampleFormat src_sample_fmt = AV_SAMPLE_FMT_S16;
// int src_nb_channels = 0;
// uint8_t **src_data = NULL;
// int src_linesize;
// int src_nb_samples = 480;
int64_t dst_ch_layout = AV_CH_LAYOUT_MONO;
int dst_rate = 48000;
enum AVSampleFormat dst_sample_fmt = AV_SAMPLE_FMT_S16;
int dst_nb_channels = 0;
uint8_t **dst_data = NULL;
int dst_linesize;
int dst_nb_samples;
int max_dst_nb_samples;
// int64_t dst_ch_layout = AV_CH_LAYOUT_MONO;
// int dst_rate = 48000;
// enum AVSampleFormat dst_sample_fmt = AV_SAMPLE_FMT_S16;
// int dst_nb_channels = 0;
// uint8_t **dst_data = NULL;
// int dst_linesize;
// int dst_nb_samples;
// int max_dst_nb_samples;
int dst_bufsize;
struct SwrContext *swr_ctx;
@@ -194,46 +187,47 @@ inline int ProcessMouseKeyEven(SDL_Event &ev) {
}
void SdlCaptureAudioIn(void *userdata, Uint8 *stream, int len) {
int64_t delay = swr_get_delay(swr_ctx, src_rate);
dst_nb_samples = (int)av_rescale_rnd(delay + src_nb_samples, dst_rate,
src_rate, AV_ROUND_UP);
if (dst_nb_samples > max_dst_nb_samples) {
av_freep(&dst_data[0]);
ret = av_samples_alloc(dst_data, &dst_linesize, dst_nb_channels,
dst_nb_samples, dst_sample_fmt, 1);
if (ret < 0) return;
max_dst_nb_samples = dst_nb_samples;
}
// int64_t delay = swr_get_delay(swr_ctx, src_rate);
// dst_nb_samples = (int)av_rescale_rnd(delay + src_nb_samples, dst_rate,
// src_rate, AV_ROUND_UP);
// if (dst_nb_samples > max_dst_nb_samples) {
// av_freep(&dst_data[0]);
// ret = av_samples_alloc(dst_data, &dst_linesize, dst_nb_channels,
// dst_nb_samples, dst_sample_fmt, 1);
// if (ret < 0) return;
// max_dst_nb_samples = dst_nb_samples;
// }
ret = swr_convert(swr_ctx, dst_data, dst_nb_samples,
(const uint8_t **)&stream, src_nb_samples);
if (ret < 0) {
fprintf(stderr, "Error while converting\n");
return;
}
dst_bufsize = av_samples_get_buffer_size(&dst_linesize, dst_nb_channels, ret,
dst_sample_fmt, 1);
if (dst_bufsize < 0) {
fprintf(stderr, "Could not get sample buffer size\n");
return;
}
// ret = swr_convert(swr_ctx, dst_data, dst_nb_samples,
// (const uint8_t **)&stream, src_nb_samples);
// if (ret < 0) {
// fprintf(stderr, "Error while converting\n");
// return;
// }
// dst_bufsize = av_samples_get_buffer_size(&dst_linesize, dst_nb_channels,
// ret,
// dst_sample_fmt, 1);
// if (dst_bufsize < 0) {
// fprintf(stderr, "Could not get sample buffer size\n");
// return;
// }
if (1) {
if ("ClientConnected" == client_connection_status_str) {
SendData(peer_client, DATA_TYPE::AUDIO, (const char *)dst_data[0],
dst_bufsize);
}
// if (1) {
// if ("ClientConnected" == client_connection_status_str) {
// SendData(peer_client, DATA_TYPE::AUDIO, (const char *)dst_data[0],
// dst_bufsize);
// }
if ("ServerConnected" == server_connection_status_str) {
SendData(peer_server, DATA_TYPE::AUDIO, (const char *)dst_data[0],
dst_bufsize);
}
} else {
memcpy(audio_buffer, dst_data[0], dst_bufsize);
audio_len = dst_bufsize;
SDL_Delay(10);
audio_buffer_fresh = true;
}
// if ("ServerConnected" == server_connection_status_str) {
// SendData(peer_server, DATA_TYPE::AUDIO, (const char *)dst_data[0],
// dst_bufsize);
// }
// } else {
// memcpy(audio_buffer, dst_data[0], dst_bufsize);
// audio_len = dst_bufsize;
// SDL_Delay(10);
// audio_buffer_fresh = true;
// }
}
void SdlCaptureAudioOut(void *userdata, Uint8 *stream, int len) {
@@ -394,52 +388,52 @@ void ClientConnectionStatus(ConnectionStatus status) {
int initResampler() {
/* create resampler context */
swr_ctx = swr_alloc();
if (!swr_ctx) {
fprintf(stderr, "Could not allocate resampler context\n");
ret = AVERROR(ENOMEM);
return -1;
}
// swr_ctx = swr_alloc();
// if (!swr_ctx) {
// fprintf(stderr, "Could not allocate resampler context\n");
// ret = AVERROR(ENOMEM);
// return -1;
// }
/* set options */
av_opt_set_int(swr_ctx, "in_channel_layout", src_ch_layout, 0);
av_opt_set_int(swr_ctx, "in_sample_rate", src_rate, 0);
av_opt_set_sample_fmt(swr_ctx, "in_sample_fmt", src_sample_fmt, 0);
// /* set options */
// av_opt_set_int(swr_ctx, "in_channel_layout", src_ch_layout, 0);
// av_opt_set_int(swr_ctx, "in_sample_rate", src_rate, 0);
// av_opt_set_sample_fmt(swr_ctx, "in_sample_fmt", src_sample_fmt, 0);
av_opt_set_int(swr_ctx, "out_channel_layout", dst_ch_layout, 0);
av_opt_set_int(swr_ctx, "out_sample_rate", dst_rate, 0);
av_opt_set_sample_fmt(swr_ctx, "out_sample_fmt", dst_sample_fmt, 0);
// av_opt_set_int(swr_ctx, "out_channel_layout", dst_ch_layout, 0);
// av_opt_set_int(swr_ctx, "out_sample_rate", dst_rate, 0);
// av_opt_set_sample_fmt(swr_ctx, "out_sample_fmt", dst_sample_fmt, 0);
/* initialize the resampling context */
if ((ret = swr_init(swr_ctx)) < 0) {
fprintf(stderr, "Failed to initialize the resampling context\n");
return -1;
}
// /* initialize the resampling context */
// if ((ret = swr_init(swr_ctx)) < 0) {
// fprintf(stderr, "Failed to initialize the resampling context\n");
// return -1;
// }
/* allocate source and destination samples buffers */
src_nb_channels = av_get_channel_layout_nb_channels(src_ch_layout);
// /* allocate source and destination samples buffers */
// src_nb_channels = av_get_channel_layout_nb_channels(src_ch_layout);
ret = av_samples_alloc_array_and_samples(&src_data, &src_linesize,
src_nb_channels, src_nb_samples,
src_sample_fmt, 0);
if (ret < 0) {
fprintf(stderr, "Could not allocate source samples\n");
return -1;
}
// ret = av_samples_alloc_array_and_samples(&src_data, &src_linesize,
// src_nb_channels, src_nb_samples,
// src_sample_fmt, 0);
// if (ret < 0) {
// fprintf(stderr, "Could not allocate source samples\n");
// return -1;
// }
max_dst_nb_samples = dst_nb_samples =
av_rescale_rnd(src_nb_samples, dst_rate, src_rate, AV_ROUND_UP);
// max_dst_nb_samples = dst_nb_samples =
// av_rescale_rnd(src_nb_samples, dst_rate, src_rate, AV_ROUND_UP);
/* buffer is going to be directly written to a rawaudio file, no alignment */
dst_nb_channels = av_get_channel_layout_nb_channels(dst_ch_layout);
// /* buffer is going to be directly written to a rawaudio file, no alignment
// */ dst_nb_channels = av_get_channel_layout_nb_channels(dst_ch_layout);
ret = av_samples_alloc_array_and_samples(&dst_data, &dst_linesize,
dst_nb_channels, dst_nb_samples,
dst_sample_fmt, 0);
if (ret < 0) {
fprintf(stderr, "Could not allocate destination samples\n");
return -1;
}
// ret = av_samples_alloc_array_and_samples(&dst_data, &dst_linesize,
// dst_nb_channels, dst_nb_samples,
// dst_sample_fmt, 0);
// if (ret < 0) {
// fprintf(stderr, "Could not allocate destination samples\n");
// return -1;
// }
return 0;
}
@@ -497,8 +491,7 @@ int main(int argc, char *argv[]) {
pixformat = SDL_PIXELFORMAT_NV12;
sdlTexture = SDL_CreateTexture(sdlRenderer, pixformat,
SDL_TEXTUREACCESS_STREAMING, pixel_w,
pixel_h);
SDL_TEXTUREACCESS_STREAMING, pixel_w, pixel_h);
// Audio
SDL_AudioSpec want_in, have_in, want_out, have_out;
@@ -617,8 +610,7 @@ int main(int argc, char *argv[]) {
// Screen capture
screen_capturer_factory = new ScreenCapturerFactory();
screen_capturer = (ScreenCapturer
*)screen_capturer_factory->Create();
screen_capturer = (ScreenCapturer *)screen_capturer_factory->Create();
last_frame_time_ = std::chrono::high_resolution_clock::now();
ScreenCapturer::RECORD_DESKTOP_RECT rect;
@@ -629,16 +621,14 @@ int main(int argc, char *argv[]) {
screen_capturer->Init(
rect, 60,
[](unsigned char *data, int size, int width, int height) ->
void {
[](unsigned char *data, int size, int width, int height) -> void {
auto now_time = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> duration =
now_time - last_frame_time_;
auto tc = duration.count() * 1000;
if (tc >= 0) {
SendData(peer_server, DATA_TYPE::VIDEO, (const char
*)data,
SendData(peer_server, DATA_TYPE::VIDEO, (const char *)data,
NV12_BUFFER_SIZE);
last_frame_time_ = now_time;
}
@@ -685,34 +675,29 @@ int main(int argc, char *argv[]) {
ImGui::InputText(
"##local_id", (char *)mac_addr_str.c_str(),
mac_addr_str.length() + 1,
ImGuiInputTextFlags_CharsUppercase |
ImGuiInputTextFlags_ReadOnly);
ImGuiInputTextFlags_CharsUppercase | ImGuiInputTextFlags_ReadOnly);
ImGui::Text(" PASSWORD:");
ImGui::SameLine();
ImGui::SetNextItemWidth(95);
char input_password_tmp[7] = "";
strncpy(input_password_tmp, input_password,
sizeof(input_password));
strncpy(input_password_tmp, input_password, sizeof(input_password));
ImGui::InputTextWithHint("##server_pwd", "max 6 chars",
input_password,
ImGui::InputTextWithHint("##server_pwd", "max 6 chars", input_password,
IM_ARRAYSIZE(input_password),
ImGuiInputTextFlags_CharsNoBlank);
if (strcmp(input_password_tmp, input_password)) {
cd_cache_file = fopen("cache.cd", "w+");
if (cd_cache_file) {
fseek(cd_cache_file, 0, SEEK_SET);
strncpy(cd_cache.password, input_password,
sizeof(input_password)); fwrite(&cd_cache.password,
sizeof(cd_cache.password), 1,
strncpy(cd_cache.password, input_password, sizeof(input_password));
fwrite(&cd_cache.password, sizeof(cd_cache.password), 1,
cd_cache_file);
fclose(cd_cache_file);
}
LeaveConnection(peer_server);
CreateConnection(peer_server, mac_addr_str.c_str(),
input_password);
CreateConnection(peer_server, mac_addr_str.c_str(), input_password);
}
ImGui::Spacing();
@@ -747,12 +732,11 @@ int main(int argc, char *argv[]) {
if ("ClientSignalConnected" == client_signal_status_str) {
if (strcmp(connect_label, "Connect") == 0 && !joined) {
std::string user_id = "C-" + mac_addr_str;
ret = JoinConnection(peer_client, remote_id,
client_password); if (0 == ret) {
ret = JoinConnection(peer_client, remote_id, client_password);
if (0 == ret) {
// joined = true;
}
} else if (strcmp(connect_label, "Disconnect") == 0 &&
joined) {
} else if (strcmp(connect_label, "Disconnect") == 0 && joined) {
ret = LeaveConnection(peer_client);
memset(audio_buffer, 0, 960);
if (0 == ret) {
@@ -830,8 +814,8 @@ int main(int argc, char *argv[]) {
if (!joined || !received_frame) {
SDL_RenderClear(sdlRenderer);
SDL_SetRenderDrawColor(
sdlRenderer, (Uint8)(clear_color.x * 0), (Uint8)(clear_color.y *
0), (Uint8)(clear_color.z * 0), (Uint8)(clear_color.w * 0));
sdlRenderer, (Uint8)(clear_color.x * 0), (Uint8)(clear_color.y * 0),
(Uint8)(clear_color.z * 0), (Uint8)(clear_color.w * 0));
}
ImGui_ImplSDLRenderer2_RenderDrawData(ImGui::GetDrawData());

View File

@@ -5,36 +5,9 @@
#include <winrt/Windows.Foundation.Metadata.h>
#include <winrt/Windows.Graphics.Capture.h>
extern "C" {
#include <libavutil/imgutils.h>
#include <libswscale/swscale.h>
};
#include <iostream>
int BGRAToNV12FFmpeg(unsigned char *src_buffer, int width, int height,
unsigned char *dst_buffer) {
AVFrame *Input_pFrame = av_frame_alloc();
AVFrame *Output_pFrame = av_frame_alloc();
struct SwsContext *img_convert_ctx =
sws_getContext(width, height, AV_PIX_FMT_BGRA, 1280, 720, AV_PIX_FMT_NV12,
SWS_FAST_BILINEAR, nullptr, nullptr, nullptr);
av_image_fill_arrays(Input_pFrame->data, Input_pFrame->linesize, src_buffer,
AV_PIX_FMT_BGRA, width, height, 1);
av_image_fill_arrays(Output_pFrame->data, Output_pFrame->linesize, dst_buffer,
AV_PIX_FMT_NV12, 1280, 720, 1);
sws_scale(img_convert_ctx, (uint8_t const **)Input_pFrame->data,
Input_pFrame->linesize, 0, height, Output_pFrame->data,
Output_pFrame->linesize);
if (Input_pFrame) av_free(Input_pFrame);
if (Output_pFrame) av_free(Output_pFrame);
if (img_convert_ctx) sws_freeContext(img_convert_ctx);
return 0;
}
#include "libyuv.h"
BOOL WINAPI EnumMonitorProc(HMONITOR hmonitor, HDC hdc, LPRECT lprc,
LPARAM data) {
@@ -81,7 +54,11 @@ int ScreenCapturerWgc::Init(const RECORD_DESKTOP_RECT &rect, const int fps,
int error = 0;
if (_inited == true) return error;
nv12_frame_ = new unsigned char[rect.right * rect.bottom * 4];
int r = rect.right;
int b = rect.bottom;
nv12_frame_ = new unsigned char[rect.right * rect.bottom * 3 / 2];
nv12_frame_scaled_ = new unsigned char[1280 * 720 * 3 / 2];
_fps = fps;
@@ -120,6 +97,11 @@ int ScreenCapturerWgc::Destroy() {
nv12_frame_ = nullptr;
}
if (nv12_frame_scaled_) {
delete nv12_frame_scaled_;
nv12_frame_scaled_ = nullptr;
}
Stop();
CleanUp();
@@ -163,12 +145,83 @@ int ScreenCapturerWgc::Stop() {
return 0;
}
void ConvertABGRtoBGRA(const uint8_t *abgr_data, uint8_t *bgra_data, int width,
int height, int abgr_stride, int bgra_stride) {
for (int i = 0; i < height; ++i) {
for (int j = 0; j < width; ++j) {
// ABGR<47><52>BGRA<52><41><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ӳ<EFBFBD><D3B3>
int abgr_index = (i * abgr_stride + j) * 4;
int bgra_index = (i * bgra_stride + j) * 4;
// ֱ<>ӽ<EFBFBD><D3BD><EFBFBD><EFBFBD><EFBFBD>ɫ<EFBFBD>ͺ<EFBFBD>ɫ<EFBFBD><C9AB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͬʱ<CDAC><CAB1><EFBFBD><EFBFBD>Alphaͨ<61><CDA8><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
bgra_data[bgra_index + 0] = abgr_data[abgr_index + 2]; // <20><>ɫ
bgra_data[bgra_index + 1] = abgr_data[abgr_index + 1]; // <20><>ɫ
bgra_data[bgra_index + 2] = abgr_data[abgr_index + 0]; // <20><>ɫ
bgra_data[bgra_index + 3] = abgr_data[abgr_index + 3]; // Alpha
}
}
}
void ConvertBGRAtoABGR(const uint8_t *bgra_data, uint8_t *abgr_data, int width,
int height, int bgra_stride, int abgr_stride) {
for (int i = 0; i < height; ++i) {
for (int j = 0; j < width; ++j) {
// BGRA<52><41>ABGR<47><52><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ӳ<EFBFBD><D3B3>
int bgra_index = (i * bgra_stride + j) * 4;
int abgr_index = (i * abgr_stride + j) * 4;
// <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ɫ<EFBFBD><C9AB><EFBFBD><EFBFBD>ɫ<EFBFBD><C9AB><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ͬʱ<CDAC><CAB1><EFBFBD><EFBFBD>Alphaͨ<61><CDA8><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ǰ<EFBFBD><C7B0>
abgr_data[abgr_index + 0] = bgra_data[bgra_index + 3]; // Alpha
abgr_data[abgr_index + 1] = bgra_data[bgra_index + 0]; // Blue
abgr_data[abgr_index + 2] = bgra_data[bgra_index + 1]; // Green
abgr_data[abgr_index + 3] = bgra_data[bgra_index + 2]; // Red
}
}
}
void ScreenCapturerWgc::OnFrame(const WgcSession::wgc_session_frame &frame) {
if (_on_data)
BGRAToNV12FFmpeg((unsigned char *)frame.data, frame.width, frame.height,
nv12_frame_);
_on_data(nv12_frame_, frame.width * frame.height * 3 / 2, frame.width,
frame.height);
if (_on_data) {
int width = 1280;
int height = 720;
// libyuv::ARGBToI420(frame.data, frame.width * 4, yuv420_frame_,
// frame.width,
// yuv420_frame_ + frame.width * frame.height,
// frame.width / 2,
// yuv420_frame_ + frame.width * frame.height * 5 / 4,
// frame.width / 2, frame.width, frame.height);
// libyuv::I420Scale(
// (const uint8_t *)yuv420_frame_, frame.width,
// (const uint8_t *)(yuv420_frame_ + frame.width * frame.height),
// frame.width / 2,
// (const uint8_t *)(yuv420_frame_ + frame.width * frame.height * 5 /
// 4), frame.width / 2, frame.width, frame.height, (uint8_t
// *)yuv420_frame_, width, (uint8_t *)(yuv420_frame_ + width * height),
// width / 2, (uint8_t *)(yuv420_frame_ + width * height * 5 / 4), width
// / 2, width, height, libyuv::FilterMode::kFilterLinear);
// libyuv::I420ToNV12(
// (const uint8_t *)yuv420_frame_, width,
// (const uint8_t *)(yuv420_frame_ + width * height), width / 2,
// (const uint8_t *)(yuv420_frame_ + width * height * 5 / 4), width / 2,
// nv12_frame_, width, nv12_frame_ + width * height, width, width,
// height);
libyuv::ARGBToNV12(frame.data, frame.width * 4, nv12_frame_, frame.width,
nv12_frame_ + frame.width * frame.height,
frame.width / 2, frame.width, frame.height);
libyuv::NV12Scale(
(const uint8_t *)nv12_frame_, frame.width,
(const uint8_t *)(nv12_frame_ + frame.width * frame.height),
frame.width / 2, frame.width, frame.height,
(uint8_t *)nv12_frame_scaled_, width,
(uint8_t *)(nv12_frame_scaled_ + width * height), width / 2, width,
height, libyuv::FilterMode::kFilterLinear);
_on_data(nv12_frame_scaled_, width * height * 3 / 2, width, height);
}
}
void ScreenCapturerWgc::CleanUp() {

View File

@@ -49,9 +49,10 @@ class ScreenCapturerWgc : public ScreenCapturer,
int _fps;
cb_desktop_data _on_data;
cb_desktop_data _on_data = nullptr;
unsigned char *nv12_frame_ = nullptr;
unsigned char *nv12_frame_scaled_ = nullptr;
};
#endif

View File

@@ -16,28 +16,22 @@ end
add_requires("sdl2 2.28.3", {system = false})
add_requires("spdlog 1.11.0", {system = false})
add_requires("imgui 1.89.9", {configs = {sdl2 = true, sdl2_renderer = true}})
add_requires("libyuv")
if is_os("windows") then
add_links("Shell32", "windowsapp", "dwmapi", "User32", "kernel32",
"SDL2-static", "SDL2main", "gdi32", "winmm", "setupapi", "version",
"Imm32", "iphlpapi")
add_requires("vcpkg::ffmpeg 5.1.2", {configs = {shared = false}})
add_packages("vcpkg::ffmpeg")
elseif is_os("linux") then
add_requires("ffmpeg 5.1.2", {system = false})
add_packages("ffmpeg")
add_syslinks("pthread", "dl")
add_linkdirs("thirdparty/projectx/thirdparty/nvcodec/Lib/x64")
add_links("SDL2", "cuda", "nvidia-encode", "nvcuvid")
add_ldflags("-lavformat", "-lavdevice", "-lavfilter", "-lavcodec",
"-lswscale", "-lavutil", "-lswresample",
"-lasound", "-lxcb-shape", "-lxcb-xfixes", "-lsndio", "-lxcb",
add_ldflags("-lasound", "-lxcb-shape", "-lxcb-xfixes", "-lsndio", "-lxcb",
"-lxcb-shm", "-lXext", "-lX11", "-lXv", "-ldl", "-lpthread",
{force = true})
elseif is_os("macosx") then
add_requires("ffmpeg 5.1.2", {system = false})
add_requires("libxcb", {system = false})
add_packages("ffmpeg", "libxcb")
add_packages("libxcb")
add_links("SDL2", "SDL2main")
add_ldflags("-Wl,-ld_classic")
add_frameworks("OpenGL")
@@ -62,6 +56,7 @@ target("common")
target("screen_capturer")
set_kind("object")
add_deps("log")
add_packages("libyuv")
add_includedirs("src/screen_capturer", {public = true})
if is_os("windows") then
add_files("src/screen_capturer/windows/*.cpp")