From 2f72e3957e53c1a48db305374cff9b125cfa6390 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Thu, 5 Sep 2024 17:29:27 +0800 Subject: [PATCH] [feat] support dynamic resolution codec --- src/screen_capturer/screen_capturer.h | 9 +---- .../windows/screen_capturer_wgc.cpp | 28 ++++++--------- .../windows/screen_capturer_wgc.h | 5 +-- src/single_window/render.cpp | 35 +++++++++++++------ src/single_window/render.h | 8 +++-- src/single_window/render_callback_func.cpp | 18 ++++++++-- thirdparty/projectx | 2 +- 7 files changed, 60 insertions(+), 45 deletions(-) diff --git a/src/screen_capturer/screen_capturer.h b/src/screen_capturer/screen_capturer.h index 8d029aa..a07158e 100644 --- a/src/screen_capturer/screen_capturer.h +++ b/src/screen_capturer/screen_capturer.h @@ -11,20 +11,13 @@ class ScreenCapturer { public: - typedef struct { - int left; - int top; - int right; - int bottom; - } RECORD_DESKTOP_RECT; typedef std::function cb_desktop_data; public: virtual ~ScreenCapturer() {} public: - virtual int Init(const RECORD_DESKTOP_RECT &rect, const int fps, - cb_desktop_data cb) = 0; + virtual int Init(const int fps, cb_desktop_data cb) = 0; virtual int Destroy() = 0; virtual int Start() = 0; diff --git a/src/screen_capturer/windows/screen_capturer_wgc.cpp b/src/screen_capturer/windows/screen_capturer_wgc.cpp index 1599367..e3f855f 100644 --- a/src/screen_capturer/windows/screen_capturer_wgc.cpp +++ b/src/screen_capturer/windows/screen_capturer_wgc.cpp @@ -62,16 +62,12 @@ bool ScreenCapturerWgc::IsWgcSupported() { } } -int ScreenCapturerWgc::Init(const RECORD_DESKTOP_RECT &rect, const int fps, - cb_desktop_data cb) { +int ScreenCapturerWgc::Init(const int fps, cb_desktop_data cb) { int error = 0; if (_inited == true) return error; - int r = rect.right; - int b = rect.bottom; - - nv12_frame_ = new unsigned char[rect.right * rect.bottom * 3 / 2]; - nv12_frame_scaled_ = new unsigned char[1280 * 720 * 3 / 2]; + // nv12_frame_ = new unsigned char[rect.right * rect.bottom * 3 / 2]; + // nv12_frame_scaled_ = new unsigned char[1280 * 720 * 3 / 2]; _fps = fps; @@ -175,22 +171,20 @@ void ConvertBGRAtoABGR(const uint8_t *bgra_data, uint8_t *abgr_data, int width, void ScreenCapturerWgc::OnFrame(const WgcSession::wgc_session_frame &frame) { if (_on_data) { - int width = 1280; - int height = 720; + // int width = 1280; + // int height = 720; + + if (!nv12_frame_) { + nv12_frame_ = new unsigned char[frame.width * frame.height * 3 / 2]; + } libyuv::ARGBToNV12((const uint8_t *)frame.data, frame.width * 4, (uint8_t *)nv12_frame_, frame.width, (uint8_t *)(nv12_frame_ + frame.width * frame.height), frame.width, frame.width, frame.height); - libyuv::NV12Scale( - (const uint8_t *)nv12_frame_, frame.width, - (const uint8_t *)(nv12_frame_ + frame.width * frame.height), - frame.width, frame.width, frame.height, (uint8_t *)nv12_frame_scaled_, - width, (uint8_t *)(nv12_frame_scaled_ + width * height), width, width, - height, libyuv::FilterMode::kFilterLinear); - - _on_data(nv12_frame_scaled_, width * height * 3 / 2, width, height); + _on_data(nv12_frame_, frame.width * frame.height * 3 / 2, frame.width, + frame.height); } } diff --git a/src/screen_capturer/windows/screen_capturer_wgc.h b/src/screen_capturer/windows/screen_capturer_wgc.h index b004a98..2d394d6 100644 --- a/src/screen_capturer/windows/screen_capturer_wgc.h +++ b/src/screen_capturer/windows/screen_capturer_wgc.h @@ -19,8 +19,7 @@ class ScreenCapturerWgc : public ScreenCapturer, public: bool IsWgcSupported(); - virtual int Init(const RECORD_DESKTOP_RECT &rect, const int fps, - cb_desktop_data cb); + virtual int Init(const int fps, cb_desktop_data cb); virtual int Destroy(); virtual int Start(); @@ -45,8 +44,6 @@ class ScreenCapturerWgc : public ScreenCapturer, std::string _device_name; - RECORD_DESKTOP_RECT _rect; - int _fps; cb_desktop_data _on_data = nullptr; diff --git a/src/single_window/render.cpp b/src/single_window/render.cpp index 6edbe0f..a9d3b70 100644 --- a/src/single_window/render.cpp +++ b/src/single_window/render.cpp @@ -156,25 +156,27 @@ int Render::LoadSettingsFromCacheFile() { int Render::StartScreenCapture() { screen_capturer_ = (ScreenCapturer *)screen_capturer_factory_->Create(); - ScreenCapturer::RECORD_DESKTOP_RECT rect; - rect.left = 0; - rect.top = 0; - rect.right = screen_width_; - rect.bottom = screen_height_; last_frame_time_ = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); int screen_capturer_init_ret = screen_capturer_->Init( - rect, 60, - [this](unsigned char *data, int size, int width, int height) -> void { + 60, [this](unsigned char *data, int size, int width, int height) -> void { auto now_time = std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()) .count(); auto duration = now_time - last_frame_time_; if (duration >= 0 && connection_established_) { - SendData(peer_, DATA_TYPE::VIDEO, (const char *)data, - NV12_BUFFER_SIZE); + // SendData(peer_, DATA_TYPE::VIDEO, (const char *)data, + // NV12_BUFFER_SIZE); + + XVideoFrame frame; + frame.data = (const char *)data; + frame.size = size; + frame.width = width; + frame.height = height; + + SendVideoFrame(peer_, &frame); last_frame_time_ = now_time; } }); @@ -275,9 +277,12 @@ int Render::CreateConnectionPeer() { ? true : false; params_.enable_turn = config_center_.IsEnableTurn(); - params_.on_receive_video_buffer = OnReceiveVideoBufferCb; + params_.on_receive_video_buffer = nullptr; params_.on_receive_audio_buffer = OnReceiveAudioBufferCb; params_.on_receive_data_buffer = OnReceiveDataBufferCb; + + params_.on_receive_video_frame = OnReceiveVideoBufferCb; + params_.on_signal_status = OnSignalStatusCb; params_.on_connection_status = OnConnectionStatusCb; params_.net_status_report = NetStatusReport; @@ -596,7 +601,15 @@ int Render::Run() { } } else if (event.type == REFRESH_EVENT) { if (stream_texture_) - SDL_UpdateTexture(stream_texture_, NULL, dst_buffer_, 1280); + if (video_width_ != texture_width_) { + texture_width_ = video_width_; + texture_height_ = video_height_; + + stream_texture_ = SDL_CreateTexture( + main_renderer_, stream_pixformat_, SDL_TEXTUREACCESS_STREAMING, + texture_width_, texture_height_); + } + SDL_UpdateTexture(stream_texture_, NULL, dst_buffer_, texture_width_); } else { if (connection_established_) { ProcessMouseKeyEven(event); diff --git a/src/single_window/render.h b/src/single_window/render.h index 0386990..48639a5 100644 --- a/src/single_window/render.h +++ b/src/single_window/render.h @@ -44,7 +44,7 @@ class Render { int ConnectionStatusWindow(); public: - static void OnReceiveVideoBufferCb(const char *data, size_t size, + static void OnReceiveVideoBufferCb(const XVideoFrame *video_frame, const char *user_id, size_t user_id_size, void *user_data); @@ -169,6 +169,9 @@ class Render { int texture_width_ = 1280; int texture_height_ = 720; + int video_width_ = 1280; + int video_height_ = 720; + SDL_Window *main_window_; SDL_Renderer *main_renderer_ = nullptr; @@ -242,7 +245,8 @@ class Render { unsigned char audio_buffer_[960]; int audio_len_ = 0; char *nv12_buffer_ = nullptr; - unsigned char *dst_buffer_ = new unsigned char[1280 * 720 * 3]; + unsigned char *dst_buffer_ = nullptr; + int dst_buffer_capacity_ = 0; private: ScreenCapturerFactory *screen_capturer_factory_ = nullptr; diff --git a/src/single_window/render_callback_func.cpp b/src/single_window/render_callback_func.cpp index fa0f8ce..25f3a53 100644 --- a/src/single_window/render_callback_func.cpp +++ b/src/single_window/render_callback_func.cpp @@ -115,12 +115,26 @@ void Render::SdlCaptureAudioOut(void *userdata, Uint8 *stream, int len) { // render->audio_buffer_fresh_ = false; } -void Render::OnReceiveVideoBufferCb(const char *data, size_t size, +void Render::OnReceiveVideoBufferCb(const XVideoFrame *video_frame, const char *user_id, size_t user_id_size, void *user_data) { Render *render = (Render *)user_data; if (render->connection_established_) { - memcpy(render->dst_buffer_, data, size); + if (!render->dst_buffer_) { + render->dst_buffer_capacity_ = video_frame->size; + render->dst_buffer_ = new unsigned char[video_frame->size]; + } + + if (render->dst_buffer_capacity_ < video_frame->size) { + delete render->dst_buffer_; + render->dst_buffer_capacity_ = video_frame->size; + render->dst_buffer_ = new unsigned char[video_frame->size]; + } + + memcpy(render->dst_buffer_, video_frame->data, video_frame->size); + render->video_width_ = video_frame->width; + render->video_height_ = video_frame->height; + SDL_Event event; event.type = REFRESH_EVENT; SDL_PushEvent(&event); diff --git a/thirdparty/projectx b/thirdparty/projectx index d285d79..c477643 160000 --- a/thirdparty/projectx +++ b/thirdparty/projectx @@ -1 +1 @@ -Subproject commit d285d7971aae1ade2705a81ea8d7d030b22e5b4d +Subproject commit c477643aed9535274371b9e1d53c514f326456c3