From e456ae357717df723a619a731ec248a60d6da193 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Tue, 7 Nov 2023 15:27:57 +0800 Subject: [PATCH] OpenH264 supports Windows platform --- .../remote_desk/remote_desk_gui/main.cpp | 5 ++ config/config.ini | 2 +- .../decode/openh264/openh264_decoder.cpp | 27 ++++---- .../video/decode/video_decoder_factory.cpp | 4 +- .../encode/ffmpeg/ffmpeg_video_encoder.cpp | 43 ++++++------- .../encode/ffmpeg/ffmpeg_video_encoder.h | 1 + .../encode/openh264/openh264_encoder.cpp | 61 ++++++++++--------- .../video/encode/openh264/openh264_encoder.h | 6 +- .../video/encode/video_encoder_factory.cpp | 4 +- thirdparty/openh264/xmake.lua | 31 ++++++++++ thirdparty/xmake.lua | 1 + 11 files changed, 116 insertions(+), 69 deletions(-) create mode 100644 thirdparty/openh264/xmake.lua diff --git a/application/remote_desk/remote_desk_gui/main.cpp b/application/remote_desk/remote_desk_gui/main.cpp index 273ae61..0142e09 100644 --- a/application/remote_desk/remote_desk_gui/main.cpp +++ b/application/remote_desk/remote_desk_gui/main.cpp @@ -88,7 +88,12 @@ ScreenCaptureX11 *screen_capture = nullptr; #endif char *nv12_buffer = nullptr; + +#ifdef __linux__ std::chrono::_V2::system_clock::time_point last_frame_time_; +#else +std::chrono::steady_clock::time_point last_frame_time_; +#endif typedef enum { mouse = 0, keyboard } ControlType; typedef enum { move = 0, left_down, left_up, right_down, right_up } MouseFlag; diff --git a/config/config.ini b/config/config.ini index 79ad05f..2939be0 100644 --- a/config/config.ini +++ b/config/config.ini @@ -13,4 +13,4 @@ username = dijunkun password = dijunkunpw [hardware acceleration] -turn_on = true \ No newline at end of file +turn_on = false \ No newline at end of file diff --git a/src/media/video/decode/openh264/openh264_decoder.cpp b/src/media/video/decode/openh264/openh264_decoder.cpp index 7c88899..ed5a0b0 100644 --- a/src/media/video/decode/openh264/openh264_decoder.cpp +++ b/src/media/video/decode/openh264/openh264_decoder.cpp @@ -134,14 +134,6 @@ int OpenH264Decoder::Init() { frame_width_ = 1280; frame_height_ = 720; - // SEncParamExt sParam; - // sParam.iPicWidth = frame_width_; - // sParam.iPicHeight = frame_height_; - // sParam.iTargetBitrate = 1000000000; - // sParam.iTemporalLayerNum = 1; - // sParam.fMaxFrameRate = 30; - // sParam.iSpatialLayerNum = 1; - decoded_frame_size_ = YUV420P_BUFFER_SIZE; decoded_frame_ = new uint8_t[YUV420P_BUFFER_SIZE]; nv12_frame_ = new uint8_t[YUV420P_BUFFER_SIZE]; @@ -192,14 +184,27 @@ int OpenH264Decoder::Decode( unsigned char *dst[3]; - int iRet = - openh264_decoder_->DecodeFrameNoDelay(data, size, dst, &sDstBufInfo); + int iRet = openh264_decoder_->DecodeFrame2(data, size, dst, &sDstBufInfo); + // int iRet = + // openh264_decoder_->DecodeFrameNoDelay(data, size, dst, &sDstBufInfo); if (iRet != 0) { return -1; } - if (sDstBufInfo.iBufferStatus == 1) { + // int num_of_buffer = 0; + // iRet = openh264_decoder_->GetOption( + // DECODER_OPTION_NUM_OF_FRAMES_REMAINING_IN_BUFFER, &num_of_buffer); + + // LOG_ERROR("Number of buffer {} {}", num_of_buffer, iRet); + + // iRet = openh264_decoder_->FlushFrame(dst, &sDstBufInfo); + // if (iRet != 0) { + // LOG_ERROR("FlushFrame state: {}", iRet); + // return -1; + // } + + if (1) { if (on_receive_decoded_frame) { CopyYUVWithStride( dst[0], dst[1], dst[2], sDstBufInfo.UsrData.sSystemBuffer.iWidth, diff --git a/src/media/video/decode/video_decoder_factory.cpp b/src/media/video/decode/video_decoder_factory.cpp index a0c440c..5e330a6 100644 --- a/src/media/video/decode/video_decoder_factory.cpp +++ b/src/media/video/decode/video_decoder_factory.cpp @@ -26,8 +26,8 @@ std::unique_ptr VideoDecoderFactory::CreateVideoDecoder( return nullptr; } } else { - // return std::make_unique(FfmpegVideoDecoder()); - return std::make_unique(OpenH264Decoder()); + return std::make_unique(FfmpegVideoDecoder()); + // return std::make_unique(OpenH264Decoder()); } #endif } diff --git a/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.cpp b/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.cpp index 5e88f37..3002af6 100644 --- a/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.cpp +++ b/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.cpp @@ -59,19 +59,20 @@ FFmpegVideoEncoder::~FFmpegVideoEncoder() { } int FFmpegVideoEncoder::Init() { - av_log_set_level(AV_LOG_VERBOSE); + // av_log_set_level(AV_LOG_VERBOSE); codec_ = avcodec_find_encoder(AV_CODEC_ID_H264); if (!codec_) { LOG_ERROR("Failed to find H.264 encoder"); return -1; - } else { - LOG_INFO("Use H264 encoder [{}]", codec_->name); - if (0 == strcmp(codec_->name, "libx264")) { - use_libx264_ = true; - } } - use_libx264_ = true; + // else { + // LOG_INFO("Use H264 encoder [{}]", codec_->name); + // if (0 == strcmp(codec_->name, "libx264")) { + // use_libx264_ = true; + // } + // } + // use_libx264_ = true; codec_ctx_ = avcodec_alloc_context3(codec_); if (!codec_ctx_) { @@ -88,17 +89,17 @@ int FFmpegVideoEncoder::Init() { if (use_libx264_) { codec_ctx_->pix_fmt = AV_PIX_FMT_YUV420P; } else { - codec_ctx_->pix_fmt = AV_PIX_FMT_YUV420P; + codec_ctx_->pix_fmt = AV_PIX_FMT_NV12; } codec_ctx_->gop_size = keyFrameInterval_; codec_ctx_->keyint_min = keyFrameInterval_; codec_ctx_->max_b_frames = 0; - codec_ctx_->bit_rate = maxBitrate_ * 500; + codec_ctx_->bit_rate = maxBitrate_ * 2500; - // av_opt_set_int(codec_ctx_->priv_data, "qp", 51, 0); - // av_opt_set_int(codec_ctx_->priv_data, "crf", 23, 0); + av_opt_set_int(codec_ctx_->priv_data, "qp", 51, 0); + av_opt_set_int(codec_ctx_->priv_data, "crf", 23, 0); - // av_opt_set(codec_ctx_->priv_data, "profile", "baseline", 0); + av_opt_set(codec_ctx_->priv_data, "profile", "baseline", 0); av_opt_set(codec_ctx_->priv_data, "preset", "ultrafast", 0); av_opt_set(codec_ctx_->priv_data, "tune", "zerolatency", 0); @@ -150,10 +151,10 @@ int FFmpegVideoEncoder::Encode( (unsigned char *)yuv420p_buffer); memcpy(frame_->data[0], yuv420p_buffer, frame_->width * frame_->height); memcpy(frame_->data[1], yuv420p_buffer + frame_->width * frame_->height, - frame_->width * frame_->height / 2); + frame_->width * frame_->height / 4); memcpy(frame_->data[2], - yuv420p_buffer + frame_->width * frame_->height * 3 / 2, - frame_->width * frame_->height / 2); + yuv420p_buffer + frame_->width * frame_->height * 5 / 4, + frame_->width * frame_->height / 4); // frame_->data[0] = yuv420p_buffer; // frame_->data[1] = yuv420p_buffer + frame_->width * frame_->height; @@ -187,12 +188,12 @@ int FFmpegVideoEncoder::Encode( } // Remove first 6 bytes in I frame, SEI ? - // if (0x00 == packet_->data[0] && 0x00 == packet_->data[1] && - // 0x00 == packet_->data[2] && 0x01 == packet_->data[3] && - // 0x09 == packet_->data[4] && 0x10 == packet_->data[5]) { - // packet_->data += 6; - // packet_->size -= 6; - // } + if (0x00 == packet_->data[0] && 0x00 == packet_->data[1] && + 0x00 == packet_->data[2] && 0x01 == packet_->data[3] && + 0x09 == packet_->data[4] && 0x10 == packet_->data[5]) { + packet_->data += 6; + packet_->size -= 6; + } if (on_encoded_image) { on_encoded_image((char *)packet_->data, packet_->size); diff --git a/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.h b/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.h index a4abea1..fdb2f99 100644 --- a/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.h +++ b/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.h @@ -7,6 +7,7 @@ extern "C" { #include "libavformat/avformat.h" #include "libavutil/imgutils.h" #include "libavutil/opt.h" +#include "libswscale/swscale.h" } #else #ifdef __cplusplus diff --git a/src/media/video/encode/openh264/openh264_encoder.cpp b/src/media/video/encode/openh264/openh264_encoder.cpp index d4843ac..5deccb8 100644 --- a/src/media/video/encode/openh264/openh264_encoder.cpp +++ b/src/media/video/encode/openh264/openh264_encoder.cpp @@ -69,9 +69,9 @@ SEncParamExt OpenH264Encoder::CreateEncoderParams() const { SEncParamExt encoder_params; openh264_encoder_->GetDefaultParams(&encoder_params); // if (codec_.mode == VideoCodecMode::kRealtimeVideo) { // - encoder_params.iUsageType = CAMERA_VIDEO_REAL_TIME; + // encoder_params.iUsageType = CAMERA_VIDEO_REAL_TIME; // } else if (codec_.mode == VideoCodecMode::kScreensharing) { - // encoder_params.iUsageType = SCREEN_CONTENT_REAL_TIME; + encoder_params.iUsageType = SCREEN_CONTENT_REAL_TIME; // } encoder_params.iPicWidth = frame_width_; @@ -80,9 +80,11 @@ SEncParamExt OpenH264Encoder::CreateEncoderParams() const { encoder_params.iMaxBitrate = max_bitrate_; encoder_params.iRCMode = RC_BITRATE_MODE; encoder_params.fMaxFrameRate = 0.000030; - encoder_params.bEnableFrameSkip = true; + encoder_params.bEnableFrameSkip = false; encoder_params.uiIntraPeriod = key_frame_interval_; encoder_params.uiMaxNalSize = 0; + encoder_params.iMaxQp = 38; + encoder_params.iMinQp = 16; // Threading model: use auto. // 0: auto (dynamic imp. internal encoder) // 1: single thread (default value) @@ -96,11 +98,6 @@ SEncParamExt OpenH264Encoder::CreateEncoderParams() const { encoder_params.iTargetBitrate; encoder_params.sSpatialLayers[0].iMaxSpatialBitrate = encoder_params.iMaxBitrate; - encoder_params.iTemporalLayerNum = 1; - if (encoder_params.iTemporalLayerNum > 1) { - encoder_params.iNumRefFrame = 1; - } - LOG_INFO("OpenH264 version is [{}.{}]", OPENH264_MAJOR, OPENH264_MINOR); // SingleNalUnit encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1; @@ -166,18 +163,12 @@ int OpenH264Encoder::Encode( fwrite(yuv420p_buffer, 1, nSize, file_nv12_); } - if (0 == seq_++ % 3) { + if (0 == seq_++ % 300) { ForceIdr(); } NV12ToYUV420PFFmpeg((unsigned char *)pData, frame_width_, frame_height_, (unsigned char *)yuv420p_buffer); - // memcpy(frame_->data[0], yuv420p_buffer, frame_width_ * frame_height_); - // memcpy(frame_->data[1], yuv420p_buffer + frame_width_ * frame_height_, - // frame_width_ * frame_height_ / 2); - // memcpy(frame_->data[2], yuv420p_buffer + frame_width_ * frame_height_ * 3 / - // 2, - // frame_width_ * frame_height_ / 2); raw_frame_ = {0}; raw_frame_.iPicWidth = frame_width_; @@ -205,23 +196,35 @@ int OpenH264Encoder::Encode( return -1; } -#if 0 - int encoded_frame_size = 0; - - for (int layer = 0; layer < info.iLayerNum; ++layer) { - const SLayerBSInfo &layerInfo = info.sLayerInfo[layer]; - size_t layer_len = 0; - memcpy(encoded_frame_ + encoded_frame_size, layerInfo.pBsBuf, layer_len); - encoded_frame_size += layer_len; +#if 0 + size_t required_capacity = 0; + size_t fragments_count = 0; + for (int layer = 0; layer < info.iLayerNum; ++layer) { + const SLayerBSInfo &layerInfo = info.sLayerInfo[layer]; + for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++fragments_count) { + required_capacity += layerInfo.pNalLengthInByte[nal]; } + } - encoded_frame_size_ = encoded_frame_size; - - if (on_encoded_image) { - on_encoded_image((char *)encoded_frame_, encoded_frame_size_); - } else { - OnEncodedImage((char *)encoded_frame_, encoded_frame_size_); + const uint8_t start_code[4] = {0, 0, 0, 1}; + size_t frag = 0; + int encoded_frame_size = 0; + for (int layer = 0; layer < info.iLayerNum; ++layer) { + const SLayerBSInfo &layerInfo = info.sLayerInfo[layer]; + size_t layer_len = 0; + for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++frag) { + layer_len += layerInfo.pNalLengthInByte[nal]; } + memcpy(encoded_frame_ + encoded_frame_size, layerInfo.pBsBuf, layer_len); + encoded_frame_size += layer_len; + } + encoded_frame_size_ = encoded_frame_size; + + if (on_encoded_image) { + on_encoded_image((char *)encoded_frame_, encoded_frame_size_); + } else { + OnEncodedImage((char *)encoded_frame_, encoded_frame_size_); + } #else if (info.eFrameType == videoFrameTypeInvalid) { LOG_ERROR("videoFrameTypeInvalid"); diff --git a/src/media/video/encode/openh264/openh264_encoder.h b/src/media/video/encode/openh264/openh264_encoder.h index 91d961e..778396b 100644 --- a/src/media/video/encode/openh264/openh264_encoder.h +++ b/src/media/video/encode/openh264/openh264_encoder.h @@ -38,10 +38,10 @@ class OpenH264Encoder : public VideoEncoder { private: int frame_width_ = 1280; int frame_height_ = 720; - int key_frame_interval_ = 5; + int key_frame_interval_ = 3000; int target_bitrate_ = 1000; - int max_bitrate_ = 1000; - int max_payload_size_ = 3000; + int max_bitrate_ = 500000; + int max_payload_size_ = 1400; int max_frame_rate_ = 30; std::vector> encoded_packets_; unsigned char* encoded_image_ = nullptr; diff --git a/src/media/video/encode/video_encoder_factory.cpp b/src/media/video/encode/video_encoder_factory.cpp index 405d7b9..fdfc588 100644 --- a/src/media/video/encode/video_encoder_factory.cpp +++ b/src/media/video/encode/video_encoder_factory.cpp @@ -26,8 +26,8 @@ std::unique_ptr VideoEncoderFactory::CreateVideoEncoder( return nullptr; } } else { - // return std::make_unique(FFmpegVideoEncoder()); - return std::make_unique(OpenH264Encoder()); + return std::make_unique(FFmpegVideoEncoder()); + // return std::make_unique(OpenH264Encoder()); } #endif } diff --git a/thirdparty/openh264/xmake.lua b/thirdparty/openh264/xmake.lua new file mode 100644 index 0000000..c99b504 --- /dev/null +++ b/thirdparty/openh264/xmake.lua @@ -0,0 +1,31 @@ +package("openh264") + + set_homepage("http://www.openh264.org/") + set_description("OpenH264 is a codec library which supports H.264 encoding and decoding.") + set_license("BSD-2-Clause") + + add_urls("https://github.com/cisco/openh264/archive/refs/tags/$(version).tar.gz") + add_versions("v2.3.1", "453afa66dacb560bc5fd0468aabee90c483741571bca820a39a1c07f0362dc32") + + add_deps("meson", "ninja", "nasm") + if is_plat("linux") then + add_syslinks("pthread", "rt") + end + on_install("windows", "linux", function (package) + import("package.tools.meson").build(package, {"-Dtests=disabled"}, {buildir = "out"}) + import("package.tools.ninja").install(package, {}, {buildir = "out"}) + -- if package:config("shared") then + -- os.tryrm(path.join(package:installdir("lib"), "libopenh264.a")) + -- else + -- os.tryrm(path.join(package:installdir("lib"), "libopenh264.so*")) + -- os.tryrm(path.join(package:installdir("lib"), "openh264.lib")) + -- os.tryrm(path.join(package:installdir("bin"), "openh264-*.dll")) + -- end + -- if package:is_plat("windows") then + -- os.trymv(path.join(package:installdir("lib"), "libopenh264.a"), path.join(package:installdir("lib"), "openh264.lib")) + -- end + end) + + on_test(function (package) + assert(package:has_cxxfuncs("WelsGetCodecVersion", {includes = "wels/codec_api.h"})) + end) \ No newline at end of file diff --git a/thirdparty/xmake.lua b/thirdparty/xmake.lua index 6e84f24..388073e 100644 --- a/thirdparty/xmake.lua +++ b/thirdparty/xmake.lua @@ -1,3 +1,4 @@ +includes("openh264") if is_plat("windows") then elseif is_plat("linux") then includes("ffmpeg")