From 3d4e1effe91182ab96eac046d1f644c442f8e176 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Wed, 20 Sep 2023 16:54:56 +0800 Subject: [PATCH] Support ffmpeg soft encode --- src/ice/ice_agent.cpp | 1 + .../video/decode/ffmpeg/ffmpeg_decoder.cpp | 2 + .../video/encode/ffmpeg/ffmpeg_encoder.cpp | 104 +++++++++++++++++- .../video/encode/ffmpeg/ffmpeg_encoder.h | 22 +++- src/media/video/encode/nvcodec/nv_encoder.cpp | 12 +- src/media/video/encode/nvcodec/nv_encoder.h | 4 +- src/pc/peer_connection.h | 6 +- xmake.lua | 17 ++- 8 files changed, 146 insertions(+), 22 deletions(-) diff --git a/src/ice/ice_agent.cpp b/src/ice/ice_agent.cpp index 9402e10..2c46142 100644 --- a/src/ice/ice_agent.cpp +++ b/src/ice/ice_agent.cpp @@ -24,6 +24,7 @@ int IceAgent::CreateIceAgent(juice_cb_state_changed_t on_state_changed, juice_cb_recv_t on_recv, void *user_ptr) { // juice_set_log_level(JUICE_LOG_LEVEL_DEBUG); + LOG_ERROR("{} {} {} {}", stun_ip_, stun_port_, turn_ip_, turn_port_); juice_config_t config; memset(&config, 0, sizeof(config)); diff --git a/src/media/video/decode/ffmpeg/ffmpeg_decoder.cpp b/src/media/video/decode/ffmpeg/ffmpeg_decoder.cpp index b1f3085..476e91e 100644 --- a/src/media/video/decode/ffmpeg/ffmpeg_decoder.cpp +++ b/src/media/video/decode/ffmpeg/ffmpeg_decoder.cpp @@ -85,8 +85,10 @@ int VideoDecoder::Decode( std::function on_receive_decoded_frame) { if (!first_) { if ((*(data + 4) & 0x1f) != 0x07) { + LOG_ERROR("1"); return -1; } else { + LOG_ERROR("2"); first_ = true; } } diff --git a/src/media/video/encode/ffmpeg/ffmpeg_encoder.cpp b/src/media/video/encode/ffmpeg/ffmpeg_encoder.cpp index 4cfeb9b..cdc6606 100644 --- a/src/media/video/encode/ffmpeg/ffmpeg_encoder.cpp +++ b/src/media/video/encode/ffmpeg/ffmpeg_encoder.cpp @@ -21,23 +21,119 @@ VideoEncoder::~VideoEncoder() { file_ = nullptr; } + av_packet_free(&packet_); + if (nv12_data_) { free(nv12_data_); nv12_data_ = nullptr; } } -int VideoEncoder::Init() { return 0; } +int VideoEncoder::Init() { + av_log_set_level(AV_LOG_ERROR); + + codec_ = avcodec_find_encoder(AV_CODEC_ID_H264); + + if (!codec_) { + LOG_ERROR("Failed to find H.264 encoder"); + return -1; + } + + codec_ctx_ = avcodec_alloc_context3(codec_); + if (!codec_ctx_) { + LOG_ERROR("Failed to allocate codec context"); + return -1; + } + + codec_ctx_->codec_id = AV_CODEC_ID_H264; + codec_ctx_->codec_type = AVMEDIA_TYPE_VIDEO; + codec_ctx_->width = frame_width_; + codec_ctx_->height = frame_height; + codec_ctx_->time_base.num = 1; + codec_ctx_->time_base.den = fps_; + codec_ctx_->pix_fmt = AV_PIX_FMT_NV12; + codec_ctx_->gop_size = keyFrameInterval_; + codec_ctx_->keyint_min = keyFrameInterval_; + codec_ctx_->max_b_frames = 0; + codec_ctx_->bit_rate = maxBitrate_ * 500; + + // av_opt_set_int(codec_ctx_->priv_data, "qp", 51, 0); + // av_opt_set_int(codec_ctx_->priv_data, "crf", 23, 0); + + av_opt_set(codec_ctx_->priv_data, "profile", "baseline", 0); + av_opt_set(codec_ctx_->priv_data, "preset", "ultrafast", 0); + av_opt_set(codec_ctx_->priv_data, "tune", "zerolatency", 0); + + if (avcodec_open2(codec_ctx_, codec_, nullptr) < 0) { + LOG_ERROR("Failed to open codec"); + return -1; + } + + frame_ = av_frame_alloc(); + frame_->format = codec_ctx_->pix_fmt; + frame_->width = codec_ctx_->width; + frame_->height = codec_ctx_->height; + + int ret = av_frame_get_buffer(frame_, 0); + if (ret < 0) { + LOG_ERROR("Could not allocate the raw frame"); + return -1; + } + + packet_ = av_packet_alloc(); + + return 0; +} int VideoEncoder::Encode( const uint8_t *pData, int nSize, std::function on_encoded_image) { - return -1; + if (!codec_ctx_) { + LOG_ERROR("Invalid codec context"); + return -1; + } + + memcpy(frame_->data[0], pData, frame_->width * frame_->height); + memcpy(frame_->data[1], pData + frame_->width * frame_->height, + frame_->width * frame_->height / 2); + + frame_->pts = pts_++; + + int ret = avcodec_send_frame(codec_ctx_, frame_); + + // frame_->pict_type = AV_PICTURE_TYPE_I; + while (ret >= 0) { + ret = avcodec_receive_packet(codec_ctx_, packet_); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { + return 0; + } else if (ret < 0) { + return -1; + } + + // Remove first 6 bytes in I frame, SEI ? + if (0x00 == packet_->data[0] && 0x00 == packet_->data[1] && + 0x00 == packet_->data[2] && 0x01 == packet_->data[3] && + 0x09 == packet_->data[4] && 0x10 == packet_->data[5]) { + packet_->data += 6; + packet_->size -= 6; + } + + if (on_encoded_image) { + on_encoded_image((char *)packet_->data, packet_->size); + if (SAVE_ENCODER_STREAM) { + fwrite(packet_->data, 1, packet_->size, file_); + } + } else { + OnEncodedImage((char *)packet_->data, packet_->size); + } + av_packet_unref(packet_); + } + + return 0; } int VideoEncoder::OnEncodedImage(char *encoded_packets, size_t size) { - LOG_INFO("output encoded image"); - fwrite(encoded_packets, 1, size, file_); + LOG_INFO("OnEncodedImage not implemented"); return 0; } diff --git a/src/media/video/encode/ffmpeg/ffmpeg_encoder.h b/src/media/video/encode/ffmpeg/ffmpeg_encoder.h index 6c9efa2..9998825 100644 --- a/src/media/video/encode/ffmpeg/ffmpeg_encoder.h +++ b/src/media/video/encode/ffmpeg/ffmpeg_encoder.h @@ -4,12 +4,20 @@ #ifdef _WIN32 extern "C" { #include "libavcodec/avcodec.h" -}; +#include "libavformat/avformat.h" +#include "libavutil/imgutils.h" +#include "libavutil/opt.h" +} #else #ifdef __cplusplus extern "C" { #endif +extern "C" { #include +#include +#include +#include +} #ifdef __cplusplus }; #endif @@ -31,10 +39,11 @@ class VideoEncoder { void ForceIdr(); private: - int frame_width = 1280; + int frame_width_ = 1280; int frame_height = 720; int keyFrameInterval_ = 3000; - int maxBitrate_ = 2000; + int maxBitrate_ = 1000; + int fps_ = 30; int max_payload_size_ = 3000; std::vector> encoded_packets_; @@ -42,6 +51,13 @@ class VideoEncoder { FILE* file_ = nullptr; unsigned char* nv12_data_ = nullptr; unsigned int seq_ = 0; + + const AVCodec* codec_ = nullptr; + AVCodecContext* codec_ctx_ = nullptr; + AVFrame* frame_ = nullptr; + AVPacket* packet_; + bool got_output_ = false; + uint32_t pts_ = 0; }; #endif \ No newline at end of file diff --git a/src/media/video/encode/nvcodec/nv_encoder.cpp b/src/media/video/encode/nvcodec/nv_encoder.cpp index 4d11db3..72f8b30 100644 --- a/src/media/video/encode/nvcodec/nv_encoder.cpp +++ b/src/media/video/encode/nvcodec/nv_encoder.cpp @@ -40,7 +40,7 @@ int VideoEncoder::Init() { if (!cuda_ctx_succeed) { } - encoder_ = new NvEncoderCuda(cuda_context_, frame_width, frame_height, + encoder_ = new NvEncoderCuda(cuda_context_, frame_width_, frame_height_, NV_ENC_BUFFER_FORMAT::NV_ENC_BUFFER_FORMAT_NV12); // Init encoder_ session @@ -52,8 +52,8 @@ int VideoEncoder::Init() { encoder_->CreateDefaultEncoderParams(&init_params, codec_guid, preset_guid, tuning_info); - init_params.encodeWidth = frame_width; - init_params.encodeHeight = frame_height; + init_params.encodeWidth = frame_width_; + init_params.encodeHeight = frame_height_; init_params.encodeConfig->profileGUID = NV_ENC_H264_PROFILE_BASELINE_GUID; init_params.encodeConfig->encodeCodecConfig.h264Config.level = NV_ENC_LEVEL::NV_ENC_LEVEL_H264_31; @@ -108,6 +108,9 @@ int VideoEncoder::Encode( for (const auto &packet : encoded_packets_) { if (on_encoded_image) { on_encoded_image((char *)packet.data(), packet.size()); + if (SAVE_ENCODER_STREAM) { + fwrite(packet.data(), 1, packet.size(), file_); + } } else { OnEncodedImage((char *)packet.data(), packet.size()); } @@ -128,8 +131,7 @@ int VideoEncoder::Encode( } int VideoEncoder::OnEncodedImage(char *encoded_packets, size_t size) { - LOG_INFO("output encoded image"); - fwrite(encoded_packets, 1, size, file_); + LOG_INFO("OnEncodedImage not implemented"); return 0; } diff --git a/src/media/video/encode/nvcodec/nv_encoder.h b/src/media/video/encode/nvcodec/nv_encoder.h index b33a376..007213e 100644 --- a/src/media/video/encode/nvcodec/nv_encoder.h +++ b/src/media/video/encode/nvcodec/nv_encoder.h @@ -25,8 +25,8 @@ class VideoEncoder { GUID preset_guid = NV_ENC_PRESET_P2_GUID; NV_ENC_TUNING_INFO tuning_info = NV_ENC_TUNING_INFO::NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY; - int frame_width = 1280; - int frame_height = 720; + int frame_width_ = 1280; + int frame_height_ = 720; int keyFrameInterval_ = 3000; int maxBitrate_ = 1000; int max_payload_size_ = 3000; diff --git a/src/pc/peer_connection.h b/src/pc/peer_connection.h index 2657880..a653e93 100644 --- a/src/pc/peer_connection.h +++ b/src/pc/peer_connection.h @@ -7,11 +7,11 @@ #include "ice_transmission.h" #ifdef _WIN32 -#include "nv_decoder.h" -#include "nv_encoder.h" -#else + #include "ffmpeg_decoder.h" #include "ffmpeg_encoder.h" +// #include "nv_decoder.h" +// #include "nv_encoder.h" #endif #include "ws_transmission.h" diff --git a/xmake.lua b/xmake.lua index fa3aed2..0602505 100644 --- a/xmake.lua +++ b/xmake.lua @@ -29,7 +29,7 @@ add_defines("ASIO_STANDALONE", "ASIO_HAS_STD_TYPE_TRAITS", "ASIO_HAS_STD_SHARED_ if is_os("windows") then add_defines("_WEBSOCKETPP_CPP11_INTERNAL_") add_links("ws2_32", "Bcrypt") - add_links("windowsapp", "User32", "Strmiids", "Mfuuid") + add_links("windowsapp", "User32", "Strmiids", "Mfuuid", "Secur32", "Bcrypt") add_requires("cuda") elseif is_os("linux") then add_links("pthread") @@ -105,16 +105,23 @@ target("media") if is_os("windows") or is_os(("linux")) then add_packages("cuda") add_files("src/media/video/encode/nvcodec/*.cpp", - "src/media/video/decode/nvcodec/*.cpp") + "src/media/video/decode/nvcodec/*.cpp", + "src/media/video/encode/ffmpeg/*.cpp", + "src/media/video/decode/ffmpeg/*.cpp" + ) add_includedirs("src/media/video/encode/nvcodec", - "src/media/video/decode/nvcodec", "src/media/video/decode/ffmpeg", + "src/media/video/decode/nvcodec", + "src/media/video/encode/ffmpeg", + "src/media/video/decode/ffmpeg", "thirdparty/nvcodec/Interface", "thirdparty/nvcodec/Samples", {public = true}) add_linkdirs("thirdparty/nvcodec/Lib/x64") add_links("cuda", "nvencodeapi", "nvcuvid") elseif is_os("macosx") then - add_files("src/media/video/encode/ffmpeg/*.cpp", "src/media/video/decode/ffmpeg/*.cpp") - add_includedirs("src/media/video/encode/ffmpeg", "src/media/video/decode/ffmpeg", {public = true}) + add_files("src/media/video/encode/ffmpeg/*.cpp", + "src/media/video/decode/ffmpeg/*.cpp") + add_includedirs("src/media/video/encode/ffmpeg", + "src/media/video/decode/ffmpeg", {public = true}) end target("qos")