Support ffmpeg soft encode

src/ice/ice_agent.cpp

@@ -24,6 +24,7 @@ int IceAgent::CreateIceAgent(juice_cb_state_changed_t on_state_changed,
                              juice_cb_recv_t on_recv, void *user_ptr) {
   // juice_set_log_level(JUICE_LOG_LEVEL_DEBUG);
 
+  LOG_ERROR("{} {} {} {}", stun_ip_, stun_port_, turn_ip_, turn_port_);
   juice_config_t config;
   memset(&config, 0, sizeof(config));
 

src/media/video/decode/ffmpeg/ffmpeg_decoder.cpp

@@ -85,8 +85,10 @@ int VideoDecoder::Decode(
     std::function<void(VideoFrame)> on_receive_decoded_frame) {
   if (!first_) {
     if ((*(data + 4) & 0x1f) != 0x07) {
+      LOG_ERROR("1");
       return -1;
     } else {
+      LOG_ERROR("2");
       first_ = true;
     }
   }

src/media/video/encode/ffmpeg/ffmpeg_encoder.cpp

@@ -21,23 +21,119 @@ VideoEncoder::~VideoEncoder() {
     file_ = nullptr;
   }
 
+  av_packet_free(&packet_);
+
   if (nv12_data_) {
     free(nv12_data_);
     nv12_data_ = nullptr;
   }
 }
 
-int VideoEncoder::Init() { return 0; }
+int VideoEncoder::Init() {
+  av_log_set_level(AV_LOG_ERROR);
+
+  codec_ = avcodec_find_encoder(AV_CODEC_ID_H264);
+
+  if (!codec_) {
+    LOG_ERROR("Failed to find H.264 encoder");
+    return -1;
+  }
+
+  codec_ctx_ = avcodec_alloc_context3(codec_);
+  if (!codec_ctx_) {
+    LOG_ERROR("Failed to allocate codec context");
+    return -1;
+  }
+
+  codec_ctx_->codec_id = AV_CODEC_ID_H264;
+  codec_ctx_->codec_type = AVMEDIA_TYPE_VIDEO;
+  codec_ctx_->width = frame_width_;
+  codec_ctx_->height = frame_height;
+  codec_ctx_->time_base.num = 1;
+  codec_ctx_->time_base.den = fps_;
+  codec_ctx_->pix_fmt = AV_PIX_FMT_NV12;
+  codec_ctx_->gop_size = keyFrameInterval_;
+  codec_ctx_->keyint_min = keyFrameInterval_;
+  codec_ctx_->max_b_frames = 0;
+  codec_ctx_->bit_rate = maxBitrate_ * 500;
+
+  // av_opt_set_int(codec_ctx_->priv_data, "qp", 51, 0);
+  // av_opt_set_int(codec_ctx_->priv_data, "crf", 23, 0);
+
+  av_opt_set(codec_ctx_->priv_data, "profile", "baseline", 0);
+  av_opt_set(codec_ctx_->priv_data, "preset", "ultrafast", 0);
+  av_opt_set(codec_ctx_->priv_data, "tune", "zerolatency", 0);
+
+  if (avcodec_open2(codec_ctx_, codec_, nullptr) < 0) {
+    LOG_ERROR("Failed to open codec");
+    return -1;
+  }
+
+  frame_ = av_frame_alloc();
+  frame_->format = codec_ctx_->pix_fmt;
+  frame_->width = codec_ctx_->width;
+  frame_->height = codec_ctx_->height;
+
+  int ret = av_frame_get_buffer(frame_, 0);
+  if (ret < 0) {
+    LOG_ERROR("Could not allocate the raw frame");
+    return -1;
+  }
+
+  packet_ = av_packet_alloc();
+
+  return 0;
+}
 
 int VideoEncoder::Encode(
     const uint8_t *pData, int nSize,
     std::function<int(char *encoded_packets, size_t size)> on_encoded_image) {
+  if (!codec_ctx_) {
+    LOG_ERROR("Invalid codec context");
     return -1;
+  }
+
+  memcpy(frame_->data[0], pData, frame_->width * frame_->height);
+  memcpy(frame_->data[1], pData + frame_->width * frame_->height,
+         frame_->width * frame_->height / 2);
+
+  frame_->pts = pts_++;
+
+  int ret = avcodec_send_frame(codec_ctx_, frame_);
+
+  // frame_->pict_type = AV_PICTURE_TYPE_I;
+  while (ret >= 0) {
+    ret = avcodec_receive_packet(codec_ctx_, packet_);
+    if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
+      return 0;
+    } else if (ret < 0) {
+      return -1;
+    }
+
+    // Remove first 6 bytes in I frame, SEI ?
+    if (0x00 == packet_->data[0] && 0x00 == packet_->data[1] &&
+        0x00 == packet_->data[2] && 0x01 == packet_->data[3] &&
+        0x09 == packet_->data[4] && 0x10 == packet_->data[5]) {
+      packet_->data += 6;
+      packet_->size -= 6;
+    }
+
+    if (on_encoded_image) {
+      on_encoded_image((char *)packet_->data, packet_->size);
+      if (SAVE_ENCODER_STREAM) {
+        fwrite(packet_->data, 1, packet_->size, file_);
+      }
+    } else {
+      OnEncodedImage((char *)packet_->data, packet_->size);
+    }
+    av_packet_unref(packet_);
+  }
+
+  return 0;
 }
 
 int VideoEncoder::OnEncodedImage(char *encoded_packets, size_t size) {
-  LOG_INFO("output encoded image");
+  LOG_INFO("OnEncodedImage not implemented");
-  fwrite(encoded_packets, 1, size, file_);
   return 0;
 }
 

src/media/video/encode/ffmpeg/ffmpeg_encoder.h

@@ -4,12 +4,20 @@
 #ifdef _WIN32
 extern "C" {
 #include "libavcodec/avcodec.h"
-};
+#include "libavformat/avformat.h"
+#include "libavutil/imgutils.h"
+#include "libavutil/opt.h"
+}
 #else
 #ifdef __cplusplus
 extern "C" {
 #endif
+extern "C" {
 #include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/imgutils.h>
+#include <libavutil/opt.h>
+}
 #ifdef __cplusplus
 };
 #endif
@@ -31,10 +39,11 @@ class VideoEncoder {
   void ForceIdr();
 
  private:
-  int frame_width = 1280;
+  int frame_width_ = 1280;
   int frame_height = 720;
   int keyFrameInterval_ = 3000;
-  int maxBitrate_ = 2000;
+  int maxBitrate_ = 1000;
+  int fps_ = 30;
   int max_payload_size_ = 3000;
 
   std::vector<std::vector<uint8_t>> encoded_packets_;
@@ -42,6 +51,13 @@ class VideoEncoder {
   FILE* file_ = nullptr;
   unsigned char* nv12_data_ = nullptr;
   unsigned int seq_ = 0;
+
+  const AVCodec* codec_ = nullptr;
+  AVCodecContext* codec_ctx_ = nullptr;
+  AVFrame* frame_ = nullptr;
+  AVPacket* packet_;
+  bool got_output_ = false;
+  uint32_t pts_ = 0;
 };
 
 #endif

src/media/video/encode/nvcodec/nv_encoder.cpp

@@ -40,7 +40,7 @@ int VideoEncoder::Init() {
   if (!cuda_ctx_succeed) {
   }
 
-  encoder_ = new NvEncoderCuda(cuda_context_, frame_width, frame_height,
+  encoder_ = new NvEncoderCuda(cuda_context_, frame_width_, frame_height_,
                                NV_ENC_BUFFER_FORMAT::NV_ENC_BUFFER_FORMAT_NV12);
 
   // Init encoder_ session
@@ -52,8 +52,8 @@ int VideoEncoder::Init() {
   encoder_->CreateDefaultEncoderParams(&init_params, codec_guid, preset_guid,
                                        tuning_info);
 
-  init_params.encodeWidth = frame_width;
+  init_params.encodeWidth = frame_width_;
-  init_params.encodeHeight = frame_height;
+  init_params.encodeHeight = frame_height_;
   init_params.encodeConfig->profileGUID = NV_ENC_H264_PROFILE_BASELINE_GUID;
   init_params.encodeConfig->encodeCodecConfig.h264Config.level =
       NV_ENC_LEVEL::NV_ENC_LEVEL_H264_31;
@@ -108,6 +108,9 @@ int VideoEncoder::Encode(
   for (const auto &packet : encoded_packets_) {
     if (on_encoded_image) {
       on_encoded_image((char *)packet.data(), packet.size());
+      if (SAVE_ENCODER_STREAM) {
+        fwrite(packet.data(), 1, packet.size(), file_);
+      }
     } else {
       OnEncodedImage((char *)packet.data(), packet.size());
     }
@@ -128,8 +131,7 @@ int VideoEncoder::Encode(
 }
 
 int VideoEncoder::OnEncodedImage(char *encoded_packets, size_t size) {
-  LOG_INFO("output encoded image");
+  LOG_INFO("OnEncodedImage not implemented");
-  fwrite(encoded_packets, 1, size, file_);
   return 0;
 }
 

src/media/video/encode/nvcodec/nv_encoder.h

@@ -25,8 +25,8 @@ class VideoEncoder {
   GUID preset_guid = NV_ENC_PRESET_P2_GUID;
   NV_ENC_TUNING_INFO tuning_info =
       NV_ENC_TUNING_INFO::NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY;
-  int frame_width = 1280;
+  int frame_width_ = 1280;
-  int frame_height = 720;
+  int frame_height_ = 720;
   int keyFrameInterval_ = 3000;
   int maxBitrate_ = 1000;
   int max_payload_size_ = 3000;

src/pc/peer_connection.h

@@ -7,11 +7,11 @@
 
 #include "ice_transmission.h"
 #ifdef _WIN32
-#include "nv_decoder.h"
+
-#include "nv_encoder.h"
-#else
 #include "ffmpeg_decoder.h"
 #include "ffmpeg_encoder.h"
+// #include "nv_decoder.h"
+// #include "nv_encoder.h"
 #endif
 
 #include "ws_transmission.h"

xmake.lua

@@ -29,7 +29,7 @@ add_defines("ASIO_STANDALONE", "ASIO_HAS_STD_TYPE_TRAITS", "ASIO_HAS_STD_SHARED_
 if is_os("windows") then
     add_defines("_WEBSOCKETPP_CPP11_INTERNAL_")
     add_links("ws2_32", "Bcrypt")
-    add_links("windowsapp", "User32", "Strmiids", "Mfuuid")
+    add_links("windowsapp", "User32", "Strmiids", "Mfuuid", "Secur32", "Bcrypt")
     add_requires("cuda")
 elseif is_os("linux") then 
     add_links("pthread")
@@ -105,16 +105,23 @@ target("media")
     if is_os("windows") or is_os(("linux")) then
         add_packages("cuda")
         add_files("src/media/video/encode/nvcodec/*.cpp",
-        "src/media/video/decode/nvcodec/*.cpp")
+        "src/media/video/decode/nvcodec/*.cpp",
+        "src/media/video/encode/ffmpeg/*.cpp",
+        "src/media/video/decode/ffmpeg/*.cpp"
+        )
         add_includedirs("src/media/video/encode/nvcodec",
-        "src/media/video/decode/nvcodec", "src/media/video/decode/ffmpeg",
+        "src/media/video/decode/nvcodec",
+        "src/media/video/encode/ffmpeg",
+        "src/media/video/decode/ffmpeg",
         "thirdparty/nvcodec/Interface",
         "thirdparty/nvcodec/Samples", {public = true})
         add_linkdirs("thirdparty/nvcodec/Lib/x64")
         add_links("cuda", "nvencodeapi", "nvcuvid")
     elseif is_os("macosx") then
-        add_files("src/media/video/encode/ffmpeg/*.cpp", "src/media/video/decode/ffmpeg/*.cpp")
+        add_files("src/media/video/encode/ffmpeg/*.cpp",
-        add_includedirs("src/media/video/encode/ffmpeg", "src/media/video/decode/ffmpeg", {public = true})
+        "src/media/video/decode/ffmpeg/*.cpp")
+        add_includedirs("src/media/video/encode/ffmpeg",
+        "src/media/video/decode/ffmpeg", {public = true})
     end
 
 target("qos")