OpenH264 supports Windows platform

application/remote_desk/remote_desk_gui/main.cpp

@@ -88,7 +88,12 @@ ScreenCaptureX11 *screen_capture = nullptr;
 #endif
 
 char *nv12_buffer = nullptr;
+
+#ifdef __linux__
 std::chrono::_V2::system_clock::time_point last_frame_time_;
+#else
+std::chrono::steady_clock::time_point last_frame_time_;
+#endif
 
 typedef enum { mouse = 0, keyboard } ControlType;
 typedef enum { move = 0, left_down, left_up, right_down, right_up } MouseFlag;

config/config.ini

@@ -13,4 +13,4 @@ username = dijunkun
 password = dijunkunpw
 
 [hardware acceleration]
-turn_on = true
+turn_on = false

src/media/video/decode/openh264/openh264_decoder.cpp

@@ -134,14 +134,6 @@ int OpenH264Decoder::Init() {
   frame_width_ = 1280;
   frame_height_ = 720;
 
-  // SEncParamExt sParam;
-  // sParam.iPicWidth = frame_width_;
-  // sParam.iPicHeight = frame_height_;
-  // sParam.iTargetBitrate = 1000000000;
-  // sParam.iTemporalLayerNum = 1;
-  // sParam.fMaxFrameRate = 30;
-  // sParam.iSpatialLayerNum = 1;
-
   decoded_frame_size_ = YUV420P_BUFFER_SIZE;
   decoded_frame_ = new uint8_t[YUV420P_BUFFER_SIZE];
   nv12_frame_ = new uint8_t[YUV420P_BUFFER_SIZE];
@@ -192,14 +184,27 @@ int OpenH264Decoder::Decode(
 
   unsigned char *dst[3];
 
-  int iRet =
+  int iRet = openh264_decoder_->DecodeFrame2(data, size, dst, &sDstBufInfo);
-      openh264_decoder_->DecodeFrameNoDelay(data, size, dst, &sDstBufInfo);
+  // int iRet =
+  //     openh264_decoder_->DecodeFrameNoDelay(data, size, dst, &sDstBufInfo);
 
   if (iRet != 0) {
     return -1;
   }
 
-  if (sDstBufInfo.iBufferStatus == 1) {
+  // int num_of_buffer = 0;
+  // iRet = openh264_decoder_->GetOption(
+  //     DECODER_OPTION_NUM_OF_FRAMES_REMAINING_IN_BUFFER, &num_of_buffer);
+
+  // LOG_ERROR("Number of buffer {} {}", num_of_buffer, iRet);
+
+  // iRet = openh264_decoder_->FlushFrame(dst, &sDstBufInfo);
+  // if (iRet != 0) {
+  //   LOG_ERROR("FlushFrame state: {}", iRet);
+  //   return -1;
+  // }
+
+  if (1) {
     if (on_receive_decoded_frame) {
       CopyYUVWithStride(
           dst[0], dst[1], dst[2], sDstBufInfo.UsrData.sSystemBuffer.iWidth,

src/media/video/decode/video_decoder_factory.cpp

@@ -26,8 +26,8 @@ std::unique_ptr<VideoDecoder> VideoDecoderFactory::CreateVideoDecoder(
       return nullptr;
     }
   } else {
-    // return std::make_unique<FfmpegVideoDecoder>(FfmpegVideoDecoder());
+    return std::make_unique<FfmpegVideoDecoder>(FfmpegVideoDecoder());
-    return std::make_unique<OpenH264Decoder>(OpenH264Decoder());
+    // return std::make_unique<OpenH264Decoder>(OpenH264Decoder());
   }
 #endif
 }

src/media/video/encode/ffmpeg/ffmpeg_video_encoder.cpp

@@ -59,19 +59,20 @@ FFmpegVideoEncoder::~FFmpegVideoEncoder() {
 }
 
 int FFmpegVideoEncoder::Init() {
-  av_log_set_level(AV_LOG_VERBOSE);
+  // av_log_set_level(AV_LOG_VERBOSE);
 
   codec_ = avcodec_find_encoder(AV_CODEC_ID_H264);
   if (!codec_) {
     LOG_ERROR("Failed to find H.264 encoder");
     return -1;
-  } else {
-    LOG_INFO("Use H264 encoder [{}]", codec_->name);
-    if (0 == strcmp(codec_->name, "libx264")) {
-      use_libx264_ = true;
-    }
   }
-  use_libx264_ = true;
+  // else {
+  //   LOG_INFO("Use H264 encoder [{}]", codec_->name);
+  //   if (0 == strcmp(codec_->name, "libx264")) {
+  //     use_libx264_ = true;
+  //   }
+  // }
+  // use_libx264_ = true;
 
   codec_ctx_ = avcodec_alloc_context3(codec_);
   if (!codec_ctx_) {
@@ -88,17 +89,17 @@ int FFmpegVideoEncoder::Init() {
   if (use_libx264_) {
     codec_ctx_->pix_fmt = AV_PIX_FMT_YUV420P;
   } else {
-    codec_ctx_->pix_fmt = AV_PIX_FMT_YUV420P;
+    codec_ctx_->pix_fmt = AV_PIX_FMT_NV12;
   }
   codec_ctx_->gop_size = keyFrameInterval_;
   codec_ctx_->keyint_min = keyFrameInterval_;
   codec_ctx_->max_b_frames = 0;
-  codec_ctx_->bit_rate = maxBitrate_ * 500;
+  codec_ctx_->bit_rate = maxBitrate_ * 2500;
 
-  // av_opt_set_int(codec_ctx_->priv_data, "qp", 51, 0);
+  av_opt_set_int(codec_ctx_->priv_data, "qp", 51, 0);
-  // av_opt_set_int(codec_ctx_->priv_data, "crf", 23, 0);
+  av_opt_set_int(codec_ctx_->priv_data, "crf", 23, 0);
 
-  // av_opt_set(codec_ctx_->priv_data, "profile", "baseline", 0);
+  av_opt_set(codec_ctx_->priv_data, "profile", "baseline", 0);
   av_opt_set(codec_ctx_->priv_data, "preset", "ultrafast", 0);
   av_opt_set(codec_ctx_->priv_data, "tune", "zerolatency", 0);
 
@@ -150,10 +151,10 @@ int FFmpegVideoEncoder::Encode(
                         (unsigned char *)yuv420p_buffer);
     memcpy(frame_->data[0], yuv420p_buffer, frame_->width * frame_->height);
     memcpy(frame_->data[1], yuv420p_buffer + frame_->width * frame_->height,
-           frame_->width * frame_->height / 2);
+           frame_->width * frame_->height / 4);
     memcpy(frame_->data[2],
-           yuv420p_buffer + frame_->width * frame_->height * 3 / 2,
+           yuv420p_buffer + frame_->width * frame_->height * 5 / 4,
-           frame_->width * frame_->height / 2);
+           frame_->width * frame_->height / 4);
 
     // frame_->data[0] = yuv420p_buffer;
     // frame_->data[1] = yuv420p_buffer + frame_->width * frame_->height;
@@ -187,12 +188,12 @@ int FFmpegVideoEncoder::Encode(
     }
 
     // Remove first 6 bytes in I frame, SEI ?
-    // if (0x00 == packet_->data[0] && 0x00 == packet_->data[1] &&
+    if (0x00 == packet_->data[0] && 0x00 == packet_->data[1] &&
-    //     0x00 == packet_->data[2] && 0x01 == packet_->data[3] &&
+        0x00 == packet_->data[2] && 0x01 == packet_->data[3] &&
-    //     0x09 == packet_->data[4] && 0x10 == packet_->data[5]) {
+        0x09 == packet_->data[4] && 0x10 == packet_->data[5]) {
-    //   packet_->data += 6;
+      packet_->data += 6;
-    //   packet_->size -= 6;
+      packet_->size -= 6;
-    // }
+    }
 
     if (on_encoded_image) {
       on_encoded_image((char *)packet_->data, packet_->size);

src/media/video/encode/ffmpeg/ffmpeg_video_encoder.h

@@ -7,6 +7,7 @@ extern "C" {
 #include "libavformat/avformat.h"
 #include "libavutil/imgutils.h"
 #include "libavutil/opt.h"
+#include "libswscale/swscale.h"
 }
 #else
 #ifdef __cplusplus

src/media/video/encode/openh264/openh264_encoder.cpp

@@ -69,9 +69,9 @@ SEncParamExt OpenH264Encoder::CreateEncoderParams() const {
   SEncParamExt encoder_params;
   openh264_encoder_->GetDefaultParams(&encoder_params);
   // if (codec_.mode == VideoCodecMode::kRealtimeVideo) {  //
-  encoder_params.iUsageType = CAMERA_VIDEO_REAL_TIME;
+  // encoder_params.iUsageType = CAMERA_VIDEO_REAL_TIME;
   // } else if (codec_.mode == VideoCodecMode::kScreensharing) {
-  // encoder_params.iUsageType = SCREEN_CONTENT_REAL_TIME;
+  encoder_params.iUsageType = SCREEN_CONTENT_REAL_TIME;
   // }
 
   encoder_params.iPicWidth = frame_width_;
@@ -80,9 +80,11 @@ SEncParamExt OpenH264Encoder::CreateEncoderParams() const {
   encoder_params.iMaxBitrate = max_bitrate_;
   encoder_params.iRCMode = RC_BITRATE_MODE;
   encoder_params.fMaxFrameRate = 0.000030;
-  encoder_params.bEnableFrameSkip = true;
+  encoder_params.bEnableFrameSkip = false;
   encoder_params.uiIntraPeriod = key_frame_interval_;
   encoder_params.uiMaxNalSize = 0;
+  encoder_params.iMaxQp = 38;
+  encoder_params.iMinQp = 16;
   // Threading model: use auto.
   //  0: auto (dynamic imp. internal encoder)
   //  1: single thread (default value)
@@ -96,11 +98,6 @@ SEncParamExt OpenH264Encoder::CreateEncoderParams() const {
       encoder_params.iTargetBitrate;
   encoder_params.sSpatialLayers[0].iMaxSpatialBitrate =
       encoder_params.iMaxBitrate;
-  encoder_params.iTemporalLayerNum = 1;
-  if (encoder_params.iTemporalLayerNum > 1) {
-    encoder_params.iNumRefFrame = 1;
-  }
-  LOG_INFO("OpenH264 version is [{}.{}]", OPENH264_MAJOR, OPENH264_MINOR);
 
   // SingleNalUnit
   encoder_params.sSpatialLayers[0].sSliceArgument.uiSliceNum = 1;
@@ -166,18 +163,12 @@ int OpenH264Encoder::Encode(
     fwrite(yuv420p_buffer, 1, nSize, file_nv12_);
   }
 
-  if (0 == seq_++ % 3) {
+  if (0 == seq_++ % 300) {
     ForceIdr();
   }
 
   NV12ToYUV420PFFmpeg((unsigned char *)pData, frame_width_, frame_height_,
                       (unsigned char *)yuv420p_buffer);
-  // memcpy(frame_->data[0], yuv420p_buffer, frame_width_ * frame_height_);
-  // memcpy(frame_->data[1], yuv420p_buffer + frame_width_ * frame_height_,
-  //        frame_width_ * frame_height_ / 2);
-  // memcpy(frame_->data[2], yuv420p_buffer + frame_width_ * frame_height_ * 3 /
-  // 2,
-  //        frame_width_ * frame_height_ / 2);
 
   raw_frame_ = {0};
   raw_frame_.iPicWidth = frame_width_;
@@ -206,22 +197,34 @@ int OpenH264Encoder::Encode(
   }
 
 #if 0
-    int encoded_frame_size = 0;
+  size_t required_capacity = 0;
-
+  size_t fragments_count = 0;
-    for (int layer = 0; layer < info.iLayerNum; ++layer) {
+  for (int layer = 0; layer < info.iLayerNum; ++layer) {
-      const SLayerBSInfo &layerInfo = info.sLayerInfo[layer];
+    const SLayerBSInfo &layerInfo = info.sLayerInfo[layer];
-      size_t layer_len = 0;
+    for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++fragments_count) {
-      memcpy(encoded_frame_ + encoded_frame_size, layerInfo.pBsBuf, layer_len);
+      required_capacity += layerInfo.pNalLengthInByte[nal];
-      encoded_frame_size += layer_len;
     }
+  }
 
-    encoded_frame_size_ = encoded_frame_size;
+  const uint8_t start_code[4] = {0, 0, 0, 1};
-
+  size_t frag = 0;
-    if (on_encoded_image) {
+  int encoded_frame_size = 0;
-      on_encoded_image((char *)encoded_frame_, encoded_frame_size_);
+  for (int layer = 0; layer < info.iLayerNum; ++layer) {
-    } else {
+    const SLayerBSInfo &layerInfo = info.sLayerInfo[layer];
-      OnEncodedImage((char *)encoded_frame_, encoded_frame_size_);
+    size_t layer_len = 0;
+    for (int nal = 0; nal < layerInfo.iNalCount; ++nal, ++frag) {
+      layer_len += layerInfo.pNalLengthInByte[nal];
     }
+    memcpy(encoded_frame_ + encoded_frame_size, layerInfo.pBsBuf, layer_len);
+    encoded_frame_size += layer_len;
+  }
+  encoded_frame_size_ = encoded_frame_size;
+
+  if (on_encoded_image) {
+    on_encoded_image((char *)encoded_frame_, encoded_frame_size_);
+  } else {
+    OnEncodedImage((char *)encoded_frame_, encoded_frame_size_);
+  }
 #else
   if (info.eFrameType == videoFrameTypeInvalid) {
     LOG_ERROR("videoFrameTypeInvalid");

src/media/video/encode/openh264/openh264_encoder.h

@@ -38,10 +38,10 @@ class OpenH264Encoder : public VideoEncoder {
  private:
   int frame_width_ = 1280;
   int frame_height_ = 720;
-  int key_frame_interval_ = 5;
+  int key_frame_interval_ = 3000;
   int target_bitrate_ = 1000;
-  int max_bitrate_ = 1000;
+  int max_bitrate_ = 500000;
-  int max_payload_size_ = 3000;
+  int max_payload_size_ = 1400;
   int max_frame_rate_ = 30;
   std::vector<std::vector<uint8_t>> encoded_packets_;
   unsigned char* encoded_image_ = nullptr;

src/media/video/encode/video_encoder_factory.cpp

@@ -26,8 +26,8 @@ std::unique_ptr<VideoEncoder> VideoEncoderFactory::CreateVideoEncoder(
       return nullptr;
     }
   } else {
-    // return std::make_unique<FFmpegVideoEncoder>(FFmpegVideoEncoder());
+    return std::make_unique<FFmpegVideoEncoder>(FFmpegVideoEncoder());
-    return std::make_unique<OpenH264Encoder>(OpenH264Encoder());
+    // return std::make_unique<OpenH264Encoder>(OpenH264Encoder());
   }
 #endif
 }

thirdparty/openh264/xmake.lua

@@ -0,0 +1,31 @@
+package("openh264")
+
+    set_homepage("http://www.openh264.org/")
+    set_description("OpenH264 is a codec library which supports H.264 encoding and decoding.")
+    set_license("BSD-2-Clause")
+
+    add_urls("https://github.com/cisco/openh264/archive/refs/tags/$(version).tar.gz")
+    add_versions("v2.3.1", "453afa66dacb560bc5fd0468aabee90c483741571bca820a39a1c07f0362dc32")
+
+    add_deps("meson", "ninja", "nasm")
+    if is_plat("linux") then
+        add_syslinks("pthread", "rt")
+    end
+    on_install("windows", "linux", function (package)
+        import("package.tools.meson").build(package, {"-Dtests=disabled"}, {buildir = "out"})
+        import("package.tools.ninja").install(package, {}, {buildir = "out"})
+        -- if package:config("shared") then
+        --     os.tryrm(path.join(package:installdir("lib"), "libopenh264.a"))
+        -- else
+        --     os.tryrm(path.join(package:installdir("lib"), "libopenh264.so*"))
+        --     os.tryrm(path.join(package:installdir("lib"), "openh264.lib"))
+        --     os.tryrm(path.join(package:installdir("bin"), "openh264-*.dll"))
+        -- end
+        -- if package:is_plat("windows") then
+        --     os.trymv(path.join(package:installdir("lib"), "libopenh264.a"), path.join(package:installdir("lib"), "openh264.lib"))
+        -- end
+    end)
+
+    on_test(function (package)
+        assert(package:has_cxxfuncs("WelsGetCodecVersion", {includes = "wels/codec_api.h"}))
+    end)

thirdparty/xmake.lua

@@ -1,3 +1,4 @@
+includes("openh264")
 if is_plat("windows") then
 elseif is_plat("linux") then
     includes("ffmpeg")