[feat] enable resolution downgrading

src/media/resolution_adapter/resolution_adapter.cpp

@@ -0,0 +1,49 @@
+#include "resolution_adapter.h"
+
+#include "libyuv.h"
+
+int ResolutionAdapter::GetResolution(int target_bitrate, int current_width,
+                                     int current_height, int& target_width,
+                                     int& target_height) {
+  for (auto& resolution : GetBitrateLimits()) {
+    if (target_bitrate >= resolution.min_start_bitrate_bps &&
+        target_bitrate <= resolution.max_bitrate_bps) {
+      if (current_width * current_height <= resolution.frame_size_pixels) {
+        target_width = current_width;
+        target_height = current_height;
+        return 0;
+      } else {
+        target_width = current_width * 3 / 5;
+        target_height = current_height * 3 / 5;
+        return 0;
+      }
+    }
+  }
+
+  return -1;
+}
+
+int ResolutionAdapter::ResolutionDowngrade(const XVideoFrame* video_frame,
+                                           int target_width, int target_height,
+                                           XVideoFrame* new_frame) {
+  if (target_width <= 0 || target_height <= 0) {
+    return -1;
+  }
+
+  target_width = video_frame->width * 3 / 5;
+  target_height = video_frame->height * 3 / 5;
+  new_frame->width = target_width;
+  new_frame->height = target_height;
+  new_frame->data = new char[target_width * target_height * 3 / 2];
+
+  libyuv::NV12Scale((const uint8_t*)(video_frame->data), video_frame->width,
+                    (const uint8_t*)(video_frame->data +
+                                     video_frame->width * video_frame->height),
+                    video_frame->width, video_frame->width, video_frame->height,
+                    (uint8_t*)(new_frame->data), target_width,
+                    (uint8_t*)(new_frame->data + target_width * target_height),
+                    target_width, target_width, target_height,
+                    libyuv::kFilterLinear);
+
+  return 0;
+}

src/media/resolution_adapter/resolution_adapter.h

@@ -0,0 +1,41 @@
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2025-03-06
+ * Copyright (c) 2025 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _RESOLUTION_ADAPTER_H_
+#define _RESOLUTION_ADAPTER_H_
+
+#include <vector>
+
+#include "resolution_bitrate_limits.h"
+#include "x.h"
+
+class ResolutionAdapter {
+ public:
+  ResolutionAdapter() = default;
+  ~ResolutionAdapter() = default;
+
+ public:
+  int GetResolution(int target_bitrate, int current_width, int current_height,
+                    int& target_width, int& target_height);
+
+  int ResolutionDowngrade(const XVideoFrame* video_frame, int target_width,
+                          int target_height, XVideoFrame* new_frame);
+
+ public:
+  std::vector<ResolutionBitrateLimits> GetBitrateLimits() {
+    return {{0 * 0, 0, 0, 0},
+            {320 * 180, 0, 30000, 300000},
+            {480 * 270, 300000, 30000, 500000},
+            {640 * 360, 500000, 30000, 800000},
+            {960 * 540, 800000, 30000, 1500000},
+            {1280 * 720, 1500000, 30000, 2500000},
+            {1920 * 1080, 2500000, 30000, 4000000}};
+  }
+
+  int SetTargetBitrate(int bitrate);
+};
+
+#endif

src/media/resolution_adapter/resolution_bitrate_limits.h

@@ -0,0 +1,27 @@
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2025-03-07
+ * Copyright (c) 2025 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _RESOLUTION_BITRATE_LIMITS_H_
+#define _RESOLUTION_BITRATE_LIMITS_H_
+
+struct ResolutionBitrateLimits {
+  ResolutionBitrateLimits(int frame_size_pixels, int min_start_bitrate_bps,
+                          int min_bitrate_bps, int max_bitrate_bps)
+      : frame_size_pixels(frame_size_pixels),
+        min_start_bitrate_bps(min_start_bitrate_bps),
+        min_bitrate_bps(min_bitrate_bps),
+        max_bitrate_bps(max_bitrate_bps) {}
+  int frame_size_pixels = 0;
+  int min_start_bitrate_bps = 0;
+  int min_bitrate_bps = 0;
+  int max_bitrate_bps = 0;
+  bool operator==(const ResolutionBitrateLimits& rhs) const;
+  bool operator!=(const ResolutionBitrateLimits& rhs) const {
+    return !(*this == rhs);
+  }
+};
+
+#endif

src/media/video/decode/nvcodec/nvidia_video_decoder.cpp

@@ -44,7 +44,8 @@ int NvidiaVideoDecoder::Init() {
     return -1;
   }
 
-  decoder = new NvDecoder(cuContext, false, cudaVideoCodec_H264, true);
+  decoder = new NvDecoder(cuContext, false, cudaVideoCodec_H264, true, false,
+                          nullptr, nullptr, false, 4096, 2160, 1000, false);
 
 #ifdef SAVE_DECODED_NV12_STREAM
   file_nv12_ = fopen("decoded_nv12_stream.yuv", "w+b");

src/media/video/encode/aom/aom_av1_encoder.h

@@ -46,6 +46,12 @@ class AomAv1Encoder : public VideoEncoder {
 
   int SetTargetBitrate(int bitrate);
 
+  int GetResolution(int& width, int& height) {
+    width = frame_width_;
+    height = frame_height_;
+    return 0;
+  }
+
   std::string GetEncoderName() { return "AomAV1"; }
 
  private:

src/media/video/encode/nvcodec/nvidia_video_encoder.h

@@ -22,6 +22,12 @@ class NvidiaVideoEncoder : public VideoEncoder {
 
   int SetTargetBitrate(int bitrate);
 
+  int GetResolution(int& width, int& height) {
+    width = frame_width_;
+    height = frame_height_;
+    return 0;
+  }
+
   std::string GetEncoderName() { return "NvidiaH264"; }
 
  private:

src/media/video/encode/openh264/openh264_encoder.cpp

@@ -367,6 +367,10 @@ int OpenH264Encoder::ForceIdr() {
 }
 
 int OpenH264Encoder::SetTargetBitrate(int bitrate) {
+  if (!openh264_encoder_) {
+    return -1;
+  }
+
   target_bitrate_ = bitrate;
   encoder_params_.iTargetBitrate = target_bitrate_;
 

src/media/video/encode/openh264/openh264_encoder.h

@@ -33,6 +33,12 @@ class OpenH264Encoder : public VideoEncoder {
 
   int SetTargetBitrate(int bitrate);
 
+  int GetResolution(int& width, int& height) {
+    width = frame_width_;
+    height = frame_height_;
+    return 0;
+  }
+
   std::string GetEncoderName() { return "OpenH264"; }
 
  private:

src/media/video/encode/video_encoder.h

@@ -24,6 +24,8 @@ class VideoEncoder {
 
   virtual int SetTargetBitrate(int bitrate) = 0;
 
+  virtual int GetResolution(int& width, int& height) = 0;
+
   virtual std::string GetEncoderName() = 0;
 
   VideoEncoder() = default;

src/qos/send_side_bandwidth_estimation.cc

@@ -413,7 +413,7 @@ void SendSideBandwidthEstimation::UpdateEstimate(Timestamp at_time) {
       // (gives a little extra increase at low rates, negligible at higher
       // rates).
       new_bitrate += DataRate::BitsPerSec(1000);
-      LOG_WARN("1 new_bitrate: [{}]", ToString(new_bitrate).c_str());
+      LOG_WARN("new_bitrate: [{}]", ToString(new_bitrate).c_str());
       UpdateTargetBitrate(new_bitrate, at_time);
       return;
     } else if (current_target_ > bitrate_threshold_) {

src/transport/ice_transport_controller.cpp

@@ -13,6 +13,8 @@ IceTransportController::IceTransportController(
     : last_report_block_time_(
           webrtc::Timestamp::Millis(webrtc_clock_->TimeInMilliseconds())),
       b_force_i_frame_(true),
+      target_width_(1280),
+      target_height_(720),
       video_codec_inited_(false),
       audio_codec_inited_(false),
       load_nvcodec_dll_success_(false),
@@ -50,6 +52,7 @@ void IceTransportController::Create(
   CreateAudioCodec();
 
   controller_ = std::make_unique<CongestionControl>();
+  resolution_adapter_ = std::make_unique<ResolutionAdapter>();
 
   video_channel_send_ = std::make_unique<VideoChannelSend>(
       clock_, ice_agent, ice_io_statistics,
@@ -126,6 +129,8 @@ int IceTransportController::SendVideo(const XVideoFrame* video_frame) {
     LOG_ERROR("Video Encoder not created");
     return -1;
   }
+  source_width_ = video_frame->width;
+  source_height_ = video_frame->height;
 
   if (b_force_i_frame_) {
     video_encoder_->ForceIdr();
@@ -133,8 +138,24 @@ int IceTransportController::SendVideo(const XVideoFrame* video_frame) {
     b_force_i_frame_ = false;
   }
 
+  bool need_to_release = false;
+
+  XVideoFrame new_frame;
+  new_frame.data = nullptr;
+  new_frame.width = video_frame->width;
+  new_frame.height = video_frame->height;
+  if (target_width_.has_value() && target_height_.has_value()) {
+    if (target_width_.value() < video_frame->width &&
+        target_height_.value() < video_frame->height) {
+      resolution_adapter_->ResolutionDowngrade(
+          video_frame, target_width_.value(), target_height_.value(),
+          &new_frame);
+      need_to_release = true;
+    }
+  }
+
   int ret = video_encoder_->Encode(
-      video_frame,
+      need_to_release ? &new_frame : video_frame,
       [this](std::shared_ptr<VideoFrameWrapper> encoded_frame) -> int {
         if (video_channel_send_) {
           video_channel_send_->SendVideo(encoded_frame);
@@ -143,6 +164,10 @@ int IceTransportController::SendVideo(const XVideoFrame* video_frame) {
         return 0;
       });
 
+  if (need_to_release) {
+    delete[] new_frame.data;
+  }
+
   if (0 != ret) {
     LOG_ERROR("Encode failed");
     return -1;
@@ -429,11 +454,33 @@ void IceTransportController::OnSentRtpPacket(
 void IceTransportController::PostUpdates(webrtc::NetworkControlUpdate update) {
   // UpdateControlState();
 
-  target_bitrate_ = update.target_rate.has_value()
+  int target_bitrate = update.target_rate.has_value()
-                        ? update.target_rate->target_rate.bps()
+                           ? (update.target_rate->target_rate.bps() == 0
-                        : 0;
+                                  ? target_bitrate_
-  // LOG_WARN("Target bitrate [{}]bps", target_bitrate_);
+                                  : update.target_rate->target_rate.bps())
-  video_encoder_->SetTargetBitrate(target_bitrate_);
+                           : target_bitrate_;
+  if (target_bitrate != target_bitrate_) {
+    target_bitrate_ = target_bitrate;
+    int width, height, target_width, target_height;
+    video_encoder_->GetResolution(width, height);
+
+    if (0 == resolution_adapter_->GetResolution(target_bitrate_, width, height,
+                                                target_width, target_height)) {
+      if (target_width != target_width_ || target_height != target_height_) {
+        target_width_ = target_width;
+        target_height_ = target_height;
+        LOG_WARN("Set target resolution [{}x{}]", target_width_.value(),
+                 target_height_.value());
+      }
+    } else {
+      target_width_.reset();
+      target_height_.reset();
+      LOG_WARN("Use original resolution [{}x{}]", source_width_,
+               source_height_);
+    }
+    video_encoder_->SetTargetBitrate(target_bitrate_);
+    // LOG_WARN("Set target bitrate [{}]bps", target_bitrate_);
+  }
 }
 
 void IceTransportController::UpdateControlState() {

src/transport/ice_transport_controller.h

@@ -20,6 +20,7 @@
 #include "data_channel_receive.h"
 #include "data_channel_send.h"
 #include "ice_agent.h"
+#include "resolution_adapter.h"
 #include "transport_feedback_adapter.h"
 #include "video_channel_receive.h"
 #include "video_channel_send.h"
@@ -108,10 +109,15 @@ class IceTransportController
  private:
   std::unique_ptr<VideoEncoder> video_encoder_ = nullptr;
   std::unique_ptr<VideoDecoder> video_decoder_ = nullptr;
+  std::unique_ptr<ResolutionAdapter> resolution_adapter_ = nullptr;
   bool b_force_i_frame_;
   bool video_codec_inited_;
   bool load_nvcodec_dll_success_;
   bool hardware_acceleration_;
+  int source_width_;
+  int source_height_;
+  std::optional<int> target_width_;
+  std::optional<int> target_height_;
 
  private:
   std::unique_ptr<AudioEncoder> audio_encoder_ = nullptr;

xmake.lua

@@ -218,9 +218,12 @@ target("media")
         "src/media/video/decode/aom", {public = true})
     end
     add_files("src/media/audio/encode/*.cpp",
-        "src/media/audio/decode/*.cpp")
+        "src/media/audio/decode/*.cpp",
+        "src/media/resolution_adapter/*.cpp")
     add_includedirs("src/media/audio/encode",
-        "src/media/audio/decode", "src/interface", {public = true})
+        "src/media/audio/decode",
+        "src/media/resolution_adapter",
+        "src/interface", {public = true})
 
 target("pc")
     set_kind("object")