diff --git a/src/media/resolution_adapter/resolution_adapter.cpp b/src/media/resolution_adapter/resolution_adapter.cpp new file mode 100644 index 0000000..67ec5f7 --- /dev/null +++ b/src/media/resolution_adapter/resolution_adapter.cpp @@ -0,0 +1,49 @@ +#include "resolution_adapter.h" + +#include "libyuv.h" + +int ResolutionAdapter::GetResolution(int target_bitrate, int current_width, + int current_height, int& target_width, + int& target_height) { + for (auto& resolution : GetBitrateLimits()) { + if (target_bitrate >= resolution.min_start_bitrate_bps && + target_bitrate <= resolution.max_bitrate_bps) { + if (current_width * current_height <= resolution.frame_size_pixels) { + target_width = current_width; + target_height = current_height; + return 0; + } else { + target_width = current_width * 3 / 5; + target_height = current_height * 3 / 5; + return 0; + } + } + } + + return -1; +} + +int ResolutionAdapter::ResolutionDowngrade(const XVideoFrame* video_frame, + int target_width, int target_height, + XVideoFrame* new_frame) { + if (target_width <= 0 || target_height <= 0) { + return -1; + } + + target_width = video_frame->width * 3 / 5; + target_height = video_frame->height * 3 / 5; + new_frame->width = target_width; + new_frame->height = target_height; + new_frame->data = new char[target_width * target_height * 3 / 2]; + + libyuv::NV12Scale((const uint8_t*)(video_frame->data), video_frame->width, + (const uint8_t*)(video_frame->data + + video_frame->width * video_frame->height), + video_frame->width, video_frame->width, video_frame->height, + (uint8_t*)(new_frame->data), target_width, + (uint8_t*)(new_frame->data + target_width * target_height), + target_width, target_width, target_height, + libyuv::kFilterLinear); + + return 0; +} \ No newline at end of file diff --git a/src/media/resolution_adapter/resolution_adapter.h b/src/media/resolution_adapter/resolution_adapter.h new file mode 100644 index 0000000..054e6ce --- /dev/null +++ b/src/media/resolution_adapter/resolution_adapter.h @@ -0,0 +1,41 @@ +/* + * @Author: DI JUNKUN + * @Date: 2025-03-06 + * Copyright (c) 2025 by DI JUNKUN, All Rights Reserved. + */ + +#ifndef _RESOLUTION_ADAPTER_H_ +#define _RESOLUTION_ADAPTER_H_ + +#include + +#include "resolution_bitrate_limits.h" +#include "x.h" + +class ResolutionAdapter { + public: + ResolutionAdapter() = default; + ~ResolutionAdapter() = default; + + public: + int GetResolution(int target_bitrate, int current_width, int current_height, + int& target_width, int& target_height); + + int ResolutionDowngrade(const XVideoFrame* video_frame, int target_width, + int target_height, XVideoFrame* new_frame); + + public: + std::vector GetBitrateLimits() { + return {{0 * 0, 0, 0, 0}, + {320 * 180, 0, 30000, 300000}, + {480 * 270, 300000, 30000, 500000}, + {640 * 360, 500000, 30000, 800000}, + {960 * 540, 800000, 30000, 1500000}, + {1280 * 720, 1500000, 30000, 2500000}, + {1920 * 1080, 2500000, 30000, 4000000}}; + } + + int SetTargetBitrate(int bitrate); +}; + +#endif \ No newline at end of file diff --git a/src/media/resolution_adapter/resolution_bitrate_limits.h b/src/media/resolution_adapter/resolution_bitrate_limits.h new file mode 100644 index 0000000..f01c3e0 --- /dev/null +++ b/src/media/resolution_adapter/resolution_bitrate_limits.h @@ -0,0 +1,27 @@ +/* + * @Author: DI JUNKUN + * @Date: 2025-03-07 + * Copyright (c) 2025 by DI JUNKUN, All Rights Reserved. + */ + +#ifndef _RESOLUTION_BITRATE_LIMITS_H_ +#define _RESOLUTION_BITRATE_LIMITS_H_ + +struct ResolutionBitrateLimits { + ResolutionBitrateLimits(int frame_size_pixels, int min_start_bitrate_bps, + int min_bitrate_bps, int max_bitrate_bps) + : frame_size_pixels(frame_size_pixels), + min_start_bitrate_bps(min_start_bitrate_bps), + min_bitrate_bps(min_bitrate_bps), + max_bitrate_bps(max_bitrate_bps) {} + int frame_size_pixels = 0; + int min_start_bitrate_bps = 0; + int min_bitrate_bps = 0; + int max_bitrate_bps = 0; + bool operator==(const ResolutionBitrateLimits& rhs) const; + bool operator!=(const ResolutionBitrateLimits& rhs) const { + return !(*this == rhs); + } +}; + +#endif \ No newline at end of file diff --git a/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp b/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp index 68735fc..96e8612 100644 --- a/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp +++ b/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp @@ -44,7 +44,8 @@ int NvidiaVideoDecoder::Init() { return -1; } - decoder = new NvDecoder(cuContext, false, cudaVideoCodec_H264, true); + decoder = new NvDecoder(cuContext, false, cudaVideoCodec_H264, true, false, + nullptr, nullptr, false, 4096, 2160, 1000, false); #ifdef SAVE_DECODED_NV12_STREAM file_nv12_ = fopen("decoded_nv12_stream.yuv", "w+b"); diff --git a/src/media/video/encode/aom/aom_av1_encoder.h b/src/media/video/encode/aom/aom_av1_encoder.h index 153da23..aac23d6 100644 --- a/src/media/video/encode/aom/aom_av1_encoder.h +++ b/src/media/video/encode/aom/aom_av1_encoder.h @@ -46,6 +46,12 @@ class AomAv1Encoder : public VideoEncoder { int SetTargetBitrate(int bitrate); + int GetResolution(int& width, int& height) { + width = frame_width_; + height = frame_height_; + return 0; + } + std::string GetEncoderName() { return "AomAV1"; } private: diff --git a/src/media/video/encode/nvcodec/nvidia_video_encoder.h b/src/media/video/encode/nvcodec/nvidia_video_encoder.h index 874b539..4da0e23 100644 --- a/src/media/video/encode/nvcodec/nvidia_video_encoder.h +++ b/src/media/video/encode/nvcodec/nvidia_video_encoder.h @@ -22,6 +22,12 @@ class NvidiaVideoEncoder : public VideoEncoder { int SetTargetBitrate(int bitrate); + int GetResolution(int& width, int& height) { + width = frame_width_; + height = frame_height_; + return 0; + } + std::string GetEncoderName() { return "NvidiaH264"; } private: diff --git a/src/media/video/encode/openh264/openh264_encoder.cpp b/src/media/video/encode/openh264/openh264_encoder.cpp index 20313ab..da10c13 100644 --- a/src/media/video/encode/openh264/openh264_encoder.cpp +++ b/src/media/video/encode/openh264/openh264_encoder.cpp @@ -367,6 +367,10 @@ int OpenH264Encoder::ForceIdr() { } int OpenH264Encoder::SetTargetBitrate(int bitrate) { + if (!openh264_encoder_) { + return -1; + } + target_bitrate_ = bitrate; encoder_params_.iTargetBitrate = target_bitrate_; diff --git a/src/media/video/encode/openh264/openh264_encoder.h b/src/media/video/encode/openh264/openh264_encoder.h index c2ae494..25aa2a3 100644 --- a/src/media/video/encode/openh264/openh264_encoder.h +++ b/src/media/video/encode/openh264/openh264_encoder.h @@ -33,6 +33,12 @@ class OpenH264Encoder : public VideoEncoder { int SetTargetBitrate(int bitrate); + int GetResolution(int& width, int& height) { + width = frame_width_; + height = frame_height_; + return 0; + } + std::string GetEncoderName() { return "OpenH264"; } private: diff --git a/src/media/video/encode/video_encoder.h b/src/media/video/encode/video_encoder.h index 0a3867d..16312fa 100644 --- a/src/media/video/encode/video_encoder.h +++ b/src/media/video/encode/video_encoder.h @@ -24,6 +24,8 @@ class VideoEncoder { virtual int SetTargetBitrate(int bitrate) = 0; + virtual int GetResolution(int& width, int& height) = 0; + virtual std::string GetEncoderName() = 0; VideoEncoder() = default; diff --git a/src/qos/send_side_bandwidth_estimation.cc b/src/qos/send_side_bandwidth_estimation.cc index 9101ad2..9a5ba91 100644 --- a/src/qos/send_side_bandwidth_estimation.cc +++ b/src/qos/send_side_bandwidth_estimation.cc @@ -413,7 +413,7 @@ void SendSideBandwidthEstimation::UpdateEstimate(Timestamp at_time) { // (gives a little extra increase at low rates, negligible at higher // rates). new_bitrate += DataRate::BitsPerSec(1000); - LOG_WARN("1 new_bitrate: [{}]", ToString(new_bitrate).c_str()); + LOG_WARN("new_bitrate: [{}]", ToString(new_bitrate).c_str()); UpdateTargetBitrate(new_bitrate, at_time); return; } else if (current_target_ > bitrate_threshold_) { diff --git a/src/transport/ice_transport_controller.cpp b/src/transport/ice_transport_controller.cpp index 836663e..d07d436 100644 --- a/src/transport/ice_transport_controller.cpp +++ b/src/transport/ice_transport_controller.cpp @@ -13,6 +13,8 @@ IceTransportController::IceTransportController( : last_report_block_time_( webrtc::Timestamp::Millis(webrtc_clock_->TimeInMilliseconds())), b_force_i_frame_(true), + target_width_(1280), + target_height_(720), video_codec_inited_(false), audio_codec_inited_(false), load_nvcodec_dll_success_(false), @@ -50,6 +52,7 @@ void IceTransportController::Create( CreateAudioCodec(); controller_ = std::make_unique(); + resolution_adapter_ = std::make_unique(); video_channel_send_ = std::make_unique( clock_, ice_agent, ice_io_statistics, @@ -126,6 +129,8 @@ int IceTransportController::SendVideo(const XVideoFrame* video_frame) { LOG_ERROR("Video Encoder not created"); return -1; } + source_width_ = video_frame->width; + source_height_ = video_frame->height; if (b_force_i_frame_) { video_encoder_->ForceIdr(); @@ -133,8 +138,24 @@ int IceTransportController::SendVideo(const XVideoFrame* video_frame) { b_force_i_frame_ = false; } + bool need_to_release = false; + + XVideoFrame new_frame; + new_frame.data = nullptr; + new_frame.width = video_frame->width; + new_frame.height = video_frame->height; + if (target_width_.has_value() && target_height_.has_value()) { + if (target_width_.value() < video_frame->width && + target_height_.value() < video_frame->height) { + resolution_adapter_->ResolutionDowngrade( + video_frame, target_width_.value(), target_height_.value(), + &new_frame); + need_to_release = true; + } + } + int ret = video_encoder_->Encode( - video_frame, + need_to_release ? &new_frame : video_frame, [this](std::shared_ptr encoded_frame) -> int { if (video_channel_send_) { video_channel_send_->SendVideo(encoded_frame); @@ -143,6 +164,10 @@ int IceTransportController::SendVideo(const XVideoFrame* video_frame) { return 0; }); + if (need_to_release) { + delete[] new_frame.data; + } + if (0 != ret) { LOG_ERROR("Encode failed"); return -1; @@ -429,11 +454,33 @@ void IceTransportController::OnSentRtpPacket( void IceTransportController::PostUpdates(webrtc::NetworkControlUpdate update) { // UpdateControlState(); - target_bitrate_ = update.target_rate.has_value() - ? update.target_rate->target_rate.bps() - : 0; - // LOG_WARN("Target bitrate [{}]bps", target_bitrate_); - video_encoder_->SetTargetBitrate(target_bitrate_); + int target_bitrate = update.target_rate.has_value() + ? (update.target_rate->target_rate.bps() == 0 + ? target_bitrate_ + : update.target_rate->target_rate.bps()) + : target_bitrate_; + if (target_bitrate != target_bitrate_) { + target_bitrate_ = target_bitrate; + int width, height, target_width, target_height; + video_encoder_->GetResolution(width, height); + + if (0 == resolution_adapter_->GetResolution(target_bitrate_, width, height, + target_width, target_height)) { + if (target_width != target_width_ || target_height != target_height_) { + target_width_ = target_width; + target_height_ = target_height; + LOG_WARN("Set target resolution [{}x{}]", target_width_.value(), + target_height_.value()); + } + } else { + target_width_.reset(); + target_height_.reset(); + LOG_WARN("Use original resolution [{}x{}]", source_width_, + source_height_); + } + video_encoder_->SetTargetBitrate(target_bitrate_); + // LOG_WARN("Set target bitrate [{}]bps", target_bitrate_); + } } void IceTransportController::UpdateControlState() { diff --git a/src/transport/ice_transport_controller.h b/src/transport/ice_transport_controller.h index 097636e..86ddcf0 100644 --- a/src/transport/ice_transport_controller.h +++ b/src/transport/ice_transport_controller.h @@ -20,6 +20,7 @@ #include "data_channel_receive.h" #include "data_channel_send.h" #include "ice_agent.h" +#include "resolution_adapter.h" #include "transport_feedback_adapter.h" #include "video_channel_receive.h" #include "video_channel_send.h" @@ -108,10 +109,15 @@ class IceTransportController private: std::unique_ptr video_encoder_ = nullptr; std::unique_ptr video_decoder_ = nullptr; + std::unique_ptr resolution_adapter_ = nullptr; bool b_force_i_frame_; bool video_codec_inited_; bool load_nvcodec_dll_success_; bool hardware_acceleration_; + int source_width_; + int source_height_; + std::optional target_width_; + std::optional target_height_; private: std::unique_ptr audio_encoder_ = nullptr; diff --git a/xmake.lua b/xmake.lua index 4c2bb6d..2feff3b 100644 --- a/xmake.lua +++ b/xmake.lua @@ -218,9 +218,12 @@ target("media") "src/media/video/decode/aom", {public = true}) end add_files("src/media/audio/encode/*.cpp", - "src/media/audio/decode/*.cpp") + "src/media/audio/decode/*.cpp", + "src/media/resolution_adapter/*.cpp") add_includedirs("src/media/audio/encode", - "src/media/audio/decode", "src/interface", {public = true}) + "src/media/audio/decode", + "src/media/resolution_adapter", + "src/interface", {public = true}) target("pc") set_kind("object")