From 36704c7e4c9b643fb1f08298763cbb667deb59b4 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Tue, 11 Feb 2025 17:25:50 +0800 Subject: [PATCH] [feat] enable congestion controller set target bitrate to video encoder --- src/channel/video_channel_send.cpp | 90 +--- src/channel/video_channel_send.h | 14 +- .../video/encode/aom/aom_av1_encoder.cpp | 5 + src/media/video/encode/aom/aom_av1_encoder.h | 2 + .../encode/nvcodec/nvidia_video_encoder.cpp | 19 + .../encode/nvcodec/nvidia_video_encoder.h | 3 + .../encode/openh264/openh264_encoder.cpp | 7 + .../video/encode/openh264/openh264_encoder.h | 2 + src/media/video/encode/video_encoder.h | 2 + src/pc/peer_connection.cpp | 14 +- src/qos/congestion_control.cpp | 176 +++++--- src/qos/congestion_control.h | 10 + src/qos/delay_based_bwe.cc | 1 + src/qos/send_side_bandwidth_estimation.cc | 10 + src/qos/send_side_bandwidth_estimation.h | 13 + src/transport/ice_transport.cpp | 321 ++------------- src/transport/ice_transport.h | 96 +---- src/transport/ice_transport_controller.cpp | 389 ++++++++++++++++++ src/transport/ice_transport_controller.h | 121 ++++++ 19 files changed, 770 insertions(+), 525 deletions(-) create mode 100644 src/transport/ice_transport_controller.cpp create mode 100644 src/transport/ice_transport_controller.h diff --git a/src/channel/video_channel_send.cpp b/src/channel/video_channel_send.cpp index 00c5c4c..88e3db5 100644 --- a/src/channel/video_channel_send.cpp +++ b/src/channel/video_channel_send.cpp @@ -9,14 +9,15 @@ VideoChannelSend::~VideoChannelSend() {} VideoChannelSend::VideoChannelSend( std::shared_ptr clock, std::shared_ptr ice_agent, - std::shared_ptr ice_io_statistics) + std::shared_ptr ice_io_statistics, + std::function + on_sent_packet_func) : ice_agent_(ice_agent), ice_io_statistics_(ice_io_statistics), - clock_(clock) {}; + on_sent_packet_func_(on_sent_packet_func), + clock_(clock){}; void VideoChannelSend::Initialize(rtp::PAYLOAD_TYPE payload_type) { - controller_ = std::make_unique(); - rtp_video_sender_ = std::make_unique(ice_io_statistics_); rtp_packetizer_ = RtpPacketizer::Create(payload_type, rtp_video_sender_->GetSsrc()); @@ -43,23 +44,7 @@ void VideoChannelSend::Initialize(rtp::PAYLOAD_TYPE payload_type) { rtp_video_sender_->SetOnSentPacketFunc( [this](const webrtc::RtpPacketToSend& packet) -> void { - webrtc::PacedPacketInfo pacing_info; - size_t transport_overhead_bytes_per_packet_ = 0; - webrtc::Timestamp creation_time = - webrtc::Timestamp::Millis(clock_->TimeInMilliseconds()); - transport_feedback_adapter_.AddPacket( - packet, pacing_info, transport_overhead_bytes_per_packet_, - creation_time); - - rtc::SentPacket sent_packet; - sent_packet.packet_id = packet.transport_sequence_number().value(); - sent_packet.send_time_ms = clock_->TimeInMilliseconds(); - sent_packet.info.included_in_feedback = true; - sent_packet.info.included_in_allocation = true; - sent_packet.info.packet_size_bytes = packet.size(); - sent_packet.info.packet_type = rtc::PacketType::kData; - - transport_feedback_adapter_.ProcessSentPacket(sent_packet); + on_sent_packet_func_(packet); }); rtp_video_sender_->Start(); @@ -80,66 +65,3 @@ int VideoChannelSend::SendVideo(char* data, size_t size) { return 0; } - -void VideoChannelSend::OnCongestionControlFeedback( - Timestamp recv_ts, - const webrtc::rtcp::CongestionControlFeedback& feedback) { - ++feedback_count_; - std::optional feedback_msg = - transport_feedback_adapter_.ProcessCongestionControlFeedback(feedback, - recv_ts); - if (feedback_msg) { - HandleTransportPacketsFeedback(*feedback_msg); - } -} - -void VideoChannelSend::HandleTransportPacketsFeedback( - const webrtc::TransportPacketsFeedback& feedback) { - // if (transport_is_ecn_capable_) { - // // If transport does not support ECN, packets should not be sent as - // ECT(1). - // // TODO: bugs.webrtc.org/42225697 - adapt to ECN feedback and - // continue to - // // send packets as ECT(1) if transport is ECN capable. - // transport_is_ecn_capable_ = false; - // LOG_INFO("Transport is {} ECN capable. Stop sending ECT(1)", - // (feedback.transport_supports_ecn ? "" : " not ")); - // } - if (controller_) - PostUpdates(controller_->OnTransportPacketsFeedback(feedback)); - - // Only update outstanding data if any packet is first time acked. - UpdateCongestedState(); -} - -void VideoChannelSend::PostUpdates(webrtc::NetworkControlUpdate update) { - // if (update.congestion_window) { - // congestion_window_size_ = *update.congestion_window; - // UpdateCongestedState(); - // } - // if (update.pacer_config) { - // pacer_.SetPacingRates(update.pacer_config->data_rate(), - // update.pacer_config->pad_rate()); - // } - // if (!update.probe_cluster_configs.empty()) { - // pacer_.CreateProbeClusters(std::move(update.probe_cluster_configs)); - // } - // if (update.target_rate) { - // control_handler_->SetTargetRate(*update.target_rate); - // UpdateControlState(); - // } -} - -void VideoChannelSend::UpdateControlState() { - // std::optional update = - // control_handler_->GetUpdate(); if (!update) return; - // retransmission_rate_limiter_.SetMaxRate(update->target_rate.bps()); - // observer_->OnTargetTransferRate(*update); -} - -void VideoChannelSend::UpdateCongestedState() { - // if (auto update = GetCongestedStateUpdate()) { - // is_congested_ = update.value(); - // pacer_.SetCongested(update.value()); - // } -} \ No newline at end of file diff --git a/src/channel/video_channel_send.h b/src/channel/video_channel_send.h index 7215a47..c9ab41f 100644 --- a/src/channel/video_channel_send.h +++ b/src/channel/video_channel_send.h @@ -23,7 +23,9 @@ class VideoChannelSend { VideoChannelSend(); VideoChannelSend(std::shared_ptr clock, std::shared_ptr ice_agent, - std::shared_ptr ice_io_statistics); + std::shared_ptr ice_io_statistics, + std::function + on_sent_packet_func_); ~VideoChannelSend(); public: @@ -50,15 +52,11 @@ class VideoChannelSend { std::unique_ptr rtp_packetizer_ = nullptr; std::unique_ptr rtp_video_sender_ = nullptr; + std::function + on_sent_packet_func_ = nullptr; + private: std::shared_ptr clock_; - int64_t current_offset_ = std::numeric_limits::min(); - // Used by RFC 8888 congestion control feedback to track base time. - std::optional last_feedback_compact_ntp_time_; - int feedback_count_ = 0; - - webrtc::TransportFeedbackAdapter transport_feedback_adapter_; - std::unique_ptr controller_; }; #endif \ No newline at end of file diff --git a/src/media/video/encode/aom/aom_av1_encoder.cpp b/src/media/video/encode/aom/aom_av1_encoder.cpp index 42cea10..2a70634 100644 --- a/src/media/video/encode/aom/aom_av1_encoder.cpp +++ b/src/media/video/encode/aom/aom_av1_encoder.cpp @@ -359,6 +359,11 @@ int AomAv1Encoder::ForceIdr() { return 0; } +int AomAv1Encoder::SetTargetBitrate(int bitrate) { + target_bitrate_ = bitrate; + return 0; +} + int AomAv1Encoder::Release() { if (frame_for_encode_ != nullptr) { aom_img_free(frame_for_encode_); diff --git a/src/media/video/encode/aom/aom_av1_encoder.h b/src/media/video/encode/aom/aom_av1_encoder.h index 191d320..8514623 100644 --- a/src/media/video/encode/aom/aom_av1_encoder.h +++ b/src/media/video/encode/aom/aom_av1_encoder.h @@ -44,6 +44,8 @@ class AomAv1Encoder : public VideoEncoder { int ForceIdr(); + int SetTargetBitrate(int bitrate); + std::string GetEncoderName() { return "AomAV1"; } private: diff --git a/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp b/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp index 987e351..acef2f7 100644 --- a/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp +++ b/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp @@ -221,6 +221,25 @@ int NvidiaVideoEncoder::ForceIdr() { return 0; } +int NvidiaVideoEncoder::SetTargetBitrate(int bitrate) { + if (!encoder_) { + return -1; + } + + NV_ENC_RECONFIGURE_PARAMS reconfig_params; + reconfig_params.version = NV_ENC_RECONFIGURE_PARAMS_VER; + NV_ENC_INITIALIZE_PARAMS init_params; + NV_ENC_CONFIG encode_config = {NV_ENC_CONFIG_VER}; + init_params.encodeConfig = &encode_config; + encoder_->GetInitializeParams(&init_params); + init_params.frameRateDen = 1; + init_params.frameRateNum = init_params.frameRateDen * fps_; + init_params.encodeConfig->rcParams.averageBitRate = average_bitrate_; + init_params.encodeConfig->rcParams.maxBitRate = bitrate; + reconfig_params.reInitEncodeParams = init_params; + return encoder_->Reconfigure(&reconfig_params) ? 0 : -1; +} + int NvidiaVideoEncoder::ResetEncodeResolution(unsigned int width, unsigned int height) { if (!encoder_) { diff --git a/src/media/video/encode/nvcodec/nvidia_video_encoder.h b/src/media/video/encode/nvcodec/nvidia_video_encoder.h index 8827d01..dfca058 100644 --- a/src/media/video/encode/nvcodec/nvidia_video_encoder.h +++ b/src/media/video/encode/nvcodec/nvidia_video_encoder.h @@ -20,6 +20,8 @@ class NvidiaVideoEncoder : public VideoEncoder { int ForceIdr(); + int SetTargetBitrate(int bitrate); + std::string GetEncoderName() { return "NvidiaH264"; } private: @@ -50,6 +52,7 @@ class NvidiaVideoEncoder : public VideoEncoder { uint32_t key_frame_interval_ = 3000; uint32_t average_bitrate_ = 2000000; uint32_t max_bitrate_ = 10000000; + uint32_t fps_ = 30; int max_payload_size_ = 3000; NvEncoder* encoder_ = nullptr; CUcontext cuda_context_ = nullptr; diff --git a/src/media/video/encode/openh264/openh264_encoder.cpp b/src/media/video/encode/openh264/openh264_encoder.cpp index eb8f425..8101728 100644 --- a/src/media/video/encode/openh264/openh264_encoder.cpp +++ b/src/media/video/encode/openh264/openh264_encoder.cpp @@ -359,6 +359,13 @@ int OpenH264Encoder::ForceIdr() { return 0; } +int OpenH264Encoder::SetTargetBitrate(int bitrate) { + target_bitrate_ = bitrate; + encoder_params_.iTargetBitrate = target_bitrate_; + + return openh264_encoder_->SetOption(ENCODER_OPTION_BITRATE, &target_bitrate_); +} + int OpenH264Encoder::Release() { if (openh264_encoder_) { openh264_encoder_->Uninitialize(); diff --git a/src/media/video/encode/openh264/openh264_encoder.h b/src/media/video/encode/openh264/openh264_encoder.h index 9e61b73..663775e 100644 --- a/src/media/video/encode/openh264/openh264_encoder.h +++ b/src/media/video/encode/openh264/openh264_encoder.h @@ -31,6 +31,8 @@ class OpenH264Encoder : public VideoEncoder { int ForceIdr(); + int SetTargetBitrate(int bitrate); + std::string GetEncoderName() { return "OpenH264"; } private: diff --git a/src/media/video/encode/video_encoder.h b/src/media/video/encode/video_encoder.h index 6041b93..b51eb5e 100644 --- a/src/media/video/encode/video_encoder.h +++ b/src/media/video/encode/video_encoder.h @@ -27,6 +27,8 @@ class VideoEncoder { virtual int ForceIdr() = 0; + virtual int SetTargetBitrate(int bitrate) = 0; + virtual std::string GetEncoderName() = 0; VideoEncoder() = default; diff --git a/src/pc/peer_connection.cpp b/src/pc/peer_connection.cpp index 6b90649..35825db 100644 --- a/src/pc/peer_connection.cpp +++ b/src/pc/peer_connection.cpp @@ -602,11 +602,8 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) { hardware_acceleration_, trickle_ice_, reliable_ice_, enable_turn_, false, video_payload_types_, audio_payload_types_); - ice_transport_list_[remote_user_id]->SetOnReceiveVideoFunc( - on_receive_video_frame_); - ice_transport_list_[remote_user_id]->SetOnReceiveAudioFunc( - on_receive_audio_buffer_); - ice_transport_list_[remote_user_id]->SetOnReceiveDataFunc( + ice_transport_list_[remote_user_id]->SetOnReceiveFunc( + on_receive_video_frame_, on_receive_audio_buffer_, on_receive_data_buffer_); ice_transport_list_[remote_user_id]->SetOnReceiveNetStatusReportFunc( @@ -649,11 +646,8 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) { hardware_acceleration_, trickle_ice_, reliable_ice_, enable_turn_, false, video_payload_types_, audio_payload_types_); - ice_transport_list_[remote_user_id]->SetOnReceiveVideoFunc( - on_receive_video_frame_); - ice_transport_list_[remote_user_id]->SetOnReceiveAudioFunc( - on_receive_audio_buffer_); - ice_transport_list_[remote_user_id]->SetOnReceiveDataFunc( + ice_transport_list_[remote_user_id]->SetOnReceiveFunc( + on_receive_video_frame_, on_receive_audio_buffer_, on_receive_data_buffer_); ice_transport_list_[remote_user_id]->SetOnReceiveNetStatusReportFunc( diff --git a/src/qos/congestion_control.cpp b/src/qos/congestion_control.cpp index 772bbbb..7be530d 100644 --- a/src/qos/congestion_control.cpp +++ b/src/qos/congestion_control.cpp @@ -21,6 +21,32 @@ constexpr float kDefaultPaceMultiplier = 2.5f; // below the current throughput estimate to drain the network queues. constexpr double kProbeDropThroughputFraction = 0.85; +BandwidthLimitedCause GetBandwidthLimitedCause(LossBasedState loss_based_state, + bool is_rtt_above_limit, + BandwidthUsage bandwidth_usage) { + if (bandwidth_usage == BandwidthUsage::kBwOverusing || + bandwidth_usage == BandwidthUsage::kBwUnderusing) { + return BandwidthLimitedCause::kDelayBasedLimitedDelayIncreased; + } else if (is_rtt_above_limit) { + return BandwidthLimitedCause::kRttBasedBackOffHighRtt; + } + switch (loss_based_state) { + case LossBasedState::kDecreasing: + // Probes may not be sent in this state. + return BandwidthLimitedCause::kLossLimitedBwe; + case webrtc::LossBasedState::kIncreaseUsingPadding: + // Probes may not be sent in this state. + return BandwidthLimitedCause::kLossLimitedBwe; + case LossBasedState::kIncreasing: + // Probes may be sent in this state. + return BandwidthLimitedCause::kLossLimitedBweIncreasing; + case LossBasedState::kDelayBasedEstimate: + return BandwidthLimitedCause::kDelayBasedLimited; + default: + return BandwidthLimitedCause::kLossLimitedBwe; + } +} + CongestionControl::CongestionControl() : packet_feedback_only_(true), use_min_allocatable_as_lower_bound_(false), @@ -179,13 +205,13 @@ NetworkControlUpdate CongestionControl::OnTransportPacketsFeedback( bandwidth_estimation_->UpdateDelayBasedEstimate(report.feedback_time, result.target_bitrate); } - // bandwidth_estimation_->UpdateLossBasedEstimator( - // report, result.delay_detector_state, probe_bitrate, - // alr_start_time.has_value()); - // if (result.updated) { - // // Update the estimate in the ProbeController, in case we want to probe. - // MaybeTriggerOnNetworkChanged(&update, report.feedback_time); - // } + bandwidth_estimation_->UpdateLossBasedEstimator( + report, result.delay_detector_state, probe_bitrate, + alr_start_time.has_value()); + if (result.updated) { + // Update the estimate in the ProbeController, in case we want to probe. + MaybeTriggerOnNetworkChanged(&update, report.feedback_time); + } recovered_from_overuse = result.recovered_from_overuse; @@ -214,14 +240,13 @@ NetworkControlUpdate CongestionControl::OnTransportPacketsFeedback( void CongestionControl::MaybeTriggerOnNetworkChanged( NetworkControlUpdate* update, Timestamp at_time) { - // uint8_t fraction_loss = bandwidth_estimation_->fraction_loss(); - // TimeDelta round_trip_time = bandwidth_estimation_->round_trip_time(); - // DataRate loss_based_target_rate = bandwidth_estimation_->target_rate(); - // LossBasedState loss_based_state = - // bandwidth_estimation_->loss_based_state(); DataRate pushback_target_rate = - // loss_based_target_rate; + uint8_t fraction_loss = bandwidth_estimation_->fraction_loss(); + TimeDelta round_trip_time = bandwidth_estimation_->round_trip_time(); + DataRate loss_based_target_rate = bandwidth_estimation_->target_rate(); + LossBasedState loss_based_state = bandwidth_estimation_->loss_based_state(); + DataRate pushback_target_rate = loss_based_target_rate; - // double cwnd_reduce_ratio = 0.0; + double cwnd_reduce_ratio = 0.0; // if (congestion_window_pushback_controller_) { // int64_t pushback_rate = // congestion_window_pushback_controller_->UpdateTargetBitrate( @@ -235,53 +260,88 @@ void CongestionControl::MaybeTriggerOnNetworkChanged( // loss_based_target_rate.bps(); // } // } - // DataRate stable_target_rate = - // bandwidth_estimation_->GetEstimatedLinkCapacity(); - // stable_target_rate = std::min(stable_target_rate, pushback_target_rate); + DataRate stable_target_rate = + bandwidth_estimation_->GetEstimatedLinkCapacity(); + stable_target_rate = std::min(stable_target_rate, pushback_target_rate); - // if ((loss_based_target_rate != last_loss_based_target_rate_) || - // (loss_based_state != last_loss_base_state_) || - // (fraction_loss != last_estimated_fraction_loss_) || - // (round_trip_time != last_estimated_round_trip_time_) || - // (pushback_target_rate != last_pushback_target_rate_) || - // (stable_target_rate != last_stable_target_rate_)) { - // last_loss_based_target_rate_ = loss_based_target_rate; - // last_pushback_target_rate_ = pushback_target_rate; - // last_estimated_fraction_loss_ = fraction_loss; - // last_estimated_round_trip_time_ = round_trip_time; - // last_stable_target_rate_ = stable_target_rate; - // last_loss_base_state_ = loss_based_state; + if ((loss_based_target_rate != last_loss_based_target_rate_) || + (loss_based_state != last_loss_base_state_) || + (fraction_loss != last_estimated_fraction_loss_) || + (round_trip_time != last_estimated_round_trip_time_) || + (pushback_target_rate != last_pushback_target_rate_) || + (stable_target_rate != last_stable_target_rate_)) { + last_loss_based_target_rate_ = loss_based_target_rate; + last_pushback_target_rate_ = pushback_target_rate; + last_estimated_fraction_loss_ = fraction_loss; + last_estimated_round_trip_time_ = round_trip_time; + last_stable_target_rate_ = stable_target_rate; + last_loss_base_state_ = loss_based_state; - // alr_detector_->SetEstimatedBitrate(loss_based_target_rate.bps()); + alr_detector_->SetEstimatedBitrate(loss_based_target_rate.bps()); - // TimeDelta bwe_period = delay_based_bwe_->GetExpectedBwePeriod(); + TimeDelta bwe_period = delay_based_bwe_->GetExpectedBwePeriod(); - // TargetTransferRate target_rate_msg; - // target_rate_msg.at_time = at_time; - // if (rate_control_settings_.UseCongestionWindowDropFrameOnly()) { - // target_rate_msg.target_rate = loss_based_target_rate; - // target_rate_msg.cwnd_reduce_ratio = cwnd_reduce_ratio; - // } else { - // target_rate_msg.target_rate = pushback_target_rate; - // } - // target_rate_msg.stable_target_rate = stable_target_rate; - // target_rate_msg.network_estimate.at_time = at_time; - // target_rate_msg.network_estimate.round_trip_time = round_trip_time; - // target_rate_msg.network_estimate.loss_rate_ratio = fraction_loss / - // 255.0f; target_rate_msg.network_estimate.bwe_period = bwe_period; + TargetTransferRate target_rate_msg; + target_rate_msg.at_time = at_time; + // if (rate_control_settings_.UseCongestionWindowDropFrameOnly()) { + // target_rate_msg.target_rate = loss_based_target_rate; + // target_rate_msg.cwnd_reduce_ratio = cwnd_reduce_ratio; + // } else + { target_rate_msg.target_rate = pushback_target_rate; } + target_rate_msg.stable_target_rate = stable_target_rate; + target_rate_msg.network_estimate.at_time = at_time; + target_rate_msg.network_estimate.round_trip_time = round_trip_time; + target_rate_msg.network_estimate.loss_rate_ratio = fraction_loss / 255.0f; + target_rate_msg.network_estimate.bwe_period = bwe_period; - // update->target_rate = target_rate_msg; + update->target_rate = target_rate_msg; - // auto probes = probe_controller_->SetEstimatedBitrate( - // loss_based_target_rate, - // GetBandwidthLimitedCause(bandwidth_estimation_->loss_based_state(), - // bandwidth_estimation_->IsRttAboveLimit(), - // delay_based_bwe_->last_state()), - // at_time); - // update->probe_cluster_configs.insert(update->probe_cluster_configs.end(), - // probes.begin(), probes.end()); - // update->pacer_config = GetPacingRates(at_time); - // LOG_INFO("bwe {} pushback_target_bps={} estimate_bps={}", at_time.ms(), - // last_pushback_target_rate_.bps(), loss_based_target_rate.bps()); - // } -} \ No newline at end of file + auto probes = probe_controller_->SetEstimatedBitrate( + loss_based_target_rate, + GetBandwidthLimitedCause(bandwidth_estimation_->loss_based_state(), + bandwidth_estimation_->IsRttAboveLimit(), + delay_based_bwe_->last_state()), + at_time); + update->probe_cluster_configs.insert(update->probe_cluster_configs.end(), + probes.begin(), probes.end()); + update->pacer_config = GetPacingRates(at_time); + // LOG_INFO("bwe {} pushback_target_bps={} estimate_bps={}", at_time.ms(), + // last_pushback_target_rate_.bps(), loss_based_target_rate.bps()); + } +} + +PacerConfig CongestionControl::GetPacingRates(Timestamp at_time) const { + // Pacing rate is based on target rate before congestion window pushback, + // because we don't want to build queues in the pacer when pushback occurs. + DataRate pacing_rate = DataRate::Zero(); + if (pace_at_max_of_bwe_and_lower_link_capacity_ && estimate_ && + !bandwidth_estimation_->PaceAtLossBasedEstimate()) { + pacing_rate = + std::max({min_total_allocated_bitrate_, estimate_->link_capacity_lower, + last_loss_based_target_rate_}) * + pacing_factor_; + } else { + pacing_rate = + std::max(min_total_allocated_bitrate_, last_loss_based_target_rate_) * + pacing_factor_; + } + if (limit_pacingfactor_by_upper_link_capacity_estimate_ && estimate_ && + estimate_->link_capacity_upper.IsFinite() && + pacing_rate > estimate_->link_capacity_upper) { + pacing_rate = + std::max({estimate_->link_capacity_upper, min_total_allocated_bitrate_, + last_loss_based_target_rate_}); + } + + DataRate padding_rate = + (last_loss_base_state_ == LossBasedState::kIncreaseUsingPadding) + ? std::max(max_padding_rate_, last_loss_based_target_rate_) + : max_padding_rate_; + padding_rate = std::min(padding_rate, last_pushback_target_rate_); + PacerConfig msg; + msg.at_time = at_time; + msg.time_window = TimeDelta::Seconds(1); + msg.data_window = pacing_rate * msg.time_window; + msg.pad_window = padding_rate * msg.time_window; + return msg; +} diff --git a/src/qos/congestion_control.h b/src/qos/congestion_control.h index 7c783f8..bf58aa4 100644 --- a/src/qos/congestion_control.h +++ b/src/qos/congestion_control.h @@ -28,6 +28,9 @@ class CongestionControl { void MaybeTriggerOnNetworkChanged(NetworkControlUpdate* update, Timestamp at_time); + private: + PacerConfig GetPacingRates(Timestamp at_time) const; + private: const bool packet_feedback_only_; const bool use_min_allocatable_as_lower_bound_; @@ -54,12 +57,19 @@ class CongestionControl { bool first_packet_sent_ = false; + std::optional estimate_; + Timestamp next_loss_update_ = Timestamp::MinusInfinity(); int lost_packets_since_last_loss_update_ = 0; int expected_packets_since_last_loss_update_ = 0; std::deque feedback_max_rtts_; + DataRate last_loss_based_target_rate_; + DataRate last_pushback_target_rate_; + DataRate last_stable_target_rate_; + LossBasedState last_loss_base_state_; + std::optional last_estimated_fraction_loss_ = 0; TimeDelta last_estimated_round_trip_time_ = TimeDelta::PlusInfinity(); diff --git a/src/qos/delay_based_bwe.cc b/src/qos/delay_based_bwe.cc index 0c2146a..8730483 100644 --- a/src/qos/delay_based_bwe.cc +++ b/src/qos/delay_based_bwe.cc @@ -206,6 +206,7 @@ DelayBasedBwe::Result DelayBasedBwe::MaybeUpdateEstimate( } result.delay_detector_state = detector_state; + return result; } diff --git a/src/qos/send_side_bandwidth_estimation.cc b/src/qos/send_side_bandwidth_estimation.cc index 31f186f..9101ad2 100644 --- a/src/qos/send_side_bandwidth_estimation.cc +++ b/src/qos/send_side_bandwidth_estimation.cc @@ -234,6 +234,10 @@ DataRate SendSideBandwidthEstimation::target_rate() const { return std::max(min_bitrate_configured_, target); } +LossBasedState SendSideBandwidthEstimation::loss_based_state() const { + return loss_based_state_; +} + bool SendSideBandwidthEstimation::IsRttAboveLimit() const { return rtt_backoff_.IsRttAboveLimit(); } @@ -511,4 +515,10 @@ void SendSideBandwidthEstimation::ApplyTargetLimits(Timestamp at_time) { UpdateTargetBitrate(current_target_, at_time); } +bool SendSideBandwidthEstimation::PaceAtLossBasedEstimate() const { + // return LossBasedBandwidthEstimatorV2ReadyForUse() && + // loss_based_bandwidth_estimator_v2_->PaceAtLossBasedEstimate(); + return false; +} + } // namespace webrtc diff --git a/src/qos/send_side_bandwidth_estimation.h b/src/qos/send_side_bandwidth_estimation.h index a401ab4..ad35797 100644 --- a/src/qos/send_side_bandwidth_estimation.h +++ b/src/qos/send_side_bandwidth_estimation.h @@ -29,6 +29,16 @@ namespace webrtc { +enum class LossBasedState { + kIncreasing = 0, + // TODO(bugs.webrtc.org/12707): Remove one of the increasing states once we + // have decided if padding is usefull for ramping up when BWE is loss + // limited. + kIncreaseUsingPadding = 1, + kDecreasing = 2, + kDelayBasedEstimate = 3 +}; + class LinkCapacityTracker { public: LinkCapacityTracker() = default; @@ -79,6 +89,7 @@ class SendSideBandwidthEstimation { void OnRouteChange(); DataRate target_rate() const; + LossBasedState loss_based_state() const; // Return whether the current rtt is higher than the rtt limited configured in // RttBasedBackoff. bool IsRttAboveLimit() const; @@ -115,6 +126,7 @@ class SendSideBandwidthEstimation { BandwidthUsage delay_detector_state, std::optional probe_bitrate, bool in_alr); + bool PaceAtLossBasedEstimate() const; private: friend class GoogCcStatePrinter; @@ -184,6 +196,7 @@ class SendSideBandwidthEstimation { float high_loss_threshold_; DataRate bitrate_threshold_; bool disable_receiver_limit_caps_only_; + LossBasedState loss_based_state_; }; } // namespace webrtc #endif // MODULES_CONGESTION_CONTROLLER_GOOG_CC_SEND_SIDE_BANDWIDTH_ESTIMATION_H_ diff --git a/src/transport/ice_transport.cpp b/src/transport/ice_transport.cpp index d3e74d4..bf40d8d 100644 --- a/src/transport/ice_transport.cpp +++ b/src/transport/ice_transport.cpp @@ -7,10 +7,6 @@ #include "common.h" #include "log.h" -#if __APPLE__ -#else -#include "nvcodec_api.h" -#endif using nlohmann::json; @@ -25,22 +21,9 @@ IceTransport::IceTransport( remote_user_id_(remote_user_id), ice_ws_transport_(ice_ws_transmission), on_ice_status_change_(on_ice_status_change), - user_data_(user_data), - clock_(webrtc::Clock::GetRealTimeClockShared()) {} + user_data_(user_data) {} -IceTransport::~IceTransport() { - user_data_ = nullptr; - video_codec_inited_ = false; - audio_codec_inited_ = false; - load_nvcodec_dll_success_ = false; - -#ifdef __APPLE__ -#else - if (hardware_acceleration_ && load_nvcodec_dll_success_) { - ReleaseNvCodecDll(); - } -#endif -} +IceTransport::~IceTransport() {} int IceTransport::SetLocalCapabilities(bool hardware_acceleration, bool use_trickle_ice, @@ -63,12 +46,24 @@ int IceTransport::InitIceTransmission( std::string &stun_ip, int stun_port, std::string &turn_ip, int turn_port, std::string &turn_username, std::string &turn_password, rtp::PAYLOAD_TYPE video_codec_payload_type) { + ice_transport_controller_ = std::make_shared(); ice_agent_ = std::make_unique( offer_peer_, use_trickle_ice_, use_reliable_ice_, enable_turn_, force_turn_, stun_ip, stun_port, turn_ip, turn_port, turn_username, turn_password); - InitializeIOStatistics(); + ice_io_statistics_ = std::make_unique( + [this](const IOStatistics::NetTrafficStats &net_traffic_stats) { + if (on_receive_net_status_report_) { + XNetTrafficStats xnet_traffic_stats; + memcpy(&xnet_traffic_stats, &net_traffic_stats, + sizeof(XNetTrafficStats)); + on_receive_net_status_report_( + user_id_.data(), user_id_.size(), TraversalMode(traversal_type_), + &xnet_traffic_stats, remote_user_id_.data(), + remote_user_id_.size(), user_data_); + } + }); ice_agent_->CreateIceAgent( [](NiceAgent *agent, guint stream_id, guint component_id, @@ -100,66 +95,6 @@ int IceTransport::InitIceTransmission( return 0; } -void IceTransport::InitializeIOStatistics() { - ice_io_statistics_ = std::make_unique( - [this](const IOStatistics::NetTrafficStats &net_traffic_stats) { - if (on_receive_net_status_report_) { - XNetTrafficStats xnet_traffic_stats; - memcpy(&xnet_traffic_stats, &net_traffic_stats, - sizeof(XNetTrafficStats)); - on_receive_net_status_report_( - user_id_.data(), user_id_.size(), TraversalMode(traversal_type_), - &xnet_traffic_stats, remote_user_id_.data(), - remote_user_id_.size(), user_data_); - } - }); -} - -void IceTransport::InitializeChannels( - rtp::PAYLOAD_TYPE video_codec_payload_type) { - video_codec_payload_type_ = video_codec_payload_type; - - video_channel_send_ = std::make_unique(clock_, ice_agent_, - ice_io_statistics_); - audio_channel_send_ = - std::make_unique(ice_agent_, ice_io_statistics_); - data_channel_send_ = - std::make_unique(ice_agent_, ice_io_statistics_); - - video_channel_send_->Initialize(video_codec_payload_type_); - audio_channel_send_->Initialize(rtp::PAYLOAD_TYPE::OPUS); - data_channel_send_->Initialize(rtp::PAYLOAD_TYPE::DATA); - - std::weak_ptr weak_self = shared_from_this(); - video_channel_receive_ = std::make_unique( - clock_, ice_agent_, ice_io_statistics_, - [this, weak_self](VideoFrame &video_frame) { - if (auto self = weak_self.lock()) { - OnReceiveCompleteFrame(video_frame); - } - }); - - audio_channel_receive_ = std::make_unique( - ice_agent_, ice_io_statistics_, - [this, weak_self](const char *data, size_t size) { - if (auto self = weak_self.lock()) { - OnReceiveCompleteAudio(data, size); - } - }); - - data_channel_receive_ = std::make_unique( - ice_agent_, ice_io_statistics_, - [this, weak_self](const char *data, size_t size) { - if (auto self = weak_self.lock()) { - OnReceiveCompleteData(data, size); - } - }); - - video_channel_receive_->Initialize(video_codec_payload_type_); - audio_channel_receive_->Initialize(rtp::PAYLOAD_TYPE::OPUS); - data_channel_receive_->Initialize(rtp::PAYLOAD_TYPE::DATA); -} - void IceTransport::OnIceStateChange(NiceAgent *agent, guint stream_id, guint component_id, NiceComponentState state, @@ -252,12 +187,13 @@ void IceTransport::OnReceiveBuffer(NiceAgent *agent, guint stream_id, gchar *buffer, gpointer user_ptr) { if (!is_closed_) { if (CheckIsRtpPacket(buffer, size)) { - if (CheckIsVideoPacket(buffer, size)) { - video_channel_receive_->OnReceiveRtpPacket(buffer, size); - } else if (CheckIsAudioPacket(buffer, size)) { - audio_channel_receive_->OnReceiveRtpPacket(buffer, size); - } else if (CheckIsDataPacket(buffer, size)) { - data_channel_receive_->OnReceiveRtpPacket(buffer, size); + if (CheckIsVideoPacket(buffer, size) && ice_transport_controller_) { + ice_transport_controller_->OnReceiveVideoRtpPacket(buffer, size); + } else if (CheckIsAudioPacket(buffer, size) && + ice_transport_controller_) { + ice_transport_controller_->OnReceiveAudioRtpPacket(buffer, size); + } else if (CheckIsDataPacket(buffer, size) && ice_transport_controller_) { + ice_transport_controller_->OnReceiveDataRtpPacket(buffer, size); } } else if (CheckIsRtcpPacket(buffer, size)) { // LOG_ERROR("Rtcp packet [{}]", (uint8_t)(buffer[1])); @@ -372,42 +308,10 @@ bool IceTransport::HandleCongestionControlFeedback( // rtcp_packet_info->congestion_control_feedback.emplace(std::move(feedback)); // } - video_channel_send_->OnCongestionControlFeedback(clock_->CurrentTime(), - feedback); - return true; -} - -void IceTransport::OnReceiveCompleteFrame(VideoFrame &video_frame) { - int num_frame_returned = video_decoder_->Decode( - (uint8_t *)video_frame.Buffer(), video_frame.Size(), - [this](VideoFrame video_frame) { - if (on_receive_video_) { - XVideoFrame x_video_frame; - x_video_frame.data = (const char *)video_frame.Buffer(); - x_video_frame.width = video_frame.Width(); - x_video_frame.height = video_frame.Height(); - x_video_frame.size = video_frame.Size(); - on_receive_video_(&x_video_frame, remote_user_id_.data(), - remote_user_id_.size(), user_data_); - } - }); -} - -void IceTransport::OnReceiveCompleteAudio(const char *data, size_t size) { - int num_frame_returned = audio_decoder_->Decode( - (uint8_t *)data, size, [this](uint8_t *data, int size) { - if (on_receive_audio_) { - on_receive_audio_((const char *)data, size, remote_user_id_.data(), - remote_user_id_.size(), user_data_); - } - }); -} - -void IceTransport::OnReceiveCompleteData(const char *data, size_t size) { - if (on_receive_data_) { - on_receive_data_(data, size, remote_user_id_.data(), remote_user_id_.size(), - user_data_); + if (ice_transport_controller_) { + ice_transport_controller_->OnCongestionControlFeedback(feedback); } + return true; } int IceTransport::DestroyIceTransmission() { @@ -422,130 +326,9 @@ int IceTransport::DestroyIceTransmission() { ice_io_statistics_->Stop(); } - if (video_channel_send_) { - video_channel_send_->Destroy(); - } - - if (audio_channel_send_) { - audio_channel_send_->Destroy(); - } - - if (data_channel_send_) { - data_channel_send_->Destroy(); - } - - if (video_channel_receive_) { - video_channel_receive_->Destroy(); - } - - if (audio_channel_receive_) { - audio_channel_receive_->Destroy(); - } - - if (data_channel_receive_) { - data_channel_receive_->Destroy(); - } - return ice_agent_->DestroyIceAgent(); } -int IceTransport::CreateVideoCodec(rtp::PAYLOAD_TYPE video_pt, - bool hardware_acceleration) { - if (video_codec_inited_) { - return 0; - } - - hardware_acceleration_ = hardware_acceleration; - - if (rtp::PAYLOAD_TYPE::AV1 == video_pt) { - if (hardware_acceleration_) { - hardware_acceleration_ = false; - LOG_WARN("Only support software codec for AV1"); - } - video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, true); - video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, true); - } else if (rtp::PAYLOAD_TYPE::H264 == video_pt) { -#ifdef __APPLE__ - if (hardware_acceleration_) { - hardware_acceleration_ = false; - LOG_WARN( - "MacOS not support hardware acceleration, use default software " - "codec"); - video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); - video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); - } else { - video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); - video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); - } -#else - if (hardware_acceleration_) { - if (0 == LoadNvCodecDll()) { - load_nvcodec_dll_success_ = true; - video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(true, false); - video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(true, false); - } else { - LOG_WARN( - "Hardware accelerated codec not available, use default software " - "codec"); - video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); - video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); - } - } else { - video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); - video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); - } -#endif - } - - if (!video_encoder_) { - video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); - LOG_ERROR("Create encoder failed, try to use software H.264 encoder"); - } - if (!video_encoder_ || 0 != video_encoder_->Init()) { - LOG_ERROR("Encoder init failed"); - return -1; - } - - if (!video_decoder_) { - video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); - LOG_ERROR("Create decoder failed, try to use software H.264 decoder"); - } - if (!video_decoder_ || video_decoder_->Init()) { - LOG_ERROR("Decoder init failed"); - return -1; - } - - video_codec_inited_ = true; - LOG_INFO("Create video codec [{}|{}] finish", - video_encoder_->GetEncoderName(), video_decoder_->GetDecoderName()); - - return 0; -} - -int IceTransport::CreateAudioCodec() { - if (audio_codec_inited_) { - return 0; - } - - audio_encoder_ = std::make_unique(AudioEncoder(48000, 1, 480)); - if (!audio_encoder_ || 0 != audio_encoder_->Init()) { - LOG_ERROR("Audio encoder init failed"); - return -1; - } - - audio_decoder_ = std::make_unique(AudioDecoder(48000, 1, 480)); - if (!audio_decoder_ || 0 != audio_decoder_->Init()) { - LOG_ERROR("Audio decoder init failed"); - return -1; - } - - audio_codec_inited_ = true; - LOG_INFO("Create audio codec [{}|{}] finish", - audio_encoder_->GetEncoderName(), audio_decoder_->GetDecoderName()); - - return 0; -} - int IceTransport::SetTransmissionId(const std::string &transmission_id) { transmission_id_ = transmission_id; @@ -742,10 +525,12 @@ std::string IceTransport::GetRemoteCapabilities(const std::string &remote_sdp) { return std::string(); } - InitializeChannels(negotiated_video_pt_); - - CreateVideoCodec(negotiated_video_pt_, hardware_acceleration_); - CreateAudioCodec(); + if (ice_transport_controller_) { + ice_transport_controller_->Create( + remote_user_id_, negotiated_video_pt_, hardware_acceleration_, + ice_agent_, ice_io_statistics_, on_receive_video_, on_receive_audio_, + on_receive_data_, user_data_); + } remote_capabilities_got_ = true; } @@ -978,29 +763,11 @@ int IceTransport::SendVideoFrame(const XVideoFrame *video_frame) { return -2; } - if (b_force_i_frame_) { - video_encoder_->ForceIdr(); - LOG_INFO("Force I frame"); - b_force_i_frame_ = false; + if (ice_transport_controller_) { + return ice_transport_controller_->SendVideo(video_frame); } - int ret = video_encoder_->Encode( - video_frame, - [this](char *encoded_frame, size_t size, - VideoEncoder::VideoFrameType frame_type) -> int { - if (video_channel_send_) { - video_channel_send_->SendVideo(encoded_frame, size); - } - - return 0; - }); - - if (0 != ret) { - LOG_ERROR("Encode failed"); - return -1; - } - - return 0; + return -1; } int IceTransport::SendAudioFrame(const char *data, size_t size) { @@ -1011,17 +778,11 @@ int IceTransport::SendAudioFrame(const char *data, size_t size) { return -2; } - int ret = audio_encoder_->Encode( - (uint8_t *)data, size, - [this](char *encoded_audio_buffer, size_t size) -> int { - if (audio_channel_send_) { - audio_channel_send_->SendAudio(encoded_audio_buffer, size); - } + if (ice_transport_controller_) { + return ice_transport_controller_->SendAudio(data, size); + } - return 0; - }); - - return ret; + return -1; } int IceTransport::SendDataFrame(const char *data, size_t size) { @@ -1032,11 +793,11 @@ int IceTransport::SendDataFrame(const char *data, size_t size) { return -2; } - if (data_channel_send_) { - data_channel_send_->SendData(data, size); + if (ice_transport_controller_) { + return ice_transport_controller_->SendData(data, size); } - return 0; + return -1; } uint8_t IceTransport::CheckIsRtpPacket(const char *buffer, size_t size) { diff --git a/src/transport/ice_transport.h b/src/transport/ice_transport.h index 30f7c6f..a8f8ee3 100644 --- a/src/transport/ice_transport.h +++ b/src/transport/ice_transport.h @@ -9,32 +9,15 @@ #include -#include "audio_decoder.h" -#include "audio_encoder.h" -// #include "congestion_control.h" -#include "audio_channel_receive.h" -#include "audio_channel_send.h" -#include "clock.h" -#include "data_channel_receive.h" -#include "data_channel_send.h" #include "ice_agent.h" +#include "ice_transport_controller.h" #include "io_statistics.h" #include "ringbuffer.h" #include "rtcp_packet_info.h" -#include "rtp_audio_receiver.h" -#include "rtp_audio_sender.h" -#include "rtp_data_receiver.h" -#include "rtp_data_sender.h" #include "rtp_packet.h" -#include "rtp_video_receiver.h" -#include "rtp_video_sender.h" -#include "video_channel_receive.h" -#include "video_channel_send.h" -#include "video_decoder_factory.h" -#include "video_encoder_factory.h" #include "ws_client.h" -class IceTransport : public std::enable_shared_from_this { +class IceTransport { public: typedef enum { VIDEO = 96, AUDIO = 97, DATA = 127 } DATA_TYPE; typedef enum { H264 = 96, AV1 = 99 } VIDEO_TYPE; @@ -70,24 +53,11 @@ class IceTransport : public std::enable_shared_from_this { int DestroyIceTransmission(); - void SetOnReceiveVideoFunc( - std::function - on_receive_video) { + void SetOnReceiveFunc(OnReceiveVideo on_receive_video, + OnReceiveAudio on_receive_audio, + OnReceiveData on_receive_data) { on_receive_video_ = on_receive_video; - } - - void SetOnReceiveAudioFunc( - std::function - on_receive_audio) { on_receive_audio_ = on_receive_audio; - } - - void SetOnReceiveDataFunc( - std::function - on_receive_data) { on_receive_data_ = on_receive_data; } @@ -131,9 +101,6 @@ class IceTransport : public std::enable_shared_from_this { bool NegotiateAudioPayloadType(const std::string &remote_sdp); bool NegotiateDataPayloadType(const std::string &remote_sdp); - int CreateVideoCodec(rtp::PAYLOAD_TYPE video_pt, bool hardware_acceleration); - int CreateAudioCodec(); - private: uint8_t CheckIsRtpPacket(const char *buffer, size_t size); uint8_t CheckIsRtcpPacket(const char *buffer, size_t size); @@ -142,10 +109,6 @@ class IceTransport : public std::enable_shared_from_this { uint8_t CheckIsDataPacket(const char *buffer, size_t size); private: - void InitializeIOStatistics(); - - void InitializeChannels(rtp::PAYLOAD_TYPE video_codec_payload_type); - void OnIceStateChange(NiceAgent *agent, guint stream_id, guint component_id, NiceComponentState state, gpointer user_ptr); @@ -162,12 +125,6 @@ class IceTransport : public std::enable_shared_from_this { void OnReceiveBuffer(NiceAgent *agent, guint stream_id, guint component_id, guint size, gchar *buffer, gpointer user_ptr); - void OnReceiveCompleteFrame(VideoFrame &video_frame); - - void OnReceiveCompleteAudio(const char *data, size_t size); - - void OnReceiveCompleteData(const char *data, size_t size); - bool ParseRtcpPacket(const uint8_t *buffer, size_t size, RtcpPacketInfo *rtcp_packet_info); @@ -176,6 +133,7 @@ class IceTransport : public std::enable_shared_from_this { RtcpPacketInfo *rtcp_packet_info); private: + bool hardware_acceleration_ = false; bool use_trickle_ice_ = true; bool enable_turn_ = false; bool use_reliable_ice_ = false; @@ -203,13 +161,10 @@ class IceTransport : public std::enable_shared_from_this { std::shared_ptr ice_agent_ = nullptr; bool is_closed_ = false; std::shared_ptr ice_ws_transport_ = nullptr; - // CongestionControl *congestion_control_ = nullptr; - std::function - on_receive_video_ = nullptr; - std::function - on_receive_audio_ = nullptr; - std::function - on_receive_data_ = nullptr; + + OnReceiveVideo on_receive_video_ = nullptr; + OnReceiveAudio on_receive_audio_ = nullptr; + OnReceiveData on_receive_data_ = nullptr; std::function on_ice_status_change_ = nullptr; @@ -220,23 +175,7 @@ class IceTransport : public std::enable_shared_from_this { on_receive_net_status_report_ = nullptr; private: - std::shared_ptr clock_; - std::unique_ptr video_channel_send_ = nullptr; - std::unique_ptr video_channel_receive_ = nullptr; - std::unique_ptr audio_channel_send_ = nullptr; - std::unique_ptr audio_channel_receive_ = nullptr; - std::unique_ptr data_channel_send_ = nullptr; - std::unique_ptr data_channel_receive_ = nullptr; - - std::unique_ptr rtp_video_receiver_ = nullptr; - std::unique_ptr rtp_video_sender_ = nullptr; - std::unique_ptr rtp_audio_receiver_ = nullptr; - std::unique_ptr rtp_audio_sender_ = nullptr; - std::unique_ptr rtp_data_receiver_ = nullptr; - std::unique_ptr rtp_data_sender_ = nullptr; - bool start_send_packet_ = false; - - uint32_t last_complete_frame_ts_ = 0; + std::shared_ptr ice_transport_controller_ = nullptr; private: std::shared_ptr ice_io_statistics_ = nullptr; @@ -250,19 +189,6 @@ class IceTransport : public std::enable_shared_from_this { rtp::PAYLOAD_TYPE negotiated_video_pt_ = rtp::PAYLOAD_TYPE::UNDEFINED; rtp::PAYLOAD_TYPE negotiated_audio_pt_ = rtp::PAYLOAD_TYPE::UNDEFINED; rtp::PAYLOAD_TYPE negotiated_data_pt_ = rtp::PAYLOAD_TYPE::UNDEFINED; - - private: - std::unique_ptr video_encoder_ = nullptr; - std::unique_ptr video_decoder_ = nullptr; - bool b_force_i_frame_ = false; - bool video_codec_inited_ = false; - bool load_nvcodec_dll_success_ = false; - bool hardware_acceleration_ = false; - - private: - std::unique_ptr audio_encoder_ = nullptr; - std::unique_ptr audio_decoder_ = nullptr; - bool audio_codec_inited_ = false; }; #endif \ No newline at end of file diff --git a/src/transport/ice_transport_controller.cpp b/src/transport/ice_transport_controller.cpp new file mode 100644 index 0000000..eaa89a9 --- /dev/null +++ b/src/transport/ice_transport_controller.cpp @@ -0,0 +1,389 @@ +#include "ice_transport_controller.h" +#if __APPLE__ +#else +#include "nvcodec_api.h" +#endif + +IceTransportController::IceTransportController() + : b_force_i_frame_(true), + video_codec_inited_(false), + audio_codec_inited_(false), + load_nvcodec_dll_success_(false), + hardware_acceleration_(false), + clock_(webrtc::Clock::GetRealTimeClockShared()) {} + +IceTransportController::~IceTransportController() { + user_data_ = nullptr; + video_codec_inited_ = false; + audio_codec_inited_ = false; + load_nvcodec_dll_success_ = false; + +#ifdef __APPLE__ +#else + if (hardware_acceleration_ && load_nvcodec_dll_success_) { + ReleaseNvCodecDll(); + } +#endif +} + +void IceTransportController::Create( + std::string remote_user_id, rtp::PAYLOAD_TYPE video_codec_payload_type, + bool hardware_acceleration, std::shared_ptr ice_agent, + std::shared_ptr ice_io_statistics, + OnReceiveVideo on_receive_video, OnReceiveAudio on_receive_audio, + OnReceiveData on_receive_data, void* user_data) { + remote_user_id_ = remote_user_id; + on_receive_video_ = on_receive_video; + on_receive_audio_ = on_receive_audio; + on_receive_data_ = on_receive_data; + user_data_ = user_data; + + CreateVideoCodec(video_codec_payload_type, hardware_acceleration); + CreateAudioCodec(); + + controller_ = std::make_unique(); + + video_channel_send_ = std::make_unique( + clock_, ice_agent, ice_io_statistics, + [this](const webrtc::RtpPacketToSend& packet) { + OnSentRtpPacket(packet); + }); + audio_channel_send_ = + std::make_unique(ice_agent, ice_io_statistics); + data_channel_send_ = + std::make_unique(ice_agent, ice_io_statistics); + + video_channel_send_->Initialize(video_codec_payload_type); + audio_channel_send_->Initialize(rtp::PAYLOAD_TYPE::OPUS); + data_channel_send_->Initialize(rtp::PAYLOAD_TYPE::DATA); + + std::weak_ptr weak_self = shared_from_this(); + video_channel_receive_ = std::make_unique( + clock_, ice_agent, ice_io_statistics, + [this, weak_self](VideoFrame& video_frame) { + if (auto self = weak_self.lock()) { + OnReceiveCompleteFrame(video_frame); + } + }); + + audio_channel_receive_ = std::make_unique( + ice_agent, ice_io_statistics, + [this, weak_self](const char* data, size_t size) { + if (auto self = weak_self.lock()) { + OnReceiveCompleteAudio(data, size); + } + }); + + data_channel_receive_ = std::make_unique( + ice_agent, ice_io_statistics, + [this, weak_self](const char* data, size_t size) { + if (auto self = weak_self.lock()) { + OnReceiveCompleteData(data, size); + } + }); + + video_channel_receive_->Initialize(video_codec_payload_type); + audio_channel_receive_->Initialize(rtp::PAYLOAD_TYPE::OPUS); + data_channel_receive_->Initialize(rtp::PAYLOAD_TYPE::DATA); +} + +void IceTransportController::Destroy() { + if (video_channel_send_) { + video_channel_send_->Destroy(); + } + + if (audio_channel_send_) { + audio_channel_send_->Destroy(); + } + + if (data_channel_send_) { + data_channel_send_->Destroy(); + } + + if (video_channel_receive_) { + video_channel_receive_->Destroy(); + } + + if (audio_channel_receive_) { + audio_channel_receive_->Destroy(); + } + + if (data_channel_receive_) { + data_channel_receive_->Destroy(); + } +} + +int IceTransportController::SendVideo(const XVideoFrame* video_frame) { + if (!video_encoder_) { + LOG_ERROR("Video Encoder not created"); + return -1; + } + + if (b_force_i_frame_) { + video_encoder_->ForceIdr(); + LOG_INFO("Force I frame"); + b_force_i_frame_ = false; + } + + int ret = video_encoder_->Encode( + video_frame, + [this](char* encoded_frame, size_t size, + VideoEncoder::VideoFrameType frame_type) -> int { + if (video_channel_send_) { + video_channel_send_->SendVideo(encoded_frame, size); + } + + return 0; + }); + + if (0 != ret) { + LOG_ERROR("Encode failed"); + return -1; + } else { + return 0; + } +} + +int IceTransportController::SendAudio(const char* data, size_t size) { + if (!audio_encoder_) { + LOG_ERROR("Audio Encoder not created"); + return -1; + } + + int ret = audio_encoder_->Encode( + (uint8_t*)data, size, + [this](char* encoded_audio_buffer, size_t size) -> int { + if (audio_channel_send_) { + audio_channel_send_->SendAudio(encoded_audio_buffer, size); + } + + return 0; + }); + + return ret; +} + +int IceTransportController::SendData(const char* data, size_t size) { + if (data_channel_send_) { + data_channel_send_->SendData(data, size); + } + + return 0; +} + +int IceTransportController::OnReceiveVideoRtpPacket(const char* data, + size_t size) { + if (video_channel_receive_) { + return video_channel_receive_->OnReceiveRtpPacket(data, size); + } + + return -1; +} + +int IceTransportController::OnReceiveAudioRtpPacket(const char* data, + size_t size) { + if (audio_channel_receive_) { + return audio_channel_receive_->OnReceiveRtpPacket(data, size); + } + + return -1; +} + +int IceTransportController::OnReceiveDataRtpPacket(const char* data, + size_t size) { + if (data_channel_receive_) { + return data_channel_receive_->OnReceiveRtpPacket(data, size); + } + + return -1; +} + +void IceTransportController::OnReceiveCompleteFrame(VideoFrame& video_frame) { + int num_frame_returned = video_decoder_->Decode( + (uint8_t*)video_frame.Buffer(), video_frame.Size(), + [this](VideoFrame video_frame) { + if (on_receive_video_) { + XVideoFrame x_video_frame; + x_video_frame.data = (const char*)video_frame.Buffer(); + x_video_frame.width = video_frame.Width(); + x_video_frame.height = video_frame.Height(); + x_video_frame.size = video_frame.Size(); + on_receive_video_(&x_video_frame, remote_user_id_.data(), + remote_user_id_.size(), user_data_); + } + }); +} + +void IceTransportController::OnReceiveCompleteAudio(const char* data, + size_t size) { + int num_frame_returned = audio_decoder_->Decode( + (uint8_t*)data, size, [this](uint8_t* data, int size) { + if (on_receive_audio_) { + on_receive_audio_((const char*)data, size, remote_user_id_.data(), + remote_user_id_.size(), user_data_); + } + }); +} + +void IceTransportController::OnReceiveCompleteData(const char* data, + size_t size) { + if (on_receive_data_) { + on_receive_data_(data, size, remote_user_id_.data(), remote_user_id_.size(), + user_data_); + } +} + +int IceTransportController::CreateVideoCodec(rtp::PAYLOAD_TYPE video_pt, + bool hardware_acceleration) { + if (video_codec_inited_) { + return 0; + } + + hardware_acceleration_ = hardware_acceleration; + + if (rtp::PAYLOAD_TYPE::AV1 == video_pt) { + if (hardware_acceleration_) { + hardware_acceleration_ = false; + LOG_WARN("Only support software codec for AV1"); + } + video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, true); + video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, true); + } else if (rtp::PAYLOAD_TYPE::H264 == video_pt) { +#ifdef __APPLE__ + if (hardware_acceleration_) { + hardware_acceleration_ = false; + LOG_WARN( + "MacOS not support hardware acceleration, use default software " + "codec"); + video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); + video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); + } else { + video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); + video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); + } +#else + if (hardware_acceleration_) { + if (0 == LoadNvCodecDll()) { + load_nvcodec_dll_success_ = true; + video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(true, false); + video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(true, false); + } else { + LOG_WARN( + "Hardware accelerated codec not available, use default software " + "codec"); + video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); + video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); + } + } else { + video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); + video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); + } +#endif + } + + if (!video_encoder_) { + video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); + LOG_ERROR("Create encoder failed, try to use software H.264 encoder"); + } + if (!video_encoder_ || 0 != video_encoder_->Init()) { + LOG_ERROR("Encoder init failed"); + return -1; + } + + if (!video_decoder_) { + video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); + LOG_ERROR("Create decoder failed, try to use software H.264 decoder"); + } + if (!video_decoder_ || video_decoder_->Init()) { + LOG_ERROR("Decoder init failed"); + return -1; + } + + video_codec_inited_ = true; + LOG_INFO("Create video codec [{}|{}] finish", + video_encoder_->GetEncoderName(), video_decoder_->GetDecoderName()); + + return 0; +} + +int IceTransportController::CreateAudioCodec() { + if (audio_codec_inited_) { + return 0; + } + + audio_encoder_ = std::make_unique(AudioEncoder(48000, 1, 480)); + if (!audio_encoder_ || 0 != audio_encoder_->Init()) { + LOG_ERROR("Audio encoder init failed"); + return -1; + } + + audio_decoder_ = std::make_unique(AudioDecoder(48000, 1, 480)); + if (!audio_decoder_ || 0 != audio_decoder_->Init()) { + LOG_ERROR("Audio decoder init failed"); + return -1; + } + + audio_codec_inited_ = true; + LOG_INFO("Create audio codec [{}|{}] finish", + audio_encoder_->GetEncoderName(), audio_decoder_->GetDecoderName()); + + return 0; +} + +void IceTransportController::OnCongestionControlFeedback( + const webrtc::rtcp::CongestionControlFeedback& feedback) { + std::optional feedback_msg = + transport_feedback_adapter_.ProcessCongestionControlFeedback( + feedback, clock_->CurrentTime()); + if (feedback_msg) { + HandleTransportPacketsFeedback(*feedback_msg); + } +} + +void IceTransportController::HandleTransportPacketsFeedback( + const webrtc::TransportPacketsFeedback& feedback) { + if (controller_) + PostUpdates(controller_->OnTransportPacketsFeedback(feedback)); + + UpdateCongestedState(); +} + +void IceTransportController::OnSentRtpPacket( + const webrtc::RtpPacketToSend& packet) { + webrtc::PacedPacketInfo pacing_info; + size_t transport_overhead_bytes_per_packet_ = 0; + webrtc::Timestamp creation_time = + webrtc::Timestamp::Millis(clock_->TimeInMilliseconds()); + transport_feedback_adapter_.AddPacket( + packet, pacing_info, transport_overhead_bytes_per_packet_, creation_time); + + rtc::SentPacket sent_packet; + sent_packet.packet_id = packet.transport_sequence_number().value(); + sent_packet.send_time_ms = clock_->TimeInMilliseconds(); + sent_packet.info.included_in_feedback = true; + sent_packet.info.included_in_allocation = true; + sent_packet.info.packet_size_bytes = packet.size(); + sent_packet.info.packet_type = rtc::PacketType::kData; + + transport_feedback_adapter_.ProcessSentPacket(sent_packet); +} + +void IceTransportController::PostUpdates(webrtc::NetworkControlUpdate update) { + // UpdateControlState(); + + target_bitrate_ = update.target_rate.has_value() + ? update.target_rate->target_rate.bps() + : 0; + // LOG_WARN("Target bitrate [{}]bps", target_bitrate_); + video_encoder_->SetTargetBitrate(target_bitrate_); +} + +void IceTransportController::UpdateControlState() { + if (controller_) { + } +} + +void IceTransportController::UpdateCongestedState() { + if (controller_) { + } +} diff --git a/src/transport/ice_transport_controller.h b/src/transport/ice_transport_controller.h new file mode 100644 index 0000000..3350fd1 --- /dev/null +++ b/src/transport/ice_transport_controller.h @@ -0,0 +1,121 @@ +/* + * @Author: DI JUNKUN + * @Date: 2025-02-11 + * Copyright (c) 2025 by DI JUNKUN, All Rights Reserved. + */ + +#ifndef _ICE_TRANSPORT_CONTROLLER_H_ +#define _ICE_TRANSPORT_CONTROLLER_H_ + +#include "api/transport/network_types.h" +#include "api/units/timestamp.h" +#include "audio_channel_receive.h" +#include "audio_channel_send.h" +#include "audio_decoder.h" +#include "audio_encoder.h" +#include "clock.h" +#include "congestion_control.h" +#include "congestion_control_feedback.h" +#include "data_channel_receive.h" +#include "data_channel_send.h" +#include "ice_agent.h" +#include "transport_feedback_adapter.h" +#include "video_channel_receive.h" +#include "video_channel_send.h" +#include "video_decoder_factory.h" +#include "video_encoder_factory.h" + +typedef void (*OnReceiveVideo)(const XVideoFrame *, const char *, const size_t, + void *); +typedef void (*OnReceiveAudio)(const char *, size_t, const char *, const size_t, + void *); +typedef void (*OnReceiveData)(const char *, size_t, const char *, const size_t, + void *); + +class IceTransportController + : public std::enable_shared_from_this { + public: + IceTransportController(); + ~IceTransportController(); + + public: + void Create(std::string remote_user_id, + rtp::PAYLOAD_TYPE video_codec_payload_type, + bool hardware_acceleration, std::shared_ptr ice_agent, + std::shared_ptr ice_io_statistics, + OnReceiveVideo on_receive_video, OnReceiveAudio on_receive_audio, + OnReceiveData on_receive_data, void *user_data); + void Destroy(); + + int SendVideo(const XVideoFrame *video_frame); + int SendAudio(const char *data, size_t size); + int SendData(const char *data, size_t size); + + int OnReceiveVideoRtpPacket(const char *data, size_t size); + int OnReceiveAudioRtpPacket(const char *data, size_t size); + int OnReceiveDataRtpPacket(const char *data, size_t size); + + void OnReceiveCompleteFrame(VideoFrame &video_frame); + void OnReceiveCompleteAudio(const char *data, size_t size); + void OnReceiveCompleteData(const char *data, size_t size); + + public: + void OnCongestionControlFeedback( + const webrtc::rtcp::CongestionControlFeedback &feedback); + + private: + int CreateVideoCodec(rtp::PAYLOAD_TYPE video_pt, bool hardware_acceleration); + int CreateAudioCodec(); + + private: + void OnSentRtpPacket(const webrtc::RtpPacketToSend &packet); + void HandleTransportPacketsFeedback( + const webrtc::TransportPacketsFeedback &feedback); + void PostUpdates(webrtc::NetworkControlUpdate update); + void UpdateControlState(); + void UpdateCongestedState(); + + private: + std::unique_ptr video_channel_send_ = nullptr; + std::unique_ptr audio_channel_send_ = nullptr; + std::unique_ptr data_channel_send_ = nullptr; + + std::unique_ptr video_channel_receive_ = nullptr; + std::unique_ptr audio_channel_receive_ = nullptr; + std::unique_ptr data_channel_receive_ = nullptr; + + OnReceiveVideo on_receive_video_ = nullptr; + OnReceiveAudio on_receive_audio_ = nullptr; + OnReceiveData on_receive_data_ = nullptr; + + private: + std::shared_ptr ice_agent_ = nullptr; + std::shared_ptr ice_io_statistics_ = nullptr; + std::unique_ptr rtp_packetizer_ = nullptr; + std::unique_ptr rtp_video_sender_ = nullptr; + std::string remote_user_id_; + void *user_data_ = nullptr; + + private: + std::shared_ptr clock_; + webrtc::TransportFeedbackAdapter transport_feedback_adapter_; + std::unique_ptr controller_; + + private: + std::unique_ptr video_encoder_ = nullptr; + std::unique_ptr video_decoder_ = nullptr; + bool b_force_i_frame_; + bool video_codec_inited_; + bool load_nvcodec_dll_success_; + bool hardware_acceleration_; + + private: + std::unique_ptr audio_encoder_ = nullptr; + std::unique_ptr audio_decoder_ = nullptr; + bool audio_codec_inited_ = false; + + private: + uint64_t target_bitrate_ = 0; +}; + +#endif \ No newline at end of file