From 092e8946227781877a0120ba63933229effef9dc Mon Sep 17 00:00:00 2001 From: dijunkun Date: Tue, 11 Mar 2025 17:36:07 +0800 Subject: [PATCH] [feat] bandwidth probing supported --- src/channel/rtp_channel/rtp_video_receiver.h | 3 +- src/common/api/transport/network_control.h | 124 +++++++++++++++++++ src/qos/congestion_control.cpp | 102 ++++++++++++++- src/qos/congestion_control.h | 8 ++ src/transport/ice_transport.cpp | 2 + src/transport/ice_transport_controller.cpp | 69 +++++++---- src/transport/ice_transport_controller.h | 7 +- 7 files changed, 281 insertions(+), 34 deletions(-) create mode 100644 src/common/api/transport/network_control.h diff --git a/src/channel/rtp_channel/rtp_video_receiver.h b/src/channel/rtp_channel/rtp_video_receiver.h index 213b5dd..1d04e8b 100644 --- a/src/channel/rtp_channel/rtp_video_receiver.h +++ b/src/channel/rtp_channel/rtp_video_receiver.h @@ -70,7 +70,8 @@ class RtpVideoReceiver : public ThreadBase, void RtcpThread(); private: - void SendNack(const std::vector& nack_list, bool buffering_allowed); + void SendNack(const std::vector& nack_list, + bool buffering_allowed) override; void SendRR(); diff --git a/src/common/api/transport/network_control.h b/src/common/api/transport/network_control.h new file mode 100644 index 0000000..06a0329 --- /dev/null +++ b/src/common/api/transport/network_control.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TRANSPORT_NETWORK_CONTROL_H_ +#define API_TRANSPORT_NETWORK_CONTROL_H_ + +#include +#include + +#include "api/transport/network_types.h" +#include "api/units/data_rate.h" +#include "api/units/time_delta.h" + +namespace webrtc { + +class TargetTransferRateObserver { + public: + virtual ~TargetTransferRateObserver() = default; + // Called to indicate target transfer rate as well as giving information about + // the current estimate of network parameters. + virtual void OnTargetTransferRate(TargetTransferRate) = 0; + // Called to provide updates to the expected target rate in case it changes + // before the first call to OnTargetTransferRate. + virtual void OnStartRateUpdate(DataRate) {} +}; + +// Configuration sent to factory create function. The parameters here are +// optional to use for a network controller implementation. +struct NetworkControllerConfig { + explicit NetworkControllerConfig() {} + + // The initial constraints to start with, these can be changed at any later + // time by calls to OnTargetRateConstraints. Note that the starting rate + // has to be set initially to provide a starting state for the network + // controller, even though the field is marked as optional. + TargetRateConstraints constraints; + // Initial stream specific configuration, these are changed at any later time + // by calls to OnStreamsConfig. + StreamsConfig stream_based_config; +}; + +// NetworkControllerInterface is implemented by network controllers. A network +// controller is a class that uses information about network state and traffic +// to estimate network parameters such as round trip time and bandwidth. Network +// controllers does not guarantee thread safety, the interface must be used in a +// non-concurrent fashion. +class NetworkControllerInterface { + public: + virtual ~NetworkControllerInterface() = default; + + // Called when network availabilty changes. + virtual NetworkControlUpdate OnNetworkAvailability(NetworkAvailability) = 0; + // Called when the receiving or sending endpoint changes address. + virtual NetworkControlUpdate OnNetworkRouteChange(NetworkRouteChange) = 0; + // Called periodically with a periodicy as specified by + // NetworkControllerFactoryInterface::GetProcessInterval. + virtual NetworkControlUpdate OnProcessInterval(ProcessInterval) = 0; + // Called when remotely calculated bitrate is received. + virtual NetworkControlUpdate OnRemoteBitrateReport(RemoteBitrateReport) = 0; + // Called round trip time has been calculated by protocol specific mechanisms. + virtual NetworkControlUpdate OnRoundTripTimeUpdate(RoundTripTimeUpdate) = 0; + // Called when a packet is sent on the network. + virtual NetworkControlUpdate OnSentPacket(SentPacket) = 0; + // Called when a packet is received from the remote client. + virtual NetworkControlUpdate OnReceivedPacket(ReceivedPacket) = 0; + // Called when the stream specific configuration has been updated. + virtual NetworkControlUpdate OnStreamsConfig(StreamsConfig) = 0; + // Called when target transfer rate constraints has been changed. + virtual NetworkControlUpdate OnTargetRateConstraints( + TargetRateConstraints) = 0; + // Called when a protocol specific calculation of packet loss has been made. + virtual NetworkControlUpdate OnTransportLossReport(TransportLossReport) = 0; + // Called with per packet feedback regarding receive time. + virtual NetworkControlUpdate OnTransportPacketsFeedback( + TransportPacketsFeedback) = 0; + // Called with network state estimate updates. + virtual NetworkControlUpdate OnNetworkStateEstimate(NetworkStateEstimate) = 0; +}; + +// NetworkControllerFactoryInterface is an interface for creating a network +// controller. +class NetworkControllerFactoryInterface { + public: + virtual ~NetworkControllerFactoryInterface() = default; + + // Used to create a new network controller, requires an observer to be + // provided to handle callbacks. + virtual std::unique_ptr Create( + NetworkControllerConfig config) = 0; + // Returns the interval by which the network controller expects + // OnProcessInterval calls. + virtual TimeDelta GetProcessInterval() const = 0; +}; + +// Under development, subject to change without notice. +class NetworkStateEstimator { + public: + // Gets the current best estimate according to the estimator. + virtual std::optional GetCurrentEstimate() = 0; + // Called with per packet feedback regarding receive time. + // Used when the NetworkStateEstimator runs in the sending endpoint. + virtual void OnTransportPacketsFeedback(const TransportPacketsFeedback&) = 0; + // Called with per packet feedback regarding receive time. + // Used when the NetworkStateEstimator runs in the receiving endpoint. + virtual void OnReceivedPacket(const PacketResult&) {} + // Called when the receiving or sending endpoint changes address. + virtual void OnRouteChange(const NetworkRouteChange&) = 0; + virtual ~NetworkStateEstimator() = default; +}; +class NetworkStateEstimatorFactory { + public: + virtual std::unique_ptr Create() = 0; + virtual ~NetworkStateEstimatorFactory() = default; +}; +} // namespace webrtc + +#endif // API_TRANSPORT_NETWORK_CONTROL_H_ diff --git a/src/qos/congestion_control.cpp b/src/qos/congestion_control.cpp index 1c1b988..1e092cb 100644 --- a/src/qos/congestion_control.cpp +++ b/src/qos/congestion_control.cpp @@ -66,12 +66,106 @@ CongestionControl::CongestionControl() AcknowledgedBitrateEstimatorInterface::Create()), pacing_factor_(kDefaultPaceMultiplier), min_total_allocated_bitrate_(DataRate::Zero()), - max_padding_rate_(DataRate::Zero()) + max_padding_rate_(DataRate::Zero()) { + NetworkControllerConfig config; -{} + config.constraints.at_time = Timestamp::PlusInfinity(); + config.constraints.min_data_rate = DataRate::BitsPerSec(300000); + config.constraints.max_data_rate = DataRate::BitsPerSec(5000000); + config.constraints.starting_rate = DataRate::BitsPerSec(2500000); + + config.stream_based_config.at_time = Timestamp::PlusInfinity(); + config.stream_based_config.requests_alr_probing = true; + config.stream_based_config.enable_repeated_initial_probing = true; + config.stream_based_config.pacing_factor = kDefaultPaceMultiplier; + config.stream_based_config.min_total_allocated_bitrate = DataRate::Zero(); + config.stream_based_config.max_padding_rate = DataRate::Zero(); + config.stream_based_config.max_total_allocated_bitrate = DataRate::Zero(); + + initial_config_ = config; +} CongestionControl::~CongestionControl() {} +NetworkControlUpdate CongestionControl::OnProcessInterval(ProcessInterval msg) { + NetworkControlUpdate update; + if (initial_config_) { + update.probe_cluster_configs = + ResetConstraints(initial_config_->constraints); + update.pacer_config = GetPacingRates(msg.at_time); + + if (initial_config_->stream_based_config.requests_alr_probing) { + probe_controller_->EnablePeriodicAlrProbing( + *initial_config_->stream_based_config.requests_alr_probing); + } + if (initial_config_->stream_based_config.enable_repeated_initial_probing) { + probe_controller_->EnableRepeatedInitialProbing( + *initial_config_->stream_based_config + .enable_repeated_initial_probing); + } + std::optional total_bitrate = + initial_config_->stream_based_config.max_total_allocated_bitrate; + if (total_bitrate) { + auto probes = probe_controller_->OnMaxTotalAllocatedBitrate( + *total_bitrate, msg.at_time); + update.probe_cluster_configs.insert(update.probe_cluster_configs.end(), + probes.begin(), probes.end()); + } + initial_config_.reset(); + } + + bandwidth_estimation_->UpdateEstimate(msg.at_time); + std::optional start_time_ms = + alr_detector_->GetApplicationLimitedRegionStartTime(); + probe_controller_->SetAlrStartTimeMs(start_time_ms); + + auto probes = probe_controller_->Process(msg.at_time); + update.probe_cluster_configs.insert(update.probe_cluster_configs.end(), + probes.begin(), probes.end()); + + update.congestion_window = current_data_window_; + + MaybeTriggerOnNetworkChanged(&update, msg.at_time); + return update; +} + +void CongestionControl::ClampConstraints() { + // TODO(holmer): We should make sure the default bitrates are set to 10 kbps, + // and that we don't try to set the min bitrate to 0 from any applications. + // The congestion controller should allow a min bitrate of 0. + min_data_rate_ = std::max(min_target_rate_, kCongestionControllerMinBitrate); + if (use_min_allocatable_as_lower_bound_) { + min_data_rate_ = std::max(min_data_rate_, min_total_allocated_bitrate_); + } + if (max_data_rate_ < min_data_rate_) { + LOG_WARN("max bitrate smaller than min bitrate"); + max_data_rate_ = min_data_rate_; + } + if (starting_rate_ && starting_rate_ < min_data_rate_) { + LOG_WARN("start bitrate smaller than min bitrate"); + starting_rate_ = min_data_rate_; + } +} + +std::vector CongestionControl::ResetConstraints( + TargetRateConstraints new_constraints) { + min_target_rate_ = new_constraints.min_data_rate.value_or(DataRate::Zero()); + max_data_rate_ = + new_constraints.max_data_rate.value_or(DataRate::PlusInfinity()); + starting_rate_ = new_constraints.starting_rate; + ClampConstraints(); + + bandwidth_estimation_->SetBitrates(starting_rate_, min_data_rate_, + max_data_rate_, new_constraints.at_time); + + if (starting_rate_) delay_based_bwe_->SetStartBitrate(*starting_rate_); + delay_based_bwe_->SetMinBitrate(min_data_rate_); + + return probe_controller_->SetBitrates( + min_data_rate_, starting_rate_.value_or(DataRate::Zero()), max_data_rate_, + new_constraints.at_time); +} + NetworkControlUpdate CongestionControl::OnTransportLossReport( TransportLossReport msg) { if (packet_feedback_only_) { @@ -236,10 +330,6 @@ NetworkControlUpdate CongestionControl::OnTransportPacketsFeedback( // No valid RTT could be because send-side BWE isn't used, in which case // we don't try to limit the outstanding packets. - // if (rate_control_settings_.UseCongestionWindow() && - // max_feedback_rtt.IsFinite()) { - // UpdateCongestionWindowSize(); - // } // if (congestion_window_pushback_controller_ && current_data_window_) { // congestion_window_pushback_controller_->SetDataWindow( // *current_data_window_); diff --git a/src/qos/congestion_control.h b/src/qos/congestion_control.h index 57ec794..39223f6 100644 --- a/src/qos/congestion_control.h +++ b/src/qos/congestion_control.h @@ -8,6 +8,7 @@ #include "acknowledged_bitrate_estimator_interface.h" #include "alr_detector.h" #include "api/network_state_predictor.h" +#include "api/transport/network_control.h" #include "api/transport/network_types.h" #include "congestion_window_pushback_controller.h" #include "delay_based_bwe.h" @@ -22,6 +23,8 @@ class CongestionControl { ~CongestionControl(); public: + NetworkControlUpdate OnProcessInterval(ProcessInterval msg); + NetworkControlUpdate OnTransportLossReport(TransportLossReport msg); NetworkControlUpdate OnTransportPacketsFeedback( @@ -31,6 +34,9 @@ class CongestionControl { Timestamp at_time); private: + void ClampConstraints(); + std::vector ResetConstraints( + TargetRateConstraints new_constraints); PacerConfig GetPacingRates(Timestamp at_time) const; private: @@ -52,6 +58,8 @@ class CongestionControl { std::unique_ptr acknowledged_bitrate_estimator_; + std::optional initial_config_; + DataRate min_target_rate_ = DataRate::Zero(); DataRate min_data_rate_ = DataRate::Zero(); DataRate max_data_rate_ = DataRate::PlusInfinity(); diff --git a/src/transport/ice_transport.cpp b/src/transport/ice_transport.cpp index 9fb50fb..7d0bc19 100644 --- a/src/transport/ice_transport.cpp +++ b/src/transport/ice_transport.cpp @@ -49,6 +49,7 @@ int IceTransport::InitIceTransmission( std::string &turn_username, std::string &turn_password, rtp::PAYLOAD_TYPE video_codec_payload_type) { ice_transport_controller_ = std::make_shared(clock_); + ice_agent_ = std::make_unique( offer_peer_, use_trickle_ice_, use_reliable_ice_, enable_turn_, force_turn_, stun_ip, stun_port, turn_ip, turn_port, turn_username, @@ -642,6 +643,7 @@ std::string IceTransport::GetRemoteCapabilities(const std::string &remote_sdp) { remote_user_id_, negotiated_video_pt_, hardware_acceleration_, ice_agent_, ice_io_statistics_, on_receive_video_, on_receive_audio_, on_receive_data_, user_data_); + ice_transport_controller_->Start(); } remote_capabilities_got_ = true; diff --git a/src/transport/ice_transport_controller.cpp b/src/transport/ice_transport_controller.cpp index b2be53a..20a88d0 100644 --- a/src/transport/ice_transport_controller.cpp +++ b/src/transport/ice_transport_controller.cpp @@ -18,7 +18,9 @@ IceTransportController::IceTransportController( load_nvcodec_dll_success_(false), hardware_acceleration_(false), clock_(clock), - webrtc_clock_(webrtc::Clock::GetWebrtcClockShared(clock)) {} + webrtc_clock_(webrtc::Clock::GetWebrtcClockShared(clock)) { + SetPeriod(std::chrono::milliseconds(25)); +} IceTransportController::~IceTransportController() { user_data_ = nullptr; @@ -429,6 +431,12 @@ void IceTransportController::HandleTransportPacketsFeedback( UpdateCongestedState(); } +void IceTransportController::UpdateControllerWithTimeInterval() { + ProcessInterval msg; + msg.at_time = Timestamp::Millis(webrtc_clock_->TimeInMilliseconds()); + PostUpdates(controller_->OnProcessInterval(msg)); +} + void IceTransportController::OnSentRtpPacket( const webrtc::RtpPacketToSend& packet) { webrtc::PacedPacketInfo pacing_info; @@ -452,35 +460,37 @@ void IceTransportController::OnSentRtpPacket( void IceTransportController::PostUpdates(webrtc::NetworkControlUpdate update) { // UpdateControlState(); - int target_bitrate = update.target_rate.has_value() - ? (update.target_rate->target_rate.bps() == 0 - ? target_bitrate_ - : update.target_rate->target_rate.bps()) - : target_bitrate_; - if (target_bitrate != target_bitrate_) { - target_bitrate_ = target_bitrate; - int width, height, target_width, target_height; - video_encoder_->GetResolution(&width, &height); + if (update.target_rate) { + int target_bitrate = update.target_rate.has_value() + ? (update.target_rate->target_rate.bps() == 0 + ? target_bitrate_ + : update.target_rate->target_rate.bps()) + : target_bitrate_; + if (target_bitrate != target_bitrate_) { + target_bitrate_ = target_bitrate; + int width, height, target_width, target_height; + video_encoder_->GetResolution(&width, &height); - if (0 == resolution_adapter_->GetResolution(target_bitrate_, width, height, - &target_width, - &target_height)) { - if (target_width != target_width_ || target_height != target_height_) { - target_width_ = target_width; - target_height_ = target_height; + if (0 == resolution_adapter_->GetResolution(target_bitrate_, width, + height, &target_width, + &target_height)) { + if (target_width != target_width_ || target_height != target_height_) { + target_width_ = target_width; + target_height_ = target_height; - b_force_i_frame_ = true; - LOG_INFO("Set target resolution [{}x{}]", target_width_.value(), - target_height_.value()); + b_force_i_frame_ = true; + // LOG_INFO("Set target resolution [{}x{}]", target_width_.value(), + // target_height_.value()); + } + } else if (target_width_.has_value() && target_height_.has_value()) { + target_width_.reset(); + target_height_.reset(); + // LOG_INFO("Use original resolution [{}x{}]", source_width_, + // source_height_); } - } else if (target_width_.has_value() && target_height_.has_value()) { - target_width_.reset(); - target_height_.reset(); - LOG_INFO("Use original resolution [{}x{}]", source_width_, - source_height_); + video_encoder_->SetTargetBitrate(target_bitrate_); + LOG_WARN("Set target bitrate [{}]bps", target_bitrate_); } - video_encoder_->SetTargetBitrate(target_bitrate_); - LOG_WARN("Set target bitrate [{}]bps", target_bitrate_); } } @@ -493,3 +503,10 @@ void IceTransportController::UpdateCongestedState() { if (controller_) { } } + +bool IceTransportController::Process() { + webrtc::ProcessInterval msg; + msg.at_time = Timestamp::Millis(webrtc_clock_->TimeInMilliseconds()); + PostUpdates(controller_->OnProcessInterval(msg)); + return true; +} \ No newline at end of file diff --git a/src/transport/ice_transport_controller.h b/src/transport/ice_transport_controller.h index 37613db..871be83 100644 --- a/src/transport/ice_transport_controller.h +++ b/src/transport/ice_transport_controller.h @@ -35,7 +35,8 @@ typedef void (*OnReceiveData)(const char *, size_t, const char *, const size_t, void *); class IceTransportController - : public std::enable_shared_from_this { + : public std::enable_shared_from_this, + public ThreadBase { public: IceTransportController(std::shared_ptr clock); ~IceTransportController(); @@ -74,6 +75,7 @@ class IceTransportController int CreateAudioCodec(); private: + void UpdateControllerWithTimeInterval(); void OnSentRtpPacket(const webrtc::RtpPacketToSend &packet); void HandleTransportPacketsFeedback( const webrtc::TransportPacketsFeedback &feedback); @@ -81,6 +83,9 @@ class IceTransportController void UpdateControlState(); void UpdateCongestedState(); + private: + bool Process() override; + private: std::unique_ptr video_channel_send_ = nullptr; std::unique_ptr audio_channel_send_ = nullptr;