[feat] bandwidth probing supported

src/channel/rtp_channel/rtp_video_receiver.h

@@ -70,7 +70,8 @@ class RtpVideoReceiver : public ThreadBase,
   void RtcpThread();
 
  private:
-  void SendNack(const std::vector<uint16_t>& nack_list, bool buffering_allowed);
+  void SendNack(const std::vector<uint16_t>& nack_list,
+                bool buffering_allowed) override;
 
   void SendRR();
 

src/common/api/transport/network_control.h

@@ -0,0 +1,124 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_TRANSPORT_NETWORK_CONTROL_H_
+#define API_TRANSPORT_NETWORK_CONTROL_H_
+
+#include <memory>
+#include <optional>
+
+#include "api/transport/network_types.h"
+#include "api/units/data_rate.h"
+#include "api/units/time_delta.h"
+
+namespace webrtc {
+
+class TargetTransferRateObserver {
+ public:
+  virtual ~TargetTransferRateObserver() = default;
+  // Called to indicate target transfer rate as well as giving information about
+  // the current estimate of network parameters.
+  virtual void OnTargetTransferRate(TargetTransferRate) = 0;
+  // Called to provide updates to the expected target rate in case it changes
+  // before the first call to OnTargetTransferRate.
+  virtual void OnStartRateUpdate(DataRate) {}
+};
+
+// Configuration sent to factory create function. The parameters here are
+// optional to use for a network controller implementation.
+struct NetworkControllerConfig {
+  explicit NetworkControllerConfig() {}
+
+  // The initial constraints to start with, these can be changed at any later
+  // time by calls to OnTargetRateConstraints. Note that the starting rate
+  // has to be set initially to provide a starting state for the network
+  // controller, even though the field is marked as optional.
+  TargetRateConstraints constraints;
+  // Initial stream specific configuration, these are changed at any later time
+  // by calls to OnStreamsConfig.
+  StreamsConfig stream_based_config;
+};
+
+// NetworkControllerInterface is implemented by network controllers. A network
+// controller is a class that uses information about network state and traffic
+// to estimate network parameters such as round trip time and bandwidth. Network
+// controllers does not guarantee thread safety, the interface must be used in a
+// non-concurrent fashion.
+class NetworkControllerInterface {
+ public:
+  virtual ~NetworkControllerInterface() = default;
+
+  // Called when network availabilty changes.
+  virtual NetworkControlUpdate OnNetworkAvailability(NetworkAvailability) = 0;
+  // Called when the receiving or sending endpoint changes address.
+  virtual NetworkControlUpdate OnNetworkRouteChange(NetworkRouteChange) = 0;
+  // Called periodically with a periodicy as specified by
+  // NetworkControllerFactoryInterface::GetProcessInterval.
+  virtual NetworkControlUpdate OnProcessInterval(ProcessInterval) = 0;
+  // Called when remotely calculated bitrate is received.
+  virtual NetworkControlUpdate OnRemoteBitrateReport(RemoteBitrateReport) = 0;
+  // Called round trip time has been calculated by protocol specific mechanisms.
+  virtual NetworkControlUpdate OnRoundTripTimeUpdate(RoundTripTimeUpdate) = 0;
+  // Called when a packet is sent on the network.
+  virtual NetworkControlUpdate OnSentPacket(SentPacket) = 0;
+  // Called when a packet is received from the remote client.
+  virtual NetworkControlUpdate OnReceivedPacket(ReceivedPacket) = 0;
+  // Called when the stream specific configuration has been updated.
+  virtual NetworkControlUpdate OnStreamsConfig(StreamsConfig) = 0;
+  // Called when target transfer rate constraints has been changed.
+  virtual NetworkControlUpdate OnTargetRateConstraints(
+      TargetRateConstraints) = 0;
+  // Called when a protocol specific calculation of packet loss has been made.
+  virtual NetworkControlUpdate OnTransportLossReport(TransportLossReport) = 0;
+  // Called with per packet feedback regarding receive time.
+  virtual NetworkControlUpdate OnTransportPacketsFeedback(
+      TransportPacketsFeedback) = 0;
+  // Called with network state estimate updates.
+  virtual NetworkControlUpdate OnNetworkStateEstimate(NetworkStateEstimate) = 0;
+};
+
+// NetworkControllerFactoryInterface is an interface for creating a network
+// controller.
+class NetworkControllerFactoryInterface {
+ public:
+  virtual ~NetworkControllerFactoryInterface() = default;
+
+  // Used to create a new network controller, requires an observer to be
+  // provided to handle callbacks.
+  virtual std::unique_ptr<NetworkControllerInterface> Create(
+      NetworkControllerConfig config) = 0;
+  // Returns the interval by which the network controller expects
+  // OnProcessInterval calls.
+  virtual TimeDelta GetProcessInterval() const = 0;
+};
+
+// Under development, subject to change without notice.
+class NetworkStateEstimator {
+ public:
+  // Gets the current best estimate according to the estimator.
+  virtual std::optional<NetworkStateEstimate> GetCurrentEstimate() = 0;
+  // Called with per packet feedback regarding receive time.
+  // Used when the NetworkStateEstimator runs in the sending endpoint.
+  virtual void OnTransportPacketsFeedback(const TransportPacketsFeedback&) = 0;
+  // Called with per packet feedback regarding receive time.
+  // Used when the NetworkStateEstimator runs in the receiving endpoint.
+  virtual void OnReceivedPacket(const PacketResult&) {}
+  // Called when the receiving or sending endpoint changes address.
+  virtual void OnRouteChange(const NetworkRouteChange&) = 0;
+  virtual ~NetworkStateEstimator() = default;
+};
+class NetworkStateEstimatorFactory {
+ public:
+  virtual std::unique_ptr<NetworkStateEstimator> Create() = 0;
+  virtual ~NetworkStateEstimatorFactory() = default;
+};
+}  // namespace webrtc
+
+#endif  // API_TRANSPORT_NETWORK_CONTROL_H_

src/qos/congestion_control.cpp

@@ -66,12 +66,106 @@ CongestionControl::CongestionControl()
           AcknowledgedBitrateEstimatorInterface::Create()),
       pacing_factor_(kDefaultPaceMultiplier),
       min_total_allocated_bitrate_(DataRate::Zero()),
-      max_padding_rate_(DataRate::Zero())
+      max_padding_rate_(DataRate::Zero()) {
+  NetworkControllerConfig config;
 
-{}
+  config.constraints.at_time = Timestamp::PlusInfinity();
+  config.constraints.min_data_rate = DataRate::BitsPerSec(300000);
+  config.constraints.max_data_rate = DataRate::BitsPerSec(5000000);
+  config.constraints.starting_rate = DataRate::BitsPerSec(2500000);
+
+  config.stream_based_config.at_time = Timestamp::PlusInfinity();
+  config.stream_based_config.requests_alr_probing = true;
+  config.stream_based_config.enable_repeated_initial_probing = true;
+  config.stream_based_config.pacing_factor = kDefaultPaceMultiplier;
+  config.stream_based_config.min_total_allocated_bitrate = DataRate::Zero();
+  config.stream_based_config.max_padding_rate = DataRate::Zero();
+  config.stream_based_config.max_total_allocated_bitrate = DataRate::Zero();
+
+  initial_config_ = config;
+}
 
 CongestionControl::~CongestionControl() {}
 
+NetworkControlUpdate CongestionControl::OnProcessInterval(ProcessInterval msg) {
+  NetworkControlUpdate update;
+  if (initial_config_) {
+    update.probe_cluster_configs =
+        ResetConstraints(initial_config_->constraints);
+    update.pacer_config = GetPacingRates(msg.at_time);
+
+    if (initial_config_->stream_based_config.requests_alr_probing) {
+      probe_controller_->EnablePeriodicAlrProbing(
+          *initial_config_->stream_based_config.requests_alr_probing);
+    }
+    if (initial_config_->stream_based_config.enable_repeated_initial_probing) {
+      probe_controller_->EnableRepeatedInitialProbing(
+          *initial_config_->stream_based_config
+               .enable_repeated_initial_probing);
+    }
+    std::optional<DataRate> total_bitrate =
+        initial_config_->stream_based_config.max_total_allocated_bitrate;
+    if (total_bitrate) {
+      auto probes = probe_controller_->OnMaxTotalAllocatedBitrate(
+          *total_bitrate, msg.at_time);
+      update.probe_cluster_configs.insert(update.probe_cluster_configs.end(),
+                                          probes.begin(), probes.end());
+    }
+    initial_config_.reset();
+  }
+
+  bandwidth_estimation_->UpdateEstimate(msg.at_time);
+  std::optional<int64_t> start_time_ms =
+      alr_detector_->GetApplicationLimitedRegionStartTime();
+  probe_controller_->SetAlrStartTimeMs(start_time_ms);
+
+  auto probes = probe_controller_->Process(msg.at_time);
+  update.probe_cluster_configs.insert(update.probe_cluster_configs.end(),
+                                      probes.begin(), probes.end());
+
+  update.congestion_window = current_data_window_;
+
+  MaybeTriggerOnNetworkChanged(&update, msg.at_time);
+  return update;
+}
+
+void CongestionControl::ClampConstraints() {
+  // TODO(holmer): We should make sure the default bitrates are set to 10 kbps,
+  // and that we don't try to set the min bitrate to 0 from any applications.
+  // The congestion controller should allow a min bitrate of 0.
+  min_data_rate_ = std::max(min_target_rate_, kCongestionControllerMinBitrate);
+  if (use_min_allocatable_as_lower_bound_) {
+    min_data_rate_ = std::max(min_data_rate_, min_total_allocated_bitrate_);
+  }
+  if (max_data_rate_ < min_data_rate_) {
+    LOG_WARN("max bitrate smaller than min bitrate");
+    max_data_rate_ = min_data_rate_;
+  }
+  if (starting_rate_ && starting_rate_ < min_data_rate_) {
+    LOG_WARN("start bitrate smaller than min bitrate");
+    starting_rate_ = min_data_rate_;
+  }
+}
+
+std::vector<ProbeClusterConfig> CongestionControl::ResetConstraints(
+    TargetRateConstraints new_constraints) {
+  min_target_rate_ = new_constraints.min_data_rate.value_or(DataRate::Zero());
+  max_data_rate_ =
+      new_constraints.max_data_rate.value_or(DataRate::PlusInfinity());
+  starting_rate_ = new_constraints.starting_rate;
+  ClampConstraints();
+
+  bandwidth_estimation_->SetBitrates(starting_rate_, min_data_rate_,
+                                     max_data_rate_, new_constraints.at_time);
+
+  if (starting_rate_) delay_based_bwe_->SetStartBitrate(*starting_rate_);
+  delay_based_bwe_->SetMinBitrate(min_data_rate_);
+
+  return probe_controller_->SetBitrates(
+      min_data_rate_, starting_rate_.value_or(DataRate::Zero()), max_data_rate_,
+      new_constraints.at_time);
+}
+
 NetworkControlUpdate CongestionControl::OnTransportLossReport(
     TransportLossReport msg) {
   if (packet_feedback_only_) {
@@ -236,10 +330,6 @@ NetworkControlUpdate CongestionControl::OnTransportPacketsFeedback(
 
   // No valid RTT could be because send-side BWE isn't used, in which case
   // we don't try to limit the outstanding packets.
-  // if (rate_control_settings_.UseCongestionWindow() &&
-  //     max_feedback_rtt.IsFinite()) {
-  //   UpdateCongestionWindowSize();
-  // }
   // if (congestion_window_pushback_controller_ && current_data_window_) {
   //   congestion_window_pushback_controller_->SetDataWindow(
   //       *current_data_window_);

src/qos/congestion_control.h

@@ -8,6 +8,7 @@
 #include "acknowledged_bitrate_estimator_interface.h"
 #include "alr_detector.h"
 #include "api/network_state_predictor.h"
+#include "api/transport/network_control.h"
 #include "api/transport/network_types.h"
 #include "congestion_window_pushback_controller.h"
 #include "delay_based_bwe.h"
@@ -22,6 +23,8 @@ class CongestionControl {
   ~CongestionControl();
 
  public:
+  NetworkControlUpdate OnProcessInterval(ProcessInterval msg);
+
   NetworkControlUpdate OnTransportLossReport(TransportLossReport msg);
 
   NetworkControlUpdate OnTransportPacketsFeedback(
@@ -31,6 +34,9 @@ class CongestionControl {
                                     Timestamp at_time);
 
  private:
+  void ClampConstraints();
+  std::vector<ProbeClusterConfig> ResetConstraints(
+      TargetRateConstraints new_constraints);
   PacerConfig GetPacingRates(Timestamp at_time) const;
 
  private:
@@ -52,6 +58,8 @@ class CongestionControl {
   std::unique_ptr<AcknowledgedBitrateEstimatorInterface>
       acknowledged_bitrate_estimator_;
 
+  std::optional<NetworkControllerConfig> initial_config_;
+
   DataRate min_target_rate_ = DataRate::Zero();
   DataRate min_data_rate_ = DataRate::Zero();
   DataRate max_data_rate_ = DataRate::PlusInfinity();

src/transport/ice_transport.cpp

@@ -49,6 +49,7 @@ int IceTransport::InitIceTransmission(
     std::string &turn_username, std::string &turn_password,
     rtp::PAYLOAD_TYPE video_codec_payload_type) {
   ice_transport_controller_ = std::make_shared<IceTransportController>(clock_);
+
   ice_agent_ = std::make_unique<IceAgent>(
       offer_peer_, use_trickle_ice_, use_reliable_ice_, enable_turn_,
       force_turn_, stun_ip, stun_port, turn_ip, turn_port, turn_username,
@@ -642,6 +643,7 @@ std::string IceTransport::GetRemoteCapabilities(const std::string &remote_sdp) {
           remote_user_id_, negotiated_video_pt_, hardware_acceleration_,
           ice_agent_, ice_io_statistics_, on_receive_video_, on_receive_audio_,
           on_receive_data_, user_data_);
+      ice_transport_controller_->Start();
     }
 
     remote_capabilities_got_ = true;

src/transport/ice_transport_controller.cpp

@@ -18,7 +18,9 @@ IceTransportController::IceTransportController(
       load_nvcodec_dll_success_(false),
       hardware_acceleration_(false),
       clock_(clock),
-      webrtc_clock_(webrtc::Clock::GetWebrtcClockShared(clock)) {}
+      webrtc_clock_(webrtc::Clock::GetWebrtcClockShared(clock)) {
+  SetPeriod(std::chrono::milliseconds(25));
+}
 
 IceTransportController::~IceTransportController() {
   user_data_ = nullptr;
@@ -429,6 +431,12 @@ void IceTransportController::HandleTransportPacketsFeedback(
   UpdateCongestedState();
 }
 
+void IceTransportController::UpdateControllerWithTimeInterval() {
+  ProcessInterval msg;
+  msg.at_time = Timestamp::Millis(webrtc_clock_->TimeInMilliseconds());
+  PostUpdates(controller_->OnProcessInterval(msg));
+}
+
 void IceTransportController::OnSentRtpPacket(
     const webrtc::RtpPacketToSend& packet) {
   webrtc::PacedPacketInfo pacing_info;
@@ -452,6 +460,7 @@ void IceTransportController::OnSentRtpPacket(
 void IceTransportController::PostUpdates(webrtc::NetworkControlUpdate update) {
   // UpdateControlState();
 
+  if (update.target_rate) {
     int target_bitrate = update.target_rate.has_value()
                              ? (update.target_rate->target_rate.bps() == 0
                                     ? target_bitrate_
@@ -462,27 +471,28 @@ void IceTransportController::PostUpdates(webrtc::NetworkControlUpdate update) {
       int width, height, target_width, target_height;
       video_encoder_->GetResolution(&width, &height);
 
-    if (0 == resolution_adapter_->GetResolution(target_bitrate_, width, height,
-                                                &target_width,
+      if (0 == resolution_adapter_->GetResolution(target_bitrate_, width,
+                                                  height, &target_width,
                                                   &target_height)) {
         if (target_width != target_width_ || target_height != target_height_) {
           target_width_ = target_width;
           target_height_ = target_height;
 
           b_force_i_frame_ = true;
-        LOG_INFO("Set target resolution [{}x{}]", target_width_.value(),
-                 target_height_.value());
+          // LOG_INFO("Set target resolution [{}x{}]", target_width_.value(),
+          //          target_height_.value());
         }
       } else if (target_width_.has_value() && target_height_.has_value()) {
         target_width_.reset();
         target_height_.reset();
-      LOG_INFO("Use original resolution [{}x{}]", source_width_,
-               source_height_);
+        // LOG_INFO("Use original resolution [{}x{}]", source_width_,
+        //          source_height_);
       }
       video_encoder_->SetTargetBitrate(target_bitrate_);
       LOG_WARN("Set target bitrate [{}]bps", target_bitrate_);
     }
   }
+}
 
 void IceTransportController::UpdateControlState() {
   if (controller_) {
@@ -493,3 +503,10 @@ void IceTransportController::UpdateCongestedState() {
   if (controller_) {
   }
 }
+
+bool IceTransportController::Process() {
+  webrtc::ProcessInterval msg;
+  msg.at_time = Timestamp::Millis(webrtc_clock_->TimeInMilliseconds());
+  PostUpdates(controller_->OnProcessInterval(msg));
+  return true;
+}

src/transport/ice_transport_controller.h

@@ -35,7 +35,8 @@ typedef void (*OnReceiveData)(const char *, size_t, const char *, const size_t,
                               void *);
 
 class IceTransportController
-    : public std::enable_shared_from_this<IceTransportController> {
+    : public std::enable_shared_from_this<IceTransportController>,
+      public ThreadBase {
  public:
   IceTransportController(std::shared_ptr<SystemClock> clock);
   ~IceTransportController();
@@ -74,6 +75,7 @@ class IceTransportController
   int CreateAudioCodec();
 
  private:
+  void UpdateControllerWithTimeInterval();
   void OnSentRtpPacket(const webrtc::RtpPacketToSend &packet);
   void HandleTransportPacketsFeedback(
       const webrtc::TransportPacketsFeedback &feedback);
@@ -81,6 +83,9 @@ class IceTransportController
   void UpdateControlState();
   void UpdateCongestedState();
 
+ private:
+  bool Process() override;
+
  private:
   std::unique_ptr<VideoChannelSend> video_channel_send_ = nullptr;
   std::unique_ptr<AudioChannelSend> audio_channel_send_ = nullptr;