diff --git a/src/channel/rtp_channel/rtp_video_receiver.cpp b/src/channel/rtp_channel/rtp_video_receiver.cpp index 559f2f0..ac5503d 100644 --- a/src/channel/rtp_channel/rtp_video_receiver.cpp +++ b/src/channel/rtp_channel/rtp_video_receiver.cpp @@ -9,18 +9,29 @@ RtpVideoReceiver::RtpVideoReceiver() : feedback_ssrc_(GenerateUniqueSsrc()), + active_remb_module_(nullptr), receive_side_congestion_controller_( + clock_, [this](std::vector> packets) { SendCombinedRtcpPacket(std::move(packets)); - }) {} + }, + [this](int64_t bitrate_bps, std::vector ssrcs) { + SendRemb(bitrate_bps, ssrcs); + }), + clock_(Clock::GetRealTimeClock()) {} RtpVideoReceiver::RtpVideoReceiver(std::shared_ptr io_statistics) : io_statistics_(io_statistics), feedback_ssrc_(GenerateUniqueSsrc()), receive_side_congestion_controller_( + clock_, [this](std::vector> packets) { SendCombinedRtcpPacket(std::move(packets)); - }) { + }, + [this](int64_t bitrate_bps, std::vector ssrcs) { + SendRemb(bitrate_bps, ssrcs); + }), + clock_(Clock::GetRealTimeClock()) { rtcp_thread_ = std::thread(&RtpVideoReceiver::RtcpThread, this); } @@ -45,16 +56,15 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) { rtp_statistics_->Start(); } - RtpPacketReceived rtp_packet_received; + webrtc::RtpPacketReceived rtp_packet_received; rtp_packet_received.Build(rtp_packet.Buffer(), rtp_packet.Size()); - rtp_packet_received.set_arrival_time( - std::chrono::system_clock::now().time_since_epoch().count()); - rtp_packet_received.set_ecn(rtc::EcnMarking::kEct0); + rtp_packet_received.set_arrival_time(clock_->CurrentTime()); + rtp_packet_received.set_ecn(EcnMarking::kEct0); rtp_packet_received.set_recovered(false); rtp_packet_received.set_payload_type_frequency(0); - receive_side_congestion_controller_.OnReceivedPacket( - rtp_packet_received, ReceiveSideCongestionController::MediaType::VIDEO); + receive_side_congestion_controller_.OnReceivedPacket(rtp_packet_received, + MediaType::VIDEO); last_recv_bytes_ = (uint32_t)rtp_packet.PayloadSize(); total_rtp_payload_recv_ += (uint32_t)rtp_packet.PayloadSize(); @@ -373,6 +383,17 @@ void RtpVideoReceiver::SendCombinedRtcpPacket( } } +void RtpVideoReceiver::SendRemb(int64_t bitrate_bps, + std::vector ssrcs) { + if (!active_remb_module_) { + return; + } + + // The Add* and Remove* methods above ensure that REMB is disabled on all + // other modules, because otherwise, they will send REMB with stale info. + active_remb_module_->SetRemb(bitrate_bps, std::move(ssrcs)); +} + bool RtpVideoReceiver::CheckIsTimeSendRR() { uint32_t now_ts = static_cast( std::chrono::duration_cast( diff --git a/src/channel/rtp_channel/rtp_video_receiver.h b/src/channel/rtp_channel/rtp_video_receiver.h index 883a6c1..610c60c 100644 --- a/src/channel/rtp_channel/rtp_video_receiver.h +++ b/src/channel/rtp_channel/rtp_video_receiver.h @@ -6,16 +6,20 @@ #include #include +#include "clock.h" #include "fec_decoder.h" #include "io_statistics.h" #include "receive_side_congestion_controller.h" #include "ringbuffer.h" #include "rtcp_receiver_report.h" #include "rtp_codec.h" +#include "rtp_rtcp_defines.h " #include "rtp_statistics.h" #include "thread_base.h" #include "video_frame.h" +using namespace webrtc; + class RtpVideoReceiver : public ThreadBase { public: RtpVideoReceiver(); @@ -47,6 +51,8 @@ class RtpVideoReceiver : public ThreadBase { void SendCombinedRtcpPacket( std::vector> rtcp_packets); + void SendRemb(int64_t bitrate_bps, std::vector ssrcs); + private: bool Process() override; void RtcpThread(); @@ -89,7 +95,9 @@ class RtpVideoReceiver : public ThreadBase { int rtcp_tcc_interval_ms_ = 200; private: + std::shared_ptr clock_; ReceiveSideCongestionController receive_side_congestion_controller_; + RtcpFeedbackSenderInterface* active_remb_module_; uint32_t feedback_ssrc_ = 0; }; diff --git a/src/channel/video_channel_send.cpp b/src/channel/video_channel_send.cpp index 9aff283..55e0eef 100644 --- a/src/channel/video_channel_send.cpp +++ b/src/channel/video_channel_send.cpp @@ -57,18 +57,18 @@ int VideoChannelSend::SendVideo(char* data, size_t size) { } void VideoChannelSend::OnCongestionControlFeedback( - int64_t recv_ts, const CongestionControlFeedback& feedback) { + int64_t recv_ts, const webrtc::rtcp::CongestionControlFeedback& feedback) { ++feedback_count_; - std::optional feedback_msg = - transport_feedback_adapter_.ProcessCongestionControlFeedback(feedback, - recv_ts); + std::optional feedback_msg = + transport_feedback_adapter_.ProcessCongestionControlFeedback( + feedback, webrtc::Timestamp::Micros(recv_ts)); if (feedback_msg) { HandleTransportPacketsFeedback(*feedback_msg); } } void VideoChannelSend::HandleTransportPacketsFeedback( - const TransportPacketsFeedback& feedback) { + const webrtc::TransportPacketsFeedback& feedback) { // if (transport_is_ecn_capable_) { // // If transport does not support ECN, packets should not be sent as // ECT(1). diff --git a/src/channel/video_channel_send.h b/src/channel/video_channel_send.h index 89c8874..bb7b673 100644 --- a/src/channel/video_channel_send.h +++ b/src/channel/video_channel_send.h @@ -7,6 +7,7 @@ #ifndef _VIDEO_CHANNEL_SEND_H_ #define _VIDEO_CHANNEL_SEND_H_ +#include "congestion_control_feedback.h" #include "ice_agent.h" #include "rtp_codec.h" #include "rtp_video_sender.h" @@ -25,10 +26,11 @@ class VideoChannelSend { int SendVideo(char* data, size_t size); - void OnCongestionControlFeedback(int64_t recv_ts, - const CongestionControlFeedback& feedback); + void OnCongestionControlFeedback( + int64_t recv_ts, const webrtc::rtcp::CongestionControlFeedback& feedback); - void HandleTransportPacketsFeedback(const TransportPacketsFeedback& feedback); + void HandleTransportPacketsFeedback( + const webrtc::TransportPacketsFeedback& feedback); private: std::shared_ptr ice_agent_ = nullptr; @@ -42,7 +44,7 @@ class VideoChannelSend { std::optional last_feedback_compact_ntp_time_; int feedback_count_ = 0; - TransportFeedbackAdapter transport_feedback_adapter_; + webrtc::TransportFeedbackAdapter transport_feedback_adapter_; }; #endif \ No newline at end of file diff --git a/src/common/api/function_view.h b/src/common/api/function_view.h new file mode 100644 index 0000000..88376bd --- /dev/null +++ b/src/common/api/function_view.h @@ -0,0 +1,127 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_FUNCTION_VIEW_H_ +#define API_FUNCTION_VIEW_H_ + +#include +#include +#include + +// Just like std::function, FunctionView will wrap any callable and hide its +// actual type, exposing only its signature. But unlike std::function, +// FunctionView doesn't own its callable---it just points to it. Thus, it's a +// good choice mainly as a function argument when the callable argument will +// not be called again once the function has returned. +// +// Its constructors are implicit, so that callers won't have to convert lambdas +// and other callables to FunctionView explicitly. This is +// safe because FunctionView is only a reference to the real callable. +// +// Example use: +// +// void SomeFunction(rtc::FunctionView index_transform); +// ... +// SomeFunction([](int i) { return 2 * i + 1; }); +// +// Note: FunctionView is tiny (essentially just two pointers) and trivially +// copyable, so it's probably cheaper to pass it by value than by const +// reference. + +namespace rtc { + +template +class FunctionView; // Undefined. + +template +class FunctionView final { + public: + // Constructor for lambdas and other callables; it accepts every type of + // argument except those noted in its enable_if call. + template < + typename F, + typename std::enable_if< + // Not for function pointers; we have another constructor for that + // below. + !std::is_function::type>::type>::value && + + // Not for nullptr; we have another constructor for that below. + !std::is_same::type>::value && + + // Not for FunctionView objects; we have another constructor for that + // (the implicitly declared copy constructor). + !std::is_same::type>::type>::value>::type* = nullptr> + FunctionView(F&& f) + : call_(CallVoidPtr::type>) { + f_.void_ptr = &f; + } + + // Constructor that accepts function pointers. If the argument is null, the + // result is an empty FunctionView. + template < + typename F, + typename std::enable_if::type>::type>::value>::type* = + nullptr> + FunctionView(F&& f) + : call_(f ? CallFunPtr::type> : nullptr) { + f_.fun_ptr = reinterpret_cast(f); + } + + // Constructor that accepts nullptr. It creates an empty FunctionView. + template ::type>:: + value>::type* = nullptr> + FunctionView(F&& /* f */) : call_(nullptr) {} + + // Default constructor. Creates an empty FunctionView. + FunctionView() : call_(nullptr) {} + + RetT operator()(ArgT... args) const { + return call_(f_, std::forward(args)...); + } + + // Returns true if we have a function, false if we don't (i.e., we're null). + explicit operator bool() const { return !!call_; } + + private: + union VoidUnion { + void* void_ptr; + void (*fun_ptr)(); + }; + + template + static RetT CallVoidPtr(VoidUnion vu, ArgT... args) { + return (*static_cast(vu.void_ptr))(std::forward(args)...); + } + template + static RetT CallFunPtr(VoidUnion vu, ArgT... args) { + return (reinterpret_cast::type>(vu.fun_ptr))( + std::forward(args)...); + } + + // A pointer to the callable thing, with type information erased. It's a + // union because we have to use separate types depending on if the callable + // thing is a function pointer or something else. + VoidUnion f_; + + // Pointer to a dispatch function that knows the type of the callable thing + // that's stored in f_, and how to call it. A FunctionView object is empty + // (null) iff call_ is null. + RetT (*call_)(VoidUnion, ArgT...); +}; + +} // namespace rtc + +#endif // API_FUNCTION_VIEW_H_ diff --git a/src/common/rtc_base/network/ecn_marking.h b/src/common/api/transport/ecn_marking.h similarity index 82% rename from src/common/rtc_base/network/ecn_marking.h rename to src/common/api/transport/ecn_marking.h index 43a4f20..bbcab6e 100644 --- a/src/common/rtc_base/network/ecn_marking.h +++ b/src/common/api/transport/ecn_marking.h @@ -7,13 +7,11 @@ * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ -#ifndef RTC_BASE_NETWORK_ECN_MARKING_H_ -#define RTC_BASE_NETWORK_ECN_MARKING_H_ -// // TODO: bugs.webrtc.org/42225697 - delete this file. -#include "ecn_marking.h" +#ifndef API_TRANSPORT_ECN_MARKING_H_ +#define API_TRANSPORT_ECN_MARKING_H_ -namespace rtc { +namespace webrtc { // TODO: bugs.webrtc.org/42225697 - L4S support is slowly being developed. // Help is appreciated. @@ -39,6 +37,6 @@ enum class EcnMarking { kCe = 3, // Congestion experienced }; -} // namespace rtc +} // namespace webrtc -#endif // RTC_BASE_NETWORK_ECN_MARKING_H_ +#endif // API_TRANSPORT_ECN_MARKING_H_ diff --git a/src/common/api/transport/network_control.h b/src/common/api/transport/network_control.h deleted file mode 100644 index 06a0329..0000000 --- a/src/common/api/transport/network_control.h +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef API_TRANSPORT_NETWORK_CONTROL_H_ -#define API_TRANSPORT_NETWORK_CONTROL_H_ - -#include -#include - -#include "api/transport/network_types.h" -#include "api/units/data_rate.h" -#include "api/units/time_delta.h" - -namespace webrtc { - -class TargetTransferRateObserver { - public: - virtual ~TargetTransferRateObserver() = default; - // Called to indicate target transfer rate as well as giving information about - // the current estimate of network parameters. - virtual void OnTargetTransferRate(TargetTransferRate) = 0; - // Called to provide updates to the expected target rate in case it changes - // before the first call to OnTargetTransferRate. - virtual void OnStartRateUpdate(DataRate) {} -}; - -// Configuration sent to factory create function. The parameters here are -// optional to use for a network controller implementation. -struct NetworkControllerConfig { - explicit NetworkControllerConfig() {} - - // The initial constraints to start with, these can be changed at any later - // time by calls to OnTargetRateConstraints. Note that the starting rate - // has to be set initially to provide a starting state for the network - // controller, even though the field is marked as optional. - TargetRateConstraints constraints; - // Initial stream specific configuration, these are changed at any later time - // by calls to OnStreamsConfig. - StreamsConfig stream_based_config; -}; - -// NetworkControllerInterface is implemented by network controllers. A network -// controller is a class that uses information about network state and traffic -// to estimate network parameters such as round trip time and bandwidth. Network -// controllers does not guarantee thread safety, the interface must be used in a -// non-concurrent fashion. -class NetworkControllerInterface { - public: - virtual ~NetworkControllerInterface() = default; - - // Called when network availabilty changes. - virtual NetworkControlUpdate OnNetworkAvailability(NetworkAvailability) = 0; - // Called when the receiving or sending endpoint changes address. - virtual NetworkControlUpdate OnNetworkRouteChange(NetworkRouteChange) = 0; - // Called periodically with a periodicy as specified by - // NetworkControllerFactoryInterface::GetProcessInterval. - virtual NetworkControlUpdate OnProcessInterval(ProcessInterval) = 0; - // Called when remotely calculated bitrate is received. - virtual NetworkControlUpdate OnRemoteBitrateReport(RemoteBitrateReport) = 0; - // Called round trip time has been calculated by protocol specific mechanisms. - virtual NetworkControlUpdate OnRoundTripTimeUpdate(RoundTripTimeUpdate) = 0; - // Called when a packet is sent on the network. - virtual NetworkControlUpdate OnSentPacket(SentPacket) = 0; - // Called when a packet is received from the remote client. - virtual NetworkControlUpdate OnReceivedPacket(ReceivedPacket) = 0; - // Called when the stream specific configuration has been updated. - virtual NetworkControlUpdate OnStreamsConfig(StreamsConfig) = 0; - // Called when target transfer rate constraints has been changed. - virtual NetworkControlUpdate OnTargetRateConstraints( - TargetRateConstraints) = 0; - // Called when a protocol specific calculation of packet loss has been made. - virtual NetworkControlUpdate OnTransportLossReport(TransportLossReport) = 0; - // Called with per packet feedback regarding receive time. - virtual NetworkControlUpdate OnTransportPacketsFeedback( - TransportPacketsFeedback) = 0; - // Called with network state estimate updates. - virtual NetworkControlUpdate OnNetworkStateEstimate(NetworkStateEstimate) = 0; -}; - -// NetworkControllerFactoryInterface is an interface for creating a network -// controller. -class NetworkControllerFactoryInterface { - public: - virtual ~NetworkControllerFactoryInterface() = default; - - // Used to create a new network controller, requires an observer to be - // provided to handle callbacks. - virtual std::unique_ptr Create( - NetworkControllerConfig config) = 0; - // Returns the interval by which the network controller expects - // OnProcessInterval calls. - virtual TimeDelta GetProcessInterval() const = 0; -}; - -// Under development, subject to change without notice. -class NetworkStateEstimator { - public: - // Gets the current best estimate according to the estimator. - virtual std::optional GetCurrentEstimate() = 0; - // Called with per packet feedback regarding receive time. - // Used when the NetworkStateEstimator runs in the sending endpoint. - virtual void OnTransportPacketsFeedback(const TransportPacketsFeedback&) = 0; - // Called with per packet feedback regarding receive time. - // Used when the NetworkStateEstimator runs in the receiving endpoint. - virtual void OnReceivedPacket(const PacketResult&) {} - // Called when the receiving or sending endpoint changes address. - virtual void OnRouteChange(const NetworkRouteChange&) = 0; - virtual ~NetworkStateEstimator() = default; -}; -class NetworkStateEstimatorFactory { - public: - virtual std::unique_ptr Create() = 0; - virtual ~NetworkStateEstimatorFactory() = default; -}; -} // namespace webrtc - -#endif // API_TRANSPORT_NETWORK_CONTROL_H_ diff --git a/src/common/api/transport/network_types.h b/src/common/api/transport/network_types.h index ba4c082..0475663 100644 --- a/src/common/api/transport/network_types.h +++ b/src/common/api/transport/network_types.h @@ -16,11 +16,11 @@ #include #include +#include "api/transport/ecn_marking.h" #include "api/units/data_rate.h" #include "api/units/data_size.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" -#include "rtc_base/network/ecn_marking.h" namespace webrtc { @@ -167,7 +167,7 @@ struct PacketResult { SentPacket sent_packet; Timestamp receive_time = Timestamp::PlusInfinity(); - rtc::EcnMarking ecn = rtc::EcnMarking::kNotEct; + EcnMarking ecn = EcnMarking::kNotEct; }; struct TransportPacketsFeedback { diff --git a/src/common/api/video/video_timing.cc b/src/common/api/video/video_timing.cc new file mode 100644 index 0000000..b1b68f8 --- /dev/null +++ b/src/common/api/video/video_timing.cc @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/video_timing.h" + +#include +#include +#include + +#include "api/array_view.h" +#include "api/units/time_delta.h" +#include "log.h" +#include "rtc_base/numerics/safe_conversions.h" + +namespace webrtc { + +uint16_t VideoSendTiming::GetDeltaCappedMs(int64_t base_ms, int64_t time_ms) { + if (time_ms < base_ms) { + LOG_ERROR("Delta {} ms expected to be positive", (time_ms - base_ms)); + } + return rtc::saturated_cast(time_ms - base_ms); +} + +uint16_t VideoSendTiming::GetDeltaCappedMs(TimeDelta delta) { + if (delta < TimeDelta::Zero()) { + LOG_ERROR("Delta {} ms expected to be positive", delta.ms()); + } + return rtc::saturated_cast(delta.ms()); +} + +TimingFrameInfo::TimingFrameInfo() + : rtp_timestamp(0), + capture_time_ms(-1), + encode_start_ms(-1), + encode_finish_ms(-1), + packetization_finish_ms(-1), + pacer_exit_ms(-1), + network_timestamp_ms(-1), + network2_timestamp_ms(-1), + receive_start_ms(-1), + receive_finish_ms(-1), + decode_start_ms(-1), + decode_finish_ms(-1), + render_time_ms(-1), + flags(VideoSendTiming::kNotTriggered) {} + +int64_t TimingFrameInfo::EndToEndDelay() const { + return capture_time_ms >= 0 ? decode_finish_ms - capture_time_ms : -1; +} + +bool TimingFrameInfo::IsLongerThan(const TimingFrameInfo& other) const { + int64_t other_delay = other.EndToEndDelay(); + return other_delay == -1 || EndToEndDelay() > other_delay; +} + +bool TimingFrameInfo::operator<(const TimingFrameInfo& other) const { + return other.IsLongerThan(*this); +} + +bool TimingFrameInfo::operator<=(const TimingFrameInfo& other) const { + return !IsLongerThan(other); +} + +bool TimingFrameInfo::IsOutlier() const { + return !IsInvalid() && (flags & VideoSendTiming::kTriggeredBySize); +} + +bool TimingFrameInfo::IsTimerTriggered() const { + return !IsInvalid() && (flags & VideoSendTiming::kTriggeredByTimer); +} + +bool TimingFrameInfo::IsInvalid() const { + return flags == VideoSendTiming::kInvalid; +} + +std::string TimingFrameInfo::ToString() const { + if (IsInvalid()) { + return ""; + } + + std::ostringstream oss; + oss << rtp_timestamp << ',' << capture_time_ms << ',' << encode_start_ms + << ',' << encode_finish_ms << ',' << packetization_finish_ms << ',' + << pacer_exit_ms << ',' << network_timestamp_ms << ',' + << network2_timestamp_ms << ',' << receive_start_ms << ',' + << receive_finish_ms << ',' << decode_start_ms << ',' << decode_finish_ms + << ',' << render_time_ms << ',' << IsOutlier() << ',' + << IsTimerTriggered(); + + return oss.str(); +} + +VideoPlayoutDelay::VideoPlayoutDelay(TimeDelta min, TimeDelta max) + : min_(std::clamp(min, TimeDelta::Zero(), kMax)), + max_(std::clamp(max, min_, kMax)) { + if (!(TimeDelta::Zero() <= min && min <= max && max <= kMax)) { + LOG_ERROR("Invalid video playout delay: [{},{}]. Clamped to [{},{}]", min, + max, this->min(), this->max()); + } +} + +bool VideoPlayoutDelay::Set(TimeDelta min, TimeDelta max) { + if (TimeDelta::Zero() <= min && min <= max && max <= kMax) { + min_ = min; + max_ = max; + return true; + } + return false; +} + +} // namespace webrtc diff --git a/src/common/api/video/video_timing.h b/src/common/api/video/video_timing.h new file mode 100644 index 0000000..500cdcb --- /dev/null +++ b/src/common/api/video/video_timing.h @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_VIDEO_TIMING_H_ +#define API_VIDEO_VIDEO_TIMING_H_ + +#include + +#include +#include + +#include "api/units/time_delta.h" + +namespace webrtc { + +// Video timing timestamps in ms counted from capture_time_ms of a frame. +// This structure represents data sent in video-timing RTP header extension. +struct VideoSendTiming { + enum TimingFrameFlags : uint8_t { + kNotTriggered = 0, // Timing info valid, but not to be transmitted. + // Used on send-side only. + kTriggeredByTimer = 1 << 0, // Frame marked for tracing by periodic timer. + kTriggeredBySize = 1 << 1, // Frame marked for tracing due to size. + kInvalid = std::numeric_limits::max() // Invalid, ignore! + }; + + // Returns |time_ms - base_ms| capped at max 16-bit value. + // Used to fill this data structure as per + // https://webrtc.org/experiments/rtp-hdrext/video-timing/ extension stores + // 16-bit deltas of timestamps from packet capture time. + static uint16_t GetDeltaCappedMs(int64_t base_ms, int64_t time_ms); + static uint16_t GetDeltaCappedMs(TimeDelta delta); + + uint16_t encode_start_delta_ms; + uint16_t encode_finish_delta_ms; + uint16_t packetization_finish_delta_ms; + uint16_t pacer_exit_delta_ms; + uint16_t network_timestamp_delta_ms; + uint16_t network2_timestamp_delta_ms; + uint8_t flags = TimingFrameFlags::kInvalid; +}; + +// Used to report precise timings of a 'timing frames'. Contains all important +// timestamps for a lifetime of that specific frame. Reported as a string via +// GetStats(). Only frame which took the longest between two GetStats calls is +// reported. +struct TimingFrameInfo { + TimingFrameInfo(); + + // Returns end-to-end delay of a frame, if sender and receiver timestamps are + // synchronized, -1 otherwise. + int64_t EndToEndDelay() const; + + // Returns true if current frame took longer to process than `other` frame. + // If other frame's clocks are not synchronized, current frame is always + // preferred. + bool IsLongerThan(const TimingFrameInfo& other) const; + + // Returns true if flags are set to indicate this frame was marked for tracing + // due to the size being outside some limit. + bool IsOutlier() const; + + // Returns true if flags are set to indicate this frame was marked fro tracing + // due to cyclic timer. + bool IsTimerTriggered() const; + + // Returns true if the timing data is marked as invalid, in which case it + // should be ignored. + bool IsInvalid() const; + + std::string ToString() const; + + bool operator<(const TimingFrameInfo& other) const; + + bool operator<=(const TimingFrameInfo& other) const; + + uint32_t rtp_timestamp; // Identifier of a frame. + // All timestamps below are in local monotonous clock of a receiver. + // If sender clock is not yet estimated, sender timestamps + // (capture_time_ms ... pacer_exit_ms) are negative values, still + // relatively correct. + int64_t capture_time_ms; // Captrue time of a frame. + int64_t encode_start_ms; // Encode start time. + int64_t encode_finish_ms; // Encode completion time. + int64_t packetization_finish_ms; // Time when frame was passed to pacer. + int64_t pacer_exit_ms; // Time when last packet was pushed out of pacer. + // Two in-network RTP processor timestamps: meaning is application specific. + int64_t network_timestamp_ms; + int64_t network2_timestamp_ms; + int64_t receive_start_ms; // First received packet time. + int64_t receive_finish_ms; // Last received packet time. + int64_t decode_start_ms; // Decode start time. + int64_t decode_finish_ms; // Decode completion time. + int64_t render_time_ms; // Proposed render time to insure smooth playback. + + uint8_t flags; // Flags indicating validity and/or why tracing was triggered. +}; + +// Minimum and maximum playout delay values from capture to render. +// These are best effort values. +// +// min = max = 0 indicates that the receiver should try and render +// frame as soon as possible. +// +// min = x, max = y indicates that the receiver is free to adapt +// in the range (x, y) based on network jitter. +// This class ensures invariant 0 <= min <= max <= kMax. +class VideoPlayoutDelay { + public: + // Maximum supported value for the delay limit. + static constexpr TimeDelta kMax = TimeDelta::Millis(10) * 0xFFF; + + // Creates delay limits that indicates receiver should try to render frame + // as soon as possible. + static VideoPlayoutDelay Minimal() { + return VideoPlayoutDelay(TimeDelta::Zero(), TimeDelta::Zero()); + } + + // Creates valid, but unspecified limits. + VideoPlayoutDelay() = default; + VideoPlayoutDelay(const VideoPlayoutDelay&) = default; + VideoPlayoutDelay& operator=(const VideoPlayoutDelay&) = default; + VideoPlayoutDelay(TimeDelta min, TimeDelta max); + + bool Set(TimeDelta min, TimeDelta max); + + TimeDelta min() const { return min_; } + TimeDelta max() const { return max_; } + + friend bool operator==(const VideoPlayoutDelay& lhs, + const VideoPlayoutDelay& rhs) { + return lhs.min_ == rhs.min_ && lhs.max_ == rhs.max_; + } + + private: + TimeDelta min_ = TimeDelta::Zero(); + TimeDelta max_ = kMax; +}; + +} // namespace webrtc + +#endif // API_VIDEO_VIDEO_TIMING_H_ diff --git a/src/common/rtc_base/network/sent_packet.cc b/src/common/rtc_base/network/sent_packet.cc new file mode 100644 index 0000000..8cc4973 --- /dev/null +++ b/src/common/rtc_base/network/sent_packet.cc @@ -0,0 +1,27 @@ +/* + * Copyright 2018 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/network/sent_packet.h" + +namespace rtc { + +PacketInfo::PacketInfo() = default; +PacketInfo::PacketInfo(const PacketInfo& info) = default; +PacketInfo::~PacketInfo() = default; + +SentPacket::SentPacket() = default; +SentPacket::SentPacket(int64_t packet_id, int64_t send_time_ms) + : packet_id(packet_id), send_time_ms(send_time_ms) {} +SentPacket::SentPacket(int64_t packet_id, + int64_t send_time_ms, + const rtc::PacketInfo& info) + : packet_id(packet_id), send_time_ms(send_time_ms), info(info) {} + +} // namespace rtc diff --git a/src/common/rtc_base/network/sent_packet.h b/src/common/rtc_base/network/sent_packet.h new file mode 100644 index 0000000..b2b938e --- /dev/null +++ b/src/common/rtc_base/network/sent_packet.h @@ -0,0 +1,70 @@ +/* + * Copyright 2018 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_NETWORK_SENT_PACKET_H_ +#define RTC_BASE_NETWORK_SENT_PACKET_H_ + +#include +#include + +#include + +namespace rtc { + +enum class PacketType { + kUnknown, + kData, + kIceConnectivityCheck, + kIceConnectivityCheckResponse, + kStunMessage, + kTurnMessage, +}; + +enum class PacketInfoProtocolType { + kUnknown, + kUdp, + kTcp, + kSsltcp, + kTls, +}; + +struct PacketInfo { + PacketInfo(); + PacketInfo(const PacketInfo& info); + ~PacketInfo(); + + bool included_in_feedback = false; + bool included_in_allocation = false; + // `is_media` is true if this is an audio or video packet, excluding + // retransmissions. + bool is_media = false; + PacketType packet_type = PacketType::kUnknown; + PacketInfoProtocolType protocol = PacketInfoProtocolType::kUnknown; + // A unique id assigned by the network manager, and std::nullopt if not set. + std::optional network_id; + size_t packet_size_bytes = 0; + size_t turn_overhead_bytes = 0; + size_t ip_overhead_bytes = 0; +}; + +struct SentPacket { + SentPacket(); + SentPacket(int64_t packet_id, int64_t send_time_ms); + SentPacket(int64_t packet_id, int64_t send_time_ms, + const rtc::PacketInfo& info); + + int64_t packet_id = -1; + int64_t send_time_ms = -1; + rtc::PacketInfo info; +}; + +} // namespace rtc + +#endif // RTC_BASE_NETWORK_SENT_PACKET_H_ diff --git a/src/common/rtc_base/network_constants.cc b/src/common/rtc_base/network_constants.cc new file mode 100644 index 0000000..c119d03 --- /dev/null +++ b/src/common/rtc_base/network_constants.cc @@ -0,0 +1,44 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/network_constants.h" + +namespace rtc { + +std::string AdapterTypeToString(AdapterType type) { + switch (type) { + case ADAPTER_TYPE_ANY: + return "Wildcard"; + case ADAPTER_TYPE_UNKNOWN: + return "Unknown"; + case ADAPTER_TYPE_ETHERNET: + return "Ethernet"; + case ADAPTER_TYPE_WIFI: + return "Wifi"; + case ADAPTER_TYPE_CELLULAR: + return "Cellular"; + case ADAPTER_TYPE_CELLULAR_2G: + return "Cellular2G"; + case ADAPTER_TYPE_CELLULAR_3G: + return "Cellular3G"; + case ADAPTER_TYPE_CELLULAR_4G: + return "Cellular4G"; + case ADAPTER_TYPE_CELLULAR_5G: + return "Cellular5G"; + case ADAPTER_TYPE_VPN: + return "VPN"; + case ADAPTER_TYPE_LOOPBACK: + return "Loopback"; + default: + return std::string(); + } +} + +} // namespace rtc diff --git a/src/common/rtc_base/network_constants.h b/src/common/rtc_base/network_constants.h new file mode 100644 index 0000000..578b971 --- /dev/null +++ b/src/common/rtc_base/network_constants.h @@ -0,0 +1,72 @@ +/* + * Copyright 2004 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_NETWORK_CONSTANTS_H_ +#define RTC_BASE_NETWORK_CONSTANTS_H_ + +#include + +#include + +namespace rtc { + +constexpr uint16_t kNetworkCostMax = 999; +constexpr uint16_t kNetworkCostCellular2G = 980; +constexpr uint16_t kNetworkCostCellular3G = 910; +constexpr uint16_t kNetworkCostCellular = 900; +constexpr uint16_t kNetworkCostCellular4G = 500; +constexpr uint16_t kNetworkCostCellular5G = 250; +constexpr uint16_t kNetworkCostUnknown = 50; +constexpr uint16_t kNetworkCostLow = 10; +constexpr uint16_t kNetworkCostMin = 0; + +// Add 1 to network cost of underlying network type +// so that e.g a "plain" WIFI is prefered over a VPN over WIFI +// everything else being equal. +constexpr uint16_t kNetworkCostVpn = 1; + +// alias +constexpr uint16_t kNetworkCostHigh = kNetworkCostCellular; + +enum AdapterType { + // This enum resembles the one in Chromium net::ConnectionType. + ADAPTER_TYPE_UNKNOWN = 0, + ADAPTER_TYPE_ETHERNET = 1 << 0, + ADAPTER_TYPE_WIFI = 1 << 1, + ADAPTER_TYPE_CELLULAR = 1 << 2, // This is CELLULAR of unknown type. + ADAPTER_TYPE_VPN = 1 << 3, + ADAPTER_TYPE_LOOPBACK = 1 << 4, + // ADAPTER_TYPE_ANY is used for a network, which only contains a single "any + // address" IP address (INADDR_ANY for IPv4 or in6addr_any for IPv6), and can + // use any/all network interfaces. Whereas ADAPTER_TYPE_UNKNOWN is used + // when the network uses a specific interface/IP, but its interface type can + // not be determined or not fit in this enum. + ADAPTER_TYPE_ANY = 1 << 5, + ADAPTER_TYPE_CELLULAR_2G = 1 << 6, + ADAPTER_TYPE_CELLULAR_3G = 1 << 7, + ADAPTER_TYPE_CELLULAR_4G = 1 << 8, + ADAPTER_TYPE_CELLULAR_5G = 1 << 9 +}; + +std::string AdapterTypeToString(AdapterType type); + +// Useful for testing! +constexpr AdapterType kAllAdapterTypes[] = { + ADAPTER_TYPE_UNKNOWN, ADAPTER_TYPE_ETHERNET, + ADAPTER_TYPE_WIFI, ADAPTER_TYPE_CELLULAR, + ADAPTER_TYPE_VPN, ADAPTER_TYPE_LOOPBACK, + ADAPTER_TYPE_ANY, ADAPTER_TYPE_CELLULAR_2G, + ADAPTER_TYPE_CELLULAR_3G, ADAPTER_TYPE_CELLULAR_4G, + ADAPTER_TYPE_CELLULAR_5G, +}; + +} // namespace rtc + +#endif // RTC_BASE_NETWORK_CONSTANTS_H_ diff --git a/src/common/rtc_base/network_route.cc b/src/common/rtc_base/network_route.cc new file mode 100644 index 0000000..9762dc2 --- /dev/null +++ b/src/common/rtc_base/network_route.cc @@ -0,0 +1,27 @@ +/* + * Copyright 2020 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtc_base/network_route.h" + +namespace rtc { + +bool RouteEndpoint::operator==(const RouteEndpoint& other) const { + return adapter_type_ == other.adapter_type_ && + adapter_id_ == other.adapter_id_ && network_id_ == other.network_id_ && + uses_turn_ == other.uses_turn_; +} + +bool NetworkRoute::operator==(const NetworkRoute& other) const { + return connected == other.connected && local == other.local && + remote == other.remote && packet_overhead == other.packet_overhead && + last_sent_packet_id == other.last_sent_packet_id; +} + +} // namespace rtc diff --git a/src/common/rtc_base/network_route.h b/src/common/rtc_base/network_route.h new file mode 100644 index 0000000..70be84c --- /dev/null +++ b/src/common/rtc_base/network_route.h @@ -0,0 +1,93 @@ +/* + * Copyright 2016 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef RTC_BASE_NETWORK_ROUTE_H_ +#define RTC_BASE_NETWORK_ROUTE_H_ + +#include + +#include +#include + +#include "rtc_base/network_constants.h" + +// TODO(honghaiz): Make a directory that describes the interfaces and structs +// the media code can rely on and the network code can implement, and both can +// depend on that, but not depend on each other. Then, move this file to that +// directory. +namespace rtc { + +class RouteEndpoint { + public: + RouteEndpoint() {} // Used by tests. + RouteEndpoint(AdapterType adapter_type, uint16_t adapter_id, + uint16_t network_id, bool uses_turn) + : adapter_type_(adapter_type), + adapter_id_(adapter_id), + network_id_(network_id), + uses_turn_(uses_turn) {} + + RouteEndpoint(const RouteEndpoint&) = default; + RouteEndpoint& operator=(const RouteEndpoint&) = default; + + // Used by tests. + static RouteEndpoint CreateWithNetworkId(uint16_t network_id) { + return RouteEndpoint(ADAPTER_TYPE_UNKNOWN, + /* adapter_id = */ 0, network_id, + /* uses_turn = */ false); + } + RouteEndpoint CreateWithTurn(bool uses_turn) const { + return RouteEndpoint(adapter_type_, adapter_id_, network_id_, uses_turn); + } + + AdapterType adapter_type() const { return adapter_type_; } + uint16_t adapter_id() const { return adapter_id_; } + uint16_t network_id() const { return network_id_; } + bool uses_turn() const { return uses_turn_; } + + bool operator==(const RouteEndpoint& other) const; + + private: + AdapterType adapter_type_ = ADAPTER_TYPE_UNKNOWN; + uint16_t adapter_id_ = 0; + uint16_t network_id_ = 0; + bool uses_turn_ = false; +}; + +struct NetworkRoute { + bool connected = false; + RouteEndpoint local; + RouteEndpoint remote; + // Last packet id sent on the PREVIOUS route. + int last_sent_packet_id = -1; + // The overhead in bytes from IP layer and above. + // This is the maximum of any part of the route. + int packet_overhead = 0; + + std::string DebugString() const { + std::ostringstream oss; + oss << "[ connected: " << connected << " local: [ " << local.adapter_id() + << "/" << local.network_id() << " " + << AdapterTypeToString(local.adapter_type()) + << " turn: " << local.uses_turn() << " ] remote: [ " + << remote.adapter_id() << "/" << remote.network_id() << " " + << AdapterTypeToString(remote.adapter_type()) + << " turn: " << remote.uses_turn() + << " ] packet_overhead_bytes: " << packet_overhead << " ]"; + return oss.str(); + } + + bool operator==(const NetworkRoute& other) const; + bool operator!=(const NetworkRoute& other) { return !operator==(other); } +}; + +} // namespace rtc + +#endif // RTC_BASE_NETWORK_ROUTE_H_ diff --git a/src/qos/acknowledged_bitrate_estimator.cc b/src/qos/acknowledged_bitrate_estimator.cc index e69b1a9..cf3e113 100644 --- a/src/qos/acknowledged_bitrate_estimator.cc +++ b/src/qos/acknowledged_bitrate_estimator.cc @@ -24,12 +24,15 @@ namespace webrtc { AcknowledgedBitrateEstimator::AcknowledgedBitrateEstimator() - : AcknowledgedBitrateEstimator(std::make_unique()) {} + : in_alr_(false), + bitrate_estimator_(std::make_unique()) {} AcknowledgedBitrateEstimator::AcknowledgedBitrateEstimator( std::unique_ptr bitrate_estimator) : in_alr_(false), bitrate_estimator_(std::move(bitrate_estimator)) {} +AcknowledgedBitrateEstimator::~AcknowledgedBitrateEstimator() {} + void AcknowledgedBitrateEstimator::IncomingPacketFeedbackVector( const std::vector& packet_feedback_vector) { for (const auto& packet : packet_feedback_vector) { diff --git a/src/qos/acknowledged_bitrate_estimator.h b/src/qos/acknowledged_bitrate_estimator.h index 3b6122b..7ec81b1 100644 --- a/src/qos/acknowledged_bitrate_estimator.h +++ b/src/qos/acknowledged_bitrate_estimator.h @@ -15,6 +15,7 @@ #include #include +#include "acknowledged_bitrate_estimator_interface.h" #include "api/transport/network_types.h" #include "api/units/data_rate.h" #include "api/units/timestamp.h" @@ -22,12 +23,13 @@ namespace webrtc { -class AcknowledgedBitrateEstimator { +class AcknowledgedBitrateEstimator + : public AcknowledgedBitrateEstimatorInterface { public: + AcknowledgedBitrateEstimator(); AcknowledgedBitrateEstimator( std::unique_ptr bitrate_estimator); - explicit AcknowledgedBitrateEstimator(); ~AcknowledgedBitrateEstimator(); void IncomingPacketFeedbackVector( diff --git a/src/qos/acknowledged_bitrate_estimator_interface.cc b/src/qos/acknowledged_bitrate_estimator_interface.cc new file mode 100644 index 0000000..802456e --- /dev/null +++ b/src/qos/acknowledged_bitrate_estimator_interface.cc @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "acknowledged_bitrate_estimator_interface.h" + +#include +#include + +#include "acknowledged_bitrate_estimator.h" +#include "api/units/time_delta.h" +#include "log.h" + +namespace webrtc { + +AcknowledgedBitrateEstimatorInterface:: + ~AcknowledgedBitrateEstimatorInterface() {} + +std::unique_ptr +AcknowledgedBitrateEstimatorInterface::Create() { + return std::make_unique(); +} + +} // namespace webrtc diff --git a/src/qos/acknowledged_bitrate_estimator_interface.h b/src/qos/acknowledged_bitrate_estimator_interface.h index c847d7a..dd9eb4e 100644 --- a/src/qos/acknowledged_bitrate_estimator_interface.h +++ b/src/qos/acknowledged_bitrate_estimator_interface.h @@ -22,44 +22,6 @@ namespace webrtc { -struct RobustThroughputEstimatorSettings { - static constexpr char kKey[] = "WebRTC-Bwe-RobustThroughputEstimatorSettings"; - - RobustThroughputEstimatorSettings(); - - // Set `enabled` to true to use the RobustThroughputEstimator, false to use - // the AcknowledgedBitrateEstimator. - bool enabled = true; - - // The estimator keeps the smallest window containing at least - // `window_packets` and at least the packets received during the last - // `min_window_duration` milliseconds. - // (This means that it may store more than `window_packets` at high bitrates, - // and a longer duration than `min_window_duration` at low bitrates.) - // However, if will never store more than kMaxPackets (for performance - // reasons), and never longer than max_window_duration (to avoid very old - // packets influencing the estimate for example when sending is paused). - unsigned window_packets = 20; - unsigned max_window_packets = 500; - TimeDelta min_window_duration = TimeDelta::Seconds(1); - TimeDelta max_window_duration = TimeDelta::Seconds(5); - - // The estimator window requires at least `required_packets` packets - // to produce an estimate. - unsigned required_packets = 10; - - // If audio packets aren't included in allocation (i.e. the - // estimated available bandwidth is divided only among the video - // streams), then `unacked_weight` should be set to 0. - // If audio packets are included in allocation, but not in bandwidth - // estimation (i.e. they don't have transport-wide sequence numbers, - // but we nevertheless divide the estimated available bandwidth among - // both audio and video streams), then `unacked_weight` should be set to 1. - // If all packets have transport-wide sequence numbers, then the value - // of `unacked_weight` doesn't matter. - double unacked_weight = 1.0; -}; - class AcknowledgedBitrateEstimatorInterface { public: static std::unique_ptr Create(); diff --git a/src/qos/clock.cc b/src/qos/clock.cc index 7775790..8b40982 100644 --- a/src/qos/clock.cc +++ b/src/qos/clock.cc @@ -55,47 +55,4 @@ class RealTimeClock : public Clock { return TimeMicrosToNtp(timestamp.us()); } }; - -Clock* Clock::GetRealTimeClock() { - static Clock* const clock = new RealTimeClock(); - return clock; -} - -SimulatedClock::SimulatedClock(int64_t initial_time_us) - : time_us_(initial_time_us) {} - -SimulatedClock::SimulatedClock(Timestamp initial_time) - : SimulatedClock(initial_time.us()) {} - -SimulatedClock::~SimulatedClock() {} - -Timestamp SimulatedClock::CurrentTime() { - return Timestamp::Micros(time_us_.load(std::memory_order_relaxed)); -} - -NtpTime SimulatedClock::ConvertTimestampToNtpTime(Timestamp timestamp) { - int64_t now_us = timestamp.us(); - uint32_t seconds = (now_us / 1'000'000) + kNtpJan1970; - uint32_t fractions = static_cast( - (now_us % 1'000'000) * kMagicNtpFractionalUnit / 1'000'000); - return NtpTime(seconds, fractions); -} - -void SimulatedClock::AdvanceTimeMilliseconds(int64_t milliseconds) { - AdvanceTime(TimeDelta::Millis(milliseconds)); -} - -void SimulatedClock::AdvanceTimeMicroseconds(int64_t microseconds) { - AdvanceTime(TimeDelta::Micros(microseconds)); -} - -// TODO(bugs.webrtc.org(12102): It's desirable to let a single thread own -// advancement of the clock. We could then replace this read-modify-write -// operation with just a thread checker. But currently, that breaks a couple of -// tests, in particular, RepeatingTaskTest.ClockIntegration and -// CallStatsTest.LastProcessedRtt. -void SimulatedClock::AdvanceTime(TimeDelta delta) { - time_us_.fetch_add(delta.us(), std::memory_order_relaxed); -} - -} // namespace webrtc +} // namespace webrtc \ No newline at end of file diff --git a/src/qos/clock.h b/src/qos/clock.h index 9946d65..c22277b 100644 --- a/src/qos/clock.h +++ b/src/qos/clock.h @@ -68,33 +68,6 @@ class Clock { static Clock* GetRealTimeClock(); }; -class SimulatedClock : public Clock { - public: - // The constructors assume an epoch of Jan 1, 1970. - explicit SimulatedClock(int64_t initial_time_us); - explicit SimulatedClock(Timestamp initial_time); - ~SimulatedClock() override; - - // Return a timestamp with an epoch of Jan 1, 1970. - Timestamp CurrentTime() override; - - NtpTime ConvertTimestampToNtpTime(Timestamp timestamp) override; - - // Advance the simulated clock with a given number of milliseconds or - // microseconds. - void AdvanceTimeMilliseconds(int64_t milliseconds); - void AdvanceTimeMicroseconds(int64_t microseconds); - void AdvanceTime(TimeDelta delta); - - private: - // The time is read and incremented with relaxed order. Each thread will see - // monotonically increasing time, and when threads post tasks or messages to - // one another, the synchronization done as part of the message passing should - // ensure that any causual chain of events on multiple threads also - // corresponds to monotonically increasing time. - std::atomic time_us_; -}; - } // namespace webrtc #endif // SYSTEM_WRAPPERS_INCLUDE_CLOCK_H_ diff --git a/src/qos/congestion_control.cpp b/src/qos/congestion_control.cpp index 8c9d93a..bb3f147 100644 --- a/src/qos/congestion_control.cpp +++ b/src/qos/congestion_control.cpp @@ -22,31 +22,24 @@ constexpr float kDefaultPaceMultiplier = 2.5f; constexpr double kProbeDropThroughputFraction = 0.85; CongestionControl::CongestionControl() - : use_min_allocatable_as_lower_bound_(false), + : packet_feedback_only_(true), + use_min_allocatable_as_lower_bound_(false), ignore_probes_lower_than_network_estimate_(false), limit_probes_lower_than_throughput_estimate_(false), pace_at_max_of_bwe_and_lower_link_capacity_(false), limit_pacingfactor_by_upper_link_capacity_estimate_(false), probe_controller_(new ProbeController()), + congestion_window_pushback_controller_( + std::make_unique()), bandwidth_estimation_(new SendSideBandwidthEstimation()), alr_detector_(new AlrDetector()), probe_bitrate_estimator_(new ProbeBitrateEstimator()), - network_estimator_(new NetworkStateEstimator()), - network_state_predictor_(new NetworkStatePredictor()), delay_based_bwe_(new DelayBasedBwe()), - acknowledged_bitrate_estimator_(new AcknowledgedBitrateEstimator()), - initial_config_(std::nullopt), - last_loss_based_target_rate_(*config.constraints.starting_rate), - last_pushback_target_rate_(last_loss_based_target_rate_), - last_stable_target_rate_(last_loss_based_target_rate_), - last_loss_base_state_(LossBasedState::kDelayBasedEstimate), - pacing_factor_(config.stream_based_config.pacing_factor.value_or( - kDefaultPaceMultiplier)), - min_total_allocated_bitrate_( - config.stream_based_config.min_total_allocated_bitrate.value_or( - DataRate::Zero())), - max_padding_rate_(config.stream_based_config.max_padding_rate.value_or( - DataRate::Zero())) + acknowledged_bitrate_estimator_( + AcknowledgedBitrateEstimatorInterface::Create()), + pacing_factor_(kDefaultPaceMultiplier), + min_total_allocated_bitrate_(DataRate::Zero()), + max_padding_rate_(DataRate::Zero()) {} @@ -119,7 +112,8 @@ NetworkControlUpdate CongestionControl::OnTransportPacketsFeedback( lost_packets_since_last_loss_update_ += 1; } if (report.feedback_time > next_loss_update_) { - next_loss_update_ = report.feedback_time + kLossUpdateInterval; + next_loss_update_ = + report.feedback_time + TimeDelta::Millis(kLossUpdateInterval); bandwidth_estimation_->UpdatePacketsLost( lost_packets_since_last_loss_update_, expected_packets_since_last_loss_update_, report.feedback_time); @@ -148,17 +142,8 @@ NetworkControlUpdate CongestionControl::OnTransportPacketsFeedback( } } - if (network_estimator_) { - network_estimator_->OnTransportPacketsFeedback(report); - SetNetworkStateEstimate(network_estimator_->GetCurrentEstimate()); - } std::optional probe_bitrate = probe_bitrate_estimator_->FetchAndResetLastEstimatedBitrate(); - if (ignore_probes_lower_than_network_estimate_ && probe_bitrate && - estimate_ && *probe_bitrate < delay_based_bwe_->last_estimate() && - *probe_bitrate < estimate_->link_capacity_lower) { - probe_bitrate.reset(); - } if (limit_probes_lower_than_throughput_estimate_ && probe_bitrate && acknowledged_bitrate) { // Limit the backoff to something slightly below the acknowledged @@ -179,8 +164,7 @@ NetworkControlUpdate CongestionControl::OnTransportPacketsFeedback( DelayBasedBwe::Result result; result = delay_based_bwe_->IncomingPacketFeedbackVector( - report, acknowledged_bitrate, probe_bitrate, estimate_, - alr_start_time.has_value()); + report, acknowledged_bitrate, probe_bitrate, alr_start_time.has_value()); if (result.updated) { if (result.probe) { @@ -192,35 +176,109 @@ NetworkControlUpdate CongestionControl::OnTransportPacketsFeedback( bandwidth_estimation_->UpdateDelayBasedEstimate(report.feedback_time, result.target_bitrate); } - bandwidth_estimation_->UpdateLossBasedEstimator( - report, result.delay_detector_state, probe_bitrate, - alr_start_time.has_value()); - if (result.updated) { - // Update the estimate in the ProbeController, in case we want to probe. - MaybeTriggerOnNetworkChanged(&update, report.feedback_time); - } + // bandwidth_estimation_->UpdateLossBasedEstimator( + // report, result.delay_detector_state, probe_bitrate, + // alr_start_time.has_value()); + // if (result.updated) { + // // Update the estimate in the ProbeController, in case we want to probe. + // MaybeTriggerOnNetworkChanged(&update, report.feedback_time); + // } - recovered_from_overuse = result.recovered_from_overuse; + // recovered_from_overuse = result.recovered_from_overuse; - if (recovered_from_overuse) { - probe_controller_->SetAlrStartTimeMs(alr_start_time); - auto probes = probe_controller_->RequestProbe(report.feedback_time); - update.probe_cluster_configs.insert(update.probe_cluster_configs.end(), - probes.begin(), probes.end()); - } + // if (recovered_from_overuse) { + // probe_controller_->SetAlrStartTimeMs(alr_start_time); + // auto probes = probe_controller_->RequestProbe(report.feedback_time); + // update.probe_cluster_configs.insert(update.probe_cluster_configs.end(), + // probes.begin(), probes.end()); + // } - // No valid RTT could be because send-side BWE isn't used, in which case - // we don't try to limit the outstanding packets. - if (rate_control_settings_.UseCongestionWindow() && - max_feedback_rtt.IsFinite()) { - UpdateCongestionWindowSize(); - } - if (congestion_window_pushback_controller_ && current_data_window_) { - congestion_window_pushback_controller_->SetDataWindow( - *current_data_window_); - } else { - update.congestion_window = current_data_window_; - } + // // No valid RTT could be because send-side BWE isn't used, in which case + // // we don't try to limit the outstanding packets. + // if (rate_control_settings_.UseCongestionWindow() && + // max_feedback_rtt.IsFinite()) { + // UpdateCongestionWindowSize(); + // } + // if (congestion_window_pushback_controller_ && current_data_window_) { + // congestion_window_pushback_controller_->SetDataWindow( + // *current_data_window_); + // } else { + // update.congestion_window = current_data_window_; + // } return update; +} + +void CongestionControl::MaybeTriggerOnNetworkChanged( + NetworkControlUpdate* update, Timestamp at_time) { + // uint8_t fraction_loss = bandwidth_estimation_->fraction_loss(); + // TimeDelta round_trip_time = bandwidth_estimation_->round_trip_time(); + // DataRate loss_based_target_rate = bandwidth_estimation_->target_rate(); + // LossBasedState loss_based_state = + // bandwidth_estimation_->loss_based_state(); DataRate pushback_target_rate = + // loss_based_target_rate; + + // double cwnd_reduce_ratio = 0.0; + // if (congestion_window_pushback_controller_) { + // int64_t pushback_rate = + // congestion_window_pushback_controller_->UpdateTargetBitrate( + // loss_based_target_rate.bps()); + // pushback_rate = std::max(bandwidth_estimation_->GetMinBitrate(), + // pushback_rate); + // pushback_target_rate = DataRate::BitsPerSec(pushback_rate); + // if (rate_control_settings_.UseCongestionWindowDropFrameOnly()) { + // cwnd_reduce_ratio = static_cast(loss_based_target_rate.bps() - + // pushback_target_rate.bps()) / + // loss_based_target_rate.bps(); + // } + // } + // DataRate stable_target_rate = + // bandwidth_estimation_->GetEstimatedLinkCapacity(); + // stable_target_rate = std::min(stable_target_rate, pushback_target_rate); + + // if ((loss_based_target_rate != last_loss_based_target_rate_) || + // (loss_based_state != last_loss_base_state_) || + // (fraction_loss != last_estimated_fraction_loss_) || + // (round_trip_time != last_estimated_round_trip_time_) || + // (pushback_target_rate != last_pushback_target_rate_) || + // (stable_target_rate != last_stable_target_rate_)) { + // last_loss_based_target_rate_ = loss_based_target_rate; + // last_pushback_target_rate_ = pushback_target_rate; + // last_estimated_fraction_loss_ = fraction_loss; + // last_estimated_round_trip_time_ = round_trip_time; + // last_stable_target_rate_ = stable_target_rate; + // last_loss_base_state_ = loss_based_state; + + // alr_detector_->SetEstimatedBitrate(loss_based_target_rate.bps()); + + // TimeDelta bwe_period = delay_based_bwe_->GetExpectedBwePeriod(); + + // TargetTransferRate target_rate_msg; + // target_rate_msg.at_time = at_time; + // if (rate_control_settings_.UseCongestionWindowDropFrameOnly()) { + // target_rate_msg.target_rate = loss_based_target_rate; + // target_rate_msg.cwnd_reduce_ratio = cwnd_reduce_ratio; + // } else { + // target_rate_msg.target_rate = pushback_target_rate; + // } + // target_rate_msg.stable_target_rate = stable_target_rate; + // target_rate_msg.network_estimate.at_time = at_time; + // target_rate_msg.network_estimate.round_trip_time = round_trip_time; + // target_rate_msg.network_estimate.loss_rate_ratio = fraction_loss / + // 255.0f; target_rate_msg.network_estimate.bwe_period = bwe_period; + + // update->target_rate = target_rate_msg; + + // auto probes = probe_controller_->SetEstimatedBitrate( + // loss_based_target_rate, + // GetBandwidthLimitedCause(bandwidth_estimation_->loss_based_state(), + // bandwidth_estimation_->IsRttAboveLimit(), + // delay_based_bwe_->last_state()), + // at_time); + // update->probe_cluster_configs.insert(update->probe_cluster_configs.end(), + // probes.begin(), probes.end()); + // update->pacer_config = GetPacingRates(at_time); + // LOG_INFO("bwe {} pushback_target_bps={} estimate_bps={}", at_time.ms(), + // last_pushback_target_rate_.bps(), loss_based_target_rate.bps()); + // } } \ No newline at end of file diff --git a/src/qos/congestion_control.h b/src/qos/congestion_control.h index 7431c26..7c783f8 100644 --- a/src/qos/congestion_control.h +++ b/src/qos/congestion_control.h @@ -8,7 +8,6 @@ #include "acknowledged_bitrate_estimator_interface.h" #include "alr_detector.h" #include "api/network_state_predictor.h" -#include "api/transport/network_control.h" #include "api/transport/network_types.h" #include "congestion_window_pushback_controller.h" #include "delay_based_bwe.h" @@ -26,7 +25,11 @@ class CongestionControl { NetworkControlUpdate OnTransportPacketsFeedback( TransportPacketsFeedback report); + void MaybeTriggerOnNetworkChanged(NetworkControlUpdate* update, + Timestamp at_time); + private: + const bool packet_feedback_only_; const bool use_min_allocatable_as_lower_bound_; const bool ignore_probes_lower_than_network_estimate_; const bool limit_probes_lower_than_throughput_estimate_; @@ -34,18 +37,16 @@ class CongestionControl { const bool limit_pacingfactor_by_upper_link_capacity_estimate_; const std::unique_ptr probe_controller_; + const std::unique_ptr + congestion_window_pushback_controller_; std::unique_ptr bandwidth_estimation_; std::unique_ptr alr_detector_; std::unique_ptr probe_bitrate_estimator_; - std::unique_ptr network_estimator_; - std::unique_ptr network_state_predictor_; std::unique_ptr delay_based_bwe_; std::unique_ptr acknowledged_bitrate_estimator_; - std::optional initial_config_; - DataRate min_target_rate_ = DataRate::Zero(); DataRate min_data_rate_ = DataRate::Zero(); DataRate max_data_rate_ = DataRate::PlusInfinity(); @@ -53,18 +54,12 @@ class CongestionControl { bool first_packet_sent_ = false; - std::optional estimate_; - Timestamp next_loss_update_ = Timestamp::MinusInfinity(); int lost_packets_since_last_loss_update_ = 0; int expected_packets_since_last_loss_update_ = 0; std::deque feedback_max_rtts_; - DataRate last_loss_based_target_rate_; - DataRate last_pushback_target_rate_; - DataRate last_stable_target_rate_; - std::optional last_estimated_fraction_loss_ = 0; TimeDelta last_estimated_round_trip_time_ = TimeDelta::PlusInfinity(); diff --git a/src/qos/congestion_control_feedback.cpp b/src/qos/congestion_control_feedback.cpp index db7baa1..1784215 100644 --- a/src/qos/congestion_control_feedback.cpp +++ b/src/qos/congestion_control_feedback.cpp @@ -17,12 +17,12 @@ #include #include "api/array_view.h" +#include "api/transport/ecn_marking.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "byte_io.h" #include "common_header.h" #include "log.h" -#include "rtc_base/network/ecn_marking.h" namespace webrtc { namespace rtcp { @@ -102,31 +102,33 @@ TimeDelta AtoToTimeDelta(uint16_t receive_info) { return TimeDelta::Seconds(ato) / 1024; } -uint16_t To2BitEcn(rtc::EcnMarking ecn_marking) { +uint16_t To2BitEcn(EcnMarking ecn_marking) { switch (ecn_marking) { - case rtc::EcnMarking::kNotEct: + case EcnMarking::kNotEct: return 0; - case rtc::EcnMarking::kEct1: + case EcnMarking::kEct1: return kEcnEct1 << 13; - case rtc::EcnMarking::kEct0: + case EcnMarking::kEct0: return kEcnEct0 << 13; - case rtc::EcnMarking::kCe: + case EcnMarking::kCe: return kEcnCe << 13; + default: + return 0; } } -rtc::EcnMarking ToEcnMarking(uint16_t receive_info) { +EcnMarking ToEcnMarking(uint16_t receive_info) { const uint16_t ecn = (receive_info >> 13) & 0b11; if (ecn == kEcnEct1) { - return rtc::EcnMarking::kEct1; + return EcnMarking::kEct1; } if (ecn == kEcnEct0) { - return rtc::EcnMarking::kEct0; + return EcnMarking::kEct0; } if (ecn == kEcnCe) { - return rtc::EcnMarking::kCe; + return EcnMarking::kCe; } - return rtc::EcnMarking::kNotEct; + return EcnMarking::kNotEct; } } // namespace @@ -146,9 +148,9 @@ bool CongestionControlFeedback::Create(uint8_t* buffer, size_t* position, const size_t position_end = *position + BlockLength(); // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // |V=2|P| FMT=11 | PT = 205 | length | + // |V=2|P| FMT=11 | PT = 205 | length | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // | SSRC of RTCP packet sender | + // | SSRC of RTCP packet sender | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ CreateHeader(kFeedbackMessageType, kPacketType, HeaderLength(), buffer, position); @@ -175,9 +177,9 @@ bool CongestionControlFeedback::Create(uint8_t* buffer, size_t* position, // num_reports uint16_t num_reports = packets.size(); - // Each report block MUST NOT include more than 16384 packet metric - // blocks (i.e., it MUST NOT report on more than one quarter of the - // sequence number space in a single report). + // Each report block MUST NOT include more than 16384 packet + // metric blocks (i.e., it MUST NOT report on more than one + // quarter of the sequence number space in a single report). if (num_reports > 16384) { LOG_ERROR("Unexpected number of reports:{}", num_reports); return; diff --git a/src/qos/congestion_control_feedback.h b/src/qos/congestion_control_feedback.h index 641b62e..20b316d 100644 --- a/src/qos/congestion_control_feedback.h +++ b/src/qos/congestion_control_feedback.h @@ -15,9 +15,9 @@ #include #include "api/array_view.h" +#include "api/transport/ecn_marking.h" #include "api/units/time_delta.h" #include "common_header.h" -#include "rtc_base/network/ecn_marking.h" #include "rtp_feedback.h" namespace webrtc { @@ -33,7 +33,7 @@ class CongestionControlFeedback : public RtpFeedback { // Time offset from report timestamp. Minus infinity if the packet has not // been received. TimeDelta arrival_time_offset = TimeDelta::MinusInfinity(); - rtc::EcnMarking ecn = rtc::EcnMarking::kNotEct; + EcnMarking ecn = EcnMarking::kNotEct; }; static constexpr uint8_t kFeedbackMessageType = 11; diff --git a/src/qos/congestion_control_feedback_generator.cc b/src/qos/congestion_control_feedback_generator.cc index e6c601a..3942c18 100644 --- a/src/qos/congestion_control_feedback_generator.cc +++ b/src/qos/congestion_control_feedback_generator.cc @@ -30,7 +30,7 @@ namespace webrtc { CongestionControlFeedbackGenerator::CongestionControlFeedbackGenerator( - std::shared_ptr clock, RtcpSender rtcp_sender) + std::shared_ptr clock, RtcpSender rtcp_sender) : clock_(clock), rtcp_sender_(std::move(rtcp_sender)), min_time_between_feedback_(TimeDelta::Millis(25)), @@ -89,8 +89,7 @@ void CongestionControlFeedbackGenerator::SetTransportOverhead( } void CongestionControlFeedbackGenerator::SendFeedback(Timestamp now) { - SimulatedClock clock(now); - uint32_t compact_ntp = CompactNtp(clock.ConvertTimestampToNtpTime(now)); + uint32_t compact_ntp = CompactNtp(clock_->ConvertTimestampToNtpTime(now)); std::vector rtcp_packet_info; for (auto& [unused, tracker] : feedback_trackers_) { tracker.AddPacketsToFeedback(now, rtcp_packet_info); diff --git a/src/qos/congestion_control_feedback_generator.h b/src/qos/congestion_control_feedback_generator.h index 2e87377..cfd637d 100644 --- a/src/qos/congestion_control_feedback_generator.h +++ b/src/qos/congestion_control_feedback_generator.h @@ -43,7 +43,7 @@ class CongestionControlFeedbackGenerator : public RtpTransportFeedbackGenerator { public: CongestionControlFeedbackGenerator( - std::shared_ptr clock, + std::shared_ptr clock, RtpTransportFeedbackGenerator::RtcpSender feedback_sender); ~CongestionControlFeedbackGenerator() = default; @@ -62,7 +62,7 @@ class CongestionControlFeedbackGenerator void CalculateNextPossibleSendTime(DataSize feedback_size, Timestamp now); - std::shared_ptr clock_; + std::shared_ptr clock_; const RtcpSender rtcp_sender_; TimeDelta min_time_between_feedback_; diff --git a/src/qos/congestion_control_feedback_tracker.cc b/src/qos/congestion_control_feedback_tracker.cc index d3735f7..37c80eb 100644 --- a/src/qos/congestion_control_feedback_tracker.cc +++ b/src/qos/congestion_control_feedback_tracker.cc @@ -66,7 +66,7 @@ void CongestionControlFeedbackTracker::AddPacketsToFeedback( for (int64_t sequence_number = *last_sequence_number_in_feedback_ + 1; sequence_number <= packets_.back().unwrapped_sequence_number; ++sequence_number) { - rtc::EcnMarking ecn = rtc::EcnMarking::kNotEct; + EcnMarking ecn = EcnMarking::kNotEct; TimeDelta arrival_time_offset = TimeDelta::MinusInfinity(); if (sequence_number == packet_it->unwrapped_sequence_number) { @@ -81,8 +81,8 @@ void CongestionControlFeedbackTracker::AddPacketsToFeedback( // any of the copies of the duplicated packet are ECN-CE marked, then // an ECN-CE mark MUST be reported for that packet; otherwise, the ECN // mark of the first copy to arrive is reported. - if (packet_it->ecn == rtc::EcnMarking::kCe) { - ecn = rtc::EcnMarking::kCe; + if (packet_it->ecn == EcnMarking::kCe) { + ecn = EcnMarking::kCe; } LOG_WARN("Received duplicate packet ssrc:{} seq:{} ecn:{}", ssrc, static_cast(sequence_number), static_cast(ecn)); diff --git a/src/qos/congestion_control_feedback_tracker.h b/src/qos/congestion_control_feedback_tracker.h index 3f95584..6d2e215 100644 --- a/src/qos/congestion_control_feedback_tracker.h +++ b/src/qos/congestion_control_feedback_tracker.h @@ -14,9 +14,9 @@ #include #include +#include "api/transport/ecn_marking.h" #include "api/units/timestamp.h" #include "congestion_control_feedback.h" -#include "rtc_base/network/ecn_marking.h" #include "rtc_base/numerics/sequence_number_unwrapper.h" #include "rtp_packet_received.h" namespace webrtc { @@ -44,7 +44,7 @@ class CongestionControlFeedbackTracker { uint32_t ssrc; int64_t unwrapped_sequence_number = 0; Timestamp arrival_time; - rtc::EcnMarking ecn = rtc::EcnMarking::kNotEct; + EcnMarking ecn = EcnMarking::kNotEct; }; std::optional last_sequence_number_in_feedback_; diff --git a/src/qos/module_common_types_public.h b/src/qos/module_common_types_public.h new file mode 100644 index 0000000..289b5d0 --- /dev/null +++ b/src/qos/module_common_types_public.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_INCLUDE_MODULE_COMMON_TYPES_PUBLIC_H_ +#define MODULES_INCLUDE_MODULE_COMMON_TYPES_PUBLIC_H_ + +#include +#include + +namespace webrtc { + +template +inline bool IsNewer(U value, U prev_value) { + static_assert(!std::numeric_limits::is_signed, "U must be unsigned"); + // kBreakpoint is the half-way mark for the type U. For instance, for a + // uint16_t it will be 0x8000, and for a uint32_t, it will be 0x8000000. + constexpr U kBreakpoint = (std::numeric_limits::max() >> 1) + 1; + // Distinguish between elements that are exactly kBreakpoint apart. + // If t1>t2 and |t1-t2| = kBreakpoint: IsNewer(t1,t2)=true, + // IsNewer(t2,t1)=false + // rather than having IsNewer(t1,t2) = IsNewer(t2,t1) = false. + if (value - prev_value == kBreakpoint) { + return value > prev_value; + } + return value != prev_value && + static_cast(value - prev_value) < kBreakpoint; +} + +// NB: Doesn't fulfill strict weak ordering requirements. +// Mustn't be used as std::map Compare function. +inline bool IsNewerSequenceNumber(uint16_t sequence_number, + uint16_t prev_sequence_number) { + return IsNewer(sequence_number, prev_sequence_number); +} + +// NB: Doesn't fulfill strict weak ordering requirements. +// Mustn't be used as std::map Compare function. +inline bool IsNewerTimestamp(uint32_t timestamp, uint32_t prev_timestamp) { + return IsNewer(timestamp, prev_timestamp); +} + +inline uint16_t LatestSequenceNumber(uint16_t sequence_number1, + uint16_t sequence_number2) { + return IsNewerSequenceNumber(sequence_number1, sequence_number2) + ? sequence_number1 + : sequence_number2; +} + +inline uint32_t LatestTimestamp(uint32_t timestamp1, uint32_t timestamp2) { + return IsNewerTimestamp(timestamp1, timestamp2) ? timestamp1 : timestamp2; +} + +} // namespace webrtc +#endif // MODULES_INCLUDE_MODULE_COMMON_TYPES_PUBLIC_H_ diff --git a/src/qos/probe_controller.cc b/src/qos/probe_controller.cc index 10408fc..0e43a5e 100644 --- a/src/qos/probe_controller.cc +++ b/src/qos/probe_controller.cc @@ -228,8 +228,10 @@ std::vector ProbeController::InitiateExponentialProbing( max_total_allocated_bitrate_.IsZero()) { last_allowed_repeated_initial_probe_ = at_time + config_.repeated_initial_probing_time_period; - LOG_INFO("Repeated initial probing enabled, last allowed probe: {} now: {}", - last_allowed_repeated_initial_probe_, at_time); + // LOG_INFO("Repeated initial probing enabled, last allowed probe: {} now: + // {}", + // ToString(last_allowed_repeated_initial_probe_), + // ToString(at_time)); } return InitiateProbing(at_time, probes, true); @@ -262,10 +264,10 @@ std::vector ProbeController::SetEstimatedBitrate( ? network_estimate_->link_capacity_upper * config_.further_probe_threshold : DataRate::PlusInfinity(); - LOG_INFO( - "Measured bitrate: {} Minimum to probe further: {} upper limit: {}", - bitrate, min_bitrate_to_probe_further_, - network_state_estimate_probe_further_limit); + // LOG_INFO( + // "Measured bitrate: {} Minimum to probe further: {} upper limit: {}", + // bitrate, ToString(min_bitrate_to_probe_further_), + // ToString(network_state_estimate_probe_further_limit)); if (bitrate > min_bitrate_to_probe_further_ && bitrate <= network_state_estimate_probe_further_limit) { diff --git a/src/qos/receive_side_congestion_controller.cc b/src/qos/receive_side_congestion_controller.cc index 6a14db0..dadef66 100644 --- a/src/qos/receive_side_congestion_controller.cc +++ b/src/qos/receive_side_congestion_controller.cc @@ -16,13 +16,13 @@ #include #include "api/media_types.h" -#include "api/transport/network_control.h" #include "api/units/data_rate.h" #include "api/units/data_size.h" #include "api/units/time_delta.h" #include "api/units/timestamp.h" #include "log.h" -#include "remote_bitrate_estimator_single_stream.h" +// #include "remote_bitrate_estimator_single_stream.h" +#include "remote_bitrate_estimator_abs_send_time.h" #include "rtp_packet_received.h" namespace webrtc { @@ -56,21 +56,24 @@ void ReceiveSideCongestionController::PickEstimator() { "WrappingBitrateEstimator: Switching to transmission " "time offset RBE."); using_absolute_send_time_ = false; - rbe_ = std::make_unique( + // rbe_ = std::make_unique( + // clock_, &remb_throttler_); + rbe_ = std::make_unique( clock_, &remb_throttler_); } } } ReceiveSideCongestionController::ReceiveSideCongestionController( - std::shared_ptr clock, + std::shared_ptr clock, RtpTransportFeedbackGenerator::RtcpSender feedback_sender, - RembThrottler::RembSender remb_sender, - std::shared_ptr network_state_estimator) + RembThrottler::RembSender remb_sender) : clock_(clock), remb_throttler_(std::move(remb_sender), clock.get()), congestion_control_feedback_generator_(clock, feedback_sender), - rbe_(std::make_unique( + // rbe_(std::make_unique( + // clock, &remb_throttler_)), + rbe_(std::make_unique( clock, &remb_throttler_)), using_absolute_send_time_(false), packets_since_absolute_send_time_(0) {} diff --git a/src/qos/receive_side_congestion_controller.h b/src/qos/receive_side_congestion_controller.h index 5e7997e..715a86e 100644 --- a/src/qos/receive_side_congestion_controller.h +++ b/src/qos/receive_side_congestion_controller.h @@ -16,7 +16,6 @@ #include #include "api/media_types.h" -#include "api/transport/network_control.h" #include "api/units/data_rate.h" #include "api/units/data_size.h" #include "api/units/time_delta.h" @@ -37,10 +36,9 @@ class RemoteBitrateEstimator; class ReceiveSideCongestionController : public CallStatsObserver { public: ReceiveSideCongestionController( - std::shared_ptr clock, + std::shared_ptr clock, RtpTransportFeedbackGenerator::RtcpSender feedback_sender, - RembThrottler::RembSender remb_sender, - std::shared_ptr network_state_estimator); + RembThrottler::RembSender remb_sender); ~ReceiveSideCongestionController() override = default; @@ -73,7 +71,7 @@ class ReceiveSideCongestionController : public CallStatsObserver { private: void PickEstimator(); - std::shared_ptr clock_; + std::shared_ptr clock_; RembThrottler remb_throttler_; CongestionControlFeedbackGenerator congestion_control_feedback_generator_; diff --git a/src/qos/remote_bitrate_estimator_abs_send_time.cc b/src/qos/remote_bitrate_estimator_abs_send_time.cc new file mode 100644 index 0000000..d4cf635 --- /dev/null +++ b/src/qos/remote_bitrate_estimator_abs_send_time.cc @@ -0,0 +1,354 @@ +/* + * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "remote_bitrate_estimator_abs_send_time.h" + +#include + +#include +#include +#include + +#include "api/units/data_rate.h" +#include "api/units/data_size.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "bwe_defines.h" +#include "log.h" +#include "remote_bitrate_estimator.h" +#include "rtp_packet_received.h" + +namespace webrtc { +namespace { + +constexpr TimeDelta kMinClusterDelta = TimeDelta::Millis(1); +constexpr TimeDelta kInitialProbingInterval = TimeDelta::Seconds(2); +constexpr int kTimestampGroupLengthMs = 5; +constexpr int kAbsSendTimeInterArrivalUpshift = 8; +static constexpr int kAbsSendTimeFraction = 18; +constexpr int kInterArrivalShift = + kAbsSendTimeFraction + kAbsSendTimeInterArrivalUpshift; +constexpr int kMinClusterSize = 4; +constexpr int kMaxProbePackets = 15; +constexpr int kExpectedNumberOfProbes = 3; +constexpr double kTimestampToMs = + 1000.0 / static_cast(1 << kInterArrivalShift); + +template +std::vector Keys(const std::map& map) { + std::vector keys; + keys.reserve(map.size()); + for (const auto& kv_pair : map) { + keys.push_back(kv_pair.first); + } + return keys; +} + +} // namespace + +RemoteBitrateEstimatorAbsSendTime::~RemoteBitrateEstimatorAbsSendTime() = + default; + +bool RemoteBitrateEstimatorAbsSendTime::IsWithinClusterBounds( + TimeDelta send_delta, const Cluster& cluster_aggregate) { + if (cluster_aggregate.count == 0) return true; + TimeDelta cluster_mean = + cluster_aggregate.send_mean / cluster_aggregate.count; + return (send_delta - cluster_mean).Abs() < TimeDelta::Micros(2'500); +} + +void RemoteBitrateEstimatorAbsSendTime::MaybeAddCluster( + const Cluster& cluster_aggregate, std::list& clusters) { + if (cluster_aggregate.count < kMinClusterSize || + cluster_aggregate.send_mean <= TimeDelta::Zero() || + cluster_aggregate.recv_mean <= TimeDelta::Zero()) { + return; + } + + Cluster cluster; + cluster.send_mean = cluster_aggregate.send_mean / cluster_aggregate.count; + cluster.recv_mean = cluster_aggregate.recv_mean / cluster_aggregate.count; + cluster.mean_size = cluster_aggregate.mean_size / cluster_aggregate.count; + cluster.count = cluster_aggregate.count; + cluster.num_above_min_delta = cluster_aggregate.num_above_min_delta; + clusters.push_back(cluster); +} + +RemoteBitrateEstimatorAbsSendTime::RemoteBitrateEstimatorAbsSendTime( + std::shared_ptr clock, RemoteBitrateObserver* observer) + : clock_(clock), observer_(observer), remote_rate_() { + LOG_INFO("RemoteBitrateEstimatorAbsSendTime: Instantiating."); +} + +std::list +RemoteBitrateEstimatorAbsSendTime::ComputeClusters() const { + std::list clusters; + Cluster cluster_aggregate; + Timestamp prev_send_time = Timestamp::MinusInfinity(); + Timestamp prev_recv_time = Timestamp::MinusInfinity(); + for (const Probe& probe : probes_) { + if (prev_send_time.IsFinite()) { + TimeDelta send_delta = probe.send_time - prev_send_time; + TimeDelta recv_delta = probe.recv_time - prev_recv_time; + if (send_delta >= kMinClusterDelta && recv_delta >= kMinClusterDelta) { + ++cluster_aggregate.num_above_min_delta; + } + if (!IsWithinClusterBounds(send_delta, cluster_aggregate)) { + MaybeAddCluster(cluster_aggregate, clusters); + cluster_aggregate = Cluster(); + } + cluster_aggregate.send_mean += send_delta; + cluster_aggregate.recv_mean += recv_delta; + cluster_aggregate.mean_size += probe.payload_size; + ++cluster_aggregate.count; + } + prev_send_time = probe.send_time; + prev_recv_time = probe.recv_time; + } + MaybeAddCluster(cluster_aggregate, clusters); + return clusters; +} + +const RemoteBitrateEstimatorAbsSendTime::Cluster* +RemoteBitrateEstimatorAbsSendTime::FindBestProbe( + const std::list& clusters) const { + DataRate highest_probe_bitrate = DataRate::Zero(); + const Cluster* best = nullptr; + for (const auto& cluster : clusters) { + if (cluster.send_mean == TimeDelta::Zero() || + cluster.recv_mean == TimeDelta::Zero()) { + continue; + } + if (cluster.num_above_min_delta > cluster.count / 2 && + (cluster.recv_mean - cluster.send_mean <= TimeDelta::Millis(2) && + cluster.send_mean - cluster.recv_mean <= TimeDelta::Millis(5))) { + DataRate probe_bitrate = + std::min(cluster.SendBitrate(), cluster.RecvBitrate()); + if (probe_bitrate > highest_probe_bitrate) { + highest_probe_bitrate = probe_bitrate; + best = &cluster; + } + } else { + LOG_INFO( + "Probe failed, sent at {} bps, received at {} bps. Mean send delta: " + "{} ms, mean recv delta: {} ms, num probes: {}", + cluster.SendBitrate().bps(), cluster.RecvBitrate().bps(), + cluster.send_mean.ms(), cluster.recv_mean.ms(), cluster.count); + break; + } + } + return best; +} + +RemoteBitrateEstimatorAbsSendTime::ProbeResult +RemoteBitrateEstimatorAbsSendTime::ProcessClusters(Timestamp now) { + std::list clusters = ComputeClusters(); + if (clusters.empty()) { + // If we reach the max number of probe packets and still have no clusters, + // we will remove the oldest one. + if (probes_.size() >= kMaxProbePackets) probes_.pop_front(); + return ProbeResult::kNoUpdate; + } + + if (const Cluster* best = FindBestProbe(clusters)) { + DataRate probe_bitrate = std::min(best->SendBitrate(), best->RecvBitrate()); + // Make sure that a probe sent on a lower bitrate than our estimate can't + // reduce the estimate. + if (IsBitrateImproving(probe_bitrate)) { + LOG_INFO( + "Probe successful, sent at {} bps, received at {} bps. Mean send " + "delta: {} ms, mean recv delta: {} ms, num probes: {}", + best->SendBitrate().bps(), best->RecvBitrate().bps(), + best->send_mean.ms(), best->recv_mean.ms(), best->count); + remote_rate_.SetEstimate(probe_bitrate, now); + return ProbeResult::kBitrateUpdated; + } + } + + // Not probing and received non-probe packet, or finished with current set + // of probes. + if (clusters.size() >= kExpectedNumberOfProbes) probes_.clear(); + return ProbeResult::kNoUpdate; +} + +bool RemoteBitrateEstimatorAbsSendTime::IsBitrateImproving( + DataRate probe_bitrate) const { + bool initial_probe = + !remote_rate_.ValidEstimate() && probe_bitrate > DataRate::Zero(); + bool bitrate_above_estimate = remote_rate_.ValidEstimate() && + probe_bitrate > remote_rate_.LatestEstimate(); + return initial_probe || bitrate_above_estimate; +} + +void RemoteBitrateEstimatorAbsSendTime::IncomingPacket( + const RtpPacketReceived& rtp_packet) { + uint32_t send_time_24bits; + if (!rtp_packet.GetAbsoluteSendTimestamp(&send_time_24bits)) { + LOG_WARN( + "RemoteBitrateEstimatorAbsSendTimeImpl: Incoming packet is missing " + "absolute send time extension!"); + return; + } + + Timestamp arrival_time = rtp_packet.arrival_time(); + DataSize payload_size = + DataSize::Bytes(rtp_packet.payload_size() + rtp_packet.padding_size()); + + if (!uma_recorded_) { + uma_recorded_ = true; + } + // Shift up send time to use the full 32 bits that inter_arrival works with, + // so wrapping works properly. + uint32_t timestamp = send_time_24bits << kAbsSendTimeInterArrivalUpshift; + Timestamp send_time = + Timestamp::Millis(static_cast(timestamp) * kTimestampToMs); + + Timestamp now = clock_->CurrentTime(); + // TODO(holmer): SSRCs are only needed for REMB, should be broken out from + // here. + + // Check if incoming bitrate estimate is valid, and if it needs to be reset. + std::optional incoming_bitrate = + incoming_bitrate_.Rate(arrival_time); + if (incoming_bitrate) { + incoming_bitrate_initialized_ = true; + } else if (incoming_bitrate_initialized_) { + // Incoming bitrate had a previous valid value, but now not enough data + // point are left within the current window. Reset incoming bitrate + // estimator so that the window size will only contain new data points. + incoming_bitrate_.Reset(); + incoming_bitrate_initialized_ = false; + } + incoming_bitrate_.Update(payload_size, arrival_time); + + if (first_packet_time_.IsInfinite()) { + first_packet_time_ = now; + } + + uint32_t ts_delta = 0; + int64_t t_delta = 0; + int size_delta = 0; + bool update_estimate = false; + DataRate target_bitrate = DataRate::Zero(); + + TimeoutStreams(now); + ssrcs_.insert_or_assign(rtp_packet.Ssrc(), now); + + // For now only try to detect probes while we don't have a valid estimate. + // We currently assume that only packets larger than 200 bytes are paced by + // the sender. + static constexpr DataSize kMinProbePacketSize = DataSize::Bytes(200); + if (payload_size > kMinProbePacketSize && + (!remote_rate_.ValidEstimate() || + now - first_packet_time_ < kInitialProbingInterval)) { + // TODO(holmer): Use a map instead to get correct order? + if (total_probes_received_ < kMaxProbePackets) { + TimeDelta send_delta = TimeDelta::Millis(-1); + TimeDelta recv_delta = TimeDelta::Millis(-1); + if (!probes_.empty()) { + send_delta = send_time - probes_.back().send_time; + recv_delta = arrival_time - probes_.back().recv_time; + } + LOG_INFO( + "Probe packet received: send time={} ms, recv time={} ms, send " + "delta={} ms, recv delta= {} ms.", + send_time.ms(), arrival_time.ms(), send_delta.ms(), recv_delta.ms()); + } + probes_.emplace_back(send_time, arrival_time, payload_size); + ++total_probes_received_; + // Make sure that a probe which updated the bitrate immediately has an + // effect by calling the OnReceiveBitrateChanged callback. + if (ProcessClusters(now) == ProbeResult::kBitrateUpdated) + update_estimate = true; + } + if (inter_arrival_->ComputeDeltas(timestamp, arrival_time.ms(), now.ms(), + payload_size.bytes(), &ts_delta, &t_delta, + &size_delta)) { + double ts_delta_ms = (1000.0 * ts_delta) / (1 << kInterArrivalShift); + estimator_->Update(t_delta, ts_delta_ms, size_delta, detector_.State(), + arrival_time.ms()); + detector_.Detect(estimator_->offset(), ts_delta_ms, + estimator_->num_of_deltas(), arrival_time.ms()); + } + + if (!update_estimate) { + // Check if it's time for a periodic update or if we should update because + // of an over-use. + if (last_update_.IsInfinite() || + now.ms() - last_update_.ms() > + remote_rate_.GetFeedbackInterval().ms()) { + update_estimate = true; + } else if (detector_.State() == BandwidthUsage::kBwOverusing) { + std::optional incoming_rate = + incoming_bitrate_.Rate(arrival_time); + if (incoming_rate.has_value() && + remote_rate_.TimeToReduceFurther(now, *incoming_rate)) { + update_estimate = true; + } + } + } + + if (update_estimate) { + // The first overuse should immediately trigger a new estimate. + // We also have to update the estimate immediately if we are overusing + // and the target bitrate is too high compared to what we are receiving. + const RateControlInput input(detector_.State(), + incoming_bitrate_.Rate(arrival_time)); + target_bitrate = remote_rate_.Update(input, now); + update_estimate = remote_rate_.ValidEstimate(); + } + + if (update_estimate) { + last_update_ = now; + observer_->OnReceiveBitrateChanged(Keys(ssrcs_), + target_bitrate.bps()); + } +} + +TimeDelta RemoteBitrateEstimatorAbsSendTime::Process() { + return TimeDelta::PlusInfinity(); +} + +void RemoteBitrateEstimatorAbsSendTime::TimeoutStreams(Timestamp now) { + for (auto it = ssrcs_.begin(); it != ssrcs_.end();) { + if (now - it->second > kStreamTimeOut) { + ssrcs_.erase(it++); + } else { + ++it; + } + } + if (ssrcs_.empty()) { + // We can't update the estimate if we don't have any active streams. + inter_arrival_ = std::make_unique( + (kTimestampGroupLengthMs << kInterArrivalShift) / 1000, kTimestampToMs); + estimator_ = std::make_unique(); + // We deliberately don't reset the first_packet_time_ms_ here for now since + // we only probe for bandwidth in the beginning of a call right now. + } +} + +void RemoteBitrateEstimatorAbsSendTime::OnRttUpdate(int64_t avg_rtt_ms, + int64_t /*max_rtt_ms*/) { + remote_rate_.SetRtt(TimeDelta::Millis(avg_rtt_ms)); +} + +void RemoteBitrateEstimatorAbsSendTime::RemoveStream(uint32_t ssrc) { + ssrcs_.erase(ssrc); +} + +DataRate RemoteBitrateEstimatorAbsSendTime::LatestEstimate() const { + // Currently accessed only from the worker thread (see Call::GetStats()). + if (!remote_rate_.ValidEstimate() || ssrcs_.empty()) { + return DataRate::Zero(); + } + return remote_rate_.LatestEstimate(); +} + +} // namespace webrtc diff --git a/src/qos/remote_bitrate_estimator_abs_send_time.h b/src/qos/remote_bitrate_estimator_abs_send_time.h new file mode 100644 index 0000000..6f3adcf --- /dev/null +++ b/src/qos/remote_bitrate_estimator_abs_send_time.h @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_REMOTE_BITRATE_ESTIMATOR_REMOTE_BITRATE_ESTIMATOR_ABS_SEND_TIME_H_ +#define MODULES_REMOTE_BITRATE_ESTIMATOR_REMOTE_BITRATE_ESTIMATOR_ABS_SEND_TIME_H_ + +#include +#include + +#include +#include +#include +#include + +#include "aimd_rate_control.h" +#include "api/units/data_rate.h" +#include "api/units/data_size.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "clock.h" +#include "inter_arrival.h" +#include "overuse_detector.h" +#include "overuse_estimator.h" +#include "remote_bitrate_estimator.h" +#include "rtc_base/bitrate_tracker.h" + +namespace webrtc { + +class RemoteBitrateEstimatorAbsSendTime : public RemoteBitrateEstimator { + public: + RemoteBitrateEstimatorAbsSendTime(std::shared_ptr clock, + RemoteBitrateObserver* observer); + + RemoteBitrateEstimatorAbsSendTime() = delete; + RemoteBitrateEstimatorAbsSendTime(const RemoteBitrateEstimatorAbsSendTime&) = + delete; + RemoteBitrateEstimatorAbsSendTime& operator=( + const RemoteBitrateEstimatorAbsSendTime&) = delete; + + ~RemoteBitrateEstimatorAbsSendTime() override; + + void IncomingPacket(const RtpPacketReceived& rtp_packet) override; + TimeDelta Process() override; + void OnRttUpdate(int64_t avg_rtt_ms, int64_t max_rtt_ms) override; + void RemoveStream(uint32_t ssrc) override; + DataRate LatestEstimate() const override; + + private: + struct Probe { + Probe(Timestamp send_time, Timestamp recv_time, DataSize payload_size) + : send_time(send_time), + recv_time(recv_time), + payload_size(payload_size) {} + + Timestamp send_time; + Timestamp recv_time; + DataSize payload_size; + }; + + struct Cluster { + DataRate SendBitrate() const { return mean_size / send_mean; } + DataRate RecvBitrate() const { return mean_size / recv_mean; } + + TimeDelta send_mean = TimeDelta::Zero(); + TimeDelta recv_mean = TimeDelta::Zero(); + // TODO(holmer): Add some variance metric as well? + DataSize mean_size = DataSize::Zero(); + int count = 0; + int num_above_min_delta = 0; + }; + + enum class ProbeResult { kBitrateUpdated, kNoUpdate }; + + static bool IsWithinClusterBounds(TimeDelta send_delta, + const Cluster& cluster_aggregate); + + static void MaybeAddCluster(const Cluster& cluster_aggregate, + std::list& clusters); + + std::list ComputeClusters() const; + + const Cluster* FindBestProbe(const std::list& clusters) const; + + // Returns true if a probe which changed the estimate was detected. + ProbeResult ProcessClusters(Timestamp now); + + bool IsBitrateImproving(DataRate probe_bitrate) const; + + void TimeoutStreams(Timestamp now); + + RemoteBitrateObserver* observer_; + std::unique_ptr inter_arrival_; + std::unique_ptr estimator_; + OveruseDetector detector_; + BitrateTracker incoming_bitrate_{kBitrateWindow}; + bool incoming_bitrate_initialized_ = false; + std::list probes_; + size_t total_probes_received_ = 0; + Timestamp first_packet_time_ = Timestamp::MinusInfinity(); + Timestamp last_update_ = Timestamp::MinusInfinity(); + bool uma_recorded_ = false; + + std::map ssrcs_; + AimdRateControl remote_rate_; + + std::shared_ptr clock_; +}; + +} // namespace webrtc + +#endif // MODULES_REMOTE_BITRATE_ESTIMATOR_REMOTE_BITRATE_ESTIMATOR_ABS_SEND_TIME_H_ diff --git a/src/qos/remote_bitrate_estimator_single_stream.cc b/src/qos/remote_bitrate_estimator_single_stream.ccxx similarity index 98% rename from src/qos/remote_bitrate_estimator_single_stream.cc rename to src/qos/remote_bitrate_estimator_single_stream.ccxx index e0c2620..9c3006f 100644 --- a/src/qos/remote_bitrate_estimator_single_stream.cc +++ b/src/qos/remote_bitrate_estimator_single_stream.ccxx @@ -36,7 +36,7 @@ RemoteBitrateEstimatorSingleStream::Detector::Detector() inter_arrival(90 * kTimestampGroupLengthMs, kTimestampToMs) {} RemoteBitrateEstimatorSingleStream::RemoteBitrateEstimatorSingleStream( - std::shared_ptr clock, RemoteBitrateObserver* observer) + std::shared_ptr clock, RemoteBitrateObserver* observer) : clock_(clock), observer_(observer), incoming_bitrate_(kBitrateWindow), diff --git a/src/qos/remote_bitrate_estimator_single_stream.h b/src/qos/remote_bitrate_estimator_single_stream.hxx similarity index 95% rename from src/qos/remote_bitrate_estimator_single_stream.h rename to src/qos/remote_bitrate_estimator_single_stream.hxx index c22b0f3..bf617b3 100644 --- a/src/qos/remote_bitrate_estimator_single_stream.h +++ b/src/qos/remote_bitrate_estimator_single_stream.hxx @@ -32,7 +32,7 @@ namespace webrtc { class RemoteBitrateEstimatorSingleStream : public RemoteBitrateEstimator { public: - RemoteBitrateEstimatorSingleStream(std::shared_ptr clock, + RemoteBitrateEstimatorSingleStream(std::shared_ptr clock, RemoteBitrateObserver* observer); RemoteBitrateEstimatorSingleStream() = delete; @@ -64,7 +64,7 @@ class RemoteBitrateEstimatorSingleStream : public RemoteBitrateEstimator { std::vector GetSsrcs() const; - std::shared_ptr clock_; + std::shared_ptr clock_; RemoteBitrateObserver* observer_; std::map overuse_detectors_; BitrateTracker incoming_bitrate_; diff --git a/src/qos/report_block.cc b/src/qos/report_block.cc new file mode 100644 index 0000000..0f7e463 --- /dev/null +++ b/src/qos/report_block.cc @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "report_block.h" + +#include "byte_io.h" +#include "log.h" + +namespace webrtc { +namespace rtcp { + +// From RFC 3550, RTP: A Transport Protocol for Real-Time Applications. +// +// RTCP report block (RFC 3550). +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ +// 0 | SSRC_1 (SSRC of first source) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 4 | fraction lost | cumulative number of packets lost | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 8 | extended highest sequence number received | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 12 | interarrival jitter | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 16 | last SR (LSR) | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 20 | delay since last SR (DLSR) | +// 24 +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ +ReportBlock::ReportBlock() + : source_ssrc_(0), + fraction_lost_(0), + cumulative_lost_(0), + extended_high_seq_num_(0), + jitter_(0), + last_sr_(0), + delay_since_last_sr_(0) {} + +bool ReportBlock::Parse(const uint8_t* buffer, size_t length) { + if (length < ReportBlock::kLength) { + LOG_ERROR("Report Block should be 24 bytes long"); + return false; + } + + source_ssrc_ = ByteReader::ReadBigEndian(&buffer[0]); + fraction_lost_ = buffer[4]; + cumulative_lost_ = ByteReader::ReadBigEndian(&buffer[5]); + extended_high_seq_num_ = ByteReader::ReadBigEndian(&buffer[8]); + jitter_ = ByteReader::ReadBigEndian(&buffer[12]); + last_sr_ = ByteReader::ReadBigEndian(&buffer[16]); + delay_since_last_sr_ = ByteReader::ReadBigEndian(&buffer[20]); + + return true; +} + +void ReportBlock::Create(uint8_t* buffer) const { + // Runtime check should be done while setting cumulative_lost. + + ByteWriter::WriteBigEndian(&buffer[0], source_ssrc()); + ByteWriter::WriteBigEndian(&buffer[4], fraction_lost()); + ByteWriter::WriteBigEndian(&buffer[5], cumulative_lost()); + ByteWriter::WriteBigEndian(&buffer[8], extended_high_seq_num()); + ByteWriter::WriteBigEndian(&buffer[12], jitter()); + ByteWriter::WriteBigEndian(&buffer[16], last_sr()); + ByteWriter::WriteBigEndian(&buffer[20], delay_since_last_sr()); +} + +bool ReportBlock::SetCumulativeLost(int32_t cumulative_lost) { + // We have only 3 bytes to store it, and it's a signed value. + if (cumulative_lost >= (1 << 23) || cumulative_lost < -(1 << 23)) { + LOG_WARN("Cumulative lost is too big to fit into Report Block"); + return false; + } + cumulative_lost_ = cumulative_lost; + return true; +} + +} // namespace rtcp +} // namespace webrtc diff --git a/src/qos/report_block.h b/src/qos/report_block.h new file mode 100644 index 0000000..9c94411 --- /dev/null +++ b/src/qos/report_block.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_REPORT_BLOCK_H_ +#define MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_REPORT_BLOCK_H_ + +#include +#include + +namespace webrtc { +namespace rtcp { + +// A ReportBlock represents the Sender Report packet from +// RFC 3550 section 6.4.1. +class ReportBlock { + public: + static constexpr size_t kLength = 24; + + ReportBlock(); + ~ReportBlock() {} + + bool Parse(const uint8_t* buffer, size_t length); + + // Fills buffer with the ReportBlock. + // Consumes ReportBlock::kLength bytes. + void Create(uint8_t* buffer) const; + + void SetMediaSsrc(uint32_t ssrc) { source_ssrc_ = ssrc; } + void SetFractionLost(uint8_t fraction_lost) { + fraction_lost_ = fraction_lost; + } + bool SetCumulativeLost(int32_t cumulative_lost); + void SetExtHighestSeqNum(uint32_t ext_highest_seq_num) { + extended_high_seq_num_ = ext_highest_seq_num; + } + void SetJitter(uint32_t jitter) { jitter_ = jitter; } + void SetLastSr(uint32_t last_sr) { last_sr_ = last_sr; } + void SetDelayLastSr(uint32_t delay_last_sr) { + delay_since_last_sr_ = delay_last_sr; + } + + uint32_t source_ssrc() const { return source_ssrc_; } + uint8_t fraction_lost() const { return fraction_lost_; } + int32_t cumulative_lost() const { return cumulative_lost_; } + uint32_t extended_high_seq_num() const { return extended_high_seq_num_; } + uint32_t jitter() const { return jitter_; } + uint32_t last_sr() const { return last_sr_; } + uint32_t delay_since_last_sr() const { return delay_since_last_sr_; } + + private: + uint32_t source_ssrc_; // 32 bits + uint8_t fraction_lost_; // 8 bits representing a fixed point value 0..1 + int32_t cumulative_lost_; // Signed 24-bit value + uint32_t extended_high_seq_num_; // 32 bits + uint32_t jitter_; // 32 bits + uint32_t last_sr_; // 32 bits + uint32_t delay_since_last_sr_; // 32 bits, units of 1/65536 seconds +}; + +} // namespace rtcp +} // namespace webrtc +#endif // MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_REPORT_BLOCK_H_ diff --git a/src/qos/report_block_data.cc b/src/qos/report_block_data.cc new file mode 100644 index 0000000..df64ce5 --- /dev/null +++ b/src/qos/report_block_data.cc @@ -0,0 +1,43 @@ +/* + * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "report_block_data.h" + +namespace webrtc { + +TimeDelta ReportBlockData::jitter(int rtp_clock_rate_hz) const { + // Conversion to TimeDelta and division are swapped to avoid conversion + // to/from floating point types. + return TimeDelta::Seconds(jitter()) / rtp_clock_rate_hz; +} + +// TODO: bugs.webrtc.org/370535296 - Remove the utc timestamp when linked +// issue is fixed. +void ReportBlockData::SetReportBlock(uint32_t sender_ssrc, + const rtcp::ReportBlock& report_block, + Timestamp report_block_timestamp_utc, + Timestamp report_block_timestamp) { + sender_ssrc_ = sender_ssrc; + source_ssrc_ = report_block.source_ssrc(); + fraction_lost_raw_ = report_block.fraction_lost(); + cumulative_lost_ = report_block.cumulative_lost(); + extended_highest_sequence_number_ = report_block.extended_high_seq_num(); + jitter_ = report_block.jitter(); + report_block_timestamp_utc_ = report_block_timestamp_utc; + report_block_timestamp_ = report_block_timestamp; +} + +void ReportBlockData::AddRoundTripTimeSample(TimeDelta rtt) { + last_rtt_ = rtt; + sum_rtt_ += rtt; + ++num_rtts_; +} + +} // namespace webrtc diff --git a/src/qos/report_block_data.h b/src/qos/report_block_data.h new file mode 100644 index 0000000..fdb6da3 --- /dev/null +++ b/src/qos/report_block_data.h @@ -0,0 +1,139 @@ +/* + * Copyright 2019 The WebRTC Project Authors. All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_RTP_RTCP_INCLUDE_REPORT_BLOCK_DATA_H_ +#define MODULES_RTP_RTCP_INCLUDE_REPORT_BLOCK_DATA_H_ + +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "report_block.h" + +namespace webrtc { + +// Represents fields and derived information received in RTCP report block +// attached to RTCP sender report or RTCP receiver report, as described in +// https://www.rfc-editor.org/rfc/rfc3550#section-6.4.1 +class ReportBlockData { + public: + ReportBlockData() = default; + + ReportBlockData(const ReportBlockData&) = default; + ReportBlockData& operator=(const ReportBlockData&) = default; + + // The SSRC identifier for the originator of this report block, + // i.e. remote receiver of the RTP stream. + uint32_t sender_ssrc() const { return sender_ssrc_; } + + // The SSRC identifier of the source to which the information in this + // reception report block pertains, i.e. local sender of the RTP stream. + uint32_t source_ssrc() const { return source_ssrc_; } + + // The fraction of RTP data packets from 'source_ssrc()' lost since the + // previous report block was sent. + // Fraction loss in range [0.0, 1.0]. + float fraction_lost() const { + return static_cast(fraction_lost_raw()) / 256.0f; + } + + // Fraction loss as was written in the raw packet: range is [0, 255] where 0 + // represents no loss, and 255 represents 99.6% loss (255/256 * 100%). + uint8_t fraction_lost_raw() const { return fraction_lost_raw_; } + + // The total number of RTP data packets from 'source_ssrc()' that have been + // lost since the beginning of reception. This number is defined to be the + // number of packets expected less the number of packets actually received, + // where the number of packets received includes any which are late or + // duplicates. Thus, packets that arrive late are not counted as lost, and the + // loss may be negative if there are duplicates. + int cumulative_lost() const { return cumulative_lost_; } + + // The low 16 bits contain the highest sequence number received in an RTP data + // packet from 'source_ssrc()', and the most significant 16 bits extend that + // sequence number with the corresponding count of sequence number cycles. + uint32_t extended_highest_sequence_number() const { + return extended_highest_sequence_number_; + } + + // An estimate of the statistical variance of the RTP data packet interarrival + // time, measured in RTP timestamp units. The interarrival jitter J is defined + // to be the mean deviation (smoothed absolute value) of the difference D in + // packet spacing at the receiver compared to the sender for a pair of + // packets. + uint32_t jitter() const { return jitter_; } + + // Jitter converted to common time units. + TimeDelta jitter(int rtp_clock_rate_hz) const; + + // Time in utc epoch (Jan 1st, 1970) the report block was received. + // TODO: bugs.webrtc.org/370535296 - Remove the utc timestamp when linked + // issue is fixed. + Timestamp report_block_timestamp_utc() const { + return report_block_timestamp_utc_; + } + + // Monotonic time when the report block was received. + Timestamp report_block_timestamp() const { return report_block_timestamp_; } + + // Round Trip Time measurments for given (sender_ssrc, source_ssrc) pair. + // Min, max, sum, number of measurements are since beginning of the call. + TimeDelta last_rtt() const { return last_rtt_; } + TimeDelta sum_rtts() const { return sum_rtt_; } + size_t num_rtts() const { return num_rtts_; } + bool has_rtt() const { return num_rtts_ != 0; } + + void set_sender_ssrc(uint32_t ssrc) { sender_ssrc_ = ssrc; } + void set_source_ssrc(uint32_t ssrc) { source_ssrc_ = ssrc; } + void set_fraction_lost_raw(uint8_t lost) { fraction_lost_raw_ = lost; } + void set_cumulative_lost(int lost) { cumulative_lost_ = lost; } + void set_extended_highest_sequence_number(uint32_t sn) { + extended_highest_sequence_number_ = sn; + } + void set_jitter(uint32_t jitter) { jitter_ = jitter; } + // TODO: bugs.webrtc.org/370535296 - Remove the utc timestamp when linked + // issue is fixed. + void set_report_block_timestamp_utc(Timestamp arrival_time) { + report_block_timestamp_utc_ = arrival_time; + } + void set_report_block_timestamp(Timestamp arrival_time) { + report_block_timestamp_ = arrival_time; + } + + void SetReportBlock(uint32_t sender_ssrc, + const rtcp::ReportBlock& report_block, + Timestamp report_block_timestamp_utc, + Timestamp report_block_timestamp); + void AddRoundTripTimeSample(TimeDelta rtt); + + private: + uint32_t sender_ssrc_ = 0; + uint32_t source_ssrc_ = 0; + uint8_t fraction_lost_raw_ = 0; + int32_t cumulative_lost_ = 0; + uint32_t extended_highest_sequence_number_ = 0; + uint32_t jitter_ = 0; + // TODO: bugs.webrtc.org/370535296 - Remove the utc timestamp when linked + // issue is fixed. + Timestamp report_block_timestamp_utc_ = Timestamp::Zero(); + Timestamp report_block_timestamp_ = Timestamp::Zero(); + TimeDelta last_rtt_ = TimeDelta::Zero(); + TimeDelta sum_rtt_ = TimeDelta::Zero(); + size_t num_rtts_ = 0; +}; + +class ReportBlockDataObserver { + public: + virtual ~ReportBlockDataObserver() = default; + + virtual void OnReportBlockDataUpdated(ReportBlockData report_block_data) = 0; +}; + +} // namespace webrtc + +#endif // MODULES_RTP_RTCP_INCLUDE_REPORT_BLOCK_DATA_H_ diff --git a/src/qos/rtp_packet_to_send.cc b/src/qos/rtp_packet_to_send.cc new file mode 100644 index 0000000..03f25a2 --- /dev/null +++ b/src/qos/rtp_packet_to_send.cc @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtp_packet_to_send.h" + +#include + +namespace webrtc { + +RtpPacketToSend::RtpPacketToSend() {} +RtpPacketToSend::RtpPacketToSend(size_t capacity) : RtpPacket(capacity) {} +RtpPacketToSend::RtpPacketToSend(const RtpPacketToSend& packet) = default; +RtpPacketToSend::RtpPacketToSend(RtpPacketToSend&& packet) = default; + +RtpPacketToSend& RtpPacketToSend::operator=(const RtpPacketToSend& packet) = + default; +RtpPacketToSend& RtpPacketToSend::operator=(RtpPacketToSend&& packet) = default; + +RtpPacketToSend::~RtpPacketToSend() = default; + +void RtpPacketToSend::set_packet_type(RtpPacketMediaType type) { + if (packet_type_ == RtpPacketMediaType::kAudio) { + original_packet_type_ = OriginalType::kAudio; + } else if (packet_type_ == RtpPacketMediaType::kVideo) { + original_packet_type_ = OriginalType::kVideo; + } + packet_type_ = type; +} + +} // namespace webrtc diff --git a/src/qos/rtp_packet_to_send.h b/src/qos/rtp_packet_to_send.h new file mode 100644 index 0000000..e3ed3e3 --- /dev/null +++ b/src/qos/rtp_packet_to_send.h @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef MODULES_RTP_RTCP_SOURCE_RTP_PACKET_TO_SEND_H_ +#define MODULES_RTP_RTCP_SOURCE_RTP_PACKET_TO_SEND_H_ + +#include +#include + +#include +#include + +#include "api/array_view.h" +#include "api/ref_counted_base.h" +#include "api/scoped_refptr.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "api/video/video_timing.h" +#include "rtp_packet.h" +#include "rtp_rtcp_defines.h" + +// Forward declare the RtpPacket class since it is not in the webrtc namespace. +class RtpPacket; + +namespace webrtc { +// Class to hold rtp packet with metadata for sender side. +// The metadata is not send over the wire, but packet sender may use it to +// create rtp header extensions or other data that is sent over the wire. +class RtpPacketToSend : public ::RtpPacket { + public: + // RtpPacketToSend::Type is deprecated. Use RtpPacketMediaType directly. + using Type = RtpPacketMediaType; + + explicit RtpPacketToSend(); + RtpPacketToSend(size_t capacity); + RtpPacketToSend(const RtpPacketToSend& packet); + RtpPacketToSend(RtpPacketToSend&& packet); + + RtpPacketToSend& operator=(const RtpPacketToSend& packet); + RtpPacketToSend& operator=(RtpPacketToSend&& packet); + + ~RtpPacketToSend(); + + // Time in local time base as close as it can to frame capture time. + webrtc::Timestamp capture_time() const { return capture_time_; } + void set_capture_time(webrtc::Timestamp time) { capture_time_ = time; } + + void set_packet_type(RtpPacketMediaType type); + + std::optional packet_type() const { return packet_type_; } + + enum class OriginalType { kAudio, kVideo }; + // Original type does not change if packet type is changed to kRetransmission. + std::optional original_packet_type() const { + return original_packet_type_; + } + + // If this is a retransmission, indicates the sequence number of the original + // media packet that this packet represents. If RTX is used this will likely + // be different from SequenceNumber(). + void set_retransmitted_sequence_number(uint16_t sequence_number) { + retransmitted_sequence_number_ = sequence_number; + } + std::optional retransmitted_sequence_number() const { + return retransmitted_sequence_number_; + } + + // If this is a retransmission, indicates the SSRC of the original + // media packet that this packet represents. + void set_original_ssrc(uint32_t ssrc) { original_ssrc_ = ssrc; } + std::optional original_ssrc() const { return original_ssrc_; } + + void set_allow_retransmission(bool allow_retransmission) { + allow_retransmission_ = allow_retransmission; + } + bool allow_retransmission() const { return allow_retransmission_; } + + // An application can attach arbitrary data to an RTP packet using + // `additional_data`. The additional data does not affect WebRTC processing. + rtc::scoped_refptr additional_data() const { + return additional_data_; + } + void set_additional_data(rtc::scoped_refptr data) { + additional_data_ = std::move(data); + } + + void set_packetization_finish_time(webrtc::Timestamp time) { + // SetExtension( + // VideoSendTiming::GetDeltaCappedMs(time - capture_time_), + // VideoTimingExtension::kPacketizationFinishDeltaOffset); + } + + void set_pacer_exit_time(webrtc::Timestamp time) { + // SetExtension( + // VideoSendTiming::GetDeltaCappedMs(time - capture_time_), + // VideoTimingExtension::kPacerExitDeltaOffset); + } + + void set_network_time(webrtc::Timestamp time) { + // SetExtension( + // VideoSendTiming::GetDeltaCappedMs(time - capture_time_), + // VideoTimingExtension::kNetworkTimestampDeltaOffset); + } + + void set_network2_time(webrtc::Timestamp time) { + // SetExtension( + // VideoSendTiming::GetDeltaCappedMs(time - capture_time_), + // VideoTimingExtension::kNetwork2TimestampDeltaOffset); + } + + // Indicates if packet is the first packet of a video frame. + void set_first_packet_of_frame(bool is_first_packet) { + is_first_packet_of_frame_ = is_first_packet; + } + bool is_first_packet_of_frame() const { return is_first_packet_of_frame_; } + + // Indicates if packet contains payload for a video key-frame. + void set_is_key_frame(bool is_key_frame) { is_key_frame_ = is_key_frame; } + bool is_key_frame() const { return is_key_frame_; } + + // Indicates if packets should be protected by FEC (Forward Error Correction). + void set_fec_protect_packet(bool protect) { fec_protect_packet_ = protect; } + bool fec_protect_packet() const { return fec_protect_packet_; } + + // Indicates if packet is using RED encapsulation, in accordance with + // https://tools.ietf.org/html/rfc2198 + void set_is_red(bool is_red) { is_red_ = is_red; } + bool is_red() const { return is_red_; } + + // The amount of time spent in the send queue, used for totalPacketSendDelay. + // https://w3c.github.io/webrtc-stats/#dom-rtcoutboundrtpstreamstats-totalpacketsenddelay + void set_time_in_send_queue(TimeDelta time_in_send_queue) { + time_in_send_queue_ = time_in_send_queue; + } + std::optional time_in_send_queue() const { + return time_in_send_queue_; + } + // A sequence number guaranteed to be monotically increasing by one for all + // packets where transport feedback is expected. + std::optional transport_sequence_number() const { + return transport_sequence_number_; + } + void set_transport_sequence_number(int64_t transport_sequence_number) { + transport_sequence_number_ = transport_sequence_number; + } + // Transport is capable of handling explicit congestion notification and the + // RTP packet should be sent as ect(1) + // https://www.rfc-editor.org/rfc/rfc9331.html + bool send_as_ect1() const { return send_as_ect1_; } + void set_send_as_ect1() { send_as_ect1_ = true; } + + private: + webrtc::Timestamp capture_time_ = webrtc::Timestamp::Zero(); + std::optional packet_type_; + std::optional original_packet_type_; + std::optional original_ssrc_; + std::optional transport_sequence_number_; + bool allow_retransmission_ = false; + std::optional retransmitted_sequence_number_; + rtc::scoped_refptr additional_data_; + bool is_first_packet_of_frame_ = false; + bool is_key_frame_ = false; + bool fec_protect_packet_ = false; + bool is_red_ = false; + bool send_as_ect1_ = false; + std::optional time_in_send_queue_; +}; + +} // namespace webrtc +#endif // MODULES_RTP_RTCP_SOURCE_RTP_PACKET_TO_SEND_H_ diff --git a/src/qos/rtp_rtcp_defines.cc b/src/qos/rtp_rtcp_defines.cc new file mode 100644 index 0000000..d812e88 --- /dev/null +++ b/src/qos/rtp_rtcp_defines.cc @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtp_rtcp_defines.h" + +#include + +#include + +#include "api/array_view.h" +#include "rtp_packet.h" +#include "rtp_packet_to_send.h" + +namespace webrtc { + +namespace { +constexpr size_t kMidRsidMaxSize = 16; + +// Check if passed character is a "token-char" from RFC 4566. +// https://datatracker.ietf.org/doc/html/rfc4566#section-9 +// token-char = %x21 / %x23-27 / %x2A-2B / %x2D-2E / %x30-39 +// / %x41-5A / %x5E-7E +bool IsTokenChar(char ch) { + return ch == 0x21 || (ch >= 0x23 && ch <= 0x27) || ch == 0x2a || ch == 0x2b || + ch == 0x2d || ch == 0x2e || (ch >= 0x30 && ch <= 0x39) || + (ch >= 0x41 && ch <= 0x5a) || (ch >= 0x5e && ch <= 0x7e); +} +} // namespace + +StreamDataCounters::StreamDataCounters() = default; + +RtpPacketCounter::RtpPacketCounter(const ::RtpPacket& packet) + : header_bytes(packet.headers_size()), + payload_bytes(packet.payload_size()), + padding_bytes(packet.padding_size()), + packets(1) {} + +RtpPacketCounter::RtpPacketCounter(const RtpPacketToSend& packet_to_send) + : RtpPacketCounter(static_cast(packet_to_send)) { + total_packet_delay = + packet_to_send.time_in_send_queue().value_or(TimeDelta::Zero()); +} + +void RtpPacketCounter::AddPacket(const ::RtpPacket& packet) { + ++packets; + header_bytes += packet.headers_size(); + padding_bytes += packet.padding_size(); + payload_bytes += packet.payload_size(); +} + +void RtpPacketCounter::AddPacket(const RtpPacketToSend& packet_to_send) { + AddPacket(static_cast(packet_to_send)); + total_packet_delay += + packet_to_send.time_in_send_queue().value_or(TimeDelta::Zero()); +} + +} // namespace webrtc diff --git a/src/qos/rtp_rtcp_defines.h b/src/qos/rtp_rtcp_defines.h new file mode 100644 index 0000000..4bc7225 --- /dev/null +++ b/src/qos/rtp_rtcp_defines.h @@ -0,0 +1,417 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_RTP_RTCP_INCLUDE_RTP_RTCP_DEFINES_H_ +#define MODULES_RTP_RTCP_INCLUDE_RTP_RTCP_DEFINES_H_ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "api/array_view.h" +#include "api/transport/network_types.h" +#include "api/units/data_rate.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "congestion_control_feedback.h" +#include "report_block_data.h" + +#define RTCP_CNAME_SIZE 256 // RFC 3550 page 44, including null termination +#define IP_PACKET_SIZE 1500 // we assume ethernet + +class RtpPacket; + +namespace webrtc { +class RtpPacketToSend; +namespace rtcp { +class TransportFeedback; +} + +const int kVideoPayloadTypeFrequency = 90000; + +// TODO(bugs.webrtc.org/6458): Remove this when all the depending projects are +// updated to correctly set rtp rate for RtcpSender. +const int kBogusRtpRateForAudioRtcp = 8000; + +// Minimum RTP header size in bytes. +const uint8_t kRtpHeaderSize = 12; + +// This enum must not have any gaps, i.e., all integers between +// kRtpExtensionNone and kRtpExtensionNumberOfExtensions must be valid enum +// entries. +enum RTPExtensionType : int { + kRtpExtensionNone, + kRtpExtensionTransmissionTimeOffset, + kRtpExtensionAudioLevel, + kRtpExtensionCsrcAudioLevel, + kRtpExtensionInbandComfortNoise, + kRtpExtensionAbsoluteSendTime, + kRtpExtensionAbsoluteCaptureTime, + kRtpExtensionVideoRotation, + kRtpExtensionTransportSequenceNumber, + kRtpExtensionTransportSequenceNumber02, + kRtpExtensionPlayoutDelay, + kRtpExtensionVideoContentType, + kRtpExtensionVideoLayersAllocation, + kRtpExtensionVideoTiming, + kRtpExtensionRtpStreamId, + kRtpExtensionRepairedRtpStreamId, + kRtpExtensionMid, + kRtpExtensionGenericFrameDescriptor, + kRtpExtensionGenericFrameDescriptor00 [[deprecated]] = + kRtpExtensionGenericFrameDescriptor, + kRtpExtensionDependencyDescriptor, + kRtpExtensionGenericFrameDescriptor02 [[deprecated]] = + kRtpExtensionDependencyDescriptor, + kRtpExtensionColorSpace, + kRtpExtensionVideoFrameTrackingId, + kRtpExtensionCorruptionDetection, + kRtpExtensionNumberOfExtensions // Must be the last entity in the enum. +}; + +enum RTCPAppSubTypes { kAppSubtypeBwe = 0x00 }; + +// TODO(sprang): Make this an enum class once rtcp_receiver has been cleaned up. +enum RTCPPacketType : uint32_t { + kRtcpReport = 0x0001, + kRtcpSr = 0x0002, + kRtcpRr = 0x0004, + kRtcpSdes = 0x0008, + kRtcpBye = 0x0010, + kRtcpPli = 0x0020, + kRtcpNack = 0x0040, + kRtcpFir = 0x0080, + kRtcpTmmbr = 0x0100, + kRtcpTmmbn = 0x0200, + kRtcpSrReq = 0x0400, + kRtcpLossNotification = 0x2000, + kRtcpRemb = 0x10000, + kRtcpTransmissionTimeOffset = 0x20000, + kRtcpXrReceiverReferenceTime = 0x40000, + kRtcpXrDlrrReportBlock = 0x80000, + kRtcpTransportFeedback = 0x100000, + kRtcpXrTargetBitrate = 0x200000, +}; + +enum class KeyFrameReqMethod : uint8_t { + kNone, // Don't request keyframes. + kPliRtcp, // Request keyframes through Picture Loss Indication. + kFirRtcp // Request keyframes through Full Intra-frame Request. +}; + +enum RtxMode { + kRtxOff = 0x0, + kRtxRetransmitted = 0x1, // Only send retransmissions over RTX. + kRtxRedundantPayloads = 0x2 // Preventively send redundant payloads + // instead of padding. +}; + +const size_t kRtxHeaderSize = 2; + +struct RtpState { + uint16_t sequence_number = 0; + uint32_t start_timestamp = 0; + uint32_t timestamp = 0; + Timestamp capture_time = Timestamp::MinusInfinity(); + Timestamp last_timestamp_time = Timestamp::MinusInfinity(); + bool ssrc_has_acked = false; +}; + +class RtcpIntraFrameObserver { + public: + virtual ~RtcpIntraFrameObserver() {} + + virtual void OnReceivedIntraFrameRequest(uint32_t ssrc) = 0; +}; + +// Observer for incoming LossNotification RTCP messages. +// See the documentation of LossNotification for details. +class RtcpLossNotificationObserver { + public: + virtual ~RtcpLossNotificationObserver() = default; + + virtual void OnReceivedLossNotification(uint32_t ssrc, + uint16_t seq_num_of_last_decodable, + uint16_t seq_num_of_last_received, + bool decodability_flag) = 0; +}; + +// Interface to watch incoming rtcp packets related to the link in general. +// All message handlers have default empty implementation. This way users only +// need to implement the ones they are interested in. +// All message handles pass `receive_time` parameter, which is receive time +// of the rtcp packet that triggered the update. +class NetworkLinkRtcpObserver { + public: + virtual ~NetworkLinkRtcpObserver() = default; + + virtual void OnTransportFeedback( + Timestamp /* receive_time */, + const rtcp::TransportFeedback& /* feedback */) {} + // RFC 8888 congestion control feedback. + virtual void OnCongestionControlFeedback( + Timestamp /* receive_time */, + const rtcp::CongestionControlFeedback& /* feedback */) {} + virtual void OnReceiverEstimatedMaxBitrate(Timestamp /* receive_time */, + DataRate /* bitrate */) {} + + // Called on an RTCP packet with sender or receiver reports with non zero + // report blocks. Report blocks are combined from all reports into one array. + virtual void OnReport( + Timestamp /* receive_time */, + rtc::ArrayView /* report_blocks */) {} + virtual void OnRttUpdate(Timestamp /* receive_time */, TimeDelta /* rtt */) {} +}; + +// NOTE! `kNumMediaTypes` must be kept in sync with RtpPacketMediaType! +static constexpr size_t kNumMediaTypes = 5; +enum class RtpPacketMediaType : size_t { + kAudio, // Audio media packets. + kVideo, // Video media packets. + kRetransmission, // Retransmisions, sent as response to NACK. + kForwardErrorCorrection, // FEC packets. + kPadding = kNumMediaTypes - 1, // RTX or plain padding sent to maintain BWE. + // Again, don't forget to update `kNumMediaTypes` if you add another value! +}; + +struct RtpPacketSendInfo { + static RtpPacketSendInfo From(const RtpPacketToSend& rtp_packet_to_send, + const PacedPacketInfo& pacing_info); + + uint16_t transport_sequence_number = 0; + std::optional media_ssrc; + uint16_t rtp_sequence_number = 0; // Only valid if `media_ssrc` is set. + uint32_t rtp_timestamp = 0; + size_t length = 0; + std::optional packet_type; + PacedPacketInfo pacing_info; +}; + +class NetworkStateEstimateObserver { + public: + virtual void OnRemoteNetworkEstimate(NetworkStateEstimate estimate) = 0; + virtual ~NetworkStateEstimateObserver() = default; +}; + +class TransportFeedbackObserver { + public: + virtual ~TransportFeedbackObserver() = default; + + virtual void OnAddPacket(const RtpPacketSendInfo& packet_info) = 0; +}; + +// Interface for PacketRouter to send rtcp feedback on behalf of +// congestion controller. +// TODO(bugs.webrtc.org/8239): Remove and use RtcpTransceiver directly +// when RtcpTransceiver always present in rtp transport. +class RtcpFeedbackSenderInterface { + public: + virtual ~RtcpFeedbackSenderInterface() = default; + virtual void SendCombinedRtcpPacket( + std::vector> rtcp_packets) = 0; + virtual void SetRemb(int64_t bitrate_bps, std::vector ssrcs) = 0; + virtual void UnsetRemb() = 0; +}; + +class StreamFeedbackObserver { + public: + struct StreamPacketInfo { + bool received; + + // `rtp_sequence_number` and `is_retransmission` are only valid if `ssrc` + // is populated. + std::optional ssrc; + uint16_t rtp_sequence_number; + bool is_retransmission; + }; + virtual ~StreamFeedbackObserver() = default; + + virtual void OnPacketFeedbackVector( + std::vector packet_feedback_vector) = 0; +}; + +class StreamFeedbackProvider { + public: + virtual void RegisterStreamFeedbackObserver( + std::vector ssrcs, StreamFeedbackObserver* observer) = 0; + virtual void DeRegisterStreamFeedbackObserver( + StreamFeedbackObserver* observer) = 0; + virtual ~StreamFeedbackProvider() = default; +}; + +class RtcpRttStats { + public: + virtual void OnRttUpdate(int64_t rtt) = 0; + + virtual int64_t LastProcessedRtt() const = 0; + + virtual ~RtcpRttStats() {} +}; + +struct RtpPacketCounter { + RtpPacketCounter() + : header_bytes(0), payload_bytes(0), padding_bytes(0), packets(0) {} + + explicit RtpPacketCounter(const ::RtpPacket& packet); + explicit RtpPacketCounter(const RtpPacketToSend& packet_to_send); + + void Add(const RtpPacketCounter& other) { + header_bytes += other.header_bytes; + payload_bytes += other.payload_bytes; + padding_bytes += other.padding_bytes; + packets += other.packets; + total_packet_delay += other.total_packet_delay; + } + + bool operator==(const RtpPacketCounter& other) const { + return header_bytes == other.header_bytes && + payload_bytes == other.payload_bytes && + padding_bytes == other.padding_bytes && packets == other.packets && + total_packet_delay == other.total_packet_delay; + } + + // Not inlined, since use of ::RtpPacket would result in circular includes. + void AddPacket(const ::RtpPacket& packet); + void AddPacket(const RtpPacketToSend& packet_to_send); + + size_t TotalBytes() const { + return header_bytes + payload_bytes + padding_bytes; + } + + size_t header_bytes; // Number of bytes used by RTP headers. + size_t payload_bytes; // Payload bytes, excluding RTP headers and padding. + size_t padding_bytes; // Number of padding bytes. + size_t packets; // Number of packets. + // The total delay of all `packets`. For RtpPacketToSend packets, this is + // `time_in_send_queue()`. For receive packets, this is zero. + webrtc::TimeDelta total_packet_delay = webrtc::TimeDelta::Zero(); +}; + +// Data usage statistics for a (rtp) stream. +struct StreamDataCounters { + StreamDataCounters(); + + void Add(const StreamDataCounters& other) { + transmitted.Add(other.transmitted); + retransmitted.Add(other.retransmitted); + fec.Add(other.fec); + if (other.first_packet_time < first_packet_time) { + // Use oldest time (excluding unsed value represented as plus infinity. + first_packet_time = other.first_packet_time; + } + } + + void MaybeSetFirstPacketTime(Timestamp now) { + if (first_packet_time == Timestamp::PlusInfinity()) { + first_packet_time = now; + } + } + + // Return time since first packet is send/received, or zero if such event + // haven't happen. + TimeDelta TimeSinceFirstPacket(Timestamp now) const { + return first_packet_time == Timestamp::PlusInfinity() + ? TimeDelta::Zero() + : now - first_packet_time; + } + + // Returns the number of bytes corresponding to the actual media payload (i.e. + // RTP headers, padding, retransmissions and fec packets are excluded). + // Note this function does not have meaning for an RTX stream. + size_t MediaPayloadBytes() const { + return transmitted.payload_bytes - retransmitted.payload_bytes - + fec.payload_bytes; + } + + // Time when first packet is sent/received. + Timestamp first_packet_time = Timestamp::PlusInfinity(); + + RtpPacketCounter transmitted; // Number of transmitted packets/bytes. + RtpPacketCounter retransmitted; // Number of retransmitted packets/bytes. + RtpPacketCounter fec; // Number of redundancy packets/bytes. +}; + +class RtpSendRates { + public: + constexpr RtpSendRates() = default; + RtpSendRates(const RtpSendRates& rhs) = default; + RtpSendRates& operator=(const RtpSendRates&) = default; + + DataRate& operator[](RtpPacketMediaType type) { + return send_rates_[static_cast(type)]; + } + const DataRate& operator[](RtpPacketMediaType type) const { + return send_rates_[static_cast(type)]; + } + DataRate Sum() const { + return std::accumulate(send_rates_.begin(), send_rates_.end(), + DataRate::Zero()); + } + + private: + std::array send_rates_; +}; + +// Callback, called whenever byte/packet counts have been updated. +class StreamDataCountersCallback { + public: + virtual ~StreamDataCountersCallback() {} + + virtual void DataCountersUpdated(const StreamDataCounters& counters, + uint32_t ssrc) = 0; +}; + +// Information exposed through the GetStats api. +struct RtpReceiveStats { + // `packets_lost` and `jitter` are defined by RFC 3550, and exposed in the + // RTCReceivedRtpStreamStats dictionary, see + // https://w3c.github.io/webrtc-stats/#receivedrtpstats-dict* + int32_t packets_lost = 0; + // Interarrival jitter in samples. + uint32_t jitter = 0; + // Interarrival jitter in time. + webrtc::TimeDelta interarrival_jitter = webrtc::TimeDelta::Zero(); + + // Time of the last packet received in unix epoch, + // i.e. Timestamp::Zero() represents 1st Jan 1970 00:00 + std::optional last_packet_received; + + // Counters exposed in RTCInboundRtpStreamStats, see + // https://w3c.github.io/webrtc-stats/#inboundrtpstats-dict* + RtpPacketCounter packet_counter; +}; + +// Callback, used to notify an observer whenever new rates have been estimated. +class BitrateStatisticsObserver { + public: + virtual ~BitrateStatisticsObserver() {} + + virtual void Notify(uint32_t total_bitrate_bps, + uint32_t retransmit_bitrate_bps, uint32_t ssrc) = 0; +}; + +// Callback, used to notify an observer whenever a packet is sent to the +// transport. +class SendPacketObserver { + public: + virtual ~SendPacketObserver() = default; + virtual void OnSendPacket(std::optional packet_id, + Timestamp capture_time, uint32_t ssrc) = 0; +}; + +} // namespace webrtc +#endif // MODULES_RTP_RTCP_INCLUDE_RTP_RTCP_DEFINES_H_ diff --git a/src/qos/rtpfb.cc b/src/qos/rtpfb.cc new file mode 100644 index 0000000..2c690fb --- /dev/null +++ b/src/qos/rtpfb.cc @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "rtpfb.h" + +#include "byte_io.h" + +namespace webrtc { +namespace rtcp { +// RFC 4585, Section 6.1: Feedback format. +// +// Common packet format: +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |V=2|P| FMT | PT | length | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 0 | SSRC of packet sender | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 4 | SSRC of media source | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// : Feedback Control Information (FCI) : +// : : + +void Rtpfb::ParseCommonFeedback(const uint8_t* payload) { + SetSenderSsrc(ByteReader::ReadBigEndian(&payload[0])); + SetMediaSsrc(ByteReader::ReadBigEndian(&payload[4])); +} + +void Rtpfb::CreateCommonFeedback(uint8_t* payload) const { + ByteWriter::WriteBigEndian(&payload[0], sender_ssrc()); + ByteWriter::WriteBigEndian(&payload[4], media_ssrc()); +} + +} // namespace rtcp +} // namespace webrtc diff --git a/src/qos/rtpfb.h b/src/qos/rtpfb.h new file mode 100644 index 0000000..544e2a4 --- /dev/null +++ b/src/qos/rtpfb.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_RTPFB_H_ +#define MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_RTPFB_H_ + +#include +#include + +#include "rtcp_packet.h" + +namespace webrtc { +namespace rtcp { + +// RTPFB: Transport layer feedback message. +// RFC4585, Section 6.2 +class Rtpfb : public RtcpPacket { + public: + static constexpr uint8_t kPacketType = 205; + + Rtpfb() = default; + ~Rtpfb() override = default; + + void SetMediaSsrc(uint32_t ssrc) { media_ssrc_ = ssrc; } + + uint32_t media_ssrc() const { return media_ssrc_; } + + protected: + static constexpr size_t kCommonFeedbackLength = 8; + void ParseCommonFeedback(const uint8_t* payload); + void CreateCommonFeedback(uint8_t* payload) const; + + private: + uint32_t media_ssrc_ = 0; +}; + +} // namespace rtcp +} // namespace webrtc +#endif // MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_RTPFB_H_ diff --git a/src/qos/send_side_bandwidth_estimation.cc b/src/qos/send_side_bandwidth_estimation.cc index 06b6766..a58cf54 100644 --- a/src/qos/send_side_bandwidth_estimation.cc +++ b/src/qos/send_side_bandwidth_estimation.cc @@ -267,6 +267,11 @@ void SendSideBandwidthEstimation::SetAcknowledgedRate( } } +void SendSideBandwidthEstimation::UpdateLossBasedEstimator( + const TransportPacketsFeedback& report, + BandwidthUsage /* delay_detector_state */, + std::optional /* probe_bitrate */, bool in_alr) {} + void SendSideBandwidthEstimation::UpdatePacketsLost(int64_t packets_lost, int64_t number_of_packets, Timestamp at_time) { diff --git a/src/qos/send_side_bandwidth_estimation.h b/src/qos/send_side_bandwidth_estimation.h index c3e3845..a401ab4 100644 --- a/src/qos/send_side_bandwidth_estimation.h +++ b/src/qos/send_side_bandwidth_estimation.h @@ -111,6 +111,10 @@ class SendSideBandwidthEstimation { int GetMinBitrate() const; void SetAcknowledgedRate(std::optional acknowledged_rate, Timestamp at_time); + void UpdateLossBasedEstimator(const TransportPacketsFeedback& report, + BandwidthUsage delay_detector_state, + std::optional probe_bitrate, + bool in_alr); private: friend class GoogCcStatePrinter; diff --git a/src/qos/transport_feedback.cc b/src/qos/transport_feedback.cc new file mode 100644 index 0000000..4ab634f --- /dev/null +++ b/src/qos/transport_feedback.cc @@ -0,0 +1,672 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "transport_feedback.h" + +#include +#include +#include +#include + +#include "byte_io.h" +#include "common_header.h" +#include "log.h" +#include "module_common_types_public.h" + +namespace webrtc { +namespace rtcp { +namespace { +// Header size: +// * 4 bytes Common RTCP Packet Header +// * 8 bytes Common Packet Format for RTCP Feedback Messages +// * 8 bytes FeedbackPacket header +constexpr size_t kTransportFeedbackHeaderSizeBytes = 4 + 8 + 8; +constexpr size_t kChunkSizeBytes = 2; +// TODO(sprang): Add support for dynamic max size for easier fragmentation, +// eg. set it to what's left in the buffer or IP_PACKET_SIZE. +// Size constraint imposed by RTCP common header: 16bit size field interpreted +// as number of four byte words minus the first header word. +constexpr size_t kMaxSizeBytes = (1 << 16) * 4; +// Payload size: +// * 8 bytes Common Packet Format for RTCP Feedback Messages +// * 8 bytes FeedbackPacket header. +// * 2 bytes for one chunk. +constexpr size_t kMinPayloadSizeBytes = 8 + 8 + 2; +constexpr TimeDelta kBaseTimeTick = TransportFeedback::kDeltaTick * (1 << 8); +constexpr TimeDelta kTimeWrapPeriod = kBaseTimeTick * (1 << 24); + +// Message format +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |V=2|P| FMT=15 | PT=205 | length | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 0 | SSRC of packet sender | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 4 | SSRC of media source | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 8 | base sequence number | packet status count | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 12 | reference time | fb pkt. count | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// 16 | packet chunk | packet chunk | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// . . +// . . +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | packet chunk | recv delta | recv delta | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// . . +// . . +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | recv delta | recv delta | zero padding | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +} // namespace + +TransportFeedback::LastChunk::LastChunk() { Clear(); } + +bool TransportFeedback::LastChunk::Empty() const { return size_ == 0; } + +void TransportFeedback::LastChunk::Clear() { + size_ = 0; + all_same_ = true; + has_large_delta_ = false; +} + +bool TransportFeedback::LastChunk::CanAdd(DeltaSize delta_size) const { + if (size_ < kMaxTwoBitCapacity) return true; + if (size_ < kMaxOneBitCapacity && !has_large_delta_ && delta_size != kLarge) + return true; + if (size_ < kMaxRunLengthCapacity && all_same_ && + delta_sizes_[0] == delta_size) + return true; + return false; +} + +void TransportFeedback::LastChunk::Add(DeltaSize delta_size) { + if (size_ < kMaxVectorCapacity) delta_sizes_[size_] = delta_size; + size_++; + all_same_ = all_same_ && delta_size == delta_sizes_[0]; + has_large_delta_ = has_large_delta_ || delta_size == kLarge; +} + +void TransportFeedback::LastChunk::AddMissingPackets(size_t num_missing) { + std::fill(delta_sizes_.begin(), delta_sizes_.end(), DeltaSize(0)); + size_ = num_missing; +} + +uint16_t TransportFeedback::LastChunk::Emit() { + if (all_same_) { + uint16_t chunk = EncodeRunLength(); + Clear(); + return chunk; + } + if (size_ == kMaxOneBitCapacity) { + uint16_t chunk = EncodeOneBit(); + Clear(); + return chunk; + } + uint16_t chunk = EncodeTwoBit(kMaxTwoBitCapacity); + // Remove `kMaxTwoBitCapacity` encoded delta sizes: + // Shift remaining delta sizes and recalculate all_same_ && has_large_delta_. + size_ -= kMaxTwoBitCapacity; + all_same_ = true; + has_large_delta_ = false; + for (size_t i = 0; i < size_; ++i) { + DeltaSize delta_size = delta_sizes_[kMaxTwoBitCapacity + i]; + delta_sizes_[i] = delta_size; + all_same_ = all_same_ && delta_size == delta_sizes_[0]; + has_large_delta_ = has_large_delta_ || delta_size == kLarge; + } + + return chunk; +} + +uint16_t TransportFeedback::LastChunk::EncodeLast() const { + if (all_same_) return EncodeRunLength(); + if (size_ <= kMaxTwoBitCapacity) return EncodeTwoBit(size_); + return EncodeOneBit(); +} + +// Appends content of the Lastchunk to `deltas`. +void TransportFeedback::LastChunk::AppendTo( + std::vector* deltas) const { + if (all_same_) { + deltas->insert(deltas->end(), size_, delta_sizes_[0]); + } else { + deltas->insert(deltas->end(), delta_sizes_.begin(), + delta_sizes_.begin() + size_); + } +} + +void TransportFeedback::LastChunk::Decode(uint16_t chunk, size_t max_size) { + if ((chunk & 0x8000) == 0) { + DecodeRunLength(chunk, max_size); + } else if ((chunk & 0x4000) == 0) { + DecodeOneBit(chunk, max_size); + } else { + DecodeTwoBit(chunk, max_size); + } +} + +// One Bit Status Vector Chunk +// +// 0 1 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |T|S| symbol list | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// +// T = 1 +// S = 0 +// Symbol list = 14 entries where 0 = not received, 1 = received 1-byte delta. +uint16_t TransportFeedback::LastChunk::EncodeOneBit() const { + uint16_t chunk = 0x8000; + for (size_t i = 0; i < size_; ++i) + chunk |= delta_sizes_[i] << (kMaxOneBitCapacity - 1 - i); + return chunk; +} + +void TransportFeedback::LastChunk::DecodeOneBit(uint16_t chunk, + size_t max_size) { + size_ = std::min(kMaxOneBitCapacity, max_size); + has_large_delta_ = false; + all_same_ = false; + for (size_t i = 0; i < size_; ++i) + delta_sizes_[i] = (chunk >> (kMaxOneBitCapacity - 1 - i)) & 0x01; +} + +// Two Bit Status Vector Chunk +// +// 0 1 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |T|S| symbol list | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// +// T = 1 +// S = 1 +// symbol list = 7 entries of two bits each. +uint16_t TransportFeedback::LastChunk::EncodeTwoBit(size_t size) const { + uint16_t chunk = 0xc000; + for (size_t i = 0; i < size; ++i) + chunk |= delta_sizes_[i] << 2 * (kMaxTwoBitCapacity - 1 - i); + return chunk; +} + +void TransportFeedback::LastChunk::DecodeTwoBit(uint16_t chunk, + size_t max_size) { + size_ = std::min(kMaxTwoBitCapacity, max_size); + has_large_delta_ = true; + all_same_ = false; + for (size_t i = 0; i < size_; ++i) + delta_sizes_[i] = (chunk >> 2 * (kMaxTwoBitCapacity - 1 - i)) & 0x03; +} + +// Run Length Status Vector Chunk +// +// 0 1 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |T| S | Run Length | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// +// T = 0 +// S = symbol +// Run Length = Unsigned integer denoting the run length of the symbol +uint16_t TransportFeedback::LastChunk::EncodeRunLength() const { + return (delta_sizes_[0] << 13) | static_cast(size_); +} + +void TransportFeedback::LastChunk::DecodeRunLength(uint16_t chunk, + size_t max_count) { + size_ = std::min(chunk & 0x1fff, max_count); + DeltaSize delta_size = (chunk >> 13) & 0x03; + has_large_delta_ = delta_size >= kLarge; + all_same_ = true; + // To make it consistent with Add function, populate delta_sizes_ beyond 1st. + for (size_t i = 0; i < std::min(size_, kMaxVectorCapacity); ++i) + delta_sizes_[i] = delta_size; +} + +TransportFeedback::TransportFeedback() + : TransportFeedback(/*include_timestamps=*/true) {} + +TransportFeedback::TransportFeedback(bool include_timestamps) + : base_seq_no_(0), + num_seq_no_(0), + base_time_ticks_(0), + feedback_seq_(0), + include_timestamps_(include_timestamps), + last_timestamp_(Timestamp::Zero()), + size_bytes_(kTransportFeedbackHeaderSizeBytes) {} + +TransportFeedback::TransportFeedback(const TransportFeedback&) = default; + +TransportFeedback::TransportFeedback(TransportFeedback&& other) + : base_seq_no_(other.base_seq_no_), + num_seq_no_(other.num_seq_no_), + base_time_ticks_(other.base_time_ticks_), + feedback_seq_(other.feedback_seq_), + include_timestamps_(other.include_timestamps_), + last_timestamp_(other.last_timestamp_), + received_packets_(std::move(other.received_packets_)), + all_packets_(std::move(other.all_packets_)), + encoded_chunks_(std::move(other.encoded_chunks_)), + last_chunk_(other.last_chunk_), + size_bytes_(other.size_bytes_) { + other.Clear(); +} + +TransportFeedback::~TransportFeedback() {} + +void TransportFeedback::SetBase(uint16_t base_sequence, + Timestamp ref_timestamp) { + base_seq_no_ = base_sequence; + base_time_ticks_ = + (ref_timestamp.us() % kTimeWrapPeriod.us()) / kBaseTimeTick.us(); + last_timestamp_ = BaseTime(); +} + +void TransportFeedback::SetFeedbackSequenceNumber(uint8_t feedback_sequence) { + feedback_seq_ = feedback_sequence; +} + +bool TransportFeedback::AddReceivedPacket(uint16_t sequence_number, + Timestamp timestamp) { + // Set delta to zero if timestamps are not included, this will simplify the + // encoding process. + int16_t delta = 0; + if (include_timestamps_) { + // Convert to ticks and round. + if (last_timestamp_ > timestamp) { + timestamp += (last_timestamp_ - timestamp).RoundUpTo(kTimeWrapPeriod); + } + int64_t delta_full = + (timestamp - last_timestamp_).us() % kTimeWrapPeriod.us(); + if (delta_full > kTimeWrapPeriod.us() / 2) { + delta_full -= kTimeWrapPeriod.us(); + delta_full -= kDeltaTick.us() / 2; + } else { + delta_full += kDeltaTick.us() / 2; + } + delta_full /= kDeltaTick.us(); + + delta = static_cast(delta_full); + // If larger than 16bit signed, we can't represent it - need new fb packet. + if (delta != delta_full) { + LOG_WARN("Delta value too large ( >= 2^16 ticks )"); + return false; + } + } + + uint16_t next_seq_no = base_seq_no_ + num_seq_no_; + if (sequence_number != next_seq_no) { + uint16_t last_seq_no = next_seq_no - 1; + if (!IsNewerSequenceNumber(sequence_number, last_seq_no)) return false; + uint16_t num_missing_packets = sequence_number - next_seq_no; + if (!AddMissingPackets(num_missing_packets)) return false; + } + + DeltaSize delta_size = (delta >= 0 && delta <= 0xff) ? 1 : 2; + if (!AddDeltaSize(delta_size)) return false; + + received_packets_.emplace_back(sequence_number, delta); + last_timestamp_ += delta * kDeltaTick; + if (include_timestamps_) { + size_bytes_ += delta_size; + } + return true; +} + +const std::vector& +TransportFeedback::GetReceivedPackets() const { + return received_packets_; +} + +void TransportFeedback::ForAllPackets( + rtc::FunctionView handler) const { + TimeDelta delta_since_base = TimeDelta::Zero(); + auto received_it = received_packets_.begin(); + const uint16_t last_seq_num = base_seq_no_ + num_seq_no_; + for (uint16_t seq_num = base_seq_no_; seq_num != last_seq_num; ++seq_num) { + if (received_it != received_packets_.end() && + received_it->sequence_number() == seq_num) { + delta_since_base += received_it->delta(); + handler(seq_num, delta_since_base); + ++received_it; + } else { + handler(seq_num, TimeDelta::PlusInfinity()); + } + } +} + +uint16_t TransportFeedback::GetBaseSequence() const { return base_seq_no_; } + +Timestamp TransportFeedback::BaseTime() const { + // Add an extra kTimeWrapPeriod to allow add received packets arrived earlier + // than the first added packet (and thus allow to record negative deltas) + // even when base_time_ticks_ == 0. + return Timestamp::Zero() + kTimeWrapPeriod + + int64_t{base_time_ticks_} * kBaseTimeTick; +} + +TimeDelta TransportFeedback::GetBaseDelta(Timestamp prev_timestamp) const { + TimeDelta delta = BaseTime() - prev_timestamp; + // Compensate for wrap around. + if ((delta - kTimeWrapPeriod).Abs() < delta.Abs()) { + delta -= kTimeWrapPeriod; // Wrap backwards. + } else if ((delta + kTimeWrapPeriod).Abs() < delta.Abs()) { + delta += kTimeWrapPeriod; // Wrap forwards. + } + return delta; +} + +// De-serialize packet. +bool TransportFeedback::Parse(const CommonHeader& packet) { + if (packet.payload_size_bytes() < kMinPayloadSizeBytes) { + LOG_WARN( + "Buffer too small ({} bytes) to fit a FeedbackPacket. Minimum size = " + "{}", + packet.payload_size_bytes(), kMinPayloadSizeBytes); + return false; + } + + const uint8_t* const payload = packet.payload(); + ParseCommonFeedback(payload); + + base_seq_no_ = ByteReader::ReadBigEndian(&payload[8]); + uint16_t status_count = ByteReader::ReadBigEndian(&payload[10]); + base_time_ticks_ = ByteReader::ReadBigEndian(&payload[12]); + feedback_seq_ = payload[15]; + Clear(); + size_t index = 16; + const size_t end_index = packet.payload_size_bytes(); + + if (status_count == 0) { + LOG_WARN("Empty feedback messages not allowed."); + return false; + } + + std::vector delta_sizes; + delta_sizes.reserve(status_count); + while (delta_sizes.size() < status_count) { + if (index + kChunkSizeBytes > end_index) { + LOG_WARN("Buffer overflow while parsing packet."); + Clear(); + return false; + } + + uint16_t chunk = ByteReader::ReadBigEndian(&payload[index]); + index += kChunkSizeBytes; + encoded_chunks_.push_back(chunk); + last_chunk_.Decode(chunk, status_count - delta_sizes.size()); + last_chunk_.AppendTo(&delta_sizes); + } + // Last chunk is stored in the `last_chunk_`. + encoded_chunks_.pop_back(); + num_seq_no_ = status_count; + + uint16_t seq_no = base_seq_no_; + size_t recv_delta_size = + std::accumulate(delta_sizes.begin(), delta_sizes.end(), 0); + + // Determine if timestamps, that is, recv_delta are included in the packet. + if (end_index >= index + recv_delta_size) { + for (size_t delta_size : delta_sizes) { + switch (delta_size) { + case 0: + break; + case 1: { + int16_t delta = payload[index]; + received_packets_.emplace_back(seq_no, delta); + last_timestamp_ += delta * kDeltaTick; + index += delta_size; + break; + } + case 2: { + int16_t delta = ByteReader::ReadBigEndian(&payload[index]); + received_packets_.emplace_back(seq_no, delta); + last_timestamp_ += delta * kDeltaTick; + index += delta_size; + break; + } + case 3: + Clear(); + LOG_WARN("Invalid delta_size for seq_no {}", seq_no); + + return false; + default: + break; + } + ++seq_no; + } + } else { + // The packet does not contain receive deltas. + include_timestamps_ = false; + for (size_t delta_size : delta_sizes) { + // Use delta sizes to detect if packet was received. + if (delta_size > 0) { + received_packets_.emplace_back(seq_no, 0); + } + ++seq_no; + } + } + size_bytes_ = RtcpPacket::kHeaderLength + index; + return true; +} + +std::unique_ptr TransportFeedback::ParseFrom( + const uint8_t* buffer, size_t length) { + CommonHeader header; + if (!header.Parse(buffer, length)) return nullptr; + if (header.type() != kPacketType || header.fmt() != kFeedbackMessageType) + return nullptr; + std::unique_ptr parsed(new TransportFeedback); + if (!parsed->Parse(header)) return nullptr; + return parsed; +} + +bool TransportFeedback::IsConsistent() const { + size_t packet_size = kTransportFeedbackHeaderSizeBytes; + std::vector delta_sizes; + LastChunk chunk_decoder; + for (uint16_t chunk : encoded_chunks_) { + chunk_decoder.Decode(chunk, kMaxReportedPackets); + chunk_decoder.AppendTo(&delta_sizes); + packet_size += kChunkSizeBytes; + } + if (!last_chunk_.Empty()) { + last_chunk_.AppendTo(&delta_sizes); + packet_size += kChunkSizeBytes; + } + if (num_seq_no_ != delta_sizes.size()) { + LOG_ERROR("{} packets encoded. Expected {}", delta_sizes.size(), + num_seq_no_); + return false; + } + Timestamp timestamp = BaseTime(); + auto packet_it = received_packets_.begin(); + uint16_t seq_no = base_seq_no_; + for (DeltaSize delta_size : delta_sizes) { + if (delta_size > 0) { + if (packet_it == received_packets_.end()) { + LOG_ERROR("Failed to find delta for seq_no {}", seq_no); + return false; + } + if (packet_it->sequence_number() != seq_no) { + LOG_ERROR("Expected to find delta for seq_no {}. Next delta is for {}", + seq_no, packet_it->sequence_number()); + return false; + } + if (delta_size == 1 && + (packet_it->delta_ticks() < 0 || packet_it->delta_ticks() > 0xff)) { + LOG_ERROR("Delta {} for seq_no {} doesn't fit into one byte", + packet_it->delta_ticks(), seq_no); + return false; + } + timestamp += packet_it->delta(); + ++packet_it; + } + if (include_timestamps_) { + packet_size += delta_size; + } + ++seq_no; + } + if (packet_it != received_packets_.end()) { + LOG_ERROR("Unencoded delta for seq_no {}", packet_it->sequence_number()); + return false; + } + if (timestamp != last_timestamp_) { + LOG_ERROR("Last timestamp mismatch. Calculated: {}. Saved: {}", + ToString(timestamp), ToString(last_timestamp_)); + return false; + } + if (size_bytes_ != packet_size) { + LOG_ERROR("Rtcp packet size mismatch. Calculated: {}. Saved: {}", + packet_size, size_bytes_); + return false; + } + return true; +} + +size_t TransportFeedback::BlockLength() const { + // Round size_bytes_ up to multiple of 32bits. + return (size_bytes_ + 3) & (~static_cast(3)); +} + +size_t TransportFeedback::PaddingLength() const { + return BlockLength() - size_bytes_; +} + +// Serialize packet. +bool TransportFeedback::Create(uint8_t* packet, size_t* position, + size_t max_length, + PacketReadyCallback callback) const { + if (num_seq_no_ == 0) return false; + + while (*position + BlockLength() > max_length) { + if (!OnBufferFull(packet, position, callback)) return false; + } + const size_t position_end = *position + BlockLength(); + const size_t padding_length = PaddingLength(); + bool has_padding = padding_length > 0; + CreateHeader(kFeedbackMessageType, kPacketType, HeaderLength(), has_padding, + packet, position); + CreateCommonFeedback(packet + *position); + *position += kCommonFeedbackLength; + + ByteWriter::WriteBigEndian(&packet[*position], base_seq_no_); + *position += 2; + + ByteWriter::WriteBigEndian(&packet[*position], num_seq_no_); + *position += 2; + + ByteWriter::WriteBigEndian(&packet[*position], base_time_ticks_); + *position += 3; + + packet[(*position)++] = feedback_seq_; + + for (uint16_t chunk : encoded_chunks_) { + ByteWriter::WriteBigEndian(&packet[*position], chunk); + *position += 2; + } + if (!last_chunk_.Empty()) { + uint16_t chunk = last_chunk_.EncodeLast(); + ByteWriter::WriteBigEndian(&packet[*position], chunk); + *position += 2; + } + + if (include_timestamps_) { + for (const auto& received_packet : received_packets_) { + int16_t delta = received_packet.delta_ticks(); + if (delta >= 0 && delta <= 0xFF) { + packet[(*position)++] = delta; + } else { + ByteWriter::WriteBigEndian(&packet[*position], delta); + *position += 2; + } + } + } + + if (padding_length > 0) { + for (size_t i = 0; i < padding_length - 1; ++i) { + packet[(*position)++] = 0; + } + packet[(*position)++] = padding_length; + } + return true; +} + +void TransportFeedback::Clear() { + num_seq_no_ = 0; + last_timestamp_ = BaseTime(); + received_packets_.clear(); + all_packets_.clear(); + encoded_chunks_.clear(); + last_chunk_.Clear(); + size_bytes_ = kTransportFeedbackHeaderSizeBytes; +} + +bool TransportFeedback::AddDeltaSize(DeltaSize delta_size) { + if (num_seq_no_ == kMaxReportedPackets) return false; + size_t add_chunk_size = last_chunk_.Empty() ? kChunkSizeBytes : 0; + if (size_bytes_ + delta_size + add_chunk_size > kMaxSizeBytes) return false; + + if (last_chunk_.CanAdd(delta_size)) { + size_bytes_ += add_chunk_size; + last_chunk_.Add(delta_size); + ++num_seq_no_; + return true; + } + if (size_bytes_ + delta_size + kChunkSizeBytes > kMaxSizeBytes) return false; + + encoded_chunks_.push_back(last_chunk_.Emit()); + size_bytes_ += kChunkSizeBytes; + last_chunk_.Add(delta_size); + ++num_seq_no_; + return true; +} + +bool TransportFeedback::AddMissingPackets(size_t num_missing_packets) { + size_t new_num_seq_no = num_seq_no_ + num_missing_packets; + if (new_num_seq_no > kMaxReportedPackets) { + return false; + } + + if (!last_chunk_.Empty()) { + while (num_missing_packets > 0 && last_chunk_.CanAdd(0)) { + last_chunk_.Add(0); + --num_missing_packets; + } + if (num_missing_packets == 0) { + num_seq_no_ = new_num_seq_no; + return true; + } + encoded_chunks_.push_back(last_chunk_.Emit()); + } + size_t full_chunks = num_missing_packets / LastChunk::kMaxRunLengthCapacity; + size_t partial_chunk = num_missing_packets % LastChunk::kMaxRunLengthCapacity; + size_t num_chunks = full_chunks + (partial_chunk > 0 ? 1 : 0); + if (size_bytes_ + kChunkSizeBytes * num_chunks > kMaxSizeBytes) { + num_seq_no_ = (new_num_seq_no - num_missing_packets); + return false; + } + size_bytes_ += kChunkSizeBytes * num_chunks; + // T = 0, S = 0, run length = kMaxRunLengthCapacity, see EncodeRunLength(). + encoded_chunks_.insert(encoded_chunks_.end(), full_chunks, + LastChunk::kMaxRunLengthCapacity); + last_chunk_.AddMissingPackets(partial_chunk); + num_seq_no_ = new_num_seq_no; + return true; +} +} // namespace rtcp +} // namespace webrtc diff --git a/src/qos/transport_feedback.h b/src/qos/transport_feedback.h new file mode 100644 index 0000000..1d1c095 --- /dev/null +++ b/src/qos/transport_feedback.h @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_TRANSPORT_FEEDBACK_H_ +#define MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_TRANSPORT_FEEDBACK_H_ + +#include +#include +#include + +#include "api/function_view.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "rtpfb.h" + +namespace webrtc { +namespace rtcp { +class CommonHeader; + +class TransportFeedback : public Rtpfb { + public: + class ReceivedPacket { + public: + ReceivedPacket(uint16_t sequence_number, int16_t delta_ticks) + : sequence_number_(sequence_number), delta_ticks_(delta_ticks) {} + ReceivedPacket(const ReceivedPacket&) = default; + ReceivedPacket& operator=(const ReceivedPacket&) = default; + + uint16_t sequence_number() const { return sequence_number_; } + int16_t delta_ticks() const { return delta_ticks_; } + TimeDelta delta() const { return delta_ticks_ * kDeltaTick; } + + private: + uint16_t sequence_number_; + int16_t delta_ticks_; + }; + // TODO(sprang): IANA reg? + static constexpr uint8_t kFeedbackMessageType = 15; + // Convert to multiples of 0.25ms. + static constexpr TimeDelta kDeltaTick = TimeDelta::Micros(250); + // Maximum number of packets (including missing) TransportFeedback can report. + static constexpr size_t kMaxReportedPackets = 0xffff; + + TransportFeedback(); + + // If `include_timestamps` is set to false, the created packet will not + // contain the receive delta block. + explicit TransportFeedback(bool include_timestamps); + TransportFeedback(const TransportFeedback&); + TransportFeedback(TransportFeedback&&); + + ~TransportFeedback() override; + + void SetBase(uint16_t base_sequence, // Seq# of first packet in this msg. + Timestamp ref_timestamp); // Reference timestamp for this msg. + + void SetFeedbackSequenceNumber(uint8_t feedback_sequence); + // NOTE: This method requires increasing sequence numbers (excepting wraps). + bool AddReceivedPacket(uint16_t sequence_number, Timestamp timestamp); + const std::vector& GetReceivedPackets() const; + + // Calls `handler` for all packets this feedback describes. + // For received packets pass receieve time as `delta_since_base` since the + // `BaseTime()`. For missed packets calls `handler` with `delta_since_base = + // PlusInfinity()`. + void ForAllPackets(rtc::FunctionView + handler) const; + + uint16_t GetBaseSequence() const; + + // Returns number of packets (including missing) this feedback describes. + size_t GetPacketStatusCount() const { return num_seq_no_; } + + // Get the reference time including any precision loss. + Timestamp BaseTime() const; + + // Get the unwrapped delta between current base time and `prev_timestamp`. + TimeDelta GetBaseDelta(Timestamp prev_timestamp) const; + + // Does the feedback packet contain timestamp information? + bool IncludeTimestamps() const { return include_timestamps_; } + + bool Parse(const CommonHeader& packet); + static std::unique_ptr ParseFrom(const uint8_t* buffer, + size_t length); + // Pre and postcondition for all public methods. Should always return true. + // This function is for tests. + bool IsConsistent() const; + + size_t BlockLength() const override; + size_t PaddingLength() const; + + bool Create(uint8_t* packet, size_t* position, size_t max_length, + PacketReadyCallback callback) const override; + + private: + // Size in bytes of a delta time in rtcp packet. + // Valid values are 0 (packet wasn't received), 1 or 2. + using DeltaSize = uint8_t; + // Keeps DeltaSizes that can be encoded into single chunk if it is last chunk. + class LastChunk { + public: + using DeltaSize = TransportFeedback::DeltaSize; + static constexpr size_t kMaxRunLengthCapacity = 0x1fff; + + LastChunk(); + + bool Empty() const; + void Clear(); + // Return if delta sizes still can be encoded into single chunk with added + // `delta_size`. + bool CanAdd(DeltaSize delta_size) const; + // Add `delta_size`, assumes `CanAdd(delta_size)`, + void Add(DeltaSize delta_size); + // Equivalent to calling Add(0) `num_missing` times. Assumes `Empty()`. + void AddMissingPackets(size_t num_missing); + + // Encode chunk as large as possible removing encoded delta sizes. + // Assume CanAdd() == false for some valid delta_size. + uint16_t Emit(); + // Encode all stored delta_sizes into single chunk, pad with 0s if needed. + uint16_t EncodeLast() const; + + // Decode up to `max_size` delta sizes from `chunk`. + void Decode(uint16_t chunk, size_t max_size); + // Appends content of the Lastchunk to `deltas`. + void AppendTo(std::vector* deltas) const; + + private: + static constexpr size_t kMaxOneBitCapacity = 14; + static constexpr size_t kMaxTwoBitCapacity = 7; + static constexpr size_t kMaxVectorCapacity = kMaxOneBitCapacity; + static constexpr DeltaSize kLarge = 2; + + uint16_t EncodeOneBit() const; + void DecodeOneBit(uint16_t chunk, size_t max_size); + + uint16_t EncodeTwoBit(size_t size) const; + void DecodeTwoBit(uint16_t chunk, size_t max_size); + + uint16_t EncodeRunLength() const; + void DecodeRunLength(uint16_t chunk, size_t max_size); + + std::array delta_sizes_; + size_t size_; + bool all_same_; + bool has_large_delta_; + }; + + // Reset packet to consistent empty state. + void Clear(); + + bool AddDeltaSize(DeltaSize delta_size); + // Adds `num_missing_packets` deltas of size 0. + bool AddMissingPackets(size_t num_missing_packets); + + uint16_t base_seq_no_; + uint16_t num_seq_no_; + uint32_t base_time_ticks_; + uint8_t feedback_seq_; + bool include_timestamps_; + + Timestamp last_timestamp_; + std::vector received_packets_; + std::vector all_packets_; + // All but last encoded packet chunks. + std::vector encoded_chunks_; + LastChunk last_chunk_; + size_t size_bytes_; +}; + +} // namespace rtcp +} // namespace webrtc +#endif // MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_TRANSPORT_FEEDBACK_H_ diff --git a/src/qos/transport_feedback_adapter.cc b/src/qos/transport_feedback_adapter.cc new file mode 100644 index 0000000..0d0327e --- /dev/null +++ b/src/qos/transport_feedback_adapter.cc @@ -0,0 +1,382 @@ +/* + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "transport_feedback_adapter.h" + +#include + +#include +#include +#include +#include +#include + +#include "api/transport/ecn_marking.h" +#include "api/units/time_delta.h" +#include "congestion_control_feedback.h" +#include "log.h" +#include "ntp_time_util.h" +#include "rtp_packet_to_send.h" +#include "transport_feedback.h" + +namespace webrtc { + +constexpr TimeDelta kSendTimeHistoryWindow = TimeDelta::Seconds(60); + +void InFlightBytesTracker::AddInFlightPacketBytes( + const PacketFeedback& packet) { + auto it = in_flight_data_.find(packet.network_route); + if (it != in_flight_data_.end()) { + it->second += packet.sent.size; + } else { + in_flight_data_.insert({packet.network_route, packet.sent.size}); + } +} + +void InFlightBytesTracker::RemoveInFlightPacketBytes( + const PacketFeedback& packet) { + if (packet.sent.send_time.IsInfinite()) return; + auto it = in_flight_data_.find(packet.network_route); + if (it != in_flight_data_.end()) { + it->second -= packet.sent.size; + if (it->second.IsZero()) in_flight_data_.erase(it); + } +} + +DataSize InFlightBytesTracker::GetOutstandingData( + const rtc::NetworkRoute& network_route) const { + auto it = in_flight_data_.find(network_route); + if (it != in_flight_data_.end()) { + return it->second; + } else { + return DataSize::Zero(); + } +} + +// Comparator for consistent map with NetworkRoute as key. +bool InFlightBytesTracker::NetworkRouteComparator::operator()( + const rtc::NetworkRoute& a, const rtc::NetworkRoute& b) const { + if (a.local.network_id() != b.local.network_id()) + return a.local.network_id() < b.local.network_id(); + if (a.remote.network_id() != b.remote.network_id()) + return a.remote.network_id() < b.remote.network_id(); + + if (a.local.adapter_id() != b.local.adapter_id()) + return a.local.adapter_id() < b.local.adapter_id(); + if (a.remote.adapter_id() != b.remote.adapter_id()) + return a.remote.adapter_id() < b.remote.adapter_id(); + + if (a.local.uses_turn() != b.local.uses_turn()) + return a.local.uses_turn() < b.local.uses_turn(); + if (a.remote.uses_turn() != b.remote.uses_turn()) + return a.remote.uses_turn() < b.remote.uses_turn(); + + return a.connected < b.connected; +} + +TransportFeedbackAdapter::TransportFeedbackAdapter() = default; + +void TransportFeedbackAdapter::AddPacket(const RtpPacketToSend& packet_to_send, + const PacedPacketInfo& pacing_info, + size_t overhead_bytes, + Timestamp creation_time) { + PacketFeedback feedback; + + feedback.creation_time = creation_time; + // Note, if transport sequence number header extension is used, transport + // sequence numbers are wrapped to 16 bit. See + // RtpSenderEgress::CompleteSendPacket. + feedback.sent.sequence_number = seq_num_unwrapper_.Unwrap( + packet_to_send.transport_sequence_number().value_or(0)); + feedback.sent.size = DataSize::Bytes(packet_to_send.size() + overhead_bytes); + feedback.sent.audio = + packet_to_send.packet_type() == RtpPacketMediaType::kAudio; + feedback.network_route = network_route_; + feedback.sent.pacing_info = pacing_info; + feedback.ssrc = packet_to_send.Ssrc(); + feedback.rtp_sequence_number = packet_to_send.SequenceNumber(); + + while (!history_.empty() && + creation_time - history_.begin()->second.creation_time > + kSendTimeHistoryWindow) { + // TODO(sprang): Warn if erasing (too many) old items? + if (history_.begin()->second.sent.sequence_number > last_ack_seq_num_) + in_flight_.RemoveInFlightPacketBytes(history_.begin()->second); + + const PacketFeedback& packet = history_.begin()->second; + rtp_to_transport_sequence_number_.erase( + {packet.ssrc, packet.rtp_sequence_number}); + history_.erase(history_.begin()); + } + // Note that it can happen that the same SSRC and sequence number is sent + // again. e.g, audio retransmission. + rtp_to_transport_sequence_number_.emplace( + SsrcAndRtpSequencenumber({feedback.ssrc, feedback.rtp_sequence_number}), + feedback.sent.sequence_number); + history_.emplace(feedback.sent.sequence_number, feedback); +} + +std::optional TransportFeedbackAdapter::ProcessSentPacket( + const rtc::SentPacket& sent_packet) { + auto send_time = Timestamp::Millis(sent_packet.send_time_ms); + // TODO(srte): Only use one way to indicate that packet feedback is used. + if (sent_packet.info.included_in_feedback || sent_packet.packet_id != -1) { + int64_t unwrapped_seq_num = + seq_num_unwrapper_.Unwrap(sent_packet.packet_id); + auto it = history_.find(unwrapped_seq_num); + if (it != history_.end()) { + bool packet_retransmit = it->second.sent.send_time.IsFinite(); + it->second.sent.send_time = send_time; + last_send_time_ = std::max(last_send_time_, send_time); + // TODO(srte): Don't do this on retransmit. + if (!pending_untracked_size_.IsZero()) { + if (send_time < last_untracked_send_time_) + LOG_WARN( + "appending acknowledged data for out of order packet. (Diff: {} " + "ms.)", + ToString(last_untracked_send_time_ - send_time)); + it->second.sent.prior_unacked_data += pending_untracked_size_; + pending_untracked_size_ = DataSize::Zero(); + } + if (!packet_retransmit) { + if (it->second.sent.sequence_number > last_ack_seq_num_) + in_flight_.AddInFlightPacketBytes(it->second); + it->second.sent.data_in_flight = GetOutstandingData(); + return it->second.sent; + } + } + } else if (sent_packet.info.included_in_allocation) { + if (send_time < last_send_time_) { + LOG_WARN("ignoring untracked data for out of order packet."); + } + pending_untracked_size_ += + DataSize::Bytes(sent_packet.info.packet_size_bytes); + last_untracked_send_time_ = std::max(last_untracked_send_time_, send_time); + } + return std::nullopt; +} + +std::optional +TransportFeedbackAdapter::ProcessTransportFeedback( + const rtcp::TransportFeedback& feedback, Timestamp feedback_receive_time) { + if (feedback.GetPacketStatusCount() == 0) { + LOG_INFO("Empty transport feedback packet received."); + return std::nullopt; + } + + // Add timestamp deltas to a local time base selected on first packet arrival. + // This won't be the true time base, but makes it easier to manually inspect + // time stamps. + if (last_transport_feedback_base_time_.IsInfinite()) { + current_offset_ = feedback_receive_time; + } else { + // TODO(srte): We shouldn't need to do rounding here. + const TimeDelta delta = + feedback.GetBaseDelta(last_transport_feedback_base_time_) + .RoundDownTo(TimeDelta::Millis(1)); + // Protect against assigning current_offset_ negative value. + if (delta < Timestamp::Zero() - current_offset_) { + LOG_WARN("Unexpected feedback timestamp received."); + current_offset_ = feedback_receive_time; + } else { + current_offset_ += delta; + } + } + last_transport_feedback_base_time_ = feedback.BaseTime(); + + std::vector packet_result_vector; + packet_result_vector.reserve(feedback.GetPacketStatusCount()); + + size_t failed_lookups = 0; + size_t ignored = 0; + + feedback.ForAllPackets([&](uint16_t sequence_number, + TimeDelta delta_since_base) { + int64_t seq_num = seq_num_unwrapper_.Unwrap(sequence_number); + std::optional packet_feedback = RetrievePacketFeedback( + seq_num, /*received=*/delta_since_base.IsFinite()); + if (!packet_feedback) { + ++failed_lookups; + return; + } + if (delta_since_base.IsFinite()) { + packet_feedback->receive_time = + current_offset_ + delta_since_base.RoundDownTo(TimeDelta::Millis(1)); + } + if (packet_feedback->network_route == network_route_) { + PacketResult result; + result.sent_packet = packet_feedback->sent; + result.receive_time = packet_feedback->receive_time; + packet_result_vector.push_back(result); + } else { + ++ignored; + } + }); + + if (failed_lookups > 0) { + LOG_WARN( + "Failed to lookup send time for {} packet {}. Packets reordered or " + "send time history too small?", + failed_lookups, (failed_lookups > 1 ? "s" : "")); + } + if (ignored > 0) { + LOG_INFO("Ignoring packets because they were sent on a different route.", + ignored); + } + return ToTransportFeedback(std::move(packet_result_vector), + feedback_receive_time, /*suports_ecn=*/false); +} + +std::optional +TransportFeedbackAdapter::ProcessCongestionControlFeedback( + const rtcp::CongestionControlFeedback& feedback, + Timestamp feedback_receive_time) { + if (feedback.packets().empty()) { + LOG_INFO("Empty congestion control feedback packet received."); + return std::nullopt; + } + if (current_offset_.IsInfinite()) { + current_offset_ = feedback_receive_time; + } + TimeDelta feedback_delta = last_feedback_compact_ntp_time_ + ? CompactNtpIntervalToTimeDelta( + feedback.report_timestamp_compact_ntp() - + *last_feedback_compact_ntp_time_) + : TimeDelta::Zero(); + last_feedback_compact_ntp_time_ = feedback.report_timestamp_compact_ntp(); + if (feedback_delta < TimeDelta::Zero()) { + LOG_WARN("Unexpected feedback ntp time delta {}", ToString(feedback_delta)); + current_offset_ = feedback_receive_time; + } else { + current_offset_ += feedback_delta; + } + + int ignored_packets = 0; + int failed_lookups = 0; + bool supports_ecn = true; + std::vector packet_result_vector; + for (const rtcp::CongestionControlFeedback::PacketInfo& packet_info : + feedback.packets()) { + std::optional packet_feedback = RetrievePacketFeedback( + {packet_info.ssrc, packet_info.sequence_number}, + /*received=*/packet_info.arrival_time_offset.IsFinite()); + if (!packet_feedback) { + ++failed_lookups; + continue; + } + if (packet_feedback->network_route != network_route_) { + ++ignored_packets; + continue; + } + PacketResult result; + result.sent_packet = packet_feedback->sent; + if (packet_info.arrival_time_offset.IsFinite()) { + result.receive_time = current_offset_ - packet_info.arrival_time_offset; + supports_ecn &= packet_info.ecn != EcnMarking::kNotEct; + } + result.ecn = packet_info.ecn; + packet_result_vector.push_back(result); + } + + if (failed_lookups > 0) { + LOG_WARN( + "Failed to lookup send time for {} packet {}. Packets reordered or " + "send time history too small?", + failed_lookups, (failed_lookups > 1 ? "s" : "")); + } + if (ignored_packets > 0) { + LOG_INFO("Ignoring {} packets because they were sent on a different route.", + ignored_packets); + } + + // Feedback is expected to be sorted in send order. + std::sort(packet_result_vector.begin(), packet_result_vector.end(), + [](const PacketResult& lhs, const PacketResult& rhs) { + return lhs.sent_packet.sequence_number < + rhs.sent_packet.sequence_number; + }); + return ToTransportFeedback(std::move(packet_result_vector), + feedback_receive_time, supports_ecn); +} + +std::optional +TransportFeedbackAdapter::ToTransportFeedback( + std::vector packet_results, Timestamp feedback_receive_time, + bool supports_ecn) { + TransportPacketsFeedback msg; + msg.feedback_time = feedback_receive_time; + if (packet_results.empty()) { + return std::nullopt; + } + msg.packet_feedbacks = std::move(packet_results); + msg.data_in_flight = in_flight_.GetOutstandingData(network_route_); + msg.transport_supports_ecn = supports_ecn; + + return msg; +} + +void TransportFeedbackAdapter::SetNetworkRoute( + const rtc::NetworkRoute& network_route) { + network_route_ = network_route; +} + +DataSize TransportFeedbackAdapter::GetOutstandingData() const { + return in_flight_.GetOutstandingData(network_route_); +} + +std::optional TransportFeedbackAdapter::RetrievePacketFeedback( + const SsrcAndRtpSequencenumber& key, bool received) { + auto it = rtp_to_transport_sequence_number_.find(key); + if (it == rtp_to_transport_sequence_number_.end()) { + return std::nullopt; + } + return RetrievePacketFeedback(it->second, received); +} + +std::optional TransportFeedbackAdapter::RetrievePacketFeedback( + int64_t transport_seq_num, bool received) { + if (transport_seq_num > last_ack_seq_num_) { + // Starts at history_.begin() if last_ack_seq_num_ < 0, since any + // valid sequence number is >= 0. + for (auto it = history_.upper_bound(last_ack_seq_num_); + it != history_.upper_bound(transport_seq_num); ++it) { + in_flight_.RemoveInFlightPacketBytes(it->second); + } + last_ack_seq_num_ = transport_seq_num; + } + + auto it = history_.find(transport_seq_num); + if (it == history_.end()) { + LOG_WARN( + "Failed to lookup send time for packet with {}. Send time history too " + "small?", + transport_seq_num); + return std::nullopt; + } + + if (it->second.sent.send_time.IsInfinite()) { + // TODO(srte): Fix the tests that makes this happen and make this a + // DCHECK. + LOG_ERROR("Received feedback before packet was indicated as sent"); + return std::nullopt; + } + + PacketFeedback packet_feedback = it->second; + if (received) { + // Note: Lost packets are not removed from history because they might + // be reported as received by a later feedback. + rtp_to_transport_sequence_number_.erase( + {packet_feedback.ssrc, packet_feedback.rtp_sequence_number}); + history_.erase(it); + } + return packet_feedback; +} + +} // namespace webrtc diff --git a/src/qos/transport_feedback_adapter.cpp b/src/qos/transport_feedback_adapter.cpp deleted file mode 100644 index 9f0257f..0000000 --- a/src/qos/transport_feedback_adapter.cpp +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include "transport_feedback_adapter.h" - -#include - -#include -#include -#include -#include -#include - -#include "log.h" - -constexpr int64_t kSendTimeHistoryWindow = 60; - -void InFlightBytesTracker::AddInFlightPacketBytes( - const PacketFeedback& packet) { - auto it = in_flight_data_.find(packet.network_route); - if (it != in_flight_data_.end()) { - it->second += packet.sent.size; - } else { - in_flight_data_.insert({packet.network_route, packet.sent.size}); - } -} - -void InFlightBytesTracker::RemoveInFlightPacketBytes( - const PacketFeedback& packet) { - if (packet.sent.send_time == std::numeric_limits::max() || - packet.sent.send_time == std::numeric_limits::min()) - return; - auto it = in_flight_data_.find(packet.network_route); - if (it != in_flight_data_.end()) { - it->second -= packet.sent.size; - if (it->second == 0) in_flight_data_.erase(it); - } -} - -int64_t InFlightBytesTracker::GetOutstandingData( - const NetworkRoute& network_route) const { - auto it = in_flight_data_.find(network_route); - if (it != in_flight_data_.end()) { - return it->second; - } else { - return 0; - } -} - -// Comparator for consistent map with NetworkRoute as key. -bool InFlightBytesTracker::NetworkRouteComparator::operator()( - const NetworkRoute& a, const NetworkRoute& b) const { - if (a.local.network_id() != b.local.network_id()) - return a.local.network_id() < b.local.network_id(); - if (a.remote.network_id() != b.remote.network_id()) - return a.remote.network_id() < b.remote.network_id(); - - if (a.local.adapter_id() != b.local.adapter_id()) - return a.local.adapter_id() < b.local.adapter_id(); - if (a.remote.adapter_id() != b.remote.adapter_id()) - return a.remote.adapter_id() < b.remote.adapter_id(); - - if (a.local.uses_turn() != b.local.uses_turn()) - return a.local.uses_turn() < b.local.uses_turn(); - if (a.remote.uses_turn() != b.remote.uses_turn()) - return a.remote.uses_turn() < b.remote.uses_turn(); - - return a.connected < b.connected; -} - -TransportFeedbackAdapter::TransportFeedbackAdapter() = default; - -std::optional -TransportFeedbackAdapter::ProcessCongestionControlFeedback( - const CongestionControlFeedback& feedback, int64_t feedback_receive_time) { - if (feedback.packets().empty()) { - LOG_INFO("Empty congestion control feedback packet received"); - return std::nullopt; - } - if (current_offset_ == std::numeric_limits::max() || - current_offset_ == std::numeric_limits::min()) { - current_offset_ = feedback_receive_time; - } - int64_t feedback_delta = last_feedback_compact_ntp_time_ - ? (feedback.report_timestamp_compact_ntp() - - *last_feedback_compact_ntp_time_) - : 0; - last_feedback_compact_ntp_time_ = feedback.report_timestamp_compact_ntp(); - if (feedback_delta < 0) { - LOG_WARN("Unexpected feedback ntp time delta {}", feedback_delta); - current_offset_ = feedback_receive_time; - } else { - current_offset_ += feedback_delta; - } - - int ignored_packets = 0; - int failed_lookups = 0; - bool supports_ecn = true; - std::vector packet_result_vector; - for (const CongestionControlFeedback::PacketInfo& packet_info : - feedback.packets()) { - std::optional packet_feedback = - RetrievePacketFeedback({packet_info.ssrc, packet_info.sequence_number}, - /*received=*/packet_info.arrival_time_offset != - std::numeric_limits::min() && - packet_info.arrival_time_offset != - std::numeric_limits::max()); - if (!packet_feedback) { - ++failed_lookups; - continue; - } - if (packet_feedback->network_route != network_route_) { - ++ignored_packets; - continue; - } - PacketResult result; - result.sent_packet = packet_feedback->sent; - if (packet_info.arrival_time_offset != - std::numeric_limits::min() && - packet_info.arrival_time_offset != - std::numeric_limits::max()) { - result.receive_time = current_offset_ - packet_info.arrival_time_offset; - supports_ecn &= packet_info.ecn != rtc::EcnMarking::kNotEct; - } - result.ecn = packet_info.ecn; - packet_result_vector.push_back(result); - } - - if (failed_lookups > 0) { - LOG_WARN( - "Failed to lookup send time for {} packet {}. Packets reordered or " - "send time history too small?", - failed_lookups, (failed_lookups > 1 ? "s" : "")); - } - if (ignored_packets > 0) { - LOG_INFO("Ignoring {} packets because they were sent on a different route", - ignored_packets); - } - - // Feedback is expected to be sorted in send order. - std::sort(packet_result_vector.begin(), packet_result_vector.end(), - [](const PacketResult& lhs, const PacketResult& rhs) { - return lhs.sent_packet.sequence_number < - rhs.sent_packet.sequence_number; - }); - return ToTransportFeedback(std::move(packet_result_vector), - feedback_receive_time, supports_ecn); -} - -std::optional -TransportFeedbackAdapter::ToTransportFeedback( - std::vector packet_results, int64_t feedback_receive_time, - bool supports_ecn) { - TransportPacketsFeedback msg; - msg.feedback_time = feedback_receive_time; - if (packet_results.empty()) { - return std::nullopt; - } - msg.packet_feedbacks = std::move(packet_results); - msg.data_in_flight = in_flight_.GetOutstandingData(network_route_); - msg.transport_supports_ecn = supports_ecn; - - return msg; -} - -void TransportFeedbackAdapter::SetNetworkRoute( - const NetworkRoute& network_route) { - network_route_ = network_route; -} - -int64_t TransportFeedbackAdapter::GetOutstandingData() const { - return in_flight_.GetOutstandingData(network_route_); -} - -std::optional TransportFeedbackAdapter::RetrievePacketFeedback( - const SsrcAndRtpSequencenumber& key, bool received) { - auto it = rtp_to_transport_sequence_number_.find(key); - if (it == rtp_to_transport_sequence_number_.end()) { - return std::nullopt; - } - return RetrievePacketFeedback(it->second, received); -} - -std::optional TransportFeedbackAdapter::RetrievePacketFeedback( - int64_t transport_seq_num, bool received) { - if (transport_seq_num > last_ack_seq_num_) { - // Starts at history_.begin() if last_ack_seq_num_ < 0, since any - // valid sequence number is >= 0. - for (auto it = history_.upper_bound(last_ack_seq_num_); - it != history_.upper_bound(transport_seq_num); ++it) { - in_flight_.RemoveInFlightPacketBytes(it->second); - } - last_ack_seq_num_ = transport_seq_num; - } - - auto it = history_.find(transport_seq_num); - if (it == history_.end()) { - LOG_WARN( - "Failed to lookup send time for packet with {}. Send time history too " - "small?", - transport_seq_num); - return std::nullopt; - } - - if (it->second.sent.send_time == std::numeric_limits::max() || - it->second.sent.send_time == std::numeric_limits::min()) { - // TODO(srte): Fix the tests that makes this happen and make this a - // DCHECK. - LOG_ERROR("Received feedback before packet was indicated as sent"); - return std::nullopt; - } - - PacketFeedback packet_feedback = it->second; - if (received) { - // Note: Lost packets are not removed from history because they might - // be reported as received by a later feedback. - rtp_to_transport_sequence_number_.erase( - {packet_feedback.ssrc, packet_feedback.rtp_sequence_number}); - history_.erase(it); - } - return packet_feedback; -} diff --git a/src/qos/transport_feedback_adapter.h b/src/qos/transport_feedback_adapter.h index 6510625..778409d 100644 --- a/src/qos/transport_feedback_adapter.h +++ b/src/qos/transport_feedback_adapter.h @@ -1,11 +1,15 @@ /* - * @Author: DI JUNKUN - * @Date: 2025-01-13 - * Copyright (c) 2025 by DI JUNKUN, All Rights Reserved. + * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. */ -#ifndef _TRANSPORT_FEEDBACK_ADAPTER_H_ -#define _TRANSPORT_FEEDBACK_ADAPTER_H_ +#ifndef MODULES_CONGESTION_CONTROLLER_RTP_TRANSPORT_FEEDBACK_ADAPTER_H_ +#define MODULES_CONGESTION_CONTROLLER_RTP_TRANSPORT_FEEDBACK_ADAPTER_H_ #include #include @@ -15,22 +19,28 @@ #include #include "api/transport/network_types.h" +#include "api/units/data_size.h" +#include "api/units/timestamp.h" #include "congestion_control_feedback.h" -#include "network_route.h" +#include "rtc_base/network/sent_packet.h" +#include "rtc_base/network_route.h" #include "rtc_base/numerics/sequence_number_unwrapper.h" +#include "rtp_packet_to_send.h" + +namespace webrtc { struct PacketFeedback { PacketFeedback() = default; // Time corresponding to when this object was created. - int64_t creation_time = std::numeric_limits::min(); + Timestamp creation_time = Timestamp::MinusInfinity(); SentPacket sent; // Time corresponding to when the packet was received. Timestamped with the - // receiver's clock. For unreceived packet, - // std::numeric_limits::max() is used. - int64_t receive_time = std::numeric_limits::max(); + // receiver's clock. For unreceived packet, Timestamp::PlusInfinity() is + // used. + Timestamp receive_time = Timestamp::PlusInfinity(); // The network route that this packet is associated with. - NetworkRoute network_route; + rtc::NetworkRoute network_route; uint32_t ssrc = 0; uint16_t rtp_sequence_number = 0; @@ -40,13 +50,14 @@ class InFlightBytesTracker { public: void AddInFlightPacketBytes(const PacketFeedback& packet); void RemoveInFlightPacketBytes(const PacketFeedback& packet); - int64_t GetOutstandingData(const NetworkRoute& network_route) const; + DataSize GetOutstandingData(const rtc::NetworkRoute& network_route) const; private: struct NetworkRouteComparator { - bool operator()(const NetworkRoute& a, const NetworkRoute& b) const; + bool operator()(const rtc::NetworkRoute& a, + const rtc::NetworkRoute& b) const; }; - std::map in_flight_data_; + std::map in_flight_data_; }; // TransportFeedbackAdapter converts RTCP feedback packets to RTCP agnostic per @@ -58,12 +69,23 @@ class TransportFeedbackAdapter { public: TransportFeedbackAdapter(); + void AddPacket(const RtpPacketToSend& packet, + const PacedPacketInfo& pacing_info, size_t overhead_bytes, + Timestamp creation_time); + + std::optional ProcessSentPacket( + const rtc::SentPacket& sent_packet); + + std::optional ProcessTransportFeedback( + const rtcp::TransportFeedback& feedback, Timestamp feedback_receive_time); + std::optional ProcessCongestionControlFeedback( - const CongestionControlFeedback& feedback, int64_t feedback_receive_time); + const rtcp::CongestionControlFeedback& feedback, + Timestamp feedback_receive_time); - void SetNetworkRoute(const NetworkRoute& network_route); + void SetNetworkRoute(const rtc::NetworkRoute& network_route); - int64_t GetOutstandingData() const; + DataSize GetOutstandingData() const; private: enum class SendTimeHistoryStatus { kNotAdded, kOk, kDuplicate }; @@ -83,26 +105,25 @@ class TransportFeedbackAdapter { std::optional RetrievePacketFeedback( const SsrcAndRtpSequencenumber& key, bool received); std::optional ToTransportFeedback( - std::vector packet_results, int64_t feedback_receive_time, + std::vector packet_results, Timestamp feedback_receive_time, bool supports_ecn); - int64_t pending_untracked_size_ = 0; - int64_t last_send_time_ = std::numeric_limits::min(); - int64_t last_untracked_send_time_ = std::numeric_limits::min(); + DataSize pending_untracked_size_ = DataSize::Zero(); + Timestamp last_send_time_ = Timestamp::MinusInfinity(); + Timestamp last_untracked_send_time_ = Timestamp::MinusInfinity(); RtpSequenceNumberUnwrapper seq_num_unwrapper_; // Sequence numbers are never negative, using -1 as it always < a real // sequence number. int64_t last_ack_seq_num_ = -1; InFlightBytesTracker in_flight_; - NetworkRoute network_route_; + rtc::NetworkRoute network_route_; - int64_t current_offset_ = std::numeric_limits::min(); + Timestamp current_offset_ = Timestamp::MinusInfinity(); // `last_transport_feedback_base_time` is only used for transport feedback to // track base time. - int64_t last_transport_feedback_base_time_ = - std::numeric_limits::min(); + Timestamp last_transport_feedback_base_time_ = Timestamp::MinusInfinity(); // Used by RFC 8888 congestion control feedback to track base time. std::optional last_feedback_compact_ntp_time_; @@ -112,4 +133,6 @@ class TransportFeedbackAdapter { std::map history_; }; -#endif \ No newline at end of file +} // namespace webrtc + +#endif // MODULES_CONGESTION_CONTROLLER_RTP_TRANSPORT_FEEDBACK_ADAPTER_H_ diff --git a/src/rtcp/rtcp_packet/rtcp_packet_info.h b/src/rtcp/rtcp_packet/rtcp_packet_info.h index c40213f..40e315f 100644 --- a/src/rtcp/rtcp_packet/rtcp_packet_info.h +++ b/src/rtcp/rtcp_packet/rtcp_packet_info.h @@ -24,7 +24,8 @@ struct RtcpPacketInfo { // std::vector report_block_datas; std::optional rtt; uint32_t receiver_estimated_max_bitrate_bps = 0; - std::optional congestion_control_feedback; + std::optional + congestion_control_feedback; // std::optional target_bitrate_allocation; // std::optional network_state_estimate; // std::unique_ptr loss_notification; diff --git a/src/rtp/rtp_packet/rtp_packet.cpp b/src/rtp/rtp_packet/rtp_packet.cpp index dc7dbc9..e982f7d 100644 --- a/src/rtp/rtp_packet/rtp_packet.cpp +++ b/src/rtp/rtp_packet/rtp_packet.cpp @@ -38,6 +38,8 @@ RtpPacket::RtpPacket() : buffer_(new uint8_t[DEFAULT_MTU]), size_(DEFAULT_MTU) { ParseRtpData(); } +RtpPacket::RtpPacket(uint32_t size) : buffer_(new uint8_t[size]), size_(size) {} + RtpPacket::RtpPacket(const uint8_t *buffer, uint32_t size) { if (size > 0) { buffer_ = (uint8_t *)malloc(size); diff --git a/src/rtp/rtp_packet/rtp_packet.h b/src/rtp/rtp_packet/rtp_packet.h index 2848bc1..43995b4 100644 --- a/src/rtp/rtp_packet/rtp_packet.h +++ b/src/rtp/rtp_packet/rtp_packet.h @@ -184,6 +184,7 @@ class RtpPacket { public: RtpPacket(); + RtpPacket(uint32_t size); RtpPacket(const uint8_t *buffer, uint32_t size); RtpPacket(const RtpPacket &rtp_packet); RtpPacket(RtpPacket &&rtp_packet); @@ -220,6 +221,31 @@ class RtpPacket { memcpy(extension_data_, extension_data, extension_len_); } + void SetAbsoluteSendTimestamp(uint32_t abs_send_time) { + // Absolute Send Time is a 24-bit field, we need to ensure it fits in 24 + // bits + abs_send_time &= 0x00FFFFFF; + + // Allocate memory for the extension data if not already allocated + if (extension_data_ == nullptr) { + extension_data_ = new uint8_t[4]; // 2 bytes for profile, 2 bytes for + // length, 3 bytes for abs_send_time + extension_len_ = 4; + } + + // Set the extension profile to 0xBEDE (one-byte header) + extension_profile_ = 0xBEDE; + + // Set the length of the extension data (in 32-bit words minus one) + extension_data_[0] = 0x00; + extension_data_[1] = 0x02; // 2 words (8 bytes) + + // Set the absolute send time in the extension data + extension_data_[2] = (abs_send_time >> 16) & 0xFF; + extension_data_[3] = (abs_send_time >> 8) & 0xFF; + extension_data_[4] = abs_send_time & 0xFF; + } + public: typedef struct { uint8_t forbidden_bit : 1; @@ -323,6 +349,18 @@ class RtpPacket { return extension_data_; } + uint32_t GetAbsoluteSendTimestamp(uint32_t *abs_send_time) const { + if (extension_data_ == nullptr || extension_len_ < 4) { + return 0; + } + + // Absolute Send Time is a 24-bit field + *abs_send_time = (extension_data_[2] << 16) | (extension_data_[3] << 8) | + extension_data_[4]; + + return *abs_send_time; + } + uint8_t FecSymbolId() { return fec_symbol_id_; } uint8_t FecSourceSymbolNum() { return fec_source_symbol_num_; } @@ -355,6 +393,7 @@ class RtpPacket { // Entire RTP buffer const uint8_t *Buffer() const { return buffer_; } size_t Size() const { return size_; } + size_t size() const { return size_; } // NAL NAL_UNIT_TYPE NalUnitType() { diff --git a/src/rtp/rtp_packet/rtp_packet_received.cpp b/src/rtp/rtp_packet/rtp_packet_received.cpp index 19e1609..eb1dddf 100644 --- a/src/rtp/rtp_packet/rtp_packet_received.cpp +++ b/src/rtp/rtp_packet/rtp_packet_received.cpp @@ -20,8 +20,7 @@ namespace webrtc { RtpPacketReceived::RtpPacketReceived() = default; -RtpPacketReceived::RtpPacketReceived( - webrtc::Timestamp arrival_time /*= webrtc::Timestamp::MinusInfinity()*/) +RtpPacketReceived::RtpPacketReceived(webrtc::Timestamp arrival_time) : RtpPacket(), arrival_time_(arrival_time) {} RtpPacketReceived::RtpPacketReceived(const RtpPacketReceived& packet) = default; RtpPacketReceived::RtpPacketReceived(RtpPacketReceived&& packet) = default; diff --git a/src/rtp/rtp_packet/rtp_packet_received.h b/src/rtp/rtp_packet/rtp_packet_received.h index 02cf0be..2b44c98 100644 --- a/src/rtp/rtp_packet/rtp_packet_received.h +++ b/src/rtp/rtp_packet/rtp_packet_received.h @@ -17,8 +17,8 @@ #include "api/array_view.h" #include "api/ref_counted_base.h" #include "api/scoped_refptr.h" +#include "api/transport/ecn_marking.h" #include "api/units/timestamp.h" -#include "rtc_base/network/ecn_marking.h" #include "rtp_header.h" #include "rtp_packet.h" @@ -29,8 +29,7 @@ namespace webrtc { class RtpPacketReceived : public RtpPacket { public: RtpPacketReceived(); - explicit RtpPacketReceived( - webrtc::Timestamp arrival_time = webrtc::Timestamp::MinusInfinity()); + explicit RtpPacketReceived(webrtc::Timestamp arrival_time); RtpPacketReceived(const RtpPacketReceived& packet); RtpPacketReceived(RtpPacketReceived&& packet); @@ -50,8 +49,8 @@ class RtpPacketReceived : public RtpPacket { // Explicit Congestion Notification (ECN), RFC-3168, Section 5. // Used by L4S: https://www.rfc-editor.org/rfc/rfc9331.html - rtc::EcnMarking ecn() const { return ecn_; } - void set_ecn(rtc::EcnMarking ecn) { ecn_ = ecn; } + EcnMarking ecn() const { return ecn_; } + void set_ecn(EcnMarking ecn) { ecn_ = ecn; } // Flag if packet was recovered via RTX or FEC. bool recovered() const { return recovered_; } @@ -73,7 +72,7 @@ class RtpPacketReceived : public RtpPacket { private: webrtc::Timestamp arrival_time_ = Timestamp::MinusInfinity(); - rtc::EcnMarking ecn_ = rtc::EcnMarking::kNotEct; + EcnMarking ecn_ = EcnMarking::kNotEct; int payload_type_frequency_ = 0; bool recovered_ = false; rtc::scoped_refptr additional_data_; diff --git a/src/transport/ice_transport.cpp b/src/transport/ice_transport.cpp index f6e7f3f..4110cba 100644 --- a/src/transport/ice_transport.cpp +++ b/src/transport/ice_transport.cpp @@ -260,7 +260,7 @@ void IceTransport::OnReceiveBuffer(NiceAgent *agent, guint stream_id, bool IceTransport::ParseRtcpPacket(const uint8_t *buffer, size_t size, RtcpPacketInfo *rtcp_packet_info) { - RtcpCommonHeader rtcp_block; + webrtc::rtcp::CommonHeader rtcp_block; // If a sender report is received but no DLRR, we need to reset the // roundTripTime stat according to the standard, see // https://www.w3.org/TR/webrtc-stats/#dom-rtcremoteoutboundrtpstreamstats-roundtriptime @@ -287,7 +287,7 @@ bool IceTransport::ParseRtcpPacket(const uint8_t *buffer, size_t size, break; case RtcpPacket::PAYLOAD_TYPE::TCC: switch (rtcp_block.fmt()) { - case CongestionControlFeedback::kFeedbackMessageType: + case webrtc::rtcp::CongestionControlFeedback::kFeedbackMessageType: LOG_INFO("Congestion Control Feedback"); valid = HandleCongestionControlFeedback(rtcp_block, rtcp_packet_info); break; @@ -350,16 +350,17 @@ bool IceTransport::ParseRtcpPacket(const uint8_t *buffer, size_t size, } bool IceTransport::HandleCongestionControlFeedback( - const RtcpCommonHeader &rtcp_block, RtcpPacketInfo *rtcp_packet_info) { - CongestionControlFeedback feedback; + const webrtc::rtcp::CommonHeader &rtcp_block, + RtcpPacketInfo *rtcp_packet_info) { + webrtc::rtcp::CongestionControlFeedback feedback; if (!feedback.Parse(rtcp_block) || feedback.packets().empty()) { return false; } - // uint32_t first_media_source_ssrc = feedback.packets()[0].ssrc; - // if (first_media_source_ssrc == local_media_ssrc() || - // registered_ssrcs_.contains(first_media_source_ssrc)) { - // rtcp_packet_info->congestion_control_feedback.emplace(std::move(feedback)); - // } + uint32_t first_media_source_ssrc = feedback.packets()[0].ssrc; + if (first_media_source_ssrc == local_media_ssrc() || + registered_ssrcs_.contains(first_media_source_ssrc)) { + rtcp_packet_info->congestion_control_feedback.emplace(std::move(feedback)); + } return true; } diff --git a/src/transport/ice_transport.h b/src/transport/ice_transport.h index 9b9219e..82e87ec 100644 --- a/src/transport/ice_transport.h +++ b/src/transport/ice_transport.h @@ -174,8 +174,9 @@ class IceTransport { bool ParseRtcpPacket(const uint8_t *buffer, size_t size, RtcpPacketInfo *rtcp_packet_info); - bool HandleCongestionControlFeedback(const RtcpCommonHeader &rtcp_block, - RtcpPacketInfo *rtcp_packet_info); + bool HandleCongestionControlFeedback( + const webrtc::rtcp::CommonHeader &rtcp_block, + RtcpPacketInfo *rtcp_packet_info); private: bool use_trickle_ice_ = true;