From ebfeaf4754fe563cd840edcafbddc28c29d74456 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Tue, 4 Mar 2025 11:17:19 +0800 Subject: [PATCH] [fix] fix receiver report building and parsing --- src/channel/audio_channel_send.h | 4 +- src/channel/data_channel_send.h | 4 +- src/channel/rtp_channel/rtp_audio_sender.h | 2 + src/channel/rtp_channel/rtp_data_sender.h | 2 + .../rtp_channel/rtp_video_receiver.cpp | 106 ++++++++++-------- src/channel/rtp_channel/rtp_video_sender.cpp | 12 ++ src/channel/rtp_channel/rtp_video_sender.h | 2 + src/channel/video_channel_send.h | 6 + src/rtcp/rtcp_packet/receiver_report.cpp | 22 +++- src/rtcp/rtcp_packet/receiver_report.h | 5 + src/rtcp/rtcp_packet/sender_report.h | 8 +- src/transport/ice_transport.cpp | 16 ++- src/transport/ice_transport.h | 3 + src/transport/ice_transport_controller.cpp | 7 ++ src/transport/ice_transport_controller.h | 1 + 15 files changed, 140 insertions(+), 60 deletions(-) diff --git a/src/channel/audio_channel_send.h b/src/channel/audio_channel_send.h index db05d4e..b8e3439 100644 --- a/src/channel/audio_channel_send.h +++ b/src/channel/audio_channel_send.h @@ -29,7 +29,9 @@ class AudioChannelSend { return 0; } - int SendAudio(char *data, size_t size); + int SendAudio(char* data, size_t size); + + void OnReceiverReport(const ReceiverReport& receiver_report) {} private: std::shared_ptr ice_agent_ = nullptr; diff --git a/src/channel/data_channel_send.h b/src/channel/data_channel_send.h index 4fe3b0f..2599453 100644 --- a/src/channel/data_channel_send.h +++ b/src/channel/data_channel_send.h @@ -29,7 +29,9 @@ class DataChannelSend { return 0; } - int SendData(const char *data, size_t size); + int SendData(const char* data, size_t size); + + void OnReceiverReport(const ReceiverReport& receiver_report) {} private: std::shared_ptr ice_agent_ = nullptr; diff --git a/src/channel/rtp_channel/rtp_audio_sender.h b/src/channel/rtp_channel/rtp_audio_sender.h index b02de7d..a57bd11 100644 --- a/src/channel/rtp_channel/rtp_audio_sender.h +++ b/src/channel/rtp_channel/rtp_audio_sender.h @@ -10,6 +10,7 @@ #include #include "io_statistics.h" +#include "receiver_report.h" #include "ringbuffer.h" #include "rtp_packet.h" #include "rtp_statistics.h" @@ -26,6 +27,7 @@ class RtpAudioSender : public ThreadBase { void Enqueue(std::vector> rtp_packets); void SetSendDataFunc(std::function data_send_func); uint32_t GetSsrc() { return ssrc_; } + void OnReceiverReport(const ReceiverReport &receiver_report) {} private: int SendRtpPacket(std::shared_ptr rtp_packet); diff --git a/src/channel/rtp_channel/rtp_data_sender.h b/src/channel/rtp_channel/rtp_data_sender.h index d23cc60..b629a05 100644 --- a/src/channel/rtp_channel/rtp_data_sender.h +++ b/src/channel/rtp_channel/rtp_data_sender.h @@ -10,6 +10,7 @@ #include #include "io_statistics.h" +#include "receiver_report.h" #include "ringbuffer.h" #include "rtp_packet.h" #include "rtp_statistics.h" @@ -26,6 +27,7 @@ class RtpDataSender : public ThreadBase { void Enqueue(std::vector> rtp_packets); void SetSendDataFunc(std::function data_send_func); uint32_t GetSsrc() { return ssrc_; } + void OnReceiverReport(const ReceiverReport &receiver_report) {} private: private: diff --git a/src/channel/rtp_channel/rtp_video_receiver.cpp b/src/channel/rtp_channel/rtp_video_receiver.cpp index 83d6c1b..372ab90 100644 --- a/src/channel/rtp_channel/rtp_video_receiver.cpp +++ b/src/channel/rtp_channel/rtp_video_receiver.cpp @@ -101,57 +101,39 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) { webrtc::Timestamp now = clock_->CurrentTime(); remote_ssrc_ = rtp_packet.Ssrc(); uint16_t sequence_number = rtp_packet.SequenceNumber(); - if (last_receive_time_.has_value() == 0) { - extended_high_seq_num_ = sequence_number; + --cumulative_loss_; + if (!last_receive_time_.has_value()) { + last_extended_high_seq_num_ = sequence_number - 1; + extended_high_seq_num_ = sequence_number - 1; } cumulative_loss_ += sequence_number - extended_high_seq_num_; extended_high_seq_num_ = sequence_number; - // Calculate fraction lost. - int64_t exp_since_last = extended_high_seq_num_ - last_extended_high_seq_num_; - int32_t lost_since_last = cumulative_loss_ - last_report_cumulative_loss_; - if (exp_since_last > 0 && lost_since_last > 0) { - // Scale 0 to 255, where 255 is 100% loss. - fraction_lost_ = 255 * lost_since_last / exp_since_last; - } - cumulative_lost_ = cumulative_loss_ + cumulative_loss_rtcp_offset_; - if (cumulative_lost_ < 0) { - // Clamp to zero. Work around to accommodate for senders that misbehave with - // negative cumulative loss. - cumulative_lost_ = 0; - cumulative_loss_rtcp_offset_ = -cumulative_loss_; - } - if (cumulative_lost_ > 0x7fffff) { - // Packets lost is a 24 bit signed field, and thus should be clamped, as - // described in https://datatracker.ietf.org/doc/html/rfc3550#appendix-A.3 - cumulative_lost_ = 0x7fffff; + if (rtp_packet_received.Timestamp() != last_received_timestamp_) { + webrtc::TimeDelta receive_diff = now - *last_receive_time_; + uint32_t receive_diff_rtp = + (receive_diff * rtp_packet_received.payload_type_frequency()) + .seconds(); + int32_t time_diff_samples = + receive_diff_rtp - + (rtp_packet_received.Timestamp() - last_received_timestamp_); + + ReviseFrequencyAndJitter(rtp_packet_received.payload_type_frequency()); + + // lib_jingle sometimes deliver crazy jumps in TS for the same stream. + // If this happens, don't update jitter value. Use 5 secs video frequency + // as the threshold. + if (time_diff_samples < 5 * kVideoPayloadTypeFrequency && + time_diff_samples > -5 * kVideoPayloadTypeFrequency) { + // Note we calculate in Q4 to avoid using float. + int32_t jitter_diff_q4 = (std::abs(time_diff_samples) << 4) - jitter_q4_; + jitter_q4_ += ((jitter_diff_q4 + 8) >> 4); + } + + jitter_ = jitter_q4_ >> 4; } - webrtc::TimeDelta receive_diff = now - *last_receive_time_; - uint32_t receive_diff_rtp = - (receive_diff * rtp_packet_received.payload_type_frequency()) - .seconds(); - int32_t time_diff_samples = - receive_diff_rtp - - (rtp_packet_received.Timestamp() - last_received_timestamp_); - - ReviseFrequencyAndJitter(rtp_packet_received.payload_type_frequency()); - - // lib_jingle sometimes deliver crazy jumps in TS for the same stream. - // If this happens, don't update jitter value. Use 5 secs video frequency - // as the threshold. - if (time_diff_samples < 5 * kVideoPayloadTypeFrequency && - time_diff_samples > -5 * kVideoPayloadTypeFrequency) { - // Note we calculate in Q4 to avoid using float. - int32_t jitter_diff_q4 = (std::abs(time_diff_samples) << 4) - jitter_q4_; - jitter_q4_ += ((jitter_diff_q4 + 8) >> 4); - } - - jitter_ = jitter_q4_ >> 4; - - last_extended_high_seq_num_ = extended_high_seq_num_; - last_report_cumulative_loss_ = cumulative_loss_; last_received_timestamp_ = rtp_packet_received.Timestamp(); last_receive_time_ = now; @@ -592,24 +574,50 @@ void RtpVideoReceiver::ReviseFrequencyAndJitter(int payload_type_frequency) { void RtpVideoReceiver::SendRR() { uint32_t now = CompactNtp(clock_->CurrentNtpTime()); + + // Calculate fraction lost. + int64_t exp_since_last = extended_high_seq_num_ - last_extended_high_seq_num_; + int32_t lost_since_last = cumulative_loss_ - last_report_cumulative_loss_; + if (exp_since_last > 0 && lost_since_last > 0) { + // Scale 0 to 255, where 255 is 100% loss. + fraction_lost_ = 255 * lost_since_last / exp_since_last; + } else { + fraction_lost_ = 0; + } + + cumulative_lost_ = cumulative_loss_ + cumulative_loss_rtcp_offset_; + if (cumulative_lost_ < 0) { + // Clamp to zero. Work around to accommodate for senders that misbehave with + // negative cumulative loss. + cumulative_lost_ = 0; + cumulative_loss_rtcp_offset_ = -cumulative_loss_; + } + if (cumulative_lost_ > 0x7fffff) { + // Packets lost is a 24 bit signed field, and thus should be clamped, as + // described in https://datatracker.ietf.org/doc/html/rfc3550#appendix-A.3 + cumulative_lost_ = 0x7fffff; + } + uint32_t receive_time = last_arrival_ntp_timestamp; uint32_t delay_since_last_sr = now - receive_time; ReceiverReport rtcp_rr; RtcpReportBlock report; - report.SetMediaSsrc(ssrc_); - report.SetMediaSsrc(fraction_lost_); + report.SetMediaSsrc(remote_ssrc_); report.SetFractionLost(fraction_lost_); + report.SetCumulativeLost(cumulative_lost_); report.SetExtHighestSeqNum(extended_high_seq_num_); report.SetJitter(jitter_); report.SetLastSr(last_remote_ntp_timestamp); report.SetDelayLastSr(delay_since_last_sr); - + rtcp_rr.SetSenderSsrc(ssrc_); rtcp_rr.SetReportBlock(report); - rtcp_rr.Build(); SendRtcpRR(rtcp_rr); + + last_extended_high_seq_num_ = extended_high_seq_num_; + last_report_cumulative_loss_ = cumulative_loss_; } void RtpVideoReceiver::RtcpThread() { @@ -626,7 +634,7 @@ void RtpVideoReceiver::RtcpThread() { auto elapsed = std::chrono::duration_cast( now - last_send_rtcp_rr_ts_) .count(); - if (elapsed >= rtcp_rr_interval_ms_) { + if (elapsed >= rtcp_rr_interval_ms_ && last_receive_time_.has_value()) { SendRR(); last_send_rtcp_rr_ts_ = now; } diff --git a/src/channel/rtp_channel/rtp_video_sender.cpp b/src/channel/rtp_channel/rtp_video_sender.cpp index eaa40e8..f5e5961 100644 --- a/src/channel/rtp_channel/rtp_video_sender.cpp +++ b/src/channel/rtp_channel/rtp_video_sender.cpp @@ -173,4 +173,16 @@ bool RtpVideoSender::Process() { } return true; +} + +void RtpVideoSender::OnReceiverReport(const ReceiverReport& receiver_report) { + std::vector reports = receiver_report.GetReportBlocks(); + + for (auto r : reports) { + LOG_WARN( + "r_ssrc [{}], f_lost [{}], c_lost [{}], h_seq [{}], jitter [{}], " + "lsr [{}], dlsr [{}] ", + r.SourceSsrc(), r.FractionLost() / 255.0, r.CumulativeLost(), + r.ExtendedHighSeqNum(), r.Jitter(), r.LastSr(), r.DelaySinceLastSr()); + } } \ No newline at end of file diff --git a/src/channel/rtp_channel/rtp_video_sender.h b/src/channel/rtp_channel/rtp_video_sender.h index 6dcdd69..2a171c6 100644 --- a/src/channel/rtp_channel/rtp_video_sender.h +++ b/src/channel/rtp_channel/rtp_video_sender.h @@ -6,6 +6,7 @@ #include "api/clock/clock.h" #include "clock/system_clock.h" #include "io_statistics.h" +#include "receiver_report.h" #include "ringbuffer.h" #include "rtp_packet.h" #include "rtp_packet_history.h" @@ -28,6 +29,7 @@ class RtpVideoSender : public ThreadBase { void SetOnSentPacketFunc( std::function on_sent_packet_func); uint32_t GetSsrc() { return ssrc_; } + void OnReceiverReport(const ReceiverReport &receiver_report); private: int SendRtpPacket( diff --git a/src/channel/video_channel_send.h b/src/channel/video_channel_send.h index 62a6991..92278dc 100644 --- a/src/channel/video_channel_send.h +++ b/src/channel/video_channel_send.h @@ -45,6 +45,12 @@ class VideoChannelSend { Timestamp recv_ts, const webrtc::rtcp::CongestionControlFeedback& feedback); + void OnReceiverReport(const ReceiverReport& receiver_report) { + if (rtp_video_sender_) { + rtp_video_sender_->OnReceiverReport(receiver_report); + } + } + private: void PostUpdates(webrtc::NetworkControlUpdate update); void UpdateControlState(); diff --git a/src/rtcp/rtcp_packet/receiver_report.cpp b/src/rtcp/rtcp_packet/receiver_report.cpp index 0ddbc7d..1dba3f9 100644 --- a/src/rtcp/rtcp_packet/receiver_report.cpp +++ b/src/rtcp/rtcp_packet/receiver_report.cpp @@ -21,7 +21,7 @@ void ReceiverReport::SetReportBlocks( const uint8_t *ReceiverReport::Build() { size_t buffer_size = - DEFAULT_SR_SIZE + reports_.size() * RtcpReportBlock::kLength; + DEFAULT_RR_SIZE + reports_.size() * RtcpReportBlock::kLength; if (!buffer_ || buffer_size != size_) { delete[] buffer_; buffer_ = nullptr; @@ -34,6 +34,12 @@ const uint8_t *ReceiverReport::Build() { rtcp_common_header_.Create(DEFAULT_RTCP_VERSION, 0, DEFAULT_RR_BLOCK_NUM, RTCP_TYPE::RR, DEFAULT_RR_SIZE, buffer_); + buffer_[pos] = sender_ssrc_ >> 24 & 0xFF; + buffer_[pos + 1] = sender_ssrc_ >> 16 & 0xFF; + buffer_[pos + 2] = sender_ssrc_ >> 8 & 0xFF; + buffer_[pos + 3] = sender_ssrc_ & 0xFF; + pos += 4; + for (const auto &report : reports_) { pos += report.Create(buffer_ + pos); } @@ -43,12 +49,18 @@ const uint8_t *ReceiverReport::Build() { size_t ReceiverReport::Parse(const RtcpCommonHeader &packet) { reports_.clear(); - size_t pos = packet.payload_size_bytes(); + rtcp_common_header_ = packet; - for (int i = 0; i < rtcp_common_header_.fmt(); i++) { + const uint8_t *payload = packet.payload(); + const uint8_t *payload_end = packet.payload() + packet.payload_size_bytes(); + size_t pos = 0; + sender_ssrc_ = (payload[pos] << 24) + (payload[pos + 1] << 16) + + (payload[pos + 2] << 8) + payload[pos + 3]; + pos += 4; + while (payload + pos < payload_end) { RtcpReportBlock report; - pos += report.Parse(buffer_ + pos); - reports_.emplace_back(std::move(report)); + pos += report.Parse(payload + pos); + reports_.push_back(std::move(report)); } return pos; diff --git a/src/rtcp/rtcp_packet/receiver_report.h b/src/rtcp/rtcp_packet/receiver_report.h index e415f6f..2dc9e80 100644 --- a/src/rtcp/rtcp_packet/receiver_report.h +++ b/src/rtcp/rtcp_packet/receiver_report.h @@ -45,9 +45,13 @@ class ReceiverReport { ~ReceiverReport(); public: + void SetSenderSsrc(uint32_t ssrc) { sender_ssrc_ = ssrc; } void SetReportBlock(RtcpReportBlock &rtcp_report_block); void SetReportBlocks(std::vector &rtcp_report_blocks); + uint32_t SenderSsrc() const { return sender_ssrc_; } + std::vector GetReportBlocks() const { return reports_; } + const uint8_t *Build(); size_t Parse(const RtcpCommonHeader &packet); @@ -55,6 +59,7 @@ class ReceiverReport { size_t Size() const { return size_; } private: + uint32_t sender_ssrc_; RtcpCommonHeader rtcp_common_header_; std::vector reports_; diff --git a/src/rtcp/rtcp_packet/sender_report.h b/src/rtcp/rtcp_packet/sender_report.h index 6f6394a..d506cb4 100644 --- a/src/rtcp/rtcp_packet/sender_report.h +++ b/src/rtcp/rtcp_packet/sender_report.h @@ -64,7 +64,10 @@ class SenderReport { ~SenderReport(); public: - void SetSenderSsrc(uint32_t ssrc) { sender_info_.sender_ssrc = ssrc; } + void SetSenderSsrc(uint32_t ssrc) { + sender_ssrc_ = ssrc; + sender_info_.sender_ssrc = ssrc; + } void SetNtpTimestamp(uint64_t ntp_timestamp) { sender_info_.ntp_ts_msw = ntp_timestamp / webrtc::NtpTime::kFractionsPerSecond; @@ -81,7 +84,7 @@ class SenderReport { void SetReportBlock(RtcpReportBlock &rtcp_report_block); void SetReportBlocks(std::vector &rtcp_report_blocks); - uint32_t SenderSsrc() const { return sender_info_.sender_ssrc; } + uint32_t SenderSsrc() const { return sender_ssrc_; } uint32_t NtpTimestamp() const { return (sender_info_.ntp_ts_msw << 16) | sender_info_.ntp_ts_lsw >> 16; } @@ -100,6 +103,7 @@ class SenderReport { size_t Size() const { return size_; } private: + uint32_t sender_ssrc_; RtcpCommonHeader rtcp_common_header_; SenderInfo sender_info_; std::vector reports_; diff --git a/src/transport/ice_transport.cpp b/src/transport/ice_transport.cpp index 5ff029e..50aed95 100644 --- a/src/transport/ice_transport.cpp +++ b/src/transport/ice_transport.cpp @@ -230,8 +230,7 @@ bool IceTransport::ParseRtcpPacket(const uint8_t *buffer, size_t size, // received_blocks[rtcp_packet_info->remote_ssrc].sender_report = true; break; case RtcpPacket::RtcpPayloadType::RR: - LOG_INFO("Receiver report"); - // valid = HandleReceiverReport(rtcp_block, rtcp_packet_info); + valid = HandleReceiverReport(rtcp_block, rtcp_packet_info); break; case RtpFeedback::kPacketType: switch (rtcp_block.fmt()) { @@ -313,6 +312,19 @@ bool IceTransport::HandleSenderReport(const RtcpCommonHeader &rtcp_block, return true; } +bool IceTransport::HandleReceiverReport(const RtcpCommonHeader &rtcp_block, + RtcpPacketInfo *rtcp_packet_info) { + ReceiverReport receiver_report; + if (!receiver_report.Parse(rtcp_block)) { + return false; + } + + if (ice_transport_controller_) { + ice_transport_controller_->OnReceiverReport(receiver_report); + } + return true; +} + bool IceTransport::HandleCongestionControlFeedback( const RtcpCommonHeader &rtcp_block, RtcpPacketInfo *rtcp_packet_info) { webrtc::rtcp::CongestionControlFeedback feedback; diff --git a/src/transport/ice_transport.h b/src/transport/ice_transport.h index 1d5ab01..c902d58 100644 --- a/src/transport/ice_transport.h +++ b/src/transport/ice_transport.h @@ -133,6 +133,9 @@ class IceTransport { bool HandleSenderReport(const RtcpCommonHeader &rtcp_block, RtcpPacketInfo *rtcp_packet_info); + bool HandleReceiverReport(const RtcpCommonHeader &rtcp_block, + RtcpPacketInfo *rtcp_packet_info); + bool HandleCongestionControlFeedback(const RtcpCommonHeader &rtcp_block, RtcpPacketInfo *rtcp_packet_info); diff --git a/src/transport/ice_transport_controller.cpp b/src/transport/ice_transport_controller.cpp index a233861..03c88bf 100644 --- a/src/transport/ice_transport_controller.cpp +++ b/src/transport/ice_transport_controller.cpp @@ -338,6 +338,13 @@ void IceTransportController::OnSenderReport(const SenderReport& sender_report) { data_channel_receive_->OnSenderReport(sender_report); } +void IceTransportController::OnReceiverReport( + const ReceiverReport& receiver_report) { + video_channel_send_->OnReceiverReport(receiver_report); + audio_channel_send_->OnReceiverReport(receiver_report); + data_channel_send_->OnReceiverReport(receiver_report); +} + void IceTransportController::OnCongestionControlFeedback( const webrtc::rtcp::CongestionControlFeedback& feedback) { std::optional feedback_msg = diff --git a/src/transport/ice_transport_controller.h b/src/transport/ice_transport_controller.h index bbf8afe..dd47268 100644 --- a/src/transport/ice_transport_controller.h +++ b/src/transport/ice_transport_controller.h @@ -62,6 +62,7 @@ class IceTransportController public: void OnSenderReport(const SenderReport &sender_report); + void OnReceiverReport(const ReceiverReport& receiver_report); void OnCongestionControlFeedback( const webrtc::rtcp::CongestionControlFeedback &feedback);