diff --git a/src/rtp/rtp_packet/rtp_packet.h b/src/rtp/rtp_packet/rtp_packet.h index f6cb7cd..c3eb1d1 100644 --- a/src/rtp/rtp_packet/rtp_packet.h +++ b/src/rtp/rtp_packet/rtp_packet.h @@ -228,6 +228,15 @@ class RtpPacket { extension.data.push_back(abs_send_time & 0xFF); } + void UpdateSequenceNumber(uint16_t sequence_number) { + // Ensure the buffer is large enough to contain the sequence number + if (buffer_.size() >= 4) { + buffer_[2] = (sequence_number >> 8) & 0xFF; + buffer_[3] = sequence_number & 0xFF; + sequence_number_ = sequence_number; + } + } + public: // Get Header uint32_t Version() const { return version_; } diff --git a/src/transport/channel/rtp_video_receiver.cpp b/src/transport/channel/rtp_video_receiver.cpp index 55a85cc..4b2ff59 100644 --- a/src/transport/channel/rtp_video_receiver.cpp +++ b/src/transport/channel/rtp_video_receiver.cpp @@ -88,7 +88,7 @@ RtpVideoReceiver::~RtpVideoReceiver() { #endif } -void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet, bool padding) { +void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) { if (!rtp_statistics_) { rtp_statistics_ = std::make_unique(); rtp_statistics_->Start(); @@ -194,16 +194,14 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet, bool padding) { // // SendRtcpRR(rtcp_rr); // } - if (padding) { - return; - } - - if (rtp_packet.PayloadType() == rtp::PAYLOAD_TYPE::AV1) { + if (rtp_packet.PayloadType() == rtp::PAYLOAD_TYPE::AV1 || + rtp_packet.PayloadType() == rtp::PAYLOAD_TYPE::AV1 - 1) { RtpPacketAv1 rtp_packet_av1; rtp_packet_av1.Build(rtp_packet.Buffer().data(), rtp_packet.Size()); rtp_packet_av1.GetFrameHeaderInfo(); ProcessAv1RtpPacket(rtp_packet_av1); - } else { + } else if (rtp_packet.PayloadType() == rtp::PAYLOAD_TYPE::H264 || + rtp_packet.PayloadType() == rtp::PAYLOAD_TYPE::H264 - 1) { RtpPacketH264 rtp_packet_h264; if (rtp_packet_h264.Build(rtp_packet.Buffer().data(), rtp_packet.Size())) { rtp_packet_h264.GetFrameHeaderInfo(); @@ -228,6 +226,8 @@ void RtpVideoReceiver::ProcessH264RtpPacket(RtpPacketH264& rtp_packet_h264) { if (!complete) { } } + } else if (rtp::PAYLOAD_TYPE::H264 - 1 == rtp_packet_h264.PayloadType()) { + padding_sequence_numbers_.insert(rtp_packet_h264.SequenceNumber()); } } // else { @@ -364,10 +364,12 @@ bool RtpVideoReceiver::CheckIsH264FrameCompleted( while (end_seq--) { auto it = incomplete_h264_frame_list_.find(end_seq); if (it == incomplete_h264_frame_list_.end()) { - // The last fragment has already received. If all fragments are in - // order, then some fragments lost in tranmission and need to be - // repaired using FEC - return false; + if (padding_sequence_numbers_.find(end_seq) == + padding_sequence_numbers_.end()) { + return false; + } else { + continue; + } } else if (!it->second.FuAStart()) { continue; } else if (it->second.FuAStart()) { @@ -380,7 +382,13 @@ bool RtpVideoReceiver::CheckIsH264FrameCompleted( uint16_t start = it->first; uint16_t end = rtp_packet_h264.SequenceNumber(); for (uint16_t seq = start; seq <= end; seq++) { - complete_frame_size += incomplete_h264_frame_list_[seq].PayloadSize(); + if (padding_sequence_numbers_.find(seq) == + padding_sequence_numbers_.end()) { + complete_frame_size += + incomplete_h264_frame_list_[seq].PayloadSize(); + } else { + padding_sequence_numbers_.erase(seq); + } } if (!nv12_data_) { @@ -393,8 +401,10 @@ bool RtpVideoReceiver::CheckIsH264FrameCompleted( uint8_t* dest = nv12_data_; for (uint16_t seq = start; seq <= end; seq++) { size_t payload_size = incomplete_h264_frame_list_[seq].PayloadSize(); - memcpy(dest, incomplete_h264_frame_list_[seq].Payload(), - payload_size); + if (payload_size) { + memcpy(dest, incomplete_h264_frame_list_[seq].Payload(), + payload_size); + } dest += payload_size; incomplete_h264_frame_list_.erase(seq); frame_fragment_count++; diff --git a/src/transport/channel/rtp_video_receiver.h b/src/transport/channel/rtp_video_receiver.h index 4f7c3ce..ffac8b2 100644 --- a/src/transport/channel/rtp_video_receiver.h +++ b/src/transport/channel/rtp_video_receiver.h @@ -5,6 +5,7 @@ #include #include #include +#include #include "api/clock/clock.h" #include "clock/system_clock.h" @@ -36,7 +37,7 @@ class RtpVideoReceiver : public ThreadBase, virtual ~RtpVideoReceiver(); public: - void InsertRtpPacket(RtpPacket& rtp_packet, bool padding); + void InsertRtpPacket(RtpPacket& rtp_packet); void SetSendDataFunc(std::function data_send_func); @@ -111,6 +112,7 @@ class RtpVideoReceiver : public ThreadBase, // std::map> fec_repair_symbol_list_; std::set incomplete_fec_frame_list_; std::map> incomplete_fec_packet_list_; + std::unordered_set padding_sequence_numbers_; private: std::thread rtcp_thread_; diff --git a/src/transport/channel/video_channel_receive.cpp b/src/transport/channel/video_channel_receive.cpp index ce72af5..4cdca32 100644 --- a/src/transport/channel/video_channel_receive.cpp +++ b/src/transport/channel/video_channel_receive.cpp @@ -50,8 +50,7 @@ void VideoChannelReceive::Destroy() { } } -int VideoChannelReceive::OnReceiveRtpPacket(const char *data, size_t size, - bool padding) { +int VideoChannelReceive::OnReceiveRtpPacket(const char *data, size_t size) { if (ice_io_statistics_) { ice_io_statistics_->UpdateVideoInboundBytes((uint32_t)size); } @@ -59,7 +58,7 @@ int VideoChannelReceive::OnReceiveRtpPacket(const char *data, size_t size, if (rtp_video_receiver_) { RtpPacket rtp_packet; rtp_packet.Build((uint8_t *)data, (uint32_t)size); - rtp_video_receiver_->InsertRtpPacket(rtp_packet, padding); + rtp_video_receiver_->InsertRtpPacket(rtp_packet); } return 0; diff --git a/src/transport/channel/video_channel_receive.h b/src/transport/channel/video_channel_receive.h index d7caffc..3fbaa27 100644 --- a/src/transport/channel/video_channel_receive.h +++ b/src/transport/channel/video_channel_receive.h @@ -39,7 +39,7 @@ class VideoChannelReceive { return 0; } - int OnReceiveRtpPacket(const char *data, size_t size, bool padding); + int OnReceiveRtpPacket(const char *data, size_t size); void OnSenderReport(const SenderReport &sender_report) { if (rtp_video_receiver_) { diff --git a/src/transport/ice_transport.cpp b/src/transport/ice_transport.cpp index eb7d262..18ac745 100644 --- a/src/transport/ice_transport.cpp +++ b/src/transport/ice_transport.cpp @@ -194,7 +194,7 @@ void IceTransport::OnReceiveBuffer(NiceAgent *agent, guint stream_id, if (!is_closed_) { if (CheckIsRtpPacket(buffer, size)) { if (CheckIsVideoPacket(buffer, size) && ice_transport_controller_) { - ice_transport_controller_->OnReceiveVideoRtpPacket(buffer, size, false); + ice_transport_controller_->OnReceiveVideoRtpPacket(buffer, size); } else if (CheckIsAudioPacket(buffer, size) && ice_transport_controller_) { ice_transport_controller_->OnReceiveAudioRtpPacket(buffer, size); @@ -205,8 +205,6 @@ void IceTransport::OnReceiveBuffer(NiceAgent *agent, guint stream_id, // LOG_ERROR("Rtcp packet [{}]", (uint8_t)(buffer[1])); RtcpPacketInfo rtcp_packet_info; ParseRtcpPacket((const uint8_t *)buffer, size, &rtcp_packet_info); - } else if (CheckIsRtpPaddingPacket(buffer, size)) { - ice_transport_controller_->OnReceiveVideoRtpPacket(buffer, size, true); } else { LOG_ERROR("Unknown packet"); } @@ -928,19 +926,8 @@ uint8_t IceTransport::CheckIsRtpPacket(const char *buffer, size_t size) { if (payload_type == 96 || payload_type == 99 || payload_type == 111 || payload_type == 127) { return payload_type; - } else { - return 0; - } -} - -uint8_t IceTransport::CheckIsRtpPaddingPacket(const char *buffer, size_t size) { - if (size < 2) { - return 0; - } - - uint8_t payload_type = buffer[1] & 0x7F; - if (payload_type == 95 || payload_type == 98 || payload_type == 110 || - payload_type == 126) { + } else if (payload_type == 95 || payload_type == 98 || payload_type == 110 || + payload_type == 126) { return payload_type; } else { return 0; @@ -972,10 +959,12 @@ uint8_t IceTransport::CheckIsVideoPacket(const char *buffer, size_t size) { } uint8_t pt = buffer[1] & 0x7F; - if (rtp::PAYLOAD_TYPE::H264 == pt || + if (rtp::PAYLOAD_TYPE::H264 == pt || (rtp::PAYLOAD_TYPE::H264 - 1) == pt || rtp::PAYLOAD_TYPE::H264_FEC_SOURCE == pt || + (rtp::PAYLOAD_TYPE::H264_FEC_SOURCE - 1) == pt || rtp::PAYLOAD_TYPE::H264_FEC_REPAIR == pt || - rtp::PAYLOAD_TYPE::AV1 == pt) { + (rtp::PAYLOAD_TYPE::H264_FEC_REPAIR - 1) == pt || + rtp::PAYLOAD_TYPE::AV1 == pt || (rtp::PAYLOAD_TYPE::AV1 - 1) == pt) { return pt; } else { return 0; diff --git a/src/transport/ice_transport.h b/src/transport/ice_transport.h index 202d434..658b6db 100644 --- a/src/transport/ice_transport.h +++ b/src/transport/ice_transport.h @@ -105,7 +105,6 @@ class IceTransport { private: uint8_t CheckIsRtpPacket(const char *buffer, size_t size); - uint8_t CheckIsRtpPaddingPacket(const char *buffer, size_t size); uint8_t CheckIsRtcpPacket(const char *buffer, size_t size); uint8_t CheckIsVideoPacket(const char *buffer, size_t size); uint8_t CheckIsAudioPacket(const char *buffer, size_t size); diff --git a/src/transport/ice_transport_controller.cpp b/src/transport/ice_transport_controller.cpp index 523c472..1d1b771 100644 --- a/src/transport/ice_transport_controller.cpp +++ b/src/transport/ice_transport_controller.cpp @@ -239,9 +239,9 @@ void IceTransportController::UpdateNetworkAvaliablity(bool network_available) { } int IceTransportController::OnReceiveVideoRtpPacket(const char* data, - size_t size, bool padding) { + size_t size) { if (video_channel_receive_) { - return video_channel_receive_->OnReceiveRtpPacket(data, size, padding); + return video_channel_receive_->OnReceiveRtpPacket(data, size); } return -1; diff --git a/src/transport/ice_transport_controller.h b/src/transport/ice_transport_controller.h index 3cf6e60..13553c6 100644 --- a/src/transport/ice_transport_controller.h +++ b/src/transport/ice_transport_controller.h @@ -61,7 +61,7 @@ class IceTransportController void UpdateNetworkAvaliablity(bool network_available); - int OnReceiveVideoRtpPacket(const char *data, size_t size, bool padding); + int OnReceiveVideoRtpPacket(const char *data, size_t size); int OnReceiveAudioRtpPacket(const char *data, size_t size); int OnReceiveDataRtpPacket(const char *data, size_t size); diff --git a/src/transport/packet_sender/packet_sender_imp.h b/src/transport/packet_sender/packet_sender_imp.h index 5d237f2..70bbbcd 100644 --- a/src/transport/packet_sender/packet_sender_imp.h +++ b/src/transport/packet_sender/packet_sender_imp.h @@ -55,6 +55,12 @@ class PacketSenderImp : public PacketSender, void SendPacket(std::unique_ptr packet, const webrtc::PacedPacketInfo& cluster_info) override { if (on_sent_packet_func_) { + if (ssrc_seq_.find(packet->Ssrc()) == ssrc_seq_.end()) { + ssrc_seq_[packet->Ssrc()] = 1; + } + + packet->UpdateSequenceNumber(ssrc_seq_[packet->Ssrc()]++); + on_sent_packet_func_(*packet); } } @@ -209,6 +215,7 @@ class PacketSenderImp : public PacketSender, TaskQueue task_queue_; int64_t transport_seq_ = 0; + std::map ssrc_seq_; }; #endif \ No newline at end of file