diff --git a/src/channel/rtp_channel/rtp_video_receiver.cpp b/src/channel/rtp_channel/rtp_video_receiver.cpp index e6d326a..8136bca 100644 --- a/src/channel/rtp_channel/rtp_video_receiver.cpp +++ b/src/channel/rtp_channel/rtp_video_receiver.cpp @@ -4,6 +4,8 @@ #include "log.h" #include "rtcp_sender.h" +#define SAVE_RTP_RECV_STREAM 1 + #define NV12_BUFFER_SIZE (1280 * 720 * 3 / 2) #define RTCP_RR_INTERVAL 1000 @@ -33,6 +35,13 @@ RtpVideoReceiver::RtpVideoReceiver(std::shared_ptr io_statistics) }), clock_(Clock::GetRealTimeClock()) { rtcp_thread_ = std::thread(&RtpVideoReceiver::RtcpThread, this); + +#ifdef SAVE_RTP_RECV_STREAM + file_rtp_recv_ = fopen("rtp_recv_stream.h264", "w+b"); + if (!file_rtp_recv_) { + LOG_WARN("Fail to open rtp_recv_stream.h264"); + } +#endif } RtpVideoReceiver::~RtpVideoReceiver() { @@ -48,6 +57,14 @@ RtpVideoReceiver::~RtpVideoReceiver() { } SSRCManager::Instance().DeleteSsrc(feedback_ssrc_); + +#ifdef SAVE_RTP_RECV_STREAM + if (file_rtp_recv_) { + fflush(file_rtp_recv_); + fclose(file_rtp_recv_); + file_rtp_recv_ = nullptr; + } +#endif } void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) { @@ -56,6 +73,14 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) { rtp_statistics_->Start(); } + // #ifdef SAVE_RTP_RECV_STREAM + // // fwrite((unsigned char*)rtp_packet.Buffer().data(), 1, + // rtp_packet.Size(), + // // file_rtp_recv_); + // fwrite((unsigned char*)rtp_packet.Payload(), 1, rtp_packet.PayloadSize(), + // file_rtp_recv_); + // #endif + webrtc::RtpPacketReceived rtp_packet_received; rtp_packet_received.Build(rtp_packet.Buffer().data(), rtp_packet.Size()); @@ -437,6 +462,10 @@ bool RtpVideoReceiver::Process() { // last_complete_frame_ts_ = now_complete_frame_ts; on_receive_complete_frame_(video_frame); +#ifdef SAVE_RTP_RECV_STREAM + fwrite((unsigned char*)video_frame.Buffer(), 1, video_frame.Size(), + file_rtp_recv_); +#endif } } diff --git a/src/channel/rtp_channel/rtp_video_receiver.h b/src/channel/rtp_channel/rtp_video_receiver.h index 05237f5..33df3c3 100644 --- a/src/channel/rtp_channel/rtp_video_receiver.h +++ b/src/channel/rtp_channel/rtp_video_receiver.h @@ -102,6 +102,9 @@ class RtpVideoReceiver : public ThreadBase { ReceiveSideCongestionController receive_side_congestion_controller_; RtcpFeedbackSenderInterface* active_remb_module_; uint32_t feedback_ssrc_ = 0; + + private: + FILE* file_rtp_recv_ = nullptr; }; #endif diff --git a/src/channel/rtp_channel/rtp_video_sender.cpp b/src/channel/rtp_channel/rtp_video_sender.cpp index 9cad148..ca09f28 100644 --- a/src/channel/rtp_channel/rtp_video_sender.cpp +++ b/src/channel/rtp_channel/rtp_video_sender.cpp @@ -4,17 +4,34 @@ #include "log.h" +#define SAVE_RTP_SENT_STREAM 1 + #define RTCP_SR_INTERVAL 1000 RtpVideoSender::RtpVideoSender() {} RtpVideoSender::RtpVideoSender(std::shared_ptr io_statistics) - : io_statistics_(io_statistics) {} + : io_statistics_(io_statistics) { +#ifdef SAVE_RTP_SENT_STREAM + file_rtp_sent_ = fopen("rtp_sent_stream.h264", "w+b"); + if (!file_rtp_sent_) { + LOG_WARN("Fail to open rtp_sent_stream.h264"); + } +#endif +} RtpVideoSender::~RtpVideoSender() { if (rtp_statistics_) { rtp_statistics_->Stop(); } + +#ifdef SAVE_RTP_SENT_STREAM + if (file_rtp_sent_) { + fflush(file_rtp_sent_); + fclose(file_rtp_sent_); + file_rtp_sent_ = nullptr; + } +#endif } void RtpVideoSender::Enqueue(std::vector& rtp_packets) { @@ -45,6 +62,13 @@ int RtpVideoSender::SendRtpPacket(RtpPacket& rtp_packet) { return -1; } +#ifdef SAVE_RTP_SENT_STREAM + // fwrite((unsigned char*)rtp_packet.Buffer().data(), 1, rtp_packet.Size(), + // file_rtp_sent_); + fwrite((unsigned char*)rtp_packet.Payload(), 1, rtp_packet.PayloadSize(), + file_rtp_sent_); +#endif + last_send_bytes_ += (uint32_t)rtp_packet.Size(); total_rtp_payload_sent_ += (uint32_t)rtp_packet.PayloadSize(); total_rtp_packets_sent_++; diff --git a/src/channel/rtp_channel/rtp_video_sender.h b/src/channel/rtp_channel/rtp_video_sender.h index 8b3f077..f6b42a4 100644 --- a/src/channel/rtp_channel/rtp_video_sender.h +++ b/src/channel/rtp_channel/rtp_video_sender.h @@ -40,6 +40,9 @@ class RtpVideoSender : public ThreadBase { uint32_t last_send_rtcp_sr_packet_ts_ = 0; uint32_t total_rtp_payload_sent_ = 0; uint32_t total_rtp_packets_sent_ = 0; + + private: + FILE *file_rtp_sent_ = nullptr; }; #endif \ No newline at end of file diff --git a/src/rtp/rtp_packet/rtp_packet.cpp b/src/rtp/rtp_packet/rtp_packet.cpp index f9d311c..d483642 100644 --- a/src/rtp/rtp_packet/rtp_packet.cpp +++ b/src/rtp/rtp_packet/rtp_packet.cpp @@ -2,7 +2,13 @@ #include -RtpPacket::RtpPacket() {} +static FILE *file_1_ = nullptr; +static FILE *file_2_ = nullptr; + +RtpPacket::RtpPacket() { + if (file_1_ == nullptr) file_1_ = fopen("file_1_.h264", "w+b"); + if (file_2_ == nullptr) file_2_ = fopen("file_2_.h264", "w+b"); +} RtpPacket::RtpPacket(size_t size) : buffer_(size) {} @@ -14,19 +20,32 @@ RtpPacket &RtpPacket::operator=(const RtpPacket &rtp_packet) = default; RtpPacket &RtpPacket::operator=(RtpPacket &&rtp_packet) = default; -RtpPacket::~RtpPacket() = default; +RtpPacket::~RtpPacket() { + // if (file_1_ != nullptr) { + // fclose(file_1_); + // file_1_ = nullptr; + // } + // if (file_2_ != nullptr) { + // fclose(file_2_); + // file_2_ = nullptr; + // } +} bool RtpPacket::Build(const uint8_t *buffer, uint32_t size) { + fwrite((unsigned char *)buffer, 1, size, file_1_); if (!Parse(buffer, size)) { LOG_WARN("RtpPacket::Build: parse failed"); return false; } buffer_.SetData(buffer, size); + fwrite((unsigned char *)Payload(), 1, PayloadSize(), file_2_); size_ = size; return true; } bool RtpPacket::Parse(const uint8_t *buffer, uint32_t size) { + payload_offset_ = 0; + if (size < kFixedHeaderSize) { LOG_WARN("RtpPacket::Parse: size is too small"); return false; @@ -45,7 +64,7 @@ bool RtpPacket::Parse(const uint8_t *buffer, uint32_t size) { LOG_WARN("RtpPacket::Parse: csrc count is too large"); return false; } - payload_offset_++; + payload_offset_ += 1; // 2nd byte marker_ = (buffer[payload_offset_] >> 7) & 0x01; @@ -99,18 +118,21 @@ bool RtpPacket::Parse(const uint8_t *buffer, uint32_t size) { (buffer[payload_offset_] << 8) | buffer[payload_offset_ + 1]; extension_len_ = (buffer[payload_offset_ + 2] << 8) | buffer[payload_offset_ + 3]; + payload_offset_ += 4; - if (payload_offset_ + extension_len_ > size) { + if (payload_offset_ + extension_len_ * 4 > size) { LOG_WARN("RtpPacket::Parse: extension len is too large"); return false; } - size_t offset = payload_offset_ + 4; - while (offset < size && extension_len_ > 0) { + size_t offset = payload_offset_; + size_t total_ext_len = extension_len_ * 4; + while (offset < payload_offset_ + total_ext_len) { uint8_t id = buffer[offset] >> 4; uint8_t len = (buffer[offset] & 0x0F) + 1; - if (offset + 1 + len > size) { - break; + if (offset + 1 + len > payload_offset_ + total_ext_len) { + LOG_WARN("RtpPacket::Parse: extension data is too large"); + return false; } Extension extension; extension.id = id; @@ -119,7 +141,7 @@ bool RtpPacket::Parse(const uint8_t *buffer, uint32_t size) { extensions_.push_back(extension); offset += 1 + len; } - payload_offset_ += extension_len_; + payload_offset_ += total_ext_len; } if (has_padding_ && payload_offset_ < size) { @@ -137,6 +159,7 @@ bool RtpPacket::Parse(const uint8_t *buffer, uint32_t size) { LOG_WARN("RtpPacket::Parse: payload size is too large"); return false; } + payload_size_ = size - payload_offset_ - padding_size_; return true; diff --git a/src/rtp/rtp_packet/rtp_packet.h b/src/rtp/rtp_packet/rtp_packet.h index 785accb..f7d6361 100644 --- a/src/rtp/rtp_packet/rtp_packet.h +++ b/src/rtp/rtp_packet/rtp_packet.h @@ -269,6 +269,10 @@ class RtpPacket { bool has_padding() const { return buffer_[0] & 0x20; } size_t padding_size() const { return padding_size_; } size_t size() const { return size_; } + void add_offset_to_payload(size_t offset) { + payload_offset_ += offset; + payload_size_ -= offset; + } private: // Common header diff --git a/src/rtp/rtp_packet/rtp_packet_h264.cpp b/src/rtp/rtp_packet/rtp_packet_h264.cpp index 92408a0..a512bba 100644 --- a/src/rtp/rtp_packet/rtp_packet_h264.cpp +++ b/src/rtp/rtp_packet/rtp_packet_h264.cpp @@ -5,16 +5,30 @@ RtpPacketH264::RtpPacketH264() {} RtpPacketH264::~RtpPacketH264() {} bool RtpPacketH264::GetFrameHeaderInfo() { + if (fu_info_got_) { + return true; + } + const uint8_t* frame_buffer = Payload(); fu_indicator_.forbidden_bit = (frame_buffer[0] >> 7) & 0x01; fu_indicator_.nal_reference_idc = (frame_buffer[0] >> 5) & 0x03; fu_indicator_.nal_unit_type = frame_buffer[0] & 0x1F; - fu_header_.start = (frame_buffer[1] >> 7) & 0x01; - fu_header_.end = (frame_buffer[1] >> 6) & 0x01; - fu_header_.remain_bit = (frame_buffer[1] >> 5) & 0x01; - fu_header_.nal_unit_type = frame_buffer[1] & 0x1F; + if (rtp::NAL_UNIT_TYPE::NALU == fu_indicator_.nal_unit_type) { + add_offset_to_payload(1); + LOG_ERROR("2 [{} {} {}]", (int)fu_indicator_.forbidden_bit, + (int)fu_indicator_.nal_reference_idc, + (int)fu_indicator_.nal_unit_type); + } else if (rtp::NAL_UNIT_TYPE::FU_A == fu_indicator_.nal_unit_type) { + fu_header_.start = (frame_buffer[1] >> 7) & 0x01; + fu_header_.end = (frame_buffer[1] >> 6) & 0x01; + fu_header_.remain_bit = (frame_buffer[1] >> 5) & 0x01; + fu_header_.nal_unit_type = frame_buffer[1] & 0x1F; + add_offset_to_payload(2); + } + + fu_info_got_ = true; return true; } \ No newline at end of file diff --git a/src/rtp/rtp_packet/rtp_packet_h264.h b/src/rtp/rtp_packet/rtp_packet_h264.h index ff1a67e..f9890e9 100644 --- a/src/rtp/rtp_packet/rtp_packet_h264.h +++ b/src/rtp/rtp_packet/rtp_packet_h264.h @@ -26,6 +26,7 @@ class RtpPacketH264 : public RtpPacket { private: rtp::FU_INDICATOR fu_indicator_; rtp::FU_HEADER fu_header_; + bool fu_info_got_ = false; }; #endif \ No newline at end of file diff --git a/src/rtp/rtp_packetizer/rtp_packetizer_h264.cpp b/src/rtp/rtp_packetizer/rtp_packetizer_h264.cpp index 9ea67b3..7920479 100644 --- a/src/rtp/rtp_packetizer/rtp_packetizer_h264.cpp +++ b/src/rtp/rtp_packetizer/rtp_packetizer_h264.cpp @@ -19,6 +19,130 @@ RtpPacketizerH264::~RtpPacketizerH264() {} std::vector RtpPacketizerH264::Build(uint8_t* payload, uint32_t payload_size) { + if (payload_size <= MAX_NALU_LEN) { + return BuildNalu(payload, payload_size); + } else { + return BuildFua(payload, payload_size); + // return std::vector(); + } +} + +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ID | L=2 | Absolute Send Time | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// ID (4 bits): The identifier of the extension header field. In WebRTC, +// the ID for Absolute Send Time is typically 3. +// L (4 bits): The length of the extension data in bytes minus 1. For +// Absolute Send Time: the length is 2 (indicating 3 bytes of data). +// Absolute Send Time (24 bits): The absolute send time, with a unit of +// 1/65536 seconds (approximately 15.258 microseconds). + +void RtpPacketizerH264::AddAbsSendTimeExtension( + std::vector& rtp_packet_frame) { + uint16_t extension_profile = 0xBEDE; // One-byte header extension + uint8_t sub_extension_id = 3; // ID for Absolute Send Time + uint8_t sub_extension_length = + 2; // Length of the extension data in bytes minus 1 + + uint32_t abs_send_time = + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + abs_send_time &= 0x00FFFFFF; // Absolute Send Time is 24 bits + + // Add extension profile + rtp_packet_frame.push_back((extension_profile >> 8) & 0xFF); + rtp_packet_frame.push_back(extension_profile & 0xFF); + + // Add extension length (in 32-bit words, minus one) + rtp_packet_frame.push_back( + 0x00); // Placeholder for length, will be updated later + rtp_packet_frame.push_back(0x01); // One 32-bit word + + // Add Absolute Send Time extension + rtp_packet_frame.push_back((sub_extension_id << 4) | sub_extension_length); + rtp_packet_frame.push_back((abs_send_time >> 16) & 0xFF); + rtp_packet_frame.push_back((abs_send_time >> 8) & 0xFF); + rtp_packet_frame.push_back(abs_send_time & 0xFF); +} + +std::vector RtpPacketizerH264::BuildNalu(uint8_t* payload, + uint32_t payload_size) { + LOG_ERROR("payload_size_ = {}", payload_size); + std::vector rtp_packets; + + version_ = kRtpVersion; + has_padding_ = false; + has_extension_ = true; + csrc_count_ = 0; + marker_ = 1; + payload_type_ = rtp::PAYLOAD_TYPE(payload_type_); + sequence_number_++; + timestamp_ = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + ssrc_ = ssrc_; + + if (!csrc_count_) { + csrcs_ = csrcs_; + } + + rtp::FU_INDICATOR fu_indicator; + fu_indicator.forbidden_bit = 0; + fu_indicator.nal_reference_idc = 1; + fu_indicator.nal_unit_type = rtp::NAL_UNIT_TYPE::NALU; + + rtp_packet_frame_.clear(); + rtp_packet_frame_.push_back((version_ << 6) | (has_padding_ << 5) | + (has_extension_ << 4) | csrc_count_); + rtp_packet_frame_.push_back((marker_ << 7) | payload_type_); + rtp_packet_frame_.push_back((sequence_number_ >> 8) & 0xFF); + rtp_packet_frame_.push_back(sequence_number_ & 0xFF); + rtp_packet_frame_.push_back((timestamp_ >> 24) & 0xFF); + rtp_packet_frame_.push_back((timestamp_ >> 16) & 0xFF); + rtp_packet_frame_.push_back((timestamp_ >> 8) & 0xFF); + rtp_packet_frame_.push_back(timestamp_ & 0xFF); + rtp_packet_frame_.push_back((ssrc_ >> 24) & 0xFF); + rtp_packet_frame_.push_back((ssrc_ >> 16) & 0xFF); + rtp_packet_frame_.push_back((ssrc_ >> 8) & 0xFF); + rtp_packet_frame_.push_back(ssrc_ & 0xFF); + + for (uint32_t index = 0; index < csrc_count_ && !csrcs_.empty(); index++) { + rtp_packet_frame_.push_back((csrcs_[index] >> 24) & 0xFF); + rtp_packet_frame_.push_back((csrcs_[index] >> 16) & 0xFF); + rtp_packet_frame_.push_back((csrcs_[index] >> 8) & 0xFF); + rtp_packet_frame_.push_back(csrcs_[index] & 0xFF); + } + + if (has_extension_) { + AddAbsSendTimeExtension(rtp_packet_frame_); + } + + rtp_packet_frame_.push_back(fu_indicator.forbidden_bit << 7 | + fu_indicator.nal_reference_idc << 6 | + fu_indicator.nal_unit_type); + + LOG_ERROR("1 [{} {} {}]", (int)fu_indicator.forbidden_bit, + (int)fu_indicator.nal_reference_idc, + (int)fu_indicator.nal_unit_type); + + rtp_packet_frame_.insert(rtp_packet_frame_.end(), payload, + payload + payload_size); + + RtpPacket rtp_packet; + rtp_packet.Build(rtp_packet_frame_.data(), rtp_packet_frame_.size()); + + rtp_packets.emplace_back(rtp_packet); + + return rtp_packets; +} + +std::vector RtpPacketizerH264::BuildFua(uint8_t* payload, + uint32_t payload_size) { + std::vector rtp_packets; + uint32_t last_packet_size = payload_size % MAX_NALU_LEN; uint32_t packet_num = payload_size / MAX_NALU_LEN + (last_packet_size ? 1 : 0); @@ -28,7 +152,6 @@ std::vector RtpPacketizerH264::Build(uint8_t* payload, std::chrono::system_clock::now().time_since_epoch()) .count(); - std::vector rtp_packets; for (uint32_t index = 0; index < packet_num; index++) { version_ = kRtpVersion; has_padding_ = false; @@ -78,41 +201,7 @@ std::vector RtpPacketizerH264::Build(uint8_t* payload, } if (has_extension_) { - // 0 1 2 3 - // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // | ID | L=2 | Absolute Send Time | - // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - // ID (4 bits): The identifier of the extension header field. In WebRTC, - // the ID for Absolute Send Time is typically 3. - // L (4 bits): The length of the extension data in bytes minus 1. For - // Absolute Send Time: the length is 2 (indicating 3 bytes of data). - // Absolute Send Time (24 bits): The absolute send time, with a unit of - // 1/65536 seconds (approximately 15.258 microseconds). - - extension_profile_ = kOneByteExtensionProfileId; - // 2 bytes for profile, 2 bytes for length, 3 bytes for abs_send_time, 1 - // byte for id and sub extension length - extension_len_ = 8; - - uint32_t abs_send_time = - std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); - - abs_send_time &= 0x00FFFFFF; - - uint8_t sub_extension_id = 0; - uint8_t sub_extension_len = 2; - - rtp_packet_frame_.push_back(extension_profile_ >> 8); - rtp_packet_frame_.push_back(extension_profile_ & 0xff); - rtp_packet_frame_.push_back((extension_len_ >> 8) & 0xFF); - rtp_packet_frame_.push_back(extension_len_ & 0xFF); - rtp_packet_frame_.push_back(sub_extension_id << 4 | sub_extension_len); - rtp_packet_frame_.push_back((abs_send_time >> 16) & 0xFF); - rtp_packet_frame_.push_back((abs_send_time >> 8) & 0xFF); - rtp_packet_frame_.push_back(abs_send_time & 0xFF); + AddAbsSendTimeExtension(rtp_packet_frame_); } rtp_packet_frame_.push_back(fu_indicator.forbidden_bit << 7 | @@ -150,7 +239,8 @@ std::vector RtpPacketizerH264::Build(uint8_t* payload, // uint8_t num_of_source_packets = 0; // unsigned int last_packet_size = 0; // fec_encoder_.GetFecPacketsParams(payload_size, num_of_total_packets, -// num_of_source_packets, last_packet_size); +// num_of_source_packets, +// last_packet_size); // timestamp_ = std::chrono::duration_cast( // std::chrono::system_clock::now().time_since_epoch()) @@ -225,7 +315,8 @@ std::vector RtpPacketizerH264::Build(uint8_t* payload, // rtp_packet.SetExtensionProfile(extension_profile_); // rtp_packet.SetExtensionData(extension_data_, extension_len_); // } -// rtp_packet.EncodeH264FecRepair(fec_packets[index], MAX_NALU_LEN, index, +// rtp_packet.EncodeH264FecRepair(fec_packets[index], MAX_NALU_LEN, +// index, // num_of_source_packets); // } // packets.emplace_back(rtp_packet); diff --git a/src/rtp/rtp_packetizer/rtp_packetizer_h264.h b/src/rtp/rtp_packetizer/rtp_packetizer_h264.h index 8076731..5aa37db 100644 --- a/src/rtp/rtp_packetizer/rtp_packetizer_h264.h +++ b/src/rtp/rtp_packetizer/rtp_packetizer_h264.h @@ -18,9 +18,16 @@ class RtpPacketizerH264 : public RtpPacketizer { std::vector Build(uint8_t* payload, uint32_t payload_size) override; + std::vector RtpPacketizerH264::BuildNalu(uint8_t* payload, + uint32_t payload_size); + + std::vector RtpPacketizerH264::BuildFua(uint8_t* payload, + uint32_t payload_size); + private: - bool RtpPacketizerH264::EncodeH264Fua(RtpPacket& rtp_packet, uint8_t* payload, - size_t payload_size); + bool EncodeH264Fua(RtpPacket& rtp_packet, uint8_t* payload, + size_t payload_size); + void AddAbsSendTimeExtension(std::vector& rtp_packet_frame); private: uint8_t version_;