[fix] fix h264 rtp packet parse

src/channel/rtp_channel/rtp_video_receiver.cpp

@@ -4,6 +4,8 @@
 #include "log.h"
 #include "rtcp_sender.h"
 
+#define SAVE_RTP_RECV_STREAM 1
+
 #define NV12_BUFFER_SIZE (1280 * 720 * 3 / 2)
 #define RTCP_RR_INTERVAL 1000
 
@@ -33,6 +35,13 @@ RtpVideoReceiver::RtpVideoReceiver(std::shared_ptr<IOStatistics> io_statistics)
           }),
       clock_(Clock::GetRealTimeClock()) {
   rtcp_thread_ = std::thread(&RtpVideoReceiver::RtcpThread, this);
+
+#ifdef SAVE_RTP_RECV_STREAM
+  file_rtp_recv_ = fopen("rtp_recv_stream.h264", "w+b");
+  if (!file_rtp_recv_) {
+    LOG_WARN("Fail to open rtp_recv_stream.h264");
+  }
+#endif
 }
 
 RtpVideoReceiver::~RtpVideoReceiver() {
@@ -48,6 +57,14 @@ RtpVideoReceiver::~RtpVideoReceiver() {
   }
 
   SSRCManager::Instance().DeleteSsrc(feedback_ssrc_);
+
+#ifdef SAVE_RTP_RECV_STREAM
+  if (file_rtp_recv_) {
+    fflush(file_rtp_recv_);
+    fclose(file_rtp_recv_);
+    file_rtp_recv_ = nullptr;
+  }
+#endif
 }
 
 void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
@@ -56,6 +73,14 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
     rtp_statistics_->Start();
   }
 
+  // #ifdef SAVE_RTP_RECV_STREAM
+  //   // fwrite((unsigned char*)rtp_packet.Buffer().data(), 1,
+  //   rtp_packet.Size(),
+  //   //        file_rtp_recv_);
+  //   fwrite((unsigned char*)rtp_packet.Payload(), 1, rtp_packet.PayloadSize(),
+  //          file_rtp_recv_);
+  // #endif
+
   webrtc::RtpPacketReceived rtp_packet_received;
   rtp_packet_received.Build(rtp_packet.Buffer().data(), rtp_packet.Size());
 
@@ -437,6 +462,10 @@ bool RtpVideoReceiver::Process() {
       // last_complete_frame_ts_ = now_complete_frame_ts;
 
       on_receive_complete_frame_(video_frame);
+#ifdef SAVE_RTP_RECV_STREAM
+      fwrite((unsigned char*)video_frame.Buffer(), 1, video_frame.Size(),
+             file_rtp_recv_);
+#endif
     }
   }
 

src/channel/rtp_channel/rtp_video_receiver.h

@@ -102,6 +102,9 @@ class RtpVideoReceiver : public ThreadBase {
   ReceiveSideCongestionController receive_side_congestion_controller_;
   RtcpFeedbackSenderInterface* active_remb_module_;
   uint32_t feedback_ssrc_ = 0;
+
+ private:
+  FILE* file_rtp_recv_ = nullptr;
 };
 
 #endif

src/channel/rtp_channel/rtp_video_sender.cpp

@@ -4,17 +4,34 @@
 
 #include "log.h"
 
+#define SAVE_RTP_SENT_STREAM 1
+
 #define RTCP_SR_INTERVAL 1000
 
 RtpVideoSender::RtpVideoSender() {}
 
 RtpVideoSender::RtpVideoSender(std::shared_ptr<IOStatistics> io_statistics)
-    : io_statistics_(io_statistics) {}
+    : io_statistics_(io_statistics) {
+#ifdef SAVE_RTP_SENT_STREAM
+  file_rtp_sent_ = fopen("rtp_sent_stream.h264", "w+b");
+  if (!file_rtp_sent_) {
+    LOG_WARN("Fail to open rtp_sent_stream.h264");
+  }
+#endif
+}
 
 RtpVideoSender::~RtpVideoSender() {
   if (rtp_statistics_) {
     rtp_statistics_->Stop();
   }
+
+#ifdef SAVE_RTP_SENT_STREAM
+  if (file_rtp_sent_) {
+    fflush(file_rtp_sent_);
+    fclose(file_rtp_sent_);
+    file_rtp_sent_ = nullptr;
+  }
+#endif
 }
 
 void RtpVideoSender::Enqueue(std::vector<RtpPacket>& rtp_packets) {
@@ -45,6 +62,13 @@ int RtpVideoSender::SendRtpPacket(RtpPacket& rtp_packet) {
     return -1;
   }
 
+#ifdef SAVE_RTP_SENT_STREAM
+  // fwrite((unsigned char*)rtp_packet.Buffer().data(), 1, rtp_packet.Size(),
+  //        file_rtp_sent_);
+  fwrite((unsigned char*)rtp_packet.Payload(), 1, rtp_packet.PayloadSize(),
+         file_rtp_sent_);
+#endif
+
   last_send_bytes_ += (uint32_t)rtp_packet.Size();
   total_rtp_payload_sent_ += (uint32_t)rtp_packet.PayloadSize();
   total_rtp_packets_sent_++;

src/channel/rtp_channel/rtp_video_sender.h

@@ -40,6 +40,9 @@ class RtpVideoSender : public ThreadBase {
   uint32_t last_send_rtcp_sr_packet_ts_ = 0;
   uint32_t total_rtp_payload_sent_ = 0;
   uint32_t total_rtp_packets_sent_ = 0;
+
+ private:
+  FILE *file_rtp_sent_ = nullptr;
 };
 
 #endif

src/rtp/rtp_packet/rtp_packet.cpp

@@ -2,7 +2,13 @@
 
 #include <string>
 
-RtpPacket::RtpPacket() {}
+static FILE *file_1_ = nullptr;
+static FILE *file_2_ = nullptr;
+
+RtpPacket::RtpPacket() {
+  if (file_1_ == nullptr) file_1_ = fopen("file_1_.h264", "w+b");
+  if (file_2_ == nullptr) file_2_ = fopen("file_2_.h264", "w+b");
+}
 
 RtpPacket::RtpPacket(size_t size) : buffer_(size) {}
 
@@ -14,19 +20,32 @@ RtpPacket &RtpPacket::operator=(const RtpPacket &rtp_packet) = default;
 
 RtpPacket &RtpPacket::operator=(RtpPacket &&rtp_packet) = default;
 
-RtpPacket::~RtpPacket() = default;
+RtpPacket::~RtpPacket() {
+  // if (file_1_ != nullptr) {
+  //   fclose(file_1_);
+  //   file_1_ = nullptr;
+  // }
+  // if (file_2_ != nullptr) {
+  //   fclose(file_2_);
+  //   file_2_ = nullptr;
+  // }
+}
 
 bool RtpPacket::Build(const uint8_t *buffer, uint32_t size) {
+  fwrite((unsigned char *)buffer, 1, size, file_1_);
   if (!Parse(buffer, size)) {
     LOG_WARN("RtpPacket::Build: parse failed");
     return false;
   }
   buffer_.SetData(buffer, size);
+  fwrite((unsigned char *)Payload(), 1, PayloadSize(), file_2_);
   size_ = size;
   return true;
 }
 
 bool RtpPacket::Parse(const uint8_t *buffer, uint32_t size) {
+  payload_offset_ = 0;
+
   if (size < kFixedHeaderSize) {
     LOG_WARN("RtpPacket::Parse: size is too small");
     return false;
@@ -45,7 +64,7 @@ bool RtpPacket::Parse(const uint8_t *buffer, uint32_t size) {
     LOG_WARN("RtpPacket::Parse: csrc count is too large");
     return false;
   }
-  payload_offset_++;
+  payload_offset_ += 1;
 
   // 2nd byte
   marker_ = (buffer[payload_offset_] >> 7) & 0x01;
@@ -99,18 +118,21 @@ bool RtpPacket::Parse(const uint8_t *buffer, uint32_t size) {
         (buffer[payload_offset_] << 8) | buffer[payload_offset_ + 1];
     extension_len_ =
         (buffer[payload_offset_ + 2] << 8) | buffer[payload_offset_ + 3];
+    payload_offset_ += 4;
 
-    if (payload_offset_ + extension_len_ > size) {
+    if (payload_offset_ + extension_len_ * 4 > size) {
       LOG_WARN("RtpPacket::Parse: extension len is too large");
       return false;
     }
 
-    size_t offset = payload_offset_ + 4;
+    size_t offset = payload_offset_;
-    while (offset < size && extension_len_ > 0) {
+    size_t total_ext_len = extension_len_ * 4;
+    while (offset < payload_offset_ + total_ext_len) {
       uint8_t id = buffer[offset] >> 4;
       uint8_t len = (buffer[offset] & 0x0F) + 1;
-      if (offset + 1 + len > size) {
+      if (offset + 1 + len > payload_offset_ + total_ext_len) {
-        break;
+        LOG_WARN("RtpPacket::Parse: extension data is too large");
+        return false;
       }
       Extension extension;
       extension.id = id;
@@ -119,7 +141,7 @@ bool RtpPacket::Parse(const uint8_t *buffer, uint32_t size) {
       extensions_.push_back(extension);
       offset += 1 + len;
     }
-    payload_offset_ += extension_len_;
+    payload_offset_ += total_ext_len;
   }
 
   if (has_padding_ && payload_offset_ < size) {
@@ -137,6 +159,7 @@ bool RtpPacket::Parse(const uint8_t *buffer, uint32_t size) {
     LOG_WARN("RtpPacket::Parse: payload size is too large");
     return false;
   }
+
   payload_size_ = size - payload_offset_ - padding_size_;
 
   return true;

src/rtp/rtp_packet/rtp_packet.h

@@ -269,6 +269,10 @@ class RtpPacket {
   bool has_padding() const { return buffer_[0] & 0x20; }
   size_t padding_size() const { return padding_size_; }
   size_t size() const { return size_; }
+  void add_offset_to_payload(size_t offset) {
+    payload_offset_ += offset;
+    payload_size_ -= offset;
+  }
 
  private:
   // Common header

src/rtp/rtp_packet/rtp_packet_h264.cpp

@@ -5,16 +5,30 @@ RtpPacketH264::RtpPacketH264() {}
 RtpPacketH264::~RtpPacketH264() {}
 
 bool RtpPacketH264::GetFrameHeaderInfo() {
+  if (fu_info_got_) {
+    return true;
+  }
+
   const uint8_t* frame_buffer = Payload();
 
   fu_indicator_.forbidden_bit = (frame_buffer[0] >> 7) & 0x01;
   fu_indicator_.nal_reference_idc = (frame_buffer[0] >> 5) & 0x03;
   fu_indicator_.nal_unit_type = frame_buffer[0] & 0x1F;
 
+  if (rtp::NAL_UNIT_TYPE::NALU == fu_indicator_.nal_unit_type) {
+    add_offset_to_payload(1);
+    LOG_ERROR("2 [{} {} {}]", (int)fu_indicator_.forbidden_bit,
+              (int)fu_indicator_.nal_reference_idc,
+              (int)fu_indicator_.nal_unit_type);
+  } else if (rtp::NAL_UNIT_TYPE::FU_A == fu_indicator_.nal_unit_type) {
     fu_header_.start = (frame_buffer[1] >> 7) & 0x01;
     fu_header_.end = (frame_buffer[1] >> 6) & 0x01;
     fu_header_.remain_bit = (frame_buffer[1] >> 5) & 0x01;
     fu_header_.nal_unit_type = frame_buffer[1] & 0x1F;
+    add_offset_to_payload(2);
+  }
+
+  fu_info_got_ = true;
 
   return true;
 }

src/rtp/rtp_packet/rtp_packet_h264.h

@@ -26,6 +26,7 @@ class RtpPacketH264 : public RtpPacket {
  private:
   rtp::FU_INDICATOR fu_indicator_;
   rtp::FU_HEADER fu_header_;
+  bool fu_info_got_ = false;
 };
 
 #endif

src/rtp/rtp_packetizer/rtp_packetizer_h264.cpp

@@ -19,6 +19,130 @@ RtpPacketizerH264::~RtpPacketizerH264() {}
 
 std::vector<RtpPacket> RtpPacketizerH264::Build(uint8_t* payload,
                                                 uint32_t payload_size) {
+  if (payload_size <= MAX_NALU_LEN) {
+    return BuildNalu(payload, payload_size);
+  } else {
+    return BuildFua(payload, payload_size);
+    // return std::vector<RtpPacket>();
+  }
+}
+
+//  0                   1                   2                   3
+//  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |  ID   |  L=2  |              Absolute Send Time               |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// ID (4 bits): The identifier of the extension header field. In WebRTC,
+// the ID for Absolute Send Time is typically 3.
+// L (4 bits): The length of the extension data in bytes minus 1. For
+// Absolute Send Time: the length is 2 (indicating 3 bytes of data).
+// Absolute Send Time (24 bits): The absolute send time, with a unit of
+// 1/65536 seconds (approximately 15.258 microseconds).
+
+void RtpPacketizerH264::AddAbsSendTimeExtension(
+    std::vector<uint8_t>& rtp_packet_frame) {
+  uint16_t extension_profile = 0xBEDE;  // One-byte header extension
+  uint8_t sub_extension_id = 3;         // ID for Absolute Send Time
+  uint8_t sub_extension_length =
+      2;  // Length of the extension data in bytes minus 1
+
+  uint32_t abs_send_time =
+      std::chrono::duration_cast<std::chrono::microseconds>(
+          std::chrono::system_clock::now().time_since_epoch())
+          .count();
+  abs_send_time &= 0x00FFFFFF;  // Absolute Send Time is 24 bits
+
+  // Add extension profile
+  rtp_packet_frame.push_back((extension_profile >> 8) & 0xFF);
+  rtp_packet_frame.push_back(extension_profile & 0xFF);
+
+  // Add extension length (in 32-bit words, minus one)
+  rtp_packet_frame.push_back(
+      0x00);  // Placeholder for length, will be updated later
+  rtp_packet_frame.push_back(0x01);  // One 32-bit word
+
+  // Add Absolute Send Time extension
+  rtp_packet_frame.push_back((sub_extension_id << 4) | sub_extension_length);
+  rtp_packet_frame.push_back((abs_send_time >> 16) & 0xFF);
+  rtp_packet_frame.push_back((abs_send_time >> 8) & 0xFF);
+  rtp_packet_frame.push_back(abs_send_time & 0xFF);
+}
+
+std::vector<RtpPacket> RtpPacketizerH264::BuildNalu(uint8_t* payload,
+                                                    uint32_t payload_size) {
+  LOG_ERROR("payload_size_ = {}", payload_size);
+  std::vector<RtpPacket> rtp_packets;
+
+  version_ = kRtpVersion;
+  has_padding_ = false;
+  has_extension_ = true;
+  csrc_count_ = 0;
+  marker_ = 1;
+  payload_type_ = rtp::PAYLOAD_TYPE(payload_type_);
+  sequence_number_++;
+  timestamp_ = std::chrono::duration_cast<std::chrono::microseconds>(
+                   std::chrono::system_clock::now().time_since_epoch())
+                   .count();
+  ssrc_ = ssrc_;
+
+  if (!csrc_count_) {
+    csrcs_ = csrcs_;
+  }
+
+  rtp::FU_INDICATOR fu_indicator;
+  fu_indicator.forbidden_bit = 0;
+  fu_indicator.nal_reference_idc = 1;
+  fu_indicator.nal_unit_type = rtp::NAL_UNIT_TYPE::NALU;
+
+  rtp_packet_frame_.clear();
+  rtp_packet_frame_.push_back((version_ << 6) | (has_padding_ << 5) |
+                              (has_extension_ << 4) | csrc_count_);
+  rtp_packet_frame_.push_back((marker_ << 7) | payload_type_);
+  rtp_packet_frame_.push_back((sequence_number_ >> 8) & 0xFF);
+  rtp_packet_frame_.push_back(sequence_number_ & 0xFF);
+  rtp_packet_frame_.push_back((timestamp_ >> 24) & 0xFF);
+  rtp_packet_frame_.push_back((timestamp_ >> 16) & 0xFF);
+  rtp_packet_frame_.push_back((timestamp_ >> 8) & 0xFF);
+  rtp_packet_frame_.push_back(timestamp_ & 0xFF);
+  rtp_packet_frame_.push_back((ssrc_ >> 24) & 0xFF);
+  rtp_packet_frame_.push_back((ssrc_ >> 16) & 0xFF);
+  rtp_packet_frame_.push_back((ssrc_ >> 8) & 0xFF);
+  rtp_packet_frame_.push_back(ssrc_ & 0xFF);
+
+  for (uint32_t index = 0; index < csrc_count_ && !csrcs_.empty(); index++) {
+    rtp_packet_frame_.push_back((csrcs_[index] >> 24) & 0xFF);
+    rtp_packet_frame_.push_back((csrcs_[index] >> 16) & 0xFF);
+    rtp_packet_frame_.push_back((csrcs_[index] >> 8) & 0xFF);
+    rtp_packet_frame_.push_back(csrcs_[index] & 0xFF);
+  }
+
+  if (has_extension_) {
+    AddAbsSendTimeExtension(rtp_packet_frame_);
+  }
+
+  rtp_packet_frame_.push_back(fu_indicator.forbidden_bit << 7 |
+                              fu_indicator.nal_reference_idc << 6 |
+                              fu_indicator.nal_unit_type);
+
+  LOG_ERROR("1 [{} {} {}]", (int)fu_indicator.forbidden_bit,
+            (int)fu_indicator.nal_reference_idc,
+            (int)fu_indicator.nal_unit_type);
+
+  rtp_packet_frame_.insert(rtp_packet_frame_.end(), payload,
+                           payload + payload_size);
+
+  RtpPacket rtp_packet;
+  rtp_packet.Build(rtp_packet_frame_.data(), rtp_packet_frame_.size());
+
+  rtp_packets.emplace_back(rtp_packet);
+
+  return rtp_packets;
+}
+
+std::vector<RtpPacket> RtpPacketizerH264::BuildFua(uint8_t* payload,
+                                                   uint32_t payload_size) {
+  std::vector<RtpPacket> rtp_packets;
+
   uint32_t last_packet_size = payload_size % MAX_NALU_LEN;
   uint32_t packet_num =
       payload_size / MAX_NALU_LEN + (last_packet_size ? 1 : 0);
@@ -28,7 +152,6 @@ std::vector<RtpPacket> RtpPacketizerH264::Build(uint8_t* payload,
                    std::chrono::system_clock::now().time_since_epoch())
                    .count();
 
-  std::vector<RtpPacket> rtp_packets;
   for (uint32_t index = 0; index < packet_num; index++) {
     version_ = kRtpVersion;
     has_padding_ = false;
@@ -78,41 +201,7 @@ std::vector<RtpPacket> RtpPacketizerH264::Build(uint8_t* payload,
     }
 
     if (has_extension_) {
-      //  0                   1                   2                   3
+      AddAbsSendTimeExtension(rtp_packet_frame_);
-      //  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-      // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-      // |  ID   |  L=2  |              Absolute Send Time               |
-      // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-      // ID (4 bits): The identifier of the extension header field. In WebRTC,
-      // the ID for Absolute Send Time is typically 3.
-      // L (4 bits): The length of the extension data in bytes minus 1. For
-      // Absolute Send Time: the length is 2 (indicating 3 bytes of data).
-      // Absolute Send Time (24 bits): The absolute send time, with a unit of
-      // 1/65536 seconds (approximately 15.258 microseconds).
-
-      extension_profile_ = kOneByteExtensionProfileId;
-      // 2 bytes for profile, 2 bytes for length, 3 bytes for abs_send_time, 1
-      // byte for id and sub extension length
-      extension_len_ = 8;
-
-      uint32_t abs_send_time =
-          std::chrono::duration_cast<std::chrono::microseconds>(
-              std::chrono::system_clock::now().time_since_epoch())
-              .count();
-
-      abs_send_time &= 0x00FFFFFF;
-
-      uint8_t sub_extension_id = 0;
-      uint8_t sub_extension_len = 2;
-
-      rtp_packet_frame_.push_back(extension_profile_ >> 8);
-      rtp_packet_frame_.push_back(extension_profile_ & 0xff);
-      rtp_packet_frame_.push_back((extension_len_ >> 8) & 0xFF);
-      rtp_packet_frame_.push_back(extension_len_ & 0xFF);
-      rtp_packet_frame_.push_back(sub_extension_id << 4 | sub_extension_len);
-      rtp_packet_frame_.push_back((abs_send_time >> 16) & 0xFF);
-      rtp_packet_frame_.push_back((abs_send_time >> 8) & 0xFF);
-      rtp_packet_frame_.push_back(abs_send_time & 0xFF);
     }
 
     rtp_packet_frame_.push_back(fu_indicator.forbidden_bit << 7 |
@@ -150,7 +239,8 @@ std::vector<RtpPacket> RtpPacketizerH264::Build(uint8_t* payload,
 //   uint8_t num_of_source_packets = 0;
 //   unsigned int last_packet_size = 0;
 //   fec_encoder_.GetFecPacketsParams(payload_size, num_of_total_packets,
-//                                    num_of_source_packets, last_packet_size);
+//                                    num_of_source_packets,
+//                                    last_packet_size);
 
 //   timestamp_ = std::chrono::duration_cast<std::chrono::microseconds>(
 //                    std::chrono::system_clock::now().time_since_epoch())
@@ -225,7 +315,8 @@ std::vector<RtpPacket> RtpPacketizerH264::Build(uint8_t* payload,
 //         rtp_packet.SetExtensionProfile(extension_profile_);
 //         rtp_packet.SetExtensionData(extension_data_, extension_len_);
 //       }
-//       rtp_packet.EncodeH264FecRepair(fec_packets[index], MAX_NALU_LEN, index,
+//       rtp_packet.EncodeH264FecRepair(fec_packets[index], MAX_NALU_LEN,
+//       index,
 //                                      num_of_source_packets);
 //     }
 //     packets.emplace_back(rtp_packet);

src/rtp/rtp_packetizer/rtp_packetizer_h264.h

@@ -18,9 +18,16 @@ class RtpPacketizerH264 : public RtpPacketizer {
   std::vector<RtpPacket> Build(uint8_t* payload,
                                uint32_t payload_size) override;
 
+  std::vector<RtpPacket> RtpPacketizerH264::BuildNalu(uint8_t* payload,
+                                                      uint32_t payload_size);
+
+  std::vector<RtpPacket> RtpPacketizerH264::BuildFua(uint8_t* payload,
+                                                     uint32_t payload_size);
+
  private:
-  bool RtpPacketizerH264::EncodeH264Fua(RtpPacket& rtp_packet, uint8_t* payload,
+  bool EncodeH264Fua(RtpPacket& rtp_packet, uint8_t* payload,
                      size_t payload_size);
+  void AddAbsSendTimeExtension(std::vector<uint8_t>& rtp_packet_frame);
 
  private:
   uint8_t version_;