[fix] fix h264 rtp packetization error

src/channel/rtp_channel/rtp_video_receiver.cpp

@@ -4,7 +4,7 @@
 #include "log.h"
 #include "rtcp_sender.h"
 
-#define SAVE_RTP_RECV_STREAM 1
+// #define SAVE_RTP_RECV_STREAM
 
 #define NV12_BUFFER_SIZE (1280 * 720 * 3 / 2)
 #define RTCP_RR_INTERVAL 1000
@@ -73,13 +73,10 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
     rtp_statistics_->Start();
   }
 
-  // #ifdef SAVE_RTP_RECV_STREAM
+#ifdef SAVE_RTP_RECV_STREAM
-  //   // fwrite((unsigned char*)rtp_packet.Buffer().data(), 1,
+  fwrite((unsigned char*)rtp_packet.Payload(), 1, rtp_packet.PayloadSize(),
-  //   rtp_packet.Size(),
+         file_rtp_recv_);
-  //   //        file_rtp_recv_);
+#endif
-  //   fwrite((unsigned char*)rtp_packet.Payload(), 1, rtp_packet.PayloadSize(),
-  //          file_rtp_recv_);
-  // #endif
 
   webrtc::RtpPacketReceived rtp_packet_received;
   rtp_packet_received.Build(rtp_packet.Buffer().data(), rtp_packet.Size());
@@ -312,6 +309,7 @@ bool RtpVideoReceiver::CheckIsH264FrameCompleted(
         }
 
         size_t complete_frame_size = 0;
+        int frame_fragment_count = 0;
         for (uint16_t start = it->first;
              start <= rtp_packet_h264.SequenceNumber(); start++) {
           memcpy(nv12_data_ + complete_frame_size,
@@ -321,6 +319,7 @@ bool RtpVideoReceiver::CheckIsH264FrameCompleted(
           complete_frame_size +=
               incomplete_h264_frame_list_[start].PayloadSize();
           incomplete_h264_frame_list_.erase(start);
+          frame_fragment_count++;
         }
         compelete_video_frame_queue_.push(
             VideoFrame(nv12_data_, complete_frame_size));
@@ -462,10 +461,11 @@ bool RtpVideoReceiver::Process() {
       // last_complete_frame_ts_ = now_complete_frame_ts;
 
       on_receive_complete_frame_(video_frame);
-#ifdef SAVE_RTP_RECV_STREAM
+      // #ifdef SAVE_RTP_RECV_STREAM
-      fwrite((unsigned char*)video_frame.Buffer(), 1, video_frame.Size(),
+      //       fwrite((unsigned char*)video_frame.Buffer(), 1,
-             file_rtp_recv_);
+      //       video_frame.Size(),
-#endif
+      //              file_rtp_recv_);
+      // #endif
     }
   }
 

src/channel/rtp_channel/rtp_video_sender.cpp

@@ -4,7 +4,7 @@
 
 #include "log.h"
 
-#define SAVE_RTP_SENT_STREAM 1
+// #define SAVE_RTP_SENT_STREAM
 
 #define RTCP_SR_INTERVAL 1000
 
@@ -63,8 +63,6 @@ int RtpVideoSender::SendRtpPacket(RtpPacket& rtp_packet) {
   }
 
 #ifdef SAVE_RTP_SENT_STREAM
-  // fwrite((unsigned char*)rtp_packet.Buffer().data(), 1, rtp_packet.Size(),
-  //        file_rtp_sent_);
   fwrite((unsigned char*)rtp_packet.Payload(), 1, rtp_packet.PayloadSize(),
          file_rtp_sent_);
 #endif

src/media/video/decode/nvcodec/nvidia_video_decoder.cpp

@@ -4,7 +4,7 @@
 #include "nvcodec_api.h"
 
 // #define SAVE_DECODED_NV12_STREAM
-#define SAVE_RECEIVED_H264_STREAM
+// #define SAVE_RECEIVED_H264_STREAM
 
 NvidiaVideoDecoder::NvidiaVideoDecoder() {}
 NvidiaVideoDecoder::~NvidiaVideoDecoder() {

src/media/video/encode/nvcodec/nvidia_video_encoder.cpp

@@ -7,7 +7,7 @@
 #include "nvcodec_common.h"
 
 // #define SAVE_RECEIVED_NV12_STREAM
-#define SAVE_ENCODED_H264_STREAM
+// #define SAVE_ENCODED_H264_STREAM
 
 NvidiaVideoEncoder::NvidiaVideoEncoder() {}
 NvidiaVideoEncoder::~NvidiaVideoEncoder() {

src/qos/transport_feedback_adapter.cc

@@ -287,7 +287,7 @@ TransportFeedbackAdapter::ProcessCongestionControlFeedback(
 
   if (failed_lookups > 0) {
     LOG_WARN(
-        "Failed to lookup send time for {} packet {}. Packets reordered or "
+        "Failed to lookup send time for {} packet{}. Packets reordered or "
         "send time history too small?",
         failed_lookups, (failed_lookups > 1 ? "s" : ""));
   }

src/rtp/rtp_packet/rtp_packet.cpp

@@ -2,13 +2,7 @@
 
 #include <string>
 
-static FILE *file_1_ = nullptr;
+RtpPacket::RtpPacket() {}
-static FILE *file_2_ = nullptr;
-
-RtpPacket::RtpPacket() {
-  if (file_1_ == nullptr) file_1_ = fopen("file_1_.h264", "w+b");
-  if (file_2_ == nullptr) file_2_ = fopen("file_2_.h264", "w+b");
-}
 
 RtpPacket::RtpPacket(size_t size) : buffer_(size) {}
 
@@ -20,25 +14,14 @@ RtpPacket &RtpPacket::operator=(const RtpPacket &rtp_packet) = default;
 
 RtpPacket &RtpPacket::operator=(RtpPacket &&rtp_packet) = default;
 
-RtpPacket::~RtpPacket() {
+RtpPacket::~RtpPacket() {}
-  // if (file_1_ != nullptr) {
-  //   fclose(file_1_);
-  //   file_1_ = nullptr;
-  // }
-  // if (file_2_ != nullptr) {
-  //   fclose(file_2_);
-  //   file_2_ = nullptr;
-  // }
-}
 
 bool RtpPacket::Build(const uint8_t *buffer, uint32_t size) {
-  fwrite((unsigned char *)buffer, 1, size, file_1_);
   if (!Parse(buffer, size)) {
     LOG_WARN("RtpPacket::Build: parse failed");
     return false;
   }
   buffer_.SetData(buffer, size);
-  fwrite((unsigned char *)Payload(), 1, PayloadSize(), file_2_);
   size_ = size;
   return true;
 }

src/rtp/rtp_packet/rtp_packet.h

@@ -263,6 +263,10 @@ class RtpPacket {
   CopyOnWriteBuffer Buffer() const { return buffer_; }
   size_t Size() const { return size_; }
 
+  // Header
+  const uint8_t *Header() { return Buffer().data(); };
+  size_t HeaderSize() { return payload_offset_; }
+
   // For webrtc module use
   size_t headers_size() const { return payload_offset_; }
   size_t payload_size() const { return payload_size_; }

src/rtp/rtp_packet/rtp_packet_h264.cpp

@@ -17,9 +17,6 @@ bool RtpPacketH264::GetFrameHeaderInfo() {
 
   if (rtp::NAL_UNIT_TYPE::NALU == fu_indicator_.nal_unit_type) {
     add_offset_to_payload(1);
-    LOG_ERROR("2 [{} {} {}]", (int)fu_indicator_.forbidden_bit,
-              (int)fu_indicator_.nal_reference_idc,
-              (int)fu_indicator_.nal_unit_type);
   } else if (rtp::NAL_UNIT_TYPE::FU_A == fu_indicator_.nal_unit_type) {
     fu_header_.start = (frame_buffer[1] >> 7) & 0x01;
     fu_header_.end = (frame_buffer[1] >> 6) & 0x01;

src/rtp/rtp_packetizer/rtp_packetizer_generic.cpp

@@ -7,7 +7,7 @@ RtpPacketizerGeneric::RtpPacketizerGeneric()
       csrc_count_(0),
       marker_(false),
       payload_type_(rtp::PAYLOAD_TYPE::DATA),
-      sequence_number_(1),
+      sequence_number_(0),
       timestamp_(0),
       ssrc_(0),
       profile_(0),
@@ -67,28 +67,7 @@ std::vector<RtpPacket> RtpPacketizerGeneric::Build(uint8_t* payload,
     }
 
     if (has_extension_) {
-      extension_profile_ = kOneByteExtensionProfileId;
+      AddAbsSendTimeExtension(rtp_packet_frame_);
-      extension_len_ = 5;  // 2 bytes for profile, 2 bytes for length, 3 bytes
-                           // for abs_send_time
-
-      uint32_t abs_send_time =
-          std::chrono::duration_cast<std::chrono::microseconds>(
-              std::chrono::system_clock::now().time_since_epoch())
-              .count();
-
-      abs_send_time &= 0x00FFFFFF;
-
-      uint8_t sub_extension_id = 0;
-      uint8_t sub_extension_len = 2;
-
-      rtp_packet_frame_.push_back(extension_profile_ >> 8);
-      rtp_packet_frame_.push_back(extension_profile_ & 0xff);
-      rtp_packet_frame_.push_back((extension_len_ >> 8) & 0xFF);
-      rtp_packet_frame_.push_back(extension_len_ & 0xFF);
-      rtp_packet_frame_.push_back(sub_extension_id << 4 | sub_extension_len);
-      rtp_packet_frame_.push_back((abs_send_time >> 16) & 0xFF);
-      rtp_packet_frame_.push_back((abs_send_time >> 8) & 0xFF);
-      rtp_packet_frame_.push_back(abs_send_time & 0xFF);
     }
 
     if (index == packet_num - 1 && last_packet_size > 0) {
@@ -107,3 +86,32 @@ std::vector<RtpPacket> RtpPacketizerGeneric::Build(uint8_t* payload,
 
   return rtp_packets;
 }
+
+void RtpPacketizerGeneric::AddAbsSendTimeExtension(
+    std::vector<uint8_t>& rtp_packet_frame) {
+  uint16_t extension_profile = 0xBEDE;  // One-byte header extension
+  uint8_t sub_extension_id = 3;         // ID for Absolute Send Time
+  uint8_t sub_extension_length =
+      2;  // Length of the extension data in bytes minus 1
+
+  uint32_t abs_send_time =
+      std::chrono::duration_cast<std::chrono::microseconds>(
+          std::chrono::system_clock::now().time_since_epoch())
+          .count();
+  abs_send_time &= 0x00FFFFFF;  // Absolute Send Time is 24 bits
+
+  // Add extension profile
+  rtp_packet_frame.push_back((extension_profile >> 8) & 0xFF);
+  rtp_packet_frame.push_back(extension_profile & 0xFF);
+
+  // Add extension length (in 32-bit words, minus one)
+  rtp_packet_frame.push_back(
+      0x00);  // Placeholder for length, will be updated later
+  rtp_packet_frame.push_back(0x01);  // One 32-bit word
+
+  // Add Absolute Send Time extension
+  rtp_packet_frame.push_back((sub_extension_id << 4) | sub_extension_length);
+  rtp_packet_frame.push_back((abs_send_time >> 16) & 0xFF);
+  rtp_packet_frame.push_back((abs_send_time >> 8) & 0xFF);
+  rtp_packet_frame.push_back(abs_send_time & 0xFF);
+}

src/rtp/rtp_packetizer/rtp_packetizer_generic.h

@@ -18,6 +18,9 @@ class RtpPacketizerGeneric : public RtpPacketizer {
   std::vector<RtpPacket> Build(uint8_t* payload,
                                uint32_t payload_size) override;
 
+ private:
+  void AddAbsSendTimeExtension(std::vector<uint8_t>& rtp_packet_frame);
+
  private:
   uint8_t version_;
   bool has_padding_;

src/rtp/rtp_packetizer/rtp_packetizer_h264.cpp

@@ -7,7 +7,7 @@ RtpPacketizerH264::RtpPacketizerH264()
       csrc_count_(0),
       marker_(false),
       payload_type_(rtp::PAYLOAD_TYPE::H264),
-      sequence_number_(1),
+      sequence_number_(0),
       timestamp_(0),
       ssrc_(0),
       profile_(0),
@@ -23,7 +23,6 @@ std::vector<RtpPacket> RtpPacketizerH264::Build(uint8_t* payload,
     return BuildNalu(payload, payload_size);
   } else {
     return BuildFua(payload, payload_size);
-    // return std::vector<RtpPacket>();
   }
 }
 
@@ -70,7 +69,6 @@ void RtpPacketizerH264::AddAbsSendTimeExtension(
 
 std::vector<RtpPacket> RtpPacketizerH264::BuildNalu(uint8_t* payload,
                                                     uint32_t payload_size) {
-  LOG_ERROR("payload_size_ = {}", payload_size);
   std::vector<RtpPacket> rtp_packets;
 
   version_ = kRtpVersion;
@@ -120,20 +118,15 @@ std::vector<RtpPacket> RtpPacketizerH264::BuildNalu(uint8_t* payload,
     AddAbsSendTimeExtension(rtp_packet_frame_);
   }
 
-  rtp_packet_frame_.push_back(fu_indicator.forbidden_bit << 7 |
+  rtp_packet_frame_.push_back((fu_indicator.forbidden_bit << 7) |
-                              fu_indicator.nal_reference_idc << 6 |
+                              (fu_indicator.nal_reference_idc << 5) |
                               fu_indicator.nal_unit_type);
 
-  LOG_ERROR("1 [{} {} {}]", (int)fu_indicator.forbidden_bit,
-            (int)fu_indicator.nal_reference_idc,
-            (int)fu_indicator.nal_unit_type);
-
   rtp_packet_frame_.insert(rtp_packet_frame_.end(), payload,
                            payload + payload_size);
 
   RtpPacket rtp_packet;
   rtp_packet.Build(rtp_packet_frame_.data(), rtp_packet_frame_.size());
-
   rtp_packets.emplace_back(rtp_packet);
 
   return rtp_packets;
@@ -157,7 +150,7 @@ std::vector<RtpPacket> RtpPacketizerH264::BuildFua(uint8_t* payload,
     has_padding_ = false;
     has_extension_ = true;
     csrc_count_ = 0;
-    marker_ = index == packet_num - 1 ? 1 : 0;
+    marker_ = (index == (packet_num - 1)) ? 1 : 0;
     payload_type_ = rtp::PAYLOAD_TYPE(payload_type_);
     sequence_number_++;
     timestamp_ = timestamp_;
@@ -193,11 +186,12 @@ std::vector<RtpPacket> RtpPacketizerH264::BuildFua(uint8_t* payload,
     rtp_packet_frame_.push_back((ssrc_ >> 8) & 0xFF);
     rtp_packet_frame_.push_back(ssrc_ & 0xFF);
 
-    for (uint32_t index = 0; index < csrc_count_ && !csrcs_.empty(); index++) {
+    for (uint32_t csrc_index = 0; csrc_index < csrc_count_ && !csrcs_.empty();
-      rtp_packet_frame_.push_back((csrcs_[index] >> 24) & 0xFF);
+         csrc_index++) {
-      rtp_packet_frame_.push_back((csrcs_[index] >> 16) & 0xFF);
+      rtp_packet_frame_.push_back((csrcs_[csrc_index] >> 24) & 0xFF);
-      rtp_packet_frame_.push_back((csrcs_[index] >> 8) & 0xFF);
+      rtp_packet_frame_.push_back((csrcs_[csrc_index] >> 16) & 0xFF);
-      rtp_packet_frame_.push_back(csrcs_[index] & 0xFF);
+      rtp_packet_frame_.push_back((csrcs_[csrc_index] >> 8) & 0xFF);
+      rtp_packet_frame_.push_back(csrcs_[csrc_index] & 0xFF);
     }
 
     if (has_extension_) {
@@ -205,19 +199,21 @@ std::vector<RtpPacket> RtpPacketizerH264::BuildFua(uint8_t* payload,
     }
 
     rtp_packet_frame_.push_back(fu_indicator.forbidden_bit << 7 |
-                                fu_indicator.nal_reference_idc << 6 |
+                                fu_indicator.nal_reference_idc << 5 |
                                 fu_indicator.nal_unit_type);
 
     rtp_packet_frame_.push_back(fu_header.start << 7 | fu_header.end << 6 |
-                                fu_header.remain_bit << 1 |
+                                fu_header.remain_bit << 5 |
                                 fu_header.nal_unit_type);
 
     if (index == packet_num - 1 && last_packet_size > 0) {
-      rtp_packet_frame_.insert(rtp_packet_frame_.end(), payload,
+      rtp_packet_frame_.insert(
-                               payload + last_packet_size);
+          rtp_packet_frame_.end(), payload + index * MAX_NALU_LEN,
+          payload + index * MAX_NALU_LEN + last_packet_size);
     } else {
-      rtp_packet_frame_.insert(rtp_packet_frame_.end(), payload,
+      rtp_packet_frame_.insert(rtp_packet_frame_.end(),
-                               payload + MAX_NALU_LEN);
+                               payload + index * MAX_NALU_LEN,
+                               payload + index * MAX_NALU_LEN + MAX_NALU_LEN);
     }
 
     RtpPacket rtp_packet;
@@ -225,6 +221,7 @@ std::vector<RtpPacket> RtpPacketizerH264::BuildFua(uint8_t* payload,
 
     rtp_packets.emplace_back(rtp_packet);
   }
+
   return rtp_packets;
 }