Implementation for FuA encode and decode

src/rtp/rtp_packet.cpp

@@ -150,9 +150,9 @@ const uint8_t *RtpPacket::EncodeH264Nalu(uint8_t *payload,
       (has_extension_ && extension_data_ ? extension_len_ : 0) +
       extension_offset;
 
-  buffer_[12 + payload_offset] = nalu_header_.forbidden_bit << 7 |
+  buffer_[12 + payload_offset] = fu_indicator_.forbidden_bit << 7 |
-                                 nalu_header_.nal_reference_idc << 6 |
+                                 fu_indicator_.nal_reference_idc << 6 |
-                                 nalu_header_.nal_unit_type;
+                                 fu_indicator_.nal_unit_type;
 
   memcpy(buffer_ + 13 + payload_offset, payload, payload_size);
   size_ = payload_size + (13 + payload_offset);
@@ -160,6 +160,57 @@ const uint8_t *RtpPacket::EncodeH264Nalu(uint8_t *payload,
   return buffer_;
 }
 
+const uint8_t *RtpPacket::EncodeH264Fua(uint8_t *payload, size_t payload_size) {
+  buffer_[0] = (version_ << 6) | (has_padding_ << 5) | (has_extension_ << 4) |
+               total_csrc_number_;
+  buffer_[1] = (marker_ << 7) | payload_type_;
+  buffer_[2] = (sequence_number_ >> 8) & 0xFF;
+  buffer_[3] = sequence_number_ & 0xFF;
+  buffer_[4] = (timestamp_ >> 24) & 0xFF;
+  buffer_[5] = (timestamp_ >> 16) & 0xFF;
+  buffer_[6] = (timestamp_ >> 8) & 0xFF;
+  buffer_[7] = timestamp_ & 0xFF;
+  buffer_[8] = (ssrc_ >> 24) & 0xFF;
+  buffer_[9] = (ssrc_ >> 16) & 0xFF;
+  buffer_[10] = (ssrc_ >> 8) & 0xFF;
+  buffer_[11] = ssrc_ & 0xFF;
+
+  for (uint32_t index = 0; index < total_csrc_number_ && !csrcs_.empty();
+       index++) {
+    buffer_[12 + index] = (csrcs_[index] >> 24) & 0xFF;
+    buffer_[13 + index] = (csrcs_[index] >> 16) & 0xFF;
+    buffer_[14 + index] = (csrcs_[index] >> 8) & 0xFF;
+    buffer_[15 + index] = csrcs_[index] & 0xFF;
+  }
+
+  uint32_t extension_offset =
+      total_csrc_number_ && !csrcs_.empty() ? total_csrc_number_ * 4 : 0;
+  if (has_extension_ && extension_data_) {
+    buffer_[12 + extension_offset] = extension_profile_ >> 8;
+    buffer_[13 + extension_offset] = extension_profile_ & 0xff;
+    buffer_[14 + extension_offset] = (extension_len_ >> 8) & 0xFF;
+    buffer_[15 + extension_offset] = extension_len_ & 0xFF;
+    memcpy(buffer_ + 16 + extension_offset, extension_data_, extension_len_);
+  }
+
+  uint32_t payload_offset =
+      (has_extension_ && extension_data_ ? extension_len_ : 0) +
+      extension_offset;
+
+  buffer_[12 + payload_offset] = fu_indicator_.forbidden_bit << 7 |
+                                 fu_indicator_.nal_reference_idc << 6 |
+                                 fu_indicator_.nal_unit_type;
+
+  buffer_[13 + payload_offset] = fu_header_.start << 7 | fu_header_.end << 6 |
+                                 fu_header_.remain_bit << 1 |
+                                 fu_header_.nal_unit_type;
+
+  memcpy(buffer_ + 14 + payload_offset, payload, payload_size);
+  size_ = payload_size + (14 + payload_offset);
+
+  return buffer_;
+}
+
 const uint8_t *RtpPacket::Decode() {
   version_ = (buffer_[0] >> 6) & 0x03;
   has_padding_ = (buffer_[0] >> 5) & 0x01;
@@ -236,12 +287,61 @@ size_t RtpPacket::DecodeH264Nalu(uint8_t *payload) {
   uint32_t payload_offset =
       (has_extension_ ? extension_len_ : 0) + extension_offset;
 
-  nalu_header_.forbidden_bit = (buffer_[12 + payload_offset] >> 7) & 0x01;
+  fu_indicator_.forbidden_bit = (buffer_[12 + payload_offset] >> 7) & 0x01;
-  nalu_header_.nal_reference_idc = (buffer_[12 + payload_offset] >> 6) & 0x01;
+  fu_indicator_.nal_reference_idc = (buffer_[12 + payload_offset] >> 5) & 0x03;
-  nalu_header_.nal_unit_type = (buffer_[12 + payload_offset]) & 0x31;
+  fu_indicator_.nal_unit_type = buffer_[12 + payload_offset] & 0x1F;
 
   payload_size_ = size_ - (13 + payload_offset);
   payload_ = buffer_ + 13 + payload_offset;
   memcpy(payload, payload_, payload_size_);
   return payload_size_;
 }
+
+size_t RtpPacket::DecodeH264Fua(uint8_t *payload) {
+  version_ = (buffer_[0] >> 6) & 0x03;
+  has_padding_ = (buffer_[0] >> 5) & 0x01;
+  has_extension_ = (buffer_[0] >> 4) & 0x01;
+  total_csrc_number_ = buffer_[0] & 0x0f;
+  marker_ = (buffer_[1] >> 7) & 0x01;
+  payload_type_ = buffer_[1] & 0x7f;
+  sequence_number_ = (buffer_[2] << 8) | buffer_[3];
+  timestamp_ =
+      (buffer_[4] << 24) | (buffer_[5] << 16) | (buffer_[6] << 8) | buffer_[7];
+  ssrc_ = (buffer_[8] << 24) | (buffer_[9] << 16) | (buffer_[10] << 8) |
+          buffer_[11];
+
+  for (uint32_t index = 0; index < total_csrc_number_; index++) {
+    uint32_t csrc = (buffer_[12 + index] << 24) | (buffer_[13 + index] << 16) |
+                    (buffer_[14 + index] << 8) | buffer_[15 + index];
+    csrcs_.push_back(csrc);
+  }
+
+  uint32_t extension_offset = total_csrc_number_ * 4;
+  if (has_extension_) {
+    extension_profile_ =
+        (buffer_[12 + extension_offset] << 8) | buffer_[13 + extension_offset];
+    extension_len_ =
+        (buffer_[14 + extension_offset] << 8) | buffer_[15 + extension_offset];
+
+    // extension_data_ = new uint8_t[extension_len_];
+    // memcpy(extension_data_, buffer_ + 16 + extension_offset, extension_len_);
+    extension_data_ = buffer_ + 16 + extension_offset;
+  }
+
+  uint32_t payload_offset =
+      (has_extension_ ? extension_len_ : 0) + extension_offset;
+
+  fu_indicator_.forbidden_bit = (buffer_[12 + payload_offset] >> 7) & 0x01;
+  fu_indicator_.nal_reference_idc = (buffer_[12 + payload_offset] >> 5) & 0x03;
+  fu_indicator_.nal_unit_type = buffer_[12 + payload_offset] & 0x1F;
+
+  fu_header_.start = (buffer_[13 + payload_offset] >> 7) & 0x01;
+  fu_header_.end = (buffer_[13 + payload_offset] >> 6) & 0x01;
+  fu_header_.remain_bit = (buffer_[13 + payload_offset] >> 5) & 0x01;
+  fu_header_.nal_unit_type = buffer_[13 + payload_offset] & 0x1F;
+
+  payload_size_ = size_ - (14 + payload_offset);
+  payload_ = buffer_ + 14 + payload_offset;
+  memcpy(payload, payload_, payload_size_);
+  return payload_size_;
+}

src/rtp/rtp_packet.h

@@ -106,43 +106,35 @@ class RtpPacket {
     unsigned char forbidden_bit : 1;
     unsigned char nal_reference_idc : 2;
     unsigned char nal_unit_type : 5;
-  } NALU_HEADER;
-
-  typedef struct {
-    unsigned char f : 1;
-    unsigned char nri : 2;
-    unsigned char type : 5;
   } FU_INDICATOR;
 
   typedef struct {
-    unsigned char s : 1;
+    unsigned char start : 1;
-    unsigned char e : 1;
+    unsigned char end : 1;
-    unsigned char r : 1;
+    unsigned char remain_bit : 1;
-    unsigned char type : 5;
+    unsigned char nal_unit_type : 5;
   } FU_HEADER;
 
-  void SetNaluHeader(NALU_HEADER nalu_header) {
+  void SetFuIndicator(FU_INDICATOR fu_indicator) {
-    nalu_header_.forbidden_bit = nalu_header.forbidden_bit;
+    fu_indicator_.forbidden_bit = fu_indicator.forbidden_bit;
-    nalu_header_.nal_reference_idc = nalu_header.nal_reference_idc;
+    fu_indicator_.nal_reference_idc = fu_indicator.nal_reference_idc;
-    nalu_header_.nal_unit_type = nalu_header.nal_unit_type;
+    fu_indicator_.nal_unit_type = fu_indicator.nal_unit_type;
   }
 
-  void FuAHeader(FU_INDICATOR fu_indicator, FU_HEADER fu_header) {
+  void SetFuHeader(FU_HEADER fu_header) {
-    fu_indicator_.f = fu_indicator.f;
+    fu_header_.start = fu_header.start;
-    fu_indicator_.nri = fu_indicator.nri;
+    fu_header_.end = fu_header.end;
-    fu_indicator_.type = fu_indicator.nri;
+    fu_header_.remain_bit = fu_header.remain_bit;
-
+    fu_header_.nal_unit_type = fu_header.nal_unit_type;
-    fu_header_.s = fu_header.s;
-    fu_header_.e = fu_header.e;
-    fu_header_.r = fu_header.r;
-    fu_header_.type = fu_header.type;
   }
 
  public:
   const uint8_t *Encode(uint8_t *payload, size_t payload_size);
   const uint8_t *EncodeH264Nalu(uint8_t *payload, size_t payload_size);
+  const uint8_t *EncodeH264Fua(uint8_t *payload, size_t payload_size);
   const uint8_t *Decode();
   size_t DecodeH264Nalu(uint8_t *payload);
+  size_t DecodeH264Fua(uint8_t *payload);
 
  public:
   // Get Header
@@ -182,7 +174,6 @@ class RtpPacket {
   uint16_t extension_profile_ = 0;
   uint16_t extension_len_ = 0;
   uint8_t *extension_data_ = nullptr;
-  NALU_HEADER nalu_header_;
   FU_INDICATOR fu_indicator_;
   FU_HEADER fu_header_;
 

src/rtp/rtp_session.cpp

@@ -5,6 +5,9 @@
 #include "log.h"
 
 #define RTP_VERSION 1
+#define NALU 1
+#define FU_A 28
+#define FU_B 29
 
 RtpSession ::RtpSession(PAYLOAD_TYPE payload_type)
     : version_(RTP_VERSION),
@@ -55,11 +58,11 @@ void RtpSession::Encode(uint8_t* buffer, size_t size,
       rtp_packet.SetExtensionData(extension_data_, extension_len_);
     }
 
-    RtpPacket::NALU_HEADER nalu_header;
+    RtpPacket::FU_INDICATOR fu_indicator;
-    nalu_header.forbidden_bit = 0;
+    fu_indicator.forbidden_bit = 0;
-    nalu_header.nal_reference_idc = 1;
+    fu_indicator.nal_reference_idc = 1;
-    nalu_header.nal_unit_type = 1;
+    fu_indicator.nal_unit_type = NALU;
-    rtp_packet.SetNaluHeader(nalu_header);
+    rtp_packet.SetFuIndicator(fu_indicator);
 
     rtp_packet.EncodeH264Nalu(buffer, size);
     packets.emplace_back(rtp_packet);
@@ -68,12 +71,11 @@ void RtpSession::Encode(uint8_t* buffer, size_t size,
     size_t last_packet_size = size % MAX_NALU_LEN;
     size_t packet_num = size / MAX_NALU_LEN + (last_packet_size ? 1 : 0);
 
-    for (size_t index = 0; index * MAX_NALU_LEN < size + MAX_NALU_LEN;
+    for (size_t index = 0; index < packet_num; index++) {
-         index++) {
       rtp_packet.SetVerion(version_);
       rtp_packet.SetHasPadding(has_padding_);
       rtp_packet.SetHasExtension(has_extension_);
-      rtp_packet.SetMarker(1);
+      rtp_packet.SetMarker(index == packet_num ? 1 : 0);
       rtp_packet.SetPayloadType(PAYLOAD_TYPE(payload_type_));
       rtp_packet.SetSequenceNumber(sequence_number_++);
 
@@ -91,12 +93,49 @@ void RtpSession::Encode(uint8_t* buffer, size_t size,
         rtp_packet.SetExtensionData(extension_data_, extension_len_);
       }
 
-      rtp_packet.Encode(buffer, size);
+      RtpPacket::FU_INDICATOR fu_indicator;
+      fu_indicator.forbidden_bit = 0;
+      fu_indicator.nal_reference_idc = 0;
+      fu_indicator.nal_unit_type = FU_A;
+
+      RtpPacket::FU_HEADER fu_header;
+      fu_header.start = index == 0 ? 1 : 0;
+      fu_header.end = index == packet_num - 1 ? 1 : 0;
+      fu_header.remain_bit = 0;
+      fu_header.nal_unit_type = FU_A;
+
+      rtp_packet.SetFuIndicator(fu_indicator);
+      rtp_packet.SetFuHeader(fu_header);
+
+      if (index == packet_num - 1 && last_packet_size > 0) {
+        rtp_packet.EncodeH264Fua(buffer + index * MAX_NALU_LEN,
+                                 last_packet_size);
+      } else {
+        rtp_packet.EncodeH264Fua(buffer + index * MAX_NALU_LEN, MAX_NALU_LEN);
+      }
       packets.emplace_back(rtp_packet);
     }
   }
 }
 
 size_t RtpSession::Decode(RtpPacket& packet, uint8_t* payload) {
-  return packet.DecodeH264Nalu(payload);
+  // if ((packet.Buffer()[13] >> 6) & 0x01) {
+  //   LOG_ERROR("End bit!!!!!!!!!!!!!!!");
+  // }
+
+  // if ((packet.Buffer()[13] >> 7) & 0x01) {
+  //   LOG_ERROR("Start bit!!!!!!!!!!!!!!!");
+  // }
+  auto nal_unit_type = packet.Buffer()[12] & 0x1F;
+
+  if (NALU == nal_unit_type) {
+    LOG_ERROR("Nalu");
+    return packet.DecodeH264Nalu(payload);
+  } else if (FU_A == nal_unit_type) {
+    LOG_ERROR("Fua");
+    return packet.DecodeH264Fua(payload);
+  } else {
+    LOG_ERROR("Default Nalu");
+    return packet.DecodeH264Nalu(payload);
+  }
 }

src/rtp/rtp_session.h

@@ -36,6 +36,7 @@ class RtpSession
 
  private:
   // RtpPacket* rtp_packet_ = nullptr;
+  RtpPacket::FU_INDICATOR fu_indicator_;
 };
 
 #endif

src/transmission/ice_transmission.cpp

@@ -287,17 +287,19 @@ int IceTransmission::SendData(const char *data, size_t size) {
 
     std::vector<RtpPacket> packets;
 
-    for (size_t num = 0, next_packet_size = 0; num * MAX_NALU_LEN < size;
+    // for (size_t num = 0, next_packet_size = 0; num * MAX_NALU_LEN < size;
-         num++) {
+    //      num++) {
-      next_packet_size = size - num * MAX_NALU_LEN;
+    //   next_packet_size = size - num * MAX_NALU_LEN;
-      if (next_packet_size < MAX_NALU_LEN) {
+    //   if (next_packet_size < MAX_NALU_LEN) {
-        video_rtp_session_->Encode((uint8_t *)(data + num * MAX_NALU_LEN),
+    //     video_rtp_session_->Encode((uint8_t *)(data + num * MAX_NALU_LEN),
-                                   next_packet_size, packets);
+    //                                next_packet_size, packets);
-      } else {
+    //   } else {
-        video_rtp_session_->Encode((uint8_t *)(data + num * MAX_NALU_LEN),
+    //     video_rtp_session_->Encode((uint8_t *)(data + num * MAX_NALU_LEN),
-                                   MAX_NALU_LEN, packets);
+    //                                MAX_NALU_LEN, packets);
-      }
+    //   }
-    }
+    // }
+
+    video_rtp_session_->Encode((uint8_t *)data, size, packets);
 
     for (auto &packet : packets) {
       send_ringbuffer_.push(packet);