Fix H264 decode error caused by fec packets

src/rtp/rtp_codec.cpp

@@ -49,18 +49,18 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size,
       fec_encoder_.GetFecPacketsParams(size, num_of_total_packets,
                                        num_of_source_packets, last_packet_size);
 
+      timestamp_ =
+          std::chrono::high_resolution_clock::now().time_since_epoch().count();
+
       for (size_t index = 0; index < num_of_total_packets; index++) {
+        if (index < num_of_source_packets) {
           RtpPacket rtp_packet;
           rtp_packet.SetVerion(version_);
           rtp_packet.SetHasPadding(has_padding_);
           rtp_packet.SetHasExtension(has_extension_);
-        rtp_packet.SetMarker(index == num_of_total_packets ? 1 : 0);
+          rtp_packet.SetMarker(index == num_of_source_packets ? 1 : 0);
-        rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_));
+          rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE::H264);
           rtp_packet.SetSequenceNumber(sequence_number_++);
-
-        timestamp_ = std::chrono::high_resolution_clock::now()
-                         .time_since_epoch()
-                         .count();
           rtp_packet.SetTimestamp(timestamp_);
           rtp_packet.SetSsrc(ssrc_);
 
@@ -80,19 +80,51 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size,
 
           RtpPacket::FU_HEADER fu_header;
           fu_header.start = index == 0 ? 1 : 0;
-        fu_header.end = index == num_of_total_packets - 1 ? 1 : 0;
+          fu_header.end = index == num_of_source_packets - 1 ? 1 : 0;
           fu_header.remain_bit = 0;
           fu_header.nal_unit_type = FU_A;
 
           rtp_packet.SetFuIndicator(fu_indicator);
           rtp_packet.SetFuHeader(fu_header);
 
-        if (index == num_of_source_packets - 1 && last_packet_size > 0) {
+          if (index == num_of_source_packets - 1) {
+            if (last_packet_size > 0) {
               rtp_packet.EncodeH264Fua(fec_packets[index], last_packet_size);
             } else {
               rtp_packet.EncodeH264Fua(fec_packets[index], MAX_NALU_LEN);
             }
+          } else {
+            rtp_packet.EncodeH264Fua(fec_packets[index], MAX_NALU_LEN);
+          }
           packets.emplace_back(rtp_packet);
+        } else if (index >= num_of_source_packets &&
+                   index < num_of_total_packets) {
+          RtpPacket rtp_packet;
+          rtp_packet.SetVerion(version_);
+          rtp_packet.SetHasPadding(has_padding_);
+          rtp_packet.SetHasExtension(has_extension_);
+          rtp_packet.SetMarker(index == num_of_total_packets ? 1 : 0);
+          rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE::H264_FEC);
+          rtp_packet.SetSequenceNumber(sequence_number_++);
+          rtp_packet.SetTimestamp(timestamp_);
+          rtp_packet.SetSsrc(ssrc_);
+
+          if (!csrcs_.empty()) {
+            rtp_packet.SetCsrcs(csrcs_);
+          }
+
+          if (has_extension_) {
+            rtp_packet.SetExtensionProfile(extension_profile_);
+            rtp_packet.SetExtensionData(extension_data_, extension_len_);
+          }
+          rtp_packet.EncodeH264Fec(fec_packets[index], MAX_NALU_LEN);
+          packets.emplace_back(rtp_packet);
+        }
+
+        // if (index < num_of_source_packets) {
+        //   rtp_packet.EncodeH264Fua(fec_packets[index], MAX_NALU_LEN);
+        //   packets.emplace_back(rtp_packet);
+        // }
       }
 
       fec_encoder_.ReleaseFecPackets(fec_packets, size);
@@ -134,6 +166,8 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size,
     } else {
       size_t last_packet_size = size % MAX_NALU_LEN;
       size_t packet_num = size / MAX_NALU_LEN + (last_packet_size ? 1 : 0);
+      timestamp_ =
+          std::chrono::high_resolution_clock::now().time_since_epoch().count();
 
       for (size_t index = 0; index < packet_num; index++) {
         RtpPacket rtp_packet;
@@ -143,10 +177,6 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size,
         rtp_packet.SetMarker(index == packet_num ? 1 : 0);
         rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_));
         rtp_packet.SetSequenceNumber(sequence_number_++);
-
-        timestamp_ = std::chrono::high_resolution_clock::now()
-                         .time_since_epoch()
-                         .count();
         rtp_packet.SetTimestamp(timestamp_);
         rtp_packet.SetSsrc(ssrc_);
 

src/rtp/rtp_packet.cpp

@@ -5,17 +5,19 @@
 #include "log.h"
 
 void RtpPacket::TryToDecodeRtpPacket() {
-  if (PAYLOAD_TYPE::H264 == NAL_UNIT_TYPE(buffer_[1] & 0x7F)) {
+  if (PAYLOAD_TYPE::H264 == PAYLOAD_TYPE(buffer_[1] & 0x7F)) {
     nal_unit_type_ = NAL_UNIT_TYPE(buffer_[12] & 0x1F);
     if (NAL_UNIT_TYPE::NALU == nal_unit_type_) {
       DecodeH264Nalu();
     } else if (NAL_UNIT_TYPE::FU_A == nal_unit_type_) {
       DecodeH264Fua();
     }
-  } else if (PAYLOAD_TYPE::DATA == NAL_UNIT_TYPE(buffer_[1] & 0x7F)) {
+  } else if (PAYLOAD_TYPE::H264_FEC == PAYLOAD_TYPE(buffer_[1] & 0x7F)) {
+    DecodeH264Fec();
+  } else if (PAYLOAD_TYPE::DATA == PAYLOAD_TYPE(buffer_[1] & 0x7F)) {
     DecodeData();
   } else {
-    LOG_ERROR("Unknown pt: {}", NAL_UNIT_TYPE(buffer_[1] & 0x7F));
+    LOG_ERROR("Unknown pt: {}", PAYLOAD_TYPE(buffer_[1] & 0x7F));
   }
 }
 
@@ -243,6 +245,59 @@ const uint8_t *RtpPacket::EncodeH264Fua(uint8_t *payload, size_t payload_size) {
   return buffer_;
 }
 
+const uint8_t *RtpPacket::EncodeH264Fec(uint8_t *payload, size_t payload_size) {
+  buffer_[0] = (version_ << 6) | (has_padding_ << 5) | (has_extension_ << 4) |
+               total_csrc_number_;
+  buffer_[1] = (marker_ << 7) | payload_type_;
+  buffer_[2] = (sequence_number_ >> 8) & 0xFF;
+  buffer_[3] = sequence_number_ & 0xFF;
+  buffer_[4] = (timestamp_ >> 24) & 0xFF;
+  buffer_[5] = (timestamp_ >> 16) & 0xFF;
+  buffer_[6] = (timestamp_ >> 8) & 0xFF;
+  buffer_[7] = timestamp_ & 0xFF;
+  buffer_[8] = (ssrc_ >> 24) & 0xFF;
+  buffer_[9] = (ssrc_ >> 16) & 0xFF;
+  buffer_[10] = (ssrc_ >> 8) & 0xFF;
+  buffer_[11] = ssrc_ & 0xFF;
+
+  for (uint32_t index = 0; index < total_csrc_number_ && !csrcs_.empty();
+       index++) {
+    buffer_[12 + index] = (csrcs_[index] >> 24) & 0xFF;
+    buffer_[13 + index] = (csrcs_[index] >> 16) & 0xFF;
+    buffer_[14 + index] = (csrcs_[index] >> 8) & 0xFF;
+    buffer_[15 + index] = csrcs_[index] & 0xFF;
+  }
+
+  uint32_t extension_offset =
+      total_csrc_number_ && !csrcs_.empty() ? total_csrc_number_ * 4 : 0;
+  if (has_extension_ && extension_data_) {
+    buffer_[12 + extension_offset] = extension_profile_ >> 8;
+    buffer_[13 + extension_offset] = extension_profile_ & 0xff;
+    buffer_[14 + extension_offset] = (extension_len_ >> 8) & 0xFF;
+    buffer_[15 + extension_offset] = extension_len_ & 0xFF;
+    memcpy(buffer_ + 16 + extension_offset, extension_data_, extension_len_);
+  }
+
+  uint32_t fec_symbol_id_offset =
+      extension_offset + (has_extension_ && extension_data_ ? 4 : 0);
+  buffer_[12 + fec_symbol_id_offset] = fec_symbol_id_;
+
+  uint32_t payload_offset = fec_symbol_id_offset + 1;
+
+  buffer_[12 + payload_offset] = fu_indicator_.forbidden_bit << 7 |
+                                 fu_indicator_.nal_reference_idc << 6 |
+                                 fu_indicator_.nal_unit_type;
+
+  buffer_[13 + payload_offset] = fu_header_.start << 7 | fu_header_.end << 6 |
+                                 fu_header_.remain_bit << 1 |
+                                 fu_header_.nal_unit_type;
+
+  memcpy(buffer_ + 14 + payload_offset, payload, payload_size);
+  size_ = payload_size + (14 + payload_offset);
+
+  return buffer_;
+}
+
 size_t RtpPacket::DecodeData(uint8_t *payload) {
   version_ = (buffer_[0] >> 6) & 0x03;
   has_padding_ = (buffer_[0] >> 5) & 0x01;
@@ -360,9 +415,6 @@ size_t RtpPacket::DecodeH264Fua(uint8_t *payload) {
     extension_len_ =
         (buffer_[14 + extension_offset] << 8) | buffer_[15 + extension_offset];
 
-    // extension_data_ = new uint8_t[extension_len_];
-    // memcpy(extension_data_, buffer_ + 16 + extension_offset,
-    // extension_len_);
     extension_data_ = buffer_ + 16 + extension_offset;
   }
 
@@ -385,3 +437,45 @@ size_t RtpPacket::DecodeH264Fua(uint8_t *payload) {
   }
   return payload_size_;
 }
+
+size_t RtpPacket::DecodeH264Fec(uint8_t *payload) {
+  version_ = (buffer_[0] >> 6) & 0x03;
+  has_padding_ = (buffer_[0] >> 5) & 0x01;
+  has_extension_ = (buffer_[0] >> 4) & 0x01;
+  total_csrc_number_ = buffer_[0] & 0x0f;
+  marker_ = (buffer_[1] >> 7) & 0x01;
+  payload_type_ = buffer_[1] & 0x7f;
+  sequence_number_ = (buffer_[2] << 8) | buffer_[3];
+  timestamp_ =
+      (buffer_[4] << 24) | (buffer_[5] << 16) | (buffer_[6] << 8) | buffer_[7];
+  ssrc_ = (buffer_[8] << 24) | (buffer_[9] << 16) | (buffer_[10] << 8) |
+          buffer_[11];
+
+  for (uint32_t index = 0; index < total_csrc_number_; index++) {
+    uint32_t csrc = (buffer_[12 + index] << 24) | (buffer_[13 + index] << 16) |
+                    (buffer_[14 + index] << 8) | buffer_[15 + index];
+    csrcs_.push_back(csrc);
+  }
+
+  uint32_t extension_offset = total_csrc_number_ * 4;
+  if (has_extension_) {
+    extension_profile_ =
+        (buffer_[12 + extension_offset] << 8) | buffer_[13 + extension_offset];
+    extension_len_ =
+        (buffer_[14 + extension_offset] << 8) | buffer_[15 + extension_offset];
+
+    extension_data_ = buffer_ + 16 + extension_offset;
+  }
+
+  uint32_t fec_symbol_id_offset = extension_offset + (has_extension_ ? 4 : 0);
+  fec_symbol_id_ = buffer_[12 + fec_symbol_id_offset];
+
+  uint32_t payload_offset = fec_symbol_id_offset + 1;
+
+  payload_size_ = size_ - (14 + payload_offset);
+  payload_ = buffer_ + 14 + payload_offset;
+  if (payload) {
+    memcpy(payload, payload_, payload_size_);
+  }
+  return payload_size_;
+}

src/rtp/rtp_packet.h

@@ -17,13 +17,13 @@
 // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 // |           synchronization source (SSRC) identifier            |
 // +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
-// |            Contributing source (CSRC) identifiers             |
+// |            Contributing source (CSRC) identifiers             |x
-// |                             ....                              |
+// |                             ....                              |x
 // +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
-// |       defined by profile      |            length             |
+// |       defined by profile      |            length             |x
 // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-// |                          Extensions                           |
+// |                          Extensions                           |x
-// |                             ....                              |
+// |                             ....                              |x
 // +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
 // |                           Payload                             |
 // |             ....              :  padding...                   |
@@ -41,13 +41,13 @@
 // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 // |           synchronization source (SSRC) identifier            |
 // +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
-// |            Contributing source (CSRC) identifiers             |
+// |            Contributing source (CSRC) identifiers             |x
-// |                             ....                              |
+// |                             ....                              |x
 // +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
-// |       defined by profile      |            length             |
+// |       defined by profile      |            length             |x
 // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-// |                          Extensions                           |
+// |                          Extensions                           |x
-// |                             ....                              |
+// |                             ....                              |x
 // +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
 // |  FU indicator |   FU header   |                               |
 // |                                                               |
@@ -63,12 +63,44 @@
 // |F|NRI|   Type  |S|E|R|   Type  |
 // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
+// H264 FEC
+//  0                   1                   2                   3
+//  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |V=2|P|X|  CC   |M|     PT      |       sequence number         |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |                           timestamp                           |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |           synchronization source (SSRC) identifier            |
+// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
+// |            Contributing source (CSRC) identifiers             |x
+// |                             ....                              |x
+// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
+// |       defined by profile      |            length             |x
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |                          Extensions                           |x
+// |                             ....                              |x
+// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
+// | FEC symbol id |                                               |
+// |                                                               |
+// |                          Fec Payload                          |
+// |                                                               |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+// |               padding         | Padding size  |
+// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
 #define DEFAULT_MTU 1500
 #define MAX_NALU_LEN 1400
 
 class RtpPacket {
  public:
-  typedef enum { H264 = 96, OPUS = 97, DATA = 127 } PAYLOAD_TYPE;
+  typedef enum {
+    H264 = 96,
+    H264_FEC = 97,
+    OPUS = 111,
+    DATA = 127
+  } PAYLOAD_TYPE;
+
   typedef enum { UNKNOWN = 0, NALU = 1, FU_A = 28, FU_B = 29 } NAL_UNIT_TYPE;
 
  public:
@@ -133,13 +165,17 @@ class RtpPacket {
     fu_header_.nal_unit_type = fu_header.nal_unit_type;
   }
 
+  void SetFecSymbolId(uint8_t fec_symbol_id) { fec_symbol_id_ = fec_symbol_id; }
+
  public:
   const uint8_t *Encode(uint8_t *payload, size_t payload_size);
   const uint8_t *EncodeH264Nalu(uint8_t *payload, size_t payload_size);
   const uint8_t *EncodeH264Fua(uint8_t *payload, size_t payload_size);
+  const uint8_t *EncodeH264Fec(uint8_t *payload, size_t payload_size);
   size_t DecodeData(uint8_t *payload = nullptr);
   size_t DecodeH264Nalu(uint8_t *payload = nullptr);
   size_t DecodeH264Fua(uint8_t *payload = nullptr);
+  size_t DecodeH264Fec(uint8_t *payload = nullptr);
 
  public:
   // Get Header
@@ -188,6 +224,8 @@ class RtpPacket {
     return extension_data_;
   }
 
+  const uint8_t FecSymbolId() { return fec_symbol_id_; }
+
   // Payload
   const uint8_t *Payload() {
     ParseRtpData();
@@ -238,6 +276,7 @@ class RtpPacket {
   uint8_t *extension_data_ = nullptr;
   FU_INDICATOR fu_indicator_;
   FU_HEADER fu_header_;
+  uint8_t fec_symbol_id_ = 0;
 
   // Payload
   uint8_t *payload_ = nullptr;

src/rtp/rtp_video_receiver.cpp

@@ -48,23 +48,12 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
 
     rtcp_rr.Encode();
 
-    // SendRtcpRR(rtcp_rr);
+    SendRtcpRR(rtcp_rr);
   }
 
   if (RtpPacket::NAL_UNIT_TYPE::NALU == rtp_packet.NalUnitType()) {
     compelete_video_frame_queue_.push(
         VideoFrame(rtp_packet.Payload(), rtp_packet.Size()));
-    // if (on_receive_complete_frame_) {
-    //   auto now_complete_frame_ts =
-    //       std::chrono::high_resolution_clock::now().time_since_epoch().count()
-    //       / 1000000;
-    //   uint32_t duration = now_complete_frame_ts - last_complete_frame_ts_;
-    //   LOG_ERROR("Duration {}", 1000 / duration);
-    //   last_complete_frame_ts_ = now_complete_frame_ts;
-
-    //   on_receive_complete_frame_(
-    //       VideoFrame(rtp_packet.Payload(), rtp_packet.Size()));
-    // }
   } else if (RtpPacket::NAL_UNIT_TYPE::FU_A == rtp_packet.NalUnitType()) {
     incomplete_frame_list_[rtp_packet.SequenceNumber()] = rtp_packet;
     bool complete = CheckIsFrameCompleted(rtp_packet);
@@ -83,6 +72,9 @@ bool RtpVideoReceiver::CheckIsFrameCompleted(RtpPacket& rtp_packet) {
       auto it = incomplete_frame_list_.find(end_seq);
       complete_frame_size += it->second.PayloadSize();
       if (it == incomplete_frame_list_.end()) {
+        // The last fragment has already received. If all fragments are in
+        // order, then some fragments lost in tranmission and need to be
+        // repaired using FEC
         return false;
       } else if (!it->second.FuAStart()) {
         continue;
@@ -105,19 +97,6 @@ bool RtpVideoReceiver::CheckIsFrameCompleted(RtpPacket& rtp_packet) {
         compelete_video_frame_queue_.push(
             VideoFrame(nv12_data_, complete_frame_size));
 
-        // if (on_receive_complete_frame_) {
-        //   auto now_complete_frame_ts =
-        //   std::chrono::high_resolution_clock::now()
-        //                                    .time_since_epoch()
-        //                                    .count() /
-        //                                1000000;
-        //   uint32_t duration = now_complete_frame_ts -
-        //   last_complete_frame_ts_; LOG_ERROR("Duration {}", 1000 / duration);
-        //   last_complete_frame_ts_ = now_complete_frame_ts;
-
-        //   on_receive_complete_frame_(
-        //       VideoFrame(nv12_data_, complete_frame_size));
-        // }
         return true;
       } else {
         LOG_WARN("What happened?")
@@ -166,8 +145,6 @@ int RtpVideoReceiver::SendRtcpRR(RtcpReceiverReport& rtcp_rr) {
     return -1;
   }
 
-  // LOG_ERROR("Send RR");
-
   return 0;
 }