Fix error during encoding obu packet into rtp packet

src/media/video/encode/aom/aom_av1_encoder.cpp

@@ -342,7 +342,7 @@ int AomAv1Encoder::Encode(const uint8_t *pData, int nSize,
 
       int qp = -1;
       SET_ENCODER_PARAM_OR_RETURN_ERROR(AOME_GET_LAST_QUANTIZER, &qp);
-      LOG_INFO("Encoded frame qp = {}", qp);
+      // LOG_INFO("Encoded frame qp = {}", qp);
 
       if (on_encoded_image) {
         on_encoded_image((char *)encoded_frame_, encoded_frame_size_,

src/rtp/obu.cpp

@@ -7,59 +7,67 @@
 Obu::Obu() {}
 
 Obu::Obu(const Obu &obu) {
+  if (obu.size_ > 0) {
+    data_ = (uint8_t *)malloc(obu.size_);
+    memcpy(data_, obu.data_, obu.size_);
+    size_ = obu.size_;
+  }
+
   if (obu.payload_size_ > 0) {
     payload_ = (uint8_t *)malloc(obu.payload_size_);
-    if (NULL == payload_) {
-      LOG_ERROR("Malloc failed");
-    } else {
     memcpy(payload_, obu.payload_, obu.payload_size_);
-    }
     payload_size_ = obu.payload_size_;
-    size_ = obu.size_;
-    header_ = obu.header_;
-    extension_header_ = obu.extension_header_;
-  } else {
-    size_ = obu.size_;
-    header_ = obu.header_;
-    extension_header_ = obu.extension_header_;
   }
+
+  header_ = obu.header_;
+  extension_header_ = obu.extension_header_;
 }
 
 Obu::Obu(Obu &&obu)
-    : payload_((uint8_t *)std::move(obu.payload_)),
+    : data_(std::move(obu.data_)),
-      payload_size_(obu.payload_size_),
       size_(obu.size_),
+      payload_((uint8_t *)std::move(obu.payload_)),
+      payload_size_(obu.payload_size_),
       header_(obu.header_),
       extension_header_(obu.extension_header_) {
+  obu.data_ = nullptr;
+  obu.size_ = 0;
   obu.payload_ = nullptr;
   obu.payload_size_ = 0;
 }
 
 Obu &Obu::operator=(const Obu &obu) {
   if (&obu != this) {
+    if (obu.size_ > 0) {
+      data_ = (uint8_t *)realloc(data_, obu.size_);
+      memcpy(data_, obu.data_, obu.size_);
+      size_ = obu.size_;
+    }
+
     if (obu.payload_size_ > 0) {
       payload_ = (uint8_t *)realloc(payload_, obu.payload_size_);
       memcpy(payload_, obu.payload_, obu.payload_size_);
       payload_size_ = obu.payload_size_;
-      size_ = obu.size_;
-      header_ = obu.header_;
-      extension_header_ = obu.extension_header_;
-    } else {
-      header_ = obu.header_;
-      extension_header_ = obu.extension_header_;
     }
+
+    header_ = obu.header_;
+    extension_header_ = obu.extension_header_;
   }
   return *this;
 }
 
 Obu &Obu::operator=(Obu &&obu) {
   if (&obu != this) {
+    data_ = std::move(obu.data_);
+    obu.data_ = nullptr;
+    size_ = obu.size_;
+    obu.size_ = 0;
+
     payload_ = std::move(obu.payload_);
     obu.payload_ = nullptr;
     payload_size_ = obu.payload_size_;
     obu.payload_size_ = 0;
-    size_ = obu.size_;
+
-    obu.size_ = 0;
     header_ = obu.header_;
     obu.header_ = 0;
     extension_header_ = obu.extension_header_;
@@ -69,12 +77,18 @@ Obu &Obu::operator=(Obu &&obu) {
 }
 
 Obu::~Obu() {
+  if (data_) {
+    free(data_);
+    data_ = nullptr;
+  }
+  size_ = 0;
+
   if (payload_) {
     free(payload_);
     payload_ = nullptr;
   }
   payload_size_ = 0;
-  size_ = 0;
+
   header_ = 0;
   extension_header_ = 0;
 }

src/rtp/obu.h

@@ -27,8 +27,9 @@ class Obu {
   uint8_t header_ = 0;
   uint8_t extension_header_ = 0;  // undefined if (header & kXbit) == 0
   uint8_t *payload_ = nullptr;
-  int size_ = 0;  // size of the header and payload combined.
   int payload_size_ = 0;
+  uint8_t *data_ = nullptr;
+  int size_ = 0;  // size of the header and payload combined.
 };
 
 #endif

src/rtp/obu_parser.cpp

@@ -49,6 +49,8 @@ std::vector<Obu> ParseObus(uint8_t* payload, int payload_size) {
   std::vector<Obu> result;
   ByteBufferReader payload_reader(reinterpret_cast<const char*>(payload),
                                   payload_size);
+
+  int pos = 0;
   while (payload_reader.Length() > 0) {
     Obu obu;
     payload_reader.ReadUInt8(&obu.header_);
@@ -87,6 +89,10 @@ std::vector<Obu> ParseObus(uint8_t* payload, int payload_size) {
       payload_reader.Consume(payload_size);
     }
     obu.size_ += obu.payload_size_;
+    obu.data_ = (uint8_t*)malloc(obu.size_);
+    memcpy(obu.data_, payload + pos, obu.size_);
+    pos += obu.size_;
+
     // Skip obus that shouldn't be transfered over rtp.
     int obu_type = ObuType(obu.header_);
     // if (obu_type != kObuTypeTemporalDelimiter &&  //

src/rtp/rtp_codec.cpp

@@ -247,7 +247,7 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size,
 
         rtp_packet.SetAv1AggrHeader(0, 0, 1, 0);
 
-        rtp_packet.EncodeAv1(obus[i].payload_, obus[i].payload_size_);
+        rtp_packet.EncodeAv1(obus[i].data_, obus[i].payload_size_);
         packets.emplace_back(rtp_packet);
       } else {
         size_t last_packet_size = obus[i].payload_size_ % MAX_NALU_LEN;
@@ -281,10 +281,10 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size,
           rtp_packet.SetAv1AggrHeader(z, y, w, n);
 
           if (index == packet_num - 1 && last_packet_size > 0) {
-            rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN,
+            rtp_packet.EncodeAv1(obus[i].data_ + index * MAX_NALU_LEN,
                                  last_packet_size);
           } else {
-            rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN,
+            rtp_packet.EncodeAv1(obus[i].data_ + index * MAX_NALU_LEN,
                                  MAX_NALU_LEN);
           }
           packets.emplace_back(rtp_packet);
@@ -510,9 +510,9 @@ void RtpCodec::Encode(VideoFrameType frame_type, uint8_t* buffer, size_t size,
     }
   } else if (RtpPacket::PAYLOAD_TYPE::AV1 == payload_type_) {
     std::vector<Obu> obus = ParseObus(buffer, size);
-    LOG_ERROR("Total size = [{}]", size);
+    // LOG_ERROR("Total size = [{}]", size);
     for (int i = 0; i < obus.size(); i++) {
-      LOG_ERROR("[{}] Obu size = [{}], Obu type [{}]", i, obus[i].size_,
+      LOG_ERROR("1 [{}] Obu size = [{}], Obu type [{}]", i, obus[i].size_,
                 ObuTypeToString((OBU_TYPE)ObuType(obus[i].header_)));
       if (obus[i].size_ <= MAX_NALU_LEN) {
         RtpPacket rtp_packet;
@@ -539,13 +539,13 @@ void RtpCodec::Encode(VideoFrameType frame_type, uint8_t* buffer, size_t size,
         }
 
         rtp_packet.SetAv1AggrHeader(0, 0, 1, 0);
-
+        rtp_packet.EncodeAv1(obus[i].data_, obus[i].size_);
-        rtp_packet.EncodeAv1(obus[i].payload_, obus[i].payload_size_);
+        LOG_ERROR("enc payload size = {}", rtp_packet.PayloadSize());
         packets.emplace_back(rtp_packet);
       } else {
-        size_t last_packet_size = obus[i].payload_size_ % MAX_NALU_LEN;
+        size_t last_packet_size = obus[i].size_ % MAX_NALU_LEN;
         size_t packet_num =
-            obus[i].payload_size_ / MAX_NALU_LEN + (last_packet_size ? 1 : 0);
+            obus[i].size_ / MAX_NALU_LEN + (last_packet_size ? 1 : 0);
         timestamp_ = std::chrono::high_resolution_clock::now()
                          .time_since_epoch()
                          .count();
@@ -577,12 +577,14 @@ void RtpCodec::Encode(VideoFrameType frame_type, uint8_t* buffer, size_t size,
           rtp_packet.SetAv1AggrHeader(z, y, w, n);
 
           if (index == packet_num - 1 && last_packet_size > 0) {
-            rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN,
+            rtp_packet.EncodeAv1(obus[i].data_ + index * MAX_NALU_LEN,
                                  last_packet_size);
           } else {
-            rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN,
+            rtp_packet.EncodeAv1(obus[i].data_ + index * MAX_NALU_LEN,
                                  MAX_NALU_LEN);
           }
+
+          LOG_ERROR("enc payload size = {}", rtp_packet.PayloadSize());
           packets.emplace_back(rtp_packet);
         }
       }

src/rtp/rtp_packet.cpp

@@ -755,13 +755,18 @@ size_t RtpPacket::DecodeAv1(uint8_t *payload) {
     extension_data_ = buffer_ + 16 + extension_offset;
   }
 
-  uint32_t payload_offset =
+  uint32_t aggr_header_offset =
       (has_extension_ ? extension_len_ : 0) + extension_offset;
 
+  av1_aggr_header_ = buffer_[12 + aggr_header_offset];
+
+  uint32_t payload_offset = aggr_header_offset;
+
   payload_size_ = size_ - (13 + payload_offset);
   payload_ = buffer_ + 13 + payload_offset;
   if (payload) {
     memcpy(payload, payload_, payload_size_);
   }
+
   return payload_size_;
 }

src/rtp/rtp_packet.h

@@ -311,6 +311,13 @@ class RtpPacket {
 
   uint8_t FecSourceSymbolNum() { return fec_source_symbol_num_; }
 
+  void GetAv1AggrHeader(int &z, int &y, int &w, int &n) {
+    z = av1_aggr_header_ >> 7;
+    y = av1_aggr_header_ >> 6 & 0x01;
+    w = av1_aggr_header_ >> 4 & 0x03;
+    n = av1_aggr_header_ >> 3 & 0x01;
+  }
+
   // Payload
   const uint8_t *Payload() {
     ParseRtpData();

src/rtp/rtp_video_receiver.cpp

@@ -1,6 +1,7 @@
 #include "rtp_video_receiver.h"
 
 #include "log.h"
+#include "obu_parser.h"
 
 #define NV12_BUFFER_SIZE (1280 * 720 * 3 / 2)
 #define RTCP_RR_INTERVAL 1000
@@ -28,12 +29,14 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
     RtcpReportBlock report;
 
     // auto duration = std::chrono::system_clock::now().time_since_epoch();
-    // auto seconds = std::chrono::duration_cast<std::chrono::seconds>(duration);
+    // auto seconds =
-    // uint32_t seconds_u32 = static_cast<uint32_t>(
+    // std::chrono::duration_cast<std::chrono::seconds>(duration); uint32_t
+    // seconds_u32 = static_cast<uint32_t>(
     //     std::chrono::duration_cast<std::chrono::seconds>(duration).count());
 
     // uint32_t fraction_u32 = static_cast<uint32_t>(
-    //     std::chrono::duration_cast<std::chrono::nanoseconds>(duration - seconds)
+    //     std::chrono::duration_cast<std::chrono::nanoseconds>(duration -
+    //     seconds)
     //         .count());
 
     report.source_ssrc = 0x00;
@@ -50,7 +53,14 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
 
     // SendRtcpRR(rtcp_rr);
   }
+  if (rtp_packet.PayloadType() == RtpPacket::PAYLOAD_TYPE::AV1) {
+    ProcessAV1RtpPacket(rtp_packet);
+  } else {
+    ProcessH264RtpPacket(rtp_packet);
+  }
+}
 
+void RtpVideoReceiver::ProcessH264RtpPacket(RtpPacket& rtp_packet) {
   if (!fec_enable_) {
     if (RtpPacket::PAYLOAD_TYPE::H264 == rtp_packet.PayloadType()) {
       if (RtpPacket::NAL_UNIT_TYPE::NALU == rtp_packet.NalUnitType()) {
@@ -58,7 +68,7 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
             VideoFrame(rtp_packet.Payload(), rtp_packet.PayloadSize()));
       } else if (RtpPacket::NAL_UNIT_TYPE::FU_A == rtp_packet.NalUnitType()) {
         incomplete_frame_list_[rtp_packet.SequenceNumber()] = rtp_packet;
-        bool complete = CheckIsFrameCompleted(rtp_packet);
+        bool complete = CheckIsH264FrameCompleted(rtp_packet);
       }
     }
   } else {
@@ -68,7 +78,7 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
             VideoFrame(rtp_packet.Payload(), rtp_packet.PayloadSize()));
       } else if (RtpPacket::NAL_UNIT_TYPE::FU_A == rtp_packet.NalUnitType()) {
         incomplete_frame_list_[rtp_packet.SequenceNumber()] = rtp_packet;
-        bool complete = CheckIsFrameCompleted(rtp_packet);
+        bool complete = CheckIsH264FrameCompleted(rtp_packet);
       }
     } else if (RtpPacket::PAYLOAD_TYPE::H264_FEC_SOURCE ==
                rtp_packet.PayloadType()) {
@@ -163,7 +173,30 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
   }
 }
 
-bool RtpVideoReceiver::CheckIsFrameCompleted(RtpPacket& rtp_packet) {
+void RtpVideoReceiver::ProcessAV1RtpPacket(RtpPacket& rtp_packet) {
+  LOG_ERROR("recv payload size = {}, sequence_number_ = {}",
+            rtp_packet.PayloadSize(), rtp_packet.SequenceNumber());
+
+  int z, y, w, n;
+  rtp_packet.GetAv1AggrHeader(z, y, w, n);
+  LOG_ERROR("z = {}, y = {}, w = {}, n = {}", z, y, w, n);
+
+  if (z == 0) {
+  }
+
+  if (y == 0) {
+  }
+
+  std::vector<Obu> obus =
+      ParseObus((uint8_t*)rtp_packet.Payload(), rtp_packet.PayloadSize());
+  for (int i = 0; i < obus.size(); i++) {
+    LOG_ERROR("2 [{}|{}] Obu size = [{}], Obu type [{}]", i, obus.size(),
+              obus[i].size_,
+              ObuTypeToString((OBU_TYPE)ObuType(obus[i].header_)));
+  }
+}
+
+bool RtpVideoReceiver::CheckIsH264FrameCompleted(RtpPacket& rtp_packet) {
   if (rtp_packet.FuAEnd()) {
     uint16_t end_seq = rtp_packet.SequenceNumber();
     if (incomplete_frame_list_.size() == end_seq) {

src/rtp/rtp_video_receiver.h

@@ -30,7 +30,13 @@ class RtpVideoReceiver : public ThreadBase {
   }
 
  private:
-  bool CheckIsFrameCompleted(RtpPacket& rtp_packet);
+  void ProcessAV1RtpPacket(RtpPacket& rtp_packet);
+
+ private:
+  void ProcessH264RtpPacket(RtpPacket& rtp_packet);
+  bool CheckIsH264FrameCompleted(RtpPacket& rtp_packet);
+
+ private:
   bool CheckIsTimeSendRR();
   int SendRtcpRR(RtcpReceiverReport& rtcp_rr);
 

src/transmission/ice_transmission.cpp

@@ -366,7 +366,8 @@ uint8_t IceTransmission::CheckIsVideoPacket(const char *buffer, size_t size) {
   uint8_t pt = buffer[1] & 0x7F;
   if (RtpPacket::PAYLOAD_TYPE::H264 == pt ||
       RtpPacket::PAYLOAD_TYPE::H264_FEC_SOURCE == pt ||
-      RtpPacket::PAYLOAD_TYPE::H264_FEC_REPAIR == pt) {
+      RtpPacket::PAYLOAD_TYPE::H264_FEC_REPAIR == pt ||
+      RtpPacket::PAYLOAD_TYPE::AV1 == pt) {
     return pt;
   } else {
     return 0;