Combine Fu-A subframes into complete h264 frame

src/rtp/rtp_packet.cpp

@@ -4,6 +4,17 @@
 
 #include "log.h"
 
+inline void RtpPacket::TryToDecodeH264RtpPacket(uint8_t *buffer) {
+  if (PAYLOAD_TYPE::H264 == NAL_UNIT_TYPE(buffer[1] & 0x7f)) {
+    nal_unit_type_ = NAL_UNIT_TYPE(buffer[12] & 0x1F);
+    if (NAL_UNIT_TYPE::NALU == nal_unit_type_) {
+      DecodeH264Nalu();
+    } else if (NAL_UNIT_TYPE::FU_A == nal_unit_type_) {
+      DecodeH264Fua();
+    }
+  }
+}
+
 RtpPacket::RtpPacket() : buffer_(new uint8_t[DEFAULT_MTU]), size_(DEFAULT_MTU) {
   memset(buffer_, 0, DEFAULT_MTU);
 }
@@ -13,6 +24,8 @@ RtpPacket::RtpPacket(const uint8_t *buffer, size_t size) {
     buffer_ = new uint8_t[size];
     memcpy(buffer_, buffer, size);
     size_ = size;
+
+    TryToDecodeH264RtpPacket(buffer_);
   }
 }
 
@@ -21,6 +34,8 @@ RtpPacket::RtpPacket(const RtpPacket &rtp_packet) {
     buffer_ = new uint8_t[rtp_packet.size_];
     memcpy(buffer_, rtp_packet.buffer_, rtp_packet.size_);
     size_ = rtp_packet.size_;
+
+    TryToDecodeH264RtpPacket(buffer_);
   }
 }
 
@@ -29,6 +44,8 @@ RtpPacket::RtpPacket(RtpPacket &&rtp_packet)
       size_(rtp_packet.size_) {
   rtp_packet.buffer_ = nullptr;
   rtp_packet.size_ = 0;
+
+  TryToDecodeH264RtpPacket(buffer_);
 }
 
 RtpPacket &RtpPacket::operator=(const RtpPacket &rtp_packet) {
@@ -40,6 +57,8 @@ RtpPacket &RtpPacket::operator=(const RtpPacket &rtp_packet) {
     buffer_ = new uint8_t[rtp_packet.size_];
     memcpy(buffer_, rtp_packet.buffer_, rtp_packet.size_);
     size_ = rtp_packet.size_;
+
+    TryToDecodeH264RtpPacket(buffer_);
   }
   return *this;
 }
@@ -50,6 +69,8 @@ RtpPacket &RtpPacket::operator=(RtpPacket &&rtp_packet) {
     rtp_packet.buffer_ = nullptr;
     size_ = rtp_packet.size_;
     rtp_packet.size_ = 0;
+
+    TryToDecodeH264RtpPacket(buffer_);
   }
   return *this;
 }
@@ -292,7 +313,9 @@ size_t RtpPacket::DecodeH264Nalu(uint8_t *payload) {
 
   payload_size_ = size_ - (13 + payload_offset);
   payload_ = buffer_ + 13 + payload_offset;
-  memcpy(payload, payload_, payload_size_);
+  if (payload) {
+    memcpy(payload, payload_, payload_size_);
+  }
   return payload_size_;
 }
 
@@ -341,6 +364,8 @@ size_t RtpPacket::DecodeH264Fua(uint8_t *payload) {
 
   payload_size_ = size_ - (14 + payload_offset);
   payload_ = buffer_ + 14 + payload_offset;
-  memcpy(payload, payload_, payload_size_);
+  if (payload) {
+    memcpy(payload, payload_, payload_size_);
+  }
   return payload_size_;
 }

src/rtp/rtp_packet.h

@@ -64,6 +64,7 @@
 #define DEFAULT_MTU 1500
 #define MAX_NALU_LEN 1400
 typedef enum { H264 = 96, OPUS = 97, USER_DEFINED = 127 } PAYLOAD_TYPE;
+typedef enum { UNKNOWN = 0, NALU = 1, FU_A = 28, FU_B = 29 } NAL_UNIT_TYPE;
 
 class RtpPacket {
  public:
@@ -133,8 +134,8 @@ class RtpPacket {
   const uint8_t *EncodeH264Nalu(uint8_t *payload, size_t payload_size);
   const uint8_t *EncodeH264Fua(uint8_t *payload, size_t payload_size);
   size_t Decode(uint8_t *payload);
-  size_t DecodeH264Nalu(uint8_t *payload);
-  size_t DecodeH264Fua(uint8_t *payload);
+  size_t DecodeH264Nalu(uint8_t *payload = nullptr);
+  size_t DecodeH264Fua(uint8_t *payload = nullptr);
 
  public:
   // Get Header
@@ -154,10 +155,18 @@ class RtpPacket {
   const uint8_t *Payload() { return payload_; };
   const size_t PayloadSize() { return payload_size_; }
 
- public:
+  // Entire RTP buffer
   const uint8_t *Buffer() { return buffer_; }
   const size_t Size() { return size_; }
 
+  // NAL
+  const NAL_UNIT_TYPE NalUnitType() { return nal_unit_type_; }
+  const bool FuAStart() { return fu_header_.start; }
+  const bool FuAEnd() { return fu_header_.end; }
+
+ private:
+  inline void TryToDecodeH264RtpPacket(uint8_t *buffer);
+
  private:
   // Header
   uint32_t version_ = 0;
@@ -184,6 +193,9 @@ class RtpPacket {
   // Entire RTP buffer
   uint8_t *buffer_ = nullptr;
   size_t size_ = 0;
+
+  // NAL
+  NAL_UNIT_TYPE nal_unit_type_ = NAL_UNIT_TYPE::UNKNOWN;
 };
 
 #endif

src/rtp/rtp_video_receiver.cpp

@@ -0,0 +1,76 @@
+#include "rtp_video_receiver.h"
+
+#include "log.h"
+
+#define NV12_BUFFER_SIZE (1280 * 720 * 3 / 2)
+
+RtpVideoReceiver::RtpVideoReceiver() {}
+
+RtpVideoReceiver::~RtpVideoReceiver() {}
+
+void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
+  if (NAL_UNIT_TYPE::NALU == rtp_packet.NalUnitType()) {
+    // compelete_video_frame_queue_.push(
+    //     VideoFrame(rtp_packet.Payload(), rtp_packet.Size()));
+    if (on_receive_complete_frame_) {
+      on_receive_complete_frame_(
+          VideoFrame(rtp_packet.Payload(), rtp_packet.Size()));
+    }
+  } else if (NAL_UNIT_TYPE::FU_A == rtp_packet.NalUnitType()) {
+    incomplete_frame_list_[rtp_packet.SequenceNumber()] = rtp_packet;
+    bool complete = CheckIsFrameCompleted(rtp_packet);
+  }
+}
+
+bool RtpVideoReceiver::CheckIsFrameCompleted(RtpPacket& rtp_packet) {
+  if (rtp_packet.FuAEnd()) {
+    size_t complete_frame_size = 0;
+    uint16_t end_seq = rtp_packet.SequenceNumber();
+    if (incomplete_frame_list_.size() == end_seq) {
+      return true;
+    }
+
+    while (end_seq--) {
+      auto it = incomplete_frame_list_.find(end_seq);
+      complete_frame_size += it->second.PayloadSize();
+      if (it == incomplete_frame_list_.end()) {
+        return false;
+      } else if (!it->second.FuAStart()) {
+        continue;
+      } else if (it->second.FuAStart()) {
+        if (!nv12_data_) {
+          nv12_data_ = new uint8_t[NV12_BUFFER_SIZE];
+        }
+
+        size_t complete_frame_size = 0;
+        for (size_t start = it->first; start <= rtp_packet.SequenceNumber();
+             start++) {
+          memcpy(nv12_data_ + complete_frame_size,
+                 incomplete_frame_list_[start].Payload(),
+                 incomplete_frame_list_[start].PayloadSize());
+
+          complete_frame_size += incomplete_frame_list_[start].PayloadSize();
+          incomplete_frame_list_.erase(start);
+        }
+
+        // compelete_video_frame_queue_.push(
+        //     VideoFrame(nv12_data_, complete_frame_size));
+
+        // LOG_ERROR("Size of compelete_video_frame_queue_ [{}]",
+        //           compelete_video_frame_queue_.size());
+
+        if (on_receive_complete_frame_) {
+          on_receive_complete_frame_(
+              VideoFrame(nv12_data_, complete_frame_size));
+        }
+        return true;
+      } else {
+        LOG_WARN("What happened?")
+        return false;
+      }
+    }
+
+    return true;
+  }
+  return false;
+}

src/rtp/rtp_video_receiver.h

@@ -0,0 +1,37 @@
+#ifndef _RTP_VIDEO_RECEIVER_H_
+#define _RTP_VIDEO_RECEIVER_H_
+
+#include <functional>
+#include <map>
+#include <queue>
+
+#include "frame.h"
+#include "rtp_video_session.h"
+
+class RtpVideoReceiver {
+ public:
+  RtpVideoReceiver();
+  ~RtpVideoReceiver();
+
+ public:
+  void InsertRtpPacket(RtpPacket& rtp_packet);
+
+  void SetOnReceiveCompleteFrame(
+      std::function<void(VideoFrame&)> on_receive_complete_frame) {
+    on_receive_complete_frame_ = on_receive_complete_frame;
+  }
+
+ private:
+  bool CheckIsFrameCompleted(RtpPacket& rtp_packet);
+
+  //  private:
+  //   void OnReceiveFrame(uint8_t* payload) {}
+
+ private:
+  std::map<uint16_t, RtpPacket> incomplete_frame_list_;
+  std::queue<VideoFrame> compelete_video_frame_queue_;
+  uint8_t* nv12_data_ = nullptr;
+  std::function<void(VideoFrame&)> on_receive_complete_frame_ = nullptr;
+};
+
+#endif

src/rtp/rtp_video_session.cpp

@@ -1,4 +1,4 @@
-#include "rtp_session.h"
+#include "rtp_video_session.h"
 
 #include <chrono>
 
@@ -9,14 +9,14 @@
 #define FU_A 28
 #define FU_B 29
 
-RtpSession ::RtpSession(PAYLOAD_TYPE payload_type)
+RtpVideoSession ::RtpVideoSession(PAYLOAD_TYPE payload_type)
     : version_(RTP_VERSION),
       has_padding_(false),
       has_extension_(false),
       payload_type_(payload_type),
       sequence_number_(0) {}
 
-RtpSession ::~RtpSession() {
+RtpVideoSession ::~RtpVideoSession() {
   if (extension_data_) {
     delete extension_data_;
     extension_data_ = nullptr;
@@ -28,8 +28,8 @@ RtpSession ::~RtpSession() {
   // }
 }
 
-void RtpSession::Encode(uint8_t* buffer, size_t size,
-                        std::vector<RtpPacket>& packets) {
+void RtpVideoSession::Encode(uint8_t* buffer, size_t size,
+                             std::vector<RtpPacket>& packets) {
   // if (!rtp_packet_) {
   //   rtp_packet_ = new RtpPacket();
   // }
@@ -118,7 +118,7 @@ void RtpSession::Encode(uint8_t* buffer, size_t size,
   }
 }
 
-size_t RtpSession::Decode(RtpPacket& packet, uint8_t* payload) {
+size_t RtpVideoSession::Decode(RtpPacket& packet, uint8_t* payload) {
   // if ((packet.Buffer()[13] >> 6) & 0x01) {
   //   LOG_ERROR("End bit!!!!!!!!!!!!!!!");
   // }

src/rtp/rtp_video_session.h

@@ -1,5 +1,5 @@
-#ifndef _RTP_SESSION_H_
-#define _RTP_SESSION_H_
+#ifndef _RTP_VIDEO_SESSION_H_
+#define _RTP_VIDEO_SESSION_H_
 
 #include <stdint.h>
 
@@ -7,17 +7,18 @@
 
 #include "rtp_packet.h"
 
-class RtpSession
-
-{
+class RtpVideoSession {
  public:
-  RtpSession(PAYLOAD_TYPE payload_type);
-  ~RtpSession();
+  RtpVideoSession(PAYLOAD_TYPE payload_type);
+  ~RtpVideoSession();
 
  public:
   void Encode(uint8_t* buffer, size_t size, std::vector<RtpPacket>& packets);
   size_t Decode(RtpPacket& packet, uint8_t* payload);
 
+  //  protected:
+  //   void OnReceiveFrame(uint8_t* payload) = 0;
+
  private:
   uint32_t version_ = 0;
   bool has_padding_ = false;