Finish AV1 rtp packetizer

src/media/video/encode/aom/aom_av1_encoder.cpp

@@ -289,9 +289,10 @@ int AomAv1Encoder::Init() {
   return 0;
 }
 
-int AomAv1Encoder::Encode(
+int AomAv1Encoder::Encode(const uint8_t *pData, int nSize,
-    const uint8_t *pData, int nSize,
+                          std::function<int(char *encoded_packets, size_t size,
-    std::function<int(char *encoded_packets, size_t size)> on_encoded_image) {
+                                            VideoFrameType frame_type)>
+                              on_encoded_image) {
   if (SAVE_NV12_STREAM) {
     fwrite(pData, 1, nSize, file_nv12_);
   }
@@ -311,10 +312,13 @@ int AomAv1Encoder::Encode(
   // NV12ToYUV420PFFmpeg((unsigned char *)pData, frame_width_, frame_height_,
   //                     (unsigned char *)yuv420p_buffer);
 
+  VideoFrameType frame_type;
   if (0 == seq_++ % 300) {
     force_i_frame_flags_ = AOM_EFLAG_FORCE_KF;
+    frame_type = VideoFrameType::kVideoFrameKey;
   } else {
     force_i_frame_flags_ = 0;
+    frame_type = VideoFrameType::kVideoFrameDelta;
   }
 
   // Encode a frame. The presentation timestamp `pts` should not use real
@@ -341,7 +345,8 @@ int AomAv1Encoder::Encode(
       LOG_INFO("Encoded frame qp = {}", qp);
 
       if (on_encoded_image) {
-        on_encoded_image((char *)encoded_frame_, encoded_frame_size_);
+        on_encoded_image((char *)encoded_frame_, encoded_frame_size_,
+                         frame_type);
         if (SAVE_H264_STREAM) {
           fwrite(encoded_frame_, 1, encoded_frame_size_, file_ivf_);
         }

src/media/video/encode/aom/aom_av1_encoder.h

@@ -36,9 +36,10 @@ class AomAv1Encoder : public VideoEncoder {
 
  public:
   int Init();
-  int Encode(
+  int Encode(const uint8_t* pData, int nSize,
-      const uint8_t* pData, int nSize,
+             std::function<int(char* encoded_packets, size_t size,
-      std::function<int(char* encoded_packets, size_t size)> on_encoded_image);
+                               VideoFrameType frame_type)>
+                 on_encoded_image);
 
   virtual int OnEncodedImage(char* encoded_packets, size_t size);
 

src/media/video/encode/ffmpeg/ffmpeg_video_encoder.cpp

@@ -149,7 +149,9 @@ int FFmpegVideoEncoder::Init() {
 
 int FFmpegVideoEncoder::Encode(
     const uint8_t *pData, int nSize,
-    std::function<int(char *encoded_packets, size_t size)> on_encoded_image) {
+    std::function<int(char *encoded_packets, size_t size,
+                      VideoFrameType frame_type)>
+        on_encoded_image) {
   if (!codec_ctx_) {
     LOG_ERROR("Invalid codec context");
     return -1;
@@ -181,6 +183,13 @@ int FFmpegVideoEncoder::Encode(
   int ret = avcodec_send_frame(codec_ctx_, frame_);
 
   // frame_->pict_type = AV_PICTURE_TYPE_I;
+  VideoFrameType frame_type;
+  if (0 == seq_++ % 300) {
+    frame_type = VideoFrameType::kVideoFrameKey;
+  } else {
+    frame_type = VideoFrameType::kVideoFrameDelta;
+  }
+
   while (ret >= 0) {
     ret = avcodec_receive_packet(codec_ctx_, packet_);
     if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
@@ -198,7 +207,7 @@ int FFmpegVideoEncoder::Encode(
     }
 
     if (on_encoded_image) {
-      on_encoded_image((char *)packet_->data, packet_->size);
+      on_encoded_image((char *)packet_->data, packet_->size, frame_type);
       if (SAVE_H264_STREAM) {
         fwrite(packet_->data, 1, packet_->size, file_h264_);
       }

src/media/video/encode/ffmpeg/ffmpeg_video_encoder.h

@@ -36,9 +36,10 @@ class FFmpegVideoEncoder : public VideoEncoder {
   virtual ~FFmpegVideoEncoder();
 
   int Init();
-  int Encode(
+  int Encode(const uint8_t* pData, int nSize,
-      const uint8_t* pData, int nSize,
+             std::function<int(char* encoded_packets, size_t size,
-      std::function<int(char* encoded_packets, size_t size)> on_encoded_image);
+                               VideoFrameType frame_type)>
+                 on_encoded_image);
 
   virtual int OnEncodedImage(char* encoded_packets, size_t size);
 

src/media/video/encode/nvcodec/nvidia_video_encoder.cpp

@@ -78,14 +78,20 @@ int NvidiaVideoEncoder::Init() {
 
 int NvidiaVideoEncoder::Encode(
     const uint8_t *pData, int nSize,
-    std::function<int(char *encoded_packets, size_t size)> on_encoded_image) {
+    std::function<int(char *encoded_packets, size_t size,
+                      VideoFrameType frame_type)>
+        on_encoded_image) {
   if (!encoder_) {
     LOG_ERROR("Invalid encoder");
     return -1;
   }
 
+  VideoFrameType frame_type;
   if (0 == seq_++ % 300) {
     ForceIdr();
+    frame_type = VideoFrameType::kVideoFrameKey;
+  } else {
+    frame_type = VideoFrameType::kVideoFrameDelta;
   }
 
 #ifdef SHOW_SUBMODULE_TIME_COST
@@ -110,7 +116,7 @@ int NvidiaVideoEncoder::Encode(
 
   for (const auto &packet : encoded_packets_) {
     if (on_encoded_image) {
-      on_encoded_image((char *)packet.data(), packet.size());
+      on_encoded_image((char *)packet.data(), packet.size(), frame_type);
       if (SAVE_ENCODER_STREAM) {
         fwrite(packet.data(), 1, packet.size(), file_);
       }

src/media/video/encode/nvcodec/nvidia_video_encoder.h

@@ -12,9 +12,10 @@ class NvidiaVideoEncoder : public VideoEncoder {
   virtual ~NvidiaVideoEncoder();
 
   int Init();
-  int Encode(
+  int Encode(const uint8_t* pData, int nSize,
-      const uint8_t* pData, int nSize,
+             std::function<int(char* encoded_packets, size_t size,
-      std::function<int(char* encoded_packets, size_t size)> on_encoded_image);
+                               VideoFrameType frame_type)>
+                 on_encoded_image);
 
   virtual int OnEncodedImage(char* encoded_packets, size_t size);
 

src/media/video/encode/openh264/openh264_encoder.cpp

@@ -155,7 +155,9 @@ int OpenH264Encoder::Init() {
 }
 int OpenH264Encoder::Encode(
     const uint8_t *pData, int nSize,
-    std::function<int(char *encoded_packets, size_t size)> on_encoded_image) {
+    std::function<int(char *encoded_packets, size_t size,
+                      VideoFrameType frame_type)>
+        on_encoded_image) {
   if (!openh264_encoder_) {
     LOG_ERROR("Invalid openh264 encoder");
     return -1;
@@ -165,8 +167,12 @@ int OpenH264Encoder::Encode(
     fwrite(yuv420p_buffer, 1, nSize, file_nv12_);
   }
 
+  VideoFrameType frame_type;
   if (0 == seq_++ % 300) {
     ForceIdr();
+    frame_type = VideoFrameType::kVideoFrameKey;
+  } else {
+    frame_type = VideoFrameType::kVideoFrameDelta;
   }
 
   NV12ToYUV420PFFmpeg((unsigned char *)pData, frame_width_, frame_height_,
@@ -222,7 +228,7 @@ int OpenH264Encoder::Encode(
   encoded_frame_size_ = encoded_frame_size;
 
   if (on_encoded_image) {
-    on_encoded_image((char *)encoded_frame_, encoded_frame_size_);
+    on_encoded_image((char *)encoded_frame_, encoded_frame_size_, frame_type);
     if (SAVE_H264_STREAM) {
       fwrite(encoded_frame_, 1, encoded_frame_size_, file_h264_);
     }
@@ -268,7 +274,7 @@ int OpenH264Encoder::Encode(
     encoded_frame_size_ = encoded_frame_size;
 
     if (on_encoded_image) {
-      on_encoded_image((char *)encoded_frame_, encoded_frame_size_);
+      on_encoded_image((char *)encoded_frame_, frame_type);
       if (SAVE_H264_STREAM) {
         fwrite(encoded_frame_, 1, encoded_frame_size_, file_h264_);
       }

src/media/video/encode/openh264/openh264_encoder.h

@@ -23,9 +23,10 @@ class OpenH264Encoder : public VideoEncoder {
   virtual ~OpenH264Encoder();
 
   int Init();
-  int Encode(
+  int Encode(const uint8_t* pData, int nSize,
-      const uint8_t* pData, int nSize,
+             std::function<int(char* encoded_packets, size_t size,
-      std::function<int(char* encoded_packets, size_t size)> on_encoded_image);
+                               VideoFrameType frame_type)>
+                 on_encoded_image);
 
   virtual int OnEncodedImage(char* encoded_packets, size_t size);
 

src/media/video/encode/video_encoder.h

@@ -4,10 +4,18 @@
 #include <functional>
 
 class VideoEncoder {
+ public:
+  enum VideoFrameType {
+    kEmptyFrame = 0,
+    kVideoFrameKey = 3,
+    kVideoFrameDelta = 4,
+  };
+
  public:
   virtual int Init() = 0;
   virtual int Encode(const uint8_t* pData, int nSize,
-                     std::function<int(char* encoded_packets, size_t size)>
+                     std::function<int(char* encoded_packets, size_t size,
+                                       VideoFrameType frame_type)>
                          on_encoded_image) = 0;
   virtual int OnEncodedImage(char* encoded_packets, size_t size) = 0;
   virtual void ForceIdr() = 0;

src/pc/peer_connection.cpp

@@ -527,11 +527,16 @@ int PeerConnection::SendVideoData(const char *data, size_t size) {
   }
 
   int ret = video_encoder_->Encode(
-      (uint8_t *)data, size, [this](char *encoded_frame, size_t size) -> int {
+      (uint8_t *)data, size,
+      [this](char *encoded_frame, size_t size,
+             VideoEncoder::VideoFrameType frame_type) -> int {
         for (auto &ice_trans : ice_transmission_list_) {
           // LOG_ERROR("H264 frame size: [{}]", size);
-          ice_trans.second->SendData(IceTransmission::DATA_TYPE::VIDEO,
+          // ice_trans.second->SendData(IceTransmission::DATA_TYPE::VIDEO,
-                                     encoded_frame, size);
+          //                            encoded_frame, size);
+          ice_trans.second->SendVideoData(
+              static_cast<IceTransmission::VideoFrameType>(frame_type),
+              encoded_frame, size);
         }
         return 0;
       });
@@ -550,8 +555,8 @@ int PeerConnection::SendAudioData(const char *data, size_t size) {
       [this](char *encoded_audio_buffer, size_t size) -> int {
         for (auto &ice_trans : ice_transmission_list_) {
           // LOG_ERROR("opus frame size: [{}]", size);
-          // ice_trans.second->SendData(IceTransmission::DATA_TYPE::AUDIO,
+          ice_trans.second->SendData(IceTransmission::DATA_TYPE::AUDIO,
-          //                            encoded_audio_buffer, size);
+                                     encoded_audio_buffer, size);
         }
         return 0;
       });

src/rtp/rtp_codec.cpp

@@ -10,6 +10,8 @@
 #define FU_A 28
 #define FU_B 29
 
+constexpr int kObuTypeSequenceHeader = 1;
+
 RtpCodec ::RtpCodec(RtpPacket::PAYLOAD_TYPE payload_type)
     : version_(RTP_VERSION),
       has_padding_(false),
@@ -33,10 +35,6 @@ RtpCodec ::~RtpCodec() {
 
 void RtpCodec::Encode(uint8_t* buffer, size_t size,
                       std::vector<RtpPacket>& packets) {
-  // if (!rtp_packet_) {
-  //   rtp_packet_ = new RtpPacket();
-  // }
-
   if (RtpPacket::PAYLOAD_TYPE::H264 == payload_type_) {
     if (fec_enable_ && IsKeyFrame((const uint8_t*)buffer, size)) {
       uint8_t** fec_packets = fec_encoder_.Encode((const char*)buffer, size);
@@ -251,78 +249,48 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size,
 
         rtp_packet.EncodeAv1(obus[i].payload_, obus[i].payload_size_);
         packets.emplace_back(rtp_packet);
+      } else {
+        size_t last_packet_size = obus[i].payload_size_ % MAX_NALU_LEN;
+        size_t packet_num =
+            obus[i].payload_size_ / MAX_NALU_LEN + (last_packet_size ? 1 : 0);
+        timestamp_ = std::chrono::high_resolution_clock::now()
+                         .time_since_epoch()
+                         .count();
+        for (size_t index = 0; index < packet_num; index++) {
+          RtpPacket rtp_packet;
+          rtp_packet.SetVerion(version_);
+          rtp_packet.SetHasPadding(has_padding_);
+          rtp_packet.SetHasExtension(has_extension_);
+          rtp_packet.SetMarker(index == packet_num - 1 ? 1 : 0);
+          rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_));
+          rtp_packet.SetSequenceNumber(sequence_number_++);
+          rtp_packet.SetTimestamp(timestamp_);
+          rtp_packet.SetSsrc(ssrc_);
+          if (!csrcs_.empty()) {
+            rtp_packet.SetCsrcs(csrcs_);
+          }
+          if (has_extension_) {
+            rtp_packet.SetExtensionProfile(extension_profile_);
+            rtp_packet.SetExtensionData(extension_data_, extension_len_);
+          }
+
+          int z = index != 0 ? 1 : 0;
+          int y = index != packet_num - 1 ? 1 : 0;
+          int w = 1;
+          int n = 0;
+          rtp_packet.SetAv1AggrHeader(z, y, w, n);
+
+          if (index == packet_num - 1 && last_packet_size > 0) {
+            rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN,
+                                 last_packet_size);
+          } else {
+            rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN,
+                                 MAX_NALU_LEN);
+          }
+          packets.emplace_back(rtp_packet);
+        }
       }
     }
-
-    // if (size <= MAX_NALU_LEN) {
-    //   RtpPacket rtp_packet;
-    //   rtp_packet.SetVerion(version_);
-    //   rtp_packet.SetHasPadding(has_padding_);
-    //   rtp_packet.SetHasExtension(has_extension_);
-    //   rtp_packet.SetMarker(1);
-    //   rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_));
-    //   rtp_packet.SetSequenceNumber(sequence_number_++);
-
-    //   timestamp_ =
-    //       std::chrono::high_resolution_clock::now().time_since_epoch().count();
-    //   rtp_packet.SetTimestamp(timestamp_);
-    //   rtp_packet.SetSsrc(ssrc_);
-
-    //   if (!csrcs_.empty()) {
-    //     rtp_packet.SetCsrcs(csrcs_);
-    //   }
-
-    //   if (has_extension_) {
-    //     rtp_packet.SetExtensionProfile(extension_profile_);
-    //     rtp_packet.SetExtensionData(extension_data_, extension_len_);
-    //   }
-
-    //   RtpPacket::AV1_AGGR_HEADER av1_aggr_header;
-    //   av1_aggr_header.z = av1_aggr_header.z;
-    //   av1_aggr_header.y = av1_aggr_header.y;
-    //   av1_aggr_header.w = av1_aggr_header.w;
-    //   av1_aggr_header.n = av1_aggr_header.n;
-
-    //   rtp_packet.SetAv1AggrHeader(av1_aggr_header);
-
-    //   rtp_packet.EncodeAv1(buffer, size);
-    //   packets.emplace_back(rtp_packet);
-
-    // } else {
-    //   size_t last_packet_size = size % MAX_NALU_LEN;
-    //   size_t packet_num = size / MAX_NALU_LEN + (last_packet_size ? 1 : 0);
-    //   timestamp_ =
-    //       std::chrono::high_resolution_clock::now().time_since_epoch().count();
-
-    //   for (size_t index = 0; index < packet_num; index++) {
-    //     RtpPacket rtp_packet;
-    //     rtp_packet.SetVerion(version_);
-    //     rtp_packet.SetHasPadding(has_padding_);
-    //     rtp_packet.SetHasExtension(has_extension_);
-    //     rtp_packet.SetMarker(index == packet_num - 1 ? 1 : 0);
-    //     rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_));
-    //     rtp_packet.SetSequenceNumber(sequence_number_++);
-    //     rtp_packet.SetTimestamp(timestamp_);
-    //     rtp_packet.SetSsrc(ssrc_);
-
-    //     if (!csrcs_.empty()) {
-    //       rtp_packet.SetCsrcs(csrcs_);
-    //     }
-
-    //     if (has_extension_) {
-    //       rtp_packet.SetExtensionProfile(extension_profile_);
-    //       rtp_packet.SetExtensionData(extension_data_, extension_len_);
-    //     }
-
-    //     if (index == packet_num - 1 && last_packet_size > 0) {
-    //       rtp_packet.EncodeAv1(buffer + index * MAX_NALU_LEN,
-    //       last_packet_size);
-    //     } else {
-    //       rtp_packet.EncodeAv1(buffer + index * MAX_NALU_LEN, MAX_NALU_LEN);
-    //     }
-    //     packets.emplace_back(rtp_packet);
-    //   }
-    // }
   } else if (RtpPacket::PAYLOAD_TYPE::OPUS == payload_type_) {
     RtpPacket rtp_packet;
     rtp_packet.SetVerion(version_);
@@ -358,6 +326,270 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size,
   }
 }
 
+void RtpCodec::Encode(VideoFrameType frame_type, uint8_t* buffer, size_t size,
+                      std::vector<RtpPacket>& packets) {
+  if (RtpPacket::PAYLOAD_TYPE::H264 == payload_type_) {
+    if (fec_enable_ && IsKeyFrame((const uint8_t*)buffer, size)) {
+      uint8_t** fec_packets = fec_encoder_.Encode((const char*)buffer, size);
+      if (nullptr == fec_packets) {
+        LOG_ERROR("Invalid fec_packets");
+        return;
+      }
+      unsigned int num_of_total_packets = 0;
+      unsigned int num_of_source_packets = 0;
+      unsigned int last_packet_size = 0;
+      fec_encoder_.GetFecPacketsParams(size, num_of_total_packets,
+                                       num_of_source_packets, last_packet_size);
+
+      timestamp_ =
+          std::chrono::high_resolution_clock::now().time_since_epoch().count();
+
+      for (unsigned int index = 0; index < num_of_total_packets; index++) {
+        RtpPacket rtp_packet;
+        if (index < num_of_source_packets) {
+          rtp_packet.SetVerion(version_);
+          rtp_packet.SetHasPadding(has_padding_);
+          rtp_packet.SetHasExtension(has_extension_);
+          rtp_packet.SetMarker(index == num_of_source_packets - 1 ? 1 : 0);
+          rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE::H264_FEC_SOURCE);
+          rtp_packet.SetSequenceNumber(sequence_number_++);
+          rtp_packet.SetTimestamp(timestamp_);
+          rtp_packet.SetSsrc(ssrc_);
+
+          if (!csrcs_.empty()) {
+            rtp_packet.SetCsrcs(csrcs_);
+          }
+
+          if (has_extension_) {
+            rtp_packet.SetExtensionProfile(extension_profile_);
+            rtp_packet.SetExtensionData(extension_data_, extension_len_);
+          }
+
+          RtpPacket::FU_INDICATOR fu_indicator;
+          fu_indicator.forbidden_bit = 0;
+          fu_indicator.nal_reference_idc = 0;
+          fu_indicator.nal_unit_type = FU_A;
+
+          RtpPacket::FU_HEADER fu_header;
+          fu_header.start = index == 0 ? 1 : 0;
+          fu_header.end = index == num_of_source_packets - 1 ? 1 : 0;
+          fu_header.remain_bit = 0;
+          fu_header.nal_unit_type = FU_A;
+
+          rtp_packet.SetFuIndicator(fu_indicator);
+          rtp_packet.SetFuHeader(fu_header);
+
+          if (index == num_of_source_packets - 1) {
+            if (last_packet_size > 0) {
+              rtp_packet.EncodeH264FecSource(fec_packets[index],
+                                             last_packet_size, index,
+                                             num_of_source_packets);
+            } else {
+              rtp_packet.EncodeH264FecSource(fec_packets[index], MAX_NALU_LEN,
+                                             index, num_of_source_packets);
+            }
+          } else {
+            rtp_packet.EncodeH264FecSource(fec_packets[index], MAX_NALU_LEN,
+                                           index, num_of_source_packets);
+          }
+
+        } else if (index >= num_of_source_packets &&
+                   index < num_of_total_packets) {
+          rtp_packet.SetVerion(version_);
+          rtp_packet.SetHasPadding(has_padding_);
+          rtp_packet.SetHasExtension(has_extension_);
+          rtp_packet.SetMarker(index == num_of_total_packets - 1 ? 1 : 0);
+          rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE::H264_FEC_REPAIR);
+          rtp_packet.SetSequenceNumber(sequence_number_++);
+          rtp_packet.SetTimestamp(timestamp_);
+          rtp_packet.SetSsrc(ssrc_);
+
+          if (!csrcs_.empty()) {
+            rtp_packet.SetCsrcs(csrcs_);
+          }
+
+          if (has_extension_) {
+            rtp_packet.SetExtensionProfile(extension_profile_);
+            rtp_packet.SetExtensionData(extension_data_, extension_len_);
+          }
+          rtp_packet.EncodeH264FecRepair(fec_packets[index], MAX_NALU_LEN,
+                                         index, num_of_source_packets);
+        }
+        packets.emplace_back(rtp_packet);
+
+        // if (index < num_of_source_packets) {
+        //   rtp_packet.EncodeH264Fua(fec_packets[index], MAX_NALU_LEN);
+        //   packets.emplace_back(rtp_packet);
+        // }
+      }
+
+      fec_encoder_.ReleaseFecPackets(fec_packets, size);
+      return;
+    }
+
+    if (size <= MAX_NALU_LEN) {
+      RtpPacket rtp_packet;
+      rtp_packet.SetVerion(version_);
+      rtp_packet.SetHasPadding(has_padding_);
+      rtp_packet.SetHasExtension(has_extension_);
+      rtp_packet.SetMarker(1);
+      rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_));
+      rtp_packet.SetSequenceNumber(sequence_number_++);
+
+      timestamp_ =
+          std::chrono::high_resolution_clock::now().time_since_epoch().count();
+      rtp_packet.SetTimestamp(timestamp_);
+      rtp_packet.SetSsrc(ssrc_);
+
+      if (!csrcs_.empty()) {
+        rtp_packet.SetCsrcs(csrcs_);
+      }
+
+      if (has_extension_) {
+        rtp_packet.SetExtensionProfile(extension_profile_);
+        rtp_packet.SetExtensionData(extension_data_, extension_len_);
+      }
+
+      RtpPacket::FU_INDICATOR fu_indicator;
+      fu_indicator.forbidden_bit = 0;
+      fu_indicator.nal_reference_idc = 1;
+      fu_indicator.nal_unit_type = NALU;
+      rtp_packet.SetFuIndicator(fu_indicator);
+
+      rtp_packet.EncodeH264Nalu(buffer, size);
+      packets.emplace_back(rtp_packet);
+
+    } else {
+      size_t last_packet_size = size % MAX_NALU_LEN;
+      size_t packet_num = size / MAX_NALU_LEN + (last_packet_size ? 1 : 0);
+      timestamp_ =
+          std::chrono::high_resolution_clock::now().time_since_epoch().count();
+
+      for (size_t index = 0; index < packet_num; index++) {
+        RtpPacket rtp_packet;
+        rtp_packet.SetVerion(version_);
+        rtp_packet.SetHasPadding(has_padding_);
+        rtp_packet.SetHasExtension(has_extension_);
+        rtp_packet.SetMarker(index == packet_num - 1 ? 1 : 0);
+        rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_));
+        rtp_packet.SetSequenceNumber(sequence_number_++);
+        rtp_packet.SetTimestamp(timestamp_);
+        rtp_packet.SetSsrc(ssrc_);
+
+        if (!csrcs_.empty()) {
+          rtp_packet.SetCsrcs(csrcs_);
+        }
+
+        if (has_extension_) {
+          rtp_packet.SetExtensionProfile(extension_profile_);
+          rtp_packet.SetExtensionData(extension_data_, extension_len_);
+        }
+
+        RtpPacket::FU_INDICATOR fu_indicator;
+        fu_indicator.forbidden_bit = 0;
+        fu_indicator.nal_reference_idc = 0;
+        fu_indicator.nal_unit_type = FU_A;
+
+        RtpPacket::FU_HEADER fu_header;
+        fu_header.start = index == 0 ? 1 : 0;
+        fu_header.end = index == packet_num - 1 ? 1 : 0;
+        fu_header.remain_bit = 0;
+        fu_header.nal_unit_type = FU_A;
+
+        rtp_packet.SetFuIndicator(fu_indicator);
+        rtp_packet.SetFuHeader(fu_header);
+
+        if (index == packet_num - 1 && last_packet_size > 0) {
+          rtp_packet.EncodeH264Fua(buffer + index * MAX_NALU_LEN,
+                                   last_packet_size);
+        } else {
+          rtp_packet.EncodeH264Fua(buffer + index * MAX_NALU_LEN, MAX_NALU_LEN);
+        }
+        packets.emplace_back(rtp_packet);
+      }
+    }
+  } else if (RtpPacket::PAYLOAD_TYPE::AV1 == payload_type_) {
+    std::vector<Obu> obus = ParseObus(buffer, size);
+    LOG_ERROR("Total size = [{}]", size);
+    for (int i = 0; i < obus.size(); i++) {
+      LOG_ERROR("[{}] Obu size = [{}], Obu type [{}]", i, obus[i].size_,
+                ObuTypeToString((OBU_TYPE)ObuType(obus[i].header_)));
+      if (obus[i].size_ <= MAX_NALU_LEN) {
+        RtpPacket rtp_packet;
+        rtp_packet.SetVerion(version_);
+        rtp_packet.SetHasPadding(has_padding_);
+        rtp_packet.SetHasExtension(has_extension_);
+        rtp_packet.SetMarker(1);
+        rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_));
+        rtp_packet.SetSequenceNumber(sequence_number_++);
+
+        timestamp_ = std::chrono::high_resolution_clock::now()
+                         .time_since_epoch()
+                         .count();
+        rtp_packet.SetTimestamp(timestamp_);
+        rtp_packet.SetSsrc(ssrc_);
+
+        if (!csrcs_.empty()) {
+          rtp_packet.SetCsrcs(csrcs_);
+        }
+
+        if (has_extension_) {
+          rtp_packet.SetExtensionProfile(extension_profile_);
+          rtp_packet.SetExtensionData(extension_data_, extension_len_);
+        }
+
+        rtp_packet.SetAv1AggrHeader(0, 0, 1, 0);
+
+        rtp_packet.EncodeAv1(obus[i].payload_, obus[i].payload_size_);
+        packets.emplace_back(rtp_packet);
+      } else {
+        size_t last_packet_size = obus[i].payload_size_ % MAX_NALU_LEN;
+        size_t packet_num =
+            obus[i].payload_size_ / MAX_NALU_LEN + (last_packet_size ? 1 : 0);
+        timestamp_ = std::chrono::high_resolution_clock::now()
+                         .time_since_epoch()
+                         .count();
+        for (size_t index = 0; index < packet_num; index++) {
+          RtpPacket rtp_packet;
+          rtp_packet.SetVerion(version_);
+          rtp_packet.SetHasPadding(has_padding_);
+          rtp_packet.SetHasExtension(has_extension_);
+          rtp_packet.SetMarker(index == packet_num - 1 ? 1 : 0);
+          rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_));
+          rtp_packet.SetSequenceNumber(sequence_number_++);
+          rtp_packet.SetTimestamp(timestamp_);
+          rtp_packet.SetSsrc(ssrc_);
+          if (!csrcs_.empty()) {
+            rtp_packet.SetCsrcs(csrcs_);
+          }
+          if (has_extension_) {
+            rtp_packet.SetExtensionProfile(extension_profile_);
+            rtp_packet.SetExtensionData(extension_data_, extension_len_);
+          }
+
+          int z = index != 0 ? 1 : 0;
+          int y = index != packet_num - 1 ? 1 : 0;
+          int w = 1;
+          int n = (frame_type == VideoFrameType::kVideoFrameKey) &&
+                          (ObuType(obus[i].header_) == kObuTypeSequenceHeader)
+                      ? 1
+                      : 0;
+          rtp_packet.SetAv1AggrHeader(z, y, w, n);
+
+          if (index == packet_num - 1 && last_packet_size > 0) {
+            rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN,
+                                 last_packet_size);
+          } else {
+            rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN,
+                                 MAX_NALU_LEN);
+          }
+          packets.emplace_back(rtp_packet);
+        }
+      }
+    }
+  }
+}
+
 size_t RtpCodec::Decode(RtpPacket& packet, uint8_t* payload) {
   // if ((packet.Buffer()[13] >> 6) & 0x01) {
   //   LOG_ERROR("End bit!!!!!!!!!!!!!!!");

src/rtp/rtp_codec.h

@@ -9,12 +9,21 @@
 #include "rtp_packet.h"
 
 class RtpCodec {
+ public:
+  enum VideoFrameType {
+    kEmptyFrame = 0,
+    kVideoFrameKey = 3,
+    kVideoFrameDelta = 4,
+  };
+
  public:
   RtpCodec(RtpPacket::PAYLOAD_TYPE payload_type);
   ~RtpCodec();
 
  public:
   void Encode(uint8_t* buffer, size_t size, std::vector<RtpPacket>& packets);
+  void Encode(VideoFrameType frame_type, uint8_t* buffer, size_t size,
+              std::vector<RtpPacket>& packets);
   size_t Decode(RtpPacket& packet, uint8_t* payload);
 
   //  protected:

src/transmission/ice_transmission.cpp

@@ -312,6 +312,23 @@ int IceTransmission::SendData(DATA_TYPE type, const char *data, size_t size) {
   return 0;
 }
 
+int IceTransmission::SendVideoData(VideoFrameType frame_type, const char *data,
+                                   size_t size) {
+  if (NiceComponentState::NICE_COMPONENT_STATE_READY == state_) {
+    std::vector<RtpPacket> packets;
+
+    if (rtp_video_sender_) {
+      if (video_rtp_codec_) {
+        video_rtp_codec_->Encode(
+            static_cast<RtpCodec::VideoFrameType>(frame_type), (uint8_t *)data,
+            size, packets);
+      }
+      rtp_video_sender_->Enqueue(packets);
+    }
+  }
+  return 0;
+}
+
 uint8_t IceTransmission::CheckIsRtcpPacket(const char *buffer, size_t size) {
   if (size < 4) {
     return 0;

src/transmission/ice_transmission.h

@@ -25,6 +25,11 @@
 class IceTransmission {
  public:
   typedef enum { VIDEO = 96, AUDIO = 97, DATA = 127 } DATA_TYPE;
+  enum VideoFrameType {
+    kEmptyFrame = 0,
+    kVideoFrameKey = 3,
+    kVideoFrameDelta = 4,
+  };
 
  public:
   IceTransmission(bool offer_peer, std::string &transmission_id,
@@ -66,6 +71,12 @@ class IceTransmission {
 
   int SendData(DATA_TYPE type, const char *data, size_t size);
 
+  int SendVideoData(VideoFrameType frame_type, const char *data, size_t size);
+
+  int SendAudioData(const char *data, size_t size);
+
+  int SendUserData(const char *data, size_t size);
+
  public:
   int GatherCandidates();