From 182c7dbec62ac95013ab5fe156465b7b4787e4c0 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Sun, 28 Apr 2024 16:13:37 +0800 Subject: [PATCH] Finish AV1 rtp packetizer --- .../video/encode/aom/aom_av1_encoder.cpp | 13 +- src/media/video/encode/aom/aom_av1_encoder.h | 7 +- .../encode/ffmpeg/ffmpeg_video_encoder.cpp | 13 +- .../encode/ffmpeg/ffmpeg_video_encoder.h | 7 +- .../encode/nvcodec/nvidia_video_encoder.cpp | 10 +- .../encode/nvcodec/nvidia_video_encoder.h | 7 +- .../encode/openh264/openh264_encoder.cpp | 12 +- .../video/encode/openh264/openh264_encoder.h | 7 +- src/media/video/encode/video_encoder.h | 10 +- src/pc/peer_connection.cpp | 15 +- src/rtp/rtp_codec.cpp | 380 ++++++++++++++---- src/rtp/rtp_codec.h | 9 + src/transmission/ice_transmission.cpp | 17 + src/transmission/ice_transmission.h | 11 + 14 files changed, 415 insertions(+), 103 deletions(-) diff --git a/src/media/video/encode/aom/aom_av1_encoder.cpp b/src/media/video/encode/aom/aom_av1_encoder.cpp index 2683b5d..25997eb 100644 --- a/src/media/video/encode/aom/aom_av1_encoder.cpp +++ b/src/media/video/encode/aom/aom_av1_encoder.cpp @@ -289,9 +289,10 @@ int AomAv1Encoder::Init() { return 0; } -int AomAv1Encoder::Encode( - const uint8_t *pData, int nSize, - std::function on_encoded_image) { +int AomAv1Encoder::Encode(const uint8_t *pData, int nSize, + std::function + on_encoded_image) { if (SAVE_NV12_STREAM) { fwrite(pData, 1, nSize, file_nv12_); } @@ -311,10 +312,13 @@ int AomAv1Encoder::Encode( // NV12ToYUV420PFFmpeg((unsigned char *)pData, frame_width_, frame_height_, // (unsigned char *)yuv420p_buffer); + VideoFrameType frame_type; if (0 == seq_++ % 300) { force_i_frame_flags_ = AOM_EFLAG_FORCE_KF; + frame_type = VideoFrameType::kVideoFrameKey; } else { force_i_frame_flags_ = 0; + frame_type = VideoFrameType::kVideoFrameDelta; } // Encode a frame. The presentation timestamp `pts` should not use real @@ -341,7 +345,8 @@ int AomAv1Encoder::Encode( LOG_INFO("Encoded frame qp = {}", qp); if (on_encoded_image) { - on_encoded_image((char *)encoded_frame_, encoded_frame_size_); + on_encoded_image((char *)encoded_frame_, encoded_frame_size_, + frame_type); if (SAVE_H264_STREAM) { fwrite(encoded_frame_, 1, encoded_frame_size_, file_ivf_); } diff --git a/src/media/video/encode/aom/aom_av1_encoder.h b/src/media/video/encode/aom/aom_av1_encoder.h index 572c1c8..4b0bd69 100644 --- a/src/media/video/encode/aom/aom_av1_encoder.h +++ b/src/media/video/encode/aom/aom_av1_encoder.h @@ -36,9 +36,10 @@ class AomAv1Encoder : public VideoEncoder { public: int Init(); - int Encode( - const uint8_t* pData, int nSize, - std::function on_encoded_image); + int Encode(const uint8_t* pData, int nSize, + std::function + on_encoded_image); virtual int OnEncodedImage(char* encoded_packets, size_t size); diff --git a/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.cpp b/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.cpp index 37dada0..4a285ec 100644 --- a/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.cpp +++ b/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.cpp @@ -149,7 +149,9 @@ int FFmpegVideoEncoder::Init() { int FFmpegVideoEncoder::Encode( const uint8_t *pData, int nSize, - std::function on_encoded_image) { + std::function + on_encoded_image) { if (!codec_ctx_) { LOG_ERROR("Invalid codec context"); return -1; @@ -181,6 +183,13 @@ int FFmpegVideoEncoder::Encode( int ret = avcodec_send_frame(codec_ctx_, frame_); // frame_->pict_type = AV_PICTURE_TYPE_I; + VideoFrameType frame_type; + if (0 == seq_++ % 300) { + frame_type = VideoFrameType::kVideoFrameKey; + } else { + frame_type = VideoFrameType::kVideoFrameDelta; + } + while (ret >= 0) { ret = avcodec_receive_packet(codec_ctx_, packet_); if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { @@ -198,7 +207,7 @@ int FFmpegVideoEncoder::Encode( } if (on_encoded_image) { - on_encoded_image((char *)packet_->data, packet_->size); + on_encoded_image((char *)packet_->data, packet_->size, frame_type); if (SAVE_H264_STREAM) { fwrite(packet_->data, 1, packet_->size, file_h264_); } diff --git a/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.h b/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.h index 33dbc11..a3835d0 100644 --- a/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.h +++ b/src/media/video/encode/ffmpeg/ffmpeg_video_encoder.h @@ -36,9 +36,10 @@ class FFmpegVideoEncoder : public VideoEncoder { virtual ~FFmpegVideoEncoder(); int Init(); - int Encode( - const uint8_t* pData, int nSize, - std::function on_encoded_image); + int Encode(const uint8_t* pData, int nSize, + std::function + on_encoded_image); virtual int OnEncodedImage(char* encoded_packets, size_t size); diff --git a/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp b/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp index 9381573..006cecc 100644 --- a/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp +++ b/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp @@ -78,14 +78,20 @@ int NvidiaVideoEncoder::Init() { int NvidiaVideoEncoder::Encode( const uint8_t *pData, int nSize, - std::function on_encoded_image) { + std::function + on_encoded_image) { if (!encoder_) { LOG_ERROR("Invalid encoder"); return -1; } + VideoFrameType frame_type; if (0 == seq_++ % 300) { ForceIdr(); + frame_type = VideoFrameType::kVideoFrameKey; + } else { + frame_type = VideoFrameType::kVideoFrameDelta; } #ifdef SHOW_SUBMODULE_TIME_COST @@ -110,7 +116,7 @@ int NvidiaVideoEncoder::Encode( for (const auto &packet : encoded_packets_) { if (on_encoded_image) { - on_encoded_image((char *)packet.data(), packet.size()); + on_encoded_image((char *)packet.data(), packet.size(), frame_type); if (SAVE_ENCODER_STREAM) { fwrite(packet.data(), 1, packet.size(), file_); } diff --git a/src/media/video/encode/nvcodec/nvidia_video_encoder.h b/src/media/video/encode/nvcodec/nvidia_video_encoder.h index 70c4cfa..cb41b6b 100644 --- a/src/media/video/encode/nvcodec/nvidia_video_encoder.h +++ b/src/media/video/encode/nvcodec/nvidia_video_encoder.h @@ -12,9 +12,10 @@ class NvidiaVideoEncoder : public VideoEncoder { virtual ~NvidiaVideoEncoder(); int Init(); - int Encode( - const uint8_t* pData, int nSize, - std::function on_encoded_image); + int Encode(const uint8_t* pData, int nSize, + std::function + on_encoded_image); virtual int OnEncodedImage(char* encoded_packets, size_t size); diff --git a/src/media/video/encode/openh264/openh264_encoder.cpp b/src/media/video/encode/openh264/openh264_encoder.cpp index 1a4a4bc..6cf267c 100644 --- a/src/media/video/encode/openh264/openh264_encoder.cpp +++ b/src/media/video/encode/openh264/openh264_encoder.cpp @@ -155,7 +155,9 @@ int OpenH264Encoder::Init() { } int OpenH264Encoder::Encode( const uint8_t *pData, int nSize, - std::function on_encoded_image) { + std::function + on_encoded_image) { if (!openh264_encoder_) { LOG_ERROR("Invalid openh264 encoder"); return -1; @@ -165,8 +167,12 @@ int OpenH264Encoder::Encode( fwrite(yuv420p_buffer, 1, nSize, file_nv12_); } + VideoFrameType frame_type; if (0 == seq_++ % 300) { ForceIdr(); + frame_type = VideoFrameType::kVideoFrameKey; + } else { + frame_type = VideoFrameType::kVideoFrameDelta; } NV12ToYUV420PFFmpeg((unsigned char *)pData, frame_width_, frame_height_, @@ -222,7 +228,7 @@ int OpenH264Encoder::Encode( encoded_frame_size_ = encoded_frame_size; if (on_encoded_image) { - on_encoded_image((char *)encoded_frame_, encoded_frame_size_); + on_encoded_image((char *)encoded_frame_, encoded_frame_size_, frame_type); if (SAVE_H264_STREAM) { fwrite(encoded_frame_, 1, encoded_frame_size_, file_h264_); } @@ -268,7 +274,7 @@ int OpenH264Encoder::Encode( encoded_frame_size_ = encoded_frame_size; if (on_encoded_image) { - on_encoded_image((char *)encoded_frame_, encoded_frame_size_); + on_encoded_image((char *)encoded_frame_, frame_type); if (SAVE_H264_STREAM) { fwrite(encoded_frame_, 1, encoded_frame_size_, file_h264_); } diff --git a/src/media/video/encode/openh264/openh264_encoder.h b/src/media/video/encode/openh264/openh264_encoder.h index 05e37ca..e225c71 100644 --- a/src/media/video/encode/openh264/openh264_encoder.h +++ b/src/media/video/encode/openh264/openh264_encoder.h @@ -23,9 +23,10 @@ class OpenH264Encoder : public VideoEncoder { virtual ~OpenH264Encoder(); int Init(); - int Encode( - const uint8_t* pData, int nSize, - std::function on_encoded_image); + int Encode(const uint8_t* pData, int nSize, + std::function + on_encoded_image); virtual int OnEncodedImage(char* encoded_packets, size_t size); diff --git a/src/media/video/encode/video_encoder.h b/src/media/video/encode/video_encoder.h index e4c668f..9fc8d87 100644 --- a/src/media/video/encode/video_encoder.h +++ b/src/media/video/encode/video_encoder.h @@ -4,10 +4,18 @@ #include class VideoEncoder { + public: + enum VideoFrameType { + kEmptyFrame = 0, + kVideoFrameKey = 3, + kVideoFrameDelta = 4, + }; + public: virtual int Init() = 0; virtual int Encode(const uint8_t* pData, int nSize, - std::function + std::function on_encoded_image) = 0; virtual int OnEncodedImage(char* encoded_packets, size_t size) = 0; virtual void ForceIdr() = 0; diff --git a/src/pc/peer_connection.cpp b/src/pc/peer_connection.cpp index 23b06be..c9552bc 100644 --- a/src/pc/peer_connection.cpp +++ b/src/pc/peer_connection.cpp @@ -527,11 +527,16 @@ int PeerConnection::SendVideoData(const char *data, size_t size) { } int ret = video_encoder_->Encode( - (uint8_t *)data, size, [this](char *encoded_frame, size_t size) -> int { + (uint8_t *)data, size, + [this](char *encoded_frame, size_t size, + VideoEncoder::VideoFrameType frame_type) -> int { for (auto &ice_trans : ice_transmission_list_) { // LOG_ERROR("H264 frame size: [{}]", size); - ice_trans.second->SendData(IceTransmission::DATA_TYPE::VIDEO, - encoded_frame, size); + // ice_trans.second->SendData(IceTransmission::DATA_TYPE::VIDEO, + // encoded_frame, size); + ice_trans.second->SendVideoData( + static_cast(frame_type), + encoded_frame, size); } return 0; }); @@ -550,8 +555,8 @@ int PeerConnection::SendAudioData(const char *data, size_t size) { [this](char *encoded_audio_buffer, size_t size) -> int { for (auto &ice_trans : ice_transmission_list_) { // LOG_ERROR("opus frame size: [{}]", size); - // ice_trans.second->SendData(IceTransmission::DATA_TYPE::AUDIO, - // encoded_audio_buffer, size); + ice_trans.second->SendData(IceTransmission::DATA_TYPE::AUDIO, + encoded_audio_buffer, size); } return 0; }); diff --git a/src/rtp/rtp_codec.cpp b/src/rtp/rtp_codec.cpp index 4d15730..c9017ee 100644 --- a/src/rtp/rtp_codec.cpp +++ b/src/rtp/rtp_codec.cpp @@ -10,6 +10,8 @@ #define FU_A 28 #define FU_B 29 +constexpr int kObuTypeSequenceHeader = 1; + RtpCodec ::RtpCodec(RtpPacket::PAYLOAD_TYPE payload_type) : version_(RTP_VERSION), has_padding_(false), @@ -33,10 +35,6 @@ RtpCodec ::~RtpCodec() { void RtpCodec::Encode(uint8_t* buffer, size_t size, std::vector& packets) { - // if (!rtp_packet_) { - // rtp_packet_ = new RtpPacket(); - // } - if (RtpPacket::PAYLOAD_TYPE::H264 == payload_type_) { if (fec_enable_ && IsKeyFrame((const uint8_t*)buffer, size)) { uint8_t** fec_packets = fec_encoder_.Encode((const char*)buffer, size); @@ -251,78 +249,48 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size, rtp_packet.EncodeAv1(obus[i].payload_, obus[i].payload_size_); packets.emplace_back(rtp_packet); + } else { + size_t last_packet_size = obus[i].payload_size_ % MAX_NALU_LEN; + size_t packet_num = + obus[i].payload_size_ / MAX_NALU_LEN + (last_packet_size ? 1 : 0); + timestamp_ = std::chrono::high_resolution_clock::now() + .time_since_epoch() + .count(); + for (size_t index = 0; index < packet_num; index++) { + RtpPacket rtp_packet; + rtp_packet.SetVerion(version_); + rtp_packet.SetHasPadding(has_padding_); + rtp_packet.SetHasExtension(has_extension_); + rtp_packet.SetMarker(index == packet_num - 1 ? 1 : 0); + rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_)); + rtp_packet.SetSequenceNumber(sequence_number_++); + rtp_packet.SetTimestamp(timestamp_); + rtp_packet.SetSsrc(ssrc_); + if (!csrcs_.empty()) { + rtp_packet.SetCsrcs(csrcs_); + } + if (has_extension_) { + rtp_packet.SetExtensionProfile(extension_profile_); + rtp_packet.SetExtensionData(extension_data_, extension_len_); + } + + int z = index != 0 ? 1 : 0; + int y = index != packet_num - 1 ? 1 : 0; + int w = 1; + int n = 0; + rtp_packet.SetAv1AggrHeader(z, y, w, n); + + if (index == packet_num - 1 && last_packet_size > 0) { + rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN, + last_packet_size); + } else { + rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN, + MAX_NALU_LEN); + } + packets.emplace_back(rtp_packet); + } } } - - // if (size <= MAX_NALU_LEN) { - // RtpPacket rtp_packet; - // rtp_packet.SetVerion(version_); - // rtp_packet.SetHasPadding(has_padding_); - // rtp_packet.SetHasExtension(has_extension_); - // rtp_packet.SetMarker(1); - // rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_)); - // rtp_packet.SetSequenceNumber(sequence_number_++); - - // timestamp_ = - // std::chrono::high_resolution_clock::now().time_since_epoch().count(); - // rtp_packet.SetTimestamp(timestamp_); - // rtp_packet.SetSsrc(ssrc_); - - // if (!csrcs_.empty()) { - // rtp_packet.SetCsrcs(csrcs_); - // } - - // if (has_extension_) { - // rtp_packet.SetExtensionProfile(extension_profile_); - // rtp_packet.SetExtensionData(extension_data_, extension_len_); - // } - - // RtpPacket::AV1_AGGR_HEADER av1_aggr_header; - // av1_aggr_header.z = av1_aggr_header.z; - // av1_aggr_header.y = av1_aggr_header.y; - // av1_aggr_header.w = av1_aggr_header.w; - // av1_aggr_header.n = av1_aggr_header.n; - - // rtp_packet.SetAv1AggrHeader(av1_aggr_header); - - // rtp_packet.EncodeAv1(buffer, size); - // packets.emplace_back(rtp_packet); - - // } else { - // size_t last_packet_size = size % MAX_NALU_LEN; - // size_t packet_num = size / MAX_NALU_LEN + (last_packet_size ? 1 : 0); - // timestamp_ = - // std::chrono::high_resolution_clock::now().time_since_epoch().count(); - - // for (size_t index = 0; index < packet_num; index++) { - // RtpPacket rtp_packet; - // rtp_packet.SetVerion(version_); - // rtp_packet.SetHasPadding(has_padding_); - // rtp_packet.SetHasExtension(has_extension_); - // rtp_packet.SetMarker(index == packet_num - 1 ? 1 : 0); - // rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_)); - // rtp_packet.SetSequenceNumber(sequence_number_++); - // rtp_packet.SetTimestamp(timestamp_); - // rtp_packet.SetSsrc(ssrc_); - - // if (!csrcs_.empty()) { - // rtp_packet.SetCsrcs(csrcs_); - // } - - // if (has_extension_) { - // rtp_packet.SetExtensionProfile(extension_profile_); - // rtp_packet.SetExtensionData(extension_data_, extension_len_); - // } - - // if (index == packet_num - 1 && last_packet_size > 0) { - // rtp_packet.EncodeAv1(buffer + index * MAX_NALU_LEN, - // last_packet_size); - // } else { - // rtp_packet.EncodeAv1(buffer + index * MAX_NALU_LEN, MAX_NALU_LEN); - // } - // packets.emplace_back(rtp_packet); - // } - // } } else if (RtpPacket::PAYLOAD_TYPE::OPUS == payload_type_) { RtpPacket rtp_packet; rtp_packet.SetVerion(version_); @@ -358,6 +326,270 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size, } } +void RtpCodec::Encode(VideoFrameType frame_type, uint8_t* buffer, size_t size, + std::vector& packets) { + if (RtpPacket::PAYLOAD_TYPE::H264 == payload_type_) { + if (fec_enable_ && IsKeyFrame((const uint8_t*)buffer, size)) { + uint8_t** fec_packets = fec_encoder_.Encode((const char*)buffer, size); + if (nullptr == fec_packets) { + LOG_ERROR("Invalid fec_packets"); + return; + } + unsigned int num_of_total_packets = 0; + unsigned int num_of_source_packets = 0; + unsigned int last_packet_size = 0; + fec_encoder_.GetFecPacketsParams(size, num_of_total_packets, + num_of_source_packets, last_packet_size); + + timestamp_ = + std::chrono::high_resolution_clock::now().time_since_epoch().count(); + + for (unsigned int index = 0; index < num_of_total_packets; index++) { + RtpPacket rtp_packet; + if (index < num_of_source_packets) { + rtp_packet.SetVerion(version_); + rtp_packet.SetHasPadding(has_padding_); + rtp_packet.SetHasExtension(has_extension_); + rtp_packet.SetMarker(index == num_of_source_packets - 1 ? 1 : 0); + rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE::H264_FEC_SOURCE); + rtp_packet.SetSequenceNumber(sequence_number_++); + rtp_packet.SetTimestamp(timestamp_); + rtp_packet.SetSsrc(ssrc_); + + if (!csrcs_.empty()) { + rtp_packet.SetCsrcs(csrcs_); + } + + if (has_extension_) { + rtp_packet.SetExtensionProfile(extension_profile_); + rtp_packet.SetExtensionData(extension_data_, extension_len_); + } + + RtpPacket::FU_INDICATOR fu_indicator; + fu_indicator.forbidden_bit = 0; + fu_indicator.nal_reference_idc = 0; + fu_indicator.nal_unit_type = FU_A; + + RtpPacket::FU_HEADER fu_header; + fu_header.start = index == 0 ? 1 : 0; + fu_header.end = index == num_of_source_packets - 1 ? 1 : 0; + fu_header.remain_bit = 0; + fu_header.nal_unit_type = FU_A; + + rtp_packet.SetFuIndicator(fu_indicator); + rtp_packet.SetFuHeader(fu_header); + + if (index == num_of_source_packets - 1) { + if (last_packet_size > 0) { + rtp_packet.EncodeH264FecSource(fec_packets[index], + last_packet_size, index, + num_of_source_packets); + } else { + rtp_packet.EncodeH264FecSource(fec_packets[index], MAX_NALU_LEN, + index, num_of_source_packets); + } + } else { + rtp_packet.EncodeH264FecSource(fec_packets[index], MAX_NALU_LEN, + index, num_of_source_packets); + } + + } else if (index >= num_of_source_packets && + index < num_of_total_packets) { + rtp_packet.SetVerion(version_); + rtp_packet.SetHasPadding(has_padding_); + rtp_packet.SetHasExtension(has_extension_); + rtp_packet.SetMarker(index == num_of_total_packets - 1 ? 1 : 0); + rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE::H264_FEC_REPAIR); + rtp_packet.SetSequenceNumber(sequence_number_++); + rtp_packet.SetTimestamp(timestamp_); + rtp_packet.SetSsrc(ssrc_); + + if (!csrcs_.empty()) { + rtp_packet.SetCsrcs(csrcs_); + } + + if (has_extension_) { + rtp_packet.SetExtensionProfile(extension_profile_); + rtp_packet.SetExtensionData(extension_data_, extension_len_); + } + rtp_packet.EncodeH264FecRepair(fec_packets[index], MAX_NALU_LEN, + index, num_of_source_packets); + } + packets.emplace_back(rtp_packet); + + // if (index < num_of_source_packets) { + // rtp_packet.EncodeH264Fua(fec_packets[index], MAX_NALU_LEN); + // packets.emplace_back(rtp_packet); + // } + } + + fec_encoder_.ReleaseFecPackets(fec_packets, size); + return; + } + + if (size <= MAX_NALU_LEN) { + RtpPacket rtp_packet; + rtp_packet.SetVerion(version_); + rtp_packet.SetHasPadding(has_padding_); + rtp_packet.SetHasExtension(has_extension_); + rtp_packet.SetMarker(1); + rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_)); + rtp_packet.SetSequenceNumber(sequence_number_++); + + timestamp_ = + std::chrono::high_resolution_clock::now().time_since_epoch().count(); + rtp_packet.SetTimestamp(timestamp_); + rtp_packet.SetSsrc(ssrc_); + + if (!csrcs_.empty()) { + rtp_packet.SetCsrcs(csrcs_); + } + + if (has_extension_) { + rtp_packet.SetExtensionProfile(extension_profile_); + rtp_packet.SetExtensionData(extension_data_, extension_len_); + } + + RtpPacket::FU_INDICATOR fu_indicator; + fu_indicator.forbidden_bit = 0; + fu_indicator.nal_reference_idc = 1; + fu_indicator.nal_unit_type = NALU; + rtp_packet.SetFuIndicator(fu_indicator); + + rtp_packet.EncodeH264Nalu(buffer, size); + packets.emplace_back(rtp_packet); + + } else { + size_t last_packet_size = size % MAX_NALU_LEN; + size_t packet_num = size / MAX_NALU_LEN + (last_packet_size ? 1 : 0); + timestamp_ = + std::chrono::high_resolution_clock::now().time_since_epoch().count(); + + for (size_t index = 0; index < packet_num; index++) { + RtpPacket rtp_packet; + rtp_packet.SetVerion(version_); + rtp_packet.SetHasPadding(has_padding_); + rtp_packet.SetHasExtension(has_extension_); + rtp_packet.SetMarker(index == packet_num - 1 ? 1 : 0); + rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_)); + rtp_packet.SetSequenceNumber(sequence_number_++); + rtp_packet.SetTimestamp(timestamp_); + rtp_packet.SetSsrc(ssrc_); + + if (!csrcs_.empty()) { + rtp_packet.SetCsrcs(csrcs_); + } + + if (has_extension_) { + rtp_packet.SetExtensionProfile(extension_profile_); + rtp_packet.SetExtensionData(extension_data_, extension_len_); + } + + RtpPacket::FU_INDICATOR fu_indicator; + fu_indicator.forbidden_bit = 0; + fu_indicator.nal_reference_idc = 0; + fu_indicator.nal_unit_type = FU_A; + + RtpPacket::FU_HEADER fu_header; + fu_header.start = index == 0 ? 1 : 0; + fu_header.end = index == packet_num - 1 ? 1 : 0; + fu_header.remain_bit = 0; + fu_header.nal_unit_type = FU_A; + + rtp_packet.SetFuIndicator(fu_indicator); + rtp_packet.SetFuHeader(fu_header); + + if (index == packet_num - 1 && last_packet_size > 0) { + rtp_packet.EncodeH264Fua(buffer + index * MAX_NALU_LEN, + last_packet_size); + } else { + rtp_packet.EncodeH264Fua(buffer + index * MAX_NALU_LEN, MAX_NALU_LEN); + } + packets.emplace_back(rtp_packet); + } + } + } else if (RtpPacket::PAYLOAD_TYPE::AV1 == payload_type_) { + std::vector obus = ParseObus(buffer, size); + LOG_ERROR("Total size = [{}]", size); + for (int i = 0; i < obus.size(); i++) { + LOG_ERROR("[{}] Obu size = [{}], Obu type [{}]", i, obus[i].size_, + ObuTypeToString((OBU_TYPE)ObuType(obus[i].header_))); + if (obus[i].size_ <= MAX_NALU_LEN) { + RtpPacket rtp_packet; + rtp_packet.SetVerion(version_); + rtp_packet.SetHasPadding(has_padding_); + rtp_packet.SetHasExtension(has_extension_); + rtp_packet.SetMarker(1); + rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_)); + rtp_packet.SetSequenceNumber(sequence_number_++); + + timestamp_ = std::chrono::high_resolution_clock::now() + .time_since_epoch() + .count(); + rtp_packet.SetTimestamp(timestamp_); + rtp_packet.SetSsrc(ssrc_); + + if (!csrcs_.empty()) { + rtp_packet.SetCsrcs(csrcs_); + } + + if (has_extension_) { + rtp_packet.SetExtensionProfile(extension_profile_); + rtp_packet.SetExtensionData(extension_data_, extension_len_); + } + + rtp_packet.SetAv1AggrHeader(0, 0, 1, 0); + + rtp_packet.EncodeAv1(obus[i].payload_, obus[i].payload_size_); + packets.emplace_back(rtp_packet); + } else { + size_t last_packet_size = obus[i].payload_size_ % MAX_NALU_LEN; + size_t packet_num = + obus[i].payload_size_ / MAX_NALU_LEN + (last_packet_size ? 1 : 0); + timestamp_ = std::chrono::high_resolution_clock::now() + .time_since_epoch() + .count(); + for (size_t index = 0; index < packet_num; index++) { + RtpPacket rtp_packet; + rtp_packet.SetVerion(version_); + rtp_packet.SetHasPadding(has_padding_); + rtp_packet.SetHasExtension(has_extension_); + rtp_packet.SetMarker(index == packet_num - 1 ? 1 : 0); + rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_)); + rtp_packet.SetSequenceNumber(sequence_number_++); + rtp_packet.SetTimestamp(timestamp_); + rtp_packet.SetSsrc(ssrc_); + if (!csrcs_.empty()) { + rtp_packet.SetCsrcs(csrcs_); + } + if (has_extension_) { + rtp_packet.SetExtensionProfile(extension_profile_); + rtp_packet.SetExtensionData(extension_data_, extension_len_); + } + + int z = index != 0 ? 1 : 0; + int y = index != packet_num - 1 ? 1 : 0; + int w = 1; + int n = (frame_type == VideoFrameType::kVideoFrameKey) && + (ObuType(obus[i].header_) == kObuTypeSequenceHeader) + ? 1 + : 0; + rtp_packet.SetAv1AggrHeader(z, y, w, n); + + if (index == packet_num - 1 && last_packet_size > 0) { + rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN, + last_packet_size); + } else { + rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN, + MAX_NALU_LEN); + } + packets.emplace_back(rtp_packet); + } + } + } + } +} + size_t RtpCodec::Decode(RtpPacket& packet, uint8_t* payload) { // if ((packet.Buffer()[13] >> 6) & 0x01) { // LOG_ERROR("End bit!!!!!!!!!!!!!!!"); diff --git a/src/rtp/rtp_codec.h b/src/rtp/rtp_codec.h index a21d0ba..a6921e8 100644 --- a/src/rtp/rtp_codec.h +++ b/src/rtp/rtp_codec.h @@ -9,12 +9,21 @@ #include "rtp_packet.h" class RtpCodec { + public: + enum VideoFrameType { + kEmptyFrame = 0, + kVideoFrameKey = 3, + kVideoFrameDelta = 4, + }; + public: RtpCodec(RtpPacket::PAYLOAD_TYPE payload_type); ~RtpCodec(); public: void Encode(uint8_t* buffer, size_t size, std::vector& packets); + void Encode(VideoFrameType frame_type, uint8_t* buffer, size_t size, + std::vector& packets); size_t Decode(RtpPacket& packet, uint8_t* payload); // protected: diff --git a/src/transmission/ice_transmission.cpp b/src/transmission/ice_transmission.cpp index b40f787..4e39e18 100644 --- a/src/transmission/ice_transmission.cpp +++ b/src/transmission/ice_transmission.cpp @@ -312,6 +312,23 @@ int IceTransmission::SendData(DATA_TYPE type, const char *data, size_t size) { return 0; } +int IceTransmission::SendVideoData(VideoFrameType frame_type, const char *data, + size_t size) { + if (NiceComponentState::NICE_COMPONENT_STATE_READY == state_) { + std::vector packets; + + if (rtp_video_sender_) { + if (video_rtp_codec_) { + video_rtp_codec_->Encode( + static_cast(frame_type), (uint8_t *)data, + size, packets); + } + rtp_video_sender_->Enqueue(packets); + } + } + return 0; +} + uint8_t IceTransmission::CheckIsRtcpPacket(const char *buffer, size_t size) { if (size < 4) { return 0; diff --git a/src/transmission/ice_transmission.h b/src/transmission/ice_transmission.h index 506d4d0..1beac6a 100644 --- a/src/transmission/ice_transmission.h +++ b/src/transmission/ice_transmission.h @@ -25,6 +25,11 @@ class IceTransmission { public: typedef enum { VIDEO = 96, AUDIO = 97, DATA = 127 } DATA_TYPE; + enum VideoFrameType { + kEmptyFrame = 0, + kVideoFrameKey = 3, + kVideoFrameDelta = 4, + }; public: IceTransmission(bool offer_peer, std::string &transmission_id, @@ -66,6 +71,12 @@ class IceTransmission { int SendData(DATA_TYPE type, const char *data, size_t size); + int SendVideoData(VideoFrameType frame_type, const char *data, size_t size); + + int SendAudioData(const char *data, size_t size); + + int SendUserData(const char *data, size_t size); + public: int GatherCandidates();