From b2e725e5647638ea0d1121776549a6c560b3c21d Mon Sep 17 00:00:00 2001 From: dijunkun Date: Thu, 7 Sep 2023 17:20:38 +0800 Subject: [PATCH] Implementation for FuA encode and decode --- src/rtp/rtp_packet.cpp | 112 ++++++++++++++++++++++++-- src/rtp/rtp_packet.h | 39 ++++----- src/rtp/rtp_session.cpp | 59 +++++++++++--- src/rtp/rtp_session.h | 1 + src/transmission/ice_transmission.cpp | 24 +++--- 5 files changed, 184 insertions(+), 51 deletions(-) diff --git a/src/rtp/rtp_packet.cpp b/src/rtp/rtp_packet.cpp index 0dd6ebe..a65ec7e 100644 --- a/src/rtp/rtp_packet.cpp +++ b/src/rtp/rtp_packet.cpp @@ -150,9 +150,9 @@ const uint8_t *RtpPacket::EncodeH264Nalu(uint8_t *payload, (has_extension_ && extension_data_ ? extension_len_ : 0) + extension_offset; - buffer_[12 + payload_offset] = nalu_header_.forbidden_bit << 7 | - nalu_header_.nal_reference_idc << 6 | - nalu_header_.nal_unit_type; + buffer_[12 + payload_offset] = fu_indicator_.forbidden_bit << 7 | + fu_indicator_.nal_reference_idc << 6 | + fu_indicator_.nal_unit_type; memcpy(buffer_ + 13 + payload_offset, payload, payload_size); size_ = payload_size + (13 + payload_offset); @@ -160,6 +160,57 @@ const uint8_t *RtpPacket::EncodeH264Nalu(uint8_t *payload, return buffer_; } +const uint8_t *RtpPacket::EncodeH264Fua(uint8_t *payload, size_t payload_size) { + buffer_[0] = (version_ << 6) | (has_padding_ << 5) | (has_extension_ << 4) | + total_csrc_number_; + buffer_[1] = (marker_ << 7) | payload_type_; + buffer_[2] = (sequence_number_ >> 8) & 0xFF; + buffer_[3] = sequence_number_ & 0xFF; + buffer_[4] = (timestamp_ >> 24) & 0xFF; + buffer_[5] = (timestamp_ >> 16) & 0xFF; + buffer_[6] = (timestamp_ >> 8) & 0xFF; + buffer_[7] = timestamp_ & 0xFF; + buffer_[8] = (ssrc_ >> 24) & 0xFF; + buffer_[9] = (ssrc_ >> 16) & 0xFF; + buffer_[10] = (ssrc_ >> 8) & 0xFF; + buffer_[11] = ssrc_ & 0xFF; + + for (uint32_t index = 0; index < total_csrc_number_ && !csrcs_.empty(); + index++) { + buffer_[12 + index] = (csrcs_[index] >> 24) & 0xFF; + buffer_[13 + index] = (csrcs_[index] >> 16) & 0xFF; + buffer_[14 + index] = (csrcs_[index] >> 8) & 0xFF; + buffer_[15 + index] = csrcs_[index] & 0xFF; + } + + uint32_t extension_offset = + total_csrc_number_ && !csrcs_.empty() ? total_csrc_number_ * 4 : 0; + if (has_extension_ && extension_data_) { + buffer_[12 + extension_offset] = extension_profile_ >> 8; + buffer_[13 + extension_offset] = extension_profile_ & 0xff; + buffer_[14 + extension_offset] = (extension_len_ >> 8) & 0xFF; + buffer_[15 + extension_offset] = extension_len_ & 0xFF; + memcpy(buffer_ + 16 + extension_offset, extension_data_, extension_len_); + } + + uint32_t payload_offset = + (has_extension_ && extension_data_ ? extension_len_ : 0) + + extension_offset; + + buffer_[12 + payload_offset] = fu_indicator_.forbidden_bit << 7 | + fu_indicator_.nal_reference_idc << 6 | + fu_indicator_.nal_unit_type; + + buffer_[13 + payload_offset] = fu_header_.start << 7 | fu_header_.end << 6 | + fu_header_.remain_bit << 1 | + fu_header_.nal_unit_type; + + memcpy(buffer_ + 14 + payload_offset, payload, payload_size); + size_ = payload_size + (14 + payload_offset); + + return buffer_; +} + const uint8_t *RtpPacket::Decode() { version_ = (buffer_[0] >> 6) & 0x03; has_padding_ = (buffer_[0] >> 5) & 0x01; @@ -236,12 +287,61 @@ size_t RtpPacket::DecodeH264Nalu(uint8_t *payload) { uint32_t payload_offset = (has_extension_ ? extension_len_ : 0) + extension_offset; - nalu_header_.forbidden_bit = (buffer_[12 + payload_offset] >> 7) & 0x01; - nalu_header_.nal_reference_idc = (buffer_[12 + payload_offset] >> 6) & 0x01; - nalu_header_.nal_unit_type = (buffer_[12 + payload_offset]) & 0x31; + fu_indicator_.forbidden_bit = (buffer_[12 + payload_offset] >> 7) & 0x01; + fu_indicator_.nal_reference_idc = (buffer_[12 + payload_offset] >> 5) & 0x03; + fu_indicator_.nal_unit_type = buffer_[12 + payload_offset] & 0x1F; payload_size_ = size_ - (13 + payload_offset); payload_ = buffer_ + 13 + payload_offset; memcpy(payload, payload_, payload_size_); return payload_size_; +} + +size_t RtpPacket::DecodeH264Fua(uint8_t *payload) { + version_ = (buffer_[0] >> 6) & 0x03; + has_padding_ = (buffer_[0] >> 5) & 0x01; + has_extension_ = (buffer_[0] >> 4) & 0x01; + total_csrc_number_ = buffer_[0] & 0x0f; + marker_ = (buffer_[1] >> 7) & 0x01; + payload_type_ = buffer_[1] & 0x7f; + sequence_number_ = (buffer_[2] << 8) | buffer_[3]; + timestamp_ = + (buffer_[4] << 24) | (buffer_[5] << 16) | (buffer_[6] << 8) | buffer_[7]; + ssrc_ = (buffer_[8] << 24) | (buffer_[9] << 16) | (buffer_[10] << 8) | + buffer_[11]; + + for (uint32_t index = 0; index < total_csrc_number_; index++) { + uint32_t csrc = (buffer_[12 + index] << 24) | (buffer_[13 + index] << 16) | + (buffer_[14 + index] << 8) | buffer_[15 + index]; + csrcs_.push_back(csrc); + } + + uint32_t extension_offset = total_csrc_number_ * 4; + if (has_extension_) { + extension_profile_ = + (buffer_[12 + extension_offset] << 8) | buffer_[13 + extension_offset]; + extension_len_ = + (buffer_[14 + extension_offset] << 8) | buffer_[15 + extension_offset]; + + // extension_data_ = new uint8_t[extension_len_]; + // memcpy(extension_data_, buffer_ + 16 + extension_offset, extension_len_); + extension_data_ = buffer_ + 16 + extension_offset; + } + + uint32_t payload_offset = + (has_extension_ ? extension_len_ : 0) + extension_offset; + + fu_indicator_.forbidden_bit = (buffer_[12 + payload_offset] >> 7) & 0x01; + fu_indicator_.nal_reference_idc = (buffer_[12 + payload_offset] >> 5) & 0x03; + fu_indicator_.nal_unit_type = buffer_[12 + payload_offset] & 0x1F; + + fu_header_.start = (buffer_[13 + payload_offset] >> 7) & 0x01; + fu_header_.end = (buffer_[13 + payload_offset] >> 6) & 0x01; + fu_header_.remain_bit = (buffer_[13 + payload_offset] >> 5) & 0x01; + fu_header_.nal_unit_type = buffer_[13 + payload_offset] & 0x1F; + + payload_size_ = size_ - (14 + payload_offset); + payload_ = buffer_ + 14 + payload_offset; + memcpy(payload, payload_, payload_size_); + return payload_size_; } \ No newline at end of file diff --git a/src/rtp/rtp_packet.h b/src/rtp/rtp_packet.h index fb29f74..58c9bb6 100644 --- a/src/rtp/rtp_packet.h +++ b/src/rtp/rtp_packet.h @@ -106,43 +106,35 @@ class RtpPacket { unsigned char forbidden_bit : 1; unsigned char nal_reference_idc : 2; unsigned char nal_unit_type : 5; - } NALU_HEADER; - - typedef struct { - unsigned char f : 1; - unsigned char nri : 2; - unsigned char type : 5; } FU_INDICATOR; typedef struct { - unsigned char s : 1; - unsigned char e : 1; - unsigned char r : 1; - unsigned char type : 5; + unsigned char start : 1; + unsigned char end : 1; + unsigned char remain_bit : 1; + unsigned char nal_unit_type : 5; } FU_HEADER; - void SetNaluHeader(NALU_HEADER nalu_header) { - nalu_header_.forbidden_bit = nalu_header.forbidden_bit; - nalu_header_.nal_reference_idc = nalu_header.nal_reference_idc; - nalu_header_.nal_unit_type = nalu_header.nal_unit_type; + void SetFuIndicator(FU_INDICATOR fu_indicator) { + fu_indicator_.forbidden_bit = fu_indicator.forbidden_bit; + fu_indicator_.nal_reference_idc = fu_indicator.nal_reference_idc; + fu_indicator_.nal_unit_type = fu_indicator.nal_unit_type; } - void FuAHeader(FU_INDICATOR fu_indicator, FU_HEADER fu_header) { - fu_indicator_.f = fu_indicator.f; - fu_indicator_.nri = fu_indicator.nri; - fu_indicator_.type = fu_indicator.nri; - - fu_header_.s = fu_header.s; - fu_header_.e = fu_header.e; - fu_header_.r = fu_header.r; - fu_header_.type = fu_header.type; + void SetFuHeader(FU_HEADER fu_header) { + fu_header_.start = fu_header.start; + fu_header_.end = fu_header.end; + fu_header_.remain_bit = fu_header.remain_bit; + fu_header_.nal_unit_type = fu_header.nal_unit_type; } public: const uint8_t *Encode(uint8_t *payload, size_t payload_size); const uint8_t *EncodeH264Nalu(uint8_t *payload, size_t payload_size); + const uint8_t *EncodeH264Fua(uint8_t *payload, size_t payload_size); const uint8_t *Decode(); size_t DecodeH264Nalu(uint8_t *payload); + size_t DecodeH264Fua(uint8_t *payload); public: // Get Header @@ -182,7 +174,6 @@ class RtpPacket { uint16_t extension_profile_ = 0; uint16_t extension_len_ = 0; uint8_t *extension_data_ = nullptr; - NALU_HEADER nalu_header_; FU_INDICATOR fu_indicator_; FU_HEADER fu_header_; diff --git a/src/rtp/rtp_session.cpp b/src/rtp/rtp_session.cpp index 52532cb..31dc75a 100644 --- a/src/rtp/rtp_session.cpp +++ b/src/rtp/rtp_session.cpp @@ -5,6 +5,9 @@ #include "log.h" #define RTP_VERSION 1 +#define NALU 1 +#define FU_A 28 +#define FU_B 29 RtpSession ::RtpSession(PAYLOAD_TYPE payload_type) : version_(RTP_VERSION), @@ -55,11 +58,11 @@ void RtpSession::Encode(uint8_t* buffer, size_t size, rtp_packet.SetExtensionData(extension_data_, extension_len_); } - RtpPacket::NALU_HEADER nalu_header; - nalu_header.forbidden_bit = 0; - nalu_header.nal_reference_idc = 1; - nalu_header.nal_unit_type = 1; - rtp_packet.SetNaluHeader(nalu_header); + RtpPacket::FU_INDICATOR fu_indicator; + fu_indicator.forbidden_bit = 0; + fu_indicator.nal_reference_idc = 1; + fu_indicator.nal_unit_type = NALU; + rtp_packet.SetFuIndicator(fu_indicator); rtp_packet.EncodeH264Nalu(buffer, size); packets.emplace_back(rtp_packet); @@ -68,12 +71,11 @@ void RtpSession::Encode(uint8_t* buffer, size_t size, size_t last_packet_size = size % MAX_NALU_LEN; size_t packet_num = size / MAX_NALU_LEN + (last_packet_size ? 1 : 0); - for (size_t index = 0; index * MAX_NALU_LEN < size + MAX_NALU_LEN; - index++) { + for (size_t index = 0; index < packet_num; index++) { rtp_packet.SetVerion(version_); rtp_packet.SetHasPadding(has_padding_); rtp_packet.SetHasExtension(has_extension_); - rtp_packet.SetMarker(1); + rtp_packet.SetMarker(index == packet_num ? 1 : 0); rtp_packet.SetPayloadType(PAYLOAD_TYPE(payload_type_)); rtp_packet.SetSequenceNumber(sequence_number_++); @@ -91,12 +93,49 @@ void RtpSession::Encode(uint8_t* buffer, size_t size, rtp_packet.SetExtensionData(extension_data_, extension_len_); } - rtp_packet.Encode(buffer, size); + RtpPacket::FU_INDICATOR fu_indicator; + fu_indicator.forbidden_bit = 0; + fu_indicator.nal_reference_idc = 0; + fu_indicator.nal_unit_type = FU_A; + + RtpPacket::FU_HEADER fu_header; + fu_header.start = index == 0 ? 1 : 0; + fu_header.end = index == packet_num - 1 ? 1 : 0; + fu_header.remain_bit = 0; + fu_header.nal_unit_type = FU_A; + + rtp_packet.SetFuIndicator(fu_indicator); + rtp_packet.SetFuHeader(fu_header); + + if (index == packet_num - 1 && last_packet_size > 0) { + rtp_packet.EncodeH264Fua(buffer + index * MAX_NALU_LEN, + last_packet_size); + } else { + rtp_packet.EncodeH264Fua(buffer + index * MAX_NALU_LEN, MAX_NALU_LEN); + } packets.emplace_back(rtp_packet); } } } size_t RtpSession::Decode(RtpPacket& packet, uint8_t* payload) { - return packet.DecodeH264Nalu(payload); + // if ((packet.Buffer()[13] >> 6) & 0x01) { + // LOG_ERROR("End bit!!!!!!!!!!!!!!!"); + // } + + // if ((packet.Buffer()[13] >> 7) & 0x01) { + // LOG_ERROR("Start bit!!!!!!!!!!!!!!!"); + // } + auto nal_unit_type = packet.Buffer()[12] & 0x1F; + + if (NALU == nal_unit_type) { + LOG_ERROR("Nalu"); + return packet.DecodeH264Nalu(payload); + } else if (FU_A == nal_unit_type) { + LOG_ERROR("Fua"); + return packet.DecodeH264Fua(payload); + } else { + LOG_ERROR("Default Nalu"); + return packet.DecodeH264Nalu(payload); + } } \ No newline at end of file diff --git a/src/rtp/rtp_session.h b/src/rtp/rtp_session.h index 4b57a34..6b7d973 100644 --- a/src/rtp/rtp_session.h +++ b/src/rtp/rtp_session.h @@ -36,6 +36,7 @@ class RtpSession private: // RtpPacket* rtp_packet_ = nullptr; + RtpPacket::FU_INDICATOR fu_indicator_; }; #endif \ No newline at end of file diff --git a/src/transmission/ice_transmission.cpp b/src/transmission/ice_transmission.cpp index 6fa2e85..9cfb2d1 100644 --- a/src/transmission/ice_transmission.cpp +++ b/src/transmission/ice_transmission.cpp @@ -287,17 +287,19 @@ int IceTransmission::SendData(const char *data, size_t size) { std::vector packets; - for (size_t num = 0, next_packet_size = 0; num * MAX_NALU_LEN < size; - num++) { - next_packet_size = size - num * MAX_NALU_LEN; - if (next_packet_size < MAX_NALU_LEN) { - video_rtp_session_->Encode((uint8_t *)(data + num * MAX_NALU_LEN), - next_packet_size, packets); - } else { - video_rtp_session_->Encode((uint8_t *)(data + num * MAX_NALU_LEN), - MAX_NALU_LEN, packets); - } - } + // for (size_t num = 0, next_packet_size = 0; num * MAX_NALU_LEN < size; + // num++) { + // next_packet_size = size - num * MAX_NALU_LEN; + // if (next_packet_size < MAX_NALU_LEN) { + // video_rtp_session_->Encode((uint8_t *)(data + num * MAX_NALU_LEN), + // next_packet_size, packets); + // } else { + // video_rtp_session_->Encode((uint8_t *)(data + num * MAX_NALU_LEN), + // MAX_NALU_LEN, packets); + // } + // } + + video_rtp_session_->Encode((uint8_t *)data, size, packets); for (auto &packet : packets) { send_ringbuffer_.push(packet);