diff --git a/src/media/video/encode/aom/aom_av1_encoder.cpp b/src/media/video/encode/aom/aom_av1_encoder.cpp index 25997eb..3d690e6 100644 --- a/src/media/video/encode/aom/aom_av1_encoder.cpp +++ b/src/media/video/encode/aom/aom_av1_encoder.cpp @@ -342,7 +342,7 @@ int AomAv1Encoder::Encode(const uint8_t *pData, int nSize, int qp = -1; SET_ENCODER_PARAM_OR_RETURN_ERROR(AOME_GET_LAST_QUANTIZER, &qp); - LOG_INFO("Encoded frame qp = {}", qp); + // LOG_INFO("Encoded frame qp = {}", qp); if (on_encoded_image) { on_encoded_image((char *)encoded_frame_, encoded_frame_size_, diff --git a/src/rtp/obu.cpp b/src/rtp/obu.cpp index 286de92..88c0d68 100644 --- a/src/rtp/obu.cpp +++ b/src/rtp/obu.cpp @@ -7,59 +7,67 @@ Obu::Obu() {} Obu::Obu(const Obu &obu) { + if (obu.size_ > 0) { + data_ = (uint8_t *)malloc(obu.size_); + memcpy(data_, obu.data_, obu.size_); + size_ = obu.size_; + } + if (obu.payload_size_ > 0) { payload_ = (uint8_t *)malloc(obu.payload_size_); - if (NULL == payload_) { - LOG_ERROR("Malloc failed"); - } else { - memcpy(payload_, obu.payload_, obu.payload_size_); - } + memcpy(payload_, obu.payload_, obu.payload_size_); payload_size_ = obu.payload_size_; - size_ = obu.size_; - header_ = obu.header_; - extension_header_ = obu.extension_header_; - } else { - size_ = obu.size_; - header_ = obu.header_; - extension_header_ = obu.extension_header_; } + + header_ = obu.header_; + extension_header_ = obu.extension_header_; } Obu::Obu(Obu &&obu) - : payload_((uint8_t *)std::move(obu.payload_)), - payload_size_(obu.payload_size_), + : data_(std::move(obu.data_)), size_(obu.size_), + payload_((uint8_t *)std::move(obu.payload_)), + payload_size_(obu.payload_size_), header_(obu.header_), extension_header_(obu.extension_header_) { + obu.data_ = nullptr; + obu.size_ = 0; obu.payload_ = nullptr; obu.payload_size_ = 0; } Obu &Obu::operator=(const Obu &obu) { if (&obu != this) { + if (obu.size_ > 0) { + data_ = (uint8_t *)realloc(data_, obu.size_); + memcpy(data_, obu.data_, obu.size_); + size_ = obu.size_; + } + if (obu.payload_size_ > 0) { payload_ = (uint8_t *)realloc(payload_, obu.payload_size_); memcpy(payload_, obu.payload_, obu.payload_size_); payload_size_ = obu.payload_size_; - size_ = obu.size_; - header_ = obu.header_; - extension_header_ = obu.extension_header_; - } else { - header_ = obu.header_; - extension_header_ = obu.extension_header_; } + + header_ = obu.header_; + extension_header_ = obu.extension_header_; } return *this; } Obu &Obu::operator=(Obu &&obu) { if (&obu != this) { + data_ = std::move(obu.data_); + obu.data_ = nullptr; + size_ = obu.size_; + obu.size_ = 0; + payload_ = std::move(obu.payload_); obu.payload_ = nullptr; payload_size_ = obu.payload_size_; obu.payload_size_ = 0; - size_ = obu.size_; - obu.size_ = 0; + header_ = obu.header_; obu.header_ = 0; extension_header_ = obu.extension_header_; @@ -69,12 +77,18 @@ Obu &Obu::operator=(Obu &&obu) { } Obu::~Obu() { + if (data_) { + free(data_); + data_ = nullptr; + } + size_ = 0; + if (payload_) { free(payload_); payload_ = nullptr; } payload_size_ = 0; - size_ = 0; + header_ = 0; extension_header_ = 0; } diff --git a/src/rtp/obu.h b/src/rtp/obu.h index 30b0ed5..0b15894 100644 --- a/src/rtp/obu.h +++ b/src/rtp/obu.h @@ -27,8 +27,9 @@ class Obu { uint8_t header_ = 0; uint8_t extension_header_ = 0; // undefined if (header & kXbit) == 0 uint8_t *payload_ = nullptr; - int size_ = 0; // size of the header and payload combined. int payload_size_ = 0; + uint8_t *data_ = nullptr; + int size_ = 0; // size of the header and payload combined. }; #endif \ No newline at end of file diff --git a/src/rtp/obu_parser.cpp b/src/rtp/obu_parser.cpp index 9ab2384..c3f982b 100644 --- a/src/rtp/obu_parser.cpp +++ b/src/rtp/obu_parser.cpp @@ -49,6 +49,8 @@ std::vector ParseObus(uint8_t* payload, int payload_size) { std::vector result; ByteBufferReader payload_reader(reinterpret_cast(payload), payload_size); + + int pos = 0; while (payload_reader.Length() > 0) { Obu obu; payload_reader.ReadUInt8(&obu.header_); @@ -87,6 +89,10 @@ std::vector ParseObus(uint8_t* payload, int payload_size) { payload_reader.Consume(payload_size); } obu.size_ += obu.payload_size_; + obu.data_ = (uint8_t*)malloc(obu.size_); + memcpy(obu.data_, payload + pos, obu.size_); + pos += obu.size_; + // Skip obus that shouldn't be transfered over rtp. int obu_type = ObuType(obu.header_); // if (obu_type != kObuTypeTemporalDelimiter && // diff --git a/src/rtp/rtp_codec.cpp b/src/rtp/rtp_codec.cpp index c9017ee..626a944 100644 --- a/src/rtp/rtp_codec.cpp +++ b/src/rtp/rtp_codec.cpp @@ -247,7 +247,7 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size, rtp_packet.SetAv1AggrHeader(0, 0, 1, 0); - rtp_packet.EncodeAv1(obus[i].payload_, obus[i].payload_size_); + rtp_packet.EncodeAv1(obus[i].data_, obus[i].payload_size_); packets.emplace_back(rtp_packet); } else { size_t last_packet_size = obus[i].payload_size_ % MAX_NALU_LEN; @@ -281,10 +281,10 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size, rtp_packet.SetAv1AggrHeader(z, y, w, n); if (index == packet_num - 1 && last_packet_size > 0) { - rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN, + rtp_packet.EncodeAv1(obus[i].data_ + index * MAX_NALU_LEN, last_packet_size); } else { - rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN, + rtp_packet.EncodeAv1(obus[i].data_ + index * MAX_NALU_LEN, MAX_NALU_LEN); } packets.emplace_back(rtp_packet); @@ -510,9 +510,9 @@ void RtpCodec::Encode(VideoFrameType frame_type, uint8_t* buffer, size_t size, } } else if (RtpPacket::PAYLOAD_TYPE::AV1 == payload_type_) { std::vector obus = ParseObus(buffer, size); - LOG_ERROR("Total size = [{}]", size); + // LOG_ERROR("Total size = [{}]", size); for (int i = 0; i < obus.size(); i++) { - LOG_ERROR("[{}] Obu size = [{}], Obu type [{}]", i, obus[i].size_, + LOG_ERROR("1 [{}] Obu size = [{}], Obu type [{}]", i, obus[i].size_, ObuTypeToString((OBU_TYPE)ObuType(obus[i].header_))); if (obus[i].size_ <= MAX_NALU_LEN) { RtpPacket rtp_packet; @@ -539,13 +539,13 @@ void RtpCodec::Encode(VideoFrameType frame_type, uint8_t* buffer, size_t size, } rtp_packet.SetAv1AggrHeader(0, 0, 1, 0); - - rtp_packet.EncodeAv1(obus[i].payload_, obus[i].payload_size_); + rtp_packet.EncodeAv1(obus[i].data_, obus[i].size_); + LOG_ERROR("enc payload size = {}", rtp_packet.PayloadSize()); packets.emplace_back(rtp_packet); } else { - size_t last_packet_size = obus[i].payload_size_ % MAX_NALU_LEN; + size_t last_packet_size = obus[i].size_ % MAX_NALU_LEN; size_t packet_num = - obus[i].payload_size_ / MAX_NALU_LEN + (last_packet_size ? 1 : 0); + obus[i].size_ / MAX_NALU_LEN + (last_packet_size ? 1 : 0); timestamp_ = std::chrono::high_resolution_clock::now() .time_since_epoch() .count(); @@ -577,12 +577,14 @@ void RtpCodec::Encode(VideoFrameType frame_type, uint8_t* buffer, size_t size, rtp_packet.SetAv1AggrHeader(z, y, w, n); if (index == packet_num - 1 && last_packet_size > 0) { - rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN, + rtp_packet.EncodeAv1(obus[i].data_ + index * MAX_NALU_LEN, last_packet_size); } else { - rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN, + rtp_packet.EncodeAv1(obus[i].data_ + index * MAX_NALU_LEN, MAX_NALU_LEN); } + + LOG_ERROR("enc payload size = {}", rtp_packet.PayloadSize()); packets.emplace_back(rtp_packet); } } diff --git a/src/rtp/rtp_packet.cpp b/src/rtp/rtp_packet.cpp index 00db11a..0f2803d 100644 --- a/src/rtp/rtp_packet.cpp +++ b/src/rtp/rtp_packet.cpp @@ -755,13 +755,18 @@ size_t RtpPacket::DecodeAv1(uint8_t *payload) { extension_data_ = buffer_ + 16 + extension_offset; } - uint32_t payload_offset = + uint32_t aggr_header_offset = (has_extension_ ? extension_len_ : 0) + extension_offset; + av1_aggr_header_ = buffer_[12 + aggr_header_offset]; + + uint32_t payload_offset = aggr_header_offset; + payload_size_ = size_ - (13 + payload_offset); payload_ = buffer_ + 13 + payload_offset; if (payload) { memcpy(payload, payload_, payload_size_); } + return payload_size_; } \ No newline at end of file diff --git a/src/rtp/rtp_packet.h b/src/rtp/rtp_packet.h index ce472f6..7033575 100644 --- a/src/rtp/rtp_packet.h +++ b/src/rtp/rtp_packet.h @@ -311,6 +311,13 @@ class RtpPacket { uint8_t FecSourceSymbolNum() { return fec_source_symbol_num_; } + void GetAv1AggrHeader(int &z, int &y, int &w, int &n) { + z = av1_aggr_header_ >> 7; + y = av1_aggr_header_ >> 6 & 0x01; + w = av1_aggr_header_ >> 4 & 0x03; + n = av1_aggr_header_ >> 3 & 0x01; + } + // Payload const uint8_t *Payload() { ParseRtpData(); diff --git a/src/rtp/rtp_video_receiver.cpp b/src/rtp/rtp_video_receiver.cpp index 8b0fda3..968a7fa 100644 --- a/src/rtp/rtp_video_receiver.cpp +++ b/src/rtp/rtp_video_receiver.cpp @@ -1,6 +1,7 @@ #include "rtp_video_receiver.h" #include "log.h" +#include "obu_parser.h" #define NV12_BUFFER_SIZE (1280 * 720 * 3 / 2) #define RTCP_RR_INTERVAL 1000 @@ -28,12 +29,14 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) { RtcpReportBlock report; // auto duration = std::chrono::system_clock::now().time_since_epoch(); - // auto seconds = std::chrono::duration_cast(duration); - // uint32_t seconds_u32 = static_cast( + // auto seconds = + // std::chrono::duration_cast(duration); uint32_t + // seconds_u32 = static_cast( // std::chrono::duration_cast(duration).count()); // uint32_t fraction_u32 = static_cast( - // std::chrono::duration_cast(duration - seconds) + // std::chrono::duration_cast(duration - + // seconds) // .count()); report.source_ssrc = 0x00; @@ -50,7 +53,14 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) { // SendRtcpRR(rtcp_rr); } + if (rtp_packet.PayloadType() == RtpPacket::PAYLOAD_TYPE::AV1) { + ProcessAV1RtpPacket(rtp_packet); + } else { + ProcessH264RtpPacket(rtp_packet); + } +} +void RtpVideoReceiver::ProcessH264RtpPacket(RtpPacket& rtp_packet) { if (!fec_enable_) { if (RtpPacket::PAYLOAD_TYPE::H264 == rtp_packet.PayloadType()) { if (RtpPacket::NAL_UNIT_TYPE::NALU == rtp_packet.NalUnitType()) { @@ -58,7 +68,7 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) { VideoFrame(rtp_packet.Payload(), rtp_packet.PayloadSize())); } else if (RtpPacket::NAL_UNIT_TYPE::FU_A == rtp_packet.NalUnitType()) { incomplete_frame_list_[rtp_packet.SequenceNumber()] = rtp_packet; - bool complete = CheckIsFrameCompleted(rtp_packet); + bool complete = CheckIsH264FrameCompleted(rtp_packet); } } } else { @@ -68,7 +78,7 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) { VideoFrame(rtp_packet.Payload(), rtp_packet.PayloadSize())); } else if (RtpPacket::NAL_UNIT_TYPE::FU_A == rtp_packet.NalUnitType()) { incomplete_frame_list_[rtp_packet.SequenceNumber()] = rtp_packet; - bool complete = CheckIsFrameCompleted(rtp_packet); + bool complete = CheckIsH264FrameCompleted(rtp_packet); } } else if (RtpPacket::PAYLOAD_TYPE::H264_FEC_SOURCE == rtp_packet.PayloadType()) { @@ -163,7 +173,30 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) { } } -bool RtpVideoReceiver::CheckIsFrameCompleted(RtpPacket& rtp_packet) { +void RtpVideoReceiver::ProcessAV1RtpPacket(RtpPacket& rtp_packet) { + LOG_ERROR("recv payload size = {}, sequence_number_ = {}", + rtp_packet.PayloadSize(), rtp_packet.SequenceNumber()); + + int z, y, w, n; + rtp_packet.GetAv1AggrHeader(z, y, w, n); + LOG_ERROR("z = {}, y = {}, w = {}, n = {}", z, y, w, n); + + if (z == 0) { + } + + if (y == 0) { + } + + std::vector obus = + ParseObus((uint8_t*)rtp_packet.Payload(), rtp_packet.PayloadSize()); + for (int i = 0; i < obus.size(); i++) { + LOG_ERROR("2 [{}|{}] Obu size = [{}], Obu type [{}]", i, obus.size(), + obus[i].size_, + ObuTypeToString((OBU_TYPE)ObuType(obus[i].header_))); + } +} + +bool RtpVideoReceiver::CheckIsH264FrameCompleted(RtpPacket& rtp_packet) { if (rtp_packet.FuAEnd()) { uint16_t end_seq = rtp_packet.SequenceNumber(); if (incomplete_frame_list_.size() == end_seq) { diff --git a/src/rtp/rtp_video_receiver.h b/src/rtp/rtp_video_receiver.h index 36bb4b6..2ad4c2a 100644 --- a/src/rtp/rtp_video_receiver.h +++ b/src/rtp/rtp_video_receiver.h @@ -30,7 +30,13 @@ class RtpVideoReceiver : public ThreadBase { } private: - bool CheckIsFrameCompleted(RtpPacket& rtp_packet); + void ProcessAV1RtpPacket(RtpPacket& rtp_packet); + + private: + void ProcessH264RtpPacket(RtpPacket& rtp_packet); + bool CheckIsH264FrameCompleted(RtpPacket& rtp_packet); + + private: bool CheckIsTimeSendRR(); int SendRtcpRR(RtcpReceiverReport& rtcp_rr); diff --git a/src/transmission/ice_transmission.cpp b/src/transmission/ice_transmission.cpp index 4e39e18..bb98630 100644 --- a/src/transmission/ice_transmission.cpp +++ b/src/transmission/ice_transmission.cpp @@ -366,7 +366,8 @@ uint8_t IceTransmission::CheckIsVideoPacket(const char *buffer, size_t size) { uint8_t pt = buffer[1] & 0x7F; if (RtpPacket::PAYLOAD_TYPE::H264 == pt || RtpPacket::PAYLOAD_TYPE::H264_FEC_SOURCE == pt || - RtpPacket::PAYLOAD_TYPE::H264_FEC_REPAIR == pt) { + RtpPacket::PAYLOAD_TYPE::H264_FEC_REPAIR == pt || + RtpPacket::PAYLOAD_TYPE::AV1 == pt) { return pt; } else { return 0;