YangWX/crossdesk
1
0
Fork 0
mirror of https://github.com/kunkundi/crossdesk.git synced 2025-10-26 12:15:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix error during encoding obu packet into rtp packet

Browse Source
This commit is contained in:
dijunkun
2024-05-06 17:21:33 +08:00
parent 182c7dbec6
commit 95da7ff52d
10 changed files with 120 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/media/video/encode/aom/aom_av1_encoder.cpp
View File

@@ -342,7 +342,7 @@ int AomAv1Encoder::Encode(const uint8_t *pData, int nSize,
int qp = -1;
SET_ENCODER_PARAM_OR_RETURN_ERROR(AOME_GET_LAST_QUANTIZER, &qp);
LOG_INFO("Encoded frame qp = {}", qp);
// LOG_INFO("Encoded frame qp = {}", qp);
if (on_encoded_image) {
on_encoded_image((char *)encoded_frame_, encoded_frame_size_,

60
src/rtp/obu.cpp
View File

@@ -7,59 +7,67 @@
Obu::Obu() {}
Obu::Obu(const Obu &obu) {
if (obu.size_ > 0) {
data_ = (uint8_t *)malloc(obu.size_);
memcpy(data_, obu.data_, obu.size_);
size_ = obu.size_;
}
if (obu.payload_size_ > 0) {
payload_ = (uint8_t *)malloc(obu.payload_size_);
if (NULL == payload_) {
LOG_ERROR("Malloc failed");
} else {
memcpy(payload_, obu.payload_, obu.payload_size_);
}
memcpy(payload_, obu.payload_, obu.payload_size_);
payload_size_ = obu.payload_size_;
size_ = obu.size_;
header_ = obu.header_;
extension_header_ = obu.extension_header_;
} else {
size_ = obu.size_;
header_ = obu.header_;
extension_header_ = obu.extension_header_;
}
header_ = obu.header_;
extension_header_ = obu.extension_header_;
}
Obu::Obu(Obu &&obu)
: payload_((uint8_t *)std::move(obu.payload_)),
payload_size_(obu.payload_size_),
: data_(std::move(obu.data_)),
size_(obu.size_),
payload_((uint8_t *)std::move(obu.payload_)),
payload_size_(obu.payload_size_),
header_(obu.header_),
extension_header_(obu.extension_header_) {
obu.data_ = nullptr;
obu.size_ = 0;
obu.payload_ = nullptr;
obu.payload_size_ = 0;
}
Obu &Obu::operator=(const Obu &obu) {
if (&obu != this) {
if (obu.size_ > 0) {
data_ = (uint8_t *)realloc(data_, obu.size_);
memcpy(data_, obu.data_, obu.size_);
size_ = obu.size_;
}
if (obu.payload_size_ > 0) {
payload_ = (uint8_t *)realloc(payload_, obu.payload_size_);
memcpy(payload_, obu.payload_, obu.payload_size_);
payload_size_ = obu.payload_size_;
size_ = obu.size_;
header_ = obu.header_;
extension_header_ = obu.extension_header_;
} else {
header_ = obu.header_;
extension_header_ = obu.extension_header_;
}
header_ = obu.header_;
extension_header_ = obu.extension_header_;
}
return *this;
}
Obu &Obu::operator=(Obu &&obu) {
if (&obu != this) {
data_ = std::move(obu.data_);
obu.data_ = nullptr;
size_ = obu.size_;
obu.size_ = 0;
payload_ = std::move(obu.payload_);
obu.payload_ = nullptr;
payload_size_ = obu.payload_size_;
obu.payload_size_ = 0;
size_ = obu.size_;
obu.size_ = 0;
header_ = obu.header_;
obu.header_ = 0;
extension_header_ = obu.extension_header_;
@@ -69,12 +77,18 @@ Obu &Obu::operator=(Obu &&obu) {
}
Obu::~Obu() {
if (data_) {
free(data_);
data_ = nullptr;
}
size_ = 0;
if (payload_) {
free(payload_);
payload_ = nullptr;
}
payload_size_ = 0;
size_ = 0;
header_ = 0;
extension_header_ = 0;
}

3
src/rtp/obu.h
View File

@@ -27,8 +27,9 @@ class Obu {
uint8_t header_ = 0;
uint8_t extension_header_ = 0; // undefined if (header & kXbit) == 0
uint8_t *payload_ = nullptr;
int size_ = 0; // size of the header and payload combined.
int payload_size_ = 0;
uint8_t *data_ = nullptr;
int size_ = 0; // size of the header and payload combined.
};
#endif

6
src/rtp/obu_parser.cpp
View File

@@ -49,6 +49,8 @@ std::vector<Obu> ParseObus(uint8_t* payload, int payload_size) {
std::vector<Obu> result;
ByteBufferReader payload_reader(reinterpret_cast<const char*>(payload),
payload_size);
int pos = 0;
while (payload_reader.Length() > 0) {
Obu obu;
payload_reader.ReadUInt8(&obu.header_);
@@ -87,6 +89,10 @@ std::vector<Obu> ParseObus(uint8_t* payload, int payload_size) {
payload_reader.Consume(payload_size);
}
obu.size_ += obu.payload_size_;
obu.data_ = (uint8_t*)malloc(obu.size_);
memcpy(obu.data_, payload + pos, obu.size_);
pos += obu.size_;
// Skip obus that shouldn't be transfered over rtp.
int obu_type = ObuType(obu.header_);
// if (obu_type != kObuTypeTemporalDelimiter && //

24
src/rtp/rtp_codec.cpp
View File

@@ -247,7 +247,7 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size,
rtp_packet.SetAv1AggrHeader(0, 0, 1, 0);
rtp_packet.EncodeAv1(obus[i].payload_, obus[i].payload_size_);
rtp_packet.EncodeAv1(obus[i].data_, obus[i].payload_size_);
packets.emplace_back(rtp_packet);
} else {
size_t last_packet_size = obus[i].payload_size_ % MAX_NALU_LEN;
@@ -281,10 +281,10 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size,
rtp_packet.SetAv1AggrHeader(z, y, w, n);
if (index == packet_num - 1 && last_packet_size > 0) {
rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN,
rtp_packet.EncodeAv1(obus[i].data_ + index * MAX_NALU_LEN,
last_packet_size);
} else {
rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN,
rtp_packet.EncodeAv1(obus[i].data_ + index * MAX_NALU_LEN,
MAX_NALU_LEN);
}
packets.emplace_back(rtp_packet);
@@ -510,9 +510,9 @@ void RtpCodec::Encode(VideoFrameType frame_type, uint8_t* buffer, size_t size,
}
} else if (RtpPacket::PAYLOAD_TYPE::AV1 == payload_type_) {
std::vector<Obu> obus = ParseObus(buffer, size);
LOG_ERROR("Total size = [{}]", size);
// LOG_ERROR("Total size = [{}]", size);
for (int i = 0; i < obus.size(); i++) {
LOG_ERROR("[{}] Obu size = [{}], Obu type [{}]", i, obus[i].size_,
LOG_ERROR("1 [{}] Obu size = [{}], Obu type [{}]", i, obus[i].size_,
ObuTypeToString((OBU_TYPE)ObuType(obus[i].header_)));
if (obus[i].size_ <= MAX_NALU_LEN) {
RtpPacket rtp_packet;
@@ -539,13 +539,13 @@ void RtpCodec::Encode(VideoFrameType frame_type, uint8_t* buffer, size_t size,
}
rtp_packet.SetAv1AggrHeader(0, 0, 1, 0);
rtp_packet.EncodeAv1(obus[i].payload_, obus[i].payload_size_);
rtp_packet.EncodeAv1(obus[i].data_, obus[i].size_);
LOG_ERROR("enc payload size = {}", rtp_packet.PayloadSize());
packets.emplace_back(rtp_packet);
} else {
size_t last_packet_size = obus[i].payload_size_ % MAX_NALU_LEN;
size_t last_packet_size = obus[i].size_ % MAX_NALU_LEN;
size_t packet_num =
obus[i].payload_size_ / MAX_NALU_LEN + (last_packet_size ? 1 : 0);
obus[i].size_ / MAX_NALU_LEN + (last_packet_size ? 1 : 0);
timestamp_ = std::chrono::high_resolution_clock::now()
.time_since_epoch()
.count();
@@ -577,12 +577,14 @@ void RtpCodec::Encode(VideoFrameType frame_type, uint8_t* buffer, size_t size,
rtp_packet.SetAv1AggrHeader(z, y, w, n);
if (index == packet_num - 1 && last_packet_size > 0) {
rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN,
rtp_packet.EncodeAv1(obus[i].data_ + index * MAX_NALU_LEN,
last_packet_size);
} else {
rtp_packet.EncodeAv1(obus[i].payload_ + index * MAX_NALU_LEN,
rtp_packet.EncodeAv1(obus[i].data_ + index * MAX_NALU_LEN,
MAX_NALU_LEN);
}
LOG_ERROR("enc payload size = {}", rtp_packet.PayloadSize());
packets.emplace_back(rtp_packet);
}
}

7
src/rtp/rtp_packet.cpp
View File

@@ -755,13 +755,18 @@ size_t RtpPacket::DecodeAv1(uint8_t *payload) {
extension_data_ = buffer_ + 16 + extension_offset;
}
uint32_t payload_offset =
uint32_t aggr_header_offset =
(has_extension_ ? extension_len_ : 0) + extension_offset;
av1_aggr_header_ = buffer_[12 + aggr_header_offset];
uint32_t payload_offset = aggr_header_offset;
payload_size_ = size_ - (13 + payload_offset);
payload_ = buffer_ + 13 + payload_offset;
if (payload) {
memcpy(payload, payload_, payload_size_);
}
return payload_size_;
}

7
src/rtp/rtp_packet.h
View File

@@ -311,6 +311,13 @@ class RtpPacket {
uint8_t FecSourceSymbolNum() { return fec_source_symbol_num_; }
void GetAv1AggrHeader(int &z, int &y, int &w, int &n) {
z = av1_aggr_header_ >> 7;
y = av1_aggr_header_ >> 6 & 0x01;
w = av1_aggr_header_ >> 4 & 0x03;
n = av1_aggr_header_ >> 3 & 0x01;
}
// Payload
const uint8_t *Payload() {
ParseRtpData();

45
src/rtp/rtp_video_receiver.cpp
View File

@@ -1,6 +1,7 @@
#include "rtp_video_receiver.h"
#include "log.h"
#include "obu_parser.h"
#define NV12_BUFFER_SIZE (1280 * 720 * 3 / 2)
#define RTCP_RR_INTERVAL 1000
@@ -28,12 +29,14 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
RtcpReportBlock report;
// auto duration = std::chrono::system_clock::now().time_since_epoch();
// auto seconds = std::chrono::duration_cast<std::chrono::seconds>(duration);
// uint32_t seconds_u32 = static_cast<uint32_t>(
// auto seconds =
// std::chrono::duration_cast<std::chrono::seconds>(duration); uint32_t
// seconds_u32 = static_cast<uint32_t>(
// std::chrono::duration_cast<std::chrono::seconds>(duration).count());
// uint32_t fraction_u32 = static_cast<uint32_t>(
// std::chrono::duration_cast<std::chrono::nanoseconds>(duration - seconds)
// std::chrono::duration_cast<std::chrono::nanoseconds>(duration -
// seconds)
// .count());
report.source_ssrc = 0x00;
@@ -50,7 +53,14 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
// SendRtcpRR(rtcp_rr);
}
if (rtp_packet.PayloadType() == RtpPacket::PAYLOAD_TYPE::AV1) {
ProcessAV1RtpPacket(rtp_packet);
} else {
ProcessH264RtpPacket(rtp_packet);
}
}
void RtpVideoReceiver::ProcessH264RtpPacket(RtpPacket& rtp_packet) {
if (!fec_enable_) {
if (RtpPacket::PAYLOAD_TYPE::H264 == rtp_packet.PayloadType()) {
if (RtpPacket::NAL_UNIT_TYPE::NALU == rtp_packet.NalUnitType()) {
@@ -58,7 +68,7 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
VideoFrame(rtp_packet.Payload(), rtp_packet.PayloadSize()));
} else if (RtpPacket::NAL_UNIT_TYPE::FU_A == rtp_packet.NalUnitType()) {
incomplete_frame_list_[rtp_packet.SequenceNumber()] = rtp_packet;
bool complete = CheckIsFrameCompleted(rtp_packet);
bool complete = CheckIsH264FrameCompleted(rtp_packet);
}
}
} else {
@@ -68,7 +78,7 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
VideoFrame(rtp_packet.Payload(), rtp_packet.PayloadSize()));
} else if (RtpPacket::NAL_UNIT_TYPE::FU_A == rtp_packet.NalUnitType()) {
incomplete_frame_list_[rtp_packet.SequenceNumber()] = rtp_packet;
bool complete = CheckIsFrameCompleted(rtp_packet);
bool complete = CheckIsH264FrameCompleted(rtp_packet);
}
} else if (RtpPacket::PAYLOAD_TYPE::H264_FEC_SOURCE ==
rtp_packet.PayloadType()) {
@@ -163,7 +173,30 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
}
}
bool RtpVideoReceiver::CheckIsFrameCompleted(RtpPacket& rtp_packet) {
void RtpVideoReceiver::ProcessAV1RtpPacket(RtpPacket& rtp_packet) {
LOG_ERROR("recv payload size = {}, sequence_number_ = {}",
rtp_packet.PayloadSize(), rtp_packet.SequenceNumber());
int z, y, w, n;
rtp_packet.GetAv1AggrHeader(z, y, w, n);
LOG_ERROR("z = {}, y = {}, w = {}, n = {}", z, y, w, n);
if (z == 0) {
}
if (y == 0) {
}
std::vector<Obu> obus =
ParseObus((uint8_t*)rtp_packet.Payload(), rtp_packet.PayloadSize());
for (int i = 0; i < obus.size(); i++) {
LOG_ERROR("2 [{}|{}] Obu size = [{}], Obu type [{}]", i, obus.size(),
obus[i].size_,
ObuTypeToString((OBU_TYPE)ObuType(obus[i].header_)));
}
}
bool RtpVideoReceiver::CheckIsH264FrameCompleted(RtpPacket& rtp_packet) {
if (rtp_packet.FuAEnd()) {
uint16_t end_seq = rtp_packet.SequenceNumber();
if (incomplete_frame_list_.size() == end_seq) {

8
src/rtp/rtp_video_receiver.h
View File

@@ -30,7 +30,13 @@ class RtpVideoReceiver : public ThreadBase {
}
private:
bool CheckIsFrameCompleted(RtpPacket& rtp_packet);
void ProcessAV1RtpPacket(RtpPacket& rtp_packet);
private:
void ProcessH264RtpPacket(RtpPacket& rtp_packet);
bool CheckIsH264FrameCompleted(RtpPacket& rtp_packet);
private:
bool CheckIsTimeSendRR();
int SendRtcpRR(RtcpReceiverReport& rtcp_rr);

3
src/transmission/ice_transmission.cpp
View File

@@ -366,7 +366,8 @@ uint8_t IceTransmission::CheckIsVideoPacket(const char *buffer, size_t size) {
uint8_t pt = buffer[1] & 0x7F;
if (RtpPacket::PAYLOAD_TYPE::H264 == pt ||
RtpPacket::PAYLOAD_TYPE::H264_FEC_SOURCE == pt ||
RtpPacket::PAYLOAD_TYPE::H264_FEC_REPAIR == pt) {
RtpPacket::PAYLOAD_TYPE::H264_FEC_REPAIR == pt ||
RtpPacket::PAYLOAD_TYPE::AV1 == pt) {
return pt;
} else {
return 0;