YangWX/crossdesk
1
0
Fork 0
mirror of https://github.com/kunkundi/crossdesk.git synced 2025-10-27 04:35:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

[fix] fix h264 rtp packet packetization and depacketization

Browse Source
This commit is contained in:
dijunkun
2025-01-23 17:28:17 +08:00
parent cd349cd98d
commit 7b839ab773
50 changed files with 871 additions and 422 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/channel/audio_channel_receive.cpp
View File

@@ -14,10 +14,8 @@ AudioChannelReceive::AudioChannelReceive(
AudioChannelReceive::~AudioChannelReceive() {}
void AudioChannelReceive::Initialize(RtpPacket::PAYLOAD_TYPE payload_type) {
audio_rtp_codec_ = std::make_unique<RtpCodec>(payload_type);
void AudioChannelReceive::Initialize(rtp::PAYLOAD_TYPE payload_type) {
rtp_audio_receiver_ = std::make_unique<RtpAudioReceiver>(ice_io_statistics_);
rtp_audio_receiver_->SetOnReceiveData(
[this](const char *data, size_t size) -> void {
if (on_receive_audio_) {
@@ -54,7 +52,9 @@ int AudioChannelReceive::OnReceiveRtpPacket(const char *data, size_t size) {
}
if (rtp_audio_receiver_) {
rtp_audio_receiver_->InsertRtpPacket(RtpPacket((uint8_t *)data, size));
RtpPacket rtp_packet;
rtp_packet.Build((uint8_t *)data, (uint32_t)size);
rtp_audio_receiver_->InsertRtpPacket(rtp_packet);
}
return 0;

4
src/channel/audio_channel_receive.h
View File

@@ -9,7 +9,6 @@
#include "ice_agent.h"
#include "rtp_audio_receiver.h"
#include "rtp_codec.h"
class AudioChannelReceive {
public:
@@ -21,14 +20,13 @@ class AudioChannelReceive {
~AudioChannelReceive();
public:
void Initialize(RtpPacket::PAYLOAD_TYPE payload_type);
void Initialize(rtp::PAYLOAD_TYPE payload_type);
void Destroy();
int OnReceiveRtpPacket(const char *data, size_t size);
private:
std::shared_ptr<IceAgent> ice_agent_ = nullptr;
std::shared_ptr<IOStatistics> ice_io_statistics_ = nullptr;
std::unique_ptr<RtpCodec> audio_rtp_codec_ = nullptr;
std::unique_ptr<RtpAudioReceiver> rtp_audio_receiver_ = nullptr;
std::function<void(const char *, size_t)> on_receive_audio_ = nullptr;
};

13
src/channel/audio_channel_send.cpp
View File

@@ -11,9 +11,8 @@ AudioChannelSend::AudioChannelSend(
std::shared_ptr<IOStatistics> ice_io_statistics)
: ice_agent_(ice_agent), ice_io_statistics_(ice_io_statistics) {}
void AudioChannelSend::Initialize(RtpPacket::PAYLOAD_TYPE payload_type) {
audio_rtp_codec_ = std::make_unique<RtpCodec>(payload_type);
void AudioChannelSend::Initialize(rtp::PAYLOAD_TYPE payload_type) {
rtp_packetizer_ = RtpPacketizer::Create(payload_type);
rtp_audio_sender_ = std::make_unique<RtpAudioSender>(ice_io_statistics_);
rtp_audio_sender_->SetSendDataFunc(
[this](const char *data, size_t size) -> int {
@@ -45,10 +44,10 @@ void AudioChannelSend::Destroy() {
}
int AudioChannelSend::SendAudio(char *data, size_t size) {
if (audio_rtp_codec_) {
std::vector<RtpPacket> packets;
audio_rtp_codec_->Encode((uint8_t *)data, (uint32_t)size, packets);
rtp_audio_sender_->Enqueue(packets);
if (rtp_audio_sender_ && rtp_packetizer_) {
std::vector<RtpPacket> rtp_packets =
rtp_packetizer_->Build((uint8_t *)data, (uint32_t)size);
rtp_audio_sender_->Enqueue(rtp_packets);
}
return 0;

6
src/channel/audio_channel_send.h
View File

@@ -9,7 +9,7 @@
#include "ice_agent.h"
#include "rtp_audio_sender.h"
#include "rtp_codec.h"
#include "rtp_packetizer.h"
class AudioChannelSend {
public:
@@ -19,14 +19,14 @@ class AudioChannelSend {
~AudioChannelSend();
public:
void Initialize(RtpPacket::PAYLOAD_TYPE payload_type);
void Initialize(rtp::PAYLOAD_TYPE payload_type);
void Destroy();
int SendAudio(char *data, size_t size);
private:
std::shared_ptr<IceAgent> ice_agent_ = nullptr;
std::shared_ptr<IOStatistics> ice_io_statistics_ = nullptr;
std::unique_ptr<RtpCodec> audio_rtp_codec_ = nullptr;
std::unique_ptr<RtpPacketizer> rtp_packetizer_ = nullptr;
std::unique_ptr<RtpAudioSender> rtp_audio_sender_ = nullptr;
};

9
src/channel/data_channel_receive.cpp
View File

@@ -14,11 +14,8 @@ DataChannelReceive::DataChannelReceive(
DataChannelReceive::~DataChannelReceive() {}
void DataChannelReceive::Initialize(RtpPacket::PAYLOAD_TYPE payload_type) {
data_rtp_codec_ = std::make_unique<RtpCodec>(payload_type);
void DataChannelReceive::Initialize(rtp::PAYLOAD_TYPE payload_type) {
rtp_data_receiver_ = std::make_unique<RtpDataReceiver>(ice_io_statistics_);
rtp_data_receiver_->SetOnReceiveData(
[this](const char *data, size_t size) -> void {
if (on_receive_data_) {
@@ -55,7 +52,9 @@ int DataChannelReceive::OnReceiveRtpPacket(const char *data, size_t size) {
}
if (rtp_data_receiver_) {
rtp_data_receiver_->InsertRtpPacket(RtpPacket((uint8_t *)data, size));
RtpPacket rtp_packet;
rtp_packet.Build((uint8_t *)data, (uint32_t)size);
rtp_data_receiver_->InsertRtpPacket(rtp_packet);
}
return -1;

4
src/channel/data_channel_receive.h
View File

@@ -8,7 +8,6 @@
#define _DATA_CHANNEL_RECEIVE_H_
#include "ice_agent.h"
#include "rtp_codec.h"
#include "rtp_data_receiver.h"
class DataChannelReceive {
@@ -20,14 +19,13 @@ class DataChannelReceive {
~DataChannelReceive();
public:
void Initialize(RtpPacket::PAYLOAD_TYPE payload_type);
void Initialize(rtp::PAYLOAD_TYPE payload_type);
void Destroy();
int OnReceiveRtpPacket(const char *data, size_t size);
private:
std::shared_ptr<IceAgent> ice_agent_ = nullptr;
std::shared_ptr<IOStatistics> ice_io_statistics_ = nullptr;
std::unique_ptr<RtpCodec> data_rtp_codec_ = nullptr;
std::unique_ptr<RtpDataReceiver> rtp_data_receiver_ = nullptr;
std::function<void(const char *, size_t)> on_receive_data_ = nullptr;
};

16
src/channel/data_channel_send.cpp
View File

@@ -11,9 +11,8 @@ DataChannelSend::DataChannelSend(
std::shared_ptr<IOStatistics> ice_io_statistics)
: ice_agent_(ice_agent), ice_io_statistics_(ice_io_statistics) {}
void DataChannelSend::Initialize(RtpPacket::PAYLOAD_TYPE payload_type) {
data_rtp_codec_ = std::make_unique<RtpCodec>(payload_type);
void DataChannelSend::Initialize(rtp::PAYLOAD_TYPE payload_type) {
rtp_packetizer_ = RtpPacketizer::Create(payload_type);
rtp_data_sender_ = std::make_unique<RtpDataSender>(ice_io_statistics_);
rtp_data_sender_->SetSendDataFunc(
[this](const char *data, size_t size) -> int {
@@ -45,13 +44,10 @@ void DataChannelSend::Destroy() {
}
int DataChannelSend::SendData(const char *data, size_t size) {
std::vector<RtpPacket> packets;
if (rtp_data_sender_) {
if (data_rtp_codec_) {
data_rtp_codec_->Encode((uint8_t *)data, (uint32_t)size, packets);
rtp_data_sender_->Enqueue(packets);
}
if (rtp_data_sender_ && rtp_packetizer_) {
std::vector<RtpPacket> rtp_packets =
rtp_packetizer_->Build((uint8_t *)data, (uint32_t)size);
rtp_data_sender_->Enqueue(rtp_packets);
}
return 0;

6
src/channel/data_channel_send.h
View File

@@ -8,8 +8,8 @@
#define _DATA_CHANNEL_SEND_H_
#include "ice_agent.h"
#include "rtp_codec.h"
#include "rtp_data_sender.h"
#include "rtp_packetizer.h"
class DataChannelSend {
public:
@@ -19,14 +19,14 @@ class DataChannelSend {
~DataChannelSend();
public:
void Initialize(RtpPacket::PAYLOAD_TYPE payload_type);
void Initialize(rtp::PAYLOAD_TYPE payload_type);
void Destroy();
int SendData(const char *data, size_t size);
private:
std::shared_ptr<IceAgent> ice_agent_ = nullptr;
std::shared_ptr<IOStatistics> ice_io_statistics_ = nullptr;
std::unique_ptr<RtpCodec> data_rtp_codec_ = nullptr;
std::unique_ptr<RtpPacketizer> rtp_packetizer_ = nullptr;
std::unique_ptr<RtpDataSender> rtp_data_sender_ = nullptr;
};

2
src/channel/rtp_channel/rtp_audio_receiver.h
View File

@@ -11,7 +11,7 @@
#include "io_statistics.h"
#include "rtcp_receiver_report.h"
#include "rtp_codec.h"
#include "rtp_packet.h"
#include "rtp_statistics.h"
class RtpAudioReceiver {

4
src/channel/rtp_channel/rtp_audio_sender.cpp
View File

@@ -39,8 +39,8 @@ int RtpAudioSender::SendRtpPacket(RtpPacket& rtp_packet) {
return -1;
}
if (0 !=
data_send_func_((const char*)rtp_packet.Buffer(), rtp_packet.Size())) {
if (0 != data_send_func_((const char*)rtp_packet.Buffer().data(),
rtp_packet.Size())) {
LOG_ERROR("Send rtp packet failed");
return -1;
}

2
src/channel/rtp_channel/rtp_data_receiver.h
View File

@@ -5,7 +5,7 @@
#include "io_statistics.h"
#include "rtcp_receiver_report.h"
#include "rtp_codec.h"
#include "rtp_packet.h"
#include "rtp_statistics.h"
class RtpDataReceiver {

4
src/channel/rtp_channel/rtp_data_sender.cpp
View File

@@ -39,8 +39,8 @@ int RtpDataSender::SendRtpPacket(RtpPacket& rtp_packet) {
return -1;
}
if (0 !=
data_send_func_((const char*)rtp_packet.Buffer(), rtp_packet.Size())) {
if (0 != data_send_func_((const char*)rtp_packet.Buffer().data(),
rtp_packet.Size())) {
LOG_ERROR("Send rtp packet failed");
return -1;
}

295
src/channel/rtp_channel/rtp_video_receiver.cpp
View File

@@ -57,7 +57,7 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
}
webrtc::RtpPacketReceived rtp_packet_received;
rtp_packet_received.Build(rtp_packet.Buffer(), rtp_packet.Size());
rtp_packet_received.Build(rtp_packet.Buffer().data(), rtp_packet.Size());
rtp_packet_received.set_arrival_time(clock_->CurrentTime());
rtp_packet_received.set_ecn(EcnMarking::kEct0);
@@ -109,137 +109,152 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
// SendRtcpRR(rtcp_rr);
}
if (rtp_packet.PayloadType() == RtpPacket::PAYLOAD_TYPE::AV1) {
ProcessAv1RtpPacket(rtp_packet);
if (rtp_packet.PayloadType() == rtp::PAYLOAD_TYPE::AV1) {
RtpPacketAv1 rtp_packet_av1;
rtp_packet_av1.Build(rtp_packet.Buffer().data(), rtp_packet.Size());
rtp_packet_av1.GetFrameHeaderInfo();
ProcessAv1RtpPacket(rtp_packet_av1);
} else {
ProcessH264RtpPacket(rtp_packet);
RtpPacketH264 rtp_packet_h264;
if (rtp_packet_h264.Build(rtp_packet.Buffer().data(), rtp_packet.Size())) {
rtp_packet_h264.GetFrameHeaderInfo();
ProcessH264RtpPacket(rtp_packet_h264);
} else {
LOG_ERROR("Invalid h264 rtp packet");
}
}
}
void RtpVideoReceiver::ProcessH264RtpPacket(RtpPacket& rtp_packet) {
void RtpVideoReceiver::ProcessH264RtpPacket(RtpPacketH264& rtp_packet_h264) {
if (!fec_enable_) {
if (RtpPacket::PAYLOAD_TYPE::H264 == rtp_packet.PayloadType()) {
if (RtpPacket::NAL_UNIT_TYPE::NALU == rtp_packet.NalUnitType()) {
compelete_video_frame_queue_.push(
VideoFrame(rtp_packet.Payload(), rtp_packet.PayloadSize()));
} else if (RtpPacket::NAL_UNIT_TYPE::FU_A == rtp_packet.NalUnitType()) {
incomplete_frame_list_[rtp_packet.SequenceNumber()] = rtp_packet;
bool complete = CheckIsH264FrameCompleted(rtp_packet);
if (rtp::PAYLOAD_TYPE::H264 == rtp_packet_h264.PayloadType()) {
rtp::NAL_UNIT_TYPE nalu_type = rtp_packet_h264.NalUnitType();
if (rtp::NAL_UNIT_TYPE::NALU == nalu_type) {
compelete_video_frame_queue_.push(VideoFrame(
rtp_packet_h264.Payload(), rtp_packet_h264.PayloadSize()));
} else if (rtp::NAL_UNIT_TYPE::FU_A == nalu_type) {
incomplete_h264_frame_list_[rtp_packet_h264.SequenceNumber()] =
rtp_packet_h264;
bool complete = CheckIsH264FrameCompleted(rtp_packet_h264);
if (!complete) {
}
}
}
} else {
if (RtpPacket::PAYLOAD_TYPE::H264 == rtp_packet.PayloadType()) {
if (RtpPacket::NAL_UNIT_TYPE::NALU == rtp_packet.NalUnitType()) {
compelete_video_frame_queue_.push(
VideoFrame(rtp_packet.Payload(), rtp_packet.PayloadSize()));
} else if (RtpPacket::NAL_UNIT_TYPE::FU_A == rtp_packet.NalUnitType()) {
incomplete_frame_list_[rtp_packet.SequenceNumber()] = rtp_packet;
bool complete = CheckIsH264FrameCompleted(rtp_packet);
if (!complete) {
}
}
} else if (RtpPacket::PAYLOAD_TYPE::H264_FEC_SOURCE ==
rtp_packet.PayloadType()) {
if (last_packet_ts_ != rtp_packet.Timestamp()) {
fec_decoder_.Init();
fec_decoder_.ResetParams(rtp_packet.FecSourceSymbolNum());
last_packet_ts_ = rtp_packet.Timestamp();
}
incomplete_fec_packet_list_[rtp_packet.Timestamp()]
[rtp_packet.SequenceNumber()] = rtp_packet;
uint8_t** complete_frame = fec_decoder_.DecodeWithNewSymbol(
(const char*)incomplete_fec_packet_list_[rtp_packet.Timestamp()]
[rtp_packet.SequenceNumber()]
.Payload(),
rtp_packet.FecSymbolId());
if (nullptr != complete_frame) {
if (!nv12_data_) {
nv12_data_ = new uint8_t[NV12_BUFFER_SIZE];
}
size_t complete_frame_size = 0;
for (int index = 0; index < rtp_packet.FecSourceSymbolNum(); index++) {
if (nullptr == complete_frame[index]) {
LOG_ERROR("Invalid complete_frame[{}]", index);
}
memcpy(nv12_data_ + complete_frame_size, complete_frame[index], 1400);
complete_frame_size += 1400;
}
fec_decoder_.ReleaseSourcePackets(complete_frame);
fec_decoder_.Release();
LOG_ERROR("Release incomplete_fec_packet_list_");
incomplete_fec_packet_list_.erase(rtp_packet.Timestamp());
if (incomplete_fec_frame_list_.end() !=
incomplete_fec_frame_list_.find(rtp_packet.Timestamp())) {
incomplete_fec_frame_list_.erase(rtp_packet.Timestamp());
}
compelete_video_frame_queue_.push(
VideoFrame(nv12_data_, complete_frame_size));
} else {
incomplete_fec_frame_list_.insert(rtp_packet.Timestamp());
}
} else if (RtpPacket::PAYLOAD_TYPE::H264_FEC_REPAIR ==
rtp_packet.PayloadType()) {
if (incomplete_fec_frame_list_.end() ==
incomplete_fec_frame_list_.find(rtp_packet.Timestamp())) {
return;
}
if (last_packet_ts_ != rtp_packet.Timestamp()) {
fec_decoder_.Init();
fec_decoder_.ResetParams(rtp_packet.FecSourceSymbolNum());
last_packet_ts_ = rtp_packet.Timestamp();
}
incomplete_fec_packet_list_[rtp_packet.Timestamp()]
[rtp_packet.SequenceNumber()] = rtp_packet;
uint8_t** complete_frame = fec_decoder_.DecodeWithNewSymbol(
(const char*)incomplete_fec_packet_list_[rtp_packet.Timestamp()]
[rtp_packet.SequenceNumber()]
.Payload(),
rtp_packet.FecSymbolId());
if (nullptr != complete_frame) {
if (!nv12_data_) {
nv12_data_ = new uint8_t[NV12_BUFFER_SIZE];
}
size_t complete_frame_size = 0;
for (int index = 0; index < rtp_packet.FecSourceSymbolNum(); index++) {
if (nullptr == complete_frame[index]) {
LOG_ERROR("Invalid complete_frame[{}]", index);
}
memcpy(nv12_data_ + complete_frame_size, complete_frame[index], 1400);
complete_frame_size += 1400;
}
fec_decoder_.ReleaseSourcePackets(complete_frame);
fec_decoder_.Release();
incomplete_fec_packet_list_.erase(rtp_packet.Timestamp());
compelete_video_frame_queue_.push(
VideoFrame(nv12_data_, complete_frame_size));
}
}
}
// else {
// if (rtp::PAYLOAD_TYPE::H264 == rtp_packet.PayloadType()) {
// if (rtp::NAL_UNIT_TYPE::NALU == rtp_packet.NalUnitType()) {
// compelete_video_frame_queue_.push(
// VideoFrame(rtp_packet.Payload(), rtp_packet.PayloadSize()));
// } else if (rtp::NAL_UNIT_TYPE::FU_A == rtp_packet.NalUnitType()) {
// incomplete_h264_frame_list_[rtp_packet.SequenceNumber()] =
// rtp_packet; bool complete = CheckIsH264FrameCompleted(rtp_packet); if
// (!complete) {
// }
// }
// } else if (rtp::PAYLOAD_TYPE::H264_FEC_SOURCE ==
// rtp_packet.PayloadType()) {
// if (last_packet_ts_ != rtp_packet.Timestamp()) {
// fec_decoder_.Init();
// fec_decoder_.ResetParams(rtp_packet.FecSourceSymbolNum());
// last_packet_ts_ = rtp_packet.Timestamp();
// }
// incomplete_fec_packet_list_[rtp_packet.Timestamp()]
// [rtp_packet.SequenceNumber()] = rtp_packet;
// uint8_t** complete_frame = fec_decoder_.DecodeWithNewSymbol(
// (const char*)incomplete_fec_packet_list_[rtp_packet.Timestamp()]
// [rtp_packet.SequenceNumber()]
// .Payload(),
// rtp_packet.FecSymbolId());
// if (nullptr != complete_frame) {
// if (!nv12_data_) {
// nv12_data_ = new uint8_t[NV12_BUFFER_SIZE];
// }
// size_t complete_frame_size = 0;
// for (int index = 0; index < rtp_packet.FecSourceSymbolNum(); index++)
// {
// if (nullptr == complete_frame[index]) {
// LOG_ERROR("Invalid complete_frame[{}]", index);
// }
// memcpy(nv12_data_ + complete_frame_size, complete_frame[index],
// 1400); complete_frame_size += 1400;
// }
// fec_decoder_.ReleaseSourcePackets(complete_frame);
// fec_decoder_.Release();
// LOG_ERROR("Release incomplete_fec_packet_list_");
// incomplete_fec_packet_list_.erase(rtp_packet.Timestamp());
// if (incomplete_fec_frame_list_.end() !=
// incomplete_fec_frame_list_.find(rtp_packet.Timestamp())) {
// incomplete_fec_frame_list_.erase(rtp_packet.Timestamp());
// }
// compelete_video_frame_queue_.push(
// VideoFrame(nv12_data_, complete_frame_size));
// } else {
// incomplete_fec_frame_list_.insert(rtp_packet.Timestamp());
// }
// } else if (rtp::PAYLOAD_TYPE::H264_FEC_REPAIR ==
// rtp_packet.PayloadType()) {
// if (incomplete_fec_frame_list_.end() ==
// incomplete_fec_frame_list_.find(rtp_packet.Timestamp())) {
// return;
// }
// if (last_packet_ts_ != rtp_packet.Timestamp()) {
// fec_decoder_.Init();
// fec_decoder_.ResetParams(rtp_packet.FecSourceSymbolNum());
// last_packet_ts_ = rtp_packet.Timestamp();
// }
// incomplete_fec_packet_list_[rtp_packet.Timestamp()]
// [rtp_packet.SequenceNumber()] = rtp_packet;
// uint8_t** complete_frame = fec_decoder_.DecodeWithNewSymbol(
// (const char*)incomplete_fec_packet_list_[rtp_packet.Timestamp()]
// [rtp_packet.SequenceNumber()]
// .Payload(),
// rtp_packet.FecSymbolId());
// if (nullptr != complete_frame) {
// if (!nv12_data_) {
// nv12_data_ = new uint8_t[NV12_BUFFER_SIZE];
// }
// size_t complete_frame_size = 0;
// for (int index = 0; index < rtp_packet.FecSourceSymbolNum(); index++)
// {
// if (nullptr == complete_frame[index]) {
// LOG_ERROR("Invalid complete_frame[{}]", index);
// }
// memcpy(nv12_data_ + complete_frame_size, complete_frame[index],
// 1400); complete_frame_size += 1400;
// }
// fec_decoder_.ReleaseSourcePackets(complete_frame);
// fec_decoder_.Release();
// incomplete_fec_packet_list_.erase(rtp_packet.Timestamp());
// compelete_video_frame_queue_.push(
// VideoFrame(nv12_data_, complete_frame_size));
// }
// }
// }
}
void RtpVideoReceiver::ProcessAv1RtpPacket(RtpPacket& rtp_packet) {
void RtpVideoReceiver::ProcessAv1RtpPacket(RtpPacketAv1& rtp_packet_av1) {
// LOG_ERROR("recv payload size = {}, sequence_number_ = {}",
// rtp_packet.PayloadSize(), rtp_packet.SequenceNumber());
if (RtpPacket::PAYLOAD_TYPE::AV1 == rtp_packet.PayloadType()) {
incomplete_frame_list_[rtp_packet.SequenceNumber()] = rtp_packet;
bool complete = CheckIsAv1FrameCompleted(rtp_packet);
if (rtp::PAYLOAD_TYPE::AV1 == rtp_packet_av1.PayloadType()) {
incomplete_av1_frame_list_[rtp_packet_av1.SequenceNumber()] =
rtp_packet_av1;
bool complete = CheckIsAv1FrameCompleted(rtp_packet_av1);
if (!complete) {
}
}
@@ -253,13 +268,13 @@ void RtpVideoReceiver::ProcessAv1RtpPacket(RtpPacket& rtp_packet) {
// }
}
bool RtpVideoReceiver::CheckIsH264FrameCompleted(RtpPacket& rtp_packet) {
if (rtp_packet.FuAEnd()) {
uint16_t end_seq = rtp_packet.SequenceNumber();
bool RtpVideoReceiver::CheckIsH264FrameCompleted(
RtpPacketH264& rtp_packet_h264) {
if (rtp_packet_h264.FuAEnd()) {
uint16_t end_seq = rtp_packet_h264.SequenceNumber();
while (end_seq--) {
auto it = incomplete_frame_list_.find(end_seq);
if (it == incomplete_frame_list_.end()) {
auto it = incomplete_h264_frame_list_.find(end_seq);
if (it == incomplete_h264_frame_list_.end()) {
// The last fragment has already received. If all fragments are in
// order, then some fragments lost in tranmission and need to be
// repaired using FEC
@@ -272,16 +287,16 @@ bool RtpVideoReceiver::CheckIsH264FrameCompleted(RtpPacket& rtp_packet) {
}
size_t complete_frame_size = 0;
for (uint16_t start = it->first; start <= rtp_packet.SequenceNumber();
start++) {
for (uint16_t start = it->first;
start <= rtp_packet_h264.SequenceNumber(); start++) {
memcpy(nv12_data_ + complete_frame_size,
incomplete_frame_list_[start].Payload(),
incomplete_frame_list_[start].PayloadSize());
incomplete_h264_frame_list_[start].Payload(),
incomplete_h264_frame_list_[start].PayloadSize());
complete_frame_size += incomplete_frame_list_[start].PayloadSize();
incomplete_frame_list_.erase(start);
complete_frame_size +=
incomplete_h264_frame_list_[start].PayloadSize();
incomplete_h264_frame_list_.erase(start);
}
compelete_video_frame_queue_.push(
VideoFrame(nv12_data_, complete_frame_size));
@@ -297,13 +312,13 @@ bool RtpVideoReceiver::CheckIsH264FrameCompleted(RtpPacket& rtp_packet) {
return false;
}
bool RtpVideoReceiver::CheckIsAv1FrameCompleted(RtpPacket& rtp_packet) {
if (rtp_packet.Av1FrameEnd()) {
uint16_t end_seq = rtp_packet.SequenceNumber();
bool RtpVideoReceiver::CheckIsAv1FrameCompleted(RtpPacketAv1& rtp_packet_av1) {
if (rtp_packet_av1.Av1FrameEnd()) {
uint16_t end_seq = rtp_packet_av1.SequenceNumber();
uint16_t start = end_seq;
while (end_seq--) {
auto it = incomplete_frame_list_.find(end_seq);
if (it == incomplete_frame_list_.end()) {
auto it = incomplete_av1_frame_list_.find(end_seq);
if (it == incomplete_av1_frame_list_.end()) {
// The last fragment has already received. If all fragments are in
// order, then some fragments lost in tranmission and need to be
// repaired using FEC
@@ -319,19 +334,19 @@ bool RtpVideoReceiver::CheckIsAv1FrameCompleted(RtpPacket& rtp_packet) {
}
}
if (start <= rtp_packet.SequenceNumber()) {
if (start <= rtp_packet_av1.SequenceNumber()) {
if (!nv12_data_) {
nv12_data_ = new uint8_t[NV12_BUFFER_SIZE];
}
size_t complete_frame_size = 0;
for (; start <= rtp_packet.SequenceNumber(); start++) {
const uint8_t* obu_frame = incomplete_frame_list_[start].Payload();
size_t obu_frame_size = incomplete_frame_list_[start].PayloadSize();
for (; start <= rtp_packet_av1.SequenceNumber(); start++) {
const uint8_t* obu_frame = incomplete_av1_frame_list_[start].Payload();
size_t obu_frame_size = incomplete_av1_frame_list_[start].PayloadSize();
memcpy(nv12_data_ + complete_frame_size, obu_frame, obu_frame_size);
complete_frame_size += obu_frame_size;
incomplete_frame_list_.erase(start);
incomplete_av1_frame_list_.erase(start);
}
compelete_video_frame_queue_.push(

13
src/channel/rtp_channel/rtp_video_receiver.h
View File

@@ -12,7 +12,8 @@
#include "receive_side_congestion_controller.h"
#include "ringbuffer.h"
#include "rtcp_receiver_report.h"
#include "rtp_codec.h"
#include "rtp_packet_av1.h"
#include "rtp_packet_h264.h"
#include "rtp_rtcp_defines.h "
#include "rtp_statistics.h"
#include "thread_base.h"
@@ -37,12 +38,12 @@ class RtpVideoReceiver : public ThreadBase {
}
private:
void ProcessAv1RtpPacket(RtpPacket& rtp_packet);
bool CheckIsAv1FrameCompleted(RtpPacket& rtp_packet);
void ProcessAv1RtpPacket(RtpPacketAv1& rtp_packet_av1);
bool CheckIsAv1FrameCompleted(RtpPacketAv1& rtp_packet_av1);
private:
void ProcessH264RtpPacket(RtpPacket& rtp_packet);
bool CheckIsH264FrameCompleted(RtpPacket& rtp_packet);
void ProcessH264RtpPacket(RtpPacketH264& rtp_packet_h264);
bool CheckIsH264FrameCompleted(RtpPacketH264& rtp_packet_h264);
private:
bool CheckIsTimeSendRR();
@@ -58,6 +59,8 @@ class RtpVideoReceiver : public ThreadBase {
void RtcpThread();
private:
std::map<uint16_t, RtpPacketH264> incomplete_h264_frame_list_;
std::map<uint16_t, RtpPacketAv1> incomplete_av1_frame_list_;
std::map<uint16_t, RtpPacket> incomplete_frame_list_;
uint8_t* nv12_data_ = nullptr;
std::function<void(VideoFrame&)> on_receive_complete_frame_ = nullptr;

4
src/channel/rtp_channel/rtp_video_sender.cpp
View File

@@ -39,8 +39,8 @@ int RtpVideoSender::SendRtpPacket(RtpPacket& rtp_packet) {
return -1;
}
if (0 !=
data_send_func_((const char*)rtp_packet.Buffer(), rtp_packet.Size())) {
if (0 != data_send_func_((const char*)rtp_packet.Buffer().data(),
rtp_packet.Size())) {
LOG_ERROR("Send rtp packet failed");
return -1;
}

8
src/channel/video_channel_receive.cpp
View File

@@ -14,10 +14,8 @@ VideoChannelReceive::VideoChannelReceive(
VideoChannelReceive::~VideoChannelReceive() {}
void VideoChannelReceive::Initialize(RtpPacket::PAYLOAD_TYPE payload_type) {
video_rtp_codec_ = std::make_unique<RtpCodec>(payload_type);
void VideoChannelReceive::Initialize(rtp::PAYLOAD_TYPE payload_type) {
rtp_video_receiver_ = std::make_unique<RtpVideoReceiver>(ice_io_statistics_);
rtp_video_receiver_->SetOnReceiveCompleteFrame(
[this](VideoFrame &video_frame) -> void {
on_receive_complete_frame_(video_frame);
@@ -54,7 +52,9 @@ int VideoChannelReceive::OnReceiveRtpPacket(const char *data, size_t size) {
}
if (rtp_video_receiver_) {
rtp_video_receiver_->InsertRtpPacket(RtpPacket((uint8_t *)data, size));
RtpPacket rtp_packet;
rtp_packet.Build((uint8_t *)data, (uint32_t)size);
rtp_video_receiver_->InsertRtpPacket(rtp_packet);
}
return 0;

4
src/channel/video_channel_receive.h
View File

@@ -8,7 +8,6 @@
#define _VIDEO_CHANNEL_RECEIVE_H_
#include "ice_agent.h"
#include "rtp_codec.h"
#include "rtp_video_receiver.h"
class VideoChannelReceive {
@@ -22,7 +21,7 @@ class VideoChannelReceive {
~VideoChannelReceive();
public:
void Initialize(RtpPacket::PAYLOAD_TYPE payload_type);
void Initialize(rtp::PAYLOAD_TYPE payload_type);
void Destroy();
int OnReceiveRtpPacket(const char *data, size_t size);
@@ -30,7 +29,6 @@ class VideoChannelReceive {
private:
std::shared_ptr<IceAgent> ice_agent_ = nullptr;
std::shared_ptr<IOStatistics> ice_io_statistics_ = nullptr;
std::unique_ptr<RtpCodec> video_rtp_codec_ = nullptr;
std::unique_ptr<RtpVideoReceiver> rtp_video_receiver_ = nullptr;
std::function<void(VideoFrame &)> on_receive_complete_frame_ = nullptr;
};

15
src/channel/video_channel_send.cpp
View File

@@ -11,9 +11,8 @@ VideoChannelSend::VideoChannelSend(
std::shared_ptr<IOStatistics> ice_io_statistics)
: ice_agent_(ice_agent), ice_io_statistics_(ice_io_statistics){};
void VideoChannelSend::Initialize(RtpPacket::PAYLOAD_TYPE payload_type) {
video_rtp_codec_ = std::make_unique<RtpCodec>(payload_type);
void VideoChannelSend::Initialize(rtp::PAYLOAD_TYPE payload_type) {
rtp_packetizer_ = RtpPacketizer::Create(payload_type);
rtp_video_sender_ = std::make_unique<RtpVideoSender>(ice_io_statistics_);
rtp_video_sender_->SetSendDataFunc(
[this](const char* data, size_t size) -> int {
@@ -45,12 +44,10 @@ void VideoChannelSend::Destroy() {
}
int VideoChannelSend::SendVideo(char* data, size_t size) {
std::vector<RtpPacket> packets;
if (rtp_video_sender_) {
if (video_rtp_codec_) {
video_rtp_codec_->Encode((uint8_t*)data, (uint32_t)size, packets);
}
rtp_video_sender_->Enqueue(packets);
if (rtp_video_sender_ && rtp_packetizer_) {
std::vector<RtpPacket> rtp_packets =
rtp_packetizer_->Build((uint8_t*)data, (uint32_t)size);
rtp_video_sender_->Enqueue(rtp_packets);
}
return 0;

6
src/channel/video_channel_send.h
View File

@@ -9,7 +9,7 @@
#include "congestion_control_feedback.h"
#include "ice_agent.h"
#include "rtp_codec.h"
#include "rtp_packetizer.h"
#include "rtp_video_sender.h"
#include "transport_feedback.h"
#include "transport_feedback_adapter.h"
@@ -22,7 +22,7 @@ class VideoChannelSend {
~VideoChannelSend();
public:
void Initialize(RtpPacket::PAYLOAD_TYPE payload_type);
void Initialize(rtp::PAYLOAD_TYPE payload_type);
void Destroy();
int SendVideo(char* data, size_t size);
@@ -36,7 +36,7 @@ class VideoChannelSend {
private:
std::shared_ptr<IceAgent> ice_agent_ = nullptr;
std::shared_ptr<IOStatistics> ice_io_statistics_ = nullptr;
std::unique_ptr<RtpCodec> video_rtp_codec_ = nullptr;
std::unique_ptr<RtpPacketizer> rtp_packetizer_ = nullptr;
std::unique_ptr<RtpVideoSender> rtp_video_sender_ = nullptr;
private: