YangWX/crossdesk
1
0
Fork 0
mirror of https://github.com/kunkundi/crossdesk.git synced 2025-10-26 12:15:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feat] complete h264 frame assember

Browse Source
This commit is contained in:
dijunkun
2025-03-27 17:35:27 +08:00
parent 0f07058fb9
commit 0d9d3c5eca
5 changed files with 86 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/media/video/assemble_frame/h264_frame_assember.cpp
View File

@@ -1,5 +1,7 @@
#include "h264_frame_assember.h"
#include "log.h"
H264FrameAssembler::H264FrameAssembler() {}
H264FrameAssembler::~H264FrameAssembler() {}
@@ -30,21 +32,22 @@ int64_t H264FrameAssembler::Unwrap(uint16_t seq_num) {
return (int64_t)seq_num;
}
std::vector<std::unique_ptr<RtpPacketH264>>& H264FrameAssembler::InsertPacket(
std::vector<std::unique_ptr<RtpPacketH264>> H264FrameAssembler::InsertPacket(
std::unique_ptr<RtpPacketH264> rtp_packet) {
std::vector<std::unique_ptr<RtpPacketH264>> result;
int64_t unwrapped_seq_num = Unwrap(rtp_packet->SequenceNumber());
int64_t unwrapped_seq_num =
rtp_seq_num_unwrapper_.Unwrap(rtp_packet->SequenceNumber());
auto& packet_slotted = GetPacketFromBuffer(unwrapped_seq_num);
if (packet_slotted != nullptr &&
AheadOrAt(packet_slotted->Timestamp(), rtp_packet->Timestamp())) {
// The incoming `packet` is old or a duplicate.
return std::move(result);
return result;
} else {
packet_slotted = std::move(rtp_packet);
}
return std::move(FindFrames(unwrapped_seq_num));
return FindFrames(unwrapped_seq_num);
}
std::unique_ptr<RtpPacketH264>& H264FrameAssembler::GetPacketFromBuffer(
@@ -98,13 +101,14 @@ std::vector<std::unique_ptr<RtpPacketH264>> H264FrameAssembler::FindFrames(
if (prev_packet == nullptr ||
prev_packet->Timestamp() != rtp_timestamp) {
const auto& current_packet = GetPacketFromBuffer(seq_num_start);
if (!current_packet->FuAStart()) {
// First packet of the frame is missing.
return result;
}
for (int64_t seq_num = seq_num_start; seq_num <= seq_num; ++seq_num) {
auto& packet = GetPacketFromBuffer(seq_num);
for (int64_t seq = seq_num_start; seq <= seq_num; ++seq) {
auto& packet = GetPacketFromBuffer(seq);
result.push_back(std::move(packet));
}
break;

5
src/media/video/assemble_frame/h264_frame_assember.h
View File

@@ -9,6 +9,7 @@
#include <array>
#include "rtc_base/numerics/sequence_number_unwrapper.h"
#include "rtp_packet_h264.h"
#define MAX_PACKET_BUFFER_SIZE 2048
@@ -20,7 +21,7 @@ class H264FrameAssembler {
~H264FrameAssembler();
public:
std::vector<std::unique_ptr<RtpPacketH264>>& InsertPacket(
std::vector<std::unique_ptr<RtpPacketH264>> InsertPacket(
std::unique_ptr<RtpPacketH264> rtp_packet);
private:
@@ -37,6 +38,8 @@ class H264FrameAssembler {
packet_buffer_;
std::array<int64_t, MAX_TRACKED_SEQUENCE_SIZE> last_continuous_in_sequence_;
int64_t last_continuous_in_sequence_index_ = 0;
webrtc::SeqNumUnwrapper<uint16_t> rtp_seq_num_unwrapper_;
};
#endif

75
src/transport/channel/rtp_video_receiver.cpp
View File

@@ -7,7 +7,7 @@
#include "nack.h"
#include "rtcp_sender.h"
// #define SAVE_RTP_RECV_STREAM
#define SAVE_RTP_RECV_STREAM
#define NV12_BUFFER_SIZE (1280 * 720 * 3 / 2)
#define RTCP_RR_INTERVAL 1000
@@ -195,39 +195,52 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
ProcessAv1RtpPacket(rtp_packet_av1);
} else if (rtp_packet.PayloadType() == rtp::PAYLOAD_TYPE::H264 ||
rtp_packet.PayloadType() == rtp::PAYLOAD_TYPE::H264 - 1) {
// std::unique_ptr<RtpPacketH264> rtp_packet_h264 =
// std::make_unique<RtpPacketH264>();
// if (rtp_packet_h264->Build(rtp_packet.Buffer().data(),
// rtp_packet.Size())) {
// std::vector<std::unique_ptr<RtpPacketH264>> complete_frame = std::move(
// h264_frame_assembler_.InsertPacket(std::move(rtp_packet_h264)));
// if (!complete_frame.empty()) {
// for (auto& frame : complete_frame) {
// ReceivedFrame received_frame(frame->Payload(),
// frame->PayloadSize());
// received_frame.SetReceivedTimestamp(clock_->CurrentTime().us());
// received_frame.SetCapturedTimestamp(
// (static_cast<int64_t>(frame->Timestamp()) /
// rtp::kMsToRtpTimestamp -
// delta_ntp_internal_ms_) *
// 1000);
// compelete_video_frame_queue_.push(received_frame);
// }
// }
// }
std::unique_ptr<RtpPacketH264> rtp_packet_h264 =
std::make_unique<RtpPacketH264>();
if (rtp_packet.Buffer().data() != nullptr && rtp_packet.Size() > 0 &&
rtp_packet_h264->Build(rtp_packet.Buffer().data(), rtp_packet.Size())) {
rtp_packet_h264->GetFrameHeaderInfo();
std::vector<std::unique_ptr<RtpPacketH264>> complete_frame =
h264_frame_assembler_.InsertPacket(std::move(rtp_packet_h264));
if (!complete_frame.empty()) {
uint8_t* nv12_data_ = new uint8_t[NV12_BUFFER_SIZE];
uint8_t* dest = nv12_data_;
size_t complete_frame_size = 0;
for (auto& frame : complete_frame) {
memcpy(dest, frame->Payload(), frame->PayloadSize());
dest += frame->PayloadSize();
complete_frame_size += frame->PayloadSize();
}
RtpPacketH264 rtp_packet_h264;
if (rtp_packet_h264.Build(rtp_packet.Buffer().data(), rtp_packet.Size())) {
rtp_packet_h264.GetFrameHeaderInfo();
bool is_missing_packet = ProcessH264RtpPacket(rtp_packet_h264);
if (!is_missing_packet) {
receive_side_congestion_controller_.OnReceivedPacket(
rtp_packet_received, MediaType::VIDEO);
nack_->OnReceivedPacket(rtp_packet.SequenceNumber(), true);
} else {
nack_->OnReceivedPacket(rtp_packet.SequenceNumber(), false);
ReceivedFrame received_frame(nv12_data_, complete_frame_size);
received_frame.SetReceivedTimestamp(clock_->CurrentTime().us());
received_frame.SetCapturedTimestamp(
(static_cast<int64_t>(complete_frame[0]->Timestamp()) /
rtp::kMsToRtpTimestamp -
delta_ntp_internal_ms_) *
1000);
compelete_video_frame_queue_.push(received_frame);
fwrite((unsigned char*)received_frame.Buffer(), 1,
received_frame.Size(), file_rtp_recv_);
delete[] nv12_data_;
}
}
// RtpPacketH264 rtp_packet_h264;
// if (rtp_packet_h264.Build(rtp_packet.Buffer().data(), rtp_packet.Size()))
// {
// rtp_packet_h264.GetFrameHeaderInfo();
// bool is_missing_packet = ProcessH264RtpPacket(rtp_packet_h264);
// if (!is_missing_packet) {
// receive_side_congestion_controller_.OnReceivedPacket(
// rtp_packet_received, MediaType::VIDEO);
// nack_->OnReceivedPacket(rtp_packet.SequenceNumber(), true);
// } else {
// nack_->OnReceivedPacket(rtp_packet.SequenceNumber(), false);
// }
// }
}
}

27
src/transport/channel/video_channel_send.cpp
View File

@@ -3,6 +3,8 @@
#include "log.h"
#include "rtc_base/network/sent_packet.h"
#define SAVE_RTP_SENT_STREAM
VideoChannelSend::VideoChannelSend(
std::shared_ptr<SystemClock> clock, std::shared_ptr<IceAgent> ice_agent,
std::shared_ptr<PacketSender> packet_sender,
@@ -16,9 +18,24 @@ VideoChannelSend::VideoChannelSend(
delta_ntp_internal_ms_(clock->CurrentNtpInMilliseconds() -
clock->CurrentTimeMs()),
rtp_packet_history_(clock),
clock_(clock){};
clock_(clock) {
#ifdef SAVE_RTP_SENT_STREAM
file_rtp_sent_ = fopen("rtp_sent_stream.h264", "w+b");
if (!file_rtp_sent_) {
LOG_WARN("Fail to open rtp_sent_stream.h264");
}
#endif
};
VideoChannelSend::~VideoChannelSend() {}
VideoChannelSend::~VideoChannelSend() {
#ifdef SAVE_RTP_SENT_STREAM
if (file_rtp_sent_) {
fflush(file_rtp_sent_);
fclose(file_rtp_sent_);
file_rtp_sent_ = nullptr;
}
#endif
}
void VideoChannelSend::Initialize(rtp::PAYLOAD_TYPE payload_type) {
rtp_video_sender_ =
@@ -114,6 +131,12 @@ int VideoChannelSend::SendVideo(const EncodedFrame& encoded_frame) {
rtp_packetizer_->Build((uint8_t*)encoded_frame.Buffer(),
(uint32_t)encoded_frame.Size(), rtp_timestamp,
true);
#ifdef SAVE_RTP_SENT_STREAM
fwrite((unsigned char*)encoded_frame.Buffer(), 1, encoded_frame.Size(),
file_rtp_sent_);
#endif
packet_sender_->EnqueueRtpPacket(std::move(rtp_packets), rtp_timestamp);
}

3
src/transport/channel/video_channel_send.h
View File

@@ -78,6 +78,9 @@ class VideoChannelSend {
std::shared_ptr<SystemClock> clock_;
RtpPacketHistory rtp_packet_history_;
int64_t delta_ntp_internal_ms_;
private:
FILE* file_rtp_sent_ = nullptr;
};
#endif