From 0d9d3c5eca9cd524c2adf9ffdfed9aa7d362cb80 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Thu, 27 Mar 2025 17:35:27 +0800 Subject: [PATCH] [feat] complete h264 frame assember --- .../assemble_frame/h264_frame_assember.cpp | 16 ++-- .../assemble_frame/h264_frame_assember.h | 5 +- src/transport/channel/rtp_video_receiver.cpp | 75 +++++++++++-------- src/transport/channel/video_channel_send.cpp | 27 ++++++- src/transport/channel/video_channel_send.h | 3 + 5 files changed, 86 insertions(+), 40 deletions(-) diff --git a/src/media/video/assemble_frame/h264_frame_assember.cpp b/src/media/video/assemble_frame/h264_frame_assember.cpp index 815d1bf..9de02a7 100644 --- a/src/media/video/assemble_frame/h264_frame_assember.cpp +++ b/src/media/video/assemble_frame/h264_frame_assember.cpp @@ -1,5 +1,7 @@ #include "h264_frame_assember.h" +#include "log.h" + H264FrameAssembler::H264FrameAssembler() {} H264FrameAssembler::~H264FrameAssembler() {} @@ -30,21 +32,22 @@ int64_t H264FrameAssembler::Unwrap(uint16_t seq_num) { return (int64_t)seq_num; } -std::vector>& H264FrameAssembler::InsertPacket( +std::vector> H264FrameAssembler::InsertPacket( std::unique_ptr rtp_packet) { std::vector> result; - int64_t unwrapped_seq_num = Unwrap(rtp_packet->SequenceNumber()); + int64_t unwrapped_seq_num = + rtp_seq_num_unwrapper_.Unwrap(rtp_packet->SequenceNumber()); auto& packet_slotted = GetPacketFromBuffer(unwrapped_seq_num); if (packet_slotted != nullptr && AheadOrAt(packet_slotted->Timestamp(), rtp_packet->Timestamp())) { // The incoming `packet` is old or a duplicate. - return std::move(result); + return result; } else { packet_slotted = std::move(rtp_packet); } - return std::move(FindFrames(unwrapped_seq_num)); + return FindFrames(unwrapped_seq_num); } std::unique_ptr& H264FrameAssembler::GetPacketFromBuffer( @@ -98,13 +101,14 @@ std::vector> H264FrameAssembler::FindFrames( if (prev_packet == nullptr || prev_packet->Timestamp() != rtp_timestamp) { const auto& current_packet = GetPacketFromBuffer(seq_num_start); + if (!current_packet->FuAStart()) { // First packet of the frame is missing. return result; } - for (int64_t seq_num = seq_num_start; seq_num <= seq_num; ++seq_num) { - auto& packet = GetPacketFromBuffer(seq_num); + for (int64_t seq = seq_num_start; seq <= seq_num; ++seq) { + auto& packet = GetPacketFromBuffer(seq); result.push_back(std::move(packet)); } break; diff --git a/src/media/video/assemble_frame/h264_frame_assember.h b/src/media/video/assemble_frame/h264_frame_assember.h index c07980b..3f1c625 100644 --- a/src/media/video/assemble_frame/h264_frame_assember.h +++ b/src/media/video/assemble_frame/h264_frame_assember.h @@ -9,6 +9,7 @@ #include +#include "rtc_base/numerics/sequence_number_unwrapper.h" #include "rtp_packet_h264.h" #define MAX_PACKET_BUFFER_SIZE 2048 @@ -20,7 +21,7 @@ class H264FrameAssembler { ~H264FrameAssembler(); public: - std::vector>& InsertPacket( + std::vector> InsertPacket( std::unique_ptr rtp_packet); private: @@ -37,6 +38,8 @@ class H264FrameAssembler { packet_buffer_; std::array last_continuous_in_sequence_; int64_t last_continuous_in_sequence_index_ = 0; + + webrtc::SeqNumUnwrapper rtp_seq_num_unwrapper_; }; #endif \ No newline at end of file diff --git a/src/transport/channel/rtp_video_receiver.cpp b/src/transport/channel/rtp_video_receiver.cpp index c68a0c1..2e15f6b 100644 --- a/src/transport/channel/rtp_video_receiver.cpp +++ b/src/transport/channel/rtp_video_receiver.cpp @@ -7,7 +7,7 @@ #include "nack.h" #include "rtcp_sender.h" -// #define SAVE_RTP_RECV_STREAM +#define SAVE_RTP_RECV_STREAM #define NV12_BUFFER_SIZE (1280 * 720 * 3 / 2) #define RTCP_RR_INTERVAL 1000 @@ -195,39 +195,52 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) { ProcessAv1RtpPacket(rtp_packet_av1); } else if (rtp_packet.PayloadType() == rtp::PAYLOAD_TYPE::H264 || rtp_packet.PayloadType() == rtp::PAYLOAD_TYPE::H264 - 1) { - // std::unique_ptr rtp_packet_h264 = - // std::make_unique(); - // if (rtp_packet_h264->Build(rtp_packet.Buffer().data(), - // rtp_packet.Size())) { - // std::vector> complete_frame = std::move( - // h264_frame_assembler_.InsertPacket(std::move(rtp_packet_h264))); - // if (!complete_frame.empty()) { - // for (auto& frame : complete_frame) { - // ReceivedFrame received_frame(frame->Payload(), - // frame->PayloadSize()); - // received_frame.SetReceivedTimestamp(clock_->CurrentTime().us()); - // received_frame.SetCapturedTimestamp( - // (static_cast(frame->Timestamp()) / - // rtp::kMsToRtpTimestamp - - // delta_ntp_internal_ms_) * - // 1000); - // compelete_video_frame_queue_.push(received_frame); - // } - // } - // } + std::unique_ptr rtp_packet_h264 = + std::make_unique(); + if (rtp_packet.Buffer().data() != nullptr && rtp_packet.Size() > 0 && + rtp_packet_h264->Build(rtp_packet.Buffer().data(), rtp_packet.Size())) { + rtp_packet_h264->GetFrameHeaderInfo(); + std::vector> complete_frame = + h264_frame_assembler_.InsertPacket(std::move(rtp_packet_h264)); + if (!complete_frame.empty()) { + uint8_t* nv12_data_ = new uint8_t[NV12_BUFFER_SIZE]; + uint8_t* dest = nv12_data_; + size_t complete_frame_size = 0; + for (auto& frame : complete_frame) { + memcpy(dest, frame->Payload(), frame->PayloadSize()); + dest += frame->PayloadSize(); + complete_frame_size += frame->PayloadSize(); + } - RtpPacketH264 rtp_packet_h264; - if (rtp_packet_h264.Build(rtp_packet.Buffer().data(), rtp_packet.Size())) { - rtp_packet_h264.GetFrameHeaderInfo(); - bool is_missing_packet = ProcessH264RtpPacket(rtp_packet_h264); - if (!is_missing_packet) { - receive_side_congestion_controller_.OnReceivedPacket( - rtp_packet_received, MediaType::VIDEO); - nack_->OnReceivedPacket(rtp_packet.SequenceNumber(), true); - } else { - nack_->OnReceivedPacket(rtp_packet.SequenceNumber(), false); + ReceivedFrame received_frame(nv12_data_, complete_frame_size); + received_frame.SetReceivedTimestamp(clock_->CurrentTime().us()); + received_frame.SetCapturedTimestamp( + (static_cast(complete_frame[0]->Timestamp()) / + rtp::kMsToRtpTimestamp - + delta_ntp_internal_ms_) * + 1000); + compelete_video_frame_queue_.push(received_frame); + + fwrite((unsigned char*)received_frame.Buffer(), 1, + received_frame.Size(), file_rtp_recv_); + + delete[] nv12_data_; } } + + // RtpPacketH264 rtp_packet_h264; + // if (rtp_packet_h264.Build(rtp_packet.Buffer().data(), rtp_packet.Size())) + // { + // rtp_packet_h264.GetFrameHeaderInfo(); + // bool is_missing_packet = ProcessH264RtpPacket(rtp_packet_h264); + // if (!is_missing_packet) { + // receive_side_congestion_controller_.OnReceivedPacket( + // rtp_packet_received, MediaType::VIDEO); + // nack_->OnReceivedPacket(rtp_packet.SequenceNumber(), true); + // } else { + // nack_->OnReceivedPacket(rtp_packet.SequenceNumber(), false); + // } + // } } } diff --git a/src/transport/channel/video_channel_send.cpp b/src/transport/channel/video_channel_send.cpp index 95c897e..11be99a 100644 --- a/src/transport/channel/video_channel_send.cpp +++ b/src/transport/channel/video_channel_send.cpp @@ -3,6 +3,8 @@ #include "log.h" #include "rtc_base/network/sent_packet.h" +#define SAVE_RTP_SENT_STREAM + VideoChannelSend::VideoChannelSend( std::shared_ptr clock, std::shared_ptr ice_agent, std::shared_ptr packet_sender, @@ -16,9 +18,24 @@ VideoChannelSend::VideoChannelSend( delta_ntp_internal_ms_(clock->CurrentNtpInMilliseconds() - clock->CurrentTimeMs()), rtp_packet_history_(clock), - clock_(clock){}; + clock_(clock) { +#ifdef SAVE_RTP_SENT_STREAM + file_rtp_sent_ = fopen("rtp_sent_stream.h264", "w+b"); + if (!file_rtp_sent_) { + LOG_WARN("Fail to open rtp_sent_stream.h264"); + } +#endif +}; -VideoChannelSend::~VideoChannelSend() {} +VideoChannelSend::~VideoChannelSend() { +#ifdef SAVE_RTP_SENT_STREAM + if (file_rtp_sent_) { + fflush(file_rtp_sent_); + fclose(file_rtp_sent_); + file_rtp_sent_ = nullptr; + } +#endif +} void VideoChannelSend::Initialize(rtp::PAYLOAD_TYPE payload_type) { rtp_video_sender_ = @@ -114,6 +131,12 @@ int VideoChannelSend::SendVideo(const EncodedFrame& encoded_frame) { rtp_packetizer_->Build((uint8_t*)encoded_frame.Buffer(), (uint32_t)encoded_frame.Size(), rtp_timestamp, true); + +#ifdef SAVE_RTP_SENT_STREAM + fwrite((unsigned char*)encoded_frame.Buffer(), 1, encoded_frame.Size(), + file_rtp_sent_); +#endif + packet_sender_->EnqueueRtpPacket(std::move(rtp_packets), rtp_timestamp); } diff --git a/src/transport/channel/video_channel_send.h b/src/transport/channel/video_channel_send.h index f87f394..5ca3ae2 100644 --- a/src/transport/channel/video_channel_send.h +++ b/src/transport/channel/video_channel_send.h @@ -78,6 +78,9 @@ class VideoChannelSend { std::shared_ptr clock_; RtpPacketHistory rtp_packet_history_; int64_t delta_ntp_internal_ms_; + + private: + FILE* file_rtp_sent_ = nullptr; }; #endif \ No newline at end of file