From df6f4321e8a936114877c1a33398c8dce1abe42a Mon Sep 17 00:00:00 2001 From: dijunkun Date: Mon, 14 Apr 2025 16:11:25 +0800 Subject: [PATCH] [feat] use smart pointer for ReceivedFrame when send it into decoder --- .../video/decode/aom/aom_av1_decoder.cpp | 10 ++-- src/media/video/decode/aom/aom_av1_decoder.h | 2 +- .../video/decode/dav1d/dav1d_av1_decoder.cpp | 10 ++-- .../video/decode/dav1d/dav1d_av1_decoder.h | 2 +- .../decode/nvcodec/nvidia_video_decoder.cpp | 10 ++-- .../decode/nvcodec/nvidia_video_decoder.h | 2 +- .../decode/openh264/openh264_decoder.cpp | 14 +++-- .../video/decode/openh264/openh264_decoder.h | 4 +- src/media/video/decode/video_decoder.h | 2 +- src/transport/channel/rtp_video_receiver.cpp | 6 ++- src/transport/channel/rtp_video_receiver.h | 7 +-- .../channel/video_channel_receive.cpp | 7 +-- src/transport/channel/video_channel_receive.h | 13 ++--- src/transport/ice_transport_controller.cpp | 54 ++++++++++--------- src/transport/ice_transport_controller.h | 2 +- 15 files changed, 80 insertions(+), 65 deletions(-) diff --git a/src/media/video/decode/aom/aom_av1_decoder.cpp b/src/media/video/decode/aom/aom_av1_decoder.cpp index 85053be..e14378e 100644 --- a/src/media/video/decode/aom/aom_av1_decoder.cpp +++ b/src/media/video/decode/aom/aom_av1_decoder.cpp @@ -66,10 +66,10 @@ int AomAv1Decoder::Init() { } int AomAv1Decoder::Decode( - const ReceivedFrame &received_frame, + std::unique_ptr received_frame, std::function on_receive_decoded_frame) { - const uint8_t *data = received_frame.Buffer(); - size_t size = received_frame.Size(); + const uint8_t *data = received_frame->Buffer(); + size_t size = received_frame->Size(); #ifdef SAVE_RECEIVED_AV1_STREAM fwrite((unsigned char *)data, 1, size, file_av1_); @@ -138,8 +138,8 @@ int AomAv1Decoder::Decode( DecodedFrame decode_frame(nv12_data.data(), nv12_size, img_->d_w, img_->d_h); - decode_frame.SetReceivedTimestamp(received_frame.ReceivedTimestamp()); - decode_frame.SetCapturedTimestamp(received_frame.CapturedTimestamp()); + decode_frame.SetReceivedTimestamp(received_frame->ReceivedTimestamp()); + decode_frame.SetCapturedTimestamp(received_frame->CapturedTimestamp()); decode_frame.SetDecodedTimestamp(clock_->CurrentTime()); #ifdef SAVE_DECODED_NV12_STREAM diff --git a/src/media/video/decode/aom/aom_av1_decoder.h b/src/media/video/decode/aom/aom_av1_decoder.h index 8199701..aa787ec 100644 --- a/src/media/video/decode/aom/aom_av1_decoder.h +++ b/src/media/video/decode/aom/aom_av1_decoder.h @@ -23,7 +23,7 @@ class AomAv1Decoder : public VideoDecoder { int Init(); int Decode( - const ReceivedFrame &received_frame, + std::unique_ptr received_frame, std::function on_receive_decoded_frame); std::string GetDecoderName() { return "AomAv1"; } diff --git a/src/media/video/decode/dav1d/dav1d_av1_decoder.cpp b/src/media/video/decode/dav1d/dav1d_av1_decoder.cpp index 2a1c848..fbcb4a3 100644 --- a/src/media/video/decode/dav1d/dav1d_av1_decoder.cpp +++ b/src/media/video/decode/dav1d/dav1d_av1_decoder.cpp @@ -107,10 +107,10 @@ int Dav1dAv1Decoder::Init() { } int Dav1dAv1Decoder::Decode( - const ReceivedFrame &received_frame, + std::unique_ptr received_frame, std::function on_receive_decoded_frame) { - const uint8_t *data = received_frame.Buffer(); - size_t size = received_frame.Size(); + const uint8_t *data = received_frame->Buffer(); + size_t size = received_frame->Size(); #ifdef SAVE_RECEIVED_AV1_STREAM fwrite((unsigned char *)data, 1, size, file_av1_); @@ -190,8 +190,8 @@ int Dav1dAv1Decoder::Decode( DecodedFrame decoded_frame(nv12_frame_, nv12_frame_capacity_, frame_width_, frame_height_); - decoded_frame.SetReceivedTimestamp(received_frame.ReceivedTimestamp()); - decoded_frame.SetCapturedTimestamp(received_frame.CapturedTimestamp()); + decoded_frame.SetReceivedTimestamp(received_frame->ReceivedTimestamp()); + decoded_frame.SetCapturedTimestamp(received_frame->CapturedTimestamp()); decoded_frame.SetDecodedTimestamp(clock_->CurrentTime()); #ifdef SAVE_DECODED_NV12_STREAM diff --git a/src/media/video/decode/dav1d/dav1d_av1_decoder.h b/src/media/video/decode/dav1d/dav1d_av1_decoder.h index ea517ff..67b3346 100644 --- a/src/media/video/decode/dav1d/dav1d_av1_decoder.h +++ b/src/media/video/decode/dav1d/dav1d_av1_decoder.h @@ -21,7 +21,7 @@ class Dav1dAv1Decoder : public VideoDecoder { int Init(); int Decode( - const ReceivedFrame &received_frame, + std::unique_ptr received_frame, std::function on_receive_decoded_frame); std::string GetDecoderName() { return "Dav1dAv1"; } diff --git a/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp b/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp index 5b1d0d9..2d78da3 100644 --- a/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp +++ b/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp @@ -69,14 +69,14 @@ int NvidiaVideoDecoder::Init() { } int NvidiaVideoDecoder::Decode( - const ReceivedFrame &received_frame, + std::unique_ptr received_frame, std::function on_receive_decoded_frame) { if (!decoder) { return -1; } - const uint8_t *data = received_frame.Buffer(); - size_t size = received_frame.Size(); + const uint8_t *data = received_frame->Buffer(); + size_t size = received_frame->Size(); #ifdef SAVE_RECEIVED_H264_STREAM fwrite((unsigned char *)data, 1, size, file_h264_); @@ -99,9 +99,9 @@ int NvidiaVideoDecoder::Decode( decoder->GetWidth() * decoder->GetHeight() * 3 / 2, decoder->GetWidth(), decoder->GetHeight()); decoded_frame.SetReceivedTimestamp( - received_frame.ReceivedTimestamp()); + received_frame->ReceivedTimestamp()); decoded_frame.SetCapturedTimestamp( - received_frame.CapturedTimestamp()); + received_frame->CapturedTimestamp()); decoded_frame.SetDecodedTimestamp(clock_->CurrentTime()); on_receive_decoded_frame(decoded_frame); #ifdef SAVE_DECODED_NV12_STREAM diff --git a/src/media/video/decode/nvcodec/nvidia_video_decoder.h b/src/media/video/decode/nvcodec/nvidia_video_decoder.h index ef9c4f1..a2f13d7 100644 --- a/src/media/video/decode/nvcodec/nvidia_video_decoder.h +++ b/src/media/video/decode/nvcodec/nvidia_video_decoder.h @@ -14,7 +14,7 @@ class NvidiaVideoDecoder : public VideoDecoder { public: int Init(); - int Decode(const ReceivedFrame& received_frame, + int Decode(std::unique_ptr received_frame, std::function on_receive_decoded_frame); std::string GetDecoderName() { return "NvidiaH264"; } diff --git a/src/media/video/decode/openh264/openh264_decoder.cpp b/src/media/video/decode/openh264/openh264_decoder.cpp index 9dbefa1..6d0336f 100644 --- a/src/media/video/decode/openh264/openh264_decoder.cpp +++ b/src/media/video/decode/openh264/openh264_decoder.cpp @@ -104,14 +104,18 @@ int OpenH264Decoder::Init() { } int OpenH264Decoder::Decode( - const ReceivedFrame &received_frame, + std::unique_ptr received_frame, std::function on_receive_decoded_frame) { if (!openh264_decoder_) { return -1; } - const uint8_t *data = received_frame.Buffer(); - size_t size = received_frame.Size(); + const uint8_t *data = received_frame->Buffer(); + size_t size = received_frame->Size(); + + if (data == nullptr) { + LOG_WARN("data is nullptr!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); + } #ifdef SAVE_RECEIVED_H264_STREAM fwrite((unsigned char *)data, 1, size, h264_stream_); @@ -180,8 +184,8 @@ int OpenH264Decoder::Decode( DecodedFrame decoded_frame(nv12_frame_, nv12_frame_capacity_, frame_width_, frame_height_); - decoded_frame.SetReceivedTimestamp(received_frame.ReceivedTimestamp()); - decoded_frame.SetCapturedTimestamp(received_frame.CapturedTimestamp()); + decoded_frame.SetReceivedTimestamp(received_frame->ReceivedTimestamp()); + decoded_frame.SetCapturedTimestamp(received_frame->CapturedTimestamp()); decoded_frame.SetDecodedTimestamp(clock_->CurrentTime()); #ifdef SAVE_DECODED_NV12_STREAM diff --git a/src/media/video/decode/openh264/openh264_decoder.h b/src/media/video/decode/openh264/openh264_decoder.h index 408ac48..4c162b3 100644 --- a/src/media/video/decode/openh264/openh264_decoder.h +++ b/src/media/video/decode/openh264/openh264_decoder.h @@ -24,7 +24,7 @@ class OpenH264Decoder : public VideoDecoder { public: int Init(); - int Decode(const ReceivedFrame& received_frame, + int Decode(std::unique_ptr received_frame, std::function on_receive_decoded_frame); std::string GetDecoderName() { return "OpenH264"; } @@ -47,7 +47,7 @@ class OpenH264Decoder : public VideoDecoder { int yuv420p_frame_capacity_ = 0; int yuv420p_frame_size_ = 0; - int nv12_frame_capacity_ = 0; + int nv12_frame_capacity_ = 0; int nv12_frame_size_ = 0; }; diff --git a/src/media/video/decode/video_decoder.h b/src/media/video/decode/video_decoder.h index a83548e..6c63055 100644 --- a/src/media/video/decode/video_decoder.h +++ b/src/media/video/decode/video_decoder.h @@ -22,7 +22,7 @@ class VideoDecoder { virtual int Init() = 0; virtual int Decode( - const ReceivedFrame& received_frame, + std::unique_ptr received_frame, std::function on_receive_decoded_frame) = 0; virtual std::string GetDecoderName() = 0; diff --git a/src/transport/channel/rtp_video_receiver.cpp b/src/transport/channel/rtp_video_receiver.cpp index 5fb9c7c..67b4479 100644 --- a/src/transport/channel/rtp_video_receiver.cpp +++ b/src/transport/channel/rtp_video_receiver.cpp @@ -733,7 +733,11 @@ bool RtpVideoReceiver::Process() { while (it != pending_frames_.end()) { if (it->second.is_complete) { if (on_receive_complete_frame_) { - on_receive_complete_frame_(*(it->second.frame)); + auto data = it->second.frame->Buffer(); + if (data == nullptr) { + LOG_WARN("data is nullptr!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); + } + on_receive_complete_frame_(std::move(it->second.frame)); } it = pending_frames_.erase(it); } else { diff --git a/src/transport/channel/rtp_video_receiver.h b/src/transport/channel/rtp_video_receiver.h index f48e2c8..9318f91 100644 --- a/src/transport/channel/rtp_video_receiver.h +++ b/src/transport/channel/rtp_video_receiver.h @@ -44,7 +44,8 @@ class RtpVideoReceiver : public ThreadBase, void SetSendDataFunc(std::function data_send_func); void SetOnReceiveCompleteFrame( - std::function on_receive_complete_frame) { + std::function)> + on_receive_complete_frame) { on_receive_complete_frame_ = on_receive_complete_frame; } uint32_t GetSsrc() { return ssrc_; } @@ -99,8 +100,8 @@ class RtpVideoReceiver : public ThreadBase, std::map incomplete_av1_frame_list_; std::map incomplete_frame_list_; uint8_t* nv12_data_ = nullptr; - std::function on_receive_complete_frame_ = - nullptr; + std::function)> + on_receive_complete_frame_ = nullptr; uint32_t last_complete_frame_ts_ = 0; RingBuffer compelete_video_frame_queue_; diff --git a/src/transport/channel/video_channel_receive.cpp b/src/transport/channel/video_channel_receive.cpp index 415b0f5..53c1383 100644 --- a/src/transport/channel/video_channel_receive.cpp +++ b/src/transport/channel/video_channel_receive.cpp @@ -7,7 +7,8 @@ VideoChannelReceive::VideoChannelReceive() {} VideoChannelReceive::VideoChannelReceive( std::shared_ptr clock, std::shared_ptr ice_agent, std::shared_ptr ice_io_statistics, - std::function on_receive_complete_frame) + std::function)> + on_receive_complete_frame) : ice_agent_(ice_agent), ice_io_statistics_(ice_io_statistics), on_receive_complete_frame_(on_receive_complete_frame), @@ -19,8 +20,8 @@ void VideoChannelReceive::Initialize(rtp::PAYLOAD_TYPE payload_type) { rtp_video_receiver_ = std::make_unique(clock_, ice_io_statistics_); rtp_video_receiver_->SetOnReceiveCompleteFrame( - [this](const ReceivedFrame &received_frame) -> void { - on_receive_complete_frame_(received_frame); + [this](std::unique_ptr received_frame) -> void { + on_receive_complete_frame_(std::move(received_frame)); }); rtp_video_receiver_->SetSendDataFunc([this](const char *data, diff --git a/src/transport/channel/video_channel_receive.h b/src/transport/channel/video_channel_receive.h index 14b70a9..5d8ed55 100644 --- a/src/transport/channel/video_channel_receive.h +++ b/src/transport/channel/video_channel_receive.h @@ -14,10 +14,11 @@ class VideoChannelReceive { public: VideoChannelReceive(); - VideoChannelReceive( - std::shared_ptr clock, std::shared_ptr ice_agent, - std::shared_ptr ice_io_statistics, - std::function on_receive_complete_frame); + VideoChannelReceive(std::shared_ptr clock, + std::shared_ptr ice_agent, + std::shared_ptr ice_io_statistics, + std::function)> + on_receive_complete_frame); ~VideoChannelReceive(); @@ -51,8 +52,8 @@ class VideoChannelReceive { std::shared_ptr ice_agent_ = nullptr; std::shared_ptr ice_io_statistics_ = nullptr; std::unique_ptr rtp_video_receiver_ = nullptr; - std::function on_receive_complete_frame_ = - nullptr; + std::function)> + on_receive_complete_frame_ = nullptr; private: std::shared_ptr clock_; diff --git a/src/transport/ice_transport_controller.cpp b/src/transport/ice_transport_controller.cpp index f46a677..78a4333 100644 --- a/src/transport/ice_transport_controller.cpp +++ b/src/transport/ice_transport_controller.cpp @@ -132,9 +132,9 @@ void IceTransportController::Create( std::weak_ptr weak_self = shared_from_this(); video_channel_receive_ = std::make_unique( clock_, ice_agent, ice_io_statistics, - [this, weak_self](const ReceivedFrame& received_frame) { + [this, weak_self](std::unique_ptr received_frame) { if (auto self = weak_self.lock()) { - OnReceiveCompleteFrame(received_frame); + OnReceiveCompleteFrame(std::move(received_frame)); } }); @@ -314,29 +314,33 @@ int IceTransportController::OnReceiveDataRtpPacket(const char* data, } void IceTransportController::OnReceiveCompleteFrame( - const ReceivedFrame& received_frame) { - task_queue_decode_->PostTask([this, received_frame]() mutable { - if (video_decoder_) { - int num_frame_returned = video_decoder_->Decode( - received_frame, [this](DecodedFrame decoded_frame) { - if (on_receive_video_) { - XVideoFrame x_video_frame; - x_video_frame.data = (const char*)decoded_frame.Buffer(); - x_video_frame.width = decoded_frame.Width(); - x_video_frame.height = decoded_frame.Height(); - x_video_frame.size = decoded_frame.Size(); - x_video_frame.captured_timestamp = - decoded_frame.CapturedTimestamp(); - x_video_frame.received_timestamp = - decoded_frame.ReceivedTimestamp(); - x_video_frame.decoded_timestamp = - decoded_frame.DecodedTimestamp(); - on_receive_video_(&x_video_frame, remote_user_id_.data(), - remote_user_id_.size(), user_data_); - } - }); - } - }); + std::unique_ptr received_frame) { + task_queue_decode_->PostTask( + [this, received_frame = std::move(received_frame)]() mutable { + if (video_decoder_) { + int num_frame_returned = video_decoder_->Decode( + std::move(received_frame), [this](DecodedFrame decoded_frame) { + if (on_receive_video_) { + XVideoFrame x_video_frame; + x_video_frame.data = (const char*)decoded_frame.Buffer(); + x_video_frame.width = decoded_frame.Width(); + x_video_frame.height = decoded_frame.Height(); + x_video_frame.size = decoded_frame.Size(); + x_video_frame.captured_timestamp = + decoded_frame.CapturedTimestamp(); + x_video_frame.received_timestamp = + decoded_frame.ReceivedTimestamp(); + x_video_frame.decoded_timestamp = + decoded_frame.DecodedTimestamp(); + + if (on_receive_video_) { + on_receive_video_(&x_video_frame, remote_user_id_.data(), + remote_user_id_.size(), user_data_); + } + } + }); + } + }); } void IceTransportController::OnReceiveCompleteAudio(const char* data, diff --git a/src/transport/ice_transport_controller.h b/src/transport/ice_transport_controller.h index ca83322..f34927c 100644 --- a/src/transport/ice_transport_controller.h +++ b/src/transport/ice_transport_controller.h @@ -65,7 +65,7 @@ class IceTransportController int OnReceiveAudioRtpPacket(const char *data, size_t size); int OnReceiveDataRtpPacket(const char *data, size_t size); - void OnReceiveCompleteFrame(const ReceivedFrame &received_frame); + void OnReceiveCompleteFrame(std::unique_ptr received_frame); void OnReceiveCompleteAudio(const char *data, size_t size); void OnReceiveCompleteData(const char *data, size_t size);