diff --git a/src/frame/decoded_frame.h b/src/frame/decoded_frame.h index 1f0138a..9712741 100644 --- a/src/frame/decoded_frame.h +++ b/src/frame/decoded_frame.h @@ -44,9 +44,9 @@ class DecodedFrame : public VideoFrame { decoded_width_ = decoded_width; } - uint32_t decodedHeight() const { return decoded_height_; } + uint32_t DecodedHeight() const { return decoded_height_; } - void SetdecodedHeight(uint32_t decoded_height) { + void SetDecodedHeight(uint32_t decoded_height) { decoded_height_ = decoded_height; } diff --git a/src/frame/video_frame.cpp b/src/frame/video_frame.cpp index 5c0aea3..568d379 100644 --- a/src/frame/video_frame.cpp +++ b/src/frame/video_frame.cpp @@ -41,4 +41,14 @@ VideoFrame &VideoFrame::operator=(const VideoFrame &video_frame) = default; VideoFrame &VideoFrame::operator=(VideoFrame &&video_frame) = default; -VideoFrame::~VideoFrame() = default; \ No newline at end of file +VideoFrame::~VideoFrame() = default; + +void VideoFrame::UpdateBuffer(const uint8_t *new_buffer, size_t new_size) { + if (new_size > size_) { + buffer_ = CopyOnWriteBuffer(new_buffer, new_size); + size_ = new_size; + } else { + std::memcpy((void *)buffer_.data(), new_buffer, new_size); + size_ = new_size; + } +} \ No newline at end of file diff --git a/src/frame/video_frame.h b/src/frame/video_frame.h index 562e1eb..2f84aa7 100644 --- a/src/frame/video_frame.h +++ b/src/frame/video_frame.h @@ -43,6 +43,8 @@ class VideoFrame { void SetWidth(uint32_t width) { width_ = width; } void SetHeight(uint32_t height) { height_ = height; } + void UpdateBuffer(const uint8_t *new_buffer, size_t new_size); + private: CopyOnWriteBuffer buffer_; size_t size_ = 0; diff --git a/src/media/video/decode/aom/aom_av1_decoder.cpp b/src/media/video/decode/aom/aom_av1_decoder.cpp index e14378e..d3a4cc4 100644 --- a/src/media/video/decode/aom/aom_av1_decoder.cpp +++ b/src/media/video/decode/aom/aom_av1_decoder.cpp @@ -9,6 +9,10 @@ AomAv1Decoder::AomAv1Decoder(std::shared_ptr clock) : clock_(clock) {} AomAv1Decoder::~AomAv1Decoder() { + if (decoded_frame_) { + delete decoded_frame_; + } + #ifdef SAVE_DECODED_NV12_STREAM if (file_nv12_) { fflush(file_nv12_); @@ -48,6 +52,11 @@ int AomAv1Decoder::Init() { aom_codec_control(&aom_av1_decoder_ctx_, AV1D_GET_IMG_FORMAT, AOM_IMG_FMT_NV12); + if (!decoded_frame_) { + decoded_frame_ = new DecodedFrame(frame_width_ * frame_height_ * 3 / 2, + frame_width_, frame_height_); + } + #ifdef SAVE_DECODED_NV12_STREAM file_nv12_ = fopen("decoded_nv12_stream.yuv", "w+b"); if (!file_nv12_) { @@ -135,18 +144,20 @@ int AomAv1Decoder::Decode( uv_data[2 * i + 1] = v_plane[i]; } - DecodedFrame decode_frame(nv12_data.data(), nv12_size, img_->d_w, - img_->d_h); - - decode_frame.SetReceivedTimestamp(received_frame->ReceivedTimestamp()); - decode_frame.SetCapturedTimestamp(received_frame->CapturedTimestamp()); - decode_frame.SetDecodedTimestamp(clock_->CurrentTime()); + decoded_frame_->UpdateBuffer(nv12_data.data(), nv12_size); + decoded_frame_->SetWidth(received_frame->Width()); + decoded_frame_->SetHeight(received_frame->Height()); + decoded_frame_->SetDecodedWidth(img_->d_w); + decoded_frame_->SetDecodedHeight(img_->d_h); + decoded_frame_->SetReceivedTimestamp(received_frame->ReceivedTimestamp()); + decoded_frame_->SetCapturedTimestamp(received_frame->CapturedTimestamp()); + decoded_frame_->SetDecodedTimestamp(clock_->CurrentTime()); #ifdef SAVE_DECODED_NV12_STREAM - fwrite((unsigned char *)decode_frame.Buffer(), 1, decode_frame.Size(), + fwrite((unsigned char *)decoded_frame_->Buffer(), 1, decoded_frame_->Size(), file_nv12_); #endif - on_receive_decoded_frame(decode_frame); + on_receive_decoded_frame(*decoded_frame_); return 0; } diff --git a/src/media/video/decode/aom/aom_av1_decoder.h b/src/media/video/decode/aom/aom_av1_decoder.h index aa787ec..e69a3ee 100644 --- a/src/media/video/decode/aom/aom_av1_decoder.h +++ b/src/media/video/decode/aom/aom_av1_decoder.h @@ -30,6 +30,7 @@ class AomAv1Decoder : public VideoDecoder { private: std::shared_ptr clock_ = nullptr; + DecodedFrame *decoded_frame_ = nullptr; unsigned char *nv12_frame_ = 0; int nv12_frame_capacity_ = 0; int nv12_frame_size_ = 0; diff --git a/src/media/video/decode/dav1d/dav1d_av1_decoder.cpp b/src/media/video/decode/dav1d/dav1d_av1_decoder.cpp index fbcb4a3..b2dd2d6 100644 --- a/src/media/video/decode/dav1d/dav1d_av1_decoder.cpp +++ b/src/media/video/decode/dav1d/dav1d_av1_decoder.cpp @@ -51,6 +51,10 @@ Dav1dAv1Decoder::Dav1dAv1Decoder(std::shared_ptr clock) : clock_(clock) {} Dav1dAv1Decoder::~Dav1dAv1Decoder() { + if (decoded_frame_) { + delete decoded_frame_; + } + #ifdef SAVE_DECODED_NV12_STREAM if (file_nv12_) { fflush(file_nv12_); @@ -89,6 +93,11 @@ int Dav1dAv1Decoder::Init() { LOG_ERROR("Dav1d AV1 decoder open failed"); } + if (!decoded_frame_) { + decoded_frame_ = new DecodedFrame(frame_width_ * frame_height_ * 3 / 2, + frame_width_, frame_height_); + } + #ifdef SAVE_DECODED_NV12_STREAM file_nv12_ = fopen("decoded_nv12_stream.yuv", "w+b"); if (!file_nv12_) { @@ -187,18 +196,20 @@ int Dav1dAv1Decoder::Decode( frame_width_, frame_height_); } - DecodedFrame decoded_frame(nv12_frame_, nv12_frame_capacity_, frame_width_, - frame_height_); - - decoded_frame.SetReceivedTimestamp(received_frame->ReceivedTimestamp()); - decoded_frame.SetCapturedTimestamp(received_frame->CapturedTimestamp()); - decoded_frame.SetDecodedTimestamp(clock_->CurrentTime()); + decoded_frame_->UpdateBuffer(nv12_frame_, nv12_frame_capacity_); + decoded_frame_->SetWidth(received_frame->Width()); + decoded_frame_->SetHeight(received_frame->Height()); + decoded_frame_->SetDecodedWidth(frame_width_); + decoded_frame_->SetDecodedHeight(frame_height_); + decoded_frame_->SetReceivedTimestamp(received_frame->ReceivedTimestamp()); + decoded_frame_->SetCapturedTimestamp(received_frame->CapturedTimestamp()); + decoded_frame_->SetDecodedTimestamp(clock_->CurrentTime()); #ifdef SAVE_DECODED_NV12_STREAM - fwrite((unsigned char *)decoded_frame.Buffer(), 1, decoded_frame.Size(), + fwrite((unsigned char *)decoded_frame_->Buffer(), 1, decoded_frame_->Size(), file_nv12_); #endif - on_receive_decoded_frame(decoded_frame); + on_receive_decoded_frame(*decoded_frame_); return 0; } \ No newline at end of file diff --git a/src/media/video/decode/dav1d/dav1d_av1_decoder.h b/src/media/video/decode/dav1d/dav1d_av1_decoder.h index 67b3346..8927186 100644 --- a/src/media/video/decode/dav1d/dav1d_av1_decoder.h +++ b/src/media/video/decode/dav1d/dav1d_av1_decoder.h @@ -28,6 +28,7 @@ class Dav1dAv1Decoder : public VideoDecoder { private: std::shared_ptr clock_ = nullptr; + DecodedFrame *decoded_frame_ = nullptr; unsigned char *nv12_frame_ = 0; size_t nv12_frame_capacity_ = 0; size_t nv12_frame_size_ = 0; diff --git a/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp b/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp index 2d78da3..49b19b2 100644 --- a/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp +++ b/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp @@ -8,6 +8,10 @@ NvidiaVideoDecoder::NvidiaVideoDecoder(std::shared_ptr clock) : clock_(clock) {} NvidiaVideoDecoder::~NvidiaVideoDecoder() { + if (decoded_frame_) { + delete decoded_frame_; + } + #ifdef SAVE_DECODED_NV12_STREAM if (file_nv12_) { fflush(file_nv12_); @@ -51,6 +55,11 @@ int NvidiaVideoDecoder::Init() { new NvDecoder(cuda_context_, false, cudaVideoCodec_H264, true, false, nullptr, nullptr, false, 4096, 2160, 1000, false); + if (!decoded_frame_) { + decoded_frame_ = new DecodedFrame(frame_width_ * frame_height_ * 3 / 2, + frame_width_, frame_height_); + } + #ifdef SAVE_DECODED_NV12_STREAM file_nv12_ = fopen("decoded_nv12_stream.yuv", "w+b"); if (!file_nv12_) { @@ -94,20 +103,24 @@ int NvidiaVideoDecoder::Decode( decoded_frame_buffer = decoder->GetFrame(); if (decoded_frame_buffer) { if (on_receive_decoded_frame) { - DecodedFrame decoded_frame( + decoded_frame_->UpdateBuffer( decoded_frame_buffer, - decoder->GetWidth() * decoder->GetHeight() * 3 / 2, - decoder->GetWidth(), decoder->GetHeight()); - decoded_frame.SetReceivedTimestamp( + decoder->GetWidth() * decoder->GetHeight() * 3 / 2); + decoded_frame_->SetWidth(received_frame->Width()); + decoded_frame_->SetHeight(received_frame->Height()); + decoded_frame_->SetDecodedWidth(decoder->GetWidth()); + decoded_frame_->SetDecodedHeight(decoder->GetHeight()); + decoded_frame_->SetReceivedTimestamp( received_frame->ReceivedTimestamp()); - decoded_frame.SetCapturedTimestamp( + decoded_frame_->SetCapturedTimestamp( received_frame->CapturedTimestamp()); - decoded_frame.SetDecodedTimestamp(clock_->CurrentTime()); - on_receive_decoded_frame(decoded_frame); + decoded_frame_->SetDecodedTimestamp(clock_->CurrentTime()); + #ifdef SAVE_DECODED_NV12_STREAM - fwrite((unsigned char *)decoded_frame.Buffer(), 1, - decoded_frame.Size(), file_nv12_); + fwrite((unsigned char *)decoded_frame_->Buffer(), 1, + decoded_frame_->Size(), file_nv12_); #endif + on_receive_decoded_frame(*decoded_frame_); } } } diff --git a/src/media/video/decode/nvcodec/nvidia_video_decoder.h b/src/media/video/decode/nvcodec/nvidia_video_decoder.h index a2f13d7..f015859 100644 --- a/src/media/video/decode/nvcodec/nvidia_video_decoder.h +++ b/src/media/video/decode/nvcodec/nvidia_video_decoder.h @@ -24,6 +24,7 @@ class NvidiaVideoDecoder : public VideoDecoder { NvDecoder* decoder = nullptr; CUcontext cuda_context_ = NULL; CUdevice cuda_device_ = 0; + DecodedFrame* decoded_frame_ = nullptr; bool get_first_keyframe_ = false; bool skip_frame_ = false; diff --git a/src/media/video/decode/openh264/openh264_decoder.cpp b/src/media/video/decode/openh264/openh264_decoder.cpp index 6d0336f..7fe0b10 100644 --- a/src/media/video/decode/openh264/openh264_decoder.cpp +++ b/src/media/video/decode/openh264/openh264_decoder.cpp @@ -46,6 +46,10 @@ OpenH264Decoder::~OpenH264Decoder() { delete[] nv12_frame_; } + if (decoded_frame_) { + delete decoded_frame_; + } + if (yuv420p_frame_) { delete[] yuv420p_frame_; } @@ -100,6 +104,11 @@ int OpenH264Decoder::Init() { int trace_level = WELS_LOG_QUIET; openh264_decoder_->SetOption(DECODER_OPTION_TRACE_LEVEL, &trace_level); + if (!decoded_frame_) { + decoded_frame_ = new DecodedFrame(frame_width_ * frame_height_ * 3 / 2, + frame_width_, frame_height_); + } + return 0; } @@ -182,17 +191,20 @@ int OpenH264Decoder::Decode( (uint8_t *)nv12_frame_ + frame_width_ * frame_height_, frame_width_, frame_width_, frame_height_); - DecodedFrame decoded_frame(nv12_frame_, nv12_frame_capacity_, - frame_width_, frame_height_); - decoded_frame.SetReceivedTimestamp(received_frame->ReceivedTimestamp()); - decoded_frame.SetCapturedTimestamp(received_frame->CapturedTimestamp()); - decoded_frame.SetDecodedTimestamp(clock_->CurrentTime()); + decoded_frame_->UpdateBuffer(nv12_frame_, nv12_frame_capacity_); + decoded_frame_->SetWidth(received_frame->Width()); + decoded_frame_->SetHeight(received_frame->Height()); + decoded_frame_->SetDecodedWidth(frame_width_); + decoded_frame_->SetDecodedHeight(frame_height_); + decoded_frame_->SetReceivedTimestamp(received_frame->ReceivedTimestamp()); + decoded_frame_->SetCapturedTimestamp(received_frame->CapturedTimestamp()); + decoded_frame_->SetDecodedTimestamp(clock_->CurrentTime()); #ifdef SAVE_DECODED_NV12_STREAM - fwrite((unsigned char *)decoded_frame.Buffer(), 1, decoded_frame.Size(), - nv12_stream_); + fwrite((unsigned char *)decoded_frame_->Buffer(), 1, + decoded_frame_->Size(), nv12_stream_); #endif - on_receive_decoded_frame(decoded_frame); + on_receive_decoded_frame(*decoded_frame_); } } diff --git a/src/media/video/decode/openh264/openh264_decoder.h b/src/media/video/decode/openh264/openh264_decoder.h index 4c162b3..fbdb09e 100644 --- a/src/media/video/decode/openh264/openh264_decoder.h +++ b/src/media/video/decode/openh264/openh264_decoder.h @@ -32,12 +32,11 @@ class OpenH264Decoder : public VideoDecoder { private: std::shared_ptr clock_ = nullptr; ISVCDecoder* openh264_decoder_ = nullptr; + DecodedFrame* decoded_frame_ = nullptr; bool get_first_keyframe_ = false; bool skip_frame_ = false; FILE* nv12_stream_ = nullptr; FILE* h264_stream_ = nullptr; - uint8_t* decoded_frame_ = nullptr; - int decoded_frame_size_ = 0; uint32_t frame_width_ = 1280; uint32_t frame_height_ = 720; diff --git a/src/transport/ice_transport_controller.cpp b/src/transport/ice_transport_controller.cpp index 93977cc..ebf1ffe 100644 --- a/src/transport/ice_transport_controller.cpp +++ b/src/transport/ice_transport_controller.cpp @@ -314,32 +314,33 @@ int IceTransportController::OnReceiveDataRtpPacket(const char* data, void IceTransportController::OnReceiveCompleteFrame( std::unique_ptr received_frame) { - task_queue_decode_->PostTask( - [this, received_frame = std::move(received_frame)]() mutable { - if (video_decoder_) { - int num_frame_returned = video_decoder_->Decode( - std::move(received_frame), [this](DecodedFrame decoded_frame) { - if (on_receive_video_) { - XVideoFrame x_video_frame; - x_video_frame.data = (const char*)decoded_frame.Buffer(); - x_video_frame.width = decoded_frame.Width(); - x_video_frame.height = decoded_frame.Height(); - x_video_frame.size = decoded_frame.Size(); - x_video_frame.captured_timestamp = - decoded_frame.CapturedTimestamp(); - x_video_frame.received_timestamp = - decoded_frame.ReceivedTimestamp(); - x_video_frame.decoded_timestamp = - decoded_frame.DecodedTimestamp(); + task_queue_decode_->PostTask([this, received_frame = + std::move(received_frame)]() mutable { + uint64_t t = clock_->CurrentTime(); + if (video_decoder_) { + int num_frame_returned = video_decoder_->Decode( + std::move(received_frame), [this](const DecodedFrame& decoded_frame) { + if (on_receive_video_) { + XVideoFrame x_video_frame; + x_video_frame.data = (const char*)decoded_frame.Buffer(); + x_video_frame.width = decoded_frame.DecodedWidth(); + x_video_frame.height = decoded_frame.DecodedHeight(); + x_video_frame.size = decoded_frame.Size(); + x_video_frame.captured_timestamp = + decoded_frame.CapturedTimestamp(); + x_video_frame.received_timestamp = + decoded_frame.ReceivedTimestamp(); + x_video_frame.decoded_timestamp = + decoded_frame.DecodedTimestamp(); - if (on_receive_video_) { - on_receive_video_(&x_video_frame, remote_user_id_.data(), - remote_user_id_.size(), user_data_); - } - } - }); - } - }); + if (on_receive_video_) { + on_receive_video_(&x_video_frame, remote_user_id_.data(), + remote_user_id_.size(), user_data_); + } + } + }); + } + }); } void IceTransportController::OnReceiveCompleteAudio(const char* data,