diff --git a/src/channel/video_channel_send.cpp b/src/channel/video_channel_send.cpp index bb1e63f..18a8c1e 100644 --- a/src/channel/video_channel_send.cpp +++ b/src/channel/video_channel_send.cpp @@ -57,10 +57,12 @@ void VideoChannelSend::Destroy() { } } -int VideoChannelSend::SendVideo(char* data, size_t size) { +int VideoChannelSend::SendVideo( + std::shared_ptr encoded_frame) { if (rtp_video_sender_ && rtp_packetizer_) { std::vector> rtp_packets = - rtp_packetizer_->Build((uint8_t*)data, (uint32_t)size, true); + rtp_packetizer_->Build((uint8_t*)encoded_frame->Buffer(), + (uint32_t)encoded_frame->Size(), true); rtp_video_sender_->Enqueue(rtp_packets); } diff --git a/src/channel/video_channel_send.h b/src/channel/video_channel_send.h index d8d6f6e..e8a659e 100644 --- a/src/channel/video_channel_send.h +++ b/src/channel/video_channel_send.h @@ -16,6 +16,7 @@ #include "rtp_packetizer.h" #include "rtp_video_sender.h" #include "transport_feedback_adapter.h" +#include "video_frame_wrapper.h" class VideoChannelSend { public: @@ -31,7 +32,7 @@ class VideoChannelSend { void Initialize(rtp::PAYLOAD_TYPE payload_type); void Destroy(); - int SendVideo(char* data, size_t size); + int SendVideo(std::shared_ptr encoded_frame); void OnCongestionControlFeedback( Timestamp recv_ts, diff --git a/src/frame/video_frame_wrapper.h b/src/frame/video_frame_wrapper.h new file mode 100644 index 0000000..e2777b5 --- /dev/null +++ b/src/frame/video_frame_wrapper.h @@ -0,0 +1,55 @@ +/* + * @Author: DI JUNKUN + * @Date: 2025-02-21 + * Copyright (c) 2025 by DI JUNKUN, All Rights Reserved. + */ + +#ifndef _VIDEO_FRAME_WRAPPER_H_ +#define _VIDEO_FRAME_WRAPPER_H_ + +#include "video_frame.h" + +enum VideoFrameType { + kEmptyFrame = 0, + kVideoFrameKey = 3, + kVideoFrameDelta = 4, +}; + +class VideoFrameWrapper : public VideoFrame { + public: + VideoFrameWrapper(const uint8_t *buffer, size_t size, uint32_t width, + uint32_t height) + : VideoFrame(buffer, size, width, height) {} + VideoFrameWrapper() = delete; + ~VideoFrameWrapper() = default; + + int64_t CaptureTimestamp() { return capture_timestamp_; } + + void SetCaptureTimestamp(int64_t capture_timestamp) { + capture_timestamp_ = capture_timestamp; + } + + VideoFrameType FrameType() { return frame_type_; } + + void SetFrameType(VideoFrameType frame_type) { frame_type_ = frame_type; } + + uint32_t EncodedWidth() { return encoded_width_; } + + void SetEncodedWidth(uint32_t encoded_width) { + encoded_width_ = encoded_width; + } + + uint32_t EncodedHeight() { return encoded_height_; } + + void SetEncodedHeight(uint32_t encoded_height) { + encoded_height_ = encoded_height; + } + + private: + int64_t capture_timestamp_ = 0; + VideoFrameType frame_type_ = VideoFrameType::kVideoFrameDelta; + uint32_t encoded_width_ = 0; + uint32_t encoded_height_ = 0; +}; + +#endif \ No newline at end of file diff --git a/src/interface/x.h b/src/interface/x.h index 2263f0c..20a6aac 100644 --- a/src/interface/x.h +++ b/src/interface/x.h @@ -44,6 +44,7 @@ typedef struct { size_t size; uint32_t width; uint32_t height; + uint64_t timestamp; } XVideoFrame; typedef struct { @@ -133,6 +134,8 @@ DLLAPI int SendAudioFrame(PeerPtr* peer_ptr, const char* data, size_t size); DLLAPI int SendDataFrame(PeerPtr* peer_ptr, const char* data, size_t size); +DLLAPI int64_t GetNowTime(PeerPtr* peer_ptr); + #ifdef __cplusplus } #endif diff --git a/src/media/video/encode/aom/aom_av1_encoder.cpp b/src/media/video/encode/aom/aom_av1_encoder.cpp index 2a70634..1f16ef3 100644 --- a/src/media/video/encode/aom/aom_av1_encoder.cpp +++ b/src/media/video/encode/aom/aom_av1_encoder.cpp @@ -266,10 +266,10 @@ int AomAv1Encoder::Init() { return 0; } -int AomAv1Encoder::Encode(const XVideoFrame *video_frame, - std::function - on_encoded_image) { +int AomAv1Encoder::Encode( + const XVideoFrame *video_frame, + std::function encoded_frame)> + on_encoded_image) { #ifdef SAVE_RECEIVED_NV12_STREAM fwrite(video_frame->data, 1, video_frame->size, file_nv12_); #endif @@ -342,8 +342,15 @@ int AomAv1Encoder::Encode(const XVideoFrame *video_frame, // LOG_INFO("Encoded frame qp = {}", qp); if (on_encoded_image) { - on_encoded_image((char *)encoded_frame_, encoded_frame_size_, - frame_type); + std::shared_ptr encoded_frame = + std::make_shared( + encoded_frame_, encoded_frame_size_, video_frame->width, + video_frame->height); + encoded_frame->SetFrameType(frame_type); + encoded_frame->SetCaptureTimestamp(video_frame->timestamp); + encoded_frame->SetEncodedWidth(video_frame->width); + encoded_frame->SetEncodedHeight(video_frame->height); + on_encoded_image(encoded_frame); #ifdef SAVE_ENCODED_AV1_STREAM fwrite(encoded_frame_, 1, encoded_frame_size_, file_av1_); #endif diff --git a/src/media/video/encode/aom/aom_av1_encoder.h b/src/media/video/encode/aom/aom_av1_encoder.h index 8514623..153da23 100644 --- a/src/media/video/encode/aom/aom_av1_encoder.h +++ b/src/media/video/encode/aom/aom_av1_encoder.h @@ -37,10 +37,10 @@ class AomAv1Encoder : public VideoEncoder { public: int Init(); - int Encode(const XVideoFrame* video_frame, - std::function - on_encoded_image); + int Encode( + const XVideoFrame* video_frame, + std::function encoded_frame)> + on_encoded_image); int ForceIdr(); diff --git a/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp b/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp index b8344dc..95ec817 100644 --- a/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp +++ b/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp @@ -130,8 +130,7 @@ int NvidiaVideoEncoder::Init() { int NvidiaVideoEncoder::Encode( const XVideoFrame *video_frame, - std::function + std::function encoded_frame)> on_encoded_image) { if (!encoder_) { LOG_ERROR("Invalid encoder"); @@ -182,7 +181,15 @@ int NvidiaVideoEncoder::Encode( for (const auto &packet : encoded_packets_) { if (on_encoded_image) { - on_encoded_image((char *)packet.data(), packet.size(), frame_type); + std::shared_ptr encoded_frame = + std::make_shared(packet.data(), packet.size(), + encoder_->GetEncodeWidth(), + encoder_->GetEncodeHeight()); + encoded_frame->SetFrameType(frame_type); + encoded_frame->SetCaptureTimestamp(video_frame->timestamp); + encoded_frame->SetEncodedWidth(encoder_->GetEncodeWidth()); + encoded_frame->SetEncodedHeight(encoder_->GetEncodeHeight()); + on_encoded_image(encoded_frame); #ifdef SAVE_ENCODED_H264_STREAM fwrite((unsigned char *)packet.data(), 1, packet.size(), file_h264_); #endif diff --git a/src/media/video/encode/nvcodec/nvidia_video_encoder.h b/src/media/video/encode/nvcodec/nvidia_video_encoder.h index dfca058..874b539 100644 --- a/src/media/video/encode/nvcodec/nvidia_video_encoder.h +++ b/src/media/video/encode/nvcodec/nvidia_video_encoder.h @@ -13,10 +13,10 @@ class NvidiaVideoEncoder : public VideoEncoder { int Init(); - int Encode(const XVideoFrame* video_frame, - std::function - on_encoded_image); + int Encode( + const XVideoFrame* video_frame, + std::function encoded_frame)> + on_encoded_image); int ForceIdr(); diff --git a/src/media/video/encode/openh264/openh264_encoder.cpp b/src/media/video/encode/openh264/openh264_encoder.cpp index 8101728..20313ab 100644 --- a/src/media/video/encode/openh264/openh264_encoder.cpp +++ b/src/media/video/encode/openh264/openh264_encoder.cpp @@ -181,8 +181,7 @@ int OpenH264Encoder::Init() { int OpenH264Encoder::Encode( const XVideoFrame *video_frame, - std::function + std::function encoded_frame)> on_encoded_image) { if (!openh264_encoder_) { LOG_ERROR("Invalid openh264 encoder"); @@ -282,7 +281,15 @@ int OpenH264Encoder::Encode( encoded_frame_size_ = encoded_frame_size; if (on_encoded_image) { - on_encoded_image((char *)encoded_frame_, encoded_frame_size_, frame_type); + std::shared_ptr encoded_frame = + std::make_shared(encoded_frame_, encoded_frame_size_, + raw_frame_.iPicWidth, + raw_frame_.iPicHeight); + encoded_frame->SetFrameType(frame_type); + encoded_frame->SetCaptureTimestamp(video_frame->timestamp); + encoded_frame->SetEncodedWidth(raw_frame_.iPicWidth); + encoded_frame->SetEncodedHeight(raw_frame_.iPicHeight); + on_encoded_image(encoded_frame); #ifdef SAVE_ENCODED_H264_STREAM fwrite(encoded_frame_, 1, encoded_frame_size_, file_h264_); #endif diff --git a/src/media/video/encode/openh264/openh264_encoder.h b/src/media/video/encode/openh264/openh264_encoder.h index 663775e..c2ae494 100644 --- a/src/media/video/encode/openh264/openh264_encoder.h +++ b/src/media/video/encode/openh264/openh264_encoder.h @@ -24,10 +24,10 @@ class OpenH264Encoder : public VideoEncoder { int Init(); - int Encode(const XVideoFrame* video_frame, - std::function - on_encoded_image); + int Encode( + const XVideoFrame* video_frame, + std::function encoded_frame)> + on_encoded_image); int ForceIdr(); diff --git a/src/media/video/encode/video_encoder.h b/src/media/video/encode/video_encoder.h index b51eb5e..0a3867d 100644 --- a/src/media/video/encode/video_encoder.h +++ b/src/media/video/encode/video_encoder.h @@ -5,25 +5,20 @@ #include #include #include +#include #include +#include "video_frame_wrapper.h" #include "x.h" class VideoEncoder { - public: - enum VideoFrameType { - kEmptyFrame = 0, - kVideoFrameKey = 3, - kVideoFrameDelta = 4, - }; - public: virtual int Init() = 0; - virtual int Encode(const XVideoFrame* video_frame, - std::function - on_encoded_image) = 0; + virtual int Encode( + const XVideoFrame* video_frame, + std::function encoded_frame)> + on_encoded_image) = 0; virtual int ForceIdr() = 0; diff --git a/src/pc/peer_connection.cpp b/src/pc/peer_connection.cpp index e727bae..7a0e266 100644 --- a/src/pc/peer_connection.cpp +++ b/src/pc/peer_connection.cpp @@ -390,6 +390,13 @@ int PeerConnection::SendDataFrame(const char *data, size_t size) { return 0; } +int64_t PeerConnection::CurrentTime() { + if (clock_) { + return clock_->CurrentTimeMs(); + } + return 0; +} + void PeerConnection::ProcessSignal(const std::string &signal) { auto j = json::parse(signal); std::string type = j["type"]; diff --git a/src/pc/peer_connection.h b/src/pc/peer_connection.h index 4f24a5e..a7f6a77 100644 --- a/src/pc/peer_connection.h +++ b/src/pc/peer_connection.h @@ -108,6 +108,8 @@ class PeerConnection { int SendAudioFrame(const char *data, size_t size); int SendDataFrame(const char *data, size_t size); + int64_t CurrentTime(); + private: int Login(); diff --git a/src/rtc/x_inner.cpp b/src/rtc/x_inner.cpp index f16aa53..9b223aa 100644 --- a/src/rtc/x_inner.cpp +++ b/src/rtc/x_inner.cpp @@ -52,8 +52,8 @@ void DestroyPeer(PeerPtr **peer_ptr) { } int Init(PeerPtr *peer_ptr, const char *user_id) { - if (!peer_ptr) { - LOG_ERROR("peer_ptr not created"); + if (!peer_ptr || !peer_ptr->peer_connection) { + LOG_ERROR("Peer connection not created"); return -1; } @@ -63,8 +63,8 @@ int Init(PeerPtr *peer_ptr, const char *user_id) { int CreateConnection(PeerPtr *peer_ptr, const char *transmission_id, const char *password) { - if (!peer_ptr) { - LOG_ERROR("peer_ptr not created"); + if (!peer_ptr || !peer_ptr->peer_connection) { + LOG_ERROR("Peer connection not created"); return -1; } @@ -77,8 +77,8 @@ int CreateConnection(PeerPtr *peer_ptr, const char *transmission_id, int JoinConnection(PeerPtr *peer_ptr, const char *transmission_id, const char *password) { int ret = -1; - if (!peer_ptr) { - LOG_ERROR("peer_ptr not created"); + if (!peer_ptr || !peer_ptr->peer_connection) { + LOG_ERROR("Peer connection not created"); return -1; } @@ -88,8 +88,8 @@ int JoinConnection(PeerPtr *peer_ptr, const char *transmission_id, } int LeaveConnection(PeerPtr *peer_ptr, const char *transmission_id) { - if (!peer_ptr) { - LOG_ERROR("peer_ptr not created"); + if (!peer_ptr || !peer_ptr->peer_connection) { + LOG_ERROR("Peer connection not created"); return -1; } @@ -99,8 +99,8 @@ int LeaveConnection(PeerPtr *peer_ptr, const char *transmission_id) { } DLLAPI int SendVideoFrame(PeerPtr *peer_ptr, const XVideoFrame *video_frame) { - if (!peer_ptr) { - LOG_ERROR("peer_ptr not created"); + if (!peer_ptr || !peer_ptr->peer_connection) { + LOG_ERROR("Peer connection not created"); return -1; } @@ -118,8 +118,8 @@ DLLAPI int SendVideoFrame(PeerPtr *peer_ptr, const XVideoFrame *video_frame) { } DLLAPI int SendAudioFrame(PeerPtr *peer_ptr, const char *data, size_t size) { - if (!peer_ptr) { - LOG_ERROR("peer_ptr not created"); + if (!peer_ptr || !peer_ptr->peer_connection) { + LOG_ERROR("Peer connection not created"); return -1; } @@ -134,8 +134,8 @@ DLLAPI int SendAudioFrame(PeerPtr *peer_ptr, const char *data, size_t size) { } int SendDataFrame(PeerPtr *peer_ptr, const char *data, size_t size) { - if (!peer_ptr) { - LOG_ERROR("peer_ptr not created"); + if (!peer_ptr || !peer_ptr->peer_connection) { + LOG_ERROR("Peer connection not created"); return -1; } @@ -147,4 +147,12 @@ int SendDataFrame(PeerPtr *peer_ptr, const char *data, size_t size) { peer_ptr->peer_connection->SendDataFrame(data, size); return 0; +} + +int64_t GetNowTime(PeerPtr *peer_ptr) { + if (!peer_ptr || !peer_ptr->peer_connection) { + LOG_ERROR("Peer connection not created"); + return -1; + } + return peer_ptr->peer_connection->CurrentTime(); } \ No newline at end of file diff --git a/src/transport/ice_transport_controller.cpp b/src/transport/ice_transport_controller.cpp index 4100e4a..f14d970 100644 --- a/src/transport/ice_transport_controller.cpp +++ b/src/transport/ice_transport_controller.cpp @@ -1,4 +1,6 @@ #include "ice_transport_controller.h" + +#include "video_frame_wrapper.h" #if __APPLE__ #else #include "nvcodec_api.h" @@ -128,10 +130,9 @@ int IceTransportController::SendVideo(const XVideoFrame* video_frame) { int ret = video_encoder_->Encode( video_frame, - [this](char* encoded_frame, size_t size, - VideoEncoder::VideoFrameType frame_type) -> int { + [this](std::shared_ptr encoded_frame) -> int { if (video_channel_send_) { - video_channel_send_->SendVideo(encoded_frame, size); + video_channel_send_->SendVideo(encoded_frame); } return 0;