From 1cd9ea1b0ece15b1df54ffaf7dc858dea7068e09 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Wed, 19 Mar 2025 14:35:48 +0800 Subject: [PATCH] [fix] fix video frame capture timestamp --- src/common/clock/system_clock.cpp | 4 ++-- src/common/clock/system_clock.h | 2 +- src/frame/video_frame_wrapper.h | 8 ++++---- src/interface/x.h | 2 +- src/pc/peer_connection.cpp | 4 ++-- src/pc/peer_connection.h | 2 +- src/rtc/x_inner.cpp | 4 ++-- src/rtp/rtp_packetizer/rtp_packetizer.h | 4 ++-- src/rtp/rtp_packetizer/rtp_packetizer_av1.cpp | 2 +- src/rtp/rtp_packetizer/rtp_packetizer_av1.h | 4 ++-- .../rtp_packetizer/rtp_packetizer_generic.cpp | 2 +- src/rtp/rtp_packetizer/rtp_packetizer_generic.h | 4 ++-- src/rtp/rtp_packetizer/rtp_packetizer_h264.cpp | 16 ++++++++-------- src/rtp/rtp_packetizer/rtp_packetizer_h264.h | 8 ++++---- src/transport/channel/rtp_video_sender.cpp | 2 +- src/transport/channel/rtp_video_sender.h | 2 +- src/transport/channel/video_channel_send.cpp | 4 ++-- src/transport/channel/video_channel_send.h | 2 +- src/transport/ice_transport_controller.cpp | 5 +++-- src/transport/packet_sender/packet_sender.h | 2 +- .../packet_sender/packet_sender_imp.cpp | 4 ++-- src/transport/packet_sender/packet_sender_imp.h | 8 +------- 22 files changed, 45 insertions(+), 50 deletions(-) diff --git a/src/common/clock/system_clock.cpp b/src/common/clock/system_clock.cpp index 3d33d4a..b3d48ed 100644 --- a/src/common/clock/system_clock.cpp +++ b/src/common/clock/system_clock.cpp @@ -68,12 +68,12 @@ int64_t SystemClock::CurrentTimeNs() { return ticks; } +int64_t SystemClock::CurrentTime() { return CurrentTimeNs() / 1000LL; } + int64_t SystemClock::CurrentTimeUs() { return CurrentTimeNs() / 1000LL; } int64_t SystemClock::CurrentTimeMs() { return CurrentTimeNs() / 1000000LL; } -int64_t SystemClock::CurrentTime() { return CurrentTimeNs() / 1000000000LL; } - int64_t SystemClock::CurrentNtpTime() { return ConvertToNtpTime(CurrentTimeNs()); } diff --git a/src/common/clock/system_clock.h b/src/common/clock/system_clock.h index 0bde1d9..e20a3c5 100644 --- a/src/common/clock/system_clock.h +++ b/src/common/clock/system_clock.h @@ -18,8 +18,8 @@ class SystemClock { ~SystemClock() = default; int64_t CurrentTime(); - int64_t CurrentTimeMs(); int64_t CurrentTimeUs(); + int64_t CurrentTimeMs(); int64_t CurrentTimeNs(); int64_t CurrentNtpTime(); diff --git a/src/frame/video_frame_wrapper.h b/src/frame/video_frame_wrapper.h index 986b9b8..5d822ba 100644 --- a/src/frame/video_frame_wrapper.h +++ b/src/frame/video_frame_wrapper.h @@ -23,10 +23,10 @@ class VideoFrameWrapper : public VideoFrame { VideoFrameWrapper() = delete; ~VideoFrameWrapper() = default; - int64_t CaptureTimestamp() { return capture_timestamp_ms_; } + int64_t CaptureTimestamp() { return capture_timestamp_us_; } - void SetCaptureTimestamp(int64_t capture_timestamp_ms) { - capture_timestamp_ms_ = capture_timestamp_ms; + void SetCaptureTimestamp(int64_t capture_timestamp_us) { + capture_timestamp_us_ = capture_timestamp_us; } VideoFrameType FrameType() { return frame_type_; } @@ -46,7 +46,7 @@ class VideoFrameWrapper : public VideoFrame { } private: - int64_t capture_timestamp_ms_ = 0; + int64_t capture_timestamp_us_ = 0; VideoFrameType frame_type_ = VideoFrameType::kVideoFrameDelta; uint32_t encoded_width_ = 0; uint32_t encoded_height_ = 0; diff --git a/src/interface/x.h b/src/interface/x.h index 7ced46c..7897d1a 100644 --- a/src/interface/x.h +++ b/src/interface/x.h @@ -134,7 +134,7 @@ DLLAPI int SendAudioFrame(PeerPtr* peer_ptr, const char* data, size_t size); DLLAPI int SendDataFrame(PeerPtr* peer_ptr, const char* data, size_t size); -DLLAPI int64_t CurrentNtpTimeMs(PeerPtr* peer_ptr); +DLLAPI int64_t GetSystemTimeMicros(PeerPtr* peer_ptr); #ifdef __cplusplus } diff --git a/src/pc/peer_connection.cpp b/src/pc/peer_connection.cpp index c4d2e22..5972bb5 100644 --- a/src/pc/peer_connection.cpp +++ b/src/pc/peer_connection.cpp @@ -390,9 +390,9 @@ int PeerConnection::SendDataFrame(const char *data, size_t size) { return 0; } -int64_t PeerConnection::CurrentNtpTimeMs() { +int64_t PeerConnection::GetSystemTimeMicros() { if (clock_) { - return clock_->CurrentNtpTimeMs(); + return clock_->CurrentTimeUs(); } return 0; } diff --git a/src/pc/peer_connection.h b/src/pc/peer_connection.h index d39f04d..bd3c19b 100644 --- a/src/pc/peer_connection.h +++ b/src/pc/peer_connection.h @@ -108,7 +108,7 @@ class PeerConnection { int SendAudioFrame(const char *data, size_t size); int SendDataFrame(const char *data, size_t size); - int64_t CurrentNtpTimeMs(); + int64_t GetSystemTimeMicros(); private: int Login(); diff --git a/src/rtc/x_inner.cpp b/src/rtc/x_inner.cpp index 6524319..6f9eaae 100644 --- a/src/rtc/x_inner.cpp +++ b/src/rtc/x_inner.cpp @@ -149,10 +149,10 @@ int SendDataFrame(PeerPtr *peer_ptr, const char *data, size_t size) { return 0; } -int64_t CurrentNtpTimeMs(PeerPtr *peer_ptr) { +int64_t GetSystemTimeMicros(PeerPtr *peer_ptr) { if (!peer_ptr || !peer_ptr->peer_connection) { LOG_ERROR("Peer connection not created"); return -1; } - return peer_ptr->peer_connection->CurrentNtpTimeMs(); + return peer_ptr->peer_connection->GetSystemTimeMicros(); } \ No newline at end of file diff --git a/src/rtp/rtp_packetizer/rtp_packetizer.h b/src/rtp/rtp_packetizer/rtp_packetizer.h index 7911afb..d3b9949 100644 --- a/src/rtp/rtp_packetizer/rtp_packetizer.h +++ b/src/rtp/rtp_packetizer/rtp_packetizer.h @@ -22,11 +22,11 @@ class RtpPacketizer { virtual ~RtpPacketizer() = default; virtual std::vector> Build( - uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_ms, + uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_us, bool use_rtp_packet_to_send) = 0; virtual std::vector> BuildPadding( - uint32_t payload_size, int64_t capture_timestamp_ms, + uint32_t payload_size, int64_t capture_timestamp_us, bool use_rtp_packet_to_send) = 0; }; diff --git a/src/rtp/rtp_packetizer/rtp_packetizer_av1.cpp b/src/rtp/rtp_packetizer/rtp_packetizer_av1.cpp index 6b97162..43f6dff 100644 --- a/src/rtp/rtp_packetizer/rtp_packetizer_av1.cpp +++ b/src/rtp/rtp_packetizer/rtp_packetizer_av1.cpp @@ -5,7 +5,7 @@ RtpPacketizerAv1::RtpPacketizerAv1(uint32_t ssrc) {} RtpPacketizerAv1::~RtpPacketizerAv1() {} std::vector> RtpPacketizerAv1::Build( - uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_ms, + uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_us, bool use_rtp_packet_to_send) { std::vector> rtp_packets; diff --git a/src/rtp/rtp_packetizer/rtp_packetizer_av1.h b/src/rtp/rtp_packetizer/rtp_packetizer_av1.h index 4c21ad9..51024c2 100644 --- a/src/rtp/rtp_packetizer/rtp_packetizer_av1.h +++ b/src/rtp/rtp_packetizer/rtp_packetizer_av1.h @@ -16,11 +16,11 @@ class RtpPacketizerAv1 : public RtpPacketizer { virtual ~RtpPacketizerAv1(); std::vector> Build( - uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_ms, + uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_us, bool use_rtp_packet_to_send) override; std::vector> BuildPadding( - uint32_t payload_size, int64_t capture_timestamp_ms, + uint32_t payload_size, int64_t capture_timestamp_us, bool use_rtp_packet_to_send) override { return std::vector>{}; }; diff --git a/src/rtp/rtp_packetizer/rtp_packetizer_generic.cpp b/src/rtp/rtp_packetizer/rtp_packetizer_generic.cpp index 8060dd0..f3c8318 100644 --- a/src/rtp/rtp_packetizer/rtp_packetizer_generic.cpp +++ b/src/rtp/rtp_packetizer/rtp_packetizer_generic.cpp @@ -47,7 +47,7 @@ void RtpPacketizerGeneric::AddAbsSendTimeExtension( } std::vector> RtpPacketizerGeneric::Build( - uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_ms, + uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_us, bool use_rtp_packet_to_send) { uint32_t last_packet_size = payload_size % MAX_NALU_LEN; uint32_t packet_num = diff --git a/src/rtp/rtp_packetizer/rtp_packetizer_generic.h b/src/rtp/rtp_packetizer/rtp_packetizer_generic.h index fed0cdb..9b647f8 100644 --- a/src/rtp/rtp_packetizer/rtp_packetizer_generic.h +++ b/src/rtp/rtp_packetizer/rtp_packetizer_generic.h @@ -16,11 +16,11 @@ class RtpPacketizerGeneric : public RtpPacketizer { virtual ~RtpPacketizerGeneric(); std::vector> Build( - uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_ms, + uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_us, bool use_rtp_packet_to_send) override; std::vector> BuildPadding( - uint32_t payload_size, int64_t capture_timestamp_ms, + uint32_t payload_size, int64_t capture_timestamp_us, bool use_rtp_packet_to_send) override { return std::vector>{}; }; diff --git a/src/rtp/rtp_packetizer/rtp_packetizer_h264.cpp b/src/rtp/rtp_packetizer/rtp_packetizer_h264.cpp index 63f51d9..451b657 100644 --- a/src/rtp/rtp_packetizer/rtp_packetizer_h264.cpp +++ b/src/rtp/rtp_packetizer/rtp_packetizer_h264.cpp @@ -61,19 +61,19 @@ void RtpPacketizerH264::AddAbsSendTimeExtension( } std::vector> RtpPacketizerH264::Build( - uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_ms, + uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_us, bool use_rtp_packet_to_send) { if (payload_size <= MAX_NALU_LEN) { - return BuildNalu(payload, payload_size, capture_timestamp_ms, + return BuildNalu(payload, payload_size, capture_timestamp_us, use_rtp_packet_to_send); } else { - return BuildFua(payload, payload_size, capture_timestamp_ms, + return BuildFua(payload, payload_size, capture_timestamp_us, use_rtp_packet_to_send); } } std::vector> RtpPacketizerH264::BuildNalu( - uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_ms, + uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_us, bool use_rtp_packet_to_send) { std::vector> rtp_packets; @@ -84,7 +84,7 @@ std::vector> RtpPacketizerH264::BuildNalu( marker_ = 1; payload_type_ = rtp::PAYLOAD_TYPE(payload_type_); sequence_number_++; - timestamp_ = kMsToRtpTimestamp * static_cast(capture_timestamp_ms); + timestamp_ = kMsToRtpTimestamp * static_cast(capture_timestamp_us); if (!csrc_count_) { } @@ -142,7 +142,7 @@ std::vector> RtpPacketizerH264::BuildNalu( } std::vector> RtpPacketizerH264::BuildFua( - uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_ms, + uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_us, bool use_rtp_packet_to_send) { std::vector> rtp_packets; @@ -242,7 +242,7 @@ std::vector> RtpPacketizerH264::BuildFua( } std::vector> RtpPacketizerH264::BuildPadding( - uint32_t payload_size, int64_t capture_timestamp_ms, + uint32_t payload_size, int64_t capture_timestamp_us, bool use_rtp_packet_to_send) { std::vector> rtp_packets; @@ -259,7 +259,7 @@ std::vector> RtpPacketizerH264::BuildPadding( uint8_t payload_type = rtp::PAYLOAD_TYPE(payload_type_ - 1); sequence_number_++; timestamp_ = - kMsToRtpTimestamp * static_cast(capture_timestamp_ms); + kMsToRtpTimestamp * static_cast(capture_timestamp_us); rtp_packet_frame_.clear(); rtp_packet_frame_.push_back((version_ << 6) | (has_padding_ << 5) | diff --git a/src/rtp/rtp_packetizer/rtp_packetizer_h264.h b/src/rtp/rtp_packetizer/rtp_packetizer_h264.h index 00fccc4..3016709 100644 --- a/src/rtp/rtp_packetizer/rtp_packetizer_h264.h +++ b/src/rtp/rtp_packetizer/rtp_packetizer_h264.h @@ -16,20 +16,20 @@ class RtpPacketizerH264 : public RtpPacketizer { virtual ~RtpPacketizerH264(); std::vector> Build( - uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_ms, + uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_us, bool use_rtp_packet_to_send) override; std::vector> BuildNalu( - uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_ms, + uint8_t* payload, uint32_t payload_size, int64_t capture_timestamp_us, bool use_rtp_packet_to_send); std::vector> BuildFua(uint8_t* payload, uint32_t payload_size, - int64_t capture_timestamp_ms, + int64_t capture_timestamp_us, bool use_rtp_packet_to_send); std::vector> BuildPadding( - uint32_t payload_size, int64_t capture_timestamp_ms, + uint32_t payload_size, int64_t capture_timestamp_us, bool use_rtp_packet_to_send) override; private: diff --git a/src/transport/channel/rtp_video_sender.cpp b/src/transport/channel/rtp_video_sender.cpp index 72b88ba..9641529 100644 --- a/src/transport/channel/rtp_video_sender.cpp +++ b/src/transport/channel/rtp_video_sender.cpp @@ -45,7 +45,7 @@ RtpVideoSender::~RtpVideoSender() { void RtpVideoSender::Enqueue( std::vector>& rtp_packets, - int64_t capture_timestamp_ms) { + int64_t capture_timestamp_us) { if (!rtp_statistics_) { rtp_statistics_ = std::make_unique(); rtp_statistics_->Start(); diff --git a/src/transport/channel/rtp_video_sender.h b/src/transport/channel/rtp_video_sender.h index 32bed4f..00e21f5 100644 --- a/src/transport/channel/rtp_video_sender.h +++ b/src/transport/channel/rtp_video_sender.h @@ -24,7 +24,7 @@ class RtpVideoSender : public ThreadBase { public: void Enqueue(std::vector> &rtp_packets, - int64_t capture_timestamp_ms); + int64_t capture_timestamp_us); void SetSendDataFunc(std::function data_send_func); void SetOnSentPacketFunc( std::function on_sent_packet_func); diff --git a/src/transport/channel/video_channel_send.cpp b/src/transport/channel/video_channel_send.cpp index 33cc575..ce50727 100644 --- a/src/transport/channel/video_channel_send.cpp +++ b/src/transport/channel/video_channel_send.cpp @@ -57,9 +57,9 @@ void VideoChannelSend::SetEnqueuePacketsFunc( } std::vector> VideoChannelSend::GeneratePadding( - uint32_t payload_size, int64_t capture_timestamp_ms) { + uint32_t payload_size, int64_t capture_timestamp_us) { if (rtp_packetizer_) { - return rtp_packetizer_->BuildPadding(payload_size, capture_timestamp_ms, + return rtp_packetizer_->BuildPadding(payload_size, capture_timestamp_us, true); } return std::vector>{}; diff --git a/src/transport/channel/video_channel_send.h b/src/transport/channel/video_channel_send.h index c808cbf..dd2db2a 100644 --- a/src/transport/channel/video_channel_send.h +++ b/src/transport/channel/video_channel_send.h @@ -36,7 +36,7 @@ class VideoChannelSend { enqueue_packets_func); std::vector> GeneratePadding( - uint32_t payload_size, int64_t capture_timestamp_ms); + uint32_t payload_size, int64_t capture_timestamp_us); int64_t GetTransportSeqAndIncrement() { int64_t transport_seq = rtp_video_sender_->GetTransportSequenceNumber(); diff --git a/src/transport/ice_transport_controller.cpp b/src/transport/ice_transport_controller.cpp index fded51b..8c94b3d 100644 --- a/src/transport/ice_transport_controller.cpp +++ b/src/transport/ice_transport_controller.cpp @@ -75,9 +75,9 @@ void IceTransportController::Create( }); packet_sender_->SetGeneratePaddingFunc( - [this](uint32_t size, int64_t capture_timestamp_ms) + [this](uint32_t size, int64_t capture_timestamp_us) -> std::vector> { - return video_channel_send_->GeneratePadding(size, capture_timestamp_ms); + return video_channel_send_->GeneratePadding(size, capture_timestamp_us); }); audio_channel_send_ = std::make_unique( @@ -170,6 +170,7 @@ int IceTransportController::SendVideo(const XVideoFrame* video_frame) { new_frame.width = video_frame->width; new_frame.height = video_frame->height; new_frame.size = video_frame->size; + new_frame.timestamp = video_frame->timestamp; if (target_width_.has_value() && target_height_.has_value()) { if (target_width_.value() < video_frame->width && target_height_.value() < video_frame->height) { diff --git a/src/transport/packet_sender/packet_sender.h b/src/transport/packet_sender/packet_sender.h index 93dd812..f0eecc5 100644 --- a/src/transport/packet_sender/packet_sender.h +++ b/src/transport/packet_sender/packet_sender.h @@ -20,7 +20,7 @@ class PacketSender { virtual int Send() = 0; virtual int EnqueueRtpPacket( std::vector> &rtp_packets, - int64_t capture_timestamp_ms) = 0; + int64_t capture_timestamp_us) = 0; }; #endif \ No newline at end of file diff --git a/src/transport/packet_sender/packet_sender_imp.cpp b/src/transport/packet_sender/packet_sender_imp.cpp index 3b6896a..f82fdeb 100644 --- a/src/transport/packet_sender/packet_sender_imp.cpp +++ b/src/transport/packet_sender/packet_sender_imp.cpp @@ -250,13 +250,13 @@ PacketSenderImp::Stats PacketSenderImp::GetStats() const { int PacketSenderImp::EnqueueRtpPacket( std::vector> &rtp_packets, - int64_t capture_timestamp_ms) { + int64_t capture_timestamp_us) { std::vector> to_send_rtp_packets; for (auto &rtp_packet : rtp_packets) { std::unique_ptr rtp_packet_to_send( static_cast(rtp_packet.release())); rtp_packet_to_send->set_capture_time( - webrtc::Timestamp::Millis(capture_timestamp_ms)); + webrtc::Timestamp::Micros(capture_timestamp_us)); rtp_packet_to_send->set_transport_sequence_number(transport_seq_++); switch (rtp_packet_to_send->PayloadType()) { diff --git a/src/transport/packet_sender/packet_sender_imp.h b/src/transport/packet_sender/packet_sender_imp.h index 03b3c2a..69a7349 100644 --- a/src/transport/packet_sender/packet_sender_imp.h +++ b/src/transport/packet_sender/packet_sender_imp.h @@ -38,7 +38,7 @@ class PacketSenderImp : public PacketSender, int Send() { return 0; } int EnqueueRtpPacket(std::vector>& rtp_packets, - int64_t capture_timestamp_ms); + int64_t capture_timestamp_us); void SetOnSentPacketFunc( std::function on_sent_packet_func) { @@ -61,12 +61,6 @@ class PacketSenderImp : public PacketSender, packet->UpdateSequenceNumber(ssrc_seq_[packet->Ssrc()]++); - webrtc::Timestamp now = clock_->CurrentTime(); - webrtc::TimeDelta interval = now - last_send_time_; - webrtc::TimeDelta delay = now - packet->capture_time(); - LOG_WARN("interval: {}, delay: {}", interval.ms(), delay.seconds()); - last_send_time_ = now; - on_sent_packet_func_(*packet); } }