From c477643aed9535274371b9e1d53c514f326456c3 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Thu, 5 Sep 2024 17:28:58 +0800 Subject: [PATCH] [feat] support dynamic resolution codec --- src/frame/video_frame.cpp | 7 ++ src/frame/video_frame.h | 4 + src/interface/x.h | 17 ++- .../video/decode/dav1d/dav1d_av1_decoder.cpp | 18 ++- .../video/decode/dav1d/dav1d_av1_decoder.h | 1 + .../video/encode/aom/aom_av1_encoder.cpp | 119 +++++++++++++++++- src/media/video/encode/aom/aom_av1_encoder.h | 8 ++ .../encode/nvcodec/nvidia_video_encoder.h | 7 ++ .../video/encode/openh264/openh264_encoder.h | 7 ++ src/media/video/encode/video_encoder.h | 8 ++ src/pc/peer_connection.cpp | 51 +++++++- src/pc/peer_connection.h | 11 ++ src/rtc/x_inner.cpp | 19 +++ src/statistics/io_statistics.cpp | 81 ++++++++++++ src/statistics/io_statistics.h | 66 ++++++++++ src/statistics/receiver_statistics.cpp | 7 -- src/statistics/receiver_statistics.h | 21 ---- src/statistics/sender_statistics.cpp | 7 -- src/statistics/sender_statistics.h | 21 ---- src/statistics/statistics_base.cpp | 5 - src/statistics/statistics_base.h | 19 --- src/transmission/ice_transmission.cpp | 85 ++++++++++--- src/transmission/ice_transmission.h | 6 +- src/ws/ws_core.cpp | 2 - xmake.lua | 2 +- 25 files changed, 481 insertions(+), 118 deletions(-) create mode 100644 src/statistics/io_statistics.cpp create mode 100644 src/statistics/io_statistics.h delete mode 100644 src/statistics/receiver_statistics.cpp delete mode 100644 src/statistics/receiver_statistics.h delete mode 100644 src/statistics/sender_statistics.cpp delete mode 100644 src/statistics/sender_statistics.h delete mode 100644 src/statistics/statistics_base.cpp delete mode 100644 src/statistics/statistics_base.h diff --git a/src/frame/video_frame.cpp b/src/frame/video_frame.cpp index c18e65b..dc872dc 100644 --- a/src/frame/video_frame.cpp +++ b/src/frame/video_frame.cpp @@ -12,6 +12,13 @@ VideoFrame::VideoFrame(size_t size) { height_ = 0; } +VideoFrame::VideoFrame(size_t size, size_t width, size_t height) { + buffer_ = new uint8_t[size]; + size_ = size; + width_ = width; + height_ = height; +} + VideoFrame::VideoFrame(const uint8_t *buffer, size_t size) { buffer_ = new uint8_t[size]; memcpy(buffer_, buffer, size); diff --git a/src/frame/video_frame.h b/src/frame/video_frame.h index 39f1e17..eb6b1ad 100644 --- a/src/frame/video_frame.h +++ b/src/frame/video_frame.h @@ -14,6 +14,7 @@ class VideoFrame { public: VideoFrame(); VideoFrame(size_t size); + VideoFrame(size_t size, size_t width, size_t height); VideoFrame(const uint8_t *buffer, size_t size); VideoFrame(const uint8_t *buffer, size_t size, size_t width, size_t height); VideoFrame(const VideoFrame &video_frame); @@ -29,6 +30,9 @@ class VideoFrame { uint8_t *GetBuffer() { return buffer_; } + size_t GetWidth() { return width_; } + size_t GetHeight() { return height_; } + private: uint8_t *buffer_ = nullptr; size_t size_ = 0; diff --git a/src/interface/x.h b/src/interface/x.h index 0056cd3..3bff163 100644 --- a/src/interface/x.h +++ b/src/interface/x.h @@ -39,11 +39,21 @@ enum TraversalMode { P2P = 0, Relay, UnknownMode }; extern "C" { #endif +typedef struct { + const char* data; + size_t size; + unsigned int width; + unsigned int height; +} XVideoFrame; + typedef struct Peer PeerPtr; -typedef void (*OnReceiveBuffer)(const char*, size_t, const char*, size_t, +typedef void (*OnReceiveBuffer)(const char*, size_t, const char*, const size_t, void*); +typedef void (*OnReceiveVideoFrame)(const XVideoFrame* video_frame, const char*, + const size_t, void*); + typedef void (*OnSignalStatus)(SignalStatus, void*); typedef void (*OnConnectionStatus)(ConnectionStatus, void*); @@ -70,6 +80,9 @@ typedef struct { OnReceiveBuffer on_receive_video_buffer; OnReceiveBuffer on_receive_audio_buffer; OnReceiveBuffer on_receive_data_buffer; + + OnReceiveVideoFrame on_receive_video_frame; + OnSignalStatus on_signal_status; OnConnectionStatus on_connection_status; NetStatusReport net_status_report; @@ -93,6 +106,8 @@ DLLAPI int LeaveConnection(PeerPtr* peer_ptr, const char* transmission_id); DLLAPI int SendData(PeerPtr* peer_ptr, DATA_TYPE data_type, const char* data, size_t size); +DLLAPI int SendVideoFrame(PeerPtr* peer_ptr, const XVideoFrame* video_frame); + #ifdef __cplusplus } #endif diff --git a/src/media/video/decode/dav1d/dav1d_av1_decoder.cpp b/src/media/video/decode/dav1d/dav1d_av1_decoder.cpp index f3dac8d..3454a7a 100644 --- a/src/media/video/decode/dav1d/dav1d_av1_decoder.cpp +++ b/src/media/video/decode/dav1d/dav1d_av1_decoder.cpp @@ -73,7 +73,7 @@ int Dav1dAv1Decoder::Init() { } decoded_frame_yuv_ = new VideoFrame(1280 * 720 * 3 / 2); - decoded_frame_nv12_ = new VideoFrame(1280 * 720 * 3 / 2); + // decoded_frame_nv12_ = new VideoFrame(1280 * 720 * 3 / 2); if (SAVE_RECEIVED_AV1_STREAM) { file_av1_ = fopen("received_av1_stream.ivf", "w+b"); @@ -157,6 +157,22 @@ int Dav1dAv1Decoder::Decode( decoded_frame_nv12_->GetBuffer(), dav1d_picture.p.w, dav1d_picture.p.h); } else { + if (!decoded_frame_nv12_) { + decoded_frame_nv12_capacity_ = + dav1d_picture.p.w * dav1d_picture.p.h * 3 / 2; + decoded_frame_nv12_ = new VideoFrame( + decoded_frame_nv12_capacity_, dav1d_picture.p.w, dav1d_picture.p.h); + } + + if (decoded_frame_nv12_capacity_ < + dav1d_picture.p.w * dav1d_picture.p.h * 3 / 2) { + delete decoded_frame_nv12_; + decoded_frame_nv12_capacity_ = + dav1d_picture.p.w * dav1d_picture.p.h * 3 / 2; + decoded_frame_nv12_ = new VideoFrame( + decoded_frame_nv12_capacity_, dav1d_picture.p.w, dav1d_picture.p.h); + } + libyuv::I420ToNV12( (const uint8_t *)dav1d_picture.data[0], dav1d_picture.p.w, (const uint8_t *)dav1d_picture.data[1], dav1d_picture.p.w / 2, diff --git a/src/media/video/decode/dav1d/dav1d_av1_decoder.h b/src/media/video/decode/dav1d/dav1d_av1_decoder.h index adcfa4c..aa183d3 100644 --- a/src/media/video/decode/dav1d/dav1d_av1_decoder.h +++ b/src/media/video/decode/dav1d/dav1d_av1_decoder.h @@ -25,6 +25,7 @@ class Dav1dAv1Decoder : public VideoDecoder { private: VideoFrame *decoded_frame_yuv_ = nullptr; VideoFrame *decoded_frame_nv12_ = nullptr; + int decoded_frame_nv12_capacity_ = 0; FILE *file_av1_ = nullptr; FILE *file_nv12_ = nullptr; diff --git a/src/media/video/encode/aom/aom_av1_encoder.cpp b/src/media/video/encode/aom/aom_av1_encoder.cpp index 6f03b17..9a5bcce 100644 --- a/src/media/video/encode/aom/aom_av1_encoder.cpp +++ b/src/media/video/encode/aom/aom_av1_encoder.cpp @@ -8,8 +8,6 @@ #define SAVE_RECEIVED_NV12_STREAM 0 #define SAVE_ENCODED_AV1_STREAM 0 -#define NV12_BUFFER_SIZE 1280 * 720 * 3 / 2 - #define SET_ENCODER_PARAM_OR_RETURN_ERROR(param_id, param_value) \ do { \ if (!SetEncoderControlParameters(param_id, param_value)) { \ @@ -84,6 +82,27 @@ int AomAv1Encoder::GetCpuSpeed(int width, int height) { return 9; } +int AomAv1Encoder::ResetEncodeResolution(unsigned int width, + unsigned int height) { + LOG_INFO("Reset encode resolution from [{}x{}] to [{}x{}]]", frame_width_, + frame_height_, width, height); + + frame_width_ = width; + frame_height_ = height; + + aom_av1_encoder_config_.g_w = width; + aom_av1_encoder_config_.g_h = height; + + if (frame_for_encode_ != nullptr) { + aom_img_free(frame_for_encode_); + frame_for_encode_ = aom_img_wrap(nullptr, AOM_IMG_FMT_NV12, frame_width_, + frame_height_, 1, nullptr); + } + + return aom_codec_enc_config_set(&aom_av1_encoder_ctx_, + &aom_av1_encoder_config_); +} + AomAv1Encoder::AomAv1Encoder() {} AomAv1Encoder::~AomAv1Encoder() { @@ -99,14 +118,13 @@ AomAv1Encoder::~AomAv1Encoder() { file_av1_ = nullptr; } - delete encoded_frame_; + delete[] encoded_frame_; + encoded_frame_ = nullptr; Release(); } int AomAv1Encoder::Init() { - encoded_frame_ = new uint8_t[NV12_BUFFER_SIZE]; - // Initialize encoder configuration structure with default values aom_codec_err_t ret = aom_codec_enc_config_default( aom_codec_av1_cx(), &aom_av1_encoder_config_, kUsageProfile); @@ -316,6 +334,97 @@ int AomAv1Encoder::Encode(const uint8_t *pData, int nSize, return 0; } +int AomAv1Encoder::Encode(const XVideoFrame *video_frame, + std::function + on_encoded_image) { + if (SAVE_RECEIVED_NV12_STREAM) { + fwrite(video_frame->data, 1, video_frame->size, file_nv12_); + } + + aom_codec_err_t ret = AOM_CODEC_OK; + + if (!encoded_frame_) { + encoded_frame_ = new uint8_t[video_frame->size]; + encoded_frame_capacity_ = video_frame->size; + } + + if (encoded_frame_capacity_ < video_frame->size) { + encoded_frame_capacity_ = video_frame->size; + delete[] encoded_frame_; + encoded_frame_ = new uint8_t[video_frame->size]; + } + + if (frame_width_ != video_frame->width || + frame_height_ != video_frame->height) { + if (AOM_CODEC_OK != + ResetEncodeResolution(video_frame->width, video_frame->height)) { + LOG_ERROR("Reset encode resolution failed"); + return -1; + } + } + + const uint32_t duration = + kRtpTicksPerSecond / static_cast(max_frame_rate_); + timestamp_ += duration; + + frame_for_encode_->planes[AOM_PLANE_Y] = (unsigned char *)(video_frame->data); + frame_for_encode_->planes[AOM_PLANE_U] = + (unsigned char *)(video_frame->data + + video_frame->width * video_frame->height); + frame_for_encode_->planes[AOM_PLANE_V] = nullptr; + frame_for_encode_->stride[AOM_PLANE_Y] = video_frame->width; + frame_for_encode_->stride[AOM_PLANE_U] = video_frame->width; + frame_for_encode_->stride[AOM_PLANE_V] = 0; + + VideoFrameType frame_type; + if (0 == seq_++ % 300) { + force_i_frame_flags_ = AOM_EFLAG_FORCE_KF; + frame_type = VideoFrameType::kVideoFrameKey; + } else { + force_i_frame_flags_ = 0; + frame_type = VideoFrameType::kVideoFrameDelta; + } + + // Encode a frame. The presentation timestamp `pts` should not use real + // timestamps from frames or the wall clock, as that can cause the rate + // controller to misbehave. + ret = aom_codec_encode(&aom_av1_encoder_ctx_, frame_for_encode_, timestamp_, + duration, force_i_frame_flags_); + if (ret != AOM_CODEC_OK) { + LOG_ERROR("Encode failed: {}, {}", + aom_codec_error_detail(&aom_av1_encoder_ctx_), + aom_codec_build_config()); + return -1; + } + + aom_codec_iter_t iter = nullptr; + int data_pkt_count = 0; + while (const aom_codec_cx_pkt_t *pkt = + aom_codec_get_cx_data(&aom_av1_encoder_ctx_, &iter)) { + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT && pkt->data.frame.sz > 0) { + memcpy(encoded_frame_, pkt->data.frame.buf, pkt->data.frame.sz); + encoded_frame_size_ = pkt->data.frame.sz; + + int qp = -1; + SET_ENCODER_PARAM_OR_RETURN_ERROR(AOME_GET_LAST_QUANTIZER, &qp); + // LOG_INFO("Encoded frame qp = {}", qp); + + if (on_encoded_image) { + on_encoded_image((char *)encoded_frame_, encoded_frame_size_, + frame_type); + if (SAVE_ENCODED_AV1_STREAM) { + fwrite(encoded_frame_, 1, encoded_frame_size_, file_av1_); + } + } else { + OnEncodedImage((char *)encoded_frame_, encoded_frame_size_); + } + } + } + + return 0; +} + int AomAv1Encoder::OnEncodedImage(char *encoded_packets, size_t size) { LOG_INFO("OnEncodedImage not implemented"); return 0; diff --git a/src/media/video/encode/aom/aom_av1_encoder.h b/src/media/video/encode/aom/aom_av1_encoder.h index 89e0ca8..7c122ec 100644 --- a/src/media/video/encode/aom/aom_av1_encoder.h +++ b/src/media/video/encode/aom/aom_av1_encoder.h @@ -41,6 +41,11 @@ class AomAv1Encoder : public VideoEncoder { VideoFrameType frame_type)> on_encoded_image); + int Encode(const XVideoFrame* video_frame, + std::function + on_encoded_image); + virtual int OnEncodedImage(char* encoded_packets, size_t size); void ForceIdr(); @@ -51,6 +56,8 @@ class AomAv1Encoder : public VideoEncoder { int NumberOfThreads(int width, int height, int number_of_cores); int GetCpuSpeed(int width, int height); + int ResetEncodeResolution(unsigned int width, unsigned int height); + int Release(); private: @@ -79,6 +86,7 @@ class AomAv1Encoder : public VideoEncoder { int64_t timestamp_ = 0; aom_enc_frame_flags_t force_i_frame_flags_ = 0; uint8_t* encoded_frame_ = nullptr; + size_t encoded_frame_capacity_ = 0; int encoded_frame_size_ = 0; }; diff --git a/src/media/video/encode/nvcodec/nvidia_video_encoder.h b/src/media/video/encode/nvcodec/nvidia_video_encoder.h index c92a839..b0c6ccb 100644 --- a/src/media/video/encode/nvcodec/nvidia_video_encoder.h +++ b/src/media/video/encode/nvcodec/nvidia_video_encoder.h @@ -17,6 +17,13 @@ class NvidiaVideoEncoder : public VideoEncoder { VideoFrameType frame_type)> on_encoded_image); + int Encode(const XVideoFrame* video_frame, + std::function + on_encoded_image) { + return 0; + } + virtual int OnEncodedImage(char* encoded_packets, size_t size); void ForceIdr(); diff --git a/src/media/video/encode/openh264/openh264_encoder.h b/src/media/video/encode/openh264/openh264_encoder.h index e225c71..8747a3e 100644 --- a/src/media/video/encode/openh264/openh264_encoder.h +++ b/src/media/video/encode/openh264/openh264_encoder.h @@ -28,6 +28,13 @@ class OpenH264Encoder : public VideoEncoder { VideoFrameType frame_type)> on_encoded_image); + int Encode(const XVideoFrame* video_frame, + std::function + on_encoded_image) { + return 0; + } + virtual int OnEncodedImage(char* encoded_packets, size_t size); void ForceIdr(); diff --git a/src/media/video/encode/video_encoder.h b/src/media/video/encode/video_encoder.h index c5edfdb..fe5eaa6 100644 --- a/src/media/video/encode/video_encoder.h +++ b/src/media/video/encode/video_encoder.h @@ -6,6 +6,8 @@ #include #include +#include "x.h" + class VideoEncoder { public: enum VideoFrameType { @@ -20,6 +22,12 @@ class VideoEncoder { std::function on_encoded_image) = 0; + + virtual int Encode(const XVideoFrame* video_frame, + std::function + on_encoded_image) = 0; + virtual int OnEncodedImage(char* encoded_packets, size_t size) = 0; virtual void ForceIdr() = 0; diff --git a/src/pc/peer_connection.cpp b/src/pc/peer_connection.cpp index 843b54b..0ee3040 100644 --- a/src/pc/peer_connection.cpp +++ b/src/pc/peer_connection.cpp @@ -103,6 +103,9 @@ int PeerConnection::Init(PeerConnectionParams params, on_receive_video_buffer_ = params.on_receive_video_buffer; on_receive_audio_buffer_ = params.on_receive_audio_buffer; on_receive_data_buffer_ = params.on_receive_data_buffer; + + on_receive_video_frame_ = params.on_receive_video_frame; + on_signal_status_ = params.on_signal_status; on_connection_status_ = params.on_connection_status; net_status_report_ = params.net_status_report; @@ -143,10 +146,14 @@ int PeerConnection::Init(PeerConnectionParams params, int num_frame_returned = video_decoder_->Decode( (uint8_t *)data, size, [this, user_id, user_id_size](VideoFrame video_frame) { - if (on_receive_video_buffer_) { - on_receive_video_buffer_((const char *)video_frame.Buffer(), - video_frame.Size(), user_id, user_id_size, - user_data_); + if (on_receive_video_frame_) { + XVideoFrame x_video_frame; + x_video_frame.data = (const char *)video_frame.Buffer(); + x_video_frame.width = video_frame.GetWidth(); + x_video_frame.height = video_frame.GetHeight(); + x_video_frame.size = video_frame.Size(); + on_receive_video_frame_(&x_video_frame, user_id, user_id_size, + user_data_); } }); }; @@ -540,6 +547,42 @@ int PeerConnection::SendUserData(const char *data, size_t size) { return 0; } +int PeerConnection::SendVideoData(const XVideoFrame *video_frame) { + if (!ice_ready_) { + return -1; + } + + if (ice_transmission_list_.empty()) { + return -1; + } + + if (b_force_i_frame_) { + video_encoder_->ForceIdr(); + LOG_INFO("Force I frame"); + b_force_i_frame_ = false; + } + + int ret = video_encoder_->Encode( + video_frame, + [this](char *encoded_frame, size_t size, + VideoEncoder::VideoFrameType frame_type) -> int { + for (auto &ice_trans : ice_transmission_list_) { + // LOG_ERROR("Send frame size: [{}]", size); + ice_trans.second->SendVideoData( + static_cast(frame_type), + encoded_frame, size); + } + return 0; + }); + + if (0 != ret) { + LOG_ERROR("Encode failed"); + return -1; + } + + return 0; +} + void PeerConnection::ProcessSignal(const std::string &signal) { auto j = json::parse(signal); std::string type = j["type"]; diff --git a/src/pc/peer_connection.h b/src/pc/peer_connection.h index 08bb9ed..b1f8901 100644 --- a/src/pc/peer_connection.h +++ b/src/pc/peer_connection.h @@ -16,6 +16,9 @@ typedef void (*OnReceiveBuffer)(const char *, size_t, const char *, const size_t, void *); +typedef void (*OnReceiveVideoFrame)(const XVideoFrame *video_frame, + const char *, const size_t, void *); + typedef void (*OnSignalStatus)(SignalStatus, void *); typedef void (*OnConnectionStatus)(ConnectionStatus, void *); @@ -42,6 +45,9 @@ typedef struct { OnReceiveBuffer on_receive_video_buffer; OnReceiveBuffer on_receive_audio_buffer; OnReceiveBuffer on_receive_data_buffer; + + OnReceiveVideoFrame on_receive_video_frame; + OnSignalStatus on_signal_status; OnConnectionStatus on_connection_status; NetStatusReport net_status_report; @@ -92,6 +98,8 @@ class PeerConnection { int SendAudioData(const char *data, size_t size); int SendUserData(const char *data, size_t size); + int SendVideoData(const XVideoFrame *video_frame); + private: int Login(); @@ -165,6 +173,9 @@ class PeerConnection { OnReceiveBuffer on_receive_video_buffer_; OnReceiveBuffer on_receive_audio_buffer_; OnReceiveBuffer on_receive_data_buffer_; + + OnReceiveVideoFrame on_receive_video_frame_; + OnSignalStatus on_signal_status_; OnConnectionStatus on_connection_status_; NetStatusReport net_status_report_; diff --git a/src/rtc/x_inner.cpp b/src/rtc/x_inner.cpp index dcdb8e3..4dd936d 100644 --- a/src/rtc/x_inner.cpp +++ b/src/rtc/x_inner.cpp @@ -32,6 +32,9 @@ PeerPtr *CreatePeer(const Params *params) { peer_ptr->pc_params.on_receive_video_buffer = params->on_receive_video_buffer; peer_ptr->pc_params.on_receive_audio_buffer = params->on_receive_audio_buffer; peer_ptr->pc_params.on_receive_data_buffer = params->on_receive_data_buffer; + + peer_ptr->pc_params.on_receive_video_frame = params->on_receive_video_frame; + peer_ptr->pc_params.on_signal_status = params->on_signal_status; peer_ptr->pc_params.on_connection_status = params->on_connection_status; peer_ptr->pc_params.net_status_report = params->net_status_report; @@ -109,5 +112,21 @@ int SendData(PeerPtr *peer_ptr, DATA_TYPE data_type, const char *data, } else if (DATA_TYPE::DATA == data_type) { peer_ptr->peer_connection->SendUserData(data, size); } + return 0; +} + +DLLAPI int SendVideoFrame(PeerPtr *peer_ptr, const XVideoFrame *video_frame) { + if (!peer_ptr) { + LOG_ERROR("peer_ptr not created"); + return -1; + } + + if (!video_frame) { + LOG_ERROR("Invaild video frame"); + return -1; + } + + peer_ptr->peer_connection->SendVideoData(video_frame); + return 0; } \ No newline at end of file diff --git a/src/statistics/io_statistics.cpp b/src/statistics/io_statistics.cpp new file mode 100644 index 0000000..cbd8ffb --- /dev/null +++ b/src/statistics/io_statistics.cpp @@ -0,0 +1,81 @@ +#include "io_statistics.h" + +IOStatistics::IOStatistics( + std::function + io_report_callback) + : io_report_callback_(io_report_callback) { + interval_ = 1000; +} + +IOStatistics::~IOStatistics() { + running_ = false; + cond_var_.notify_one(); + if (statistics_thread_.joinable()) { + statistics_thread_.join(); + } +} + +void IOStatistics::Process() { + while (running_) { + std::unique_lock lock(mtx_); + cond_var_.wait_for(lock, std::chrono::milliseconds(interval_), + [this] { return !running_; }); + + video_inbound_bitrate_ = video_inbound_bytes_ * 1000 * 8 / interval_; + video_outbound_bitrate_ = video_outbound_bytes_ * 1000 * 8 / interval_; + audio_inbound_bitrate_ = audio_inbound_bytes_ * 1000 * 8 / interval_; + audio_outbound_bitrate_ = audio_outbound_bytes_ * 1000 * 8 / interval_; + data_inbound_bitrate_ = data_inbound_bytes_ * 1000 * 8 / interval_; + data_outbound_bitrate_ = data_outbound_bytes_ * 1000 * 8 / interval_; + total_inbound_bitrate_ = + video_inbound_bitrate_ + audio_inbound_bitrate_ + data_inbound_bitrate_; + total_outbound_bitrate_ = video_outbound_bitrate_ + + audio_outbound_bitrate_ + data_outbound_bitrate_; + + video_inbound_bytes_ = 0; + video_outbound_bytes_ = 0; + audio_inbound_bytes_ = 0; + audio_outbound_bytes_ = 0; + data_inbound_bytes_ = 0; + data_outbound_bytes_ = 0; + + if (io_report_callback_) { + io_report_callback_(video_inbound_bitrate_, video_outbound_bitrate_, + audio_inbound_bitrate_, audio_outbound_bitrate_, + data_inbound_bitrate_, data_outbound_bitrate_, + total_inbound_bitrate_, total_outbound_bitrate_); + } + } +} + +void IOStatistics::Start() { + if (!running_) { + running_ = true; + statistics_thread_ = std::thread(&IOStatistics::Process, this); + } +} + +void IOStatistics::UpdateVideoInboundBytes(uint32_t bytes) { + video_inbound_bytes_ += bytes; +} + +void IOStatistics::UpdateVideoOutboundBytes(uint32_t bytes) { + video_outbound_bytes_ += bytes; +} + +void IOStatistics::UpdateAudioInboundBytes(uint32_t bytes) { + audio_inbound_bytes_ += bytes; +} + +void IOStatistics::UpdateAudioOutboundBytes(uint32_t bytes) { + audio_outbound_bytes_ += bytes; +} + +void IOStatistics::UpdateDataInboundBytes(uint32_t bytes) { + data_inbound_bytes_ += bytes; +} + +void IOStatistics::UpdateDataOutboundBytes(uint32_t bytes) { + data_outbound_bytes_ += bytes; +} \ No newline at end of file diff --git a/src/statistics/io_statistics.h b/src/statistics/io_statistics.h new file mode 100644 index 0000000..1f7a2ad --- /dev/null +++ b/src/statistics/io_statistics.h @@ -0,0 +1,66 @@ +/* + * @Author: DI JUNKUN + * @Date: 2024-09-05 + * Copyright (c) 2024 by DI JUNKUN, All Rights Reserved. + */ + +#ifndef _STATISTICS_H_ +#define _STATISTICS_H_ + +#include +#include +#include +#include + +class IOStatistics { + public: + IOStatistics(std::function + io_report_callback); + ~IOStatistics(); + + public: + void Start(); + + void UpdateVideoInboundBytes(uint32_t bytes); + void UpdateVideoOutboundBytes(uint32_t bytes); + + void UpdateAudioInboundBytes(uint32_t bytes); + void UpdateAudioOutboundBytes(uint32_t bytes); + + void UpdateDataInboundBytes(uint32_t bytes); + void UpdateDataOutboundBytes(uint32_t bytes); + + private: + void Process(); + + private: + std::function + io_report_callback_ = nullptr; + std::thread statistics_thread_; + std::mutex mtx_; + uint32_t interval_ = 1000; + std::condition_variable cond_var_; + std::atomic running_{false}; + + std::atomic video_inbound_bytes_ = 0; + std::atomic video_outbound_bytes_ = 0; + std::atomic audio_inbound_bytes_ = 0; + std::atomic audio_outbound_bytes_ = 0; + std::atomic data_inbound_bytes_ = 0; + std::atomic data_outbound_bytes_ = 0; + std::atomic total_inbound_bytes_ = 0; + std::atomic total_outbound_bytes_ = 0; + + std::atomic video_inbound_bitrate_ = 0; + std::atomic video_outbound_bitrate_ = 0; + std::atomic audio_inbound_bitrate_ = 0; + std::atomic audio_outbound_bitrate_ = 0; + std::atomic data_inbound_bitrate_ = 0; + std::atomic data_outbound_bitrate_ = 0; + std::atomic total_inbound_bitrate_ = 0; + std::atomic total_outbound_bitrate_ = 0; +}; + +#endif \ No newline at end of file diff --git a/src/statistics/receiver_statistics.cpp b/src/statistics/receiver_statistics.cpp deleted file mode 100644 index c873ba9..0000000 --- a/src/statistics/receiver_statistics.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "receiver_statistics.h" - -#include "log.h" - -ReceiverStatistics::ReceiverStatistics() {} - -ReceiverStatistics::~ReceiverStatistics() {} \ No newline at end of file diff --git a/src/statistics/receiver_statistics.h b/src/statistics/receiver_statistics.h deleted file mode 100644 index 590554e..0000000 --- a/src/statistics/receiver_statistics.h +++ /dev/null @@ -1,21 +0,0 @@ -/* - * @Author: DI JUNKUN - * @Date: 2024-09-04 - * Copyright (c) 2024 by DI JUNKUN, All Rights Reserved. - */ - -#ifndef _RECEIVER_STATISTICS_H_ -#define _RECEIVER_STATISTICS_H_ - -#include "statistics_base.h" - -class ReceiverStatistics : public StatisticsBase { - public: - ReceiverStatistics(); - virtual ~ReceiverStatistics(); - - private: - /* data */ -}; - -#endif \ No newline at end of file diff --git a/src/statistics/sender_statistics.cpp b/src/statistics/sender_statistics.cpp deleted file mode 100644 index 27b53d9..0000000 --- a/src/statistics/sender_statistics.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "sender_statistics.h" - -#include "log.h" - -SenderStatistics::SenderStatistics() {} - -SenderStatistics::~SenderStatistics() {} \ No newline at end of file diff --git a/src/statistics/sender_statistics.h b/src/statistics/sender_statistics.h deleted file mode 100644 index db0bcf3..0000000 --- a/src/statistics/sender_statistics.h +++ /dev/null @@ -1,21 +0,0 @@ -/* - * @Author: DI JUNKUN - * @Date: 2024-09-04 - * Copyright (c) 2024 by DI JUNKUN, All Rights Reserved. - */ - -#ifndef _SENDER_STATISTICS_H_ -#define _SENDER_STATISTICS_H_ - -#include "statistics_base.h" - -class SenderStatistics : public StatisticsBase { - public: - SenderStatistics(); - virtual ~SenderStatistics(); - - private: - /* data */ -}; - -#endif \ No newline at end of file diff --git a/src/statistics/statistics_base.cpp b/src/statistics/statistics_base.cpp deleted file mode 100644 index 0f2cc30..0000000 --- a/src/statistics/statistics_base.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "statistics_base.h" - -StatisticsBase::StatisticsBase() {} - -StatisticsBase::~StatisticsBase() {} \ No newline at end of file diff --git a/src/statistics/statistics_base.h b/src/statistics/statistics_base.h deleted file mode 100644 index 8a89ede..0000000 --- a/src/statistics/statistics_base.h +++ /dev/null @@ -1,19 +0,0 @@ -/* - * @Author: DI JUNKUN - * @Date: 2024-09-04 - * Copyright (c) 2024 by DI JUNKUN, All Rights Reserved. - */ - -#ifndef _STATISTICS_BASE_H_ -#define _STATISTICS_BASE_H_ - -class StatisticsBase { - public: - StatisticsBase(); - virtual ~StatisticsBase(); - - private: - /* data */ -}; - -#endif \ No newline at end of file diff --git a/src/transmission/ice_transmission.cpp b/src/transmission/ice_transmission.cpp index d4c4321..f987a89 100644 --- a/src/transmission/ice_transmission.cpp +++ b/src/transmission/ice_transmission.cpp @@ -44,11 +44,26 @@ int IceTransmission::InitIceTransmission( std::string &stun_ip, int stun_port, std::string &turn_ip, int turn_port, std::string &turn_username, std::string &turn_password, RtpPacket::PAYLOAD_TYPE video_codec_payload_type) { + ice_io_statistics_ = std::make_unique( + [](uint32_t video_inbound_bitrate, uint32_t video_outbound_bitrate, + uint32_t audio_inbound_bitrate, uint32_t audio_outbound_bitrate, + uint32_t data_inbound_bitrate, uint32_t data_outbound_bitrate, + uint32_t total_inbound_bitrate, uint32_t total_outbound_bitrate) { + LOG_ERROR( + "video in: [{}] kbps, video out: [{}] kbps, audio in: [{}] kbps, " + "audio out: [{}] kbps, data in: [{}] kbps, data out: [{}] kbps, " + "total in: [{}] kbps, total out: [{}] kbps", + video_inbound_bitrate / 1000, video_outbound_bitrate / 1000, + audio_inbound_bitrate / 1000, audio_outbound_bitrate / 1000, + data_inbound_bitrate / 1000, data_outbound_bitrate / 1000, + total_inbound_bitrate / 1000, total_outbound_bitrate / 1000); + }); video_rtp_codec_ = std::make_unique(video_codec_payload_type); audio_rtp_codec_ = std::make_unique(RtpPacket::PAYLOAD_TYPE::OPUS); data_rtp_codec_ = std::make_unique(RtpPacket::PAYLOAD_TYPE::DATA); rtp_video_receiver_ = std::make_unique(); + // rr sender rtp_video_receiver_->SetSendDataFunc( [this](const char *data, size_t size) -> int { if (!ice_agent_) { @@ -68,6 +83,7 @@ int IceTransmission::InitIceTransmission( rtp_video_receiver_->SetOnReceiveCompleteFrame( [this](VideoFrame &video_frame) -> void { // LOG_ERROR("OnReceiveCompleteFrame {}", video_frame.Size()); + ice_io_statistics_->UpdateVideoInboundBytes(video_frame.Size()); on_receive_video_((const char *)video_frame.Buffer(), video_frame.Size(), remote_user_id_.data(), remote_user_id_.size()); @@ -90,14 +106,33 @@ int IceTransmission::InitIceTransmission( return -2; } + ice_io_statistics_->UpdateVideoOutboundBytes(size); return ice_agent_->Send(data, size); }); rtp_video_sender_->Start(); rtp_audio_receiver_ = std::make_unique(); + // rr sender + rtp_audio_receiver_->SetSendDataFunc( + [this](const char *data, size_t size) -> int { + if (!ice_agent_) { + LOG_ERROR("ice_agent_ is nullptr"); + return -1; + } + + if (state_ != NICE_COMPONENT_STATE_CONNECTED && + state_ != NICE_COMPONENT_STATE_READY) { + LOG_ERROR("Ice is not connected, state = [{}]", + nice_component_state_to_string(state_)); + return -2; + } + + return ice_agent_->Send(data, size); + }); rtp_audio_receiver_->SetOnReceiveData( [this](const char *data, size_t size) -> void { + ice_io_statistics_->UpdateAudioInboundBytes(size); on_receive_audio_(data, size, remote_user_id_.data(), remote_user_id_.size()); }); @@ -117,32 +152,14 @@ int IceTransmission::InitIceTransmission( return -2; } + ice_io_statistics_->UpdateAudioOutboundBytes(size); return ice_agent_->Send(data, size); }); rtp_audio_sender_->Start(); - rtp_data_sender_ = std::make_unique(); - rtp_data_sender_->SetSendDataFunc( - [this](const char *data, size_t size) -> int { - if (!ice_agent_) { - LOG_ERROR("ice_agent_ is nullptr"); - return -1; - } - - if (state_ != NICE_COMPONENT_STATE_CONNECTED && - state_ != NICE_COMPONENT_STATE_READY) { - LOG_ERROR("Ice is not connected, state = [{}]", - nice_component_state_to_string(state_)); - return -2; - } - - return ice_agent_->Send(data, size); - }); - - rtp_data_sender_->Start(); - rtp_data_receiver_ = std::make_unique(); + // rr sender rtp_data_receiver_->SetSendDataFunc( [this](const char *data, size_t size) -> int { if (!ice_agent_) { @@ -161,10 +178,32 @@ int IceTransmission::InitIceTransmission( }); rtp_data_receiver_->SetOnReceiveData( [this](const char *data, size_t size) -> void { + ice_io_statistics_->UpdateDataInboundBytes(size); on_receive_data_(data, size, remote_user_id_.data(), remote_user_id_.size()); }); + rtp_data_sender_ = std::make_unique(); + rtp_data_sender_->SetSendDataFunc( + [this](const char *data, size_t size) -> int { + if (!ice_agent_) { + LOG_ERROR("ice_agent_ is nullptr"); + return -1; + } + + if (state_ != NICE_COMPONENT_STATE_CONNECTED && + state_ != NICE_COMPONENT_STATE_READY) { + LOG_ERROR("Ice is not connected, state = [{}]", + nice_component_state_to_string(state_)); + return -2; + } + + ice_io_statistics_->UpdateDataOutboundBytes(size); + return ice_agent_->Send(data, size); + }); + + rtp_data_sender_->Start(); + ice_agent_ = std::make_unique( enable_turn_, trickle_ice_, offer_peer_, stun_ip, stun_port, turn_ip, turn_port, turn_username, turn_password); @@ -179,6 +218,12 @@ int IceTransmission::InitIceTransmission( ice_transmission_obj->remote_user_id_, nice_component_state_to_string(state)); ice_transmission_obj->state_ = state; + + if (state == NICE_COMPONENT_STATE_READY || + state == NICE_COMPONENT_STATE_CONNECTED) { + ice_transmission_obj->ice_io_statistics_->Start(); + } + ice_transmission_obj->on_ice_status_change_( nice_component_state_to_string(state)); } else { diff --git a/src/transmission/ice_transmission.h b/src/transmission/ice_transmission.h index ac72ecc..71de389 100644 --- a/src/transmission/ice_transmission.h +++ b/src/transmission/ice_transmission.h @@ -11,7 +11,7 @@ #include "congestion_control.h" #include "ice_agent.h" -#include "receiver_statistics.h" +#include "io_statistics.h" #include "ringbuffer.h" #include "rtp_audio_receiver.h" #include "rtp_audio_sender.h" @@ -21,7 +21,6 @@ #include "rtp_packet.h" #include "rtp_video_receiver.h" #include "rtp_video_sender.h" -#include "sender_statistics.h" #include "ws_transmission.h" class IceTransmission { @@ -154,8 +153,7 @@ class IceTransmission { uint32_t last_complete_frame_ts_ = 0; private: - std::unique_ptr sender_statistics_ = nullptr; - std::unique_ptr receiver_statistics_ = nullptr; + std::unique_ptr ice_io_statistics_ = nullptr; }; #endif \ No newline at end of file diff --git a/src/ws/ws_core.cpp b/src/ws/ws_core.cpp index b4dd99e..1cbf184 100644 --- a/src/ws/ws_core.cpp +++ b/src/ws/ws_core.cpp @@ -123,8 +123,6 @@ void WsCore::Ping(websocketpp::connection_hdl hdl) { } } } - - if (!running_) break; } } diff --git a/xmake.lua b/xmake.lua index 78133ec..1d93ba0 100644 --- a/xmake.lua +++ b/xmake.lua @@ -162,7 +162,7 @@ target("media") add_files("src/media/audio/encode/*.cpp", "src/media/audio/decode/*.cpp") add_includedirs("src/media/audio/encode", - "src/media/audio/decode", {public = true}) + "src/media/audio/decode", "src/interface", {public = true}) target("qos") set_kind("object")