From 09c0ab923504779214c511378b8495c47ca7b05a Mon Sep 17 00:00:00 2001 From: dijunkun Date: Tue, 26 Nov 2024 15:06:05 +0800 Subject: [PATCH] [fix] move codecs from pc into transmission module --- src/interface/x.h | 7 +- src/pc/peer_connection.cpp | 293 +++----------------------- src/pc/peer_connection.h | 21 +- src/rtc/x_inner.cpp | 52 +++-- src/transmission/ice_transmission.cpp | 253 ++++++++++++++++++---- src/transmission/ice_transmission.h | 52 +++-- xmake.lua | 4 +- 7 files changed, 323 insertions(+), 359 deletions(-) diff --git a/src/interface/x.h b/src/interface/x.h index e435e6d..2089ab1 100644 --- a/src/interface/x.h +++ b/src/interface/x.h @@ -115,11 +115,12 @@ DLLAPI int JoinConnection(PeerPtr* peer_ptr, const char* transmission_id, DLLAPI int LeaveConnection(PeerPtr* peer_ptr, const char* transmission_id); -DLLAPI int SendData(PeerPtr* peer_ptr, DATA_TYPE data_type, const char* data, - size_t size); - DLLAPI int SendVideoFrame(PeerPtr* peer_ptr, const XVideoFrame* video_frame); +DLLAPI int SendAudioFrame(PeerPtr* peer_ptr, const char* data, size_t size); + +DLLAPI int SendData(PeerPtr* peer_ptr, const char* data, size_t size); + #ifdef __cplusplus } #endif diff --git a/src/pc/peer_connection.cpp b/src/pc/peer_connection.cpp index 68a0b9d..0001d6a 100644 --- a/src/pc/peer_connection.cpp +++ b/src/pc/peer_connection.cpp @@ -7,11 +7,6 @@ #include "log.h" #include "nlohmann/json.hpp" -#if __APPLE__ -#else -#include "nvcodec_api.h" -#endif - using nlohmann::json; PeerConnection::PeerConnection() {} @@ -22,10 +17,6 @@ PeerConnection::~PeerConnection() { nv12_data_ = nullptr; } - video_codec_inited_ = false; - audio_codec_inited_ = false; - load_nvcodec_dll_success = false; - user_data_ = nullptr; } @@ -143,41 +134,6 @@ int PeerConnection::Init(PeerConnectionParams params, } }; - on_receive_video_ = [this](const char *data, size_t size, - const std::string &user_id) { - int num_frame_returned = video_decoder_->Decode( - (uint8_t *)data, size, [this, user_id](VideoFrame video_frame) { - if (on_receive_video_frame_) { - XVideoFrame x_video_frame; - x_video_frame.data = (const char *)video_frame.Buffer(); - x_video_frame.width = video_frame.Width(); - x_video_frame.height = video_frame.Height(); - x_video_frame.size = video_frame.Size(); - on_receive_video_frame_(&x_video_frame, user_id.data(), - user_id.size(), user_data_); - } - }); - }; - - on_receive_audio_ = [this](const char *data, size_t size, - const std::string &user_id) { - int num_frame_returned = audio_decoder_->Decode( - (uint8_t *)data, size, [this, user_id](uint8_t *data, int size) { - if (on_receive_audio_buffer_) { - on_receive_audio_buffer_((const char *)data, size, user_id.data(), - user_id.size(), user_data_); - } - }); - }; - - on_receive_data_ = [this](const char *data, size_t size, - const std::string &user_id) { - if (on_receive_data_buffer_) { - on_receive_data_buffer_(data, size, user_id.data(), user_id.size(), - user_data_); - } - }; - on_ice_status_change_ = [this](std::string ice_status, const std::string &user_id) { if ("connecting" == ice_status) { @@ -277,102 +233,6 @@ int PeerConnection::Init(PeerConnectionParams params, return 0; } -int PeerConnection::CreateVideoCodec(bool av1, bool hardware_acceleration) { - if (video_codec_inited_) { - return 0; - } - - hardware_acceleration_ = hardware_acceleration; - - if (av1) { - if (hardware_acceleration_) { - hardware_acceleration_ = false; - LOG_WARN("Only support software codec for AV1"); - } - video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, true); - video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, true); - } else { -#ifdef __APPLE__ - if (hardware_acceleration_) { - hardware_acceleration_ = false; - LOG_WARN( - "MacOS not support hardware acceleration, use default software " - "codec"); - video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); - video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); - } else { - video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); - video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); - } -#else - if (hardware_acceleration_) { - if (0 == LoadNvCodecDll()) { - load_nvcodec_dll_success = true; - video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(true, false); - video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(true, false); - } else { - LOG_WARN( - "Hardware accelerated codec not available, use default software " - "codec"); - video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); - video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); - } - } else { - video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); - video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); - } -#endif - } - - if (!video_encoder_) { - video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); - LOG_ERROR("Create encoder failed, try to use software H.264 encoder"); - } - if (!video_encoder_ || 0 != video_encoder_->Init()) { - LOG_ERROR("Encoder init failed"); - return -1; - } - - if (!video_decoder_) { - video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); - LOG_ERROR("Create decoder failed, try to use software H.264 decoder"); - } - if (!video_decoder_ || video_decoder_->Init()) { - LOG_ERROR("Decoder init failed"); - return -1; - } - - video_codec_inited_ = true; - LOG_INFO("Create video codec [{}|{}] finish", - video_encoder_->GetEncoderName(), video_decoder_->GetDecoderName()); - - return 0; -} - -int PeerConnection::CreateAudioCodec() { - if (audio_codec_inited_) { - return 0; - } - - audio_encoder_ = std::make_unique(AudioEncoder(48000, 1, 480)); - if (!audio_encoder_ || 0 != audio_encoder_->Init()) { - LOG_ERROR("Audio encoder init failed"); - return -1; - } - - audio_decoder_ = std::make_unique(AudioDecoder(48000, 1, 480)); - if (!audio_decoder_ || 0 != audio_decoder_->Init()) { - LOG_ERROR("Audio decoder init failed"); - return -1; - } - - audio_codec_inited_ = true; - LOG_INFO("Create audio codec [{}|{}] finish", - audio_encoder_->GetEncoderName(), audio_decoder_->GetDecoderName()); - - return 0; -} - int PeerConnection::Login() { if (WsStatus::WsOpened != ws_status_) { LOG_ERROR("Websocket not opened"); @@ -494,12 +354,6 @@ int PeerConnection::Destroy() { nv12_data_ = nullptr; } -#ifdef __APPLE__ -#else - if (hardware_acceleration_ && load_nvcodec_dll_success) { - ReleaseNvCodecDll(); - } -#endif return 0; } @@ -527,36 +381,18 @@ SignalStatus PeerConnection::GetSignalStatus() { return signal_status_; } -int PeerConnection::SendVideoData(const char *data, size_t size) { +// media send +int PeerConnection::SendVideoData(const XVideoFrame *video_frame) { if (ice_transmission_list_.empty()) { return -1; } - if (b_force_i_frame_) { - LOG_INFO("Force I frame"); - video_encoder_->ForceIdr(); - b_force_i_frame_ = false; - } + for (auto &ice_trans : ice_transmission_list_) { + if (!is_ice_transmission_ready_[ice_trans.first]) { + continue; + } - int ret = video_encoder_->Encode( - (uint8_t *)data, size, - [this](char *encoded_frame, size_t size, - VideoEncoder::VideoFrameType frame_type) -> int { - for (auto &ice_trans : ice_transmission_list_) { - if (!is_ice_transmission_ready_[ice_trans.first]) { - continue; - } - // LOG_ERROR("Send frame size: [{}]", size); - ice_trans.second->SendVideoData( - static_cast(frame_type), - encoded_frame, size); - } - return 0; - }); - - if (0 != ret) { - LOG_ERROR("Encode failed"); - return -1; + ice_trans.second->SendVideoData(video_frame); } return 0; @@ -567,19 +403,12 @@ int PeerConnection::SendAudioData(const char *data, size_t size) { return -1; } - int ret = audio_encoder_->Encode( - (uint8_t *)data, size, - [this](char *encoded_audio_buffer, size_t size) -> int { - for (auto &ice_trans : ice_transmission_list_) { - if (!is_ice_transmission_ready_[ice_trans.first]) { - continue; - } - // LOG_ERROR("opus frame size: [{}]", size); - ice_trans.second->SendData(IceTransmission::DATA_TYPE::AUDIO, - encoded_audio_buffer, size); - } - return 0; - }); + for (auto &ice_trans : ice_transmission_list_) { + if (!is_ice_transmission_ready_[ice_trans.first]) { + continue; + } + ice_trans.second->SendAudioData(data, size); + } return 0; } @@ -589,46 +418,11 @@ int PeerConnection::SendUserData(const char *data, size_t size) { if (!is_ice_transmission_ready_[ice_trans.first]) { continue; } - ice_trans.second->SendData(IceTransmission::DATA_TYPE::DATA, data, size); + ice_trans.second->SendUserData(data, size); } return 0; } -int PeerConnection::SendVideoData(const XVideoFrame *video_frame) { - if (ice_transmission_list_.empty()) { - return -1; - } - - if (b_force_i_frame_) { - video_encoder_->ForceIdr(); - LOG_INFO("Force I frame"); - b_force_i_frame_ = false; - } - - int ret = video_encoder_->Encode( - video_frame, - [this](char *encoded_frame, size_t size, - VideoEncoder::VideoFrameType frame_type) -> int { - for (auto &ice_trans : ice_transmission_list_) { - if (!is_ice_transmission_ready_[ice_trans.first]) { - continue; - } - // LOG_ERROR("Send frame size: [{}]", size); - ice_trans.second->SendVideoData( - static_cast(frame_type), - encoded_frame, size); - } - return 0; - }); - - if (0 != ret) { - LOG_ERROR("Encode failed"); - return -1; - } - - return 0; -} - void PeerConnection::ProcessSignal(const std::string &signal) { auto j = json::parse(signal); std::string type = j["type"]; @@ -842,20 +636,21 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) { for (auto &remote_user_id : user_id_list) { ice_transmission_list_[remote_user_id] = - std::make_unique(true, transmission_id, user_id_, - remote_user_id, ws_transport_, - on_ice_status_change_); + std::make_unique( + true, transmission_id, user_id_, remote_user_id, ws_transport_, + on_ice_status_change_, user_data_); ice_transmission_list_[remote_user_id]->SetLocalCapabilities( - trickle_ice_, reliable_ice_, enable_turn_, false, - video_payload_types_, audio_payload_types_); + hardware_acceleration_, trickle_ice_, reliable_ice_, enable_turn_, + false, video_payload_types_, audio_payload_types_); ice_transmission_list_[remote_user_id]->SetOnReceiveVideoFunc( - on_receive_video_); + on_receive_video_frame_); ice_transmission_list_[remote_user_id]->SetOnReceiveAudioFunc( - on_receive_audio_); + on_receive_audio_buffer_); ice_transmission_list_[remote_user_id]->SetOnReceiveDataFunc( - on_receive_data_); + on_receive_data_buffer_); + ice_transmission_list_[remote_user_id]->SetOnReceiveNetStatusReportFunc( on_net_status_report_); @@ -889,20 +684,21 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) { ice_transmission_list_.find(remote_user_id)) { // Enable TURN for answer peer by default ice_transmission_list_[remote_user_id] = - std::make_unique(false, transmission_id, user_id_, - remote_user_id, ws_transport_, - on_ice_status_change_); + std::make_unique( + false, transmission_id, user_id_, remote_user_id, ws_transport_, + on_ice_status_change_, user_data_); ice_transmission_list_[remote_user_id]->SetLocalCapabilities( - trickle_ice_, reliable_ice_, enable_turn_, false, - video_payload_types_, audio_payload_types_); + hardware_acceleration_, trickle_ice_, reliable_ice_, enable_turn_, + false, video_payload_types_, audio_payload_types_); ice_transmission_list_[remote_user_id]->SetOnReceiveVideoFunc( - on_receive_video_); + on_receive_video_frame_); ice_transmission_list_[remote_user_id]->SetOnReceiveAudioFunc( - on_receive_audio_); + on_receive_audio_buffer_); ice_transmission_list_[remote_user_id]->SetOnReceiveDataFunc( - on_receive_data_); + on_receive_data_buffer_); + ice_transmission_list_[remote_user_id]->SetOnReceiveNetStatusReportFunc( on_net_status_report_); @@ -921,18 +717,6 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) { if (0 != ret) { NegotiationFailed(); break; - } else { - std::vector negotiated_payload_types = - ice_transmission_list_[remote_user_id]->GetNegotiatedCapabilities(); - if (0 != CreateVideoCodec(RtpPacket::PAYLOAD_TYPE::AV1 == - negotiated_payload_types[0], - hardware_acceleration_)) { - LOG_ERROR("Create video codec failed"); - } - - if (0 != CreateAudioCodec()) { - LOG_ERROR("Create audio codec failed"); - } } if (trickle_ice_) { @@ -953,19 +737,6 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) { if (0 != ret) { Leave(remote_transmission_id_); break; - } else { - std::vector negotiated_payload_types = - ice_transmission_list_[remote_user_id] - ->GetNegotiatedCapabilities(); - if (0 != CreateVideoCodec(RtpPacket::PAYLOAD_TYPE::AV1 == - negotiated_payload_types[0], - hardware_acceleration_)) { - LOG_ERROR("Create video codec failed"); - } - - if (0 != CreateAudioCodec()) { - LOG_ERROR("Create audio codec failed"); - } } if (trickle_ice_) { diff --git a/src/pc/peer_connection.h b/src/pc/peer_connection.h index 87b1c65..aae591e 100644 --- a/src/pc/peer_connection.h +++ b/src/pc/peer_connection.h @@ -104,9 +104,6 @@ class PeerConnection { private: int Login(); - int CreateVideoCodec(bool av1, bool hardware_acceleration); - int CreateAudioCodec(); - void ProcessSignal(const std::string &signal); int RequestTransmissionMemberList(const std::string &transmission_id, @@ -168,12 +165,7 @@ class PeerConnection { std::map> ice_transmission_list_; std::map is_ice_transmission_ready_; - std::function - on_receive_video_ = nullptr; - std::function - on_receive_audio_ = nullptr; - std::function - on_receive_data_ = nullptr; + std::function on_ice_status_change_ = nullptr; std::function video_encoder_ = nullptr; - std::unique_ptr video_decoder_ = nullptr; - bool hardware_accelerated_encode_ = false; - bool hardware_accelerated_decode_ = false; bool b_force_i_frame_ = false; - bool video_codec_inited_ = false; - bool load_nvcodec_dll_success = false; - - private: - std::unique_ptr audio_encoder_ = nullptr; - std::unique_ptr audio_decoder_ = nullptr; - bool audio_codec_inited_ = false; private: std::thread ice_worker_; diff --git a/src/rtc/x_inner.cpp b/src/rtc/x_inner.cpp index 7bb384b..35590e7 100644 --- a/src/rtc/x_inner.cpp +++ b/src/rtc/x_inner.cpp @@ -98,23 +98,6 @@ int LeaveConnection(PeerPtr *peer_ptr, const char *transmission_id) { return 0; } -int SendData(PeerPtr *peer_ptr, DATA_TYPE data_type, const char *data, - size_t size) { - if (!peer_ptr) { - LOG_ERROR("peer_ptr not created"); - return -1; - } - - if (DATA_TYPE::VIDEO == data_type) { - peer_ptr->peer_connection->SendVideoData(data, size); - } else if (DATA_TYPE::AUDIO == data_type) { - peer_ptr->peer_connection->SendAudioData(data, size); - } else if (DATA_TYPE::DATA == data_type) { - peer_ptr->peer_connection->SendUserData(data, size); - } - return 0; -} - DLLAPI int SendVideoFrame(PeerPtr *peer_ptr, const XVideoFrame *video_frame) { if (!peer_ptr) { LOG_ERROR("peer_ptr not created"); @@ -124,9 +107,44 @@ DLLAPI int SendVideoFrame(PeerPtr *peer_ptr, const XVideoFrame *video_frame) { if (!video_frame) { LOG_ERROR("Invaild video frame"); return -1; + } else if (!video_frame->data || video_frame->size <= 0) { + LOG_ERROR("Invaild video frame"); + return -1; } peer_ptr->peer_connection->SendVideoData(video_frame); + return 0; +} + +DLLAPI int SendAudioFrame(PeerPtr *peer_ptr, const char *data, size_t size) { + if (!peer_ptr) { + LOG_ERROR("peer_ptr not created"); + return -1; + } + + if (!data || size <= 0) { + LOG_ERROR("Invaild video frame"); + return -1; + } + + peer_ptr->peer_connection->SendAudioData(data, size); + + return 0; +} + +int SendData(PeerPtr *peer_ptr, const char *data, size_t size) { + if (!peer_ptr) { + LOG_ERROR("peer_ptr not created"); + return -1; + } + + if (!data || size <= 0) { + LOG_ERROR("Invaild data"); + return -1; + } + + peer_ptr->peer_connection->SendUserData(data, size); + return 0; } \ No newline at end of file diff --git a/src/transmission/ice_transmission.cpp b/src/transmission/ice_transmission.cpp index bbf6636..3be3efd 100644 --- a/src/transmission/ice_transmission.cpp +++ b/src/transmission/ice_transmission.cpp @@ -8,26 +8,44 @@ #include "common.h" #include "ikcp.h" #include "log.h" +#if __APPLE__ +#else +#include "nvcodec_api.h" +#endif using nlohmann::json; IceTransmission::IceTransmission( bool offer_peer, std::string &transmission_id, std::string &user_id, std::string &remote_user_id, std::shared_ptr ice_ws_transmission, - std::function on_ice_status_change) + std::function on_ice_status_change, + void *user_data) : offer_peer_(offer_peer), transmission_id_(transmission_id), user_id_(user_id), remote_user_id_(remote_user_id), ice_ws_transport_(ice_ws_transmission), - on_ice_status_change_(on_ice_status_change) {} + on_ice_status_change_(on_ice_status_change), + user_data_(user_data) {} -IceTransmission::~IceTransmission() {} +IceTransmission::~IceTransmission() { + video_codec_inited_ = false; + audio_codec_inited_ = false; + load_nvcodec_dll_success_ = false; + +#ifdef __APPLE__ +#else + if (hardware_acceleration_ && load_nvcodec_dll_success_) { + ReleaseNvCodecDll(); + } +#endif +} int IceTransmission::SetLocalCapabilities( - bool use_trickle_ice, bool use_reliable_ice, bool enable_turn, - bool force_turn, std::vector &video_payload_types, + bool hardware_acceleration, bool use_trickle_ice, bool use_reliable_ice, + bool enable_turn, bool force_turn, std::vector &video_payload_types, std::vector &audio_payload_types) { + hardware_acceleration_ = hardware_acceleration; use_trickle_ice_ = use_trickle_ice; use_reliable_ice_ = use_reliable_ice; enable_turn_ = force_turn; @@ -81,8 +99,20 @@ int IceTransmission::InitIceTransmission( [this](VideoFrame &video_frame) -> void { // LOG_ERROR("OnReceiveCompleteFrame {}", video_frame.Size()); ice_io_statistics_->UpdateVideoInboundBytes(video_frame.Size()); - on_receive_video_((const char *)video_frame.Buffer(), - video_frame.Size(), remote_user_id_); + + int num_frame_returned = video_decoder_->Decode( + (uint8_t *)video_frame.Buffer(), video_frame.Size(), + [this](VideoFrame video_frame) { + if (on_receive_video_) { + XVideoFrame x_video_frame; + x_video_frame.data = (const char *)video_frame.Buffer(); + x_video_frame.width = video_frame.Width(); + x_video_frame.height = video_frame.Height(); + x_video_frame.size = video_frame.Size(); + on_receive_video_(&x_video_frame, remote_user_id_.data(), + remote_user_id_.size(), user_data_); + } + }); }); rtp_video_receiver_->Start(); @@ -126,11 +156,18 @@ int IceTransmission::InitIceTransmission( return ice_agent_->Send(data, size); }); - rtp_audio_receiver_->SetOnReceiveData( - [this](const char *data, size_t size) -> void { - ice_io_statistics_->UpdateAudioInboundBytes(size); - on_receive_audio_(data, size, remote_user_id_); - }); + rtp_audio_receiver_->SetOnReceiveData([this](const char *data, + size_t size) -> void { + ice_io_statistics_->UpdateAudioInboundBytes(size); + + int num_frame_returned = audio_decoder_->Decode( + (uint8_t *)data, size, [this](uint8_t *data, int size) { + if (on_receive_audio_) { + on_receive_audio_((const char *)data, size, remote_user_id_.data(), + remote_user_id_.size(), user_data_); + } + }); + }); rtp_audio_sender_ = std::make_unique(); rtp_audio_sender_->SetSendDataFunc( @@ -174,7 +211,11 @@ int IceTransmission::InitIceTransmission( rtp_data_receiver_->SetOnReceiveData( [this](const char *data, size_t size) -> void { ice_io_statistics_->UpdateDataInboundBytes(size); - on_receive_data_(data, size, remote_user_id_); + + if (on_receive_data_) { + on_receive_data_(data, size, remote_user_id_.data(), + remote_user_id_.size(), user_data_); + } }); rtp_data_sender_ = std::make_unique(); @@ -372,6 +413,103 @@ int IceTransmission::CreateMediaCodec() { return 0; } +int IceTransmission::CreateVideoCodec(RtpPacket::PAYLOAD_TYPE video_pt, + bool hardware_acceleration) { + if (video_codec_inited_) { + return 0; + } + + hardware_acceleration_ = hardware_acceleration; + + if (RtpPacket::PAYLOAD_TYPE::AV1 == video_pt) { + if (hardware_acceleration_) { + hardware_acceleration_ = false; + LOG_WARN("Only support software codec for AV1"); + } + video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, true); + video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, true); + } else if (RtpPacket::PAYLOAD_TYPE::H264 == video_pt) { +#ifdef __APPLE__ + if (hardware_acceleration_) { + hardware_acceleration_ = false; + LOG_WARN( + "MacOS not support hardware acceleration, use default software " + "codec"); + video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); + video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); + } else { + video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); + video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); + } +#else + if (hardware_acceleration_) { + if (0 == LoadNvCodecDll()) { + load_nvcodec_dll_success_ = true; + video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(true, false); + video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(true, false); + } else { + LOG_WARN( + "Hardware accelerated codec not available, use default software " + "codec"); + video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); + video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); + } + } else { + video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); + video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); + } +#endif + } + + if (!video_encoder_) { + video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false); + LOG_ERROR("Create encoder failed, try to use software H.264 encoder"); + } + if (!video_encoder_ || 0 != video_encoder_->Init()) { + LOG_ERROR("Encoder init failed"); + return -1; + } + + if (!video_decoder_) { + video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false); + LOG_ERROR("Create decoder failed, try to use software H.264 decoder"); + } + if (!video_decoder_ || video_decoder_->Init()) { + LOG_ERROR("Decoder init failed"); + return -1; + } + + video_codec_inited_ = true; + LOG_INFO("Create video codec [{}|{}] finish", + video_encoder_->GetEncoderName(), video_decoder_->GetDecoderName()); + + return 0; +} + +int IceTransmission::CreateAudioCodec() { + if (audio_codec_inited_) { + return 0; + } + + audio_encoder_ = std::make_unique(AudioEncoder(48000, 1, 480)); + if (!audio_encoder_ || 0 != audio_encoder_->Init()) { + LOG_ERROR("Audio encoder init failed"); + return -1; + } + + audio_decoder_ = std::make_unique(AudioDecoder(48000, 1, 480)); + if (!audio_decoder_ || 0 != audio_decoder_->Init()) { + LOG_ERROR("Audio decoder init failed"); + return -1; + } + + audio_codec_inited_ = true; + LOG_INFO("Create audio codec [{}|{}] finish", + audio_encoder_->GetEncoderName(), audio_decoder_->GetDecoderName()); + + return 0; +} + int IceTransmission::SetTransmissionId(const std::string &transmission_id) { transmission_id_ = transmission_id; @@ -570,6 +708,8 @@ std::string IceTransmission::GetRemoteCapabilities( } CreateMediaCodec(); + CreateVideoCodec(negotiated_video_pt_, hardware_acceleration_); + CreateAudioCodec(); remote_capabilities_got_ = true; } @@ -795,7 +935,7 @@ IceTransmission::GetNegotiatedCapabilities() { return {negotiated_video_pt_, negotiated_audio_pt_, negotiated_data_pt_}; } -int IceTransmission::SendData(DATA_TYPE type, const char *data, size_t size) { +int IceTransmission::SendVideoData(const XVideoFrame *video_frame) { if (state_ != NICE_COMPONENT_STATE_CONNECTED && state_ != NICE_COMPONENT_STATE_READY) { LOG_ERROR("Ice is not connected, state = [{}]", @@ -803,36 +943,63 @@ int IceTransmission::SendData(DATA_TYPE type, const char *data, size_t size) { return -2; } - std::vector packets; + if (b_force_i_frame_) { + video_encoder_->ForceIdr(); + LOG_INFO("Force I frame"); + b_force_i_frame_ = false; + } - if (DATA_TYPE::VIDEO == type) { - if (rtp_video_sender_) { - if (video_rtp_codec_) { - video_rtp_codec_->Encode((uint8_t *)data, size, packets); - } - rtp_video_sender_->Enqueue(packets); - } - } else if (DATA_TYPE::AUDIO == type) { - if (rtp_audio_sender_) { - if (audio_rtp_codec_) { - audio_rtp_codec_->Encode((uint8_t *)data, size, packets); - rtp_audio_sender_->Enqueue(packets); - } - } - } else if (DATA_TYPE::DATA == type) { - if (rtp_data_sender_) { - if (data_rtp_codec_) { - data_rtp_codec_->Encode((uint8_t *)data, size, packets); - rtp_data_sender_->Enqueue(packets); - } - } + int ret = video_encoder_->Encode( + video_frame, + [this](char *encoded_frame, size_t size, + VideoEncoder::VideoFrameType frame_type) -> int { + std::vector packets; + if (rtp_video_sender_) { + if (video_rtp_codec_) { + video_rtp_codec_->Encode( + static_cast(frame_type), + (uint8_t *)encoded_frame, size, packets); + } + rtp_video_sender_->Enqueue(packets); + } + + return 0; + }); + + if (0 != ret) { + LOG_ERROR("Encode failed"); + return -1; } return 0; } -int IceTransmission::SendVideoData(VideoFrameType frame_type, const char *data, - size_t size) { +int IceTransmission::SendAudioData(const char *data, size_t size) { + if (state_ != NICE_COMPONENT_STATE_CONNECTED && + state_ != NICE_COMPONENT_STATE_READY) { + LOG_ERROR("Ice is not connected, state = [{}]", + nice_component_state_to_string(state_)); + return -2; + } + + int ret = audio_encoder_->Encode( + (uint8_t *)data, size, + [this](char *encoded_audio_buffer, size_t size) -> int { + if (rtp_audio_sender_) { + if (audio_rtp_codec_) { + std::vector packets; + audio_rtp_codec_->Encode((uint8_t *)encoded_audio_buffer, size, + packets); + rtp_audio_sender_->Enqueue(packets); + } + } + return 0; + }); + + return 0; +} + +int IceTransmission::SendUserData(const char *data, size_t size) { if (state_ != NICE_COMPONENT_STATE_CONNECTED && state_ != NICE_COMPONENT_STATE_READY) { LOG_ERROR("Ice is not connected, state = [{}]", @@ -842,13 +1009,11 @@ int IceTransmission::SendVideoData(VideoFrameType frame_type, const char *data, std::vector packets; - if (rtp_video_sender_) { - if (video_rtp_codec_) { - video_rtp_codec_->Encode( - static_cast(frame_type), (uint8_t *)data, - size, packets); + if (rtp_data_sender_) { + if (data_rtp_codec_) { + data_rtp_codec_->Encode((uint8_t *)data, size, packets); + rtp_data_sender_->Enqueue(packets); } - rtp_video_sender_->Enqueue(packets); } return 0; diff --git a/src/transmission/ice_transmission.h b/src/transmission/ice_transmission.h index fec5bc3..be09797 100644 --- a/src/transmission/ice_transmission.h +++ b/src/transmission/ice_transmission.h @@ -9,6 +9,8 @@ #include +#include "audio_decoder.h" +#include "audio_encoder.h" #include "congestion_control.h" #include "ice_agent.h" #include "io_statistics.h" @@ -21,11 +23,14 @@ #include "rtp_packet.h" #include "rtp_video_receiver.h" #include "rtp_video_sender.h" +#include "video_decoder_factory.h" +#include "video_encoder_factory.h" #include "ws_client.h" class IceTransmission { public: typedef enum { VIDEO = 96, AUDIO = 97, DATA = 127 } DATA_TYPE; + typedef enum { H264 = 96, AV1 = 99 } VIDEO_TYPE; enum VideoFrameType { kEmptyFrame = 0, kVideoFrameKey = 3, @@ -39,12 +44,14 @@ class IceTransmission { std::string &user_id, std::string &remote_user_id, std::shared_ptr ice_ws_transmission, std::function - on_ice_status_change); + on_ice_status_change, + void *user_data); ~IceTransmission(); public: - int SetLocalCapabilities(bool use_trickle_ice, bool use_reliable_ice, - bool enable_turn, bool force_turn, + int SetLocalCapabilities(bool hardware_acceleration, bool use_trickle_ice, + bool use_reliable_ice, bool enable_turn, + bool force_turn, std::vector &video_payload_types, std::vector &audio_payload_types); @@ -57,19 +64,22 @@ class IceTransmission { int DestroyIceTransmission(); void SetOnReceiveVideoFunc( - std::function + std::function on_receive_video) { on_receive_video_ = on_receive_video; } void SetOnReceiveAudioFunc( - std::function + std::function on_receive_audio) { on_receive_audio_ = on_receive_audio; } void SetOnReceiveDataFunc( - std::function + std::function on_receive_data) { on_receive_data_ = on_receive_data; } @@ -87,9 +97,7 @@ class IceTransmission { int SetTransmissionId(const std::string &transmission_id); - int SendData(DATA_TYPE type, const char *data, size_t size); - - int SendVideoData(VideoFrameType frame_type, const char *data, size_t size); + int SendVideoData(const XVideoFrame *video_frame); int SendAudioData(const char *data, size_t size); @@ -119,6 +127,10 @@ class IceTransmission { int CreateMediaCodec(); + int CreateVideoCodec(RtpPacket::PAYLOAD_TYPE video_pt, + bool hardware_acceleration); + int CreateAudioCodec(); + private: uint8_t CheckIsRtcpPacket(const char *buffer, size_t size); uint8_t CheckIsVideoPacket(const char *buffer, size_t size); @@ -147,20 +159,23 @@ class IceTransmission { std::string remote_ice_username_ = ""; NiceComponentState state_ = NICE_COMPONENT_STATE_DISCONNECTED; TraversalType traversal_type_ = TraversalType::TP2P; + void *user_data_ = nullptr; private: std::unique_ptr ice_agent_ = nullptr; bool is_closed_ = false; std::shared_ptr ice_ws_transport_ = nullptr; CongestionControl *congestion_control_ = nullptr; - std::function + std::function on_receive_video_ = nullptr; - std::function + std::function on_receive_audio_ = nullptr; - std::function + std::function on_receive_data_ = nullptr; + std::function on_ice_status_change_ = nullptr; + std::function video_encoder_ = nullptr; + std::unique_ptr video_decoder_ = nullptr; + bool b_force_i_frame_ = false; + bool video_codec_inited_ = false; + bool load_nvcodec_dll_success_ = false; + bool hardware_acceleration_ = false; + + private: + std::unique_ptr audio_encoder_ = nullptr; + std::unique_ptr audio_decoder_ = nullptr; + bool audio_codec_inited_ = false; }; #endif \ No newline at end of file diff --git a/xmake.lua b/xmake.lua index 39184e2..9a9ea35 100644 --- a/xmake.lua +++ b/xmake.lua @@ -184,13 +184,13 @@ target("statistics") target("transmission") set_kind("object") - add_deps("log", "ws", "ice", "qos", "rtp", "rtcp", "statistics") + add_deps("log", "ws", "ice", "qos", "rtp", "rtcp", "statistics", "media") add_files("src/transmission/*.cpp") add_includedirs("src/ws", "src/ice", "src/qos", {public = true}) target("pc") set_kind("object") - add_deps("log", "ws", "ice", "transmission", "inih", "common", "media") + add_deps("log", "ws", "ice", "transmission", "inih", "common") add_files("src/pc/*.cpp") add_includedirs("src/transmission", "src/interface", {public = true})