YangWX/crossdesk
1
0
Fork 0
mirror of https://github.com/kunkundi/crossdesk.git synced 2025-10-26 20:25:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

[fix] move codecs from pc into transmission module

Browse Source
This commit is contained in:
dijunkun
2024-11-26 15:06:05 +08:00
parent 98c6501846
commit 09c0ab9235
7 changed files with 323 additions and 359 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/interface/x.h
View File

@@ -115,11 +115,12 @@ DLLAPI int JoinConnection(PeerPtr* peer_ptr, const char* transmission_id,
DLLAPI int LeaveConnection(PeerPtr* peer_ptr, const char* transmission_id);
DLLAPI int SendData(PeerPtr* peer_ptr, DATA_TYPE data_type, const char* data,
size_t size);
DLLAPI int SendVideoFrame(PeerPtr* peer_ptr, const XVideoFrame* video_frame);
DLLAPI int SendAudioFrame(PeerPtr* peer_ptr, const char* data, size_t size);
DLLAPI int SendData(PeerPtr* peer_ptr, const char* data, size_t size);
#ifdef __cplusplus
}
#endif

293
src/pc/peer_connection.cpp
View File

@@ -7,11 +7,6 @@
#include "log.h"
#include "nlohmann/json.hpp"
#if __APPLE__
#else
#include "nvcodec_api.h"
#endif
using nlohmann::json;
PeerConnection::PeerConnection() {}
@@ -22,10 +17,6 @@ PeerConnection::~PeerConnection() {
nv12_data_ = nullptr;
}
video_codec_inited_ = false;
audio_codec_inited_ = false;
load_nvcodec_dll_success = false;
user_data_ = nullptr;
}
@@ -143,41 +134,6 @@ int PeerConnection::Init(PeerConnectionParams params,
}
};
on_receive_video_ = [this](const char *data, size_t size,
const std::string &user_id) {
int num_frame_returned = video_decoder_->Decode(
(uint8_t *)data, size, [this, user_id](VideoFrame video_frame) {
if (on_receive_video_frame_) {
XVideoFrame x_video_frame;
x_video_frame.data = (const char *)video_frame.Buffer();
x_video_frame.width = video_frame.Width();
x_video_frame.height = video_frame.Height();
x_video_frame.size = video_frame.Size();
on_receive_video_frame_(&x_video_frame, user_id.data(),
user_id.size(), user_data_);
}
});
};
on_receive_audio_ = [this](const char *data, size_t size,
const std::string &user_id) {
int num_frame_returned = audio_decoder_->Decode(
(uint8_t *)data, size, [this, user_id](uint8_t *data, int size) {
if (on_receive_audio_buffer_) {
on_receive_audio_buffer_((const char *)data, size, user_id.data(),
user_id.size(), user_data_);
}
});
};
on_receive_data_ = [this](const char *data, size_t size,
const std::string &user_id) {
if (on_receive_data_buffer_) {
on_receive_data_buffer_(data, size, user_id.data(), user_id.size(),
user_data_);
}
};
on_ice_status_change_ = [this](std::string ice_status,
const std::string &user_id) {
if ("connecting" == ice_status) {
@@ -277,102 +233,6 @@ int PeerConnection::Init(PeerConnectionParams params,
return 0;
}
int PeerConnection::CreateVideoCodec(bool av1, bool hardware_acceleration) {
if (video_codec_inited_) {
return 0;
}
hardware_acceleration_ = hardware_acceleration;
if (av1) {
if (hardware_acceleration_) {
hardware_acceleration_ = false;
LOG_WARN("Only support software codec for AV1");
}
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, true);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, true);
} else {
#ifdef __APPLE__
if (hardware_acceleration_) {
hardware_acceleration_ = false;
LOG_WARN(
"MacOS not support hardware acceleration, use default software "
"codec");
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
} else {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
#else
if (hardware_acceleration_) {
if (0 == LoadNvCodecDll()) {
load_nvcodec_dll_success = true;
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(true, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(true, false);
} else {
LOG_WARN(
"Hardware accelerated codec not available, use default software "
"codec");
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
} else {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
#endif
}
if (!video_encoder_) {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
LOG_ERROR("Create encoder failed, try to use software H.264 encoder");
}
if (!video_encoder_ || 0 != video_encoder_->Init()) {
LOG_ERROR("Encoder init failed");
return -1;
}
if (!video_decoder_) {
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
LOG_ERROR("Create decoder failed, try to use software H.264 decoder");
}
if (!video_decoder_ || video_decoder_->Init()) {
LOG_ERROR("Decoder init failed");
return -1;
}
video_codec_inited_ = true;
LOG_INFO("Create video codec [{}|{}] finish",
video_encoder_->GetEncoderName(), video_decoder_->GetDecoderName());
return 0;
}
int PeerConnection::CreateAudioCodec() {
if (audio_codec_inited_) {
return 0;
}
audio_encoder_ = std::make_unique<AudioEncoder>(AudioEncoder(48000, 1, 480));
if (!audio_encoder_ || 0 != audio_encoder_->Init()) {
LOG_ERROR("Audio encoder init failed");
return -1;
}
audio_decoder_ = std::make_unique<AudioDecoder>(AudioDecoder(48000, 1, 480));
if (!audio_decoder_ || 0 != audio_decoder_->Init()) {
LOG_ERROR("Audio decoder init failed");
return -1;
}
audio_codec_inited_ = true;
LOG_INFO("Create audio codec [{}|{}] finish",
audio_encoder_->GetEncoderName(), audio_decoder_->GetDecoderName());
return 0;
}
int PeerConnection::Login() {
if (WsStatus::WsOpened != ws_status_) {
LOG_ERROR("Websocket not opened");
@@ -494,12 +354,6 @@ int PeerConnection::Destroy() {
nv12_data_ = nullptr;
}
#ifdef __APPLE__
#else
if (hardware_acceleration_ && load_nvcodec_dll_success) {
ReleaseNvCodecDll();
}
#endif
return 0;
}
@@ -527,36 +381,18 @@ SignalStatus PeerConnection::GetSignalStatus() {
return signal_status_;
}
int PeerConnection::SendVideoData(const char *data, size_t size) {
// media send
int PeerConnection::SendVideoData(const XVideoFrame *video_frame) {
if (ice_transmission_list_.empty()) {
return -1;
}
if (b_force_i_frame_) {
LOG_INFO("Force I frame");
video_encoder_->ForceIdr();
b_force_i_frame_ = false;
}
for (auto &ice_trans : ice_transmission_list_) {
if (!is_ice_transmission_ready_[ice_trans.first]) {
continue;
}
int ret = video_encoder_->Encode(
(uint8_t *)data, size,
[this](char *encoded_frame, size_t size,
VideoEncoder::VideoFrameType frame_type) -> int {
for (auto &ice_trans : ice_transmission_list_) {
if (!is_ice_transmission_ready_[ice_trans.first]) {
continue;
}
// LOG_ERROR("Send frame size: [{}]", size);
ice_trans.second->SendVideoData(
static_cast<IceTransmission::VideoFrameType>(frame_type),
encoded_frame, size);
}
return 0;
});
if (0 != ret) {
LOG_ERROR("Encode failed");
return -1;
ice_trans.second->SendVideoData(video_frame);
}
return 0;
@@ -567,19 +403,12 @@ int PeerConnection::SendAudioData(const char *data, size_t size) {
return -1;
}
int ret = audio_encoder_->Encode(
(uint8_t *)data, size,
[this](char *encoded_audio_buffer, size_t size) -> int {
for (auto &ice_trans : ice_transmission_list_) {
if (!is_ice_transmission_ready_[ice_trans.first]) {
continue;
}
// LOG_ERROR("opus frame size: [{}]", size);
ice_trans.second->SendData(IceTransmission::DATA_TYPE::AUDIO,
encoded_audio_buffer, size);
}
return 0;
});
for (auto &ice_trans : ice_transmission_list_) {
if (!is_ice_transmission_ready_[ice_trans.first]) {
continue;
}
ice_trans.second->SendAudioData(data, size);
}
return 0;
}
@@ -589,46 +418,11 @@ int PeerConnection::SendUserData(const char *data, size_t size) {
if (!is_ice_transmission_ready_[ice_trans.first]) {
continue;
}
ice_trans.second->SendData(IceTransmission::DATA_TYPE::DATA, data, size);
ice_trans.second->SendUserData(data, size);
}
return 0;
}
int PeerConnection::SendVideoData(const XVideoFrame *video_frame) {
if (ice_transmission_list_.empty()) {
return -1;
}
if (b_force_i_frame_) {
video_encoder_->ForceIdr();
LOG_INFO("Force I frame");
b_force_i_frame_ = false;
}
int ret = video_encoder_->Encode(
video_frame,
[this](char *encoded_frame, size_t size,
VideoEncoder::VideoFrameType frame_type) -> int {
for (auto &ice_trans : ice_transmission_list_) {
if (!is_ice_transmission_ready_[ice_trans.first]) {
continue;
}
// LOG_ERROR("Send frame size: [{}]", size);
ice_trans.second->SendVideoData(
static_cast<IceTransmission::VideoFrameType>(frame_type),
encoded_frame, size);
}
return 0;
});
if (0 != ret) {
LOG_ERROR("Encode failed");
return -1;
}
return 0;
}
void PeerConnection::ProcessSignal(const std::string &signal) {
auto j = json::parse(signal);
std::string type = j["type"];
@@ -842,20 +636,21 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) {
for (auto &remote_user_id : user_id_list) {
ice_transmission_list_[remote_user_id] =
std::make_unique<IceTransmission>(true, transmission_id, user_id_,
remote_user_id, ws_transport_,
on_ice_status_change_);
std::make_unique<IceTransmission>(
true, transmission_id, user_id_, remote_user_id, ws_transport_,
on_ice_status_change_, user_data_);
ice_transmission_list_[remote_user_id]->SetLocalCapabilities(
trickle_ice_, reliable_ice_, enable_turn_, false,
video_payload_types_, audio_payload_types_);
hardware_acceleration_, trickle_ice_, reliable_ice_, enable_turn_,
false, video_payload_types_, audio_payload_types_);
ice_transmission_list_[remote_user_id]->SetOnReceiveVideoFunc(
on_receive_video_);
on_receive_video_frame_);
ice_transmission_list_[remote_user_id]->SetOnReceiveAudioFunc(
on_receive_audio_);
on_receive_audio_buffer_);
ice_transmission_list_[remote_user_id]->SetOnReceiveDataFunc(
on_receive_data_);
on_receive_data_buffer_);
ice_transmission_list_[remote_user_id]->SetOnReceiveNetStatusReportFunc(
on_net_status_report_);
@@ -889,20 +684,21 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) {
ice_transmission_list_.find(remote_user_id)) {
// Enable TURN for answer peer by default
ice_transmission_list_[remote_user_id] =
std::make_unique<IceTransmission>(false, transmission_id, user_id_,
remote_user_id, ws_transport_,
on_ice_status_change_);
std::make_unique<IceTransmission>(
false, transmission_id, user_id_, remote_user_id, ws_transport_,
on_ice_status_change_, user_data_);
ice_transmission_list_[remote_user_id]->SetLocalCapabilities(
trickle_ice_, reliable_ice_, enable_turn_, false,
video_payload_types_, audio_payload_types_);
hardware_acceleration_, trickle_ice_, reliable_ice_, enable_turn_,
false, video_payload_types_, audio_payload_types_);
ice_transmission_list_[remote_user_id]->SetOnReceiveVideoFunc(
on_receive_video_);
on_receive_video_frame_);
ice_transmission_list_[remote_user_id]->SetOnReceiveAudioFunc(
on_receive_audio_);
on_receive_audio_buffer_);
ice_transmission_list_[remote_user_id]->SetOnReceiveDataFunc(
on_receive_data_);
on_receive_data_buffer_);
ice_transmission_list_[remote_user_id]->SetOnReceiveNetStatusReportFunc(
on_net_status_report_);
@@ -921,18 +717,6 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) {
if (0 != ret) {
NegotiationFailed();
break;
} else {
std::vector<RtpPacket::PAYLOAD_TYPE> negotiated_payload_types =
ice_transmission_list_[remote_user_id]->GetNegotiatedCapabilities();
if (0 != CreateVideoCodec(RtpPacket::PAYLOAD_TYPE::AV1 ==
negotiated_payload_types[0],
hardware_acceleration_)) {
LOG_ERROR("Create video codec failed");
}
if (0 != CreateAudioCodec()) {
LOG_ERROR("Create audio codec failed");
}
}
if (trickle_ice_) {
@@ -953,19 +737,6 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) {
if (0 != ret) {
Leave(remote_transmission_id_);
break;
} else {
std::vector<RtpPacket::PAYLOAD_TYPE> negotiated_payload_types =
ice_transmission_list_[remote_user_id]
->GetNegotiatedCapabilities();
if (0 != CreateVideoCodec(RtpPacket::PAYLOAD_TYPE::AV1 ==
negotiated_payload_types[0],
hardware_acceleration_)) {
LOG_ERROR("Create video codec failed");
}
if (0 != CreateAudioCodec()) {
LOG_ERROR("Create audio codec failed");
}
}
if (trickle_ice_) {

21
src/pc/peer_connection.h
View File

@@ -104,9 +104,6 @@ class PeerConnection {
private:
int Login();
int CreateVideoCodec(bool av1, bool hardware_acceleration);
int CreateAudioCodec();
void ProcessSignal(const std::string &signal);
int RequestTransmissionMemberList(const std::string &transmission_id,
@@ -168,12 +165,7 @@ class PeerConnection {
std::map<std::string, std::unique_ptr<IceTransmission>>
ice_transmission_list_;
std::map<std::string, bool> is_ice_transmission_ready_;
std::function<void(const char *, size_t, const std::string &)>
on_receive_video_ = nullptr;
std::function<void(const char *, size_t, const std::string &)>
on_receive_audio_ = nullptr;
std::function<void(const char *, size_t, const std::string &)>
on_receive_data_ = nullptr;
std::function<void(std::string, const std::string &)> on_ice_status_change_ =
nullptr;
std::function<void(const std::string &, IceTransmission::TraversalType,
@@ -199,18 +191,7 @@ class PeerConnection {
std::string password_;
private:
std::unique_ptr<VideoEncoder> video_encoder_ = nullptr;
std::unique_ptr<VideoDecoder> video_decoder_ = nullptr;
bool hardware_accelerated_encode_ = false;
bool hardware_accelerated_decode_ = false;
bool b_force_i_frame_ = false;
bool video_codec_inited_ = false;
bool load_nvcodec_dll_success = false;
private:
std::unique_ptr<AudioEncoder> audio_encoder_ = nullptr;
std::unique_ptr<AudioDecoder> audio_decoder_ = nullptr;
bool audio_codec_inited_ = false;
private:
std::thread ice_worker_;

52
src/rtc/x_inner.cpp
View File

@@ -98,23 +98,6 @@ int LeaveConnection(PeerPtr *peer_ptr, const char *transmission_id) {
return 0;
}
int SendData(PeerPtr *peer_ptr, DATA_TYPE data_type, const char *data,
size_t size) {
if (!peer_ptr) {
LOG_ERROR("peer_ptr not created");
return -1;
}
if (DATA_TYPE::VIDEO == data_type) {
peer_ptr->peer_connection->SendVideoData(data, size);
} else if (DATA_TYPE::AUDIO == data_type) {
peer_ptr->peer_connection->SendAudioData(data, size);
} else if (DATA_TYPE::DATA == data_type) {
peer_ptr->peer_connection->SendUserData(data, size);
}
return 0;
}
DLLAPI int SendVideoFrame(PeerPtr *peer_ptr, const XVideoFrame *video_frame) {
if (!peer_ptr) {
LOG_ERROR("peer_ptr not created");
@@ -124,9 +107,44 @@ DLLAPI int SendVideoFrame(PeerPtr *peer_ptr, const XVideoFrame *video_frame) {
if (!video_frame) {
LOG_ERROR("Invaild video frame");
return -1;
} else if (!video_frame->data || video_frame->size <= 0) {
LOG_ERROR("Invaild video frame");
return -1;
}
peer_ptr->peer_connection->SendVideoData(video_frame);
return 0;
}
DLLAPI int SendAudioFrame(PeerPtr *peer_ptr, const char *data, size_t size) {
if (!peer_ptr) {
LOG_ERROR("peer_ptr not created");
return -1;
}
if (!data || size <= 0) {
LOG_ERROR("Invaild video frame");
return -1;
}
peer_ptr->peer_connection->SendAudioData(data, size);
return 0;
}
int SendData(PeerPtr *peer_ptr, const char *data, size_t size) {
if (!peer_ptr) {
LOG_ERROR("peer_ptr not created");
return -1;
}
if (!data || size <= 0) {
LOG_ERROR("Invaild data");
return -1;
}
peer_ptr->peer_connection->SendUserData(data, size);
return 0;
}

253
src/transmission/ice_transmission.cpp
View File

@@ -8,26 +8,44 @@
#include "common.h"
#include "ikcp.h"
#include "log.h"
#if __APPLE__
#else
#include "nvcodec_api.h"
#endif
using nlohmann::json;
IceTransmission::IceTransmission(
bool offer_peer, std::string &transmission_id, std::string &user_id,
std::string &remote_user_id, std::shared_ptr<WsClient> ice_ws_transmission,
std::function<void(std::string, const std::string &)> on_ice_status_change)
std::function<void(std::string, const std::string &)> on_ice_status_change,
void *user_data)
: offer_peer_(offer_peer),
transmission_id_(transmission_id),
user_id_(user_id),
remote_user_id_(remote_user_id),
ice_ws_transport_(ice_ws_transmission),
on_ice_status_change_(on_ice_status_change) {}
on_ice_status_change_(on_ice_status_change),
user_data_(user_data) {}
IceTransmission::~IceTransmission() {}
IceTransmission::~IceTransmission() {
video_codec_inited_ = false;
audio_codec_inited_ = false;
load_nvcodec_dll_success_ = false;
#ifdef __APPLE__
#else
if (hardware_acceleration_ && load_nvcodec_dll_success_) {
ReleaseNvCodecDll();
}
#endif
}
int IceTransmission::SetLocalCapabilities(
bool use_trickle_ice, bool use_reliable_ice, bool enable_turn,
bool force_turn, std::vector<int> &video_payload_types,
bool hardware_acceleration, bool use_trickle_ice, bool use_reliable_ice,
bool enable_turn, bool force_turn, std::vector<int> &video_payload_types,
std::vector<int> &audio_payload_types) {
hardware_acceleration_ = hardware_acceleration;
use_trickle_ice_ = use_trickle_ice;
use_reliable_ice_ = use_reliable_ice;
enable_turn_ = force_turn;
@@ -81,8 +99,20 @@ int IceTransmission::InitIceTransmission(
[this](VideoFrame &video_frame) -> void {
// LOG_ERROR("OnReceiveCompleteFrame {}", video_frame.Size());
ice_io_statistics_->UpdateVideoInboundBytes(video_frame.Size());
on_receive_video_((const char *)video_frame.Buffer(),
video_frame.Size(), remote_user_id_);
int num_frame_returned = video_decoder_->Decode(
(uint8_t *)video_frame.Buffer(), video_frame.Size(),
[this](VideoFrame video_frame) {
if (on_receive_video_) {
XVideoFrame x_video_frame;
x_video_frame.data = (const char *)video_frame.Buffer();
x_video_frame.width = video_frame.Width();
x_video_frame.height = video_frame.Height();
x_video_frame.size = video_frame.Size();
on_receive_video_(&x_video_frame, remote_user_id_.data(),
remote_user_id_.size(), user_data_);
}
});
});
rtp_video_receiver_->Start();
@@ -126,11 +156,18 @@ int IceTransmission::InitIceTransmission(
return ice_agent_->Send(data, size);
});
rtp_audio_receiver_->SetOnReceiveData(
[this](const char *data, size_t size) -> void {
ice_io_statistics_->UpdateAudioInboundBytes(size);
on_receive_audio_(data, size, remote_user_id_);
});
rtp_audio_receiver_->SetOnReceiveData([this](const char *data,
size_t size) -> void {
ice_io_statistics_->UpdateAudioInboundBytes(size);
int num_frame_returned = audio_decoder_->Decode(
(uint8_t *)data, size, [this](uint8_t *data, int size) {
if (on_receive_audio_) {
on_receive_audio_((const char *)data, size, remote_user_id_.data(),
remote_user_id_.size(), user_data_);
}
});
});
rtp_audio_sender_ = std::make_unique<RtpAudioSender>();
rtp_audio_sender_->SetSendDataFunc(
@@ -174,7 +211,11 @@ int IceTransmission::InitIceTransmission(
rtp_data_receiver_->SetOnReceiveData(
[this](const char *data, size_t size) -> void {
ice_io_statistics_->UpdateDataInboundBytes(size);
on_receive_data_(data, size, remote_user_id_);
if (on_receive_data_) {
on_receive_data_(data, size, remote_user_id_.data(),
remote_user_id_.size(), user_data_);
}
});
rtp_data_sender_ = std::make_unique<RtpDataSender>();
@@ -372,6 +413,103 @@ int IceTransmission::CreateMediaCodec() {
return 0;
}
int IceTransmission::CreateVideoCodec(RtpPacket::PAYLOAD_TYPE video_pt,
bool hardware_acceleration) {
if (video_codec_inited_) {
return 0;
}
hardware_acceleration_ = hardware_acceleration;
if (RtpPacket::PAYLOAD_TYPE::AV1 == video_pt) {
if (hardware_acceleration_) {
hardware_acceleration_ = false;
LOG_WARN("Only support software codec for AV1");
}
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, true);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, true);
} else if (RtpPacket::PAYLOAD_TYPE::H264 == video_pt) {
#ifdef __APPLE__
if (hardware_acceleration_) {
hardware_acceleration_ = false;
LOG_WARN(
"MacOS not support hardware acceleration, use default software "
"codec");
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
} else {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
#else
if (hardware_acceleration_) {
if (0 == LoadNvCodecDll()) {
load_nvcodec_dll_success_ = true;
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(true, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(true, false);
} else {
LOG_WARN(
"Hardware accelerated codec not available, use default software "
"codec");
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
} else {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
#endif
}
if (!video_encoder_) {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
LOG_ERROR("Create encoder failed, try to use software H.264 encoder");
}
if (!video_encoder_ || 0 != video_encoder_->Init()) {
LOG_ERROR("Encoder init failed");
return -1;
}
if (!video_decoder_) {
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
LOG_ERROR("Create decoder failed, try to use software H.264 decoder");
}
if (!video_decoder_ || video_decoder_->Init()) {
LOG_ERROR("Decoder init failed");
return -1;
}
video_codec_inited_ = true;
LOG_INFO("Create video codec [{}|{}] finish",
video_encoder_->GetEncoderName(), video_decoder_->GetDecoderName());
return 0;
}
int IceTransmission::CreateAudioCodec() {
if (audio_codec_inited_) {
return 0;
}
audio_encoder_ = std::make_unique<AudioEncoder>(AudioEncoder(48000, 1, 480));
if (!audio_encoder_ || 0 != audio_encoder_->Init()) {
LOG_ERROR("Audio encoder init failed");
return -1;
}
audio_decoder_ = std::make_unique<AudioDecoder>(AudioDecoder(48000, 1, 480));
if (!audio_decoder_ || 0 != audio_decoder_->Init()) {
LOG_ERROR("Audio decoder init failed");
return -1;
}
audio_codec_inited_ = true;
LOG_INFO("Create audio codec [{}|{}] finish",
audio_encoder_->GetEncoderName(), audio_decoder_->GetDecoderName());
return 0;
}
int IceTransmission::SetTransmissionId(const std::string &transmission_id) {
transmission_id_ = transmission_id;
@@ -570,6 +708,8 @@ std::string IceTransmission::GetRemoteCapabilities(
}
CreateMediaCodec();
CreateVideoCodec(negotiated_video_pt_, hardware_acceleration_);
CreateAudioCodec();
remote_capabilities_got_ = true;
}
@@ -795,7 +935,7 @@ IceTransmission::GetNegotiatedCapabilities() {
return {negotiated_video_pt_, negotiated_audio_pt_, negotiated_data_pt_};
}
int IceTransmission::SendData(DATA_TYPE type, const char *data, size_t size) {
int IceTransmission::SendVideoData(const XVideoFrame *video_frame) {
if (state_ != NICE_COMPONENT_STATE_CONNECTED &&
state_ != NICE_COMPONENT_STATE_READY) {
LOG_ERROR("Ice is not connected, state = [{}]",
@@ -803,36 +943,63 @@ int IceTransmission::SendData(DATA_TYPE type, const char *data, size_t size) {
return -2;
}
std::vector<RtpPacket> packets;
if (b_force_i_frame_) {
video_encoder_->ForceIdr();
LOG_INFO("Force I frame");
b_force_i_frame_ = false;
}
if (DATA_TYPE::VIDEO == type) {
if (rtp_video_sender_) {
if (video_rtp_codec_) {
video_rtp_codec_->Encode((uint8_t *)data, size, packets);
}
rtp_video_sender_->Enqueue(packets);
}
} else if (DATA_TYPE::AUDIO == type) {
if (rtp_audio_sender_) {
if (audio_rtp_codec_) {
audio_rtp_codec_->Encode((uint8_t *)data, size, packets);
rtp_audio_sender_->Enqueue(packets);
}
}
} else if (DATA_TYPE::DATA == type) {
if (rtp_data_sender_) {
if (data_rtp_codec_) {
data_rtp_codec_->Encode((uint8_t *)data, size, packets);
rtp_data_sender_->Enqueue(packets);
}
}
int ret = video_encoder_->Encode(
video_frame,
[this](char *encoded_frame, size_t size,
VideoEncoder::VideoFrameType frame_type) -> int {
std::vector<RtpPacket> packets;
if (rtp_video_sender_) {
if (video_rtp_codec_) {
video_rtp_codec_->Encode(
static_cast<RtpCodec::VideoFrameType>(frame_type),
(uint8_t *)encoded_frame, size, packets);
}
rtp_video_sender_->Enqueue(packets);
}
return 0;
});
if (0 != ret) {
LOG_ERROR("Encode failed");
return -1;
}
return 0;
}
int IceTransmission::SendVideoData(VideoFrameType frame_type, const char *data,
size_t size) {
int IceTransmission::SendAudioData(const char *data, size_t size) {
if (state_ != NICE_COMPONENT_STATE_CONNECTED &&
state_ != NICE_COMPONENT_STATE_READY) {
LOG_ERROR("Ice is not connected, state = [{}]",
nice_component_state_to_string(state_));
return -2;
}
int ret = audio_encoder_->Encode(
(uint8_t *)data, size,
[this](char *encoded_audio_buffer, size_t size) -> int {
if (rtp_audio_sender_) {
if (audio_rtp_codec_) {
std::vector<RtpPacket> packets;
audio_rtp_codec_->Encode((uint8_t *)encoded_audio_buffer, size,
packets);
rtp_audio_sender_->Enqueue(packets);
}
}
return 0;
});
return 0;
}
int IceTransmission::SendUserData(const char *data, size_t size) {
if (state_ != NICE_COMPONENT_STATE_CONNECTED &&
state_ != NICE_COMPONENT_STATE_READY) {
LOG_ERROR("Ice is not connected, state = [{}]",
@@ -842,13 +1009,11 @@ int IceTransmission::SendVideoData(VideoFrameType frame_type, const char *data,
std::vector<RtpPacket> packets;
if (rtp_video_sender_) {
if (video_rtp_codec_) {
video_rtp_codec_->Encode(
static_cast<RtpCodec::VideoFrameType>(frame_type), (uint8_t *)data,
size, packets);
if (rtp_data_sender_) {
if (data_rtp_codec_) {
data_rtp_codec_->Encode((uint8_t *)data, size, packets);
rtp_data_sender_->Enqueue(packets);
}
rtp_video_sender_->Enqueue(packets);
}
return 0;

52
src/transmission/ice_transmission.h
View File

@@ -9,6 +9,8 @@
#include <iostream>
#include "audio_decoder.h"
#include "audio_encoder.h"
#include "congestion_control.h"
#include "ice_agent.h"
#include "io_statistics.h"
@@ -21,11 +23,14 @@
#include "rtp_packet.h"
#include "rtp_video_receiver.h"
#include "rtp_video_sender.h"
#include "video_decoder_factory.h"
#include "video_encoder_factory.h"
#include "ws_client.h"
class IceTransmission {
public:
typedef enum { VIDEO = 96, AUDIO = 97, DATA = 127 } DATA_TYPE;
typedef enum { H264 = 96, AV1 = 99 } VIDEO_TYPE;
enum VideoFrameType {
kEmptyFrame = 0,
kVideoFrameKey = 3,
@@ -39,12 +44,14 @@ class IceTransmission {
std::string &user_id, std::string &remote_user_id,
std::shared_ptr<WsClient> ice_ws_transmission,
std::function<void(std::string, const std::string &)>
on_ice_status_change);
on_ice_status_change,
void *user_data);
~IceTransmission();
public:
int SetLocalCapabilities(bool use_trickle_ice, bool use_reliable_ice,
bool enable_turn, bool force_turn,
int SetLocalCapabilities(bool hardware_acceleration, bool use_trickle_ice,
bool use_reliable_ice, bool enable_turn,
bool force_turn,
std::vector<int> &video_payload_types,
std::vector<int> &audio_payload_types);
@@ -57,19 +64,22 @@ class IceTransmission {
int DestroyIceTransmission();
void SetOnReceiveVideoFunc(
std::function<void(const char *, size_t, const std::string &)>
std::function<void(const XVideoFrame *, const char *, const size_t,
void *)>
on_receive_video) {
on_receive_video_ = on_receive_video;
}
void SetOnReceiveAudioFunc(
std::function<void(const char *, size_t, const std::string &)>
std::function<void(const char *, size_t, const char *, const size_t,
void *)>
on_receive_audio) {
on_receive_audio_ = on_receive_audio;
}
void SetOnReceiveDataFunc(
std::function<void(const char *, size_t, const std::string &)>
std::function<void(const char *, size_t, const char *, const size_t,
void *)>
on_receive_data) {
on_receive_data_ = on_receive_data;
}
@@ -87,9 +97,7 @@ class IceTransmission {
int SetTransmissionId(const std::string &transmission_id);
int SendData(DATA_TYPE type, const char *data, size_t size);
int SendVideoData(VideoFrameType frame_type, const char *data, size_t size);
int SendVideoData(const XVideoFrame *video_frame);
int SendAudioData(const char *data, size_t size);
@@ -119,6 +127,10 @@ class IceTransmission {
int CreateMediaCodec();
int CreateVideoCodec(RtpPacket::PAYLOAD_TYPE video_pt,
bool hardware_acceleration);
int CreateAudioCodec();
private:
uint8_t CheckIsRtcpPacket(const char *buffer, size_t size);
uint8_t CheckIsVideoPacket(const char *buffer, size_t size);
@@ -147,20 +159,23 @@ class IceTransmission {
std::string remote_ice_username_ = "";
NiceComponentState state_ = NICE_COMPONENT_STATE_DISCONNECTED;
TraversalType traversal_type_ = TraversalType::TP2P;
void *user_data_ = nullptr;
private:
std::unique_ptr<IceAgent> ice_agent_ = nullptr;
bool is_closed_ = false;
std::shared_ptr<WsClient> ice_ws_transport_ = nullptr;
CongestionControl *congestion_control_ = nullptr;
std::function<void(const char *, size_t, const std::string &)>
std::function<void(const XVideoFrame *, const char *, const size_t, void *)>
on_receive_video_ = nullptr;
std::function<void(const char *, size_t, const std::string &)>
std::function<void(const char *, size_t, const char *, const size_t, void *)>
on_receive_audio_ = nullptr;
std::function<void(const char *, size_t, const std::string &)>
std::function<void(const char *, size_t, const char *, const size_t, void *)>
on_receive_data_ = nullptr;
std::function<void(std::string, const std::string &)> on_ice_status_change_ =
nullptr;
std::function<void(const std::string &, TraversalType, const uint64_t,
const uint64_t, const uint64_t, const uint64_t,
const uint64_t, const uint64_t, const uint64_t,
@@ -200,6 +215,19 @@ class IceTransmission {
RtpPacket::PAYLOAD_TYPE::UNDEFINED;
RtpPacket::PAYLOAD_TYPE negotiated_data_pt_ =
RtpPacket::PAYLOAD_TYPE::UNDEFINED;
private:
std::unique_ptr<VideoEncoder> video_encoder_ = nullptr;
std::unique_ptr<VideoDecoder> video_decoder_ = nullptr;
bool b_force_i_frame_ = false;
bool video_codec_inited_ = false;
bool load_nvcodec_dll_success_ = false;
bool hardware_acceleration_ = false;
private:
std::unique_ptr<AudioEncoder> audio_encoder_ = nullptr;
std::unique_ptr<AudioDecoder> audio_decoder_ = nullptr;
bool audio_codec_inited_ = false;
};
#endif