YangWX/crossdesk
1
0
Fork 0
mirror of https://github.com/kunkundi/crossdesk.git synced 2025-10-27 04:35:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feat] enable congestion controller set target bitrate to video encoder

Browse Source
This commit is contained in:
dijunkun
2025-02-11 17:25:50 +08:00
parent 7a8be01e7b
commit 36704c7e4c
19 changed files with 770 additions and 525 deletions
Download Patch File Download Diff File Expand all files Collapse all files

321
src/transport/ice_transport.cpp
View File

@@ -7,10 +7,6 @@
#include "common.h"
#include "log.h"
#if __APPLE__
#else
#include "nvcodec_api.h"
#endif
using nlohmann::json;
@@ -25,22 +21,9 @@ IceTransport::IceTransport(
remote_user_id_(remote_user_id),
ice_ws_transport_(ice_ws_transmission),
on_ice_status_change_(on_ice_status_change),
user_data_(user_data),
clock_(webrtc::Clock::GetRealTimeClockShared()) {}
user_data_(user_data) {}
IceTransport::~IceTransport() {
user_data_ = nullptr;
video_codec_inited_ = false;
audio_codec_inited_ = false;
load_nvcodec_dll_success_ = false;
#ifdef __APPLE__
#else
if (hardware_acceleration_ && load_nvcodec_dll_success_) {
ReleaseNvCodecDll();
}
#endif
}
IceTransport::~IceTransport() {}
int IceTransport::SetLocalCapabilities(bool hardware_acceleration,
bool use_trickle_ice,
@@ -63,12 +46,24 @@ int IceTransport::InitIceTransmission(
std::string &stun_ip, int stun_port, std::string &turn_ip, int turn_port,
std::string &turn_username, std::string &turn_password,
rtp::PAYLOAD_TYPE video_codec_payload_type) {
ice_transport_controller_ = std::make_shared<IceTransportController>();
ice_agent_ = std::make_unique<IceAgent>(
offer_peer_, use_trickle_ice_, use_reliable_ice_, enable_turn_,
force_turn_, stun_ip, stun_port, turn_ip, turn_port, turn_username,
turn_password);
InitializeIOStatistics();
ice_io_statistics_ = std::make_unique<IOStatistics>(
[this](const IOStatistics::NetTrafficStats &net_traffic_stats) {
if (on_receive_net_status_report_) {
XNetTrafficStats xnet_traffic_stats;
memcpy(&xnet_traffic_stats, &net_traffic_stats,
sizeof(XNetTrafficStats));
on_receive_net_status_report_(
user_id_.data(), user_id_.size(), TraversalMode(traversal_type_),
&xnet_traffic_stats, remote_user_id_.data(),
remote_user_id_.size(), user_data_);
}
});
ice_agent_->CreateIceAgent(
[](NiceAgent *agent, guint stream_id, guint component_id,
@@ -100,66 +95,6 @@ int IceTransport::InitIceTransmission(
return 0;
}
void IceTransport::InitializeIOStatistics() {
ice_io_statistics_ = std::make_unique<IOStatistics>(
[this](const IOStatistics::NetTrafficStats &net_traffic_stats) {
if (on_receive_net_status_report_) {
XNetTrafficStats xnet_traffic_stats;
memcpy(&xnet_traffic_stats, &net_traffic_stats,
sizeof(XNetTrafficStats));
on_receive_net_status_report_(
user_id_.data(), user_id_.size(), TraversalMode(traversal_type_),
&xnet_traffic_stats, remote_user_id_.data(),
remote_user_id_.size(), user_data_);
}
});
}
void IceTransport::InitializeChannels(
rtp::PAYLOAD_TYPE video_codec_payload_type) {
video_codec_payload_type_ = video_codec_payload_type;
video_channel_send_ = std::make_unique<VideoChannelSend>(clock_, ice_agent_,
ice_io_statistics_);
audio_channel_send_ =
std::make_unique<AudioChannelSend>(ice_agent_, ice_io_statistics_);
data_channel_send_ =
std::make_unique<DataChannelSend>(ice_agent_, ice_io_statistics_);
video_channel_send_->Initialize(video_codec_payload_type_);
audio_channel_send_->Initialize(rtp::PAYLOAD_TYPE::OPUS);
data_channel_send_->Initialize(rtp::PAYLOAD_TYPE::DATA);
std::weak_ptr<IceTransport> weak_self = shared_from_this();
video_channel_receive_ = std::make_unique<VideoChannelReceive>(
clock_, ice_agent_, ice_io_statistics_,
[this, weak_self](VideoFrame &video_frame) {
if (auto self = weak_self.lock()) {
OnReceiveCompleteFrame(video_frame);
}
});
audio_channel_receive_ = std::make_unique<AudioChannelReceive>(
ice_agent_, ice_io_statistics_,
[this, weak_self](const char *data, size_t size) {
if (auto self = weak_self.lock()) {
OnReceiveCompleteAudio(data, size);
}
});
data_channel_receive_ = std::make_unique<DataChannelReceive>(
ice_agent_, ice_io_statistics_,
[this, weak_self](const char *data, size_t size) {
if (auto self = weak_self.lock()) {
OnReceiveCompleteData(data, size);
}
});
video_channel_receive_->Initialize(video_codec_payload_type_);
audio_channel_receive_->Initialize(rtp::PAYLOAD_TYPE::OPUS);
data_channel_receive_->Initialize(rtp::PAYLOAD_TYPE::DATA);
}
void IceTransport::OnIceStateChange(NiceAgent *agent, guint stream_id,
guint component_id,
NiceComponentState state,
@@ -252,12 +187,13 @@ void IceTransport::OnReceiveBuffer(NiceAgent *agent, guint stream_id,
gchar *buffer, gpointer user_ptr) {
if (!is_closed_) {
if (CheckIsRtpPacket(buffer, size)) {
if (CheckIsVideoPacket(buffer, size)) {
video_channel_receive_->OnReceiveRtpPacket(buffer, size);
} else if (CheckIsAudioPacket(buffer, size)) {
audio_channel_receive_->OnReceiveRtpPacket(buffer, size);
} else if (CheckIsDataPacket(buffer, size)) {
data_channel_receive_->OnReceiveRtpPacket(buffer, size);
if (CheckIsVideoPacket(buffer, size) && ice_transport_controller_) {
ice_transport_controller_->OnReceiveVideoRtpPacket(buffer, size);
} else if (CheckIsAudioPacket(buffer, size) &&
ice_transport_controller_) {
ice_transport_controller_->OnReceiveAudioRtpPacket(buffer, size);
} else if (CheckIsDataPacket(buffer, size) && ice_transport_controller_) {
ice_transport_controller_->OnReceiveDataRtpPacket(buffer, size);
}
} else if (CheckIsRtcpPacket(buffer, size)) {
// LOG_ERROR("Rtcp packet [{}]", (uint8_t)(buffer[1]));
@@ -372,42 +308,10 @@ bool IceTransport::HandleCongestionControlFeedback(
// rtcp_packet_info->congestion_control_feedback.emplace(std::move(feedback));
// }
video_channel_send_->OnCongestionControlFeedback(clock_->CurrentTime(),
feedback);
return true;
}
void IceTransport::OnReceiveCompleteFrame(VideoFrame &video_frame) {
int num_frame_returned = video_decoder_->Decode(
(uint8_t *)video_frame.Buffer(), video_frame.Size(),
[this](VideoFrame video_frame) {
if (on_receive_video_) {
XVideoFrame x_video_frame;
x_video_frame.data = (const char *)video_frame.Buffer();
x_video_frame.width = video_frame.Width();
x_video_frame.height = video_frame.Height();
x_video_frame.size = video_frame.Size();
on_receive_video_(&x_video_frame, remote_user_id_.data(),
remote_user_id_.size(), user_data_);
}
});
}
void IceTransport::OnReceiveCompleteAudio(const char *data, size_t size) {
int num_frame_returned = audio_decoder_->Decode(
(uint8_t *)data, size, [this](uint8_t *data, int size) {
if (on_receive_audio_) {
on_receive_audio_((const char *)data, size, remote_user_id_.data(),
remote_user_id_.size(), user_data_);
}
});
}
void IceTransport::OnReceiveCompleteData(const char *data, size_t size) {
if (on_receive_data_) {
on_receive_data_(data, size, remote_user_id_.data(), remote_user_id_.size(),
user_data_);
if (ice_transport_controller_) {
ice_transport_controller_->OnCongestionControlFeedback(feedback);
}
return true;
}
int IceTransport::DestroyIceTransmission() {
@@ -422,130 +326,9 @@ int IceTransport::DestroyIceTransmission() {
ice_io_statistics_->Stop();
}
if (video_channel_send_) {
video_channel_send_->Destroy();
}
if (audio_channel_send_) {
audio_channel_send_->Destroy();
}
if (data_channel_send_) {
data_channel_send_->Destroy();
}
if (video_channel_receive_) {
video_channel_receive_->Destroy();
}
if (audio_channel_receive_) {
audio_channel_receive_->Destroy();
}
if (data_channel_receive_) {
data_channel_receive_->Destroy();
}
return ice_agent_->DestroyIceAgent();
}
int IceTransport::CreateVideoCodec(rtp::PAYLOAD_TYPE video_pt,
bool hardware_acceleration) {
if (video_codec_inited_) {
return 0;
}
hardware_acceleration_ = hardware_acceleration;
if (rtp::PAYLOAD_TYPE::AV1 == video_pt) {
if (hardware_acceleration_) {
hardware_acceleration_ = false;
LOG_WARN("Only support software codec for AV1");
}
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, true);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, true);
} else if (rtp::PAYLOAD_TYPE::H264 == video_pt) {
#ifdef __APPLE__
if (hardware_acceleration_) {
hardware_acceleration_ = false;
LOG_WARN(
"MacOS not support hardware acceleration, use default software "
"codec");
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
} else {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
#else
if (hardware_acceleration_) {
if (0 == LoadNvCodecDll()) {
load_nvcodec_dll_success_ = true;
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(true, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(true, false);
} else {
LOG_WARN(
"Hardware accelerated codec not available, use default software "
"codec");
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
} else {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
#endif
}
if (!video_encoder_) {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
LOG_ERROR("Create encoder failed, try to use software H.264 encoder");
}
if (!video_encoder_ || 0 != video_encoder_->Init()) {
LOG_ERROR("Encoder init failed");
return -1;
}
if (!video_decoder_) {
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
LOG_ERROR("Create decoder failed, try to use software H.264 decoder");
}
if (!video_decoder_ || video_decoder_->Init()) {
LOG_ERROR("Decoder init failed");
return -1;
}
video_codec_inited_ = true;
LOG_INFO("Create video codec [{}|{}] finish",
video_encoder_->GetEncoderName(), video_decoder_->GetDecoderName());
return 0;
}
int IceTransport::CreateAudioCodec() {
if (audio_codec_inited_) {
return 0;
}
audio_encoder_ = std::make_unique<AudioEncoder>(AudioEncoder(48000, 1, 480));
if (!audio_encoder_ || 0 != audio_encoder_->Init()) {
LOG_ERROR("Audio encoder init failed");
return -1;
}
audio_decoder_ = std::make_unique<AudioDecoder>(AudioDecoder(48000, 1, 480));
if (!audio_decoder_ || 0 != audio_decoder_->Init()) {
LOG_ERROR("Audio decoder init failed");
return -1;
}
audio_codec_inited_ = true;
LOG_INFO("Create audio codec [{}|{}] finish",
audio_encoder_->GetEncoderName(), audio_decoder_->GetDecoderName());
return 0;
}
int IceTransport::SetTransmissionId(const std::string &transmission_id) {
transmission_id_ = transmission_id;
@@ -742,10 +525,12 @@ std::string IceTransport::GetRemoteCapabilities(const std::string &remote_sdp) {
return std::string();
}
InitializeChannels(negotiated_video_pt_);
CreateVideoCodec(negotiated_video_pt_, hardware_acceleration_);
CreateAudioCodec();
if (ice_transport_controller_) {
ice_transport_controller_->Create(
remote_user_id_, negotiated_video_pt_, hardware_acceleration_,
ice_agent_, ice_io_statistics_, on_receive_video_, on_receive_audio_,
on_receive_data_, user_data_);
}
remote_capabilities_got_ = true;
}
@@ -978,29 +763,11 @@ int IceTransport::SendVideoFrame(const XVideoFrame *video_frame) {
return -2;
}
if (b_force_i_frame_) {
video_encoder_->ForceIdr();
LOG_INFO("Force I frame");
b_force_i_frame_ = false;
if (ice_transport_controller_) {
return ice_transport_controller_->SendVideo(video_frame);
}
int ret = video_encoder_->Encode(
video_frame,
[this](char *encoded_frame, size_t size,
VideoEncoder::VideoFrameType frame_type) -> int {
if (video_channel_send_) {
video_channel_send_->SendVideo(encoded_frame, size);
}
return 0;
});
if (0 != ret) {
LOG_ERROR("Encode failed");
return -1;
}
return 0;
return -1;
}
int IceTransport::SendAudioFrame(const char *data, size_t size) {
@@ -1011,17 +778,11 @@ int IceTransport::SendAudioFrame(const char *data, size_t size) {
return -2;
}
int ret = audio_encoder_->Encode(
(uint8_t *)data, size,
[this](char *encoded_audio_buffer, size_t size) -> int {
if (audio_channel_send_) {
audio_channel_send_->SendAudio(encoded_audio_buffer, size);
}
if (ice_transport_controller_) {
return ice_transport_controller_->SendAudio(data, size);
}
return 0;
});
return ret;
return -1;
}
int IceTransport::SendDataFrame(const char *data, size_t size) {
@@ -1032,11 +793,11 @@ int IceTransport::SendDataFrame(const char *data, size_t size) {
return -2;
}
if (data_channel_send_) {
data_channel_send_->SendData(data, size);
if (ice_transport_controller_) {
return ice_transport_controller_->SendData(data, size);
}
return 0;
return -1;
}
uint8_t IceTransport::CheckIsRtpPacket(const char *buffer, size_t size) {

96
src/transport/ice_transport.h
View File

@@ -9,32 +9,15 @@
#include <iostream>
#include "audio_decoder.h"
#include "audio_encoder.h"
// #include "congestion_control.h"
#include "audio_channel_receive.h"
#include "audio_channel_send.h"
#include "clock.h"
#include "data_channel_receive.h"
#include "data_channel_send.h"
#include "ice_agent.h"
#include "ice_transport_controller.h"
#include "io_statistics.h"
#include "ringbuffer.h"
#include "rtcp_packet_info.h"
#include "rtp_audio_receiver.h"
#include "rtp_audio_sender.h"
#include "rtp_data_receiver.h"
#include "rtp_data_sender.h"
#include "rtp_packet.h"
#include "rtp_video_receiver.h"
#include "rtp_video_sender.h"
#include "video_channel_receive.h"
#include "video_channel_send.h"
#include "video_decoder_factory.h"
#include "video_encoder_factory.h"
#include "ws_client.h"
class IceTransport : public std::enable_shared_from_this<IceTransport> {
class IceTransport {
public:
typedef enum { VIDEO = 96, AUDIO = 97, DATA = 127 } DATA_TYPE;
typedef enum { H264 = 96, AV1 = 99 } VIDEO_TYPE;
@@ -70,24 +53,11 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
int DestroyIceTransmission();
void SetOnReceiveVideoFunc(
std::function<void(const XVideoFrame *, const char *, const size_t,
void *)>
on_receive_video) {
void SetOnReceiveFunc(OnReceiveVideo on_receive_video,
OnReceiveAudio on_receive_audio,
OnReceiveData on_receive_data) {
on_receive_video_ = on_receive_video;
}
void SetOnReceiveAudioFunc(
std::function<void(const char *, size_t, const char *, const size_t,
void *)>
on_receive_audio) {
on_receive_audio_ = on_receive_audio;
}
void SetOnReceiveDataFunc(
std::function<void(const char *, size_t, const char *, const size_t,
void *)>
on_receive_data) {
on_receive_data_ = on_receive_data;
}
@@ -131,9 +101,6 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
bool NegotiateAudioPayloadType(const std::string &remote_sdp);
bool NegotiateDataPayloadType(const std::string &remote_sdp);
int CreateVideoCodec(rtp::PAYLOAD_TYPE video_pt, bool hardware_acceleration);
int CreateAudioCodec();
private:
uint8_t CheckIsRtpPacket(const char *buffer, size_t size);
uint8_t CheckIsRtcpPacket(const char *buffer, size_t size);
@@ -142,10 +109,6 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
uint8_t CheckIsDataPacket(const char *buffer, size_t size);
private:
void InitializeIOStatistics();
void InitializeChannels(rtp::PAYLOAD_TYPE video_codec_payload_type);
void OnIceStateChange(NiceAgent *agent, guint stream_id, guint component_id,
NiceComponentState state, gpointer user_ptr);
@@ -162,12 +125,6 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
void OnReceiveBuffer(NiceAgent *agent, guint stream_id, guint component_id,
guint size, gchar *buffer, gpointer user_ptr);
void OnReceiveCompleteFrame(VideoFrame &video_frame);
void OnReceiveCompleteAudio(const char *data, size_t size);
void OnReceiveCompleteData(const char *data, size_t size);
bool ParseRtcpPacket(const uint8_t *buffer, size_t size,
RtcpPacketInfo *rtcp_packet_info);
@@ -176,6 +133,7 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
RtcpPacketInfo *rtcp_packet_info);
private:
bool hardware_acceleration_ = false;
bool use_trickle_ice_ = true;
bool enable_turn_ = false;
bool use_reliable_ice_ = false;
@@ -203,13 +161,10 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
std::shared_ptr<IceAgent> ice_agent_ = nullptr;
bool is_closed_ = false;
std::shared_ptr<WsClient> ice_ws_transport_ = nullptr;
// CongestionControl *congestion_control_ = nullptr;
std::function<void(const XVideoFrame *, const char *, const size_t, void *)>
on_receive_video_ = nullptr;
std::function<void(const char *, size_t, const char *, const size_t, void *)>
on_receive_audio_ = nullptr;
std::function<void(const char *, size_t, const char *, const size_t, void *)>
on_receive_data_ = nullptr;
OnReceiveVideo on_receive_video_ = nullptr;
OnReceiveAudio on_receive_audio_ = nullptr;
OnReceiveData on_receive_data_ = nullptr;
std::function<void(std::string, const std::string &)> on_ice_status_change_ =
nullptr;
@@ -220,23 +175,7 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
on_receive_net_status_report_ = nullptr;
private:
std::shared_ptr<webrtc::Clock> clock_;
std::unique_ptr<VideoChannelSend> video_channel_send_ = nullptr;
std::unique_ptr<VideoChannelReceive> video_channel_receive_ = nullptr;
std::unique_ptr<AudioChannelSend> audio_channel_send_ = nullptr;
std::unique_ptr<AudioChannelReceive> audio_channel_receive_ = nullptr;
std::unique_ptr<DataChannelSend> data_channel_send_ = nullptr;
std::unique_ptr<DataChannelReceive> data_channel_receive_ = nullptr;
std::unique_ptr<RtpVideoReceiver> rtp_video_receiver_ = nullptr;
std::unique_ptr<RtpVideoSender> rtp_video_sender_ = nullptr;
std::unique_ptr<RtpAudioReceiver> rtp_audio_receiver_ = nullptr;
std::unique_ptr<RtpAudioSender> rtp_audio_sender_ = nullptr;
std::unique_ptr<RtpDataReceiver> rtp_data_receiver_ = nullptr;
std::unique_ptr<RtpDataSender> rtp_data_sender_ = nullptr;
bool start_send_packet_ = false;
uint32_t last_complete_frame_ts_ = 0;
std::shared_ptr<IceTransportController> ice_transport_controller_ = nullptr;
private:
std::shared_ptr<IOStatistics> ice_io_statistics_ = nullptr;
@@ -250,19 +189,6 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
rtp::PAYLOAD_TYPE negotiated_video_pt_ = rtp::PAYLOAD_TYPE::UNDEFINED;
rtp::PAYLOAD_TYPE negotiated_audio_pt_ = rtp::PAYLOAD_TYPE::UNDEFINED;
rtp::PAYLOAD_TYPE negotiated_data_pt_ = rtp::PAYLOAD_TYPE::UNDEFINED;
private:
std::unique_ptr<VideoEncoder> video_encoder_ = nullptr;
std::unique_ptr<VideoDecoder> video_decoder_ = nullptr;
bool b_force_i_frame_ = false;
bool video_codec_inited_ = false;
bool load_nvcodec_dll_success_ = false;
bool hardware_acceleration_ = false;
private:
std::unique_ptr<AudioEncoder> audio_encoder_ = nullptr;
std::unique_ptr<AudioDecoder> audio_decoder_ = nullptr;
bool audio_codec_inited_ = false;
};
#endif

389
src/transport/ice_transport_controller.cpp Normal file
View File

@@ -0,0 +1,389 @@
#include "ice_transport_controller.h"
#if __APPLE__
#else
#include "nvcodec_api.h"
#endif
IceTransportController::IceTransportController()
: b_force_i_frame_(true),
video_codec_inited_(false),
audio_codec_inited_(false),
load_nvcodec_dll_success_(false),
hardware_acceleration_(false),
clock_(webrtc::Clock::GetRealTimeClockShared()) {}
IceTransportController::~IceTransportController() {
user_data_ = nullptr;
video_codec_inited_ = false;
audio_codec_inited_ = false;
load_nvcodec_dll_success_ = false;
#ifdef __APPLE__
#else
if (hardware_acceleration_ && load_nvcodec_dll_success_) {
ReleaseNvCodecDll();
}
#endif
}
void IceTransportController::Create(
std::string remote_user_id, rtp::PAYLOAD_TYPE video_codec_payload_type,
bool hardware_acceleration, std::shared_ptr<IceAgent> ice_agent,
std::shared_ptr<IOStatistics> ice_io_statistics,
OnReceiveVideo on_receive_video, OnReceiveAudio on_receive_audio,
OnReceiveData on_receive_data, void* user_data) {
remote_user_id_ = remote_user_id;
on_receive_video_ = on_receive_video;
on_receive_audio_ = on_receive_audio;
on_receive_data_ = on_receive_data;
user_data_ = user_data;
CreateVideoCodec(video_codec_payload_type, hardware_acceleration);
CreateAudioCodec();
controller_ = std::make_unique<CongestionControl>();
video_channel_send_ = std::make_unique<VideoChannelSend>(
clock_, ice_agent, ice_io_statistics,
[this](const webrtc::RtpPacketToSend& packet) {
OnSentRtpPacket(packet);
});
audio_channel_send_ =
std::make_unique<AudioChannelSend>(ice_agent, ice_io_statistics);
data_channel_send_ =
std::make_unique<DataChannelSend>(ice_agent, ice_io_statistics);
video_channel_send_->Initialize(video_codec_payload_type);
audio_channel_send_->Initialize(rtp::PAYLOAD_TYPE::OPUS);
data_channel_send_->Initialize(rtp::PAYLOAD_TYPE::DATA);
std::weak_ptr<IceTransportController> weak_self = shared_from_this();
video_channel_receive_ = std::make_unique<VideoChannelReceive>(
clock_, ice_agent, ice_io_statistics,
[this, weak_self](VideoFrame& video_frame) {
if (auto self = weak_self.lock()) {
OnReceiveCompleteFrame(video_frame);
}
});
audio_channel_receive_ = std::make_unique<AudioChannelReceive>(
ice_agent, ice_io_statistics,
[this, weak_self](const char* data, size_t size) {
if (auto self = weak_self.lock()) {
OnReceiveCompleteAudio(data, size);
}
});
data_channel_receive_ = std::make_unique<DataChannelReceive>(
ice_agent, ice_io_statistics,
[this, weak_self](const char* data, size_t size) {
if (auto self = weak_self.lock()) {
OnReceiveCompleteData(data, size);
}
});
video_channel_receive_->Initialize(video_codec_payload_type);
audio_channel_receive_->Initialize(rtp::PAYLOAD_TYPE::OPUS);
data_channel_receive_->Initialize(rtp::PAYLOAD_TYPE::DATA);
}
void IceTransportController::Destroy() {
if (video_channel_send_) {
video_channel_send_->Destroy();
}
if (audio_channel_send_) {
audio_channel_send_->Destroy();
}
if (data_channel_send_) {
data_channel_send_->Destroy();
}
if (video_channel_receive_) {
video_channel_receive_->Destroy();
}
if (audio_channel_receive_) {
audio_channel_receive_->Destroy();
}
if (data_channel_receive_) {
data_channel_receive_->Destroy();
}
}
int IceTransportController::SendVideo(const XVideoFrame* video_frame) {
if (!video_encoder_) {
LOG_ERROR("Video Encoder not created");
return -1;
}
if (b_force_i_frame_) {
video_encoder_->ForceIdr();
LOG_INFO("Force I frame");
b_force_i_frame_ = false;
}
int ret = video_encoder_->Encode(
video_frame,
[this](char* encoded_frame, size_t size,
VideoEncoder::VideoFrameType frame_type) -> int {
if (video_channel_send_) {
video_channel_send_->SendVideo(encoded_frame, size);
}
return 0;
});
if (0 != ret) {
LOG_ERROR("Encode failed");
return -1;
} else {
return 0;
}
}
int IceTransportController::SendAudio(const char* data, size_t size) {
if (!audio_encoder_) {
LOG_ERROR("Audio Encoder not created");
return -1;
}
int ret = audio_encoder_->Encode(
(uint8_t*)data, size,
[this](char* encoded_audio_buffer, size_t size) -> int {
if (audio_channel_send_) {
audio_channel_send_->SendAudio(encoded_audio_buffer, size);
}
return 0;
});
return ret;
}
int IceTransportController::SendData(const char* data, size_t size) {
if (data_channel_send_) {
data_channel_send_->SendData(data, size);
}
return 0;
}
int IceTransportController::OnReceiveVideoRtpPacket(const char* data,
size_t size) {
if (video_channel_receive_) {
return video_channel_receive_->OnReceiveRtpPacket(data, size);
}
return -1;
}
int IceTransportController::OnReceiveAudioRtpPacket(const char* data,
size_t size) {
if (audio_channel_receive_) {
return audio_channel_receive_->OnReceiveRtpPacket(data, size);
}
return -1;
}
int IceTransportController::OnReceiveDataRtpPacket(const char* data,
size_t size) {
if (data_channel_receive_) {
return data_channel_receive_->OnReceiveRtpPacket(data, size);
}
return -1;
}
void IceTransportController::OnReceiveCompleteFrame(VideoFrame& video_frame) {
int num_frame_returned = video_decoder_->Decode(
(uint8_t*)video_frame.Buffer(), video_frame.Size(),
[this](VideoFrame video_frame) {
if (on_receive_video_) {
XVideoFrame x_video_frame;
x_video_frame.data = (const char*)video_frame.Buffer();
x_video_frame.width = video_frame.Width();
x_video_frame.height = video_frame.Height();
x_video_frame.size = video_frame.Size();
on_receive_video_(&x_video_frame, remote_user_id_.data(),
remote_user_id_.size(), user_data_);
}
});
}
void IceTransportController::OnReceiveCompleteAudio(const char* data,
size_t size) {
int num_frame_returned = audio_decoder_->Decode(
(uint8_t*)data, size, [this](uint8_t* data, int size) {
if (on_receive_audio_) {
on_receive_audio_((const char*)data, size, remote_user_id_.data(),
remote_user_id_.size(), user_data_);
}
});
}
void IceTransportController::OnReceiveCompleteData(const char* data,
size_t size) {
if (on_receive_data_) {
on_receive_data_(data, size, remote_user_id_.data(), remote_user_id_.size(),
user_data_);
}
}
int IceTransportController::CreateVideoCodec(rtp::PAYLOAD_TYPE video_pt,
bool hardware_acceleration) {
if (video_codec_inited_) {
return 0;
}
hardware_acceleration_ = hardware_acceleration;
if (rtp::PAYLOAD_TYPE::AV1 == video_pt) {
if (hardware_acceleration_) {
hardware_acceleration_ = false;
LOG_WARN("Only support software codec for AV1");
}
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, true);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, true);
} else if (rtp::PAYLOAD_TYPE::H264 == video_pt) {
#ifdef __APPLE__
if (hardware_acceleration_) {
hardware_acceleration_ = false;
LOG_WARN(
"MacOS not support hardware acceleration, use default software "
"codec");
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
} else {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
#else
if (hardware_acceleration_) {
if (0 == LoadNvCodecDll()) {
load_nvcodec_dll_success_ = true;
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(true, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(true, false);
} else {
LOG_WARN(
"Hardware accelerated codec not available, use default software "
"codec");
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
} else {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
#endif
}
if (!video_encoder_) {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
LOG_ERROR("Create encoder failed, try to use software H.264 encoder");
}
if (!video_encoder_ || 0 != video_encoder_->Init()) {
LOG_ERROR("Encoder init failed");
return -1;
}
if (!video_decoder_) {
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
LOG_ERROR("Create decoder failed, try to use software H.264 decoder");
}
if (!video_decoder_ || video_decoder_->Init()) {
LOG_ERROR("Decoder init failed");
return -1;
}
video_codec_inited_ = true;
LOG_INFO("Create video codec [{}|{}] finish",
video_encoder_->GetEncoderName(), video_decoder_->GetDecoderName());
return 0;
}
int IceTransportController::CreateAudioCodec() {
if (audio_codec_inited_) {
return 0;
}
audio_encoder_ = std::make_unique<AudioEncoder>(AudioEncoder(48000, 1, 480));
if (!audio_encoder_ || 0 != audio_encoder_->Init()) {
LOG_ERROR("Audio encoder init failed");
return -1;
}
audio_decoder_ = std::make_unique<AudioDecoder>(AudioDecoder(48000, 1, 480));
if (!audio_decoder_ || 0 != audio_decoder_->Init()) {
LOG_ERROR("Audio decoder init failed");
return -1;
}
audio_codec_inited_ = true;
LOG_INFO("Create audio codec [{}|{}] finish",
audio_encoder_->GetEncoderName(), audio_decoder_->GetDecoderName());
return 0;
}
void IceTransportController::OnCongestionControlFeedback(
const webrtc::rtcp::CongestionControlFeedback& feedback) {
std::optional<webrtc::TransportPacketsFeedback> feedback_msg =
transport_feedback_adapter_.ProcessCongestionControlFeedback(
feedback, clock_->CurrentTime());
if (feedback_msg) {
HandleTransportPacketsFeedback(*feedback_msg);
}
}
void IceTransportController::HandleTransportPacketsFeedback(
const webrtc::TransportPacketsFeedback& feedback) {
if (controller_)
PostUpdates(controller_->OnTransportPacketsFeedback(feedback));
UpdateCongestedState();
}
void IceTransportController::OnSentRtpPacket(
const webrtc::RtpPacketToSend& packet) {
webrtc::PacedPacketInfo pacing_info;
size_t transport_overhead_bytes_per_packet_ = 0;
webrtc::Timestamp creation_time =
webrtc::Timestamp::Millis(clock_->TimeInMilliseconds());
transport_feedback_adapter_.AddPacket(
packet, pacing_info, transport_overhead_bytes_per_packet_, creation_time);
rtc::SentPacket sent_packet;
sent_packet.packet_id = packet.transport_sequence_number().value();
sent_packet.send_time_ms = clock_->TimeInMilliseconds();
sent_packet.info.included_in_feedback = true;
sent_packet.info.included_in_allocation = true;
sent_packet.info.packet_size_bytes = packet.size();
sent_packet.info.packet_type = rtc::PacketType::kData;
transport_feedback_adapter_.ProcessSentPacket(sent_packet);
}
void IceTransportController::PostUpdates(webrtc::NetworkControlUpdate update) {
// UpdateControlState();
target_bitrate_ = update.target_rate.has_value()
? update.target_rate->target_rate.bps()
: 0;
// LOG_WARN("Target bitrate [{}]bps", target_bitrate_);
video_encoder_->SetTargetBitrate(target_bitrate_);
}
void IceTransportController::UpdateControlState() {
if (controller_) {
}
}
void IceTransportController::UpdateCongestedState() {
if (controller_) {
}
}

121
src/transport/ice_transport_controller.h Normal file
View File

@@ -0,0 +1,121 @@
/*
* @Author: DI JUNKUN
* @Date: 2025-02-11
* Copyright (c) 2025 by DI JUNKUN, All Rights Reserved.
*/
#ifndef _ICE_TRANSPORT_CONTROLLER_H_
#define _ICE_TRANSPORT_CONTROLLER_H_
#include "api/transport/network_types.h"
#include "api/units/timestamp.h"
#include "audio_channel_receive.h"
#include "audio_channel_send.h"
#include "audio_decoder.h"
#include "audio_encoder.h"
#include "clock.h"
#include "congestion_control.h"
#include "congestion_control_feedback.h"
#include "data_channel_receive.h"
#include "data_channel_send.h"
#include "ice_agent.h"
#include "transport_feedback_adapter.h"
#include "video_channel_receive.h"
#include "video_channel_send.h"
#include "video_decoder_factory.h"
#include "video_encoder_factory.h"
typedef void (*OnReceiveVideo)(const XVideoFrame *, const char *, const size_t,
void *);
typedef void (*OnReceiveAudio)(const char *, size_t, const char *, const size_t,
void *);
typedef void (*OnReceiveData)(const char *, size_t, const char *, const size_t,
void *);
class IceTransportController
: public std::enable_shared_from_this<IceTransportController> {
public:
IceTransportController();
~IceTransportController();
public:
void Create(std::string remote_user_id,
rtp::PAYLOAD_TYPE video_codec_payload_type,
bool hardware_acceleration, std::shared_ptr<IceAgent> ice_agent,
std::shared_ptr<IOStatistics> ice_io_statistics,
OnReceiveVideo on_receive_video, OnReceiveAudio on_receive_audio,
OnReceiveData on_receive_data, void *user_data);
void Destroy();
int SendVideo(const XVideoFrame *video_frame);
int SendAudio(const char *data, size_t size);
int SendData(const char *data, size_t size);
int OnReceiveVideoRtpPacket(const char *data, size_t size);
int OnReceiveAudioRtpPacket(const char *data, size_t size);
int OnReceiveDataRtpPacket(const char *data, size_t size);
void OnReceiveCompleteFrame(VideoFrame &video_frame);
void OnReceiveCompleteAudio(const char *data, size_t size);
void OnReceiveCompleteData(const char *data, size_t size);
public:
void OnCongestionControlFeedback(
const webrtc::rtcp::CongestionControlFeedback &feedback);
private:
int CreateVideoCodec(rtp::PAYLOAD_TYPE video_pt, bool hardware_acceleration);
int CreateAudioCodec();
private:
void OnSentRtpPacket(const webrtc::RtpPacketToSend &packet);
void HandleTransportPacketsFeedback(
const webrtc::TransportPacketsFeedback &feedback);
void PostUpdates(webrtc::NetworkControlUpdate update);
void UpdateControlState();
void UpdateCongestedState();
private:
std::unique_ptr<VideoChannelSend> video_channel_send_ = nullptr;
std::unique_ptr<AudioChannelSend> audio_channel_send_ = nullptr;
std::unique_ptr<DataChannelSend> data_channel_send_ = nullptr;
std::unique_ptr<VideoChannelReceive> video_channel_receive_ = nullptr;
std::unique_ptr<AudioChannelReceive> audio_channel_receive_ = nullptr;
std::unique_ptr<DataChannelReceive> data_channel_receive_ = nullptr;
OnReceiveVideo on_receive_video_ = nullptr;
OnReceiveAudio on_receive_audio_ = nullptr;
OnReceiveData on_receive_data_ = nullptr;
private:
std::shared_ptr<IceAgent> ice_agent_ = nullptr;
std::shared_ptr<IOStatistics> ice_io_statistics_ = nullptr;
std::unique_ptr<RtpPacketizer> rtp_packetizer_ = nullptr;
std::unique_ptr<RtpVideoSender> rtp_video_sender_ = nullptr;
std::string remote_user_id_;
void *user_data_ = nullptr;
private:
std::shared_ptr<Clock> clock_;
webrtc::TransportFeedbackAdapter transport_feedback_adapter_;
std::unique_ptr<CongestionControl> controller_;
private:
std::unique_ptr<VideoEncoder> video_encoder_ = nullptr;
std::unique_ptr<VideoDecoder> video_decoder_ = nullptr;
bool b_force_i_frame_;
bool video_codec_inited_;
bool load_nvcodec_dll_success_;
bool hardware_acceleration_;
private:
std::unique_ptr<AudioEncoder> audio_encoder_ = nullptr;
std::unique_ptr<AudioDecoder> audio_decoder_ = nullptr;
bool audio_codec_inited_ = false;
private:
uint64_t target_bitrate_ = 0;
};
#endif