YangWX/crossdesk
1
0
Fork 0
mirror of https://github.com/kunkundi/crossdesk.git synced 2025-10-26 20:25:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feat] enable congestion controller set target bitrate to video encoder

Browse Source
This commit is contained in:
dijunkun
2025-02-11 17:25:50 +08:00
parent 7a8be01e7b
commit 36704c7e4c
19 changed files with 770 additions and 525 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
src/channel/video_channel_send.cpp
View File

@@ -9,14 +9,15 @@ VideoChannelSend::~VideoChannelSend() {}
VideoChannelSend::VideoChannelSend(
std::shared_ptr<webrtc::Clock> clock, std::shared_ptr<IceAgent> ice_agent,
std::shared_ptr<IOStatistics> ice_io_statistics)
std::shared_ptr<IOStatistics> ice_io_statistics,
std::function<void(const webrtc::RtpPacketToSend& packet)>
on_sent_packet_func)
: ice_agent_(ice_agent),
ice_io_statistics_(ice_io_statistics),
clock_(clock) {};
on_sent_packet_func_(on_sent_packet_func),
clock_(clock){};
void VideoChannelSend::Initialize(rtp::PAYLOAD_TYPE payload_type) {
controller_ = std::make_unique<CongestionControl>();
rtp_video_sender_ = std::make_unique<RtpVideoSender>(ice_io_statistics_);
rtp_packetizer_ =
RtpPacketizer::Create(payload_type, rtp_video_sender_->GetSsrc());
@@ -43,23 +44,7 @@ void VideoChannelSend::Initialize(rtp::PAYLOAD_TYPE payload_type) {
rtp_video_sender_->SetOnSentPacketFunc(
[this](const webrtc::RtpPacketToSend& packet) -> void {
webrtc::PacedPacketInfo pacing_info;
size_t transport_overhead_bytes_per_packet_ = 0;
webrtc::Timestamp creation_time =
webrtc::Timestamp::Millis(clock_->TimeInMilliseconds());
transport_feedback_adapter_.AddPacket(
packet, pacing_info, transport_overhead_bytes_per_packet_,
creation_time);
rtc::SentPacket sent_packet;
sent_packet.packet_id = packet.transport_sequence_number().value();
sent_packet.send_time_ms = clock_->TimeInMilliseconds();
sent_packet.info.included_in_feedback = true;
sent_packet.info.included_in_allocation = true;
sent_packet.info.packet_size_bytes = packet.size();
sent_packet.info.packet_type = rtc::PacketType::kData;
transport_feedback_adapter_.ProcessSentPacket(sent_packet);
on_sent_packet_func_(packet);
});
rtp_video_sender_->Start();
@@ -80,66 +65,3 @@ int VideoChannelSend::SendVideo(char* data, size_t size) {
return 0;
}
void VideoChannelSend::OnCongestionControlFeedback(
Timestamp recv_ts,
const webrtc::rtcp::CongestionControlFeedback& feedback) {
++feedback_count_;
std::optional<webrtc::TransportPacketsFeedback> feedback_msg =
transport_feedback_adapter_.ProcessCongestionControlFeedback(feedback,
recv_ts);
if (feedback_msg) {
HandleTransportPacketsFeedback(*feedback_msg);
}
}
void VideoChannelSend::HandleTransportPacketsFeedback(
const webrtc::TransportPacketsFeedback& feedback) {
// if (transport_is_ecn_capable_) {
// // If transport does not support ECN, packets should not be sent as
// ECT(1).
// // TODO: bugs.webrtc.org/42225697 - adapt to ECN feedback and
// continue to
// // send packets as ECT(1) if transport is ECN capable.
// transport_is_ecn_capable_ = false;
// LOG_INFO("Transport is {} ECN capable. Stop sending ECT(1)",
// (feedback.transport_supports_ecn ? "" : " not "));
// }
if (controller_)
PostUpdates(controller_->OnTransportPacketsFeedback(feedback));
// Only update outstanding data if any packet is first time acked.
UpdateCongestedState();
}
void VideoChannelSend::PostUpdates(webrtc::NetworkControlUpdate update) {
// if (update.congestion_window) {
// congestion_window_size_ = *update.congestion_window;
// UpdateCongestedState();
// }
// if (update.pacer_config) {
// pacer_.SetPacingRates(update.pacer_config->data_rate(),
// update.pacer_config->pad_rate());
// }
// if (!update.probe_cluster_configs.empty()) {
// pacer_.CreateProbeClusters(std::move(update.probe_cluster_configs));
// }
// if (update.target_rate) {
// control_handler_->SetTargetRate(*update.target_rate);
// UpdateControlState();
// }
}
void VideoChannelSend::UpdateControlState() {
// std::optional<TargetTransferRate> update =
// control_handler_->GetUpdate(); if (!update) return;
// retransmission_rate_limiter_.SetMaxRate(update->target_rate.bps());
// observer_->OnTargetTransferRate(*update);
}
void VideoChannelSend::UpdateCongestedState() {
// if (auto update = GetCongestedStateUpdate()) {
// is_congested_ = update.value();
// pacer_.SetCongested(update.value());
// }
}

14
src/channel/video_channel_send.h
View File

@@ -23,7 +23,9 @@ class VideoChannelSend {
VideoChannelSend();
VideoChannelSend(std::shared_ptr<webrtc::Clock> clock,
std::shared_ptr<IceAgent> ice_agent,
std::shared_ptr<IOStatistics> ice_io_statistics);
std::shared_ptr<IOStatistics> ice_io_statistics,
std::function<void(const webrtc::RtpPacketToSend& packet)>
on_sent_packet_func_);
~VideoChannelSend();
public:
@@ -50,15 +52,11 @@ class VideoChannelSend {
std::unique_ptr<RtpPacketizer> rtp_packetizer_ = nullptr;
std::unique_ptr<RtpVideoSender> rtp_video_sender_ = nullptr;
std::function<void(const webrtc::RtpPacketToSend& packet)>
on_sent_packet_func_ = nullptr;
private:
std::shared_ptr<Clock> clock_;
int64_t current_offset_ = std::numeric_limits<int64_t>::min();
// Used by RFC 8888 congestion control feedback to track base time.
std::optional<uint32_t> last_feedback_compact_ntp_time_;
int feedback_count_ = 0;
webrtc::TransportFeedbackAdapter transport_feedback_adapter_;
std::unique_ptr<CongestionControl> controller_;
};
#endif

5
src/media/video/encode/aom/aom_av1_encoder.cpp
View File

@@ -359,6 +359,11 @@ int AomAv1Encoder::ForceIdr() {
return 0;
}
int AomAv1Encoder::SetTargetBitrate(int bitrate) {
target_bitrate_ = bitrate;
return 0;
}
int AomAv1Encoder::Release() {
if (frame_for_encode_ != nullptr) {
aom_img_free(frame_for_encode_);

2
src/media/video/encode/aom/aom_av1_encoder.h
View File

@@ -44,6 +44,8 @@ class AomAv1Encoder : public VideoEncoder {
int ForceIdr();
int SetTargetBitrate(int bitrate);
std::string GetEncoderName() { return "AomAV1"; }
private:

19
src/media/video/encode/nvcodec/nvidia_video_encoder.cpp
View File

@@ -221,6 +221,25 @@ int NvidiaVideoEncoder::ForceIdr() {
return 0;
}
int NvidiaVideoEncoder::SetTargetBitrate(int bitrate) {
if (!encoder_) {
return -1;
}
NV_ENC_RECONFIGURE_PARAMS reconfig_params;
reconfig_params.version = NV_ENC_RECONFIGURE_PARAMS_VER;
NV_ENC_INITIALIZE_PARAMS init_params;
NV_ENC_CONFIG encode_config = {NV_ENC_CONFIG_VER};
init_params.encodeConfig = &encode_config;
encoder_->GetInitializeParams(&init_params);
init_params.frameRateDen = 1;
init_params.frameRateNum = init_params.frameRateDen * fps_;
init_params.encodeConfig->rcParams.averageBitRate = average_bitrate_;
init_params.encodeConfig->rcParams.maxBitRate = bitrate;
reconfig_params.reInitEncodeParams = init_params;
return encoder_->Reconfigure(&reconfig_params) ? 0 : -1;
}
int NvidiaVideoEncoder::ResetEncodeResolution(unsigned int width,
unsigned int height) {
if (!encoder_) {

3
src/media/video/encode/nvcodec/nvidia_video_encoder.h
View File

@@ -20,6 +20,8 @@ class NvidiaVideoEncoder : public VideoEncoder {
int ForceIdr();
int SetTargetBitrate(int bitrate);
std::string GetEncoderName() { return "NvidiaH264"; }
private:
@@ -50,6 +52,7 @@ class NvidiaVideoEncoder : public VideoEncoder {
uint32_t key_frame_interval_ = 3000;
uint32_t average_bitrate_ = 2000000;
uint32_t max_bitrate_ = 10000000;
uint32_t fps_ = 30;
int max_payload_size_ = 3000;
NvEncoder* encoder_ = nullptr;
CUcontext cuda_context_ = nullptr;

7
src/media/video/encode/openh264/openh264_encoder.cpp
View File

@@ -359,6 +359,13 @@ int OpenH264Encoder::ForceIdr() {
return 0;
}
int OpenH264Encoder::SetTargetBitrate(int bitrate) {
target_bitrate_ = bitrate;
encoder_params_.iTargetBitrate = target_bitrate_;
return openh264_encoder_->SetOption(ENCODER_OPTION_BITRATE, &target_bitrate_);
}
int OpenH264Encoder::Release() {
if (openh264_encoder_) {
openh264_encoder_->Uninitialize();

2
src/media/video/encode/openh264/openh264_encoder.h
View File

@@ -31,6 +31,8 @@ class OpenH264Encoder : public VideoEncoder {
int ForceIdr();
int SetTargetBitrate(int bitrate);
std::string GetEncoderName() { return "OpenH264"; }
private:

2
src/media/video/encode/video_encoder.h
View File

@@ -27,6 +27,8 @@ class VideoEncoder {
virtual int ForceIdr() = 0;
virtual int SetTargetBitrate(int bitrate) = 0;
virtual std::string GetEncoderName() = 0;
VideoEncoder() = default;

14
src/pc/peer_connection.cpp
View File

@@ -602,11 +602,8 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) {
hardware_acceleration_, trickle_ice_, reliable_ice_, enable_turn_,
false, video_payload_types_, audio_payload_types_);
ice_transport_list_[remote_user_id]->SetOnReceiveVideoFunc(
on_receive_video_frame_);
ice_transport_list_[remote_user_id]->SetOnReceiveAudioFunc(
on_receive_audio_buffer_);
ice_transport_list_[remote_user_id]->SetOnReceiveDataFunc(
ice_transport_list_[remote_user_id]->SetOnReceiveFunc(
on_receive_video_frame_, on_receive_audio_buffer_,
on_receive_data_buffer_);
ice_transport_list_[remote_user_id]->SetOnReceiveNetStatusReportFunc(
@@ -649,11 +646,8 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) {
hardware_acceleration_, trickle_ice_, reliable_ice_, enable_turn_,
false, video_payload_types_, audio_payload_types_);
ice_transport_list_[remote_user_id]->SetOnReceiveVideoFunc(
on_receive_video_frame_);
ice_transport_list_[remote_user_id]->SetOnReceiveAudioFunc(
on_receive_audio_buffer_);
ice_transport_list_[remote_user_id]->SetOnReceiveDataFunc(
ice_transport_list_[remote_user_id]->SetOnReceiveFunc(
on_receive_video_frame_, on_receive_audio_buffer_,
on_receive_data_buffer_);
ice_transport_list_[remote_user_id]->SetOnReceiveNetStatusReportFunc(

164
src/qos/congestion_control.cpp
View File

@@ -21,6 +21,32 @@ constexpr float kDefaultPaceMultiplier = 2.5f;
// below the current throughput estimate to drain the network queues.
constexpr double kProbeDropThroughputFraction = 0.85;
BandwidthLimitedCause GetBandwidthLimitedCause(LossBasedState loss_based_state,
bool is_rtt_above_limit,
BandwidthUsage bandwidth_usage) {
if (bandwidth_usage == BandwidthUsage::kBwOverusing ||
bandwidth_usage == BandwidthUsage::kBwUnderusing) {
return BandwidthLimitedCause::kDelayBasedLimitedDelayIncreased;
} else if (is_rtt_above_limit) {
return BandwidthLimitedCause::kRttBasedBackOffHighRtt;
}
switch (loss_based_state) {
case LossBasedState::kDecreasing:
// Probes may not be sent in this state.
return BandwidthLimitedCause::kLossLimitedBwe;
case webrtc::LossBasedState::kIncreaseUsingPadding:
// Probes may not be sent in this state.
return BandwidthLimitedCause::kLossLimitedBwe;
case LossBasedState::kIncreasing:
// Probes may be sent in this state.
return BandwidthLimitedCause::kLossLimitedBweIncreasing;
case LossBasedState::kDelayBasedEstimate:
return BandwidthLimitedCause::kDelayBasedLimited;
default:
return BandwidthLimitedCause::kLossLimitedBwe;
}
}
CongestionControl::CongestionControl()
: packet_feedback_only_(true),
use_min_allocatable_as_lower_bound_(false),
@@ -179,13 +205,13 @@ NetworkControlUpdate CongestionControl::OnTransportPacketsFeedback(
bandwidth_estimation_->UpdateDelayBasedEstimate(report.feedback_time,
result.target_bitrate);
}
// bandwidth_estimation_->UpdateLossBasedEstimator(
// report, result.delay_detector_state, probe_bitrate,
// alr_start_time.has_value());
// if (result.updated) {
// // Update the estimate in the ProbeController, in case we want to probe.
// MaybeTriggerOnNetworkChanged(&update, report.feedback_time);
// }
bandwidth_estimation_->UpdateLossBasedEstimator(
report, result.delay_detector_state, probe_bitrate,
alr_start_time.has_value());
if (result.updated) {
// Update the estimate in the ProbeController, in case we want to probe.
MaybeTriggerOnNetworkChanged(&update, report.feedback_time);
}
recovered_from_overuse = result.recovered_from_overuse;
@@ -214,14 +240,13 @@ NetworkControlUpdate CongestionControl::OnTransportPacketsFeedback(
void CongestionControl::MaybeTriggerOnNetworkChanged(
NetworkControlUpdate* update, Timestamp at_time) {
// uint8_t fraction_loss = bandwidth_estimation_->fraction_loss();
// TimeDelta round_trip_time = bandwidth_estimation_->round_trip_time();
// DataRate loss_based_target_rate = bandwidth_estimation_->target_rate();
// LossBasedState loss_based_state =
// bandwidth_estimation_->loss_based_state(); DataRate pushback_target_rate =
// loss_based_target_rate;
uint8_t fraction_loss = bandwidth_estimation_->fraction_loss();
TimeDelta round_trip_time = bandwidth_estimation_->round_trip_time();
DataRate loss_based_target_rate = bandwidth_estimation_->target_rate();
LossBasedState loss_based_state = bandwidth_estimation_->loss_based_state();
DataRate pushback_target_rate = loss_based_target_rate;
// double cwnd_reduce_ratio = 0.0;
double cwnd_reduce_ratio = 0.0;
// if (congestion_window_pushback_controller_) {
// int64_t pushback_rate =
// congestion_window_pushback_controller_->UpdateTargetBitrate(
@@ -235,53 +260,88 @@ void CongestionControl::MaybeTriggerOnNetworkChanged(
// loss_based_target_rate.bps();
// }
// }
// DataRate stable_target_rate =
// bandwidth_estimation_->GetEstimatedLinkCapacity();
// stable_target_rate = std::min(stable_target_rate, pushback_target_rate);
DataRate stable_target_rate =
bandwidth_estimation_->GetEstimatedLinkCapacity();
stable_target_rate = std::min(stable_target_rate, pushback_target_rate);
// if ((loss_based_target_rate != last_loss_based_target_rate_) ||
// (loss_based_state != last_loss_base_state_) ||
// (fraction_loss != last_estimated_fraction_loss_) ||
// (round_trip_time != last_estimated_round_trip_time_) ||
// (pushback_target_rate != last_pushback_target_rate_) ||
// (stable_target_rate != last_stable_target_rate_)) {
// last_loss_based_target_rate_ = loss_based_target_rate;
// last_pushback_target_rate_ = pushback_target_rate;
// last_estimated_fraction_loss_ = fraction_loss;
// last_estimated_round_trip_time_ = round_trip_time;
// last_stable_target_rate_ = stable_target_rate;
// last_loss_base_state_ = loss_based_state;
if ((loss_based_target_rate != last_loss_based_target_rate_) ||
(loss_based_state != last_loss_base_state_) ||
(fraction_loss != last_estimated_fraction_loss_) ||
(round_trip_time != last_estimated_round_trip_time_) ||
(pushback_target_rate != last_pushback_target_rate_) ||
(stable_target_rate != last_stable_target_rate_)) {
last_loss_based_target_rate_ = loss_based_target_rate;
last_pushback_target_rate_ = pushback_target_rate;
last_estimated_fraction_loss_ = fraction_loss;
last_estimated_round_trip_time_ = round_trip_time;
last_stable_target_rate_ = stable_target_rate;
last_loss_base_state_ = loss_based_state;
// alr_detector_->SetEstimatedBitrate(loss_based_target_rate.bps());
alr_detector_->SetEstimatedBitrate(loss_based_target_rate.bps());
// TimeDelta bwe_period = delay_based_bwe_->GetExpectedBwePeriod();
TimeDelta bwe_period = delay_based_bwe_->GetExpectedBwePeriod();
// TargetTransferRate target_rate_msg;
// target_rate_msg.at_time = at_time;
TargetTransferRate target_rate_msg;
target_rate_msg.at_time = at_time;
// if (rate_control_settings_.UseCongestionWindowDropFrameOnly()) {
// target_rate_msg.target_rate = loss_based_target_rate;
// target_rate_msg.cwnd_reduce_ratio = cwnd_reduce_ratio;
// } else {
// target_rate_msg.target_rate = pushback_target_rate;
// }
// target_rate_msg.stable_target_rate = stable_target_rate;
// target_rate_msg.network_estimate.at_time = at_time;
// target_rate_msg.network_estimate.round_trip_time = round_trip_time;
// target_rate_msg.network_estimate.loss_rate_ratio = fraction_loss /
// 255.0f; target_rate_msg.network_estimate.bwe_period = bwe_period;
// } else
{ target_rate_msg.target_rate = pushback_target_rate; }
target_rate_msg.stable_target_rate = stable_target_rate;
target_rate_msg.network_estimate.at_time = at_time;
target_rate_msg.network_estimate.round_trip_time = round_trip_time;
target_rate_msg.network_estimate.loss_rate_ratio = fraction_loss / 255.0f;
target_rate_msg.network_estimate.bwe_period = bwe_period;
// update->target_rate = target_rate_msg;
update->target_rate = target_rate_msg;
// auto probes = probe_controller_->SetEstimatedBitrate(
// loss_based_target_rate,
// GetBandwidthLimitedCause(bandwidth_estimation_->loss_based_state(),
// bandwidth_estimation_->IsRttAboveLimit(),
// delay_based_bwe_->last_state()),
// at_time);
// update->probe_cluster_configs.insert(update->probe_cluster_configs.end(),
// probes.begin(), probes.end());
// update->pacer_config = GetPacingRates(at_time);
auto probes = probe_controller_->SetEstimatedBitrate(
loss_based_target_rate,
GetBandwidthLimitedCause(bandwidth_estimation_->loss_based_state(),
bandwidth_estimation_->IsRttAboveLimit(),
delay_based_bwe_->last_state()),
at_time);
update->probe_cluster_configs.insert(update->probe_cluster_configs.end(),
probes.begin(), probes.end());
update->pacer_config = GetPacingRates(at_time);
// LOG_INFO("bwe {} pushback_target_bps={} estimate_bps={}", at_time.ms(),
// last_pushback_target_rate_.bps(), loss_based_target_rate.bps());
// }
}
}
PacerConfig CongestionControl::GetPacingRates(Timestamp at_time) const {
// Pacing rate is based on target rate before congestion window pushback,
// because we don't want to build queues in the pacer when pushback occurs.
DataRate pacing_rate = DataRate::Zero();
if (pace_at_max_of_bwe_and_lower_link_capacity_ && estimate_ &&
!bandwidth_estimation_->PaceAtLossBasedEstimate()) {
pacing_rate =
std::max({min_total_allocated_bitrate_, estimate_->link_capacity_lower,
last_loss_based_target_rate_}) *
pacing_factor_;
} else {
pacing_rate =
std::max(min_total_allocated_bitrate_, last_loss_based_target_rate_) *
pacing_factor_;
}
if (limit_pacingfactor_by_upper_link_capacity_estimate_ && estimate_ &&
estimate_->link_capacity_upper.IsFinite() &&
pacing_rate > estimate_->link_capacity_upper) {
pacing_rate =
std::max({estimate_->link_capacity_upper, min_total_allocated_bitrate_,
last_loss_based_target_rate_});
}
DataRate padding_rate =
(last_loss_base_state_ == LossBasedState::kIncreaseUsingPadding)
? std::max(max_padding_rate_, last_loss_based_target_rate_)
: max_padding_rate_;
padding_rate = std::min(padding_rate, last_pushback_target_rate_);
PacerConfig msg;
msg.at_time = at_time;
msg.time_window = TimeDelta::Seconds(1);
msg.data_window = pacing_rate * msg.time_window;
msg.pad_window = padding_rate * msg.time_window;
return msg;
}

10
src/qos/congestion_control.h
View File

@@ -28,6 +28,9 @@ class CongestionControl {
void MaybeTriggerOnNetworkChanged(NetworkControlUpdate* update,
Timestamp at_time);
private:
PacerConfig GetPacingRates(Timestamp at_time) const;
private:
const bool packet_feedback_only_;
const bool use_min_allocatable_as_lower_bound_;
@@ -54,12 +57,19 @@ class CongestionControl {
bool first_packet_sent_ = false;
std::optional<NetworkStateEstimate> estimate_;
Timestamp next_loss_update_ = Timestamp::MinusInfinity();
int lost_packets_since_last_loss_update_ = 0;
int expected_packets_since_last_loss_update_ = 0;
std::deque<int64_t> feedback_max_rtts_;
DataRate last_loss_based_target_rate_;
DataRate last_pushback_target_rate_;
DataRate last_stable_target_rate_;
LossBasedState last_loss_base_state_;
std::optional<uint8_t> last_estimated_fraction_loss_ = 0;
TimeDelta last_estimated_round_trip_time_ = TimeDelta::PlusInfinity();

1
src/qos/delay_based_bwe.cc
View File

@@ -206,6 +206,7 @@ DelayBasedBwe::Result DelayBasedBwe::MaybeUpdateEstimate(
}
result.delay_detector_state = detector_state;
return result;
}

10
src/qos/send_side_bandwidth_estimation.cc
View File

@@ -234,6 +234,10 @@ DataRate SendSideBandwidthEstimation::target_rate() const {
return std::max(min_bitrate_configured_, target);
}
LossBasedState SendSideBandwidthEstimation::loss_based_state() const {
return loss_based_state_;
}
bool SendSideBandwidthEstimation::IsRttAboveLimit() const {
return rtt_backoff_.IsRttAboveLimit();
}
@@ -511,4 +515,10 @@ void SendSideBandwidthEstimation::ApplyTargetLimits(Timestamp at_time) {
UpdateTargetBitrate(current_target_, at_time);
}
bool SendSideBandwidthEstimation::PaceAtLossBasedEstimate() const {
// return LossBasedBandwidthEstimatorV2ReadyForUse() &&
// loss_based_bandwidth_estimator_v2_->PaceAtLossBasedEstimate();
return false;
}
} // namespace webrtc

13
src/qos/send_side_bandwidth_estimation.h
View File

@@ -29,6 +29,16 @@
namespace webrtc {
enum class LossBasedState {
kIncreasing = 0,
// TODO(bugs.webrtc.org/12707): Remove one of the increasing states once we
// have decided if padding is usefull for ramping up when BWE is loss
// limited.
kIncreaseUsingPadding = 1,
kDecreasing = 2,
kDelayBasedEstimate = 3
};
class LinkCapacityTracker {
public:
LinkCapacityTracker() = default;
@@ -79,6 +89,7 @@ class SendSideBandwidthEstimation {
void OnRouteChange();
DataRate target_rate() const;
LossBasedState loss_based_state() const;
// Return whether the current rtt is higher than the rtt limited configured in
// RttBasedBackoff.
bool IsRttAboveLimit() const;
@@ -115,6 +126,7 @@ class SendSideBandwidthEstimation {
BandwidthUsage delay_detector_state,
std::optional<DataRate> probe_bitrate,
bool in_alr);
bool PaceAtLossBasedEstimate() const;
private:
friend class GoogCcStatePrinter;
@@ -184,6 +196,7 @@ class SendSideBandwidthEstimation {
float high_loss_threshold_;
DataRate bitrate_threshold_;
bool disable_receiver_limit_caps_only_;
LossBasedState loss_based_state_;
};
} // namespace webrtc
#endif // MODULES_CONGESTION_CONTROLLER_GOOG_CC_SEND_SIDE_BANDWIDTH_ESTIMATION_H_

317
src/transport/ice_transport.cpp
View File

@@ -7,10 +7,6 @@
#include "common.h"
#include "log.h"
#if __APPLE__
#else
#include "nvcodec_api.h"
#endif
using nlohmann::json;
@@ -25,22 +21,9 @@ IceTransport::IceTransport(
remote_user_id_(remote_user_id),
ice_ws_transport_(ice_ws_transmission),
on_ice_status_change_(on_ice_status_change),
user_data_(user_data),
clock_(webrtc::Clock::GetRealTimeClockShared()) {}
user_data_(user_data) {}
IceTransport::~IceTransport() {
user_data_ = nullptr;
video_codec_inited_ = false;
audio_codec_inited_ = false;
load_nvcodec_dll_success_ = false;
#ifdef __APPLE__
#else
if (hardware_acceleration_ && load_nvcodec_dll_success_) {
ReleaseNvCodecDll();
}
#endif
}
IceTransport::~IceTransport() {}
int IceTransport::SetLocalCapabilities(bool hardware_acceleration,
bool use_trickle_ice,
@@ -63,12 +46,24 @@ int IceTransport::InitIceTransmission(
std::string &stun_ip, int stun_port, std::string &turn_ip, int turn_port,
std::string &turn_username, std::string &turn_password,
rtp::PAYLOAD_TYPE video_codec_payload_type) {
ice_transport_controller_ = std::make_shared<IceTransportController>();
ice_agent_ = std::make_unique<IceAgent>(
offer_peer_, use_trickle_ice_, use_reliable_ice_, enable_turn_,
force_turn_, stun_ip, stun_port, turn_ip, turn_port, turn_username,
turn_password);
InitializeIOStatistics();
ice_io_statistics_ = std::make_unique<IOStatistics>(
[this](const IOStatistics::NetTrafficStats &net_traffic_stats) {
if (on_receive_net_status_report_) {
XNetTrafficStats xnet_traffic_stats;
memcpy(&xnet_traffic_stats, &net_traffic_stats,
sizeof(XNetTrafficStats));
on_receive_net_status_report_(
user_id_.data(), user_id_.size(), TraversalMode(traversal_type_),
&xnet_traffic_stats, remote_user_id_.data(),
remote_user_id_.size(), user_data_);
}
});
ice_agent_->CreateIceAgent(
[](NiceAgent *agent, guint stream_id, guint component_id,
@@ -100,66 +95,6 @@ int IceTransport::InitIceTransmission(
return 0;
}
void IceTransport::InitializeIOStatistics() {
ice_io_statistics_ = std::make_unique<IOStatistics>(
[this](const IOStatistics::NetTrafficStats &net_traffic_stats) {
if (on_receive_net_status_report_) {
XNetTrafficStats xnet_traffic_stats;
memcpy(&xnet_traffic_stats, &net_traffic_stats,
sizeof(XNetTrafficStats));
on_receive_net_status_report_(
user_id_.data(), user_id_.size(), TraversalMode(traversal_type_),
&xnet_traffic_stats, remote_user_id_.data(),
remote_user_id_.size(), user_data_);
}
});
}
void IceTransport::InitializeChannels(
rtp::PAYLOAD_TYPE video_codec_payload_type) {
video_codec_payload_type_ = video_codec_payload_type;
video_channel_send_ = std::make_unique<VideoChannelSend>(clock_, ice_agent_,
ice_io_statistics_);
audio_channel_send_ =
std::make_unique<AudioChannelSend>(ice_agent_, ice_io_statistics_);
data_channel_send_ =
std::make_unique<DataChannelSend>(ice_agent_, ice_io_statistics_);
video_channel_send_->Initialize(video_codec_payload_type_);
audio_channel_send_->Initialize(rtp::PAYLOAD_TYPE::OPUS);
data_channel_send_->Initialize(rtp::PAYLOAD_TYPE::DATA);
std::weak_ptr<IceTransport> weak_self = shared_from_this();
video_channel_receive_ = std::make_unique<VideoChannelReceive>(
clock_, ice_agent_, ice_io_statistics_,
[this, weak_self](VideoFrame &video_frame) {
if (auto self = weak_self.lock()) {
OnReceiveCompleteFrame(video_frame);
}
});
audio_channel_receive_ = std::make_unique<AudioChannelReceive>(
ice_agent_, ice_io_statistics_,
[this, weak_self](const char *data, size_t size) {
if (auto self = weak_self.lock()) {
OnReceiveCompleteAudio(data, size);
}
});
data_channel_receive_ = std::make_unique<DataChannelReceive>(
ice_agent_, ice_io_statistics_,
[this, weak_self](const char *data, size_t size) {
if (auto self = weak_self.lock()) {
OnReceiveCompleteData(data, size);
}
});
video_channel_receive_->Initialize(video_codec_payload_type_);
audio_channel_receive_->Initialize(rtp::PAYLOAD_TYPE::OPUS);
data_channel_receive_->Initialize(rtp::PAYLOAD_TYPE::DATA);
}
void IceTransport::OnIceStateChange(NiceAgent *agent, guint stream_id,
guint component_id,
NiceComponentState state,
@@ -252,12 +187,13 @@ void IceTransport::OnReceiveBuffer(NiceAgent *agent, guint stream_id,
gchar *buffer, gpointer user_ptr) {
if (!is_closed_) {
if (CheckIsRtpPacket(buffer, size)) {
if (CheckIsVideoPacket(buffer, size)) {
video_channel_receive_->OnReceiveRtpPacket(buffer, size);
} else if (CheckIsAudioPacket(buffer, size)) {
audio_channel_receive_->OnReceiveRtpPacket(buffer, size);
} else if (CheckIsDataPacket(buffer, size)) {
data_channel_receive_->OnReceiveRtpPacket(buffer, size);
if (CheckIsVideoPacket(buffer, size) && ice_transport_controller_) {
ice_transport_controller_->OnReceiveVideoRtpPacket(buffer, size);
} else if (CheckIsAudioPacket(buffer, size) &&
ice_transport_controller_) {
ice_transport_controller_->OnReceiveAudioRtpPacket(buffer, size);
} else if (CheckIsDataPacket(buffer, size) && ice_transport_controller_) {
ice_transport_controller_->OnReceiveDataRtpPacket(buffer, size);
}
} else if (CheckIsRtcpPacket(buffer, size)) {
// LOG_ERROR("Rtcp packet [{}]", (uint8_t)(buffer[1]));
@@ -372,44 +308,12 @@ bool IceTransport::HandleCongestionControlFeedback(
// rtcp_packet_info->congestion_control_feedback.emplace(std::move(feedback));
// }
video_channel_send_->OnCongestionControlFeedback(clock_->CurrentTime(),
feedback);
if (ice_transport_controller_) {
ice_transport_controller_->OnCongestionControlFeedback(feedback);
}
return true;
}
void IceTransport::OnReceiveCompleteFrame(VideoFrame &video_frame) {
int num_frame_returned = video_decoder_->Decode(
(uint8_t *)video_frame.Buffer(), video_frame.Size(),
[this](VideoFrame video_frame) {
if (on_receive_video_) {
XVideoFrame x_video_frame;
x_video_frame.data = (const char *)video_frame.Buffer();
x_video_frame.width = video_frame.Width();
x_video_frame.height = video_frame.Height();
x_video_frame.size = video_frame.Size();
on_receive_video_(&x_video_frame, remote_user_id_.data(),
remote_user_id_.size(), user_data_);
}
});
}
void IceTransport::OnReceiveCompleteAudio(const char *data, size_t size) {
int num_frame_returned = audio_decoder_->Decode(
(uint8_t *)data, size, [this](uint8_t *data, int size) {
if (on_receive_audio_) {
on_receive_audio_((const char *)data, size, remote_user_id_.data(),
remote_user_id_.size(), user_data_);
}
});
}
void IceTransport::OnReceiveCompleteData(const char *data, size_t size) {
if (on_receive_data_) {
on_receive_data_(data, size, remote_user_id_.data(), remote_user_id_.size(),
user_data_);
}
}
int IceTransport::DestroyIceTransmission() {
LOG_INFO("[{}->{}] Destroy ice transmission", user_id_, remote_user_id_);
is_closed_ = true;
@@ -422,130 +326,9 @@ int IceTransport::DestroyIceTransmission() {
ice_io_statistics_->Stop();
}
if (video_channel_send_) {
video_channel_send_->Destroy();
}
if (audio_channel_send_) {
audio_channel_send_->Destroy();
}
if (data_channel_send_) {
data_channel_send_->Destroy();
}
if (video_channel_receive_) {
video_channel_receive_->Destroy();
}
if (audio_channel_receive_) {
audio_channel_receive_->Destroy();
}
if (data_channel_receive_) {
data_channel_receive_->Destroy();
}
return ice_agent_->DestroyIceAgent();
}
int IceTransport::CreateVideoCodec(rtp::PAYLOAD_TYPE video_pt,
bool hardware_acceleration) {
if (video_codec_inited_) {
return 0;
}
hardware_acceleration_ = hardware_acceleration;
if (rtp::PAYLOAD_TYPE::AV1 == video_pt) {
if (hardware_acceleration_) {
hardware_acceleration_ = false;
LOG_WARN("Only support software codec for AV1");
}
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, true);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, true);
} else if (rtp::PAYLOAD_TYPE::H264 == video_pt) {
#ifdef __APPLE__
if (hardware_acceleration_) {
hardware_acceleration_ = false;
LOG_WARN(
"MacOS not support hardware acceleration, use default software "
"codec");
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
} else {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
#else
if (hardware_acceleration_) {
if (0 == LoadNvCodecDll()) {
load_nvcodec_dll_success_ = true;
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(true, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(true, false);
} else {
LOG_WARN(
"Hardware accelerated codec not available, use default software "
"codec");
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
} else {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
#endif
}
if (!video_encoder_) {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
LOG_ERROR("Create encoder failed, try to use software H.264 encoder");
}
if (!video_encoder_ || 0 != video_encoder_->Init()) {
LOG_ERROR("Encoder init failed");
return -1;
}
if (!video_decoder_) {
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
LOG_ERROR("Create decoder failed, try to use software H.264 decoder");
}
if (!video_decoder_ || video_decoder_->Init()) {
LOG_ERROR("Decoder init failed");
return -1;
}
video_codec_inited_ = true;
LOG_INFO("Create video codec [{}|{}] finish",
video_encoder_->GetEncoderName(), video_decoder_->GetDecoderName());
return 0;
}
int IceTransport::CreateAudioCodec() {
if (audio_codec_inited_) {
return 0;
}
audio_encoder_ = std::make_unique<AudioEncoder>(AudioEncoder(48000, 1, 480));
if (!audio_encoder_ || 0 != audio_encoder_->Init()) {
LOG_ERROR("Audio encoder init failed");
return -1;
}
audio_decoder_ = std::make_unique<AudioDecoder>(AudioDecoder(48000, 1, 480));
if (!audio_decoder_ || 0 != audio_decoder_->Init()) {
LOG_ERROR("Audio decoder init failed");
return -1;
}
audio_codec_inited_ = true;
LOG_INFO("Create audio codec [{}|{}] finish",
audio_encoder_->GetEncoderName(), audio_decoder_->GetDecoderName());
return 0;
}
int IceTransport::SetTransmissionId(const std::string &transmission_id) {
transmission_id_ = transmission_id;
@@ -742,10 +525,12 @@ std::string IceTransport::GetRemoteCapabilities(const std::string &remote_sdp) {
return std::string();
}
InitializeChannels(negotiated_video_pt_);
CreateVideoCodec(negotiated_video_pt_, hardware_acceleration_);
CreateAudioCodec();
if (ice_transport_controller_) {
ice_transport_controller_->Create(
remote_user_id_, negotiated_video_pt_, hardware_acceleration_,
ice_agent_, ice_io_statistics_, on_receive_video_, on_receive_audio_,
on_receive_data_, user_data_);
}
remote_capabilities_got_ = true;
}
@@ -978,29 +763,11 @@ int IceTransport::SendVideoFrame(const XVideoFrame *video_frame) {
return -2;
}
if (b_force_i_frame_) {
video_encoder_->ForceIdr();
LOG_INFO("Force I frame");
b_force_i_frame_ = false;
if (ice_transport_controller_) {
return ice_transport_controller_->SendVideo(video_frame);
}
int ret = video_encoder_->Encode(
video_frame,
[this](char *encoded_frame, size_t size,
VideoEncoder::VideoFrameType frame_type) -> int {
if (video_channel_send_) {
video_channel_send_->SendVideo(encoded_frame, size);
}
return 0;
});
if (0 != ret) {
LOG_ERROR("Encode failed");
return -1;
}
return 0;
}
int IceTransport::SendAudioFrame(const char *data, size_t size) {
@@ -1011,17 +778,11 @@ int IceTransport::SendAudioFrame(const char *data, size_t size) {
return -2;
}
int ret = audio_encoder_->Encode(
(uint8_t *)data, size,
[this](char *encoded_audio_buffer, size_t size) -> int {
if (audio_channel_send_) {
audio_channel_send_->SendAudio(encoded_audio_buffer, size);
if (ice_transport_controller_) {
return ice_transport_controller_->SendAudio(data, size);
}
return 0;
});
return ret;
return -1;
}
int IceTransport::SendDataFrame(const char *data, size_t size) {
@@ -1032,11 +793,11 @@ int IceTransport::SendDataFrame(const char *data, size_t size) {
return -2;
}
if (data_channel_send_) {
data_channel_send_->SendData(data, size);
if (ice_transport_controller_) {
return ice_transport_controller_->SendData(data, size);
}
return 0;
return -1;
}
uint8_t IceTransport::CheckIsRtpPacket(const char *buffer, size_t size) {

96
src/transport/ice_transport.h
View File

@@ -9,32 +9,15 @@
#include <iostream>
#include "audio_decoder.h"
#include "audio_encoder.h"
// #include "congestion_control.h"
#include "audio_channel_receive.h"
#include "audio_channel_send.h"
#include "clock.h"
#include "data_channel_receive.h"
#include "data_channel_send.h"
#include "ice_agent.h"
#include "ice_transport_controller.h"
#include "io_statistics.h"
#include "ringbuffer.h"
#include "rtcp_packet_info.h"
#include "rtp_audio_receiver.h"
#include "rtp_audio_sender.h"
#include "rtp_data_receiver.h"
#include "rtp_data_sender.h"
#include "rtp_packet.h"
#include "rtp_video_receiver.h"
#include "rtp_video_sender.h"
#include "video_channel_receive.h"
#include "video_channel_send.h"
#include "video_decoder_factory.h"
#include "video_encoder_factory.h"
#include "ws_client.h"
class IceTransport : public std::enable_shared_from_this<IceTransport> {
class IceTransport {
public:
typedef enum { VIDEO = 96, AUDIO = 97, DATA = 127 } DATA_TYPE;
typedef enum { H264 = 96, AV1 = 99 } VIDEO_TYPE;
@@ -70,24 +53,11 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
int DestroyIceTransmission();
void SetOnReceiveVideoFunc(
std::function<void(const XVideoFrame *, const char *, const size_t,
void *)>
on_receive_video) {
void SetOnReceiveFunc(OnReceiveVideo on_receive_video,
OnReceiveAudio on_receive_audio,
OnReceiveData on_receive_data) {
on_receive_video_ = on_receive_video;
}
void SetOnReceiveAudioFunc(
std::function<void(const char *, size_t, const char *, const size_t,
void *)>
on_receive_audio) {
on_receive_audio_ = on_receive_audio;
}
void SetOnReceiveDataFunc(
std::function<void(const char *, size_t, const char *, const size_t,
void *)>
on_receive_data) {
on_receive_data_ = on_receive_data;
}
@@ -131,9 +101,6 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
bool NegotiateAudioPayloadType(const std::string &remote_sdp);
bool NegotiateDataPayloadType(const std::string &remote_sdp);
int CreateVideoCodec(rtp::PAYLOAD_TYPE video_pt, bool hardware_acceleration);
int CreateAudioCodec();
private:
uint8_t CheckIsRtpPacket(const char *buffer, size_t size);
uint8_t CheckIsRtcpPacket(const char *buffer, size_t size);
@@ -142,10 +109,6 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
uint8_t CheckIsDataPacket(const char *buffer, size_t size);
private:
void InitializeIOStatistics();
void InitializeChannels(rtp::PAYLOAD_TYPE video_codec_payload_type);
void OnIceStateChange(NiceAgent *agent, guint stream_id, guint component_id,
NiceComponentState state, gpointer user_ptr);
@@ -162,12 +125,6 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
void OnReceiveBuffer(NiceAgent *agent, guint stream_id, guint component_id,
guint size, gchar *buffer, gpointer user_ptr);
void OnReceiveCompleteFrame(VideoFrame &video_frame);
void OnReceiveCompleteAudio(const char *data, size_t size);
void OnReceiveCompleteData(const char *data, size_t size);
bool ParseRtcpPacket(const uint8_t *buffer, size_t size,
RtcpPacketInfo *rtcp_packet_info);
@@ -176,6 +133,7 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
RtcpPacketInfo *rtcp_packet_info);
private:
bool hardware_acceleration_ = false;
bool use_trickle_ice_ = true;
bool enable_turn_ = false;
bool use_reliable_ice_ = false;
@@ -203,13 +161,10 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
std::shared_ptr<IceAgent> ice_agent_ = nullptr;
bool is_closed_ = false;
std::shared_ptr<WsClient> ice_ws_transport_ = nullptr;
// CongestionControl *congestion_control_ = nullptr;
std::function<void(const XVideoFrame *, const char *, const size_t, void *)>
on_receive_video_ = nullptr;
std::function<void(const char *, size_t, const char *, const size_t, void *)>
on_receive_audio_ = nullptr;
std::function<void(const char *, size_t, const char *, const size_t, void *)>
on_receive_data_ = nullptr;
OnReceiveVideo on_receive_video_ = nullptr;
OnReceiveAudio on_receive_audio_ = nullptr;
OnReceiveData on_receive_data_ = nullptr;
std::function<void(std::string, const std::string &)> on_ice_status_change_ =
nullptr;
@@ -220,23 +175,7 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
on_receive_net_status_report_ = nullptr;
private:
std::shared_ptr<webrtc::Clock> clock_;
std::unique_ptr<VideoChannelSend> video_channel_send_ = nullptr;
std::unique_ptr<VideoChannelReceive> video_channel_receive_ = nullptr;
std::unique_ptr<AudioChannelSend> audio_channel_send_ = nullptr;
std::unique_ptr<AudioChannelReceive> audio_channel_receive_ = nullptr;
std::unique_ptr<DataChannelSend> data_channel_send_ = nullptr;
std::unique_ptr<DataChannelReceive> data_channel_receive_ = nullptr;
std::unique_ptr<RtpVideoReceiver> rtp_video_receiver_ = nullptr;
std::unique_ptr<RtpVideoSender> rtp_video_sender_ = nullptr;
std::unique_ptr<RtpAudioReceiver> rtp_audio_receiver_ = nullptr;
std::unique_ptr<RtpAudioSender> rtp_audio_sender_ = nullptr;
std::unique_ptr<RtpDataReceiver> rtp_data_receiver_ = nullptr;
std::unique_ptr<RtpDataSender> rtp_data_sender_ = nullptr;
bool start_send_packet_ = false;
uint32_t last_complete_frame_ts_ = 0;
std::shared_ptr<IceTransportController> ice_transport_controller_ = nullptr;
private:
std::shared_ptr<IOStatistics> ice_io_statistics_ = nullptr;
@@ -250,19 +189,6 @@ class IceTransport : public std::enable_shared_from_this<IceTransport> {
rtp::PAYLOAD_TYPE negotiated_video_pt_ = rtp::PAYLOAD_TYPE::UNDEFINED;
rtp::PAYLOAD_TYPE negotiated_audio_pt_ = rtp::PAYLOAD_TYPE::UNDEFINED;
rtp::PAYLOAD_TYPE negotiated_data_pt_ = rtp::PAYLOAD_TYPE::UNDEFINED;
private:
std::unique_ptr<VideoEncoder> video_encoder_ = nullptr;
std::unique_ptr<VideoDecoder> video_decoder_ = nullptr;
bool b_force_i_frame_ = false;
bool video_codec_inited_ = false;
bool load_nvcodec_dll_success_ = false;
bool hardware_acceleration_ = false;
private:
std::unique_ptr<AudioEncoder> audio_encoder_ = nullptr;
std::unique_ptr<AudioDecoder> audio_decoder_ = nullptr;
bool audio_codec_inited_ = false;
};
#endif

389
src/transport/ice_transport_controller.cpp Normal file
View File

@@ -0,0 +1,389 @@
#include "ice_transport_controller.h"
#if __APPLE__
#else
#include "nvcodec_api.h"
#endif
IceTransportController::IceTransportController()
: b_force_i_frame_(true),
video_codec_inited_(false),
audio_codec_inited_(false),
load_nvcodec_dll_success_(false),
hardware_acceleration_(false),
clock_(webrtc::Clock::GetRealTimeClockShared()) {}
IceTransportController::~IceTransportController() {
user_data_ = nullptr;
video_codec_inited_ = false;
audio_codec_inited_ = false;
load_nvcodec_dll_success_ = false;
#ifdef __APPLE__
#else
if (hardware_acceleration_ && load_nvcodec_dll_success_) {
ReleaseNvCodecDll();
}
#endif
}
void IceTransportController::Create(
std::string remote_user_id, rtp::PAYLOAD_TYPE video_codec_payload_type,
bool hardware_acceleration, std::shared_ptr<IceAgent> ice_agent,
std::shared_ptr<IOStatistics> ice_io_statistics,
OnReceiveVideo on_receive_video, OnReceiveAudio on_receive_audio,
OnReceiveData on_receive_data, void* user_data) {
remote_user_id_ = remote_user_id;
on_receive_video_ = on_receive_video;
on_receive_audio_ = on_receive_audio;
on_receive_data_ = on_receive_data;
user_data_ = user_data;
CreateVideoCodec(video_codec_payload_type, hardware_acceleration);
CreateAudioCodec();
controller_ = std::make_unique<CongestionControl>();
video_channel_send_ = std::make_unique<VideoChannelSend>(
clock_, ice_agent, ice_io_statistics,
[this](const webrtc::RtpPacketToSend& packet) {
OnSentRtpPacket(packet);
});
audio_channel_send_ =
std::make_unique<AudioChannelSend>(ice_agent, ice_io_statistics);
data_channel_send_ =
std::make_unique<DataChannelSend>(ice_agent, ice_io_statistics);
video_channel_send_->Initialize(video_codec_payload_type);
audio_channel_send_->Initialize(rtp::PAYLOAD_TYPE::OPUS);
data_channel_send_->Initialize(rtp::PAYLOAD_TYPE::DATA);
std::weak_ptr<IceTransportController> weak_self = shared_from_this();
video_channel_receive_ = std::make_unique<VideoChannelReceive>(
clock_, ice_agent, ice_io_statistics,
[this, weak_self](VideoFrame& video_frame) {
if (auto self = weak_self.lock()) {
OnReceiveCompleteFrame(video_frame);
}
});
audio_channel_receive_ = std::make_unique<AudioChannelReceive>(
ice_agent, ice_io_statistics,
[this, weak_self](const char* data, size_t size) {
if (auto self = weak_self.lock()) {
OnReceiveCompleteAudio(data, size);
}
});
data_channel_receive_ = std::make_unique<DataChannelReceive>(
ice_agent, ice_io_statistics,
[this, weak_self](const char* data, size_t size) {
if (auto self = weak_self.lock()) {
OnReceiveCompleteData(data, size);
}
});
video_channel_receive_->Initialize(video_codec_payload_type);
audio_channel_receive_->Initialize(rtp::PAYLOAD_TYPE::OPUS);
data_channel_receive_->Initialize(rtp::PAYLOAD_TYPE::DATA);
}
void IceTransportController::Destroy() {
if (video_channel_send_) {
video_channel_send_->Destroy();
}
if (audio_channel_send_) {
audio_channel_send_->Destroy();
}
if (data_channel_send_) {
data_channel_send_->Destroy();
}
if (video_channel_receive_) {
video_channel_receive_->Destroy();
}
if (audio_channel_receive_) {
audio_channel_receive_->Destroy();
}
if (data_channel_receive_) {
data_channel_receive_->Destroy();
}
}
int IceTransportController::SendVideo(const XVideoFrame* video_frame) {
if (!video_encoder_) {
LOG_ERROR("Video Encoder not created");
return -1;
}
if (b_force_i_frame_) {
video_encoder_->ForceIdr();
LOG_INFO("Force I frame");
b_force_i_frame_ = false;
}
int ret = video_encoder_->Encode(
video_frame,
[this](char* encoded_frame, size_t size,
VideoEncoder::VideoFrameType frame_type) -> int {
if (video_channel_send_) {
video_channel_send_->SendVideo(encoded_frame, size);
}
return 0;
});
if (0 != ret) {
LOG_ERROR("Encode failed");
return -1;
} else {
return 0;
}
}
int IceTransportController::SendAudio(const char* data, size_t size) {
if (!audio_encoder_) {
LOG_ERROR("Audio Encoder not created");
return -1;
}
int ret = audio_encoder_->Encode(
(uint8_t*)data, size,
[this](char* encoded_audio_buffer, size_t size) -> int {
if (audio_channel_send_) {
audio_channel_send_->SendAudio(encoded_audio_buffer, size);
}
return 0;
});
return ret;
}
int IceTransportController::SendData(const char* data, size_t size) {
if (data_channel_send_) {
data_channel_send_->SendData(data, size);
}
return 0;
}
int IceTransportController::OnReceiveVideoRtpPacket(const char* data,
size_t size) {
if (video_channel_receive_) {
return video_channel_receive_->OnReceiveRtpPacket(data, size);
}
return -1;
}
int IceTransportController::OnReceiveAudioRtpPacket(const char* data,
size_t size) {
if (audio_channel_receive_) {
return audio_channel_receive_->OnReceiveRtpPacket(data, size);
}
return -1;
}
int IceTransportController::OnReceiveDataRtpPacket(const char* data,
size_t size) {
if (data_channel_receive_) {
return data_channel_receive_->OnReceiveRtpPacket(data, size);
}
return -1;
}
void IceTransportController::OnReceiveCompleteFrame(VideoFrame& video_frame) {
int num_frame_returned = video_decoder_->Decode(
(uint8_t*)video_frame.Buffer(), video_frame.Size(),
[this](VideoFrame video_frame) {
if (on_receive_video_) {
XVideoFrame x_video_frame;
x_video_frame.data = (const char*)video_frame.Buffer();
x_video_frame.width = video_frame.Width();
x_video_frame.height = video_frame.Height();
x_video_frame.size = video_frame.Size();
on_receive_video_(&x_video_frame, remote_user_id_.data(),
remote_user_id_.size(), user_data_);
}
});
}
void IceTransportController::OnReceiveCompleteAudio(const char* data,
size_t size) {
int num_frame_returned = audio_decoder_->Decode(
(uint8_t*)data, size, [this](uint8_t* data, int size) {
if (on_receive_audio_) {
on_receive_audio_((const char*)data, size, remote_user_id_.data(),
remote_user_id_.size(), user_data_);
}
});
}
void IceTransportController::OnReceiveCompleteData(const char* data,
size_t size) {
if (on_receive_data_) {
on_receive_data_(data, size, remote_user_id_.data(), remote_user_id_.size(),
user_data_);
}
}
int IceTransportController::CreateVideoCodec(rtp::PAYLOAD_TYPE video_pt,
bool hardware_acceleration) {
if (video_codec_inited_) {
return 0;
}
hardware_acceleration_ = hardware_acceleration;
if (rtp::PAYLOAD_TYPE::AV1 == video_pt) {
if (hardware_acceleration_) {
hardware_acceleration_ = false;
LOG_WARN("Only support software codec for AV1");
}
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, true);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, true);
} else if (rtp::PAYLOAD_TYPE::H264 == video_pt) {
#ifdef __APPLE__
if (hardware_acceleration_) {
hardware_acceleration_ = false;
LOG_WARN(
"MacOS not support hardware acceleration, use default software "
"codec");
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
} else {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
#else
if (hardware_acceleration_) {
if (0 == LoadNvCodecDll()) {
load_nvcodec_dll_success_ = true;
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(true, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(true, false);
} else {
LOG_WARN(
"Hardware accelerated codec not available, use default software "
"codec");
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
} else {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
}
#endif
}
if (!video_encoder_) {
video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
LOG_ERROR("Create encoder failed, try to use software H.264 encoder");
}
if (!video_encoder_ || 0 != video_encoder_->Init()) {
LOG_ERROR("Encoder init failed");
return -1;
}
if (!video_decoder_) {
video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
LOG_ERROR("Create decoder failed, try to use software H.264 decoder");
}
if (!video_decoder_ || video_decoder_->Init()) {
LOG_ERROR("Decoder init failed");
return -1;
}
video_codec_inited_ = true;
LOG_INFO("Create video codec [{}|{}] finish",
video_encoder_->GetEncoderName(), video_decoder_->GetDecoderName());
return 0;
}
int IceTransportController::CreateAudioCodec() {
if (audio_codec_inited_) {
return 0;
}
audio_encoder_ = std::make_unique<AudioEncoder>(AudioEncoder(48000, 1, 480));
if (!audio_encoder_ || 0 != audio_encoder_->Init()) {
LOG_ERROR("Audio encoder init failed");
return -1;
}
audio_decoder_ = std::make_unique<AudioDecoder>(AudioDecoder(48000, 1, 480));
if (!audio_decoder_ || 0 != audio_decoder_->Init()) {
LOG_ERROR("Audio decoder init failed");
return -1;
}
audio_codec_inited_ = true;
LOG_INFO("Create audio codec [{}|{}] finish",
audio_encoder_->GetEncoderName(), audio_decoder_->GetDecoderName());
return 0;
}
void IceTransportController::OnCongestionControlFeedback(
const webrtc::rtcp::CongestionControlFeedback& feedback) {
std::optional<webrtc::TransportPacketsFeedback> feedback_msg =
transport_feedback_adapter_.ProcessCongestionControlFeedback(
feedback, clock_->CurrentTime());
if (feedback_msg) {
HandleTransportPacketsFeedback(*feedback_msg);
}
}
void IceTransportController::HandleTransportPacketsFeedback(
const webrtc::TransportPacketsFeedback& feedback) {
if (controller_)
PostUpdates(controller_->OnTransportPacketsFeedback(feedback));
UpdateCongestedState();
}
void IceTransportController::OnSentRtpPacket(
const webrtc::RtpPacketToSend& packet) {
webrtc::PacedPacketInfo pacing_info;
size_t transport_overhead_bytes_per_packet_ = 0;
webrtc::Timestamp creation_time =
webrtc::Timestamp::Millis(clock_->TimeInMilliseconds());
transport_feedback_adapter_.AddPacket(
packet, pacing_info, transport_overhead_bytes_per_packet_, creation_time);
rtc::SentPacket sent_packet;
sent_packet.packet_id = packet.transport_sequence_number().value();
sent_packet.send_time_ms = clock_->TimeInMilliseconds();
sent_packet.info.included_in_feedback = true;
sent_packet.info.included_in_allocation = true;
sent_packet.info.packet_size_bytes = packet.size();
sent_packet.info.packet_type = rtc::PacketType::kData;
transport_feedback_adapter_.ProcessSentPacket(sent_packet);
}
void IceTransportController::PostUpdates(webrtc::NetworkControlUpdate update) {
// UpdateControlState();
target_bitrate_ = update.target_rate.has_value()
? update.target_rate->target_rate.bps()
: 0;
// LOG_WARN("Target bitrate [{}]bps", target_bitrate_);
video_encoder_->SetTargetBitrate(target_bitrate_);
}
void IceTransportController::UpdateControlState() {
if (controller_) {
}
}
void IceTransportController::UpdateCongestedState() {
if (controller_) {
}
}

121
src/transport/ice_transport_controller.h Normal file
View File

@@ -0,0 +1,121 @@
/*
* @Author: DI JUNKUN
* @Date: 2025-02-11
* Copyright (c) 2025 by DI JUNKUN, All Rights Reserved.
*/
#ifndef _ICE_TRANSPORT_CONTROLLER_H_
#define _ICE_TRANSPORT_CONTROLLER_H_
#include "api/transport/network_types.h"
#include "api/units/timestamp.h"
#include "audio_channel_receive.h"
#include "audio_channel_send.h"
#include "audio_decoder.h"
#include "audio_encoder.h"
#include "clock.h"
#include "congestion_control.h"
#include "congestion_control_feedback.h"
#include "data_channel_receive.h"
#include "data_channel_send.h"
#include "ice_agent.h"
#include "transport_feedback_adapter.h"
#include "video_channel_receive.h"
#include "video_channel_send.h"
#include "video_decoder_factory.h"
#include "video_encoder_factory.h"
typedef void (*OnReceiveVideo)(const XVideoFrame *, const char *, const size_t,
void *);
typedef void (*OnReceiveAudio)(const char *, size_t, const char *, const size_t,
void *);
typedef void (*OnReceiveData)(const char *, size_t, const char *, const size_t,
void *);
class IceTransportController
: public std::enable_shared_from_this<IceTransportController> {
public:
IceTransportController();
~IceTransportController();
public:
void Create(std::string remote_user_id,
rtp::PAYLOAD_TYPE video_codec_payload_type,
bool hardware_acceleration, std::shared_ptr<IceAgent> ice_agent,
std::shared_ptr<IOStatistics> ice_io_statistics,
OnReceiveVideo on_receive_video, OnReceiveAudio on_receive_audio,
OnReceiveData on_receive_data, void *user_data);
void Destroy();
int SendVideo(const XVideoFrame *video_frame);
int SendAudio(const char *data, size_t size);
int SendData(const char *data, size_t size);
int OnReceiveVideoRtpPacket(const char *data, size_t size);
int OnReceiveAudioRtpPacket(const char *data, size_t size);
int OnReceiveDataRtpPacket(const char *data, size_t size);
void OnReceiveCompleteFrame(VideoFrame &video_frame);
void OnReceiveCompleteAudio(const char *data, size_t size);
void OnReceiveCompleteData(const char *data, size_t size);
public:
void OnCongestionControlFeedback(
const webrtc::rtcp::CongestionControlFeedback &feedback);
private:
int CreateVideoCodec(rtp::PAYLOAD_TYPE video_pt, bool hardware_acceleration);
int CreateAudioCodec();
private:
void OnSentRtpPacket(const webrtc::RtpPacketToSend &packet);
void HandleTransportPacketsFeedback(
const webrtc::TransportPacketsFeedback &feedback);
void PostUpdates(webrtc::NetworkControlUpdate update);
void UpdateControlState();
void UpdateCongestedState();
private:
std::unique_ptr<VideoChannelSend> video_channel_send_ = nullptr;
std::unique_ptr<AudioChannelSend> audio_channel_send_ = nullptr;
std::unique_ptr<DataChannelSend> data_channel_send_ = nullptr;
std::unique_ptr<VideoChannelReceive> video_channel_receive_ = nullptr;
std::unique_ptr<AudioChannelReceive> audio_channel_receive_ = nullptr;
std::unique_ptr<DataChannelReceive> data_channel_receive_ = nullptr;
OnReceiveVideo on_receive_video_ = nullptr;
OnReceiveAudio on_receive_audio_ = nullptr;
OnReceiveData on_receive_data_ = nullptr;
private:
std::shared_ptr<IceAgent> ice_agent_ = nullptr;
std::shared_ptr<IOStatistics> ice_io_statistics_ = nullptr;
std::unique_ptr<RtpPacketizer> rtp_packetizer_ = nullptr;
std::unique_ptr<RtpVideoSender> rtp_video_sender_ = nullptr;
std::string remote_user_id_;
void *user_data_ = nullptr;
private:
std::shared_ptr<Clock> clock_;
webrtc::TransportFeedbackAdapter transport_feedback_adapter_;
std::unique_ptr<CongestionControl> controller_;
private:
std::unique_ptr<VideoEncoder> video_encoder_ = nullptr;
std::unique_ptr<VideoDecoder> video_decoder_ = nullptr;
bool b_force_i_frame_;
bool video_codec_inited_;
bool load_nvcodec_dll_success_;
bool hardware_acceleration_;
private:
std::unique_ptr<AudioEncoder> audio_encoder_ = nullptr;
std::unique_ptr<AudioDecoder> audio_decoder_ = nullptr;
bool audio_codec_inited_ = false;
private:
uint64_t target_bitrate_ = 0;
};
#endif