YangWX/crossdesk
1
0
Fork 0
mirror of https://github.com/kunkundi/crossdesk.git synced 2025-10-26 20:25:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feat] support FIR

Browse Source
This commit is contained in:
dijunkun
2025-03-07 18:36:55 +08:00
parent 19fc8cda89
commit de0386f08a
21 changed files with 247 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/channel/rtp_channel/rtp_video_receiver.cpp
View File

@@ -2,6 +2,7 @@
#include "api/ntp/ntp_time_util.h"
#include "common.h"
#include "fir.h"
#include "log.h"
#include "nack.h"
#include "rtcp_sender.h"
@@ -670,7 +671,15 @@ void RtpVideoReceiver::SendNack(const std::vector<uint16_t>& nack_list,
}
}
void RtpVideoReceiver::RequestKeyFrame() {}
void RtpVideoReceiver::RequestKeyFrame() {
++sequence_number_fir_;
webrtc::rtcp::Fir fir;
fir.SetSenderSsrc(ssrc_);
fir.AddRequestTo(remote_ssrc_, sequence_number_fir_);
rtcp_sender_->AppendPacket(fir);
rtcp_sender_->Send();
}
void RtpVideoReceiver::SendLossNotification(uint16_t last_decoded_seq_num,
uint16_t last_received_seq_num,

2
src/channel/rtp_channel/rtp_video_receiver.h
View File

@@ -155,6 +155,8 @@ class RtpVideoReceiver : public ThreadBase,
uint32_t bytes_sent = 0;
uint32_t reports_count = 0;
uint8_t sequence_number_fir_ = 0;
private:
FILE* file_rtp_recv_ = nullptr;
};

23
src/media/resolution_adapter/resolution_adapter.cpp
View File

@@ -1,25 +1,36 @@
#include "resolution_adapter.h"
#include "libyuv.h"
#include "log.h"
int ResolutionAdapter::GetResolution(int target_bitrate, int current_width,
int current_height, int& target_width,
int& target_height) {
int current_height, int* target_width,
int* target_height) {
for (auto& resolution : GetBitrateLimits()) {
if (target_bitrate >= resolution.min_start_bitrate_bps &&
target_bitrate <= resolution.max_bitrate_bps) {
if (current_width * current_height <= resolution.frame_size_pixels) {
target_width = current_width;
target_height = current_height;
*target_width = current_width;
*target_height = current_height;
LOG_INFO("1 source resolution {}x{}, target resolution {}x{}",
current_width, current_height, *target_width, *target_height);
return 0;
} else {
target_width = current_width * 3 / 5;
target_height = current_height * 3 / 5;
*target_width = current_width * 3 / 5;
*target_height = current_height * 3 / 5;
LOG_INFO("2 source resolution {}x{}, target resolution {}x{}",
current_width, current_height, *target_width, *target_height);
return 0;
}
}
}
*target_width = -1;
*target_height = -1;
LOG_INFO("3 source resolution {}x{}, target resolution {}x{}", current_width,
current_height, *target_width, *target_height);
return -1;
}

2
src/media/resolution_adapter/resolution_adapter.h
View File

@@ -19,7 +19,7 @@ class ResolutionAdapter {
public:
int GetResolution(int target_bitrate, int current_width, int current_height,
int& target_width, int& target_height);
int* target_width, int* target_height);
int ResolutionDowngrade(const XVideoFrame* video_frame, int target_width,
int target_height, XVideoFrame* new_frame);

2
src/media/video/decode/nvcodec/nvidia_video_decoder.cpp
View File

@@ -75,7 +75,7 @@ int NvidiaVideoDecoder::Decode(
#endif
if ((*(data + 4) & 0x1f) == 0x07) {
// LOG_WARN("Receive key frame");
LOG_WARN("Receive key frame");
}
int num_frame_returned = decoder->Decode(data, (int)size);

2
src/media/video/encode/aom/aom_av1_encoder.cpp
View File

@@ -310,7 +310,7 @@ int AomAv1Encoder::Encode(
frame_for_encode_->stride[AOM_PLANE_V] = 0;
VideoFrameType frame_type;
if (0 == seq_++ % 300) {
if (0 == seq_++ % key_frame_interval_) {
force_i_frame_flags_ = AOM_EFLAG_FORCE_KF;
frame_type = VideoFrameType::kVideoFrameKey;
} else {

8
src/media/video/encode/aom/aom_av1_encoder.h
View File

@@ -46,9 +46,9 @@ class AomAv1Encoder : public VideoEncoder {
int SetTargetBitrate(int bitrate);
int GetResolution(int& width, int& height) {
width = frame_width_;
height = frame_height_;
int GetResolution(int* width, int* height) {
*width = frame_width_;
*height = frame_height_;
return 0;
}
@@ -67,7 +67,7 @@ class AomAv1Encoder : public VideoEncoder {
private:
uint32_t frame_width_ = 1280;
uint32_t frame_height_ = 720;
int key_frame_interval_ = 300;
int key_frame_interval_ = I_FRAME_INTERVAL;
int target_bitrate_ = 1000;
int max_bitrate_ = 2500000;
int max_payload_size_ = 1400;

3
src/media/video/encode/nvcodec/nvidia_video_encoder.cpp
View File

@@ -151,7 +151,7 @@ int NvidiaVideoEncoder::Encode(
}
VideoFrameType frame_type;
if (0 == seq_++ % 300) {
if (0 == seq_++ % key_frame_interval_) {
ForceIdr();
frame_type = VideoFrameType::kVideoFrameKey;
} else {
@@ -263,6 +263,7 @@ int NvidiaVideoEncoder::ResetEncodeResolution(unsigned int width,
frame_width_ = width;
frame_height_ = height;
LOG_WARN("Reset resolution to [{}x{}]", frame_width_, frame_height_);
NV_ENC_RECONFIGURE_PARAMS reconfig_params = {NV_ENC_RECONFIGURE_PARAMS_VER};
NV_ENC_INITIALIZE_PARAMS init_params = {NV_ENC_INITIALIZE_PARAMS_VER};

7
src/media/video/encode/nvcodec/nvidia_video_encoder.h
View File

@@ -4,6 +4,7 @@
#include <functional>
#include "NvEncoderCuda.h"
#include "log.h"
#include "video_encoder.h"
class NvidiaVideoEncoder : public VideoEncoder {
@@ -22,9 +23,9 @@ class NvidiaVideoEncoder : public VideoEncoder {
int SetTargetBitrate(int bitrate);
int GetResolution(int& width, int& height) {
width = frame_width_;
height = frame_height_;
int GetResolution(int* width, int* height) {
*width = frame_width_;
*height = frame_height_;
return 0;
}

2
src/media/video/encode/openh264/openh264_encoder.cpp
View File

@@ -223,7 +223,7 @@ int OpenH264Encoder::Encode(
video_frame->height, yuv420p_frame_);
VideoFrameType frame_type;
if (0 == seq_++ % 300) {
if (0 == seq_++ % key_frame_interval_) {
ForceIdr();
frame_type = VideoFrameType::kVideoFrameKey;
} else {

8
src/media/video/encode/openh264/openh264_encoder.h
View File

@@ -33,9 +33,9 @@ class OpenH264Encoder : public VideoEncoder {
int SetTargetBitrate(int bitrate);
int GetResolution(int& width, int& height) {
width = frame_width_;
height = frame_height_;
int GetResolution(int* width, int* height) {
*width = frame_width_;
*height = frame_height_;
return 0;
}
@@ -50,7 +50,7 @@ class OpenH264Encoder : public VideoEncoder {
private:
uint32_t frame_width_ = 1280;
uint32_t frame_height_ = 720;
int key_frame_interval_ = 300;
int key_frame_interval_ = 3000;
int target_bitrate_ = 10000000;
int max_bitrate_ = 10000000;
int max_payload_size_ = 1400;

3
src/media/video/encode/video_encoder.h
View File

@@ -11,6 +11,7 @@
#include "video_frame_wrapper.h"
#include "x.h"
#define I_FRAME_INTERVAL 3000
class VideoEncoder {
public:
virtual int Init() = 0;
@@ -24,7 +25,7 @@ class VideoEncoder {
virtual int SetTargetBitrate(int bitrate) = 0;
virtual int GetResolution(int& width, int& height) = 0;
virtual int GetResolution(int* width, int* height) = 0;
virtual std::string GetEncoderName() = 0;

2
src/qos/bwe_defines.h
View File

@@ -22,7 +22,7 @@
namespace webrtc {
inline constexpr DataRate kCongestionControllerMinBitrate =
DataRate::BitsPerSec(500'000);
DataRate::BitsPerSec(5'000);
inline constexpr TimeDelta kBitrateWindow = TimeDelta::Seconds(1);
extern const char kBweTypeHistogram[];

8
src/qos/send_side_bandwidth_estimation.cc
View File

@@ -394,7 +394,7 @@ void SendSideBandwidthEstimation::UpdateEstimate(Timestamp at_time) {
// We only make decisions based on loss when the bitrate is above a
// threshold. This is a crude way of handling loss which is uncorrelated
// to congestion.
LOG_WARN("loss: [{}]", loss);
// LOG_WARN("loss: [{}]", loss);
if (current_target_ < bitrate_threshold_ || loss <= low_loss_threshold_) {
// Loss < 2%: Increase rate by 8% of the min bitrate in the last
// kBweIncreaseInterval.
@@ -407,13 +407,13 @@ void SendSideBandwidthEstimation::UpdateEstimate(Timestamp at_time) {
// it would take over one second since the lower packet loss to achieve
// 108kbps.
DataRate new_bitrate = DataRate::BitsPerSec(
min_bitrate_history_.front().second.bps() * 1.08 + 0.5);
min_bitrate_history_.front().second.bps() * 1.5 + 0.5);
// Add 1 kbps extra, just to make sure that we do not get stuck
// (gives a little extra increase at low rates, negligible at higher
// rates).
new_bitrate += DataRate::BitsPerSec(1000);
LOG_WARN("new_bitrate: [{}]", ToString(new_bitrate).c_str());
// LOG_WARN("new_bitrate: [{}]", ToString(new_bitrate).c_str());
UpdateTargetBitrate(new_bitrate, at_time);
return;
} else if (current_target_ > bitrate_threshold_) {
@@ -435,7 +435,7 @@ void SendSideBandwidthEstimation::UpdateEstimate(Timestamp at_time) {
static_cast<double>(512 - last_fraction_loss_)) /
512.0);
has_decreased_since_last_fraction_loss_ = true;
LOG_WARN("2 new_bitrate: [{}]", ToString(new_bitrate).c_str());
// LOG_WARN("new_bitrate: [{}]", ToString(new_bitrate).c_str());
UpdateTargetBitrate(new_bitrate, at_time);
return;
}

103
src/rtcp/fir.cpp Normal file
View File

@@ -0,0 +1,103 @@
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "fir.h"
#include "byte_io.h"
#include "log.h"
namespace webrtc {
namespace rtcp {
// RFC 4585: Feedback format.
// Common packet format:
//
// 0 1 2 3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |V=2|P| FMT | PT | length |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | SSRC of packet sender |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | SSRC of media source (unused) = 0 |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// : Feedback Control Information (FCI) :
// : :
// Full intra request (FIR) (RFC 5104).
// The Feedback Control Information (FCI) for the Full Intra Request
// consists of one or more FCI entries.
// FCI:
// 0 1 2 3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | SSRC |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | Seq nr. | Reserved = 0 |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
Fir::Fir() = default;
Fir::Fir(const Fir& fir) = default;
Fir::~Fir() = default;
bool Fir::Parse(const RtcpCommonHeader& packet) {
// The FCI field MUST contain one or more FIR entries.
if (packet.payload_size_bytes() < kCommonFeedbackLength + kFciLength) {
LOG_WARN("Packet is too small to be a valid FIR packet.");
return false;
}
if ((packet.payload_size_bytes() - kCommonFeedbackLength) % kFciLength != 0) {
LOG_WARN("Invalid size for a valid FIR packet.");
return false;
}
ParseCommonFeedback(packet.payload());
size_t number_of_fci_items =
(packet.payload_size_bytes() - kCommonFeedbackLength) / kFciLength;
const uint8_t* next_fci = packet.payload() + kCommonFeedbackLength;
items_.resize(number_of_fci_items);
for (Request& request : items_) {
request.ssrc = ByteReader<uint32_t>::ReadBigEndian(next_fci);
request.seq_nr = ByteReader<uint8_t>::ReadBigEndian(next_fci + 4);
next_fci += kFciLength;
}
return true;
}
size_t Fir::BlockLength() const {
return kHeaderLength + kCommonFeedbackLength + kFciLength * items_.size();
}
bool Fir::Create(uint8_t* packet, size_t* index, size_t max_length,
PacketReadyCallback callback) const {
while (*index + BlockLength() > max_length) {
if (!OnBufferFull(packet, index, callback)) return false;
}
size_t index_end = *index + BlockLength();
CreateHeader(kFeedbackMessageType, kPacketType, HeaderLength(), packet,
index);
CreateCommonFeedback(packet + *index);
*index += kCommonFeedbackLength;
constexpr uint32_t kReserved = 0;
for (const Request& request : items_) {
ByteWriter<uint32_t>::WriteBigEndian(packet + *index, request.ssrc);
ByteWriter<uint8_t>::WriteBigEndian(packet + *index + 4, request.seq_nr);
ByteWriter<uint32_t, 3>::WriteBigEndian(packet + *index + 5, kReserved);
*index += kFciLength;
}
return true;
}
} // namespace rtcp
} // namespace webrtc

60
src/rtcp/fir.h Normal file
View File

@@ -0,0 +1,60 @@
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_FIR_H_
#define MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_FIR_H_
#include <vector>
#include "rtcp_common_header.h"
#include "rtp_feedback.h"
namespace webrtc {
namespace rtcp {
// Full intra request (FIR) (RFC 5104).
class Fir : public RtpFeedback {
public:
static constexpr uint8_t kFeedbackMessageType = 4;
struct Request {
Request() : ssrc(0), seq_nr(0) {}
Request(uint32_t ssrc, uint8_t seq_nr) : ssrc(ssrc), seq_nr(seq_nr) {}
uint32_t ssrc;
uint8_t seq_nr;
};
Fir();
Fir(const Fir& fir);
~Fir() override;
// Parse assumes header is already parsed and validated.
bool Parse(const RtcpCommonHeader& packet);
void AddRequestTo(uint32_t ssrc, uint8_t seq_num) {
items_.emplace_back(ssrc, seq_num);
}
const std::vector<Request>& requests() const { return items_; }
size_t BlockLength() const override;
bool Create(uint8_t* packet, size_t* index, size_t max_length,
PacketReadyCallback callback) const override;
private:
static constexpr size_t kFciLength = 8;
// SSRC of media source is not used in FIR packet. Shadow base functions.
void SetMediaSsrc(uint32_t ssrc);
uint32_t media_ssrc() const;
std::vector<Request> items_;
};
} // namespace rtcp
} // namespace webrtc
#endif // MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_FIR_H_

1
src/rtcp/rtcp_packet/rtcp_packet_info.h
View File

@@ -15,6 +15,7 @@
#include <vector>
#include "congestion_control_feedback.h"
#include "fir.h"
#include "nack.h"
#include "report_block_data.h"

18
src/transport/ice_transport.cpp
View File

@@ -241,6 +241,9 @@ bool IceTransport::ParseRtcpPacket(const uint8_t *buffer, size_t size,
case webrtc::rtcp::Nack::kFeedbackMessageType:
valid = HandleNack(rtcp_block, rtcp_packet_info);
break;
case webrtc::rtcp::Fir::kFeedbackMessageType:
valid = HandleFir(rtcp_block, rtcp_packet_info);
break;
default:
break;
}
@@ -408,6 +411,21 @@ bool IceTransport::HandleNack(const RtcpCommonHeader &rtcp_block,
return true;
}
bool IceTransport::HandleFir(const RtcpCommonHeader &rtcp_block,
RtcpPacketInfo *rtcp_packet_info) {
webrtc::rtcp::Fir fir;
if (!fir.Parse(rtcp_block)) {
return false;
}
if (ice_transport_controller_) {
ice_transport_controller_->FullIntraRequest();
return true;
}
return false;
}
int IceTransport::DestroyIceTransmission() {
LOG_INFO("[{}->{}] Destroy ice transmission", user_id_, remote_user_id_);
is_closed_ = true;

3
src/transport/ice_transport.h
View File

@@ -146,6 +146,9 @@ class IceTransport {
bool HandleNack(const RtcpCommonHeader &rtcp_block,
RtcpPacketInfo *rtcp_packet_info);
bool HandleFir(const RtcpCommonHeader &rtcp_block,
RtcpPacketInfo *rtcp_packet_info);
private:
bool hardware_acceleration_ = false;
bool use_trickle_ice_ = true;

9
src/transport/ice_transport_controller.cpp
View File

@@ -462,13 +462,16 @@ void IceTransportController::PostUpdates(webrtc::NetworkControlUpdate update) {
if (target_bitrate != target_bitrate_) {
target_bitrate_ = target_bitrate;
int width, height, target_width, target_height;
video_encoder_->GetResolution(width, height);
video_encoder_->GetResolution(&width, &height);
if (0 == resolution_adapter_->GetResolution(target_bitrate_, width, height,
target_width, target_height)) {
&target_width,
&target_height)) {
if (target_width != target_width_ || target_height != target_height_) {
target_width_ = target_width;
target_height_ = target_height;
b_force_i_frame_ = true;
LOG_WARN("Set target resolution [{}x{}]", target_width_.value(),
target_height_.value());
}
@@ -479,7 +482,7 @@ void IceTransportController::PostUpdates(webrtc::NetworkControlUpdate update) {
source_height_);
}
video_encoder_->SetTargetBitrate(target_bitrate_);
// LOG_WARN("Set target bitrate [{}]bps", target_bitrate_);
LOG_WARN("Set target bitrate [{}]bps", target_bitrate_);
}
}

2
src/transport/ice_transport_controller.h
View File

@@ -53,6 +53,8 @@ class IceTransportController
int SendAudio(const char *data, size_t size);
int SendData(const char *data, size_t size);
void FullIntraRequest() { b_force_i_frame_ = true; }
int OnReceiveVideoRtpPacket(const char *data, size_t size);
int OnReceiveAudioRtpPacket(const char *data, size_t size);
int OnReceiveDataRtpPacket(const char *data, size_t size);