diff --git a/src/channel/rtp_channel/rtp_video_receiver.cpp b/src/channel/rtp_channel/rtp_video_receiver.cpp index f4d78d9..0d29923 100644 --- a/src/channel/rtp_channel/rtp_video_receiver.cpp +++ b/src/channel/rtp_channel/rtp_video_receiver.cpp @@ -2,6 +2,7 @@ #include "api/ntp/ntp_time_util.h" #include "common.h" +#include "fir.h" #include "log.h" #include "nack.h" #include "rtcp_sender.h" @@ -670,7 +671,15 @@ void RtpVideoReceiver::SendNack(const std::vector& nack_list, } } -void RtpVideoReceiver::RequestKeyFrame() {} +void RtpVideoReceiver::RequestKeyFrame() { + ++sequence_number_fir_; + webrtc::rtcp::Fir fir; + fir.SetSenderSsrc(ssrc_); + fir.AddRequestTo(remote_ssrc_, sequence_number_fir_); + + rtcp_sender_->AppendPacket(fir); + rtcp_sender_->Send(); +} void RtpVideoReceiver::SendLossNotification(uint16_t last_decoded_seq_num, uint16_t last_received_seq_num, diff --git a/src/channel/rtp_channel/rtp_video_receiver.h b/src/channel/rtp_channel/rtp_video_receiver.h index ba48bd9..213b5dd 100644 --- a/src/channel/rtp_channel/rtp_video_receiver.h +++ b/src/channel/rtp_channel/rtp_video_receiver.h @@ -155,6 +155,8 @@ class RtpVideoReceiver : public ThreadBase, uint32_t bytes_sent = 0; uint32_t reports_count = 0; + uint8_t sequence_number_fir_ = 0; + private: FILE* file_rtp_recv_ = nullptr; }; diff --git a/src/media/resolution_adapter/resolution_adapter.cpp b/src/media/resolution_adapter/resolution_adapter.cpp index 67ec5f7..f2ae930 100644 --- a/src/media/resolution_adapter/resolution_adapter.cpp +++ b/src/media/resolution_adapter/resolution_adapter.cpp @@ -1,25 +1,36 @@ #include "resolution_adapter.h" #include "libyuv.h" +#include "log.h" int ResolutionAdapter::GetResolution(int target_bitrate, int current_width, - int current_height, int& target_width, - int& target_height) { + int current_height, int* target_width, + int* target_height) { for (auto& resolution : GetBitrateLimits()) { if (target_bitrate >= resolution.min_start_bitrate_bps && target_bitrate <= resolution.max_bitrate_bps) { if (current_width * current_height <= resolution.frame_size_pixels) { - target_width = current_width; - target_height = current_height; + *target_width = current_width; + *target_height = current_height; + LOG_INFO("1 source resolution {}x{}, target resolution {}x{}", + current_width, current_height, *target_width, *target_height); return 0; } else { - target_width = current_width * 3 / 5; - target_height = current_height * 3 / 5; + *target_width = current_width * 3 / 5; + *target_height = current_height * 3 / 5; + LOG_INFO("2 source resolution {}x{}, target resolution {}x{}", + current_width, current_height, *target_width, *target_height); return 0; } } } + *target_width = -1; + *target_height = -1; + + LOG_INFO("3 source resolution {}x{}, target resolution {}x{}", current_width, + current_height, *target_width, *target_height); + return -1; } diff --git a/src/media/resolution_adapter/resolution_adapter.h b/src/media/resolution_adapter/resolution_adapter.h index 054e6ce..bd9c336 100644 --- a/src/media/resolution_adapter/resolution_adapter.h +++ b/src/media/resolution_adapter/resolution_adapter.h @@ -19,7 +19,7 @@ class ResolutionAdapter { public: int GetResolution(int target_bitrate, int current_width, int current_height, - int& target_width, int& target_height); + int* target_width, int* target_height); int ResolutionDowngrade(const XVideoFrame* video_frame, int target_width, int target_height, XVideoFrame* new_frame); diff --git a/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp b/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp index 96e8612..3dd6091 100644 --- a/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp +++ b/src/media/video/decode/nvcodec/nvidia_video_decoder.cpp @@ -75,7 +75,7 @@ int NvidiaVideoDecoder::Decode( #endif if ((*(data + 4) & 0x1f) == 0x07) { - // LOG_WARN("Receive key frame"); + LOG_WARN("Receive key frame"); } int num_frame_returned = decoder->Decode(data, (int)size); diff --git a/src/media/video/encode/aom/aom_av1_encoder.cpp b/src/media/video/encode/aom/aom_av1_encoder.cpp index 1f16ef3..d8400e9 100644 --- a/src/media/video/encode/aom/aom_av1_encoder.cpp +++ b/src/media/video/encode/aom/aom_av1_encoder.cpp @@ -310,7 +310,7 @@ int AomAv1Encoder::Encode( frame_for_encode_->stride[AOM_PLANE_V] = 0; VideoFrameType frame_type; - if (0 == seq_++ % 300) { + if (0 == seq_++ % key_frame_interval_) { force_i_frame_flags_ = AOM_EFLAG_FORCE_KF; frame_type = VideoFrameType::kVideoFrameKey; } else { diff --git a/src/media/video/encode/aom/aom_av1_encoder.h b/src/media/video/encode/aom/aom_av1_encoder.h index aac23d6..e165881 100644 --- a/src/media/video/encode/aom/aom_av1_encoder.h +++ b/src/media/video/encode/aom/aom_av1_encoder.h @@ -46,9 +46,9 @@ class AomAv1Encoder : public VideoEncoder { int SetTargetBitrate(int bitrate); - int GetResolution(int& width, int& height) { - width = frame_width_; - height = frame_height_; + int GetResolution(int* width, int* height) { + *width = frame_width_; + *height = frame_height_; return 0; } @@ -67,7 +67,7 @@ class AomAv1Encoder : public VideoEncoder { private: uint32_t frame_width_ = 1280; uint32_t frame_height_ = 720; - int key_frame_interval_ = 300; + int key_frame_interval_ = I_FRAME_INTERVAL; int target_bitrate_ = 1000; int max_bitrate_ = 2500000; int max_payload_size_ = 1400; diff --git a/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp b/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp index 95ec817..c7b4881 100644 --- a/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp +++ b/src/media/video/encode/nvcodec/nvidia_video_encoder.cpp @@ -151,7 +151,7 @@ int NvidiaVideoEncoder::Encode( } VideoFrameType frame_type; - if (0 == seq_++ % 300) { + if (0 == seq_++ % key_frame_interval_) { ForceIdr(); frame_type = VideoFrameType::kVideoFrameKey; } else { @@ -263,6 +263,7 @@ int NvidiaVideoEncoder::ResetEncodeResolution(unsigned int width, frame_width_ = width; frame_height_ = height; + LOG_WARN("Reset resolution to [{}x{}]", frame_width_, frame_height_); NV_ENC_RECONFIGURE_PARAMS reconfig_params = {NV_ENC_RECONFIGURE_PARAMS_VER}; NV_ENC_INITIALIZE_PARAMS init_params = {NV_ENC_INITIALIZE_PARAMS_VER}; diff --git a/src/media/video/encode/nvcodec/nvidia_video_encoder.h b/src/media/video/encode/nvcodec/nvidia_video_encoder.h index 4da0e23..788827c 100644 --- a/src/media/video/encode/nvcodec/nvidia_video_encoder.h +++ b/src/media/video/encode/nvcodec/nvidia_video_encoder.h @@ -4,6 +4,7 @@ #include #include "NvEncoderCuda.h" +#include "log.h" #include "video_encoder.h" class NvidiaVideoEncoder : public VideoEncoder { @@ -22,9 +23,9 @@ class NvidiaVideoEncoder : public VideoEncoder { int SetTargetBitrate(int bitrate); - int GetResolution(int& width, int& height) { - width = frame_width_; - height = frame_height_; + int GetResolution(int* width, int* height) { + *width = frame_width_; + *height = frame_height_; return 0; } diff --git a/src/media/video/encode/openh264/openh264_encoder.cpp b/src/media/video/encode/openh264/openh264_encoder.cpp index da10c13..a496fef 100644 --- a/src/media/video/encode/openh264/openh264_encoder.cpp +++ b/src/media/video/encode/openh264/openh264_encoder.cpp @@ -223,7 +223,7 @@ int OpenH264Encoder::Encode( video_frame->height, yuv420p_frame_); VideoFrameType frame_type; - if (0 == seq_++ % 300) { + if (0 == seq_++ % key_frame_interval_) { ForceIdr(); frame_type = VideoFrameType::kVideoFrameKey; } else { diff --git a/src/media/video/encode/openh264/openh264_encoder.h b/src/media/video/encode/openh264/openh264_encoder.h index 25aa2a3..fed5ec9 100644 --- a/src/media/video/encode/openh264/openh264_encoder.h +++ b/src/media/video/encode/openh264/openh264_encoder.h @@ -33,9 +33,9 @@ class OpenH264Encoder : public VideoEncoder { int SetTargetBitrate(int bitrate); - int GetResolution(int& width, int& height) { - width = frame_width_; - height = frame_height_; + int GetResolution(int* width, int* height) { + *width = frame_width_; + *height = frame_height_; return 0; } @@ -50,7 +50,7 @@ class OpenH264Encoder : public VideoEncoder { private: uint32_t frame_width_ = 1280; uint32_t frame_height_ = 720; - int key_frame_interval_ = 300; + int key_frame_interval_ = 3000; int target_bitrate_ = 10000000; int max_bitrate_ = 10000000; int max_payload_size_ = 1400; diff --git a/src/media/video/encode/video_encoder.h b/src/media/video/encode/video_encoder.h index 16312fa..66cbea2 100644 --- a/src/media/video/encode/video_encoder.h +++ b/src/media/video/encode/video_encoder.h @@ -11,6 +11,7 @@ #include "video_frame_wrapper.h" #include "x.h" +#define I_FRAME_INTERVAL 3000 class VideoEncoder { public: virtual int Init() = 0; @@ -24,7 +25,7 @@ class VideoEncoder { virtual int SetTargetBitrate(int bitrate) = 0; - virtual int GetResolution(int& width, int& height) = 0; + virtual int GetResolution(int* width, int* height) = 0; virtual std::string GetEncoderName() = 0; diff --git a/src/qos/bwe_defines.h b/src/qos/bwe_defines.h index 878115e..e3f4e8c 100644 --- a/src/qos/bwe_defines.h +++ b/src/qos/bwe_defines.h @@ -22,7 +22,7 @@ namespace webrtc { inline constexpr DataRate kCongestionControllerMinBitrate = - DataRate::BitsPerSec(500'000); + DataRate::BitsPerSec(5'000); inline constexpr TimeDelta kBitrateWindow = TimeDelta::Seconds(1); extern const char kBweTypeHistogram[]; diff --git a/src/qos/send_side_bandwidth_estimation.cc b/src/qos/send_side_bandwidth_estimation.cc index 9a5ba91..6275a4d 100644 --- a/src/qos/send_side_bandwidth_estimation.cc +++ b/src/qos/send_side_bandwidth_estimation.cc @@ -394,7 +394,7 @@ void SendSideBandwidthEstimation::UpdateEstimate(Timestamp at_time) { // We only make decisions based on loss when the bitrate is above a // threshold. This is a crude way of handling loss which is uncorrelated // to congestion. - LOG_WARN("loss: [{}]", loss); + // LOG_WARN("loss: [{}]", loss); if (current_target_ < bitrate_threshold_ || loss <= low_loss_threshold_) { // Loss < 2%: Increase rate by 8% of the min bitrate in the last // kBweIncreaseInterval. @@ -407,13 +407,13 @@ void SendSideBandwidthEstimation::UpdateEstimate(Timestamp at_time) { // it would take over one second since the lower packet loss to achieve // 108kbps. DataRate new_bitrate = DataRate::BitsPerSec( - min_bitrate_history_.front().second.bps() * 1.08 + 0.5); + min_bitrate_history_.front().second.bps() * 1.5 + 0.5); // Add 1 kbps extra, just to make sure that we do not get stuck // (gives a little extra increase at low rates, negligible at higher // rates). new_bitrate += DataRate::BitsPerSec(1000); - LOG_WARN("new_bitrate: [{}]", ToString(new_bitrate).c_str()); + // LOG_WARN("new_bitrate: [{}]", ToString(new_bitrate).c_str()); UpdateTargetBitrate(new_bitrate, at_time); return; } else if (current_target_ > bitrate_threshold_) { @@ -435,7 +435,7 @@ void SendSideBandwidthEstimation::UpdateEstimate(Timestamp at_time) { static_cast(512 - last_fraction_loss_)) / 512.0); has_decreased_since_last_fraction_loss_ = true; - LOG_WARN("2 new_bitrate: [{}]", ToString(new_bitrate).c_str()); + // LOG_WARN("new_bitrate: [{}]", ToString(new_bitrate).c_str()); UpdateTargetBitrate(new_bitrate, at_time); return; } diff --git a/src/rtcp/fir.cpp b/src/rtcp/fir.cpp new file mode 100644 index 0000000..53e7f4d --- /dev/null +++ b/src/rtcp/fir.cpp @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "fir.h" + +#include "byte_io.h" +#include "log.h" + +namespace webrtc { +namespace rtcp { +// RFC 4585: Feedback format. +// Common packet format: +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |V=2|P| FMT | PT | length | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | SSRC of packet sender | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | SSRC of media source (unused) = 0 | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// : Feedback Control Information (FCI) : +// : : +// Full intra request (FIR) (RFC 5104). +// The Feedback Control Information (FCI) for the Full Intra Request +// consists of one or more FCI entries. +// FCI: +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | SSRC | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Seq nr. | Reserved = 0 | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +Fir::Fir() = default; + +Fir::Fir(const Fir& fir) = default; + +Fir::~Fir() = default; + +bool Fir::Parse(const RtcpCommonHeader& packet) { + // The FCI field MUST contain one or more FIR entries. + if (packet.payload_size_bytes() < kCommonFeedbackLength + kFciLength) { + LOG_WARN("Packet is too small to be a valid FIR packet."); + return false; + } + + if ((packet.payload_size_bytes() - kCommonFeedbackLength) % kFciLength != 0) { + LOG_WARN("Invalid size for a valid FIR packet."); + return false; + } + + ParseCommonFeedback(packet.payload()); + + size_t number_of_fci_items = + (packet.payload_size_bytes() - kCommonFeedbackLength) / kFciLength; + const uint8_t* next_fci = packet.payload() + kCommonFeedbackLength; + items_.resize(number_of_fci_items); + for (Request& request : items_) { + request.ssrc = ByteReader::ReadBigEndian(next_fci); + request.seq_nr = ByteReader::ReadBigEndian(next_fci + 4); + next_fci += kFciLength; + } + return true; +} + +size_t Fir::BlockLength() const { + return kHeaderLength + kCommonFeedbackLength + kFciLength * items_.size(); +} + +bool Fir::Create(uint8_t* packet, size_t* index, size_t max_length, + PacketReadyCallback callback) const { + while (*index + BlockLength() > max_length) { + if (!OnBufferFull(packet, index, callback)) return false; + } + size_t index_end = *index + BlockLength(); + CreateHeader(kFeedbackMessageType, kPacketType, HeaderLength(), packet, + index); + + CreateCommonFeedback(packet + *index); + *index += kCommonFeedbackLength; + + constexpr uint32_t kReserved = 0; + for (const Request& request : items_) { + ByteWriter::WriteBigEndian(packet + *index, request.ssrc); + ByteWriter::WriteBigEndian(packet + *index + 4, request.seq_nr); + ByteWriter::WriteBigEndian(packet + *index + 5, kReserved); + *index += kFciLength; + } + + return true; +} +} // namespace rtcp +} // namespace webrtc diff --git a/src/rtcp/fir.h b/src/rtcp/fir.h new file mode 100644 index 0000000..0f3a8b5 --- /dev/null +++ b/src/rtcp/fir.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_FIR_H_ +#define MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_FIR_H_ + +#include + +#include "rtcp_common_header.h" +#include "rtp_feedback.h" + +namespace webrtc { +namespace rtcp { +// Full intra request (FIR) (RFC 5104). +class Fir : public RtpFeedback { + public: + static constexpr uint8_t kFeedbackMessageType = 4; + struct Request { + Request() : ssrc(0), seq_nr(0) {} + Request(uint32_t ssrc, uint8_t seq_nr) : ssrc(ssrc), seq_nr(seq_nr) {} + uint32_t ssrc; + uint8_t seq_nr; + }; + + Fir(); + Fir(const Fir& fir); + ~Fir() override; + + // Parse assumes header is already parsed and validated. + bool Parse(const RtcpCommonHeader& packet); + + void AddRequestTo(uint32_t ssrc, uint8_t seq_num) { + items_.emplace_back(ssrc, seq_num); + } + const std::vector& requests() const { return items_; } + + size_t BlockLength() const override; + + bool Create(uint8_t* packet, size_t* index, size_t max_length, + PacketReadyCallback callback) const override; + + private: + static constexpr size_t kFciLength = 8; + + // SSRC of media source is not used in FIR packet. Shadow base functions. + void SetMediaSsrc(uint32_t ssrc); + uint32_t media_ssrc() const; + + std::vector items_; +}; +} // namespace rtcp +} // namespace webrtc +#endif // MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_FIR_H_ diff --git a/src/rtcp/rtcp_packet/rtcp_packet_info.h b/src/rtcp/rtcp_packet/rtcp_packet_info.h index 8e784ed..0605136 100644 --- a/src/rtcp/rtcp_packet/rtcp_packet_info.h +++ b/src/rtcp/rtcp_packet/rtcp_packet_info.h @@ -15,6 +15,7 @@ #include #include "congestion_control_feedback.h" +#include "fir.h" #include "nack.h" #include "report_block_data.h" diff --git a/src/transport/ice_transport.cpp b/src/transport/ice_transport.cpp index b12a857..9fb50fb 100644 --- a/src/transport/ice_transport.cpp +++ b/src/transport/ice_transport.cpp @@ -241,6 +241,9 @@ bool IceTransport::ParseRtcpPacket(const uint8_t *buffer, size_t size, case webrtc::rtcp::Nack::kFeedbackMessageType: valid = HandleNack(rtcp_block, rtcp_packet_info); break; + case webrtc::rtcp::Fir::kFeedbackMessageType: + valid = HandleFir(rtcp_block, rtcp_packet_info); + break; default: break; } @@ -408,6 +411,21 @@ bool IceTransport::HandleNack(const RtcpCommonHeader &rtcp_block, return true; } +bool IceTransport::HandleFir(const RtcpCommonHeader &rtcp_block, + RtcpPacketInfo *rtcp_packet_info) { + webrtc::rtcp::Fir fir; + if (!fir.Parse(rtcp_block)) { + return false; + } + + if (ice_transport_controller_) { + ice_transport_controller_->FullIntraRequest(); + return true; + } + + return false; +} + int IceTransport::DestroyIceTransmission() { LOG_INFO("[{}->{}] Destroy ice transmission", user_id_, remote_user_id_); is_closed_ = true; diff --git a/src/transport/ice_transport.h b/src/transport/ice_transport.h index b2fea76..658b6db 100644 --- a/src/transport/ice_transport.h +++ b/src/transport/ice_transport.h @@ -146,6 +146,9 @@ class IceTransport { bool HandleNack(const RtcpCommonHeader &rtcp_block, RtcpPacketInfo *rtcp_packet_info); + bool HandleFir(const RtcpCommonHeader &rtcp_block, + RtcpPacketInfo *rtcp_packet_info); + private: bool hardware_acceleration_ = false; bool use_trickle_ice_ = true; diff --git a/src/transport/ice_transport_controller.cpp b/src/transport/ice_transport_controller.cpp index d07d436..1689768 100644 --- a/src/transport/ice_transport_controller.cpp +++ b/src/transport/ice_transport_controller.cpp @@ -462,13 +462,16 @@ void IceTransportController::PostUpdates(webrtc::NetworkControlUpdate update) { if (target_bitrate != target_bitrate_) { target_bitrate_ = target_bitrate; int width, height, target_width, target_height; - video_encoder_->GetResolution(width, height); + video_encoder_->GetResolution(&width, &height); if (0 == resolution_adapter_->GetResolution(target_bitrate_, width, height, - target_width, target_height)) { + &target_width, + &target_height)) { if (target_width != target_width_ || target_height != target_height_) { target_width_ = target_width; target_height_ = target_height; + + b_force_i_frame_ = true; LOG_WARN("Set target resolution [{}x{}]", target_width_.value(), target_height_.value()); } @@ -479,7 +482,7 @@ void IceTransportController::PostUpdates(webrtc::NetworkControlUpdate update) { source_height_); } video_encoder_->SetTargetBitrate(target_bitrate_); - // LOG_WARN("Set target bitrate [{}]bps", target_bitrate_); + LOG_WARN("Set target bitrate [{}]bps", target_bitrate_); } } diff --git a/src/transport/ice_transport_controller.h b/src/transport/ice_transport_controller.h index 86ddcf0..37613db 100644 --- a/src/transport/ice_transport_controller.h +++ b/src/transport/ice_transport_controller.h @@ -53,6 +53,8 @@ class IceTransportController int SendAudio(const char *data, size_t size); int SendData(const char *data, size_t size); + void FullIntraRequest() { b_force_i_frame_ = true; } + int OnReceiveVideoRtpPacket(const char *data, size_t size); int OnReceiveAudioRtpPacket(const char *data, size_t size); int OnReceiveDataRtpPacket(const char *data, size_t size);