[feat] support FIR

src/channel/rtp_channel/rtp_video_receiver.cpp

@@ -2,6 +2,7 @@
 
 #include "api/ntp/ntp_time_util.h"
 #include "common.h"
+#include "fir.h"
 #include "log.h"
 #include "nack.h"
 #include "rtcp_sender.h"
@@ -670,7 +671,15 @@ void RtpVideoReceiver::SendNack(const std::vector<uint16_t>& nack_list,
   }
 }
 
-void RtpVideoReceiver::RequestKeyFrame() {}
+void RtpVideoReceiver::RequestKeyFrame() {
+  ++sequence_number_fir_;
+  webrtc::rtcp::Fir fir;
+  fir.SetSenderSsrc(ssrc_);
+  fir.AddRequestTo(remote_ssrc_, sequence_number_fir_);
+
+  rtcp_sender_->AppendPacket(fir);
+  rtcp_sender_->Send();
+}
 
 void RtpVideoReceiver::SendLossNotification(uint16_t last_decoded_seq_num,
                                             uint16_t last_received_seq_num,

src/channel/rtp_channel/rtp_video_receiver.h

@@ -155,6 +155,8 @@ class RtpVideoReceiver : public ThreadBase,
   uint32_t bytes_sent = 0;
   uint32_t reports_count = 0;
 
+  uint8_t sequence_number_fir_ = 0;
+
  private:
   FILE* file_rtp_recv_ = nullptr;
 };

src/media/resolution_adapter/resolution_adapter.cpp

@@ -1,25 +1,36 @@
 #include "resolution_adapter.h"
 
 #include "libyuv.h"
+#include "log.h"
 
 int ResolutionAdapter::GetResolution(int target_bitrate, int current_width,
-                                     int current_height, int& target_width,
+                                     int current_height, int* target_width,
-                                     int& target_height) {
+                                     int* target_height) {
   for (auto& resolution : GetBitrateLimits()) {
     if (target_bitrate >= resolution.min_start_bitrate_bps &&
         target_bitrate <= resolution.max_bitrate_bps) {
       if (current_width * current_height <= resolution.frame_size_pixels) {
-        target_width = current_width;
+        *target_width = current_width;
-        target_height = current_height;
+        *target_height = current_height;
+        LOG_INFO("1 source resolution {}x{}, target resolution {}x{}",
+                 current_width, current_height, *target_width, *target_height);
         return 0;
       } else {
-        target_width = current_width * 3 / 5;
+        *target_width = current_width * 3 / 5;
-        target_height = current_height * 3 / 5;
+        *target_height = current_height * 3 / 5;
+        LOG_INFO("2 source resolution {}x{}, target resolution {}x{}",
+                 current_width, current_height, *target_width, *target_height);
         return 0;
       }
     }
   }
 
+  *target_width = -1;
+  *target_height = -1;
+
+  LOG_INFO("3 source resolution {}x{}, target resolution {}x{}", current_width,
+           current_height, *target_width, *target_height);
+
   return -1;
 }
 

src/media/resolution_adapter/resolution_adapter.h

@@ -19,7 +19,7 @@ class ResolutionAdapter {
 
  public:
   int GetResolution(int target_bitrate, int current_width, int current_height,
-                    int& target_width, int& target_height);
+                    int* target_width, int* target_height);
 
   int ResolutionDowngrade(const XVideoFrame* video_frame, int target_width,
                           int target_height, XVideoFrame* new_frame);

src/media/video/decode/nvcodec/nvidia_video_decoder.cpp

@@ -75,7 +75,7 @@ int NvidiaVideoDecoder::Decode(
 #endif
 
   if ((*(data + 4) & 0x1f) == 0x07) {
-    // LOG_WARN("Receive key frame");
+    LOG_WARN("Receive key frame");
   }
 
   int num_frame_returned = decoder->Decode(data, (int)size);

src/media/video/encode/aom/aom_av1_encoder.cpp

@@ -310,7 +310,7 @@ int AomAv1Encoder::Encode(
   frame_for_encode_->stride[AOM_PLANE_V] = 0;
 
   VideoFrameType frame_type;
-  if (0 == seq_++ % 300) {
+  if (0 == seq_++ % key_frame_interval_) {
     force_i_frame_flags_ = AOM_EFLAG_FORCE_KF;
     frame_type = VideoFrameType::kVideoFrameKey;
   } else {

src/media/video/encode/aom/aom_av1_encoder.h

@@ -46,9 +46,9 @@ class AomAv1Encoder : public VideoEncoder {
 
   int SetTargetBitrate(int bitrate);
 
-  int GetResolution(int& width, int& height) {
+  int GetResolution(int* width, int* height) {
-    width = frame_width_;
+    *width = frame_width_;
-    height = frame_height_;
+    *height = frame_height_;
     return 0;
   }
 
@@ -67,7 +67,7 @@ class AomAv1Encoder : public VideoEncoder {
  private:
   uint32_t frame_width_ = 1280;
   uint32_t frame_height_ = 720;
-  int key_frame_interval_ = 300;
+  int key_frame_interval_ = I_FRAME_INTERVAL;
   int target_bitrate_ = 1000;
   int max_bitrate_ = 2500000;
   int max_payload_size_ = 1400;

src/media/video/encode/nvcodec/nvidia_video_encoder.cpp

@@ -151,7 +151,7 @@ int NvidiaVideoEncoder::Encode(
   }
 
   VideoFrameType frame_type;
-  if (0 == seq_++ % 300) {
+  if (0 == seq_++ % key_frame_interval_) {
     ForceIdr();
     frame_type = VideoFrameType::kVideoFrameKey;
   } else {
@@ -263,6 +263,7 @@ int NvidiaVideoEncoder::ResetEncodeResolution(unsigned int width,
 
   frame_width_ = width;
   frame_height_ = height;
+  LOG_WARN("Reset resolution to [{}x{}]", frame_width_, frame_height_);
 
   NV_ENC_RECONFIGURE_PARAMS reconfig_params = {NV_ENC_RECONFIGURE_PARAMS_VER};
   NV_ENC_INITIALIZE_PARAMS init_params = {NV_ENC_INITIALIZE_PARAMS_VER};

src/media/video/encode/nvcodec/nvidia_video_encoder.h

@@ -4,6 +4,7 @@
 #include <functional>
 
 #include "NvEncoderCuda.h"
+#include "log.h"
 #include "video_encoder.h"
 
 class NvidiaVideoEncoder : public VideoEncoder {
@@ -22,9 +23,9 @@ class NvidiaVideoEncoder : public VideoEncoder {
 
   int SetTargetBitrate(int bitrate);
 
-  int GetResolution(int& width, int& height) {
+  int GetResolution(int* width, int* height) {
-    width = frame_width_;
+    *width = frame_width_;
-    height = frame_height_;
+    *height = frame_height_;
     return 0;
   }
 

src/media/video/encode/openh264/openh264_encoder.cpp

@@ -223,7 +223,7 @@ int OpenH264Encoder::Encode(
              video_frame->height, yuv420p_frame_);
 
   VideoFrameType frame_type;
-  if (0 == seq_++ % 300) {
+  if (0 == seq_++ % key_frame_interval_) {
     ForceIdr();
     frame_type = VideoFrameType::kVideoFrameKey;
   } else {

src/media/video/encode/openh264/openh264_encoder.h

@@ -33,9 +33,9 @@ class OpenH264Encoder : public VideoEncoder {
 
   int SetTargetBitrate(int bitrate);
 
-  int GetResolution(int& width, int& height) {
+  int GetResolution(int* width, int* height) {
-    width = frame_width_;
+    *width = frame_width_;
-    height = frame_height_;
+    *height = frame_height_;
     return 0;
   }
 
@@ -50,7 +50,7 @@ class OpenH264Encoder : public VideoEncoder {
  private:
   uint32_t frame_width_ = 1280;
   uint32_t frame_height_ = 720;
-  int key_frame_interval_ = 300;
+  int key_frame_interval_ = 3000;
   int target_bitrate_ = 10000000;
   int max_bitrate_ = 10000000;
   int max_payload_size_ = 1400;

src/media/video/encode/video_encoder.h

@@ -11,6 +11,7 @@
 #include "video_frame_wrapper.h"
 #include "x.h"
 
+#define I_FRAME_INTERVAL 3000
 class VideoEncoder {
  public:
   virtual int Init() = 0;
@@ -24,7 +25,7 @@ class VideoEncoder {
 
   virtual int SetTargetBitrate(int bitrate) = 0;
 
-  virtual int GetResolution(int& width, int& height) = 0;
+  virtual int GetResolution(int* width, int* height) = 0;
 
   virtual std::string GetEncoderName() = 0;
 

src/qos/bwe_defines.h

@@ -22,7 +22,7 @@
 namespace webrtc {
 
 inline constexpr DataRate kCongestionControllerMinBitrate =
-    DataRate::BitsPerSec(500'000);
+    DataRate::BitsPerSec(5'000);
 inline constexpr TimeDelta kBitrateWindow = TimeDelta::Seconds(1);
 
 extern const char kBweTypeHistogram[];

src/qos/send_side_bandwidth_estimation.cc

@@ -394,7 +394,7 @@ void SendSideBandwidthEstimation::UpdateEstimate(Timestamp at_time) {
     // We only make decisions based on loss when the bitrate is above a
     // threshold. This is a crude way of handling loss which is uncorrelated
     // to congestion.
-    LOG_WARN("loss: [{}]", loss);
+    // LOG_WARN("loss: [{}]", loss);
     if (current_target_ < bitrate_threshold_ || loss <= low_loss_threshold_) {
       // Loss < 2%: Increase rate by 8% of the min bitrate in the last
       // kBweIncreaseInterval.
@@ -407,13 +407,13 @@ void SendSideBandwidthEstimation::UpdateEstimate(Timestamp at_time) {
       //   it would take over one second since the lower packet loss to achieve
       //   108kbps.
       DataRate new_bitrate = DataRate::BitsPerSec(
-          min_bitrate_history_.front().second.bps() * 1.08 + 0.5);
+          min_bitrate_history_.front().second.bps() * 1.5 + 0.5);
 
       // Add 1 kbps extra, just to make sure that we do not get stuck
       // (gives a little extra increase at low rates, negligible at higher
       // rates).
       new_bitrate += DataRate::BitsPerSec(1000);
-      LOG_WARN("new_bitrate: [{}]", ToString(new_bitrate).c_str());
+      // LOG_WARN("new_bitrate: [{}]", ToString(new_bitrate).c_str());
       UpdateTargetBitrate(new_bitrate, at_time);
       return;
     } else if (current_target_ > bitrate_threshold_) {
@@ -435,7 +435,7 @@ void SendSideBandwidthEstimation::UpdateEstimate(Timestamp at_time) {
                static_cast<double>(512 - last_fraction_loss_)) /
               512.0);
           has_decreased_since_last_fraction_loss_ = true;
-          LOG_WARN("2 new_bitrate: [{}]", ToString(new_bitrate).c_str());
+          // LOG_WARN("new_bitrate: [{}]", ToString(new_bitrate).c_str());
           UpdateTargetBitrate(new_bitrate, at_time);
           return;
         }

src/rtcp/fir.cpp

@@ -0,0 +1,103 @@
+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "fir.h"
+
+#include "byte_io.h"
+#include "log.h"
+
+namespace webrtc {
+namespace rtcp {
+// RFC 4585: Feedback format.
+// Common packet format:
+//
+//   0                   1                   2                   3
+//   0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+//  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//  |V=2|P|   FMT   |       PT      |          length               |
+//  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//  |                  SSRC of packet sender                        |
+//  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//  |             SSRC of media source (unused) = 0                 |
+//  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//  :            Feedback Control Information (FCI)                 :
+//  :                                                               :
+// Full intra request (FIR) (RFC 5104).
+// The Feedback Control Information (FCI) for the Full Intra Request
+// consists of one or more FCI entries.
+// FCI:
+//   0                   1                   2                   3
+//   0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+//  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//  |                              SSRC                             |
+//  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+//  | Seq nr.       |    Reserved = 0                               |
+//  +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+Fir::Fir() = default;
+
+Fir::Fir(const Fir& fir) = default;
+
+Fir::~Fir() = default;
+
+bool Fir::Parse(const RtcpCommonHeader& packet) {
+  // The FCI field MUST contain one or more FIR entries.
+  if (packet.payload_size_bytes() < kCommonFeedbackLength + kFciLength) {
+    LOG_WARN("Packet is too small to be a valid FIR packet.");
+    return false;
+  }
+
+  if ((packet.payload_size_bytes() - kCommonFeedbackLength) % kFciLength != 0) {
+    LOG_WARN("Invalid size for a valid FIR packet.");
+    return false;
+  }
+
+  ParseCommonFeedback(packet.payload());
+
+  size_t number_of_fci_items =
+      (packet.payload_size_bytes() - kCommonFeedbackLength) / kFciLength;
+  const uint8_t* next_fci = packet.payload() + kCommonFeedbackLength;
+  items_.resize(number_of_fci_items);
+  for (Request& request : items_) {
+    request.ssrc = ByteReader<uint32_t>::ReadBigEndian(next_fci);
+    request.seq_nr = ByteReader<uint8_t>::ReadBigEndian(next_fci + 4);
+    next_fci += kFciLength;
+  }
+  return true;
+}
+
+size_t Fir::BlockLength() const {
+  return kHeaderLength + kCommonFeedbackLength + kFciLength * items_.size();
+}
+
+bool Fir::Create(uint8_t* packet, size_t* index, size_t max_length,
+                 PacketReadyCallback callback) const {
+  while (*index + BlockLength() > max_length) {
+    if (!OnBufferFull(packet, index, callback)) return false;
+  }
+  size_t index_end = *index + BlockLength();
+  CreateHeader(kFeedbackMessageType, kPacketType, HeaderLength(), packet,
+               index);
+
+  CreateCommonFeedback(packet + *index);
+  *index += kCommonFeedbackLength;
+
+  constexpr uint32_t kReserved = 0;
+  for (const Request& request : items_) {
+    ByteWriter<uint32_t>::WriteBigEndian(packet + *index, request.ssrc);
+    ByteWriter<uint8_t>::WriteBigEndian(packet + *index + 4, request.seq_nr);
+    ByteWriter<uint32_t, 3>::WriteBigEndian(packet + *index + 5, kReserved);
+    *index += kFciLength;
+  }
+
+  return true;
+}
+}  // namespace rtcp
+}  // namespace webrtc

src/rtcp/fir.h

@@ -0,0 +1,60 @@
+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_FIR_H_
+#define MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_FIR_H_
+
+#include <vector>
+
+#include "rtcp_common_header.h"
+#include "rtp_feedback.h"
+
+namespace webrtc {
+namespace rtcp {
+// Full intra request (FIR) (RFC 5104).
+class Fir : public RtpFeedback {
+ public:
+  static constexpr uint8_t kFeedbackMessageType = 4;
+  struct Request {
+    Request() : ssrc(0), seq_nr(0) {}
+    Request(uint32_t ssrc, uint8_t seq_nr) : ssrc(ssrc), seq_nr(seq_nr) {}
+    uint32_t ssrc;
+    uint8_t seq_nr;
+  };
+
+  Fir();
+  Fir(const Fir& fir);
+  ~Fir() override;
+
+  // Parse assumes header is already parsed and validated.
+  bool Parse(const RtcpCommonHeader& packet);
+
+  void AddRequestTo(uint32_t ssrc, uint8_t seq_num) {
+    items_.emplace_back(ssrc, seq_num);
+  }
+  const std::vector<Request>& requests() const { return items_; }
+
+  size_t BlockLength() const override;
+
+  bool Create(uint8_t* packet, size_t* index, size_t max_length,
+              PacketReadyCallback callback) const override;
+
+ private:
+  static constexpr size_t kFciLength = 8;
+
+  // SSRC of media source is not used in FIR packet. Shadow base functions.
+  void SetMediaSsrc(uint32_t ssrc);
+  uint32_t media_ssrc() const;
+
+  std::vector<Request> items_;
+};
+}  // namespace rtcp
+}  // namespace webrtc
+#endif  // MODULES_RTP_RTCP_SOURCE_RTCP_PACKET_FIR_H_

src/rtcp/rtcp_packet/rtcp_packet_info.h

@@ -15,6 +15,7 @@
 #include <vector>
 
 #include "congestion_control_feedback.h"
+#include "fir.h"
 #include "nack.h"
 #include "report_block_data.h"
 

src/transport/ice_transport.cpp

@@ -241,6 +241,9 @@ bool IceTransport::ParseRtcpPacket(const uint8_t *buffer, size_t size,
         case webrtc::rtcp::Nack::kFeedbackMessageType:
           valid = HandleNack(rtcp_block, rtcp_packet_info);
           break;
+        case webrtc::rtcp::Fir::kFeedbackMessageType:
+          valid = HandleFir(rtcp_block, rtcp_packet_info);
+          break;
         default:
           break;
       }
@@ -408,6 +411,21 @@ bool IceTransport::HandleNack(const RtcpCommonHeader &rtcp_block,
   return true;
 }
 
+bool IceTransport::HandleFir(const RtcpCommonHeader &rtcp_block,
+                             RtcpPacketInfo *rtcp_packet_info) {
+  webrtc::rtcp::Fir fir;
+  if (!fir.Parse(rtcp_block)) {
+    return false;
+  }
+
+  if (ice_transport_controller_) {
+    ice_transport_controller_->FullIntraRequest();
+    return true;
+  }
+
+  return false;
+}
+
 int IceTransport::DestroyIceTransmission() {
   LOG_INFO("[{}->{}] Destroy ice transmission", user_id_, remote_user_id_);
   is_closed_ = true;

src/transport/ice_transport.h

@@ -146,6 +146,9 @@ class IceTransport {
   bool HandleNack(const RtcpCommonHeader &rtcp_block,
                   RtcpPacketInfo *rtcp_packet_info);
 
+  bool HandleFir(const RtcpCommonHeader &rtcp_block,
+                 RtcpPacketInfo *rtcp_packet_info);
+
  private:
   bool hardware_acceleration_ = false;
   bool use_trickle_ice_ = true;

src/transport/ice_transport_controller.cpp

@@ -462,13 +462,16 @@ void IceTransportController::PostUpdates(webrtc::NetworkControlUpdate update) {
   if (target_bitrate != target_bitrate_) {
     target_bitrate_ = target_bitrate;
     int width, height, target_width, target_height;
-    video_encoder_->GetResolution(width, height);
+    video_encoder_->GetResolution(&width, &height);
 
     if (0 == resolution_adapter_->GetResolution(target_bitrate_, width, height,
-                                                target_width, target_height)) {
+                                                &target_width,
+                                                &target_height)) {
       if (target_width != target_width_ || target_height != target_height_) {
         target_width_ = target_width;
         target_height_ = target_height;
+
+        b_force_i_frame_ = true;
         LOG_WARN("Set target resolution [{}x{}]", target_width_.value(),
                  target_height_.value());
       }
@@ -479,7 +482,7 @@ void IceTransportController::PostUpdates(webrtc::NetworkControlUpdate update) {
                source_height_);
     }
     video_encoder_->SetTargetBitrate(target_bitrate_);
-    // LOG_WARN("Set target bitrate [{}]bps", target_bitrate_);
+    LOG_WARN("Set target bitrate [{}]bps", target_bitrate_);
   }
 }
 

src/transport/ice_transport_controller.h

@@ -53,6 +53,8 @@ class IceTransportController
   int SendAudio(const char *data, size_t size);
   int SendData(const char *data, size_t size);
 
+  void FullIntraRequest() { b_force_i_frame_ = true; }
+
   int OnReceiveVideoRtpPacket(const char *data, size_t size);
   int OnReceiveAudioRtpPacket(const char *data, size_t size);
   int OnReceiveDataRtpPacket(const char *data, size_t size);