Implementation for rtcp receiver report

src/media/video/encode/nvcodec/nv_encoder.cpp

@@ -100,7 +100,6 @@ int VideoEncoder::Encode(const uint8_t *pData, int nSize) {
   encoder_->EncodeFrame(encoded_packets_);
 
   if (encoded_packets_.size() < 1) {
-    LOG_WARN("empty encoded_packets_");
     return -1;
   }
 

src/pc/peer_connection.cpp

@@ -266,7 +266,7 @@ int PeerConnection::SendVideoData(const char *data, size_t size) {
 
 int PeerConnection::OnEncodedImage(char *encoded_packets, size_t size) {
   for (auto &ice_trans : ice_transmission_list_) {
-    LOG_ERROR("H264 frame size: [{}]", size);
+    // LOG_ERROR("H264 frame size: [{}]", size);
     ice_trans.second->SendData(encoded_packets, size);
   }
 

src/rtcp/rtcp_header.h

@@ -12,8 +12,7 @@
 // |V=2|P|   RC    |   PT=SR=200   |            length             |
 // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
-#define DEFAULT_RTCP_VERSION 2
+#include "rtcp_typedef.h"
-#define DEFAULT_RTCP_HEADER_SIZE 4
 
 class RtcpHeader {
  public:

src/rtcp/rtcp_packet.h

@@ -1,16 +0,0 @@
-#ifndef _RTCP_PACKET_H_
-#define _RTCP_PACKET_H_
-
-class RtcpPacket {
- public:
-  typedef enum {
-    UNKNOWN = 0,
-    SR = 200,
-    RR = 201,
-    SDES = 202,
-    BYE = 203,
-    APP = 204
-  } RTCP_TYPE;
-};
-
-#endif

src/rtcp/rtcp_receiver_report.cpp

@@ -1,5 +1,31 @@
 #include "rtcp_receiver_report.h"
 
-RtcpReceiverReport::RtcpReceiverReport() {}
+RtcpReceiverReport::RtcpReceiverReport() {
+  buffer_ = new uint8_t[DEFAULT_RR_SIZE];
+  size_ = DEFAULT_RR_SIZE;
+}
 
-RtcpReceiverReport::~RtcpReceiverReport() {}
+RtcpReceiverReport::~RtcpReceiverReport() {
+  if (buffer_) {
+    delete buffer_;
+    buffer_ = nullptr;
+  }
+
+  size_ = 0;
+}
+
+void RtcpReceiverReport::SetReportBlock(RtcpReportBlock &rtcp_report_block) {
+  reports_.push_back(rtcp_report_block);
+}
+
+void RtcpReceiverReport::SetReportBlock(
+    std::vector<RtcpReportBlock> &rtcp_report_blocks) {
+  reports_ = rtcp_report_blocks;
+}
+
+const uint8_t *RtcpReceiverReport::Encode() {
+  int pos = rtcp_header_.Encode(DEFAULT_RTCP_VERSION, 0, DEFAULT_RR_BLOCK_NUM,
+                                RTCP_TYPE::RR, DEFAULT_RR_SIZE, buffer_);
+
+  return buffer_;
+}

src/rtcp/rtcp_receiver_report.h

@@ -31,9 +31,7 @@
 #include <vector>
 
 #include "rtcp_header.h"
-#include "rtcp_report_block.h"
+#include "rtcp_typedef.h"
-
-#define DEFAULT_SR_SIZE 32
 
 class RtcpReceiverReport {
  public:
@@ -41,12 +39,24 @@ class RtcpReceiverReport {
   ~RtcpReceiverReport();
 
  public:
-  const uint8_t *Encode(uint8_t *payload, size_t payload_size);
+  void SetReportBlock(RtcpReportBlock &rtcp_report_block);
-  size_t Decode(uint8_t *payload);
+  void SetReportBlock(std::vector<RtcpReportBlock> &rtcp_report_blocks);
+
+ public:
+  const uint8_t *Encode();
+  size_t Decode();
+
+  // Entire RTP buffer
+  const uint8_t *Buffer() const { return buffer_; }
+  size_t Size() const { return size_; }
 
  private:
   RtcpHeader rtcp_header_;
   std::vector<RtcpReportBlock> reports_;
+
+  // Entire RTCP buffer
+  uint8_t *buffer_ = nullptr;
+  size_t size_ = 0;
 };
 
 #endif

src/rtcp/rtcp_report_block.cpp

@@ -1,5 +0,0 @@
-#include "rtcp_report_block.h"
-
-RtcpReportBlock::RtcpReportBlock() {}
-
-RtcpReportBlock::~RtcpReportBlock() {}

src/rtcp/rtcp_report_block.h

@@ -1,53 +0,0 @@
-#ifndef _RTCP_REPORT_BLOCK_H_
-#define _RTCP_REPORT_BLOCK_H_
-
-// Report block 1
-//  0                   1                   2                   3
-//  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-// |                 SSRC_1 (SSRC of first source)                 |
-// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-// | fraction lost |       cumulative number of packets lost       |
-// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-// |           extended highest sequence number received           |
-// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-// |                      interarrival jitter                      |
-// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-// |                         last SR (LSR)                         |
-// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-// |                   delay since last SR (DLSR)                  |
-// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-#include <stdint.h>
-
-class RtcpReportBlock {
- public:
-  RtcpReportBlock();
-  ~RtcpReportBlock();
-
- public:
-  typedef struct {
-    uint32_t source_ssrc : 32;
-    uint8_t fraction_lost : 8;
-    uint32_t cumulative_lost : 24;
-    uint32_t extended_high_seq_num : 32;
-    uint32_t jitter : 32;
-    uint32_t lsr : 32;
-    uint32_t dlsr : 32;
-  } REPORT;
-
-  void SetReport(REPORT &report) {
-    report_.source_ssrc = report.source_ssrc;
-    report_.fraction_lost = report.fraction_lost;
-    report_.cumulative_lost = report.cumulative_lost;
-    report_.extended_high_seq_num = report.extended_high_seq_num;
-    report_.jitter = report.jitter;
-    report_.lsr = report.lsr;
-    report_.dlsr = report.dlsr;
-  }
-
- private:
-  REPORT report_;
-};
-
-#endif

src/rtcp/rtcp_sender_report.cpp

@@ -14,24 +14,41 @@ RtcpSenderReport::~RtcpSenderReport() {
   size_ = 0;
 }
 
+void RtcpSenderReport::SetSenderInfo(SenderInfo &sender_info) {
+  sender_info_.sender_ssrc = sender_info.sender_ssrc;
+  sender_info_.ntp_ts_msw = sender_info.ntp_ts_msw;
+  sender_info_.ntp_ts_lsw = sender_info.ntp_ts_lsw;
+  sender_info_.rtp_ts = sender_info.rtp_ts;
+  sender_info_.sender_packet_count = sender_info.sender_packet_count;
+  sender_info_.sender_octet_count = sender_info.sender_octet_count;
+}
+
+void RtcpSenderReport::SetReportBlock(RtcpReportBlock &rtcp_report_block) {
+  reports_.push_back(rtcp_report_block);
+}
+
+void RtcpSenderReport::SetReportBlock(
+    std::vector<RtcpReportBlock> &rtcp_report_blocks) {
+  reports_ = rtcp_report_blocks;
+}
+
 const uint8_t *RtcpSenderReport::Encode() {
-  int pos =
+  int pos = rtcp_header_.Encode(DEFAULT_RTCP_VERSION, 0, DEFAULT_SR_BLOCK_NUM,
-      rtcp_header_.Encode(DEFAULT_RTCP_VERSION, 0, DEFAULT_SR_BLOCK_NUM,
+                                RTCP_TYPE::SR, DEFAULT_SR_SIZE, buffer_);
-                          RtcpPacket::RTCP_TYPE::SR, DEFAULT_SR_SIZE, buffer_);
 
   buffer_[pos] = sender_info_.sender_ssrc >> 24 & 0xFF;
   buffer_[pos + 1] = sender_info_.sender_ssrc >> 16 & 0xFF;
   buffer_[pos + 2] = sender_info_.sender_ssrc >> 8 & 0xFF;
   buffer_[pos + 3] = sender_info_.sender_ssrc & 0xFF;
 
-  buffer_[pos + 4] = sender_info_.ntp_ts >> 56 & 0xFF;
+  buffer_[pos + 4] = sender_info_.ntp_ts_msw >> 56 & 0xFF;
-  buffer_[pos + 5] = sender_info_.ntp_ts >> 48 & 0xFF;
+  buffer_[pos + 5] = sender_info_.ntp_ts_msw >> 48 & 0xFF;
-  buffer_[pos + 6] = sender_info_.ntp_ts >> 40 & 0xFF;
+  buffer_[pos + 6] = sender_info_.ntp_ts_msw >> 40 & 0xFF;
-  buffer_[pos + 7] = sender_info_.ntp_ts >> 32 & 0xFF;
+  buffer_[pos + 7] = sender_info_.ntp_ts_msw >> 32 & 0xFF;
-  buffer_[pos + 8] = sender_info_.ntp_ts >> 24 & 0xFF;
+  buffer_[pos + 8] = sender_info_.ntp_ts_lsw >> 24 & 0xFF;
-  buffer_[pos + 9] = sender_info_.ntp_ts >> 16 & 0xFF;
+  buffer_[pos + 9] = sender_info_.ntp_ts_lsw >> 16 & 0xFF;
-  buffer_[pos + 10] = sender_info_.ntp_ts >> 8 & 0xFF;
+  buffer_[pos + 10] = sender_info_.ntp_ts_lsw >> 8 & 0xFF;
-  buffer_[pos + 11] = sender_info_.ntp_ts & 0xFF;
+  buffer_[pos + 11] = sender_info_.ntp_ts_lsw & 0xFF;
 
   buffer_[pos + 12] = sender_info_.rtp_ts >> 24 & 0xFF;
   buffer_[pos + 13] = sender_info_.rtp_ts >> 16 & 0xFF;

src/rtcp/rtcp_sender_report.h

@@ -39,11 +39,7 @@
 #include <vector>
 
 #include "rtcp_header.h"
-#include "rtcp_packet.h"
+#include "rtcp_typedef.h"
-#include "rtcp_report_block.h"
-
-#define DEFAULT_SR_BLOCK_NUM 1
-#define DEFAULT_SR_SIZE 52
 
 class RtcpSenderReport {
  public:
@@ -51,21 +47,9 @@ class RtcpSenderReport {
   ~RtcpSenderReport();
 
  public:
-  typedef struct {
+  void SetSenderInfo(SenderInfo &sender_info);
-    uint32_t sender_ssrc : 32;
+  void SetReportBlock(RtcpReportBlock &rtcp_report_block);
-    uint64_t ntp_ts : 64;
+  void SetReportBlock(std::vector<RtcpReportBlock> &rtcp_report_blocks);
-    uint32_t rtp_ts : 32;
-    uint32_t sender_packet_count : 32;
-    uint32_t sender_octet_count : 32;
-  } SENDER_INFO;
-
-  void SetSenderInfo(SENDER_INFO &sender_info) {
-    sender_info_.sender_ssrc = sender_info.sender_ssrc;
-    sender_info_.ntp_ts = sender_info.ntp_ts;
-    sender_info_.rtp_ts = sender_info.rtp_ts;
-    sender_info_.sender_packet_count = sender_info.sender_packet_count;
-    sender_info_.sender_octet_count = sender_info.sender_octet_count;
-  }
 
  public:
   const uint8_t *Encode();
@@ -77,7 +61,7 @@ class RtcpSenderReport {
 
  private:
   RtcpHeader rtcp_header_;
-  SENDER_INFO sender_info_;
+  SenderInfo sender_info_;
   std::vector<RtcpReportBlock> reports_;
 
   // Entire RTCP buffer

src/rtcp/rtcp_typedef.h

@@ -0,0 +1,42 @@
+#ifndef _RTCP_TYPEDEF_H_
+#define _RTCP_TYPEDEF_H_
+
+#include <stdint.h>
+
+#define DEFAULT_RTCP_VERSION 2
+#define DEFAULT_RTCP_HEADER_SIZE 4
+
+#define DEFAULT_SR_BLOCK_NUM 1
+#define DEFAULT_SR_SIZE 52
+#define DEFAULT_RR_BLOCK_NUM 1
+#define DEFAULT_RR_SIZE 32
+
+typedef enum {
+  UNKNOWN = 0,
+  SR = 200,
+  RR = 201,
+  SDES = 202,
+  BYE = 203,
+  APP = 204
+} RTCP_TYPE;
+
+typedef struct {
+  uint32_t sender_ssrc : 32;
+  uint64_t ntp_ts_msw : 64;
+  uint64_t ntp_ts_lsw : 64;
+  uint32_t rtp_ts : 32;
+  uint32_t sender_packet_count : 32;
+  uint32_t sender_octet_count : 32;
+} SenderInfo;
+
+typedef struct {
+  uint32_t source_ssrc : 32;
+  uint8_t fraction_lost : 8;
+  uint32_t cumulative_lost : 24;
+  uint32_t extended_high_seq_num : 32;
+  uint32_t jitter : 32;
+  uint32_t lsr : 32;
+  uint32_t dlsr : 32;
+} RtcpReportBlock;
+
+#endif

src/rtp/rtp_video_receiver.cpp

@@ -3,6 +3,7 @@
 #include "log.h"
 
 #define NV12_BUFFER_SIZE (1280 * 720 * 3 / 2)
+#define RTCP_RR_INTERVAL 1000
 
 RtpVideoReceiver::RtpVideoReceiver() {}
 
@@ -19,6 +20,34 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) {
     rtp_video_receive_statistics_->UpdateReceiveBytes(rtp_packet.Size());
   }
 
+  if (CheckIsTimeSendRR()) {
+    RtcpReceiverReport rtcp_rr;
+    RtcpReportBlock report;
+
+    auto duration = std::chrono::system_clock::now().time_since_epoch();
+    auto seconds = std::chrono::duration_cast<std::chrono::seconds>(duration);
+    uint32_t seconds_u32 = static_cast<uint32_t>(
+        std::chrono::duration_cast<std::chrono::seconds>(duration).count());
+
+    uint32_t fraction_u32 = static_cast<uint32_t>(
+        std::chrono::duration_cast<std::chrono::nanoseconds>(duration - seconds)
+            .count());
+
+    report.source_ssrc = 0x00;
+    report.fraction_lost = 0;
+    report.cumulative_lost = 0;
+    report.extended_high_seq_num = 0;
+    report.jitter = 0;
+    report.lsr = 0;
+    report.dlsr = 0;
+
+    rtcp_rr.SetReportBlock(report);
+
+    rtcp_rr.Encode();
+
+    SendRtcpRR(rtcp_rr);
+  }
+
   if (RtpPacket::NAL_UNIT_TYPE::NALU == rtp_packet.NalUnitType()) {
     compelete_video_frame_queue_.push(
         VideoFrame(rtp_packet.Payload(), rtp_packet.Size()));
@@ -117,3 +146,38 @@ bool RtpVideoReceiver::Process() {
   std::this_thread::sleep_for(std::chrono::milliseconds(13));
   return true;
 }
+
+void RtpVideoReceiver::SetUdpSender(
+    std::function<int(const char*, size_t)> udp_sender) {
+  udp_sender_ = udp_sender;
+}
+
+int RtpVideoReceiver::SendRtcpRR(RtcpReceiverReport& rtcp_rr) {
+  if (!udp_sender_) {
+    LOG_ERROR("udp_sender_ is nullptr");
+    return -1;
+  }
+
+  if (udp_sender_((const char*)rtcp_rr.Buffer(), rtcp_rr.Size())) {
+    LOG_ERROR("Send RR failed");
+    return -1;
+  }
+
+  LOG_ERROR("Send RR");
+
+  return 0;
+}
+
+bool RtpVideoReceiver::CheckIsTimeSendRR() {
+  uint32_t now_ts = static_cast<uint32_t>(
+      std::chrono::duration_cast<std::chrono::milliseconds>(
+          std::chrono::high_resolution_clock::now().time_since_epoch())
+          .count());
+
+  if (now_ts - last_send_rtcp_rr_packet_ts_ >= RTCP_RR_INTERVAL) {
+    last_send_rtcp_rr_packet_ts_ = now_ts;
+    return true;
+  } else {
+    return false;
+  }
+}

src/rtp/rtp_video_receiver.h

@@ -7,6 +7,7 @@
 
 #include "frame.h"
 #include "ringbuffer.h"
+#include "rtcp_receiver_report.h"
 #include "rtp_codec.h"
 #include "rtp_video_receive_statistics.h"
 #include "thread_base.h"
@@ -19,6 +20,9 @@ class RtpVideoReceiver : public ThreadBase {
  public:
   void InsertRtpPacket(RtpPacket& rtp_packet);
 
+  void SetUdpSender(
+      std::function<int(const char*, size_t)> rtp_packet_send_func);
+
   void SetOnReceiveCompleteFrame(
       std::function<void(VideoFrame&)> on_receive_complete_frame) {
     on_receive_complete_frame_ = on_receive_complete_frame;
@@ -26,7 +30,8 @@ class RtpVideoReceiver : public ThreadBase {
 
  private:
   bool CheckIsFrameCompleted(RtpPacket& rtp_packet);
-  //   void OnReceiveFrame(uint8_t* payload) {}
+  bool CheckIsTimeSendRR();
+  int SendRtcpRR(RtcpReceiverReport& rtcp_rr);
 
  private:
   bool Process() override;
@@ -41,6 +46,8 @@ class RtpVideoReceiver : public ThreadBase {
  private:
   std::unique_ptr<RtpVideoReceiveStatistics> rtp_video_receive_statistics_ =
       nullptr;
+  uint32_t last_send_rtcp_rr_packet_ts_ = 0;
+  std::function<int(const char*, size_t)> udp_sender_ = nullptr;
 };
 
 #endif

src/rtp/rtp_video_sender.cpp

@@ -4,7 +4,7 @@
 
 #include "log.h"
 
-#define RTCP_INTERVAL 1000
+#define RTCP_SR_INTERVAL 1000
 
 RtpVideoSender::RtpVideoSender() {}
 
@@ -45,7 +45,8 @@ int RtpVideoSender::SendRtpPacket(RtpPacket& rtp_packet) {
 
   if (CheckIsTimeSendSR()) {
     RtcpSenderReport rtcp_sr;
-    RtcpSenderReport::SENDER_INFO sender_info;
+    SenderInfo sender_info;
+    RtcpReportBlock report;
 
     auto duration = std::chrono::system_clock::now().time_since_epoch();
     auto seconds = std::chrono::duration_cast<std::chrono::seconds>(duration);
@@ -57,7 +58,8 @@ int RtpVideoSender::SendRtpPacket(RtpPacket& rtp_packet) {
             .count());
 
     sender_info.sender_ssrc = 0x00;
-    sender_info.ntp_ts = (uint64_t)seconds_u32 << 32 | (uint64_t)fraction_u32;
+    sender_info.ntp_ts_msw = (uint32_t)seconds_u32;
+    sender_info.ntp_ts_lsw = (uint32_t)fraction_u32;
     sender_info.rtp_ts =
         std::chrono::high_resolution_clock::now().time_since_epoch().count() *
         1000000;
@@ -66,6 +68,16 @@ int RtpVideoSender::SendRtpPacket(RtpPacket& rtp_packet) {
 
     rtcp_sr.SetSenderInfo(sender_info);
 
+    report.source_ssrc = 0x00;
+    report.fraction_lost = 0;
+    report.cumulative_lost = 0;
+    report.extended_high_seq_num = 0;
+    report.jitter = 0;
+    report.lsr = 0;
+    report.dlsr = 0;
+
+    rtcp_sr.SetReportBlock(report);
+
     rtcp_sr.Encode();
 
     SendRtcpSR(rtcp_sr);
@@ -96,8 +108,8 @@ bool RtpVideoSender::CheckIsTimeSendSR() {
           std::chrono::high_resolution_clock::now().time_since_epoch())
           .count());
 
-  if (now_ts - last_send_rtcp_packet_ts_ >= RTCP_INTERVAL) {
+  if (now_ts - last_send_rtcp_sr_packet_ts_ >= RTCP_SR_INTERVAL) {
-    last_send_rtcp_packet_ts_ = now_ts;
+    last_send_rtcp_sr_packet_ts_ = now_ts;
     return true;
   } else {
     return false;

src/rtp/rtp_video_sender.h

@@ -4,7 +4,6 @@
 #include <functional>
 
 #include "ringbuffer.h"
-#include "rtcp_packet.h"
 #include "rtcp_sender_report.h"
 #include "rtp_packet.h"
 #include "rtp_video_send_statistics.h"
@@ -38,7 +37,7 @@ class RtpVideoSender : public ThreadBase {
  private:
   std::unique_ptr<RtpVideoSendStatistics> rtp_video_send_statistics_ = nullptr;
   uint32_t last_send_bytes_ = 0;
-  uint32_t last_send_rtcp_packet_ts_ = 0;
+  uint32_t last_send_rtcp_sr_packet_ts_ = 0;
   uint32_t total_rtp_packets_sent_ = 0;
   uint32_t total_rtp_payload_sent_ = 0;
 };

src/transmission/ice_transmission.cpp

@@ -49,9 +49,18 @@ IceTransmission::~IceTransmission() {
 int IceTransmission::InitIceTransmission(std::string &ip, int port) {
   rtp_codec_ = std::make_unique<RtpCodec>(RtpPacket::PAYLOAD_TYPE::H264);
   rtp_video_receiver_ = std::make_unique<RtpVideoReceiver>();
+  rtp_video_receiver_->SetUdpSender(
+      [this](const char *data, size_t size) -> int {
+        if (!ice_agent_) {
+          LOG_ERROR("ice_agent_ is nullptr");
+          return -1;
+        }
+
+        return ice_agent_->Send(data, size);
+      });
   rtp_video_receiver_->SetOnReceiveCompleteFrame(
       [this](VideoFrame &video_frame) -> void {
-        LOG_ERROR("OnReceiveCompleteFrame {}", video_frame.Size());
+        // LOG_ERROR("OnReceiveCompleteFrame {}", video_frame.Size());
         on_receive_ice_msg_cb_((const char *)video_frame.Buffer(),
                                video_frame.Size(), remote_user_id_.data(),
                                remote_user_id_.size());