diff --git a/src/rtp/rtp_codec.cpp b/src/rtp/rtp_codec.cpp index cefd106..7f6ecc2 100644 --- a/src/rtp/rtp_codec.cpp +++ b/src/rtp/rtp_codec.cpp @@ -49,50 +49,82 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size, fec_encoder_.GetFecPacketsParams(size, num_of_total_packets, num_of_source_packets, last_packet_size); + timestamp_ = + std::chrono::high_resolution_clock::now().time_since_epoch().count(); + for (size_t index = 0; index < num_of_total_packets; index++) { - RtpPacket rtp_packet; - rtp_packet.SetVerion(version_); - rtp_packet.SetHasPadding(has_padding_); - rtp_packet.SetHasExtension(has_extension_); - rtp_packet.SetMarker(index == num_of_total_packets ? 1 : 0); - rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_)); - rtp_packet.SetSequenceNumber(sequence_number_++); + if (index < num_of_source_packets) { + RtpPacket rtp_packet; + rtp_packet.SetVerion(version_); + rtp_packet.SetHasPadding(has_padding_); + rtp_packet.SetHasExtension(has_extension_); + rtp_packet.SetMarker(index == num_of_source_packets ? 1 : 0); + rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE::H264); + rtp_packet.SetSequenceNumber(sequence_number_++); + rtp_packet.SetTimestamp(timestamp_); + rtp_packet.SetSsrc(ssrc_); - timestamp_ = std::chrono::high_resolution_clock::now() - .time_since_epoch() - .count(); - rtp_packet.SetTimestamp(timestamp_); - rtp_packet.SetSsrc(ssrc_); + if (!csrcs_.empty()) { + rtp_packet.SetCsrcs(csrcs_); + } - if (!csrcs_.empty()) { - rtp_packet.SetCsrcs(csrcs_); + if (has_extension_) { + rtp_packet.SetExtensionProfile(extension_profile_); + rtp_packet.SetExtensionData(extension_data_, extension_len_); + } + + RtpPacket::FU_INDICATOR fu_indicator; + fu_indicator.forbidden_bit = 0; + fu_indicator.nal_reference_idc = 0; + fu_indicator.nal_unit_type = FU_A; + + RtpPacket::FU_HEADER fu_header; + fu_header.start = index == 0 ? 1 : 0; + fu_header.end = index == num_of_source_packets - 1 ? 1 : 0; + fu_header.remain_bit = 0; + fu_header.nal_unit_type = FU_A; + + rtp_packet.SetFuIndicator(fu_indicator); + rtp_packet.SetFuHeader(fu_header); + + if (index == num_of_source_packets - 1) { + if (last_packet_size > 0) { + rtp_packet.EncodeH264Fua(fec_packets[index], last_packet_size); + } else { + rtp_packet.EncodeH264Fua(fec_packets[index], MAX_NALU_LEN); + } + } else { + rtp_packet.EncodeH264Fua(fec_packets[index], MAX_NALU_LEN); + } + packets.emplace_back(rtp_packet); + } else if (index >= num_of_source_packets && + index < num_of_total_packets) { + RtpPacket rtp_packet; + rtp_packet.SetVerion(version_); + rtp_packet.SetHasPadding(has_padding_); + rtp_packet.SetHasExtension(has_extension_); + rtp_packet.SetMarker(index == num_of_total_packets ? 1 : 0); + rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE::H264_FEC); + rtp_packet.SetSequenceNumber(sequence_number_++); + rtp_packet.SetTimestamp(timestamp_); + rtp_packet.SetSsrc(ssrc_); + + if (!csrcs_.empty()) { + rtp_packet.SetCsrcs(csrcs_); + } + + if (has_extension_) { + rtp_packet.SetExtensionProfile(extension_profile_); + rtp_packet.SetExtensionData(extension_data_, extension_len_); + } + rtp_packet.EncodeH264Fec(fec_packets[index], MAX_NALU_LEN); + packets.emplace_back(rtp_packet); } - if (has_extension_) { - rtp_packet.SetExtensionProfile(extension_profile_); - rtp_packet.SetExtensionData(extension_data_, extension_len_); - } - - RtpPacket::FU_INDICATOR fu_indicator; - fu_indicator.forbidden_bit = 0; - fu_indicator.nal_reference_idc = 0; - fu_indicator.nal_unit_type = FU_A; - - RtpPacket::FU_HEADER fu_header; - fu_header.start = index == 0 ? 1 : 0; - fu_header.end = index == num_of_total_packets - 1 ? 1 : 0; - fu_header.remain_bit = 0; - fu_header.nal_unit_type = FU_A; - - rtp_packet.SetFuIndicator(fu_indicator); - rtp_packet.SetFuHeader(fu_header); - - if (index == num_of_source_packets - 1 && last_packet_size > 0) { - rtp_packet.EncodeH264Fua(fec_packets[index], last_packet_size); - } else { - rtp_packet.EncodeH264Fua(fec_packets[index], MAX_NALU_LEN); - } - packets.emplace_back(rtp_packet); + // if (index < num_of_source_packets) { + // rtp_packet.EncodeH264Fua(fec_packets[index], MAX_NALU_LEN); + // packets.emplace_back(rtp_packet); + // } } fec_encoder_.ReleaseFecPackets(fec_packets, size); @@ -134,6 +166,8 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size, } else { size_t last_packet_size = size % MAX_NALU_LEN; size_t packet_num = size / MAX_NALU_LEN + (last_packet_size ? 1 : 0); + timestamp_ = + std::chrono::high_resolution_clock::now().time_since_epoch().count(); for (size_t index = 0; index < packet_num; index++) { RtpPacket rtp_packet; @@ -143,10 +177,6 @@ void RtpCodec::Encode(uint8_t* buffer, size_t size, rtp_packet.SetMarker(index == packet_num ? 1 : 0); rtp_packet.SetPayloadType(RtpPacket::PAYLOAD_TYPE(payload_type_)); rtp_packet.SetSequenceNumber(sequence_number_++); - - timestamp_ = std::chrono::high_resolution_clock::now() - .time_since_epoch() - .count(); rtp_packet.SetTimestamp(timestamp_); rtp_packet.SetSsrc(ssrc_); diff --git a/src/rtp/rtp_packet.cpp b/src/rtp/rtp_packet.cpp index 165316d..1b948b8 100644 --- a/src/rtp/rtp_packet.cpp +++ b/src/rtp/rtp_packet.cpp @@ -5,17 +5,19 @@ #include "log.h" void RtpPacket::TryToDecodeRtpPacket() { - if (PAYLOAD_TYPE::H264 == NAL_UNIT_TYPE(buffer_[1] & 0x7F)) { + if (PAYLOAD_TYPE::H264 == PAYLOAD_TYPE(buffer_[1] & 0x7F)) { nal_unit_type_ = NAL_UNIT_TYPE(buffer_[12] & 0x1F); if (NAL_UNIT_TYPE::NALU == nal_unit_type_) { DecodeH264Nalu(); } else if (NAL_UNIT_TYPE::FU_A == nal_unit_type_) { DecodeH264Fua(); } - } else if (PAYLOAD_TYPE::DATA == NAL_UNIT_TYPE(buffer_[1] & 0x7F)) { + } else if (PAYLOAD_TYPE::H264_FEC == PAYLOAD_TYPE(buffer_[1] & 0x7F)) { + DecodeH264Fec(); + } else if (PAYLOAD_TYPE::DATA == PAYLOAD_TYPE(buffer_[1] & 0x7F)) { DecodeData(); } else { - LOG_ERROR("Unknown pt: {}", NAL_UNIT_TYPE(buffer_[1] & 0x7F)); + LOG_ERROR("Unknown pt: {}", PAYLOAD_TYPE(buffer_[1] & 0x7F)); } } @@ -243,6 +245,59 @@ const uint8_t *RtpPacket::EncodeH264Fua(uint8_t *payload, size_t payload_size) { return buffer_; } +const uint8_t *RtpPacket::EncodeH264Fec(uint8_t *payload, size_t payload_size) { + buffer_[0] = (version_ << 6) | (has_padding_ << 5) | (has_extension_ << 4) | + total_csrc_number_; + buffer_[1] = (marker_ << 7) | payload_type_; + buffer_[2] = (sequence_number_ >> 8) & 0xFF; + buffer_[3] = sequence_number_ & 0xFF; + buffer_[4] = (timestamp_ >> 24) & 0xFF; + buffer_[5] = (timestamp_ >> 16) & 0xFF; + buffer_[6] = (timestamp_ >> 8) & 0xFF; + buffer_[7] = timestamp_ & 0xFF; + buffer_[8] = (ssrc_ >> 24) & 0xFF; + buffer_[9] = (ssrc_ >> 16) & 0xFF; + buffer_[10] = (ssrc_ >> 8) & 0xFF; + buffer_[11] = ssrc_ & 0xFF; + + for (uint32_t index = 0; index < total_csrc_number_ && !csrcs_.empty(); + index++) { + buffer_[12 + index] = (csrcs_[index] >> 24) & 0xFF; + buffer_[13 + index] = (csrcs_[index] >> 16) & 0xFF; + buffer_[14 + index] = (csrcs_[index] >> 8) & 0xFF; + buffer_[15 + index] = csrcs_[index] & 0xFF; + } + + uint32_t extension_offset = + total_csrc_number_ && !csrcs_.empty() ? total_csrc_number_ * 4 : 0; + if (has_extension_ && extension_data_) { + buffer_[12 + extension_offset] = extension_profile_ >> 8; + buffer_[13 + extension_offset] = extension_profile_ & 0xff; + buffer_[14 + extension_offset] = (extension_len_ >> 8) & 0xFF; + buffer_[15 + extension_offset] = extension_len_ & 0xFF; + memcpy(buffer_ + 16 + extension_offset, extension_data_, extension_len_); + } + + uint32_t fec_symbol_id_offset = + extension_offset + (has_extension_ && extension_data_ ? 4 : 0); + buffer_[12 + fec_symbol_id_offset] = fec_symbol_id_; + + uint32_t payload_offset = fec_symbol_id_offset + 1; + + buffer_[12 + payload_offset] = fu_indicator_.forbidden_bit << 7 | + fu_indicator_.nal_reference_idc << 6 | + fu_indicator_.nal_unit_type; + + buffer_[13 + payload_offset] = fu_header_.start << 7 | fu_header_.end << 6 | + fu_header_.remain_bit << 1 | + fu_header_.nal_unit_type; + + memcpy(buffer_ + 14 + payload_offset, payload, payload_size); + size_ = payload_size + (14 + payload_offset); + + return buffer_; +} + size_t RtpPacket::DecodeData(uint8_t *payload) { version_ = (buffer_[0] >> 6) & 0x03; has_padding_ = (buffer_[0] >> 5) & 0x01; @@ -360,9 +415,6 @@ size_t RtpPacket::DecodeH264Fua(uint8_t *payload) { extension_len_ = (buffer_[14 + extension_offset] << 8) | buffer_[15 + extension_offset]; - // extension_data_ = new uint8_t[extension_len_]; - // memcpy(extension_data_, buffer_ + 16 + extension_offset, - // extension_len_); extension_data_ = buffer_ + 16 + extension_offset; } @@ -378,6 +430,48 @@ size_t RtpPacket::DecodeH264Fua(uint8_t *payload) { fu_header_.remain_bit = (buffer_[13 + payload_offset] >> 5) & 0x01; fu_header_.nal_unit_type = buffer_[13 + payload_offset] & 0x1F; + payload_size_ = size_ - (14 + payload_offset); + payload_ = buffer_ + 14 + payload_offset; + if (payload) { + memcpy(payload, payload_, payload_size_); + } + return payload_size_; +} + +size_t RtpPacket::DecodeH264Fec(uint8_t *payload) { + version_ = (buffer_[0] >> 6) & 0x03; + has_padding_ = (buffer_[0] >> 5) & 0x01; + has_extension_ = (buffer_[0] >> 4) & 0x01; + total_csrc_number_ = buffer_[0] & 0x0f; + marker_ = (buffer_[1] >> 7) & 0x01; + payload_type_ = buffer_[1] & 0x7f; + sequence_number_ = (buffer_[2] << 8) | buffer_[3]; + timestamp_ = + (buffer_[4] << 24) | (buffer_[5] << 16) | (buffer_[6] << 8) | buffer_[7]; + ssrc_ = (buffer_[8] << 24) | (buffer_[9] << 16) | (buffer_[10] << 8) | + buffer_[11]; + + for (uint32_t index = 0; index < total_csrc_number_; index++) { + uint32_t csrc = (buffer_[12 + index] << 24) | (buffer_[13 + index] << 16) | + (buffer_[14 + index] << 8) | buffer_[15 + index]; + csrcs_.push_back(csrc); + } + + uint32_t extension_offset = total_csrc_number_ * 4; + if (has_extension_) { + extension_profile_ = + (buffer_[12 + extension_offset] << 8) | buffer_[13 + extension_offset]; + extension_len_ = + (buffer_[14 + extension_offset] << 8) | buffer_[15 + extension_offset]; + + extension_data_ = buffer_ + 16 + extension_offset; + } + + uint32_t fec_symbol_id_offset = extension_offset + (has_extension_ ? 4 : 0); + fec_symbol_id_ = buffer_[12 + fec_symbol_id_offset]; + + uint32_t payload_offset = fec_symbol_id_offset + 1; + payload_size_ = size_ - (14 + payload_offset); payload_ = buffer_ + 14 + payload_offset; if (payload) { diff --git a/src/rtp/rtp_packet.h b/src/rtp/rtp_packet.h index a042f5a..cd07452 100644 --- a/src/rtp/rtp_packet.h +++ b/src/rtp/rtp_packet.h @@ -17,13 +17,13 @@ // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ // | synchronization source (SSRC) identifier | // +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ -// | Contributing source (CSRC) identifiers | -// | .... | +// | Contributing source (CSRC) identifiers |x +// | .... |x // +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ -// | defined by profile | length | +// | defined by profile | length |x // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | Extensions | -// | .... | +// | Extensions |x +// | .... |x // +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ // | Payload | // | .... : padding... | @@ -41,13 +41,13 @@ // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ // | synchronization source (SSRC) identifier | // +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ -// | Contributing source (CSRC) identifiers | -// | .... | +// | Contributing source (CSRC) identifiers |x +// | .... |x // +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ -// | defined by profile | length | +// | defined by profile | length |x // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | Extensions | -// | .... | +// | Extensions |x +// | .... |x // +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ // | FU indicator | FU header | | // | | @@ -63,12 +63,44 @@ // |F|NRI| Type |S|E|R| Type | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// H264 FEC +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// |V=2|P|X| CC |M| PT | sequence number | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | timestamp | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | synchronization source (SSRC) identifier | +// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ +// | Contributing source (CSRC) identifiers |x +// | .... |x +// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ +// | defined by profile | length |x +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | Extensions |x +// | .... |x +// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ +// | FEC symbol id | | +// | | +// | Fec Payload | +// | | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | padding | Padding size | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + #define DEFAULT_MTU 1500 #define MAX_NALU_LEN 1400 class RtpPacket { public: - typedef enum { H264 = 96, OPUS = 97, DATA = 127 } PAYLOAD_TYPE; + typedef enum { + H264 = 96, + H264_FEC = 97, + OPUS = 111, + DATA = 127 + } PAYLOAD_TYPE; + typedef enum { UNKNOWN = 0, NALU = 1, FU_A = 28, FU_B = 29 } NAL_UNIT_TYPE; public: @@ -133,13 +165,17 @@ class RtpPacket { fu_header_.nal_unit_type = fu_header.nal_unit_type; } + void SetFecSymbolId(uint8_t fec_symbol_id) { fec_symbol_id_ = fec_symbol_id; } + public: const uint8_t *Encode(uint8_t *payload, size_t payload_size); const uint8_t *EncodeH264Nalu(uint8_t *payload, size_t payload_size); const uint8_t *EncodeH264Fua(uint8_t *payload, size_t payload_size); + const uint8_t *EncodeH264Fec(uint8_t *payload, size_t payload_size); size_t DecodeData(uint8_t *payload = nullptr); size_t DecodeH264Nalu(uint8_t *payload = nullptr); size_t DecodeH264Fua(uint8_t *payload = nullptr); + size_t DecodeH264Fec(uint8_t *payload = nullptr); public: // Get Header @@ -188,6 +224,8 @@ class RtpPacket { return extension_data_; } + const uint8_t FecSymbolId() { return fec_symbol_id_; } + // Payload const uint8_t *Payload() { ParseRtpData(); @@ -238,6 +276,7 @@ class RtpPacket { uint8_t *extension_data_ = nullptr; FU_INDICATOR fu_indicator_; FU_HEADER fu_header_; + uint8_t fec_symbol_id_ = 0; // Payload uint8_t *payload_ = nullptr; diff --git a/src/rtp/rtp_video_receiver.cpp b/src/rtp/rtp_video_receiver.cpp index 81d547f..bc7096c 100644 --- a/src/rtp/rtp_video_receiver.cpp +++ b/src/rtp/rtp_video_receiver.cpp @@ -48,23 +48,12 @@ void RtpVideoReceiver::InsertRtpPacket(RtpPacket& rtp_packet) { rtcp_rr.Encode(); - // SendRtcpRR(rtcp_rr); + SendRtcpRR(rtcp_rr); } if (RtpPacket::NAL_UNIT_TYPE::NALU == rtp_packet.NalUnitType()) { compelete_video_frame_queue_.push( VideoFrame(rtp_packet.Payload(), rtp_packet.Size())); - // if (on_receive_complete_frame_) { - // auto now_complete_frame_ts = - // std::chrono::high_resolution_clock::now().time_since_epoch().count() - // / 1000000; - // uint32_t duration = now_complete_frame_ts - last_complete_frame_ts_; - // LOG_ERROR("Duration {}", 1000 / duration); - // last_complete_frame_ts_ = now_complete_frame_ts; - - // on_receive_complete_frame_( - // VideoFrame(rtp_packet.Payload(), rtp_packet.Size())); - // } } else if (RtpPacket::NAL_UNIT_TYPE::FU_A == rtp_packet.NalUnitType()) { incomplete_frame_list_[rtp_packet.SequenceNumber()] = rtp_packet; bool complete = CheckIsFrameCompleted(rtp_packet); @@ -83,6 +72,9 @@ bool RtpVideoReceiver::CheckIsFrameCompleted(RtpPacket& rtp_packet) { auto it = incomplete_frame_list_.find(end_seq); complete_frame_size += it->second.PayloadSize(); if (it == incomplete_frame_list_.end()) { + // The last fragment has already received. If all fragments are in + // order, then some fragments lost in tranmission and need to be + // repaired using FEC return false; } else if (!it->second.FuAStart()) { continue; @@ -105,19 +97,6 @@ bool RtpVideoReceiver::CheckIsFrameCompleted(RtpPacket& rtp_packet) { compelete_video_frame_queue_.push( VideoFrame(nv12_data_, complete_frame_size)); - // if (on_receive_complete_frame_) { - // auto now_complete_frame_ts = - // std::chrono::high_resolution_clock::now() - // .time_since_epoch() - // .count() / - // 1000000; - // uint32_t duration = now_complete_frame_ts - - // last_complete_frame_ts_; LOG_ERROR("Duration {}", 1000 / duration); - // last_complete_frame_ts_ = now_complete_frame_ts; - - // on_receive_complete_frame_( - // VideoFrame(nv12_data_, complete_frame_size)); - // } return true; } else { LOG_WARN("What happened?") @@ -166,8 +145,6 @@ int RtpVideoReceiver::SendRtcpRR(RtcpReceiverReport& rtcp_rr) { return -1; } - // LOG_ERROR("Send RR"); - return 0; }