YangWX/crossdesk
1
0
Fork 0
mirror of https://github.com/kunkundi/crossdesk.git synced 2025-10-26 20:25:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implementation for NALU

Browse Source
This commit is contained in:
dijunkun
2023-09-06 17:30:58 +08:00
parent cea0cfdb95
commit 952bb02df5
8 changed files with 290 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
config/config.ini
View File

@@ -1,6 +1,6 @@
[signal server]
ip = localhost
port = 9090
ip = 120.77.216.215
port = 9099
[stun server]
ip = 120.77.216.215

125
src/rtp/rtp_packet.cpp
View File

@@ -2,33 +2,9 @@
#include <string>
// 0 1 2 3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |V=2|P|X| CC |M| PT | sequence number |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | timestamp |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | synchronization source (SSRC) identifier |
// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
// | Contributing source (CSRC) identifiers |
// | .... |
// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
// | defined by profile | length |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | Extensions |
// | .... |
// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
// | Payload |
// | .... : padding... |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | padding | Padding size |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
constexpr size_t kDefaultMtu = 1400;
RtpPacket::RtpPacket() : buffer_(new uint8_t[kDefaultMtu]), size_(kDefaultMtu) {
memset(buffer_, 0, kDefaultMtu);
RtpPacket::RtpPacket()
: buffer_(new uint8_t[MAX_NALU_LEN]), size_(MAX_NALU_LEN) {
memset(buffer_, 0, MAX_NALU_LEN);
}
RtpPacket::RtpPacket(const uint8_t *buffer, size_t size) {
@@ -122,6 +98,54 @@ const uint8_t *RtpPacket::Encode(uint8_t *payload, size_t payload_size) {
return buffer_;
}
const uint8_t *RtpPacket::EncodeH264Nalu(uint8_t *payload,
size_t payload_size) {
buffer_[0] = (version_ << 6) | (has_padding_ << 5) | (has_extension_ << 4) |
total_csrc_number_;
buffer_[1] = (marker_ << 7) | payload_type_;
buffer_[2] = (sequence_number_ >> 8) & 0xFF;
buffer_[3] = sequence_number_ & 0xFF;
buffer_[4] = (timestamp_ >> 24) & 0xFF;
buffer_[5] = (timestamp_ >> 16) & 0xFF;
buffer_[6] = (timestamp_ >> 8) & 0xFF;
buffer_[7] = timestamp_ & 0xFF;
buffer_[8] = (ssrc_ >> 24) & 0xFF;
buffer_[9] = (ssrc_ >> 16) & 0xFF;
buffer_[10] = (ssrc_ >> 8) & 0xFF;
buffer_[11] = ssrc_ & 0xFF;
for (uint32_t index = 0; index < total_csrc_number_ && !csrcs_.empty();
index++) {
buffer_[12 + index] = (csrcs_[index] >> 24) & 0xFF;
buffer_[13 + index] = (csrcs_[index] >> 16) & 0xFF;
buffer_[14 + index] = (csrcs_[index] >> 8) & 0xFF;
buffer_[15 + index] = csrcs_[index] & 0xFF;
}
uint32_t extension_offset =
total_csrc_number_ && !csrcs_.empty() ? total_csrc_number_ * 4 : 0;
if (has_extension_ && extension_data_) {
buffer_[12 + extension_offset] = extension_profile_ >> 8;
buffer_[13 + extension_offset] = extension_profile_ & 0xff;
buffer_[14 + extension_offset] = (extension_len_ >> 8) & 0xFF;
buffer_[15 + extension_offset] = extension_len_ & 0xFF;
memcpy(buffer_ + 16 + extension_offset, extension_data_, extension_len_);
}
uint32_t payload_offset =
(has_extension_ && extension_data_ ? extension_len_ : 0) +
extension_offset;
buffer_[12 + payload_offset] = nalu_header_.forbidden_bit << 7 |
nalu_header_.nal_reference_idc << 6 |
nalu_header_.nal_unit_type;
memcpy(buffer_ + 13 + payload_offset, payload, payload_size);
size_ = payload_size + (13 + payload_offset);
return buffer_;
}
const uint8_t *RtpPacket::Decode() {
version_ = (buffer_[0] >> 6) & 0x03;
has_padding_ = (buffer_[0] >> 5) & 0x01;
@@ -163,3 +187,50 @@ const uint8_t *RtpPacket::Decode() {
return payload_;
}
size_t RtpPacket::DecodeH264Nalu(uint8_t *payload) {
version_ = (buffer_[0] >> 6) & 0x03;
has_padding_ = (buffer_[0] >> 5) & 0x01;
has_extension_ = (buffer_[0] >> 4) & 0x01;
total_csrc_number_ = buffer_[0] & 0x0f;
marker_ = (buffer_[1] >> 7) & 0x01;
payload_type_ = buffer_[1] & 0x7f;
sequence_number_ = (buffer_[2] << 8) | buffer_[3];
timestamp_ =
(buffer_[4] << 24) | (buffer_[5] << 16) | (buffer_[6] << 8) | buffer_[7];
ssrc_ = (buffer_[8] << 24) | (buffer_[9] << 16) | (buffer_[10] << 8) |
buffer_[11];
for (uint32_t index = 0; index < total_csrc_number_; index++) {
uint32_t csrc = (buffer_[12 + index] << 24) | (buffer_[13 + index] << 16) |
(buffer_[14 + index] << 8) | buffer_[15 + index];
csrcs_.push_back(csrc);
}
uint32_t extension_offset = total_csrc_number_ * 4;
if (has_extension_) {
extension_profile_ =
(buffer_[12 + extension_offset] << 8) | buffer_[13 + extension_offset];
extension_len_ =
(buffer_[14 + extension_offset] << 8) | buffer_[15 + extension_offset];
// extension_data_ = new uint8_t[extension_len_];
// memcpy(extension_data_, buffer_ + 16 + extension_offset, extension_len_);
extension_data_ = buffer_ + 16 + extension_offset;
}
uint32_t payload_offset =
(has_extension_ ? extension_len_ : 0) + extension_offset;
nalu_header_.forbidden_bit = (buffer_[12 + payload_offset] >> 7) & 0x01;
nalu_header_.nal_reference_idc = (buffer_[12 + payload_offset] >> 6) & 0x01;
nalu_header_.nal_unit_type = (buffer_[12 + payload_offset]) & 0x31;
payload_size_ = size_ - (13 + payload_offset);
// payload_ = new uint8_t[payload_size_];
// memcpy(payload_, buffer_ + 12 + payload_offset, payload_size_);
payload_ = buffer_ + 13 + payload_offset;
memcpy(payload, payload_, payload_size_);
return payload_size_;
}

82
src/rtp/rtp_packet.h
View File

@@ -5,6 +5,7 @@
#include <vector>
// Common
// 0 1 2 3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -28,6 +29,41 @@
// | padding | Padding size |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// H264
// 0 1 2 3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |V=2|P|X| CC |M| PT | sequence number |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | timestamp |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | synchronization source (SSRC) identifier |
// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
// | Contributing source (CSRC) identifiers |
// | .... |
// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
// | defined by profile | length |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | Extensions |
// | .... |
// +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
// | FU indicator | FU header | |
// | |
// | FU Payload |
// | |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | padding | Padding size |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | FU indicator | FU header |
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |F|NRI| Type |S|E|R| Type |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
#define MAX_NALU_LEN 1400
typedef enum { H264 = 96, OPUS = 97, USER_DEFINED = 127 } PAYLOAD_TYPE;
class RtpPacket {
public:
RtpPacket();
@@ -44,7 +80,9 @@ class RtpPacket {
void SetHasPadding(bool has_padding) { has_padding_ = has_padding; }
void SetHasExtension(bool has_extension) { has_extension_ = has_extension; }
void SetMarker(bool marker) { marker_ = marker; }
void SetPayloadType(uint32_t payload_type) { payload_type_ = payload_type; }
void SetPayloadType(PAYLOAD_TYPE payload_type) {
payload_type_ = payload_type;
}
void SetSequenceNumber(uint16_t sequence_number) {
sequence_number_ = sequence_number;
}
@@ -61,9 +99,48 @@ class RtpPacket {
memcpy(extension_data_, extension_data, extension_len_);
}
public:
typedef struct {
unsigned char forbidden_bit : 1;
unsigned char nal_reference_idc : 2;
unsigned char nal_unit_type : 5;
} NALU_HEADER;
typedef struct {
unsigned char f : 1;
unsigned char nri : 2;
unsigned char type : 5;
} FU_INDICATOR;
typedef struct {
unsigned char s : 1;
unsigned char e : 1;
unsigned char r : 1;
unsigned char type : 5;
} FU_HEADER;
void SetNaluHeader(NALU_HEADER nalu_header) {
nalu_header_.forbidden_bit = nalu_header.forbidden_bit;
nalu_header_.nal_reference_idc = nalu_header.nal_reference_idc;
nalu_header_.nal_unit_type = nalu_header.nal_unit_type;
}
void FuAHeader(FU_INDICATOR fu_indicator, FU_HEADER fu_header) {
fu_indicator_.f = fu_indicator.f;
fu_indicator_.nri = fu_indicator.nri;
fu_indicator_.type = fu_indicator.nri;
fu_header_.s = fu_header.s;
fu_header_.e = fu_header.e;
fu_header_.r = fu_header.r;
fu_header_.type = fu_header.type;
}
public:
const uint8_t *Encode(uint8_t *payload, size_t payload_size);
const uint8_t *EncodeH264Nalu(uint8_t *payload, size_t payload_size);
const uint8_t *Decode();
size_t DecodeH264Nalu(uint8_t *payload);
public:
// Get Header
@@ -103,6 +180,9 @@ class RtpPacket {
uint16_t extension_profile_ = 0;
uint16_t extension_len_ = 0;
uint8_t *extension_data_ = nullptr;
NALU_HEADER nalu_header_;
FU_INDICATOR fu_indicator_;
FU_HEADER fu_header_;
// Payload
uint8_t *payload_ = nullptr;

61
src/rtp/rtp_session.cpp
View File

@@ -2,9 +2,11 @@
#include <chrono>
#include "log.h"
#define RTP_VERSION 1
RtpSession ::RtpSession(uint32_t payload_type)
RtpSession ::RtpSession(PAYLOAD_TYPE payload_type)
: version_(RTP_VERSION),
has_padding_(false),
has_extension_(false),
@@ -23,16 +25,55 @@ RtpSession ::~RtpSession() {
}
}
RtpPacket RtpSession::Encode(uint8_t* buffer, size_t size) {
std::vector<RtpPacket> RtpSession::Encode(uint8_t* buffer, size_t size) {
if (!rtp_packet_) {
rtp_packet_ = new RtpPacket();
}
std::vector<RtpPacket> packets;
if (size <= MAX_NALU_LEN) {
rtp_packet_->SetVerion(version_);
rtp_packet_->SetHasPadding(has_padding_);
rtp_packet_->SetHasExtension(has_extension_);
rtp_packet_->SetMarker(marker_);
rtp_packet_->SetPayloadType(payload_type_);
rtp_packet_->SetMarker(1);
rtp_packet_->SetPayloadType(PAYLOAD_TYPE(payload_type_));
rtp_packet_->SetSequenceNumber(sequence_number_++);
timestamp_ =
std::chrono::high_resolution_clock::now().time_since_epoch().count();
rtp_packet_->SetTimestamp(timestamp_);
rtp_packet_->SetSsrc(ssrc_);
if (!csrcs_.empty()) {
rtp_packet_->SetCsrcs(csrcs_);
}
if (has_extension_) {
rtp_packet_->SetExtensionProfile(extension_profile_);
rtp_packet_->SetExtensionData(extension_data_, extension_len_);
}
RtpPacket::NALU_HEADER nalu_header;
nalu_header.forbidden_bit = 0;
nalu_header.nal_reference_idc = 1;
nalu_header.nal_unit_type = 1;
rtp_packet_->SetNaluHeader(nalu_header);
rtp_packet_->EncodeH264Nalu(buffer, size);
packets.push_back(*rtp_packet_);
} else {
size_t last_packet_size = size % MAX_NALU_LEN;
size_t packet_num = size / MAX_NALU_LEN + (last_packet_size ? 1 : 0);
for (size_t index = 0; index * MAX_NALU_LEN < size + MAX_NALU_LEN;
index++) {
rtp_packet_->SetVerion(version_);
rtp_packet_->SetHasPadding(has_padding_);
rtp_packet_->SetHasExtension(has_extension_);
rtp_packet_->SetMarker(1);
rtp_packet_->SetPayloadType(PAYLOAD_TYPE(payload_type_));
rtp_packet_->SetSequenceNumber(sequence_number_++);
timestamp_ =
@@ -50,6 +91,14 @@ RtpPacket RtpSession::Encode(uint8_t* buffer, size_t size) {
}
rtp_packet_->Encode(buffer, size);
return *rtp_packet_;
packets.push_back(*rtp_packet_);
}
}
return std::move(packets);
}
size_t RtpSession::Decode(uint8_t* buffer, size_t size, uint8_t* payload) {
*rtp_packet_ = std::move(RtpPacket(buffer, size));
return rtp_packet_->DecodeH264Nalu(payload);
}

7
src/rtp/rtp_session.h
View File

@@ -11,11 +11,12 @@ class RtpSession
{
public:
RtpSession(uint32_t payload_type);
RtpSession(PAYLOAD_TYPE payload_type);
~RtpSession();
public:
RtpPacket Encode(uint8_t* buffer, size_t size);
std::vector<RtpPacket> Encode(uint8_t* buffer, size_t size);
size_t Decode(uint8_t* buffer, size_t size, uint8_t* payload);
private:
uint32_t version_ = 0;
@@ -23,7 +24,7 @@ class RtpSession
bool has_extension_ = false;
uint32_t total_csrc_number_ = 0;
bool marker_ = false;
uint8_t payload_type_ = 0;
uint32_t payload_type_ = 0;
uint16_t sequence_number_ = 0;
uint32_t timestamp_ = 0;
uint32_t ssrc_ = 0;

43
src/transmission/ice_transmission.cpp
View File

@@ -29,9 +29,14 @@ IceTransmission::IceTransmission(
on_receive_ice_msg_cb_(on_receive_ice_msg) {}
IceTransmission::~IceTransmission() {
if (rtp_session_) {
delete rtp_session_;
rtp_session_ = nullptr;
if (video_rtp_session_) {
delete video_rtp_session_;
video_rtp_session_ = nullptr;
}
if (rtp_payload_) {
delete rtp_payload_;
rtp_payload_ = nullptr;
}
if (ice_agent_) {
@@ -84,8 +89,14 @@ int IceTransmission::InitIceTransmission(std::string &ip, int port) {
// Data buffer;
RtpPacket buffer;
recv_ringbuffer_.pop(buffer);
if (!rtp_payload_) {
rtp_payload_ = new uint8_t[1400];
}
size_t rtp_payload_size = video_rtp_session_->Decode(
(uint8_t *)buffer.Buffer(), buffer.Size(), rtp_payload_);
// ret = ikcp_input(kcp, buffer.data(), buffer.size());
ret = ikcp_input(kcp, (const char *)buffer.Buffer(), buffer.Size());
// ret = ikcp_input(kcp, (const char *)buffer.Buffer(), buffer.Size());
ret = ikcp_input(kcp, (const char *)rtp_payload_, rtp_payload_size);
}
int len = 0;
@@ -97,7 +108,7 @@ int IceTransmission::InitIceTransmission(std::string &ip, int port) {
if (len <= 0) {
if (on_receive_ice_msg_cb_ && total_len > 0) {
// LOG_ERROR("Receive size: {}", total_len);
LOG_ERROR("Receive size: {}", total_len);
on_receive_ice_msg_cb_(kcp_complete_buffer_, total_len,
remote_user_id_.data(),
remote_user_id_.size());
@@ -112,7 +123,7 @@ int IceTransmission::InitIceTransmission(std::string &ip, int port) {
ikcp_release(kcp);
});
rtp_session_ = new RtpSession(1);
video_rtp_session_ = new RtpSession(PAYLOAD_TYPE::H264);
ice_agent_ = new IceAgent(ip, port);
ice_agent_->CreateIceAgent(
@@ -154,7 +165,7 @@ int IceTransmission::InitIceTransmission(std::string &ip, int port) {
if (user_ptr) {
IceTransmission *ice_transmission_obj =
static_cast<IceTransmission *>(user_ptr);
if (ice_transmission_obj->on_receive_ice_msg_cb_) {
if (ice_transmission_obj) {
// ice_transmission_obj->recv_ringbuffer_.push(
// std::move(Data(data, size)));
@@ -273,17 +284,19 @@ int IceTransmission::SendData(const char *data, size_t size) {
if (JUICE_STATE_COMPLETED == state_) {
// send_ringbuffer_.push(std::move(Data(data, size)));
int num = 0;
for (num = 0; num * 1400 < size; num++) {
RtpPacket packet =
rtp_session_->Encode((uint8_t *)(data + num * 1400), 1400);
for (int num = 0; num * 1400 < size + 1400; num++) {
std::vector<RtpPacket> packets =
video_rtp_session_->Encode((uint8_t *)(data + num * 1400), 1400);
for (auto &packet : packets) {
send_ringbuffer_.push(packet);
}
if (size > num * 1400 && size - num * 1400 > 0) {
RtpPacket packet = rtp_session_->Encode((uint8_t *)(data + num * 1400),
size - num * 1400);
send_ringbuffer_.push(packet);
// std::vector<RtpPacket> packets =
// video_rtp_session_->Encode((uint8_t *)(data), size);
// send_ringbuffer_.insert(send_ringbuffer_.end(), packets.begin(),
// packets.end());
}
}
return 0;

3
src/transmission/ice_transmission.h
View File

@@ -75,7 +75,8 @@ class IceTransmission {
std::thread *kcp_update_thread_ = nullptr;
private:
RtpSession *rtp_session_ = nullptr;
RtpSession *video_rtp_session_ = nullptr;
uint8_t *rtp_payload_ = nullptr;
};
#endif

48
thirdparty/nvcodec/Interface/nvEncodeAPI.h vendored
View File

@@ -1520,52 +1520,10 @@ typedef struct _NV_ENC_RC_PARAMS {
uint16_t lookaheadDepth; /**< [in]: Maximum depth of lookahead with range
0-(31 - number of B frames). lookaheadDepth is
only used if enableLookahead=1.*/
uint8_t lowDelayKeyFrameScale; /**< [in]: Specifies the ratio of I frame bits
to P frame bits in case of single frame VBV
and CBR rate control mode, is set 2 by
default for low latency tuning info and 1 by
default for ultra low latency tuning info */
uint8_t lowDelayKeyFrameScale;
uint8_t reserved1[3];
NV_ENC_QP_MAP_MODE
qpMapMode; /**< [in]: This flag is used to interpret values in array
specified by NV_ENC_PIC_PARAMS::qpDeltaMap. Set this to
NV_ENC_QP_MAP_EMPHASIS to treat values specified by
NV_ENC_PIC_PARAMS::qpDeltaMap as Emphasis Level Map.
Emphasis Level can be assigned any value specified
in enum NV_ENC_EMPHASIS_MAP_LEVEL. Emphasis Level Map is
used to specify regions to be encoded at varying levels of
quality. The hardware encoder adjusts the quantization
within the image as per the provided emphasis map, by
adjusting the quantization parameter (QP) assigned to each
macroblock. This adjustment is commonly called “Delta QP”.
The adjustment depends on the absolute QP decided by
the rate control algorithm, and is applied after the rate
control has decided each macroblock’s QP. Since the Delta QP
overrides rate control, enabling Emphasis Level Map may
violate bitrate and VBV buffer size constraints. Emphasis
Level Map is useful in situations where client has a priori
knowledge of the image complexity (e.g. via use of NVFBC's
Classification feature) and encoding those high-complexity
areas at higher quality (lower QP) is important, even at the
possible cost of violating bitrate/VBV buffer size
constraints This feature is not supported when AQ(
Spatial/Temporal) is enabled. This feature is only supported
for H264 codec currently.
Set this to NV_ENC_QP_MAP_DELTA to treat values
specified by NV_ENC_PIC_PARAMS::qpDeltaMap as QP Delta. This
specifies QP modifier to be applied on top of the QP chosen
by rate control
Set this to NV_ENC_QP_MAP_DISABLED to ignore
NV_ENC_PIC_PARAMS::qpDeltaMap values. In this case,
qpDeltaMap should be set to NULL.
Other values are reserved for future use.*/
NV_ENC_MULTI_PASS
multiPass; /**< [in]: This flag is used to enable multi-pass encoding for
a given ::NV_ENC_PARAMS_RC_MODE. This flag is not valid for
H264 and HEVC MEOnly mode */
NV_ENC_QP_MAP_MODE qpMapMode;
NV_ENC_MULTI_PASS multiPass;
uint32_t
alphaLayerBitrateRatio; /**< [in]: Specifies the ratio in which bitrate
should be split between base and alpha layer. A