From 5eb455b6c80217f24dc238bd51581ee9af9935fc Mon Sep 17 00:00:00 2001 From: dijunkun Date: Mon, 23 Sep 2024 11:13:53 +0800 Subject: [PATCH] [feat] support using negotiated sdp to create media codecs --- src/pc/peer_connection.cpp | 78 +++++++++++++++++++++++---- src/pc/peer_connection.h | 5 +- src/transmission/ice_transmission.cpp | 48 ++++++++++++----- src/transmission/ice_transmission.h | 4 ++ 4 files changed, 110 insertions(+), 25 deletions(-) diff --git a/src/pc/peer_connection.cpp b/src/pc/peer_connection.cpp index 59ca7a9..7ce2431 100644 --- a/src/pc/peer_connection.cpp +++ b/src/pc/peer_connection.cpp @@ -243,14 +243,6 @@ int PeerConnection::Init(PeerConnectionParams params, StartIceWorker(); - if (0 != CreateVideoCodec(hardware_acceleration_)) { - LOG_ERROR("Create video codec failed"); - } - - if (0 != CreateAudioCodec()) { - LOG_ERROR("Create audio codec failed"); - } - // do { // } while (SignalStatus::SignalConnected != GetSignalStatus()); @@ -262,7 +254,7 @@ int PeerConnection::Init(PeerConnectionParams params, return 0; } -int PeerConnection::CreateVideoCodec(bool hardware_acceleration) { +int PeerConnection::CreateVideoCodec(bool av1, bool hardware_acceleration) { if (video_codec_inited_) { return 0; } @@ -412,6 +404,30 @@ int PeerConnection::Join(const std::string &transmission_id, return ret; } +int PeerConnection::NegotiationFailed() { + if (SignalStatus::SignalConnected != GetSignalStatus()) { + LOG_ERROR("Signal not connected"); + return -1; + } + + json message = {{"type", "negotiation_failed"}, + {"user_id", user_id_}, + {"transmission_id", local_transmission_id_}}; + if (ws_transport_) { + ws_transport_->Send(message.dump()); + LOG_INFO( + "[{}] sends negotiation failed notification to [{}] for transmission " + "id [{}]", + user_id_, remote_user_id_, local_transmission_id_); + } + + IceWorkMsg msg; + msg.type = IceWorkMsg::Type::Destroy; + PushIceWorkMsg(msg); + + return 0; +} + int PeerConnection::Leave(const std::string &transmission_id) { if (SignalStatus::SignalConnected != GetSignalStatus()) { LOG_ERROR("Signal not connected"); @@ -656,6 +672,8 @@ void PeerConnection::ProcessSignal(const std::string &signal) { case "offer"_H: { std::string transmission_id = j["transmission_id"].get(); std::string remote_user_id = j["remote_user_id"].get(); + remote_user_id_ = remote_user_id; + if (j.contains("sdp")) { std::string remote_sdp = j["sdp"].get(); LOG_INFO("[{}] receive offer from [{}]", user_id_, remote_user_id); @@ -676,6 +694,7 @@ void PeerConnection::ProcessSignal(const std::string &signal) { case "answer"_H: { std::string transmission_id = j["transmission_id"].get(); std::string remote_user_id = j["remote_user_id"].get(); + remote_user_id_ = remote_user_id; if (j.contains("sdp")) { std::string remote_sdp = j["sdp"].get(); @@ -853,7 +872,25 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) { } std::string remote_sdp = msg.remote_sdp; - ice_transmission_list_[remote_user_id]->SetRemoteSdp(remote_sdp); + int ret = + ice_transmission_list_[remote_user_id]->SetRemoteSdp(remote_sdp); + if (0 != ret) { + NegotiationFailed(); + break; + } else { + std::vector negotiated_payload_types = + ice_transmission_list_[remote_user_id]->GetNegotiatedCapabilities(); + if (0 != CreateVideoCodec(RtpPacket::PAYLOAD_TYPE::AV1 == + negotiated_payload_types[0], + hardware_acceleration_)) { + LOG_ERROR("Create video codec failed"); + } + + if (0 != CreateAudioCodec()) { + LOG_ERROR("Create audio codec failed"); + } + } + if (trickle_ice_) { sdp_without_cands_ = remote_sdp; ice_transmission_list_[remote_user_id]->SendAnswer(); @@ -867,7 +904,26 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) { std::string remote_sdp = msg.remote_sdp; if (ice_transmission_list_.find(remote_user_id) != ice_transmission_list_.end()) { - ice_transmission_list_[remote_user_id]->SetRemoteSdp(remote_sdp); + int ret = + ice_transmission_list_[remote_user_id]->SetRemoteSdp(remote_sdp); + if (0 != ret) { + Leave(remote_transmission_id_); + break; + } else { + std::vector negotiated_payload_types = + ice_transmission_list_[remote_user_id] + ->GetNegotiatedCapabilities(); + if (0 != CreateVideoCodec(RtpPacket::PAYLOAD_TYPE::AV1 == + negotiated_payload_types[0], + hardware_acceleration_)) { + LOG_ERROR("Create video codec failed"); + } + + if (0 != CreateAudioCodec()) { + LOG_ERROR("Create audio codec failed"); + } + } + if (trickle_ice_) { sdp_without_cands_ = remote_sdp; ice_transmission_list_[remote_user_id]->GatherCandidates(); diff --git a/src/pc/peer_connection.h b/src/pc/peer_connection.h index 7e4c145..99d2b34 100644 --- a/src/pc/peer_connection.h +++ b/src/pc/peer_connection.h @@ -103,7 +103,7 @@ class PeerConnection { private: int Login(); - int CreateVideoCodec(bool hardware_acceleration); + int CreateVideoCodec(bool av1, bool hardware_acceleration); int CreateAudioCodec(); void ProcessSignal(const std::string &signal); @@ -111,6 +111,8 @@ class PeerConnection { int RequestTransmissionMemberList(const std::string &transmission_id, const std::string &password); + int NegotiationFailed(); + private: void StartIceWorker(); void StopIceWorker(); @@ -150,6 +152,7 @@ class PeerConnection { unsigned int ws_connection_id_ = 0; bool offer_peer_ = false; std::string user_id_ = ""; + std::string remote_user_id_ = ""; std::string local_transmission_id_ = ""; std::string remote_transmission_id_ = ""; std::vector user_id_list_; diff --git a/src/transmission/ice_transmission.cpp b/src/transmission/ice_transmission.cpp index bc8c94a..6fdd321 100644 --- a/src/transmission/ice_transmission.cpp +++ b/src/transmission/ice_transmission.cpp @@ -68,10 +68,8 @@ int IceTransmission::InitIceTransmission( // data_inbound_bitrate / 1000, data_outbound_bitrate / 1000, // total_inbound_bitrate / 1000, total_outbound_bitrate / 1000); }); + video_codec_payload_type_ = video_codec_payload_type; - video_rtp_codec_ = std::make_unique(video_codec_payload_type); - audio_rtp_codec_ = std::make_unique(RtpPacket::PAYLOAD_TYPE::OPUS); - data_rtp_codec_ = std::make_unique(RtpPacket::PAYLOAD_TYPE::DATA); rtp_video_receiver_ = std::make_unique(); // rr sender @@ -356,6 +354,13 @@ int IceTransmission::DestroyIceTransmission() { return ice_agent_->DestroyIceAgent(); } +int IceTransmission::CreateMediaCodec() { + video_rtp_codec_ = std::make_unique(negotiated_video_pt_); + audio_rtp_codec_ = std::make_unique(negotiated_audio_pt_); + data_rtp_codec_ = std::make_unique(negotiated_data_pt_); + return 0; +} + int IceTransmission::SetTransmissionId(const std::string &transmission_id) { transmission_id_ = transmission_id; @@ -432,24 +437,24 @@ int IceTransmission::AppendLocalCapabilitiesToOffer( const std::string &remote_sdp) { std::string preferred_video_pt; std::string to_replace = "ICE/SDP"; - std::string video_capabilities = "UDP/TLS/RTP/SAVPF"; + std::string video_capabilities = "UDP/TLS/RTP/SAVPF "; std::string audio_capabilities = "UDP/TLS/RTP/SAVPF 111"; std::string data_capabilities = "UDP/TLS/RTP/SAVPF 127"; switch (video_codec_payload_type_) { case RtpPacket::PAYLOAD_TYPE::H264: { preferred_video_pt = std::to_string(RtpPacket::PAYLOAD_TYPE::H264); - video_capabilities = preferred_video_pt + " 97 98 99"; + video_capabilities += preferred_video_pt + " 97 98 99"; break; } case RtpPacket::PAYLOAD_TYPE::AV1: { preferred_video_pt = std::to_string(RtpPacket::PAYLOAD_TYPE::AV1); - video_capabilities = preferred_video_pt + " 96 97 98"; + video_capabilities += preferred_video_pt + " 96 97 98"; break; } default: { preferred_video_pt = std::to_string(RtpPacket::PAYLOAD_TYPE::H264); - video_capabilities = preferred_video_pt + " 97 98 99"; + video_capabilities += preferred_video_pt + " 97 98 99"; break; } } @@ -543,18 +548,18 @@ std::string IceTransmission::GetRemoteCapabilities( std::size_t candidate_start = data_end; if (!remote_capabilities_got_) { - if (NegotiateVideoPayloadType(remote_sdp)) { - remote_capabilities_got_ = true; + if (!NegotiateVideoPayloadType(remote_sdp)) { return std::string(); } - if (NegotiateAudioPayloadType(remote_sdp)) { - remote_capabilities_got_ = true; + if (!NegotiateAudioPayloadType(remote_sdp)) { return std::string(); } - if (NegotiateDataPayloadType(remote_sdp)) { - remote_capabilities_got_ = true; + if (!NegotiateDataPayloadType(remote_sdp)) { return std::string(); } + + CreateMediaCodec(); + remote_capabilities_got_ = true; } @@ -584,6 +589,7 @@ std::string IceTransmission::GetRemoteCapabilities( bool IceTransmission::NegotiateVideoPayloadType(const std::string &remote_sdp) { std::string remote_video_capabilities; + std::string local_video_capabilities; std::string remote_prefered_video_pt; std::size_t start = @@ -600,6 +606,17 @@ bool IceTransmission::NegotiateVideoPayloadType(const std::string &remote_sdp) { } LOG_INFO("remote video capabilities [{}]", remote_video_capabilities.c_str()); + for (size_t index = 0; index < support_video_payload_types_.size(); ++index) { + if (index == support_video_payload_types_.size() - 1) { + local_video_capabilities += + std::to_string(support_video_payload_types_[index]); + } else { + local_video_capabilities += + std::to_string(support_video_payload_types_[index]) + " "; + } + } + LOG_INFO("local video capabilities [{}]", local_video_capabilities.c_str()); + std::size_t prefered_pt_start = 0; while (prefered_pt_start <= remote_video_capabilities.length()) { @@ -762,6 +779,11 @@ bool IceTransmission::NegotiateDataPayloadType(const std::string &remote_sdp) { } } +std::vector +IceTransmission::GetNegotiatedCapabilities() { + return {negotiated_video_pt_, negotiated_audio_pt_, negotiated_data_pt_}; +} + int IceTransmission::SendData(DATA_TYPE type, const char *data, size_t size) { if (state_ != NICE_COMPONENT_STATE_CONNECTED && state_ != NICE_COMPONENT_STATE_READY) { diff --git a/src/transmission/ice_transmission.h b/src/transmission/ice_transmission.h index c638470..fefdda5 100644 --- a/src/transmission/ice_transmission.h +++ b/src/transmission/ice_transmission.h @@ -103,6 +103,8 @@ class IceTransmission { int SendAnswer(); + std::vector GetNegotiatedCapabilities(); + private: int AppendLocalCapabilitiesToOffer(const std::string &remote_sdp); int AppendLocalCapabilitiesToAnswer(const std::string &remote_sdp); @@ -112,6 +114,8 @@ class IceTransmission { bool NegotiateAudioPayloadType(const std::string &remote_sdp); bool NegotiateDataPayloadType(const std::string &remote_sdp); + int CreateMediaCodec(); + private: uint8_t CheckIsRtcpPacket(const char *buffer, size_t size); uint8_t CheckIsVideoPacket(const char *buffer, size_t size);