[feat] support using negotiated sdp to create media codecs

src/pc/peer_connection.cpp

@@ -243,14 +243,6 @@ int PeerConnection::Init(PeerConnectionParams params,
 
   StartIceWorker();
 
-  if (0 != CreateVideoCodec(hardware_acceleration_)) {
-    LOG_ERROR("Create video codec failed");
-  }
-
-  if (0 != CreateAudioCodec()) {
-    LOG_ERROR("Create audio codec failed");
-  }
-
   // do {
   // } while (SignalStatus::SignalConnected != GetSignalStatus());
 
@@ -262,7 +254,7 @@ int PeerConnection::Init(PeerConnectionParams params,
   return 0;
 }
 
-int PeerConnection::CreateVideoCodec(bool hardware_acceleration) {
+int PeerConnection::CreateVideoCodec(bool av1, bool hardware_acceleration) {
   if (video_codec_inited_) {
     return 0;
   }
@@ -412,6 +404,30 @@ int PeerConnection::Join(const std::string &transmission_id,
   return ret;
 }
 
+int PeerConnection::NegotiationFailed() {
+  if (SignalStatus::SignalConnected != GetSignalStatus()) {
+    LOG_ERROR("Signal not connected");
+    return -1;
+  }
+
+  json message = {{"type", "negotiation_failed"},
+                  {"user_id", user_id_},
+                  {"transmission_id", local_transmission_id_}};
+  if (ws_transport_) {
+    ws_transport_->Send(message.dump());
+    LOG_INFO(
+        "[{}] sends negotiation failed notification to [{}] for transmission "
+        "id [{}]",
+        user_id_, remote_user_id_, local_transmission_id_);
+  }
+
+  IceWorkMsg msg;
+  msg.type = IceWorkMsg::Type::Destroy;
+  PushIceWorkMsg(msg);
+
+  return 0;
+}
+
 int PeerConnection::Leave(const std::string &transmission_id) {
   if (SignalStatus::SignalConnected != GetSignalStatus()) {
     LOG_ERROR("Signal not connected");
@@ -656,6 +672,8 @@ void PeerConnection::ProcessSignal(const std::string &signal) {
     case "offer"_H: {
       std::string transmission_id = j["transmission_id"].get<std::string>();
       std::string remote_user_id = j["remote_user_id"].get<std::string>();
+      remote_user_id_ = remote_user_id;
+
       if (j.contains("sdp")) {
         std::string remote_sdp = j["sdp"].get<std::string>();
         LOG_INFO("[{}] receive offer from [{}]", user_id_, remote_user_id);
@@ -676,6 +694,7 @@ void PeerConnection::ProcessSignal(const std::string &signal) {
     case "answer"_H: {
       std::string transmission_id = j["transmission_id"].get<std::string>();
       std::string remote_user_id = j["remote_user_id"].get<std::string>();
+      remote_user_id_ = remote_user_id;
 
       if (j.contains("sdp")) {
         std::string remote_sdp = j["sdp"].get<std::string>();
@@ -853,7 +872,25 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) {
       }
 
       std::string remote_sdp = msg.remote_sdp;
-      ice_transmission_list_[remote_user_id]->SetRemoteSdp(remote_sdp);
+      int ret =
+          ice_transmission_list_[remote_user_id]->SetRemoteSdp(remote_sdp);
+      if (0 != ret) {
+        NegotiationFailed();
+        break;
+      } else {
+        std::vector<RtpPacket::PAYLOAD_TYPE> negotiated_payload_types =
+            ice_transmission_list_[remote_user_id]->GetNegotiatedCapabilities();
+        if (0 != CreateVideoCodec(RtpPacket::PAYLOAD_TYPE::AV1 ==
+                                      negotiated_payload_types[0],
+                                  hardware_acceleration_)) {
+          LOG_ERROR("Create video codec failed");
+        }
+
+        if (0 != CreateAudioCodec()) {
+          LOG_ERROR("Create audio codec failed");
+        }
+      }
+
       if (trickle_ice_) {
         sdp_without_cands_ = remote_sdp;
         ice_transmission_list_[remote_user_id]->SendAnswer();
@@ -867,7 +904,26 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) {
       std::string remote_sdp = msg.remote_sdp;
       if (ice_transmission_list_.find(remote_user_id) !=
           ice_transmission_list_.end()) {
-        ice_transmission_list_[remote_user_id]->SetRemoteSdp(remote_sdp);
+        int ret =
+            ice_transmission_list_[remote_user_id]->SetRemoteSdp(remote_sdp);
+        if (0 != ret) {
+          Leave(remote_transmission_id_);
+          break;
+        } else {
+          std::vector<RtpPacket::PAYLOAD_TYPE> negotiated_payload_types =
+              ice_transmission_list_[remote_user_id]
+                  ->GetNegotiatedCapabilities();
+          if (0 != CreateVideoCodec(RtpPacket::PAYLOAD_TYPE::AV1 ==
+                                        negotiated_payload_types[0],
+                                    hardware_acceleration_)) {
+            LOG_ERROR("Create video codec failed");
+          }
+
+          if (0 != CreateAudioCodec()) {
+            LOG_ERROR("Create audio codec failed");
+          }
+        }
+
         if (trickle_ice_) {
           sdp_without_cands_ = remote_sdp;
           ice_transmission_list_[remote_user_id]->GatherCandidates();

src/pc/peer_connection.h

@@ -103,7 +103,7 @@ class PeerConnection {
  private:
   int Login();
 
-  int CreateVideoCodec(bool hardware_acceleration);
+  int CreateVideoCodec(bool av1, bool hardware_acceleration);
   int CreateAudioCodec();
 
   void ProcessSignal(const std::string &signal);
@@ -111,6 +111,8 @@ class PeerConnection {
   int RequestTransmissionMemberList(const std::string &transmission_id,
                                     const std::string &password);
 
+  int NegotiationFailed();
+
  private:
   void StartIceWorker();
   void StopIceWorker();
@@ -150,6 +152,7 @@ class PeerConnection {
   unsigned int ws_connection_id_ = 0;
   bool offer_peer_ = false;
   std::string user_id_ = "";
+  std::string remote_user_id_ = "";
   std::string local_transmission_id_ = "";
   std::string remote_transmission_id_ = "";
   std::vector<std::string> user_id_list_;

src/transmission/ice_transmission.cpp

@@ -68,10 +68,8 @@ int IceTransmission::InitIceTransmission(
         //     data_inbound_bitrate / 1000, data_outbound_bitrate / 1000,
         //     total_inbound_bitrate / 1000, total_outbound_bitrate / 1000);
       });
+
   video_codec_payload_type_ = video_codec_payload_type;
-  video_rtp_codec_ = std::make_unique<RtpCodec>(video_codec_payload_type);
-  audio_rtp_codec_ = std::make_unique<RtpCodec>(RtpPacket::PAYLOAD_TYPE::OPUS);
-  data_rtp_codec_ = std::make_unique<RtpCodec>(RtpPacket::PAYLOAD_TYPE::DATA);
 
   rtp_video_receiver_ = std::make_unique<RtpVideoReceiver>();
   // rr sender
@@ -356,6 +354,13 @@ int IceTransmission::DestroyIceTransmission() {
   return ice_agent_->DestroyIceAgent();
 }
 
+int IceTransmission::CreateMediaCodec() {
+  video_rtp_codec_ = std::make_unique<RtpCodec>(negotiated_video_pt_);
+  audio_rtp_codec_ = std::make_unique<RtpCodec>(negotiated_audio_pt_);
+  data_rtp_codec_ = std::make_unique<RtpCodec>(negotiated_data_pt_);
+  return 0;
+}
+
 int IceTransmission::SetTransmissionId(const std::string &transmission_id) {
   transmission_id_ = transmission_id;
 
@@ -432,24 +437,24 @@ int IceTransmission::AppendLocalCapabilitiesToOffer(
     const std::string &remote_sdp) {
   std::string preferred_video_pt;
   std::string to_replace = "ICE/SDP";
-  std::string video_capabilities = "UDP/TLS/RTP/SAVPF";
+  std::string video_capabilities = "UDP/TLS/RTP/SAVPF ";
   std::string audio_capabilities = "UDP/TLS/RTP/SAVPF 111";
   std::string data_capabilities = "UDP/TLS/RTP/SAVPF 127";
 
   switch (video_codec_payload_type_) {
     case RtpPacket::PAYLOAD_TYPE::H264: {
       preferred_video_pt = std::to_string(RtpPacket::PAYLOAD_TYPE::H264);
-      video_capabilities = preferred_video_pt + " 97 98 99";
+      video_capabilities += preferred_video_pt + " 97 98 99";
       break;
     }
     case RtpPacket::PAYLOAD_TYPE::AV1: {
       preferred_video_pt = std::to_string(RtpPacket::PAYLOAD_TYPE::AV1);
-      video_capabilities = preferred_video_pt + " 96 97 98";
+      video_capabilities += preferred_video_pt + " 96 97 98";
       break;
     }
     default: {
       preferred_video_pt = std::to_string(RtpPacket::PAYLOAD_TYPE::H264);
-      video_capabilities = preferred_video_pt + " 97 98 99";
+      video_capabilities += preferred_video_pt + " 97 98 99";
       break;
     }
   }
@@ -543,18 +548,18 @@ std::string IceTransmission::GetRemoteCapabilities(
   std::size_t candidate_start = data_end;
 
   if (!remote_capabilities_got_) {
-    if (NegotiateVideoPayloadType(remote_sdp)) {
+    if (!NegotiateVideoPayloadType(remote_sdp)) {
-      remote_capabilities_got_ = true;
       return std::string();
     }
-    if (NegotiateAudioPayloadType(remote_sdp)) {
+    if (!NegotiateAudioPayloadType(remote_sdp)) {
-      remote_capabilities_got_ = true;
       return std::string();
     }
-    if (NegotiateDataPayloadType(remote_sdp)) {
+    if (!NegotiateDataPayloadType(remote_sdp)) {
-      remote_capabilities_got_ = true;
       return std::string();
     }
+
+    CreateMediaCodec();
+
     remote_capabilities_got_ = true;
   }
 
@@ -584,6 +589,7 @@ std::string IceTransmission::GetRemoteCapabilities(
 
 bool IceTransmission::NegotiateVideoPayloadType(const std::string &remote_sdp) {
   std::string remote_video_capabilities;
+  std::string local_video_capabilities;
   std::string remote_prefered_video_pt;
 
   std::size_t start =
@@ -600,6 +606,17 @@ bool IceTransmission::NegotiateVideoPayloadType(const std::string &remote_sdp) {
   }
   LOG_INFO("remote video capabilities [{}]", remote_video_capabilities.c_str());
 
+  for (size_t index = 0; index < support_video_payload_types_.size(); ++index) {
+    if (index == support_video_payload_types_.size() - 1) {
+      local_video_capabilities +=
+          std::to_string(support_video_payload_types_[index]);
+    } else {
+      local_video_capabilities +=
+          std::to_string(support_video_payload_types_[index]) + " ";
+    }
+  }
+  LOG_INFO("local video capabilities [{}]", local_video_capabilities.c_str());
+
   std::size_t prefered_pt_start = 0;
 
   while (prefered_pt_start <= remote_video_capabilities.length()) {
@@ -762,6 +779,11 @@ bool IceTransmission::NegotiateDataPayloadType(const std::string &remote_sdp) {
   }
 }
 
+std::vector<RtpPacket::PAYLOAD_TYPE>
+IceTransmission::GetNegotiatedCapabilities() {
+  return {negotiated_video_pt_, negotiated_audio_pt_, negotiated_data_pt_};
+}
+
 int IceTransmission::SendData(DATA_TYPE type, const char *data, size_t size) {
   if (state_ != NICE_COMPONENT_STATE_CONNECTED &&
       state_ != NICE_COMPONENT_STATE_READY) {

src/transmission/ice_transmission.h

@@ -103,6 +103,8 @@ class IceTransmission {
 
   int SendAnswer();
 
+  std::vector<RtpPacket::PAYLOAD_TYPE> GetNegotiatedCapabilities();
+
  private:
   int AppendLocalCapabilitiesToOffer(const std::string &remote_sdp);
   int AppendLocalCapabilitiesToAnswer(const std::string &remote_sdp);
@@ -112,6 +114,8 @@ class IceTransmission {
   bool NegotiateAudioPayloadType(const std::string &remote_sdp);
   bool NegotiateDataPayloadType(const std::string &remote_sdp);
 
+  int CreateMediaCodec();
+
  private:
   uint8_t CheckIsRtcpPacket(const char *buffer, size_t size);
   uint8_t CheckIsVideoPacket(const char *buffer, size_t size);