[fix] move codecs from pc into transmission module

src/interface/x.h

@@ -115,11 +115,12 @@ DLLAPI int JoinConnection(PeerPtr* peer_ptr, const char* transmission_id,
 
 DLLAPI int LeaveConnection(PeerPtr* peer_ptr, const char* transmission_id);
 
-DLLAPI int SendData(PeerPtr* peer_ptr, DATA_TYPE data_type, const char* data,
-                    size_t size);
-
 DLLAPI int SendVideoFrame(PeerPtr* peer_ptr, const XVideoFrame* video_frame);
 
+DLLAPI int SendAudioFrame(PeerPtr* peer_ptr, const char* data, size_t size);
+
+DLLAPI int SendData(PeerPtr* peer_ptr, const char* data, size_t size);
+
 #ifdef __cplusplus
 }
 #endif

src/pc/peer_connection.cpp

@@ -7,11 +7,6 @@
 #include "log.h"
 #include "nlohmann/json.hpp"
 
-#if __APPLE__
-#else
-#include "nvcodec_api.h"
-#endif
-
 using nlohmann::json;
 
 PeerConnection::PeerConnection() {}
@@ -22,10 +17,6 @@ PeerConnection::~PeerConnection() {
     nv12_data_ = nullptr;
   }
 
-  video_codec_inited_ = false;
-  audio_codec_inited_ = false;
-  load_nvcodec_dll_success = false;
-
   user_data_ = nullptr;
 }
 
@@ -143,41 +134,6 @@ int PeerConnection::Init(PeerConnectionParams params,
     }
   };
 
-  on_receive_video_ = [this](const char *data, size_t size,
-                             const std::string &user_id) {
-    int num_frame_returned = video_decoder_->Decode(
-        (uint8_t *)data, size, [this, user_id](VideoFrame video_frame) {
-          if (on_receive_video_frame_) {
-            XVideoFrame x_video_frame;
-            x_video_frame.data = (const char *)video_frame.Buffer();
-            x_video_frame.width = video_frame.Width();
-            x_video_frame.height = video_frame.Height();
-            x_video_frame.size = video_frame.Size();
-            on_receive_video_frame_(&x_video_frame, user_id.data(),
-                                    user_id.size(), user_data_);
-          }
-        });
-  };
-
-  on_receive_audio_ = [this](const char *data, size_t size,
-                             const std::string &user_id) {
-    int num_frame_returned = audio_decoder_->Decode(
-        (uint8_t *)data, size, [this, user_id](uint8_t *data, int size) {
-          if (on_receive_audio_buffer_) {
-            on_receive_audio_buffer_((const char *)data, size, user_id.data(),
-                                     user_id.size(), user_data_);
-          }
-        });
-  };
-
-  on_receive_data_ = [this](const char *data, size_t size,
-                            const std::string &user_id) {
-    if (on_receive_data_buffer_) {
-      on_receive_data_buffer_(data, size, user_id.data(), user_id.size(),
-                              user_data_);
-    }
-  };
-
   on_ice_status_change_ = [this](std::string ice_status,
                                  const std::string &user_id) {
     if ("connecting" == ice_status) {
@@ -277,102 +233,6 @@ int PeerConnection::Init(PeerConnectionParams params,
   return 0;
 }
 
-int PeerConnection::CreateVideoCodec(bool av1, bool hardware_acceleration) {
-  if (video_codec_inited_) {
-    return 0;
-  }
-
-  hardware_acceleration_ = hardware_acceleration;
-
-  if (av1) {
-    if (hardware_acceleration_) {
-      hardware_acceleration_ = false;
-      LOG_WARN("Only support software codec for AV1");
-    }
-    video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, true);
-    video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, true);
-  } else {
-#ifdef __APPLE__
-    if (hardware_acceleration_) {
-      hardware_acceleration_ = false;
-      LOG_WARN(
-          "MacOS not support hardware acceleration, use default software "
-          "codec");
-      video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
-      video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
-    } else {
-      video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
-      video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
-    }
-#else
-    if (hardware_acceleration_) {
-      if (0 == LoadNvCodecDll()) {
-        load_nvcodec_dll_success = true;
-        video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(true, false);
-        video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(true, false);
-      } else {
-        LOG_WARN(
-            "Hardware accelerated codec not available, use default software "
-            "codec");
-        video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
-        video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
-      }
-    } else {
-      video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
-      video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
-    }
-#endif
-  }
-
-  if (!video_encoder_) {
-    video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
-    LOG_ERROR("Create encoder failed, try to use software H.264 encoder");
-  }
-  if (!video_encoder_ || 0 != video_encoder_->Init()) {
-    LOG_ERROR("Encoder init failed");
-    return -1;
-  }
-
-  if (!video_decoder_) {
-    video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
-    LOG_ERROR("Create decoder failed, try to use software H.264 decoder");
-  }
-  if (!video_decoder_ || video_decoder_->Init()) {
-    LOG_ERROR("Decoder init failed");
-    return -1;
-  }
-
-  video_codec_inited_ = true;
-  LOG_INFO("Create video codec [{}|{}] finish",
-           video_encoder_->GetEncoderName(), video_decoder_->GetDecoderName());
-
-  return 0;
-}
-
-int PeerConnection::CreateAudioCodec() {
-  if (audio_codec_inited_) {
-    return 0;
-  }
-
-  audio_encoder_ = std::make_unique<AudioEncoder>(AudioEncoder(48000, 1, 480));
-  if (!audio_encoder_ || 0 != audio_encoder_->Init()) {
-    LOG_ERROR("Audio encoder init failed");
-    return -1;
-  }
-
-  audio_decoder_ = std::make_unique<AudioDecoder>(AudioDecoder(48000, 1, 480));
-  if (!audio_decoder_ || 0 != audio_decoder_->Init()) {
-    LOG_ERROR("Audio decoder init failed");
-    return -1;
-  }
-
-  audio_codec_inited_ = true;
-  LOG_INFO("Create audio codec [{}|{}] finish",
-           audio_encoder_->GetEncoderName(), audio_decoder_->GetDecoderName());
-
-  return 0;
-}
-
 int PeerConnection::Login() {
   if (WsStatus::WsOpened != ws_status_) {
     LOG_ERROR("Websocket not opened");
@@ -494,12 +354,6 @@ int PeerConnection::Destroy() {
     nv12_data_ = nullptr;
   }
 
-#ifdef __APPLE__
-#else
-  if (hardware_acceleration_ && load_nvcodec_dll_success) {
-    ReleaseNvCodecDll();
-  }
-#endif
   return 0;
 }
 
@@ -527,36 +381,18 @@ SignalStatus PeerConnection::GetSignalStatus() {
   return signal_status_;
 }
 
-int PeerConnection::SendVideoData(const char *data, size_t size) {
+// media send
+int PeerConnection::SendVideoData(const XVideoFrame *video_frame) {
   if (ice_transmission_list_.empty()) {
     return -1;
   }
 
-  if (b_force_i_frame_) {
+  for (auto &ice_trans : ice_transmission_list_) {
-    LOG_INFO("Force I frame");
+    if (!is_ice_transmission_ready_[ice_trans.first]) {
-    video_encoder_->ForceIdr();
+      continue;
-    b_force_i_frame_ = false;
+    }
-  }
 
-  int ret = video_encoder_->Encode(
+    ice_trans.second->SendVideoData(video_frame);
-      (uint8_t *)data, size,
-      [this](char *encoded_frame, size_t size,
-             VideoEncoder::VideoFrameType frame_type) -> int {
-        for (auto &ice_trans : ice_transmission_list_) {
-          if (!is_ice_transmission_ready_[ice_trans.first]) {
-            continue;
-          }
-          // LOG_ERROR("Send frame size: [{}]", size);
-          ice_trans.second->SendVideoData(
-              static_cast<IceTransmission::VideoFrameType>(frame_type),
-              encoded_frame, size);
-        }
-        return 0;
-      });
-
-  if (0 != ret) {
-    LOG_ERROR("Encode failed");
-    return -1;
   }
 
   return 0;
@@ -567,19 +403,12 @@ int PeerConnection::SendAudioData(const char *data, size_t size) {
     return -1;
   }
 
-  int ret = audio_encoder_->Encode(
+  for (auto &ice_trans : ice_transmission_list_) {
-      (uint8_t *)data, size,
+    if (!is_ice_transmission_ready_[ice_trans.first]) {
-      [this](char *encoded_audio_buffer, size_t size) -> int {
+      continue;
-        for (auto &ice_trans : ice_transmission_list_) {
+    }
-          if (!is_ice_transmission_ready_[ice_trans.first]) {
+    ice_trans.second->SendAudioData(data, size);
-            continue;
+  }
-          }
-          // LOG_ERROR("opus frame size: [{}]", size);
-          ice_trans.second->SendData(IceTransmission::DATA_TYPE::AUDIO,
-                                     encoded_audio_buffer, size);
-        }
-        return 0;
-      });
 
   return 0;
 }
@@ -589,46 +418,11 @@ int PeerConnection::SendUserData(const char *data, size_t size) {
     if (!is_ice_transmission_ready_[ice_trans.first]) {
       continue;
     }
-    ice_trans.second->SendData(IceTransmission::DATA_TYPE::DATA, data, size);
+    ice_trans.second->SendUserData(data, size);
   }
   return 0;
 }
 
-int PeerConnection::SendVideoData(const XVideoFrame *video_frame) {
-  if (ice_transmission_list_.empty()) {
-    return -1;
-  }
-
-  if (b_force_i_frame_) {
-    video_encoder_->ForceIdr();
-    LOG_INFO("Force I frame");
-    b_force_i_frame_ = false;
-  }
-
-  int ret = video_encoder_->Encode(
-      video_frame,
-      [this](char *encoded_frame, size_t size,
-             VideoEncoder::VideoFrameType frame_type) -> int {
-        for (auto &ice_trans : ice_transmission_list_) {
-          if (!is_ice_transmission_ready_[ice_trans.first]) {
-            continue;
-          }
-          // LOG_ERROR("Send frame size: [{}]", size);
-          ice_trans.second->SendVideoData(
-              static_cast<IceTransmission::VideoFrameType>(frame_type),
-              encoded_frame, size);
-        }
-        return 0;
-      });
-
-  if (0 != ret) {
-    LOG_ERROR("Encode failed");
-    return -1;
-  }
-
-  return 0;
-}
-
 void PeerConnection::ProcessSignal(const std::string &signal) {
   auto j = json::parse(signal);
   std::string type = j["type"];
@@ -842,20 +636,21 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) {
 
       for (auto &remote_user_id : user_id_list) {
         ice_transmission_list_[remote_user_id] =
-            std::make_unique<IceTransmission>(true, transmission_id, user_id_,
+            std::make_unique<IceTransmission>(
-                                              remote_user_id, ws_transport_,
+                true, transmission_id, user_id_, remote_user_id, ws_transport_,
-                                              on_ice_status_change_);
+                on_ice_status_change_, user_data_);
 
         ice_transmission_list_[remote_user_id]->SetLocalCapabilities(
-            trickle_ice_, reliable_ice_, enable_turn_, false,
+            hardware_acceleration_, trickle_ice_, reliable_ice_, enable_turn_,
-            video_payload_types_, audio_payload_types_);
+            false, video_payload_types_, audio_payload_types_);
 
         ice_transmission_list_[remote_user_id]->SetOnReceiveVideoFunc(
-            on_receive_video_);
+            on_receive_video_frame_);
         ice_transmission_list_[remote_user_id]->SetOnReceiveAudioFunc(
-            on_receive_audio_);
+            on_receive_audio_buffer_);
         ice_transmission_list_[remote_user_id]->SetOnReceiveDataFunc(
-            on_receive_data_);
+            on_receive_data_buffer_);
+
         ice_transmission_list_[remote_user_id]->SetOnReceiveNetStatusReportFunc(
             on_net_status_report_);
 
@@ -889,20 +684,21 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) {
           ice_transmission_list_.find(remote_user_id)) {
         // Enable TURN for answer peer by default
         ice_transmission_list_[remote_user_id] =
-            std::make_unique<IceTransmission>(false, transmission_id, user_id_,
+            std::make_unique<IceTransmission>(
-                                              remote_user_id, ws_transport_,
+                false, transmission_id, user_id_, remote_user_id, ws_transport_,
-                                              on_ice_status_change_);
+                on_ice_status_change_, user_data_);
 
         ice_transmission_list_[remote_user_id]->SetLocalCapabilities(
-            trickle_ice_, reliable_ice_, enable_turn_, false,
+            hardware_acceleration_, trickle_ice_, reliable_ice_, enable_turn_,
-            video_payload_types_, audio_payload_types_);
+            false, video_payload_types_, audio_payload_types_);
 
         ice_transmission_list_[remote_user_id]->SetOnReceiveVideoFunc(
-            on_receive_video_);
+            on_receive_video_frame_);
         ice_transmission_list_[remote_user_id]->SetOnReceiveAudioFunc(
-            on_receive_audio_);
+            on_receive_audio_buffer_);
         ice_transmission_list_[remote_user_id]->SetOnReceiveDataFunc(
-            on_receive_data_);
+            on_receive_data_buffer_);
+
         ice_transmission_list_[remote_user_id]->SetOnReceiveNetStatusReportFunc(
             on_net_status_report_);
 
@@ -921,18 +717,6 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) {
       if (0 != ret) {
         NegotiationFailed();
         break;
-      } else {
-        std::vector<RtpPacket::PAYLOAD_TYPE> negotiated_payload_types =
-            ice_transmission_list_[remote_user_id]->GetNegotiatedCapabilities();
-        if (0 != CreateVideoCodec(RtpPacket::PAYLOAD_TYPE::AV1 ==
-                                      negotiated_payload_types[0],
-                                  hardware_acceleration_)) {
-          LOG_ERROR("Create video codec failed");
-        }
-
-        if (0 != CreateAudioCodec()) {
-          LOG_ERROR("Create audio codec failed");
-        }
       }
 
       if (trickle_ice_) {
@@ -953,19 +737,6 @@ void PeerConnection::ProcessIceWorkMsg(const IceWorkMsg &msg) {
         if (0 != ret) {
           Leave(remote_transmission_id_);
           break;
-        } else {
-          std::vector<RtpPacket::PAYLOAD_TYPE> negotiated_payload_types =
-              ice_transmission_list_[remote_user_id]
-                  ->GetNegotiatedCapabilities();
-          if (0 != CreateVideoCodec(RtpPacket::PAYLOAD_TYPE::AV1 ==
-                                        negotiated_payload_types[0],
-                                    hardware_acceleration_)) {
-            LOG_ERROR("Create video codec failed");
-          }
-
-          if (0 != CreateAudioCodec()) {
-            LOG_ERROR("Create audio codec failed");
-          }
         }
 
         if (trickle_ice_) {

src/pc/peer_connection.h

@@ -104,9 +104,6 @@ class PeerConnection {
  private:
   int Login();
 
-  int CreateVideoCodec(bool av1, bool hardware_acceleration);
-  int CreateAudioCodec();
-
   void ProcessSignal(const std::string &signal);
 
   int RequestTransmissionMemberList(const std::string &transmission_id,
@@ -168,12 +165,7 @@ class PeerConnection {
   std::map<std::string, std::unique_ptr<IceTransmission>>
       ice_transmission_list_;
   std::map<std::string, bool> is_ice_transmission_ready_;
-  std::function<void(const char *, size_t, const std::string &)>
+
-      on_receive_video_ = nullptr;
-  std::function<void(const char *, size_t, const std::string &)>
-      on_receive_audio_ = nullptr;
-  std::function<void(const char *, size_t, const std::string &)>
-      on_receive_data_ = nullptr;
   std::function<void(std::string, const std::string &)> on_ice_status_change_ =
       nullptr;
   std::function<void(const std::string &, IceTransmission::TraversalType,
@@ -199,18 +191,7 @@ class PeerConnection {
   std::string password_;
 
  private:
-  std::unique_ptr<VideoEncoder> video_encoder_ = nullptr;
-  std::unique_ptr<VideoDecoder> video_decoder_ = nullptr;
-  bool hardware_accelerated_encode_ = false;
-  bool hardware_accelerated_decode_ = false;
   bool b_force_i_frame_ = false;
-  bool video_codec_inited_ = false;
-  bool load_nvcodec_dll_success = false;
-
- private:
-  std::unique_ptr<AudioEncoder> audio_encoder_ = nullptr;
-  std::unique_ptr<AudioDecoder> audio_decoder_ = nullptr;
-  bool audio_codec_inited_ = false;
 
  private:
   std::thread ice_worker_;

src/rtc/x_inner.cpp

@@ -98,23 +98,6 @@ int LeaveConnection(PeerPtr *peer_ptr, const char *transmission_id) {
   return 0;
 }
 
-int SendData(PeerPtr *peer_ptr, DATA_TYPE data_type, const char *data,
-             size_t size) {
-  if (!peer_ptr) {
-    LOG_ERROR("peer_ptr not created");
-    return -1;
-  }
-
-  if (DATA_TYPE::VIDEO == data_type) {
-    peer_ptr->peer_connection->SendVideoData(data, size);
-  } else if (DATA_TYPE::AUDIO == data_type) {
-    peer_ptr->peer_connection->SendAudioData(data, size);
-  } else if (DATA_TYPE::DATA == data_type) {
-    peer_ptr->peer_connection->SendUserData(data, size);
-  }
-  return 0;
-}
-
 DLLAPI int SendVideoFrame(PeerPtr *peer_ptr, const XVideoFrame *video_frame) {
   if (!peer_ptr) {
     LOG_ERROR("peer_ptr not created");
@@ -124,9 +107,44 @@ DLLAPI int SendVideoFrame(PeerPtr *peer_ptr, const XVideoFrame *video_frame) {
   if (!video_frame) {
     LOG_ERROR("Invaild video frame");
     return -1;
+  } else if (!video_frame->data || video_frame->size <= 0) {
+    LOG_ERROR("Invaild video frame");
+    return -1;
   }
 
   peer_ptr->peer_connection->SendVideoData(video_frame);
 
+  return 0;
+}
+
+DLLAPI int SendAudioFrame(PeerPtr *peer_ptr, const char *data, size_t size) {
+  if (!peer_ptr) {
+    LOG_ERROR("peer_ptr not created");
+    return -1;
+  }
+
+  if (!data || size <= 0) {
+    LOG_ERROR("Invaild video frame");
+    return -1;
+  }
+
+  peer_ptr->peer_connection->SendAudioData(data, size);
+
+  return 0;
+}
+
+int SendData(PeerPtr *peer_ptr, const char *data, size_t size) {
+  if (!peer_ptr) {
+    LOG_ERROR("peer_ptr not created");
+    return -1;
+  }
+
+  if (!data || size <= 0) {
+    LOG_ERROR("Invaild data");
+    return -1;
+  }
+
+  peer_ptr->peer_connection->SendUserData(data, size);
+
   return 0;
 }

src/transmission/ice_transmission.cpp

@@ -8,26 +8,44 @@
 #include "common.h"
 #include "ikcp.h"
 #include "log.h"
+#if __APPLE__
+#else
+#include "nvcodec_api.h"
+#endif
 
 using nlohmann::json;
 
 IceTransmission::IceTransmission(
     bool offer_peer, std::string &transmission_id, std::string &user_id,
     std::string &remote_user_id, std::shared_ptr<WsClient> ice_ws_transmission,
-    std::function<void(std::string, const std::string &)> on_ice_status_change)
+    std::function<void(std::string, const std::string &)> on_ice_status_change,
+    void *user_data)
     : offer_peer_(offer_peer),
       transmission_id_(transmission_id),
       user_id_(user_id),
       remote_user_id_(remote_user_id),
       ice_ws_transport_(ice_ws_transmission),
-      on_ice_status_change_(on_ice_status_change) {}
+      on_ice_status_change_(on_ice_status_change),
+      user_data_(user_data) {}
 
-IceTransmission::~IceTransmission() {}
+IceTransmission::~IceTransmission() {
+  video_codec_inited_ = false;
+  audio_codec_inited_ = false;
+  load_nvcodec_dll_success_ = false;
+
+#ifdef __APPLE__
+#else
+  if (hardware_acceleration_ && load_nvcodec_dll_success_) {
+    ReleaseNvCodecDll();
+  }
+#endif
+}
 
 int IceTransmission::SetLocalCapabilities(
-    bool use_trickle_ice, bool use_reliable_ice, bool enable_turn,
+    bool hardware_acceleration, bool use_trickle_ice, bool use_reliable_ice,
-    bool force_turn, std::vector<int> &video_payload_types,
+    bool enable_turn, bool force_turn, std::vector<int> &video_payload_types,
     std::vector<int> &audio_payload_types) {
+  hardware_acceleration_ = hardware_acceleration;
   use_trickle_ice_ = use_trickle_ice;
   use_reliable_ice_ = use_reliable_ice;
   enable_turn_ = force_turn;
@@ -81,8 +99,20 @@ int IceTransmission::InitIceTransmission(
       [this](VideoFrame &video_frame) -> void {
         // LOG_ERROR("OnReceiveCompleteFrame {}", video_frame.Size());
         ice_io_statistics_->UpdateVideoInboundBytes(video_frame.Size());
-        on_receive_video_((const char *)video_frame.Buffer(),
+
-                          video_frame.Size(), remote_user_id_);
+        int num_frame_returned = video_decoder_->Decode(
+            (uint8_t *)video_frame.Buffer(), video_frame.Size(),
+            [this](VideoFrame video_frame) {
+              if (on_receive_video_) {
+                XVideoFrame x_video_frame;
+                x_video_frame.data = (const char *)video_frame.Buffer();
+                x_video_frame.width = video_frame.Width();
+                x_video_frame.height = video_frame.Height();
+                x_video_frame.size = video_frame.Size();
+                on_receive_video_(&x_video_frame, remote_user_id_.data(),
+                                  remote_user_id_.size(), user_data_);
+              }
+            });
       });
 
   rtp_video_receiver_->Start();
@@ -126,11 +156,18 @@ int IceTransmission::InitIceTransmission(
 
         return ice_agent_->Send(data, size);
       });
-  rtp_audio_receiver_->SetOnReceiveData(
+  rtp_audio_receiver_->SetOnReceiveData([this](const char *data,
-      [this](const char *data, size_t size) -> void {
+                                               size_t size) -> void {
-        ice_io_statistics_->UpdateAudioInboundBytes(size);
+    ice_io_statistics_->UpdateAudioInboundBytes(size);
-        on_receive_audio_(data, size, remote_user_id_);
+
-      });
+    int num_frame_returned = audio_decoder_->Decode(
+        (uint8_t *)data, size, [this](uint8_t *data, int size) {
+          if (on_receive_audio_) {
+            on_receive_audio_((const char *)data, size, remote_user_id_.data(),
+                              remote_user_id_.size(), user_data_);
+          }
+        });
+  });
 
   rtp_audio_sender_ = std::make_unique<RtpAudioSender>();
   rtp_audio_sender_->SetSendDataFunc(
@@ -174,7 +211,11 @@ int IceTransmission::InitIceTransmission(
   rtp_data_receiver_->SetOnReceiveData(
       [this](const char *data, size_t size) -> void {
         ice_io_statistics_->UpdateDataInboundBytes(size);
-        on_receive_data_(data, size, remote_user_id_);
+
+        if (on_receive_data_) {
+          on_receive_data_(data, size, remote_user_id_.data(),
+                           remote_user_id_.size(), user_data_);
+        }
       });
 
   rtp_data_sender_ = std::make_unique<RtpDataSender>();
@@ -372,6 +413,103 @@ int IceTransmission::CreateMediaCodec() {
   return 0;
 }
 
+int IceTransmission::CreateVideoCodec(RtpPacket::PAYLOAD_TYPE video_pt,
+                                      bool hardware_acceleration) {
+  if (video_codec_inited_) {
+    return 0;
+  }
+
+  hardware_acceleration_ = hardware_acceleration;
+
+  if (RtpPacket::PAYLOAD_TYPE::AV1 == video_pt) {
+    if (hardware_acceleration_) {
+      hardware_acceleration_ = false;
+      LOG_WARN("Only support software codec for AV1");
+    }
+    video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, true);
+    video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, true);
+  } else if (RtpPacket::PAYLOAD_TYPE::H264 == video_pt) {
+#ifdef __APPLE__
+    if (hardware_acceleration_) {
+      hardware_acceleration_ = false;
+      LOG_WARN(
+          "MacOS not support hardware acceleration, use default software "
+          "codec");
+      video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
+      video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
+    } else {
+      video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
+      video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
+    }
+#else
+    if (hardware_acceleration_) {
+      if (0 == LoadNvCodecDll()) {
+        load_nvcodec_dll_success_ = true;
+        video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(true, false);
+        video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(true, false);
+      } else {
+        LOG_WARN(
+            "Hardware accelerated codec not available, use default software "
+            "codec");
+        video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
+        video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
+      }
+    } else {
+      video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
+      video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
+    }
+#endif
+  }
+
+  if (!video_encoder_) {
+    video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
+    LOG_ERROR("Create encoder failed, try to use software H.264 encoder");
+  }
+  if (!video_encoder_ || 0 != video_encoder_->Init()) {
+    LOG_ERROR("Encoder init failed");
+    return -1;
+  }
+
+  if (!video_decoder_) {
+    video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
+    LOG_ERROR("Create decoder failed, try to use software H.264 decoder");
+  }
+  if (!video_decoder_ || video_decoder_->Init()) {
+    LOG_ERROR("Decoder init failed");
+    return -1;
+  }
+
+  video_codec_inited_ = true;
+  LOG_INFO("Create video codec [{}|{}] finish",
+           video_encoder_->GetEncoderName(), video_decoder_->GetDecoderName());
+
+  return 0;
+}
+
+int IceTransmission::CreateAudioCodec() {
+  if (audio_codec_inited_) {
+    return 0;
+  }
+
+  audio_encoder_ = std::make_unique<AudioEncoder>(AudioEncoder(48000, 1, 480));
+  if (!audio_encoder_ || 0 != audio_encoder_->Init()) {
+    LOG_ERROR("Audio encoder init failed");
+    return -1;
+  }
+
+  audio_decoder_ = std::make_unique<AudioDecoder>(AudioDecoder(48000, 1, 480));
+  if (!audio_decoder_ || 0 != audio_decoder_->Init()) {
+    LOG_ERROR("Audio decoder init failed");
+    return -1;
+  }
+
+  audio_codec_inited_ = true;
+  LOG_INFO("Create audio codec [{}|{}] finish",
+           audio_encoder_->GetEncoderName(), audio_decoder_->GetDecoderName());
+
+  return 0;
+}
+
 int IceTransmission::SetTransmissionId(const std::string &transmission_id) {
   transmission_id_ = transmission_id;
 
@@ -570,6 +708,8 @@ std::string IceTransmission::GetRemoteCapabilities(
     }
 
     CreateMediaCodec();
+    CreateVideoCodec(negotiated_video_pt_, hardware_acceleration_);
+    CreateAudioCodec();
 
     remote_capabilities_got_ = true;
   }
@@ -795,7 +935,7 @@ IceTransmission::GetNegotiatedCapabilities() {
   return {negotiated_video_pt_, negotiated_audio_pt_, negotiated_data_pt_};
 }
 
-int IceTransmission::SendData(DATA_TYPE type, const char *data, size_t size) {
+int IceTransmission::SendVideoData(const XVideoFrame *video_frame) {
   if (state_ != NICE_COMPONENT_STATE_CONNECTED &&
       state_ != NICE_COMPONENT_STATE_READY) {
     LOG_ERROR("Ice is not connected, state = [{}]",
@@ -803,36 +943,63 @@ int IceTransmission::SendData(DATA_TYPE type, const char *data, size_t size) {
     return -2;
   }
 
-  std::vector<RtpPacket> packets;
+  if (b_force_i_frame_) {
+    video_encoder_->ForceIdr();
+    LOG_INFO("Force I frame");
+    b_force_i_frame_ = false;
+  }
 
-  if (DATA_TYPE::VIDEO == type) {
+  int ret = video_encoder_->Encode(
-    if (rtp_video_sender_) {
+      video_frame,
-      if (video_rtp_codec_) {
+      [this](char *encoded_frame, size_t size,
-        video_rtp_codec_->Encode((uint8_t *)data, size, packets);
+             VideoEncoder::VideoFrameType frame_type) -> int {
-      }
+        std::vector<RtpPacket> packets;
-      rtp_video_sender_->Enqueue(packets);
+        if (rtp_video_sender_) {
-    }
+          if (video_rtp_codec_) {
-  } else if (DATA_TYPE::AUDIO == type) {
+            video_rtp_codec_->Encode(
-    if (rtp_audio_sender_) {
+                static_cast<RtpCodec::VideoFrameType>(frame_type),
-      if (audio_rtp_codec_) {
+                (uint8_t *)encoded_frame, size, packets);
-        audio_rtp_codec_->Encode((uint8_t *)data, size, packets);
+          }
-        rtp_audio_sender_->Enqueue(packets);
+          rtp_video_sender_->Enqueue(packets);
-      }
+        }
-    }
+
-  } else if (DATA_TYPE::DATA == type) {
+        return 0;
-    if (rtp_data_sender_) {
+      });
-      if (data_rtp_codec_) {
+
-        data_rtp_codec_->Encode((uint8_t *)data, size, packets);
+  if (0 != ret) {
-        rtp_data_sender_->Enqueue(packets);
+    LOG_ERROR("Encode failed");
-      }
+    return -1;
-    }
   }
 
   return 0;
 }
 
-int IceTransmission::SendVideoData(VideoFrameType frame_type, const char *data,
+int IceTransmission::SendAudioData(const char *data, size_t size) {
-                                   size_t size) {
+  if (state_ != NICE_COMPONENT_STATE_CONNECTED &&
+      state_ != NICE_COMPONENT_STATE_READY) {
+    LOG_ERROR("Ice is not connected, state = [{}]",
+              nice_component_state_to_string(state_));
+    return -2;
+  }
+
+  int ret = audio_encoder_->Encode(
+      (uint8_t *)data, size,
+      [this](char *encoded_audio_buffer, size_t size) -> int {
+        if (rtp_audio_sender_) {
+          if (audio_rtp_codec_) {
+            std::vector<RtpPacket> packets;
+            audio_rtp_codec_->Encode((uint8_t *)encoded_audio_buffer, size,
+                                     packets);
+            rtp_audio_sender_->Enqueue(packets);
+          }
+        }
+        return 0;
+      });
+
+  return 0;
+}
+
+int IceTransmission::SendUserData(const char *data, size_t size) {
   if (state_ != NICE_COMPONENT_STATE_CONNECTED &&
       state_ != NICE_COMPONENT_STATE_READY) {
     LOG_ERROR("Ice is not connected, state = [{}]",
@@ -842,13 +1009,11 @@ int IceTransmission::SendVideoData(VideoFrameType frame_type, const char *data,
 
   std::vector<RtpPacket> packets;
 
-  if (rtp_video_sender_) {
+  if (rtp_data_sender_) {
-    if (video_rtp_codec_) {
+    if (data_rtp_codec_) {
-      video_rtp_codec_->Encode(
+      data_rtp_codec_->Encode((uint8_t *)data, size, packets);
-          static_cast<RtpCodec::VideoFrameType>(frame_type), (uint8_t *)data,
+      rtp_data_sender_->Enqueue(packets);
-          size, packets);
     }
-    rtp_video_sender_->Enqueue(packets);
   }
 
   return 0;

src/transmission/ice_transmission.h

@@ -9,6 +9,8 @@
 
 #include <iostream>
 
+#include "audio_decoder.h"
+#include "audio_encoder.h"
 #include "congestion_control.h"
 #include "ice_agent.h"
 #include "io_statistics.h"
@@ -21,11 +23,14 @@
 #include "rtp_packet.h"
 #include "rtp_video_receiver.h"
 #include "rtp_video_sender.h"
+#include "video_decoder_factory.h"
+#include "video_encoder_factory.h"
 #include "ws_client.h"
 
 class IceTransmission {
  public:
   typedef enum { VIDEO = 96, AUDIO = 97, DATA = 127 } DATA_TYPE;
+  typedef enum { H264 = 96, AV1 = 99 } VIDEO_TYPE;
   enum VideoFrameType {
     kEmptyFrame = 0,
     kVideoFrameKey = 3,
@@ -39,12 +44,14 @@ class IceTransmission {
                   std::string &user_id, std::string &remote_user_id,
                   std::shared_ptr<WsClient> ice_ws_transmission,
                   std::function<void(std::string, const std::string &)>
-                      on_ice_status_change);
+                      on_ice_status_change,
+                  void *user_data);
   ~IceTransmission();
 
  public:
-  int SetLocalCapabilities(bool use_trickle_ice, bool use_reliable_ice,
+  int SetLocalCapabilities(bool hardware_acceleration, bool use_trickle_ice,
-                           bool enable_turn, bool force_turn,
+                           bool use_reliable_ice, bool enable_turn,
+                           bool force_turn,
                            std::vector<int> &video_payload_types,
                            std::vector<int> &audio_payload_types);
 
@@ -57,19 +64,22 @@ class IceTransmission {
   int DestroyIceTransmission();
 
   void SetOnReceiveVideoFunc(
-      std::function<void(const char *, size_t, const std::string &)>
+      std::function<void(const XVideoFrame *, const char *, const size_t,
+                         void *)>
           on_receive_video) {
     on_receive_video_ = on_receive_video;
   }
 
   void SetOnReceiveAudioFunc(
-      std::function<void(const char *, size_t, const std::string &)>
+      std::function<void(const char *, size_t, const char *, const size_t,
+                         void *)>
           on_receive_audio) {
     on_receive_audio_ = on_receive_audio;
   }
 
   void SetOnReceiveDataFunc(
-      std::function<void(const char *, size_t, const std::string &)>
+      std::function<void(const char *, size_t, const char *, const size_t,
+                         void *)>
           on_receive_data) {
     on_receive_data_ = on_receive_data;
   }
@@ -87,9 +97,7 @@ class IceTransmission {
 
   int SetTransmissionId(const std::string &transmission_id);
 
-  int SendData(DATA_TYPE type, const char *data, size_t size);
+  int SendVideoData(const XVideoFrame *video_frame);
-
-  int SendVideoData(VideoFrameType frame_type, const char *data, size_t size);
 
   int SendAudioData(const char *data, size_t size);
 
@@ -119,6 +127,10 @@ class IceTransmission {
 
   int CreateMediaCodec();
 
+  int CreateVideoCodec(RtpPacket::PAYLOAD_TYPE video_pt,
+                       bool hardware_acceleration);
+  int CreateAudioCodec();
+
  private:
   uint8_t CheckIsRtcpPacket(const char *buffer, size_t size);
   uint8_t CheckIsVideoPacket(const char *buffer, size_t size);
@@ -147,20 +159,23 @@ class IceTransmission {
   std::string remote_ice_username_ = "";
   NiceComponentState state_ = NICE_COMPONENT_STATE_DISCONNECTED;
   TraversalType traversal_type_ = TraversalType::TP2P;
+  void *user_data_ = nullptr;
 
  private:
   std::unique_ptr<IceAgent> ice_agent_ = nullptr;
   bool is_closed_ = false;
   std::shared_ptr<WsClient> ice_ws_transport_ = nullptr;
   CongestionControl *congestion_control_ = nullptr;
-  std::function<void(const char *, size_t, const std::string &)>
+  std::function<void(const XVideoFrame *, const char *, const size_t, void *)>
       on_receive_video_ = nullptr;
-  std::function<void(const char *, size_t, const std::string &)>
+  std::function<void(const char *, size_t, const char *, const size_t, void *)>
       on_receive_audio_ = nullptr;
-  std::function<void(const char *, size_t, const std::string &)>
+  std::function<void(const char *, size_t, const char *, const size_t, void *)>
       on_receive_data_ = nullptr;
+
   std::function<void(std::string, const std::string &)> on_ice_status_change_ =
       nullptr;
+
   std::function<void(const std::string &, TraversalType, const uint64_t,
                      const uint64_t, const uint64_t, const uint64_t,
                      const uint64_t, const uint64_t, const uint64_t,
@@ -200,6 +215,19 @@ class IceTransmission {
       RtpPacket::PAYLOAD_TYPE::UNDEFINED;
   RtpPacket::PAYLOAD_TYPE negotiated_data_pt_ =
       RtpPacket::PAYLOAD_TYPE::UNDEFINED;
+
+ private:
+  std::unique_ptr<VideoEncoder> video_encoder_ = nullptr;
+  std::unique_ptr<VideoDecoder> video_decoder_ = nullptr;
+  bool b_force_i_frame_ = false;
+  bool video_codec_inited_ = false;
+  bool load_nvcodec_dll_success_ = false;
+  bool hardware_acceleration_ = false;
+
+ private:
+  std::unique_ptr<AudioEncoder> audio_encoder_ = nullptr;
+  std::unique_ptr<AudioDecoder> audio_decoder_ = nullptr;
+  bool audio_codec_inited_ = false;
 };
 
 #endif

xmake.lua

@@ -184,13 +184,13 @@ target("statistics")
 
 target("transmission")
     set_kind("object")
-    add_deps("log", "ws", "ice", "qos", "rtp", "rtcp", "statistics")
+    add_deps("log", "ws", "ice", "qos", "rtp", "rtcp", "statistics", "media")
     add_files("src/transmission/*.cpp")
     add_includedirs("src/ws", "src/ice", "src/qos", {public = true})
 
 target("pc")
     set_kind("object")
-    add_deps("log", "ws", "ice", "transmission", "inih", "common", "media")
+    add_deps("log", "ws", "ice", "transmission", "inih", "common")
     add_files("src/pc/*.cpp")
     add_includedirs("src/transmission", "src/interface", {public = true})