Remote desk client supports MacOS

src/frame/frame.h

@@ -2,6 +2,7 @@
 #define _FRAME_H_
 
 #include <stdint.h>
+#include <stddef.h>
 
 class VideoFrame {
  public:

src/ice/ice_agent.cpp

@@ -6,7 +6,15 @@
 
 #include "log.h"
 
-IceAgent::IceAgent(std::string &ip, uint16_t port) : ip_(ip), port_(port) {}
+IceAgent::IceAgent(std::string &stun_ip, uint16_t stun_port,
+                   std::string &turn_ip, uint16_t turn_port,
+                   std::string &turn_username, std::string &turn_password)
+    : stun_ip_(stun_ip),
+      stun_port_(stun_port),
+      turn_ip_(turn_ip),
+      turn_port_(turn_port),
+      turn_username_(turn_username),
+      turn_password_(turn_password) {}
 
 IceAgent::~IceAgent() {}
 
@@ -20,8 +28,20 @@ int IceAgent::CreateIceAgent(juice_cb_state_changed_t on_state_changed,
   memset(&config, 0, sizeof(config));
 
   // STUN server example
-  config.stun_server_host = ip_.c_str();
-  config.stun_server_port = port_;
+  config.stun_server_host = stun_ip_.c_str();
+  config.stun_server_port = stun_port_;
+
+  if (!turn_ip_.empty() && -1 != turn_port_ && !turn_username_.empty() &&
+      !turn_password_.empty()) {
+    juice_turn_server_t turn_server;
+    memset(&turn_server, 0, sizeof(turn_server));
+    turn_server.host = turn_ip_.c_str();
+    turn_server.port = turn_port_;
+    turn_server.username = turn_username_.c_str();
+    turn_server.password = turn_password_.c_str();
+    config.turn_servers = &turn_server;
+    config.turn_servers_count = 1;
+  }
 
   config.cb_state_changed = on_state_changed;
   config.cb_candidate = on_candidate;

src/ice/ice_agent.h

@@ -7,7 +7,9 @@
 
 class IceAgent {
  public:
-  IceAgent(std::string& ip, uint16_t port);
+  IceAgent(std::string& stun_ip, uint16_t stun_port, std::string& turn_ip,
+           uint16_t turn_port, std::string& turn_username,
+           std::string& turn_password);
   ~IceAgent();
 
   int CreateIceAgent(juice_cb_state_changed_t on_state_changed,
@@ -36,8 +38,12 @@ class IceAgent {
   int Send(const char* data, size_t size);
 
  private:
-  std::string ip_ = "";
-  uint16_t port_ = 0;
+  std::string stun_ip_ = "";
+  uint16_t stun_port_ = 0;
+  std::string turn_ip_ = "";
+  uint16_t turn_port_ = 0;
+  std::string turn_username_ = "";
+  std::string turn_password_ = "";
   juice_agent_t* agent_ = nullptr;
   char local_sdp_[JUICE_MAX_SDP_STRING_LEN];
   juice_state_t state_;

src/media/video/decode/ffmpeg/ffmpeg_decoder.h

@@ -2,12 +2,10 @@
 #define _FFMPEG_DECODER_H_
 
 #ifdef _WIN32
-// Windows
 extern "C" {
 #include "libavcodec/avcodec.h"
 };
 #else
-// Linux...
 #ifdef __cplusplus
 extern "C" {
 #endif

src/media/video/encode/ffmpeg/ffmpeg_encoder.cpp

@@ -0,0 +1,44 @@
+#include "ffmpeg_encoder.h"
+
+#include <chrono>
+
+#include "log.h"
+
+#define SAVE_ENCODER_STREAM 0
+
+VideoEncoder::VideoEncoder() {
+  if (SAVE_ENCODER_STREAM) {
+    file_ = fopen("encode_stream.h264", "w+b");
+    if (!file_) {
+      LOG_WARN("Fail to open stream.h264");
+    }
+  }
+}
+VideoEncoder::~VideoEncoder() {
+  if (SAVE_ENCODER_STREAM && file_) {
+    fflush(file_);
+    fclose(file_);
+    file_ = nullptr;
+  }
+
+  if (nv12_data_) {
+    free(nv12_data_);
+    nv12_data_ = nullptr;
+  }
+}
+
+int VideoEncoder::Init() { return 0; }
+
+int VideoEncoder::Encode(
+    const uint8_t *pData, int nSize,
+    std::function<int(char *encoded_packets, size_t size)> on_encoded_image) {
+  return -1;
+}
+
+int VideoEncoder::OnEncodedImage(char *encoded_packets, size_t size) {
+  LOG_INFO("output encoded image");
+  fwrite(encoded_packets, 1, size, file_);
+  return 0;
+}
+
+void VideoEncoder::ForceIdr() {}

src/media/video/encode/ffmpeg/ffmpeg_encoder.h

@@ -0,0 +1,47 @@
+#ifndef _FFMPEG_ENCODER_H_
+#define _FFMPEG_ENCODER_H_
+
+#ifdef _WIN32
+extern "C" {
+#include "libavcodec/avcodec.h"
+};
+#else
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include <libavcodec/avcodec.h>
+#ifdef __cplusplus
+};
+#endif
+#endif
+#include <functional>
+
+class VideoEncoder {
+ public:
+  VideoEncoder();
+  ~VideoEncoder();
+
+  int Init();
+  int Encode(
+      const uint8_t* pData, int nSize,
+      std::function<int(char* encoded_packets, size_t size)> on_encoded_image);
+
+  virtual int OnEncodedImage(char* encoded_packets, size_t size);
+
+  void ForceIdr();
+
+ private:
+  int frame_width = 1280;
+  int frame_height = 720;
+  int keyFrameInterval_ = 3000;
+  int maxBitrate_ = 2000;
+  int max_payload_size_ = 3000;
+
+  std::vector<std::vector<uint8_t>> encoded_packets_;
+  unsigned char* encoded_image_ = nullptr;
+  FILE* file_ = nullptr;
+  unsigned char* nv12_data_ = nullptr;
+  unsigned int seq_ = 0;
+};
+
+#endif

src/media/video/encode/nvcodec/nv_encoder.cpp

@@ -60,11 +60,11 @@ int VideoEncoder::Init() {
   // TO TEST: not tested yet
   // init_params.encodeConfig->gopLength = NVENC_INFINITE_GOPLENGTH;
   init_params.encodeConfig->gopLength = keyFrameInterval_;
-  // Donot use B-frame for realtime application
+  // Do not use B-frame for realtime application
   init_params.encodeConfig->frameIntervalP = 1;
   init_params.encodeConfig->rcParams.rateControlMode =
       NV_ENC_PARAMS_RC_MODE::NV_ENC_PARAMS_RC_CBR;
-  init_params.encodeConfig->rcParams.maxBitRate = maxBitrate_ * 1000;
+  init_params.encodeConfig->rcParams.maxBitRate = maxBitrate_ * 500;
   init_params.encodeConfig->encodeCodecConfig.h264Config.sliceMode = 1;
   init_params.encodeConfig->encodeCodecConfig.h264Config.sliceModeData =
       max_payload_size_;
@@ -81,7 +81,7 @@ int VideoEncoder::Encode(
     return -1;
   }
 
-  if (0 == seq_++ % (30)) {
+  if (0 == seq_++ % (300)) {
     ForceIdr();
   }
 

src/media/video/encode/nvcodec/nv_encoder.h

@@ -28,7 +28,7 @@ class VideoEncoder {
   int frame_width = 1280;
   int frame_height = 720;
   int keyFrameInterval_ = 3000;
-  int maxBitrate_ = 2000;
+  int maxBitrate_ = 1000;
   int max_payload_size_ = 3000;
   NvEncoder* encoder_ = nullptr;
   CUcontext cuda_context_ = nullptr;

src/pc/peer_connection.cpp

@@ -29,16 +29,29 @@ int PeerConnection::Init(PeerConnectionParams params,
   cfg_signal_server_port_ = reader.Get("signal server", "port", "-1");
   cfg_stun_server_ip_ = reader.Get("stun server", "ip", "-1");
   cfg_stun_server_port_ = reader.Get("stun server", "port", "-1");
+  cfg_turn_server_ip_ = reader.Get("turn server", "ip", "");
+  cfg_turn_server_port_ = reader.Get("turn server", "port", "-1");
+  cfg_turn_server_username_ = reader.Get("turn server", "username", "");
+  cfg_turn_server_password_ = reader.Get("turn server", "password", "");
   std::regex regex("\n");
 
   LOG_INFO("Read config success");
 
   signal_server_port_ = stoi(cfg_signal_server_port_);
   stun_server_port_ = stoi(cfg_stun_server_port_);
+  turn_server_port_ = stoi(cfg_turn_server_port_);
 
   LOG_INFO("stun server ip [{}] port [{}]", cfg_stun_server_ip_,
            stun_server_port_);
 
+  if (!cfg_turn_server_ip_.empty() && 0 != turn_server_port_ &&
+      !cfg_turn_server_username_.empty() &&
+      !cfg_turn_server_password_.empty()) {
+    LOG_INFO("turn server ip [{}] port [{}] username [{}] password [{}]",
+             cfg_turn_server_ip_, turn_server_port_, cfg_turn_server_username_,
+             cfg_turn_server_password_);
+  }
+
   on_receive_video_buffer_ = params.on_receive_video_buffer;
   on_receive_audio_buffer_ = params.on_receive_audio_buffer;
   on_receive_data_buffer_ = params.on_receive_data_buffer;
@@ -74,10 +87,9 @@ int PeerConnection::Init(PeerConnectionParams params,
   on_ice_status_change_ = [this](std::string ice_status) {
     if ("JUICE_STATE_COMPLETED" == ice_status) {
       ice_ready_ = true;
-      LOG_INFO("Ice connected");
+      LOG_INFO("Ice finish");
     } else {
       ice_ready_ = false;
-      LOG_INFO("Ice not useable");
     }
   };
 
@@ -135,6 +147,7 @@ void PeerConnection::ProcessSignal(const std::string &signal) {
       ws_connection_id_ = j["ws_connection_id"].get<unsigned int>();
       LOG_INFO("Receive local peer websocket connection id [{}]",
                ws_connection_id_);
+      std::lock_guard<std::mutex> l(signal_status_mutex_);
       signal_status_ = SignalStatus::Connected;
       break;
     }
@@ -179,7 +192,9 @@ void PeerConnection::ProcessSignal(const std::string &signal) {
             on_receive_data_);
 
         ice_transmission_list_[remote_user_id]->InitIceTransmission(
-            cfg_stun_server_ip_, stun_server_port_);
+            cfg_stun_server_ip_, stun_server_port_, cfg_turn_server_ip_,
+            turn_server_port_, cfg_turn_server_username_,
+            cfg_turn_server_password_);
         ice_transmission_list_[remote_user_id]->JoinTransmission();
       }
 
@@ -221,7 +236,9 @@ void PeerConnection::ProcessSignal(const std::string &signal) {
             on_receive_data_);
 
         ice_transmission_list_[remote_user_id]->InitIceTransmission(
-            cfg_stun_server_ip_, stun_server_port_);
+            cfg_stun_server_ip_, stun_server_port_, cfg_turn_server_ip_,
+            turn_server_port_, cfg_turn_server_username_,
+            cfg_turn_server_password_);
 
         ice_transmission_list_[remote_user_id]->SetTransmissionId(
             transmission_id_);
@@ -271,7 +288,10 @@ int PeerConnection::RequestTransmissionMemberList(
 
 int PeerConnection::Destroy() { return 0; }
 
-SignalStatus PeerConnection::GetSignalStatus() { return signal_status_; }
+SignalStatus PeerConnection::GetSignalStatus() {
+  std::lock_guard<std::mutex> l(signal_status_mutex_);
+  return signal_status_;
+}
 
 int PeerConnection::SendVideoData(const char *data, size_t size) {
   if (!ice_ready_) {

src/pc/peer_connection.h

@@ -3,10 +3,17 @@
 
 #include <iostream>
 #include <map>
+#include <mutex>
 
-#include "ffmpeg_decoder.h"
 #include "ice_transmission.h"
+#ifdef _WIN32
+#include "nv_decoder.h"
 #include "nv_encoder.h"
+#else
+#include "ffmpeg_decoder.h"
+#include "ffmpeg_encoder.h"
+#endif
+
 #include "ws_transmission.h"
 
 enum SignalStatus { Connecting = 0, Connected, Closed };
@@ -59,8 +66,13 @@ class PeerConnection : public VideoEncoder, VideoDecoder {
   std::string cfg_signal_server_port_;
   std::string cfg_stun_server_ip_;
   std::string cfg_stun_server_port_;
+  std::string cfg_turn_server_ip_;
+  std::string cfg_turn_server_port_;
+  std::string cfg_turn_server_username_;
+  std::string cfg_turn_server_password_;
   int signal_server_port_ = 0;
   int stun_server_port_ = 0;
+  int turn_server_port_ = 0;
 
  private:
   std::shared_ptr<WsTransmission> ws_transport_ = nullptr;
@@ -70,6 +82,7 @@ class PeerConnection : public VideoEncoder, VideoDecoder {
   std::string transmission_id_ = "";
   std::vector<std::string> user_id_list_;
   SignalStatus signal_status_ = SignalStatus::Closed;
+  std::mutex signal_status_mutex_;
 
  private:
   std::map<std::string, std::unique_ptr<IceTransmission>>

src/transmission/ice_transmission.cpp

@@ -47,7 +47,10 @@ IceTransmission::~IceTransmission() {
   }
 }
 
-int IceTransmission::InitIceTransmission(std::string &ip, int port) {
+int IceTransmission::InitIceTransmission(std::string &stun_ip, int stun_port,
+                                         std::string &turn_ip, int turn_port,
+                                         std::string &turn_username,
+                                         std::string &turn_password) {
   video_rtp_codec_ = std::make_unique<RtpCodec>(RtpPacket::PAYLOAD_TYPE::H264);
   data_rtp_codec_ = std::make_unique<RtpCodec>(RtpPacket::PAYLOAD_TYPE::DATA);
 
@@ -113,7 +116,8 @@ int IceTransmission::InitIceTransmission(std::string &ip, int port) {
                          remote_user_id_.size());
       });
 
-  ice_agent_ = std::make_unique<IceAgent>(ip, port);
+  ice_agent_ = std::make_unique<IceAgent>(
+      stun_ip, stun_port, turn_ip, turn_port, turn_username, turn_password);
 
   ice_agent_->CreateIceAgent(
       [](juice_agent_t *agent, juice_state_t state, void *user_ptr) {

src/transmission/ice_transmission.h

@@ -26,7 +26,10 @@ class IceTransmission {
   ~IceTransmission();
 
  public:
-  int InitIceTransmission(std::string &ip, int port);
+  int InitIceTransmission(std::string &stun_ip, int stun_port,
+                          std::string &turn_ip, int turn_port,
+                          std::string &turn_username,
+                          std::string &turn_password);
 
   int DestroyIceTransmission();