Support cursor control on MacOS

src/gui/main.cpp

@@ -6,6 +6,7 @@
 #include <Winsock2.h>
 #include <iphlpapi.h>
 #elif __APPLE__
+#include <ApplicationServices/ApplicationServices.h>
 #include <ifaddrs.h>
 #include <net/if_dl.h>
 #include <net/if_types.h>
@@ -49,6 +50,8 @@ extern "C" {
 #include "screen_capture_wgc.h"
 #elif __linux__
 #include "screen_capture_x11.h"
+#elif __APPLE__
+#include "screen_capture_avf.h"
 #endif
 #include "../../thirdparty/projectx/src/interface/x.h"
 
@@ -85,7 +88,7 @@ static std::atomic<bool> audio_buffer_fresh = false;
 static uint32_t last_ts = 0;
 
 char *out = "audio_old.pcm";
-// FILE *outfile = fopen(out, "wb+");
+FILE *outfile = fopen(out, "wb+");
 
 int64_t src_ch_layout = AV_CH_LAYOUT_MONO;
 int src_rate = 48000;
@@ -125,10 +128,17 @@ unsigned char pcm_bytes[MAX_FRAME_SIZE * CHANNELS * 2];
 
 std::string window_title = "Remote Desk Client";
 
-std::string server_connection_status = "-";
-std::string client_connection_status = "-";
-std::string server_signal_status = "-";
-std::string client_signal_status = "-";
+std::string server_connection_status_str = "-";
+std::string client_connection_status_str = "-";
+std::string server_signal_status_str = "-";
+std::string client_signal_status_str = "-";
+
+std::atomic<ConnectionStatus> server_connection_status =
+    ConnectionStatus::Closed;
+std::atomic<ConnectionStatus> client_connection_status =
+    ConnectionStatus::Closed;
+std::atomic<SignalStatus> server_signal_status = SignalStatus::SignalClosed;
+std::atomic<SignalStatus> client_signal_status = SignalStatus::SignalClosed;
 
 // Refresh Event
 #define REFRESH_EVENT (SDL_USEREVENT + 1)
@@ -157,6 +167,8 @@ static bool done = false;
 ScreenCaptureWgc *screen_capture = nullptr;
 #elif __linux__
 ScreenCaptureX11 *screen_capture = nullptr;
+#elif __APPLE__
+ScreenCaptureAvf *screen_capture = nullptr;
 #endif
 
 char *nv12_buffer = nullptr;
@@ -313,12 +325,12 @@ void SdlCaptureAudioIn(void *userdata, Uint8 *stream, int len) {
   }
 
   if (1) {
-    if ("ClientConnected" == client_connection_status) {
+    if ("ClientConnected" == client_connection_status_str) {
       SendData(peer_client, DATA_TYPE::AUDIO, (const char *)dst_data[0],
                dst_bufsize);
     }
 
-    if ("ServerConnected" == server_connection_status) {
+    if ("ServerConnected" == server_connection_status_str) {
       SendData(peer_server, DATA_TYPE::AUDIO, (const char *)dst_data[0],
                dst_bufsize);
     }
@@ -331,7 +343,7 @@ void SdlCaptureAudioIn(void *userdata, Uint8 *stream, int len) {
 }
 
 void SdlCaptureAudioOut(void *userdata, Uint8 *stream, int len) {
-  // if ("ClientConnected" != client_connection_status) {
+  // if ("ClientConnected" != client_connection_status_str) {
   //   return;
   // }
 
@@ -396,6 +408,10 @@ void ServerReceiveDataBuffer(const char *data, size_t size, const char *user_id,
   // std::cout << "remote_action: " << remote_action.type << " "
   //           << remote_action.m.flag << " " << remote_action.m.x << " "
   //           << remote_action.m.y << std::endl;
+
+  int mouse_pos_x = remote_action.m.x * screen_w / 1280;
+  int mouse_pos_y = remote_action.m.y * screen_h / 720;
+#if 1
 #ifdef _WIN32
   INPUT ip;
 
@@ -417,18 +433,43 @@ void ServerReceiveDataBuffer(const char *data, size_t size, const char *user_id,
     ip.mi.mouseData = 0;
     ip.mi.time = 0;
 
-#if MOUSE_CONTROL
     // Set cursor pos
     SetCursorPos(ip.mi.dx, ip.mi.dy);
     // Send the press
     if (ip.mi.dwFlags != MOUSEEVENTF_MOVE) {
       SendInput(1, &ip, sizeof(INPUT));
     }
-#endif
+
     // std::cout << "Receive data from [" << user << "], " << ip.type << " "
     //           << ip.mi.dwFlags << " " << ip.mi.dx << " " << ip.mi.dy
     //           << std::endl;
   }
+
+#elif __APPLE__
+  if (remote_action.type == ControlType::mouse) {
+    CGEventRef mouse_event;
+    CGEventType mouse_type;
+
+    if (remote_action.m.flag == MouseFlag::left_down) {
+      mouse_type = kCGEventLeftMouseDown;
+    } else if (remote_action.m.flag == MouseFlag::left_up) {
+      mouse_type = kCGEventLeftMouseUp;
+    } else if (remote_action.m.flag == MouseFlag::right_down) {
+      mouse_type = kCGEventRightMouseDown;
+    } else if (remote_action.m.flag == MouseFlag::right_up) {
+      mouse_type = kCGEventRightMouseUp;
+    } else {
+      mouse_type = kCGEventMouseMoved;
+    }
+
+    mouse_event = CGEventCreateMouseEvent(NULL, mouse_type,
+                                          CGPointMake(mouse_pos_x, mouse_pos_y),
+                                          kCGMouseButtonLeft);
+
+    CGEventPost(kCGHIDEventTap, mouse_event);
+    CFRelease(mouse_event);
+  }
+#endif
 #endif
 }
 
@@ -439,16 +480,20 @@ void ClientReceiveDataBuffer(const char *data, size_t size, const char *user_id,
   RemoteAction remote_action;
   memcpy(&remote_action, data, sizeof(remote_action));
 
-  // std::cout << "remote_action: " << remote_action.type << " "
-  //           << remote_action.m.flag << " " << remote_action.m.x << " "
-  //           << remote_action.m.y << std::endl;
+  std::cout << "remote_action: " << remote_action.type << " "
+            << remote_action.m.flag << " " << remote_action.m.x << " "
+            << remote_action.m.y << std::endl;
+
+  int mouse_pos_x = remote_action.m.x * screen_w / 1280;
+  int mouse_pos_y = remote_action.m.y * screen_h / 720;
+
 #ifdef _WIN32
   INPUT ip;
 
   if (remote_action.type == ControlType::mouse) {
     ip.type = INPUT_MOUSE;
-    ip.mi.dx = remote_action.m.x * screen_w / 1280;
-    ip.mi.dy = remote_action.m.y * screen_h / 720;
+    ip.mi.dx = mouse_pos_x;
+    ip.mi.dy = mouse_pos_y;
     if (remote_action.m.flag == MouseFlag::left_down) {
       ip.mi.dwFlags = MOUSEEVENTF_LEFTDOWN | MOUSEEVENTF_ABSOLUTE;
     } else if (remote_action.m.flag == MouseFlag::left_up) {
@@ -475,50 +520,62 @@ void ClientReceiveDataBuffer(const char *data, size_t size, const char *user_id,
     //           << ip.mi.dwFlags << " " << ip.mi.dx << " " << ip.mi.dy
     //           << std::endl;
   }
+
+#elif __APPLE__
+  CGEventRef mouse_event;
+  mouse_event = CGEventCreateMouseEvent(NULL, kCGEventMouseMoved,
+                                        CGPointMake(mouse_pos_x, mouse_pos_y),
+                                        kCGMouseButtonLeft);
+  CGEventPost(kCGHIDEventTap, mouse_event);
+  CFRelease(mouse_event);
+
 #endif
 }
 
 void ServerSignalStatus(SignalStatus status) {
+  server_signal_status = status;
   if (SignalStatus::SignalConnecting == status) {
-    server_signal_status = "ServerSignalConnecting";
+    server_signal_status_str = "ServerSignalConnecting";
   } else if (SignalStatus::SignalConnected == status) {
-    server_signal_status = "ServerSignalConnected";
+    server_signal_status_str = "ServerSignalConnected";
   } else if (SignalStatus::SignalFailed == status) {
-    server_signal_status = "ServerSignalFailed";
+    server_signal_status_str = "ServerSignalFailed";
   } else if (SignalStatus::SignalClosed == status) {
-    server_signal_status = "ServerSignalClosed";
+    server_signal_status_str = "ServerSignalClosed";
   } else if (SignalStatus::SignalReconnecting == status) {
-    server_signal_status = "ServerSignalReconnecting";
+    server_signal_status_str = "ServerSignalReconnecting";
   }
 }
 
 void ClientSignalStatus(SignalStatus status) {
+  client_signal_status = status;
   if (SignalStatus::SignalConnecting == status) {
-    client_signal_status = "ClientSignalConnecting";
+    client_signal_status_str = "ClientSignalConnecting";
   } else if (SignalStatus::SignalConnected == status) {
-    client_signal_status = "ClientSignalConnected";
+    client_signal_status_str = "ClientSignalConnected";
   } else if (SignalStatus::SignalFailed == status) {
-    client_signal_status = "ClientSignalFailed";
+    client_signal_status_str = "ClientSignalFailed";
   } else if (SignalStatus::SignalClosed == status) {
-    client_signal_status = "ClientSignalClosed";
+    client_signal_status_str = "ClientSignalClosed";
   } else if (SignalStatus::SignalReconnecting == status) {
-    client_signal_status = "ClientSignalReconnecting";
+    client_signal_status_str = "ClientSignalReconnecting";
   }
 }
 
 void ServerConnectionStatus(ConnectionStatus status) {
+  server_connection_status = status;
   if (ConnectionStatus::Connecting == status) {
-    server_connection_status = "ServerConnecting";
+    server_connection_status_str = "ServerConnecting";
   } else if (ConnectionStatus::Connected == status) {
-    server_connection_status = "ServerConnected";
+    server_connection_status_str = "ServerConnected";
   } else if (ConnectionStatus::Disconnected == status) {
-    server_connection_status = "ServerDisconnected";
+    server_connection_status_str = "ServerDisconnected";
   } else if (ConnectionStatus::Failed == status) {
-    server_connection_status = "ServerFailed";
+    server_connection_status_str = "ServerFailed";
   } else if (ConnectionStatus::Closed == status) {
-    server_connection_status = "ServerClosed";
+    server_connection_status_str = "ServerClosed";
   } else if (ConnectionStatus::IncorrectPassword == status) {
-    server_connection_status = "Incorrect password";
+    server_connection_status_str = "Incorrect password";
     if (connect_button_pressed) {
       connect_button_pressed = false;
       joined = false;
@@ -528,18 +585,26 @@ void ServerConnectionStatus(ConnectionStatus status) {
 }
 
 void ClientConnectionStatus(ConnectionStatus status) {
+  client_connection_status = status;
   if (ConnectionStatus::Connecting == status) {
-    client_connection_status = "ClientConnecting";
+    client_connection_status_str = "ClientConnecting";
   } else if (ConnectionStatus::Connected == status) {
-    client_connection_status = "ClientConnected";
+    client_connection_status_str = "ClientConnected";
   } else if (ConnectionStatus::Disconnected == status) {
-    client_connection_status = "ClientDisconnected";
+    client_connection_status_str = "ClientDisconnected";
   } else if (ConnectionStatus::Failed == status) {
-    client_connection_status = "ClientFailed";
+    client_connection_status_str = "ClientFailed";
   } else if (ConnectionStatus::Closed == status) {
-    client_connection_status = "ClientClosed";
+    client_connection_status_str = "ClientClosed";
   } else if (ConnectionStatus::IncorrectPassword == status) {
-    client_connection_status = "Incorrect password";
+    client_connection_status_str = "Incorrect password";
+    if (connect_button_pressed) {
+      connect_button_pressed = false;
+      joined = false;
+      connect_label = connect_button_pressed ? "Disconnect" : "Connect";
+    }
+  } else if (ConnectionStatus::NoSuchTransmissionId == status) {
+    client_connection_status_str = "No such transmission id";
     if (connect_button_pressed) {
       connect_button_pressed = false;
       joined = false;
@@ -772,7 +837,7 @@ int main() {
     LOG_INFO("peer_client init finish");
 
     {
-      while ("ServerSignalConnected" != server_signal_status && !done) {
+      while (SignalStatus::SignalConnected != server_signal_status && !done) {
       }
 
       if (done) {
@@ -824,6 +889,31 @@ int main() {
       rect.right = 0;
       rect.bottom = 0;
 
+      last_frame_time_ = std::chrono::high_resolution_clock::now();
+      screen_capture->Init(
+          rect, 60,
+          [](unsigned char *data, int size, int width, int height) -> void {
+            auto now_time = std::chrono::high_resolution_clock::now();
+            std::chrono::duration<double> duration =
+                now_time - last_frame_time_;
+            auto tc = duration.count() * 1000;
+
+            if (tc >= 0) {
+              SendData(peer_server, DATA_TYPE::VIDEO, (const char *)data,
+                       NV12_BUFFER_SIZE);
+              last_frame_time_ = now_time;
+            }
+          });
+      screen_capture->Start();
+#elif __APPLE__
+      screen_capture = new ScreenCaptureAvf();
+
+      RECORD_DESKTOP_RECT rect;
+      rect.left = 0;
+      rect.top = 0;
+      rect.right = 0;
+      rect.bottom = 0;
+
       last_frame_time_ = std::chrono::high_resolution_clock::now();
       screen_capture->Init(
           rect, 60,
@@ -991,15 +1081,15 @@ int main() {
         {
           {
             static char remote_id[20] = "";
-            if (strcmp(remote_id, "") == 0) {
-              strcpy(remote_id, GetMac(mac_addr).c_str());
-            }
+            // if (strcmp(remote_id, "") == 0) {
+            //   strcpy(remote_id, GetMac(mac_addr).c_str());
+            // }
             ImGui::Text("REMOTE ID:");
             ImGui::SameLine();
             ImGui::SetNextItemWidth(95);
             ImGui::InputTextWithHint("##remote_id", mac_addr, remote_id,
                                      IM_ARRAYSIZE(remote_id),
-                                     ImGuiInputTextFlags_AllowTabInput);
+                                     ImGuiInputTextFlags_CharsNoBlank);
 
             ImGui::Spacing();
 
@@ -1014,7 +1104,7 @@ int main() {
 
             if (ImGui::Button(connect_label)) {
               int ret = -1;
-              if ("ClientSignalConnected" == client_signal_status) {
+              if ("ClientSignalConnected" == client_signal_status_str) {
                 if (strcmp(connect_label, "Connect") == 0 && !joined) {
                   std::string user_id = "C-" + std::string(GetMac(mac_addr));
                   ret = JoinConnection(peer_client, remote_id, client_password);
@@ -1113,9 +1203,10 @@ int main() {
       fps = frame_count / (elapsed_time / 1000);
       frame_count = 0;
       window_title = "Remote Desk Client FPS [" + std::to_string(fps) +
-                     "] status [" + server_signal_status + "|" +
-                     client_signal_status + "|" + server_connection_status +
-                     "|" + client_connection_status + "]";
+                     "] status [" + server_signal_status_str + "|" +
+                     client_signal_status_str + "|" +
+                     server_connection_status_str + "|" +
+                     client_connection_status_str + "]";
       // For MacOS, UI frameworks can only be called from the main thread
       SDL_SetWindowTitle(window, window_title.c_str());
       start_time = end_time;

src/screen_capture/linux/screen_capture_x11.cpp

@@ -21,6 +21,8 @@ int ScreenCaptureX11::Init(const RECORD_DESKTOP_RECT &rect, const int fps,
     _on_data = cb;
   }
 
+  av_log_set_level(AV_LOG_QUIET);
+
   pFormatCtx_ = avformat_alloc_context();
 
   avdevice_register_all();

src/screen_capture/macosx/screen_capture_avf.cpp

@@ -0,0 +1,140 @@
+#include "screen_capture_avf.h"
+
+#include <iostream>
+
+#include "log.h"
+
+#define NV12_BUFFER_SIZE 1280 * 720 * 3 / 2
+unsigned char nv12_buffer_[NV12_BUFFER_SIZE];
+
+ScreenCaptureAvf::ScreenCaptureAvf() {}
+
+ScreenCaptureAvf::~ScreenCaptureAvf() {
+  if (capture_thread_->joinable()) {
+    capture_thread_->join();
+  }
+}
+
+int ScreenCaptureAvf::Init(const RECORD_DESKTOP_RECT &rect, const int fps,
+                           cb_desktop_data cb) {
+  if (cb) {
+    _on_data = cb;
+  }
+
+  av_log_set_level(AV_LOG_QUIET);
+
+  pFormatCtx_ = avformat_alloc_context();
+
+  avdevice_register_all();
+
+  // grabbing frame rate
+  av_dict_set(&options_, "framerate", "60", 0);
+  av_dict_set(&options_, "pixel_format", "nv12", 0);
+  // show remote cursor
+  av_dict_set(&options_, "capture_cursor", "1", 0);
+  // Make the grabbed area follow the mouse
+  // av_dict_set(&options_, "follow_mouse", "centered", 0);
+  // Video frame size. The default is to capture the full screen
+  // av_dict_set(&options_, "video_size", "1280x720", 0);
+  ifmt_ = (AVInputFormat *)av_find_input_format("avfoundation");
+  if (!ifmt_) {
+    printf("Couldn't find_input_format\n");
+  }
+
+  // Grab at position 10,20
+  if (avformat_open_input(&pFormatCtx_, "Capture screen 0", ifmt_, &options_) !=
+      0) {
+    printf("Couldn't open input stream.\n");
+    return -1;
+  }
+
+  if (avformat_find_stream_info(pFormatCtx_, NULL) < 0) {
+    printf("Couldn't find stream information.\n");
+    return -1;
+  }
+
+  videoindex_ = -1;
+  for (i_ = 0; i_ < pFormatCtx_->nb_streams; i_++)
+    if (pFormatCtx_->streams[i_]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+      videoindex_ = i_;
+      break;
+    }
+  if (videoindex_ == -1) {
+    printf("Didn't find a video stream.\n");
+    return -1;
+  }
+
+  pCodecParam_ = pFormatCtx_->streams[videoindex_]->codecpar;
+
+  pCodecCtx_ = avcodec_alloc_context3(NULL);
+  avcodec_parameters_to_context(pCodecCtx_, pCodecParam_);
+
+  pCodec_ = const_cast<AVCodec *>(avcodec_find_decoder(pCodecCtx_->codec_id));
+  if (pCodec_ == NULL) {
+    printf("Codec not found.\n");
+    return -1;
+  }
+  if (avcodec_open2(pCodecCtx_, pCodec_, NULL) < 0) {
+    printf("Could not open codec.\n");
+    return -1;
+  }
+
+  const int screen_w = pFormatCtx_->streams[videoindex_]->codecpar->width;
+  const int screen_h = pFormatCtx_->streams[videoindex_]->codecpar->height;
+
+  pFrame_ = av_frame_alloc();
+  pFrameNv12_ = av_frame_alloc();
+
+  pFrame_->width = screen_w;
+  pFrame_->height = screen_h;
+  pFrameNv12_->width = 1280;
+  pFrameNv12_->height = 720;
+
+  packet_ = (AVPacket *)av_malloc(sizeof(AVPacket));
+
+  img_convert_ctx_ = sws_getContext(
+      pFrame_->width, pFrame_->height, pCodecCtx_->pix_fmt, pFrameNv12_->width,
+      pFrameNv12_->height, AV_PIX_FMT_NV12, SWS_BICUBIC, NULL, NULL, NULL);
+
+  return 0;
+}
+
+int ScreenCaptureAvf::Start() {
+  capture_thread_.reset(new std::thread([this]() {
+    while (1) {
+      if (av_read_frame(pFormatCtx_, packet_) >= 0) {
+        if (packet_->stream_index == videoindex_) {
+          avcodec_send_packet(pCodecCtx_, packet_);
+          av_packet_unref(packet_);
+          got_picture_ = avcodec_receive_frame(pCodecCtx_, pFrame_);
+
+          if (!got_picture_) {
+            av_image_fill_arrays(pFrameNv12_->data, pFrameNv12_->linesize,
+                                 nv12_buffer_, AV_PIX_FMT_NV12,
+                                 pFrameNv12_->width, pFrameNv12_->height, 1);
+
+            sws_scale(img_convert_ctx_, pFrame_->data, pFrame_->linesize, 0,
+                      pFrame_->height, pFrameNv12_->data,
+                      pFrameNv12_->linesize);
+
+            _on_data((unsigned char *)nv12_buffer_,
+                     pFrameNv12_->width * pFrameNv12_->height * 3 / 2,
+                     pFrameNv12_->width, pFrameNv12_->height);
+          }
+        }
+      }
+    }
+  }));
+
+  return 0;
+}
+
+int ScreenCaptureAvf::Pause() { return 0; }
+
+int ScreenCaptureAvf::Resume() { return 0; }
+
+int ScreenCaptureAvf::Stop() { return 0; }
+
+void ScreenCaptureAvf::OnFrame() {}
+
+void ScreenCaptureAvf::CleanUp() {}

src/screen_capture/macosx/screen_capture_avf.h

@@ -0,0 +1,91 @@
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2023-12-01
+ * Copyright (c) 2023 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _SCREEN_CAPTURE_AVF_H_
+#define _SCREEN_CAPTURE_AVF_H_
+
+#include <atomic>
+#include <functional>
+#include <string>
+#include <thread>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include <libavcodec/avcodec.h>
+#include <libavdevice/avdevice.h>
+#include <libavformat/avformat.h>
+#include <libavutil/imgutils.h>
+#include <libswscale/swscale.h>
+#ifdef __cplusplus
+};
+#endif
+
+typedef struct {
+  int left;
+  int top;
+  int right;
+  int bottom;
+} RECORD_DESKTOP_RECT;
+
+typedef std::function<void(unsigned char *, int, int, int)> cb_desktop_data;
+typedef std::function<void(int)> cb_desktop_error;
+
+class ScreenCaptureAvf {
+ public:
+  ScreenCaptureAvf();
+  ~ScreenCaptureAvf();
+
+ public:
+  int Init(const RECORD_DESKTOP_RECT &rect, const int fps, cb_desktop_data cb);
+
+  int Start();
+  int Pause();
+  int Resume();
+  int Stop();
+
+  void OnFrame();
+
+ protected:
+  void CleanUp();
+
+ private:
+  std::atomic_bool _running;
+  std::atomic_bool _paused;
+  std::atomic_bool _inited;
+
+  std::thread _thread;
+
+  std::string _device_name;
+
+  RECORD_DESKTOP_RECT _rect;
+
+  int _fps;
+
+  cb_desktop_data _on_data;
+  cb_desktop_error _on_error;
+
+ private:
+  int i_ = 0;
+  int videoindex_ = 0;
+  int got_picture_ = 0;
+  // ffmpeg
+  AVFormatContext *pFormatCtx_ = nullptr;
+  AVCodecContext *pCodecCtx_ = nullptr;
+  AVCodec *pCodec_ = nullptr;
+  AVCodecParameters *pCodecParam_ = nullptr;
+  AVDictionary *options_ = nullptr;
+  AVInputFormat *ifmt_ = nullptr;
+  AVFrame *pFrame_ = nullptr;
+  AVFrame *pFrameNv12_ = nullptr;
+  AVPacket *packet_ = nullptr;
+  struct SwsContext *img_convert_ctx_ = nullptr;
+
+  // thread
+  std::unique_ptr<std::thread> capture_thread_ = nullptr;
+};
+
+#endif

src/screen_capture/macosx/screen_capture_wgc.cpp

@@ -1,26 +0,0 @@
-#include "screen_capture_wgc.h"
-
-#include <iostream>
-
-ScreenCaptureWgc::ScreenCaptureWgc() {}
-
-ScreenCaptureWgc::~ScreenCaptureWgc() {}
-
-bool ScreenCaptureWgc::IsWgcSupported() { return false; }
-
-int ScreenCaptureWgc::Init(const RECORD_DESKTOP_RECT &rect, const int fps,
-                           cb_desktop_data cb) {
-  return 0;
-}
-
-int ScreenCaptureWgc::Start() { return 0; }
-
-int ScreenCaptureWgc::Pause() { return 0; }
-
-int ScreenCaptureWgc::Resume() { return 0; }
-
-int ScreenCaptureWgc::Stop() { return 0; }
-
-void ScreenCaptureWgc::OnFrame() {}
-
-void ScreenCaptureWgc::CleanUp() {}

src/screen_capture/macosx/screen_capture_wgc.h

@@ -1,56 +0,0 @@
-#ifndef _SCREEN_CAPTURE_WGC_H_
-#define _SCREEN_CAPTURE_WGC_H_
-
-#include <atomic>
-#include <functional>
-#include <string>
-#include <thread>
-
-typedef struct {
-  int left;
-  int top;
-  int right;
-  int bottom;
-} RECORD_DESKTOP_RECT;
-
-typedef std::function<void(unsigned char *, int, int, int)> cb_desktop_data;
-typedef std::function<void(int)> cb_desktop_error;
-
-class ScreenCaptureWgc {
- public:
-  ScreenCaptureWgc();
-  ~ScreenCaptureWgc();
-
- public:
-  bool IsWgcSupported();
-
-  int Init(const RECORD_DESKTOP_RECT &rect, const int fps, cb_desktop_data cb);
-
-  int Start();
-  int Pause();
-  int Resume();
-  int Stop();
-
-  void OnFrame();
-
- protected:
-  void CleanUp();
-
- private:
-  std::atomic_bool _running;
-  std::atomic_bool _paused;
-  std::atomic_bool _inited;
-
-  std::thread _thread;
-
-  std::string _device_name;
-
-  RECORD_DESKTOP_RECT _rect;
-
-  int _fps;
-
-  cb_desktop_data _on_data;
-  cb_desktop_error _on_error;
-};
-
-#endif

test/screen_capture/mac_capture.cpp

@@ -0,0 +1,286 @@
+#include <stdio.h>
+
+#define __STDC_CONSTANT_MACROS
+
+#ifdef _WIN32
+// Windows
+extern "C" {
+#include "SDL/SDL.h"
+#include "libavcodec/avcodec.h"
+#include "libavdevice/avdevice.h"
+#include "libavformat/avformat.h"
+#include "libswscale/swscale.h"
+};
+#else
+// Linux...
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include <SDL2/SDL.h>
+#include <libavcodec/avcodec.h>
+#include <libavdevice/avdevice.h>
+#include <libavformat/avformat.h>
+#include <libswscale/swscale.h>
+#ifdef __cplusplus
+};
+#endif
+#endif
+
+// Output YUV420P
+#define OUTPUT_YUV420P 0
+//'1' Use Dshow
+//'0' Use GDIgrab
+#define USE_DSHOW 0
+
+// Refresh Event
+#define SFM_REFRESH_EVENT (SDL_USEREVENT + 1)
+
+#define SFM_BREAK_EVENT (SDL_USEREVENT + 2)
+
+int thread_exit = 0;
+
+int sfp_refresh_thread(void *opaque) {
+  thread_exit = 0;
+  while (!thread_exit) {
+    SDL_Event event;
+    event.type = SFM_REFRESH_EVENT;
+    SDL_PushEvent(&event);
+    SDL_Delay(40);
+  }
+  thread_exit = 0;
+  // Break
+  SDL_Event event;
+  event.type = SFM_BREAK_EVENT;
+  SDL_PushEvent(&event);
+
+  return 0;
+}
+
+// Show AVFoundation Device
+void show_avfoundation_device() {
+  AVFormatContext *pFormatCtx = avformat_alloc_context();
+  AVDictionary *options = NULL;
+  av_dict_set(&options, "list_devices", "true", 0);
+  AVInputFormat *iformat =
+      (AVInputFormat *)av_find_input_format("avfoundation");
+  printf("==AVFoundation Device Info===\n");
+  avformat_open_input(&pFormatCtx, "", iformat, &options);
+  printf("=============================\n");
+}
+
+int main(int argc, char *argv[]) {
+  AVFormatContext *pFormatCtx;
+  int i, videoindex;
+  AVCodecContext *pCodecCtx;
+  AVCodec *pCodec;
+
+  avformat_network_init();
+  pFormatCtx = avformat_alloc_context();
+
+  // Open File
+  // char filepath[]="src01_480x272_22.h265";
+  // avformat_open_input(&pFormatCtx,filepath,NULL,NULL)
+
+  // Register Device
+  avdevice_register_all();
+  // Windows
+#ifdef _WIN32
+#if USE_DSHOW
+  // Use dshow
+  //
+  // Need to Install screen-capture-recorder
+  // screen-capture-recorder
+  // Website: http://sourceforge.net/projects/screencapturer/
+  //
+  AVInputFormat *ifmt = av_find_input_format("dshow");
+  if (avformat_open_input(&pFormatCtx, "video=screen-capture-recorder", ifmt,
+                          NULL) != 0) {
+    printf("Couldn't open input stream.\n");
+    return -1;
+  }
+#else
+  // Use gdigrab
+  AVDictionary *options = NULL;
+  // Set some options
+  // grabbing frame rate
+  // av_dict_set(&options,"framerate","5",0);
+  // The distance from the left edge of the screen or desktop
+  // av_dict_set(&options,"offset_x","20",0);
+  // The distance from the top edge of the screen or desktop
+  // av_dict_set(&options,"offset_y","40",0);
+  // Video frame size. The default is to capture the full screen
+  // av_dict_set(&options,"video_size","640x480",0);
+  AVInputFormat *ifmt = av_find_input_format("gdigrab");
+  if (avformat_open_input(&pFormatCtx, "desktop", ifmt, &options) != 0) {
+    printf("Couldn't open input stream.\n");
+    return -1;
+  }
+
+#endif
+#elif defined linux
+  // Linux
+  AVDictionary *options = NULL;
+  // Set some options
+  // grabbing frame rate
+  // av_dict_set(&options,"framerate","5",0);
+  // Make the grabbed area follow the mouse
+  // av_dict_set(&options,"follow_mouse","centered",0);
+  // Video frame size. The default is to capture the full screen
+  // av_dict_set(&options,"video_size","640x480",0);
+  AVInputFormat *ifmt = av_find_input_format("x11grab");
+  // Grab at position 10,20
+  if (avformat_open_input(&pFormatCtx, ":0.0+10,20", ifmt, &options) != 0) {
+    printf("Couldn't open input stream.\n");
+    return -1;
+  }
+#else
+  show_avfoundation_device();
+  // Mac
+  AVInputFormat *ifmt = (AVInputFormat *)av_find_input_format("avfoundation");
+  // Avfoundation
+  //[video]:[audio]
+  if (avformat_open_input(&pFormatCtx, "1", ifmt, NULL) != 0) {
+    printf("Couldn't open input stream.\n");
+    return -1;
+  }
+#endif
+
+  if (avformat_find_stream_info(pFormatCtx, NULL) < 0) {
+    printf("Couldn't find stream information.\n");
+    return -1;
+  }
+  videoindex = -1;
+  for (i = 0; i < pFormatCtx->nb_streams; i++)
+    if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO) {
+      videoindex = i;
+      break;
+    }
+  if (videoindex == -1) {
+    printf("Didn't find a video stream.\n");
+    return -1;
+  }
+  pCodecCtx = pFormatCtx->streams[videoindex]->codec;
+  pCodec = avcodec_find_decoder(pCodecCtx->codec_id);
+  if (pCodec == NULL) {
+    printf("Codec not found.\n");
+    return -1;
+  }
+  if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0) {
+    printf("Could not open codec.\n");
+    return -1;
+  }
+  AVFrame *pFrame, *pFrameYUV;
+  pFrame = av_frame_alloc();
+  pFrameYUV = av_frame_alloc();
+  // unsigned char *out_buffer=(unsigned char
+  // *)av_malloc(avpicture_get_size(AV_PIX_FMT_YUV420P, pCodecCtx->width,
+  // pCodecCtx->height)); avpicture_fill((AVPicture *)pFrameYUV, out_buffer,
+  // AV_PIX_FMT_YUV420P, pCodecCtx->width, pCodecCtx->height);
+  // SDL----------------------------
+  if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER)) {
+    printf("Could not initialize SDL - %s\n", SDL_GetError());
+    return -1;
+  }
+  int screen_w = 640, screen_h = 360;
+  const SDL_VideoInfo *vi = SDL_GetVideoInfo();
+  // Half of the Desktop's width and height.
+  screen_w = vi->current_w / 2;
+  screen_h = vi->current_h / 2;
+  SDL_Surface *screen;
+  screen = SDL_SetVideoMode(screen_w, screen_h, 0, 0);
+
+  if (!screen) {
+    printf("SDL: could not set video mode - exiting:%s\n", SDL_GetError());
+    return -1;
+  }
+  SDL_Overlay *bmp;
+  bmp = SDL_CreateYUVOverlay(pCodecCtx->width, pCodecCtx->height,
+                             SDL_YV12_OVERLAY, screen);
+  SDL_Rect rect;
+  rect.x = 0;
+  rect.y = 0;
+  rect.w = screen_w;
+  rect.h = screen_h;
+  // SDL End------------------------
+  int ret, got_picture;
+
+  AVPacket *packet = (AVPacket *)av_malloc(sizeof(AVPacket));
+
+#if OUTPUT_YUV420P
+  FILE *fp_yuv = fopen("output.yuv", "wb+");
+#endif
+
+  struct SwsContext *img_convert_ctx;
+  img_convert_ctx = sws_getContext(
+      pCodecCtx->width, pCodecCtx->height, pCodecCtx->pix_fmt, pCodecCtx->width,
+      pCodecCtx->height, AV_PIX_FMT_YUV420P, SWS_BICUBIC, NULL, NULL, NULL);
+  //------------------------------
+  SDL_Thread *video_tid = SDL_CreateThread(sfp_refresh_thread, NULL);
+  //
+  SDL_WM_SetCaption("Simplest FFmpeg Grab Desktop", NULL);
+  // Event Loop
+  SDL_Event event;
+
+  for (;;) {
+    // Wait
+    SDL_WaitEvent(&event);
+    if (event.type == SFM_REFRESH_EVENT) {
+      //------------------------------
+      if (av_read_frame(pFormatCtx, packet) >= 0) {
+        if (packet->stream_index == videoindex) {
+          ret = avcodec_decode_video2(pCodecCtx, pFrame, &got_picture, packet);
+          if (ret < 0) {
+            printf("Decode Error.\n");
+            return -1;
+          }
+          if (got_picture) {
+            SDL_LockYUVOverlay(bmp);
+            pFrameYUV->data[0] = bmp->pixels[0];
+            pFrameYUV->data[1] = bmp->pixels[2];
+            pFrameYUV->data[2] = bmp->pixels[1];
+            pFrameYUV->linesize[0] = bmp->pitches[0];
+            pFrameYUV->linesize[1] = bmp->pitches[2];
+            pFrameYUV->linesize[2] = bmp->pitches[1];
+            sws_scale(img_convert_ctx,
+                      (const unsigned char *const *)pFrame->data,
+                      pFrame->linesize, 0, pCodecCtx->height, pFrameYUV->data,
+                      pFrameYUV->linesize);
+
+#if OUTPUT_YUV420P
+            int y_size = pCodecCtx->width * pCodecCtx->height;
+            fwrite(pFrameYUV->data[0], 1, y_size, fp_yuv);      // Y
+            fwrite(pFrameYUV->data[1], 1, y_size / 4, fp_yuv);  // U
+            fwrite(pFrameYUV->data[2], 1, y_size / 4, fp_yuv);  // V
+#endif
+            SDL_UnlockYUVOverlay(bmp);
+
+            SDL_DisplayYUVOverlay(bmp, &rect);
+          }
+        }
+        av_free_packet(packet);
+      } else {
+        // Exit Thread
+        thread_exit = 1;
+      }
+    } else if (event.type == SDL_QUIT) {
+      thread_exit = 1;
+    } else if (event.type == SFM_BREAK_EVENT) {
+      break;
+    }
+  }
+
+  sws_freeContext(img_convert_ctx);
+
+#if OUTPUT_YUV420P
+  fclose(fp_yuv);
+#endif
+
+  SDL_Quit();
+
+  // av_free(out_buffer);
+  av_free(pFrameYUV);
+  avcodec_close(pCodecCtx);
+  avformat_close_input(&pFormatCtx);
+
+  return 0;
+}

thirdparty/ffmpeg/xmake.lua

@@ -45,7 +45,7 @@ package("ffmpeg")
         add_configs("libopenh264",      {description = "Enable libopenh264", default = false, type = "boolean"})
     end
 
-    add_links("avfilter", "avdevice", "avformat", "avcodec", "swscale", "swresample", "avutil")
+    add_links("avfilter", "avdevice", "avformat", "avcodec", "swscale", "swresample", "avutil", "postproc")
     if is_plat("macosx") then
         add_frameworks("CoreFoundation", "Foundation", "CoreVideo", "CoreMedia", "AudioToolbox", "VideoToolbox", "Security")
     elseif is_plat("linux") then

thirdparty/xmake.lua

@@ -2,4 +2,6 @@ includes("sdl2", "projectx")
 if is_plat("windows") then
 elseif is_plat("linux") then
     includes("ffmpeg")
+elseif is_plat("macosx") then
+    includes("ffmpeg")
 end

xmake.lua

@@ -23,7 +23,6 @@ if is_os("windows") then
 elseif is_os("linux") then
     add_requires("ffmpeg 5.1.2", {system = false})
     add_packages("ffmpeg")
-    add_links("SDL2")
     add_syslinks("pthread", "dl")
     add_links("SDL2")
     add_ldflags("-lavformat", "-lavdevice", "-lavfilter", "-lavcodec",
@@ -33,8 +32,11 @@ elseif is_os("linux") then
         {force = true})
 elseif is_os("macosx") then
     add_requires("ffmpeg 5.1.2", {system = false})
-    add_packages("ffmpeg")
-    add_links("SDL2")
+    add_requires("libxcb", {system = false})
+    add_packages("ffmpeg", "libxcb")
+    add_links("SDL2", "SDL2main")
+    add_ldflags("-Wl,-ld_classic")
+    add_frameworks("OpenGL")
 end
 
 add_packages("spdlog", "sdl2", "imgui")
@@ -55,7 +57,7 @@ target("screen_capture")
         add_includedirs("src/screen_capture/windows", {public = true})
     elseif is_os("macosx") then
          add_files("src/screen_capture/macosx/*.cpp")
-         add_includedirs("ssrc/creen_capture/macosx", {public = true})
+         add_includedirs("src/screen_capture/macosx", {public = true})
     elseif is_os("linux") then
          add_files("src/screen_capture/linux/*.cpp")
          add_includedirs("src/screen_capture/linux", {public = true})
@@ -84,29 +86,39 @@ target("remote_desk")
 --     "-lxcb-shm", "-lXext", "-lX11", "-lXv", "-lpthread", "-lSDL2", "-lopenh264",
 --     "-ldl", {force = true})
 
-target("audio_capture")
-    set_kind("binary")
-    add_packages("ffmpeg")
-    add_files("test/audio_capture/sdl2_audio_capture.cpp")
-    add_includedirs("test/audio_capture")
-    add_ldflags("-lavformat", "-lavdevice", "-lavfilter", "-lavcodec",
-    "-lswscale", "-lavutil", "-lswresample",
-    "-lasound", "-lxcb-shape", "-lxcb-xfixes", "-lsndio", "-lxcb", 
-    "-lxcb-shm", "-lXext", "-lX11", "-lXv", "-lpthread", "-lSDL2", "-lopenh264",
-    "-ldl", {force = true})
-    add_links("Shlwapi", "Strmiids", "Vfw32", "Secur32", "Mfuuid")
+-- target("screen_capture")
+--     set_kind("binary")
+--     add_packages("sdl2", "imgui",  "ffmpeg", "openh264")
+--     add_files("test/screen_capture/mac_capture.cpp")
+--     add_ldflags("-lavformat", "-lavdevice", "-lavfilter", "-lavcodec",
+--     "-lswscale", "-lavutil", "-lswresample",
+--     "-lasound", "-lxcb-shape", "-lxcb-xfixes", "-lsndio", "-lxcb", 
+--     "-lxcb-shm", "-lXext", "-lX11", "-lXv", "-lpthread", "-lSDL2", "-lopenh264",
+--     "-ldl", {force = true})
 
-target("play_audio")
-    set_kind("binary")
-    add_packages("ffmpeg")
-    add_files("test/audio_capture/play_loopback.cpp")
-    add_includedirs("test/audio_capture")
-    add_ldflags("-lavformat", "-lavdevice", "-lavfilter", "-lavcodec",
-    "-lswscale", "-lavutil", "-lswresample",
-    "-lasound", "-lxcb-shape", "-lxcb-xfixes", "-lsndio", "-lxcb", 
-    "-lxcb-shm", "-lXext", "-lX11", "-lXv", "-lpthread", "-lSDL2", "-lopenh264",
-    "-ldl", {force = true})
-    add_links("Shlwapi", "Strmiids", "Vfw32", "Secur32", "Mfuuid")
+-- target("audio_capture")
+--     set_kind("binary")
+--     add_packages("ffmpeg")
+--     add_files("test/audio_capture/sdl2_audio_capture.cpp")
+--     add_includedirs("test/audio_capture")
+--     add_ldflags("-lavformat", "-lavdevice", "-lavfilter", "-lavcodec",
+--     "-lswscale", "-lavutil", "-lswresample",
+--     "-lasound", "-lxcb-shape", "-lxcb-xfixes", "-lsndio", "-lxcb", 
+--     "-lxcb-shm", "-lXext", "-lX11", "-lXv", "-lpthread", "-lSDL2", "-lopenh264",
+--     "-ldl", {force = true})
+--     add_links("Shlwapi", "Strmiids", "Vfw32", "Secur32", "Mfuuid")
+
+-- target("play_audio")
+--     set_kind("binary")
+--     add_packages("ffmpeg")
+--     add_files("test/audio_capture/play_loopback.cpp")
+--     add_includedirs("test/audio_capture")
+--     add_ldflags("-lavformat", "-lavdevice", "-lavfilter", "-lavcodec",
+--     "-lswscale", "-lavutil", "-lswresample",
+--     "-lasound", "-lxcb-shape", "-lxcb-xfixes", "-lsndio", "-lxcb", 
+--     "-lxcb-shm", "-lXext", "-lX11", "-lXv", "-lpthread", "-lSDL2", "-lopenh264",
+--     "-ldl", {force = true})
+--     add_links("Shlwapi", "Strmiids", "Vfw32", "Secur32", "Mfuuid")
 
 -- target("audio_capture")
 --     set_kind("binary")