Update x11 capture module

README.md

@@ -4,3 +4,8 @@ vcpkg/buildtrees/versioning_/versions/pcre/69e232f12c4e3eab4115f0672466a6661978b
 
 -    URLS "https://ftp.pcre.org/pub/pcre/pcre-${PCRE_VERSION}.zip"
 +    URLS "https://sourceforge.net/projects/pcre/files/pcre/${PCRE_VERSION}/pcre-${PCRE_VERSION}.zip"
+
+linux
+
+apt-get install nvidia-cuda-toolkit
+solve <cuda.h>

application/remote_desk/remote_desk_gui/main.cpp

@@ -26,6 +26,9 @@
 #include <thread>
 
 extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavdevice/avdevice.h>
+#include <libavfilter/avfilter.h>
 #include <libavformat/avformat.h>
 #include <libavutil/imgutils.h>
 #include <libswscale/swscale.h>
@@ -35,7 +38,11 @@ extern "C" {
 #include "imgui_impl_sdl2.h"
 #include "imgui_impl_sdlrenderer2.h"
 #include "log.h"
+#ifdef _WIN32
 #include "screen_capture_wgc.h"
+#elif __linux__
+#include "screen_capture_x11.h"
+#endif
 #include "x.h"
 
 #define NV12_BUFFER_SIZE 1280 * 720 * 3 / 2
@@ -74,10 +81,14 @@ bool received_frame = false;
 static bool connect_button_pressed = false;
 static const char *connect_label = "Connect";
 
+#ifdef _WIN32
 ScreenCaptureWgc *screen_capture = nullptr;
+#elif __linux__
+ScreenCaptureX11 *screen_capture = nullptr;
+#endif
 
-char *nv12_buffer_ = nullptr;
+char *nv12_buffer = nullptr;
-std::chrono::steady_clock::time_point last_frame_time_;
+std::chrono::_V2::system_clock::time_point last_frame_time_;
 
 typedef enum { mouse = 0, keyboard } ControlType;
 typedef enum { move = 0, left_down, left_up, right_down, right_up } MouseFlag;
@@ -527,7 +538,7 @@ int main() {
             if (strcmp(online_label, "Online") == 0) {
               CreateConnection(peer_server, mac_addr, server_password);
 
-              nv12_buffer_ = new char[NV12_BUFFER_SIZE];
+              nv12_buffer = new char[NV12_BUFFER_SIZE];
 #ifdef _WIN32
               screen_capture = new ScreenCaptureWgc();
 
@@ -549,9 +560,38 @@ int main() {
 
                     if (tc >= 0) {
                       BGRAToNV12FFmpeg(data, width, height,
-                                       (unsigned char *)nv12_buffer_);
+                                       (unsigned char *)nv12_buffer);
                       SendData(peer_server, DATA_TYPE::VIDEO,
-                               (const char *)nv12_buffer_, NV12_BUFFER_SIZE);
+                               (const char *)nv12_buffer, NV12_BUFFER_SIZE);
+                      // std::cout << "Send" << std::endl;
+                      last_frame_time_ = now_time;
+                    }
+                  });
+
+              screen_capture->Start();
+
+#elif __linux__
+              screen_capture = new ScreenCaptureX11();
+
+              RECORD_DESKTOP_RECT rect;
+              rect.left = 0;
+              rect.top = 0;
+              rect.right = 0;
+              rect.bottom = 0;
+
+              last_frame_time_ = std::chrono::high_resolution_clock::now();
+              screen_capture->Init(
+                  rect, 60,
+                  [](unsigned char *data, int size, int width,
+                     int height) -> void {
+                    auto now_time = std::chrono::high_resolution_clock::now();
+                    std::chrono::duration<double> duration =
+                        now_time - last_frame_time_;
+                    auto tc = duration.count() * 1000;
+
+                    if (tc >= 0) {
+                      SendData(peer_server, DATA_TYPE::VIDEO,
+                               (const char *)nv12_buffer, NV12_BUFFER_SIZE);
                       // std::cout << "Send" << std::endl;
                       last_frame_time_ = now_time;
                     }

application/remote_desk/screen_capture/linux/screen_capture_x11.cpp

@@ -11,16 +11,20 @@ ScreenCaptureX11::~ScreenCaptureX11() {}
 
 int ScreenCaptureX11::Init(const RECORD_DESKTOP_RECT &rect, const int fps,
                            cb_desktop_data cb) {
+  if (cb) {
+    _on_data = cb;
+  }
+
   pFormatCtx_ = avformat_alloc_context();
 
   avdevice_register_all();
 
   // grabbing frame rate
-  av_dict_set(&options_, "framerate", "5", 0);
+  av_dict_set(&options_, "framerate", "30", 0);
   // Make the grabbed area follow the mouse
   av_dict_set(&options_, "follow_mouse", "centered", 0);
   // Video frame size. The default is to capture the full screen
-  av_dict_set(&options_, "video_size", "1280x720", 0);
+  // av_dict_set(&options_, "video_size", "1280x720", 0);
   ifmt_ = (AVInputFormat *)av_find_input_format("x11grab");
   if (!ifmt_) {
     printf("Couldn't find_input_format\n");
@@ -103,6 +107,10 @@ int ScreenCaptureX11::Start() {
             sws_scale(img_convert_ctx_, pFrame_->data, pFrame_->linesize, 0,
                       pFrame_->height, pFrameNV12_->data,
                       pFrameNV12_->linesize);
+
+            _on_data((unsigned char *)nv12_buffer_,
+                     pFrame_->width * pFrame_->height * 3 / 2, pFrame_->width,
+                     pFrame_->height);
           }
         }
       }

application/remote_desk/screen_capture/linux/x11_session_impl.cpp

@@ -38,14 +38,12 @@ int X11SessionImpl::Start() {
 
 int X11SessionImpl::Stop() { return 0; }
 
-int X11SessionImpl::Pause() {}
+int X11SessionImpl::Pause() { return 0; }
 
-int X11SessionImpl::Resume() {}
+int X11SessionImpl::Resume() { return 0; }
 
 void X11SessionImpl::OnFrame() {}
 
 void X11SessionImpl::OnClosed() {}
 
-int X11SessionImpl::Initialize() { return 0; }
-
 void X11SessionImpl::CleanUp() {}

application/remote_desk/screen_capture/linux/x11_session_impl.h

@@ -28,7 +28,6 @@ class X11SessionImpl : public X11Session {
   void OnFrame();
   void OnClosed();
 
-  int Initialize();
   void CleanUp();
 
   // void message_func();

application/remote_desk/xmake.lua

@@ -38,7 +38,7 @@ target("log")
 
 target("screen_capture")
     set_kind("static")
-    add_packages("log")
+    add_packages("log", "ffmpeg")
     if is_os("windows") then
         add_files("screen_capture/windows/*.cpp")
         add_includedirs("screen_capture/windows", {public = true})
@@ -63,4 +63,9 @@ target("remote_desk")
         add_links("SDL2")
     elseif is_os("linux") then
         add_links("SDL2")
+        -- add_ldflags("-lsndio", "-lasound", "-lxcb", "-lX11", "-lXext", "-lXv", 
+        -- "-lxcb-shape", "-lxcb-xfixes", "-lxcb-shm", "-lavfilter", "-ldl", 
+        -- "-lavdevice", "-lavformat", "-lavcodec", "-lswscale", "-lswresample", 
+        -- "-lavutil", {force = true})
+        -- -- add_ldflags("-lasound", "-lX11", "-lXext", "-lxcb", "-lsndio", "-lpostproc", "-ldl", {force = true})
     end

tests/peerconnection/linux_capture.cpp

@@ -29,6 +29,8 @@ extern "C" {
 #endif
 #endif
 
+#include <chrono>
+
 // Output YUV420P
 #define OUTPUT_YUV420P 0
 //'1' Use Dshow
@@ -47,6 +49,10 @@ SDL_Texture *sdlTexture = nullptr;
 SDL_Renderer *sdlRenderer = nullptr;
 SDL_Rect sdlRect;
 unsigned char nv12_buffer[NV12_BUFFER_SIZE];
+std::chrono::_V2::system_clock::time_point last_frame_time;
+const int pixel_w = 1280, pixel_h = 720;
+int screen_w = 1280, screen_h = 720;
+bool done = false;
 
 int YUV420ToNV12FFmpeg(unsigned char *src_buffer, int width, int height,
                        unsigned char *des_buffer) {
@@ -78,7 +84,7 @@ int sfp_refresh_thread(void *opaque) {
     SDL_Event event;
     event.type = SFM_REFRESH_EVENT;
     SDL_PushEvent(&event);
-    SDL_Delay(40);
+    SDL_Delay(30);
     printf("sfp_refresh_thread\n");
   }
   thread_exit = 0;
@@ -113,7 +119,7 @@ int main(int argc, char *argv[]) {
   AVDictionary *options = NULL;
   // Set some options
   // grabbing frame rate
-  av_dict_set(&options, "framerate", "5", 0);
+  av_dict_set(&options, "framerate", "30", 0);
   // Make the grabbed area follow the mouse
   av_dict_set(&options, "follow_mouse", "centered", 0);
   // Video frame size. The default is to capture the full screen
@@ -173,8 +179,7 @@ int main(int argc, char *argv[]) {
     printf("Could not initialize SDL - %s\n", SDL_GetError());
     return -1;
   }
-  const int pixel_w = 1280, pixel_h = 720;
+
-  int screen_w = 1280, screen_h = 720;
   // const SDL_VideoInfo *vi = SDL_GetVideoInfo();
   // Half of the Desktop's width and height.
   screen_w = 1280;
@@ -223,34 +228,32 @@ int main(int argc, char *argv[]) {
   // SDL_WM_SetCaption("Simplest FFmpeg Grab Desktop", NULL);
   // Event Loop
   SDL_Event event;
-  printf("111111111\n");
+
+  last_frame_time = std::chrono::high_resolution_clock::now();
+
   for (;;) {
     // Wait
     SDL_WaitEvent(&event);
-    printf("11112222\n");
+
-    if (event.type == SFM_REFRESH_EVENT) {
+    if (1) {
-      printf("11111113333333\n");
       //------------------------------
       if (av_read_frame(pFormatCtx, packet) >= 0) {
-        printf("111111444444\n");
         if (packet->stream_index == videoindex) {
-          printf("11111155555\n");
           avcodec_send_packet(pCodecCtx, packet);
           got_picture = avcodec_receive_frame(pCodecCtx, pFrame);
-          printf("33333333\n");
           // ret = avcodec_decode_video2(pCodecCtx, pFrame, &got_picture,
           // packet);
           if (ret < 0) {
             printf("Decode Error.\n");
             return -1;
           }
+          printf("xxxxxxxxxxxxxxxxxxx\n");
           if (!got_picture) {
-            printf("44444444444\n");
+            auto now_time = std::chrono::high_resolution_clock::now();
-
+            std::chrono::duration<double> duration = now_time - last_frame_time;
-            // memcpy(nv12_buffer, pFrame->data[0],
+            auto tc = duration.count() * 1000;
-            //        pFrame->width * pFrame->height);
+            printf("duration: %f\n", tc);
-            // memcpy(nv12_buffer + pFrame->width * pFrame->height,
+            last_frame_time = now_time;
-            //        pFrame->data[1], pFrame->width * pFrame->height / 2);
 
             av_image_fill_arrays(pFrameNV12->data, pFrameNV12->linesize,
                                  nv12_buffer, AV_PIX_FMT_NV12, pFrame->width,
@@ -291,7 +294,7 @@ int main(int argc, char *argv[]) {
 #if OUTPUT_YUV420P
   fclose(fp_yuv);
 #endif
-  printf("222222222\n");
+
   SDL_Quit();
 
   // av_free(out_buffer);

xmake.lua

@@ -7,23 +7,6 @@ set_languages("c++17")
 
 add_rules("mode.release", "mode.debug")
 
-add_requires("asio 1.24.0", "nlohmann_json", "spdlog 1.11.0")
-
-if is_os("windows") then
-    add_requires("vcpkg::ffmpeg 5.1.2", {configs = {shared = false}})
-    add_packages("vcpkg::ffmpeg")
-    add_requires("vcpkg::libnice 0.1.21")
-    add_packages("vcpkg::libnice")
-elseif is_os("linux") then
-    add_requires("vcpkg::ffmpeg", {configs = {shared = false}})
-    add_requires("glib", {system = true})
-    add_requires("vcpkg::libnice 0.1.21")
-    add_packages("ffmpeg", "glib", "vcpkg::libnice")
-elseif is_os("macosx") then
-    add_requires("ffmpeg 5.1.2", {system = false})
-    add_requires("brew::libnice", "brew::glib")
-end
-
 add_defines("ASIO_STANDALONE", "ASIO_HAS_STD_TYPE_TRAITS", "ASIO_HAS_STD_SHARED_PTR", 
     "ASIO_HAS_STD_ADDRESSOF", "ASIO_HAS_STD_ATOMIC", "ASIO_HAS_STD_CHRONO", "ASIO_HAS_CSTDINT", "ASIO_HAS_STD_ARRAY",
     "ASIO_HAS_STD_SYSTEM_ERROR")
@@ -40,6 +23,22 @@ elseif is_os("macosx") then
     add_ldflags("-ld_classic", {force = true})
 end
 
+add_requires("asio 1.24.0", "nlohmann_json", "spdlog 1.11.0")
+
+if is_os("windows") then
+    add_requires("vcpkg::ffmpeg 5.1.2", {configs = {shared = false}})
+    add_requires("vcpkg::libnice 0.1.21")
+    add_packages("vcpkg::libnice")
+elseif is_os("linux") then
+    add_requires("ffmpeg 5.1.2", {shared = true})
+    add_requires("glib", {system = true})
+    add_requires("vcpkg::libnice 0.1.21")
+    add_packages("glib", "vcpkg::libnice")
+elseif is_os("macosx") then
+    add_requires("ffmpeg 5.1.2", {system = false})
+    add_requires("brew::libnice", "brew::glib")
+end
+
 add_packages("spdlog")
 
 includes("thirdparty")
@@ -125,7 +124,7 @@ target("media")
     set_kind("static")
     add_deps("log", "frame")
     if is_os("windows") then
-        add_packages("cuda")
+        add_packages("cuda", "vcpkg::ffmpeg")
         add_files("src/media/video/encode/*.cpp",
         "src/media/video/decode/*.cpp",
         "src/media/video/encode/nvcodec/*.cpp",
@@ -143,7 +142,7 @@ target("media")
         add_linkdirs("thirdparty/nvcodec/Lib/x64")
         add_links("cuda", "nvencodeapi", "nvcuvid")
     elseif is_os(("linux")) then
-        add_packages("cuda")
+        add_packages("cuda", "ffmpeg")
         add_files("src/media/video/encode/*.cpp",
         "src/media/video/decode/*.cpp",
         "src/media/video/encode/nvcodec/*.cpp",
@@ -230,6 +229,6 @@ target("nicetest")
 
 target("linux_capture")
     set_kind("binary")
-    add_packages("vcpkg::ffmpeg", "sdl2", "asound")
+    add_packages("ffmpeg", "sdl2", "asound")
     add_files("tests/peerconnection/linux_capture.cpp")
     add_ldflags("-lasound", "-lX11", "-lXext", "-lxcb", "-lsndio", "-lpostproc", "-ldl", {force = true})