From df686461a556a84347a9850e39c92468cc774932 Mon Sep 17 00:00:00 2001 From: dijunkun Date: Thu, 14 Sep 2023 16:32:22 +0800 Subject: [PATCH] Fix data parse error --- .../remote_desk_client/remote_desk_client.cpp | 101 +++++++++++++++--- .../remote_desk_server/remote_desk_server.cpp | 96 ++++++++++++++--- .../remote_desk_server/remote_desk_server.h | 6 +- application/remote_desk/xmake.lua | 17 ++- src/interface/x.h | 4 +- src/pc/peer_connection.cpp | 58 +++++++--- src/pc/peer_connection.h | 16 ++- src/rtc/x_inner.cpp | 6 +- src/rtp/rtp_codec.cpp | 2 +- src/rtp/rtp_data_receiver.cpp | 87 ++++++++++++++- src/rtp/rtp_data_receiver.h | 28 ++++- src/rtp/rtp_data_sender.cpp | 2 +- src/rtp/rtp_packet.cpp | 42 +++++--- src/rtp/rtp_packet.h | 87 +++++++++++---- src/rtp/rtp_statistics.cpp | 4 +- src/rtp/rtp_video_receiver.cpp | 2 +- src/rtp/rtp_video_sender.cpp | 2 +- src/transmission/ice_transmission.cpp | 45 ++++++-- src/transmission/ice_transmission.h | 36 +++++-- 19 files changed, 526 insertions(+), 115 deletions(-) diff --git a/application/remote_desk/remote_desk_client/remote_desk_client.cpp b/application/remote_desk/remote_desk_client/remote_desk_client.cpp index 3c1bf9c..4cf8f68 100644 --- a/application/remote_desk/remote_desk_client/remote_desk_client.cpp +++ b/application/remote_desk/remote_desk_client/remote_desk_client.cpp @@ -4,6 +4,7 @@ #endif #include +#include #include #include #include @@ -34,11 +35,26 @@ std::string window_title = "Remote Desk Client"; int thread_exit = 0; PeerPtr *peer = nullptr; +typedef enum { mouse = 0, keyboard } ControlType; +typedef enum { move = 0, left_down, left_up, right_down, right_up } MouseFlag; +typedef enum { key_down = 0, key_up } KeyFlag; typedef struct { - SDL_KeyCode key; - SDL_EventType action; - int px; - int py; + long x; + long y; + MouseFlag flag; +} Mouse; + +typedef struct { + long key_value; + KeyFlag flag; +} Key; + +typedef struct { + ControlType type; + union { + Mouse m; + Key k; + }; } RemoteAction; inline void FreshVideo() { @@ -55,13 +71,6 @@ inline void FreshVideo() { inline int ProcessMouseKeyEven(SDL_Event &ev) { RemoteAction remote_action; - remote_action.key = SDL_KeyCode(ev.key.keysym.sym); - remote_action.action = SDL_EventType(ev.type); - remote_action.px = ev.button.x; - remote_action.py = ev.button.y; - - SendData(peer, DATA_TYPE::DATA, (const char *)&remote_action, - sizeof(remote_action)); if (SDL_KEYDOWN == ev.type) // SDL_KEYUP { @@ -84,10 +93,20 @@ inline int ProcessMouseKeyEven(SDL_Event &ev) { int py = ev.button.y; printf("SDL_MOUSEBUTTONDOWN x, y %d %d \n", px, py); + remote_action.type = ControlType::mouse; + remote_action.m.flag = MouseFlag::left_down; + remote_action.m.x = ev.button.x; + remote_action.m.y = ev.button.y; + } else if (SDL_BUTTON_RIGHT == ev.button.button) { int px = ev.button.x; int py = ev.button.y; printf("SDL_BUTTON_RIGHT x, y %d %d \n", px, py); + + remote_action.type = ControlType::mouse; + remote_action.m.flag = MouseFlag::right_down; + remote_action.m.x = ev.button.x; + remote_action.m.y = ev.button.y; } } else if (SDL_MOUSEBUTTONUP == ev.type) { if (SDL_BUTTON_LEFT == ev.button.button) { @@ -95,16 +114,31 @@ inline int ProcessMouseKeyEven(SDL_Event &ev) { int py = ev.button.y; printf("SDL_MOUSEBUTTONUP x, y %d %d \n", px, py); + remote_action.type = ControlType::mouse; + remote_action.m.flag = MouseFlag::left_up; + remote_action.m.x = ev.button.x; + remote_action.m.y = ev.button.y; + } else if (SDL_BUTTON_RIGHT == ev.button.button) { int px = ev.button.x; int py = ev.button.y; printf("SDL_MOUSEBUTTONUP x, y %d %d \n", px, py); + + remote_action.type = ControlType::mouse; + remote_action.m.flag = MouseFlag::right_up; + remote_action.m.x = ev.button.x; + remote_action.m.y = ev.button.y; } } else if (SDL_MOUSEMOTION == ev.type) { int px = ev.motion.x; int py = ev.motion.y; printf("SDL_MOUSEMOTION x, y %d %d \n", px, py); + + remote_action.type = ControlType::mouse; + remote_action.m.flag = MouseFlag::move; + remote_action.m.x = ev.button.x; + remote_action.m.y = ev.button.y; } else if (SDL_QUIT == ev.type) { SDL_Event event; event.type = SDL_QUIT; @@ -113,10 +147,15 @@ inline int ProcessMouseKeyEven(SDL_Event &ev) { return 0; } + SendData(peer, DATA_TYPE::DATA, (const char *)&remote_action, + sizeof(remote_action)); + // std::cout << remote_action.type << " " << remote_action.type << " " + // << remote_action.px << " " << remote_action.py << std::endl; + return 0; } -void GuestReceiveBuffer(const char *data, size_t size, const char *user_id, +void ReceiveVideoBuffer(const char *data, size_t size, const char *user_id, size_t user_id_size) { // std::cout << "Receive: [" << user_id << "] " << std::endl; memcpy(dst_buffer, data, size); @@ -136,6 +175,18 @@ void GuestReceiveBuffer(const char *data, size_t size, const char *user_id, } } +void ReceiveAudioBuffer(const char *data, size_t size, const char *user_id, + size_t user_id_size) { + std::cout << "Receive audio, size " << size << ", user [" << user_id << "] " + << std::endl; +} + +void ReceiveDataBuffer(const char *data, size_t size, const char *user_id, + size_t user_id_size) { + std::cout << "Receive data, size " << size << ", user [" << user_id << "] " + << std::endl; +} + std::string GetMac() { IP_ADAPTER_INFO adapterInfo[16]; DWORD bufferSize = sizeof(adapterInfo); @@ -157,11 +208,16 @@ std::string GetMac() { } int main() { - Params params; - params.cfg_path = "../../../../config/config.ini"; - params.on_receive_buffer = GuestReceiveBuffer; + std::string default_cfg_path = "../../../../config/config.ini"; + std::ifstream f(default_cfg_path.c_str()); - std::string transmission_id = "000000"; + Params params; + params.cfg_path = f.good() ? "../../../../config/config.ini" : "config.ini"; + params.on_receive_video_buffer = ReceiveVideoBuffer; + params.on_receive_audio_buffer = ReceiveAudioBuffer; + params.on_receive_data_buffer = ReceiveDataBuffer; + + std::string transmission_id = "000001"; std::string user_id = GetMac(); peer = CreatePeer(¶ms); @@ -199,7 +255,18 @@ int main() { if (event.type == REFRESH_EVENT) { FreshVideo(); } else if (event.type == SDL_WINDOWEVENT) { - SDL_GetWindowSize(screen, &screen_w, &screen_h); + int new_screen_w = 0; + int new_screen_h = 0; + SDL_GetWindowSize(screen, &new_screen_w, &new_screen_h); + + if (new_screen_w != screen_w) { + screen_w = new_screen_w; + screen_h = new_screen_w * 9 / 16; + } else if (new_screen_h != screen_h) { + screen_w = new_screen_h * 16 / 9; + screen_h = new_screen_h; + } + SDL_SetWindowSize(screen, screen_w, screen_h); printf("Resize windows: %dx%d\n", screen_w, screen_h); } else if (event.type == SDL_QUIT) { return 0; diff --git a/application/remote_desk/remote_desk_server/remote_desk_server.cpp b/application/remote_desk/remote_desk_server/remote_desk_server.cpp index 326abba..b95a68a 100644 --- a/application/remote_desk/remote_desk_server/remote_desk_server.cpp +++ b/application/remote_desk/remote_desk_server/remote_desk_server.cpp @@ -3,10 +3,16 @@ #ifdef _WIN32 #include #include +#include #endif +#define SDL_MAIN_HANDLED + +#include #include +#include "SDL2/SDL.h" + extern "C" { #include #include @@ -15,6 +21,31 @@ extern "C" { #define NV12_BUFFER_SIZE 1280 * 720 * 3 / 2 +int screen_w = 0; +int screen_h = 0; + +typedef enum { mouse = 0, keyboard } ControlType; +typedef enum { move = 0, left_down, left_up, right_down, right_up } MouseFlag; +typedef enum { key_down = 0, key_up } KeyFlag; +typedef struct { + long x; + long y; + MouseFlag flag; +} Mouse; + +typedef struct { + long key_value; + KeyFlag flag; +} Key; + +typedef struct { + ControlType type; + union { + Mouse m; + Key k; + }; +} RemoteAction; + RemoteDeskServer ::RemoteDeskServer() {} RemoteDeskServer ::~RemoteDeskServer() { @@ -48,15 +79,54 @@ int BGRAToNV12FFmpeg(unsigned char *src_buffer, int width, int height, return 0; } -void RemoteDeskServer::HostReceiveBuffer(const char *data, size_t size, - const char *user_id, - size_t user_id_size) { +void RemoteDeskServer::ReceiveVideoBuffer(const char *data, size_t size, + const char *user_id, + size_t user_id_size) { std::string msg(data, size); std::string user(user_id, user_id_size); std::cout << "Receive: [" << user << "] " << msg << std::endl; } +void RemoteDeskServer::ReceiveAudioBuffer(const char *data, size_t size, + const char *user_id, + size_t user_id_size) {} + +void RemoteDeskServer::ReceiveDataBuffer(const char *data, size_t size, + const char *user_id, + size_t user_id_size) { + std::string user(user_id, user_id_size); + + RemoteAction remote_action; + memcpy(&remote_action, data, sizeof(remote_action)); + + INPUT ip; + + if (remote_action.type == ControlType::mouse) { + ip.type = INPUT_MOUSE; + ip.mi.dx = remote_action.m.x * screen_w / 1280; + ip.mi.dy = remote_action.m.y * screen_h / 720; + if (remote_action.m.flag == MouseFlag::left_down) { + ip.mi.dwFlags = MOUSEEVENTF_LEFTDOWN | MOUSEEVENTF_ABSOLUTE; + } else if (remote_action.m.flag == MouseFlag::left_up) { + ip.mi.dwFlags = MOUSEEVENTF_LEFTUP | MOUSEEVENTF_ABSOLUTE; + } else if (remote_action.m.flag == MouseFlag::right_down) { + ip.mi.dwFlags = MOUSEEVENTF_RIGHTDOWN | MOUSEEVENTF_ABSOLUTE; + } else if (remote_action.m.flag == MouseFlag::right_up) { + ip.mi.dwFlags = MOUSEEVENTF_RIGHTUP | MOUSEEVENTF_ABSOLUTE; + } + ip.mi.mouseData = 0; + ip.mi.time = 0; + + // Send the press + SendInput(1, &ip, sizeof(INPUT)); + + std::cout << "Receive data from [" << user << "], " << ip.type << " " + << ip.mi.dwFlags << " " << ip.mi.dx << " " << ip.mi.dy + << std::endl; + } +} + std::string GetMac() { IP_ADAPTER_INFO adapterInfo[16]; DWORD bufferSize = sizeof(adapterInfo); @@ -78,17 +148,16 @@ std::string GetMac() { } int RemoteDeskServer::Init() { + std::string default_cfg_path = "../../../../config/config.ini"; + std::ifstream f(default_cfg_path.c_str()); + Params params; - params.cfg_path = "../../../../config/config.ini"; - params.on_receive_buffer = [](const char *data, size_t size, - const char *user_id, size_t user_id_size) { - // std::string msg(data, size); - // std::string user(user_id, user_id_size); + params.cfg_path = f.good() ? "../../../../config/config.ini" : "config.ini"; + params.on_receive_video_buffer = ReceiveVideoBuffer; + params.on_receive_audio_buffer = ReceiveAudioBuffer; + params.on_receive_data_buffer = ReceiveDataBuffer; - // std::cout << "Receive: [" << user << "] " << msg << std::endl; - }; - - std::string transmission_id = "000000"; + std::string transmission_id = "000001"; std::string user_id = "Server-" + GetMac(); peer = CreatePeer(¶ms); CreateConnection(peer, transmission_id.c_str(), user_id.c_str()); @@ -103,6 +172,9 @@ int RemoteDeskServer::Init() { rect.right = GetSystemMetrics(SM_CXSCREEN); rect.bottom = GetSystemMetrics(SM_CYSCREEN); + screen_w = GetSystemMetrics(SM_CXSCREEN); + screen_h = GetSystemMetrics(SM_CYSCREEN); + last_frame_time_ = std::chrono::high_resolution_clock::now(); screen_capture->Init( rect, 60, diff --git a/application/remote_desk/remote_desk_server/remote_desk_server.h b/application/remote_desk/remote_desk_server/remote_desk_server.h index 7aa5e44..fdd9c3c 100644 --- a/application/remote_desk/remote_desk_server/remote_desk_server.h +++ b/application/remote_desk/remote_desk_server/remote_desk_server.h @@ -12,7 +12,11 @@ class RemoteDeskServer { public: int Init(); - static void HostReceiveBuffer(const char* data, size_t size, + static void ReceiveVideoBuffer(const char* data, size_t size, + const char* user_id, size_t user_id_size); + static void ReceiveAudioBuffer(const char* data, size_t size, + const char* user_id, size_t user_id_size); + static void ReceiveDataBuffer(const char* data, size_t size, const char* user_id, size_t user_id_size); private: diff --git a/application/remote_desk/xmake.lua b/application/remote_desk/xmake.lua index 72e059a..62ec3e7 100644 --- a/application/remote_desk/xmake.lua +++ b/application/remote_desk/xmake.lua @@ -37,27 +37,22 @@ target("screen_capture") target("remote_desk_server") set_kind("binary") - add_packages("log", "vcpkg::ffmpeg") + add_packages("log", "vcpkg::ffmpeg", "vcpkg::sdl2") add_deps("projectx", "screen_capture") add_files("remote_desk_server/*.cpp") add_includedirs("../../src/interface") add_links("swscale", "avutil") add_defines("WIN32_LEAN_AND_MEAN") - -- if is_os("windows") then - -- add_links("iphlpapi") - -- add_includedirs("../../thirdparty/ffmpeg/include") - -- if is_mode("debug") then - -- add_linkdirs("../../thirdparty/ffmpeg/lib/debug/win") - -- else - -- add_linkdirs("../../thirdparty/ffmpeg/lib/release/win") - -- end - -- end + if is_os("windows") then + add_links("SDL2-static", "SDL2main", "gdi32", "winmm", + "setupapi", "version", "Imm32", "iphlpapi") + end + target("remote_desk_client") set_kind("binary") add_deps("projectx") add_packages("log") - add_packages("ffmpeg") add_packages("vcpkg::sdl2") add_files("remote_desk_client/*.cpp") add_includedirs("../../src/interface") diff --git a/src/interface/x.h b/src/interface/x.h index a2dbaa2..1adab52 100644 --- a/src/interface/x.h +++ b/src/interface/x.h @@ -19,7 +19,9 @@ typedef void (*NetStatusReport)(const unsigned short, const unsigned short); typedef struct { const char* cfg_path; - OnReceiveBuffer on_receive_buffer; + OnReceiveBuffer on_receive_video_buffer; + OnReceiveBuffer on_receive_audio_buffer; + OnReceiveBuffer on_receive_data_buffer; NetStatusReport net_status_report; } Params; diff --git a/src/pc/peer_connection.cpp b/src/pc/peer_connection.cpp index 58033fb..aa2b341 100644 --- a/src/pc/peer_connection.cpp +++ b/src/pc/peer_connection.cpp @@ -9,8 +9,7 @@ using nlohmann::json; -PeerConnection::PeerConnection(OnReceiveBuffer on_receive_buffer) - : on_receive_buffer_(on_receive_buffer) {} +PeerConnection::PeerConnection() {} PeerConnection::~PeerConnection() { if (nv12_data_) { @@ -40,10 +39,14 @@ int PeerConnection::Init(PeerConnectionParams params, LOG_INFO("stun server ip [{}] port [{}]", cfg_stun_server_ip_, stun_server_port_); + on_receive_video_buffer_ = params.on_receive_video_buffer; + on_receive_audio_buffer_ = params.on_receive_audio_buffer; + on_receive_data_buffer_ = params.on_receive_data_buffer; + on_receive_ws_msg_ = [this](const std::string &msg) { ProcessSignal(msg); }; - on_receive_ice_msg_ = [this](const char *data, size_t size, - const char *user_id, size_t user_id_size) { + on_receive_video_ = [this](const char *data, size_t size, const char *user_id, + size_t user_id_size) { int num_frame_returned = Decode((uint8_t *)data, size); uint32_t width = 0; uint32_t height = 0; @@ -51,8 +54,24 @@ int PeerConnection::Init(PeerConnectionParams params, for (size_t i = 0; i < num_frame_returned; ++i) { int ret = GetFrame((uint8_t *)nv12_data_, width, height, frame_size); - on_receive_buffer_(nv12_data_, width * height * 3 / 2, user_id, - user_id_size); + if (on_receive_video_buffer_) { + on_receive_video_buffer_(nv12_data_, width * height * 3 / 2, user_id, + user_id_size); + } + } + }; + + on_receive_audio_ = [this](const char *data, size_t size, const char *user_id, + size_t user_id_size) { + if (on_receive_audio_buffer_) { + on_receive_audio_buffer_(data, size, user_id, user_id_size); + } + }; + + on_receive_data_ = [this](const char *data, size_t size, const char *user_id, + size_t user_id_size) { + if (on_receive_data_buffer_) { + on_receive_data_buffer_(data, size, user_id, user_id_size); } }; @@ -152,9 +171,17 @@ void PeerConnection::ProcessSignal(const std::string &signal) { for (auto &remote_user_id : user_id_list_) { ice_transmission_list_[remote_user_id] = - std::make_unique( - true, transmission_id, user_id_, remote_user_id, ws_transport_, - on_receive_ice_msg_, on_ice_status_change_); + std::make_unique(true, transmission_id, user_id_, + remote_user_id, ws_transport_, + on_ice_status_change_); + + ice_transmission_list_[remote_user_id]->SetOnReceiveVideoFunc( + on_receive_video_); + ice_transmission_list_[remote_user_id]->SetOnReceiveAudioFunc( + on_receive_audio_); + ice_transmission_list_[remote_user_id]->SetOnReceiveDataFunc( + on_receive_data_); + ice_transmission_list_[remote_user_id]->InitIceTransmission( cfg_stun_server_ip_, stun_server_port_); ice_transmission_list_[remote_user_id]->JoinTransmission(); @@ -186,9 +213,16 @@ void PeerConnection::ProcessSignal(const std::string &signal) { LOG_INFO("[{}] receive offer from [{}]", user_id_, remote_user_id); ice_transmission_list_[remote_user_id] = - std::make_unique( - false, transmission_id, user_id_, remote_user_id, ws_transport_, - on_receive_ice_msg_, on_ice_status_change_); + std::make_unique(false, transmission_id, user_id_, + remote_user_id, ws_transport_, + on_ice_status_change_); + + ice_transmission_list_[remote_user_id]->SetOnReceiveVideoFunc( + on_receive_video_); + ice_transmission_list_[remote_user_id]->SetOnReceiveAudioFunc( + on_receive_audio_); + ice_transmission_list_[remote_user_id]->SetOnReceiveDataFunc( + on_receive_data_); ice_transmission_list_[remote_user_id]->InitIceTransmission( cfg_stun_server_ip_, stun_server_port_); diff --git a/src/pc/peer_connection.h b/src/pc/peer_connection.h index f130462..2d6a728 100644 --- a/src/pc/peer_connection.h +++ b/src/pc/peer_connection.h @@ -18,13 +18,15 @@ typedef void (*NetStatusReport)(const unsigned short, const unsigned short); typedef struct { const char *cfg_path; - OnReceiveBuffer on_receive_buffer; + OnReceiveBuffer on_receive_video_buffer; + OnReceiveBuffer on_receive_audio_buffer; + OnReceiveBuffer on_receive_data_buffer; NetStatusReport net_status_report; } PeerConnectionParams; class PeerConnection : public VideoEncoder, VideoDecoder { public: - PeerConnection(OnReceiveBuffer on_receive_buffer); + PeerConnection(); ~PeerConnection(); public: @@ -73,11 +75,17 @@ class PeerConnection : public VideoEncoder, VideoDecoder { std::map> ice_transmission_list_; std::function - on_receive_ice_msg_ = nullptr; + on_receive_video_ = nullptr; + std::function + on_receive_audio_ = nullptr; + std::function + on_receive_data_ = nullptr; std::function on_ice_status_change_ = nullptr; bool ice_ready_ = false; - OnReceiveBuffer on_receive_buffer_; + OnReceiveBuffer on_receive_video_buffer_; + OnReceiveBuffer on_receive_audio_buffer_; + OnReceiveBuffer on_receive_data_buffer_; char *nv12_data_ = nullptr; }; diff --git a/src/rtc/x_inner.cpp b/src/rtc/x_inner.cpp index 9e6f7dc..bd53d2a 100644 --- a/src/rtc/x_inner.cpp +++ b/src/rtc/x_inner.cpp @@ -14,9 +14,11 @@ static PeerConnection *peer_connection; PeerPtr *CreatePeer(const Params *params) { PeerPtr *peer_ptr = new PeerPtr; - peer_ptr->peer_connection = new PeerConnection(params->on_receive_buffer); + peer_ptr->peer_connection = new PeerConnection(); peer_ptr->pc_params.cfg_path = params->cfg_path; - peer_ptr->pc_params.on_receive_buffer = params->on_receive_buffer; + peer_ptr->pc_params.on_receive_video_buffer = params->on_receive_video_buffer; + peer_ptr->pc_params.on_receive_audio_buffer = params->on_receive_audio_buffer; + peer_ptr->pc_params.on_receive_data_buffer = params->on_receive_data_buffer; peer_ptr->pc_params.net_status_report = params->net_status_report; return peer_ptr; diff --git a/src/rtp/rtp_codec.cpp b/src/rtp/rtp_codec.cpp index 4bf5acf..2bb539c 100644 --- a/src/rtp/rtp_codec.cpp +++ b/src/rtp/rtp_codec.cpp @@ -154,6 +154,6 @@ size_t RtpCodec::Decode(RtpPacket& packet, uint8_t* payload) { return packet.DecodeH264Fua(payload); } else { LOG_ERROR("Default"); - return packet.Decode(payload); + return packet.DecodeData(payload); } } \ No newline at end of file diff --git a/src/rtp/rtp_data_receiver.cpp b/src/rtp/rtp_data_receiver.cpp index c4fd976..2185fd2 100644 --- a/src/rtp/rtp_data_receiver.cpp +++ b/src/rtp/rtp_data_receiver.cpp @@ -1,5 +1,90 @@ #include "rtp_data_receiver.h" +#define RTCP_RR_INTERVAL 1000 + RtpDataReceiver::RtpDataReceiver() {} -RtpDataReceiver::~RtpDataReceiver() {} \ No newline at end of file +RtpDataReceiver::~RtpDataReceiver() { + if (rtp_statistics_) { + rtp_statistics_->Stop(); + } +} + +void RtpDataReceiver::InsertRtpPacket(RtpPacket& rtp_packet) { + if (!rtp_statistics_) { + rtp_statistics_ = std::make_unique(); + rtp_statistics_->Start(); + } + + if (rtp_statistics_) { + rtp_statistics_->UpdateReceiveBytes(rtp_packet.Size()); + } + + if (CheckIsTimeSendRR()) { + RtcpReceiverReport rtcp_rr; + RtcpReportBlock report; + + auto duration = std::chrono::system_clock::now().time_since_epoch(); + auto seconds = std::chrono::duration_cast(duration); + uint32_t seconds_u32 = static_cast( + std::chrono::duration_cast(duration).count()); + + uint32_t fraction_u32 = static_cast( + std::chrono::duration_cast(duration - seconds) + .count()); + + report.source_ssrc = 0x00; + report.fraction_lost = 0; + report.cumulative_lost = 0; + report.extended_high_seq_num = 0; + report.jitter = 0; + report.lsr = 0; + report.dlsr = 0; + + rtcp_rr.SetReportBlock(report); + + rtcp_rr.Encode(); + + SendRtcpRR(rtcp_rr); + } + + if (on_receive_data_) { + on_receive_data_((const char*)rtp_packet.Payload(), + rtp_packet.PayloadSize()); + } +} + +void RtpDataReceiver::SetSendDataFunc( + std::function data_send_func) { + data_send_func_ = data_send_func; +} + +int RtpDataReceiver::SendRtcpRR(RtcpReceiverReport& rtcp_rr) { + if (!data_send_func_) { + LOG_ERROR("data_send_func_ is nullptr"); + return -1; + } + + if (data_send_func_((const char*)rtcp_rr.Buffer(), rtcp_rr.Size())) { + LOG_ERROR("Send RR failed"); + return -1; + } + + // LOG_ERROR("Send RR"); + + return 0; +} + +bool RtpDataReceiver::CheckIsTimeSendRR() { + uint32_t now_ts = static_cast( + std::chrono::duration_cast( + std::chrono::high_resolution_clock::now().time_since_epoch()) + .count()); + + if (now_ts - last_send_rtcp_rr_packet_ts_ >= RTCP_RR_INTERVAL) { + last_send_rtcp_rr_packet_ts_ = now_ts; + return true; + } else { + return false; + } +} \ No newline at end of file diff --git a/src/rtp/rtp_data_receiver.h b/src/rtp/rtp_data_receiver.h index 22197e4..d3c925d 100644 --- a/src/rtp/rtp_data_receiver.h +++ b/src/rtp/rtp_data_receiver.h @@ -1,13 +1,39 @@ #ifndef _RTP_DATA_RECEIVER_H_ #define _RTP_DATA_RECEIVER_H_ +#include + +#include "rtcp_receiver_report.h" +#include "rtp_codec.h" +#include "rtp_statistics.h" + class RtpDataReceiver { public: RtpDataReceiver(); ~RtpDataReceiver(); + public: + void InsertRtpPacket(RtpPacket& rtp_packet); + + void SetSendDataFunc(std::function data_send_func); + + void SetOnReceiveData( + std::function on_receive_data) { + on_receive_data_ = on_receive_data; + } + private: - /* data */ + bool CheckIsTimeSendRR(); + int SendRtcpRR(RtcpReceiverReport& rtcp_rr); + + private: + std::function on_receive_data_ = nullptr; + uint32_t last_complete_frame_ts_ = 0; + + private: + std::unique_ptr rtp_statistics_ = nullptr; + uint32_t last_send_rtcp_rr_packet_ts_ = 0; + std::function data_send_func_ = nullptr; }; #endif \ No newline at end of file diff --git a/src/rtp/rtp_data_sender.cpp b/src/rtp/rtp_data_sender.cpp index 2ff6923..767f1bd 100644 --- a/src/rtp/rtp_data_sender.cpp +++ b/src/rtp/rtp_data_sender.cpp @@ -102,7 +102,7 @@ int RtpDataSender::SendRtcpSR(RtcpSenderReport& rtcp_sr) { return -1; } - LOG_ERROR("Send SR"); + // LOG_ERROR("Send SR"); return 0; } diff --git a/src/rtp/rtp_packet.cpp b/src/rtp/rtp_packet.cpp index 2506a4e..a85afa2 100644 --- a/src/rtp/rtp_packet.cpp +++ b/src/rtp/rtp_packet.cpp @@ -4,14 +4,25 @@ #include "log.h" -inline void RtpPacket::TryToDecodeH264RtpPacket(uint8_t *buffer) { - if (PAYLOAD_TYPE::H264 == NAL_UNIT_TYPE(buffer[1] & 0x7f)) { - nal_unit_type_ = NAL_UNIT_TYPE(buffer[12] & 0x1F); +void RtpPacket::TryToDecodeRtpPacket() { + if (PAYLOAD_TYPE::H264 == NAL_UNIT_TYPE(buffer_[1] & 0x7F)) { + nal_unit_type_ = NAL_UNIT_TYPE(buffer_[12] & 0x1F); if (NAL_UNIT_TYPE::NALU == nal_unit_type_) { DecodeH264Nalu(); } else if (NAL_UNIT_TYPE::FU_A == nal_unit_type_) { DecodeH264Fua(); } + } else if (PAYLOAD_TYPE::DATA == NAL_UNIT_TYPE(buffer_[1] & 0x7F)) { + DecodeData(); + } else { + LOG_ERROR("Unknown pt: {}", NAL_UNIT_TYPE(buffer_[1] & 0x7F)); + } +} + +void RtpPacket::ParseRtpData() { + if (!parsed_) { + TryToDecodeRtpPacket(); + parsed_ = true; } } @@ -25,7 +36,7 @@ RtpPacket::RtpPacket(const uint8_t *buffer, size_t size) { memcpy(buffer_, buffer, size); size_ = size; - TryToDecodeH264RtpPacket(buffer_); + // TryToDecodeH264RtpPacket(buffer_); } } @@ -35,7 +46,7 @@ RtpPacket::RtpPacket(const RtpPacket &rtp_packet) { memcpy(buffer_, rtp_packet.buffer_, rtp_packet.size_); size_ = rtp_packet.size_; - TryToDecodeH264RtpPacket(buffer_); + // TryToDecodeH264RtpPacket(buffer_); } } @@ -45,7 +56,7 @@ RtpPacket::RtpPacket(RtpPacket &&rtp_packet) rtp_packet.buffer_ = nullptr; rtp_packet.size_ = 0; - TryToDecodeH264RtpPacket(buffer_); + // TryToDecodeH264RtpPacket(buffer_); } RtpPacket &RtpPacket::operator=(const RtpPacket &rtp_packet) { @@ -58,7 +69,7 @@ RtpPacket &RtpPacket::operator=(const RtpPacket &rtp_packet) { memcpy(buffer_, rtp_packet.buffer_, rtp_packet.size_); size_ = rtp_packet.size_; - TryToDecodeH264RtpPacket(buffer_); + // TryToDecodeH264RtpPacket(buffer_); } return *this; } @@ -70,7 +81,7 @@ RtpPacket &RtpPacket::operator=(RtpPacket &&rtp_packet) { size_ = rtp_packet.size_; rtp_packet.size_ = 0; - TryToDecodeH264RtpPacket(buffer_); + // TryToDecodeH264RtpPacket(buffer_); } return *this; } @@ -232,7 +243,7 @@ const uint8_t *RtpPacket::EncodeH264Fua(uint8_t *payload, size_t payload_size) { return buffer_; } -size_t RtpPacket::Decode(uint8_t *payload) { +size_t RtpPacket::DecodeData(uint8_t *payload) { version_ = (buffer_[0] >> 6) & 0x03; has_padding_ = (buffer_[0] >> 5) & 0x01; has_extension_ = (buffer_[0] >> 4) & 0x01; @@ -259,7 +270,8 @@ size_t RtpPacket::Decode(uint8_t *payload) { (buffer_[14 + extension_offset] << 8) | buffer_[15 + extension_offset]; // extension_data_ = new uint8_t[extension_len_]; - // memcpy(extension_data_, buffer_ + 16 + extension_offset, extension_len_); + // memcpy(extension_data_, buffer_ + 16 + extension_offset, + // extension_len_); extension_data_ = buffer_ + 16 + extension_offset; } @@ -268,7 +280,9 @@ size_t RtpPacket::Decode(uint8_t *payload) { payload_size_ = size_ - (12 + payload_offset); payload_ = buffer_ + 12 + payload_offset; - memcpy(payload, payload_, payload_size_); + if (payload) { + memcpy(payload, payload_, payload_size_); + } return payload_size_; } @@ -300,7 +314,8 @@ size_t RtpPacket::DecodeH264Nalu(uint8_t *payload) { (buffer_[14 + extension_offset] << 8) | buffer_[15 + extension_offset]; // extension_data_ = new uint8_t[extension_len_]; - // memcpy(extension_data_, buffer_ + 16 + extension_offset, extension_len_); + // memcpy(extension_data_, buffer_ + 16 + extension_offset, + // extension_len_); extension_data_ = buffer_ + 16 + extension_offset; } @@ -346,7 +361,8 @@ size_t RtpPacket::DecodeH264Fua(uint8_t *payload) { (buffer_[14 + extension_offset] << 8) | buffer_[15 + extension_offset]; // extension_data_ = new uint8_t[extension_len_]; - // memcpy(extension_data_, buffer_ + 16 + extension_offset, extension_len_); + // memcpy(extension_data_, buffer_ + 16 + extension_offset, + // extension_len_); extension_data_ = buffer_ + 16 + extension_offset; } diff --git a/src/rtp/rtp_packet.h b/src/rtp/rtp_packet.h index 60041ff..32770ac 100644 --- a/src/rtp/rtp_packet.h +++ b/src/rtp/rtp_packet.h @@ -135,39 +135,88 @@ class RtpPacket { const uint8_t *Encode(uint8_t *payload, size_t payload_size); const uint8_t *EncodeH264Nalu(uint8_t *payload, size_t payload_size); const uint8_t *EncodeH264Fua(uint8_t *payload, size_t payload_size); - size_t Decode(uint8_t *payload); + size_t DecodeData(uint8_t *payload = nullptr); size_t DecodeH264Nalu(uint8_t *payload = nullptr); size_t DecodeH264Fua(uint8_t *payload = nullptr); public: // Get Header - uint32_t Verion() const { return version_; } - bool HasPadding() const { return has_padding_; } - bool HasExtension() const { return has_extension_; } - bool Marker() const { return marker_; } - PAYLOAD_TYPE PayloadType() const { return PAYLOAD_TYPE(payload_type_); } - uint16_t SequenceNumber() const { return sequence_number_; } - uint32_t Timestamp() const { return timestamp_; } - uint32_t Ssrc() const { return ssrc_; } - std::vector Csrcs() const { return csrcs_; }; - uint16_t ExtensionProfile() const { return extension_profile_; } - const uint8_t *ExtensionData() const { return extension_data_; } + uint32_t Verion() { + ParseRtpData(); + return version_; + } + bool HasPadding() { + ParseRtpData(); + return has_padding_; + } + bool HasExtension() { + ParseRtpData(); + return has_extension_; + } + bool Marker() { + ParseRtpData(); + return marker_; + } + PAYLOAD_TYPE PayloadType() { + ParseRtpData(); + return PAYLOAD_TYPE(payload_type_); + } + uint16_t SequenceNumber() { + ParseRtpData(); + return sequence_number_; + } + uint32_t Timestamp() { + ParseRtpData(); + return timestamp_; + } + uint32_t Ssrc() { + ParseRtpData(); + return ssrc_; + } + std::vector Csrcs() { + ParseRtpData(); + return csrcs_; + }; + uint16_t ExtensionProfile() { + ParseRtpData(); + return extension_profile_; + } + const uint8_t *ExtensionData() { + ParseRtpData(); + return extension_data_; + } // Payload - const uint8_t *Payload() const { return payload_; }; - size_t PayloadSize() const { return payload_size_; } + const uint8_t *Payload() { + ParseRtpData(); + return payload_; + }; + size_t PayloadSize() { + ParseRtpData(); + return payload_size_; + } // Entire RTP buffer const uint8_t *Buffer() const { return buffer_; } size_t Size() const { return size_; } // NAL - NAL_UNIT_TYPE NalUnitType() const { return nal_unit_type_; } - bool FuAStart() const { return fu_header_.start; } - bool FuAEnd() const { return fu_header_.end; } + NAL_UNIT_TYPE NalUnitType() { + ParseRtpData(); + return nal_unit_type_; + } + bool FuAStart() { + ParseRtpData(); + return fu_header_.start; + } + bool FuAEnd() { + ParseRtpData(); + return fu_header_.end; + } private: - inline void TryToDecodeH264RtpPacket(uint8_t *buffer); + void TryToDecodeRtpPacket(); + void ParseRtpData(); private: // Header @@ -198,6 +247,8 @@ class RtpPacket { // NAL NAL_UNIT_TYPE nal_unit_type_ = NAL_UNIT_TYPE::UNKNOWN; + + bool parsed_ = false; }; #endif \ No newline at end of file diff --git a/src/rtp/rtp_statistics.cpp b/src/rtp/rtp_statistics.cpp index 85b2d0e..96afce3 100644 --- a/src/rtp/rtp_statistics.cpp +++ b/src/rtp/rtp_statistics.cpp @@ -16,11 +16,11 @@ void RtpStatistics::UpdateReceiveBytes(uint32_t received_bytes) { bool RtpStatistics::Process() { if (!sent_bytes_) { - LOG_INFO("rtp statistics: Send [{} bps]", sent_bytes_); + // LOG_INFO("rtp statistics: Send [{} bps]", sent_bytes_); } if (!received_bytes_) { - LOG_INFO("rtp statistics: Receive [{} bps]", received_bytes_); + // LOG_INFO("rtp statistics: Receive [{} bps]", received_bytes_); } sent_bytes_ = 0; diff --git a/src/rtp/rtp_video_receiver.cpp b/src/rtp/rtp_video_receiver.cpp index 36ef803..695f1f8 100644 --- a/src/rtp/rtp_video_receiver.cpp +++ b/src/rtp/rtp_video_receiver.cpp @@ -166,7 +166,7 @@ int RtpVideoReceiver::SendRtcpRR(RtcpReceiverReport& rtcp_rr) { return -1; } - LOG_ERROR("Send RR"); + // LOG_ERROR("Send RR"); return 0; } diff --git a/src/rtp/rtp_video_sender.cpp b/src/rtp/rtp_video_sender.cpp index 34fc62b..2bac991 100644 --- a/src/rtp/rtp_video_sender.cpp +++ b/src/rtp/rtp_video_sender.cpp @@ -102,7 +102,7 @@ int RtpVideoSender::SendRtcpSR(RtcpSenderReport& rtcp_sr) { return -1; } - LOG_ERROR("Send SR"); + // LOG_ERROR("Send SR"); return 0; } diff --git a/src/transmission/ice_transmission.cpp b/src/transmission/ice_transmission.cpp index 38c79c8..97efb53 100644 --- a/src/transmission/ice_transmission.cpp +++ b/src/transmission/ice_transmission.cpp @@ -20,15 +20,12 @@ IceTransmission::IceTransmission( bool offer_peer, std::string &transmission_id, std::string &user_id, std::string &remote_user_id, std::shared_ptr ice_ws_transmission, - std::function - on_receive_ice_msg, std::function on_ice_status_change) : offer_peer_(offer_peer), transmission_id_(transmission_id), user_id_(user_id), remote_user_id_(remote_user_id), ice_ws_transport_(ice_ws_transmission), - on_receive_ice_msg_cb_(on_receive_ice_msg), on_ice_status_change_(on_ice_status_change) {} IceTransmission::~IceTransmission() { @@ -67,9 +64,9 @@ int IceTransmission::InitIceTransmission(std::string &ip, int port) { rtp_video_receiver_->SetOnReceiveCompleteFrame( [this](VideoFrame &video_frame) -> void { // LOG_ERROR("OnReceiveCompleteFrame {}", video_frame.Size()); - on_receive_ice_msg_cb_((const char *)video_frame.Buffer(), - video_frame.Size(), remote_user_id_.data(), - remote_user_id_.size()); + on_receive_video_((const char *)video_frame.Buffer(), + video_frame.Size(), remote_user_id_.data(), + remote_user_id_.size()); }); rtp_video_receiver_->Start(); @@ -100,6 +97,22 @@ int IceTransmission::InitIceTransmission(std::string &ip, int port) { rtp_data_sender_->Start(); + rtp_data_receiver_ = std::make_unique(); + rtp_data_receiver_->SetSendDataFunc( + [this](const char *data, size_t size) -> int { + if (!ice_agent_) { + LOG_ERROR("ice_agent_ is nullptr"); + return -1; + } + + return ice_agent_->Send(data, size); + }); + rtp_data_receiver_->SetOnReceiveData( + [this](const char *data, size_t size) -> void { + on_receive_data_(data, size, remote_user_id_.data(), + remote_user_id_.size()); + }); + ice_agent_ = std::make_unique(ip, port); ice_agent_->CreateIceAgent( @@ -148,8 +161,11 @@ int IceTransmission::InitIceTransmission(std::string &ip, int port) { RtpPacket packet((uint8_t *)buffer, size); ice_transmission_obj->rtp_video_receiver_->InsertRtpPacket( packet); + } else if (ice_transmission_obj->CheckIsDataPacket(buffer, size)) { + RtpPacket packet((uint8_t *)buffer, size); + ice_transmission_obj->rtp_data_receiver_->InsertRtpPacket(packet); } else if (ice_transmission_obj->CheckIsRtcpPacket(buffer, size)) { - LOG_ERROR("Rtcp packet [{}]", (uint8_t)(buffer[1])); + // LOG_ERROR("Rtcp packet [{}]", (uint8_t)(buffer[1])); } } } @@ -317,4 +333,17 @@ uint8_t IceTransmission::CheckIsAudioPacket(const char *buffer, size_t size) { } else { return 0; } -} \ No newline at end of file +} + +uint8_t IceTransmission::CheckIsDataPacket(const char *buffer, size_t size) { + if (size < 4) { + return 0; + } + + uint8_t pt = buffer[1] & 0x7F; + if (RtpPacket::PAYLOAD_TYPE::DATA == pt) { + return pt; + } else { + return 0; + } +} diff --git a/src/transmission/ice_transmission.h b/src/transmission/ice_transmission.h index 2cb1b22..80e4feb 100644 --- a/src/transmission/ice_transmission.h +++ b/src/transmission/ice_transmission.h @@ -19,13 +19,10 @@ class IceTransmission { typedef enum { VIDEO = 96, AUDIO = 97, DATA = 127 } DATA_TYPE; public: - IceTransmission( - bool offer_peer, std::string &transmission_id, std::string &user_id, - std::string &remote_user_id, - std::shared_ptr ice_ws_transmission, - std::function - on_receive_ice_msg, - std::function on_ice_status_change); + IceTransmission(bool offer_peer, std::string &transmission_id, + std::string &user_id, std::string &remote_user_id, + std::shared_ptr ice_ws_transmission, + std::function on_ice_status_change); ~IceTransmission(); public: @@ -33,6 +30,24 @@ class IceTransmission { int DestroyIceTransmission(); + void SetOnReceiveVideoFunc( + std::function + on_receive_video) { + on_receive_video_ = on_receive_video; + } + + void SetOnReceiveAudioFunc( + std::function + on_receive_audio) { + on_receive_audio_ = on_receive_audio; + } + + void SetOnReceiveDataFunc( + std::function + on_receive_data) { + on_receive_data_ = on_receive_data; + } + int JoinTransmission(); int SetTransmissionId(const std::string &transmission_id); @@ -60,13 +75,18 @@ class IceTransmission { uint8_t CheckIsRtcpPacket(const char *buffer, size_t size); uint8_t CheckIsVideoPacket(const char *buffer, size_t size); uint8_t CheckIsAudioPacket(const char *buffer, size_t size); + uint8_t CheckIsDataPacket(const char *buffer, size_t size); private: std::unique_ptr ice_agent_ = nullptr; std::shared_ptr ice_ws_transport_ = nullptr; CongestionControl *congestion_control_ = nullptr; std::function - on_receive_ice_msg_cb_ = nullptr; + on_receive_video_ = nullptr; + std::function + on_receive_audio_ = nullptr; + std::function + on_receive_data_ = nullptr; std::function on_ice_status_change_ = nullptr; std::string local_sdp_; std::string remote_sdp_;