[feat] use VideoFrameWrapper to store frame info

src/channel/video_channel_send.cpp

@@ -57,10 +57,12 @@ void VideoChannelSend::Destroy() {
   }
 }
 
-int VideoChannelSend::SendVideo(char* data, size_t size) {
+int VideoChannelSend::SendVideo(
+    std::shared_ptr<VideoFrameWrapper> encoded_frame) {
   if (rtp_video_sender_ && rtp_packetizer_) {
     std::vector<std::shared_ptr<RtpPacket>> rtp_packets =
-        rtp_packetizer_->Build((uint8_t*)data, (uint32_t)size, true);
+        rtp_packetizer_->Build((uint8_t*)encoded_frame->Buffer(),
+                               (uint32_t)encoded_frame->Size(), true);
     rtp_video_sender_->Enqueue(rtp_packets);
   }
 

src/channel/video_channel_send.h

@@ -16,6 +16,7 @@
 #include "rtp_packetizer.h"
 #include "rtp_video_sender.h"
 #include "transport_feedback_adapter.h"
+#include "video_frame_wrapper.h"
 
 class VideoChannelSend {
  public:
@@ -31,7 +32,7 @@ class VideoChannelSend {
   void Initialize(rtp::PAYLOAD_TYPE payload_type);
   void Destroy();
 
-  int SendVideo(char* data, size_t size);
+  int SendVideo(std::shared_ptr<VideoFrameWrapper> encoded_frame);
 
   void OnCongestionControlFeedback(
       Timestamp recv_ts,

src/frame/video_frame_wrapper.h

@@ -0,0 +1,55 @@
+/*
+ * @Author: DI JUNKUN
+ * @Date: 2025-02-21
+ * Copyright (c) 2025 by DI JUNKUN, All Rights Reserved.
+ */
+
+#ifndef _VIDEO_FRAME_WRAPPER_H_
+#define _VIDEO_FRAME_WRAPPER_H_
+
+#include "video_frame.h"
+
+enum VideoFrameType {
+  kEmptyFrame = 0,
+  kVideoFrameKey = 3,
+  kVideoFrameDelta = 4,
+};
+
+class VideoFrameWrapper : public VideoFrame {
+ public:
+  VideoFrameWrapper(const uint8_t *buffer, size_t size, uint32_t width,
+                    uint32_t height)
+      : VideoFrame(buffer, size, width, height) {}
+  VideoFrameWrapper() = delete;
+  ~VideoFrameWrapper() = default;
+
+  int64_t CaptureTimestamp() { return capture_timestamp_; }
+
+  void SetCaptureTimestamp(int64_t capture_timestamp) {
+    capture_timestamp_ = capture_timestamp;
+  }
+
+  VideoFrameType FrameType() { return frame_type_; }
+
+  void SetFrameType(VideoFrameType frame_type) { frame_type_ = frame_type; }
+
+  uint32_t EncodedWidth() { return encoded_width_; }
+
+  void SetEncodedWidth(uint32_t encoded_width) {
+    encoded_width_ = encoded_width;
+  }
+
+  uint32_t EncodedHeight() { return encoded_height_; }
+
+  void SetEncodedHeight(uint32_t encoded_height) {
+    encoded_height_ = encoded_height;
+  }
+
+ private:
+  int64_t capture_timestamp_ = 0;
+  VideoFrameType frame_type_ = VideoFrameType::kVideoFrameDelta;
+  uint32_t encoded_width_ = 0;
+  uint32_t encoded_height_ = 0;
+};
+
+#endif

src/interface/x.h

@@ -44,6 +44,7 @@ typedef struct {
   size_t size;
   uint32_t width;
   uint32_t height;
+  uint64_t timestamp;
 } XVideoFrame;
 
 typedef struct {
@@ -133,6 +134,8 @@ DLLAPI int SendAudioFrame(PeerPtr* peer_ptr, const char* data, size_t size);
 
 DLLAPI int SendDataFrame(PeerPtr* peer_ptr, const char* data, size_t size);
 
+DLLAPI int64_t GetNowTime(PeerPtr* peer_ptr);
+
 #ifdef __cplusplus
 }
 #endif

src/media/video/encode/aom/aom_av1_encoder.cpp

@@ -266,10 +266,10 @@ int AomAv1Encoder::Init() {
   return 0;
 }
 
-int AomAv1Encoder::Encode(const XVideoFrame *video_frame,
+int AomAv1Encoder::Encode(
-                          std::function<int(char *encoded_packets, size_t size,
+    const XVideoFrame *video_frame,
-                                            VideoFrameType frame_type)>
+    std::function<int(std::shared_ptr<VideoFrameWrapper> encoded_frame)>
-                              on_encoded_image) {
+        on_encoded_image) {
 #ifdef SAVE_RECEIVED_NV12_STREAM
   fwrite(video_frame->data, 1, video_frame->size, file_nv12_);
 #endif
@@ -342,8 +342,15 @@ int AomAv1Encoder::Encode(const XVideoFrame *video_frame,
       // LOG_INFO("Encoded frame qp = {}", qp);
 
       if (on_encoded_image) {
-        on_encoded_image((char *)encoded_frame_, encoded_frame_size_,
+        std::shared_ptr<VideoFrameWrapper> encoded_frame =
-                         frame_type);
+            std::make_shared<VideoFrameWrapper>(
+                encoded_frame_, encoded_frame_size_, video_frame->width,
+                video_frame->height);
+        encoded_frame->SetFrameType(frame_type);
+        encoded_frame->SetCaptureTimestamp(video_frame->timestamp);
+        encoded_frame->SetEncodedWidth(video_frame->width);
+        encoded_frame->SetEncodedHeight(video_frame->height);
+        on_encoded_image(encoded_frame);
 #ifdef SAVE_ENCODED_AV1_STREAM
         fwrite(encoded_frame_, 1, encoded_frame_size_, file_av1_);
 #endif

src/media/video/encode/aom/aom_av1_encoder.h

@@ -37,10 +37,10 @@ class AomAv1Encoder : public VideoEncoder {
  public:
   int Init();
 
-  int Encode(const XVideoFrame* video_frame,
+  int Encode(
-             std::function<int(char* encoded_packets, size_t size,
+      const XVideoFrame* video_frame,
-                               VideoFrameType frame_type)>
+      std::function<int(std::shared_ptr<VideoFrameWrapper> encoded_frame)>
-                 on_encoded_image);
+          on_encoded_image);
 
   int ForceIdr();
 

src/media/video/encode/nvcodec/nvidia_video_encoder.cpp

@@ -130,8 +130,7 @@ int NvidiaVideoEncoder::Init() {
 
 int NvidiaVideoEncoder::Encode(
     const XVideoFrame *video_frame,
-    std::function<int(char *encoded_packets, size_t size,
+    std::function<int(std::shared_ptr<VideoFrameWrapper> encoded_frame)>
-                      VideoFrameType frame_type)>
         on_encoded_image) {
   if (!encoder_) {
     LOG_ERROR("Invalid encoder");
@@ -182,7 +181,15 @@ int NvidiaVideoEncoder::Encode(
 
   for (const auto &packet : encoded_packets_) {
     if (on_encoded_image) {
-      on_encoded_image((char *)packet.data(), packet.size(), frame_type);
+      std::shared_ptr<VideoFrameWrapper> encoded_frame =
+          std::make_shared<VideoFrameWrapper>(packet.data(), packet.size(),
+                                              encoder_->GetEncodeWidth(),
+                                              encoder_->GetEncodeHeight());
+      encoded_frame->SetFrameType(frame_type);
+      encoded_frame->SetCaptureTimestamp(video_frame->timestamp);
+      encoded_frame->SetEncodedWidth(encoder_->GetEncodeWidth());
+      encoded_frame->SetEncodedHeight(encoder_->GetEncodeHeight());
+      on_encoded_image(encoded_frame);
 #ifdef SAVE_ENCODED_H264_STREAM
       fwrite((unsigned char *)packet.data(), 1, packet.size(), file_h264_);
 #endif

src/media/video/encode/nvcodec/nvidia_video_encoder.h

@@ -13,10 +13,10 @@ class NvidiaVideoEncoder : public VideoEncoder {
 
   int Init();
 
-  int Encode(const XVideoFrame* video_frame,
+  int Encode(
-             std::function<int(char* encoded_packets, size_t size,
+      const XVideoFrame* video_frame,
-                               VideoFrameType frame_type)>
+      std::function<int(std::shared_ptr<VideoFrameWrapper> encoded_frame)>
-                 on_encoded_image);
+          on_encoded_image);
 
   int ForceIdr();
 

src/media/video/encode/openh264/openh264_encoder.cpp

@@ -181,8 +181,7 @@ int OpenH264Encoder::Init() {
 
 int OpenH264Encoder::Encode(
     const XVideoFrame *video_frame,
-    std::function<int(char *encoded_packets, size_t size,
+    std::function<int(std::shared_ptr<VideoFrameWrapper> encoded_frame)>
-                      VideoFrameType frame_type)>
         on_encoded_image) {
   if (!openh264_encoder_) {
     LOG_ERROR("Invalid openh264 encoder");
@@ -282,7 +281,15 @@ int OpenH264Encoder::Encode(
   encoded_frame_size_ = encoded_frame_size;
 
   if (on_encoded_image) {
-    on_encoded_image((char *)encoded_frame_, encoded_frame_size_, frame_type);
+    std::shared_ptr<VideoFrameWrapper> encoded_frame =
+        std::make_shared<VideoFrameWrapper>(encoded_frame_, encoded_frame_size_,
+                                            raw_frame_.iPicWidth,
+                                            raw_frame_.iPicHeight);
+    encoded_frame->SetFrameType(frame_type);
+    encoded_frame->SetCaptureTimestamp(video_frame->timestamp);
+    encoded_frame->SetEncodedWidth(raw_frame_.iPicWidth);
+    encoded_frame->SetEncodedHeight(raw_frame_.iPicHeight);
+    on_encoded_image(encoded_frame);
 #ifdef SAVE_ENCODED_H264_STREAM
     fwrite(encoded_frame_, 1, encoded_frame_size_, file_h264_);
 #endif

src/media/video/encode/openh264/openh264_encoder.h

@@ -24,10 +24,10 @@ class OpenH264Encoder : public VideoEncoder {
 
   int Init();
 
-  int Encode(const XVideoFrame* video_frame,
+  int Encode(
-             std::function<int(char* encoded_packets, size_t size,
+      const XVideoFrame* video_frame,
-                               VideoFrameType frame_type)>
+      std::function<int(std::shared_ptr<VideoFrameWrapper> encoded_frame)>
-                 on_encoded_image);
+          on_encoded_image);
 
   int ForceIdr();
 

src/media/video/encode/video_encoder.h

@@ -5,25 +5,20 @@
 #include <cstdint>
 #include <cstdio>
 #include <functional>
+#include <memory>
 #include <string>
 
+#include "video_frame_wrapper.h"
 #include "x.h"
 
 class VideoEncoder {
- public:
-  enum VideoFrameType {
-    kEmptyFrame = 0,
-    kVideoFrameKey = 3,
-    kVideoFrameDelta = 4,
-  };
-
  public:
   virtual int Init() = 0;
 
-  virtual int Encode(const XVideoFrame* video_frame,
+  virtual int Encode(
-                     std::function<int(char* encoded_packets, size_t size,
+      const XVideoFrame* video_frame,
-                                       VideoFrameType frame_type)>
+      std::function<int(std::shared_ptr<VideoFrameWrapper> encoded_frame)>
-                         on_encoded_image) = 0;
+          on_encoded_image) = 0;
 
   virtual int ForceIdr() = 0;
 

src/pc/peer_connection.cpp

@@ -390,6 +390,13 @@ int PeerConnection::SendDataFrame(const char *data, size_t size) {
   return 0;
 }
 
+int64_t PeerConnection::CurrentTime() {
+  if (clock_) {
+    return clock_->CurrentTimeMs();
+  }
+  return 0;
+}
+
 void PeerConnection::ProcessSignal(const std::string &signal) {
   auto j = json::parse(signal);
   std::string type = j["type"];

src/pc/peer_connection.h

@@ -108,6 +108,8 @@ class PeerConnection {
   int SendAudioFrame(const char *data, size_t size);
   int SendDataFrame(const char *data, size_t size);
 
+  int64_t CurrentTime();
+
  private:
   int Login();
 

src/rtc/x_inner.cpp

@@ -52,8 +52,8 @@ void DestroyPeer(PeerPtr **peer_ptr) {
 }
 
 int Init(PeerPtr *peer_ptr, const char *user_id) {
-  if (!peer_ptr) {
+  if (!peer_ptr || !peer_ptr->peer_connection) {
-    LOG_ERROR("peer_ptr not created");
+    LOG_ERROR("Peer connection not created");
     return -1;
   }
 
@@ -63,8 +63,8 @@ int Init(PeerPtr *peer_ptr, const char *user_id) {
 
 int CreateConnection(PeerPtr *peer_ptr, const char *transmission_id,
                      const char *password) {
-  if (!peer_ptr) {
+  if (!peer_ptr || !peer_ptr->peer_connection) {
-    LOG_ERROR("peer_ptr not created");
+    LOG_ERROR("Peer connection not created");
     return -1;
   }
 
@@ -77,8 +77,8 @@ int CreateConnection(PeerPtr *peer_ptr, const char *transmission_id,
 int JoinConnection(PeerPtr *peer_ptr, const char *transmission_id,
                    const char *password) {
   int ret = -1;
-  if (!peer_ptr) {
+  if (!peer_ptr || !peer_ptr->peer_connection) {
-    LOG_ERROR("peer_ptr not created");
+    LOG_ERROR("Peer connection not created");
     return -1;
   }
 
@@ -88,8 +88,8 @@ int JoinConnection(PeerPtr *peer_ptr, const char *transmission_id,
 }
 
 int LeaveConnection(PeerPtr *peer_ptr, const char *transmission_id) {
-  if (!peer_ptr) {
+  if (!peer_ptr || !peer_ptr->peer_connection) {
-    LOG_ERROR("peer_ptr not created");
+    LOG_ERROR("Peer connection not created");
     return -1;
   }
 
@@ -99,8 +99,8 @@ int LeaveConnection(PeerPtr *peer_ptr, const char *transmission_id) {
 }
 
 DLLAPI int SendVideoFrame(PeerPtr *peer_ptr, const XVideoFrame *video_frame) {
-  if (!peer_ptr) {
+  if (!peer_ptr || !peer_ptr->peer_connection) {
-    LOG_ERROR("peer_ptr not created");
+    LOG_ERROR("Peer connection not created");
     return -1;
   }
 
@@ -118,8 +118,8 @@ DLLAPI int SendVideoFrame(PeerPtr *peer_ptr, const XVideoFrame *video_frame) {
 }
 
 DLLAPI int SendAudioFrame(PeerPtr *peer_ptr, const char *data, size_t size) {
-  if (!peer_ptr) {
+  if (!peer_ptr || !peer_ptr->peer_connection) {
-    LOG_ERROR("peer_ptr not created");
+    LOG_ERROR("Peer connection not created");
     return -1;
   }
 
@@ -134,8 +134,8 @@ DLLAPI int SendAudioFrame(PeerPtr *peer_ptr, const char *data, size_t size) {
 }
 
 int SendDataFrame(PeerPtr *peer_ptr, const char *data, size_t size) {
-  if (!peer_ptr) {
+  if (!peer_ptr || !peer_ptr->peer_connection) {
-    LOG_ERROR("peer_ptr not created");
+    LOG_ERROR("Peer connection not created");
     return -1;
   }
 
@@ -148,3 +148,11 @@ int SendDataFrame(PeerPtr *peer_ptr, const char *data, size_t size) {
 
   return 0;
 }
+
+int64_t GetNowTime(PeerPtr *peer_ptr) {
+  if (!peer_ptr || !peer_ptr->peer_connection) {
+    LOG_ERROR("Peer connection not created");
+    return -1;
+  }
+  return peer_ptr->peer_connection->CurrentTime();
+}

src/transport/ice_transport_controller.cpp

@@ -1,4 +1,6 @@
 #include "ice_transport_controller.h"
+
+#include "video_frame_wrapper.h"
 #if __APPLE__
 #else
 #include "nvcodec_api.h"
@@ -128,10 +130,9 @@ int IceTransportController::SendVideo(const XVideoFrame* video_frame) {
 
   int ret = video_encoder_->Encode(
       video_frame,
-      [this](char* encoded_frame, size_t size,
+      [this](std::shared_ptr<VideoFrameWrapper> encoded_frame) -> int {
-             VideoEncoder::VideoFrameType frame_type) -> int {
         if (video_channel_send_) {
-          video_channel_send_->SendVideo(encoded_frame, size);
+          video_channel_send_->SendVideo(encoded_frame);
         }
 
         return 0;