[fix] fix crash due to codec init failed

src/pc/peer_connection.cpp

@@ -233,56 +233,39 @@ int PeerConnection::CreateVideoCodec(bool hardware_acceleration) {
 
   if (av1_encoding_) {
     video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, true);
-    LOG_INFO("Only support software encoding for AV1");
+    video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, true);
+    LOG_WARN("Only support software codec for AV1");
   } else {
-    video_encoder_ =
+    if (hardware_acceleration_) {
-        VideoEncoderFactory::CreateVideoEncoder(hardware_acceleration_, false);
+      if (0 == InitNvCodecApi()) {
-    if (hardware_acceleration_ && !video_encoder_) {
+        video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(true, false);
-      LOG_WARN(
+        video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(true, false);
-          "Hardware accelerated encoder not available, use default software "
+      } else {
-          "encoder");
+        LOG_WARN(
-      video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
+            "Hardware accelerated codec not available, use default software "
-      if (!video_encoder_) {
+            "codec");
-        LOG_ERROR(
+        video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
-            "Hardware accelerated encoder and software encoder both not "
+        video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
-            "available");
-        return -1;
       }
     }
   }
-  if (0 != video_encoder_->Init()) {
-    video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
-    if (!video_encoder_ || 0 != video_encoder_->Init()) {
-      LOG_ERROR("Encoder init failed");
-      return -1;
-    }
-  }
 
-  if (av1_encoding_) {
+  if (!video_encoder_) {
-    video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, true);
+    video_encoder_ = VideoEncoderFactory::CreateVideoEncoder(false, false);
-    LOG_INFO("Only support software decoding for AV1");
+    LOG_ERROR("Create encoder failed, try to use software H.264 encoder");
-  } else {
-    video_decoder_ =
-        VideoDecoderFactory::CreateVideoDecoder(hardware_acceleration_, false);
-    if (hardware_acceleration_ && !video_decoder_) {
-      LOG_WARN(
-          "Hardware accelerated decoder not available, use default software "
-          "decoder");
-      video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
-      if (!video_decoder_) {
-        LOG_ERROR(
-            "Hardware accelerated decoder and software decoder both not "
-            "available");
-        return -1;
-      }
-    }
   }
-  if (0 != video_decoder_->Init()) {
+  if (!video_encoder_ || 0 != video_encoder_->Init()) {
+    LOG_ERROR("Encoder init failed");
+    return -1;
+  }
+
+  if (!video_decoder_) {
     video_decoder_ = VideoDecoderFactory::CreateVideoDecoder(false, false);
-    if (!video_decoder_ || video_decoder_->Init()) {
+    LOG_ERROR("Create decoder failed, try to use software H.264 decoder");
-      LOG_ERROR("Decoder init failed");
+  }
-      return -1;
+  if (!video_decoder_ || video_decoder_->Init()) {
-    }
+    LOG_ERROR("Decoder init failed");
+    return -1;
   }
 
   video_codec_inited_ = true;
@@ -622,6 +605,10 @@ int PeerConnection::RequestTransmissionMemberList(
 }
 
 int PeerConnection::Destroy() {
+  if (ws_transport_) {
+    ws_transport_->Close(0, "destroy");
+  }
+
   ice_transmission_list_.clear();
   if (nv12_data_) {
     delete nv12_data_;

src/rtc/x_inner.cpp

@@ -43,6 +43,7 @@ void DestroyPeer(PeerPtr *peer_ptr) {
   peer_ptr->peer_connection->Destroy();
   delete peer_ptr;
   peer_ptr = nullptr;
+  LOG_INFO("Destroy peer");
 }
 
 int Init(PeerPtr *peer_ptr, const char *user_id) {

thirdparty/nvcodec/Interface/nvcodec_api.h

@@ -118,21 +118,21 @@ static int InitNvCodecApi() {
   HMODULE nvcuda_dll = LoadLibrary(TEXT("nvcuda.dll"));
   if (nvcuda_dll == NULL) {
     std::cerr << "Unable to load nvcuda.dll!" << std::endl;
-    return 1;
+    return -1;
   }
 
   cuInit_ld = (TcuInit)GetProcAddress(nvcuda_dll, "cuInit");
   if (cuInit_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuDeviceGet_ld = (TcuDeviceGet)GetProcAddress(nvcuda_dll, "cuDeviceGet");
   if (cuDeviceGet_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuDeviceGetCount_ld =
@@ -140,14 +140,14 @@ static int InitNvCodecApi() {
   if (cuDeviceGetCount_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuCtxCreate_ld = (TcuCtxCreate)GetProcAddress(nvcuda_dll, "cuCtxCreate");
   if (cuCtxCreate_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuGetErrorName_ld =
@@ -155,7 +155,7 @@ static int InitNvCodecApi() {
   if (cuGetErrorName_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuCtxPushCurrent_ld =
@@ -163,7 +163,7 @@ static int InitNvCodecApi() {
   if (cuCtxPushCurrent_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuCtxPopCurrent_ld =
@@ -171,13 +171,13 @@ static int InitNvCodecApi() {
   if (cuCtxPopCurrent_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
   cuMemAlloc_ld = (TcuMemAlloc)GetProcAddress(nvcuda_dll, "cuMemAlloc");
   if (cuMemAlloc_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuMemAllocPitch_ld =
@@ -185,14 +185,14 @@ static int InitNvCodecApi() {
   if (cuMemAllocPitch_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuMemFree_ld = (TcuMemFree)GetProcAddress(nvcuda_dll, "cuMemFree");
   if (cuMemFree_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuMemcpy2DAsync_ld =
@@ -200,7 +200,7 @@ static int InitNvCodecApi() {
   if (cuMemcpy2DAsync_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuStreamSynchronize_ld =
@@ -208,14 +208,14 @@ static int InitNvCodecApi() {
   if (cuStreamSynchronize_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuMemcpy2D_ld = (TcuMemcpy2D)GetProcAddress(nvcuda_dll, "cuMemcpy2D");
   if (cuMemcpy2D_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuMemcpy2DUnaligned_ld =
@@ -223,14 +223,14 @@ static int InitNvCodecApi() {
   if (cuMemcpy2DUnaligned_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   //
   HMODULE nvcuvid_dll = LoadLibrary(TEXT("nvcuvid.dll"));
   if (nvcuvid_dll == NULL) {
     std::cerr << "Unable to load nvcuvid.dll!" << std::endl;
-    return 1;
+    return -1;
   }
 
   cuvidCtxLockCreate_ld =
@@ -238,7 +238,7 @@ static int InitNvCodecApi() {
   if (cuvidCtxLockCreate_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuvidGetDecoderCaps_ld =
@@ -246,7 +246,7 @@ static int InitNvCodecApi() {
   if (cuvidGetDecoderCaps_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuvidCreateDecoder_ld =
@@ -254,7 +254,7 @@ static int InitNvCodecApi() {
   if (cuvidCreateDecoder_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuvidDestroyDecoder_ld =
@@ -262,7 +262,7 @@ static int InitNvCodecApi() {
   if (cuvidDestroyDecoder_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuvidDecodePicture_ld =
@@ -270,7 +270,7 @@ static int InitNvCodecApi() {
   if (cuvidDecodePicture_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuvidGetDecodeStatus_ld =
@@ -278,7 +278,7 @@ static int InitNvCodecApi() {
   if (cuvidGetDecodeStatus_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuvidReconfigureDecoder_ld = (TcuvidReconfigureDecoder)GetProcAddress(
@@ -286,7 +286,7 @@ static int InitNvCodecApi() {
   if (cuvidReconfigureDecoder_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuvidMapVideoFrame64_ld =
@@ -294,7 +294,7 @@ static int InitNvCodecApi() {
   if (cuvidMapVideoFrame64_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuvidUnmapVideoFrame64_ld = (TcuvidUnmapVideoFrame64)GetProcAddress(
@@ -302,7 +302,7 @@ static int InitNvCodecApi() {
   if (cuvidUnmapVideoFrame64_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuvidCtxLockDestroy_ld =
@@ -310,7 +310,7 @@ static int InitNvCodecApi() {
   if (cuvidCtxLockDestroy_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuvidCreateVideoParser_ld = (TcuvidCreateVideoParser)GetProcAddress(
@@ -318,7 +318,7 @@ static int InitNvCodecApi() {
   if (cuvidCreateVideoParser_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuvidParseVideoData_ld =
@@ -326,7 +326,7 @@ static int InitNvCodecApi() {
   if (cuvidParseVideoData_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   cuvidDestroyVideoParser_ld = (TcuvidDestroyVideoParser)GetProcAddress(
@@ -334,14 +334,14 @@ static int InitNvCodecApi() {
   if (cuvidDestroyVideoParser_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   //
   HMODULE nvEncodeAPI64_dll = LoadLibrary(TEXT("nvEncodeAPI64.dll"));
   if (nvEncodeAPI64_dll == NULL) {
     std::cerr << "Unable to load nvEncodeAPI64.dll!" << std::endl;
-    return 1;
+    return -1;
   }
 
   NvEncodeAPICreateInstance_ld = (TNvEncodeAPICreateInstance)GetProcAddress(
@@ -349,7 +349,7 @@ static int InitNvCodecApi() {
   if (NvEncodeAPICreateInstance_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   NvEncodeAPIGetMaxSupportedVersion_ld =
@@ -358,7 +358,7 @@ static int InitNvCodecApi() {
   if (NvEncodeAPIGetMaxSupportedVersion_ld == NULL) {
     std::cerr << "Unable to find function!" << std::endl;
     FreeLibrary(nvcuda_dll);
-    return 1;
+    return -1;
   }
 
   return 0;