Fix GPU sync stream Segmentation fault

mindspore/ccsrc/runtime/device/gpu/gpu_device_manager.cc

@@ -39,6 +39,7 @@ void GPUDeviceManager::InitDevice() {
     cusolverDnSetStream(cusolver_dn_handle_, reinterpret_cast<cudaStream_t>(default_stream())),
     "Failed to set stream for cusolver dn handle");
   CHECK_OP_RET_WITH_EXCEPT(GPUMemoryAllocator::GetInstance().Init(), "Failed to Init gpu memory allocator")
+  dev_alive_ = true;
 }
 
 void GPUDeviceManager::ReleaseDevice() {
@@ -57,6 +58,7 @@ void GPUDeviceManager::ReleaseDevice() {
     CHECK_CUSOLVER_RET_WITH_ERROR(cusolverDnDestroy(cusolver_dn_handle_), "Failed to destroy cusolver dn handle.");
   }
   CHECK_OP_RET_WITH_ERROR(GPUMemoryAllocator::GetInstance().Finalize(), "Failed to destroy gpu memory allocator");
+  dev_alive_ = false;
 }
 
 bool GPUDeviceManager::CreateStream(DeviceStream *stream) {
@@ -89,7 +91,9 @@ const cudnnHandle_t &GPUDeviceManager::GetCudnnHandle() const { return cudnn_han
 
 const cublasHandle_t &GPUDeviceManager::GetCublasHandle() const { return cublas_handle_; }
 const cusolverDnHandle_t &GPUDeviceManager::GetCusolverDnHandle() const { return cusolver_dn_handle_; }
-bool GPUDeviceManager::SyncStream(const DeviceStream &stream) const { return CudaDriver::SyncStream(stream); }
+bool GPUDeviceManager::SyncStream(const DeviceStream &stream) const {
+  return dev_alive_ ? CudaDriver::SyncStream(stream) : false;
+}
 
 bool GPUDeviceManager::CopyDeviceMemToHost(const HostMemPtr &dst, const DeviceMemPtr &src, size_t size) const {
   return CudaDriver::CopyDeviceMemToHost(dst, src, size);

mindspore/ccsrc/runtime/device/gpu/gpu_device_manager.h

@@ -60,7 +60,7 @@ class GPUDeviceManager {
   }
 
  private:
-  GPUDeviceManager() : dev_id_init_(false), cur_dev_id_(0) {}
+  GPUDeviceManager() : dev_id_init_(false), cur_dev_id_(0), dev_alive_(false) {}
   ~GPUDeviceManager() = default;
   GPUDeviceManager(const GPUDeviceManager &) = delete;
   GPUDeviceManager &operator=(const GPUDeviceManager &) = delete;
@@ -81,6 +81,7 @@ class GPUDeviceManager {
   cusolverDnHandle_t cusolver_dn_handle_{nullptr};
   bool dev_id_init_;
   uint32_t cur_dev_id_;
+  bool dev_alive_;
 };
 }  // namespace gpu
 }  // namespace device