Browse Source

!10192 fix the bug of Profiler module do not print meaningful error when training process unexpectedly stopped

From: @gzhcv
Reviewed-by: @wangyue01,@wuxuejian
Signed-off-by: @wuxuejian
tags/v1.2.0-rc1
mindspore-ci-bot Gitee 5 years ago
parent
commit
f1579bc5d1
1 changed files with 4 additions and 0 deletions
  1. +4
    -0
      mindspore/ccsrc/profiler/device/gpu/gpu_profiling.cc

+ 4
- 0
mindspore/ccsrc/profiler/device/gpu/gpu_profiling.cc View File

@@ -70,6 +70,10 @@ uint32_t GetStreamID(const CUcontext context, const void *stream) {
uint32_t stream_id = 0;
if (stream != nullptr) {
CHECK_CUPTI_RET_WITH_ERROR(CuptiGetStreamId(context, (CUstream)stream, &stream_id), "CuptiGetStreamId");
if (CuptiGetStreamId(context, (CUstream)stream, &stream_id) != CUPTI_SUCCESS) {
MS_LOG(ERROR) << "Training process unexpectedly stopped, profiling data cannot be write to file"
<< "To obtain the profiling data, do not interrupt the training process.";
}
}
return stream_id;
}


Loading…
Cancel
Save