|
|
|
@@ -49,7 +49,7 @@ const ( |
|
|
|
|
|
|
|
func GrampusTrainJobGPUNew(ctx *context.Context) { |
|
|
|
ctx.Data["IsCreate"] = true |
|
|
|
err := GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
err := grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
if err != nil { |
|
|
|
ctx.ServerError("get new train-job info failed", err) |
|
|
|
return |
|
|
|
@@ -60,7 +60,7 @@ func GrampusTrainJobGPUNew(ctx *context.Context) { |
|
|
|
|
|
|
|
func GrampusTrainJobNPUNew(ctx *context.Context) { |
|
|
|
ctx.Data["IsCreate"] = true |
|
|
|
err := GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
err := grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
if err != nil { |
|
|
|
ctx.ServerError("get new train-job info failed", err) |
|
|
|
return |
|
|
|
@@ -68,7 +68,7 @@ func GrampusTrainJobNPUNew(ctx *context.Context) { |
|
|
|
ctx.HTML(200, tplGrampusTrainJobNPUNew) |
|
|
|
} |
|
|
|
|
|
|
|
func GrampusTrainJobNewDataPrepare(ctx *context.Context, processType string) error { |
|
|
|
func grampusTrainJobNewDataPrepare(ctx *context.Context, processType string) error { |
|
|
|
ctx.Data["PageIsCloudBrain"] = true |
|
|
|
|
|
|
|
t := time.Now() |
|
|
|
@@ -176,14 +176,14 @@ func GrampusTrainJobVersionNew(ctx *context.Context) { |
|
|
|
task := ctx.Cloudbrain |
|
|
|
ctx.Data["IsCreate"] = false |
|
|
|
if task.ComputeResource == models.GPUResource { |
|
|
|
err := GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
err := grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
if err != nil { |
|
|
|
ctx.ServerError("get new train-job version info failed", err) |
|
|
|
return |
|
|
|
} |
|
|
|
ctx.HTML(http.StatusOK, tplGrampusTrainJobGPUNew) |
|
|
|
} else if task.ComputeResource == models.NPUResource { |
|
|
|
err := GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
err := grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
if err != nil { |
|
|
|
ctx.ServerError("get new train-job version info failed", err) |
|
|
|
return |
|
|
|
@@ -279,14 +279,14 @@ func grampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
isOk, err := lock.Lock(models.CloudbrainKeyDuration) |
|
|
|
if !isOk { |
|
|
|
log.Error("lock processed failed:%v", err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tplGrampusTrainJobGPUNew, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
defer lock.UnLock() |
|
|
|
|
|
|
|
if !jobNamePattern.MatchString(displayJobName) { |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -294,7 +294,7 @@ func grampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName) |
|
|
|
if err != nil || !bootFileExist { |
|
|
|
log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -303,13 +303,13 @@ func grampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
count, err := models.GetGrampusCountByUserID(ctx.User.ID, string(models.JobTypeTrain), models.GPUResource) |
|
|
|
if err != nil { |
|
|
|
log.Error("GetGrampusCountByUserID failed:%v", err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr("system error", tpl, &form) |
|
|
|
return |
|
|
|
} else { |
|
|
|
if count >= 1 { |
|
|
|
log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr("you have already a running or waiting task, can not create more", tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -318,7 +318,7 @@ func grampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
//check param |
|
|
|
if err := grampusParamCheckCreateTrainJob(form); err != nil { |
|
|
|
log.Error("paramCheckCreateTrainJob failed:(%v)", err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr(err.Error(), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -328,14 +328,14 @@ func grampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
if err == nil { |
|
|
|
if len(tasks) != 0 { |
|
|
|
log.Error("the job name did already exist", ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr("the job name did already exist", tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
} else { |
|
|
|
if !models.IsErrJobNotExist(err) { |
|
|
|
log.Error("system error, %v", err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr("system error", tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -348,14 +348,14 @@ func grampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
Cluster: models.C2NetCluster, |
|
|
|
}) |
|
|
|
if err != nil || spec == nil { |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr("Resource specification not available", tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { |
|
|
|
log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("points.insufficient_points_balance"), tplGrampusTrainJobGPUNew, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -365,7 +365,7 @@ func grampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid, models.GPU) |
|
|
|
if err != nil { |
|
|
|
log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -378,7 +378,7 @@ func grampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
|
|
|
|
if err := downloadZipCode(ctx, codeLocalPath, branchName); err != nil { |
|
|
|
log.Error("downloadZipCode failed, server timed out: %s (%v)", repo.FullName(), err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -387,7 +387,7 @@ func grampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
//upload code |
|
|
|
if err := uploadCodeToMinio(codeLocalPath+"/", jobName, cloudbrain.CodeMountPath+"/"); err != nil { |
|
|
|
log.Error("Failed to uploadCodeToMinio: %s (%v)", repo.FullName(), err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -395,7 +395,7 @@ func grampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/" |
|
|
|
if err := mkModelPath(modelPath); err != nil { |
|
|
|
log.Error("Failed to mkModelPath: %s (%v)", repo.FullName(), err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -403,7 +403,7 @@ func grampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
//init model readme |
|
|
|
if err := uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/"); err != nil { |
|
|
|
log.Error("Failed to uploadCodeToMinio: %s (%v)", repo.FullName(), err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -426,7 +426,7 @@ func grampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
command, err := generateCommand(repo.Name, grampus.ProcessorTypeGPU, codeMinioPath+cloudbrain.DefaultBranchName+".zip", datasetRemotePath, bootFile, params, setting.CBCodePathPrefix+jobName+cloudbrain.ModelMountPath+"/", allFileName, preTrainModelPath, form.CkptName) |
|
|
|
if err != nil { |
|
|
|
log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr("Create task failed, internal error", tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -468,7 +468,7 @@ func grampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
err = grampus.GenerateTrainJob(ctx, req) |
|
|
|
if err != nil { |
|
|
|
log.Error("GenerateTrainJob failed:%v", err.Error(), ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) |
|
|
|
ctx.RenderWithErr(err.Error(), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -549,14 +549,14 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
isOk, err := lock.Lock(models.CloudbrainKeyDuration) |
|
|
|
if !isOk { |
|
|
|
log.Error("lock processed failed:%v", err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tplGrampusTrainJobNPUNew, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
defer lock.UnLock() |
|
|
|
|
|
|
|
if !jobNamePattern.MatchString(displayJobName) { |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -564,7 +564,7 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName) |
|
|
|
if err != nil || !bootFileExist { |
|
|
|
log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -573,13 +573,13 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
count, err := models.GetGrampusCountByUserID(ctx.User.ID, string(models.JobTypeTrain), models.NPUResource) |
|
|
|
if err != nil { |
|
|
|
log.Error("GetGrampusCountByUserID failed:%v", err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr("system error", tpl, &form) |
|
|
|
return |
|
|
|
} else { |
|
|
|
if count >= 1 { |
|
|
|
log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr("you have already a running or waiting task, can not create more", tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -588,7 +588,7 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
//check param |
|
|
|
if err := grampusParamCheckCreateTrainJob(form); err != nil { |
|
|
|
log.Error("paramCheckCreateTrainJob failed:(%v)", err) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr(err.Error(), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -598,14 +598,14 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
if err == nil { |
|
|
|
if len(tasks) != 0 { |
|
|
|
log.Error("the job name did already exist", ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr("the job name did already exist", tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
} else { |
|
|
|
if !models.IsErrJobNotExist(err) { |
|
|
|
log.Error("system error, %v", err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr("system error", tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -618,13 +618,13 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
Cluster: models.C2NetCluster, |
|
|
|
}) |
|
|
|
if err != nil || spec == nil { |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr("Resource specification not available", tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { |
|
|
|
log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("points.insufficient_points_balance"), tplGrampusTrainJobNPUNew, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -633,7 +633,7 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid, models.NPU) |
|
|
|
if err != nil { |
|
|
|
log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -646,7 +646,7 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
|
|
|
|
if err := downloadZipCode(ctx, codeLocalPath, branchName); err != nil { |
|
|
|
log.Error("downloadZipCode failed, server timed out: %s (%v)", repo.FullName(), err) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -654,14 +654,14 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
//todo: upload code (send to file_server todo this work?) |
|
|
|
if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath); err != nil { |
|
|
|
log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { |
|
|
|
log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -683,7 +683,7 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
command, err := generateCommand(repo.Name, grampus.ProcessorTypeNPU, codeObsPath+cloudbrain.DefaultBranchName+".zip", datasetRemotePath, bootFile, params, setting.CodePathPrefix+jobName+modelarts.OutputPath, allFileName, preTrainModelPath, form.CkptName) |
|
|
|
if err != nil { |
|
|
|
log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"]) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr("Create task failed, internal error", tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -727,7 +727,7 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain |
|
|
|
err = grampus.GenerateTrainJob(ctx, req) |
|
|
|
if err != nil { |
|
|
|
log.Error("GenerateTrainJob failed:%v", err.Error()) |
|
|
|
GrampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) |
|
|
|
ctx.RenderWithErr(err.Error(), tpl, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
|