From 87bf34ca29ed1581766544759882cc52df173121 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 1 Jun 2022 16:12:40 +0800 Subject: [PATCH] select dataset --- routers/repo/grampus.go | 157 ++++--------------- templates/custom/select_dataset.tmpl | 8 +- templates/custom/select_dataset_train.tmpl | 4 +- templates/repo/grampus/trainjob/gpu/new.tmpl | 35 +---- templates/repo/grampus/trainjob/npu/new.tmpl | 46 +----- web_src/js/index.js | 43 ++--- 6 files changed, 74 insertions(+), 219 deletions(-) mode change 100644 => 100755 templates/custom/select_dataset.tmpl mode change 100644 => 100755 templates/custom/select_dataset_train.tmpl diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index d25a75c68..7df2d2819 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -37,7 +37,8 @@ const ( ) func GrampusTrainJobGPUNew(ctx *context.Context) { - err := grampusGpuNewDataPrepare(ctx) + ctx.Data["cloudbraintype"] = models.TypeCloudBrainOne + err := grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) if err != nil { ctx.ServerError("get new train-job info failed", err) return @@ -45,110 +46,9 @@ func GrampusTrainJobGPUNew(ctx *context.Context) { ctx.HTML(http.StatusOK, tplGrampusTrainJobGPUNew) } -func grampusGpuNewDataPrepare(ctx *context.Context) error { - ctx.Data["PageIsCloudBrain"] = true - t := time.Now() - var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] - ctx.Data["display_job_name"] = displayJobName - - //get valid images - result, err := cloudbrain.GetImages() - if err != nil { - ctx.Data["error"] = err.Error() - log.Error("cloudbrain.GetImages failed:", err.Error(), ctx.Data["MsgID"]) - } - - for i, payload := range result.Payload.ImageInfo { - if strings.HasPrefix(result.Payload.ImageInfo[i].Place, "192.168") { - result.Payload.ImageInfo[i].PlaceView = payload.Place[strings.Index(payload.Place, "/"):len(payload.Place)] - } else { - result.Payload.ImageInfo[i].PlaceView = payload.Place - } - } - - ctx.Data["images"] = result.Payload.ImageInfo - - resultPublic, err := cloudbrain.GetPublicImages() - if err != nil { - ctx.Data["error"] = err.Error() - log.Error("cloudbrain.GetPublicImages failed:", err.Error(), ctx.Data["MsgID"]) - } - - for i, payload := range resultPublic.Payload.ImageInfo { - if strings.HasPrefix(resultPublic.Payload.ImageInfo[i].Place, "192.168") { - resultPublic.Payload.ImageInfo[i].PlaceView = payload.Place[strings.Index(payload.Place, "/"):len(payload.Place)] - } else { - resultPublic.Payload.ImageInfo[i].PlaceView = payload.Place - } - } - - ctx.Data["public_images"] = resultPublic.Payload.ImageInfo - - //get valid dataset - attachs, err := models.GetAllUserAttachments(ctx.User.ID) - if err != nil { - log.Error("GetAllUserAttachments failed: %v", err, ctx.Data["MsgID"]) - return err - } - - ctx.Data["attachments"] = attachs - ctx.Data["command"] = cloudbrain.Command - ctx.Data["code_path"] = cloudbrain.CodeMountPath - ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath - ctx.Data["model_path"] = cloudbrain.ModelMountPath - ctx.Data["benchmark_path"] = cloudbrain.BenchMarkMountPath - ctx.Data["is_benchmark_enabled"] = setting.IsBenchmarkEnabled - - //get valid resource specs - if categories == nil { - json.Unmarshal([]byte(setting.BenchmarkCategory), &categories) - } - ctx.Data["benchmark_categories"] = categories.Category - - ctx.Data["benchmark_types"] = GetBenchmarkTypes(ctx).BenchmarkType - - if gpuInfos == nil { - json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos) - } - ctx.Data["gpu_types"] = gpuInfos.GpuInfo - - if trainGpuInfos == nil { - json.Unmarshal([]byte(setting.TrainGpuTypes), &trainGpuInfos) - } - ctx.Data["train_gpu_types"] = trainGpuInfos.GpuInfo - - if benchmarkGpuInfos == nil { - json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos) - } - ctx.Data["benchmark_gpu_types"] = benchmarkGpuInfos.GpuInfo - - if benchmarkResourceSpecs == nil { - json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &benchmarkResourceSpecs) - } - ctx.Data["benchmark_resource_specs"] = benchmarkResourceSpecs.ResourceSpec - - if cloudbrain.ResourceSpecs == nil { - json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs) - } - ctx.Data["resource_specs"] = cloudbrain.ResourceSpecs.ResourceSpec - - if cloudbrain.TrainResourceSpecs == nil { - json.Unmarshal([]byte(setting.TrainResourceSpecs), &cloudbrain.TrainResourceSpecs) - } - ctx.Data["train_resource_specs"] = cloudbrain.TrainResourceSpecs.ResourceSpec - - branches, _, err := ctx.Repo.GitRepo.GetBranches(0, 0) - if err != nil { - log.Error("GetBranches error:", err) - } - ctx.Data["branches"] = branches - ctx.Data["branchName"] = ctx.Repo.BranchName - - return nil -} - func GrampusTrainJobNPUNew(ctx *context.Context) { - err := grampusTrainJobNpuNewDataPrepare(ctx) + ctx.Data["cloudbraintype"] = models.TypeCloudBrainTwo + err := grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) if err != nil { ctx.ServerError("get new train-job info failed", err) return @@ -156,7 +56,7 @@ func GrampusTrainJobNPUNew(ctx *context.Context) { ctx.HTML(200, tplGrampusTrainJobNPUNew) } -func grampusTrainJobNpuNewDataPrepare(ctx *context.Context) error { +func grampusTrainJobNewDataPrepare(ctx *context.Context, processType string) error { ctx.Data["PageIsCloudBrain"] = true t := time.Now() @@ -164,23 +64,32 @@ func grampusTrainJobNpuNewDataPrepare(ctx *context.Context) error { ctx.Data["display_job_name"] = displayJobName //get valid dataset - attachs, err := models.GetModelArtsTrainAttachments(ctx.User.ID) - if err != nil { - log.Error("GetModelArtsTrainAttachments failed:", err.Error()) + if processType == grampus.ProcessorTypeNPU { + attachs, err := models.GetModelArtsTrainAttachments(ctx.User.ID) + if err != nil { + log.Error("GetModelArtsTrainAttachments failed:", err.Error()) + } else { + ctx.Data["attachments"] = attachs + } } else { - ctx.Data["attachments"] = attachs + attachs, err := models.GetAllUserAttachments(ctx.User.ID) + if err != nil { + log.Error("GetAllUserAttachments failed: %v", err, ctx.Data["MsgID"]) + } else { + ctx.Data["attachments"] = attachs + } } - //get valid engines - images, err := grampus.GetImages(grampus.ProcessorTypeNPU) + //get valid images + images, err := grampus.GetImages(processType) if err != nil { - log.Error("GetResourceSpecs failed:", err.Error()) + log.Error("GetImages failed:", err.Error()) } else { - ctx.Data["engine_versions"] = images.Infos + ctx.Data["images"] = images.Infos } //get valid resource specs - specs, err := grampus.GetResourceSpecs(grampus.ProcessorTypeNPU) + specs, err := grampus.GetResourceSpecs(processType) if err != nil { log.Error("GetResourceSpecs failed:", err.Error()) } else { @@ -235,13 +144,13 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain count, err := models.GetGrampusCountByUserID(ctx.User.ID, string(models.JobTypeTrain), models.NPUResource) if err != nil { log.Error("GetGrampusCountByUserID failed:%v", err, ctx.Data["MsgID"]) - grampusTrainJobNpuNewDataPrepare(ctx) + grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) ctx.RenderWithErr("system error", tplGrampusTrainJobNPUNew, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - grampusTrainJobNpuNewDataPrepare(ctx) + grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplGrampusTrainJobNPUNew, &form) return } @@ -250,7 +159,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain //check param if err := grampusParamCheckCreateTrainJob(form); err != nil { log.Error("paramCheckCreateTrainJob failed:(%v)", err) - grampusTrainJobNpuNewDataPrepare(ctx) + grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) ctx.RenderWithErr(err.Error(), tplGrampusTrainJobNPUNew, &form) return } @@ -260,14 +169,14 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) - grampusTrainJobNpuNewDataPrepare(ctx) + grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) ctx.RenderWithErr("the job name did already exist", tplGrampusTrainJobNPUNew, &form) return } } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) - grampusTrainJobNpuNewDataPrepare(ctx) + grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) ctx.RenderWithErr("system error", tplGrampusTrainJobNPUNew, &form) return } @@ -277,7 +186,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain attachment, err := models.GetAttachmentByUUID(uuid) if err != nil { log.Error("GetAttachmentByUUID failed:", err.Error(), ctx.Data["MsgID"]) - grampusTrainJobNpuNewDataPrepare(ctx) + grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) ctx.RenderWithErr("dataset is not exist", tplGrampusTrainJobNPUNew, &form) return } @@ -290,7 +199,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain if err := downloadZipCode(ctx, codeLocalPath, branchName); err != nil { log.Error("downloadZipCode failed, server timed out: %s (%v)", repo.FullName(), err) - grampusTrainJobNpuNewDataPrepare(ctx) + grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) ctx.RenderWithErr("Create task failed, server timed out", tplGrampusTrainJobNPUNew, &form) return } @@ -298,14 +207,14 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain //todo: upload code (send to file_server todo this work?) if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath); err != nil { log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err) - grampusTrainJobNpuNewDataPrepare(ctx) + grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) ctx.RenderWithErr("Failed to obsMkdir_output", tplGrampusTrainJobNPUNew, &form) return } if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) - grampusTrainJobNpuNewDataPrepare(ctx) + grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) ctx.RenderWithErr("Failed to uploadCodeToObs", tplGrampusTrainJobNPUNew, &form) return } @@ -344,7 +253,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain err = grampus.GenerateTrainJob(ctx, req) if err != nil { log.Error("GenerateTrainJob failed:%v", err.Error()) - grampusTrainJobNpuNewDataPrepare(ctx) + grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) ctx.RenderWithErr(err.Error(), tplGrampusTrainJobNPUNew, &form) return } diff --git a/templates/custom/select_dataset.tmpl b/templates/custom/select_dataset.tmpl old mode 100644 new mode 100755 index d545f487e..0108327d5 --- a/templates/custom/select_dataset.tmpl +++ b/templates/custom/select_dataset.tmpl @@ -1,9 +1,9 @@ -
+ data-dataset-type="{{.datasetType}}">
+
- {{if eq .cloudbraintype 0}} + {{if eq .datasetType 0}} {{else}} @@ -18,7 +18,7 @@
- +
diff --git a/templates/custom/select_dataset_train.tmpl b/templates/custom/select_dataset_train.tmpl old mode 100644 new mode 100755 index f1d2abaf3..e96bfa3a8 --- a/templates/custom/select_dataset_train.tmpl +++ b/templates/custom/select_dataset_train.tmpl @@ -1,5 +1,5 @@ + data-dataset-type="{{.datasetType}}">
{{if or (.benchmarkMode) (.newInference)}}
- +
diff --git a/templates/repo/grampus/trainjob/gpu/new.tmpl b/templates/repo/grampus/trainjob/gpu/new.tmpl index 2eef1c5f2..7556937d8 100755 --- a/templates/repo/grampus/trainjob/gpu/new.tmpl +++ b/templates/repo/grampus/trainjob/gpu/new.tmpl @@ -156,29 +156,6 @@
-
- - -
- - -
@@ -198,7 +175,7 @@ {{template "custom/select_dataset_train" .}} - 训练脚本存储在/code中，数据集存储在/dataset中，训练输出请存储在/model中以供后续下载。 + {{.i18n.Tr "repo.grampus.dataset_path_rule"}}
{{.i18n.Tr "repo.modelarts.train_job.add_run_parameter"}} @@ -225,11 +202,11 @@
-
- - + {{range .flavor_infos}} + {{end}}
diff --git a/templates/repo/grampus/trainjob/npu/new.tmpl b/templates/repo/grampus/trainjob/npu/new.tmpl index 4840e6192..d32605e88 100755 --- a/templates/repo/grampus/trainjob/npu/new.tmpl +++ b/templates/repo/grampus/trainjob/npu/new.tmpl @@ -145,19 +145,13 @@ - - -
- -
- - -
- +
+ +
@@ -183,32 +177,6 @@
- - - - -