From 7279a21a819b739fa1b37ab2e99c27af3811748d Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 5 May 2022 17:41:55 +0800 Subject: [PATCH] =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E8=AF=84=E6=B5=8B=E5=90=8E?= =?UTF-8?q?=E7=AB=AF=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 1 + options/locale/locale_en-US.ini | 2 + options/locale/locale_zh-CN.ini | 3 + routers/admin/cloudbrains.go | 4 +- routers/repo/cloudbrain.go | 114 ++++++++++++++++++++++++++++- routers/repo/modelarts.go | 2 +- routers/user/home.go | 4 +- templates/repo/cloudbrain/new.tmpl | 6 -- 8 files changed, 122 insertions(+), 14 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 480350c8d..e1dc2cbd0 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -160,6 +160,7 @@ type Cloudbrain struct { User *User `xorm:"-"` Repo *Repository `xorm:"-"` + BenchmarkType string `xorm:"-"` //算法评测，模型评测 BenchmarkTypeName string `xorm:"-"` BenchmarkTypeRankLink string `xorm:"-"` StartTime timeutil.TimeStamp diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index ca16093a6..8ff34f9ae 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1091,6 +1091,8 @@ modelarts.train_job_para.connfirm=train_job_para.connfirm modelarts.evaluate_job=Model Evaluation modelarts.evaluate_job.new_job=New Model Evaluation cloudbrain.benchmark.evaluate_type=Evaluation Type +cloudbrain.benchmark.algorithm=Algorithm Evaluation +cloudbrain.benchmark.model=Model Evaluation cloudbrain.benchmark.evaluate_child_type=Child Type cloudbrain.benchmark.evaluate_mirror=Mirror cloudbrain.benchmark.evaluate_train=Train Script diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 474b96c3f..a01fdd94a 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1102,11 +1102,14 @@ modelarts.train_job_para.connfirm=确定 modelarts.evaluate_job=评测任务 modelarts.evaluate_job.new_job=新建评测任务 cloudbrain.benchmark.evaluate_type=评测类型 +cloudbrain.benchmark.algorithm=算法评测 +cloudbrain.benchmark.model=模型评测 cloudbrain.benchmark.evaluate_child_type=子类型 cloudbrain.benchmark.evaluate_mirror=镜像 cloudbrain.benchmark.evaluate_train=训练程序 cloudbrain.benchmark.evaluate_test=测试程序 cloudbrain.benchmark.types={"type":[{"id":1,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=detection","first":"目标检测","second":[{"id":1,"value":"无","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"yangzhx","repo_name":"detection_benchmark_script"}]},{"id":2,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=reid","first":"目标重识别","second":[{"id":1,"value":"车辆重识别","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"JiahongXu","repo_name":"benchmark_reID_script"},{"id":2,"value":"基于图像的行人重识别","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"JiahongXu","repo_name":"benchmark_reID_script"}]},{"id":3,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=tracking","first":"多目标跟踪","second":[{"id":1,"value":"无","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"lix07","repo_name":"MOT_benchmark_script"}]}]} +cloudbrain.benchmark.model.types={"type":[{"id":1,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=detection","first":"目标检测","second":[{"id":1,"value":"无","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"yangzhx","repo_name":"detection_benchmark_script"}]},{"id":2,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=reid","first":"目标重识别","second":[{"id":1,"value":"车辆重识别","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"JiahongXu","repo_name":"benchmark_reID_script"},{"id":2,"value":"基于图像的行人重识别","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"JiahongXu","repo_name":"benchmark_reID_script"}]},{"id":3,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=tracking","first":"多目标跟踪","second":[{"id":1,"value":"无","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"lix07","repo_name":"MOT_benchmark_script"}]}]} modelarts.infer_job_model = 模型名称 modelarts.infer_job_model_file = 模型文件 diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index 6687b990a..0481e6743 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -52,8 +52,8 @@ func CloudBrains(ctx *context.Context) { var jobTypes []string jobTypeNot := false - if jobType == string(models.JobTypeDebug) { - jobTypes = append(jobTypes, string(models.JobTypeSnn4imagenet), string(models.JobTypeBrainScore), string(models.JobTypeDebug)) + if jobType == string(models.JobTypeBenchmark) { + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index b3b07f352..0520c2dbd 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -249,7 +249,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { return } - if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) && jobType != string(models.JobTypeTrain) { + if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeTrain) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("jobtype error", tpl, &form) @@ -1646,7 +1646,7 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { } var jobTypes []string - jobTypes = append(jobTypes, string(models.JobTypeBenchmark)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, @@ -1676,7 +1676,12 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { } ciTasks[i].TrainJobDuration = models.ConvertDurationToStr(ciTasks[i].Duration) } - + if ciTasks[i].JobType == string(models.JobTypeBenchmark) { + ciTasks[i].BenchmarkType = ctx.Tr("cloudbrain.benchmark.algorithm") + } else if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) || ciTasks[i].JobType == string(models.JobTypeBrainScore) { + ciTasks[i].BenchmarkType = ctx.Tr("cloudbrain.benchmark.model") + ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType + } ciTasks[i].BenchmarkTypeName = "" if task.BenchmarkTypeID > 0 { for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType { @@ -1813,6 +1818,16 @@ func getBenchmarkResourceSpec(resourceSpecID int) (int, error) { } func CloudBrainBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { + jobType := form.JobType + if jobType == string(models.JobTypeBenchmark) { + BenchMarkAlgorithmCreate(ctx, form) + } else { + ModelBenchmarkCreate(ctx, form) + } + +} + +func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { ctx.Data["PageIsCloudBrain"] = true displayJobName := form.DisplayJobName jobName := util.ConvertDisplayJobNameToJobName(displayJobName) @@ -1965,6 +1980,99 @@ func CloudBrainBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainF ctx.RenderWithErr(err.Error(), tplCloudBrainBenchmarkNew, &form) return } + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") + +} + +func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { + ctx.Data["PageIsCloudBrain"] = true + displayJobName := form.DisplayJobName + jobName := util.ConvertDisplayJobNameToJobName(displayJobName) + image := form.Image + uuid := form.Attachment + jobType := form.JobType + gpuQueue := form.GpuType + codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath + resourceSpecId := form.ResourceSpecId + branchName := cloudbrain.DefaultBranchName + repo := ctx.Repo.Repository + + tpl := tplCloudBrainBenchmarkNew + command := cloudbrain.Command + + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) + if err == nil { + if len(tasks) != 0 { + log.Error("the job name did already exist", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("the job name did already exist", tpl, &form) + return + } + } else { + if !models.IsErrJobNotExist(err) { + log.Error("system error, %v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tpl, &form) + return + } + } + + if !jobNamePattern.MatchString(displayJobName) { + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form) + return + } + + if jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) { + log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("jobtype error", tpl, &form) + return + } + + count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType) + if err != nil { + log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tpl, &form) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("you have already a running or waiting task, can not create more", tpl, &form) + return + } + } + + downloadCode(repo, codePath, branchName) + uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/") + + modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/" + mkModelPath(modelPath) + uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/") + + snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath + if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { + downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "") + uploadCodeToMinio(snn4imagenetPath+"/", jobName, cloudbrain.Snn4imagenetMountPath+"/") + } + + brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath + if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { + downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "") + uploadCodeToMinio(brainScorePath+"/", jobName, cloudbrain.BrainScoreMountPath+"/") + } + + err = cloudbrain.GenerateTask(ctx, displayJobName, jobName, image, command, uuid, storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), + storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), + storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), + storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), jobType, gpuQueue, form.Description, branchName, form.BootFile, form.Params, + 0, 0, resourceSpecId) + if err != nil { + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr(err.Error(), tpl, &form) + return + } ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 81e1664a4..bcc6f6156 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -76,7 +76,7 @@ func DebugJobIndex(ctx *context.Context) { } var jobTypes []string - jobTypes = append(jobTypes, string(models.JobTypeSnn4imagenet), string(models.JobTypeBrainScore), string(models.JobTypeDebug)) + jobTypes = append(jobTypes, string(models.JobTypeDebug)) ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, diff --git a/routers/user/home.go b/routers/user/home.go index d92bc31c6..53aff19b8 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -778,8 +778,8 @@ func Cloudbrains(ctx *context.Context) { var jobTypes []string jobTypeNot := false - if jobType == string(models.JobTypeDebug) { - jobTypes = append(jobTypes, string(models.JobTypeSnn4imagenet), string(models.JobTypeBrainScore), string(models.JobTypeDebug)) + if jobType == string(models.JobTypeBenchmark) { + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index e3e3dbd1c..c4596de37 100755 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -156,12 +156,6 @@