From 72ab6629b19e74f25d99da47402171b32d854267 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 24 May 2021 18:01:09 +0800 Subject: [PATCH 1/3] add gpu type --- models/cloudbrain.go | 1 + routers/repo/cloudbrain.go | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 00d4b00e6..7d9d951a9 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -167,6 +167,7 @@ type TaskInfo struct { CodeName string `json:"code_name"` BenchmarkCategory []string `json:"selected_category"` CodeLink string `json:"code_link"` + GpuType string `json:"gpu_type"` } func ConvertToTaskPod(input map[string]interface{}) (TaskPod, error) { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 0c1b34836..f3d28eb5f 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -170,12 +170,12 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath if setting.IsBenchmarkEnabled && jobType == string(models.JobTypeBenchmark) { gpuType = form.GpuType - downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory) + downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType) } snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { - downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "") + downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "") } err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, jobType, gpuType) @@ -344,7 +344,7 @@ func downloadCode(repo *models.Repository, codePath string) error { return nil } -func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benchmarkCategory string) error { +func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benchmarkCategory, gpuType string) error { err := os.MkdirAll(codePath, os.ModePerm) if err != nil { log.Error("mkdir codePath failed", err.Error()) @@ -375,6 +375,7 @@ func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benc CodeName: repo.Name, BenchmarkCategory: strings.Split(benchmarkCategory, ","), CodeLink: strings.TrimSuffix(repo.CloneLink().HTTPS, ".git"), + GpuType: gpuType, }) if err != nil { log.Error("json.Marshal failed", err.Error()) From 9c140c4d2447eb67ecd43e207c37b9dbe30e4ba7 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 25 May 2021 11:56:47 +0800 Subject: [PATCH 2/3] optimization --- models/cloudbrain.go | 6 +++--- modules/cloudbrain/cloudbrain.go | 4 ++-- routers/repo/cloudbrain.go | 24 ++++++++++++++++++------ 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 7d9d951a9..c8000c5a6 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -277,11 +277,11 @@ type Category struct { Value string `json:"value"` } -type GpuTypes struct { - GpuType []*GpuType `json:"gpu_type"` +type GpuInfos struct { + GpuInfo []*GpuInfo `json:"gpu_type"` } -type GpuType struct { +type GpuInfo struct { Id int `json:"id"` Value string `json:"value"` Queue string `json:"queue"` diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 2f22f9770..9fda84ee8 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -23,7 +23,7 @@ const ( Success = "S000" ) -func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, jobType, gpuType string) error { +func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, jobType, gpuQueue string) error { dataActualPath := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.Attachment.Minio.BasePath + @@ -32,7 +32,7 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, jobResult, err := CreateJob(jobName, models.CreateJobParams{ JobName: jobName, RetryCount: 1, - GpuType: gpuType, + GpuType: gpuQueue, Image: image, TaskRoles: []models.TaskRole{ { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index f3d28eb5f..e6e56d2c8 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -25,6 +25,10 @@ const ( tplCloudBrainShow base.TplName = "repo/cloudbrain/show" ) +var ( + gpuInfos *models.GpuInfos +) + // MustEnableDataset check if repository enable internal cb func MustEnableCloudbrain(ctx *context.Context) { if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) { @@ -134,9 +138,11 @@ func CloudBrainNew(ctx *context.Context) { var categories *models.Categories json.Unmarshal([]byte(setting.BenchmarkCategory), &categories) ctx.Data["benchmark_categories"] = categories.Category - var gpuTypes *models.GpuTypes - json.Unmarshal([]byte(setting.GpuTypes), &gpuTypes) - ctx.Data["gpu_types"] = gpuTypes.GpuType + + if gpuInfos == nil { + json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos) + } + ctx.Data["gpu_types"] = gpuInfos.GpuInfo ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled ctx.HTML(200, tplCloudBrainNew) @@ -149,7 +155,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { command := form.Command uuid := form.Attachment jobType := form.JobType - gpuType := setting.JobType + gpuQueue := setting.JobType codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) { @@ -169,7 +175,13 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath if setting.IsBenchmarkEnabled && jobType == string(models.JobTypeBenchmark) { - gpuType = form.GpuType + gpuQueue = form.GpuType + var gpuType string + for _, gpuInfo := range gpuInfos.GpuInfo { + if gpuInfo.Queue == gpuQueue { + gpuType = gpuInfo.Value + } + } downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType) } @@ -178,7 +190,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "") } - err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, jobType, gpuType) + err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, jobType, gpuQueue) if err != nil { ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) return From 4f1b6c3df1187f16aa2af1cdf68fcf5ffa77ce02 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 25 May 2021 15:56:09 +0800 Subject: [PATCH 3/3] add type info into task info --- routers/repo/cloudbrain.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index e6e56d2c8..8d8e97377 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -26,7 +26,8 @@ const ( ) var ( - gpuInfos *models.GpuInfos + gpuInfos *models.GpuInfos + categories *models.Categories ) // MustEnableDataset check if repository enable internal cb @@ -135,8 +136,9 @@ func CloudBrainNew(ctx *context.Context) { ctx.Data["benchmark_path"] = cloudbrain.BenchMarkMountPath ctx.Data["is_benchmark_enabled"] = setting.IsBenchmarkEnabled - var categories *models.Categories - json.Unmarshal([]byte(setting.BenchmarkCategory), &categories) + if categories == nil { + json.Unmarshal([]byte(setting.BenchmarkCategory), &categories) + } ctx.Data["benchmark_categories"] = categories.Category if gpuInfos == nil {