diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index bf1761955..d79b6ace6 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -44,13 +44,17 @@ var ( TrainResourceSpecs *models.ResourceSpecs ) +type DatasetInfo struct { + DataLocalPath string + Name string +} + type GenerateCloudBrainTaskReq struct { Ctx *context.Context DisplayJobName string JobName string Image string Command string - Uuids string CodePath string ModelPath string BenchmarkPath string @@ -63,7 +67,9 @@ type GenerateCloudBrainTaskReq struct { BootFile string Params string CommitID string - DataLocalPath string + Uuids string + DatasetNames string + DatasetInfos map[string]DatasetInfo BenchmarkTypeID int BenchmarkChildTypeID int ResourceSpecId int @@ -213,15 +219,9 @@ func AdminOrImageCreaterRight(ctx *context.Context) { } func GenerateTask(req GenerateCloudBrainTaskReq) error { - - dataActualPath := setting.Attachment.Minio.RealPath + - setting.Attachment.Minio.Bucket + "/" + - setting.Attachment.Minio.BasePath + - models.AttachmentRelativePath(req.Uuids) + - req.Uuids - var resourceSpec *models.ResourceSpec var versionCount int + if req.JobType == string(models.JobTypeTrain) { versionCount = 1 if TrainResourceSpecs == nil { @@ -249,14 +249,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) error { return errors.New("no such resourceSpec") } - var datasetName string - attach, err := models.GetAttachmentByUUID(req.Uuids) - if err != nil { - //for benchmark, do not return error - log.Error("GetAttachmentByUUID failed:%v", err, req.Ctx.Data["MsgID"]) - } else { - datasetName = attach.Name - } + log.Info(req.DatasetInfos[req.Uuids].DataLocalPath) createTime := timeutil.TimeStampNow() jobResult, err := CreateJob(req.JobName, models.CreateJobParams{ @@ -290,7 +283,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) error { }, { HostPath: models.StHostPath{ - Path: dataActualPath, + Path: req.DatasetInfos[req.Uuids].DataLocalPath, MountPath: DataSetMountPath, ReadOnly: true, }, @@ -357,7 +350,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) error { VersionCount: versionCount, BranchName: req.BranchName, BootFile: req.BootFile, - DatasetName: datasetName, + DatasetName: req.DatasetNames, Parameters: req.Params, CreatedUnix: createTime, UpdatedUnix: createTime, diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index f3f2fddb0..90c2146d1 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -273,10 +273,11 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } } - if err = checkDatasetLimit(uuids); err != nil { - log.Error("checkDatasetLimit failed: %v", err, ctx.Data["MsgID"]) + datasetInfos, datasetNames, err := getDatasetInfo(uuids) + if err != nil { + log.Error("getDatasetInfo failed: %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("checkDatasetLimit failed", tpl, &form) + ctx.RenderWithErr("getDatasetInfo failed", tpl, &form) return } @@ -299,6 +300,8 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { Image: image, Command: command, Uuids: uuids, + DatasetNames: datasetNames, + DatasetInfos: datasetInfos, CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), @@ -2009,11 +2012,14 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo //return } - dataActualPath := setting.Attachment.Minio.RealPath + - setting.Attachment.Minio.Bucket + "/" + - setting.Attachment.Minio.BasePath + - models.AttachmentRelativePath(childInfo.Attachment) + - childInfo.Attachment + uuid := childInfo.Attachment + datasetInfos, datasetNames, err := getDatasetInfo(uuid) + if err != nil { + log.Error("getDatasetInfo failed: %v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("getDatasetInfo failed", tplCloudBrainBenchmarkNew, &form) + return + } req := cloudbrain.GenerateCloudBrainTaskReq{ Ctx: ctx, @@ -2021,7 +2027,9 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo JobName: jobName, Image: image, Command: command, - Uuids: childInfo.Attachment, + Uuids: uuid, + DatasetNames: datasetNames, + DatasetInfos: datasetInfos, CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), @@ -2037,7 +2045,6 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo BenchmarkTypeID: benchmarkTypeID, BenchmarkChildTypeID: benchmarkChildTypeID, ResourceSpecId: resourceSpecId, - DataLocalPath: dataActualPath, } err = cloudbrain.GenerateTask(req) @@ -2134,11 +2141,13 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description)) } - dataActualPath := setting.Attachment.Minio.RealPath + - setting.Attachment.Minio.Bucket + "/" + - setting.Attachment.Minio.BasePath + - models.AttachmentRelativePath(uuid) + - uuid + datasetInfos, datasetNames, err := getDatasetInfo(uuid) + if err != nil { + log.Error("getDatasetInfo failed: %v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("getDatasetInfo failed", tpl, &form) + return + } req := cloudbrain.GenerateCloudBrainTaskReq{ Ctx: ctx, @@ -2147,6 +2156,8 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) Image: image, Command: command, Uuids: uuid, + DatasetNames: datasetNames, + DatasetInfos: datasetInfos, CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), @@ -2162,7 +2173,6 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) BenchmarkTypeID: 0, BenchmarkChildTypeID: benchmarkChildTypeID, ResourceSpecId: resourceSpecId, - DataLocalPath: dataActualPath, } err = cloudbrain.GenerateTask(req) @@ -2277,26 +2287,45 @@ func GetBenchmarkTypes(ctx *context.Context) *models.BenchmarkTypes { return benchmarkTypesMap[lang] } -func checkDatasetLimit(uuidStr string) error { +func getDatasetInfo(uuidStr string) (map[string]cloudbrain.DatasetInfo, string, error) { + var datasetNames string uuids := strings.Split(uuidStr, ";") if len(uuids) > 5 { log.Error("the dataset count(%d) exceed the limit", len(uuids)) - return errors.New("the dataset count exceed the limit") + return nil, datasetNames, errors.New("the dataset count exceed the limit") } - attachNames := make(map[string]string) - for _, uuid := range uuids { + datasetInfos := make(map[string]cloudbrain.DatasetInfo) + for i, uuid := range uuids { attach, err := models.GetAttachmentByUUID(uuid) if err != nil { log.Error("GetAttachmentByUUID failed: %v", err) - return err + return nil, datasetNames, err } - if _, ok := attachNames[attach.Name]; ok { + if _, ok := datasetInfos[uuid]; ok { log.Error("the dataset name is same: %v", attach.Name) - return errors.New("the dataset name is same") + return nil, datasetNames, errors.New("the dataset name is same") + } + + dataLocalPath := setting.Attachment.Minio.RealPath + + setting.Attachment.Minio.Bucket + "/" + + setting.Attachment.Minio.BasePath + + models.AttachmentRelativePath(uuid) + + uuid + + datasetInfos[uuid] = cloudbrain.DatasetInfo{ + DataLocalPath: dataLocalPath, + Name: attach.Name, + } + if i == 0 { + datasetNames = attach.Name + } else { + datasetNames += ";" + attach.Name } - attachNames[attach.Name] = attach.Name } - return nil + + log.Info(datasetNames) + + return datasetInfos, datasetNames, nil }