debug

modules/cloudbrain/cloudbrain.go

@@ -44,13 +44,17 @@ var (
 	TrainResourceSpecs *models.ResourceSpecs
 )
 
+type DatasetInfo struct {
+	DataLocalPath string
+	Name          string
+}
+
 type GenerateCloudBrainTaskReq struct {
 	Ctx                  *context.Context
 	DisplayJobName       string
 	JobName              string
 	Image                string
 	Command              string
-	Uuids                string
 	CodePath             string
 	ModelPath            string
 	BenchmarkPath        string
@@ -63,7 +67,9 @@ type GenerateCloudBrainTaskReq struct {
 	BootFile             string
 	Params               string
 	CommitID             string
-	DataLocalPath        string
+	Uuids                string
+	DatasetNames         string
+	DatasetInfos         map[string]DatasetInfo
 	BenchmarkTypeID      int
 	BenchmarkChildTypeID int
 	ResourceSpecId       int
@@ -213,15 +219,9 @@ func AdminOrImageCreaterRight(ctx *context.Context) {
 }
 
 func GenerateTask(req GenerateCloudBrainTaskReq) error {
-
-	dataActualPath := setting.Attachment.Minio.RealPath +
-		setting.Attachment.Minio.Bucket + "/" +
-		setting.Attachment.Minio.BasePath +
-		models.AttachmentRelativePath(req.Uuids) +
-		req.Uuids
-
 	var resourceSpec *models.ResourceSpec
 	var versionCount int
+
 	if req.JobType == string(models.JobTypeTrain) {
 		versionCount = 1
 		if TrainResourceSpecs == nil {
@@ -249,14 +249,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) error {
 		return errors.New("no such resourceSpec")
 	}
 
-	var datasetName string
-	attach, err := models.GetAttachmentByUUID(req.Uuids)
-	if err != nil {
-		//for benchmark, do not return error
-		log.Error("GetAttachmentByUUID failed:%v", err, req.Ctx.Data["MsgID"])
-	} else {
-		datasetName = attach.Name
-	}
+	log.Info(req.DatasetInfos[req.Uuids].DataLocalPath)
 
 	createTime := timeutil.TimeStampNow()
 	jobResult, err := CreateJob(req.JobName, models.CreateJobParams{
@@ -290,7 +283,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) error {
 			},
 			{
 				HostPath: models.StHostPath{
-					Path:      dataActualPath,
+					Path:      req.DatasetInfos[req.Uuids].DataLocalPath,
 					MountPath: DataSetMountPath,
 					ReadOnly:  true,
 				},
@@ -357,7 +350,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) error {
 		VersionCount:         versionCount,
 		BranchName:           req.BranchName,
 		BootFile:             req.BootFile,
-		DatasetName:          datasetName,
+		DatasetName:          req.DatasetNames,
 		Parameters:           req.Params,
 		CreatedUnix:          createTime,
 		UpdatedUnix:          createTime,

routers/repo/cloudbrain.go

@@ -273,10 +273,11 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
 		}
 	}
 
-	if err = checkDatasetLimit(uuids); err != nil {
-		log.Error("checkDatasetLimit failed: %v", err, ctx.Data["MsgID"])
+	datasetInfos, datasetNames, err := getDatasetInfo(uuids)
+	if err != nil {
+		log.Error("getDatasetInfo failed: %v", err, ctx.Data["MsgID"])
 		cloudBrainNewDataPrepare(ctx)
-		ctx.RenderWithErr("checkDatasetLimit failed", tpl, &form)
+		ctx.RenderWithErr("getDatasetInfo failed", tpl, &form)
 		return
 	}
 
@@ -299,6 +300,8 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
 		Image:                image,
 		Command:              command,
 		Uuids:                uuids,
+		DatasetNames:         datasetNames,
+		DatasetInfos:         datasetInfos,
 		CodePath:             storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
 		ModelPath:            storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
 		BenchmarkPath:        storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
@@ -2009,11 +2012,14 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo
 		//return
 	}
 
-	dataActualPath := setting.Attachment.Minio.RealPath +
-		setting.Attachment.Minio.Bucket + "/" +
-		setting.Attachment.Minio.BasePath +
-		models.AttachmentRelativePath(childInfo.Attachment) +
-		childInfo.Attachment
+	uuid := childInfo.Attachment
+	datasetInfos, datasetNames, err := getDatasetInfo(uuid)
+	if err != nil {
+		log.Error("getDatasetInfo failed: %v", err, ctx.Data["MsgID"])
+		cloudBrainNewDataPrepare(ctx)
+		ctx.RenderWithErr("getDatasetInfo failed", tplCloudBrainBenchmarkNew, &form)
+		return
+	}
 
 	req := cloudbrain.GenerateCloudBrainTaskReq{
 		Ctx:                  ctx,
@@ -2021,7 +2027,9 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo
 		JobName:              jobName,
 		Image:                image,
 		Command:              command,
-		Uuids:                childInfo.Attachment,
+		Uuids:                uuid,
+		DatasetNames:         datasetNames,
+		DatasetInfos:         datasetInfos,
 		CodePath:             storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
 		ModelPath:            storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
 		BenchmarkPath:        storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
@@ -2037,7 +2045,6 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo
 		BenchmarkTypeID:      benchmarkTypeID,
 		BenchmarkChildTypeID: benchmarkChildTypeID,
 		ResourceSpecId:       resourceSpecId,
-		DataLocalPath:        dataActualPath,
 	}
 
 	err = cloudbrain.GenerateTask(req)
@@ -2134,11 +2141,13 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
 		command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description))
 	}
 
-	dataActualPath := setting.Attachment.Minio.RealPath +
-		setting.Attachment.Minio.Bucket + "/" +
-		setting.Attachment.Minio.BasePath +
-		models.AttachmentRelativePath(uuid) +
-		uuid
+	datasetInfos, datasetNames, err := getDatasetInfo(uuid)
+	if err != nil {
+		log.Error("getDatasetInfo failed: %v", err, ctx.Data["MsgID"])
+		cloudBrainNewDataPrepare(ctx)
+		ctx.RenderWithErr("getDatasetInfo failed", tpl, &form)
+		return
+	}
 
 	req := cloudbrain.GenerateCloudBrainTaskReq{
 		Ctx:                  ctx,
@@ -2147,6 +2156,8 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
 		Image:                image,
 		Command:              command,
 		Uuids:                uuid,
+		DatasetNames:         datasetNames,
+		DatasetInfos:         datasetInfos,
 		CodePath:             storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
 		ModelPath:            storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
 		BenchmarkPath:        storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
@@ -2162,7 +2173,6 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
 		BenchmarkTypeID:      0,
 		BenchmarkChildTypeID: benchmarkChildTypeID,
 		ResourceSpecId:       resourceSpecId,
-		DataLocalPath:        dataActualPath,
 	}
 
 	err = cloudbrain.GenerateTask(req)
@@ -2277,26 +2287,45 @@ func GetBenchmarkTypes(ctx *context.Context) *models.BenchmarkTypes {
 	return benchmarkTypesMap[lang]
 }
 
-func checkDatasetLimit(uuidStr string) error {
+func getDatasetInfo(uuidStr string) (map[string]cloudbrain.DatasetInfo, string, error) {
+	var datasetNames string
 	uuids := strings.Split(uuidStr, ";")
 	if len(uuids) > 5 {
 		log.Error("the dataset count(%d) exceed the limit", len(uuids))
-		return errors.New("the dataset count exceed the limit")
+		return nil, datasetNames, errors.New("the dataset count exceed the limit")
 	}
 
-	attachNames := make(map[string]string)
-	for _, uuid := range uuids {
+	datasetInfos := make(map[string]cloudbrain.DatasetInfo)
+	for i, uuid := range uuids {
 		attach, err := models.GetAttachmentByUUID(uuid)
 		if err != nil {
 			log.Error("GetAttachmentByUUID failed: %v", err)
-			return err
+			return nil, datasetNames, err
 		}
 
-		if _, ok := attachNames[attach.Name]; ok {
+		if _, ok := datasetInfos[uuid]; ok {
 			log.Error("the dataset name is same: %v", attach.Name)
-			return errors.New("the dataset name is same")
+			return nil, datasetNames, errors.New("the dataset name is same")
+		}
+
+		dataLocalPath := setting.Attachment.Minio.RealPath +
+			setting.Attachment.Minio.Bucket + "/" +
+			setting.Attachment.Minio.BasePath +
+			models.AttachmentRelativePath(uuid) +
+			uuid
+
+		datasetInfos[uuid] = cloudbrain.DatasetInfo{
+			DataLocalPath: dataLocalPath,
+			Name:          attach.Name,
+		}
+		if i == 0 {
+			datasetNames = attach.Name
+		} else {
+			datasetNames += ";" + attach.Name
 		}
-		attachNames[attach.Name] = attach.Name
 	}
-	return nil
+
+	log.Info(datasetNames)
+
+	return datasetInfos, datasetNames, nil
 }