Merge branch 'V20220815' into issue-zpz-0815

modules/storage/minio_ext.go

@@ -214,47 +214,6 @@ func GetOneLevelAllObjectUnderDirMinio(bucket string, prefixRootPath string, rel
 				ParenDir: ParenDir,
 			}
 			fileInfos = append(fileInfos, fileInfo)
-
-			// log.Info("val key=" + val.Key)
-			// var isDir bool
-			// var fileName string
-			// if val.Key == Prefix {
-			// 	continue
-			// }
-
-			// fileName = val.Key[prefixLen:]
-			// log.Info("fileName =" + fileName)
-			// files := strings.Split(fileName, "/")
-			// if fileMap[files[0]] {
-			// 	continue
-			// } else {
-			// 	fileMap[files[0]] = true
-			// }
-			// ParenDir := relativePath
-			// fileName = files[0]
-			// if len(files) > 1 {
-			// 	isDir = true
-			// 	ParenDir += fileName + "/"
-			// } else {
-			// 	isDir = false
-			// }
-
-			// // if strings.HasSuffix(val.Key, "/") {
-			// // 	isDir = true
-			// // 	fileName = val.Key[prefixLen : len(val.Key)-1]
-			// // 	relativePath += val.Key[prefixLen:]
-			// // } else {
-			// // 	isDir = false
-			// // 	fileName = val.Key[prefixLen:]
-			// // }
-			// fileInfo := FileInfo{
-			// 	ModTime:  val.LastModified.Local().Format("2006-01-02 15:04:05"),
-			// 	FileName: fileName,
-			// 	Size:     val.Size,
-			// 	IsDir:    isDir,
-			// 	ParenDir: relativePath,
-			// }
-			// fileInfos = append(fileInfos, fileInfo)
 		}
 		return fileInfos, err
 	} else {

options/locale/locale_en-US.ini

@@ -1073,6 +1073,7 @@ cloudbrain_operate = Operate
 cloudbrain_status_createtime = Status/Createtime
 cloudbrain_status_runtime = Running Time
 cloudbrain_jobname_err=Name must start with a lowercase letter or number,can include lowercase letter,number,_ and -,can not end with _,  and can be up to 36 characters long.
+cloudbrain_bootfile_err=The bootfile does not exist in the repository
 cloudbrain_query_fail=Failed to query cloudbrain information.
 cloudbrain.mirror_tag = Mirror Tag
 cloudbrain.mirror_description = Mirror Description

options/locale/locale_zh-CN.ini

@@ -1076,6 +1076,7 @@ cloudbrain_operate=操作
 cloudbrain_status_createtime=状态/创建时间
 cloudbrain_status_runtime = 运行时长
 cloudbrain_jobname_err=只能以小写字母或数字开头且只包含小写字母、数字、_和-，不能以_结尾，最长36个字符。
+cloudbrain_bootfile_err=仓库中不存在启动文件
 cloudbrain_query_fail=查询云脑任务失败。
 cloudbrain.mirror_tag = 镜像标签
 cloudbrain.mirror_description = 镜像描述

routers/repo/cloudbrain.go

@@ -239,6 +239,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
 	codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
 	resourceSpecId := form.ResourceSpecId
 	branchName := form.BranchName
+	bootFile := strings.TrimSpace(form.BootFile)
 	repo := ctx.Repo.Repository
 	tpl := tplCloudBrainNew
 
@@ -305,6 +306,13 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
 
 	command := cloudbrain.GetCloudbrainDebugCommand()
 	if jobType == string(models.JobTypeTrain) {
+		bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName)
+		if err != nil || !bootFileExist {
+			log.Error("Get bootfile error:", err, ctx.Data["MsgID"])
+			cloudBrainNewDataPrepare(ctx)
+			ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form)
+			return
+		}
 		tpl = tplCloudBrainTrainJobNew
 		commandTrain, err := getTrainJobCommand(form)
 		if err != nil {
@@ -413,6 +421,7 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra
 	codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
 	resourceSpecId := form.ResourceSpecId
 	branchName := form.BranchName
+	bootFile := strings.TrimSpace(form.BootFile)
 	labelName := form.LabelName
 	repo := ctx.Repo.Repository
 
@@ -450,6 +459,14 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra
 		return
 	}
 
+	bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName)
+	if err != nil || !bootFileExist {
+		log.Error("Get bootfile error:", err, ctx.Data["MsgID"])
+		cloudBrainNewDataPrepare(ctx)
+		ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form)
+		return
+	}
+
 	count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType)
 	if err != nil {
 		log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])

routers/repo/grampus.go

@@ -215,6 +215,14 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 		return
 	}
 
+	bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName)
+	if err != nil || !bootFileExist {
+		log.Error("Get bootfile error:", err, ctx.Data["MsgID"])
+		grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
+		ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplGrampusTrainJobGPUNew, &form)
+		return
+	}
+
 	errStr := checkSpecialPool(ctx, "GPU")
 	if errStr != "" {
 		grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
@@ -399,6 +407,14 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 		return
 	}
 
+	bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName)
+	if err != nil || !bootFileExist {
+		log.Error("Get bootfile error:", err, ctx.Data["MsgID"])
+		grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
+		ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplGrampusTrainJobNPUNew, &form)
+		return
+	}
+
 	errStr := checkSpecialPool(ctx, "NPU")
 	if errStr != "" {
 		grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
@@ -789,7 +805,13 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo
 		}
 	}
 
-	commandCode := "cd " + workDir + "code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";"
+	var commandCode string
+	if processorType == grampus.ProcessorTypeNPU {
+		commandCode = "/bin/bash /home/work/run_train_for_openi.sh " + workDir + "code/" + strings.ToLower(repoName) + "/" + bootFile + " /tmp/log/train.log" + paramCode + ";"
+	} else if processorType == grampus.ProcessorTypeGPU {
+		commandCode = "cd " + workDir + "code/" + strings.ToLower(repoName) + ";python " + bootFile + paramCode + ";"
+	}
+
 	command += commandCode
 
 	//get exec result

routers/repo/modelarts.go

@@ -1089,7 +1089,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 	outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/"
 	logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/"
 	// dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
-	branch_name := form.BranchName
+	branchName := form.BranchName
 	isLatestVersion := modelarts.IsLatestVersion
 	FlavorName := form.FlavorName
 	VersionCount := modelarts.VersionCount
@@ -1117,6 +1117,14 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 		return
 	}
 
+	bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName)
+	if err != nil || !bootFileExist {
+		log.Error("Get bootfile error:", err)
+		trainJobErrorNewDataPrepare(ctx, form)
+		ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplModelArtsTrainJobNew, &form)
+		return
+	}
+
 	errStr := checkModelArtsSpecialPool(ctx, flavorCode, string(models.JobTypeTrain))
 	if errStr != "" {
 		trainJobErrorNewDataPrepare(ctx, form)
@@ -1148,9 +1156,9 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 	}
 
 	gitRepo, _ := git.OpenRepository(repo.RepoPath())
-	commitID, _ := gitRepo.GetBranchCommitID(branch_name)
+	commitID, _ := gitRepo.GetBranchCommitID(branchName)
 
-	if err := downloadCode(repo, codeLocalPath, branch_name); err != nil {
+	if err := downloadCode(repo, codeLocalPath, branchName); err != nil {
 		log.Error("downloadCode failed, server timed out: %s (%v)", repo.FullName(), err)
 		trainJobErrorNewDataPrepare(ctx, form)
 		ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tplModelArtsTrainJobNew, &form)
@@ -1292,7 +1300,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm)
 		Parameters:        param,
 		CommitID:          commitID,
 		IsLatestVersion:   isLatestVersion,
-		BranchName:        branch_name,
+		BranchName:        branchName,
 		Params:            form.Params,
 		FlavorName:        FlavorName,
 		EngineName:        EngineName,
@@ -1394,7 +1402,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 	outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath + VersionOutputPath + "/"
 	logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/"
 	// dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
-	branch_name := form.BranchName
+	branchName := form.BranchName
 	PreVersionName := form.VersionName
 	FlavorName := form.FlavorName
 	EngineName := form.EngineName
@@ -1414,6 +1422,14 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 		return
 	}
 
+	bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName)
+	if err != nil || !bootFileExist {
+		log.Error("Get bootfile error:", err)
+		versionErrorDataPrepare(ctx, form)
+		ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplModelArtsTrainJobVersionNew, &form)
+		return
+	}
+
 	errStr := checkModelArtsSpecialPool(ctx, flavorCode, string(models.JobTypeTrain))
 	if errStr != "" {
 		versionErrorDataPrepare(ctx, form)
@@ -1428,8 +1444,8 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 	}
 
 	gitRepo, _ := git.OpenRepository(repo.RepoPath())
-	commitID, _ := gitRepo.GetBranchCommitID(branch_name)
-	if err := downloadCode(repo, codeLocalPath, branch_name); err != nil {
+	commitID, _ := gitRepo.GetBranchCommitID(branchName)
+	if err := downloadCode(repo, codeLocalPath, branchName); err != nil {
 		log.Error("Failed git clone repo to local(！: %s (%v)", repo.FullName(), err)
 		versionErrorDataPrepare(ctx, form)
 		ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tplModelArtsTrainJobVersionNew, &form)
@@ -1582,7 +1598,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ
 		Parameters:        param,
 		PreVersionId:      task.VersionID,
 		CommitID:          commitID,
-		BranchName:        branch_name,
+		BranchName:        branchName,
 		FlavorName:        FlavorName,
 		EngineName:        EngineName,
 		PreVersionName:    PreVersionName,
@@ -2025,7 +2041,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
 	resultObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.ResultPath + VersionOutputPath + "/"
 	logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath + VersionOutputPath + "/"
 	dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/"
-	branch_name := form.BranchName
+	branchName := form.BranchName
 	FlavorName := form.FlavorName
 	EngineName := form.EngineName
 	LabelName := form.LabelName
@@ -2060,6 +2076,14 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
 		return
 	}
 
+	bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName)
+	if err != nil || !bootFileExist {
+		log.Error("Get bootfile error:", err)
+		inferenceJobErrorNewDataPrepare(ctx, form)
+		ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplModelArtsInferenceJobNew, &form)
+		return
+	}
+
 	//Determine whether the task name of the task in the project is duplicated
 	tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeInference), displayJobName)
 	if err == nil {
@@ -2092,9 +2116,9 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
 	}
 
 	gitRepo, _ := git.OpenRepository(repo.RepoPath())
-	commitID, _ := gitRepo.GetBranchCommitID(branch_name)
+	commitID, _ := gitRepo.GetBranchCommitID(branchName)
 
-	if err := downloadCode(repo, codeLocalPath, branch_name); err != nil {
+	if err := downloadCode(repo, codeLocalPath, branchName); err != nil {
 		log.Error("Create task failed, server timed out: %s (%v)", repo.FullName(), err)
 		inferenceJobErrorNewDataPrepare(ctx, form)
 		ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tplModelArtsInferenceJobNew, &form)
@@ -2178,7 +2202,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
 		Uuid:              uuid,
 		Parameters:        param, //modelarts train parameters
 		CommitID:          commitID,
-		BranchName:        branch_name,
+		BranchName:        branchName,
 		Params:            form.Params,
 		FlavorName:        FlavorName,
 		EngineName:        EngineName,

templates/repo/grampus/trainjob/show.tmpl

@@ -773,7 +773,7 @@
                              }
                               if (srcEngine == 'mindspore') {
                                 $('#choice_Engine .default.text').text("MindSpore");
-                                $('#choice_Engine input[name="Engine"]').val(1)
+                                $('#choice_Engine input[name="Engine"]').val(2)
                               }
                         }
                    }else{

templates/repo/modelmanage/index.tmpl

@@ -325,6 +325,7 @@
             $('#name').val(modelName)
             $('#version').val("0.0.1")
         }
+        let dirKey="isOnlyDir--:&";
         function showcreate(obj) {
             $('.ui.modal.second')
                 .modal({
@@ -460,7 +461,7 @@
                     var zNodes=[];
                     var nodesMap={};
                     for (let i=0;i<n_length;i++){
-                        parentNodeMap = nodesMap;
+                        var parentNodeMap = nodesMap;
                         var fileSplits = data[i].FileName.split("/");
                         for(let j=0;j < fileSplits.length;j++){
                             if(fileSplits[j] == ""){
@@ -469,7 +470,22 @@
                             if(parentNodeMap[fileSplits[j]] == null){
                                 parentNodeMap[fileSplits[j]] = {};
                             }
-                            parentNodeMap = parentNodeMap[fileSplits[j]]
+                            parentNodeMap = parentNodeMap[fileSplits[j]];
+                        }
+                    }
+                    for (let i=0;i<n_length;i++){
+                        var parentNodeMap = nodesMap;
+                        var fileSplits = data[i].FileName.split("/");
+                        for(let j=0;j < fileSplits.length;j++){
+                            if(fileSplits[j] == ""){
+                                if(data[i].FileName[data[i].FileName.length -1] =="/"){
+                                    if(Object.keys(parentNodeMap).length ==0){
+                                        parentNodeMap[dirKey]="true";
+                                    }
+                                }
+                                break;
+                            }
+                            parentNodeMap = parentNodeMap[fileSplits[j]];
                         }
                     }
                     convertToNode(zNodes,nodesMap);
@@ -488,12 +504,18 @@
                 node["name"] = keyList[i];
                 nodeList.push(node);
                 if(nodesMap[keyList[i]] != null && Object.keys(nodesMap[keyList[i]]).length >0){
-                    node["children"]=[];
-                    if(isFirst){
-                        node["open"] = true;
-                        isFirst= false;
+                    if(nodesMap[keyList[i]][dirKey] != null){
+                        node["open"] = false;
+                        node["isParent"] = true;
+                    }else{
+                        node["children"]=[];
+                        if(isFirst){
+                            node["open"] = true;
+                            isFirst= false;
+                        }
+                        convertToNode(node["children"],nodesMap[keyList[i]]);
                     }
-                    convertToNode(node["children"],nodesMap[keyList[i]]);
+                    
                 }
             }
         }