diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 0ab1bbc88..f7b78235b 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -451,18 +451,21 @@ var ( //benchmark config IsBenchmarkEnabled bool - BenchmarkCode string + BenchmarkOwner string + BenchmarkName string BenchmarkServerHost string BenchmarkCategory string //snn4imagenet config IsSnn4imagenetEnabled bool - Snn4imagenetCode string + Snn4imagenetOwner string + Snn4imagenetName string Snn4imagenetServerHost string //snn4imagenet config IsBrainScoreEnabled bool - BrainScoreCode string + BrainScoreOwner string + BrainScoreName string BrainScoreServerHost string //blockchain config @@ -1238,18 +1241,21 @@ func NewContext() { sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) - BenchmarkCode = sec.Key("BENCHMARKCODE").MustString("") + BenchmarkOwner = sec.Key("OWNER").MustString("") + BenchmarkName = sec.Key("NAME").MustString("") BenchmarkServerHost = sec.Key("HOST").MustString("") BenchmarkCategory = sec.Key("CATEGORY").MustString("") sec = Cfg.Section("snn4imagenet") IsSnn4imagenetEnabled = sec.Key("ENABLED").MustBool(false) - Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("") + Snn4imagenetOwner = sec.Key("OWNER").MustString("") + Snn4imagenetName = sec.Key("NAME").MustString("") Snn4imagenetServerHost = sec.Key("HOST").MustString("") sec = Cfg.Section("brainscore") IsBrainScoreEnabled = sec.Key("ENABLED").MustBool(false) - BrainScoreCode = sec.Key("BRAINSCORECODE").MustString("") + BrainScoreOwner = sec.Key("OWNER").MustString("") + BrainScoreName = sec.Key("NAME").MustString("") BrainScoreServerHost = sec.Key("HOST").MustString("") sec = Cfg.Section("blockchain") diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 07a3a9e8d..5c2f5b5b7 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -8,7 +8,6 @@ import ( "io" "net/http" "os" - "os/exec" "regexp" "sort" "strconv" @@ -251,13 +250,9 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { downloadCode(repo, codePath) uploadCodeToMinio(codePath + "/", jobName, "/code/") - modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath - err = os.MkdirAll(modelPath, os.ModePerm) - if err != nil { - cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) - return - } + modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/" + mkModelPath(modelPath) + uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath + "/") benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath if setting.IsBenchmarkEnabled && jobType == string(models.JobTypeBenchmark) { @@ -267,23 +262,25 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { gpuType = gpuInfo.Value } } - downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType) + downloadRateCode(repo, jobName, setting.BenchmarkOwner, setting.BrainScoreName, benchmarkPath, form.BenchmarkCategory, gpuType) uploadCodeToMinio(benchmarkPath + "/", jobName, cloudbrain.BenchMarkMountPath + "/") } snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { - downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "") + downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "") uploadCodeToMinio(snn4imagenetPath + "/", jobName, cloudbrain.Snn4imagenetMountPath + "/") } brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { - downloadRateCode(repo, jobName, setting.BrainScoreCode, brainScorePath, "", "") + downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "") uploadCodeToMinio(brainScorePath + "/", jobName, cloudbrain.BrainScoreMountPath + "/") } - err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, resourceSpecId) + err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, getMinioPath(jobName, cloudbrain.ModelMountPath + "/"), + getMinioPath(jobName, cloudbrain.BenchMarkMountPath + "/"), getMinioPath(jobName, cloudbrain.Snn4imagenetMountPath + "/"), + getMinioPath(jobName, cloudbrain.BrainScoreMountPath + "/"), jobType, gpuQueue, resourceSpecId) if err != nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) @@ -607,7 +604,7 @@ func getImages(ctx *context.Context, imageType string) { func GetModelDirs(jobName string, parentDir string) (string, error) { var req string - modelActualPath := setting.JobPath + jobName + "/model/" + modelActualPath := getMinioPath(jobName, cloudbrain.ModelMountPath + "/") if parentDir == "" { req = "baseDir=" + modelActualPath } else { @@ -617,6 +614,10 @@ func GetModelDirs(jobName string, parentDir string) (string, error) { return getDirs(req) } +func getMinioPath(jobName, suffixPath string) string { + return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + jobName + suffixPath +} + func CloudBrainDownloadModel(ctx *context.Context) { parentDir := ctx.Query("parentDir") fileName := ctx.Query("fileName") @@ -698,19 +699,21 @@ func downloadCode(repo *models.Repository, codePath string) error { return nil } -func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benchmarkCategory, gpuType string) error { +func downloadRateCode(repo *models.Repository, taskName, rateOwnerName, rateRepoName, codePath, benchmarkCategory, gpuType string) error { err := os.MkdirAll(codePath, os.ModePerm) if err != nil { log.Error("mkdir codePath failed", err.Error()) return err } - command := "git clone " + gitPath + " " + codePath - cmd := exec.Command("/bin/bash", "-c", command) - _, err = cmd.Output() - + repoExt, err := models.GetRepositoryByOwnerAndName(rateOwnerName, rateRepoName) if err != nil { - log.Error("exec.Command(%s) failed:%v", command, err) + log.Error("GetRepositoryByOwnerAndName(%s) failed", rateRepoName, err.Error()) + return err + } + + if err := git.Clone(repoExt.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil { + log.Error("Failed to clone repository: %s (%v)", repoExt.FullName(), err) return err } @@ -772,6 +775,32 @@ func uploadCodeToMinio(codePath, jobName, parentDir string) error { return nil } +func mkModelPath(modelPath string) error { + err := os.MkdirAll(modelPath, os.ModePerm) + if err != nil { + log.Error("MkdirAll(%s) failed:%v", modelPath, err) + return err + } + + fileName := modelPath + "README" + f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) + if err != nil { + log.Error("OpenFile failed", err.Error()) + return err + } + + defer f.Close() + + _, err = f.WriteString("You can put the model file into this directory and download it by the web page.") + if err != nil { + log.Error("WriteString failed", err.Error()) + return err + } + + return nil +} + + func SyncCloudbrainStatus() { cloudBrains, err := models.GetCloudBrainUnStoppedJob() if err != nil {