|
|
|
@@ -2,8 +2,10 @@ package repo |
|
|
|
|
|
|
|
import ( |
|
|
|
"code.gitea.io/gitea/modules/git" |
|
|
|
"code.gitea.io/gitea/modules/storage" |
|
|
|
"encoding/json" |
|
|
|
"errors" |
|
|
|
"net/http" |
|
|
|
"os" |
|
|
|
"os/exec" |
|
|
|
"strconv" |
|
|
|
@@ -23,6 +25,12 @@ const ( |
|
|
|
tplCloudBrainIndex base.TplName = "repo/cloudbrain/index" |
|
|
|
tplCloudBrainNew base.TplName = "repo/cloudbrain/new" |
|
|
|
tplCloudBrainShow base.TplName = "repo/cloudbrain/show" |
|
|
|
tplCloudBrainShowModels base.TplName = "repo/cloudbrain/models/index" |
|
|
|
) |
|
|
|
|
|
|
|
var ( |
|
|
|
gpuInfos *models.GpuInfos |
|
|
|
categories *models.Categories |
|
|
|
) |
|
|
|
|
|
|
|
// MustEnableDataset check if repository enable internal cb |
|
|
|
@@ -88,7 +96,7 @@ func CloudBrainNew(ctx *context.Context) { |
|
|
|
result, err := cloudbrain.GetImages() |
|
|
|
if err != nil { |
|
|
|
ctx.Data["error"] = err.Error() |
|
|
|
log.Error("cloudbrain.GetImages failed:", err.Error()) |
|
|
|
log.Error("cloudbrain.GetImages failed:", err.Error(), ctx.Data["msgID"]) |
|
|
|
} |
|
|
|
|
|
|
|
for i, payload := range result.Payload.ImageInfo { |
|
|
|
@@ -104,7 +112,7 @@ func CloudBrainNew(ctx *context.Context) { |
|
|
|
resultPublic, err := cloudbrain.GetPublicImages() |
|
|
|
if err != nil { |
|
|
|
ctx.Data["error"] = err.Error() |
|
|
|
log.Error("cloudbrain.GetPublicImages failed:", err.Error()) |
|
|
|
log.Error("cloudbrain.GetPublicImages failed:", err.Error(), ctx.Data["msgID"]) |
|
|
|
} |
|
|
|
|
|
|
|
for i, payload := range resultPublic.Payload.ImageInfo { |
|
|
|
@@ -131,9 +139,15 @@ func CloudBrainNew(ctx *context.Context) { |
|
|
|
ctx.Data["benchmark_path"] = cloudbrain.BenchMarkMountPath |
|
|
|
ctx.Data["is_benchmark_enabled"] = setting.IsBenchmarkEnabled |
|
|
|
|
|
|
|
var categories *models.Categories |
|
|
|
json.Unmarshal([]byte(setting.BenchmarkCategory), &categories) |
|
|
|
if categories == nil { |
|
|
|
json.Unmarshal([]byte(setting.BenchmarkCategory), &categories) |
|
|
|
} |
|
|
|
ctx.Data["benchmark_categories"] = categories.Category |
|
|
|
|
|
|
|
if gpuInfos == nil { |
|
|
|
json.Unmarshal([]byte(setting.GpuTypes), &gpuInfos) |
|
|
|
} |
|
|
|
ctx.Data["gpu_types"] = gpuInfos.GpuInfo |
|
|
|
ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath |
|
|
|
ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled |
|
|
|
ctx.HTML(200, tplCloudBrainNew) |
|
|
|
@@ -146,10 +160,11 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { |
|
|
|
command := form.Command |
|
|
|
uuid := form.Attachment |
|
|
|
jobType := form.JobType |
|
|
|
gpuQueue := setting.JobType |
|
|
|
codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath |
|
|
|
|
|
|
|
if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) { |
|
|
|
log.Error("jobtype error:", jobType) |
|
|
|
log.Error("jobtype error:", jobType, ctx.Data["msgID"]) |
|
|
|
ctx.RenderWithErr("jobtype error", tplCloudBrainNew, &form) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -165,15 +180,22 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { |
|
|
|
|
|
|
|
benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath |
|
|
|
if setting.IsBenchmarkEnabled && jobType == string(models.JobTypeBenchmark) { |
|
|
|
downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory) |
|
|
|
gpuQueue = form.GpuType |
|
|
|
var gpuType string |
|
|
|
for _, gpuInfo := range gpuInfos.GpuInfo { |
|
|
|
if gpuInfo.Queue == gpuQueue { |
|
|
|
gpuType = gpuInfo.Value |
|
|
|
} |
|
|
|
} |
|
|
|
downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType) |
|
|
|
} |
|
|
|
|
|
|
|
snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath |
|
|
|
if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { |
|
|
|
downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "") |
|
|
|
downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "") |
|
|
|
} |
|
|
|
|
|
|
|
err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, jobType) |
|
|
|
err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, jobType, gpuQueue) |
|
|
|
if err != nil { |
|
|
|
ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) |
|
|
|
return |
|
|
|
@@ -245,7 +267,7 @@ func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrain |
|
|
|
ImageTag: form.Tag, |
|
|
|
}) |
|
|
|
if err != nil { |
|
|
|
log.Error("CommitImage(%s) failed:", task.JobName, err.Error()) |
|
|
|
log.Error("CommitImage(%s) failed:%v", task.JobName, err.Error(), ctx.Data["msgID"]) |
|
|
|
ctx.JSON(200, map[string]string{ |
|
|
|
"result_code": "-1", |
|
|
|
"error_msg": "CommitImage failed", |
|
|
|
@@ -268,14 +290,14 @@ func CloudBrainStop(ctx *context.Context) { |
|
|
|
} |
|
|
|
|
|
|
|
if task.Status == string(models.JobStopped) { |
|
|
|
log.Error("the job(%s) has been stopped", task.JobName) |
|
|
|
log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"]) |
|
|
|
ctx.ServerError("the job has been stopped", errors.New("the job has been stopped")) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
err = cloudbrain.StopJob(jobID) |
|
|
|
if err != nil { |
|
|
|
log.Error("StopJob(%s) failed:%v", task.JobName, err.Error()) |
|
|
|
log.Error("StopJob(%s) failed:%v", task.JobName, err.Error(), ctx.Data["msgID"]) |
|
|
|
ctx.ServerError("StopJob failed", err) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -299,7 +321,7 @@ func CloudBrainDel(ctx *context.Context) { |
|
|
|
} |
|
|
|
|
|
|
|
if task.Status != string(models.JobStopped) { |
|
|
|
log.Error("the job(%s) has not been stopped", task.JobName) |
|
|
|
log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"]) |
|
|
|
ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped")) |
|
|
|
return |
|
|
|
} |
|
|
|
@@ -313,6 +335,69 @@ func CloudBrainDel(ctx *context.Context) { |
|
|
|
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain") |
|
|
|
} |
|
|
|
|
|
|
|
func CloudBrainShowModels(ctx *context.Context) { |
|
|
|
ctx.Data["PageIsCloudBrain"] = true |
|
|
|
|
|
|
|
jobID := ctx.Params(":jobid") |
|
|
|
parentDir := ctx.Query("parentDir") |
|
|
|
dirArray := strings.Split(parentDir, "/") |
|
|
|
task, err := models.GetCloudbrainByJobID(jobID) |
|
|
|
if err != nil { |
|
|
|
log.Error("no such job!", ctx.Data["msgID"]) |
|
|
|
ctx.ServerError("no such job:", err) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
//get dirs |
|
|
|
dirs, err := getModelDirs(task.JobName, parentDir) |
|
|
|
if err != nil { |
|
|
|
log.Error("getModelDirs failed:%v", err.Error(), ctx.Data["msgID"]) |
|
|
|
ctx.ServerError("getModelDirs failed:", err) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
var fileInfos []FileInfo |
|
|
|
err = json.Unmarshal([]byte(dirs), &fileInfos) |
|
|
|
if err != nil { |
|
|
|
log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"]) |
|
|
|
ctx.ServerError("json.Unmarshal failed:", err) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
ctx.Data["Path"] = dirArray |
|
|
|
ctx.Data["Dirs"] = fileInfos |
|
|
|
ctx.Data["task"] = task |
|
|
|
ctx.Data["JobID"] = jobID |
|
|
|
ctx.HTML(200, tplCloudBrainShowModels) |
|
|
|
} |
|
|
|
|
|
|
|
func getModelDirs(jobName string, parentDir string) (string, error) { |
|
|
|
var req string |
|
|
|
modelActualPath := setting.JobPath + jobName + "/model/" |
|
|
|
if parentDir == "" { |
|
|
|
req = "baseDir=" + modelActualPath |
|
|
|
} else { |
|
|
|
req = "baseDir=" + modelActualPath + "&parentDir=" + parentDir |
|
|
|
} |
|
|
|
|
|
|
|
return getDirs(req) |
|
|
|
} |
|
|
|
|
|
|
|
func CloudBrainDownloadModel(ctx *context.Context) { |
|
|
|
parentDir := ctx.Query("parentDir") |
|
|
|
fileName := ctx.Query("fileName") |
|
|
|
jobName := ctx.Query("jobName") |
|
|
|
filePath := "jobs/" +jobName + "/model/" + parentDir |
|
|
|
url, err := storage.Attachments.PresignedGetURL(filePath, fileName) |
|
|
|
if err != nil { |
|
|
|
log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"]) |
|
|
|
ctx.ServerError("PresignedGetURL", err) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) |
|
|
|
} |
|
|
|
|
|
|
|
func GetRate(ctx *context.Context) { |
|
|
|
var jobID = ctx.Params(":jobid") |
|
|
|
job, err := models.GetCloudbrainByJobID(jobID) |
|
|
|
@@ -322,11 +407,11 @@ func GetRate(ctx *context.Context) { |
|
|
|
} |
|
|
|
|
|
|
|
if job.JobType == string(models.JobTypeBenchmark) { |
|
|
|
ctx.Redirect(setting.BenchmarkServerHost) |
|
|
|
ctx.Redirect(setting.BenchmarkServerHost + "?username=" + ctx.User.Name) |
|
|
|
} else if job.JobType == string(models.JobTypeSnn4imagenet) { |
|
|
|
ctx.Redirect(setting.Snn4imagenetServerHost) |
|
|
|
} else { |
|
|
|
log.Error("JobType error:", job.JobType) |
|
|
|
log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"]) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
@@ -339,7 +424,7 @@ func downloadCode(repo *models.Repository, codePath string) error { |
|
|
|
return nil |
|
|
|
} |
|
|
|
|
|
|
|
func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benchmarkCategory string) error { |
|
|
|
func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benchmarkCategory, gpuType string) error { |
|
|
|
err := os.MkdirAll(codePath, os.ModePerm) |
|
|
|
if err != nil { |
|
|
|
log.Error("mkdir codePath failed", err.Error()) |
|
|
|
@@ -370,6 +455,7 @@ func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benc |
|
|
|
CodeName: repo.Name, |
|
|
|
BenchmarkCategory: strings.Split(benchmarkCategory, ","), |
|
|
|
CodeLink: strings.TrimSuffix(repo.CloneLink().HTTPS, ".git"), |
|
|
|
GpuType: gpuType, |
|
|
|
}) |
|
|
|
if err != nil { |
|
|
|
log.Error("json.Marshal failed", err.Error()) |
|
|
|
|
Gitea
4 years ago
