|
|
|
@@ -4,13 +4,21 @@ import ( |
|
|
|
"encoding/json" |
|
|
|
"io/ioutil" |
|
|
|
"net/http" |
|
|
|
"os" |
|
|
|
"strings" |
|
|
|
|
|
|
|
"code.gitea.io/gitea/models" |
|
|
|
"code.gitea.io/gitea/modules/aisafety" |
|
|
|
"code.gitea.io/gitea/modules/cloudbrain" |
|
|
|
"code.gitea.io/gitea/modules/context" |
|
|
|
"code.gitea.io/gitea/modules/log" |
|
|
|
"code.gitea.io/gitea/modules/setting" |
|
|
|
"code.gitea.io/gitea/modules/storage" |
|
|
|
"code.gitea.io/gitea/modules/util" |
|
|
|
"code.gitea.io/gitea/services/cloudbrain/resource" |
|
|
|
) |
|
|
|
|
|
|
|
func CloudBrainAiSafetyCreate(ctx *context.Context) { |
|
|
|
func CloudBrainAiSafetyCreateTest(ctx *context.Context) { |
|
|
|
log.Info("start to create CloudBrainAiSafetyCreate") |
|
|
|
//jobType := ctx.Query("jobType") |
|
|
|
//if jobType == string(models.JobTypeBenchmark) { |
|
|
|
@@ -23,6 +31,143 @@ func CloudBrainAiSafetyCreate(ctx *context.Context) { |
|
|
|
//} |
|
|
|
} |
|
|
|
|
|
|
|
func CloudBrainAiSafetyCreate(ctx *context.Context) { |
|
|
|
ctx.Data["PageIsCloudBrain"] = true |
|
|
|
displayJobName := ctx.Query("DisplayJobName") |
|
|
|
jobName := util.ConvertDisplayJobNameToJobName(displayJobName) |
|
|
|
image := strings.TrimSpace(ctx.Query("Image")) |
|
|
|
command := "python /code/inferench.py > " + cloudbrain.ModelMountPath + "/" + displayJobName + "-" + cloudbrain.LogFile |
|
|
|
codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath |
|
|
|
description := ctx.Query("Description") |
|
|
|
specId := ctx.QueryInt64("SpecId") |
|
|
|
ctx.Data["description"] = description |
|
|
|
|
|
|
|
repo := ctx.Repo.Repository |
|
|
|
|
|
|
|
tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeBenchmark), displayJobName) |
|
|
|
if err == nil { |
|
|
|
if len(tasks) != 0 { |
|
|
|
log.Error("the job name did already exist", ctx.Data["MsgID"]) |
|
|
|
cloudBrainNewDataPrepare(ctx) |
|
|
|
ctx.RenderWithErr("the job name did already exist", tplCloudBrainBenchmarkNew, nil) |
|
|
|
return |
|
|
|
} |
|
|
|
} else { |
|
|
|
if !models.IsErrJobNotExist(err) { |
|
|
|
log.Error("system error, %v", err, ctx.Data["MsgID"]) |
|
|
|
cloudBrainNewDataPrepare(ctx) |
|
|
|
ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, nil) |
|
|
|
return |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
if !jobNamePattern.MatchString(jobName) { |
|
|
|
cloudBrainNewDataPrepare(ctx) |
|
|
|
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplCloudBrainBenchmarkNew, nil) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
spec, err := resource.GetAndCheckSpec(ctx.User.ID, specId, models.FindSpecsOptions{ |
|
|
|
JobType: models.JobTypeBenchmark, |
|
|
|
ComputeResource: models.GPU, |
|
|
|
Cluster: models.OpenICluster, |
|
|
|
AiCenterCode: models.AICenterOfCloudBrainOne}) |
|
|
|
if err != nil || spec == nil { |
|
|
|
cloudBrainNewDataPrepare(ctx) |
|
|
|
ctx.RenderWithErr("Resource specification not available", tplCloudBrainBenchmarkNew, nil) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
count, err := models.GetBenchmarkCountByUserID(ctx.User.ID) |
|
|
|
if err != nil { |
|
|
|
log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) |
|
|
|
cloudBrainNewDataPrepare(ctx) |
|
|
|
ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, nil) |
|
|
|
return |
|
|
|
} else { |
|
|
|
if count >= 1 { |
|
|
|
log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) |
|
|
|
cloudBrainNewDataPrepare(ctx) |
|
|
|
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tplCloudBrainBenchmarkNew, nil) |
|
|
|
return |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
os.RemoveAll(codePath) |
|
|
|
|
|
|
|
if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil { |
|
|
|
log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"]) |
|
|
|
cloudBrainNewDataPrepare(ctx) |
|
|
|
ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, nil) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
if _, err := os.Stat(codePath + "/inference.py"); err != nil { |
|
|
|
if os.IsNotExist(err) { |
|
|
|
// file does not exist |
|
|
|
log.Error("inference.py does not exist, %v", err, ctx.Data["MsgID"]) |
|
|
|
cloudBrainNewDataPrepare(ctx) |
|
|
|
ctx.RenderWithErr("inference.py does not exist", tplCloudBrainBenchmarkNew, nil) |
|
|
|
} else { |
|
|
|
log.Error("Stat failed, %v", err, ctx.Data["MsgID"]) |
|
|
|
cloudBrainNewDataPrepare(ctx) |
|
|
|
ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, nil) |
|
|
|
} |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
if err := uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/"); err != nil { |
|
|
|
log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"]) |
|
|
|
cloudBrainNewDataPrepare(ctx) |
|
|
|
ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, nil) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
uuid := "dee79f68-19f1-42dd-b004-bc9ce08415ca" |
|
|
|
datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid) |
|
|
|
if err != nil { |
|
|
|
log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) |
|
|
|
cloudBrainNewDataPrepare(ctx) |
|
|
|
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tplCloudBrainBenchmarkNew, nil) |
|
|
|
return |
|
|
|
} |
|
|
|
log.Info("Command=" + command) |
|
|
|
log.Info("ModelPath=" + storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/")) |
|
|
|
req := cloudbrain.GenerateCloudBrainTaskReq{ |
|
|
|
Ctx: ctx, |
|
|
|
DisplayJobName: displayJobName, |
|
|
|
JobName: jobName, |
|
|
|
Image: image, |
|
|
|
Command: command, |
|
|
|
Uuids: uuid, |
|
|
|
DatasetNames: datasetNames, |
|
|
|
DatasetInfos: datasetInfos, |
|
|
|
CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), |
|
|
|
ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), |
|
|
|
BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), |
|
|
|
Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), |
|
|
|
BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), |
|
|
|
JobType: string(models.JobTypeModelSafety), |
|
|
|
Description: description, |
|
|
|
BranchName: cloudbrain.DefaultBranchName, |
|
|
|
BootFile: "", |
|
|
|
Params: "", |
|
|
|
CommitID: "", |
|
|
|
ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), |
|
|
|
Spec: spec, |
|
|
|
} |
|
|
|
|
|
|
|
err = cloudbrain.GenerateTask(req) |
|
|
|
if err != nil { |
|
|
|
cloudBrainNewDataPrepare(ctx) |
|
|
|
ctx.RenderWithErr(err.Error(), tplCloudBrainBenchmarkNew, nil) |
|
|
|
return |
|
|
|
} |
|
|
|
|
|
|
|
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
func getJsonContent(url string) (map[string]interface{}, error) { |
|
|
|
|
|
|
|
resp, err := http.Get(url) |
|
|
|
|