diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 24be7b989..cb0fb3421 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -175,26 +175,26 @@ type Cloudbrain struct { ImageID string //grampus image_id AiCenter string //grampus ai center: center_id+center_name - TrainUrl string //输出模型的obs路径 - BranchName string //分支名称 - Parameters string //传给modelarts的param参数 - BootFile string //启动文件 - DataUrl string //数据集的obs路径 - LogUrl string //日志输出的obs路径 - PreVersionId int64 //父版本的版本id - FlavorCode string //modelarts上的规格id - Description string `xorm:"varchar(256)"` //描述 - WorkServerNumber int //节点数 - FlavorName string //规格名称 - EngineName string //引擎名称 - TotalVersionCount int //任务的所有版本数量，包括删除的 - LabelName string //标签名称 - ModelName string //模型名称 - ModelVersion string //模型版本 - CkptName string //权重文件名称 - PreTrainModelUrl string //预训练模型地址 - ResultUrl string //推理结果的obs路径 - + TrainUrl string //输出模型的obs路径 + BranchName string //分支名称 + Parameters string //传给modelarts的param参数 + BootFile string //启动文件 + DataUrl string //数据集的obs路径 + LogUrl string //日志输出的obs路径 + PreVersionId int64 //父版本的版本id + FlavorCode string //modelarts上的规格id + Description string `xorm:"varchar(256)"` //描述 + WorkServerNumber int //节点数 + FlavorName string //规格名称 + EngineName string //引擎名称 + TotalVersionCount int //任务的所有版本数量，包括删除的 + LabelName string //标签名称 + ModelName string //模型名称 + ModelVersion string //模型版本 + CkptName string //权重文件名称 + PreTrainModelUrl string //预训练模型地址 + ResultUrl string //推理结果的obs路径 + ResultJson string `xorm:"varchar(4000)"` User *User `xorm:"-"` Repo *Repository `xorm:"-"` BenchmarkType string `xorm:"-"` //算法评测，模型评测 diff --git a/modules/aisafety/resty.go b/modules/aisafety/resty.go index 868adeb91..be6468529 100644 --- a/modules/aisafety/resty.go +++ b/modules/aisafety/resty.go @@ -38,11 +38,13 @@ type ReturnMsg struct { Data ReturnData `json:"data"` Times int64 `json:"times"` } + type ReturnData struct { ID int `json:"id"` No string `json:"no"` StandardJson string `json:"standardJson"` Code int `json:"code"` + Msg string `json:"msg"` Status int `json:"status"` } @@ -223,7 +225,7 @@ func GetAlgorithmList() (map[string]interface{}, error) { func GetTaskStatus(jobID string) (*ReturnMsg, error) { checkSetting() client := getRestyClient() - var taskResult string + var reMap ReturnMsg params := make(map[string]interface{}) params["serialNo"] = jobID @@ -231,11 +233,10 @@ func GetTaskStatus(jobID string) (*ReturnMsg, error) { SetHeader("Content-Type", "application/json"). SetHeader("appId", APPID). SetHeader("sign", createSign(params, KEY)). - SetResult(&taskResult). + SetResult(&reMap). Get(HOST + "/v1/external/eval-standard/query?serialNo=" + jobID) log.Info("url=" + HOST + "/v1/external/eval-standard/query?serialNo=" + jobID) - responseStr := string(res.Body()) log.Info("GetTaskStatus responseStr=" + responseStr + " res code=" + fmt.Sprint(res.StatusCode())) @@ -243,12 +244,6 @@ func GetTaskStatus(jobID string) (*ReturnMsg, error) { log.Info("error =" + err.Error()) return nil, fmt.Errorf("Get task status error: %v", err) } else { - var reMap *ReturnMsg - err = json.Unmarshal(res.Body(), reMap) - if err == nil { - return reMap, nil - } else { - return nil, fmt.Errorf("get error,code not 0") - } + return &reMap, nil } } diff --git a/routers/repo/aisafety.go b/routers/repo/aisafety.go index ab4065916..f845c3918 100644 --- a/routers/repo/aisafety.go +++ b/routers/repo/aisafety.go @@ -28,7 +28,8 @@ import ( ) const ( - tplModelSafetyTestCreate = "repo/modelsafety/new" + tplModelSafetyTestCreateGpu = "repo/modelsafety/newgpu" + tplModelSafetyTestCreateNpu = "repo/modelsafety/newnpu" ) func CloudBrainAiSafetyCreateTest(ctx *context.Context) { @@ -75,19 +76,17 @@ func GetAiSafetyTask(ctx *context.Context) { } func syncAiSafetyTaskStatus(job *models.Cloudbrain) { - if job.Type == models.TypeCloudBrainTwo { - if isTaskNotFinished(job.Status) { + if isTaskNotFinished(job.Status) { + if job.Type == models.TypeCloudBrainTwo { queryTaskStatusFromCloudbrainTwo(job) - } - } else if job.Type == models.TypeCloudBrainOne { - if isTaskNotFinished(job.Status) { + } else if job.Type == models.TypeCloudBrainOne { queryTaskStatusFromCloudbrain(job) + } + } else { + if job.Status == string(models.ModelSafetyTesting) { + queryTaskStatusFromModelSafetyTestServer(job) } else { - if job.Status == string(models.ModelSafetyTesting) { - queryTaskStatusFromModelSafetyTestServer(job) - } else { - log.Info("The job is finished. status=" + job.Status) - } + log.Info("The job is finished. status=" + job.Status) } } } @@ -150,7 +149,28 @@ func sendNPUInferenceResultToTest(job *models.Cloudbrain) { BDName: datasetnames[0], } jsonContent := "" - //TODO + VersionOutputPath := modelarts.GetOutputPathByCount(modelarts.TotalVersionCount) + resultPath := modelarts.JobPath + job.JobName + modelarts.ResultPath + VersionOutputPath + "/result.json" + body, err := storage.ObsDownloadAFile(setting.Bucket, resultPath) + if err != nil { + log.Info("ObsDownloadAFile error." + err.Error() + " resultPath=" + resultPath) + } else { + defer body.Close() + var data []byte + p := make([]byte, 4096) + var readErr error + var readCount int + for { + readCount, readErr = body.Read(p) + if readCount > 0 { + data = append(data, p[:readCount]...) + } + if readErr != nil || readCount == 0 { + break + } + } + jsonContent = string(data) + } if jsonContent != "" { serialNo, err := aisafety.CreateSafetyTask(req, jsonContent) @@ -211,8 +231,27 @@ func queryTaskStatusFromModelSafetyTestServer(job *models.Cloudbrain) { result, err := aisafety.GetTaskStatus(job.PreVersionName) if err == nil { if result.Code == "0" { - + if result.Data.Status == 1 { + log.Info("The task is running....") + } else { + if result.Data.Code == 0 { + job.ResultJson = result.Data.StandardJson + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } + } + } + } else { + log.Info("The task is failed.") + job.Status = string(models.JobFailed) + err = models.UpdateJob(job) + if err != nil { + log.Error("UpdateJob failed:", err) + } } + } else { + log.Info("The task not found.....") } } @@ -309,13 +348,46 @@ func DelAiSafetyTask(ctx *context.Context) { } -func AiSafetyCreateForGet(ctx *context.Context) { +func AiSafetyCreateForGetGPU(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["IsCreate"] = true + ctx.Data["BaseDataSetName"] = setting.ModelSafetyTest.BaseDataSetName + ctx.Data["BaseDataSetUUID"] = setting.ModelSafetyTest.BaseDataSetUUID + ctx.Data["CombatDataSetName"] = setting.ModelSafetyTest.CombatDataSetName + ctx.Data["CombatDataSetUUID"] = setting.ModelSafetyTest.CombatDataSetUUID + prepareCloudbrainOneSpecs(ctx) + ctx.HTML(200, tplModelSafetyTestCreateGpu) +} + +func AiSafetyCreateForGetNPU(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true + ctx.Data["IsCreate"] = true ctx.Data["BaseDataSetName"] = setting.ModelSafetyTest.BaseDataSetName ctx.Data["BaseDataSetUUID"] = setting.ModelSafetyTest.BaseDataSetUUID ctx.Data["CombatDataSetName"] = setting.ModelSafetyTest.CombatDataSetName ctx.Data["CombatDataSetUUID"] = setting.ModelSafetyTest.CombatDataSetUUID - ctx.HTML(200, tplModelSafetyTestCreate) + + var resourcePools modelarts.ResourcePool + if err := json.Unmarshal([]byte(setting.ResourcePools), &resourcePools); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + } + ctx.Data["resource_pools"] = resourcePools.Info + + var engines modelarts.Engine + if err := json.Unmarshal([]byte(setting.Engines), &engines); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + } + ctx.Data["engines"] = engines.Info + + var versionInfos modelarts.VersionInfo + if err := json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil { + ctx.ServerError("json.Unmarshal failed:", err) + } + ctx.Data["engine_versions"] = versionInfos.Version + + prepareCloudbrainTwoInferenceSpecs(ctx) + + ctx.HTML(200, tplModelSafetyTestCreateNpu) } func AiSafetyCreateForPost(ctx *context.Context) { @@ -329,26 +401,31 @@ func AiSafetyCreateForPost(ctx *context.Context) { repo := ctx.Repo.Repository + tpname := tplCloudBrainModelSafetyNewNpu + if taskType == models.TypeCloudBrainOne { + tpname = tplCloudBrainModelSafetyNewGpu + } + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeModelSafety), displayJobName) if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr("the job name did already exist", tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr("the job name did already exist", tpname, nil) return } } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr("system error", tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr("system error", tpname, nil) return } } if !jobNamePattern.MatchString(jobName) { modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpname, nil) return } @@ -356,13 +433,13 @@ func AiSafetyCreateForPost(ctx *context.Context) { if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr("system error", tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr("system error", tpname, nil) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpname, nil) return } } @@ -371,7 +448,7 @@ func AiSafetyCreateForPost(ctx *context.Context) { if err != nil || !bootFileExist { log.Error("Get bootfile error:", err) modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tplModelArtsInferenceJobNew, nil) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpname, nil) return } @@ -424,13 +501,13 @@ func createForNPU(ctx *context.Context, jobName string) { AiCenterCode: models.AICenterOfCloudBrainTwo}) if err != nil || spec == nil { modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr("Resource specification not available", tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr("Resource specification not available", tplCloudBrainModelSafetyNewNpu, nil) return } if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { log.Error("point balance is not enough,userId=%d specId=%d ", ctx.User.ID, spec.ID) modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr(ctx.Tr("points.insufficient_points_balance"), tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr(ctx.Tr("points.insufficient_points_balance"), tplCloudBrainModelSafetyNewNpu, nil) return } @@ -446,7 +523,7 @@ func createForNPU(ctx *context.Context, jobName string) { if err := downloadCode(repo, codeLocalPath, cloudbrain.DefaultBranchName); err != nil { log.Error("Create task failed, server timed out: %s (%v)", repo.FullName(), err) modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tplCloudBrainModelSafetyNewNpu, nil) return } @@ -454,21 +531,21 @@ func createForNPU(ctx *context.Context, jobName string) { if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.ResultPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_result: %s (%v)", repo.FullName(), err) modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr("Failed to obsMkdir_result", tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr("Failed to obsMkdir_result", tplCloudBrainModelSafetyNewNpu, nil) return } if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err) modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr("Failed to obsMkdir_log", tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr("Failed to obsMkdir_log", tplCloudBrainModelSafetyNewNpu, nil) return } if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr(ctx.Tr("cloudbrain.load_code_failed"), tplCloudBrainModelSafetyNewNpu, nil) return } @@ -485,7 +562,7 @@ func createForNPU(ctx *context.Context, jobName string) { datasUrlList, dataUrl, datasetNames, isMultiDataset, err := getDatasUrlListByUUIDS(uuid) if err != nil { modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr(err.Error(), tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr(err.Error(), tplCloudBrainModelSafetyNewNpu, nil) return } dataPath := dataUrl @@ -493,7 +570,7 @@ func createForNPU(ctx *context.Context, jobName string) { if err != nil { log.Error("Failed to Marshal: %v", err) modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr("json error:"+err.Error(), tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr("json error:"+err.Error(), tplCloudBrainModelSafetyNewNpu, nil) return } if isMultiDataset { @@ -509,7 +586,7 @@ func createForNPU(ctx *context.Context, jobName string) { if err != nil { log.Error("Failed to Unmarshal params: %s (%v)", Params, err) modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr("运行参数错误", tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr("运行参数错误", tplCloudBrainModelSafetyNewNpu, nil) return } @@ -569,7 +646,7 @@ func createForNPU(ctx *context.Context, jobName string) { if err != nil { log.Error("GenerateTrainJob failed:%v", err.Error()) modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr(err.Error(), tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr(err.Error(), tplCloudBrainModelSafetyNewNpu, nil) return } } @@ -595,7 +672,7 @@ func createForGPU(ctx *context.Context, jobName string) { AiCenterCode: models.AICenterOfCloudBrainOne}) if err != nil || spec == nil { modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr("Resource specification not available", tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr("Resource specification not available", tplCloudBrainModelSafetyNewGpu, nil) return } @@ -606,7 +683,7 @@ func createForGPU(ctx *context.Context, jobName string) { if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil { log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"]) modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr("system error", tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr("system error", tplCloudBrainModelSafetyNewGpu, nil) return } @@ -614,7 +691,7 @@ func createForGPU(ctx *context.Context, jobName string) { if err != nil { log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"]) modelSafetyNewDataPrepare(ctx) - ctx.RenderWithErr("system error", tplCloudBrainModelSafetyNew, nil) + ctx.RenderWithErr("system error", tplCloudBrainModelSafetyNewGpu, nil) return } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index f24d33cd0..1f6732174 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -51,7 +51,8 @@ const ( tplCloudBrainBenchmarkNew base.TplName = "repo/cloudbrain/benchmark/new" tplCloudBrainBenchmarkShow base.TplName = "repo/cloudbrain/benchmark/show" - tplCloudBrainModelSafetyNew base.TplName = "repo/cloudbrain/modelsafety/new" + tplCloudBrainModelSafetyNewGpu base.TplName = "repo/cloudbrain/modelsafety/newgpu" + tplCloudBrainModelSafetyNewNpu base.TplName = "repo/cloudbrain/modelsafety/newnpu" tplCloudBrainImageSubmit base.TplName = "repo/cloudbrain/image/submit" tplCloudBrainImageEdit base.TplName = "repo/cloudbrain/image/edit" diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 09f6a1b1c..e249fafbe 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -1233,7 +1233,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.StopAiSafetyTask) m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.DelAiSafetyTask) }) - m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, repo.AiSafetyCreateForGet) + m.Get("/create_gpu", reqWechatBind, reqRepoCloudBrainWriter, repo.AiSafetyCreateForGetGPU) + m.Get("/create_npu", reqWechatBind, reqRepoCloudBrainWriter, repo.AiSafetyCreateForGetNPU) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, repo.AiSafetyCreateForPost) }, context.RepoRef())