From 303db82ac04cbac982f367168025e5a5b2fc7d1a Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 22 Dec 2021 16:20:12 +0800 Subject: [PATCH] update --- models/cloudbrain.go | 8 ++- modules/auth/modelarts.go | 2 + modules/modelarts/modelarts.go | 105 +++++++++++++++++---------------- routers/repo/modelarts.go | 59 ++++++++---------- 4 files changed, 87 insertions(+), 87 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 509f4a9ed..8927147af 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -30,6 +30,7 @@ const ( JobTypeSnn4imagenet JobType = "SNN4IMAGENET" JobTypeBrainScore JobType = "BRAINSCORE" JobTypeTrain JobType = "TRAIN" + JobTypeInference JobType = "INFERENCE" //notebook ModelArtsCreateQueue ModelArtsJobStatus = "CREATE_QUEUING" //免费资源创建排队中 @@ -105,7 +106,7 @@ type Cloudbrain struct { ComputeResource string //计算资源，例如npu EngineID int64 //引擎id - TrainUrl string //输出的obs路径 + TrainUrl string //输出模型的obs路径 BranchName string //分支名称 Parameters string //传给modelarts的param参数 BootFile string //启动文件 @@ -119,6 +120,11 @@ type Cloudbrain struct { EngineName string //引擎名称 TotalVersionCount int //任务的所有版本数量，包括删除的 + ModelName string //模型名称 + ModelVersion string //模型版本 + CkptName string //权重文件名称 + ResultUrl string //推理结果的obs路径 + User *User `xorm:"-"` Repo *Repository `xorm:"-"` } diff --git a/modules/auth/modelarts.go b/modules/auth/modelarts.go index a34c87870..7d727f182 100755 --- a/modules/auth/modelarts.go +++ b/modules/auth/modelarts.go @@ -63,6 +63,8 @@ type CreateModelArtsInferenceJobForm struct { FlavorName string `form:"flaver_names" binding:"Required"` EngineName string `form:"engine_names" binding:"Required"` TrainUrl string `form:"train_url" binding:"Required"` + ModelName string `form:"model_name" binding:"Required"` + ModelVersion string `form:"model_version" binding:"Required"` CkptName string `form:"ckpt_name" binding:"Required"` } diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index c0a003450..1225ea82b 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -46,6 +46,8 @@ const ( Lines = 500 TrainUrl = "train_url" DataUrl = "data_url" + ResultUrl = "result_url" + CkptName = "ckptName" PerPage = 10 IsLatestVersion = "1" NotLatestVersion = "0" @@ -119,30 +121,29 @@ type GenerateTrainJobVersionReq struct { } type GenerateInferenceJobReq struct { - JobName string - Uuid string - Description string - CodeObsPath string - BootFile string - BootFileUrl string - DataUrl string - TrainUrl string - FlavorCode string - LogUrl string - PoolID string - WorkServerNumber int - EngineID int64 - Parameters []models.Parameter - CommitID string - IsLatestVersion string - Params string - BranchName string - PreVersionId int64 - PreVersionName string - FlavorName string - VersionCount int - EngineName string - TotalVersionCount int + JobName string + Uuid string + Description string + CodeObsPath string + BootFile string + BootFileUrl string + DataUrl string + TrainUrl string + FlavorCode string + LogUrl string + PoolID string + WorkServerNumber int + EngineID int64 + Parameters []models.Parameter + CommitID string + Params string + BranchName string + FlavorName string + EngineName string + ModelName string + ModelVersion string + CkptName string + ResultUrl string } type VersionInfo struct { @@ -510,34 +511,34 @@ func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (e } err = models.CreateCloudbrain(&models.Cloudbrain{ - Status: TransTrainJobStatus(jobResult.Status), - UserID: ctx.User.ID, - RepoID: ctx.Repo.Repository.ID, - JobID: strconv.FormatInt(jobResult.JobID, 10), - JobName: req.JobName, - JobType: string(models.JobTypeTrain), - Type: models.TypeCloudBrainTwo, - VersionID: jobResult.VersionID, - VersionName: jobResult.VersionName, - Uuid: req.Uuid, - DatasetName: attach.Name, - CommitID: req.CommitID, - IsLatestVersion: req.IsLatestVersion, - ComputeResource: NPUResource, - EngineID: req.EngineID, - TrainUrl: req.TrainUrl, - BranchName: req.BranchName, - Parameters: req.Params, - BootFile: req.BootFile, - DataUrl: req.DataUrl, - LogUrl: req.LogUrl, - FlavorCode: req.FlavorCode, - Description: req.Description, - WorkServerNumber: req.WorkServerNumber, - FlavorName: req.FlavorName, - EngineName: req.EngineName, - VersionCount: req.VersionCount, - TotalVersionCount: req.TotalVersionCount, + Status: TransTrainJobStatus(jobResult.Status), + UserID: ctx.User.ID, + RepoID: ctx.Repo.Repository.ID, + JobID: strconv.FormatInt(jobResult.JobID, 10), + JobName: req.JobName, + JobType: string(models.JobTypeInference), + Type: models.TypeCloudBrainTwo, + VersionID: jobResult.VersionID, + VersionName: jobResult.VersionName, + Uuid: req.Uuid, + DatasetName: attach.Name, + CommitID: req.CommitID, + EngineID: req.EngineID, + TrainUrl: req.TrainUrl, + BranchName: req.BranchName, + Parameters: req.Params, + BootFile: req.BootFile, + DataUrl: req.DataUrl, + LogUrl: req.LogUrl, + FlavorCode: req.FlavorCode, + Description: req.Description, + WorkServerNumber: req.WorkServerNumber, + FlavorName: req.FlavorName, + EngineName: req.EngineName, + ModelName: req.ModelName, + ModelVersion: req.ModelVersion, + CkptName: req.CkptName, + ResultUrl: req.ResultUrl, }) if err != nil { diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 6dbe6740b..0ae433654 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1259,7 +1259,7 @@ func TrainJobShow(ctx *context.Context) { ctx.Data["canNewJob"] = canNewJob //将运行参数转化为epoch_size = 3, device_target = Ascend的格式 - for i, task := range VersionListTasks { + for i, _ := range VersionListTasks { var parameters models.Parameters @@ -1280,9 +1280,6 @@ func TrainJobShow(ctx *context.Context) { } else { VersionListTasks[i].Parameters = "" } - - VersionListTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) - VersionListTasks[i].CanModify = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain) } pager := context.NewPagination(VersionListCount, setting.UI.IssuePagingNum, page, 5) @@ -1518,7 +1515,6 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference flavorCode := form.Flavor params := form.Params poolID := form.PoolID - isSaveParam := form.IsSaveParam repo := ctx.Repo.Repository codeLocalPath := setting.JobPath + jobName + modelarts.CodePath codeObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.CodePath @@ -1526,9 +1522,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference logObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.LogPath dataPath := "/" + setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + uuid + "/" branch_name := form.BranchName - isLatestVersion := modelarts.IsLatestVersion FlavorName := form.FlavorName - VersionCount := modelarts.VersionCount EngineName := form.EngineName trainUrl := form.TrainUrl ckptName := form.CkptName @@ -1608,11 +1602,11 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference var parameters models.Parameters param := make([]models.Parameter, 0) param = append(param, models.Parameter{ - Label: modelarts.TrainUrl, - Value: trainUrl, + Label: modelarts.ResultUrl, + Value: "s3:/" + resultObsPath, }, models.Parameter{ - Label: modelarts.DataUrl, - Value: dataPath, + Label: modelarts.CkptName, + Value: ckptName, }) if len(params) != 0 { err := json.Unmarshal([]byte(params), ¶meters) @@ -1634,28 +1628,25 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference } req := &modelarts.GenerateInferenceJobReq{ - JobName: jobName, - DataUrl: dataPath, - Description: description, - CodeObsPath: codeObsPath, - BootFileUrl: codeObsPath + bootFile, - BootFile: bootFile, - TrainUrl: trainUrl, - FlavorCode: flavorCode, - WorkServerNumber: workServerNumber, - EngineID: int64(engineID), - LogUrl: logObsPath, - PoolID: poolID, - Uuid: uuid, - Parameters: parameters.Parameter, - CommitID: commitID, - IsLatestVersion: isLatestVersion, - BranchName: branch_name, - Params: form.Params, - FlavorName: FlavorName, - EngineName: EngineName, - VersionCount: VersionCount, - TotalVersionCount: modelarts.TotalVersionCount, + JobName: jobName, + DataUrl: dataPath, + Description: description, + CodeObsPath: codeObsPath, + BootFileUrl: codeObsPath + bootFile, + BootFile: bootFile, + TrainUrl: trainUrl, + FlavorCode: flavorCode, + WorkServerNumber: workServerNumber, + EngineID: int64(engineID), + LogUrl: logObsPath, + PoolID: poolID, + Uuid: uuid, + Parameters: param, //modelarts训练时用到 + CommitID: commitID, + BranchName: branch_name, + Params: form.Params, + FlavorName: FlavorName, + EngineName: EngineName, } //将params转换Parameters.Parameter，出错时返回给前端 @@ -1665,7 +1656,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference return } - err = modelarts.GenerateTrainJob(ctx, req) + err = modelarts.GenerateInferenceJob(ctx, req) if err != nil { log.Error("GenerateTrainJob failed:%v", err.Error()) inferenceJobErrorNewDataPrepare(ctx, form)