diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 59cff4238..63baa910f 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -237,13 +237,13 @@ func TransTrainJobStatus(status int) string { case 8: return "RUNNING" case 9: - return "STOPED" + return "STOPPED" case 10: return "COMPLETED" case 11: return "FAILED" case 12: - return "STOPED" + return "STOPPED" case 13: return "CANCELED" case 14: diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 2e512eae0..05d8c94b7 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -473,6 +473,8 @@ var ( Bucket string Location string BasePath string + JobModelPath string + OutPutPath string CodePathPrefix string UserBasePath string @@ -488,11 +490,12 @@ var ( ProfileID string PoolInfos string Flavor string + FlavorInfos string //train-job - ResourcePools string - Engines string - EngineVersions string - FlavorInfos string + ResourcePools string + Engines string + EngineVersions string + TrainJobFlavorInfos string //elk config ElkUrl string @@ -1252,6 +1255,8 @@ func NewContext() { Bucket = sec.Key("BUCKET").MustString("testopendata") Location = sec.Key("LOCATION").MustString("cn-south-222") BasePath = sec.Key("BASE_PATH").MustString("attachment/") + JobModelPath = sec.Key("Job_Model_PATH").MustString("job/") + OutPutPath = sec.Key("OutPut_PATH").MustString("output/") CodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("code/") UserBasePath = sec.Key("BASE_PATH_USER").MustString("users/") PROXYURL = sec.Key("PROXY_URL").MustString("") @@ -1272,6 +1277,7 @@ func NewContext() { Engines = sec.Key("Engines").MustString("") EngineVersions = sec.Key("Engine_Versions").MustString("") FlavorInfos = sec.Key("FLAVOR_INFOS").MustString("") + TrainJobFlavorInfos = sec.Key("TrainJob_FLAVOR_INFOS").MustString("") sec = Cfg.Section("elk") ElkUrl = sec.Key("ELKURL").MustString("http://192.168.207.35:5601/internal/bsearch") diff --git a/modules/storage/obs.go b/modules/storage/obs.go index d5817fd3f..f622bfd93 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -134,6 +134,7 @@ func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { input := &obs.GetObjectInput{} input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") + // input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") output, err := ObsCli.GetObject(input) if err == nil { log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n", @@ -148,6 +149,54 @@ func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { } } +func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { + input := &obs.GetObjectInput{} + input.Bucket = setting.Bucket + input.Key = strings.TrimPrefix(path.Join(setting.JobModelPath, JobName, setting.OutPutPath, fileName), "/") + // input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") + output, err := ObsCli.GetObject(input) + if err == nil { + log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n", + output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified) + return output.Body, nil + } else if obsError, ok := err.(obs.ObsError); ok { + fmt.Printf("Code:%s\n", obsError.Code) + fmt.Printf("Message:%s\n", obsError.Message) + return nil, obsError + } else { + return nil, err + } +} + +func GetObsListObject(jobName string) ([]string, error) { + // jobName = "liuzx202110271830856" + input := &obs.ListObjectsInput{} + input.Bucket = setting.Bucket + input.Prefix = strings.TrimPrefix(path.Join(setting.JobModelPath, jobName, setting.OutPutPath), "/") + log.Info("input.Prefix:", input.Prefix) + output, err := ObsCli.ListObjects(input) + ModelListArr := make([]string, 0) + if err == nil { + fmt.Printf("RequestId:%s\n", output.RequestId) + for index, val := range output.Contents { + fmt.Printf("Content[%d]-OwnerId:%s, ETag:%s, Key:%s, LastModified:%s, Size:%d\n", + index, val.Owner.ID, val.ETag, val.Key, val.LastModified, val.Size) + str1 := strings.Split(val.Key, "/") + ModelList := str1[len(str1)-1] + ModelListArr = append(ModelListArr, ModelList) + } + return ModelListArr, err + } else { + if obsError, ok := err.(obs.ObsError); ok { + fmt.Println(obsError.Code) + fmt.Println(obsError.Message) + } else { + fmt.Println(err) + } + return nil, err + } +} + func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) { input := &obs.CreateSignedUrlInput{} diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index d6f78a80e..42d0bffbe 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -580,7 +580,6 @@ func CloudBrainDownloadModel(ctx *context.Context) { ctx.ServerError("PresignedGetURL", err) return } - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } @@ -599,11 +598,34 @@ func CloudBrainDownloadModel(ctx *context.Context) { // http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) // } -func TrainJobloadModel(ctx *context.Context) { - uuid := ctx.Query("uuid") +func TrainJobListModel(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + jobID := ctx.Params(":jobid") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + log.Error("no such job!", ctx.Data["err"]) + ctx.ServerError("no such job:", err) + return + } + TrainJobListModel, err := storage.GetObsListObject(task.JobName) + if err != nil { + log.Info("get TrainJobListModel failed:", err) + return + } + ctx.Data["task"] = task + ctx.Data["JobID"] = jobID + ctx.Data["TrainJobListModel"] = TrainJobListModel + ctx.HTML(200, tplModelArtsTrainJobListModel) +} + +func TrainJobDownloadModel(ctx *context.Context) { + JobName := ctx.Query("JobName") fileName := ctx.Query("file_name") - body, err := storage.ObsDownload(uuid, fileName) + // JobName = "liuzx202110271830856" + // fileName = "Untitled.ipynb" + + body, err := storage.ObsModelDownload(JobName, fileName) if err != nil { log.Info("download error.") } else { diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 414550800..a6392ff38 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -36,9 +36,10 @@ const ( tplModelArtsNew base.TplName = "repo/modelarts/new" tplModelArtsShow base.TplName = "repo/modelarts/show" - tplModelArtsTrainJobIndex base.TplName = "repo/modelarts/trainjob/index" - tplModelArtsTrainJobNew base.TplName = "repo/modelarts/trainjob/new" - tplModelArtsTrainJobShow base.TplName = "repo/modelarts/trainjob/show" + tplModelArtsTrainJobIndex base.TplName = "repo/modelarts/trainjob/index" + tplModelArtsTrainJobNew base.TplName = "repo/modelarts/trainjob/new" + tplModelArtsTrainJobShow base.TplName = "repo/modelarts/trainjob/show" + tplModelArtsTrainJobListModel base.TplName = "repo/modelarts/trainjob/list_model" ) // MustEnableDataset check if repository enable internal cb @@ -331,6 +332,7 @@ func NotebookNew(ctx *context.Context) { json.Unmarshal([]byte(setting.FlavorInfos), &modelarts.FlavorInfos) } ctx.Data["flavors"] = modelarts.FlavorInfos.FlavorInfo + ctx.HTML(200, tplModelArtsNotebookNew) } @@ -610,7 +612,7 @@ func TrainJobNew(ctx *context.Context) { ctx.Data["engine_versions"] = versionInfos.Version var flavorInfos modelarts.Flavor - if err = json.Unmarshal([]byte(setting.FlavorInfos), &flavorInfos); err != nil { + if err = json.Unmarshal([]byte(setting.TrainJobFlavorInfos), &flavorInfos); err != nil { ctx.ServerError("json.Unmarshal failed:", err) return } @@ -899,6 +901,11 @@ func TrainJobShow(ctx *context.Context) { } else { result.TrainJobDuration = "00:00:00" } + err = models.SetTrainJobStatusByJobID(jobID, modelarts.TransTrainJobStatus(result.IntStatus), int(result.Duration), string(result.TrainJobDuration)) + if err != nil { + ctx.ServerError("UpdateJob failed", err) + return + } result.Status = modelarts.TransTrainJobStatus(result.IntStatus) result.DatasetName = attach.Name } diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 69c03c42f..90d204a82 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -991,7 +991,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop) m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel) m.Get("/log", reqRepoCloudBrainReader, repo.TrainJobGetLog) - m.Get("/download_model", reqRepoCloudBrainReader, repo.TrainJobloadModel) + m.Get("/models", reqRepoCloudBrainReader, repo.TrainJobListModel) + m.Get("/download_model", reqRepoCloudBrainReader, repo.TrainJobDownloadModel) }) m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNew) m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate) diff --git a/templates/repo/modelarts/trainjob/index.tmpl b/templates/repo/modelarts/trainjob/index.tmpl index 69d6dc2f7..0d15638ad 100755 --- a/templates/repo/modelarts/trainjob/index.tmpl +++ b/templates/repo/modelarts/trainjob/index.tmpl @@ -360,7 +360,7 @@
- + 模型下载 diff --git a/templates/repo/modelarts/trainjob/list_model.tmpl b/templates/repo/modelarts/trainjob/list_model.tmpl new file mode 100644 index 000000000..aa90ec3ad --- /dev/null +++ b/templates/repo/modelarts/trainjob/list_model.tmpl @@ -0,0 +1 @@ +shuchu \ No newline at end of file