From fa43cb7687651cb2faca9b809cf11e4ba01c82be Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 17 Nov 2021 15:18:08 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E6=A8=A1=E5=9E=8B=E5=88=86?= =?UTF-8?q?=E7=A6=BB=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/storage/obs.go | 59 ++++++++++++++++++++++++ routers/api/v1/api.go | 2 + routers/api/v1/repo/modelarts.go | 78 ++++++++++++++++++++++++-------- 3 files changed, 121 insertions(+), 18 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index caa77c48e..3a7277c41 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -235,6 +235,65 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { } } +func GetObsListObjectVersion(jobName, parentDir string, VersionOutputPath string) ([]FileInfo, error) { + input := &obs.ListObjectsInput{} + input.Bucket = setting.Bucket + input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, VersionOutputPath, parentDir), "/") + strPrefix := strings.Split(input.Prefix, "/") + output, err := ObsCli.ListObjects(input) + fileInfos := make([]FileInfo, 0) + if err == nil { + for _, val := range output.Contents { + str1 := strings.Split(val.Key, "/") + var isDir bool + var fileName, nextParentDir string + if strings.HasSuffix(val.Key, "/") { + //dirs in next level dir + if len(str1)-len(strPrefix) > 2 { + continue + } + fileName = str1[len(str1)-2] + isDir = true + if parentDir == "" { + nextParentDir = fileName + } else { + nextParentDir = parentDir + "/" + fileName + } + + if fileName == strPrefix[len(strPrefix)-1] || (fileName+"/") == setting.OutPutPath { + continue + } + } else { + //files in next level dir + if len(str1)-len(strPrefix) > 1 { + continue + } + fileName = str1[len(str1)-1] + isDir = false + nextParentDir = parentDir + } + + fileInfo := FileInfo{ + ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), + FileName: fileName, + Size: val.Size, + IsDir: isDir, + ParenDir: nextParentDir, + } + fileInfos = append(fileInfos, fileInfo) + } + sort.Slice(fileInfos, func(i, j int) bool { + return fileInfos[i].ModTime > fileInfos[j].ModTime + }) + return fileInfos, err + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) + } + return nil, err + } +} + func GetVersionObsListObject(jobName, parentDir string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = setting.Bucket diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index dde17c059..d66aa698d 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -880,6 +880,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/log", repo.TrainJobGetLog) m.Post("/del_version", repo.DelTrainJobVersion) m.Post("/stop_version", repo.StopTrainJobVersion) + m.Get("/model_list", repo.ModelList) + m.Get("/model_download", repo.ModelDownload) // m.Group("/:version-name", func() { // m.Get("", repo.GetModelArtsTrainJobVersion) // }) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 6137eb6b7..d8824b19e 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -8,12 +8,14 @@ package repo import ( "net/http" "strconv" + "strings" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" ) func GetModelArtsNotebook(ctx *context.APIContext) { @@ -157,15 +159,6 @@ func TrainJobGetLog(ctx *context.APIContext) { return } - // task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) - // if err != nil { - // log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", jobID, err.Error()) - // ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - // "err_msg": "GetCloudbrainByJobIDAndVersionName failed", - // }) - // return - // } - resultLogFile, result, err := trainJobGetLogContent(jobID, versionName, baseLine, order, lines_int) if err != nil { log.Error("trainJobGetLog(%s) failed:%v", jobID, err.Error()) @@ -175,15 +168,6 @@ func TrainJobGetLog(ctx *context.APIContext) { ctx.Data["log_file_name"] = resultLogFile.LogFileList[0] - // result, err := modelarts.GetTrainJobLog(jobID, strconv.FormatInt(task.VersionID, 10), baseLine, logFileName, order, modelarts.Lines) - // if err != nil { - // log.Error("GetTrainJobLog(%s) failed:%v", jobID, err.Error()) - // ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - // "err_msg": "GetTrainJobLog failed", - // }) - // return - // } - ctx.JSON(http.StatusOK, map[string]interface{}{ "JobID": jobID, "LogFileName": resultLogFile.LogFileList[0], @@ -316,3 +300,61 @@ func StopTrainJobVersion(ctx *context.APIContext) { "StatusOK": 0, }) } + +func ModelList(ctx *context.APIContext) { + var ( + err error + ) + + var jobID = ctx.Params(":jobid") + var versionName = ctx.Query("version_name") + parentDir := ctx.Query("parentDir") + dirArray := strings.Split(parentDir, "/") + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) + return + } + VersionOutputPath := "V" + strconv.Itoa(task.TotalVersionCount) + models, err := storage.GetObsListObjectVersion(task.JobName, parentDir, VersionOutputPath) + if err != nil { + log.Info("get TrainJobListModel failed:", err) + ctx.ServerError("GetObsListObject:", err) + return + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobID": jobID, + "VersionName": versionName, + "StatusOK": 0, + "Path": dirArray, + "Dirs": models, + "task": task, + "PageIsCloudBrain": true, + }) +} + +func ModelDownload(ctx *context.APIContext) { + var ( + err error + ) + + // var jobID = ctx.Params(":jobid") + // var versionName = ctx.Query("version_name") + // parentDir := ctx.Query("parentDir") + // task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + // if err != nil { + // log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) + // return + // } + parentDir := ctx.Query("parentDir") + fileName := ctx.Query("fileName") + jobName := ctx.Query("jobName") + url, err := storage.GetObsCreateSignedUrl(jobName, parentDir, fileName) + if err != nil { + log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetObsCreateSignedUrl", err) + return + } + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) +}