From 9cf92312e65c3c0ede25c98195d7bfd5440f85ac Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Fri, 29 Oct 2021 18:24:05 +0800 Subject: [PATCH] download model --- modules/storage/obs.go | 47 ++++++----- routers/repo/cloudbrain.go | 42 +--------- routers/repo/dir.go | 11 +-- routers/repo/modelarts.go | 78 ++++++++++++------- routers/routes/routes.go | 2 +- templates/repo/modelarts/notebook/index.tmpl | 4 +- templates/repo/modelarts/trainjob/index.tmpl | 4 +- .../modelarts/trainjob/models/dir_list.tmpl | 27 +++++++ .../repo/modelarts/trainjob/models/index.tmpl | 29 +++++++ 9 files changed, 141 insertions(+), 103 deletions(-) create mode 100755 templates/repo/modelarts/trainjob/models/dir_list.tmpl create mode 100755 templates/repo/modelarts/trainjob/models/index.tmpl diff --git a/modules/storage/obs.go b/modules/storage/obs.go index e5e02952f..697ced6c5 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -5,7 +5,6 @@ package storage import ( - "fmt" "io" "path" "strconv" @@ -18,6 +17,15 @@ import ( "code.gitea.io/gitea/modules/setting" ) +type FileInfo struct { + FileName string `json:"FileName"` + ModTime string `json:"ModTime"` + IsDir bool `json:"IsDir"` + Size int64 `json:"Size"` + ParenDir string `json:"ParenDir"` + UUID string `json:"UUID"` +} + //check if has the object //todo：修改查询方式 func ObsHasObject(path string) (bool, error) { @@ -141,8 +149,7 @@ func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified) return output.Body, nil } else if obsError, ok := err.(obs.ObsError); ok { - fmt.Printf("Code:%s\n", obsError.Code) - fmt.Printf("Message:%s\n", obsError.Message) + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) return nil, obsError } else { return nil, err @@ -160,40 +167,38 @@ func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified) return output.Body, nil } else if obsError, ok := err.(obs.ObsError); ok { - fmt.Printf("Code:%s\n", obsError.Code) - fmt.Printf("Message:%s\n", obsError.Message) + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) return nil, obsError } else { return nil, err } } -func GetObsListObject(jobName string) ([]string, error) { - // jobName = "liuzx202110271830856" +func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = setting.Bucket - input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath), "/") - log.Info("input.Prefix:", input.Prefix) + input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") output, err := ObsCli.ListObjects(input) - log.Info("output.Prefix:", output) - ModelListArr := make([]string, 0) + fileInfos := make([]FileInfo, 0) if err == nil { - fmt.Printf("RequestId:%s\n", output.RequestId) for index, val := range output.Contents { - fmt.Printf("Content[%d]-OwnerId:%s, ETag:%s, Key:%s, LastModified:%s, Size:%d\n", + log.Info("Content[%d]-OwnerId:%s, ETag:%s, Key:%s, LastModified:%s, Size:%d\n", index, val.Owner.ID, val.ETag, val.Key, val.LastModified, val.Size) str1 := strings.Split(val.Key, "/") - ModelList := str1[len(str1)-1] - ModelListArr = append(ModelListArr, ModelList) - log.Info("ModelListArr.Prefix:", ModelListArr) + fileName := str1[len(str1)-1] + log.Info("", fileName) + fileInfo := FileInfo{ + ModTime: val.LastModified.Format("2006-01-02 15:04:05"), + FileName: fileName, + Size: val.Size, + IsDir:false, + } + fileInfos = append(fileInfos, fileInfo) } - return ModelListArr, err + return fileInfos, err } else { if obsError, ok := err.(obs.ObsError); ok { - fmt.Println(obsError.Code) - fmt.Println(obsError.Message) - } else { - fmt.Println(err) + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) } return nil, err } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 782a51000..d5bc85a30 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -510,7 +510,7 @@ func CloudBrainShowModels(ctx *context.Context) { return } - var fileInfos []FileInfo + var fileInfos []storage.FileInfo err = json.Unmarshal([]byte(dirs), &fileInfos) if err != nil { log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"]) @@ -583,50 +583,10 @@ func CloudBrainDownloadModel(ctx *context.Context) { http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } -// func TrainJobloadModel(ctx *context.Context) { -// parentDir := ctx.Query("parentDir") -// fileName := ctx.Query("fileName") -// jobName := ctx.Query("jobName") -// filePath := "jobs/" + jobName + "/model/" + parentDir -// url, err := storage.Attachments.PresignedGetURL(filePath, fileName) -// if err != nil { -// log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"]) -// ctx.ServerError("PresignedGetURL", err) -// return -// } - -// http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) -// } - -func TrainJobListModel(ctx *context.Context) { - ctx.Data["PageIsCloudBrain"] = true - jobID := ctx.Params(":jobid") - task, err := models.GetCloudbrainByJobID(jobID) - if err != nil { - log.Error("no such job!", ctx.Data["err"]) - ctx.ServerError("no such job:", err) - return - } - TrainJobListModel, err := storage.GetObsListObject(task.JobName) - log.Info("TrainJobListModel", TrainJobListModel) - fmt.Println("TrainJobListModel:", TrainJobListModel) - if err != nil { - log.Info("get TrainJobListModel failed:", err) - return - } - ctx.Data["task"] = task - ctx.Data["JobID"] = jobID - ctx.Data["ListModel"] = TrainJobListModel - ctx.HTML(200, tplModelArtsTrainJobListModel) -} - func TrainJobDownloadModel(ctx *context.Context) { JobName := ctx.Query("JobName") fileName := ctx.Query("file_name") - // JobName = "liuzx202110271830856" - // fileName = "Untitled.ipynb" - body, err := storage.ObsModelDownload(JobName, fileName) if err != nil { log.Info("download error.") diff --git a/routers/repo/dir.go b/routers/repo/dir.go index 612019d46..406f3dc73 100755 --- a/routers/repo/dir.go +++ b/routers/repo/dir.go @@ -21,15 +21,6 @@ const ( tplDirIndex base.TplName = "repo/datasets/dirs/index" ) -type FileInfo struct { - FileName string `json:"FileName"` - ModTime string `json:"ModTime"` - IsDir bool `json:"IsDir"` - Size int64 `json:"Size"` - ParenDir string `json:"ParenDir"` - UUID string `json:"UUID"` -} - type RespGetDirs struct { ResultCode string `json:"resultCode"` FileInfos string `json:"fileInfos"` @@ -59,7 +50,7 @@ func DeleteAllUnzipFile(attachment *models.Attachment, parentDir string) { return } - var fileInfos []FileInfo + var fileInfos []storage.FileInfo err = json.Unmarshal([]byte(dirs), &fileInfos) if err != nil { log.Error("json.Unmarshal failed:", err.Error()) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 396457077..2c8f8f617 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -3,7 +3,6 @@ package repo import ( "encoding/json" "errors" - "fmt" "io" "net/http" "os" @@ -12,18 +11,18 @@ import ( "strings" "time" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/modelarts" - "code.gitea.io/gitea/modules/obs" - "code.gitea.io/gitea/modules/storage" - "github.com/unknwon/com" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/modules/obs" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + + "github.com/unknwon/com" ) const ( @@ -39,7 +38,7 @@ const ( tplModelArtsTrainJobIndex base.TplName = "repo/modelarts/trainjob/index" tplModelArtsTrainJobNew base.TplName = "repo/modelarts/trainjob/new" tplModelArtsTrainJobShow base.TplName = "repo/modelarts/trainjob/show" - tplModelArtsTrainJobListModel base.TplName = "repo/modelarts/trainjob/list_model" + tplModelArtsTrainJobShowModels base.TplName = "repo/modelarts/trainjob/models/index" ) // MustEnableDataset check if repository enable internal cb @@ -492,13 +491,13 @@ func NotebookDel(ctx *context.Context) { func TrainJobIndex(ctx *context.Context) { MustEnableModelArts(ctx) - can, err := canUserCreateTrainJob(ctx.User.ID) - if err != nil { - ctx.ServerError("canUserCreateTrainJob", err) - return - } - - ctx.Data["CanCreate"] = can + //can, err := canUserCreateTrainJob(ctx.User.ID) + //if err != nil { + // ctx.ServerError("canUserCreateTrainJob", err) + // return + //} + // + //ctx.Data["CanCreate"] = can repo := ctx.Repo.Repository page := ctx.QueryInt("page") @@ -531,17 +530,17 @@ func TrainJobIndex(ctx *context.Context) { func TrainJobNew(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true - can, err := canUserCreateTrainJob(ctx.User.ID) - if err != nil { - ctx.ServerError("canUserCreateTrainJob", err) - return - } - - if !can { - log.Error("the user can not create train-job") - ctx.ServerError("the user can not create train-job", fmt.Errorf("the user can not create train-job")) - return - } + //can, err := canUserCreateTrainJob(ctx.User.ID) + //if err != nil { + // ctx.ServerError("canUserCreateTrainJob", err) + // return + //} + // + //if !can { + // log.Error("the user can not create train-job") + // ctx.ServerError("the user can not create train-job", fmt.Errorf("the user can not create train-job")) + // return + //} t := time.Now() var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] @@ -1060,3 +1059,30 @@ func getConfigList(perPage, page int, sortBy, order, searchContent, configType s return list, nil } + +func TrainJobShowModels(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + + jobID := ctx.Params(":jobid") + parentDir := ctx.Query("parentDir") + dirArray := strings.Split(parentDir, "/") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + log.Error("no such job!", ctx.Data["msgID"]) + ctx.ServerError("no such job:", err) + return + } + + models, err := storage.GetObsListObject(task.JobName, parentDir) + if err != nil { + log.Info("get TrainJobListModel failed:", err) + ctx.ServerError("GetObsListObject:", err) + return + } + + ctx.Data["Path"] = dirArray + ctx.Data["Dirs"] = models + ctx.Data["task"] = task + ctx.Data["JobID"] = jobID + ctx.HTML(200, tplModelArtsTrainJobShowModels) +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 90d204a82..7e7d0642a 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -991,7 +991,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop) m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel) m.Get("/log", reqRepoCloudBrainReader, repo.TrainJobGetLog) - m.Get("/models", reqRepoCloudBrainReader, repo.TrainJobListModel) + m.Get("/models", reqRepoCloudBrainReader, repo.TrainJobShowModels) m.Get("/download_model", reqRepoCloudBrainReader, repo.TrainJobDownloadModel) }) m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNew) diff --git a/templates/repo/modelarts/notebook/index.tmpl b/templates/repo/modelarts/notebook/index.tmpl index 841954214..29c132580 100755 --- a/templates/repo/modelarts/notebook/index.tmpl +++ b/templates/repo/modelarts/notebook/index.tmpl @@ -423,7 +423,7 @@ // 加载任务状态 var timeid = window.setInterval(loadJobStatus, 15000); - $(document).ready(loadJobStatus); + $(document).ready(loadJobStatus); function loadJobStatus() { $(".job-status").each((index, job) => { const jobID = job.dataset.jobid; @@ -435,7 +435,7 @@ $.get(`/api/v1/repos/${repoPath}/modelarts/notebook/${jobID}`, (data) => { const jobID = data.JobID const status = data.JobStatus - if (status != job.textContent.trim() || status=='RUNNING') { + if (status != job.textContent.trim()) { //$('#' + jobID).text(status) //if (status == 'STOPPED') { window.location.reload() diff --git a/templates/repo/modelarts/trainjob/index.tmpl b/templates/repo/modelarts/trainjob/index.tmpl index 56df053ad..4945e4781 100755 --- a/templates/repo/modelarts/trainjob/index.tmpl +++ b/templates/repo/modelarts/trainjob/index.tmpl @@ -459,7 +459,7 @@ // 加载任务状态 var timeid = window.setInterval(loadJobStatus, 15000); - $(document).ready(loadJobStatus); + $(document).ready(loadJobStatus); function loadJobStatus() { $(".job-status").each((index, job) => { const jobID = job.dataset.jobid; @@ -473,7 +473,7 @@ $.get(`/api/v1/repos/${repoPath}/modelarts/train-job/${jobID}`, (data) => { const jobID = data.JobID const status = data.JobStatus - if (status != job.textContent.trim() || status=='RUNNING') { + if (status != job.textContent.trim()) { //$('#' + jobID).text(status) //if (status == 'STOPPED') { window.location.reload() diff --git a/templates/repo/modelarts/trainjob/models/dir_list.tmpl b/templates/repo/modelarts/trainjob/models/dir_list.tmpl new file mode 100755 index 000000000..a9683de77 --- /dev/null +++ b/templates/repo/modelarts/trainjob/models/dir_list.tmpl @@ -0,0 +1,27 @@ +{{if .Dirs}} + + + {{range .Dirs}} + + + + + + {{end}} + +
+ + + + {{if .IsDir}} {{svg "octicon-file-directory" 16}}{{else}}{{svg "octicon-file" 16}}{{end}} {{.FileName}} + + + + + {{.Size | FileSize}} + + + {{.ModTime}} +
+ +{{end}} diff --git a/templates/repo/modelarts/trainjob/models/index.tmpl b/templates/repo/modelarts/trainjob/models/index.tmpl new file mode 100755 index 000000000..3b53ad78e --- /dev/null +++ b/templates/repo/modelarts/trainjob/models/index.tmpl @@ -0,0 +1,29 @@ +{{template "base/head" .}} +
+ {{template "repo/header" .}} +
+
+
+
+

+ {{ range $index, $item := .Path }}{{ $item }}/{{ end }} +

+
+
+
+ +
+
+
+
+ {{template "repo/cloudbrain/models/dir_list" .}} +
+
+
+
+
+
+ + + +{{template "base/footer" .}}