From 76ea89deeb02d0a350c8f8a86a2fefaf7db5aaa6 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 4 Nov 2021 15:57:06 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/storage/obs.go | 31 ++++++++--- routers/repo/ai_model_manage.go | 94 +++++++++++++++++++-------------- routers/repo/modelarts.go | 3 ++ 3 files changed, 81 insertions(+), 47 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index bd73281d0..87e972572 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -174,6 +174,25 @@ func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { } } +func ObsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKeyName string) error { + input := &obs.CopyObjectInput{} + input.Bucket = srcBucket + input.Key = srcKeyName + input.CopySourceBucket = destBucket + input.CopySourceKey = destKeyName + _, err := ObsCli.CopyObject(input) + if err == nil { + log.Info("copy success,destBuckName:%s, destkeyname:%s", destBucket, destKeyName) + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Info(obsError.Code) + log.Info(obsError.Message) + } + return err + } + return nil +} + func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = setting.Bucket @@ -184,12 +203,12 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { for _, val := range output.Contents { str1 := strings.Split(val.Key, "/") var isDir bool - var fileName,nextParentDir string + var fileName, nextParentDir string if strings.HasSuffix(val.Key, "/") { fileName = str1[len(str1)-2] isDir = true nextParentDir = fileName - if fileName == parentDir || (fileName + "/") == setting.OutPutPath { + if fileName == parentDir || (fileName+"/") == setting.OutPutPath { continue } } else { @@ -198,10 +217,10 @@ func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) { } fileInfo := FileInfo{ - ModTime: val.LastModified.Format("2006-01-02 15:04:05"), + ModTime: val.LastModified.Format("2006-01-02 15:04:05"), FileName: fileName, - Size: val.Size, - IsDir:isDir, + Size: val.Size, + IsDir: isDir, ParenDir: nextParentDir, } fileInfos = append(fileInfos, fileInfo) @@ -242,7 +261,7 @@ func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) input := &obs.CreateSignedUrlInput{} input.Bucket = setting.Bucket input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/") - + input.Expires = 60 * 60 input.Method = obs.HttpMethodGet diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 2e75c5634..1dd1e82b2 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -13,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" uuid "github.com/satori/go.uuid" ) @@ -55,16 +56,9 @@ func SaveModel(ctx *context.Context) { } } cloudType = aiTask.Cloudbrain.Type - //download model zip - if cloudType == models.TypeCloudBrainOne { - modelPath, modelSize, err = downloadModelFromCloudBrainOne(id, aiTask.JobName, "") - if err != nil { - log.Info("download model from CloudBrainOne faild." + err.Error()) - ctx.Error(500, fmt.Sprintf("%v", err)) - return - } - } else if cloudType == models.TypeCloudBrainTwo { - modelPath, err = downloadModelFromCloudBrainTwo(id) + //download model zip //train type + if cloudType == models.TypeCloudBrainTrainJob { + modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "") if err == nil { } else { @@ -94,6 +88,55 @@ func SaveModel(ctx *context.Context) { log.Info("save model end.") } +func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string) (string, int64, error) { + dataActualPath := setting.Bucket + "/" + + "aimodels/" + + models.AttachmentRelativePath(modelUUID) + + "/" + + models, err := storage.GetObsListObject(jobName, parentDir) + if err != nil { + log.Info("get TrainJobListModel failed:", err) + return "", 0, err + } + if len(models) == 0 { + return "", 0, errors.New("cannot create model, as model is empty.") + } + + for _, modelFile := range models { + log.Info("copy file, bucket=%s, src keyname=%s,dest keyname=%s", setting.Bucket, modelFile.ParenDir+modelFile.FileName, dataActualPath+modelFile.FileName) + // err := storage.ObsCopyFile(setting.Bucket, modelFile.ParenDir+modelFile.FileName, setting.Bucket, dataActualPath+modelFile.FileName) + // if err != nil { + // log.Info("copy failed.") + // } + } + + return dataActualPath, 0, nil +} + +func DeleteModel(ctx *context.Context) { + log.Info("delete model start.") + id := ctx.Query("ID") + err := models.DeleteModelById(id) + if err != nil { + ctx.JSON(500, err.Error()) + } else { + ctx.JSON(200, map[string]string{ + "result_code": "0", + }) + } +} + +func DownloadModel(ctx *context.Context) { + log.Info("download model start.") + +} + +func ShowModelInfo(ctx *context.Context) { + log.Info("ShowModelInfo.") + +} + func downloadModelFromCloudBrainOne(modelUUID string, jobName string, parentDir string) (string, int64, error) { modelActualPath := setting.Attachment.Minio.RealPath + @@ -161,34 +204,3 @@ func zipDir(dir, zipFile string) error { } return nil } - -func downloadModelFromCloudBrainTwo(modelUUID string) (string, error) { - dataActualPath := setting.Bucket + "/" + - "aimodels/" + - models.AttachmentRelativePath(modelUUID) + - "/" - return dataActualPath, nil -} - -func DeleteModel(ctx *context.Context) { - log.Info("delete model start.") - id := ctx.Query("ID") - err := models.DeleteModelById(id) - if err != nil { - ctx.JSON(500, err.Error()) - } else { - ctx.JSON(200, map[string]string{ - "result_code": "0", - }) - } -} - -func DownloadModel(ctx *context.Context) { - log.Info("download model start.") - -} - -func ShowModelInfo(ctx *context.Context) { - log.Info("ShowModelInfo.") - -} diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index ea8ab0c06..ad50c743a 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1107,6 +1107,9 @@ func TrainJobShowModels(ctx *context.Context) { jobID := ctx.Params(":jobid") parentDir := ctx.Query("parentDir") + + log.Info("parentDir=" + parentDir) + dirArray := strings.Split(parentDir, "/") task, err := models.GetCloudbrainByJobID(jobID) if err != nil {