From 32a846f928cbdcaa1532d45a21ee03a403eb75c3 Mon Sep 17 00:00:00 2001 From: zouap Date: Tue, 9 Nov 2021 11:38:15 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/ai_model_manage.go | 27 +++++++++-- modules/storage/obs.go | 57 ++++++++++++++++++---- routers/repo/ai_model_manage.go | 83 ++++++++++++++++++++++----------- 3 files changed, 127 insertions(+), 40 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index cab4201c7..dfeaba287 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -14,7 +14,7 @@ type AiModelManage struct { ID string `xorm:"pk"` Name string `xorm:"NOT NULL"` Version string `xorm:"NOT NULL"` - Parent string `xorm:"NOT NULL"` + New int `xorm:"NOT NULL"` Type int `xorm:"NOT NULL"` Size int64 `xorm:"NOT NULL"` Description string `xorm:"varchar(2000)"` @@ -41,6 +41,7 @@ type AiModelQueryOptions struct { UserID int64 ModelID string SortType string + New int // JobStatus CloudbrainStatus Type int } @@ -99,14 +100,28 @@ func ModifyModelDescription(id string, description string) error { } log.Info("success to update description from db.re=" + fmt.Sprint((re))) return nil +} + +func ModifyModelNewProperty(id string, new int) error { + var sess *xorm.Session + sess = x.ID(id) + defer sess.Close() + re, err := sess.Cols("new").Update(&AiModelManage{ + New: new, + }) + if err != nil { + return err + } + log.Info("success to update new property from db.re=" + fmt.Sprint((re))) + return nil } -func QueryModelByName(name string, uid int64) []*AiModelManage { +func QueryModelByName(name string, repoId int64) []*AiModelManage { sess := x.NewSession() defer sess.Close() sess.Select("*").Table("ai_model_manage"). - Where("name='" + name + "' and user_id=" + fmt.Sprint(uid)) + Where("name='" + name + "' and repo_id=" + fmt.Sprint(repoId)).OrderBy("version desc") aiModelManageList := make([]*AiModelManage, 0) sess.Find(&aiModelManageList) return aiModelManageList @@ -129,6 +144,12 @@ func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) { ) } + if opts.New >= 0 { + cond = cond.And( + builder.Eq{"ai_model_manage.new": opts.New}, + ) + } + if len(opts.ModelID) > 0 { cond = cond.And( builder.Eq{"ai_model_manage.id": opts.ModelID}, diff --git a/modules/storage/obs.go b/modules/storage/obs.go index e7de0c77d..a0fa56407 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -220,6 +220,47 @@ func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { } } +func ObsCopyManyFile(srcBucket string, srcPath string, destBucket string, destPath string) (int64, error) { + input := &obs.ListObjectsInput{} + input.Bucket = srcBucket + // 设置每页100个对象 + input.MaxKeys = 100 + input.Prefix = srcPath + index := 1 + length := len(srcPath) + var fileTotalSize int64 + log.Info("prefix=" + input.Prefix) + for { + output, err := ObsCli.ListObjects(input) + if err == nil { + log.Info("Page:%d\n", index) + index++ + for _, val := range output.Contents { + if strings.HasSuffix(val.Key, "/") { + log.Info("copy file, src key=" + val.Key + " is dir, not copy.") + continue + } + destKey := destPath + val.Key[length:] + log.Info("copy file,bucket=" + srcBucket + " src key=" + val.Key + "destbucket=" + destBucket + " dest key=" + destKey) + ObsCopyFile(srcBucket, val.Key, destBucket, destKey) + fileTotalSize += val.Size + } + if output.IsTruncated { + input.Marker = output.NextMarker + } else { + break + } + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Info("Code:%s\n", obsError.Code) + log.Info("Message:%s\n", obsError.Message) + } + return 0, err + } + } + return fileTotalSize, nil +} + func ObsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKeyName string) error { input := &obs.CopyObjectInput{} input.Bucket = destBucket @@ -249,7 +290,7 @@ func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, erro for _, val := range output.Contents { var isDir bool if strings.HasSuffix(val.Key, "/") { - continue + isDir = true } else { isDir = false } @@ -272,30 +313,28 @@ func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, erro } -func GetObsListObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) { +func GetObsListObjectByBucketAndPrefix(bucket string, prefix string, parentDir string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = bucket input.Prefix = prefix output, err := ObsCli.ListObjects(input) fileInfos := make([]FileInfo, 0) if err == nil { - var nextParentDir string for _, val := range output.Contents { str1 := strings.Split(val.Key, "/") var isDir bool - var fileName string + var fileName, nextParentDir string if strings.HasSuffix(val.Key, "/") { - if prefix == val.Key { - continue - } fileName = str1[len(str1)-2] isDir = true - nextParentDir = nextParentDir + fileName + "/" + nextParentDir = fileName + if fileName == parentDir || (fileName+"/") == setting.OutPutPath { + continue + } } else { fileName = str1[len(str1)-1] isDir = false } - fileInfo := FileInfo{ ModTime: val.LastModified.Format("2006-01-02 15:04:05"), FileName: fileName, diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index b63763fa3..3b85a0b20 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -19,6 +19,8 @@ const ( Model_prefix = "aimodels/" tplModelManageIndex = "repo/modelmanage/index" tplModelManageDownload = "repo/modelmanage/download" + MODEL_LATEST = 1 + MODEL_NOT_LATEST = 0 ) func SaveModelByParameters(jobId string, name string, version string, label string, description string, userId int64) error { @@ -31,16 +33,16 @@ func SaveModelByParameters(jobId string, name string, version string, label stri uuid := uuid.NewV4() id := uuid.String() modelPath := id - parent := id + var lastNewModelId string var modelSize int64 cloudType := models.TypeCloudBrainTwo log.Info("find task name:" + aiTask.JobName) - aimodels := models.QueryModelByName(name, userId) + aimodels := models.QueryModelByName(name, aiTask.RepoID) if len(aimodels) > 0 { for _, model := range aimodels { - if model.ID == model.Parent { - parent = model.ID + if model.New == MODEL_LATEST { + lastNewModelId = model.ID } } } @@ -60,7 +62,7 @@ func SaveModelByParameters(jobId string, name string, version string, label stri Label: label, Name: name, Description: description, - Parent: parent, + New: MODEL_LATEST, Type: cloudType, Path: modelPath, Size: modelSize, @@ -71,6 +73,11 @@ func SaveModelByParameters(jobId string, name string, version string, label stri models.SaveModelToDb(model) + if len(lastNewModelId) > 0 { + //udpate status + models.ModifyModelNewProperty(lastNewModelId, MODEL_NOT_LATEST) + } + log.Info("save model end.") return nil @@ -96,9 +103,7 @@ func SaveModel(ctx *context.Context) { } func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string) (string, int64, error) { - dataActualPath := setting.Bucket + "/" + Model_prefix + - models.AttachmentRelativePath(modelUUID) + - "/" + objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") modelDbResult, err := storage.GetAllObsListObjectUnderDir(setting.Bucket, objectkey) if err != nil { @@ -108,25 +113,28 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir if len(modelDbResult) == 0 { return "", 0, errors.New("cannot create model, as model is empty.") } - var size int64 - prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + "/" + + prefix := objectkey + "/" destKeyNamePrefix := Model_prefix + models.AttachmentRelativePath(modelUUID) + "/" - for _, modelFile := range modelDbResult { - if modelFile.IsDir { - log.Info("copy dir, continue. dir=" + modelFile.FileName) - continue - } - srcKeyName := prefix + modelFile.ParenDir + modelFile.FileName - log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + srcKeyName) - destKeyName := destKeyNamePrefix + modelFile.ParenDir + modelFile.FileName - log.Info("Dest key name=" + destKeyName) - err := storage.ObsCopyFile(setting.Bucket, srcKeyName, setting.Bucket, destKeyName) - if err != nil { - log.Info("copy failed.") - } - size += modelFile.Size - } + size, err := storage.ObsCopyManyFile(setting.Bucket, prefix, setting.Bucket, destKeyNamePrefix) + + // for _, modelFile := range modelDbResult { + // if modelFile.IsDir { + // log.Info("copy dir, continue. dir=" + modelFile.FileName) + // continue + // } + // srcKeyName := prefix + modelFile.FileName + // log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + srcKeyName) + // destKeyName := destKeyNamePrefix + modelFile.FileName + // log.Info("Dest key name=" + destKeyName) + // err := storage.ObsCopyFile(setting.Bucket, srcKeyName, setting.Bucket, destKeyName) + // if err != nil { + // log.Info("copy failed.") + // } + // size += modelFile.Size + // } + dataActualPath := setting.Bucket + "/" + destKeyNamePrefix return dataActualPath, size, nil } @@ -156,8 +164,15 @@ func DeleteModelByID(id string) error { return err } } - - return models.DeleteModelById(id) + err = models.DeleteModelById(id) + if err == nil { //find a model to change new + if model.New == MODEL_LATEST { + aimodels := models.QueryModelByName(model.Name, model.RepoId) + if len(aimodels) > 0 { + models.ModifyModelNewProperty(aimodels[0].ID, MODEL_LATEST) + } + } + } } return err } @@ -175,6 +190,7 @@ func QueryModelByParameters(repoId int64, page int) ([]*models.AiModelManage, in }, RepoID: repoId, Type: -1, + New: MODEL_LATEST, }) } @@ -221,6 +237,17 @@ func ShowSingleModel(ctx *context.Context) { ctx.HTML(200, tplModelManageDownload) } +func ShowOneVersionOtherModel(ctx *context.Context) { + repoId := ctx.Repo.Repository.ID + name := ctx.Query("name") + aimodels := models.QueryModelByName(name, repoId) + if len(aimodels) > 0 { + ctx.JSON(200, aimodels[1:]) + } else { + ctx.JSON(200, aimodels) + } +} + func ShowModelPageInfo(ctx *context.Context) { log.Info("ShowModelInfo start.") @@ -237,6 +264,7 @@ func ShowModelPageInfo(ctx *context.Context) { }, RepoID: repoId, Type: Type, + New: MODEL_LATEST, }) if err != nil { ctx.ServerError("Cloudbrain", err) @@ -250,7 +278,6 @@ func ShowModelPageInfo(ctx *context.Context) { ctx.Data["Tasks"] = modelResult ctx.HTML(200, tplModelManageIndex) - } func ModifyModel(id string, description string) error {