提交代码。

Signed-off-by: zouap <zouap@pcl.ac.cn>

models/ai_model_manage.go

@@ -14,7 +14,7 @@ type AiModelManage struct {
 	ID            string             `xorm:"pk"`
 	Name          string             `xorm:"NOT NULL"`
 	Version       string             `xorm:"NOT NULL"`
-	Parent        string             `xorm:"NOT NULL"`
+	New           int                `xorm:"NOT NULL"`
 	Type          int                `xorm:"NOT NULL"`
 	Size          int64              `xorm:"NOT NULL"`
 	Description   string             `xorm:"varchar(2000)"`
@@ -41,6 +41,7 @@ type AiModelQueryOptions struct {
 	UserID   int64
 	ModelID  string
 	SortType string
+	New      int
 	// JobStatus     CloudbrainStatus
 	Type int
 }
@@ -99,14 +100,28 @@ func ModifyModelDescription(id string, description string) error {
 	}
 	log.Info("success to update description from  db.re=" + fmt.Sprint((re)))
 	return nil
+}
+
+func ModifyModelNewProperty(id string, new int) error {
+	var sess *xorm.Session
+	sess = x.ID(id)
+	defer sess.Close()
+	re, err := sess.Cols("new").Update(&AiModelManage{
+		New: new,
+	})
+	if err != nil {
+		return err
+	}
+	log.Info("success to update new property from  db.re=" + fmt.Sprint((re)))
+	return nil
 
 }
 
-func QueryModelByName(name string, uid int64) []*AiModelManage {
+func QueryModelByName(name string, repoId int64) []*AiModelManage {
 	sess := x.NewSession()
 	defer sess.Close()
 	sess.Select("*").Table("ai_model_manage").
-		Where("name='" + name + "' and user_id=" + fmt.Sprint(uid))
+		Where("name='" + name + "' and repo_id=" + fmt.Sprint(repoId)).OrderBy("version desc")
 	aiModelManageList := make([]*AiModelManage, 0)
 	sess.Find(&aiModelManageList)
 	return aiModelManageList
@@ -129,6 +144,12 @@ func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) {
 		)
 	}
 
+	if opts.New >= 0 {
+		cond = cond.And(
+			builder.Eq{"ai_model_manage.new": opts.New},
+		)
+	}
+
 	if len(opts.ModelID) > 0 {
 		cond = cond.And(
 			builder.Eq{"ai_model_manage.id": opts.ModelID},

modules/storage/obs.go

@@ -220,6 +220,47 @@ func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) {
 	}
 }
 
+func ObsCopyManyFile(srcBucket string, srcPath string, destBucket string, destPath string) (int64, error) {
+	input := &obs.ListObjectsInput{}
+	input.Bucket = srcBucket
+	// 设置每页100个对象
+	input.MaxKeys = 100
+	input.Prefix = srcPath
+	index := 1
+	length := len(srcPath)
+	var fileTotalSize int64
+	log.Info("prefix=" + input.Prefix)
+	for {
+		output, err := ObsCli.ListObjects(input)
+		if err == nil {
+			log.Info("Page:%d\n", index)
+			index++
+			for _, val := range output.Contents {
+				if strings.HasSuffix(val.Key, "/") {
+					log.Info("copy file, src key=" + val.Key + "  is dir, not copy.")
+					continue
+				}
+				destKey := destPath + val.Key[length:]
+				log.Info("copy file,bucket=" + srcBucket + " src key=" + val.Key + "destbucket=" + destBucket + " dest key=" + destKey)
+				ObsCopyFile(srcBucket, val.Key, destBucket, destKey)
+				fileTotalSize += val.Size
+			}
+			if output.IsTruncated {
+				input.Marker = output.NextMarker
+			} else {
+				break
+			}
+		} else {
+			if obsError, ok := err.(obs.ObsError); ok {
+				log.Info("Code:%s\n", obsError.Code)
+				log.Info("Message:%s\n", obsError.Message)
+			}
+			return 0, err
+		}
+	}
+	return fileTotalSize, nil
+}
+
 func ObsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKeyName string) error {
 	input := &obs.CopyObjectInput{}
 	input.Bucket = destBucket
@@ -249,7 +290,7 @@ func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, erro
 		for _, val := range output.Contents {
 			var isDir bool
 			if strings.HasSuffix(val.Key, "/") {
-				continue
+				isDir = true
 			} else {
 				isDir = false
 			}
@@ -272,30 +313,28 @@ func GetAllObsListObjectUnderDir(bucket string, prefix string) ([]FileInfo, erro
 
 }
 
-func GetObsListObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) {
+func GetObsListObjectByBucketAndPrefix(bucket string, prefix string, parentDir string) ([]FileInfo, error) {
 	input := &obs.ListObjectsInput{}
 	input.Bucket = bucket
 	input.Prefix = prefix
 	output, err := ObsCli.ListObjects(input)
 	fileInfos := make([]FileInfo, 0)
 	if err == nil {
-		var nextParentDir string
 		for _, val := range output.Contents {
 			str1 := strings.Split(val.Key, "/")
 			var isDir bool
-			var fileName string
+			var fileName, nextParentDir string
 			if strings.HasSuffix(val.Key, "/") {
-				if prefix == val.Key {
-					continue
-				}
 				fileName = str1[len(str1)-2]
 				isDir = true
-				nextParentDir = nextParentDir + fileName + "/"
+				nextParentDir = fileName
+				if fileName == parentDir || (fileName+"/") == setting.OutPutPath {
+					continue
+				}
 			} else {
 				fileName = str1[len(str1)-1]
 				isDir = false
 			}
-
 			fileInfo := FileInfo{
 				ModTime:  val.LastModified.Format("2006-01-02 15:04:05"),
 				FileName: fileName,

routers/repo/ai_model_manage.go

@@ -19,6 +19,8 @@ const (
 	Model_prefix           = "aimodels/"
 	tplModelManageIndex    = "repo/modelmanage/index"
 	tplModelManageDownload = "repo/modelmanage/download"
+	MODEL_LATEST           = 1
+	MODEL_NOT_LATEST       = 0
 )
 
 func SaveModelByParameters(jobId string, name string, version string, label string, description string, userId int64) error {
@@ -31,16 +33,16 @@ func SaveModelByParameters(jobId string, name string, version string, label stri
 	uuid := uuid.NewV4()
 	id := uuid.String()
 	modelPath := id
-	parent := id
+	var lastNewModelId string
 	var modelSize int64
 	cloudType := models.TypeCloudBrainTwo
 
 	log.Info("find task name:" + aiTask.JobName)
-	aimodels := models.QueryModelByName(name, userId)
+	aimodels := models.QueryModelByName(name, aiTask.RepoID)
 	if len(aimodels) > 0 {
 		for _, model := range aimodels {
-			if model.ID == model.Parent {
-				parent = model.ID
+			if model.New == MODEL_LATEST {
+				lastNewModelId = model.ID
 			}
 		}
 	}
@@ -60,7 +62,7 @@ func SaveModelByParameters(jobId string, name string, version string, label stri
 		Label:        label,
 		Name:         name,
 		Description:  description,
-		Parent:       parent,
+		New:          MODEL_LATEST,
 		Type:         cloudType,
 		Path:         modelPath,
 		Size:         modelSize,
@@ -71,6 +73,11 @@ func SaveModelByParameters(jobId string, name string, version string, label stri
 
 	models.SaveModelToDb(model)
 
+	if len(lastNewModelId) > 0 {
+		//udpate status
+		models.ModifyModelNewProperty(lastNewModelId, MODEL_NOT_LATEST)
+	}
+
 	log.Info("save model end.")
 
 	return nil
@@ -96,9 +103,7 @@ func SaveModel(ctx *context.Context) {
 }
 
 func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string) (string, int64, error) {
-	dataActualPath := setting.Bucket + "/" + Model_prefix +
-		models.AttachmentRelativePath(modelUUID) +
-		"/"
+
 	objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/")
 	modelDbResult, err := storage.GetAllObsListObjectUnderDir(setting.Bucket, objectkey)
 	if err != nil {
@@ -108,25 +113,28 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir
 	if len(modelDbResult) == 0 {
 		return "", 0, errors.New("cannot create model, as model is empty.")
 	}
-	var size int64
-	prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + "/"
+
+	prefix := objectkey + "/"
 	destKeyNamePrefix := Model_prefix + models.AttachmentRelativePath(modelUUID) + "/"
-	for _, modelFile := range modelDbResult {
-		if modelFile.IsDir {
-			log.Info("copy dir, continue. dir=" + modelFile.FileName)
-			continue
-		}
-		srcKeyName := prefix + modelFile.ParenDir + modelFile.FileName
-		log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + srcKeyName)
-		destKeyName := destKeyNamePrefix + modelFile.ParenDir + modelFile.FileName
-		log.Info("Dest key name=" + destKeyName)
-		err := storage.ObsCopyFile(setting.Bucket, srcKeyName, setting.Bucket, destKeyName)
-		if err != nil {
-			log.Info("copy failed.")
-		}
-		size += modelFile.Size
-	}
 
+	size, err := storage.ObsCopyManyFile(setting.Bucket, prefix, setting.Bucket, destKeyNamePrefix)
+
+	// for _, modelFile := range modelDbResult {
+	// 	if modelFile.IsDir {
+	// 		log.Info("copy dir, continue. dir=" + modelFile.FileName)
+	// 		continue
+	// 	}
+	// 	srcKeyName := prefix + modelFile.FileName
+	// 	log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + srcKeyName)
+	// 	destKeyName := destKeyNamePrefix + modelFile.FileName
+	// 	log.Info("Dest key name=" + destKeyName)
+	// 	err := storage.ObsCopyFile(setting.Bucket, srcKeyName, setting.Bucket, destKeyName)
+	// 	if err != nil {
+	// 		log.Info("copy failed.")
+	// 	}
+	// 	size += modelFile.Size
+	// }
+	dataActualPath := setting.Bucket + "/" + destKeyNamePrefix
 	return dataActualPath, size, nil
 }
 
@@ -156,8 +164,15 @@ func DeleteModelByID(id string) error {
 				return err
 			}
 		}
-
-		return models.DeleteModelById(id)
+		err = models.DeleteModelById(id)
+		if err == nil { //find a model to change new
+			if model.New == MODEL_LATEST {
+				aimodels := models.QueryModelByName(model.Name, model.RepoId)
+				if len(aimodels) > 0 {
+					models.ModifyModelNewProperty(aimodels[0].ID, MODEL_LATEST)
+				}
+			}
+		}
 	}
 	return err
 }
@@ -175,6 +190,7 @@ func QueryModelByParameters(repoId int64, page int) ([]*models.AiModelManage, in
 		},
 		RepoID: repoId,
 		Type:   -1,
+		New:    MODEL_LATEST,
 	})
 }
 
@@ -221,6 +237,17 @@ func ShowSingleModel(ctx *context.Context) {
 	ctx.HTML(200, tplModelManageDownload)
 }
 
+func ShowOneVersionOtherModel(ctx *context.Context) {
+	repoId := ctx.Repo.Repository.ID
+	name := ctx.Query("name")
+	aimodels := models.QueryModelByName(name, repoId)
+	if len(aimodels) > 0 {
+		ctx.JSON(200, aimodels[1:])
+	} else {
+		ctx.JSON(200, aimodels)
+	}
+}
+
 func ShowModelPageInfo(ctx *context.Context) {
 	log.Info("ShowModelInfo start.")
 
@@ -237,6 +264,7 @@ func ShowModelPageInfo(ctx *context.Context) {
 		},
 		RepoID: repoId,
 		Type:   Type,
+		New:    MODEL_LATEST,
 	})
 	if err != nil {
 		ctx.ServerError("Cloudbrain", err)
@@ -250,7 +278,6 @@ func ShowModelPageInfo(ctx *context.Context) {
 	ctx.Data["Tasks"] = modelResult
 
 	ctx.HTML(200, tplModelManageIndex)
-
 }
 
 func ModifyModel(id string, description string) error {