提交代码。

Signed-off-by: zouap <zouap@pcl.ac.cn>

models/ai_model_manage.go

@@ -56,6 +56,16 @@ func SaveModelToDb(model *AiModelManage) error {
 	return nil
 }
 
+func QueryModelById(id string) AiModelManage {
+	sess := x.NewSession()
+	defer sess.Close()
+	sess.Select("*").Table("ai_model_manage").
+		Where("id='" + id + "'")
+	var aiModelManage AiModelManage
+	sess.Find(&aiModelManage)
+	return aiModelManage
+}
+
 func DeleteModelById(id string) error {
 	sess := x.NewSession()
 	defer sess.Close()

modules/storage/obs.go

@@ -138,6 +138,43 @@ func ObsMultiPartUpload(uuid string, uploadId string, partNumber int, fileName s
 
 }
 
+//delete all file under the dir path
+func ObsRemoveObject(bucket string, path string) error {
+	input := &obs.ListObjectsInput{}
+	input.Bucket = bucket
+	// 设置每页100个对象
+	input.MaxKeys = 100
+	input.Prefix = path
+	index := 1
+	log.Info("prefix=" + input.Prefix)
+	for {
+		output, err := ObsCli.ListObjects(input)
+		if err == nil {
+			log.Info("Page:%d\n", index)
+			index++
+			for _, val := range output.Contents {
+				log.Info("delete obs file:" + val.Key)
+				delObj := &obs.DeleteObjectInput{}
+				delObj.Bucket = setting.Bucket
+				delObj.Key = val.Key
+				ObsCli.DeleteObject(delObj)
+			}
+			if output.IsTruncated {
+				input.Marker = output.NextMarker
+			} else {
+				break
+			}
+		} else {
+			if obsError, ok := err.(obs.ObsError); ok {
+				log.Info("Code:%s\n", obsError.Code)
+				log.Info("Message:%s\n", obsError.Message)
+			}
+			return err
+		}
+	}
+	return nil
+}
+
 func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) {
 	input := &obs.GetObjectInput{}
 	input.Bucket = setting.Bucket

routers/repo/ai_model_manage.go

@@ -1,14 +1,9 @@
 package repo
 
 import (
-	"archive/zip"
 	"errors"
 	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
 	"path"
-	"path/filepath"
 	"strings"
 
 	"code.gitea.io/gitea/models"
@@ -19,12 +14,11 @@ import (
 	uuid "github.com/satori/go.uuid"
 )
 
-func SaveModelByParameters(trainTaskId string, name string, version string, label string, description string, userId int64) {
-	aiTask, err := models.GetCloudbrainByJobID(trainTaskId)
+func SaveModelByParameters(jobId string, name string, version string, label string, description string, userId int64) error {
+	aiTask, err := models.GetCloudbrainByJobID(jobId)
 	if err != nil {
 		log.Info("query task error." + err.Error())
-		//ctx.Error(500, fmt.Sprintf("query cloud brain train task error. %v", err))
-		return
+		return err
 	}
 
 	uuid := uuid.NewV4()
@@ -47,12 +41,9 @@ func SaveModelByParameters(trainTaskId string, name string, version string, labe
 	//download model zip   //train type
 	if cloudType == models.TypeCloudBrainTwo {
 		modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "")
-		if err == nil {
-
-		} else {
+		if err != nil {
 			log.Info("download model from CloudBrainTwo faild." + err.Error())
-			//ctx.Error(500, fmt.Sprintf("%v", err))
-			return
+			return err
 		}
 	}
 
@@ -74,75 +65,25 @@ func SaveModelByParameters(trainTaskId string, name string, version string, labe
 	models.SaveModelToDb(model)
 
 	log.Info("save model end.")
+
+	return nil
 }
 
 func SaveModel(ctx *context.Context) {
 	log.Info("save model start.")
-	trainTaskId := ctx.QueryInt64("TrainTask")
+	JobId := ctx.Query("JobId")
 	name := ctx.Query("Name")
 	version := ctx.Query("Version")
 	label := ctx.Query("Label")
 	description := ctx.Query("Description")
 
-	aiTasks, _, err := models.Cloudbrains(&models.CloudbrainsOptions{
-		JobID: trainTaskId,
-	})
-	if err != nil {
-		log.Info("query task error." + err.Error())
-		ctx.Error(500, fmt.Sprintf("query cloud brain train task error. %v", err))
-		return
-	}
-	uuid := uuid.NewV4()
-	id := uuid.String()
-	modelPath := id
-	parent := id
-	var modelSize int64
-	cloudType := models.TypeCloudBrainTwo
+	err := SaveModelByParameters(JobId, name, version, label, description, ctx.User.ID)
 
-	if len(aiTasks) != 1 {
-		log.Info("query task error. len=" + fmt.Sprint(len(aiTasks)))
-		ctx.Error(500, fmt.Sprintf("query cloud brain train task error. %v", err))
+	if err != nil {
+		log.Info("save model error." + err.Error())
+		ctx.Error(500, fmt.Sprintf("save model error. %v", err))
 		return
 	}
-	aiTask := aiTasks[0]
-	log.Info("find task name:" + aiTask.JobName)
-	aimodels := models.QueryModelByName(name, ctx.User.ID)
-	if len(aimodels) > 0 {
-		for _, model := range aimodels {
-			if model.ID == model.Parent {
-				parent = model.ID
-			}
-		}
-	}
-	cloudType = aiTask.Cloudbrain.Type
-	//download model zip   //train type
-	if cloudType == models.TypeCloudBrainTrainJob {
-		modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "")
-		if err == nil {
-
-		} else {
-			log.Info("download model from CloudBrainTwo faild." + err.Error())
-			ctx.Error(500, fmt.Sprintf("%v", err))
-			return
-		}
-	}
-
-	model := &models.AiModelManage{
-		ID:           id,
-		Version:      version,
-		Label:        label,
-		Name:         name,
-		Description:  description,
-		Parent:       parent,
-		Type:         cloudType,
-		Path:         modelPath,
-		Size:         modelSize,
-		AttachmentId: aiTask.Uuid,
-		RepoId:       aiTask.RepoID,
-		UserId:       ctx.User.ID,
-	}
-
-	models.SaveModelToDb(model)
 
 	log.Info("save model end.")
 }
@@ -161,6 +102,7 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir
 	if len(modelDbResult) == 0 {
 		return "", 0, errors.New("cannot create model, as model is empty.")
 	}
+	var size int64
 	prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + "/"
 	for _, modelFile := range modelDbResult {
 		destKeyNamePrefix := "aimodels/" + models.AttachmentRelativePath(modelUUID) + "/"
@@ -171,9 +113,10 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir
 		if err != nil {
 			log.Info("copy failed.")
 		}
+		size += modelFile.Size
 	}
 
-	return dataActualPath, 0, nil
+	return dataActualPath, size, nil
 }
 
 func DeleteModel(ctx *context.Context) {
@@ -191,6 +134,14 @@ func DeleteModel(ctx *context.Context) {
 
 func DeleteModelByID(id string) error {
 	log.Info("delete model start. id=" + id)
+
+	model := models.QueryModelById(id)
+
+	err := storage.ObsRemoveObject(setting.Bucket, model.Path)
+	if err != nil {
+		log.Info("Failed to delete model. id=" + id)
+		return err
+	}
 	return models.DeleteModelById(id)
 }
 
@@ -240,71 +191,3 @@ func ShowModelInfo(ctx *context.Context) {
 	ctx.Data["Tasks"] = modelResult
 	ctx.HTML(200, "")
 }
-
-func downloadModelFromCloudBrainOne(modelUUID string, jobName string, parentDir string) (string, int64, error) {
-
-	modelActualPath := setting.Attachment.Minio.RealPath +
-		setting.Attachment.Minio.Bucket + "/" +
-		"aimodels/" +
-		models.AttachmentRelativePath(modelUUID) +
-		"/"
-	os.MkdirAll(modelActualPath, 0755)
-	zipFile := modelActualPath + "model.zip"
-
-	modelDir := setting.JobPath + jobName + "/model/"
-
-	dir, _ := ioutil.ReadDir(modelDir)
-	if len(dir) == 0 {
-		return "", 0, errors.New("cannot create model, as model is empty.")
-	}
-
-	err := zipDir(modelDir, zipFile)
-	if err != nil {
-		return "", 0, err
-	}
-
-	fi, err := os.Stat(zipFile)
-	if err == nil {
-		return modelActualPath, fi.Size(), nil
-	} else {
-		return "", 0, err
-	}
-}
-
-func zipDir(dir, zipFile string) error {
-	fz, err := os.Create(zipFile)
-	if err != nil {
-		log.Info("Create zip file failed: %s\n", err.Error())
-		return err
-	}
-	defer fz.Close()
-
-	w := zip.NewWriter(fz)
-	defer w.Close()
-
-	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-		if !info.IsDir() {
-			fDest, err := w.Create(path[len(dir)+1:])
-			if err != nil {
-				log.Info("Create failed: %s\n", err.Error())
-				return err
-			}
-			fSrc, err := os.Open(path)
-			if err != nil {
-				log.Info("Open failed: %s\n", err.Error())
-				return err
-			}
-			defer fSrc.Close()
-			_, err = io.Copy(fDest, fSrc)
-			if err != nil {
-				log.Info("Copy failed: %s\n", err.Error())
-				return err
-			}
-		}
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-	return nil
-}

routers/repo/dir.go

@@ -12,7 +12,6 @@ import (
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/obs"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
 )
@@ -70,40 +69,10 @@ func DeleteAllUnzipFile(attachment *models.Attachment, parentDir string) {
 		}
 	}
 	if attachment.Type == models.TypeCloudBrainTwo {
-
-		input := &obs.ListObjectsInput{}
-		input.Bucket = setting.Bucket
-		// 设置每页100个对象
-		input.MaxKeys = 100
-		input.Prefix = setting.BasePath + attachment.RelativePath() + attachment.UUID
-		index := 1
-		log.Info("prefix=" + input.Prefix)
-		for {
-			output, err := storage.ObsCli.ListObjects(input)
-			if err == nil {
-				log.Info("Page:%d\n", index)
-				index++
-				for _, val := range output.Contents {
-					log.Info("delete obs file:" + val.Key)
-					delObj := &obs.DeleteObjectInput{}
-					delObj.Bucket = setting.Bucket
-					delObj.Key = val.Key
-					storage.ObsCli.DeleteObject(delObj)
-				}
-				if output.IsTruncated {
-					input.Marker = output.NextMarker
-				} else {
-					break
-				}
-			} else {
-				if obsError, ok := err.(obs.ObsError); ok {
-					log.Info("Code:%s\n", obsError.Code)
-					log.Info("Message:%s\n", obsError.Message)
-				}
-				break
-			}
+		err := storage.ObsRemoveObject(setting.Bucket, setting.BasePath+attachment.RelativePath()+attachment.UUID)
+		if err != nil {
+			log.Info("delete file error.")
 		}
-
 	}
 
 }