Merge pull request '删除云脑任务处理' (#1170) from fix-1068 into V20211228

Reviewed-on: https://git.openi.org.cn/OpenI/aiforge/pulls/1170
Reviewed-by: ychao_1983 <ychao_1983@sina.com>

modules/storage/local.go

@@ -80,3 +80,7 @@ func (l *LocalStorage) HasObject(path string) (bool, error) {
 func (l *LocalStorage) UploadObject(fileName, filePath string) error {
 	return nil
 }
+
+func (l *LocalStorage) DeleteDir(dir string) error {
+	return nil
+}

modules/storage/minio.go

@@ -11,7 +11,9 @@ import (
 	"strings"
 	"time"
 
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+
 	"github.com/minio/minio-go"
 )
 
@@ -77,6 +79,29 @@ func (m *MinioStorage) Delete(path string) error {
 	return m.client.RemoveObject(m.bucket, m.buildMinioPath(path))
 }
 
+// Delete delete a file
+func (m *MinioStorage) DeleteDir(dir string) error {
+	objectsCh := make(chan string)
+
+	// Send object names that are needed to be removed to objectsCh
+	go func() {
+		defer close(objectsCh)
+		// List all objects from a bucket-name with a matching prefix.
+		for object := range m.client.ListObjects(m.bucket, dir, true, nil) {
+			if object.Err != nil {
+				log.Error("ListObjects failed:%v", object.Err)
+			}
+			objectsCh <- object.Key
+		}
+	}()
+
+	for rErr := range m.client.RemoveObjects(m.bucket, objectsCh) {
+		log.Error("Error detected during deletion: ", rErr)
+	}
+
+	return nil
+}
+
 //Get Presigned URL for get object
 func (m *MinioStorage) PresignedGetURL(path string, fileName string) (string, error) {
 	// Set request parameters for content-disposition.

modules/storage/storage.go

@@ -23,6 +23,7 @@ type ObjectStorage interface {
 	Save(path string, r io.Reader) (int64, error)
 	Open(path string) (io.ReadCloser, error)
 	Delete(path string) error
+	DeleteDir(dir string) error
 	PresignedGetURL(path string, fileName string) (string, error)
 	PresignedPutURL(path string) (string, error)
 	HasObject(path string) (bool, error)

routers/api/v1/repo/modelarts.go

@@ -7,6 +7,7 @@ package repo
 
 import (
 	"net/http"
+	"os"
 	"strconv"
 	"strings"
 
@@ -14,6 +15,7 @@ import (
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/modelarts"
+	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
 )
 
@@ -237,7 +239,7 @@ func DelTrainJobVersion(ctx *context.APIContext) {
 		JobID:   jobID,
 	})
 	if err != nil {
-		ctx.ServerError("get VersionListCount faild", err)
+		ctx.ServerError("get VersionListCount failed", err)
 		return
 	}
 	if VersionListCount > 0 {
@@ -255,6 +257,8 @@ func DelTrainJobVersion(ctx *context.APIContext) {
 				return
 			}
 		}
+	} else { //已删除该任务下的所有版本
+		deleteJobStorage(task.JobName)
 	}
 
 	ctx.JSON(http.StatusOK, map[string]interface{}{
@@ -321,3 +325,21 @@ func ModelList(ctx *context.APIContext) {
 		"PageIsCloudBrain": true,
 	})
 }
+
+func deleteJobStorage(jobName string) error {
+	//delete local
+	localJobPath := setting.JobPath + jobName
+	err := os.RemoveAll(localJobPath)
+	if err != nil {
+		log.Error("RemoveAll(%s) failed:%v", localJobPath, err)
+	}
+
+	//delete oss
+	dirPath := setting.CodePathPrefix + jobName + "/"
+	err = storage.ObsRemoveObject(setting.Bucket, dirPath)
+	if err != nil {
+		log.Error("ObsRemoveObject(%s) failed:%v", localJobPath, err)
+	}
+
+	return nil
+}

routers/repo/cloudbrain.go

@@ -566,6 +566,8 @@ func CloudBrainDel(ctx *context.Context) {
 		ctx.ServerError("DeleteJob failed", err)
 		return
 	}
+
+	deleteJobStorage(task.JobName, models.TypeCloudBrainOne)
 	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob")
 }
 
@@ -840,6 +842,35 @@ func mkModelPath(modelPath string) error {
 	return nil
 }
 
+func deleteJobStorage(jobName string, cloudbrainType int) error {
+	//delete local
+	localJobPath := setting.JobPath + jobName
+	err := os.RemoveAll(localJobPath)
+	if err != nil {
+		log.Error("RemoveAll(%s) failed:%v", localJobPath, err)
+	}
+
+	//delete oss
+	if cloudbrainType == models.TypeCloudBrainOne {
+		dirPath := setting.CBCodePathPrefix + jobName + "/"
+		err = storage.Attachments.DeleteDir(dirPath)
+		if err != nil {
+			log.Error("DeleteDir(%s) failed:%v", localJobPath, err)
+		}
+	} else if cloudbrainType == models.TypeCloudBrainTwo {
+		//dirPath := setting.CodePathPrefix + jobName + "/"
+		//err = storage.ObsRemoveObject(setting.Bucket, dirPath)
+		//if err != nil {
+		//	log.Error("ObsRemoveObject(%s) failed:%v", localJobPath, err)
+		//}
+		log.Info("no need to delete")
+	} else {
+		log.Error("cloudbrainType(%d) error", cloudbrainType)
+	}
+
+	return nil
+}
+
 func SyncCloudbrainStatus() {
 	cloudBrains, err := models.GetCloudBrainUnStoppedJob()
 	if err != nil {