From a3a78ee8858d069cd01f4e48fc608e6c4d95a103 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 14 Dec 2021 17:52:55 +0800 Subject: [PATCH 1/4] del job --- modules/storage/local.go | 4 ++++ modules/storage/minio.go | 25 +++++++++++++++++++++++++ modules/storage/storage.go | 1 + routers/repo/cloudbrain.go | 29 +++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+) mode change 100644 => 100755 modules/storage/local.go diff --git a/modules/storage/local.go b/modules/storage/local.go old mode 100644 new mode 100755 index d46a5528d..df5807f87 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -80,3 +80,7 @@ func (l *LocalStorage) HasObject(path string) (bool, error) { func (l *LocalStorage) UploadObject(fileName, filePath string) error { return nil } + +func (l *LocalStorage) DeleteDir(dir string) error { + return nil +} diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 664e58d1b..bdc2b6eb0 100755 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/log" + "github.com/minio/minio-go" ) @@ -76,6 +78,29 @@ func (m *MinioStorage) Delete(path string) error { return m.client.RemoveObject(m.bucket, m.buildMinioPath(path)) } +// Delete delete a file +func (m *MinioStorage) DeleteDir(dir string) error { + objectsCh := make(chan string) + + // Send object names that are needed to be removed to objectsCh + go func() { + defer close(objectsCh) + // List all objects from a bucket-name with a matching prefix. + for object := range m.client.ListObjects(m.bucket, dir, true, nil) { + if object.Err != nil { + log.Error("ListObjects failed:%v", object.Err) + } + objectsCh <- object.Key + } + }() + + for rErr := range m.client.RemoveObjects(m.bucket, objectsCh) { + log.Error("Error detected during deletion: ", rErr) + } + + return nil +} + //Get Presigned URL for get object func (m *MinioStorage) PresignedGetURL(path string, fileName string) (string, error) { // Set request parameters for content-disposition. diff --git a/modules/storage/storage.go b/modules/storage/storage.go index d364346f0..fc9d38020 100755 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -23,6 +23,7 @@ type ObjectStorage interface { Save(path string, r io.Reader) (int64, error) Open(path string) (io.ReadCloser, error) Delete(path string) error + DeleteDir(dir string) error PresignedGetURL(path string, fileName string) (string, error) PresignedPutURL(path string) (string, error) HasObject(path string) (bool, error) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 3f5fce013..cc4eddd51 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -478,6 +478,9 @@ func CloudBrainDel(ctx *context.Context) { ctx.ServerError("DeleteJob failed", err) return } + + //todo: delete local and oss's job + deleteJobName(task.JobName, models.TypeCloudBrainOne) ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob") } @@ -756,6 +759,32 @@ func mkModelPath(modelPath string) error { return nil } +func deleteJobName(jobName string, cloudbrainType int) error { + //delete local + localJobPath := setting.JobPath + jobName + log.Info("%s", localJobPath) + err := os.RemoveAll(localJobPath) + if err != nil { + log.Error("RemoveAll(%s) failed:%v", localJobPath, err) + } + + //delete oss + if cloudbrainType == models.TypeCloudBrainOne { + dirPath := setting.CBCodePathPrefix + jobName + "/" + log.Info("%s", dirPath) + err = storage.Attachments.DeleteDir(dirPath) + if err != nil { + log.Error("Delete(%s) failed:%v", localJobPath, err) + } + } else if cloudbrainType == models.TypeCloudBrainTwo { + + } else { + log.Error("cloudbrainType(%d) error", cloudbrainType) + } + + return nil +} + func SyncCloudbrainStatus() { cloudBrains, err := models.GetCloudBrainUnStoppedJob() if err != nil { From 2d0b53b901cc0d061d9800fd419a6cc3414ff14c Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 15 Dec 2021 16:23:35 +0800 Subject: [PATCH 2/4] delete cb --- routers/repo/cloudbrain.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index cc4eddd51..3686419be 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -479,7 +479,6 @@ func CloudBrainDel(ctx *context.Context) { return } - //todo: delete local and oss's job deleteJobName(task.JobName, models.TypeCloudBrainOne) ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob") } @@ -762,7 +761,6 @@ func mkModelPath(modelPath string) error { func deleteJobName(jobName string, cloudbrainType int) error { //delete local localJobPath := setting.JobPath + jobName - log.Info("%s", localJobPath) err := os.RemoveAll(localJobPath) if err != nil { log.Error("RemoveAll(%s) failed:%v", localJobPath, err) @@ -771,13 +769,17 @@ func deleteJobName(jobName string, cloudbrainType int) error { //delete oss if cloudbrainType == models.TypeCloudBrainOne { dirPath := setting.CBCodePathPrefix + jobName + "/" - log.Info("%s", dirPath) err = storage.Attachments.DeleteDir(dirPath) if err != nil { - log.Error("Delete(%s) failed:%v", localJobPath, err) + log.Error("DeleteDir(%s) failed:%v", localJobPath, err) } } else if cloudbrainType == models.TypeCloudBrainTwo { - + //dirPath := setting.CodePathPrefix + jobName + "/" + //err = storage.ObsRemoveObject(setting.Bucket, dirPath) + //if err != nil { + // log.Error("ObsRemoveObject(%s) failed:%v", localJobPath, err) + //} + log.Info("no need to delete") } else { log.Error("cloudbrainType(%d) error", cloudbrainType) } From 027d476a66d72d2c8763e398f2c420eb22c608c4 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 22 Dec 2021 17:22:24 +0800 Subject: [PATCH 3/4] del train job storage --- routers/api/v1/repo/modelarts.go | 24 +++++++++++++++++++++++- routers/repo/cloudbrain.go | 4 ++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index a18da3fe5..82d32951b 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -6,7 +6,9 @@ package repo import ( + "code.gitea.io/gitea/modules/setting" "net/http" + "os" "strconv" "strings" @@ -237,7 +239,7 @@ func DelTrainJobVersion(ctx *context.APIContext) { JobID: jobID, }) if err != nil { - ctx.ServerError("get VersionListCount faild", err) + ctx.ServerError("get VersionListCount failed", err) return } if VersionListCount > 0 { @@ -255,6 +257,8 @@ func DelTrainJobVersion(ctx *context.APIContext) { return } } + } else { //已删除该任务下的所有版本 + deleteJobStorage(task.JobName) } ctx.JSON(http.StatusOK, map[string]interface{}{ @@ -321,3 +325,21 @@ func ModelList(ctx *context.APIContext) { "PageIsCloudBrain": true, }) } + +func deleteJobStorage(jobName string) error { + //delete local + localJobPath := setting.JobPath + jobName + err := os.RemoveAll(localJobPath) + if err != nil { + log.Error("RemoveAll(%s) failed:%v", localJobPath, err) + } + + //delete oss + dirPath := setting.CodePathPrefix + jobName + "/" + err = storage.ObsRemoveObject(setting.Bucket, dirPath) + if err != nil { + log.Error("ObsRemoveObject(%s) failed:%v", localJobPath, err) + } + + return nil +} diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 5a68c298f..5503c7b35 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -567,7 +567,7 @@ func CloudBrainDel(ctx *context.Context) { return } - deleteJobName(task.JobName, models.TypeCloudBrainOne) + deleteJobStorage(task.JobName, models.TypeCloudBrainOne) ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob") } @@ -842,7 +842,7 @@ func mkModelPath(modelPath string) error { return nil } -func deleteJobName(jobName string, cloudbrainType int) error { +func deleteJobStorage(jobName string, cloudbrainType int) error { //delete local localJobPath := setting.JobPath + jobName err := os.RemoveAll(localJobPath) From 2d86005bf6fa2516bd3db6f0e9d3e7a63022d357 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 22 Dec 2021 17:26:40 +0800 Subject: [PATCH 4/4] opt --- routers/api/v1/repo/modelarts.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 82d32951b..676097465 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -6,7 +6,6 @@ package repo import ( - "code.gitea.io/gitea/modules/setting" "net/http" "os" "strconv" @@ -16,6 +15,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" ) @@ -257,7 +257,7 @@ func DelTrainJobVersion(ctx *context.APIContext) { return } } - } else { //已删除该任务下的所有版本 + } else { //已删除该任务下的所有版本 deleteJobStorage(task.JobName) } @@ -340,6 +340,6 @@ func deleteJobStorage(jobName string) error { if err != nil { log.Error("ObsRemoveObject(%s) failed:%v", localJobPath, err) } - + return nil }