From 85d03023bd7baaa53f3e8addad8368e758313585 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 2 Dec 2022 11:13:09 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cron/tasks_basic.go | 17 +++++- modules/setting/setting.go | 5 ++ services/cloudbrain/clear.go | 108 +++++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 services/cloudbrain/clear.go diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 985a82cdb..9b74c1637 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -5,10 +5,12 @@ package cron import ( - "code.gitea.io/gitea/modules/urfs_client/urchin" "context" "time" + "code.gitea.io/gitea/modules/urfs_client/urchin" + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" + "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/services/cloudbrain/resource" "code.gitea.io/gitea/services/reward" @@ -190,6 +192,17 @@ func registerHandleRepoAndUserStatistic() { }) } +func registerHandleClearCloudbrainResult() { + RegisterTaskFatal("handle_cloudbrain_one_result_clear", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "* 0,30 2-8 * * ? *", + }, func(ctx context.Context, _ *models.User, _ Config) error { + cloudbrainService.ClearCloudbrainResultSpace() + return nil + }) +} + func registerHandleSummaryStatistic() { RegisterTaskFatal("handle_summary_statistic", &BaseConfig{ Enabled: true, @@ -317,6 +330,6 @@ func initBasicTasks() { registerHandleModelSafetyTask() - registerHandleScheduleRecord() + registerHandleScheduleRecord() registerHandleCloudbrainDurationStatistic() } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index a473dad6a..2b2ded678 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -518,6 +518,8 @@ var ( MaxDatasetNum int CullIdleTimeout string CullInterval string + ResultSaveDays int64 + ResultBatchSize int //benchmark config IsBenchmarkEnabled bool @@ -1480,6 +1482,9 @@ func NewContext() { CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") CullInterval = sec.Key("CULL_INTERVAL").MustString("60") + ResultSaveDays = sec.Key("RESULT_SAVE_DAYS").MustInt64(30) + ResultBatchSize = sec.Key("BATCH_SIZE").MustInt(500) + sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) BenchmarkOwner = sec.Key("OWNER").MustString("") diff --git a/services/cloudbrain/clear.go b/services/cloudbrain/clear.go new file mode 100644 index 000000000..52e5c5cf1 --- /dev/null +++ b/services/cloudbrain/clear.go @@ -0,0 +1,108 @@ +package cloudbrain + +import ( + "io/ioutil" + "os" + "sort" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" +) + +func ClearCloudbrainResultSpace() { + + tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ResultSaveDays, setting.ResultBatchSize) + + if err != nil { + log.Warn("Failed to get cloudbrain, clear result failed.", err) + return + } + var ids []int64 + for _, task := range tasks { + err := DeleteCloudbrainOneJobStorage(task.JobName) + if err == nil { + ids = append(ids, task.ID) + } + } + + err = models.UpdateCloudBrainRecordsCleared(ids) + if err != nil { + log.Warn("Failed to set cloudbrain cleared status", err) + } + if len(tasks) < setting.ResultBatchSize { //云脑表处理完了，处理历史垃圾数据，如果存在 + files, err := ioutil.ReadDir(setting.JobPath) + if err != nil { + log.Warn("Can not browser local job path.") + } else { + SortModTimeAscend(files) + for _, file := range files { + //清理4个月前的任务 + if file.ModTime().Before(time.Now().AddDate(0, -4, 0)) { + //os.RemoveAll(setting.JobPath + file.Name()) + } else { + break + } + } + + } + + minioFiles, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, setting.CBCodePathPrefix) + + if err != nil { + log.Warn("Can not browser minio job path.") + } else { + SortModTimeAscendForMinio(minioFiles) + for _, file := range minioFiles { + //清理4个月前的任务 + timeConvert, err := time.Parse("2006-01-02 15:04:05", file.ModTime) + if err == nil && timeConvert.Before(time.Now().AddDate(0, -4, 0)) { + dirPath := setting.CBCodePathPrefix + file.FileName + "/" + log.Info(dirPath) + //storage.Attachments.DeleteDir(dirPath) + } else { + break + } + } + + } + + } + +} + +func SortModTimeAscend(files []os.FileInfo) { + sort.Slice(files, func(i, j int) bool { + return files[i].ModTime().Before(files[j].ModTime()) + }) +} +func SortModTimeAscendForMinio(files []storage.FileInfo) { + sort.Slice(files, func(i, j int) bool { + timeI, _ := time.Parse("2006-01-02 15:04:05", files[i].ModTime) + timeJ, _ := time.Parse("2006-01-02 15:04:05", files[i].ModTime) + return timeI.Before(timeJ) + }) +} + +func DeleteCloudbrainOneJobStorage(jobName string) error { + //delete local + localJobPath := setting.JobPath + jobName + err := os.RemoveAll(localJobPath) + if err != nil { + log.Error("RemoveAll(%s) failed:%v", localJobPath, err) + } + + dirPath := setting.CBCodePathPrefix + jobName + "/" + err1 := storage.Attachments.DeleteDir(dirPath) + + if err1 != nil { + log.Error("DeleteDir(%s) failed:%v", localJobPath, err) + } + if err == nil { + err = err1 + } + + return err +} From 9b565469d1d02ce020cfd4a7a36159ae32080198 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 2 Dec 2022 11:19:55 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 58 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 1e7c702ab..e492d9eb9 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2050,6 +2050,64 @@ func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) { Find(&cloudbrains) } +func GetCloudBrainOneStoppedJobDaysAgo(days int64, limit int) ([]*Cloudbrain, error) { + cloudbrains := make([]*Cloudbrain, 0, 10) + endTimeBefore := time.Now().Unix() - days*24*3600 + missEndTimeBefore := endTimeBefore - 24*3600 + return cloudbrains, x.Cols("id,job_name,job_id"). + In("status", + JobStopped, JobSucceeded, JobFailed, ModelArtsCreateFailed, ModelArtsStartFailed, ModelArtsUnavailable, ModelArtsResizFailed, ModelArtsDeleted, + ModelArtsStopped, ModelArtsTrainJobCanceled, ModelArtsTrainJobCheckFailed, ModelArtsTrainJobCompleted, ModelArtsTrainJobDeleteFailed, ModelArtsTrainJobDeployServiceFailed, + ModelArtsTrainJobFailed, ModelArtsTrainJobImageFailed, ModelArtsTrainJobKilled, ModelArtsTrainJobLost, ModelArtsTrainJobSubmitFailed, ModelArtsTrainJobSubmitModelFailed). + Where("((end_time is null and created_unix 0 { + idsIn := "" + for i, id := range tempIds { + if i == 0 { + idsIn += strconv.FormatInt(id, 10) + } else { + idsIn += "," + strconv.FormatInt(id, 10) + } + } + _, errTemp := x.Exec("update cloudbrain set cleared=true where id in (?)", ids) + if errTemp != nil { + err = errTemp + } + + } + + } + return err + +} + +func getPageIds(ids []int64, page int, pagesize int) []int64 { + begin := (page - 1) * pagesize + end := (page) * pagesize + + if begin > len(ids)-1 { + return []int64{} + } + if end > len(ids)-1 { + return ids[begin:] + } else { + return ids[begin:end] + } + +} + func GetStoppedJobWithNoDurationJob() ([]*Cloudbrain, error) { cloudbrains := make([]*Cloudbrain, 0) return cloudbrains, x. From 0111ef5d923ccbe1a46dcb0bff30a606ccb734eb Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 6 Dec 2022 08:52:00 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 4 ++-- modules/cron/tasks_basic.go | 4 +++- modules/setting/setting.go | 25 ++++++++++++++++++++----- services/cloudbrain/clear.go | 29 +++++++++++++++++------------ 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index e492d9eb9..abb804eef 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2050,9 +2050,9 @@ func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) { Find(&cloudbrains) } -func GetCloudBrainOneStoppedJobDaysAgo(days int64, limit int) ([]*Cloudbrain, error) { +func GetCloudBrainOneStoppedJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) { cloudbrains := make([]*Cloudbrain, 0, 10) - endTimeBefore := time.Now().Unix() - days*24*3600 + endTimeBefore := time.Now().Unix() - int64(days)*24*3600 missEndTimeBefore := endTimeBefore - 24*3600 return cloudbrains, x.Cols("id,job_name,job_id"). In("status", diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 9b74c1637..1813b1d50 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -5,6 +5,7 @@ package cron import ( + "code.gitea.io/gitea/modules/setting" "context" "time" @@ -196,7 +197,7 @@ func registerHandleClearCloudbrainResult() { RegisterTaskFatal("handle_cloudbrain_one_result_clear", &BaseConfig{ Enabled: true, RunAtStart: false, - Schedule: "* 0,30 2-8 * * ? *", + Schedule: setting.ClearStrategy.Cron, }, func(ctx context.Context, _ *models.User, _ Config) error { cloudbrainService.ClearCloudbrainResultSpace() return nil @@ -319,6 +320,7 @@ func initBasicTasks() { registerHandleRepoAndUserStatistic() registerHandleSummaryStatistic() + registerHandleClearCloudbrainResult() registerSyncCloudbrainStatus() registerHandleOrgStatistic() diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 2b2ded678..b5ec488a8 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -518,8 +518,7 @@ var ( MaxDatasetNum int CullIdleTimeout string CullInterval string - ResultSaveDays int64 - ResultBatchSize int + //benchmark config IsBenchmarkEnabled bool @@ -615,6 +614,15 @@ var ( UsageRateBeginTime string }{} + ClearStrategy= struct { + Enabled bool + ResultSaveDays int + ResultBatchSize int + BatchSize int + TrashSaveDays int + Cron string + }{} + C2NetInfos *C2NetSqInfos CenterInfos *AiCenterInfos C2NetMapInfo map[string]*C2NetSequenceInfo @@ -1482,9 +1490,6 @@ func NewContext() { CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") CullInterval = sec.Key("CULL_INTERVAL").MustString("60") - ResultSaveDays = sec.Key("RESULT_SAVE_DAYS").MustInt64(30) - ResultBatchSize = sec.Key("BATCH_SIZE").MustInt(500) - sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) BenchmarkOwner = sec.Key("OWNER").MustString("") @@ -1684,6 +1689,16 @@ func getModelartsCDConfig() { getNotebookFlavorInfos() } +func getClearStrategy(){ + + sec := Cfg.Section("clear_strategy") + ClearStrategy.Enabled=sec.Key("ENABLED").MustBool(false) + ClearStrategy.ResultSaveDays=sec.Key("RESULT_SAVE_DAYS").MustInt(30) + ClearStrategy.BatchSize=sec.Key("BATCH_SIZE").MustInt(500) + ClearStrategy.TrashSaveDays=sec.Key("TRASH_SAVE_DAYS").MustInt(90) + ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ? *") +} + func getGrampusConfig() { sec := Cfg.Section("grampus") diff --git a/services/cloudbrain/clear.go b/services/cloudbrain/clear.go index 52e5c5cf1..f8dfae30e 100644 --- a/services/cloudbrain/clear.go +++ b/services/cloudbrain/clear.go @@ -14,7 +14,7 @@ import ( func ClearCloudbrainResultSpace() { - tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ResultSaveDays, setting.ResultBatchSize) + tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.ResultBatchSize) if err != nil { log.Warn("Failed to get cloudbrain, clear result failed.", err) @@ -32,17 +32,20 @@ func ClearCloudbrainResultSpace() { if err != nil { log.Warn("Failed to set cloudbrain cleared status", err) } - if len(tasks) < setting.ResultBatchSize { //云脑表处理完了，处理历史垃圾数据，如果存在 + if len(tasks) < setting.ClearStrategy.ResultBatchSize { //如果云脑表处理完了，通过遍历minio对象处理历史垃圾数据，如果存在 files, err := ioutil.ReadDir(setting.JobPath) + processCount:=0 if err != nil { log.Warn("Can not browser local job path.") } else { SortModTimeAscend(files) for _, file := range files { - //清理4个月前的任务 - if file.ModTime().Before(time.Now().AddDate(0, -4, 0)) { - //os.RemoveAll(setting.JobPath + file.Name()) - } else { + //清理n天前的历史垃圾数据，清理job目录 + if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { + os.RemoveAll(setting.JobPath + file.Name()) + processCount++ + } + if processCount==setting.ClearStrategy.ResultBatchSize{ break } } @@ -50,20 +53,22 @@ func ClearCloudbrainResultSpace() { } minioFiles, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, setting.CBCodePathPrefix) - + processCount=0 if err != nil { log.Warn("Can not browser minio job path.") } else { SortModTimeAscendForMinio(minioFiles) for _, file := range minioFiles { - //清理4个月前的任务 + timeConvert, err := time.Parse("2006-01-02 15:04:05", file.ModTime) - if err == nil && timeConvert.Before(time.Now().AddDate(0, -4, 0)) { + if err == nil && timeConvert.Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { dirPath := setting.CBCodePathPrefix + file.FileName + "/" log.Info(dirPath) - //storage.Attachments.DeleteDir(dirPath) - } else { - break + storage.Attachments.DeleteDir(dirPath) + processCount++ + if processCount==setting.ClearStrategy.ResultBatchSize{ + break + } } } From 47913c8be53cfee7b07e75403203c5445e69adc6 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 7 Dec 2022 17:44:26 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E6=B8=85=E7=90=86=E4=BA=91=E8=84=91?= =?UTF-8?q?=E4=B8=80=E5=8E=86=E5=8F=B2=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 4 +- modules/setting/setting.go | 4 +- options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 2 + routers/repo/cloudbrain.go | 12 +++++ services/cloudbrain/clear.go | 83 ++++++++++++++++++++------------- 6 files changed, 72 insertions(+), 34 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index abb804eef..2de66a834 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -204,6 +204,7 @@ type Cloudbrain struct { BenchmarkTypeRankLink string `xorm:"-"` StartTime timeutil.TimeStamp EndTime timeutil.TimeStamp + Cleared bool `xorm:"DEFAULT false"` Spec *Specification `xorm:"-"` } @@ -2081,7 +2082,8 @@ func UpdateCloudBrainRecordsCleared(ids []int64) error { idsIn += "," + strconv.FormatInt(id, 10) } } - _, errTemp := x.Exec("update cloudbrain set cleared=true where id in (?)", ids) + + _, errTemp := x.Unscoped().Exec("update cloudbrain set cleared=true where id in (" + idsIn + ")") if errTemp != nil { err = errTemp } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index b5ec488a8..8815c6dcf 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -617,10 +617,10 @@ var ( ClearStrategy= struct { Enabled bool ResultSaveDays int - ResultBatchSize int BatchSize int TrashSaveDays int Cron string + RunAtStart bool }{} C2NetInfos *C2NetSqInfos @@ -1629,6 +1629,7 @@ func NewContext() { getModelConvertConfig() getModelSafetyConfig() getModelAppConfig() + getClearStrategy() } func getModelSafetyConfig() { @@ -1697,6 +1698,7 @@ func getClearStrategy(){ ClearStrategy.BatchSize=sec.Key("BATCH_SIZE").MustInt(500) ClearStrategy.TrashSaveDays=sec.Key("TRASH_SAVE_DAYS").MustInt(90) ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ? *") + ClearStrategy.RunAtStart=sec.Key("RUN_AT_START").MustBool(false) } func getGrampusConfig() { diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 647bdb1ad..1ac1a0bd7 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3246,6 +3246,7 @@ specification = specification select_specification = select specification description = description wrong_specification=You cannot use this specification, please choose another item. +result_cleared=The files of the task have been cleared, can not restart any more, please create a new debug task instead. resource_use=Resource Occupancy job_name_rule = Please enter letters, numbers, _ and - up to 64 characters and cannot end with a dash (-). diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 8f9e6b664..792981c86 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3266,6 +3266,8 @@ card_duration = 运行卡时 card_type = 卡类型 wrong_specification=您目前不能使用这个资源规格，请选择其他资源规格。 +result_cleared=本任务的文件已被清理，无法再次调试，请新建调试任务。 + job_name_rule = 请输入字母、数字、_和-，最长64个字符，且不能以中划线（-）结尾。 train_dataset_path_rule = 数据集位置存储在运行参数 data_url 中，预训练模型存放在运行参数 ckpt_url 中，训练输出路径存储在运行参数 train_url 中。 infer_dataset_path_rule = 数据集位置存储在运行参数 data_url 中，推理输出路径存储在运行参数 result_url 中。 diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index d3d76f440..a23cd5462 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -670,6 +670,13 @@ func CloudBrainRestart(ctx *context.Context) { break } + if _, err := os.Stat(getOldJobPath(task)); err != nil { + log.Error("Can not find job minio path", err) + resultCode = "-1" + errorMsg = ctx.Tr("cloudbrain.result_cleared") + break + } + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, string(models.JobTypeDebug)) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -704,6 +711,11 @@ func CloudBrainRestart(ctx *context.Context) { }) } + +func getOldJobPath(task *models.Cloudbrain) string { + return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + task.JobName +} + func CloudBrainBenchMarkShow(ctx *context.Context) { cloudBrainShow(ctx, tplCloudBrainBenchmarkShow, models.JobTypeBenchmark) } diff --git a/services/cloudbrain/clear.go b/services/cloudbrain/clear.go index f8dfae30e..2cefb8341 100644 --- a/services/cloudbrain/clear.go +++ b/services/cloudbrain/clear.go @@ -13,8 +13,11 @@ import ( ) func ClearCloudbrainResultSpace() { + if !setting.ClearStrategy.Enabled{ + return + } - tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.ResultBatchSize) + tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.BatchSize) if err != nil { log.Warn("Failed to get cloudbrain, clear result failed.", err) @@ -32,44 +35,60 @@ func ClearCloudbrainResultSpace() { if err != nil { log.Warn("Failed to set cloudbrain cleared status", err) } - if len(tasks) < setting.ClearStrategy.ResultBatchSize { //如果云脑表处理完了，通过遍历minio对象处理历史垃圾数据，如果存在 - files, err := ioutil.ReadDir(setting.JobPath) - processCount:=0 - if err != nil { - log.Warn("Can not browser local job path.") - } else { - SortModTimeAscend(files) - for _, file := range files { - //清理n天前的历史垃圾数据，清理job目录 - if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { - os.RemoveAll(setting.JobPath + file.Name()) - processCount++ - } - if processCount==setting.ClearStrategy.ResultBatchSize{ + //如果云脑表处理完了，通过遍历minio对象处理历史垃圾数据，如果存在的话 + if len(tasks) < setting.ClearStrategy.BatchSize { + clearLocalHistoryTrashFile() + clearMinioHistoryTrashFile() + + } + +} + +func clearMinioHistoryTrashFile() { + JobRealPrefix := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + + miniofiles, err := ioutil.ReadDir(JobRealPrefix) + + processCount := 0 + if err != nil { + log.Warn("Can not browser minio job path.") + } else { + SortModTimeAscend(miniofiles) + for _, file := range miniofiles { + + if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { + dirPath := setting.CBCodePathPrefix + file.Name() + "/" + storage.Attachments.DeleteDir(dirPath) + processCount++ + if processCount == setting.ClearStrategy.BatchSize { break } + } else { + break } } - minioFiles, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, setting.CBCodePathPrefix) - processCount=0 - if err != nil { - log.Warn("Can not browser minio job path.") - } else { - SortModTimeAscendForMinio(minioFiles) - for _, file := range minioFiles { - - timeConvert, err := time.Parse("2006-01-02 15:04:05", file.ModTime) - if err == nil && timeConvert.Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { - dirPath := setting.CBCodePathPrefix + file.FileName + "/" - log.Info(dirPath) - storage.Attachments.DeleteDir(dirPath) - processCount++ - if processCount==setting.ClearStrategy.ResultBatchSize{ - break - } + } +} + +func clearLocalHistoryTrashFile() { + files, err := ioutil.ReadDir(setting.JobPath) + processCount := 0 + if err != nil { + log.Warn("Can not browser local job path.") + } else { + SortModTimeAscend(files) + for _, file := range files { + //清理n天前的历史垃圾数据，清理job目录 + if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { + os.RemoveAll(setting.JobPath + file.Name()) + processCount++ + if processCount == setting.ClearStrategy.BatchSize { + break } + } else { + break } } From f9c9b888eb7435619b1cc01d12d837255977425a Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 7 Dec 2022 17:46:13 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/cron/tasks_basic.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 1813b1d50..6a1fc6e39 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -196,7 +196,7 @@ func registerHandleRepoAndUserStatistic() { func registerHandleClearCloudbrainResult() { RegisterTaskFatal("handle_cloudbrain_one_result_clear", &BaseConfig{ Enabled: true, - RunAtStart: false, + RunAtStart: setting.ClearStrategy.RunAtStart, Schedule: setting.ClearStrategy.Cron, }, func(ctx context.Context, _ *models.User, _ Config) error { cloudbrainService.ClearCloudbrainResultSpace()