diff --git a/models/cloudbrain.go b/models/cloudbrain.go index abb804eef..2de66a834 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -204,6 +204,7 @@ type Cloudbrain struct { BenchmarkTypeRankLink string `xorm:"-"` StartTime timeutil.TimeStamp EndTime timeutil.TimeStamp + Cleared bool `xorm:"DEFAULT false"` Spec *Specification `xorm:"-"` } @@ -2081,7 +2082,8 @@ func UpdateCloudBrainRecordsCleared(ids []int64) error { idsIn += "," + strconv.FormatInt(id, 10) } } - _, errTemp := x.Exec("update cloudbrain set cleared=true where id in (?)", ids) + + _, errTemp := x.Unscoped().Exec("update cloudbrain set cleared=true where id in (" + idsIn + ")") if errTemp != nil { err = errTemp } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index b5ec488a8..8815c6dcf 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -617,10 +617,10 @@ var ( ClearStrategy= struct { Enabled bool ResultSaveDays int - ResultBatchSize int BatchSize int TrashSaveDays int Cron string + RunAtStart bool }{} C2NetInfos *C2NetSqInfos @@ -1629,6 +1629,7 @@ func NewContext() { getModelConvertConfig() getModelSafetyConfig() getModelAppConfig() + getClearStrategy() } func getModelSafetyConfig() { @@ -1697,6 +1698,7 @@ func getClearStrategy(){ ClearStrategy.BatchSize=sec.Key("BATCH_SIZE").MustInt(500) ClearStrategy.TrashSaveDays=sec.Key("TRASH_SAVE_DAYS").MustInt(90) ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ? *") + ClearStrategy.RunAtStart=sec.Key("RUN_AT_START").MustBool(false) } func getGrampusConfig() { diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 647bdb1ad..1ac1a0bd7 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3246,6 +3246,7 @@ specification = specification select_specification = select specification description = description wrong_specification=You cannot use this specification, please choose another item. +result_cleared=The files of the task have been cleared, can not restart any more, please create a new debug task instead. resource_use=Resource Occupancy job_name_rule = Please enter letters, numbers, _ and - up to 64 characters and cannot end with a dash (-). diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 8f9e6b664..792981c86 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3266,6 +3266,8 @@ card_duration = 运行卡时 card_type = 卡类型 wrong_specification=您目前不能使用这个资源规格，请选择其他资源规格。 +result_cleared=本任务的文件已被清理，无法再次调试，请新建调试任务。 + job_name_rule = 请输入字母、数字、_和-，最长64个字符，且不能以中划线（-）结尾。 train_dataset_path_rule = 数据集位置存储在运行参数 data_url 中，预训练模型存放在运行参数 ckpt_url 中，训练输出路径存储在运行参数 train_url 中。 infer_dataset_path_rule = 数据集位置存储在运行参数 data_url 中，推理输出路径存储在运行参数 result_url 中。 diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index d3d76f440..a23cd5462 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -670,6 +670,13 @@ func CloudBrainRestart(ctx *context.Context) { break } + if _, err := os.Stat(getOldJobPath(task)); err != nil { + log.Error("Can not find job minio path", err) + resultCode = "-1" + errorMsg = ctx.Tr("cloudbrain.result_cleared") + break + } + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, string(models.JobTypeDebug)) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -704,6 +711,11 @@ func CloudBrainRestart(ctx *context.Context) { }) } + +func getOldJobPath(task *models.Cloudbrain) string { + return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + task.JobName +} + func CloudBrainBenchMarkShow(ctx *context.Context) { cloudBrainShow(ctx, tplCloudBrainBenchmarkShow, models.JobTypeBenchmark) } diff --git a/services/cloudbrain/clear.go b/services/cloudbrain/clear.go index f8dfae30e..2cefb8341 100644 --- a/services/cloudbrain/clear.go +++ b/services/cloudbrain/clear.go @@ -13,8 +13,11 @@ import ( ) func ClearCloudbrainResultSpace() { + if !setting.ClearStrategy.Enabled{ + return + } - tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.ResultBatchSize) + tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.BatchSize) if err != nil { log.Warn("Failed to get cloudbrain, clear result failed.", err) @@ -32,44 +35,60 @@ func ClearCloudbrainResultSpace() { if err != nil { log.Warn("Failed to set cloudbrain cleared status", err) } - if len(tasks) < setting.ClearStrategy.ResultBatchSize { //如果云脑表处理完了，通过遍历minio对象处理历史垃圾数据，如果存在 - files, err := ioutil.ReadDir(setting.JobPath) - processCount:=0 - if err != nil { - log.Warn("Can not browser local job path.") - } else { - SortModTimeAscend(files) - for _, file := range files { - //清理n天前的历史垃圾数据，清理job目录 - if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { - os.RemoveAll(setting.JobPath + file.Name()) - processCount++ - } - if processCount==setting.ClearStrategy.ResultBatchSize{ + //如果云脑表处理完了，通过遍历minio对象处理历史垃圾数据，如果存在的话 + if len(tasks) < setting.ClearStrategy.BatchSize { + clearLocalHistoryTrashFile() + clearMinioHistoryTrashFile() + + } + +} + +func clearMinioHistoryTrashFile() { + JobRealPrefix := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + + miniofiles, err := ioutil.ReadDir(JobRealPrefix) + + processCount := 0 + if err != nil { + log.Warn("Can not browser minio job path.") + } else { + SortModTimeAscend(miniofiles) + for _, file := range miniofiles { + + if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { + dirPath := setting.CBCodePathPrefix + file.Name() + "/" + storage.Attachments.DeleteDir(dirPath) + processCount++ + if processCount == setting.ClearStrategy.BatchSize { break } + } else { + break } } - minioFiles, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, setting.CBCodePathPrefix) - processCount=0 - if err != nil { - log.Warn("Can not browser minio job path.") - } else { - SortModTimeAscendForMinio(minioFiles) - for _, file := range minioFiles { - - timeConvert, err := time.Parse("2006-01-02 15:04:05", file.ModTime) - if err == nil && timeConvert.Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { - dirPath := setting.CBCodePathPrefix + file.FileName + "/" - log.Info(dirPath) - storage.Attachments.DeleteDir(dirPath) - processCount++ - if processCount==setting.ClearStrategy.ResultBatchSize{ - break - } + } +} + +func clearLocalHistoryTrashFile() { + files, err := ioutil.ReadDir(setting.JobPath) + processCount := 0 + if err != nil { + log.Warn("Can not browser local job path.") + } else { + SortModTimeAscend(files) + for _, file := range files { + //清理n天前的历史垃圾数据，清理job目录 + if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { + os.RemoveAll(setting.JobPath + file.Name()) + processCount++ + if processCount == setting.ClearStrategy.BatchSize { + break } + } else { + break } }