清理云脑一历史文件

models/cloudbrain.go

@@ -204,6 +204,7 @@ type Cloudbrain struct {
 	BenchmarkTypeRankLink string      `xorm:"-"`
 	StartTime             timeutil.TimeStamp
 	EndTime               timeutil.TimeStamp
+	Cleared               bool   `xorm:"DEFAULT false"`
 	Spec                  *Specification `xorm:"-"`
 }
 
@@ -2081,7 +2082,8 @@ func UpdateCloudBrainRecordsCleared(ids []int64) error {
 					idsIn += "," + strconv.FormatInt(id, 10)
 				}
 			}
-			_, errTemp := x.Exec("update cloudbrain set cleared=true where id in (?)", ids)
+
+			_, errTemp := x.Unscoped().Exec("update cloudbrain set cleared=true where id in (" + idsIn + ")")
 			if errTemp != nil {
 				err = errTemp
 			}

modules/setting/setting.go

@@ -617,10 +617,10 @@ var (
 	ClearStrategy= struct {
 		Enabled bool
 		ResultSaveDays         int
-		ResultBatchSize        int
 		BatchSize int
 		TrashSaveDays int
 		Cron string
+		RunAtStart bool
 	}{}
 
 	C2NetInfos                 *C2NetSqInfos
@@ -1629,6 +1629,7 @@ func NewContext() {
 	getModelConvertConfig()
 	getModelSafetyConfig()
 	getModelAppConfig()
+	getClearStrategy()
 }
 
 func getModelSafetyConfig() {
@@ -1697,6 +1698,7 @@ func getClearStrategy(){
 	ClearStrategy.BatchSize=sec.Key("BATCH_SIZE").MustInt(500)
 	ClearStrategy.TrashSaveDays=sec.Key("TRASH_SAVE_DAYS").MustInt(90)
 	ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ? *")
+	ClearStrategy.RunAtStart=sec.Key("RUN_AT_START").MustBool(false)
 }
 
 func getGrampusConfig() {

options/locale/locale_en-US.ini

@@ -3246,6 +3246,7 @@ specification = specification
 select_specification = select specification
 description = description
 wrong_specification=You cannot use this specification, please choose another item.
+result_cleared=The files of the task have been cleared, can not restart any more, please create a new debug task instead.
 resource_use=Resource Occupancy
 
 job_name_rule = Please enter letters, numbers, _ and - up to 64 characters and cannot end with a dash (-).

options/locale/locale_zh-CN.ini

@@ -3266,6 +3266,8 @@ card_duration = 运行卡时
 card_type = 卡类型
 wrong_specification=您目前不能使用这个资源规格，请选择其他资源规格。
 
+result_cleared=本任务的文件已被清理，无法再次调试，请新建调试任务。
+
 job_name_rule = 请输入字母、数字、_和-，最长64个字符，且不能以中划线（-）结尾。
 train_dataset_path_rule = 数据集位置存储在运行参数 <strong style="color:#010101">data_url</strong> 中，预训练模型存放在运行参数 <strong style="color:#010101">ckpt_url</strong> 中，训练输出路径存储在运行参数 <strong style="color:#010101">train_url</strong> 中。
 infer_dataset_path_rule = 数据集位置存储在运行参数 <strong style="color:#010101">data_url</strong> 中，推理输出路径存储在运行参数 <strong style="color:#010101">result_url</strong> 中。

routers/repo/cloudbrain.go

@@ -670,6 +670,13 @@ func CloudBrainRestart(ctx *context.Context) {
 			break
 		}
 
+		if _, err := os.Stat(getOldJobPath(task)); err != nil {
+			log.Error("Can not find job minio path", err)
+			resultCode = "-1"
+			errorMsg = ctx.Tr("cloudbrain.result_cleared")
+			break
+		}
+
 		count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, string(models.JobTypeDebug))
 		if err != nil {
 			log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
@@ -704,6 +711,11 @@ func CloudBrainRestart(ctx *context.Context) {
 	})
 }
 
+
+func getOldJobPath(task *models.Cloudbrain) string {
+	return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + task.JobName
+}
+
 func CloudBrainBenchMarkShow(ctx *context.Context) {
 	cloudBrainShow(ctx, tplCloudBrainBenchmarkShow, models.JobTypeBenchmark)
 }

services/cloudbrain/clear.go

@@ -13,8 +13,11 @@ import (
 )
 
 func ClearCloudbrainResultSpace() {
+	if !setting.ClearStrategy.Enabled{
+		return
+	}
 
-	tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.ResultBatchSize)
+	tasks, err := models.GetCloudBrainOneStoppedJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.BatchSize)
 
 	if err != nil {
 		log.Warn("Failed to get cloudbrain, clear result failed.", err)
@@ -32,44 +35,60 @@ func ClearCloudbrainResultSpace() {
 	if err != nil {
 		log.Warn("Failed to set cloudbrain cleared status", err)
 	}
-	if len(tasks) < setting.ClearStrategy.ResultBatchSize { //如果云脑表处理完了，通过遍历minio对象处理历史垃圾数据，如果存在
-		files, err := ioutil.ReadDir(setting.JobPath)
-		processCount:=0
-		if err != nil {
-			log.Warn("Can not browser local job path.")
-		} else {
-			SortModTimeAscend(files)
-			for _, file := range files {
-				//清理n天前的历史垃圾数据，清理job目录
-				if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {
-					os.RemoveAll(setting.JobPath + file.Name())
-					processCount++
-				}
-				if processCount==setting.ClearStrategy.ResultBatchSize{
+	//如果云脑表处理完了，通过遍历minio对象处理历史垃圾数据，如果存在的话
+	if len(tasks) < setting.ClearStrategy.BatchSize {
+		clearLocalHistoryTrashFile()
+		clearMinioHistoryTrashFile()
+
+	}
+
+}
+
+func clearMinioHistoryTrashFile() {
+	JobRealPrefix := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix
+
+	miniofiles, err := ioutil.ReadDir(JobRealPrefix)
+
+	processCount := 0
+	if err != nil {
+		log.Warn("Can not browser minio job path.")
+	} else {
+		SortModTimeAscend(miniofiles)
+		for _, file := range miniofiles {
+
+			if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {
+				dirPath := setting.CBCodePathPrefix + file.Name() + "/"
+				storage.Attachments.DeleteDir(dirPath)
+				processCount++
+				if processCount == setting.ClearStrategy.BatchSize {
 					break
 				}
+			} else {
+				break
 			}
 
 		}
 
-		minioFiles, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, setting.CBCodePathPrefix)
-		processCount=0
-		if err != nil {
-			log.Warn("Can not browser minio job path.")
-		} else {
-			SortModTimeAscendForMinio(minioFiles)
-			for _, file := range minioFiles {
-
-				timeConvert, err := time.Parse("2006-01-02 15:04:05", file.ModTime)
-				if err == nil && timeConvert.Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {
-					dirPath := setting.CBCodePathPrefix + file.FileName + "/"
-					log.Info(dirPath)
-					storage.Attachments.DeleteDir(dirPath)
-					processCount++
-					if processCount==setting.ClearStrategy.ResultBatchSize{
-						break
-					}
+	}
+}
+
+func clearLocalHistoryTrashFile()  {
+	files, err := ioutil.ReadDir(setting.JobPath)
+	processCount := 0
+	if err != nil {
+		log.Warn("Can not browser local job path.")
+	} else {
+		SortModTimeAscend(files)
+		for _, file := range files {
+			//清理n天前的历史垃圾数据，清理job目录
+			if file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) {
+				os.RemoveAll(setting.JobPath + file.Name())
+				processCount++
+				if processCount == setting.ClearStrategy.BatchSize {
+					break
 				}
+			} else {
+				break
 			}
 
 		}