Merge remote-tracking branch 'origin/V20221130' into fix-csh

models/cloudbrain_static.go

@@ -1,7 +1,6 @@
 package models
 
 import (
-	"fmt"
 	"strconv"
 	"time"
 
@@ -313,10 +312,6 @@ func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDu
 	return xStatistic.Insert(cloudbrainDurationStatistic)
 }
 
-func GetDurationStatisticByDate(date string, hour int, aiCenterCode string, accCardType string) (*CloudbrainDurationStatistic, error) {
-	cb := &CloudbrainDurationStatistic{DayTime: date, HourTime: hour, AiCenterCode: aiCenterCode, AccCardType: accCardType}
-	return getDurationStatistic(cb)
-}
 func getDurationStatistic(cb *CloudbrainDurationStatistic) (*CloudbrainDurationStatistic, error) {
 	has, err := x.Get(cb)
 	if err != nil {
@@ -327,26 +322,6 @@ func getDurationStatistic(cb *CloudbrainDurationStatistic) (*CloudbrainDurationS
 	return cb, nil
 }
 
-func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode string, accCardType string) error {
-	sess := xStatistic.NewSession()
-	defer sess.Close()
-	if err := sess.Begin(); err != nil {
-		return fmt.Errorf("Begin: %v", err)
-	}
-
-	if _, err := sess.Where("day_time = ? AND hour_time = ? AND ai_center_code = ? AND acc_card_type = ?", date, hour, aiCenterCode, accCardType).Delete(&CloudbrainDurationStatistic{}); err != nil {
-		return fmt.Errorf("Delete: %v", err)
-	}
-
-	if err := sess.Commit(); err != nil {
-		sess.Close()
-		return fmt.Errorf("Commit: %v", err)
-	}
-
-	sess.Close()
-	return nil
-}
-
 func GetCanUseCardInfo() ([]*ResourceQueue, error) {
 	sess := x.NewSession()
 	defer sess.Close()
@@ -402,11 +377,11 @@ func GetDurationRecordUpdateTime() ([]*CloudbrainDurationStatistic, error) {
 	return CloudbrainDurationStatistics, nil
 }
 
-func DeleteCloudbrainDurationStatistic() error {
+func DeleteCloudbrainDurationStatistic(beginTime timeutil.TimeStamp, endTime timeutil.TimeStamp) error {
 	sess := xStatistic.NewSession()
 	defer sess.Close()
-	if _, err := sess.Exec("TRUNCATE TABLE cloudbrain_duration_statistic"); err != nil {
-		log.Info("TRUNCATE cloudbrain_duration_statistic error.")
+	if _, err := sess.Exec("DELETE FROM cloudbrain_duration_statistic WHERE cloudbrain_duration_statistic.date_time BETWEEN ? AND ?", beginTime, endTime); err != nil {
+		log.Info("DELETE cloudbrain_duration_statistic data error.")
 		return err
 	}
 	return nil

routers/api/v1/repo/cloudbrain_dashboard.go

@@ -123,8 +123,9 @@ func GetOverviewDuration(ctx *context.Context) {
 	recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix
 	now := time.Now()
 	endTime := now
-	// worker_server_num := 1
-	// cardNum := 1
+	var workServerNumber int64
+	var cardNum int64
+
 	durationAllSum := int64(0)
 	cardDuSum := int64(0)
 
@@ -138,52 +139,60 @@ func GetOverviewDuration(ctx *context.Context) {
 	c2NetDuration := int64(0)
 	cDCenterDuration := int64(0)
 
-	cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{
-		Type:          models.TypeCloudBrainAll,
-		BeginTimeUnix: int64(recordBeginTime),
-		EndTimeUnix:   endTime.Unix(),
-	})
-	if err != nil {
-		ctx.ServerError("Get cloudbrains failed:", err)
-		return
-	}
-	models.LoadSpecs4CloudbrainInfo(cloudbrains)
-
-	for _, cloudbrain := range cloudbrains {
-		cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain)
-		CardDurationString := repo.GetCloudbrainCardDuration(cloudbrain.Cloudbrain)
-		CardDuration := models.ConvertStrToDuration(CardDurationString)
-		// if cloudbrain.Cloudbrain.WorkServerNumber >= 1 {
-		// 	worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber
-		// } else {
-		// 	worker_server_num = 1
-		// }
-		// if cloudbrain.Cloudbrain.Spec == nil {
-		// 	cardNum = 1
-		// } else {
-		// 	cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum
-		// }
-		// duration := cloudbrain.Duration
-		// duration := cloudbrain.Duration
-		duration := models.ConvertStrToDuration(cloudbrain.TrainJobDuration)
-		// CardDuration := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum)
-
-		if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne {
-			cloudBrainOneDuration += duration
-			cloudBrainOneCardDuSum += CardDuration
-		} else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo {
-			cloudBrainTwoDuration += duration
-			cloudBrainTwoCardDuSum += CardDuration
-		} else if cloudbrain.Cloudbrain.Type == models.TypeC2Net {
-			c2NetDuration += duration
-			c2NetCardDuSum += CardDuration
-		} else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter {
-			cDCenterDuration += duration
-			cDNetCardDuSum += CardDuration
+	page := 1
+	pagesize := 10000
+	count := pagesize
+	// Each time a maximum of 10000 pieces of data are detected to the memory, batch processing
+	for count == pagesize && count != 0 {
+		cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{
+			ListOptions: models.ListOptions{
+				Page:     page,
+				PageSize: pagesize,
+			},
+			Type:          models.TypeCloudBrainAll,
+			BeginTimeUnix: int64(recordBeginTime),
+			EndTimeUnix:   endTime.Unix(),
+		})
+		if err != nil {
+			ctx.ServerError("Get cloudbrains failed:", err)
+			return
 		}
+		models.LoadSpecs4CloudbrainInfo(cloudbrains)
+
+		for _, cloudbrain := range cloudbrains {
+			cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain)
+			if cloudbrain.Cloudbrain.Spec != nil {
+				cardNum = int64(cloudbrain.Cloudbrain.Spec.AccCardsNum)
+			} else {
+				cardNum = 1
+			}
+			if cloudbrain.Cloudbrain.WorkServerNumber >= 1 {
+				workServerNumber = int64(cloudbrain.Cloudbrain.WorkServerNumber)
+			} else {
+				workServerNumber = 1
+			}
+			duration := models.ConvertStrToDuration(cloudbrain.TrainJobDuration)
+			CardDuration := workServerNumber * int64(cardNum) * duration
+
+			if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne {
+				cloudBrainOneDuration += duration
+				cloudBrainOneCardDuSum += CardDuration
+			} else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo {
+				cloudBrainTwoDuration += duration
+				cloudBrainTwoCardDuSum += CardDuration
+			} else if cloudbrain.Cloudbrain.Type == models.TypeC2Net {
+				c2NetDuration += duration
+				c2NetCardDuSum += CardDuration
+			} else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter {
+				cDCenterDuration += duration
+				cDNetCardDuSum += CardDuration
+			}
 
-		durationAllSum += duration
-		cardDuSum += CardDuration
+			durationAllSum += duration
+			cardDuSum += CardDuration
+		}
+		count = len(cloudbrains)
+		page += 1
 	}
 	ctx.JSON(http.StatusOK, map[string]interface{}{
 		"cloudBrainOneCardDuSum": cloudBrainOneCardDuSum,

routers/repo/cloudbrain_statistic.go

@@ -8,7 +8,6 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
 	cloudbrainService "code.gitea.io/gitea/services/cloudbrain"
 )
@@ -16,19 +15,23 @@ import (
 func CloudbrainDurationStatisticHour() {
 	var statisticTime time.Time
 	var count int64
-	recordBeginTime, _ := time.ParseInLocation("2006-01-02 15:04:05", setting.Grampus.UsageRateBeginTime, time.Local)
 	recordDurationUpdateTime, err := models.GetDurationRecordUpdateTime()
 	if err != nil {
 		log.Error("Can not get GetDurationRecordBeginTime", err)
-		return
-	}
-	if recordDurationUpdateTime == nil {
-		statisticTime = recordBeginTime
-	} else {
-		statisticTime = time.Unix(int64(recordDurationUpdateTime[0].DateTime), 0)
 	}
 	now := time.Now()
 	currentTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location())
+	if err == nil && len(recordDurationUpdateTime) > 0 {
+		statisticTime = time.Unix(int64(recordDurationUpdateTime[0].DateTime), 0).Add(+1 * time.Hour)
+	} else {
+		statisticTime = currentTime
+	}
+	deleteBeginTime := time.Unix(int64(recordDurationUpdateTime[0].DateTime), 0)
+
+	err = models.DeleteCloudbrainDurationStatistic(timeutil.TimeStamp(deleteBeginTime.Unix()), timeutil.TimeStamp(currentTime.Unix()))
+	if err != nil {
+		log.Error("DeleteCloudbrainDurationStatistic failed", err)
+	}
 
 	for statisticTime.Before(currentTime) || statisticTime.Equal(currentTime) {
 		countEach := summaryDurationStat(statisticTime)
@@ -37,13 +40,10 @@ func CloudbrainDurationStatisticHour() {
 	}
 	log.Info("summaryDurationStat count: %v", count)
 }
-func UpdateDurationStatisticHistoryData() int64 {
+func UpdateDurationStatisticHistoryData(beginTime time.Time, endTime time.Time) int64 {
 	var count int64
-	recordBeginTime, _ := time.ParseInLocation("2006-01-02 15:04:05", setting.Grampus.UsageRateBeginTime, time.Local)
-	now := time.Now()
-	currentTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location())
-	statisticTime := recordBeginTime.Add(+1 * time.Hour)
-
+	statisticTime := beginTime
+	currentTime := endTime
 	for statisticTime.Before(currentTime) || statisticTime.Equal(currentTime) {
 		countEach := summaryDurationStat(statisticTime)
 		count += countEach
@@ -107,13 +107,6 @@ func summaryDurationStat(statisticTime time.Time) int64 {
 		for cardType, cardDuration := range CardTypes {
 			cloudbrainTable := cloudbrainMap[centerCode+"/"+cardType]
 			if cloudbrainTable != nil {
-				if _, err := models.GetDurationStatisticByDate(dayTime, hourTime, centerCode, cardType); err == nil {
-					if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType); err != nil {
-						log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error())
-						return 0
-					}
-				}
-
 				if _, ok := cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType]; !ok {
 					cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType] = 0
 				}
@@ -139,12 +132,6 @@ func summaryDurationStat(statisticTime time.Time) int64 {
 	}
 
 	for key, cardsTotalDuration := range cardsTotalDurationMap {
-		if _, err := models.GetDurationStatisticByDate(dayTime, hourTime, strings.Split(key, "/")[1], strings.Split(key, "/")[2]); err == nil {
-			if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, strings.Split(key, "/")[1], strings.Split(key, "/")[2]); err != nil {
-				log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error())
-				return 0
-			}
-		}
 		cloudbrainDurationStat := models.CloudbrainDurationStatistic{
 			DateTime:           dateTime,
 			DayTime:            dayTime,
@@ -257,8 +244,22 @@ func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, be
 }
 
 func CloudbrainUpdateHistoryData(ctx *context.Context) {
-	err := models.DeleteCloudbrainDurationStatistic()
-	count := UpdateDurationStatisticHistoryData()
+	beginTimeStr := ctx.QueryTrim("beginTime")
+	endTimeStr := ctx.QueryTrim("endTime")
+	var count int64
+	var err error
+	if beginTimeStr != "" && endTimeStr != "" {
+		beginTime, _ := time.ParseInLocation("2006-01-02 15:04:05", beginTimeStr, time.Local)
+		endTime, _ := time.ParseInLocation("2006-01-02 15:04:05", endTimeStr, time.Local)
+		if time.Now().Before(endTime) {
+			endTime = time.Now()
+		}
+		beginTimeUnix := timeutil.TimeStamp(beginTime.Unix())
+		endTimeUnix := timeutil.TimeStamp(endTime.Unix())
+
+		err = models.DeleteCloudbrainDurationStatistic(beginTimeUnix, endTimeUnix)
+		count = UpdateDurationStatisticHistoryData(beginTime.Add(+1*time.Hour), endTime)
+	}
 	ctx.JSON(http.StatusOK, map[string]interface{}{
 		"message": 0,
 		"count":   count,