update

models/cloudbrain_static.go

@@ -1,9 +1,11 @@
 package models
 
 import (
+	"fmt"
 	"strconv"
 	"time"
 
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/timeutil"
 )
 
@@ -11,12 +13,24 @@ import (
 type CloudbrainStatistic struct {
 	ID              int64  `xorm:"pk autoincr" json:"-"`
 	Date            string `xorm:"unique(s) NOT NULL" json:"date"`
+	WhichHour       int64  `xorm:"NOT NULL DEFAULT -1" json:"whichHour"`
 	NumDubugOne     int64  `xorm:"NOT NULL DEFAULT 0" json:"numDubugOne"`
 	NumBenchmarkOne int64  `xorm:"NOT NULL DEFAULT 0" json:"numBenchmarkOne"`
 	NumTrainOne     int64  `xorm:"NOT NULL DEFAULT 0" json:"numTrainOne"`
 	NumDubugTwo     int64  `xorm:"NOT NULL DEFAULT 0" json:"numDubugTwo"`
 	NumTrainTwo     int64  `xorm:"NOT NULL DEFAULT 0" json:"numTrainTwo"`
 	NumInferenceTwo int64  `xorm:"NOT NULL DEFAULT 0" json:"numInferenceTwo"`
+	NumCloudOneAll  int64  `xorm:"NOT NULL DEFAULT 0" json:"numCloudOneAll"`
+	NumCloudTwoAll  int64  `xorm:"NOT NULL DEFAULT 0" json:"numCloudTwoAll"`
+
+	DurationDubugOne     int64 `xorm:"NOT NULL DEFAULT 0" json:"durationDubugOne"`
+	DurationBenchmarkOne int64 `xorm:"NOT NULL DEFAULT 0" json:"durationBenchmarkOne"`
+	DurationTrainOne     int64 `xorm:"NOT NULL DEFAULT 0" json:"durationTrainOne"`
+	DurationDebugTwo     int64 `xorm:"NOT NULL DEFAULT 0" json:"durationDebugTwo"`
+	DurationTrainTwo     int64 `xorm:"NOT NULL DEFAULT 0" json:"durationTrainTwo"`
+	DurationInferenceTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"durationInferenceTwo"`
+	DurationCloudOneAll  int64 `xorm:"NOT NULL DEFAULT 0" json:"durationCloudOneAll"`
+	DurationCloudTwoAll  int64 `xorm:"NOT NULL DEFAULT 0" json:"durationCloudTwoAll"`
 
 	CreatedUnix timeutil.TimeStamp `xorm:"INDEX created" json:"-"`
 	UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"-"`
@@ -31,6 +45,32 @@ func GenerateDebugOneCount(beginTime time.Time, endTime time.Time) (int64, error
 
 	return x.SQL(countSql).Count()
 }
+func GetDebugOneDuration(beginTime time.Time, endTime time.Time) (int64, error) {
+	total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeDebug, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration")
+	if err != nil {
+		return 0, err
+	}
+
+	return total, nil
+}
+
+func GenerateTrainOneCount(beginTime time.Time, endTime time.Time) (int64, error) {
+	countSql := "SELECT count(*) FROM  " +
+		"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
+		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
+		" and job_type ='" + string(JobTypeTrain) + "'" +
+		" and type='" + strconv.Itoa(TypeCloudBrainOne) + "'"
+
+	return x.SQL(countSql).Count()
+}
+func GetTrainOneDuration(beginTime time.Time, endTime time.Time) (int64, error) {
+	total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeTrain, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration")
+	if err != nil {
+		return 0, err
+	}
+
+	return total, nil
+}
 
 func GenerateBenchmarkOneCount(beginTime time.Time, endTime time.Time) (int64, error) {
 	countSql := "SELECT count(*) FROM  " +
@@ -40,6 +80,14 @@ func GenerateBenchmarkOneCount(beginTime time.Time, endTime time.Time) (int64, e
 		" and type='" + strconv.Itoa(TypeCloudBrainOne) + "'"
 	return x.SQL(countSql).Count()
 }
+func GetBenchmarkOneDuration(beginTime time.Time, endTime time.Time) (int64, error) {
+	total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeBenchmark, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration")
+	if err != nil {
+		return 0, err
+	}
+
+	return total, nil
+}
 func GenerateDebugTwoCount(beginTime time.Time, endTime time.Time) (int64, error) {
 	countSql := "SELECT count(*) FROM  " +
 		"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
@@ -48,6 +96,13 @@ func GenerateDebugTwoCount(beginTime time.Time, endTime time.Time) (int64, error
 		" and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'"
 	return x.SQL(countSql).Count()
 }
+func GetDebugTwoDuration(beginTime time.Time, endTime time.Time) (int64, error) {
+	total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeDebug, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration")
+	if err != nil {
+		return 0, err
+	}
+	return total, nil
+}
 func GenerateTrainTwoCount(beginTime time.Time, endTime time.Time) (int64, error) {
 	countSql := "SELECT count(*) FROM  " +
 		"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
@@ -56,6 +111,13 @@ func GenerateTrainTwoCount(beginTime time.Time, endTime time.Time) (int64, error
 		" and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'"
 	return x.SQL(countSql).Count()
 }
+func GetTrainTwoDuration(beginTime time.Time, endTime time.Time) (int64, error) {
+	total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeTrain, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration")
+	if err != nil {
+		return 0, err
+	}
+	return total, nil
+}
 func GenerateInferenceTwoCount(beginTime time.Time, endTime time.Time) (int64, error) {
 	countSql := "SELECT count(*) FROM  " +
 		"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
@@ -64,3 +126,60 @@ func GenerateInferenceTwoCount(beginTime time.Time, endTime time.Time) (int64, e
 		" and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'"
 	return x.SQL(countSql).Count()
 }
+func GetInferenceTwoDuration(beginTime time.Time, endTime time.Time) (int64, error) {
+	total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeInference, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration")
+	if err != nil {
+		return 0, err
+	}
+	return total, nil
+}
+
+func DeleteCloudbrainStatisticDaily(date string) error {
+	sess := xStatistic.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return fmt.Errorf("Begin: %v", err)
+	}
+
+	if _, err := sess.Where("date = ?", date).Delete(&CloudbrainStatistic{}); err != nil {
+		return fmt.Errorf("Delete: %v", err)
+	}
+
+	if err := sess.Commit(); err != nil {
+		sess.Close()
+		return fmt.Errorf("Commit: %v", err)
+	}
+
+	sess.Close()
+	return nil
+}
+
+func GetHourStatTime(timeStr string, whichHour int64) (time.Time, time.Time) {
+	t, _ := time.Parse("2006-01-02", timeStr)
+	timeNumber := t.Unix()
+	beginTimeNumber := timeNumber - 8*60*60 + whichHour*60*60
+	endTimeNumber := beginTimeNumber + 1*60*60
+	beginTime := time.Unix(beginTimeNumber, 0)
+	endTime := time.Unix(endTimeNumber, 0)
+	log.Info("%s, %s", beginTime, endTime)
+
+	return beginTime, endTime
+}
+func GetDayStatTime(timeStr string) (time.Time, time.Time) {
+	t, _ := time.Parse("2006-01-02", timeStr)
+	timeNumber := t.Unix()
+	beginTimeNumber := timeNumber - 8*60*60
+	endTimeNumber := beginTimeNumber + 16*60*60
+	beginTime := time.Unix(beginTimeNumber, 0)
+	endTime := time.Unix(endTimeNumber, 0)
+	log.Info("%s, %s", beginTime, endTime)
+
+	return beginTime, endTime
+}
+
+func CreateCloudbrainStatistic(cloudbrainStat *CloudbrainStatistic) (err error) {
+	if _, err = xStatistic.Insert(cloudbrainStat); err != nil {
+		return err
+	}
+	return nil
+}

routers/repo/cloudbrain_statistic.go

@@ -0,0 +1,115 @@
+package repo
+
+import (
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/services/mailer"
+)
+
+func CloudbrainStatisticAuto() {
+	yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
+	CloudbrainStatisticDaily(yesterday)
+}
+
+func CloudbrainStatisticDaily(date string) {
+	log.Info("%s", date)
+	log.Info("begin Repo Statistic")
+	warnEmailMessage := "云脑任务统计信息入库失败，请尽快定位。"
+	if err := models.DeleteCloudbrainStatisticDaily(date); err != nil {
+		log.Error("DeleteCloudbrainStatisticDaily failed: %v", err.Error())
+		mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage)
+		return
+	}
+
+	//0 to 23 for each hour, 25 for all 24 hours
+	var WhichHour [25]int64 = [25]int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25}
+	for _, whichHour := range WhichHour {
+		log.Info("start statistic: %s", date)
+		var beginTime, endTime time.Time
+		beginTime, endTime = models.GetHourStatTime(date, whichHour)
+		if whichHour == 25 {
+			beginTime, endTime = models.GetDayStatTime(date)
+		}
+		numDubugOne, err := models.GenerateDebugOneCount(beginTime, endTime)
+		if err != nil {
+			log.Error("GenerateDebugOneCount failed(%s): %v", numDubugOne, err)
+		}
+		numBenchmarkOne, err := models.GenerateBenchmarkOneCount(beginTime, endTime)
+		if err != nil {
+			log.Error("GenerateBenchmarkOneCount failed(%s): %v", numBenchmarkOne, err)
+		}
+		numTrainOne, err := models.GenerateTrainOneCount(beginTime, endTime)
+		if err != nil {
+			log.Error("GenerateTrainOneCount failed(%s): %v", numTrainOne, err)
+		}
+		numDebugTwo, err := models.GenerateDebugTwoCount(beginTime, endTime)
+		if err != nil {
+			log.Error("GenerateDebugTwoCount failed(%s): %v", numDebugTwo, err)
+		}
+		numTrainTwo, err := models.GenerateTrainTwoCount(beginTime, endTime)
+		if err != nil {
+			log.Error("GenerateTrainTwoCount failed(%s): %v", numTrainTwo, err)
+		}
+		numInferenceTwo, err := models.GenerateInferenceTwoCount(beginTime, endTime)
+		if err != nil {
+			log.Error("GenerateInferenceTwoCount failed(%s): %v", numInferenceTwo, err)
+		}
+		numCloudOneAll := numDubugOne + numBenchmarkOne + numTrainOne
+		numCloudTwoAll := numDebugTwo + numTrainTwo + numInferenceTwo
+
+		durationDubugOne, err := models.GetDebugOneDuration(beginTime, endTime)
+		if err != nil {
+			log.Error("GetDebugOneDuration failed(%s): %v", durationDubugOne, err)
+		}
+		durationBenchmarkOne, err := models.GetBenchmarkOneDuration(beginTime, endTime)
+		if err != nil {
+			log.Error("GetBenchmarkOneDuration failed(%s): %v", durationBenchmarkOne, err)
+		}
+		durationTrainOne, err := models.GetTrainOneDuration(beginTime, endTime)
+		if err != nil {
+			log.Error("GetTrainOneDuration failed(%s): %v", durationTrainOne, err)
+		}
+		durationDebugTwo, err := models.GetDebugTwoDuration(beginTime, endTime)
+		if err != nil {
+			log.Error("GetDebugTwoDuration failed(%s): %v", durationDebugTwo, err)
+		}
+		durationTrainTwo, err := models.GetTrainTwoDuration(beginTime, endTime)
+		if err != nil {
+			log.Error("GetTrainTwoDuration failed(%s): %v", durationTrainTwo, err)
+		}
+		durationInferenceTwo, err := models.GetInferenceTwoDuration(beginTime, endTime)
+		if err != nil {
+			log.Error("GetInferenceTwoDuration failed(%s): %v", durationInferenceTwo, err)
+		}
+		durationCloudOneAll := durationDubugOne + durationBenchmarkOne + durationTrainOne
+		durationCloudTwoAll := durationDebugTwo + durationTrainTwo + durationInferenceTwo
+
+		err = models.CreateCloudbrainStatistic(&models.CloudbrainStatistic{
+			Date:                 date,
+			WhichHour:            whichHour,
+			NumDubugOne:          numDubugOne,
+			NumBenchmarkOne:      numBenchmarkOne,
+			NumTrainOne:          numTrainOne,
+			NumDubugTwo:          numDebugTwo,
+			NumTrainTwo:          numTrainTwo,
+			NumInferenceTwo:      numInferenceTwo,
+			NumCloudOneAll:       numCloudOneAll,
+			NumCloudTwoAll:       numCloudTwoAll,
+			DurationDubugOne:     durationDubugOne,
+			DurationBenchmarkOne: durationBenchmarkOne,
+			DurationTrainOne:     durationTrainOne,
+			DurationDebugTwo:     durationDebugTwo,
+			DurationTrainTwo:     durationTrainTwo,
+			DurationInferenceTwo: durationInferenceTwo,
+			DurationCloudOneAll:  durationCloudOneAll,
+			DurationCloudTwoAll:  durationCloudTwoAll,
+		})
+		if err != nil {
+			log.Error("CreateCloudbrainStatistic(%s) failed:%v", date, err.Error())
+			return
+		}
+	}
+}

routers/repo/repo_statistic.go

@@ -334,12 +334,3 @@ func UpdateRepoVisits(ctx *macaron.Context, repo *models.Repository, date string
 	}
 	return nil
 }
-
-func CloudbrainStatisticAuto() {
-	yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
-	CloudbrainStatisticDaily(yesterday)
-}
-
-func CloudbrainStatisticDaily(date string) {
-	return
-}