merge fix-1656

models/cloudbrain_static.go

@@ -0,0 +1,185 @@
+package models
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/timeutil"
+)
+
+// Cloudbrain statistic info of all CloudbrainTasks
+type CloudbrainStatistic struct {
+	ID              int64  `xorm:"pk autoincr" json:"-"`
+	Date            string `xorm:"NOT NULL DEFAULT 0" json:"date"`
+	WhichHour       int64  `xorm:"NOT NULL DEFAULT -1" json:"whichHour"`
+	NumDubugOne     int64  `xorm:"NOT NULL DEFAULT 0" json:"numDubugOne"`
+	NumBenchmarkOne int64  `xorm:"NOT NULL DEFAULT 0" json:"numBenchmarkOne"`
+	NumTrainOne     int64  `xorm:"NOT NULL DEFAULT 0" json:"numTrainOne"`
+	NumDubugTwo     int64  `xorm:"NOT NULL DEFAULT 0" json:"numDubugTwo"`
+	NumTrainTwo     int64  `xorm:"NOT NULL DEFAULT 0" json:"numTrainTwo"`
+	NumInferenceTwo int64  `xorm:"NOT NULL DEFAULT 0" json:"numInferenceTwo"`
+	NumCloudOneAll  int64  `xorm:"NOT NULL DEFAULT 0" json:"numCloudOneAll"`
+	NumCloudTwoAll  int64  `xorm:"NOT NULL DEFAULT 0" json:"numCloudTwoAll"`
+
+	DurationDubugOne     int64 `xorm:"NOT NULL DEFAULT 0" json:"durationDubugOne"`
+	DurationBenchmarkOne int64 `xorm:"NOT NULL DEFAULT 0" json:"durationBenchmarkOne"`
+	DurationTrainOne     int64 `xorm:"NOT NULL DEFAULT 0" json:"durationTrainOne"`
+	DurationDebugTwo     int64 `xorm:"NOT NULL DEFAULT 0" json:"durationDebugTwo"`
+	DurationTrainTwo     int64 `xorm:"NOT NULL DEFAULT 0" json:"durationTrainTwo"`
+	DurationInferenceTwo int64 `xorm:"NOT NULL DEFAULT 0" json:"durationInferenceTwo"`
+	DurationCloudOneAll  int64 `xorm:"NOT NULL DEFAULT 0" json:"durationCloudOneAll"`
+	DurationCloudTwoAll  int64 `xorm:"NOT NULL DEFAULT 0" json:"durationCloudTwoAll"`
+
+	CreatedUnix timeutil.TimeStamp `xorm:"INDEX created" json:"-"`
+	UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated" json:"-"`
+}
+
+func GenerateDebugOneCount(beginTime time.Time, endTime time.Time) (int64, error) {
+	countSql := "SELECT count(*) FROM  " +
+		"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
+		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
+		" and job_type ='" + string(JobTypeDebug) + "'" +
+		" and type='" + strconv.Itoa(TypeCloudBrainOne) + "'"
+
+	return x.SQL(countSql).Count()
+}
+func GetDebugOneDuration(beginTime time.Time, endTime time.Time) (int64, error) {
+	total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeDebug, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration")
+	if err != nil {
+		return 0, err
+	}
+
+	return total, nil
+}
+
+func GenerateTrainOneCount(beginTime time.Time, endTime time.Time) (int64, error) {
+	countSql := "SELECT count(*) FROM  " +
+		"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
+		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
+		" and job_type ='" + string(JobTypeTrain) + "'" +
+		" and type='" + strconv.Itoa(TypeCloudBrainOne) + "'"
+
+	return x.SQL(countSql).Count()
+}
+func GetTrainOneDuration(beginTime time.Time, endTime time.Time) (int64, error) {
+	total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeTrain, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration")
+	if err != nil {
+		return 0, err
+	}
+
+	return total, nil
+}
+
+func GenerateBenchmarkOneCount(beginTime time.Time, endTime time.Time) (int64, error) {
+	countSql := "SELECT count(*) FROM  " +
+		"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
+		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
+		" and job_type ='" + string(JobTypeBenchmark) + "'" +
+		" and type='" + strconv.Itoa(TypeCloudBrainOne) + "'"
+	return x.SQL(countSql).Count()
+}
+func GetBenchmarkOneDuration(beginTime time.Time, endTime time.Time) (int64, error) {
+	total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeBenchmark, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration")
+	if err != nil {
+		return 0, err
+	}
+
+	return total, nil
+}
+func GenerateDebugTwoCount(beginTime time.Time, endTime time.Time) (int64, error) {
+	countSql := "SELECT count(*) FROM  " +
+		"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
+		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
+		" and job_type ='" + string(JobTypeDebug) + "'" +
+		" and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'"
+	return x.SQL(countSql).Count()
+}
+func GetDebugTwoDuration(beginTime time.Time, endTime time.Time) (int64, error) {
+	total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeDebug, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration")
+	if err != nil {
+		return 0, err
+	}
+	return total, nil
+}
+func GenerateTrainTwoCount(beginTime time.Time, endTime time.Time) (int64, error) {
+	countSql := "SELECT count(*) FROM  " +
+		"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
+		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
+		" and job_type ='" + string(JobTypeTrain) + "'" +
+		" and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'"
+	return x.SQL(countSql).Count()
+}
+func GetTrainTwoDuration(beginTime time.Time, endTime time.Time) (int64, error) {
+	total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeTrain, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration")
+	if err != nil {
+		return 0, err
+	}
+	return total, nil
+}
+func GenerateInferenceTwoCount(beginTime time.Time, endTime time.Time) (int64, error) {
+	countSql := "SELECT count(*) FROM  " +
+		"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
+		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
+		" and job_type ='" + string(JobTypeInference) + "'" +
+		" and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'"
+	return x.SQL(countSql).Count()
+}
+func GetInferenceTwoDuration(beginTime time.Time, endTime time.Time) (int64, error) {
+	total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeInference, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration")
+	if err != nil {
+		return 0, err
+	}
+	return total, nil
+}
+
+func DeleteCloudbrainStatisticDaily(date string) error {
+	sess := xStatistic.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return fmt.Errorf("Begin: %v", err)
+	}
+
+	if _, err := sess.Where("date = ?", date).Delete(&CloudbrainStatistic{}); err != nil {
+		return fmt.Errorf("Delete: %v", err)
+	}
+
+	if err := sess.Commit(); err != nil {
+		sess.Close()
+		return fmt.Errorf("Commit: %v", err)
+	}
+
+	sess.Close()
+	return nil
+}
+
+func GetHourStatTime(timeStr string, whichHour int64) (time.Time, time.Time) {
+	t, _ := time.Parse("2006-01-02", timeStr)
+	timeNumber := t.Unix()
+	beginTimeNumber := timeNumber - 8*60*60 + whichHour*60*60
+	endTimeNumber := beginTimeNumber + 1*60*60
+	beginTime := time.Unix(beginTimeNumber, 0)
+	endTime := time.Unix(endTimeNumber, 0)
+	log.Info("%s, %s", beginTime, endTime)
+
+	return beginTime, endTime
+}
+func GetDayStatTime(timeStr string) (time.Time, time.Time) {
+	t, _ := time.Parse("2006-01-02", timeStr)
+	timeNumber := t.Unix()
+	beginTimeNumber := timeNumber - 8*60*60
+	endTimeNumber := beginTimeNumber + 16*60*60
+	beginTime := time.Unix(beginTimeNumber, 0)
+	endTime := time.Unix(endTimeNumber, 0)
+	log.Info("%s, %s", beginTime, endTime)
+
+	return beginTime, endTime
+}
+
+func CreateCloudbrainStatistic(cloudbrainStat *CloudbrainStatistic) (err error) {
+	if _, err = xStatistic.Insert(cloudbrainStat); err != nil {
+		return err
+	}
+	return nil
+}

models/models.go

@@ -160,6 +160,7 @@ func init() {
 		new(UserLoginLog),
 		new(UserMetrics),
 		new(UserAnalysisPara),
+		new(CloudbrainStatistic),
 	)
 
 	gonicNames := []string{"SSL", "UID"}

modules/cron/tasks_basic.go

@@ -173,6 +173,16 @@ func registerHandleRepoAndUserStatistic() {
 		return nil
 	})
 }
+func registerHandleCloudbrainStatistic() {
+	RegisterTaskFatal("handle_cloudbrain_statistic", &BaseConfig{
+		Enabled:    true,
+		RunAtStart: false,
+		Schedule:   "@daily",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		repo.CloudbrainStatisticAuto()
+		return nil
+	})
+}
 
 func registerHandleSummaryStatistic() {
 	RegisterTaskFatal("handle_summary_statistic", &BaseConfig{
@@ -223,6 +233,7 @@ func initBasicTasks() {
 	registerHandleBlockChainUnSuccessCommits()
 
 	registerHandleRepoAndUserStatistic()
+	registerHandleCloudbrainStatistic()
 	registerHandleSummaryStatistic()
 
 	registerSyncCloudbrainStatus()

modules/setting/setting.go

@@ -453,18 +453,19 @@ var (
 	DecompressOBSTaskName   string
 
 	//cloudbrain config
-	CBAuthUser         string
-	CBAuthPassword     string
-	RestServerHost     string
-	JobPath            string
-	CBCodePathPrefix   string
-	JobType            string
-	GpuTypes           string
-	DebugServerHost    string
-	ResourceSpecs      string
-	MaxDuration        int64
-	TrainGpuTypes      string
-	TrainResourceSpecs string
+	CBAuthUser           string
+	CBAuthPassword       string
+	RestServerHost       string
+	JobPath              string
+	CBCodePathPrefix     string
+	JobType              string
+	GpuTypes             string
+	DebugServerHost      string
+	ResourceSpecs        string
+	MaxDuration          int64
+	TrainGpuTypes        string
+	TrainResourceSpecs   string
+	BrainRecordBeginTime string
 
 	//benchmark config
 	IsBenchmarkEnabled     bool
@@ -1292,6 +1293,7 @@ func NewContext() {
 	MaxDuration = sec.Key("MAX_DURATION").MustInt64(14400)
 	TrainGpuTypes = sec.Key("TRAIN_GPU_TYPES").MustString("")
 	TrainResourceSpecs = sec.Key("TRAIN_RESOURCE_SPECS").MustString("")
+	BrainRecordBeginTime = sec.Key("brain_record_beigin_time").MustString("2021-01-01")
 
 	sec = Cfg.Section("benchmark")
 	IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false)

routers/api/v1/api.go

@@ -559,8 +559,14 @@ func RegisterRoutes(m *macaron.Macaron) {
 		m.Get("/query_user_all", operationReq, repo_ext.QueryUserStaticAll)
 		//cloudbrain board
 		m.Group("/cloudbrainboard", func() {
+			// m.Get("/downloadStatistics", repo.ServeCloudbrainPeriodStatisticsFile)
 			m.Get("/downloadAll", repo.DownloadCloudBrainBoard)
+			m.Group("/cloudbrain", func() {
+				m.Get("/trend", repo.GetAllCloudbrainsTrend)
+				m.Get("/statistics", repo.GetAllCloudbrainsPeriodStatistics)
+			})
 		}, operationReq)
+
 		// Users
 		m.Group("/users", func() {
 			m.Get("/search", user.Search)

routers/api/v1/repo/cloudbrain_dashboard.go

@@ -1,6 +1,7 @@
 package repo
 
 import (
+	"fmt"
 	"net/http"
 	"net/url"
 	"time"
@@ -11,6 +12,535 @@ import (
 	"github.com/360EntSecGroup-Skylar/excelize/v2"
 )
 
+	"code.gitea.io/gitea/modules/setting"
+	"github.com/360EntSecGroup-Skylar/excelize/v2"
+)
+
+type CloudbrainsPeriodData struct {
+	DebugOneCount     int64 `json:"debugOneCount"`
+	BenchmarkOneCount int64 `json:"benchmarkOneCount"`
+	TrainOneCount     int64 `json:"trainOneCount"`
+	DebugTwoCount     int64 `json:"debugTwoCount"`
+	TrainTwoCount     int64 `json:"trainTwoCount"`
+	InferenceTwoCount int64 `json:"inferenceTwoCount"`
+}
+type TimeCloudbrainsNum struct {
+	TimeCloudbrainNum []DateCloudbrainNum `json:"dateCloudbrainNum"`
+}
+type DateCloudbrainNum struct {
+	Date               string `json:"date"`
+	DebugOneCount      int64  `json:"debugOneCount"`
+	BenchmarkOneCount  int64  `json:"benchmarkOneCount"`
+	TrainOneCount      int64  `json:"trainOneCount"`
+	DebugTwoCount      int64  `json:"debugTwoCount"`
+	TrainTwoCount      int64  `json:"trainTwoCount"`
+	InferenceTwoCount  int64  `json:"inferenceTwoCount"`
+	CloudbrainOneCount int64  `json:"cloudbrainOneCount"`
+	CloudbrainTwoCount int64  `json:"cloudbrainTwoCount"`
+	CloudbrainCount    int64  `json:"cloudbrainCount"`
+}
+
+func GetAllCloudbrainsTrend(ctx *context.Context) {
+	brainRecordBeginTime, err := getBrainRecordBeginTime()
+	if err != nil {
+		log.Error("Can not get brain record begin time", err)
+		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.brain_record_begintime_get_err"))
+		return
+	}
+	queryType := ctx.QueryTrim("type")
+	now := time.Now()
+
+	beginTimeStr := ctx.QueryTrim("beginTime")
+	endTimeStr := ctx.QueryTrim("endTime")
+	var beginTime time.Time
+	var endTime time.Time
+	var endTimeTemp time.Time
+	dateCloudbrainNum := make([]DateCloudbrainNum, 0)
+	if queryType != "" {
+		if queryType == "all" {
+			beginTime = brainRecordBeginTime
+			endTime = now
+			endTimeTemp = beginTime.AddDate(0, 1, 0)
+			dateCloudbrainNum, err = getYearCloudbrainNum(beginTime, endTimeTemp, endTime)
+			if err != nil {
+				log.Error("Can not query getYearCloudbrainNum.", err)
+				ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error"))
+				return
+			}
+		} else if queryType == "yesterday" {
+			beginTime = now.AddDate(0, 0, -1)
+			beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
+			endTimeTemp = beginTime.Add(time.Hour)
+			endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
+			dateCloudbrainNum, err = getHourCloudbrainNum(beginTime, endTimeTemp, endTime)
+			if err != nil {
+				log.Error("Can not query getHourCloudbrainNum.", err)
+				ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error"))
+				return
+			}
+
+		} else if queryType == "current_week" {
+			beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+1) //begin from monday
+			beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
+			endTime = now
+			endTimeTemp = beginTime.AddDate(0, 0, 1)
+			dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime)
+			if err != nil {
+				log.Error("Can not query getDayCloudbrainNum.", err)
+				ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error"))
+				return
+			}
+		} else if queryType == "current_month" {
+			endTime = now
+			beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location())
+			endTimeTemp = beginTime.AddDate(0, 0, 1)
+			dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime)
+			if err != nil {
+				log.Error("Can not query getDayCloudbrainNum.", err)
+				ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error"))
+				return
+			}
+
+		} else if queryType == "monthly" {
+			endTime = now
+			beginTime = now.AddDate(0, -1, 0)
+			beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
+			endTimeTemp = beginTime.AddDate(0, 0, 1)
+			dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime)
+			if err != nil {
+				log.Error("Can not query getDayCloudbrainNum.", err)
+				ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error"))
+				return
+			}
+
+		} else if queryType == "current_year" {
+			endTime = now
+			beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location())
+			endTimeTemp = beginTime.AddDate(0, 1, 0)
+			dateCloudbrainNum, err = getYearCloudbrainNum(beginTime, endTimeTemp, endTime)
+			if err != nil {
+				log.Error("Can not query getDayCloudbrainNum.", err)
+				ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error"))
+				return
+			}
+
+		} else if queryType == "last_month" {
+
+			lastMonthTime := now.AddDate(0, -1, 0)
+			beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location())
+			endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location())
+			endTimeTemp = beginTime.AddDate(0, 0, 1)
+			dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime)
+			if err != nil {
+				log.Error("Can not query getDayCloudbrainNum.", err)
+				ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error"))
+				return
+			}
+
+		}
+
+	} else {
+		if beginTimeStr == "" || endTimeStr == "" {
+			//如果查询类型和开始时间结束时间都未设置，按queryType=all处理
+			beginTime = brainRecordBeginTime
+			endTime = now
+			endTimeTemp = beginTime.AddDate(0, 1, 0)
+			dateCloudbrainNum, err = getYearCloudbrainNum(beginTime, endTimeTemp, endTime)
+			if err != nil {
+				log.Error("Can not query getDayCloudbrainNum.", err)
+				ctx.Error(http.StatusBadRequest, ctx.Tr("getYearCloudbrainNum_get_error"))
+				return
+			}
+		} else {
+			beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local)
+			if err != nil {
+				log.Error("Can not ParseInLocation.", err)
+				ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error"))
+				return
+			}
+			endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local)
+			if err != nil {
+				log.Error("Can not ParseInLocation.", err)
+				ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error"))
+				return
+			}
+			days := (endTime.Unix() - beginTime.Unix()) / 3600 / 24
+			if 1 < days {
+				endTimeTemp = beginTime.AddDate(0, 0, 1)
+				endTime = endTime.AddDate(0, 0, 2)
+				dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTimeTemp, endTime)
+				if err != nil {
+					log.Error("Can not query getDayCloudbrainNum.", err)
+					ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error"))
+					return
+				}
+			} else if 0 < days || days <= 1 {
+				endTimeTemp = beginTime.Add(time.Hour)
+				dateCloudbrainNum, err = getHourCloudbrainNum(beginTime, endTimeTemp, endTime)
+				if err != nil {
+					log.Error("Can not query getHourCloudbrainNum.", err)
+					ctx.Error(http.StatusBadRequest, ctx.Tr("getHourCloudbrainNum_get_error"))
+					return
+				}
+			} else {
+				return
+			}
+		}
+
+	}
+	cloudbrainsPeriodData := TimeCloudbrainsNum{
+		TimeCloudbrainNum: dateCloudbrainNum,
+	}
+
+	ctx.JSON(http.StatusOK, cloudbrainsPeriodData)
+
+}
+
+func GetAllCloudbrainsPeriodStatistics(ctx *context.Context) {
+
+	recordBeginTime, err := getBrainRecordBeginTime()
+	if err != nil {
+		log.Error("Can not get record begin time", err)
+		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err"))
+		return
+	}
+	beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime)
+	if err != nil {
+		log.Error("Parameter is wrong", err)
+		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong"))
+		return
+	}
+	page := ctx.QueryInt("page")
+	if page <= 0 {
+		page = 1
+	}
+	pageSize := ctx.QueryInt("pagesize")
+	if pageSize <= 0 {
+		pageSize = DEFAULT_PAGE_SIZE
+	}
+
+	debugOneCount, err := models.GenerateDebugOneCount(beginTime, endTime)
+	if err != nil {
+		log.Error("Can not query debugOneCount.", err)
+		ctx.Error(http.StatusBadRequest, ctx.Tr("debugOneCount_get_error"))
+		return
+	}
+	benchmarkOneCount, err := models.GenerateBenchmarkOneCount(beginTime, endTime)
+	if err != nil {
+		log.Error("Can not query benchmarkCount.", err)
+		ctx.Error(http.StatusBadRequest, ctx.Tr("benchmarkOneCount_get_error"))
+		return
+	}
+	trainOneCount, err := models.GenerateTrainOneCount(beginTime, endTime)
+	if err != nil {
+		log.Error("Can not query trainOneCount.", err)
+		ctx.Error(http.StatusBadRequest, ctx.Tr("trainOneCount_get_error"))
+		return
+	}
+	debugTwoCount, err := models.GenerateDebugTwoCount(beginTime, endTime)
+	if err != nil {
+		log.Error("Can not query debugTwoCount.", err)
+		ctx.Error(http.StatusBadRequest, ctx.Tr("debugTwoCount_get_error"))
+		return
+	}
+	trainTwoCount, err := models.GenerateTrainTwoCount(beginTime, endTime)
+	if err != nil {
+		log.Error("Can not query DebugOneTotal count.", err)
+		ctx.Error(http.StatusBadRequest, ctx.Tr("total_count_get_error"))
+		return
+	}
+	inferenceTwoCount, err := models.GenerateInferenceTwoCount(beginTime, endTime)
+	if err != nil {
+		log.Error("Can not query inferenceTwoCount.", err)
+		ctx.Error(http.StatusBadRequest, ctx.Tr("inferenceTwoCount_get_error"))
+		return
+	}
+
+	cloudbrainsPeriodData := CloudbrainsPeriodData{
+		DebugOneCount:     debugOneCount,
+		BenchmarkOneCount: benchmarkOneCount,
+		TrainOneCount:     trainOneCount,
+		DebugTwoCount:     debugTwoCount,
+		TrainTwoCount:     trainTwoCount,
+		InferenceTwoCount: inferenceTwoCount,
+	}
+	now := time.Now()
+	endTime = now
+	beginTime = time.Date(endTime.Year(), endTime.Month(), 2, 0, 0, 0, 0, now.Location())
+	testtime := now.AddDate(0, -1, 1)
+	fmt.Printf("beginTime is: %s", beginTime)
+	fmt.Printf("endTime is: %s", endTime)
+	fmt.Printf("testtime is:%s", testtime)
+
+	ctx.JSON(http.StatusOK, cloudbrainsPeriodData)
+
+}
+
+// func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (time.Time, time.Time, error) {
+// 	queryType := ctx.QueryTrim("type")
+// 	now := time.Now()
+// 	recordBeginTimeTemp := recordBeginTime.AddDate(0, 0, 1)
+
+// 	beginTimeStr := ctx.QueryTrim("beginTime")
+// 	endTimeStr := ctx.QueryTrim("endTime")
+// 	var beginTime time.Time
+// 	var endTime time.Time
+// 	var err error
+// 	if queryType != "" {
+
+// 		if queryType == "all" {
+// 			beginTime = recordBeginTimeTemp
+// 			endTime = now
+// 		} else if queryType == "yesterday" {
+// 			endTime = now
+// 			beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location())
+
+// 		} else if queryType == "current_week" {
+// 			beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+2) //begin from monday
+// 			beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
+// 			endTime = now
+// 		} else if queryType == "current_month" {
+// 			endTime = now
+// 			beginTime = time.Date(endTime.Year(), endTime.Month(), 2, 0, 0, 0, 0, now.Location())
+// 		} else if queryType == "monthly" {
+// 			endTime = now
+// 			beginTime = now.AddDate(0, -1, 1)
+// 			beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
+
+// 		} else if queryType == "current_year" {
+// 			endTime = now
+// 			beginTime = time.Date(endTime.Year(), 1, 2, 0, 0, 0, 0, now.Location())
+
+// 		} else if queryType == "last_month" {
+
+// 			lastMonthTime := now.AddDate(0, -1, 0)
+// 			beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 2, 0, 0, 0, 0, now.Location())
+// 			endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location())
+
+// 		} else {
+// 			return now, now, fmt.Errorf("The value of type parameter is wrong.")
+
+// 		}
+
+// 	} else {
+// 		if beginTimeStr == "" || endTimeStr == "" {
+// 			//如果查询类型和开始时间结束时间都未设置，按queryType=all处理
+// 			beginTime = recordBeginTimeTemp
+// 			endTime = now
+
+// 		} else {
+
+// 			beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local)
+// 			if err != nil {
+// 				return now, now, err
+// 			}
+
+// 			endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local)
+// 			if err != nil {
+// 				return now, now, err
+// 			}
+
+// 			beginTime = beginTime.AddDate(0, 0, 1)
+// 			endTime = endTime.AddDate(0, 0, 1)
+// 		}
+
+// 	}
+
+// 	if beginTime.Before(recordBeginTimeTemp) {
+// 		beginTime = recordBeginTimeTemp
+// 	}
+
+// 	return beginTime, endTime, nil
+
+// }
+
+func getCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, int64, int64, int64, int64, int64, int64, int64, int64, error) {
+	debugOneCount, err := models.GenerateDebugOneCount(beginTime, endTime)
+	if err != nil {
+		log.Error("Can not query debugOneCount.", err)
+		return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
+	}
+	benchmarkOneCount, err := models.GenerateBenchmarkOneCount(beginTime, endTime)
+	if err != nil {
+		log.Error("Can not query benchmarkCount.", err)
+		return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
+	}
+	trainOneCount, err := models.GenerateTrainOneCount(beginTime, endTime)
+	if err != nil {
+		log.Error("Can not query trainOneCount.", err)
+		return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
+	}
+	debugTwoCount, err := models.GenerateDebugTwoCount(beginTime, endTime)
+	if err != nil {
+		log.Error("Can not query debugTwoCount.", err)
+		return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
+	}
+	trainTwoCount, err := models.GenerateTrainTwoCount(beginTime, endTime)
+	if err != nil {
+		log.Error("Can not query DebugOneTotal count.", err)
+		return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
+	}
+	inferenceTwoCount, err := models.GenerateInferenceTwoCount(beginTime, endTime)
+	if err != nil {
+		log.Error("Can not query inferenceTwoCount.", err)
+		return 0, 0, 0, 0, 0, 0, 0, 0, 0, err
+	}
+	cloudbrainOneCount := debugOneCount + benchmarkOneCount + trainOneCount
+	cloudbrainTwoCount := debugTwoCount + trainTwoCount + inferenceTwoCount
+	cloudbrainCount := cloudbrainOneCount + cloudbrainTwoCount
+	return debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err
+}
+func getHourCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) {
+	dayCloudbrainNum := make([]DateCloudbrainNum, 0)
+	for endTimeTemp.Before(endTime) || endTimeTemp.Equal(endTime) {
+		debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp)
+		if err != nil {
+			log.Error("Can not query getCloudbrainCount.", err)
+			return nil, err
+		}
+		dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{
+			Date:               beginTime.Format(time.RFC3339),
+			DebugOneCount:      debugOneCount,
+			BenchmarkOneCount:  benchmarkOneCount,
+			TrainOneCount:      trainOneCount,
+			DebugTwoCount:      debugTwoCount,
+			TrainTwoCount:      trainTwoCount,
+			InferenceTwoCount:  inferenceTwoCount,
+			CloudbrainOneCount: cloudbrainOneCount,
+			CloudbrainTwoCount: cloudbrainTwoCount,
+			CloudbrainCount:    cloudbrainCount,
+		})
+		beginTime = endTimeTemp
+		endTimeTemp = beginTime.Add(time.Hour)
+	}
+	return dayCloudbrainNum, nil
+}
+func getDayCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) {
+	dayCloudbrainNum := make([]DateCloudbrainNum, 0)
+	for endTimeTemp.Before(endTime) {
+		debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp)
+		if err != nil {
+			log.Error("Can not query getCloudbrainCount.", err)
+			return nil, err
+		}
+		dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{
+			Date:               beginTime.Format("2006-01-02"),
+			DebugOneCount:      debugOneCount,
+			BenchmarkOneCount:  benchmarkOneCount,
+			TrainOneCount:      trainOneCount,
+			DebugTwoCount:      debugTwoCount,
+			TrainTwoCount:      trainTwoCount,
+			InferenceTwoCount:  inferenceTwoCount,
+			CloudbrainOneCount: cloudbrainOneCount,
+			CloudbrainTwoCount: cloudbrainTwoCount,
+			CloudbrainCount:    cloudbrainCount,
+		})
+		beginTime = endTimeTemp
+		endTimeTemp = beginTime.AddDate(0, 0, 1)
+	}
+	return dayCloudbrainNum, nil
+}
+func getYearCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) {
+	yearCloudbrainNum := make([]DateCloudbrainNum, 0)
+	for endTimeTemp.Before(endTime) {
+		debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp)
+		if err != nil {
+			log.Error("Can not query getCloudbrainCount.", err)
+			return nil, err
+		}
+		yearCloudbrainNum = append(yearCloudbrainNum, DateCloudbrainNum{
+			Date:               beginTime.Format("2006-01"),
+			DebugOneCount:      debugOneCount,
+			BenchmarkOneCount:  benchmarkOneCount,
+			TrainOneCount:      trainOneCount,
+			DebugTwoCount:      debugTwoCount,
+			TrainTwoCount:      trainTwoCount,
+			InferenceTwoCount:  inferenceTwoCount,
+			CloudbrainOneCount: cloudbrainOneCount,
+			CloudbrainTwoCount: cloudbrainTwoCount,
+			CloudbrainCount:    cloudbrainCount,
+		})
+		beginTime = endTimeTemp
+		endTimeTemp = beginTime.AddDate(0, 1, 0)
+	}
+	return yearCloudbrainNum, nil
+}
+func getBrainRecordBeginTime() (time.Time, error) {
+	return time.ParseInLocation(DATE_FORMAT, setting.BrainRecordBeginTime, time.Local)
+}
+
+// func ServeCloudbrainPeriodStatisticsFile(ctx *context.Context) {
+
+// 	recordBeginTime, err := getRecordBeginTime()
+// 	if err != nil {
+// 		log.Error("Can not get record begin time", err)
+// 		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err"))
+// 		return
+// 	}
+// 	beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime)
+// 	if err != nil {
+// 		log.Error("Parameter is wrong", err)
+// 		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong"))
+// 		return
+// 	}
+// 	q := ctx.QueryTrim("q")
+// 	page := ctx.QueryInt("page")
+// 	if page <= 0 {
+// 		page = 1
+// 	}
+// 	pageSize := 1000
+// 	orderBy := getOrderBy(ctx)
+
+// 	_, latestDate, err := models.GetRepoStatLastUpdatedTime()
+// 	if err != nil {
+// 		log.Error("Can not query the last updated time.", err)
+// 		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error"))
+// 		return
+// 	}
+
+// 	countSql := generateCountSql(beginTime, endTime, latestDate, q)
+// 	total, err := models.CountRepoStatByRawSql(countSql)
+// 	if err != nil {
+// 		log.Error("Can not query total count.", err)
+// 		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error"))
+// 		return
+// 	}
+// 	var projectAnalysis = ctx.Tr("repo.repo_stat_inspect")
+// 	fileName := getFileName(ctx, beginTime, endTime, projectAnalysis)
+
+// 	totalPage := getTotalPage(total, pageSize)
+
+// 	f := excelize.NewFile()
+
+// 	index := f.NewSheet(projectAnalysis)
+// 	f.DeleteSheet("Sheet1")
+
+// 	for k, v := range allProjectsPeroidHeader(ctx) {
+// 		f.SetCellValue(projectAnalysis, k, v)
+// 	}
+
+// 	var row = 2
+// 	for i := 0; i <= totalPage; i++ {
+
+// 		pageRecords := models.GetRepoStatisticByRawSql(generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, i+1, pageSize))
+// 		for _, record := range pageRecords {
+
+// 			for k, v := range allProjectsPeroidValues(row, record, ctx) {
+// 				f.SetCellValue(projectAnalysis, k, v)
+// 			}
+// 			row++
+
+// 		}
+
+// 	}
+// 	f.SetActiveSheet(index)
+
+// 	ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName))
+// 	ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
+
+// 	f.WriteTo(ctx.Resp)
+
+// }
 func DownloadCloudBrainBoard(ctx *context.Context) {
 
 	page := 1
@@ -36,6 +566,8 @@ func DownloadCloudBrainBoard(ctx *context.Context) {
 	}
 
 	totalPage := getTotalPage(total, pageSize)
+	fmt.Printf("total:%v", total)
+	fmt.Printf("totalPage:%v", totalPage)
 
 	f := excelize.NewFile()
 
@@ -79,6 +611,7 @@ func DownloadCloudBrainBoard(ctx *context.Context) {
 
 	f.WriteTo(ctx.Resp)
 }
+
 func getCloudbrainFileName(baseName string) string {
 	return baseName + "_" + time.Now().Format(EXCEL_DATE_FORMAT) + ".xlsx"
 

routers/repo/cloudbrain_statistic.go

@@ -0,0 +1,115 @@
+package repo
+
+import (
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/services/mailer"
+)
+
+func CloudbrainStatisticAuto() {
+	yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
+	CloudbrainStatisticDaily(yesterday)
+}
+
+func CloudbrainStatisticDaily(date string) {
+	log.Info("%s", date)
+	log.Info("begin Cloudbrain Statistic")
+	warnEmailMessage := "云脑任务统计信息入库失败，请尽快定位。"
+	if err := models.DeleteCloudbrainStatisticDaily(date); err != nil {
+		log.Error("DeleteCloudbrainStatisticDaily failed: %v", err.Error())
+		mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage)
+		return
+	}
+
+	//0 to 23 for each hour, 25 for all 24 hours
+	var WhichHour [25]int64 = [25]int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 25}
+	for _, whichHour := range WhichHour {
+		log.Info("start statistic: %s", date)
+		var beginTime, endTime time.Time
+		beginTime, endTime = models.GetHourStatTime(date, whichHour)
+		if whichHour == 25 {
+			beginTime, endTime = models.GetDayStatTime(date)
+		}
+		numDubugOne, err := models.GenerateDebugOneCount(beginTime, endTime)
+		if err != nil {
+			log.Error("GenerateDebugOneCount failed(%s): %v", numDubugOne, err)
+		}
+		numBenchmarkOne, err := models.GenerateBenchmarkOneCount(beginTime, endTime)
+		if err != nil {
+			log.Error("GenerateBenchmarkOneCount failed(%s): %v", numBenchmarkOne, err)
+		}
+		numTrainOne, err := models.GenerateTrainOneCount(beginTime, endTime)
+		if err != nil {
+			log.Error("GenerateTrainOneCount failed(%s): %v", numTrainOne, err)
+		}
+		numDebugTwo, err := models.GenerateDebugTwoCount(beginTime, endTime)
+		if err != nil {
+			log.Error("GenerateDebugTwoCount failed(%s): %v", numDebugTwo, err)
+		}
+		numTrainTwo, err := models.GenerateTrainTwoCount(beginTime, endTime)
+		if err != nil {
+			log.Error("GenerateTrainTwoCount failed(%s): %v", numTrainTwo, err)
+		}
+		numInferenceTwo, err := models.GenerateInferenceTwoCount(beginTime, endTime)
+		if err != nil {
+			log.Error("GenerateInferenceTwoCount failed(%s): %v", numInferenceTwo, err)
+		}
+		numCloudOneAll := numDubugOne + numBenchmarkOne + numTrainOne
+		numCloudTwoAll := numDebugTwo + numTrainTwo + numInferenceTwo
+
+		durationDubugOne, err := models.GetDebugOneDuration(beginTime, endTime)
+		if err != nil {
+			log.Error("GetDebugOneDuration failed(%s): %v", durationDubugOne, err)
+		}
+		durationBenchmarkOne, err := models.GetBenchmarkOneDuration(beginTime, endTime)
+		if err != nil {
+			log.Error("GetBenchmarkOneDuration failed(%s): %v", durationBenchmarkOne, err)
+		}
+		durationTrainOne, err := models.GetTrainOneDuration(beginTime, endTime)
+		if err != nil {
+			log.Error("GetTrainOneDuration failed(%s): %v", durationTrainOne, err)
+		}
+		durationDebugTwo, err := models.GetDebugTwoDuration(beginTime, endTime)
+		if err != nil {
+			log.Error("GetDebugTwoDuration failed(%s): %v", durationDebugTwo, err)
+		}
+		durationTrainTwo, err := models.GetTrainTwoDuration(beginTime, endTime)
+		if err != nil {
+			log.Error("GetTrainTwoDuration failed(%s): %v", durationTrainTwo, err)
+		}
+		durationInferenceTwo, err := models.GetInferenceTwoDuration(beginTime, endTime)
+		if err != nil {
+			log.Error("GetInferenceTwoDuration failed(%s): %v", durationInferenceTwo, err)
+		}
+		durationCloudOneAll := durationDubugOne + durationBenchmarkOne + durationTrainOne
+		durationCloudTwoAll := durationDebugTwo + durationTrainTwo + durationInferenceTwo
+
+		err = models.CreateCloudbrainStatistic(&models.CloudbrainStatistic{
+			Date:                 date,
+			WhichHour:            whichHour,
+			NumDubugOne:          numDubugOne,
+			NumBenchmarkOne:      numBenchmarkOne,
+			NumTrainOne:          numTrainOne,
+			NumDubugTwo:          numDebugTwo,
+			NumTrainTwo:          numTrainTwo,
+			NumInferenceTwo:      numInferenceTwo,
+			NumCloudOneAll:       numCloudOneAll,
+			NumCloudTwoAll:       numCloudTwoAll,
+			DurationDubugOne:     durationDubugOne,
+			DurationBenchmarkOne: durationBenchmarkOne,
+			DurationTrainOne:     durationTrainOne,
+			DurationDebugTwo:     durationDebugTwo,
+			DurationTrainTwo:     durationTrainTwo,
+			DurationInferenceTwo: durationInferenceTwo,
+			DurationCloudOneAll:  durationCloudOneAll,
+			DurationCloudTwoAll:  durationCloudTwoAll,
+		})
+		if err != nil {
+			log.Error("CreateCloudbrainStatistic(%s) failed:%v", date, err.Error())
+			return
+		}
+	}
+}

routers/routes/routes.go

@@ -1031,6 +1031,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 		}, context.RepoRef())
 
 		m.Group("/cloudbrain", func() {
+			m.Get("/test", repo.CloudbrainStatisticAuto)
 			m.Group("/:id", func() {
 				m.Get("", reqRepoCloudBrainReader, repo.CloudBrainShow)
 				m.Get("/debug", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDebug)