From cc8136a372f80ceb5916e6232f92aa16950c5436 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 26 May 2022 16:14:10 +0800 Subject: [PATCH] update --- models/cloudbrain.go | 2 + models/cloudbrain_static.go | 134 +++-- modules/setting/setting.go | 2 +- routers/api/v1/repo/cloudbrain_dashboard.go | 536 +++++++++++--------- 4 files changed, 367 insertions(+), 307 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 03eef25e1..10d57df2e 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -327,6 +327,8 @@ type CloudbrainsOptions struct { JobTypeNot bool NeedRepoInfo bool RepoIDList []int64 + BeginTime time.Time + EndTime time.Time } type TaskPod struct { diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index dfc1b6dd6..7239c5d89 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -5,48 +5,69 @@ import ( "time" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" ) -func GetJobWaitingPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { - countSql := "SELECT count(*) FROM " + - "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and status ='" + string(JobWaiting) + "'" - - return x.SQL(countSql).Count() +type AllTask struct { + List []TaskDetail `json:"list"` +} +type TaskDetail struct { + ID int64 `json:"ID"` + JobName string `json:"JobName"` + DisplayJobName string `json:"DisplayJobName"` + Status string `json:"Status"` + JobType string `json:"JobType"` + CreatedUnix timeutil.TimeStamp `json:"CreatedUnix"` + WaitTime timeutil.TimeStamp `json:"WaitTime"` + RunTime timeutil.TimeStamp `json:"RunTime"` + StartTime timeutil.TimeStamp `json:"StartTime"` + EndTime timeutil.TimeStamp `json:"EndTime"` + ComputeResource string `json:"ComputeResource"` + Type int `json:"Type"` + UserID int64 `json:"UserID"` + RepoID int64 `json:"RepoID"` } -func GetJobRunningPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { - countSql := "SELECT count(*) FROM " + - "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and status ='" + string(JobRunning) + "'" - return x.SQL(countSql).Count() -} -func GetJobStoppedPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { - countSql := "SELECT count(*) FROM " + - "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and status ='" + string(JobStopped) + "'" +// func GetJobWaitingPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { +// countSql := "SELECT count(*) FROM " + +// "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + +// " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + +// " and status ='" + string(JobWaiting) + "'" - return x.SQL(countSql).Count() -} -func GetJobSucceededPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { - countSql := "SELECT count(*) FROM " + - "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and status ='" + string(JobSucceeded) + "'" +// return x.SQL(countSql).Count() +// } +// func GetJobRunningPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { +// countSql := "SELECT count(*) FROM " + +// "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + +// " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + +// " and status ='" + string(JobRunning) + "'" - return x.SQL(countSql).Count() -} -func GetJobFailedPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { - countSql := "SELECT count(*) FROM " + - "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + - " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + - " and status ='" + string(JobFailed) + "'" +// return x.SQL(countSql).Count() +// } +// func GetJobStoppedPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { +// countSql := "SELECT count(*) FROM " + +// "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + +// " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + +// " and status ='" + string(JobStopped) + "'" - return x.SQL(countSql).Count() -} +// return x.SQL(countSql).Count() +// } +// func GetJobSucceededPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { +// countSql := "SELECT count(*) FROM " + +// "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + +// " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + +// " and status ='" + string(JobSucceeded) + "'" + +// return x.SQL(countSql).Count() +// } +// func GetJobFailedPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { +// countSql := "SELECT count(*) FROM " + +// "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + +// " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + +// " and status ='" + string(JobFailed) + "'" + +// return x.SQL(countSql).Count() +// } func GetDebugOnePeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(*) FROM " + @@ -175,36 +196,7 @@ func GetCloudBrainTwoPeriodDuration(beginTime time.Time, endTime time.Time) (int return total, nil } -func GetCloudBrainOneCount() (int64, error) { - countSql := "SELECT count(*) FROM " + - "public.cloudbrain where type='" + strconv.Itoa(TypeCloudBrainOne) + "'" - - return x.SQL(countSql).Count() -} -func GetCloudBrainOneDuration() (int64, error) { - total, err := x.Where("type = ? ", TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration") - if err != nil { - return 0, err - } - - return total, nil -} -func GetCloudBrainTwoCount() (int64, error) { - countSql := "SELECT count(*) FROM " + - "public.cloudbrain where type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" - - return x.SQL(countSql).Count() -} -func GetCloudBrainTwoDuration() (int64, error) { - total, err := x.Where("type = ? ", TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration") - if err != nil { - return 0, err - } - - return total, nil -} - -func GetCreatorPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { +func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(distinct user_id) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) @@ -215,7 +207,7 @@ func GetCreatorCount() (int64, error) { return x.SQL(countSql).Count() } -func GetAllStatusCloudBrain() map[string]int { +func GetAllCloudBrain() ([]*CloudbrainInfo, error) { sess := x.NewSession() defer sess.Close() cloudbrains := make([]*CloudbrainInfo, 0) @@ -223,15 +215,7 @@ func GetAllStatusCloudBrain() map[string]int { Find(&cloudbrains); err != nil { log.Info("find error.") } - cloudBrainStatusResult := make(map[string]int) - for _, cloudbrain := range cloudbrains { - if _, ok := cloudBrainStatusResult[cloudbrain.Status]; !ok { - cloudBrainStatusResult[cloudbrain.Status] = 1 - } else { - cloudBrainStatusResult[cloudbrain.Status] += 1 - } - } - return cloudBrainStatusResult + return cloudbrains, nil } func getCreatePeriodCount(dateBeginTime string, dateEndTime string, hourBeginTime string, hourEndTime string) (int64, error) { diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 12bae39c0..cd7357dc1 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1293,7 +1293,7 @@ func NewContext() { MaxDuration = sec.Key("MAX_DURATION").MustInt64(14400) TrainGpuTypes = sec.Key("TRAIN_GPU_TYPES").MustString("") TrainResourceSpecs = sec.Key("TRAIN_RESOURCE_SPECS").MustString("") - BrainRecordBeginTime = sec.Key("brain_record_beigin_time").MustString("2021-01-01") + BrainRecordBeginTime = sec.Key("brain_record_begin_time").MustString("2021-01-01") sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 01c78ff87..83fdb612e 100644 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1,7 +1,6 @@ package repo import ( - "fmt" "net/http" "net/url" "strings" @@ -16,126 +15,99 @@ import ( "code.gitea.io/gitea/modules/setting" ) -type CloudbrainsPeriodData struct { - DebugOnePeriodCount int64 `json:"debugOnePeriodCount"` - BenchmarkOnePeriodCount int64 `json:"benchmarkOnePeriodCount"` - TrainOnePeriodCount int64 `json:"trainOnePeriodCount"` - DebugTwoPeriodCount int64 `json:"debugTwoPeriodCount"` - TrainTwoPeriodCount int64 `json:"trainTwoPeriodCount"` - InferenceTwoPeriodCount int64 `json:"inferenceTwoPeriodCount"` - JobWaitingPeriodCount int64 `json:"jobWaitingPeriodCount"` - JobRunningPeriodCount int64 `json:"jobRunningPeriodCount"` - JobSucceededPeriodCount int64 `json:"jobRunningPeriodCount"` - CloudBrainOnePeriodCount int64 `json:"cloudBrainOnePeriodCount"` - CloudBrainTwoPeriodCount int64 `json:"cloudBrainTwoPeriodCount"` -} -type CloudbrainsOverviewData struct { - JobWaitingPeriodCount int64 `json:"jobWaitingPeriodCount"` - JobRunningPeriodCount int64 `json:"jobRunningPeriodCount"` - JobStoppedPeriodCount int64 `json:"jobStoppedPeriodCount"` - JobSucceededPeriodCount int64 `json:"jobSucceededPeriodCount"` - JobFailedPeriodCount int64 `json:"jobFailedPeriodCount"` - CloudBrainOneCount int64 `json:"cloudBrainOneCount"` - CloudBrainTwoCount int64 `json:"cloudBrainTwoCount"` - CloudBrainOneDuration int64 `json:"cloudBrainOneDuration"` - CloudBrainTwoDuration int64 `json:"cloudBrainTwoDuration"` - CreatorPeriodCount int64 `json:"creatorPeriodCount"` - CreatorCount int64 `json:"creatorCount"` -} - type TimeCloudbrainsNum struct { TimeCloudbrainNum []DateCloudbrainNum `json:"dateCloudbrainNum"` } type DateCloudbrainNum struct { - Date string `json:"date"` - DebugOneCount int64 `json:"debugOneCount"` - BenchmarkOneCount int64 `json:"benchmarkOneCount"` - TrainOneCount int64 `json:"trainOneCount"` - DebugTwoCount int64 `json:"debugTwoCount"` - TrainTwoCount int64 `json:"trainTwoCount"` - InferenceTwoCount int64 `json:"inferenceTwoCount"` - CloudbrainOneCount int64 `json:"cloudbrainOneCount"` - CloudbrainTwoCount int64 `json:"cloudbrainTwoCount"` - CloudbrainCount int64 `json:"cloudbrainCount"` + Date string `json:"date"` + CloudOneJobTypeRes map[string]int `json:"cloudOneJobTypeRes"` + CloudTwoJobTypeRes map[string]int `json:"cloudTwoJobTypeRes"` + IntelligentNetJobTypeRes map[string]int `json:"intelligentNetJobTypeRes"` + CloudBrainPeriodNum map[int]int `json:"cloudBrainPeriodNum"` + CloudBrainComputeResource map[string]int `json:"cloudBrainComputeResource"` } func GetAllCloudbrainsOverview(ctx *context.Context) { + recordBeginTime, err := getBrainRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } now := time.Now() //today overview beginTime := now.AddDate(0, 0, 0) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime := now - jobWaitingPeriodCount, err := models.GetJobWaitingPeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query jobWaitingCount.", err) - return - } - jobRunningPeriodCount, err := models.GetJobRunningPeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query jobRunningCount.", err) - return - } - jobStoppedPeriodCount, err := models.GetJobStoppedPeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query jobStoppedCount.", err) - return - } - jobSucceededPeriodCount, err := models.GetJobSucceededPeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query jobSucceededCount.", err) - return - } - jobFailedPeriodCount, err := models.GetJobFailedPeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query jobFailedCount.", err) - return - } - cloudBrainOneCount, err := models.GetCloudBrainOneCount() + todayCreatorCount, err := models.GetTodayCreatorCount(beginTime, endTime) if err != nil { - log.Error("Can not query cloudBrainOneCount.", err) + log.Error("Can not query todayCreatorCount.", err) return } - cloudBrainOneDuration, err := models.GetCloudBrainOneDuration() - if err != nil { - log.Error("Can not query cloudBrainOneDuration.", err) - return - } - cloudBrainTwoCount, err := models.GetCloudBrainTwoCount() - if err != nil { - log.Error("Can not query cloudBrainTwoCount.", err) - return - } - cloudBrainTwoDuration, err := models.GetCloudBrainTwoDuration() + creatorCount, err := models.GetCreatorCount() if err != nil { - log.Error("Can not query cloudBrainTwoDuration.", err) + log.Error("Can not query creatorCount.", err) return } - creatorPeriodCount, err := models.GetCreatorPeriodCount(beginTime, endTime) + + cloudbrains, err := models.GetAllCloudBrain() if err != nil { - log.Error("Can not query creatorPeriodCount.", err) + log.Error("Getcloudbrains failed:%v", err) return } - creatorCount, err := models.GetCreatorCount() - if err != nil { - log.Error("Can not query creatorCount.", err) - return + + todayStatusResult := make(map[string]int) + cloudBrainNum := make(map[int]int) + cloudBrainOneDuration := int64(0) + cloudBrainTwoDuration := int64(0) + for _, cloudbrain := range cloudbrains { + if int64(cloudbrain.Cloudbrain.CreatedUnix) >= beginTime.Unix() && int64(cloudbrain.Cloudbrain.CreatedUnix) < endTime.Unix() { + if _, ok := todayStatusResult[cloudbrain.Status]; !ok { + todayStatusResult[cloudbrain.Status] = 1 + } else { + todayStatusResult[cloudbrain.Status] += 1 + } + } + + if _, ok := cloudBrainNum[cloudbrain.Cloudbrain.Type]; !ok { + cloudBrainNum[cloudbrain.Cloudbrain.Type] = 1 + } else { + cloudBrainNum[cloudbrain.Cloudbrain.Type] += 1 + } + + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { + cloudBrainOneDuration = cloudBrainOneDuration + cloudbrain.Cloudbrain.Duration + } + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + cloudBrainTwoDuration = cloudBrainTwoDuration + cloudbrain.Cloudbrain.Duration + } + } - CloudbrainsOverviewData := CloudbrainsOverviewData{ - JobWaitingPeriodCount: jobWaitingPeriodCount, - JobRunningPeriodCount: jobRunningPeriodCount, - JobStoppedPeriodCount: jobStoppedPeriodCount, - JobSucceededPeriodCount: jobSucceededPeriodCount, - JobFailedPeriodCount: jobFailedPeriodCount, - CloudBrainOneCount: cloudBrainOneCount, - CloudBrainTwoCount: cloudBrainTwoCount, - CloudBrainOneDuration: cloudBrainOneDuration, - CloudBrainTwoDuration: cloudBrainTwoDuration, - CreatorPeriodCount: creatorPeriodCount, - CreatorCount: creatorCount, + statusNameList := []string{"COMPLETED", "FAILED", "INIT", "RUNNING", "START_FAILED", "STOPPED", "SUCCEEDED", "WAITING", "KILLED"} + for _, v := range statusNameList { + if _, ok := todayStatusResult[v]; !ok { + todayStatusResult[v] = 0 + } } - ctx.JSON(http.StatusOK, CloudbrainsOverviewData) + todayRunningCount := todayStatusResult["RUNNING"] + todayCompletedCount := todayStatusResult["COMPLETED"] + todayStatusResult["FAILED"] + todayStatusResult["START_FAILED"] + todayStatusResult["STOPPED"] + todayStatusResult["SUCCEEDED"] + todayStatusResult["KILLED"] + todayWaitingCount := todayStatusResult["INIT"] + todayStatusResult["WAITING"] + + ctx.JSON(http.StatusOK, map[string]interface{}{ + // "todayStatusResult": todayStatusResult, + "recordBeginTime": recordBeginTime, + "cloudBrainNum": cloudBrainNum, + "cloudBrainOneDuration": cloudBrainOneDuration, + "cloudBrainTwoDuration": cloudBrainTwoDuration, + "intelligentNetDuration": 0, + "todayCreatorCount": todayCreatorCount, + "creatorCount": creatorCount, + "todayRunningCount": todayRunningCount, + "todayCompletedCount": todayCompletedCount, + "todayWaitingCount": todayWaitingCount, + }) } func GetAllCloudbrainsTrend(ctx *context.Context) { @@ -308,71 +280,128 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) return } - debugOnePeriodCount, err := models.GetDebugOnePeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query debugOneCount.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("debugOneCount_get_error")) - return - } - benchmarkOnePeriodCount, err := models.GetBenchmarkOnePeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query benchmarkCount.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("benchmarkOneCount_get_error")) - return - } - trainOnePeriodCount, err := models.GetTrainOnePeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query trainOneCount.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("trainOneCount_get_error")) - return - } - debugTwoPeriodCount, err := models.GetDebugTwoPeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query debugTwoCount.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("debugTwoCount_get_error")) + // debugOnePeriodCount, err := models.GetDebugOnePeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query debugOneCount.", err) + // ctx.Error(http.StatusBadRequest, ctx.Tr("debugOneCount_get_error")) + // return + // } + // benchmarkOnePeriodCount, err := models.GetBenchmarkOnePeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query benchmarkCount.", err) + // ctx.Error(http.StatusBadRequest, ctx.Tr("benchmarkOneCount_get_error")) + // return + // } + // trainOnePeriodCount, err := models.GetTrainOnePeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query trainOneCount.", err) + // ctx.Error(http.StatusBadRequest, ctx.Tr("trainOneCount_get_error")) + // return + // } + // debugTwoPeriodCount, err := models.GetDebugTwoPeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query debugTwoCount.", err) + // ctx.Error(http.StatusBadRequest, ctx.Tr("debugTwoCount_get_error")) + // return + // } + // trainTwoPeriodCount, err := models.GetTrainTwoPeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query DebugOneTotal count.", err) + // ctx.Error(http.StatusBadRequest, ctx.Tr("total_count_get_error")) + // return + // } + // inferenceTwoPeriodCount, err := models.GetInferenceTwoPeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query inferenceTwoCount.", err) + // ctx.Error(http.StatusBadRequest, ctx.Tr("inferenceTwoCount_get_error")) + // return + // } + // cloudBrainOnePeriodCount, err := models.GetCloudBrainOnePeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query cloudBrainOnePeriodCount.", err) + // ctx.Error(http.StatusBadRequest, ctx.Tr("cloudBrainOnePeriodCount_get_error")) + // return + // } + // cloudBrainTwoPeriodCount, err := models.GetCloudBrainTwoPeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query cloudBrainTwoPeriodCount.", err) + // ctx.Error(http.StatusBadRequest, ctx.Tr("cloudBrainTwoPeriodCount_get_error")) + // return + // } + + cloudbrains, err := models.GetAllCloudBrain() + if err != nil { + log.Error("Getcloudbrains failed:%v", err) return } - trainTwoPeriodCount, err := models.GetTrainTwoPeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query DebugOneTotal count.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("total_count_get_error")) - return - } - inferenceTwoPeriodCount, err := models.GetInferenceTwoPeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query inferenceTwoCount.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("inferenceTwoCount_get_error")) - return - } - cloudBrainOnePeriodCount, err := models.GetCloudBrainOnePeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query cloudBrainOnePeriodCount.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("cloudBrainOnePeriodCount_get_error")) - return - } - cloudBrainTwoPeriodCount, err := models.GetCloudBrainTwoPeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query cloudBrainTwoPeriodCount.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("cloudBrainTwoPeriodCount_get_error")) - return + cloudOneJobTypeRes := make(map[string]int) + cloudTwoJobTypeRes := make(map[string]int) + intelligentNetJobTypeRes := make(map[string]int) + cloudBrainPeriodNum := make(map[int]int) + for _, cloudbrain := range cloudbrains { + if int64(cloudbrain.Cloudbrain.CreatedUnix) >= beginTime.Unix() && int64(cloudbrain.Cloudbrain.CreatedUnix) < endTime.Unix() { + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { + if _, ok := cloudOneJobTypeRes[cloudbrain.JobType]; !ok { + cloudOneJobTypeRes[cloudbrain.JobType] = 1 + } else { + cloudOneJobTypeRes[cloudbrain.JobType] += 1 + } + } + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + if _, ok := cloudTwoJobTypeRes[cloudbrain.JobType]; !ok { + cloudTwoJobTypeRes[cloudbrain.JobType] = 1 + } else { + cloudTwoJobTypeRes[cloudbrain.JobType] += 1 + } + } + + if _, ok := cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type]; !ok { + cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] = 1 + } else { + cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] += 1 + } + + } } - cloudbrainsPeriodData := CloudbrainsPeriodData{ - DebugOnePeriodCount: debugOnePeriodCount, - BenchmarkOnePeriodCount: benchmarkOnePeriodCount, - TrainOnePeriodCount: trainOnePeriodCount, - DebugTwoPeriodCount: debugTwoPeriodCount, - TrainTwoPeriodCount: trainTwoPeriodCount, - InferenceTwoPeriodCount: inferenceTwoPeriodCount, - CloudBrainOnePeriodCount: cloudBrainOnePeriodCount, - CloudBrainTwoPeriodCount: cloudBrainTwoPeriodCount, + jobTypeList := []string{"DEBUG", "BENCHMARK", "INFERENCE", "TRAIN", "SNN4IMAGENET", "BRAINSCORE"} + for _, v := range jobTypeList { + if _, ok := cloudOneJobTypeRes[v]; !ok { + cloudOneJobTypeRes[v] = 0 + } + if _, ok := cloudTwoJobTypeRes[v]; !ok { + cloudTwoJobTypeRes[v] = 0 + } + if _, ok := intelligentNetJobTypeRes[v]; !ok { + intelligentNetJobTypeRes[v] = 0 + } } - ctx.JSON(http.StatusOK, cloudbrainsPeriodData) + cloudOneJobTypeRes["EVALUATION"] = cloudOneJobTypeRes["BENCHMARK"] + cloudOneJobTypeRes["SNN4IMAGENET"] + cloudOneJobTypeRes["BRAINSCORE"] + cloudTwoJobTypeRes["EVALUATION"] = cloudTwoJobTypeRes["BENCHMARK"] + cloudTwoJobTypeRes["SNN4IMAGENET"] + cloudTwoJobTypeRes["BRAINSCORE"] + intelligentNetJobTypeRes["EVALUATION"] = intelligentNetJobTypeRes["BENCHMARK"] + intelligentNetJobTypeRes["SNN4IMAGENET"] + intelligentNetJobTypeRes["BRAINSCORE"] + ctx.JSON(http.StatusOK, map[string]interface{}{ + "cloudOneJobTypeRes": cloudOneJobTypeRes, + "cloudTwoJobTypeRes": cloudTwoJobTypeRes, + "intelligentNetJobTypeRes": intelligentNetJobTypeRes, + "cloudBrainPeriodNum": cloudBrainPeriodNum, + }) } func GetCloudbrainsStatusAnalysis(ctx *context.Context) { - cloudBrainStatusResult := models.GetAllStatusCloudBrain() + cloudbrains, err := models.GetAllCloudBrain() + if err != nil { + log.Error("Getcloudbrains failed:%v", err) + return + } + cloudBrainStatusResult := make(map[string]int) + for _, cloudbrain := range cloudbrains { + if _, ok := cloudBrainStatusResult[cloudbrain.Status]; !ok { + cloudBrainStatusResult[cloudbrain.Status] = 1 + } else { + cloudBrainStatusResult[cloudbrain.Status] += 1 + } + } ctx.JSON(http.StatusOK, map[string]interface{}{ "cloudBrainStatusResult": cloudBrainStatusResult, }) @@ -434,11 +463,13 @@ func GetCloudbrainsDetailData(ctx *context.Context) { ctx.ServerError("Get job failed:", err) return } + // var taskList := []*models.TaskDetail for i, task := range ciTasks { ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource + } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, getTotalPage(count, setting.UI.IssuePagingNum)) @@ -598,70 +629,127 @@ func GetCloudbrainsRunHoursData(ctx *context.Context) { } -func getCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, int64, int64, int64, int64, int64, int64, int64, int64, error) { - debugOneCount, err := models.GetDebugOnePeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query debugOneCount.", err) - return 0, 0, 0, 0, 0, 0, 0, 0, 0, err - } - benchmarkOneCount, err := models.GetBenchmarkOnePeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query benchmarkCount.", err) - return 0, 0, 0, 0, 0, 0, 0, 0, 0, err - } - trainOneCount, err := models.GetTrainOnePeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query trainOneCount.", err) - return 0, 0, 0, 0, 0, 0, 0, 0, 0, err - } - debugTwoCount, err := models.GetDebugTwoPeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query debugTwoCount.", err) - return 0, 0, 0, 0, 0, 0, 0, 0, 0, err - } - trainTwoCount, err := models.GetTrainTwoPeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query DebugOneTotal count.", err) - return 0, 0, 0, 0, 0, 0, 0, 0, 0, err - } - inferenceTwoCount, err := models.GetInferenceTwoPeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query inferenceTwoCount.", err) - return 0, 0, 0, 0, 0, 0, 0, 0, 0, err +func getCloudbrainCount(beginTime time.Time, endTime time.Time) (map[string]int, map[string]int, map[string]int, map[int]int, map[string]int, error) { + // debugOneCount, err := models.GetDebugOnePeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query debugOneCount.", err) + // return 0, 0, 0, 0, 0, 0, 0, 0, 0, err + // } + // benchmarkOneCount, err := models.GetBenchmarkOnePeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query benchmarkCount.", err) + // return 0, 0, 0, 0, 0, 0, 0, 0, 0, err + // } + // trainOneCount, err := models.GetTrainOnePeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query trainOneCount.", err) + // return 0, 0, 0, 0, 0, 0, 0, 0, 0, err + // } + // debugTwoCount, err := models.GetDebugTwoPeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query debugTwoCount.", err) + // return 0, 0, 0, 0, 0, 0, 0, 0, 0, err + // } + // trainTwoCount, err := models.GetTrainTwoPeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query DebugOneTotal count.", err) + // return 0, 0, 0, 0, 0, 0, 0, 0, 0, err + // } + // inferenceTwoCount, err := models.GetInferenceTwoPeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query inferenceTwoCount.", err) + // return 0, 0, 0, 0, 0, 0, 0, 0, 0, err + // } + // cloudbrainOneCount, err := models.GetCloudBrainOnePeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query cloudbrainOneCount.", err) + // return 0, 0, 0, 0, 0, 0, 0, 0, 0, err + // } + // cloudbrainTwoCount, err := models.GetCloudBrainTwoPeriodCount(beginTime, endTime) + // if err != nil { + // log.Error("Can not query cloudbrainOneCount.", err) + // return 0, 0, 0, 0, 0, 0, 0, 0, 0, err + // } + // cloudbrainCount := cloudbrainOneCount + cloudbrainTwoCount + + cloudbrains, err := models.GetAllCloudBrain() + if err != nil { + log.Error("Getcloudbrains failed:%v", err) + return nil, nil, nil, nil, nil, err + } + cloudOneJobTypeRes := make(map[string]int) + cloudTwoJobTypeRes := make(map[string]int) + intelligentNetJobTypeRes := make(map[string]int) + cloudBrainPeriodNum := make(map[int]int) + cloudBrainComputeResource := make(map[string]int) + for _, cloudbrain := range cloudbrains { + if int64(cloudbrain.Cloudbrain.CreatedUnix) >= beginTime.Unix() && int64(cloudbrain.Cloudbrain.CreatedUnix) < endTime.Unix() { + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { + if _, ok := cloudOneJobTypeRes[cloudbrain.Cloudbrain.JobType]; !ok { + cloudOneJobTypeRes[cloudbrain.Cloudbrain.JobType] = 1 + } else { + cloudOneJobTypeRes[cloudbrain.Cloudbrain.JobType] += 1 + } + } + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + if _, ok := cloudTwoJobTypeRes[cloudbrain.JobType]; !ok { + cloudTwoJobTypeRes[cloudbrain.Cloudbrain.JobType] = 1 + } else { + cloudTwoJobTypeRes[cloudbrain.Cloudbrain.JobType] += 1 + } + } + + if _, ok := cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type]; !ok { + cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] = 1 + } else { + cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] += 1 + } + + if _, ok := cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource]; !ok { + cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource] = 1 + } else { + cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource] += 1 + } + + } } - cloudbrainOneCount, err := models.GetCloudBrainOnePeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query cloudbrainOneCount.", err) - return 0, 0, 0, 0, 0, 0, 0, 0, 0, err + + jobTypeList := []string{"DEBUG", "BENCHMARK", "INFERENCE", "TRAIN", "SNN4IMAGENET", "BRAINSCORE"} + cloudBrainTypeList := []int{0, 1, 2} + for _, v := range jobTypeList { + if _, ok := cloudOneJobTypeRes[v]; !ok { + cloudOneJobTypeRes[v] = 0 + } + if _, ok := cloudTwoJobTypeRes[v]; !ok { + cloudTwoJobTypeRes[v] = 0 + } + if _, ok := intelligentNetJobTypeRes[v]; !ok { + intelligentNetJobTypeRes[v] = 0 + } } - cloudbrainTwoCount, err := models.GetCloudBrainTwoPeriodCount(beginTime, endTime) - if err != nil { - log.Error("Can not query cloudbrainOneCount.", err) - return 0, 0, 0, 0, 0, 0, 0, 0, 0, err + for _, v := range cloudBrainTypeList { + if _, ok := cloudBrainPeriodNum[v]; !ok { + cloudBrainPeriodNum[v] = 0 + } } - cloudbrainCount := cloudbrainOneCount + cloudbrainTwoCount - return debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err + return cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource, err } func getHourCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { dayCloudbrainNum := make([]DateCloudbrainNum, 0) for endTimeTemp.Before(endTime) || endTimeTemp.Equal(endTime) { - debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp) + cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource, err := getCloudbrainCount(beginTime, endTimeTemp) if err != nil { log.Error("Can not query getCloudbrainCount.", err) return nil, err } dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{ - Date: beginTime.Format(time.RFC3339), - DebugOneCount: debugOneCount, - BenchmarkOneCount: benchmarkOneCount, - TrainOneCount: trainOneCount, - DebugTwoCount: debugTwoCount, - TrainTwoCount: trainTwoCount, - InferenceTwoCount: inferenceTwoCount, - CloudbrainOneCount: cloudbrainOneCount, - CloudbrainTwoCount: cloudbrainTwoCount, - CloudbrainCount: cloudbrainCount, + Date: beginTime.Format(time.RFC3339), + CloudOneJobTypeRes: cloudOneJobTypeRes, + CloudTwoJobTypeRes: cloudTwoJobTypeRes, + IntelligentNetJobTypeRes: intelligentNetJobTypeRes, + CloudBrainPeriodNum: cloudBrainPeriodNum, + CloudBrainComputeResource: cloudBrainComputeResource, }) beginTime = endTimeTemp endTimeTemp = beginTime.Add(time.Hour) @@ -671,23 +759,18 @@ func getHourCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime ti func getDayCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { dayCloudbrainNum := make([]DateCloudbrainNum, 0) for endTimeTemp.Before(endTime) { - debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, - cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp) + cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource, err := getCloudbrainCount(beginTime, endTimeTemp) if err != nil { log.Error("Can not query getCloudbrainCount.", err) return nil, err } dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{ - Date: beginTime.Format("2006-01-02"), - DebugOneCount: debugOneCount, - BenchmarkOneCount: benchmarkOneCount, - TrainOneCount: trainOneCount, - DebugTwoCount: debugTwoCount, - TrainTwoCount: trainTwoCount, - InferenceTwoCount: inferenceTwoCount, - CloudbrainOneCount: cloudbrainOneCount, - CloudbrainTwoCount: cloudbrainTwoCount, - CloudbrainCount: cloudbrainCount, + Date: beginTime.Format("2006-01-02"), + CloudOneJobTypeRes: cloudOneJobTypeRes, + CloudTwoJobTypeRes: cloudTwoJobTypeRes, + IntelligentNetJobTypeRes: intelligentNetJobTypeRes, + CloudBrainPeriodNum: cloudBrainPeriodNum, + CloudBrainComputeResource: cloudBrainComputeResource, }) beginTime = endTimeTemp endTimeTemp = beginTime.AddDate(0, 0, 1) @@ -697,23 +780,18 @@ func getDayCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime tim func getYearCloudbrainNum(beginTime time.Time, endTimeTemp time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { yearCloudbrainNum := make([]DateCloudbrainNum, 0) for endTimeTemp.Before(endTime) { - debugOneCount, benchmarkOneCount, trainOneCount, debugTwoCount, trainTwoCount, inferenceTwoCount, cloudbrainOneCount, - cloudbrainTwoCount, cloudbrainCount, err := getCloudbrainCount(beginTime, endTimeTemp) + cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource, err := getCloudbrainCount(beginTime, endTimeTemp) if err != nil { log.Error("Can not query getCloudbrainCount.", err) return nil, err } yearCloudbrainNum = append(yearCloudbrainNum, DateCloudbrainNum{ - Date: beginTime.Format("2006-01"), - DebugOneCount: debugOneCount, - BenchmarkOneCount: benchmarkOneCount, - TrainOneCount: trainOneCount, - DebugTwoCount: debugTwoCount, - TrainTwoCount: trainTwoCount, - InferenceTwoCount: inferenceTwoCount, - CloudbrainOneCount: cloudbrainOneCount, - CloudbrainTwoCount: cloudbrainTwoCount, - CloudbrainCount: cloudbrainCount, + Date: beginTime.Format("2006-01"), + CloudOneJobTypeRes: cloudOneJobTypeRes, + CloudTwoJobTypeRes: cloudTwoJobTypeRes, + IntelligentNetJobTypeRes: intelligentNetJobTypeRes, + CloudBrainPeriodNum: cloudBrainPeriodNum, + CloudBrainComputeResource: cloudBrainComputeResource, }) beginTime = endTimeTemp endTimeTemp = beginTime.AddDate(0, 1, 0) @@ -747,11 +825,7 @@ func DownloadCloudBrainBoard(ctx *context.Context) { ctx.Error(http.StatusBadRequest, ctx.Tr("repo.cloudbrain_query_fail")) return } - totalPage := getTotalPage(total, pageSize) - fmt.Printf("total:%v", total) - fmt.Printf("totalPage:%v", totalPage) - f := excelize.NewFile() index := f.NewSheet(cloudBrain)