diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 48df111a0..bd2bbaef8 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -38,6 +38,25 @@ type TaskDetail struct { Spec *Specification `json:"Spec"` } +type CloudbrainDurationStatistic struct { + ID int64 `xorm:"pk autoincr"` + Cluster string `xorm:"notnull"` + AiCenterCode string + AiCenterName string + ComputeResource string + AccCardType string + QueueCode string + CardsTotalNum int + + DateTime string + HourTime int + CardsTotalDuration int + + DeletedTime timeutil.TimeStamp `xorm:"deleted"` + CreatedTime timeutil.TimeStamp `xorm:"created"` + UpdatedTime timeutil.TimeStamp `xorm:"updated"` +} + func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(distinct user_id) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + @@ -199,3 +218,39 @@ func GetRunHourPeriodCount(dateBeginTime string, dateEndTime string) (map[string } return dateHourMap, nil } + +func GetCloudbrainRunning() ([]*CloudbrainInfo, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.Eq{"cloudbrain.status": string(JobRunning)}, + ) + sess.OrderBy("cloudbrain.created_unix ASC") + cloudbrains := make([]*CloudbrainInfo, 0, 10) + if err := sess.Table(&Cloudbrain{}).Where(cond). + Find(&cloudbrains); err != nil { + log.Info("find error.") + } + return cloudbrains, nil +} + +func GetCloudbrainCompleteByTime(beginTime int64, endTime int64) ([]*CloudbrainInfo, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.And(builder.Gte{"cloudbrain.end_time": beginTime}, builder.Lte{"cloudbrain.end_time": endTime}), + ) + sess.OrderBy("cloudbrain.created_unix ASC") + cloudbrains := make([]*CloudbrainInfo, 0, 10) + if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). + Find(&cloudbrains); err != nil { + log.Info("find error.") + } + return cloudbrains, nil +} + +func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDurationStatistic) (int64, error) { + return xStatistic.Insert(cloudbrainDurationStatistic) +} diff --git a/models/models.go b/models/models.go index 4c2079cd8..7665c3b0f 100755 --- a/models/models.go +++ b/models/models.go @@ -179,6 +179,7 @@ func init() { new(UserMetrics), new(UserAnalysisPara), new(Invitation), + new(CloudbrainDurationStatistic), ) gonicNames := []string{"SSL", "UID"} diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 04cd7fe41..1fe10cb2c 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -5,12 +5,13 @@ package cron import ( - "code.gitea.io/gitea/services/reward" - "code.gitea.io/gitea/services/cloudbrain/resource" - "code.gitea.io/gitea/modules/modelarts" "context" "time" + "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/services/cloudbrain/resource" + "code.gitea.io/gitea/services/reward" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/migrations" repository_service "code.gitea.io/gitea/modules/repository" @@ -254,6 +255,17 @@ func registerSyncModelArtsTempJobs() { }) } +func registerHandleCloudbrainDurationStatistic() { + RegisterTaskFatal("handle_cloudbrain_duration_statistic", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@every 60m", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.CloudbrainDurationStatistic() + return nil + }) +} + func initBasicTasks() { registerUpdateMirrorTask() registerRepoHealthCheck() @@ -271,6 +283,7 @@ func initBasicTasks() { registerHandleRepoAndUserStatistic() registerHandleSummaryStatistic() + registerHandleCloudbrainDurationStatistic() registerSyncCloudbrainStatus() registerHandleOrgStatistic() diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go new file mode 100644 index 000000000..1aa943932 --- /dev/null +++ b/routers/repo/cloudbrain_statistic.go @@ -0,0 +1,188 @@ +package repo + +import ( + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" +) + +func CloudbrainDurationStatistic() { + log.Info("Generate Cloudbrain Duration statistic begin") + // yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + // CloudbrainDurationStatisticHour(yesterday) + log.Info("Generate Cloudbrain Duration statistic end") +} + +func CloudbrainDurationStatisticHour(ctx *context.Context) { + //获取规定时间段的云脑任务列表 + // page := ctx.QueryInt("page") + // pageSize := ctx.QueryInt("pagesize") + // if page <= 0 { + // page = 1 + // } + // if pageSize <= 0 { + // pageSize = 10 + // } + + // cloudBrainDurationRes := make(map[string]map[string]int) + // cloudBrainOneCardRes := make(map[string]int) + // cloudBrainTwoCardRes := make(map[string]int) + // c2NetCardRes := make(map[string]int) + // cDNetCenterCardRes := make(map[string]int) + // var WorkServerNumber int + // var AccCardsNum int + // endTime := time.Now().Unix() + // beginTime := time.Now().AddDate(0, 0, -1).Unix() + + // hour := time.Now().Hour() + // tStr := time.Now().Format("2006-01-02 15:04:05") + + currentTime := time.Now() + m, _ := time.ParseDuration("-1h") + beginTime := currentTime.Add(m).Unix() + endTime := currentTime.Unix() + // fmt.Println(beginTime) + + ciTasks1, err := models.GetCloudbrainRunning() + if err != nil { + // ctx.ServerError("Get job failed:", err) + log.Info("GetCloudbrainRunning err: %v", err) + return + } + ciTasks2, err := models.GetCloudbrainCompleteByTime(beginTime, endTime) + ciTasks := append(ciTasks1, ciTasks2...) + log.Info("beginTime: %s", beginTime) + log.Info("endTime: %s", endTime) + if err != nil { + // ctx.ServerError("Get job failed:", err) + log.Info("GetCloudbrainCompleteByTime err: %v", err) + return + } + models.LoadSpecs4CloudbrainInfo(ciTasks) + log.Info("ciTasks here: %s", ciTasks) + log.Info("count here: %s", len(ciTasks)) + cloudBrainCardRes := getCloudBrainCardRes(ciTasks, beginTime, endTime) + + //根据云脑任务列表获取云脑任务已使用的卡时,并区分是哪个智算中心,哪个卡类型的卡时,将这些信息存入新表 + + // cloudbrainDurationStat := models.CloudbrainDurationStatistic{ + // DateTime: date, + // HourTime: userNumber, + // Cluster: repositorySize, + // AiCenterName: allDatasetSize, + // AiCenterCode: organizationNumber, + // ComputeResource: repositoryNumer, + // AccCardType: forkRepositoryNumber, + // CardsTotalNum: mirrorRepositoryNumber, + // CardsTotalDuration: privateRepositoryNumer, + // QueueCode: publicRepositoryNumer, + // CreatedTime: privateRepositoryNumer, + // UpdatedTime: publicRepositoryNumer, + // } + + // if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { + // log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) + // } + // log.Info("cloudBrainDurationRes2: %s", cloudBrainDurationRes) + // cloudBrainDurationRes = append(cloudBrainDurationRes, cloudBrainOneCardRes) + // log.Info("cloudBrainDurationRes: %s", cloudBrainDurationRes) + // log.Info("cloudBrainOneCardRes: %s", cloudBrainOneCardRes) + log.Info("cloudBrainCardRes: %s", cloudBrainCardRes) + // log.Info("c2NetCardRes: %s", c2NetCardRes) + log.Info("finish summary cloudbrainDurationStat") +} + +func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) map[string]map[string]int { + var WorkServerNumber int + var AccCardsNum int + // cloudBrainCardRes := make(map[string]int) + cloudBrainAicenterNameList := make(map[string]string) + cloudBrainCardTypeList := make(map[string]string) + cloudBrainCenterNameAndCardType := make(map[string]map[string]int) + // var cloudbrainDurationInfo models.CloudbrainDurationInfo + for _, cloudbrain := range ciTasks { + + if _, ok := cloudBrainAicenterNameList[cloudbrain.Cloudbrain.Spec.AiCenterName]; !ok { + cloudBrainAicenterNameList[cloudbrain.Cloudbrain.Spec.AiCenterName] = cloudbrain.Cloudbrain.Spec.AiCenterName + } + + if cloudbrain.Cloudbrain.StartTime == 0 { + cloudbrain.Cloudbrain.StartTime = cloudbrain.Cloudbrain.CreatedUnix + } + if cloudbrain.Cloudbrain.EndTime == 0 { + cloudbrain.Cloudbrain.EndTime = cloudbrain.Cloudbrain.UpdatedUnix + } + if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { + WorkServerNumber = cloudbrain.Cloudbrain.WorkServerNumber + } else { + WorkServerNumber = 1 + } + if cloudbrain.Cloudbrain.Spec == nil { + AccCardsNum = 1 + } else { + AccCardsNum = cloudbrain.Cloudbrain.Spec.AccCardsNum + } + for k, _ := range cloudBrainAicenterNameList { + if cloudbrain.Cloudbrain.Spec.AiCenterName == cloudBrainAicenterNameList[k] { + if _, ok := cloudBrainCardTypeList[cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { + cloudBrainCardTypeList[cloudbrain.Cloudbrain.Spec.AccCardType] = cloudbrain.Cloudbrain.Spec.AccCardType + } + for i, _ := range cloudBrainCardTypeList { + if cloudBrainCenterNameAndCardType == nil { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] = 0 + } + if cloudbrain.Cloudbrain.Spec.AccCardType == cloudBrainCardTypeList[i] { + if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { + if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + } else { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + } + } else { + if int64(cloudbrain.Cloudbrain.StartTime) < beginTime && int64(cloudbrain.Cloudbrain.EndTime) < endTime { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + } else { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + } + } + } + } + // cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] = cloudBrainCardRes[cloudBrainCardTypeList[i]] + + // if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { + // if _, ok := cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType]; ok { + // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + // } else { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + // } + // } else { + // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + // } else { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + // } + // } + // } else { + // if _, ok := cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType]; ok { + // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime && int64(cloudbrain.Cloudbrain.EndTime) < endTime { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + // } else { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + // } + // } else { + // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime && int64(cloudbrain.Cloudbrain.EndTime) < endTime { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + // } else { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + // } + // } + // } + } + } + } + + return cloudBrainCenterNameAndCardType +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 66a357c79..e52686ee4 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -6,15 +6,16 @@ package routes import ( "bytes" - "code.gitea.io/gitea/routers/reward/point" - "code.gitea.io/gitea/routers/task" - "code.gitea.io/gitea/services/reward" "encoding/gob" "net/http" "path" "text/template" "time" + "code.gitea.io/gitea/routers/reward/point" + "code.gitea.io/gitea/routers/task" + "code.gitea.io/gitea/services/reward" + "code.gitea.io/gitea/modules/slideimage" "code.gitea.io/gitea/routers/image" @@ -374,6 +375,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/^:type(issues|pulls)$", reqSignIn, user.Issues) m.Get("/milestones", reqSignIn, reqMilestonesDashboardPageEnabled, user.Milestones) m.Get("/cloudbrains", reqSignIn, user.Cloudbrains) + m.Get("/duration", repo.CloudbrainDurationStatisticHour) // ***** START: User ***** m.Group("/user", func() {