From 36402724026517361a106eaabf31d45c587e0c59 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 13 Oct 2022 14:42:52 +0800 Subject: [PATCH 01/82] update --- models/cloudbrain_static.go | 55 ++++++++ models/models.go | 1 + modules/cron/tasks_basic.go | 19 ++- routers/repo/cloudbrain_statistic.go | 188 +++++++++++++++++++++++++++ routers/routes/routes.go | 8 +- 5 files changed, 265 insertions(+), 6 deletions(-) create mode 100644 routers/repo/cloudbrain_statistic.go diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 48df111a0..bd2bbaef8 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -38,6 +38,25 @@ type TaskDetail struct { Spec *Specification `json:"Spec"` } +type CloudbrainDurationStatistic struct { + ID int64 `xorm:"pk autoincr"` + Cluster string `xorm:"notnull"` + AiCenterCode string + AiCenterName string + ComputeResource string + AccCardType string + QueueCode string + CardsTotalNum int + + DateTime string + HourTime int + CardsTotalDuration int + + DeletedTime timeutil.TimeStamp `xorm:"deleted"` + CreatedTime timeutil.TimeStamp `xorm:"created"` + UpdatedTime timeutil.TimeStamp `xorm:"updated"` +} + func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { countSql := "SELECT count(distinct user_id) FROM " + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + @@ -199,3 +218,39 @@ func GetRunHourPeriodCount(dateBeginTime string, dateEndTime string) (map[string } return dateHourMap, nil } + +func GetCloudbrainRunning() ([]*CloudbrainInfo, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.Eq{"cloudbrain.status": string(JobRunning)}, + ) + sess.OrderBy("cloudbrain.created_unix ASC") + cloudbrains := make([]*CloudbrainInfo, 0, 10) + if err := sess.Table(&Cloudbrain{}).Where(cond). + Find(&cloudbrains); err != nil { + log.Info("find error.") + } + return cloudbrains, nil +} + +func GetCloudbrainCompleteByTime(beginTime int64, endTime int64) ([]*CloudbrainInfo, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.And(builder.Gte{"cloudbrain.end_time": beginTime}, builder.Lte{"cloudbrain.end_time": endTime}), + ) + sess.OrderBy("cloudbrain.created_unix ASC") + cloudbrains := make([]*CloudbrainInfo, 0, 10) + if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). + Find(&cloudbrains); err != nil { + log.Info("find error.") + } + return cloudbrains, nil +} + +func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDurationStatistic) (int64, error) { + return xStatistic.Insert(cloudbrainDurationStatistic) +} diff --git a/models/models.go b/models/models.go index 4c2079cd8..7665c3b0f 100755 --- a/models/models.go +++ b/models/models.go @@ -179,6 +179,7 @@ func init() { new(UserMetrics), new(UserAnalysisPara), new(Invitation), + new(CloudbrainDurationStatistic), ) gonicNames := []string{"SSL", "UID"} diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 04cd7fe41..1fe10cb2c 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -5,12 +5,13 @@ package cron import ( - "code.gitea.io/gitea/services/reward" - "code.gitea.io/gitea/services/cloudbrain/resource" - "code.gitea.io/gitea/modules/modelarts" "context" "time" + "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/services/cloudbrain/resource" + "code.gitea.io/gitea/services/reward" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/migrations" repository_service "code.gitea.io/gitea/modules/repository" @@ -254,6 +255,17 @@ func registerSyncModelArtsTempJobs() { }) } +func registerHandleCloudbrainDurationStatistic() { + RegisterTaskFatal("handle_cloudbrain_duration_statistic", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@every 60m", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.CloudbrainDurationStatistic() + return nil + }) +} + func initBasicTasks() { registerUpdateMirrorTask() registerRepoHealthCheck() @@ -271,6 +283,7 @@ func initBasicTasks() { registerHandleRepoAndUserStatistic() registerHandleSummaryStatistic() + registerHandleCloudbrainDurationStatistic() registerSyncCloudbrainStatus() registerHandleOrgStatistic() diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go new file mode 100644 index 000000000..1aa943932 --- /dev/null +++ b/routers/repo/cloudbrain_statistic.go @@ -0,0 +1,188 @@ +package repo + +import ( + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" +) + +func CloudbrainDurationStatistic() { + log.Info("Generate Cloudbrain Duration statistic begin") + // yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + // CloudbrainDurationStatisticHour(yesterday) + log.Info("Generate Cloudbrain Duration statistic end") +} + +func CloudbrainDurationStatisticHour(ctx *context.Context) { + //获取规定时间段的云脑任务列表 + // page := ctx.QueryInt("page") + // pageSize := ctx.QueryInt("pagesize") + // if page <= 0 { + // page = 1 + // } + // if pageSize <= 0 { + // pageSize = 10 + // } + + // cloudBrainDurationRes := make(map[string]map[string]int) + // cloudBrainOneCardRes := make(map[string]int) + // cloudBrainTwoCardRes := make(map[string]int) + // c2NetCardRes := make(map[string]int) + // cDNetCenterCardRes := make(map[string]int) + // var WorkServerNumber int + // var AccCardsNum int + // endTime := time.Now().Unix() + // beginTime := time.Now().AddDate(0, 0, -1).Unix() + + // hour := time.Now().Hour() + // tStr := time.Now().Format("2006-01-02 15:04:05") + + currentTime := time.Now() + m, _ := time.ParseDuration("-1h") + beginTime := currentTime.Add(m).Unix() + endTime := currentTime.Unix() + // fmt.Println(beginTime) + + ciTasks1, err := models.GetCloudbrainRunning() + if err != nil { + // ctx.ServerError("Get job failed:", err) + log.Info("GetCloudbrainRunning err: %v", err) + return + } + ciTasks2, err := models.GetCloudbrainCompleteByTime(beginTime, endTime) + ciTasks := append(ciTasks1, ciTasks2...) + log.Info("beginTime: %s", beginTime) + log.Info("endTime: %s", endTime) + if err != nil { + // ctx.ServerError("Get job failed:", err) + log.Info("GetCloudbrainCompleteByTime err: %v", err) + return + } + models.LoadSpecs4CloudbrainInfo(ciTasks) + log.Info("ciTasks here: %s", ciTasks) + log.Info("count here: %s", len(ciTasks)) + cloudBrainCardRes := getCloudBrainCardRes(ciTasks, beginTime, endTime) + + //根据云脑任务列表获取云脑任务已使用的卡时，并区分是哪个智算中心，哪个卡类型的卡时，将这些信息存入新表 + + // cloudbrainDurationStat := models.CloudbrainDurationStatistic{ + // DateTime: date, + // HourTime: userNumber, + // Cluster: repositorySize, + // AiCenterName: allDatasetSize, + // AiCenterCode: organizationNumber, + // ComputeResource: repositoryNumer, + // AccCardType: forkRepositoryNumber, + // CardsTotalNum: mirrorRepositoryNumber, + // CardsTotalDuration: privateRepositoryNumer, + // QueueCode: publicRepositoryNumer, + // CreatedTime: privateRepositoryNumer, + // UpdatedTime: publicRepositoryNumer, + // } + + // if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { + // log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) + // } + // log.Info("cloudBrainDurationRes2: %s", cloudBrainDurationRes) + // cloudBrainDurationRes = append(cloudBrainDurationRes, cloudBrainOneCardRes) + // log.Info("cloudBrainDurationRes: %s", cloudBrainDurationRes) + // log.Info("cloudBrainOneCardRes: %s", cloudBrainOneCardRes) + log.Info("cloudBrainCardRes: %s", cloudBrainCardRes) + // log.Info("c2NetCardRes: %s", c2NetCardRes) + log.Info("finish summary cloudbrainDurationStat") +} + +func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) map[string]map[string]int { + var WorkServerNumber int + var AccCardsNum int + // cloudBrainCardRes := make(map[string]int) + cloudBrainAicenterNameList := make(map[string]string) + cloudBrainCardTypeList := make(map[string]string) + cloudBrainCenterNameAndCardType := make(map[string]map[string]int) + // var cloudbrainDurationInfo models.CloudbrainDurationInfo + for _, cloudbrain := range ciTasks { + + if _, ok := cloudBrainAicenterNameList[cloudbrain.Cloudbrain.Spec.AiCenterName]; !ok { + cloudBrainAicenterNameList[cloudbrain.Cloudbrain.Spec.AiCenterName] = cloudbrain.Cloudbrain.Spec.AiCenterName + } + + if cloudbrain.Cloudbrain.StartTime == 0 { + cloudbrain.Cloudbrain.StartTime = cloudbrain.Cloudbrain.CreatedUnix + } + if cloudbrain.Cloudbrain.EndTime == 0 { + cloudbrain.Cloudbrain.EndTime = cloudbrain.Cloudbrain.UpdatedUnix + } + if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { + WorkServerNumber = cloudbrain.Cloudbrain.WorkServerNumber + } else { + WorkServerNumber = 1 + } + if cloudbrain.Cloudbrain.Spec == nil { + AccCardsNum = 1 + } else { + AccCardsNum = cloudbrain.Cloudbrain.Spec.AccCardsNum + } + for k, _ := range cloudBrainAicenterNameList { + if cloudbrain.Cloudbrain.Spec.AiCenterName == cloudBrainAicenterNameList[k] { + if _, ok := cloudBrainCardTypeList[cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { + cloudBrainCardTypeList[cloudbrain.Cloudbrain.Spec.AccCardType] = cloudbrain.Cloudbrain.Spec.AccCardType + } + for i, _ := range cloudBrainCardTypeList { + if cloudBrainCenterNameAndCardType == nil { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] = 0 + } + if cloudbrain.Cloudbrain.Spec.AccCardType == cloudBrainCardTypeList[i] { + if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { + if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + } else { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + } + } else { + if int64(cloudbrain.Cloudbrain.StartTime) < beginTime && int64(cloudbrain.Cloudbrain.EndTime) < endTime { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + } else { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + } + } + } + } + // cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] = cloudBrainCardRes[cloudBrainCardTypeList[i]] + + // if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { + // if _, ok := cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType]; ok { + // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + // } else { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + // } + // } else { + // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + // } else { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + // } + // } + // } else { + // if _, ok := cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType]; ok { + // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime && int64(cloudbrain.Cloudbrain.EndTime) < endTime { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + // } else { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + // } + // } else { + // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime && int64(cloudbrain.Cloudbrain.EndTime) < endTime { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + // } else { + // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + // } + // } + // } + } + } + } + + return cloudBrainCenterNameAndCardType +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 66a357c79..e52686ee4 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -6,15 +6,16 @@ package routes import ( "bytes" - "code.gitea.io/gitea/routers/reward/point" - "code.gitea.io/gitea/routers/task" - "code.gitea.io/gitea/services/reward" "encoding/gob" "net/http" "path" "text/template" "time" + "code.gitea.io/gitea/routers/reward/point" + "code.gitea.io/gitea/routers/task" + "code.gitea.io/gitea/services/reward" + "code.gitea.io/gitea/modules/slideimage" "code.gitea.io/gitea/routers/image" @@ -374,6 +375,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/^:type(issues|pulls)$", reqSignIn, user.Issues) m.Get("/milestones", reqSignIn, reqMilestonesDashboardPageEnabled, user.Milestones) m.Get("/cloudbrains", reqSignIn, user.Cloudbrains) + m.Get("/duration", repo.CloudbrainDurationStatisticHour) // ***** START: User ***** m.Group("/user", func() { From ebd4c78e290e1919b89bbea3214978713b9d3e3f Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 13 Oct 2022 15:28:25 +0800 Subject: [PATCH 02/82] update --- routers/repo/cloudbrain_statistic.go | 39 +++------------------------- 1 file changed, 3 insertions(+), 36 deletions(-) diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 1aa943932..550e290d0 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -101,9 +101,7 @@ func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, end cloudBrainAicenterNameList := make(map[string]string) cloudBrainCardTypeList := make(map[string]string) cloudBrainCenterNameAndCardType := make(map[string]map[string]int) - // var cloudbrainDurationInfo models.CloudbrainDurationInfo for _, cloudbrain := range ciTasks { - if _, ok := cloudBrainAicenterNameList[cloudbrain.Cloudbrain.Spec.AiCenterName]; !ok { cloudBrainAicenterNameList[cloudbrain.Cloudbrain.Spec.AiCenterName] = cloudbrain.Cloudbrain.Spec.AiCenterName } @@ -125,14 +123,14 @@ func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, end AccCardsNum = cloudbrain.Cloudbrain.Spec.AccCardsNum } for k, _ := range cloudBrainAicenterNameList { + if cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]] == nil { + cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]] = make(map[string]int) + } if cloudbrain.Cloudbrain.Spec.AiCenterName == cloudBrainAicenterNameList[k] { if _, ok := cloudBrainCardTypeList[cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { cloudBrainCardTypeList[cloudbrain.Cloudbrain.Spec.AccCardType] = cloudbrain.Cloudbrain.Spec.AccCardType } for i, _ := range cloudBrainCardTypeList { - if cloudBrainCenterNameAndCardType == nil { - cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] = 0 - } if cloudbrain.Cloudbrain.Spec.AccCardType == cloudBrainCardTypeList[i] { if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { @@ -149,37 +147,6 @@ func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, end } } } - // cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] = cloudBrainCardRes[cloudBrainCardTypeList[i]] - - // if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { - // if _, ok := cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType]; ok { - // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) - // } else { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) - // } - // } else { - // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) - // } else { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) - // } - // } - // } else { - // if _, ok := cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType]; ok { - // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime && int64(cloudbrain.Cloudbrain.EndTime) < endTime { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) - // } else { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) - // } - // } else { - // if int64(cloudbrain.Cloudbrain.StartTime) < beginTime && int64(cloudbrain.Cloudbrain.EndTime) < endTime { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) - // } else { - // cloudBrainCardRes[cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) - // } - // } - // } } } } From d3bc16efa423a4e048cbd0aace4e59c0bbb9a496 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 13 Oct 2022 16:01:38 +0800 Subject: [PATCH 03/82] update --- models/cloudbrain_static.go | 19 ++++++++++ routers/repo/cloudbrain_statistic.go | 56 ++++++++++++++++++++-------- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index bd2bbaef8..b760fdc31 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -251,6 +251,25 @@ func GetCloudbrainCompleteByTime(beginTime int64, endTime int64) ([]*CloudbrainI return cloudbrains, nil } +func GetSpecByAiCenterCodeAndType(aiCenterCode string, accCardType string) ([]*CloudbrainSpec, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.Eq{"cloudbrain_spec.ai_center_code": aiCenterCode}, + ) + cond = cond.And( + builder.Eq{"cloudbrain_spec.acc_card_type": accCardType}, + ) + sess.OrderBy("cloudbrain_spec.created_unix ASC limit 1") + cloudbrainSpecs := make([]*CloudbrainSpec, 0, 10) + if err := sess.Table(&CloudbrainSpec{}).Where(cond). + Find(&cloudbrainSpecs); err != nil { + log.Info("find error.") + } + return cloudbrainSpecs, nil +} + func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDurationStatistic) (int64, error) { return xStatistic.Insert(cloudbrainDurationStatistic) } diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 550e290d0..0bc4938ff 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -63,7 +63,31 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { models.LoadSpecs4CloudbrainInfo(ciTasks) log.Info("ciTasks here: %s", ciTasks) log.Info("count here: %s", len(ciTasks)) - cloudBrainCardRes := getCloudBrainCardRes(ciTasks, beginTime, endTime) + cloudBrainCenterCodeAndCardTypeInfo := getcloudBrainCenterCodeAndCardTypeInfo(ciTasks, beginTime, endTime) + for centerCode, CardTypeInfo := range cloudBrainCenterCodeAndCardTypeInfo { + for cardType, cardDuration := range CardTypeInfo { + cloudbrain, err := models.GetSpecByAiCenterCodeAndType(centerCode, cardType) + if err != nil { + log.Info("GetSpecByAiCenterCodeAndType err: %v", err) + return + } + cloudbrainDurationStat := models.CloudbrainDurationStatistic{ + DateTime: date, + HourTime: userNumber, + Cluster: cloudbrain[0].Cluster, + AiCenterName: cloudbrain[0].AiCenterName, + AiCenterCode: centerCode, + ComputeResource: cloudbrain[0].ComputeResource, + AccCardType: cardType, + CardsTotalNum: mirrorRepositoryNumber, + CardsTotalDuration: cardDuration, + QueueCode: cloudbrain[0].QueueCode, + CreatedTime: privateRepositoryNumer, + UpdatedTime: publicRepositoryNumer, + } + } + + } //根据云脑任务列表获取云脑任务已使用的卡时，并区分是哪个智算中心，哪个卡类型的卡时，将这些信息存入新表 @@ -89,21 +113,21 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { // cloudBrainDurationRes = append(cloudBrainDurationRes, cloudBrainOneCardRes) // log.Info("cloudBrainDurationRes: %s", cloudBrainDurationRes) // log.Info("cloudBrainOneCardRes: %s", cloudBrainOneCardRes) - log.Info("cloudBrainCardRes: %s", cloudBrainCardRes) + // log.Info("cloudBrainCardRes: %s", cloudBrainCardRes) // log.Info("c2NetCardRes: %s", c2NetCardRes) log.Info("finish summary cloudbrainDurationStat") } -func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) map[string]map[string]int { +func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) map[string]map[string]int { var WorkServerNumber int var AccCardsNum int // cloudBrainCardRes := make(map[string]int) - cloudBrainAicenterNameList := make(map[string]string) + cloudBrainAiCenterCodeList := make(map[string]string) cloudBrainCardTypeList := make(map[string]string) - cloudBrainCenterNameAndCardType := make(map[string]map[string]int) + cloudBrainCenterCodeAndCardType := make(map[string]map[string]int) for _, cloudbrain := range ciTasks { - if _, ok := cloudBrainAicenterNameList[cloudbrain.Cloudbrain.Spec.AiCenterName]; !ok { - cloudBrainAicenterNameList[cloudbrain.Cloudbrain.Spec.AiCenterName] = cloudbrain.Cloudbrain.Spec.AiCenterName + if _, ok := cloudBrainAiCenterCodeList[cloudbrain.Cloudbrain.Spec.AiCenterCode]; !ok { + cloudBrainAiCenterCodeList[cloudbrain.Cloudbrain.Spec.AiCenterCode] = cloudbrain.Cloudbrain.Spec.AiCenterCode } if cloudbrain.Cloudbrain.StartTime == 0 { @@ -122,11 +146,11 @@ func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, end } else { AccCardsNum = cloudbrain.Cloudbrain.Spec.AccCardsNum } - for k, _ := range cloudBrainAicenterNameList { - if cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]] == nil { - cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]] = make(map[string]int) + for k, _ := range cloudBrainAiCenterCodeList { + if cloudBrainCenterCodeAndCardType[cloudBrainAiCenterCodeList[k]] == nil { + cloudBrainCenterCodeAndCardType[cloudBrainAiCenterCodeList[k]] = make(map[string]int) } - if cloudbrain.Cloudbrain.Spec.AiCenterName == cloudBrainAicenterNameList[k] { + if cloudbrain.Cloudbrain.Spec.AiCenterCode == cloudBrainAiCenterCodeList[k] { if _, ok := cloudBrainCardTypeList[cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { cloudBrainCardTypeList[cloudbrain.Cloudbrain.Spec.AccCardType] = cloudbrain.Cloudbrain.Spec.AccCardType } @@ -134,15 +158,15 @@ func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, end if cloudbrain.Cloudbrain.Spec.AccCardType == cloudBrainCardTypeList[i] { if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { - cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + cloudBrainCenterCodeAndCardType[cloudBrainAiCenterCodeList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) } else { - cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + cloudBrainCenterCodeAndCardType[cloudBrainAiCenterCodeList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) } } else { if int64(cloudbrain.Cloudbrain.StartTime) < beginTime && int64(cloudbrain.Cloudbrain.EndTime) < endTime { - cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + cloudBrainCenterCodeAndCardType[cloudBrainAiCenterCodeList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) } else { - cloudBrainCenterNameAndCardType[cloudBrainAicenterNameList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + cloudBrainCenterCodeAndCardType[cloudBrainAiCenterCodeList[k]][cloudBrainCardTypeList[i]] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) } } } @@ -151,5 +175,5 @@ func getCloudBrainCardRes(ciTasks []*models.CloudbrainInfo, beginTime int64, end } } - return cloudBrainCenterNameAndCardType + return cloudBrainCenterCodeAndCardType } From a8341fcb425841b4babac0087c4794dfea024d1b Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 14 Oct 2022 17:51:05 +0800 Subject: [PATCH 04/82] update --- models/cloudbrain_spec.go | 4 +- models/cloudbrain_static.go | 69 +++++++++-- routers/api/v1/api.go | 4 + routers/api/v1/repo/cloudbrain_dashboard.go | 81 ++++++++++++- routers/repo/cloudbrain_statistic.go | 123 ++++++++++---------- 5 files changed, 208 insertions(+), 73 deletions(-) diff --git a/models/cloudbrain_spec.go b/models/cloudbrain_spec.go index c32e4b0fd..49a4d603e 100644 --- a/models/cloudbrain_spec.go +++ b/models/cloudbrain_spec.go @@ -9,7 +9,7 @@ type CloudbrainSpec struct { SpecId int64 `xorm:"index"` SourceSpecId string AccCardsNum int - AccCardType string + AccCardType string `xorm:"index"` CpuCores int MemGiB float32 GPUMemGiB float32 @@ -19,7 +19,7 @@ type CloudbrainSpec struct { QueueId int64 QueueCode string Cluster string - AiCenterCode string + AiCenterCode string `xorm:"index"` AiCenterName string IsExclusive bool ExclusiveOrg string diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index b760fdc31..b8f1c8e62 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -1,6 +1,7 @@ package models import ( + "fmt" "strconv" "time" @@ -39,18 +40,20 @@ type TaskDetail struct { } type CloudbrainDurationStatistic struct { - ID int64 `xorm:"pk autoincr"` - Cluster string `xorm:"notnull"` + ID int64 `xorm:"pk autoincr"` + Cluster string AiCenterCode string AiCenterName string ComputeResource string AccCardType string - QueueCode string - CardsTotalNum int + TotalUse bool + TotalCanUse bool DateTime string + DayTime string HourTime int CardsTotalDuration int + CardsTotalNum int DeletedTime timeutil.TimeStamp `xorm:"deleted"` CreatedTime timeutil.TimeStamp `xorm:"created"` @@ -256,12 +259,8 @@ func GetSpecByAiCenterCodeAndType(aiCenterCode string, accCardType string) ([]*C defer sess.Close() var cond = builder.NewCond() cond = cond.And( - builder.Eq{"cloudbrain_spec.ai_center_code": aiCenterCode}, + builder.And(builder.Eq{"cloudbrain_spec.ai_center_code": aiCenterCode}, builder.Eq{"cloudbrain_spec.acc_card_type": accCardType}), ) - cond = cond.And( - builder.Eq{"cloudbrain_spec.acc_card_type": accCardType}, - ) - sess.OrderBy("cloudbrain_spec.created_unix ASC limit 1") cloudbrainSpecs := make([]*CloudbrainSpec, 0, 10) if err := sess.Table(&CloudbrainSpec{}).Where(cond). Find(&cloudbrainSpecs); err != nil { @@ -273,3 +272,55 @@ func GetSpecByAiCenterCodeAndType(aiCenterCode string, accCardType string) ([]*C func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDurationStatistic) (int64, error) { return xStatistic.Insert(cloudbrainDurationStatistic) } + +func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode string, accCardType string, tatalUse bool, totalCanUse bool) error { + sess := xStatistic.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Begin: %v", err) + } + + if _, err := sess.Where("day_time = ? AND hour_time = ? AND ai_center_code = ? AND acc_card_type = ? And total_use = ? And total_can_use = ?", date, hour, aiCenterCode, accCardType, tatalUse, totalCanUse).Delete(&CloudbrainDurationStatistic{}); err != nil { + return fmt.Errorf("Delete: %v", err) + } + + if err := sess.Commit(); err != nil { + sess.Close() + return fmt.Errorf("Commit: %v", err) + } + + sess.Close() + return nil +} + +func GetCanUseCardInfo() ([]*ResourceQueue, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.And(builder.Eq{"resource_queue.is_automatic_sync": false}), + ) + ResourceQueues := make([]*ResourceQueue, 0, 10) + if err := sess.Table(&ResourceQueue{}).Where(cond). + Find(&ResourceQueues); err != nil { + log.Info("find error.") + } + return ResourceQueues, nil +} +func GetCardDurationStatistics(beginTime time.Time, endTime time.Time, totalUse bool, totalCanUse bool) ([]*CloudbrainDurationStatistic, error) { + sess := xStatistic.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.And(builder.Gte{"cloudbrain_duration_statistic.created_time": beginTime.Unix()}, builder.Lte{"cloudbrain_duration_statistic.created_time": endTime.Unix()}), + ) + cond = cond.And( + builder.And(builder.Eq{"cloudbrain_duration_statistic.total_use": totalUse}, builder.Eq{"cloudbrain_duration_statistic.total_can_use": totalCanUse}), + ) + CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0, 10) + if err := sess.Table(&CloudbrainDurationStatistic{}).Where(cond). + Find(&CloudbrainDurationStatistics); err != nil { + log.Info("find error.") + } + return CloudbrainDurationStatistics, nil +} diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 8e1d725ed..06be12e92 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -599,6 +599,10 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/hours_data", repo.GetCloudbrainsCreateHoursData) m.Get("/waitting_top_data", repo.GetWaittingTop) m.Get("/running_top_data", repo.GetRunningTop) + + m.Get("/overview_resource", repo.GetCloudbrainResourceOverview) + m.Get("/resource_usage", repo.GetCloudbrainResourceUsage) + m.Get("/resource_usage_detail", repo.GetCloudbrainResourceUsageDetail) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 54c0ddc20..7d348d578 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -545,7 +545,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { recordBeginTime := time.Unix(int64(recordCloudbrain[0].Cloudbrain.CreatedUnix), 0) beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) if err != nil { - log.Error("Parameter is wrong", err) + log.Error("getCloudbrainTimePeroid error:", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) return } @@ -1403,3 +1403,82 @@ func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (t return beginTime, endTime, nil } + +func GetCloudbrainResourceOverview(ctx *context.Context) { + resourceQueues, err := models.GetCanUseCardInfo() + if err != nil { + log.Info("GetCanUseCardInfo err: %v", err) + return + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "resourceQueues": resourceQueues, + }) + +} + +func GetCloudbrainResourceUsage(ctx *context.Context) { + recordBeginTime := time.Now().AddDate(0, 0, -6) + beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) + if err != nil { + log.Error("getCloudbrainTimePeroid error:", err) + return + } + cardUsageRes := make(map[string]int) + cardCanUsageRes := make(map[string]int) + cardUseInfo, err := models.GetCardDurationStatistics(beginTime, endTime, true, false) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return + } + cardCanUseInfo, err := models.GetCardDurationStatistics(beginTime, endTime, false, true) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return + } + + for _, cloudbrainStat := range cardUseInfo { + if _, ok := cardUsageRes[cloudbrainStat.AiCenterCode]; !ok { + cardUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration + } else { + cardUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration + } + } + + for _, cloudbrainStat := range cardCanUseInfo { + if _, ok := cardCanUsageRes[cloudbrainStat.AiCenterCode]; !ok { + cardCanUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration + } else { + cardCanUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration + } + } + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "cardUseInfo": cardUseInfo, + "cardCanUseInfo": cardCanUseInfo, + "cardUsageRes": cardUsageRes, + "cardCanUsageRes": cardCanUsageRes, + }) + +} +func GetCloudbrainResourceUsageDetail(ctx *context.Context) { + recordBeginTime := time.Now().AddDate(0, 0, -6) + beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) + if err != nil { + log.Error("getCloudbrainTimePeroid error:", err) + return + } + totalUse := true + totalCanUse := false + cardDurationStatisticsInfo, err := models.GetCardDurationStatistics(beginTime, endTime, totalUse, totalCanUse) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return + } + ctx.JSON(http.StatusOK, map[string]interface{}{ + "cardDurationStatisticsInfo": cardDurationStatisticsInfo, + "beginTime": beginTime, + "endTime": endTime, + }) + +} diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 0bc4938ff..c7546c1c6 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -6,6 +6,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" ) func CloudbrainDurationStatistic() { @@ -16,28 +17,9 @@ func CloudbrainDurationStatistic() { } func CloudbrainDurationStatisticHour(ctx *context.Context) { - //获取规定时间段的云脑任务列表 - // page := ctx.QueryInt("page") - // pageSize := ctx.QueryInt("pagesize") - // if page <= 0 { - // page = 1 - // } - // if pageSize <= 0 { - // pageSize = 10 - // } - - // cloudBrainDurationRes := make(map[string]map[string]int) - // cloudBrainOneCardRes := make(map[string]int) - // cloudBrainTwoCardRes := make(map[string]int) - // c2NetCardRes := make(map[string]int) - // cDNetCenterCardRes := make(map[string]int) - // var WorkServerNumber int - // var AccCardsNum int - // endTime := time.Now().Unix() - // beginTime := time.Now().AddDate(0, 0, -1).Unix() - - // hour := time.Now().Hour() - // tStr := time.Now().Format("2006-01-02 15:04:05") + hourTime := time.Now().Hour() + dateTime := time.Now().Format("2006-01-02 15:04:05") + dayTime := time.Now().Format("2006-01-02") currentTime := time.Now() m, _ := time.ParseDuration("-1h") @@ -71,50 +53,69 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { log.Info("GetSpecByAiCenterCodeAndType err: %v", err) return } - cloudbrainDurationStat := models.CloudbrainDurationStatistic{ - DateTime: date, - HourTime: userNumber, - Cluster: cloudbrain[0].Cluster, - AiCenterName: cloudbrain[0].AiCenterName, - AiCenterCode: centerCode, - ComputeResource: cloudbrain[0].ComputeResource, - AccCardType: cardType, - CardsTotalNum: mirrorRepositoryNumber, - CardsTotalDuration: cardDuration, - QueueCode: cloudbrain[0].QueueCode, - CreatedTime: privateRepositoryNumer, - UpdatedTime: publicRepositoryNumer, + log.Info("cloudbrain: %s", cloudbrain) + if cloudbrain != nil { + totalUse := true + totalCanUse := false + if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType, totalUse, totalCanUse); err != nil { + log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) + return + } + cloudbrainDurationStat := models.CloudbrainDurationStatistic{ + DateTime: dateTime, + DayTime: dayTime, + HourTime: hourTime, + Cluster: cloudbrain[0].Cluster, + AiCenterName: cloudbrain[0].AiCenterName, + AiCenterCode: centerCode, + ComputeResource: cloudbrain[0].ComputeResource, + AccCardType: cardType, + CardsTotalDuration: cardDuration, + CreatedTime: timeutil.TimeStampNow(), + TotalUse: true, + TotalCanUse: false, + } + if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { + log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) + } } } } - //根据云脑任务列表获取云脑任务已使用的卡时，并区分是哪个智算中心，哪个卡类型的卡时，将这些信息存入新表 - - // cloudbrainDurationStat := models.CloudbrainDurationStatistic{ - // DateTime: date, - // HourTime: userNumber, - // Cluster: repositorySize, - // AiCenterName: allDatasetSize, - // AiCenterCode: organizationNumber, - // ComputeResource: repositoryNumer, - // AccCardType: forkRepositoryNumber, - // CardsTotalNum: mirrorRepositoryNumber, - // CardsTotalDuration: privateRepositoryNumer, - // QueueCode: publicRepositoryNumer, - // CreatedTime: privateRepositoryNumer, - // UpdatedTime: publicRepositoryNumer, - // } - - // if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { - // log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) - // } - // log.Info("cloudBrainDurationRes2: %s", cloudBrainDurationRes) - // cloudBrainDurationRes = append(cloudBrainDurationRes, cloudBrainOneCardRes) - // log.Info("cloudBrainDurationRes: %s", cloudBrainDurationRes) - // log.Info("cloudBrainOneCardRes: %s", cloudBrainOneCardRes) - // log.Info("cloudBrainCardRes: %s", cloudBrainCardRes) - // log.Info("c2NetCardRes: %s", c2NetCardRes) + resourceQueues, err := models.GetCanUseCardInfo() + if err != nil { + log.Info("GetCanUseCardInfo err: %v", err) + return + } + log.Info("resourceQueues here: %s", resourceQueues) + for _, resourceQueue := range resourceQueues { + totalUse := false + totalCanUse := true + if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, resourceQueue.AiCenterCode, resourceQueue.AccCardType, totalUse, totalCanUse); err != nil { + log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) + return + } + cardsTotalDuration := resourceQueue.CardsTotalNum * 1 * 60 * 60 + cloudbrainDurationStat := models.CloudbrainDurationStatistic{ + DateTime: dateTime, + DayTime: dayTime, + HourTime: hourTime, + Cluster: resourceQueue.Cluster, + AiCenterName: resourceQueue.AiCenterName, + AiCenterCode: resourceQueue.AiCenterCode, + ComputeResource: resourceQueue.ComputeResource, + AccCardType: resourceQueue.AccCardType, + CardsTotalDuration: cardsTotalDuration, + CardsTotalNum: resourceQueue.CardsTotalNum, + CreatedTime: timeutil.TimeStampNow(), + TotalUse: false, + TotalCanUse: true, + } + if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { + log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) + } + } log.Info("finish summary cloudbrainDurationStat") } From 52d69f1287693f5b010b1a83150a0018d2286bcd Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 17 Oct 2022 18:02:14 +0800 Subject: [PATCH 05/82] update --- models/cloudbrain_static.go | 53 +++- modules/cron/tasks_basic.go | 2 +- routers/api/v1/repo/cloudbrain_dashboard.go | 301 +++++++++++++++++--- routers/repo/cloudbrain_statistic.go | 25 +- 4 files changed, 313 insertions(+), 68 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index b8f1c8e62..ae9358dc1 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -46,7 +46,6 @@ type CloudbrainDurationStatistic struct { AiCenterName string ComputeResource string AccCardType string - TotalUse bool TotalCanUse bool DateTime string @@ -55,9 +54,20 @@ type CloudbrainDurationStatistic struct { CardsTotalDuration int CardsTotalNum int - DeletedTime timeutil.TimeStamp `xorm:"deleted"` - CreatedTime timeutil.TimeStamp `xorm:"created"` - UpdatedTime timeutil.TimeStamp `xorm:"updated"` + DeletedUnix timeutil.TimeStamp `xorm:"deleted"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` +} +type DurationStatisticOptions struct { + BeginTime time.Time + EndTime time.Time + AiCenterCode string +} +type DateCloudbrainStatistic struct { + Date string `json:"date"` + AiCenterUsageDuration map[string]int `json:"aiCenterUsageDuration"` + AiCenterTotalDuration map[string]int `json:"aiCenterTotalDuration"` + AiCenterUsageRate map[string]int `json:"aiCenterUsageRate"` } func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { @@ -273,14 +283,14 @@ func InsertCloudbrainDurationStatistic(cloudbrainDurationStatistic *CloudbrainDu return xStatistic.Insert(cloudbrainDurationStatistic) } -func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode string, accCardType string, tatalUse bool, totalCanUse bool) error { +func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode string, accCardType string, totalCanUse bool) error { sess := xStatistic.NewSession() defer sess.Close() if err := sess.Begin(); err != nil { return fmt.Errorf("Begin: %v", err) } - if _, err := sess.Where("day_time = ? AND hour_time = ? AND ai_center_code = ? AND acc_card_type = ? And total_use = ? And total_can_use = ?", date, hour, aiCenterCode, accCardType, tatalUse, totalCanUse).Delete(&CloudbrainDurationStatistic{}); err != nil { + if _, err := sess.Where("day_time = ? AND hour_time = ? AND ai_center_code = ? AND acc_card_type = ? And total_can_use = ?", date, hour, aiCenterCode, accCardType, totalCanUse).Delete(&CloudbrainDurationStatistic{}); err != nil { return fmt.Errorf("Delete: %v", err) } @@ -307,16 +317,21 @@ func GetCanUseCardInfo() ([]*ResourceQueue, error) { } return ResourceQueues, nil } -func GetCardDurationStatistics(beginTime time.Time, endTime time.Time, totalUse bool, totalCanUse bool) ([]*CloudbrainDurationStatistic, error) { + +func GetCardDurationStatistics(opts *DurationStatisticOptions) ([]*CloudbrainDurationStatistic, error) { sess := xStatistic.NewSession() defer sess.Close() var cond = builder.NewCond() - cond = cond.And( - builder.And(builder.Gte{"cloudbrain_duration_statistic.created_time": beginTime.Unix()}, builder.Lte{"cloudbrain_duration_statistic.created_time": endTime.Unix()}), - ) - cond = cond.And( - builder.And(builder.Eq{"cloudbrain_duration_statistic.total_use": totalUse}, builder.Eq{"cloudbrain_duration_statistic.total_can_use": totalCanUse}), - ) + if opts.BeginTime.Unix() > 0 && opts.EndTime.Unix() > 0 { + cond = cond.And( + builder.And(builder.Gte{"cloudbrain_duration_statistic.created_unix": opts.BeginTime.Unix()}, builder.Lte{"cloudbrain_duration_statistic.created_unix": opts.EndTime.Unix()}), + ) + } + if opts.AiCenterCode != "" { + cond = cond.And( + builder.Eq{"cloudbrain_duration_statistic.ai_center_code": opts.AiCenterCode}, + ) + } CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0, 10) if err := sess.Table(&CloudbrainDurationStatistic{}).Where(cond). Find(&CloudbrainDurationStatistics); err != nil { @@ -324,3 +339,15 @@ func GetCardDurationStatistics(beginTime time.Time, endTime time.Time, totalUse } return CloudbrainDurationStatistics, nil } + +func GetDurationRecordBeginTime() ([]*CloudbrainDurationStatistic, error) { + sess := xStatistic.NewSession() + defer sess.Close() + sess.OrderBy("cloudbrain_duration_statistic.id ASC limit 1") + CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0) + if err := sess.Table(&CloudbrainDurationStatistic{}).Unscoped(). + Find(&CloudbrainDurationStatistics); err != nil { + log.Info("find error.") + } + return CloudbrainDurationStatistics, nil +} diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 1fe10cb2c..b68d747d8 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -261,7 +261,7 @@ func registerHandleCloudbrainDurationStatistic() { RunAtStart: false, Schedule: "@every 60m", }, func(ctx context.Context, _ *models.User, _ Config) error { - repo.CloudbrainDurationStatistic() + repo.CloudbrainDurationStatisticHour() return nil }) } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 7d348d578..caac24aad 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -532,6 +532,21 @@ func getPageDateCloudbrainInfo(dateCloudbrainInfo []DateCloudbrainInfo, page int } +func getPageDateCloudbrainDuration(dateCloudbrainDuration []models.DateCloudbrainStatistic, page int, pagesize int) []models.DateCloudbrainStatistic { + begin := (page - 1) * pagesize + end := (page) * pagesize + + if begin > len(dateCloudbrainDuration)-1 { + return nil + } + if end > len(dateCloudbrainDuration)-1 { + return dateCloudbrainDuration[begin:] + } else { + return dateCloudbrainDuration[begin:end] + } + +} + func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { queryType := ctx.QueryTrim("type") beginTimeStr := ctx.QueryTrim("beginTime") @@ -1426,59 +1441,277 @@ func GetCloudbrainResourceUsage(ctx *context.Context) { } cardUsageRes := make(map[string]int) cardCanUsageRes := make(map[string]int) - cardUseInfo, err := models.GetCardDurationStatistics(beginTime, endTime, true, false) - if err != nil { - log.Error("GetCardDurationStatistics error:", err) - return - } - cardCanUseInfo, err := models.GetCardDurationStatistics(beginTime, endTime, false, true) + cardUtilizationRate := make(map[string]int) + // cardDurationStatistics, err := models.GetCardDurationStatistics(beginTime, endTime) + cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ + BeginTime: beginTime, + EndTime: endTime, + }) if err != nil { log.Error("GetCardDurationStatistics error:", err) return } - for _, cloudbrainStat := range cardUseInfo { - if _, ok := cardUsageRes[cloudbrainStat.AiCenterCode]; !ok { - cardUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration + for _, cloudbrainStat := range cardDurationStatistics { + if cloudbrainStat.TotalCanUse { + if _, ok := cardCanUsageRes[cloudbrainStat.AiCenterCode]; !ok { + cardCanUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration + } else { + cardCanUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration + } } else { - cardUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration + if _, ok := cardUsageRes[cloudbrainStat.AiCenterCode]; !ok { + cardUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration + } else { + cardUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration + } } } - - for _, cloudbrainStat := range cardCanUseInfo { - if _, ok := cardCanUsageRes[cloudbrainStat.AiCenterCode]; !ok { - cardCanUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration - } else { - cardCanUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration + for k, v := range cardCanUsageRes { + for j, i := range cardUsageRes { + if k == j { + cardUtilizationRate[k] = i / v + } } } ctx.JSON(http.StatusOK, map[string]interface{}{ - "cardUseInfo": cardUseInfo, - "cardCanUseInfo": cardCanUseInfo, - "cardUsageRes": cardUsageRes, - "cardCanUsageRes": cardCanUsageRes, + "cardDurationStatistics": cardDurationStatistics, + "cardUsageRes": cardUsageRes, + "cardCanUsageRes": cardCanUsageRes, + "cardUtilizationRate": cardUtilizationRate, }) } + func GetCloudbrainResourceUsageDetail(ctx *context.Context) { - recordBeginTime := time.Now().AddDate(0, 0, -6) - beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) - if err != nil { - log.Error("getCloudbrainTimePeroid error:", err) - return + queryType := ctx.QueryTrim("type") + now := time.Now() + + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + var beginTime time.Time + var endTime time.Time + var endTimeTemp time.Time + dayCloudbrainDuration := make([]models.DateCloudbrainStatistic, 0) + var err error + var count int + if queryType != "" { + if queryType == "all" { + recordCloudbrainDuration, err := models.GetDurationRecordBeginTime() + if err != nil { + log.Error("Can not get GetDurationRecordBeginTime", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() + beginTime = brainRecordBeginTime + endTime = now + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + } else if queryType == "today" { + beginTime = now.AddDate(0, 0, 0) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + + } else if queryType == "yesterday" { + beginTime = now.AddDate(0, 0, -1) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + + } else if queryType == "last_7day" { + beginTime = now.AddDate(0, 0, -6) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + endTimeTemp = time.Date(endTimeTemp.Year(), endTimeTemp.Month(), endTimeTemp.Day(), 0, 0, 0, 0, now.Location()) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + } else if queryType == "last_30day" { + beginTime = now.AddDate(0, 0, -29) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + } else if queryType == "current_month" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + + } else if queryType == "current_year" { + endTime = now + beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location()) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + + } else if queryType == "last_month" { + + lastMonthTime := now.AddDate(0, -1, 0) + beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + + } + + } else { + if beginTimeStr == "" || endTimeStr == "" { + //如果查询类型和开始时间结束时间都未设置，按queryType=all处理 + recordCloudbrainDuration, err := models.GetDurationRecordBeginTime() + if err != nil { + log.Error("Can not get recordCloudbrain", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() + beginTime = brainRecordBeginTime + endTime = now + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + } else { + beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + if endTime.After(time.Now()) { + endTime = time.Now() + } + endTimeTemp = beginTime.AddDate(0, 0, 1) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + } + } - totalUse := true - totalCanUse := false - cardDurationStatisticsInfo, err := models.GetCardDurationStatistics(beginTime, endTime, totalUse, totalCanUse) - if err != nil { - log.Error("GetCardDurationStatistics error:", err) - return + + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pagesize := ctx.QueryInt("pagesize") + if pagesize <= 0 { + pagesize = 5 } + pageDateCloudbrainDuration := getPageDateCloudbrainDuration(dayCloudbrainDuration, page, pagesize) + ctx.JSON(http.StatusOK, map[string]interface{}{ - "cardDurationStatisticsInfo": cardDurationStatisticsInfo, - "beginTime": beginTime, - "endTime": endTime, + "totalCount": count, + "pageDateCloudbrainDuration": pageDateCloudbrainDuration, }) } + +func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrainStatistics []*models.CloudbrainDurationStatistic) (map[string]int, map[string]int, map[string]int) { + + aiCenterTotalDuration := make(map[string]int) + aiCenterUsageDuration := make(map[string]int) + aiCenterUsageRate := make(map[string]int) + for _, cloudbrainStatistic := range cloudbrainStatistics { + if int64(cloudbrainStatistic.CreatedUnix) >= beginTime.Unix() && int64(cloudbrainStatistic.CreatedUnix) < endTime.Unix() { + if cloudbrainStatistic.TotalCanUse { + if _, ok := aiCenterTotalDuration[cloudbrainStatistic.AiCenterCode]; !ok { + aiCenterTotalDuration[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration + } else { + aiCenterTotalDuration[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration + } + } else { + if _, ok := aiCenterUsageDuration[cloudbrainStatistic.AiCenterCode]; !ok { + aiCenterUsageDuration[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration + } else { + aiCenterUsageDuration[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration + } + } + } + } + for k, v := range aiCenterTotalDuration { + for i, j := range aiCenterUsageDuration { + if k == i { + aiCenterUsageRate[k] = j / v + } + } + } + + return aiCenterUsageDuration, aiCenterTotalDuration, aiCenterUsageRate +} + +func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time) ([]models.DateCloudbrainStatistic, int, error) { + now := time.Now() + endTimeTemp := time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + if endTimeTemp.Equal(endTime) { + endTimeTemp = endTimeTemp.AddDate(0, 0, -1) + } + cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ + BeginTime: beginTime, + EndTime: endTime, + }) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return nil, 0, err + } + dayCloudbrainInfo := make([]models.DateCloudbrainStatistic, 0) + count := 0 + for beginTime.Before(endTimeTemp) || beginTime.Equal(endTimeTemp) { + aiCenterUsageDuration, aiCenterTotalDuration, aiCenterUsageRate := getAiCenterUsageDuration(endTimeTemp, endTime, cardDurationStatistics) + dayCloudbrainInfo = append(dayCloudbrainInfo, models.DateCloudbrainStatistic{ + Date: endTimeTemp.Format("2006/01/02"), + AiCenterUsageDuration: aiCenterUsageDuration, + AiCenterTotalDuration: aiCenterTotalDuration, + AiCenterUsageRate: aiCenterUsageRate, + }) + endTime = endTimeTemp + endTimeTemp = endTimeTemp.AddDate(0, 0, -1) + count += 1 + } + return dayCloudbrainInfo, count, nil +} diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index c7546c1c6..da1849096 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -4,19 +4,11 @@ import ( "time" "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" ) -func CloudbrainDurationStatistic() { - log.Info("Generate Cloudbrain Duration statistic begin") - // yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") - // CloudbrainDurationStatisticHour(yesterday) - log.Info("Generate Cloudbrain Duration statistic end") -} - -func CloudbrainDurationStatisticHour(ctx *context.Context) { +func CloudbrainDurationStatisticHour() { hourTime := time.Now().Hour() dateTime := time.Now().Format("2006-01-02 15:04:05") dayTime := time.Now().Format("2006-01-02") @@ -25,11 +17,9 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { m, _ := time.ParseDuration("-1h") beginTime := currentTime.Add(m).Unix() endTime := currentTime.Unix() - // fmt.Println(beginTime) ciTasks1, err := models.GetCloudbrainRunning() if err != nil { - // ctx.ServerError("Get job failed:", err) log.Info("GetCloudbrainRunning err: %v", err) return } @@ -38,7 +28,6 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { log.Info("beginTime: %s", beginTime) log.Info("endTime: %s", endTime) if err != nil { - // ctx.ServerError("Get job failed:", err) log.Info("GetCloudbrainCompleteByTime err: %v", err) return } @@ -55,9 +44,8 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { } log.Info("cloudbrain: %s", cloudbrain) if cloudbrain != nil { - totalUse := true totalCanUse := false - if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType, totalUse, totalCanUse); err != nil { + if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType, totalCanUse); err != nil { log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) return } @@ -71,8 +59,7 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { ComputeResource: cloudbrain[0].ComputeResource, AccCardType: cardType, CardsTotalDuration: cardDuration, - CreatedTime: timeutil.TimeStampNow(), - TotalUse: true, + CreatedUnix: timeutil.TimeStampNow(), TotalCanUse: false, } if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { @@ -90,9 +77,8 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { } log.Info("resourceQueues here: %s", resourceQueues) for _, resourceQueue := range resourceQueues { - totalUse := false totalCanUse := true - if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, resourceQueue.AiCenterCode, resourceQueue.AccCardType, totalUse, totalCanUse); err != nil { + if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, resourceQueue.AiCenterCode, resourceQueue.AccCardType, totalCanUse); err != nil { log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) return } @@ -108,8 +94,7 @@ func CloudbrainDurationStatisticHour(ctx *context.Context) { AccCardType: resourceQueue.AccCardType, CardsTotalDuration: cardsTotalDuration, CardsTotalNum: resourceQueue.CardsTotalNum, - CreatedTime: timeutil.TimeStampNow(), - TotalUse: false, + CreatedUnix: timeutil.TimeStampNow(), TotalCanUse: true, } if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { From 9d1ceabf9cfab9a6fd581803bc8a7f8e88e18ccf Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 18 Oct 2022 18:04:22 +0800 Subject: [PATCH 06/82] update --- models/cloudbrain_static.go | 23 ++- modules/cron/tasks_basic.go | 4 +- routers/api/v1/repo/cloudbrain_dashboard.go | 148 +++++++++++++++++++- 3 files changed, 161 insertions(+), 14 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index ae9358dc1..3c5da2fd3 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -63,11 +63,23 @@ type DurationStatisticOptions struct { EndTime time.Time AiCenterCode string } + +type DurationRateStatistic struct { + AiCenterTotalDurationStat map[string]int `json:"aiCenterTotalDurationStat"` + AiCenterUsageDurationStat map[string]int `json:"aiCenterUsageDurationStat"` + TotalUsageRate float64 `json:"totalUsageRate"` +} type DateCloudbrainStatistic struct { - Date string `json:"date"` - AiCenterUsageDuration map[string]int `json:"aiCenterUsageDuration"` - AiCenterTotalDuration map[string]int `json:"aiCenterTotalDuration"` - AiCenterUsageRate map[string]int `json:"aiCenterUsageRate"` + Date string `json:"date"` + AiCenterUsageDuration map[string]int `json:"aiCenterUsageDuration"` + AiCenterTotalDuration map[string]int `json:"aiCenterTotalDuration"` + AiCenterUsageRate map[string]float64 `json:"aiCenterUsageRate"` +} + +type HourTimeStatistic struct { + HourTimeUsageDuration map[int]int `json:"hourTimeUsageDuration"` + HourTimeTotalDuration map[int]int `json:"hourTimeTotalDuration"` + HourTimeUsageRate map[int]float64 `json:"hourTimeUsageRate"` } func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { @@ -345,8 +357,7 @@ func GetDurationRecordBeginTime() ([]*CloudbrainDurationStatistic, error) { defer sess.Close() sess.OrderBy("cloudbrain_duration_statistic.id ASC limit 1") CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0) - if err := sess.Table(&CloudbrainDurationStatistic{}).Unscoped(). - Find(&CloudbrainDurationStatistics); err != nil { + if err := sess.Table(&CloudbrainDurationStatistic{}).Find(&CloudbrainDurationStatistics); err != nil { log.Info("find error.") } return CloudbrainDurationStatistics, nil diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index b68d747d8..38ac37852 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -259,7 +259,7 @@ func registerHandleCloudbrainDurationStatistic() { RegisterTaskFatal("handle_cloudbrain_duration_statistic", &BaseConfig{ Enabled: true, RunAtStart: false, - Schedule: "@every 60m", + Schedule: "55 59 * * * ?", }, func(ctx context.Context, _ *models.User, _ Config) error { repo.CloudbrainDurationStatisticHour() return nil @@ -283,7 +283,6 @@ func initBasicTasks() { registerHandleRepoAndUserStatistic() registerHandleSummaryStatistic() - registerHandleCloudbrainDurationStatistic() registerSyncCloudbrainStatus() registerHandleOrgStatistic() @@ -292,4 +291,5 @@ func initBasicTasks() { //registerRewardPeriodTask() registerCloudbrainPointDeductTask() + registerHandleCloudbrainDurationStatistic() } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index caac24aad..468692663 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1487,13 +1487,15 @@ func GetCloudbrainResourceUsage(ctx *context.Context) { func GetCloudbrainResourceUsageDetail(ctx *context.Context) { queryType := ctx.QueryTrim("type") now := time.Now() - beginTimeStr := ctx.QueryTrim("beginTime") endTimeStr := ctx.QueryTrim("endTime") + aiCenterCode := ctx.QueryTrim("aiCenterCode") + var beginTime time.Time var endTime time.Time var endTimeTemp time.Time dayCloudbrainDuration := make([]models.DateCloudbrainStatistic, 0) + hourCloudbrainDuration := models.HourTimeStatistic{} var err error var count int if queryType != "" { @@ -1505,14 +1507,22 @@ func GetCloudbrainResourceUsageDetail(ctx *context.Context) { return } brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() + log.Info("recordCloudbrainDuration:", recordCloudbrainDuration) + log.Info("brainRecordBeginTime:", brainRecordBeginTime) beginTime = brainRecordBeginTime endTime = now - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime, aiCenterCode) if err != nil { log.Error("Can not query dayCloudbrainDuration.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) return } + hourCloudbrainDuration = getHourCloudbrainDuration(beginTime, endTime, aiCenterCode) + if err != nil { + log.Error("Can not query hourCloudbrainDuration.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } } else if queryType == "today" { beginTime = now.AddDate(0, 0, 0) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) @@ -1644,19 +1654,21 @@ func GetCloudbrainResourceUsageDetail(ctx *context.Context) { pagesize = 5 } pageDateCloudbrainDuration := getPageDateCloudbrainDuration(dayCloudbrainDuration, page, pagesize) + durationRateStatistic := getDurationStatistic(beginTime, endTime) ctx.JSON(http.StatusOK, map[string]interface{}{ "totalCount": count, "pageDateCloudbrainDuration": pageDateCloudbrainDuration, + "durationRateStatistic": durationRateStatistic, + "hourCloudbrainDuration": hourCloudbrainDuration, }) } -func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrainStatistics []*models.CloudbrainDurationStatistic) (map[string]int, map[string]int, map[string]int) { - +func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrainStatistics []*models.CloudbrainDurationStatistic) (map[string]int, map[string]int, map[string]float64) { aiCenterTotalDuration := make(map[string]int) aiCenterUsageDuration := make(map[string]int) - aiCenterUsageRate := make(map[string]int) + aiCenterUsageRate := make(map[string]float64) for _, cloudbrainStatistic := range cloudbrainStatistics { if int64(cloudbrainStatistic.CreatedUnix) >= beginTime.Unix() && int64(cloudbrainStatistic.CreatedUnix) < endTime.Unix() { if cloudbrainStatistic.TotalCanUse { @@ -1674,10 +1686,21 @@ func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrain } } } + ResourceAiCenterRes, err := models.GetResourceAiCenters() + if err != nil { + log.Error("Can not get ResourceAiCenterRes.", err) + return nil, nil, nil + } + for _, v := range ResourceAiCenterRes { + if _, ok := aiCenterUsageDuration[v.AiCenterCode]; !ok { + aiCenterUsageDuration[v.AiCenterCode] = 0 + } + } + for k, v := range aiCenterTotalDuration { for i, j := range aiCenterUsageDuration { if k == i { - aiCenterUsageRate[k] = j / v + aiCenterUsageRate[k] = float64(j) / float64(v) } } } @@ -1685,6 +1708,61 @@ func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrain return aiCenterUsageDuration, aiCenterTotalDuration, aiCenterUsageRate } +func getDurationStatistic(beginTime time.Time, endTime time.Time) models.DurationRateStatistic { + aiCenterTotalDurationStat := make(map[string]int) + aiCenterUsageDurationStat := make(map[string]int) + durationRateStatistic := models.DurationRateStatistic{} + cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ + BeginTime: beginTime, + EndTime: endTime, + }) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return durationRateStatistic + } + for _, cloudbrainStatistic := range cardDurationStatistics { + if cloudbrainStatistic.TotalCanUse { + if _, ok := aiCenterTotalDurationStat[cloudbrainStatistic.AiCenterCode]; !ok { + aiCenterTotalDurationStat[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration + } else { + aiCenterTotalDurationStat[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration + } + } else { + if _, ok := aiCenterUsageDurationStat[cloudbrainStatistic.AiCenterCode]; !ok { + aiCenterUsageDurationStat[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration + } else { + aiCenterUsageDurationStat[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration + } + } + } + ResourceAiCenterRes, err := models.GetResourceAiCenters() + if err != nil { + log.Error("Can not get ResourceAiCenterRes.", err) + return durationRateStatistic + } + for _, v := range ResourceAiCenterRes { + if _, ok := aiCenterUsageDurationStat[v.AiCenterCode]; !ok { + aiCenterUsageDurationStat[v.AiCenterCode] = 0 + } + } + totalCanUse := float64(0) + totalUse := float64(0) + for k, v := range aiCenterTotalDurationStat { + for i, j := range aiCenterUsageDurationStat { + if k == i { + totalUse += float64(j) + totalCanUse += float64(v) + } + } + } + totalUsageRate := totalUse / totalCanUse + + durationRateStatistic.AiCenterTotalDurationStat = aiCenterTotalDurationStat + durationRateStatistic.AiCenterUsageDurationStat = aiCenterUsageDurationStat + durationRateStatistic.TotalUsageRate = totalUsageRate + return durationRateStatistic +} + func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time) ([]models.DateCloudbrainStatistic, int, error) { now := time.Now() endTimeTemp := time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) @@ -1699,6 +1777,7 @@ func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time) ([]models. log.Error("GetCardDurationStatistics error:", err) return nil, 0, err } + dayCloudbrainInfo := make([]models.DateCloudbrainStatistic, 0) count := 0 for beginTime.Before(endTimeTemp) || beginTime.Equal(endTimeTemp) { @@ -1711,7 +1790,64 @@ func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time) ([]models. }) endTime = endTimeTemp endTimeTemp = endTimeTemp.AddDate(0, 0, -1) + if endTimeTemp.Before(beginTime) && beginTime.Before(endTime) { + endTimeTemp = beginTime + } count += 1 } return dayCloudbrainInfo, count, nil } + +func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) models.HourTimeStatistic { + hourTimeTotalDuration := make(map[int]int) + hourTimeUsageDuration := make(map[int]int) + hourTimeUsageRate := make(map[int]float64) + hourTimeStatistic := models.HourTimeStatistic{} + + cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ + BeginTime: beginTime, + EndTime: endTime, + }) + if err != nil { + log.Error("GetCardDurationStatistics error:", err) + return hourTimeStatistic + } + for _, cloudbrainStatistic := range cardDurationStatistics { + if cloudbrainStatistic.AiCenterCode == aiCenterCode { + if cloudbrainStatistic.TotalCanUse { + if _, ok := hourTimeTotalDuration[cloudbrainStatistic.HourTime]; !ok { + hourTimeTotalDuration[cloudbrainStatistic.HourTime] = cloudbrainStatistic.CardsTotalDuration + } else { + hourTimeTotalDuration[cloudbrainStatistic.HourTime] += cloudbrainStatistic.CardsTotalDuration + } + } else { + if _, ok := hourTimeUsageDuration[cloudbrainStatistic.HourTime]; !ok { + hourTimeUsageDuration[cloudbrainStatistic.HourTime] = cloudbrainStatistic.CardsTotalDuration + } else { + hourTimeUsageDuration[cloudbrainStatistic.HourTime] += cloudbrainStatistic.CardsTotalDuration + } + } + hourTimeList := []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23} + for _, v := range hourTimeList { + if _, ok := hourTimeUsageDuration[v]; !ok { + hourTimeUsageDuration[v] = 0 + } + if _, ok := hourTimeTotalDuration[v]; !ok { + hourTimeTotalDuration[v] = 0 + } + } + + for k, v := range hourTimeTotalDuration { + for i, j := range hourTimeUsageDuration { + if k == i { + hourTimeUsageRate[k] = float64(j) / float64(v) + } + } + } + } + } + hourTimeStatistic.HourTimeTotalDuration = hourTimeTotalDuration + hourTimeStatistic.HourTimeUsageDuration = hourTimeUsageDuration + hourTimeStatistic.HourTimeUsageRate = hourTimeUsageRate + return hourTimeStatistic +} From c0d643a2b41fa2fd8d858a40a27837cfd7faf80f Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 18 Oct 2022 18:05:31 +0800 Subject: [PATCH 07/82] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 468692663..781bc6280 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1511,7 +1511,7 @@ func GetCloudbrainResourceUsageDetail(ctx *context.Context) { log.Info("brainRecordBeginTime:", brainRecordBeginTime) beginTime = brainRecordBeginTime endTime = now - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime, aiCenterCode) + dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) if err != nil { log.Error("Can not query dayCloudbrainDuration.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) From f1ad30cd11d2b2e2380e748bff73e1625f1bef14 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 19 Oct 2022 16:13:36 +0800 Subject: [PATCH 08/82] update --- models/cloudbrain_static.go | 33 +- routers/api/v1/api.go | 5 +- routers/api/v1/repo/cloudbrain_dashboard.go | 368 ++++++++++---------- 3 files changed, 214 insertions(+), 192 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 3c5da2fd3..ea93015b3 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -38,6 +38,16 @@ type TaskDetail struct { WorkServerNum int64 `json:"WorkServerNum"` Spec *Specification `json:"Spec"` } +type CardTypeAndNum struct { + CardType string `json:"CardType"` + Num int `json:"Num"` + ComputeResource string `json:"computeResource"` +} +type ResourceOverview struct { + Cluster string `json:"cluster"` + AiCenterCode string `json:"aiCenterCode"` + CardTypeAndNum []CardTypeAndNum `json:"cardTypeAndNum"` +} type CloudbrainDurationStatistic struct { ID int64 `xorm:"pk autoincr"` @@ -69,17 +79,24 @@ type DurationRateStatistic struct { AiCenterUsageDurationStat map[string]int `json:"aiCenterUsageDurationStat"` TotalUsageRate float64 `json:"totalUsageRate"` } -type DateCloudbrainStatistic struct { - Date string `json:"date"` - AiCenterUsageDuration map[string]int `json:"aiCenterUsageDuration"` - AiCenterTotalDuration map[string]int `json:"aiCenterTotalDuration"` - AiCenterUsageRate map[string]float64 `json:"aiCenterUsageRate"` + +// type DateCloudbrainStatistic struct { +// Date string `json:"date"` +// AiCenterUsageDuration map[string]int `json:"aiCenterUsageDuration"` +// AiCenterTotalDuration map[string]int `json:"aiCenterTotalDuration"` +// AiCenterUsageRate map[string]float64 `json:"aiCenterUsageRate"` +// } +type DateUsageStatistic struct { + Date string `json:"date"` + UsageDuration int `json:"usageDuration"` + TotalDuration int `json:"totalDuration"` + UsageRate float64 `json:"usageRate"` } type HourTimeStatistic struct { - HourTimeUsageDuration map[int]int `json:"hourTimeUsageDuration"` - HourTimeTotalDuration map[int]int `json:"hourTimeTotalDuration"` - HourTimeUsageRate map[int]float64 `json:"hourTimeUsageRate"` + HourTimeUsageDuration map[string]int `json:"hourTimeUsageDuration"` + HourTimeTotalDuration map[string]int `json:"hourTimeTotalDuration"` + HourTimeUsageRate map[string]float64 `json:"hourTimeUsageRate"` } func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 06be12e92..2964e87ce 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -601,8 +601,9 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/running_top_data", repo.GetRunningTop) m.Get("/overview_resource", repo.GetCloudbrainResourceOverview) - m.Get("/resource_usage", repo.GetCloudbrainResourceUsage) - m.Get("/resource_usage_detail", repo.GetCloudbrainResourceUsageDetail) + m.Get("/resource_usage_statistic", repo.GetDurationRateStatistic) + m.Get("/resource_usage_rate", repo.GetCloudbrainResourceUsage) + m.Get("/resource_usage_rate_detail", repo.GetCloudbrainResourceUsageDetail) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 781bc6280..2e5e77078 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "net/url" + "strconv" "strings" "time" @@ -532,17 +533,17 @@ func getPageDateCloudbrainInfo(dateCloudbrainInfo []DateCloudbrainInfo, page int } -func getPageDateCloudbrainDuration(dateCloudbrainDuration []models.DateCloudbrainStatistic, page int, pagesize int) []models.DateCloudbrainStatistic { +func getPageDateCloudbrainDuration(dateUsageStatistic []models.DateUsageStatistic, page int, pagesize int) []models.DateUsageStatistic { begin := (page - 1) * pagesize end := (page) * pagesize - if begin > len(dateCloudbrainDuration)-1 { + if begin > len(dateUsageStatistic)-1 { return nil } - if end > len(dateCloudbrainDuration)-1 { - return dateCloudbrainDuration[begin:] + if end > len(dateUsageStatistic)-1 { + return dateUsageStatistic[begin:] } else { - return dateCloudbrainDuration[begin:end] + return dateUsageStatistic[begin:end] } } @@ -1425,9 +1426,53 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { log.Info("GetCanUseCardInfo err: %v", err) return } + // ResourceAiCenterRes, err := models.GetResourceAiCenters() + // if err != nil { + // log.Error("Can not get ResourceAiCenterRes.", err) + // return + // } + resourceOverviews := []models.ResourceOverview{} + resourceOpenIOne := models.ResourceOverview{} + resourceOpenITwo := models.ResourceOverview{} + // resourceChengdu := models.ResourceOverview{} + + for _, resourceQueue := range resourceQueues { + if resourceQueue.Cluster == models.OpenICluster { + if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainOne { + resourceOpenIOne.Cluster = models.OpenICluster + resourceOpenIOne.AiCenterCode = models.AICenterOfCloudBrainOne + cardTypeNum := models.CardTypeAndNum{} + cardTypeNum.CardType = resourceQueue.AccCardType + cardTypeNum.Num = resourceQueue.CardsTotalNum + cardTypeNum.ComputeResource = resourceQueue.ComputeResource + resourceOpenIOne.CardTypeAndNum = append(resourceOpenIOne.CardTypeAndNum, cardTypeNum) + } + if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainTwo { + resourceOpenITwo.Cluster = models.OpenICluster + resourceOpenITwo.AiCenterCode = models.AICenterOfCloudBrainTwo + cardTypeNum := models.CardTypeAndNum{} + cardTypeNum.CardType = resourceQueue.AccCardType + cardTypeNum.Num = resourceQueue.CardsTotalNum + cardTypeNum.ComputeResource = resourceQueue.ComputeResource + resourceOpenITwo.CardTypeAndNum = append(resourceOpenITwo.CardTypeAndNum, cardTypeNum) + } + // if resourceQueue.AiCenterCode == models.AICenterOfChengdu { + // resourceChengdu.Cluster = models.OpenICluster + // resourceChengdu.AiCenterCode = models.AICenterOfChengdu + // cardTypeNum := models.CardTypeAndNum{} + // cardTypeNum.CardType = resourceQueue.AccCardType + // cardTypeNum.Num = resourceQueue.CardsTotalNum + // cardTypeNum.ComputeResource = resourceQueue.ComputeResource + // resourceChengdu.CardTypeAndNum = append(resourceChengdu.CardTypeAndNum, cardTypeNum) + // } + } + } + resourceOverviews = append(resourceOverviews, resourceOpenIOne) + resourceOverviews = append(resourceOverviews, resourceOpenITwo) + // resourceOverviews = append(resourceOverviews, resourceChengdu) ctx.JSON(http.StatusOK, map[string]interface{}{ - "resourceQueues": resourceQueues, + "resourceOverviews": resourceOverviews, }) } @@ -1485,119 +1530,96 @@ func GetCloudbrainResourceUsage(ctx *context.Context) { } func GetCloudbrainResourceUsageDetail(ctx *context.Context) { + aiCenterCode := ctx.QueryTrim("aiCenterCode") + log.Info("aiCenterCode: %v", aiCenterCode) + if aiCenterCode == "" { + aiCenterCode = "OpenIOne" + } + beginTime, endTime := getBeginAndEndTime(ctx) + dayCloudbrainDuration, count, err := getDayCloudbrainDuration(beginTime, endTime, aiCenterCode) + if err != nil { + log.Error("Can not query dayCloudbrainDuration.", err) + return + } + hourCloudbrainDuration, err := getHourCloudbrainDuration(beginTime, endTime, aiCenterCode) + if err != nil { + log.Error("Can not query hourCloudbrainDuration.", err) + return + } + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pagesize := ctx.QueryInt("pagesize") + if pagesize <= 0 { + pagesize = 36500 + } + pageDateCloudbrainDuration := getPageDateCloudbrainDuration(dayCloudbrainDuration, page, pagesize) + ctx.JSON(http.StatusOK, map[string]interface{}{ + "totalCount": count, + "pageDateCloudbrainDuration": pageDateCloudbrainDuration, + "hourCloudbrainDuration": hourCloudbrainDuration, + }) +} + +func GetDurationRateStatistic(ctx *context.Context) { + beginTime, endTime := getBeginAndEndTime(ctx) + durationRateStatistic := getDurationStatistic(beginTime, endTime) + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "durationRateStatistic": durationRateStatistic, + }) + +} + +func getBeginAndEndTime(ctx *context.Context) (time.Time, time.Time) { queryType := ctx.QueryTrim("type") now := time.Now() beginTimeStr := ctx.QueryTrim("beginTime") endTimeStr := ctx.QueryTrim("endTime") - aiCenterCode := ctx.QueryTrim("aiCenterCode") var beginTime time.Time var endTime time.Time - var endTimeTemp time.Time - dayCloudbrainDuration := make([]models.DateCloudbrainStatistic, 0) - hourCloudbrainDuration := models.HourTimeStatistic{} var err error - var count int if queryType != "" { if queryType == "all" { recordCloudbrainDuration, err := models.GetDurationRecordBeginTime() if err != nil { log.Error("Can not get GetDurationRecordBeginTime", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) - return + return beginTime, endTime } brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() - log.Info("recordCloudbrainDuration:", recordCloudbrainDuration) - log.Info("brainRecordBeginTime:", brainRecordBeginTime) beginTime = brainRecordBeginTime endTime = now - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } - hourCloudbrainDuration = getHourCloudbrainDuration(beginTime, endTime, aiCenterCode) - if err != nil { - log.Error("Can not query hourCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } } else if queryType == "today" { beginTime = now.AddDate(0, 0, 0) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime = now - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } } else if queryType == "yesterday" { beginTime = now.AddDate(0, 0, -1) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } - } else if queryType == "last_7day" { beginTime = now.AddDate(0, 0, -6) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime = now - endTimeTemp = time.Date(endTimeTemp.Year(), endTimeTemp.Month(), endTimeTemp.Day(), 0, 0, 0, 0, now.Location()) - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } } else if queryType == "last_30day" { beginTime = now.AddDate(0, 0, -29) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime = now - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } } else if queryType == "current_month" { endTime = now beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } } else if queryType == "current_year" { endTime = now beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location()) - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } - } else if queryType == "last_month" { - lastMonthTime := now.AddDate(0, -1, 0) beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) endTime = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } - } } else { @@ -1607,105 +1629,81 @@ func GetCloudbrainResourceUsageDetail(ctx *context.Context) { if err != nil { log.Error("Can not get recordCloudbrain", err) ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) - return + return beginTime, endTime } brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() beginTime = brainRecordBeginTime endTime = now - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } } else { beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) if err != nil { log.Error("Can not ParseInLocation.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) - return + return beginTime, endTime } endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) if err != nil { log.Error("Can not ParseInLocation.", err) ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) - return + return beginTime, endTime } if endTime.After(time.Now()) { endTime = time.Now() } - endTimeTemp = beginTime.AddDate(0, 0, 1) - dayCloudbrainDuration, count, err = getDayCloudbrainDuration(beginTime, endTime) - if err != nil { - log.Error("Can not query dayCloudbrainDuration.", err) - ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) - return - } } } - - page := ctx.QueryInt("page") - if page <= 0 { - page = 1 - } - pagesize := ctx.QueryInt("pagesize") - if pagesize <= 0 { - pagesize = 5 - } - pageDateCloudbrainDuration := getPageDateCloudbrainDuration(dayCloudbrainDuration, page, pagesize) - durationRateStatistic := getDurationStatistic(beginTime, endTime) - - ctx.JSON(http.StatusOK, map[string]interface{}{ - "totalCount": count, - "pageDateCloudbrainDuration": pageDateCloudbrainDuration, - "durationRateStatistic": durationRateStatistic, - "hourCloudbrainDuration": hourCloudbrainDuration, - }) - + return beginTime, endTime } -func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrainStatistics []*models.CloudbrainDurationStatistic) (map[string]int, map[string]int, map[string]float64) { - aiCenterTotalDuration := make(map[string]int) - aiCenterUsageDuration := make(map[string]int) - aiCenterUsageRate := make(map[string]float64) +func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrainStatistics []*models.CloudbrainDurationStatistic) (int, int, float64) { + totalDuration := int(0) + usageDuration := int(0) + usageRate := float64(0) + for _, cloudbrainStatistic := range cloudbrainStatistics { if int64(cloudbrainStatistic.CreatedUnix) >= beginTime.Unix() && int64(cloudbrainStatistic.CreatedUnix) < endTime.Unix() { if cloudbrainStatistic.TotalCanUse { - if _, ok := aiCenterTotalDuration[cloudbrainStatistic.AiCenterCode]; !ok { - aiCenterTotalDuration[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration - } else { - aiCenterTotalDuration[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration - } + totalDuration += cloudbrainStatistic.CardsTotalDuration } else { - if _, ok := aiCenterUsageDuration[cloudbrainStatistic.AiCenterCode]; !ok { - aiCenterUsageDuration[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration - } else { - aiCenterUsageDuration[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration - } - } - } - } - ResourceAiCenterRes, err := models.GetResourceAiCenters() - if err != nil { - log.Error("Can not get ResourceAiCenterRes.", err) - return nil, nil, nil - } - for _, v := range ResourceAiCenterRes { - if _, ok := aiCenterUsageDuration[v.AiCenterCode]; !ok { - aiCenterUsageDuration[v.AiCenterCode] = 0 + usageDuration += cloudbrainStatistic.CardsTotalDuration + } + // if cloudbrainStatistic.TotalCanUse { + // if _, ok := aiCenterTotalDuration[Date]; !ok { + // aiCenterTotalDuration[Date] = cloudbrainStatistic.CardsTotalDuration + // } else { + // aiCenterTotalDuration[Date] += cloudbrainStatistic.CardsTotalDuration + // } + // } else { + // if _, ok := aiCenterUsageDuration[Date]; !ok { + // aiCenterUsageDuration[Date] = cloudbrainStatistic.CardsTotalDuration + // } else { + // aiCenterUsageDuration[Date] += cloudbrainStatistic.CardsTotalDuration + // } + // } } } - - for k, v := range aiCenterTotalDuration { - for i, j := range aiCenterUsageDuration { - if k == i { - aiCenterUsageRate[k] = float64(j) / float64(v) - } - } - } - - return aiCenterUsageDuration, aiCenterTotalDuration, aiCenterUsageRate + // ResourceAiCenterRes, err := models.GetResourceAiCenters() + // if err != nil { + // log.Error("Can not get ResourceAiCenterRes.", err) + // return nil, nil, nil + // } + // for _, v := range ResourceAiCenterRes { + // if _, ok := aiCenterUsageDuration[v.AiCenterCode]; !ok { + // aiCenterUsageDuration[v.AiCenterCode] = 0 + // } + // } + + // for k, v := range aiCenterTotalDuration { + // for i, j := range aiCenterUsageDuration { + // if k == i { + // aiCenterUsageRate[k] = float64(j) / float64(v) + // } + // } + // } + // usageRate = float64(usageDuration) / float64(totalDuration) + + return totalDuration, usageDuration, usageRate } func getDurationStatistic(beginTime time.Time, endTime time.Time) models.DurationRateStatistic { @@ -1763,30 +1761,31 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) models.Duratio return durationRateStatistic } -func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time) ([]models.DateCloudbrainStatistic, int, error) { +func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) ([]models.DateUsageStatistic, int, error) { now := time.Now() endTimeTemp := time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) if endTimeTemp.Equal(endTime) { endTimeTemp = endTimeTemp.AddDate(0, 0, -1) } cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ - BeginTime: beginTime, - EndTime: endTime, + BeginTime: beginTime, + EndTime: endTime, + AiCenterCode: aiCenterCode, }) if err != nil { log.Error("GetCardDurationStatistics error:", err) return nil, 0, err } - dayCloudbrainInfo := make([]models.DateCloudbrainStatistic, 0) + dayCloudbrainInfo := make([]models.DateUsageStatistic, 0) count := 0 for beginTime.Before(endTimeTemp) || beginTime.Equal(endTimeTemp) { - aiCenterUsageDuration, aiCenterTotalDuration, aiCenterUsageRate := getAiCenterUsageDuration(endTimeTemp, endTime, cardDurationStatistics) - dayCloudbrainInfo = append(dayCloudbrainInfo, models.DateCloudbrainStatistic{ - Date: endTimeTemp.Format("2006/01/02"), - AiCenterUsageDuration: aiCenterUsageDuration, - AiCenterTotalDuration: aiCenterTotalDuration, - AiCenterUsageRate: aiCenterUsageRate, + TotalDuration, UsageDuration, UsageRate := getAiCenterUsageDuration(endTimeTemp, endTime, cardDurationStatistics) + dayCloudbrainInfo = append(dayCloudbrainInfo, models.DateUsageStatistic{ + Date: endTimeTemp.Format("2006/01/02"), + UsageDuration: UsageDuration, + TotalDuration: TotalDuration, + UsageRate: UsageRate, }) endTime = endTimeTemp endTimeTemp = endTimeTemp.AddDate(0, 0, -1) @@ -1798,10 +1797,10 @@ func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time) ([]models. return dayCloudbrainInfo, count, nil } -func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) models.HourTimeStatistic { - hourTimeTotalDuration := make(map[int]int) - hourTimeUsageDuration := make(map[int]int) - hourTimeUsageRate := make(map[int]float64) +func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) (models.HourTimeStatistic, error) { + hourTimeTotalDuration := make(map[string]int) + hourTimeUsageDuration := make(map[string]int) + hourTimeUsageRate := make(map[string]float64) hourTimeStatistic := models.HourTimeStatistic{} cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ @@ -1810,44 +1809,49 @@ func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterC }) if err != nil { log.Error("GetCardDurationStatistics error:", err) - return hourTimeStatistic + return hourTimeStatistic, err } for _, cloudbrainStatistic := range cardDurationStatistics { if cloudbrainStatistic.AiCenterCode == aiCenterCode { if cloudbrainStatistic.TotalCanUse { - if _, ok := hourTimeTotalDuration[cloudbrainStatistic.HourTime]; !ok { - hourTimeTotalDuration[cloudbrainStatistic.HourTime] = cloudbrainStatistic.CardsTotalDuration + if _, ok := hourTimeTotalDuration[strconv.Itoa(cloudbrainStatistic.HourTime)]; !ok { + hourTimeTotalDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] = cloudbrainStatistic.CardsTotalDuration } else { - hourTimeTotalDuration[cloudbrainStatistic.HourTime] += cloudbrainStatistic.CardsTotalDuration + hourTimeTotalDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] += cloudbrainStatistic.CardsTotalDuration } } else { - if _, ok := hourTimeUsageDuration[cloudbrainStatistic.HourTime]; !ok { - hourTimeUsageDuration[cloudbrainStatistic.HourTime] = cloudbrainStatistic.CardsTotalDuration + if _, ok := hourTimeUsageDuration[strconv.Itoa(cloudbrainStatistic.HourTime)]; !ok { + hourTimeUsageDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] = cloudbrainStatistic.CardsTotalDuration } else { - hourTimeUsageDuration[cloudbrainStatistic.HourTime] += cloudbrainStatistic.CardsTotalDuration - } - } - hourTimeList := []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23} - for _, v := range hourTimeList { - if _, ok := hourTimeUsageDuration[v]; !ok { - hourTimeUsageDuration[v] = 0 - } - if _, ok := hourTimeTotalDuration[v]; !ok { - hourTimeTotalDuration[v] = 0 - } - } - - for k, v := range hourTimeTotalDuration { - for i, j := range hourTimeUsageDuration { - if k == i { - hourTimeUsageRate[k] = float64(j) / float64(v) - } + hourTimeUsageDuration[strconv.Itoa(cloudbrainStatistic.HourTime)] += cloudbrainStatistic.CardsTotalDuration } } } } + hourTimeList := []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23"} + for _, v := range hourTimeList { + if _, ok := hourTimeUsageDuration[v]; !ok { + hourTimeUsageDuration[v] = 0 + } + if _, ok := hourTimeTotalDuration[v]; !ok { + hourTimeTotalDuration[v] = 0 + } + // if _, ok := hourTimeUsageRate[v]; !ok { + // hourTimeUsageRate[v] = 0 + // } + + } + + // for k, v := range hourTimeTotalDuration { + // for i, j := range hourTimeUsageDuration { + // if k == i { + // hourTimeUsageRate[k] = float64(j) / float64(v) + // } + // } + // } + hourTimeStatistic.HourTimeTotalDuration = hourTimeTotalDuration hourTimeStatistic.HourTimeUsageDuration = hourTimeUsageDuration hourTimeStatistic.HourTimeUsageRate = hourTimeUsageRate - return hourTimeStatistic + return hourTimeStatistic, nil } From ebc00c2bfaa7806a6e0b761606db369543e66fc5 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 19 Oct 2022 17:12:41 +0800 Subject: [PATCH 09/82] update --- models/cloudbrain_static.go | 1 + routers/api/v1/repo/cloudbrain_dashboard.go | 31 +++++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index ea93015b3..a693e0555 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -45,6 +45,7 @@ type CardTypeAndNum struct { } type ResourceOverview struct { Cluster string `json:"cluster"` + AiCenterName string `json:"aiCenterName"` AiCenterCode string `json:"aiCenterCode"` CardTypeAndNum []CardTypeAndNum `json:"cardTypeAndNum"` } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 2e5e77078..517905c9b 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1426,11 +1426,11 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { log.Info("GetCanUseCardInfo err: %v", err) return } - // ResourceAiCenterRes, err := models.GetResourceAiCenters() - // if err != nil { - // log.Error("Can not get ResourceAiCenterRes.", err) - // return - // } + ResourceAiCenterRes, err := models.GetResourceAiCenters() + if err != nil { + log.Error("Can not get ResourceAiCenterRes.", err) + return + } resourceOverviews := []models.ResourceOverview{} resourceOpenIOne := models.ResourceOverview{} resourceOpenITwo := models.ResourceOverview{} @@ -1439,8 +1439,9 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { for _, resourceQueue := range resourceQueues { if resourceQueue.Cluster == models.OpenICluster { if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainOne { - resourceOpenIOne.Cluster = models.OpenICluster - resourceOpenIOne.AiCenterCode = models.AICenterOfCloudBrainOne + resourceOpenIOne.Cluster = resourceQueue.Cluster + resourceOpenIOne.AiCenterCode = resourceQueue.AiCenterCode + resourceOpenIOne.AiCenterName = resourceQueue.AiCenterName cardTypeNum := models.CardTypeAndNum{} cardTypeNum.CardType = resourceQueue.AccCardType cardTypeNum.Num = resourceQueue.CardsTotalNum @@ -1448,8 +1449,9 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { resourceOpenIOne.CardTypeAndNum = append(resourceOpenIOne.CardTypeAndNum, cardTypeNum) } if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainTwo { - resourceOpenITwo.Cluster = models.OpenICluster - resourceOpenITwo.AiCenterCode = models.AICenterOfCloudBrainTwo + resourceOpenITwo.Cluster = resourceQueue.Cluster + resourceOpenITwo.AiCenterCode = resourceQueue.AiCenterCode + resourceOpenITwo.AiCenterName = resourceQueue.AiCenterName cardTypeNum := models.CardTypeAndNum{} cardTypeNum.CardType = resourceQueue.AccCardType cardTypeNum.Num = resourceQueue.CardsTotalNum @@ -1470,6 +1472,17 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { resourceOverviews = append(resourceOverviews, resourceOpenIOne) resourceOverviews = append(resourceOverviews, resourceOpenITwo) // resourceOverviews = append(resourceOverviews, resourceChengdu) + for _, resourceAiCenterRes := range ResourceAiCenterRes { + if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainOne { + if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainTwo { + test := models.ResourceOverview{} + test.Cluster = models.C2NetCluster + test.AiCenterCode = resourceAiCenterRes.AiCenterCode + test.AiCenterName = resourceAiCenterRes.AiCenterName + resourceOverviews = append(resourceOverviews, test) + } + } + } ctx.JSON(http.StatusOK, map[string]interface{}{ "resourceOverviews": resourceOverviews, From 0a4875c4cb8e9727add429c5393d897a4b0c5d85 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 19 Oct 2022 17:58:35 +0800 Subject: [PATCH 10/82] update --- models/cloudbrain_static.go | 6 +- routers/api/v1/repo/cloudbrain_dashboard.go | 92 ++++++++++++++------- 2 files changed, 66 insertions(+), 32 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index a693e0555..84b659dd4 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -76,9 +76,9 @@ type DurationStatisticOptions struct { } type DurationRateStatistic struct { - AiCenterTotalDurationStat map[string]int `json:"aiCenterTotalDurationStat"` - AiCenterUsageDurationStat map[string]int `json:"aiCenterUsageDurationStat"` - TotalUsageRate float64 `json:"totalUsageRate"` + AiCenterTotalDurationStat map[string]int `json:"aiCenterTotalDurationStat"` + AiCenterUsageDurationStat map[string]int `json:"aiCenterUsageDurationStat"` + UsageRate map[string]float32 `json:"UsageRate"` } // type DateCloudbrainStatistic struct { diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 517905c9b..82bdb7be4 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1577,10 +1577,12 @@ func GetCloudbrainResourceUsageDetail(ctx *context.Context) { func GetDurationRateStatistic(ctx *context.Context) { beginTime, endTime := getBeginAndEndTime(ctx) - durationRateStatistic := getDurationStatistic(beginTime, endTime) + OpenIDurationRate, C2NetDurationRate, totalUsageRate := getDurationStatistic(beginTime, endTime) ctx.JSON(http.StatusOK, map[string]interface{}{ - "durationRateStatistic": durationRateStatistic, + "openIDurationRate": OpenIDurationRate, + "c2NetDurationRate": C2NetDurationRate, + "totalUsageRate": totalUsageRate, }) } @@ -1719,59 +1721,91 @@ func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrain return totalDuration, usageDuration, usageRate } -func getDurationStatistic(beginTime time.Time, endTime time.Time) models.DurationRateStatistic { - aiCenterTotalDurationStat := make(map[string]int) - aiCenterUsageDurationStat := make(map[string]int) - durationRateStatistic := models.DurationRateStatistic{} +func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.DurationRateStatistic, models.DurationRateStatistic, float32) { + OpenITotalDuration := make(map[string]int) + OpenIUsageDuration := make(map[string]int) + OpenIUsageRate := make(map[string]float32) + C2NetTotalDuration := make(map[string]int) + C2NetUsageDuration := make(map[string]int) + OpenIDurationRate := models.DurationRateStatistic{} + C2NetDurationRate := models.DurationRateStatistic{} cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ BeginTime: beginTime, EndTime: endTime, }) if err != nil { log.Error("GetCardDurationStatistics error:", err) - return durationRateStatistic + return OpenIDurationRate, C2NetDurationRate, 0 } for _, cloudbrainStatistic := range cardDurationStatistics { - if cloudbrainStatistic.TotalCanUse { - if _, ok := aiCenterTotalDurationStat[cloudbrainStatistic.AiCenterCode]; !ok { - aiCenterTotalDurationStat[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration + if cloudbrainStatistic.Cluster == models.OpenICluster { + if cloudbrainStatistic.TotalCanUse { + if _, ok := OpenITotalDuration[cloudbrainStatistic.AiCenterName]; !ok { + OpenITotalDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration + } else { + OpenITotalDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration + } } else { - aiCenterTotalDurationStat[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration + if _, ok := OpenIUsageDuration[cloudbrainStatistic.AiCenterName]; !ok { + OpenIUsageDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration + } else { + OpenIUsageDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration + } } - } else { - if _, ok := aiCenterUsageDurationStat[cloudbrainStatistic.AiCenterCode]; !ok { - aiCenterUsageDurationStat[cloudbrainStatistic.AiCenterCode] = cloudbrainStatistic.CardsTotalDuration + } + if cloudbrainStatistic.Cluster == models.C2NetCluster { + if cloudbrainStatistic.TotalCanUse { + if _, ok := C2NetTotalDuration[cloudbrainStatistic.AiCenterName]; !ok { + C2NetTotalDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration + } else { + C2NetTotalDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration + } } else { - aiCenterUsageDurationStat[cloudbrainStatistic.AiCenterCode] += cloudbrainStatistic.CardsTotalDuration + if _, ok := C2NetUsageDuration[cloudbrainStatistic.AiCenterName]; !ok { + C2NetUsageDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration + } else { + C2NetUsageDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration + } } } } ResourceAiCenterRes, err := models.GetResourceAiCenters() if err != nil { log.Error("Can not get ResourceAiCenterRes.", err) - return durationRateStatistic + return OpenIDurationRate, C2NetDurationRate, 0 } for _, v := range ResourceAiCenterRes { - if _, ok := aiCenterUsageDurationStat[v.AiCenterCode]; !ok { - aiCenterUsageDurationStat[v.AiCenterCode] = 0 + if v.AiCenterCode != models.AICenterOfCloudBrainOne && v.AiCenterCode != models.AICenterOfCloudBrainTwo { + // if v.AiCenterCode != models.AICenterOfCloudBrainTwo { + if _, ok := C2NetUsageDuration[v.AiCenterName]; !ok { + C2NetUsageDuration[v.AiCenterName] = 0 + } + // } + } else { + if _, ok := OpenIUsageDuration[v.AiCenterName]; !ok { + OpenIUsageDuration[v.AiCenterName] = 0 + } } } - totalCanUse := float64(0) - totalUse := float64(0) - for k, v := range aiCenterTotalDurationStat { - for i, j := range aiCenterUsageDurationStat { + // totalCanUse := float64(0) + // totalUse := float64(0) + totalUsageRate := float32(0) + for k, v := range OpenITotalDuration { + for i, j := range OpenIUsageDuration { if k == i { - totalUse += float64(j) - totalCanUse += float64(v) + OpenIUsageRate[k] = float32(j) / float32(v) } } } - totalUsageRate := totalUse / totalCanUse + // totalUsageRate := totalUse / totalCanUse - durationRateStatistic.AiCenterTotalDurationStat = aiCenterTotalDurationStat - durationRateStatistic.AiCenterUsageDurationStat = aiCenterUsageDurationStat - durationRateStatistic.TotalUsageRate = totalUsageRate - return durationRateStatistic + OpenIDurationRate.AiCenterTotalDurationStat = OpenITotalDuration + OpenIDurationRate.AiCenterUsageDurationStat = OpenIUsageDuration + OpenIDurationRate.UsageRate = OpenIUsageRate + C2NetDurationRate.AiCenterTotalDurationStat = C2NetTotalDuration + C2NetDurationRate.AiCenterUsageDurationStat = C2NetUsageDuration + // C2NetDurationRate.TotalUsageRate = totalUsageRate + return OpenIDurationRate, C2NetDurationRate, totalUsageRate } func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) ([]models.DateUsageStatistic, int, error) { From e25291a069936602bece9f051f5c7f9cf5ce583e Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 20 Oct 2022 09:56:26 +0800 Subject: [PATCH 11/82] update --- models/cloudbrain_static.go | 33 ++++- routers/api/v1/repo/cloudbrain_dashboard.go | 132 ++++++++++++-------- 2 files changed, 110 insertions(+), 55 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 84b659dd4..3f2dcf94a 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -80,6 +80,17 @@ type DurationRateStatistic struct { AiCenterUsageDurationStat map[string]int `json:"aiCenterUsageDurationStat"` UsageRate map[string]float32 `json:"UsageRate"` } +type ResourceDetail struct { + QueueCode string + Cluster string `xorm:"notnull"` + AiCenterCode string + AiCenterName string + ComputeResource string + AccCardType string + CardsTotalNum int + IsAutomaticSync bool + // CardTypeAndNum []CardTypeAndNum `json:"cardTypeAndNum"` +} // type DateCloudbrainStatistic struct { // Date string `json:"date"` @@ -336,16 +347,26 @@ func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode s func GetCanUseCardInfo() ([]*ResourceQueue, error) { sess := x.NewSession() defer sess.Close() - var cond = builder.NewCond() - cond = cond.And( - builder.And(builder.Eq{"resource_queue.is_automatic_sync": false}), - ) + // var cond = builder.NewCond() + // cond = cond.And( + // builder.And(builder.Eq{"resource_queue.is_automatic_sync": false}), + // ) + sess.OrderBy("resource_queue.id ASC") ResourceQueues := make([]*ResourceQueue, 0, 10) - if err := sess.Table(&ResourceQueue{}).Where(cond). - Find(&ResourceQueues); err != nil { + if err := sess.Table(&ResourceQueue{}).Find(&ResourceQueues); err != nil { log.Info("find error.") } return ResourceQueues, nil + // Cols("queue_code", "cluster", "ai_center_name", "ai_center_code", "compute_resource", "acc_card_type", "cards_total_num") + // sess := x.NewSession() + // defer sess.Close() + // sess.OrderBy("resource_queue.id ASC limit 1") + // cloudbrains := make([]*CloudbrainInfo, 0) + // if err := sess.Table(&Cloudbrain{}).Unscoped(). + // Find(&cloudbrains); err != nil { + // log.Info("find error.") + // } + // return cloudbrains, nil } func GetCardDurationStatistics(opts *DurationStatisticOptions) ([]*CloudbrainDurationStatistic, error) { diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 82bdb7be4..fe3e015ac 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1426,68 +1426,102 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { log.Info("GetCanUseCardInfo err: %v", err) return } - ResourceAiCenterRes, err := models.GetResourceAiCenters() - if err != nil { - log.Error("Can not get ResourceAiCenterRes.", err) - return - } - resourceOverviews := []models.ResourceOverview{} - resourceOpenIOne := models.ResourceOverview{} - resourceOpenITwo := models.ResourceOverview{} - // resourceChengdu := models.ResourceOverview{} - + log.Info("resourceQueues:", resourceQueues) + OpenIResourceDetail := []models.ResourceDetail{} + C2NetResourceDetail := []models.ResourceDetail{} for _, resourceQueue := range resourceQueues { if resourceQueue.Cluster == models.OpenICluster { - if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainOne { - resourceOpenIOne.Cluster = resourceQueue.Cluster - resourceOpenIOne.AiCenterCode = resourceQueue.AiCenterCode - resourceOpenIOne.AiCenterName = resourceQueue.AiCenterName - cardTypeNum := models.CardTypeAndNum{} - cardTypeNum.CardType = resourceQueue.AccCardType - cardTypeNum.Num = resourceQueue.CardsTotalNum - cardTypeNum.ComputeResource = resourceQueue.ComputeResource - resourceOpenIOne.CardTypeAndNum = append(resourceOpenIOne.CardTypeAndNum, cardTypeNum) - } - if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainTwo { - resourceOpenITwo.Cluster = resourceQueue.Cluster - resourceOpenITwo.AiCenterCode = resourceQueue.AiCenterCode - resourceOpenITwo.AiCenterName = resourceQueue.AiCenterName - cardTypeNum := models.CardTypeAndNum{} - cardTypeNum.CardType = resourceQueue.AccCardType - cardTypeNum.Num = resourceQueue.CardsTotalNum - cardTypeNum.ComputeResource = resourceQueue.ComputeResource - resourceOpenITwo.CardTypeAndNum = append(resourceOpenITwo.CardTypeAndNum, cardTypeNum) - } - // if resourceQueue.AiCenterCode == models.AICenterOfChengdu { - // resourceChengdu.Cluster = models.OpenICluster - // resourceChengdu.AiCenterCode = models.AICenterOfChengdu + // var resourceDetail models.ResourceDetail + // if _, ok := resourceDetail[resourceQueue.AiCenterCode]; !ok { + // resourceDetail.AiCenterCode = resourceQueue.AiCenterCode + // } else { // cardTypeNum := models.CardTypeAndNum{} // cardTypeNum.CardType = resourceQueue.AccCardType // cardTypeNum.Num = resourceQueue.CardsTotalNum // cardTypeNum.ComputeResource = resourceQueue.ComputeResource - // resourceChengdu.CardTypeAndNum = append(resourceChengdu.CardTypeAndNum, cardTypeNum) // } + + var resourceDetail models.ResourceDetail + resourceDetail.QueueCode = resourceQueue.QueueCode + resourceDetail.Cluster = resourceQueue.Cluster + resourceDetail.AiCenterCode = resourceQueue.AiCenterCode + resourceDetail.AiCenterName = resourceQueue.AiCenterName + resourceDetail.ComputeResource = resourceQueue.ComputeResource + resourceDetail.AccCardType = resourceQueue.AccCardType + resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum + resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync + OpenIResourceDetail = append(OpenIResourceDetail, resourceDetail) + // } else { + } - } - resourceOverviews = append(resourceOverviews, resourceOpenIOne) - resourceOverviews = append(resourceOverviews, resourceOpenITwo) - // resourceOverviews = append(resourceOverviews, resourceChengdu) - for _, resourceAiCenterRes := range ResourceAiCenterRes { - if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainOne { - if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainTwo { - test := models.ResourceOverview{} - test.Cluster = models.C2NetCluster - test.AiCenterCode = resourceAiCenterRes.AiCenterCode - test.AiCenterName = resourceAiCenterRes.AiCenterName - resourceOverviews = append(resourceOverviews, test) - } + if resourceQueue.Cluster == models.C2NetCluster { + var resourceDetail models.ResourceDetail + resourceDetail.QueueCode = resourceQueue.QueueCode + resourceDetail.Cluster = resourceQueue.Cluster + resourceDetail.AiCenterCode = resourceQueue.AiCenterCode + resourceDetail.AiCenterName = resourceQueue.AiCenterName + resourceDetail.ComputeResource = resourceQueue.ComputeResource + resourceDetail.AccCardType = resourceQueue.AccCardType + resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum + resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync + C2NetResourceDetail = append(C2NetResourceDetail, resourceDetail) } + + // ResourceAiCenterRes, err := models.GetResourceAiCenters() + // if err != nil { + // log.Error("Can not get ResourceAiCenterRes.", err) + // return + // } + // resourceOverviews := []models.ResourceOverview{} + // resourceOpenIOne := models.ResourceOverview{} + // resourceOpenITwo := models.ResourceOverview{} + // // resourceChengdu := models.ResourceOverview{} + + // for _, resourceQueue := range resourceQueues { + // if resourceQueue.Cluster == models.OpenICluster { + // if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainOne { + // resourceOpenIOne.Cluster = resourceQueue.Cluster + // resourceOpenIOne.AiCenterCode = resourceQueue.AiCenterCode + // resourceOpenIOne.AiCenterName = resourceQueue.AiCenterName + // cardTypeNum := models.CardTypeAndNum{} + // cardTypeNum.CardType = resourceQueue.AccCardType + // cardTypeNum.Num = resourceQueue.CardsTotalNum + // cardTypeNum.ComputeResource = resourceQueue.ComputeResource + // resourceOpenIOne.CardTypeAndNum = append(resourceOpenIOne.CardTypeAndNum, cardTypeNum) + // } + // if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainTwo { + // resourceOpenITwo.Cluster = resourceQueue.Cluster + // resourceOpenITwo.AiCenterCode = resourceQueue.AiCenterCode + // resourceOpenITwo.AiCenterName = resourceQueue.AiCenterName + // cardTypeNum := models.CardTypeAndNum{} + // cardTypeNum.CardType = resourceQueue.AccCardType + // cardTypeNum.Num = resourceQueue.CardsTotalNum + // cardTypeNum.ComputeResource = resourceQueue.ComputeResource + // resourceOpenITwo.CardTypeAndNum = append(resourceOpenITwo.CardTypeAndNum, cardTypeNum) + // } + // } + // } + // resourceOverviews = append(resourceOverviews, resourceOpenIOne) + // resourceOverviews = append(resourceOverviews, resourceOpenITwo) + // // resourceOverviews = append(resourceOverviews, resourceChengdu) + // for _, resourceAiCenterRes := range ResourceAiCenterRes { + // if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainOne { + // if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainTwo { + // test := models.ResourceOverview{} + // test.Cluster = models.C2NetCluster + // test.AiCenterCode = resourceAiCenterRes.AiCenterCode + // test.AiCenterName = resourceAiCenterRes.AiCenterName + // resourceOverviews = append(resourceOverviews, test) + // } + // } } ctx.JSON(http.StatusOK, map[string]interface{}{ - "resourceOverviews": resourceOverviews, + // "resourceOverviews": resourceOverviews, + "OpenIResourceDetail": OpenIResourceDetail, + "C2NetResourceDetail": C2NetResourceDetail, + // "resourceQueues": resourceQueues, }) - } func GetCloudbrainResourceUsage(ctx *context.Context) { From c1a63edd2cfd3516ca8a4e9d89472fac076a4a3f Mon Sep 17 00:00:00 2001 From: Gitea Date: Thu, 20 Oct 2022 10:37:13 +0800 Subject: [PATCH 12/82] add --- modules/setting/setting.go | 8 ++++++++ routers/repo/ai_model_convert.go | 18 +++++++++++++++++- templates/repo/modelmanage/convertIndex.tmpl | 6 ++++-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index c6afae05a..89a0fd2d6 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -694,8 +694,12 @@ var ( GPU_PYTORCH_IMAGE string GpuQueue string GPU_TENSORFLOW_IMAGE string + GPU_PADDLE_IMAGE string + GPU_MXNET_IMAGE string NPU_MINDSPORE_16_IMAGE string PytorchOnnxBootFile string + PaddleOnnxBootFile string + MXnetOnnxBootFile string PytorchTrTBootFile string MindsporeBootFile string TensorFlowNpuBootFile string @@ -1576,6 +1580,10 @@ func getModelConvertConfig() { ModelConvert.NPU_PoolID = sec.Key("NPU_PoolID").MustString("pool7908321a") ModelConvert.NPU_MINDSPORE_IMAGE_ID = sec.Key("NPU_MINDSPORE_IMAGE_ID").MustInt(121) ModelConvert.NPU_TENSORFLOW_IMAGE_ID = sec.Key("NPU_TENSORFLOW_IMAGE_ID").MustInt(35) + ModelConvert.GPU_PADDLE_IMAGE = sec.Key("GPU_PADDLE_IMAGE").MustString("dockerhub.pcl.ac.cn:5000/user-images/openi:paddle2.3.0_gpu_cuda11.2_cudnn8") + ModelConvert.GPU_MXNET_IMAGE = sec.Key("GPU_MXNET_IMAGE").MustString("dockerhub.pcl.ac.cn:5000/user-images/openi:mxnet191cu_cuda102_py37") + ModelConvert.PaddleOnnxBootFile = sec.Key("PaddleOnnxBootFile").MustString("convert_paddle.py") + ModelConvert.MXnetOnnxBootFile = sec.Key("MXnetOnnxBootFile").MustString("convert_mxnet.py") } func getModelAppConfig() { diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go index 9a5874956..bd6a01072 100644 --- a/routers/repo/ai_model_convert.go +++ b/routers/repo/ai_model_convert.go @@ -29,7 +29,9 @@ const ( tplModelConvertInfo = "repo/modelmanage/convertshowinfo" PYTORCH_ENGINE = 0 TENSORFLOW_ENGINE = 1 - MINDSPORE_ENGIN = 2 + MINDSPORE_ENGINE = 2 + PADDLE_ENGINE = 4 + MXNET_ENGINE = 6 ModelMountPath = "/model" CodeMountPath = "/code" DataSetMountPath = "/dataset" @@ -395,6 +397,20 @@ func createGpuTrainJob(modelConvert *models.AiModelConvert, ctx *context.Context deleteLocalDir(relatetiveModelPath) dataActualPath = setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + modelConvert.ID + "/dataset" } + } else if modelConvert.SrcEngine == PADDLE_ENGINE { + IMAGE_URL = setting.ModelConvert.GPU_PADDLE_IMAGE + if modelConvert.DestFormat == CONVERT_FORMAT_ONNX { + command = getGpuModelConvertCommand(modelConvert.ID, modelConvert.ModelPath, modelConvert, setting.ModelConvert.PaddleOnnxBootFile) + } else { + return errors.New("Not support the format.") + } + } else if modelConvert.SrcEngine == MXNET_ENGINE { + IMAGE_URL = setting.ModelConvert.GPU_MXNET_IMAGE + if modelConvert.DestFormat == CONVERT_FORMAT_ONNX { + command = getGpuModelConvertCommand(modelConvert.ID, modelConvert.ModelPath, modelConvert, setting.ModelConvert.MXnetOnnxBootFile) + } else { + return errors.New("Not support the format.") + } } log.Info("dataActualPath=" + dataActualPath) diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index 92eefca2e..26e79b04c 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -103,7 +103,7 @@
- {{if eq .SrcEngine 0}}PyTorch {{else if eq .SrcEngine 1}}TensorFlow{{else if eq .SrcEngine 2}}MindSpore {{end}} + {{if eq .SrcEngine 0}}PyTorch {{else if eq .SrcEngine 1}}TensorFlow {{else if eq .SrcEngine 2}}MindSpore {{else if eq .SrcEngine 4}}PaddlePaddle {{else if eq .SrcEngine 6}}MXNet {{end}}
{{if eq .DestFormat 0}}ONNX {{else if eq .DestFormat 1}}TensorRT {{end}} @@ -532,7 +532,7 @@ } } function isModel(filename){ - var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb"]; + var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel","pdparams",".params",".json"]; for(var i =0; iPyTorch"; html +=""; html +=""; + html +=""; + html +=""; $('#SrcEngine').html(html); srcEngineChanged(); } From 5f471bf4d8e531e8006b9e69ff9c1988d45d8a6d Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 20 Oct 2022 12:02:38 +0800 Subject: [PATCH 13/82] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 49 ++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index fe3e015ac..7acb82ade 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1431,6 +1431,10 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { C2NetResourceDetail := []models.ResourceDetail{} for _, resourceQueue := range resourceQueues { if resourceQueue.Cluster == models.OpenICluster { + // for _, openIResourceDetail := range OpenIResourceDetail { + // aiCenterCode := reflect.ValueOf(&openIResourceDetail.AiCenterCode).Elem() + // } + // var resourceDetail models.ResourceDetail // if _, ok := resourceDetail[resourceQueue.AiCenterCode]; !ok { // resourceDetail.AiCenterCode = resourceQueue.AiCenterCode @@ -1466,6 +1470,17 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync C2NetResourceDetail = append(C2NetResourceDetail, resourceDetail) } + // for _, resourceDetail := range OpenIResourceDetail { + // for _, resourceDetails := range OpenIResourceDetail { + // if resourceDetail.AiCenterCode == resourceDetails.AiCenterCode { + // cardTypeNum := models.CardTypeAndNum{} + // cardTypeNum.CardType = resourceDetails.AccCardType + // cardTypeNum.Num = resourceDetails.CardsTotalNum + // cardTypeNum.ComputeResource = resourceDetails.ComputeResource + // resourceDetail.CardTypeAndNum = append(resourceDetail.CardTypeAndNum, cardTypeNum) + // } + // } + // } // ResourceAiCenterRes, err := models.GetResourceAiCenters() // if err != nil { @@ -1515,12 +1530,44 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { // } // } } + // aiCenterNum := make(map[string]map[string]int) + // // cardTypeNum := make(map[string]int) + // for _, openIResourceDetail := range OpenIResourceDetail { + // if _, ok := aiCenterNum[openIResourceDetail.AiCenterCode]; !ok { + // aiCenterNum[openIResourceDetail.AiCenterCode][openIResourceDetail.AccCardType] = openIResourceDetail.CardsTotalNum + // } else { + // aiCenterNum[openIResourceDetail.AiCenterCode][openIResourceDetail.AccCardType] += openIResourceDetail.CardsTotalNum + // } + // } + AiCenterCodeList := make(map[string]string) + CardTypeList := make(map[string]string) + AiCenterCardTypeNum := make(map[string]map[string]int) + for _, openIResourceDetail := range OpenIResourceDetail { + if _, ok := AiCenterCodeList[openIResourceDetail.AiCenterCode]; !ok { + AiCenterCodeList[openIResourceDetail.AiCenterCode] = openIResourceDetail.AiCenterCode + } + for k, _ := range AiCenterCodeList { + if AiCenterCardTypeNum[AiCenterCodeList[k]] == nil { + AiCenterCardTypeNum[AiCenterCodeList[k]] = make(map[string]int) + } + if openIResourceDetail.AiCenterCode == AiCenterCodeList[k] { + if _, ok := CardTypeList[openIResourceDetail.AccCardType]; !ok { + CardTypeList[openIResourceDetail.AccCardType] = openIResourceDetail.AccCardType + } + for i, _ := range CardTypeList { + if openIResourceDetail.AccCardType == CardTypeList[i] { + AiCenterCardTypeNum[AiCenterCodeList[k]][CardTypeList[i]] += openIResourceDetail.CardsTotalNum + } + } + } + } + } ctx.JSON(http.StatusOK, map[string]interface{}{ // "resourceOverviews": resourceOverviews, "OpenIResourceDetail": OpenIResourceDetail, "C2NetResourceDetail": C2NetResourceDetail, - // "resourceQueues": resourceQueues, + "AiCenterCardTypeNum": AiCenterCardTypeNum, }) } From 66a28d19117775e3636d367847b813146922ad43 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 20 Oct 2022 16:25:14 +0800 Subject: [PATCH 14/82] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 137 ++++++-------------- 1 file changed, 40 insertions(+), 97 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 7acb82ade..ca414f805 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1431,27 +1431,13 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { C2NetResourceDetail := []models.ResourceDetail{} for _, resourceQueue := range resourceQueues { if resourceQueue.Cluster == models.OpenICluster { - // for _, openIResourceDetail := range OpenIResourceDetail { - // aiCenterCode := reflect.ValueOf(&openIResourceDetail.AiCenterCode).Elem() - // } - - // var resourceDetail models.ResourceDetail - // if _, ok := resourceDetail[resourceQueue.AiCenterCode]; !ok { - // resourceDetail.AiCenterCode = resourceQueue.AiCenterCode - // } else { - // cardTypeNum := models.CardTypeAndNum{} - // cardTypeNum.CardType = resourceQueue.AccCardType - // cardTypeNum.Num = resourceQueue.CardsTotalNum - // cardTypeNum.ComputeResource = resourceQueue.ComputeResource - // } - var resourceDetail models.ResourceDetail resourceDetail.QueueCode = resourceQueue.QueueCode resourceDetail.Cluster = resourceQueue.Cluster resourceDetail.AiCenterCode = resourceQueue.AiCenterCode resourceDetail.AiCenterName = resourceQueue.AiCenterName resourceDetail.ComputeResource = resourceQueue.ComputeResource - resourceDetail.AccCardType = resourceQueue.AccCardType + resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")" resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync OpenIResourceDetail = append(OpenIResourceDetail, resourceDetail) @@ -1465,98 +1451,54 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { resourceDetail.AiCenterCode = resourceQueue.AiCenterCode resourceDetail.AiCenterName = resourceQueue.AiCenterName resourceDetail.ComputeResource = resourceQueue.ComputeResource - resourceDetail.AccCardType = resourceQueue.AccCardType + resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")" resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync C2NetResourceDetail = append(C2NetResourceDetail, resourceDetail) } - // for _, resourceDetail := range OpenIResourceDetail { - // for _, resourceDetails := range OpenIResourceDetail { - // if resourceDetail.AiCenterCode == resourceDetails.AiCenterCode { - // cardTypeNum := models.CardTypeAndNum{} - // cardTypeNum.CardType = resourceDetails.AccCardType - // cardTypeNum.Num = resourceDetails.CardsTotalNum - // cardTypeNum.ComputeResource = resourceDetails.ComputeResource - // resourceDetail.CardTypeAndNum = append(resourceDetail.CardTypeAndNum, cardTypeNum) - // } - // } - // } - - // ResourceAiCenterRes, err := models.GetResourceAiCenters() - // if err != nil { - // log.Error("Can not get ResourceAiCenterRes.", err) - // return - // } - // resourceOverviews := []models.ResourceOverview{} - // resourceOpenIOne := models.ResourceOverview{} - // resourceOpenITwo := models.ResourceOverview{} - // // resourceChengdu := models.ResourceOverview{} - - // for _, resourceQueue := range resourceQueues { - // if resourceQueue.Cluster == models.OpenICluster { - // if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainOne { - // resourceOpenIOne.Cluster = resourceQueue.Cluster - // resourceOpenIOne.AiCenterCode = resourceQueue.AiCenterCode - // resourceOpenIOne.AiCenterName = resourceQueue.AiCenterName - // cardTypeNum := models.CardTypeAndNum{} - // cardTypeNum.CardType = resourceQueue.AccCardType - // cardTypeNum.Num = resourceQueue.CardsTotalNum - // cardTypeNum.ComputeResource = resourceQueue.ComputeResource - // resourceOpenIOne.CardTypeAndNum = append(resourceOpenIOne.CardTypeAndNum, cardTypeNum) - // } - // if resourceQueue.AiCenterCode == models.AICenterOfCloudBrainTwo { - // resourceOpenITwo.Cluster = resourceQueue.Cluster - // resourceOpenITwo.AiCenterCode = resourceQueue.AiCenterCode - // resourceOpenITwo.AiCenterName = resourceQueue.AiCenterName - // cardTypeNum := models.CardTypeAndNum{} - // cardTypeNum.CardType = resourceQueue.AccCardType - // cardTypeNum.Num = resourceQueue.CardsTotalNum - // cardTypeNum.ComputeResource = resourceQueue.ComputeResource - // resourceOpenITwo.CardTypeAndNum = append(resourceOpenITwo.CardTypeAndNum, cardTypeNum) - // } - // } - // } - // resourceOverviews = append(resourceOverviews, resourceOpenIOne) - // resourceOverviews = append(resourceOverviews, resourceOpenITwo) - // // resourceOverviews = append(resourceOverviews, resourceChengdu) - // for _, resourceAiCenterRes := range ResourceAiCenterRes { - // if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainOne { - // if resourceAiCenterRes.AiCenterCode != models.AICenterOfCloudBrainTwo { - // test := models.ResourceOverview{} - // test.Cluster = models.C2NetCluster - // test.AiCenterCode = resourceAiCenterRes.AiCenterCode - // test.AiCenterName = resourceAiCenterRes.AiCenterName - // resourceOverviews = append(resourceOverviews, test) - // } - // } - } - // aiCenterNum := make(map[string]map[string]int) - // // cardTypeNum := make(map[string]int) - // for _, openIResourceDetail := range OpenIResourceDetail { - // if _, ok := aiCenterNum[openIResourceDetail.AiCenterCode]; !ok { - // aiCenterNum[openIResourceDetail.AiCenterCode][openIResourceDetail.AccCardType] = openIResourceDetail.CardsTotalNum - // } else { - // aiCenterNum[openIResourceDetail.AiCenterCode][openIResourceDetail.AccCardType] += openIResourceDetail.CardsTotalNum - // } - // } - AiCenterCodeList := make(map[string]string) + } + AiCenterNameList := make(map[string]string) CardTypeList := make(map[string]string) - AiCenterCardTypeNum := make(map[string]map[string]int) + openIResourceNum := make(map[string]map[string]int) for _, openIResourceDetail := range OpenIResourceDetail { - if _, ok := AiCenterCodeList[openIResourceDetail.AiCenterCode]; !ok { - AiCenterCodeList[openIResourceDetail.AiCenterCode] = openIResourceDetail.AiCenterCode + if _, ok := AiCenterNameList[openIResourceDetail.AiCenterName]; !ok { + AiCenterNameList[openIResourceDetail.AiCenterName] = openIResourceDetail.AiCenterName } - for k, _ := range AiCenterCodeList { - if AiCenterCardTypeNum[AiCenterCodeList[k]] == nil { - AiCenterCardTypeNum[AiCenterCodeList[k]] = make(map[string]int) + for k, _ := range AiCenterNameList { + if openIResourceNum[AiCenterNameList[k]] == nil { + openIResourceNum[AiCenterNameList[k]] = make(map[string]int) } - if openIResourceDetail.AiCenterCode == AiCenterCodeList[k] { + if openIResourceDetail.AiCenterName == AiCenterNameList[k] { if _, ok := CardTypeList[openIResourceDetail.AccCardType]; !ok { CardTypeList[openIResourceDetail.AccCardType] = openIResourceDetail.AccCardType } for i, _ := range CardTypeList { if openIResourceDetail.AccCardType == CardTypeList[i] { - AiCenterCardTypeNum[AiCenterCodeList[k]][CardTypeList[i]] += openIResourceDetail.CardsTotalNum + openIResourceNum[AiCenterNameList[k]][CardTypeList[i]] += openIResourceDetail.CardsTotalNum + } + } + } + } + } + + c2NetAiCenterNameList := make(map[string]string) + c2NetCardTypeList := make(map[string]string) + c2NetResourceNum := make(map[string]map[string]int) + for _, c2NetResourceDetail := range C2NetResourceDetail { + if _, ok := c2NetAiCenterNameList[c2NetResourceDetail.AiCenterName]; !ok { + c2NetAiCenterNameList[c2NetResourceDetail.AiCenterName] = c2NetResourceDetail.AiCenterName + } + for k, _ := range c2NetAiCenterNameList { + if c2NetResourceNum[c2NetAiCenterNameList[k]] == nil { + c2NetResourceNum[c2NetAiCenterNameList[k]] = make(map[string]int) + } + if c2NetResourceDetail.AiCenterName == c2NetAiCenterNameList[k] { + if _, ok := c2NetCardTypeList[c2NetResourceDetail.AccCardType]; !ok { + c2NetCardTypeList[c2NetResourceDetail.AccCardType] = c2NetResourceDetail.AccCardType + } + for i, _ := range c2NetCardTypeList { + if c2NetResourceDetail.AccCardType == c2NetCardTypeList[i] { + c2NetResourceNum[c2NetAiCenterNameList[k]][c2NetCardTypeList[i]] += c2NetResourceDetail.CardsTotalNum } } } @@ -1565,9 +1507,10 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { ctx.JSON(http.StatusOK, map[string]interface{}{ // "resourceOverviews": resourceOverviews, - "OpenIResourceDetail": OpenIResourceDetail, - "C2NetResourceDetail": C2NetResourceDetail, - "AiCenterCardTypeNum": AiCenterCardTypeNum, + // "OpenIResourceDetail": OpenIResourceDetail, + // "C2NetResourceDetail": C2NetResourceDetail, + "openI": openIResourceNum, + "c2Net": c2NetResourceNum, }) } From d8f40329c650e53539bc1d1d925c0fe869af5aa3 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 20 Oct 2022 16:30:35 +0800 Subject: [PATCH 15/82] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index ca414f805..de6ae9eca 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1421,6 +1421,12 @@ func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (t } func GetCloudbrainResourceOverview(ctx *context.Context) { + recordCloudbrainDuration, err := models.GetDurationRecordBeginTime() + if err != nil { + log.Error("Can not get GetDurationRecordBeginTime", err) + return + } + brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() resourceQueues, err := models.GetCanUseCardInfo() if err != nil { log.Info("GetCanUseCardInfo err: %v", err) @@ -1509,8 +1515,9 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { // "resourceOverviews": resourceOverviews, // "OpenIResourceDetail": OpenIResourceDetail, // "C2NetResourceDetail": C2NetResourceDetail, - "openI": openIResourceNum, - "c2Net": c2NetResourceNum, + "openI": openIResourceNum, + "c2Net": c2NetResourceNum, + "brainRecordBeginTime": brainRecordBeginTime, }) } From 4eeead973bbd379d6c8a751ca5402501a3e1b7fe Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 20 Oct 2022 16:50:19 +0800 Subject: [PATCH 16/82] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index de6ae9eca..3ce46c60a 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1426,7 +1426,8 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { log.Error("Can not get GetDurationRecordBeginTime", err) return } - brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() + recordBeginTime := recordCloudbrainDuration[0].CreatedUnix + recordUpdateTime := time.Now().Unix() resourceQueues, err := models.GetCanUseCardInfo() if err != nil { log.Info("GetCanUseCardInfo err: %v", err) @@ -1512,12 +1513,10 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { } ctx.JSON(http.StatusOK, map[string]interface{}{ - // "resourceOverviews": resourceOverviews, - // "OpenIResourceDetail": OpenIResourceDetail, - // "C2NetResourceDetail": C2NetResourceDetail, - "openI": openIResourceNum, - "c2Net": c2NetResourceNum, - "brainRecordBeginTime": brainRecordBeginTime, + "openI": openIResourceNum, + "c2Net": c2NetResourceNum, + "recordUpdateTime": recordUpdateTime, + "recordBeginTime": recordBeginTime, }) } From 4206670658a168bfdc2a32e49f5494af9a3de1ca Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 21 Oct 2022 10:04:12 +0800 Subject: [PATCH 17/82] update --- routers/api/v1/api.go | 1 - routers/api/v1/repo/cloudbrain_dashboard.go | 131 ++++++-------------- 2 files changed, 35 insertions(+), 97 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 2964e87ce..b719db71e 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -602,7 +602,6 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/overview_resource", repo.GetCloudbrainResourceOverview) m.Get("/resource_usage_statistic", repo.GetDurationRateStatistic) - m.Get("/resource_usage_rate", repo.GetCloudbrainResourceUsage) m.Get("/resource_usage_rate_detail", repo.GetCloudbrainResourceUsageDetail) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 3ce46c60a..6824858fb 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1520,63 +1520,10 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { }) } -func GetCloudbrainResourceUsage(ctx *context.Context) { - recordBeginTime := time.Now().AddDate(0, 0, -6) - beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) - if err != nil { - log.Error("getCloudbrainTimePeroid error:", err) - return - } - cardUsageRes := make(map[string]int) - cardCanUsageRes := make(map[string]int) - cardUtilizationRate := make(map[string]int) - // cardDurationStatistics, err := models.GetCardDurationStatistics(beginTime, endTime) - cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{ - BeginTime: beginTime, - EndTime: endTime, - }) - if err != nil { - log.Error("GetCardDurationStatistics error:", err) - return - } - - for _, cloudbrainStat := range cardDurationStatistics { - if cloudbrainStat.TotalCanUse { - if _, ok := cardCanUsageRes[cloudbrainStat.AiCenterCode]; !ok { - cardCanUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration - } else { - cardCanUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration - } - } else { - if _, ok := cardUsageRes[cloudbrainStat.AiCenterCode]; !ok { - cardUsageRes[cloudbrainStat.AiCenterCode] = cloudbrainStat.CardsTotalDuration - } else { - cardUsageRes[cloudbrainStat.AiCenterCode] += cloudbrainStat.CardsTotalDuration - } - } - } - for k, v := range cardCanUsageRes { - for j, i := range cardUsageRes { - if k == j { - cardUtilizationRate[k] = i / v - } - } - } - - ctx.JSON(http.StatusOK, map[string]interface{}{ - "cardDurationStatistics": cardDurationStatistics, - "cardUsageRes": cardUsageRes, - "cardCanUsageRes": cardCanUsageRes, - "cardUtilizationRate": cardUtilizationRate, - }) - -} - func GetCloudbrainResourceUsageDetail(ctx *context.Context) { aiCenterCode := ctx.QueryTrim("aiCenterCode") - log.Info("aiCenterCode: %v", aiCenterCode) if aiCenterCode == "" { - aiCenterCode = "OpenIOne" + aiCenterCode = models.AICenterOfCloudBrainOne } beginTime, endTime := getBeginAndEndTime(ctx) dayCloudbrainDuration, count, err := getDayCloudbrainDuration(beginTime, endTime, aiCenterCode) @@ -1713,40 +1660,16 @@ func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrain } else { usageDuration += cloudbrainStatistic.CardsTotalDuration } - // if cloudbrainStatistic.TotalCanUse { - // if _, ok := aiCenterTotalDuration[Date]; !ok { - // aiCenterTotalDuration[Date] = cloudbrainStatistic.CardsTotalDuration - // } else { - // aiCenterTotalDuration[Date] += cloudbrainStatistic.CardsTotalDuration - // } - // } else { - // if _, ok := aiCenterUsageDuration[Date]; !ok { - // aiCenterUsageDuration[Date] = cloudbrainStatistic.CardsTotalDuration - // } else { - // aiCenterUsageDuration[Date] += cloudbrainStatistic.CardsTotalDuration - // } - // } } } - // ResourceAiCenterRes, err := models.GetResourceAiCenters() - // if err != nil { - // log.Error("Can not get ResourceAiCenterRes.", err) - // return nil, nil, nil - // } - // for _, v := range ResourceAiCenterRes { - // if _, ok := aiCenterUsageDuration[v.AiCenterCode]; !ok { - // aiCenterUsageDuration[v.AiCenterCode] = 0 - // } - // } - - // for k, v := range aiCenterTotalDuration { - // for i, j := range aiCenterUsageDuration { - // if k == i { - // aiCenterUsageRate[k] = float64(j) / float64(v) - // } - // } - // } - // usageRate = float64(usageDuration) / float64(totalDuration) + if totalDuration == 0 || usageDuration == 0 { + usageRate = 0 + } else { + usageRate = float64(usageDuration) / float64(totalDuration) + } + // if + // usageRate, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", float32(usageDuration)/float32(totalDuration)), 64) + // totalUsageRate = totalUse / totalCanUse return totalDuration, usageDuration, usageRate } @@ -1817,8 +1740,8 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati } } } - // totalCanUse := float64(0) - // totalUse := float64(0) + totalCanUse := float32(0) + totalUse := float32(0) totalUsageRate := float32(0) for k, v := range OpenITotalDuration { for i, j := range OpenIUsageDuration { @@ -1827,7 +1750,19 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati } } } - // totalUsageRate := totalUse / totalCanUse + for _, v := range OpenITotalDuration { + totalCanUse += float32(v) + } + for _, v := range OpenIUsageRate { + totalUse += float32(v) + } + if totalCanUse == 0 || totalUse == 0 { + totalUsageRate = 0 + } else { + totalUsageRate = totalUse / totalCanUse + } + // totalUsageRate = totalUse / totalCanUse + // strconv.FormatFloat(*100, 'f', 4, 64) + "%" OpenIDurationRate.AiCenterTotalDurationStat = OpenITotalDuration OpenIDurationRate.AiCenterUsageDurationStat = OpenIUsageDuration @@ -1919,13 +1854,17 @@ func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterC } - // for k, v := range hourTimeTotalDuration { - // for i, j := range hourTimeUsageDuration { - // if k == i { - // hourTimeUsageRate[k] = float64(j) / float64(v) - // } - // } - // } + for k, v := range hourTimeTotalDuration { + for i, j := range hourTimeUsageDuration { + if k == i { + if v == 0 || j == 0 { + hourTimeUsageRate[k] = 0 + } else { + hourTimeUsageRate[k] = float64(j) / float64(v) + } + } + } + } hourTimeStatistic.HourTimeTotalDuration = hourTimeTotalDuration hourTimeStatistic.HourTimeUsageDuration = hourTimeUsageDuration From 6e65b7144d1e8dd36d1589244e84fd170dd2ff10 Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 21 Oct 2022 10:53:53 +0800 Subject: [PATCH 18/82] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 23 ++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 6824858fb..691fe5002 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1728,16 +1728,18 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati return OpenIDurationRate, C2NetDurationRate, 0 } for _, v := range ResourceAiCenterRes { - if v.AiCenterCode != models.AICenterOfCloudBrainOne && v.AiCenterCode != models.AICenterOfCloudBrainTwo { - // if v.AiCenterCode != models.AICenterOfCloudBrainTwo { - if _, ok := C2NetUsageDuration[v.AiCenterName]; !ok { - C2NetUsageDuration[v.AiCenterName] = 0 - } - // } - } else { + // if v.AiCenterCode != models.AICenterOfCloudBrainOne && v.AiCenterCode != models.AICenterOfCloudBrainTwo { + if cutString(v.AiCenterCode, 4) == cutString(models.AICenterOfCloudBrainOne, 4) { if _, ok := OpenIUsageDuration[v.AiCenterName]; !ok { OpenIUsageDuration[v.AiCenterName] = 0 } + if _, ok := OpenITotalDuration[v.AiCenterName]; !ok { + OpenITotalDuration[v.AiCenterName] = 0 + } + } else { + if _, ok := C2NetUsageDuration[v.AiCenterName]; !ok { + C2NetUsageDuration[v.AiCenterName] = 0 + } } } totalCanUse := float32(0) @@ -1773,6 +1775,13 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati return OpenIDurationRate, C2NetDurationRate, totalUsageRate } +func cutString(str string, lens int) string { + if len(str) < lens { + return str + } + return str[:lens] +} + func getDayCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterCode string) ([]models.DateUsageStatistic, int, error) { now := time.Now() endTimeTemp := time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) From a529e612948c834e4fcc50a8750f232e973afdac Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Fri, 21 Oct 2022 15:51:34 +0800 Subject: [PATCH 19/82] =?UTF-8?q?=E8=B5=84=E6=BA=90=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/home.go | 5 + routers/routes/routes.go | 1 + templates/base/footer_content.tmpl | 3 +- templates/resource_desc.tmpl | 235 +++++++++++++++++++++++++++++ 4 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 templates/resource_desc.tmpl diff --git a/routers/home.go b/routers/home.go index ac607b5be..aab760611 100755 --- a/routers/home.go +++ b/routers/home.go @@ -41,6 +41,7 @@ const ( tplExploreExploreDataAnalysis base.TplName = "explore/data_analysis" tplHomeTerm base.TplName = "terms" tplHomePrivacy base.TplName = "privacy" + tplResoruceDesc base.TplName = "resource_desc" ) // Home render home page @@ -820,3 +821,7 @@ func HomeTerm(ctx *context.Context) { func HomePrivacy(ctx *context.Context) { ctx.HTML(200, tplHomePrivacy) } + +func HomeResoruceDesc(ctx *context.Context) { + ctx.HTML(200, tplResoruceDesc) +} \ No newline at end of file diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 9a523ea48..fd8bae7d7 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -338,6 +338,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/action/notification", routers.ActionNotification) m.Get("/recommend/home", routers.RecommendHomeInfo) m.Get("/dashboard/invitation", routers.GetMapInfo) + m.Get("/resource_desc", routers.HomeResoruceDesc) //m.Get("/recommend/org", routers.RecommendOrgFromPromote) //m.Get("/recommend/repo", routers.RecommendRepoFromPromote) m.Get("/recommend/userrank/:index", routers.GetUserRankFromPromote) diff --git a/templates/base/footer_content.tmpl b/templates/base/footer_content.tmpl index cdc2cf549..ce4ea5cc6 100755 --- a/templates/base/footer_content.tmpl +++ b/templates/base/footer_content.tmpl @@ -36,7 +36,8 @@ {{else}} {{.i18n.Tr "custom.foot.advice_feedback"}} {{end}} - + {{.i18n.Tr "custom.Platform_Tutorial"}} + {{template "custom/extra_links_footer" .}}
diff --git a/templates/resource_desc.tmpl b/templates/resource_desc.tmpl new file mode 100644 index 000000000..e809541ad --- /dev/null +++ b/templates/resource_desc.tmpl @@ -0,0 +1,235 @@ +{{template "base/head_home" .}} + +
+

平台资源说明

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
集群计算资源任务类型卡类型可用镜像网络类型数据集处理方式容器目录说明示例代码仓备注
启智集群GPU调试任务T4 +
    +
  • + 外部公开镜像，如：dockerhub镜像； +
    • +
  • 平台镜像；
    • +
+ 		能连外网平台可解压数据集 + 数据集存放路径/dataset，模型存放路径/model，代码存放路径/code +
训练任务V100 +
    +
  • 平台镜像；
    • +
+ 		不能连外网 + 训练脚本存储在/code中，数据集存储在/dataset中，预训练模型存放在环境变量ckpt_url中，训练输出请存储在/model中 + 以供后续下载。 + + https://git.openi.org.cn/OpenIO + SSG/MNIST_PytorchExample_GPU + + 启智集群V100不能连外网，只能使用平台的镜像，不可使用外部公开镜像，，否则任务会一直处于waiting + 状态 +
A100 +
    +
  • + 外部公开镜像，如：dockerhub镜像； +
    • +
  • 平台镜像；
    • +
+ 		能连外网
推理任务V100 +
    +
  • 平台镜像；
    • +
+ 		不能连外网 + 数据集存储在/dataset中，模型文件存储在/model中，推理输出请存储在/result中 + 以供后续下载。 + + https://git.openi.org.cn/OpenIO + SSG/MNIST_PytorchExample_GPU/src/branch/master/inference.py +
评测任务V100 +
    +
  • 平台镜像；
    • +
+ 		不能连外网 + 模型评测时，先使用数据集功能上传模型，然后从数据集列表选模型。 +
NPU调试任务Ascend 910 +
    +
  • 平台镜像；
    • +
+ 		能连外网
训练任务Ascend 910 + 数据集位置存储在环境变量data_url中，预训练模型存放在环境变量ckpt_url中，训练输出路径存储在环境变量train_url中。 + + https://git.openi.org.cn/OpenIOSSG/MNIST_Example +
推理任务Ascend 910 + 数据集位置存储在环境变量data_url中，推理输出路径存储在环境变量result_url中。 + + https://git.openi.org.cn/OpenIOSSG/MNIST_Example +
智算网络GPU训练任务V100 +
    +
  • + 外部公开镜像，如：dockerhub镜像； +
    • +
  • 平台镜像；
    • +
+ 		能连外网用户自行解压数据 集 + 训练脚本存储在/tmp/code中，数据集存储在/tmp/dataset中，预训练模型存放在环境变量ckpt_url中，训练输出请存储在/tmp/output中以供后续下载。 + + https://git.openi.org.cn/OpenIO + SSG/MNIST_PytorchExample_GPU/src/branch/master/train_for_c + A100 2net.py +
A100 +
    +
  • + 外部公开镜像，如：dockerhub镜像； +
    • +
  • 平台镜像；
    • +
+ 		能连外网
NPU训练任务Ascend 910 +
    +
  • 平台镜像；
    • +
+ 		能连外网 + 训练脚本存储在/cache/code中，预训练模型存放在环境变量ckpt_url中，训练输出请存储在/cache/output中以供后续下载。 + + https://git.openi.org.cn/OpenIO + SSG/MNIST_Example/src/branch/master/train_for_c2net.py +
+
+ +{{template "base/footer" .}} From 482b4bffea0cf233ddca1823c68933918c336bce Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 21 Oct 2022 16:01:32 +0800 Subject: [PATCH 20/82] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 137 +------------------- routers/repo/aisafety.go | 6 + routers/repo/cloudbrain.go | 136 +++++++++---------- routers/repo/grampus.go | 11 +- routers/repo/modelarts.go | 70 +++------- services/cloudbrain/cloudbrainTask/count.go | 86 ++++++++++++ 6 files changed, 196 insertions(+), 250 deletions(-) create mode 100644 services/cloudbrain/cloudbrainTask/count.go diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 6135dac40..f0e27995e 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2015,11 +2015,6 @@ func GetModelSafetyTestTask() ([]*Cloudbrain, error) { return cloudbrains, err } -func GetCloudbrainCountByUserID(userID int64, jobType string) (int, error) { - count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ? and type = ?", jobType, userID, TypeCloudBrainOne).Count(new(Cloudbrain)) - return int(count), err -} - func GetCloudbrainRunCountByRepoID(repoID int64) (int, error) { count, err := x.In("status", JobWaiting, JobRunning, ModelArtsCreateQueue, ModelArtsCreating, ModelArtsStarting, ModelArtsReadyToStart, ModelArtsResizing, ModelArtsStartQueuing, ModelArtsRunning, ModelArtsDeleting, ModelArtsRestarting, ModelArtsTrainJobInit, @@ -2028,11 +2023,6 @@ func GetCloudbrainRunCountByRepoID(repoID int64) (int, error) { return int(count), err } -func GetBenchmarkCountByUserID(userID int64) (int, error) { - count, err := x.In("status", JobWaiting, JobRunning).And("(job_type = ? or job_type = ? or job_type = ?) and user_id = ? and type = ?", string(JobTypeBenchmark), string(JobTypeModelSafety), string(JobTypeBrainScore), string(JobTypeSnn4imagenet), userID, TypeCloudBrainOne).Count(new(Cloudbrain)) - return int(count), err -} - func GetModelSafetyCountByUserID(userID int64) (int, error) { count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ?", string(JobTypeModelSafety), userID).Count(new(Cloudbrain)) return int(count), err @@ -2048,40 +2038,14 @@ func GetWaitingCloudbrainCount(cloudbrainType int, computeResource string, jobTy } return sess.Count(new(Cloudbrain)) } - -func GetCloudbrainNotebookCountByUserID(userID int64) (int, error) { - count, err := x.In("status", ModelArtsCreateQueue, ModelArtsCreating, ModelArtsStarting, ModelArtsReadyToStart, ModelArtsResizing, ModelArtsStartQueuing, ModelArtsRunning, ModelArtsRestarting). - And("job_type = ? and user_id = ? and type in (?,?)", JobTypeDebug, userID, TypeCloudBrainTwo, TypeCDCenter).Count(new(Cloudbrain)) - return int(count), err -} - -func GetCloudbrainTrainJobCountByUserID(userID int64) (int, error) { - count, err := x.In("status", ModelArtsTrainJobInit, ModelArtsTrainJobImageCreating, ModelArtsTrainJobSubmitTrying, ModelArtsTrainJobWaiting, ModelArtsTrainJobRunning, ModelArtsTrainJobScaling, ModelArtsTrainJobCheckInit, ModelArtsTrainJobCheckRunning, ModelArtsTrainJobCheckRunningCompleted). - And("job_type = ? and user_id = ? and type = ?", JobTypeTrain, userID, TypeCloudBrainTwo).Count(new(Cloudbrain)) - return int(count), err -} - -func GetCloudbrainInferenceJobCountByUserID(userID int64) (int, error) { - count, err := x.In("status", ModelArtsTrainJobInit, ModelArtsTrainJobImageCreating, ModelArtsTrainJobSubmitTrying, ModelArtsTrainJobWaiting, ModelArtsTrainJobRunning, ModelArtsTrainJobScaling, ModelArtsTrainJobCheckInit, ModelArtsTrainJobCheckRunning, ModelArtsTrainJobCheckRunningCompleted). - And("job_type = ? and user_id = ? and type = ?", JobTypeInference, userID, TypeCloudBrainTwo).Count(new(Cloudbrain)) - return int(count), err -} - -func GetGrampusCountByUserID(userID int64, jobType, computeResource string) (int, error) { - count, err := x.In("status", GrampusStatusWaiting, GrampusStatusRunning).And("job_type = ? and user_id = ? and type = ?", jobType, userID, TypeC2Net).And("compute_resource = ?", computeResource).Count(new(Cloudbrain)) +func GetNotFinalStatusTaskCount(userID int64, notFinalStatus []string, jobTypes []JobType, cloudbrainTypes []int, computeResource string) (int, error) { + count, err := x.In("status", notFinalStatus). + In("job_type", jobTypes). + In("type", cloudbrainTypes). + And("user_id = ? and compute_resource = ?", userID, computeResource).Count(new(Cloudbrain)) return int(count), err } -func UpdateInferenceJob(job *Cloudbrain) error { - return updateInferenceJob(x, job) -} - -func updateInferenceJob(e Engine, job *Cloudbrain) error { - var sess *xorm.Session - sess = e.Where("job_id = ?", job.JobID) - _, err := sess.Cols("status", "train_job_duration", "duration", "start_time", "end_time", "created_unix").Update(job) - return err -} func RestartCloudbrain(old *Cloudbrain, new *Cloudbrain) (err error) { sess := x.NewSession() defer sess.Close() @@ -2411,97 +2375,6 @@ var ( CloudbrainSpecialGpuInfosMap map[string]*GpuInfo ) -func InitCloudbrainOneResourceSpecMap() { - if CloudbrainDebugResourceSpecsMap == nil || len(CloudbrainDebugResourceSpecsMap) == 0 { - t := ResourceSpecs{} - json.Unmarshal([]byte(setting.ResourceSpecs), &t) - CloudbrainDebugResourceSpecsMap = make(map[int]*ResourceSpec, len(t.ResourceSpec)) - for _, spec := range t.ResourceSpec { - CloudbrainDebugResourceSpecsMap[spec.Id] = spec - } - } - if CloudbrainTrainResourceSpecsMap == nil || len(CloudbrainTrainResourceSpecsMap) == 0 { - t := ResourceSpecs{} - json.Unmarshal([]byte(setting.TrainResourceSpecs), &t) - CloudbrainTrainResourceSpecsMap = make(map[int]*ResourceSpec, len(t.ResourceSpec)) - for _, spec := range t.ResourceSpec { - CloudbrainTrainResourceSpecsMap[spec.Id] = spec - } - } - if CloudbrainInferenceResourceSpecsMap == nil || len(CloudbrainInferenceResourceSpecsMap) == 0 { - t := ResourceSpecs{} - json.Unmarshal([]byte(setting.InferenceResourceSpecs), &t) - CloudbrainInferenceResourceSpecsMap = make(map[int]*ResourceSpec, len(t.ResourceSpec)) - for _, spec := range t.ResourceSpec { - CloudbrainInferenceResourceSpecsMap[spec.Id] = spec - } - } - if CloudbrainBenchmarkResourceSpecsMap == nil || len(CloudbrainBenchmarkResourceSpecsMap) == 0 { - t := ResourceSpecs{} - json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &t) - CloudbrainBenchmarkResourceSpecsMap = make(map[int]*ResourceSpec, len(t.ResourceSpec)) - for _, spec := range t.ResourceSpec { - CloudbrainBenchmarkResourceSpecsMap[spec.Id] = spec - } - } - if CloudbrainSpecialResourceSpecsMap == nil || len(CloudbrainSpecialResourceSpecsMap) == 0 { - t := SpecialPools{} - json.Unmarshal([]byte(setting.SpecialPools), &t) - for _, pool := range t.Pools { - CloudbrainSpecialResourceSpecsMap = make(map[int]*ResourceSpec, len(pool.ResourceSpec)) - for _, spec := range pool.ResourceSpec { - CloudbrainSpecialResourceSpecsMap[spec.Id] = spec - } - } - } - SpecsMapInitFlag = true -} - -func InitCloudbrainOneGpuInfoMap() { - if CloudbrainDebugGpuInfosMap == nil || len(CloudbrainDebugGpuInfosMap) == 0 { - t := GpuInfos{} - json.Unmarshal([]byte(setting.GpuTypes), &t) - CloudbrainDebugGpuInfosMap = make(map[string]*GpuInfo, len(t.GpuInfo)) - for _, GpuInfo := range t.GpuInfo { - CloudbrainDebugGpuInfosMap[GpuInfo.Queue] = GpuInfo - } - } - if CloudbrainTrainGpuInfosMap == nil || len(CloudbrainTrainGpuInfosMap) == 0 { - t := GpuInfos{} - json.Unmarshal([]byte(setting.TrainGpuTypes), &t) - CloudbrainTrainGpuInfosMap = make(map[string]*GpuInfo, len(t.GpuInfo)) - for _, GpuInfo := range t.GpuInfo { - CloudbrainTrainGpuInfosMap[GpuInfo.Queue] = GpuInfo - } - } - if CloudbrainInferenceGpuInfosMap == nil || len(CloudbrainInferenceGpuInfosMap) == 0 { - t := GpuInfos{} - json.Unmarshal([]byte(setting.InferenceGpuTypes), &t) - CloudbrainInferenceGpuInfosMap = make(map[string]*GpuInfo, len(t.GpuInfo)) - for _, GpuInfo := range t.GpuInfo { - CloudbrainInferenceGpuInfosMap[GpuInfo.Queue] = GpuInfo - } - } - if CloudbrainBenchmarkGpuInfosMap == nil || len(CloudbrainBenchmarkGpuInfosMap) == 0 { - t := GpuInfos{} - json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &t) - CloudbrainBenchmarkGpuInfosMap = make(map[string]*GpuInfo, len(t.GpuInfo)) - for _, GpuInfo := range t.GpuInfo { - CloudbrainBenchmarkGpuInfosMap[GpuInfo.Queue] = GpuInfo - } - } - if CloudbrainSpecialGpuInfosMap == nil || len(CloudbrainSpecialGpuInfosMap) == 0 { - t := SpecialPools{} - json.Unmarshal([]byte(setting.SpecialPools), &t) - for _, pool := range t.Pools { - CloudbrainSpecialGpuInfosMap = make(map[string]*GpuInfo, len(pool.Pool)) - for _, GpuInfo := range pool.Pool { - CloudbrainSpecialGpuInfosMap[GpuInfo.Queue] = GpuInfo - } - } - } - GpuInfosMapInitFlag = true -} func GetNewestJobsByAiCenter() ([]int64, error) { ids := make([]int64, 0) return ids, x. diff --git a/routers/repo/aisafety.go b/routers/repo/aisafety.go index 5102a6722..63f50592b 100644 --- a/routers/repo/aisafety.go +++ b/routers/repo/aisafety.go @@ -535,6 +535,8 @@ func AiSafetyCreateForGetGPU(ctx *context.Context) { } else { log.Info("The GPU WaitCount not get") } + NotStopTaskCount, _ := models.GetModelSafetyCountByUserID(ctx.User.ID) + ctx.Data["NotStopTaskCount"] = NotStopTaskCount ctx.HTML(200, tplModelSafetyTestCreateGpu) } @@ -578,6 +580,8 @@ func AiSafetyCreateForGetNPU(ctx *context.Context) { waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") ctx.Data["WaitCount"] = waitCount log.Info("The NPU WaitCount is " + fmt.Sprint(waitCount)) + NotStopTaskCount, _ := models.GetModelSafetyCountByUserID(ctx.User.ID) + ctx.Data["NotStopTaskCount"] = NotStopTaskCount ctx.HTML(200, tplModelSafetyTestCreateNpu) } @@ -980,6 +984,8 @@ func modelSafetyNewDataPrepare(ctx *context.Context) error { ctx.Data["ckpt_name"] = ctx.Query("ckpt_name") ctx.Data["model_name"] = ctx.Query("model_name") ctx.Data["model_version"] = ctx.Query("model_version") + NotStopTaskCount, _ := models.GetModelSafetyCountByUserID(ctx.User.ID) + ctx.Data["NotStopTaskCount"] = NotStopTaskCount if ctx.QueryInt("type") == models.TypeCloudBrainOne { ctx.Data["type"] = models.TypeCloudBrainOne diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index a2ea7d51b..f5a43e697 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/dataset" + "code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" "code.gitea.io/gitea/services/cloudbrain/resource" "code.gitea.io/gitea/services/reward/point/account" @@ -107,7 +108,7 @@ func jobNamePrefixValid(s string) string { } -func cloudBrainNewDataPrepare(ctx *context.Context) error { +func cloudBrainNewDataPrepare(ctx *context.Context, jobType string) error { ctx.Data["PageIsCloudBrain"] = true t := time.Now() var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] @@ -148,6 +149,8 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error { defaultMode = "alogrithm" } ctx.Data["benchmarkMode"] = defaultMode + NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType, models.GPUResource) + ctx.Data["NotStopTaskCount"] = NotStopTaskCount if ctx.Cloudbrain != nil { ctx.Data["branch_name"] = ctx.Cloudbrain.BranchName @@ -210,7 +213,7 @@ func prepareCloudbrainOneSpecs(ctx *context.Context) { } func CloudBrainNew(ctx *context.Context) { - err := cloudBrainNewDataPrepare(ctx) + err := cloudBrainNewDataPrepare(ctx, string(models.JobTypeDebug)) if err != nil { ctx.ServerError("get new cloudbrain info failed", err) return @@ -244,7 +247,7 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { isOk, err := lock.Lock(models.CloudbrainKeyDuration) if !isOk { log.Error("lock processed failed:%v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tpl, &form) return } @@ -254,42 +257,42 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("the job name did already exist", tpl, &form) return } } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("system error", tpl, &form) return } } if !jobNamePattern.MatchString(displayJobName) { - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form) return } if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeTrain) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("jobtype error", tpl, &form) return } - count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType, models.GPUResource) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("system error", tpl, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) return } @@ -301,7 +304,7 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { datasetInfos, datasetNames, err = models.GetDatasetInfo(uuids) if err != nil { log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) return } @@ -312,7 +315,7 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName) if err != nil || !bootFileExist { log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form) return } @@ -320,7 +323,7 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { commandTrain, err := getTrainJobCommand(form) if err != nil { log.Error("getTrainJobCommand failed: %v", err) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(err.Error(), tpl, &form) return } @@ -333,7 +336,7 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } errStr := loadCodeAndMakeModelPath(repo, codePath, branchName, jobName, cloudbrain.ModelMountPath) if errStr != "" { - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr(errStr), tpl, &form) return } @@ -346,14 +349,14 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { Cluster: models.OpenICluster, AiCenterCode: models.AICenterOfCloudBrainOne}) if err != nil || spec == nil { - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("Resource specification not available", tpl, &form) return } if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("points.insufficient_points_balance"), tpl, &form) return } @@ -396,7 +399,7 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { err = cloudbrain.GenerateTask(req) if err != nil { - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(err.Error(), tpl, &form) return } @@ -454,7 +457,7 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra isOk, err := lock.Lock(models.CloudbrainKeyDuration) if !isOk { log.Error("lock processed failed:%v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tpl, &form) return } @@ -465,7 +468,7 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra command, err := getInferenceJobCommand(form) if err != nil { log.Error("getTrainJobCommand failed: %v", err) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(err.Error(), tpl, &form) return } @@ -474,21 +477,21 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("the job name did already exist", tpl, &form) return } } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("system error", tpl, &form) return } } if !jobNamePattern.MatchString(displayJobName) { - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form) return } @@ -496,21 +499,21 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName) if err != nil || !bootFileExist { log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form) return } - count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType, models.GPUResource) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("system error", tpl, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) return } @@ -521,7 +524,7 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra } errStr := loadCodeAndMakeModelPath(repo, codePath, branchName, jobName, cloudbrain.ResultPath) if errStr != "" { - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr(errStr), tpl, &form) return } @@ -531,7 +534,7 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid) if err != nil { log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) return } @@ -541,13 +544,13 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra Cluster: models.OpenICluster, AiCenterCode: models.AICenterOfCloudBrainOne}) if err != nil || spec == nil { - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("Resource specification not available", tpl, &form) return } if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("points.insufficient_points_balance"), tpl, &form) return } @@ -582,7 +585,7 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra err = cloudbrain.GenerateTask(req) if err != nil { - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(err.Error(), tpl, &form) return } @@ -682,7 +685,7 @@ func CloudBrainRestart(ctx *context.Context) { break } - count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, string(models.JobTypeDebug)) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, string(models.JobTypeDebug), models.GPUResource) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) resultCode = "-1" @@ -2222,7 +2225,7 @@ func CloudBrainBenchmarkNew(ctx *context.Context) { ctx.Data["description"] = "" ctx.Data["benchmarkTypeID"] = -1 ctx.Data["benchmark_child_types_id_hidden"] = -1 - err := cloudBrainNewDataPrepare(ctx) + err := cloudBrainNewDataPrepare(ctx, string(models.JobTypeBenchmark)) if err != nil { ctx.ServerError("get new cloudbrain info failed", err) return @@ -2327,6 +2330,7 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo benchmarkTypeID := form.BenchmarkTypeID benchmarkChildTypeID := form.BenchmarkChildTypeID repo := ctx.Repo.Repository + jobType := form.JobType ctx.Data["description"] = form.Description ctx.Data["benchmarkTypeID"] = benchmarkTypeID @@ -2336,31 +2340,31 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo isOk, err := lock.Lock(models.CloudbrainKeyDuration) if !isOk { log.Error("lock processed failed:%v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tplCloudBrainBenchmarkNew, &form) return } defer lock.UnLock() - tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeBenchmark), displayJobName) + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("the job name did already exist", tplCloudBrainBenchmarkNew, &form) return } } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) return } } if !jobNamePattern.MatchString(jobName) { - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplCloudBrainBenchmarkNew, &form) return } @@ -2368,7 +2372,7 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo childInfo, err := getBenchmarkAttachment(benchmarkTypeID, benchmarkChildTypeID, ctx) if err != nil { log.Error("getBenchmarkAttachment failed:%v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("benchmark type error", tplCloudBrainBenchmarkNew, &form) return } @@ -2379,27 +2383,27 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo Cluster: models.OpenICluster, AiCenterCode: models.AICenterOfCloudBrainOne}) if err != nil || spec == nil { - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("Resource specification not available", tplCloudBrainBenchmarkNew, &form) return } if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("points.insufficient_points_balance"), tplCloudBrainBenchmarkNew, &form) return } - count, err := models.GetBenchmarkCountByUserID(ctx.User.ID) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType, models.GPUResource) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tplCloudBrainBenchmarkNew, &form) return } @@ -2408,7 +2412,7 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo os.RemoveAll(codePath) if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil { log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) return } @@ -2417,11 +2421,11 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo if os.IsNotExist(err) { // file does not exist log.Error("train.py does not exist, %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("train.py does not exist", tplCloudBrainBenchmarkNew, &form) } else { log.Error("Stat failed, %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) } return @@ -2429,11 +2433,11 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo if os.IsNotExist(err) { // file does not exist log.Error("test.py does not exist, %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("test.py does not exist", tplCloudBrainBenchmarkNew, &form) } else { log.Error("Stat failed, %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) } return @@ -2441,7 +2445,7 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo if err := uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/"); err != nil { log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form) return } @@ -2466,7 +2470,7 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid) if err != nil { log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tplCloudBrainBenchmarkNew, &form) return } @@ -2500,7 +2504,7 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo err = cloudbrain.GenerateTask(req) if err != nil { - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(err.Error(), tplCloudBrainBenchmarkNew, &form) return } @@ -2526,7 +2530,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) isOk, err := lock.Lock(models.CloudbrainKeyDuration) if !isOk { log.Error("lock processed failed:%v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tpl, &form) return } @@ -2536,42 +2540,42 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("the job name did already exist", tpl, &form) return } } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("system error", tpl, &form) return } } if !jobNamePattern.MatchString(displayJobName) { - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form) return } if jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("jobtype error", tpl, &form) return } - count, err := models.GetBenchmarkCountByUserID(ctx.User.ID) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType, models.GPUResource) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("system error", tpl, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) return } @@ -2603,7 +2607,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid) if err != nil { log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) return } @@ -2613,14 +2617,14 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) Cluster: models.OpenICluster, AiCenterCode: models.AICenterOfCloudBrainOne}) if err != nil || spec == nil { - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("Resource specification not available", tpl, &form) return } if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID) - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(ctx.Tr("points.insufficient_points_balance"), tpl, &form) return } @@ -2654,7 +2658,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) err = cloudbrain.GenerateTask(req) if err != nil { - cloudBrainNewDataPrepare(ctx) + cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr(err.Error(), tpl, &form) return } @@ -2701,7 +2705,7 @@ func CloudBrainTrainJobVersionNew(ctx *context.Context) { } func cloudBrainTrainJobCreate(ctx *context.Context) { - err := cloudBrainNewDataPrepare(ctx) + err := cloudBrainNewDataPrepare(ctx, string(models.JobTypeTrain)) if err != nil { ctx.ServerError("get new train-job info failed", err) return @@ -2710,7 +2714,7 @@ func cloudBrainTrainJobCreate(ctx *context.Context) { } func InferenceCloudBrainJobNew(ctx *context.Context) { - err := cloudBrainNewDataPrepare(ctx) + err := cloudBrainNewDataPrepare(ctx, string(models.JobTypeInference)) if err != nil { ctx.ServerError("get new train-job info failed", err) return diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index b78bdebd3..d901298b7 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -12,6 +12,8 @@ import ( "strings" "time" + "code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" + "code.gitea.io/gitea/modules/dataset" "code.gitea.io/gitea/services/cloudbrain/resource" @@ -135,10 +137,15 @@ func grampusTrainJobNewDataPrepare(ctx *context.Context, processType string) err ctx.Data["datasetType"] = models.TypeCloudBrainOne waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeC2Net, models.GPUResource, models.JobTypeTrain) ctx.Data["WaitCount"] = waitCount + NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeC2Net, string(models.JobTypeTrain), models.GPUResource) + ctx.Data["NotStopTaskCount"] = NotStopTaskCount + } else if processType == grampus.ProcessorTypeNPU { ctx.Data["datasetType"] = models.TypeCloudBrainTwo waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeC2Net, models.NPUResource, models.JobTypeTrain) ctx.Data["WaitCount"] = waitCount + NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeC2Net, string(models.JobTypeTrain), models.NPUResource) + ctx.Data["NotStopTaskCount"] = NotStopTaskCount } if ctx.Cloudbrain != nil { @@ -300,7 +307,7 @@ func grampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain } //check count limit - count, err := models.GetGrampusCountByUserID(ctx.User.ID, string(models.JobTypeTrain), models.GPUResource) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeC2Net, string(models.JobTypeTrain), models.GPUResource) if err != nil { log.Error("GetGrampusCountByUserID failed:%v", err, ctx.Data["MsgID"]) grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) @@ -570,7 +577,7 @@ func grampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain } //check count limit - count, err := models.GetGrampusCountByUserID(ctx.User.ID, string(models.JobTypeTrain), models.NPUResource) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeC2Net, string(models.JobTypeTrain), models.NPUResource) if err != nil { log.Error("GetGrampusCountByUserID failed:%v", err, ctx.Data["MsgID"]) grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 6e44b3cd2..2a10da264 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -15,6 +15,8 @@ import ( "time" "unicode/utf8" + "code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" + "code.gitea.io/gitea/modules/dataset" "code.gitea.io/gitea/modules/modelarts_cd" @@ -144,6 +146,8 @@ func notebookNewDataPrepare(ctx *context.Context) error { waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") ctx.Data["WaitCount"] = waitCount + NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeDebug), models.NPUResource) + ctx.Data["NotStopTaskCount"] = NotStopTaskCount return nil } @@ -162,50 +166,6 @@ func prepareCloudbrainTwoDebugSpecs(ctx *context.Context) { ctx.Data["Specs"] = noteBookSpecs } -func NotebookCreate(ctx *context.Context, form auth.CreateModelArtsNotebookForm) { - ctx.Data["PageIsNotebook"] = true - jobName := form.JobName - uuid := form.Attachment - description := form.Description - flavor := form.Flavor - - count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID) - if err != nil { - log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("system error", tplModelArtsNotebookNew, &form) - return - } else { - if count >= 1 { - log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsNotebookNew, &form) - return - } - } - _, err = models.GetCloudbrainByName(jobName) - if err == nil { - log.Error("the job name did already exist", ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("the job name did already exist", tplModelArtsNotebookNew, &form) - return - } else { - if !models.IsErrJobNotExist(err) { - log.Error("system error, %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("system error", tplModelArtsNotebookNew, &form) - return - } - } - - err = modelarts.GenerateTask(ctx, jobName, uuid, description, flavor) - if err != nil { - ctx.RenderWithErr(err.Error(), tplModelArtsNotebookNew, &form) - return - } - ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all") -} - func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm) { ctx.Data["PageIsNotebook"] = true displayJobName := form.DisplayJobName @@ -225,7 +185,8 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm } defer lock.UnLock() - count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeDebug), models.NPUResource) + if err != nil { log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) notebookNewDataPrepare(ctx) @@ -272,7 +233,7 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm } if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { log.Error("point balance is not enough,userId=%d specId=%d ", ctx.User.ID, spec.ID) - cloudBrainNewDataPrepare(ctx) + notebookNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("points.insufficient_points_balance"), tplModelArtsNotebookNew, &form) return } @@ -450,7 +411,8 @@ func NotebookRestart(ctx *context.Context) { break } - count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeDebug), models.NPUResource) + if err != nil { log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) errorMsg = "system error" @@ -798,6 +760,8 @@ func trainJobNewDataPrepare(ctx *context.Context) error { ctx.Data["datasetType"] = models.TypeCloudBrainTwo waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") ctx.Data["WaitCount"] = waitCount + NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeTrain), models.NPUResource) + ctx.Data["NotStopTaskCount"] = NotStopTaskCount setMultiNodeIfConfigureMatch(ctx) @@ -966,6 +930,8 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error { ctx.Data["config_list"] = configList.ParaConfigs waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") ctx.Data["WaitCount"] = waitCount + NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeTrain), models.NPUResource) + ctx.Data["NotStopTaskCount"] = NotStopTaskCount return nil } @@ -1012,7 +978,8 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) } defer lock.UnLock() - count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeTrain), models.NPUResource) + if err != nil { log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) trainJobNewDataPrepare(ctx) @@ -1356,7 +1323,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ return } - count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeTrain), models.NPUResource) if err != nil { log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) trainJobNewVersionDataPrepare(ctx) @@ -2007,7 +1974,8 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference } defer lock.UnLock() - count, err := models.GetCloudbrainInferenceJobCountByUserID(ctx.User.ID) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeInference), models.NPUResource) + if err != nil { log.Error("GetCloudbrainInferenceJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) inferenceJobErrorNewDataPrepare(ctx, form) @@ -2409,6 +2377,8 @@ func inferenceJobNewDataPrepare(ctx *context.Context) error { ctx.Data["datasetType"] = models.TypeCloudBrainTwo waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") ctx.Data["WaitCount"] = waitCount + NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeInference), models.NPUResource) + ctx.Data["NotStopTaskCount"] = NotStopTaskCount return nil } diff --git a/services/cloudbrain/cloudbrainTask/count.go b/services/cloudbrain/cloudbrainTask/count.go new file mode 100644 index 000000000..25d56b0b6 --- /dev/null +++ b/services/cloudbrain/cloudbrainTask/count.go @@ -0,0 +1,86 @@ +package cloudbrainTask + +import ( + "fmt" + "strconv" + + "code.gitea.io/gitea/models" +) + +type StatusInfo struct { + CloudBrainTypes []int + JobType []models.JobType + NotFinalStatuses []string + ComputeResource string +} + +var cloudbrainOneNotFinalStatuses = []string{string(models.JobWaiting), string(models.JobRunning)} +var cloudbrainTwoNotFinalStatuses = []string{string(models.ModelArtsTrainJobInit), string(models.ModelArtsTrainJobImageCreating), string(models.ModelArtsTrainJobSubmitTrying), string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobRunning), string(models.ModelArtsTrainJobScaling), string(models.ModelArtsTrainJobCheckInit), string(models.ModelArtsTrainJobCheckRunning), string(models.ModelArtsTrainJobCheckRunningCompleted)} +var grampusTwoNotFinalStatuses = []string{models.GrampusStatusWaiting, models.GrampusStatusRunning} +var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { + CloudBrainTypes: []int{models.TypeCloudBrainOne}, + JobType: []models.JobType{models.JobTypeDebug}, + NotFinalStatuses: cloudbrainOneNotFinalStatuses, + ComputeResource: models.GPUResource, +}, string(models.JobTypeTrain) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { + CloudBrainTypes: []int{models.TypeCloudBrainOne}, + JobType: []models.JobType{models.JobTypeTrain}, + NotFinalStatuses: cloudbrainOneNotFinalStatuses, + ComputeResource: models.GPUResource, +}, string(models.JobTypeInference) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { + CloudBrainTypes: []int{models.TypeCloudBrainOne}, + JobType: []models.JobType{models.JobTypeInference}, + NotFinalStatuses: cloudbrainOneNotFinalStatuses, + ComputeResource: models.GPUResource, +}, string(models.JobTypeBenchmark) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { + CloudBrainTypes: []int{models.TypeCloudBrainOne}, + JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeModelSafety, models.JobTypeBrainScore, models.JobTypeSnn4imagenet}, + NotFinalStatuses: cloudbrainOneNotFinalStatuses, + ComputeResource: models.GPUResource, +}, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { + CloudBrainTypes: []int{models.TypeCloudBrainTwo, models.TypeCDCenter}, + JobType: []models.JobType{models.JobTypeDebug}, + NotFinalStatuses: []string{string(models.ModelArtsCreateQueue), string(models.ModelArtsCreating), string(models.ModelArtsStarting), string(models.ModelArtsReadyToStart), string(models.ModelArtsResizing), string(models.ModelArtsStartQueuing), string(models.ModelArtsRunning), string(models.ModelArtsRestarting)}, + ComputeResource: models.NPUResource, +}, string(models.JobTypeTrain) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { + CloudBrainTypes: []int{models.TypeCloudBrainTwo}, + JobType: []models.JobType{models.JobTypeTrain}, + NotFinalStatuses: cloudbrainTwoNotFinalStatuses, + ComputeResource: models.NPUResource, +}, string(models.JobTypeInference) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { + CloudBrainTypes: []int{models.TypeCloudBrainTwo}, + JobType: []models.JobType{models.JobTypeTrain}, + NotFinalStatuses: cloudbrainTwoNotFinalStatuses, + ComputeResource: models.NPUResource, +}, string(models.JobTypeTrain) + "-" + strconv.Itoa(models.TypeC2Net) + "-" + models.GPUResource: { + CloudBrainTypes: []int{models.TypeC2Net}, + JobType: []models.JobType{models.JobTypeTrain}, + NotFinalStatuses: grampusTwoNotFinalStatuses, + ComputeResource: models.GPUResource, +}, string(models.JobTypeTrain) + "-" + strconv.Itoa(models.TypeC2Net) + "-" + models.NPUResource: { + CloudBrainTypes: []int{models.TypeC2Net}, + JobType: []models.JobType{models.JobTypeTrain}, + NotFinalStatuses: grampusTwoNotFinalStatuses, + ComputeResource: models.NPUResource, +}} + +func GetNotFinalStatusTaskCount(uid int64, cloudbrainType int, jobType string, computeResource ...string) (int, error) { + jobNewType := jobType + if jobType == string(models.JobTypeSnn4imagenet) || jobType == string(models.JobTypeBrainScore) { + jobNewType = string(models.JobTypeBenchmark) + } + + key := jobNewType + "-" + strconv.Itoa(cloudbrainType) + if len(computeResource) > 0 { + key = key + "-" + computeResource[0] + } + + if statusInfo, ok := StatusInfoDict[key]; ok { + + return models.GetNotFinalStatusTaskCount(uid, statusInfo.NotFinalStatuses, statusInfo.JobType, statusInfo.CloudBrainTypes, statusInfo.ComputeResource) + + } else { + return 0, fmt.Errorf("Can not find the status info.") + } + +} From c634b0e80359e1f01fdd5199da19c32aaa2e842a Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 21 Oct 2022 16:18:17 +0800 Subject: [PATCH 21/82] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/cloudbrain.go | 12 ++++++------ routers/repo/modelarts.go | 18 +++++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index f5a43e697..92c95de4e 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -149,7 +149,7 @@ func cloudBrainNewDataPrepare(ctx *context.Context, jobType string) error { defaultMode = "alogrithm" } ctx.Data["benchmarkMode"] = defaultMode - NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType, models.GPUResource) + NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType) ctx.Data["NotStopTaskCount"] = NotStopTaskCount if ctx.Cloudbrain != nil { @@ -283,7 +283,7 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { return } - count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType, models.GPUResource) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx, jobType) @@ -504,7 +504,7 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra return } - count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType, models.GPUResource) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx, jobType) @@ -685,7 +685,7 @@ func CloudBrainRestart(ctx *context.Context) { break } - count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, string(models.JobTypeDebug), models.GPUResource) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, string(models.JobTypeDebug)) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) resultCode = "-1" @@ -2394,7 +2394,7 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo return } - count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType, models.GPUResource) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx, jobType) @@ -2566,7 +2566,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) return } - count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType, models.GPUResource) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType) if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx, jobType) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 2a10da264..07c1fcd3e 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -146,7 +146,7 @@ func notebookNewDataPrepare(ctx *context.Context) error { waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") ctx.Data["WaitCount"] = waitCount - NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeDebug), models.NPUResource) + NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeDebug)) ctx.Data["NotStopTaskCount"] = NotStopTaskCount return nil @@ -185,7 +185,7 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm } defer lock.UnLock() - count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeDebug), models.NPUResource) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeDebug)) if err != nil { log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -411,7 +411,7 @@ func NotebookRestart(ctx *context.Context) { break } - count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeDebug), models.NPUResource) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeDebug)) if err != nil { log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -760,7 +760,7 @@ func trainJobNewDataPrepare(ctx *context.Context) error { ctx.Data["datasetType"] = models.TypeCloudBrainTwo waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") ctx.Data["WaitCount"] = waitCount - NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeTrain), models.NPUResource) + NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeTrain)) ctx.Data["NotStopTaskCount"] = NotStopTaskCount setMultiNodeIfConfigureMatch(ctx) @@ -930,7 +930,7 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error { ctx.Data["config_list"] = configList.ParaConfigs waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") ctx.Data["WaitCount"] = waitCount - NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeTrain), models.NPUResource) + NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeTrain)) ctx.Data["NotStopTaskCount"] = NotStopTaskCount return nil @@ -978,7 +978,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) } defer lock.UnLock() - count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeTrain), models.NPUResource) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeTrain)) if err != nil { log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -1323,7 +1323,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ return } - count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeTrain), models.NPUResource) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeTrain)) if err != nil { log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) trainJobNewVersionDataPrepare(ctx) @@ -1974,7 +1974,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference } defer lock.UnLock() - count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeInference), models.NPUResource) + count, err := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeInference)) if err != nil { log.Error("GetCloudbrainInferenceJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -2377,7 +2377,7 @@ func inferenceJobNewDataPrepare(ctx *context.Context) error { ctx.Data["datasetType"] = models.TypeCloudBrainTwo waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") ctx.Data["WaitCount"] = waitCount - NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeInference), models.NPUResource) + NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeInference)) ctx.Data["NotStopTaskCount"] = NotStopTaskCount return nil From 3695fd996c94557a72e78a1f22055e82afbe0955 Mon Sep 17 00:00:00 2001 From: Gitea Date: Fri, 21 Oct 2022 16:40:38 +0800 Subject: [PATCH 22/82] added --- models/ai_model_manage.go | 2 +- routers/repo/ai_model_convert.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index a88da8fe5..d9adda2dc 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -88,7 +88,7 @@ type AiModelQueryOptions struct { } func (a *AiModelConvert) IsGpuTrainTask() bool { - if a.SrcEngine == 0 || a.SrcEngine == 1 { + if a.SrcEngine == 0 || a.SrcEngine == 1 || a.SrcEngine == 4 || a.SrcEngine == 6 { return true } return false diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go index bd6a01072..13b02f9d0 100644 --- a/routers/repo/ai_model_convert.go +++ b/routers/repo/ai_model_convert.go @@ -18,7 +18,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" - "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/set ting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" uuid "github.com/satori/go.uuid" From e1a454d5704835a3a78802eb2b8ffac74eaee892 Mon Sep 17 00:00:00 2001 From: Gitea Date: Fri, 21 Oct 2022 16:43:59 +0800 Subject: [PATCH 23/82] add --- routers/repo/ai_model_convert.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go index 13b02f9d0..bd6a01072 100644 --- a/routers/repo/ai_model_convert.go +++ b/routers/repo/ai_model_convert.go @@ -18,7 +18,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" - "code.gitea.io/gitea/modules/set ting" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" uuid "github.com/satori/go.uuid" From dc36c0d3a899e12203bf00d6fa3ce831f017b100 Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 21 Oct 2022 16:53:05 +0800 Subject: [PATCH 24/82] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 691fe5002..1f4ea1ca3 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -122,7 +122,7 @@ func GetOverviewDuration(ctx *context.Context) { now := time.Now() endTime := now page := 1 - pagesize := 10000 + pagesize := 1000 count := pagesize worker_server_num := 1 cardNum := 1 From 2d93c784f3f8e2f583f95c954fc9586ff0c80c22 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 21 Oct 2022 16:58:50 +0800 Subject: [PATCH 25/82] =?UTF-8?q?=E8=B0=83=E7=94=A8=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E5=AE=89=E5=85=A8=E7=9A=84=E5=9C=B0=E5=9D=80=E6=94=BE=E5=88=B0?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6=E4=B8=AD=E8=BF=9B=E8=A1=8C?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- modules/aisafety/resty.go | 5 +++-- modules/setting/setting.go | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/aisafety/resty.go b/modules/aisafety/resty.go index be6468529..ce1fa736e 100644 --- a/modules/aisafety/resty.go +++ b/modules/aisafety/resty.go @@ -10,6 +10,7 @@ import ( "strings" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "github.com/go-resty/resty/v2" ) @@ -71,8 +72,8 @@ func checkSetting() { } func loginCloudbrain() error { - HOST = "http://221.122.70.196:8081/atp-api" - KEY = "1" + HOST = setting.ModelSafetyTest.HOST + KEY = setting.ModelSafetyTest.KEY return nil } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index c6afae05a..6a2520162 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -718,6 +718,9 @@ var ( GPUBaseDataSetUUID string GPUCombatDataSetName string GPUCombatDataSetUUID string + + HOST string + KEY string }{} ModelApp = struct { @@ -1557,6 +1560,8 @@ func getModelSafetyConfig() { ModelSafetyTest.NPUBaseDataSetUUID = sec.Key("NPUBaseDataSetUUID").MustString("") ModelSafetyTest.NPUCombatDataSetName = sec.Key("NPUCombatDataSetName").MustString("") ModelSafetyTest.NPUCombatDataSetUUID = sec.Key("NPUCombatDataSetUUID").MustString("") + ModelSafetyTest.HOST = sec.Key("HOST").MustString("") + ModelSafetyTest.KEY = sec.Key("KEY").MustString("") } func getModelConvertConfig() { From 5b64d7871097a80c6354d8b936e5a09313d61b84 Mon Sep 17 00:00:00 2001 From: zouap Date: Fri, 21 Oct 2022 17:00:47 +0800 Subject: [PATCH 26/82] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E5=88=A0=E9=99=A4=E5=A4=9A=E4=BD=99=E7=9A=84=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/aisafety.go | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/routers/repo/aisafety.go b/routers/repo/aisafety.go index 5102a6722..6881b4640 100644 --- a/routers/repo/aisafety.go +++ b/routers/repo/aisafety.go @@ -26,7 +26,6 @@ import ( "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/cloudbrain/resource" "code.gitea.io/gitea/services/reward/point/account" - uuid "github.com/satori/go.uuid" ) const ( @@ -37,39 +36,6 @@ const ( tplModelSafetyTestShow = "repo/modelsafety/show" ) -func CloudBrainAiSafetyCreateTest(ctx *context.Context) { - log.Info("start to create CloudBrainAiSafetyCreate") - uuid := uuid.NewV4() - id := uuid.String() - seriaNoParas := ctx.Query("serialNo") - fileName := ctx.Query("fileName") - - //if jobType == string(models.JobTypeBenchmark) { - req := aisafety.TaskReq{ - UnionId: id, - EvalName: "test1", - EvalContent: "test1", - TLPath: "test1", - Indicators: []string{"ACC", "ASS"}, - CDName: "CIFAR10_1000_FGSM", - BDName: "CIFAR10_1000基础数据集", - } - aisafety.GetAlgorithmList() - if seriaNoParas != "" { - aisafety.GetTaskStatus(seriaNoParas) - } else { - jsonStr, err := getJsonContent("http://192.168.207.34:8065/Test_zap1234/openi_aisafety/raw/branch/master/result/" + fileName) - serialNo, err := aisafety.CreateSafetyTask(req, jsonStr) - if err == nil { - log.Info("serialNo=" + serialNo) - time.Sleep(time.Duration(2) * time.Second) - aisafety.GetTaskStatus(serialNo) - } else { - log.Info("CreateSafetyTask error," + err.Error()) - } - } -} - func GetAiSafetyTaskByJob(job *models.Cloudbrain) { if job == nil { log.Error("GetCloudbrainByJobID failed") From 97b60991194d44421f5de1a90eebe64e7f73c010 Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 21 Oct 2022 18:02:51 +0800 Subject: [PATCH 27/82] fix-bug --- models/cloudbrain.go | 4 +- routers/api/v1/repo/cloudbrain_dashboard.go | 87 +++++++++------------ 2 files changed, 40 insertions(+), 51 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 6135dac40..3f58284fd 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2296,9 +2296,9 @@ func CloudbrainAllStatic(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, er } sess.Limit(opts.PageSize, start) } - sess.OrderBy("cloudbrain.created_unix DESC") + // sess.OrderBy("cloudbrain.created_unix DESC") cloudbrains := make([]*CloudbrainInfo, 0, setting.UI.IssuePagingNum) - if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). + if err := sess.Cols("status", "type", "job_type", "train_job_duration", "duration", "compute_resource", "created_unix", "start_time", "end_time", "work_server_number").Table(&Cloudbrain{}).Unscoped().Where(cond). Find(&cloudbrains); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 935006476..81c795087 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -120,9 +120,6 @@ func GetOverviewDuration(ctx *context.Context) { recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix now := time.Now() endTime := now - page := 1 - pagesize := 1000 - count := pagesize worker_server_num := 1 cardNum := 1 durationAllSum := int64(0) @@ -138,54 +135,46 @@ func GetOverviewDuration(ctx *context.Context) { c2NetDuration := int64(0) cDCenterDuration := int64(0) - for count == pagesize && count != 0 { - cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ - ListOptions: models.ListOptions{ - Page: page, - PageSize: pagesize, - }, - Type: models.TypeCloudBrainAll, - BeginTimeUnix: int64(recordBeginTime), - EndTimeUnix: endTime.Unix(), - }) - if err != nil { - ctx.ServerError("Get cloudbrains failed:", err) - return - } - models.LoadSpecs4CloudbrainInfo(cloudbrains) - - for _, cloudbrain := range cloudbrains { - if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { - worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber - } else { - worker_server_num = 1 - } - if cloudbrain.Cloudbrain.Spec == nil { - cardNum = 1 - } else { - cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum - } - duration := cloudbrain.Duration - durationSum := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum) - if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { - cloudBrainOneDuration += duration - cloudBrainOneCardDuSum += durationSum - } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { - cloudBrainTwoDuration += duration - cloudBrainTwoCardDuSum += durationSum - } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net { - c2NetDuration += duration - c2NetCardDuSum += durationSum - } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { - cDCenterDuration += duration - cDNetCardDuSum += durationSum - } + cloudbrains, _, err := models.CloudbrainAllStatic(&models.CloudbrainsOptions{ + Type: models.TypeCloudBrainAll, + BeginTimeUnix: int64(recordBeginTime), + EndTimeUnix: endTime.Unix(), + }) + if err != nil { + ctx.ServerError("Get cloudbrains failed:", err) + return + } + models.LoadSpecs4CloudbrainInfo(cloudbrains) - durationAllSum += duration - cardDuSum += durationSum - count = len(cloudbrains) - page += 1 + for _, cloudbrain := range cloudbrains { + if cloudbrain.Cloudbrain.WorkServerNumber >= 1 { + worker_server_num = cloudbrain.Cloudbrain.WorkServerNumber + } else { + worker_server_num = 1 } + if cloudbrain.Cloudbrain.Spec == nil { + cardNum = 1 + } else { + cardNum = cloudbrain.Cloudbrain.Spec.AccCardsNum + } + duration := cloudbrain.Duration + durationSum := cloudbrain.Duration * int64(worker_server_num) * int64(cardNum) + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { + cloudBrainOneDuration += duration + cloudBrainOneCardDuSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + cloudBrainTwoDuration += duration + cloudBrainTwoCardDuSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeC2Net { + c2NetDuration += duration + c2NetCardDuSum += durationSum + } else if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { + cDCenterDuration += duration + cDNetCardDuSum += durationSum + } + + durationAllSum += duration + cardDuSum += durationSum } ctx.JSON(http.StatusOK, map[string]interface{}{ "cloudBrainOneCardDuSum": cloudBrainOneCardDuSum, From 4964fe5364d432cf57119e1113f7d9e2c545d355 Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 24 Oct 2022 10:27:56 +0800 Subject: [PATCH 28/82] merge --- routers/routes/routes.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index be603785f..899918339 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -20,10 +20,6 @@ import ( "code.gitea.io/gitea/routers/modelapp" - "code.gitea.io/gitea/routers/reward/point" - "code.gitea.io/gitea/routers/task" - "code.gitea.io/gitea/services/reward" - "code.gitea.io/gitea/modules/slideimage" "code.gitea.io/gitea/routers/image" From fc5692e781ce2abea4af8386c968a1ae75038e8e Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 24 Oct 2022 11:48:12 +0800 Subject: [PATCH 29/82] update --- models/cloudbrain_static.go | 34 +-------------------- routers/api/v1/repo/cloudbrain_dashboard.go | 24 +++++---------- routers/repo/cloudbrain_statistic.go | 2 -- 3 files changed, 9 insertions(+), 51 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 3f2dcf94a..3bc6ad296 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -38,17 +38,6 @@ type TaskDetail struct { WorkServerNum int64 `json:"WorkServerNum"` Spec *Specification `json:"Spec"` } -type CardTypeAndNum struct { - CardType string `json:"CardType"` - Num int `json:"Num"` - ComputeResource string `json:"computeResource"` -} -type ResourceOverview struct { - Cluster string `json:"cluster"` - AiCenterName string `json:"aiCenterName"` - AiCenterCode string `json:"aiCenterCode"` - CardTypeAndNum []CardTypeAndNum `json:"cardTypeAndNum"` -} type CloudbrainDurationStatistic struct { ID int64 `xorm:"pk autoincr"` @@ -78,7 +67,7 @@ type DurationStatisticOptions struct { type DurationRateStatistic struct { AiCenterTotalDurationStat map[string]int `json:"aiCenterTotalDurationStat"` AiCenterUsageDurationStat map[string]int `json:"aiCenterUsageDurationStat"` - UsageRate map[string]float32 `json:"UsageRate"` + UsageRate map[string]float64 `json:"UsageRate"` } type ResourceDetail struct { QueueCode string @@ -89,15 +78,8 @@ type ResourceDetail struct { AccCardType string CardsTotalNum int IsAutomaticSync bool - // CardTypeAndNum []CardTypeAndNum `json:"cardTypeAndNum"` } -// type DateCloudbrainStatistic struct { -// Date string `json:"date"` -// AiCenterUsageDuration map[string]int `json:"aiCenterUsageDuration"` -// AiCenterTotalDuration map[string]int `json:"aiCenterTotalDuration"` -// AiCenterUsageRate map[string]float64 `json:"aiCenterUsageRate"` -// } type DateUsageStatistic struct { Date string `json:"date"` UsageDuration int `json:"usageDuration"` @@ -347,26 +329,12 @@ func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode s func GetCanUseCardInfo() ([]*ResourceQueue, error) { sess := x.NewSession() defer sess.Close() - // var cond = builder.NewCond() - // cond = cond.And( - // builder.And(builder.Eq{"resource_queue.is_automatic_sync": false}), - // ) sess.OrderBy("resource_queue.id ASC") ResourceQueues := make([]*ResourceQueue, 0, 10) if err := sess.Table(&ResourceQueue{}).Find(&ResourceQueues); err != nil { log.Info("find error.") } return ResourceQueues, nil - // Cols("queue_code", "cluster", "ai_center_name", "ai_center_code", "compute_resource", "acc_card_type", "cards_total_num") - // sess := x.NewSession() - // defer sess.Close() - // sess.OrderBy("resource_queue.id ASC limit 1") - // cloudbrains := make([]*CloudbrainInfo, 0) - // if err := sess.Table(&Cloudbrain{}).Unscoped(). - // Find(&cloudbrains); err != nil { - // log.Info("find error.") - // } - // return cloudbrains, nil } func GetCardDurationStatistics(opts *DurationStatisticOptions) ([]*CloudbrainDurationStatistic, error) { diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 0ae516750..0bfd2a825 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1663,10 +1663,10 @@ func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrain return totalDuration, usageDuration, usageRate } -func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.DurationRateStatistic, models.DurationRateStatistic, float32) { +func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.DurationRateStatistic, models.DurationRateStatistic, float64) { OpenITotalDuration := make(map[string]int) OpenIUsageDuration := make(map[string]int) - OpenIUsageRate := make(map[string]float32) + OpenIUsageRate := make(map[string]float64) C2NetTotalDuration := make(map[string]int) C2NetUsageDuration := make(map[string]int) OpenIDurationRate := models.DurationRateStatistic{} @@ -1717,7 +1717,6 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati return OpenIDurationRate, C2NetDurationRate, 0 } for _, v := range ResourceAiCenterRes { - // if v.AiCenterCode != models.AICenterOfCloudBrainOne && v.AiCenterCode != models.AICenterOfCloudBrainTwo { if cutString(v.AiCenterCode, 4) == cutString(models.AICenterOfCloudBrainOne, 4) { if _, ok := OpenIUsageDuration[v.AiCenterName]; !ok { OpenIUsageDuration[v.AiCenterName] = 0 @@ -1731,36 +1730,33 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati } } } - totalCanUse := float32(0) - totalUse := float32(0) - totalUsageRate := float32(0) + totalCanUse := float64(0) + totalUse := float64(0) + totalUsageRate := float64(0) for k, v := range OpenITotalDuration { for i, j := range OpenIUsageDuration { if k == i { - OpenIUsageRate[k] = float32(j) / float32(v) + OpenIUsageRate[k] = float64(j) / float64(v) } } } for _, v := range OpenITotalDuration { - totalCanUse += float32(v) + totalCanUse += float64(v) } for _, v := range OpenIUsageRate { - totalUse += float32(v) + totalUse += float64(v) } if totalCanUse == 0 || totalUse == 0 { totalUsageRate = 0 } else { totalUsageRate = totalUse / totalCanUse } - // totalUsageRate = totalUse / totalCanUse - // strconv.FormatFloat(*100, 'f', 4, 64) + "%" OpenIDurationRate.AiCenterTotalDurationStat = OpenITotalDuration OpenIDurationRate.AiCenterUsageDurationStat = OpenIUsageDuration OpenIDurationRate.UsageRate = OpenIUsageRate C2NetDurationRate.AiCenterTotalDurationStat = C2NetTotalDuration C2NetDurationRate.AiCenterUsageDurationStat = C2NetUsageDuration - // C2NetDurationRate.TotalUsageRate = totalUsageRate return OpenIDurationRate, C2NetDurationRate, totalUsageRate } @@ -1846,10 +1842,6 @@ func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterC if _, ok := hourTimeTotalDuration[v]; !ok { hourTimeTotalDuration[v] = 0 } - // if _, ok := hourTimeUsageRate[v]; !ok { - // hourTimeUsageRate[v] = 0 - // } - } for k, v := range hourTimeTotalDuration { diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index da1849096..0cc3fc317 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -42,7 +42,6 @@ func CloudbrainDurationStatisticHour() { log.Info("GetSpecByAiCenterCodeAndType err: %v", err) return } - log.Info("cloudbrain: %s", cloudbrain) if cloudbrain != nil { totalCanUse := false if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType, totalCanUse); err != nil { @@ -107,7 +106,6 @@ func CloudbrainDurationStatisticHour() { func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) map[string]map[string]int { var WorkServerNumber int var AccCardsNum int - // cloudBrainCardRes := make(map[string]int) cloudBrainAiCenterCodeList := make(map[string]string) cloudBrainCardTypeList := make(map[string]string) cloudBrainCenterCodeAndCardType := make(map[string]map[string]int) From 8808452bb2f8b6879acc9320954d598b254f7a7c Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Mon, 24 Oct 2022 12:28:16 +0800 Subject: [PATCH 30/82] fix issue --- templates/resource_desc.tmpl | 267 ++++++++--------------------------- 1 file changed, 62 insertions(+), 205 deletions(-) diff --git a/templates/resource_desc.tmpl b/templates/resource_desc.tmpl index e809541ad..30ae3ff14 100644 --- a/templates/resource_desc.tmpl +++ b/templates/resource_desc.tmpl @@ -24,212 +24,69 @@ 备注 - - - 启智集群 - GPU - 调试任务 - T4 - -
    -
  • - 外部公开镜像，如：dockerhub镜像； -
    • -
  • 平台镜像；
    • -
- - 能连外网 - 平台可解压数据集 - - 数据集存放路径/dataset，模型存放路径/model，代码存放路径/code - - - - - - 训练任务 - V100 - -
    -
  • 平台镜像；
    • -
- - 不能连外网 - - 训练脚本存储在/code中，数据集存储在/dataset中，预训练模型存放在环境变量ckpt_url中，训练输出请存储在/model中 - 以供后续下载。 - - - https://git.openi.org.cn/OpenIO - SSG/MNIST_PytorchExample_GPU - - - 启智集群V100不能连外网，只能使用平台的镜像，不可使用外部公开镜像，，否则任务会一直处于waiting - 状态 - - - - A100 - -
    -
  • - 外部公开镜像，如：dockerhub镜像； -
    • -
  • 平台镜像；
    • -
- - 能连外网 - - - - 推理任务 - V100 - -
    -
  • 平台镜像；
    • -
- - 不能连外网 - - 数据集存储在/dataset中，模型文件存储在/model中，推理输出请存储在/result中 - 以供后续下载。 - - - https://git.openi.org.cn/OpenIO - SSG/MNIST_PytorchExample_GPU/src/branch/master/inference.py - - - - - 评测任务 - V100 - -
    -
  • 平台镜像；
    • -
- - 不能连外网 - - - - 模型评测时，先使用数据集功能上传模型，然后从数据集列表选模型。 - - - - NPU - 调试任务 - Ascend 910 - -
    -
  • 平台镜像；
    • -
- - 能连外网 - - - - - - 训练任务 - Ascend 910 - - 数据集位置存储在环境变量data_url中，预训练模型存放在环境变量ckpt_url中，训练输出路径存储在环境变量train_url中。 - - - https://git.openi.org.cn/OpenIOSSG/MNIST_Example - - - - - 推理任务 - Ascend 910 - - 数据集位置存储在环境变量data_url中，推理输出路径存储在环境变量result_url中。 - - - https://git.openi.org.cn/OpenIOSSG/MNIST_Example - - - - - 智算网络 - GPU - 训练任务 - V100 - -
    -
  • - 外部公开镜像，如：dockerhub镜像； -
    • -
  • 平台镜像；
    • -
- - 能连外网 - 用户自行解压数据 集 - - 训练脚本存储在/tmp/code中，数据集存储在/tmp/dataset中，预训练模型存放在环境变量ckpt_url中，训练输出请存储在/tmp/output中以供后续下载。 - - - https://git.openi.org.cn/OpenIO - SSG/MNIST_PytorchExample_GPU/src/branch/master/train_for_c - A100 2net.py - - - - - A100 - -
    -
  • - 外部公开镜像，如：dockerhub镜像； -
    • -
  • 平台镜像；
    • -
- - 能连外网 - - - - NPU - 训练任务 - Ascend 910 - -
    -
  • 平台镜像；
    • -
- - 能连外网 - - 训练脚本存储在/cache/code中，预训练模型存放在环境变量ckpt_url中，训练输出请存储在/cache/output中以供后续下载。 - - - https://git.openi.org.cn/OpenIO - SSG/MNIST_Example/src/branch/master/train_for_c2net.py - - - - + {{template "base/footer" .}} + From 409b4be0dc5f653babad9d3dc36e9da54a8e7e0f Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 24 Oct 2022 16:25:01 +0800 Subject: [PATCH 31/82] update --- modules/cron/tasks_basic.go | 2 +- routers/api/v1/api.go | 1 + routers/api/v1/repo/cloudbrain_dashboard.go | 13 ++++++------- routers/repo/cloudbrain_statistic.go | 4 ---- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 958595960..f9661b892 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -270,7 +270,7 @@ func registerHandleCloudbrainDurationStatistic() { RegisterTaskFatal("handle_cloudbrain_duration_statistic", &BaseConfig{ Enabled: true, RunAtStart: false, - Schedule: "55 59 * * * ?", + Schedule: "59 59 * * * ?", }, func(ctx context.Context, _ *models.User, _ Config) error { repo.CloudbrainDurationStatisticHour() return nil diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index b719db71e..c464f252c 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -603,6 +603,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/overview_resource", repo.GetCloudbrainResourceOverview) m.Get("/resource_usage_statistic", repo.GetDurationRateStatistic) m.Get("/resource_usage_rate_detail", repo.GetCloudbrainResourceUsageDetail) + m.Get("/apitest_for_statistic", repo.CloudbrainDurationStatisticForTest) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 0bfd2a825..6da731f5e 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1431,21 +1431,19 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { resourceDetail.QueueCode = resourceQueue.QueueCode resourceDetail.Cluster = resourceQueue.Cluster resourceDetail.AiCenterCode = resourceQueue.AiCenterCode - resourceDetail.AiCenterName = resourceQueue.AiCenterName + resourceDetail.AiCenterName = resourceQueue.AiCenterName + "/" + resourceQueue.AiCenterCode resourceDetail.ComputeResource = resourceQueue.ComputeResource resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")" resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum resourceDetail.IsAutomaticSync = resourceQueue.IsAutomaticSync OpenIResourceDetail = append(OpenIResourceDetail, resourceDetail) - // } else { - } if resourceQueue.Cluster == models.C2NetCluster { var resourceDetail models.ResourceDetail resourceDetail.QueueCode = resourceQueue.QueueCode resourceDetail.Cluster = resourceQueue.Cluster resourceDetail.AiCenterCode = resourceQueue.AiCenterCode - resourceDetail.AiCenterName = resourceQueue.AiCenterName + resourceDetail.AiCenterName = resourceQueue.AiCenterName + "/" + resourceQueue.AiCenterCode resourceDetail.ComputeResource = resourceQueue.ComputeResource resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")" resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum @@ -1553,6 +1551,10 @@ func GetDurationRateStatistic(ctx *context.Context) { } +func CloudbrainDurationStatisticForTest(ctx *context.Context) { + repo.CloudbrainDurationStatisticHour() +} + func getBeginAndEndTime(ctx *context.Context) (time.Time, time.Time) { queryType := ctx.QueryTrim("type") now := time.Now() @@ -1656,9 +1658,6 @@ func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrain } else { usageRate = float64(usageDuration) / float64(totalDuration) } - // if - // usageRate, _ = strconv.ParseFloat(fmt.Sprintf("%.4f", float32(usageDuration)/float32(totalDuration)), 64) - // totalUsageRate = totalUse / totalCanUse return totalDuration, usageDuration, usageRate } diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 0cc3fc317..703e0635f 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -25,15 +25,11 @@ func CloudbrainDurationStatisticHour() { } ciTasks2, err := models.GetCloudbrainCompleteByTime(beginTime, endTime) ciTasks := append(ciTasks1, ciTasks2...) - log.Info("beginTime: %s", beginTime) - log.Info("endTime: %s", endTime) if err != nil { log.Info("GetCloudbrainCompleteByTime err: %v", err) return } models.LoadSpecs4CloudbrainInfo(ciTasks) - log.Info("ciTasks here: %s", ciTasks) - log.Info("count here: %s", len(ciTasks)) cloudBrainCenterCodeAndCardTypeInfo := getcloudBrainCenterCodeAndCardTypeInfo(ciTasks, beginTime, endTime) for centerCode, CardTypeInfo := range cloudBrainCenterCodeAndCardTypeInfo { for cardType, cardDuration := range CardTypeInfo { From ff4873129000f36993fe1a68a4ff8418cdccb0f3 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Mon, 24 Oct 2022 16:31:07 +0800 Subject: [PATCH 32/82] fix issue --- templates/repo/modelarts/notebook/new.tmpl | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/templates/repo/modelarts/notebook/new.tmpl b/templates/repo/modelarts/notebook/new.tmpl index f9c4670a5..0ada241bc 100755 --- a/templates/repo/modelarts/notebook/new.tmpl +++ b/templates/repo/modelarts/notebook/new.tmpl @@ -179,17 +179,7 @@ } } - $('select.dropdown') - .dropdown(); - $(function() { - $("#cloudbrain_job_type").change(function() { - if ($(this).val() == 'BENCHMARK') { - $(".cloudbrain_benchmark").show(); - } else { - $(".cloudbrain_benchmark").hide(); - } - }) - }) + $(document).ready(function(){ $(document).keydown(function(event){ if(event.keyCode==13){ @@ -209,4 +199,5 @@ shared_memory: {{$.i18n.Tr "cloudbrain.shared_memory"}}, }); })(); + console.log("-------------:",{{.NotStopTaskCount}}) From b306e35a31ddb8b45d5aac47c96eac8e30829b91 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Mon, 24 Oct 2022 17:38:51 +0800 Subject: [PATCH 33/82] fix issue --- templates/custom/alert_cb.tmpl | 12 ++++++++++++ templates/repo/cloudbrain/new.tmpl | 1 + templates/repo/modelarts/notebook/new.tmpl | 14 +++++++++++++- web_src/js/features/cloudrbanin.js | 3 +++ web_src/less/_form.less | 15 +++++++++++---- 5 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 templates/custom/alert_cb.tmpl diff --git a/templates/custom/alert_cb.tmpl b/templates/custom/alert_cb.tmpl new file mode 100644 index 000000000..1e3ed5873 --- /dev/null +++ b/templates/custom/alert_cb.tmpl @@ -0,0 +1,12 @@ +{{if .NotStopTaskCount}} +
+ +
+ +
+
您已经有 同类任务 正在等待或运行中，请等待任务结束再创建
+
可以在 “个人中心 > 云脑任务” 查看您所有的云脑任务
+
+
+
+{{end}} \ No newline at end of file diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index fb7ccbed1..48cadf278 100755 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -28,6 +28,7 @@ + {{template "custom/alert_cb" .}}
{{.CsrfTokenHtml}} diff --git a/templates/repo/modelarts/notebook/new.tmpl b/templates/repo/modelarts/notebook/new.tmpl index 0ada241bc..4eaaf3cd4 100755 --- a/templates/repo/modelarts/notebook/new.tmpl +++ b/templates/repo/modelarts/notebook/new.tmpl @@ -15,6 +15,8 @@

+ + {{template "custom/alert_cb" .}} {{.CsrfTokenHtml}}

@@ -178,8 +180,18 @@ document.getElementById("mask").style.display = "none" } } + $('select.dropdown') + .dropdown(); - + $(function() { + $("#cloudbrain_job_type").change(function() { + if ($(this).val() == 'BENCHMARK') { + $(".cloudbrain_benchmark").show(); + } else { + $(".cloudbrain_benchmark").hide(); + } + }) + }) $(document).ready(function(){ $(document).keydown(function(event){ if(event.keyCode==13){ diff --git a/web_src/js/features/cloudrbanin.js b/web_src/js/features/cloudrbanin.js index 698523d11..657ca1381 100644 --- a/web_src/js/features/cloudrbanin.js +++ b/web_src/js/features/cloudrbanin.js @@ -575,3 +575,6 @@ function AdaminSearchControll() { } userSearchControll(); AdaminSearchControll(); +$(".message .close").on("click", function () { + $(this).closest(".message").transition("fade"); +}); diff --git a/web_src/less/_form.less b/web_src/less/_form.less index e41c428c8..d481d1cee 100644 --- a/web_src/less/_form.less +++ b/web_src/less/_form.less @@ -1,8 +1,8 @@ .form { .help { color: #999999; - padding-top: .6em; - + padding-top: 0.6em; + display: inline-block; } } @@ -109,7 +109,7 @@ @media screen and (max-height: 575px) { #rc-imageselect, .g-recaptcha { - transform: scale(.77); + transform: scale(0.77); transform-origin: 0 0; } } @@ -141,7 +141,7 @@ } } - input[type=number] { + input[type="number"] { -moz-appearance: textfield; } @@ -157,6 +157,13 @@ &.new.repo, &.new.migrate, &.new.fork { + .ui.message { + @media only screen and (min-width: 768px) { + width: 800px !important; + } + margin: 0 auto; + margin-bottom: 1rem; + } #create-page-form; form { From 9c0acb703172c378526f20f72c6082f9ff734c93 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Mon, 24 Oct 2022 18:15:13 +0800 Subject: [PATCH 34/82] fix issue --- templates/repo/cloudbrain/benchmark/new.tmpl | 1 + templates/repo/cloudbrain/inference/new.tmpl | 1 + templates/repo/cloudbrain/trainjob/new.tmpl | 1 + templates/repo/modelarts/inferencejob/new.tmpl | 1 + templates/repo/modelarts/trainjob/new.tmpl | 1 + 5 files changed, 5 insertions(+) diff --git a/templates/repo/cloudbrain/benchmark/new.tmpl b/templates/repo/cloudbrain/benchmark/new.tmpl index 13665c036..d337db460 100755 --- a/templates/repo/cloudbrain/benchmark/new.tmpl +++ b/templates/repo/cloudbrain/benchmark/new.tmpl @@ -33,6 +33,7 @@ {{template "repo/header" .}}
{{template "base/alert" .}} + {{template "custom/alert_cb" .}}

{{.i18n.Tr "repo.modelarts.evaluate_job.new_job"}}

diff --git a/templates/repo/cloudbrain/inference/new.tmpl b/templates/repo/cloudbrain/inference/new.tmpl index 630df7a2e..0d5a5005a 100644 --- a/templates/repo/cloudbrain/inference/new.tmpl +++ b/templates/repo/cloudbrain/inference/new.tmpl @@ -41,6 +41,7 @@ {{template "repo/header" .}}
{{template "base/alert" .}} + {{template "custom/alert_cb" .}}

{{.i18n.Tr "repo.modelarts.train_job.new_infer"}} diff --git a/templates/repo/cloudbrain/trainjob/new.tmpl b/templates/repo/cloudbrain/trainjob/new.tmpl index 607af5f07..58ec9dd36 100755 --- a/templates/repo/cloudbrain/trainjob/new.tmpl +++ b/templates/repo/cloudbrain/trainjob/new.tmpl @@ -72,6 +72,7 @@
{{template "base/alert" .}} + {{template "custom/alert_cb" .}}

{{.i18n.Tr "repo.modelarts.train_job.new"}}

diff --git a/templates/repo/modelarts/inferencejob/new.tmpl b/templates/repo/modelarts/inferencejob/new.tmpl index d2f6a8194..2ffb16c57 100644 --- a/templates/repo/modelarts/inferencejob/new.tmpl +++ b/templates/repo/modelarts/inferencejob/new.tmpl @@ -40,6 +40,7 @@ {{template "repo/header" .}}
{{template "base/alert" .}} + {{template "custom/alert_cb" .}}

{{.i18n.Tr "repo.modelarts.train_job.new_infer"}} diff --git a/templates/repo/modelarts/trainjob/new.tmpl b/templates/repo/modelarts/trainjob/new.tmpl index 0e1e8eedb..29d91e633 100755 --- a/templates/repo/modelarts/trainjob/new.tmpl +++ b/templates/repo/modelarts/trainjob/new.tmpl @@ -64,6 +64,7 @@
{{template "base/alert" .}} + {{template "custom/alert_cb" .}}

{{.i18n.Tr "repo.modelarts.train_job.new"}}

From be3810d608cfb7139f1aeb6eb0dc94f190d4b6b4 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Tue, 25 Oct 2022 09:09:34 +0800 Subject: [PATCH 35/82] fix issue --- templates/custom/alert_cb.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/custom/alert_cb.tmpl b/templates/custom/alert_cb.tmpl index 1e3ed5873..3ea03b8e5 100644 --- a/templates/custom/alert_cb.tmpl +++ b/templates/custom/alert_cb.tmpl @@ -5,7 +5,7 @@
您已经有 同类任务 正在等待或运行中，请等待任务结束再创建
-
可以在 “个人中心 > 云脑任务” 查看您所有的云脑任务
+
可以在 “个人中心 > 云脑任务” 查看您所有的云脑任务

From 91e93b293b9daf6cb2d3fe65372e0774c244cff5 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Tue, 25 Oct 2022 09:16:00 +0800 Subject: [PATCH 36/82] fix issue --- templates/custom/alert_cb.tmpl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/templates/custom/alert_cb.tmpl b/templates/custom/alert_cb.tmpl index 3ea03b8e5..7961c2b16 100644 --- a/templates/custom/alert_cb.tmpl +++ b/templates/custom/alert_cb.tmpl @@ -9,4 +9,7 @@

-{{end}} \ No newline at end of file +{{end}} + \ No newline at end of file From e0a6f88c04aa2c3bb4068b417d31e2c05c4bb2c6 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Tue, 25 Oct 2022 09:19:59 +0800 Subject: [PATCH 37/82] fix issue --- templates/custom/alert_cb.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/custom/alert_cb.tmpl b/templates/custom/alert_cb.tmpl index 7961c2b16..f8e46486a 100644 --- a/templates/custom/alert_cb.tmpl +++ b/templates/custom/alert_cb.tmpl @@ -11,5 +11,5 @@ {{end}} \ No newline at end of file From ea7d8ce9b743aa59cf8d8bafa93647c81571149c Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Tue, 25 Oct 2022 10:00:54 +0800 Subject: [PATCH 38/82] fix-2982 issue --- templates/custom/alert_cb.tmpl | 5 +---- templates/repo/grampus/trainjob/gpu/new.tmpl | 1 + templates/repo/grampus/trainjob/npu/new.tmpl | 1 + templates/repo/modelsafety/new.tmpl | 1 + 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/custom/alert_cb.tmpl b/templates/custom/alert_cb.tmpl index f8e46486a..3ea03b8e5 100644 --- a/templates/custom/alert_cb.tmpl +++ b/templates/custom/alert_cb.tmpl @@ -9,7 +9,4 @@ -{{end}} - \ No newline at end of file +{{end}} \ No newline at end of file diff --git a/templates/repo/grampus/trainjob/gpu/new.tmpl b/templates/repo/grampus/trainjob/gpu/new.tmpl index e97ccfe59..439991813 100755 --- a/templates/repo/grampus/trainjob/gpu/new.tmpl +++ b/templates/repo/grampus/trainjob/gpu/new.tmpl @@ -64,6 +64,7 @@
{{template "base/alert" .}} + {{template "custom/alert_cb" .}}

{{.i18n.Tr "repo.modelarts.train_job.new"}}

diff --git a/templates/repo/grampus/trainjob/npu/new.tmpl b/templates/repo/grampus/trainjob/npu/new.tmpl index 51a561d3d..4b5666c37 100755 --- a/templates/repo/grampus/trainjob/npu/new.tmpl +++ b/templates/repo/grampus/trainjob/npu/new.tmpl @@ -59,6 +59,7 @@
{{template "base/alert" .}} + {{template "custom/alert_cb" .}}

{{.i18n.Tr "repo.modelarts.train_job.new"}}

diff --git a/templates/repo/modelsafety/new.tmpl b/templates/repo/modelsafety/new.tmpl index 07bcd311d..9383c2f2a 100644 --- a/templates/repo/modelsafety/new.tmpl +++ b/templates/repo/modelsafety/new.tmpl @@ -56,6 +56,7 @@ {{$Grampus := (or (eq (index (SubJumpablePath .Link) 1) "create_grampus_gpu") (eq (index (SubJumpablePath .Link) 1) "create_grampus_npu"))}} {{template "base/alert" .}} + {{template "custom/alert_cb" .}}

{{.i18n.Tr "repo.modelarts.evaluate_job.new_job"}}

From 1567e9cff0640e9b3b918e18e82b1cf2b8242a29 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Tue, 25 Oct 2022 10:54:32 +0800 Subject: [PATCH 39/82] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/cloudbrain/cloudbrainTask/count.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/cloudbrain/cloudbrainTask/count.go b/services/cloudbrain/cloudbrainTask/count.go index 25d56b0b6..c601edca6 100644 --- a/services/cloudbrain/cloudbrainTask/count.go +++ b/services/cloudbrain/cloudbrainTask/count.go @@ -49,7 +49,7 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s ComputeResource: models.NPUResource, }, string(models.JobTypeInference) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { CloudBrainTypes: []int{models.TypeCloudBrainTwo}, - JobType: []models.JobType{models.JobTypeTrain}, + JobType: []models.JobType{models.JobTypeInference}, NotFinalStatuses: cloudbrainTwoNotFinalStatuses, ComputeResource: models.NPUResource, }, string(models.JobTypeTrain) + "-" + strconv.Itoa(models.TypeC2Net) + "-" + models.GPUResource: { From 674f4701c998cbd6f81e31d28c9f4a424f9fd31a Mon Sep 17 00:00:00 2001 From: Gitea Date: Tue, 25 Oct 2022 10:57:32 +0800 Subject: [PATCH 40/82] add convert --- templates/repo/modelmanage/convertIndex.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index 26e79b04c..1802b3fdd 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -532,7 +532,7 @@ } } function isModel(filename){ - var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel","pdparams",".params",".json"]; + var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel","pdiparams",".params",".json"]; for(var i =0; i Date: Tue, 25 Oct 2022 11:04:55 +0800 Subject: [PATCH 41/82] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/cloudbrain.go | 2 +- services/cloudbrain/cloudbrainTask/count.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index f0e27995e..1eaeffcd3 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2024,7 +2024,7 @@ func GetCloudbrainRunCountByRepoID(repoID int64) (int, error) { } func GetModelSafetyCountByUserID(userID int64) (int, error) { - count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ?", string(JobTypeModelSafety), userID).Count(new(Cloudbrain)) + count, err := x.In("status", JobWaiting, JobRunning,ModelArtsTrainJobInit,ModelArtsTrainJobImageCreating,ModelArtsTrainJobSubmitTrying,ModelArtsTrainJobScaling,ModelArtsTrainJobCheckInit,ModelArtsTrainJobCheckRunning,ModelArtsTrainJobCheckRunningCompleted).And("job_type = ? and user_id = ?", string(JobTypeModelSafety), userID).Count(new(Cloudbrain)) return int(count), err } diff --git a/services/cloudbrain/cloudbrainTask/count.go b/services/cloudbrain/cloudbrainTask/count.go index c601edca6..a9b254618 100644 --- a/services/cloudbrain/cloudbrainTask/count.go +++ b/services/cloudbrain/cloudbrainTask/count.go @@ -34,7 +34,7 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s ComputeResource: models.GPUResource, }, string(models.JobTypeBenchmark) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { CloudBrainTypes: []int{models.TypeCloudBrainOne}, - JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeModelSafety, models.JobTypeBrainScore, models.JobTypeSnn4imagenet}, + JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet}, NotFinalStatuses: cloudbrainOneNotFinalStatuses, ComputeResource: models.GPUResource, }, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { From 9f7b111750b7652ddeeafe086852f806f3e21676 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Tue, 25 Oct 2022 15:19:53 +0800 Subject: [PATCH 42/82] =?UTF-8?q?=E8=B5=84=E6=BA=90=E8=AF=B4=E6=98=8E?= =?UTF-8?q?=E6=89=80=E6=9C=89=E5=85=A5=E5=8F=A3=E6=B7=BB=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- options/locale/locale_en-US.ini | 2 +- options/locale/locale_zh-CN.ini | 1 + templates/base/footer_content.tmpl | 2 +- templates/base/footer_content_fluid.tmpl | 1 + templates/repo/cloudbrain/benchmark/new.tmpl | 6 ++++++ templates/repo/cloudbrain/inference/new.tmpl | 2 ++ templates/repo/cloudbrain/new.tmpl | 2 ++ templates/repo/cloudbrain/trainjob/new.tmpl | 2 ++ templates/repo/grampus/trainjob/gpu/new.tmpl | 2 ++ templates/repo/grampus/trainjob/npu/new.tmpl | 2 ++ templates/repo/modelarts/inferencejob/new.tmpl | 2 ++ templates/repo/modelarts/notebook/new.tmpl | 2 ++ templates/repo/modelarts/trainjob/new.tmpl | 2 ++ templates/repo/modelarts/trainjob/version_new.tmpl | 2 ++ templates/repo/modelsafety/new.tmpl | 2 ++ 15 files changed, 30 insertions(+), 2 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 773a338c1..6aca6bd20 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3177,7 +3177,7 @@ foot.help = help foot.copyright= Copyright: New Generation Artificial Intelligence Open Source Open Platform (OpenI) Platform_Tutorial = Tutorial foot.advice_feedback = Feedback - +resource_description = Resource Note [cloudbrain] all_resource_cluster=All Cluster all_ai_center=All Computing NET diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 8ba4d252d..9a99fa5b5 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3194,6 +3194,7 @@ foot.help=帮助 foot.copyright= 版权所有：新一代人工智能开源开放平台（OpenI） Platform_Tutorial=新手指引 foot.advice_feedback = 意见反馈 +resource_description = 资源说明 [cloudbrain] all_resource_cluster=全部集群 diff --git a/templates/base/footer_content.tmpl b/templates/base/footer_content.tmpl index ce4ea5cc6..8786af653 100755 --- a/templates/base/footer_content.tmpl +++ b/templates/base/footer_content.tmpl @@ -36,7 +36,7 @@ {{else}} {{.i18n.Tr "custom.foot.advice_feedback"}} {{end}} - {{.i18n.Tr "custom.Platform_Tutorial"}} + {{.i18n.Tr "custom.resource_description"}} {{template "custom/extra_links_footer" .}}
diff --git a/templates/base/footer_content_fluid.tmpl b/templates/base/footer_content_fluid.tmpl index 24b18e94d..723c78045 100755 --- a/templates/base/footer_content_fluid.tmpl +++ b/templates/base/footer_content_fluid.tmpl @@ -33,6 +33,7 @@ {{else}} {{.i18n.Tr "custom.foot.advice_feedback"}} {{end}} + {{.i18n.Tr "custom.resource_description"}} {{template "custom/extra_links_footer" .}}
diff --git a/templates/repo/cloudbrain/benchmark/new.tmpl b/templates/repo/cloudbrain/benchmark/new.tmpl index 13665c036..23ac60427 100755 --- a/templates/repo/cloudbrain/benchmark/new.tmpl +++ b/templates/repo/cloudbrain/benchmark/new.tmpl @@ -136,6 +136,8 @@ {{if .CloudBrainPaySwitch}}blance="{{.PointAccount.Balance}}"{{end}} name="spec_id"> + + {{.i18n.Tr "custom.resource_description"}} {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} @@ -248,6 +250,8 @@ {{if .CloudBrainPaySwitch}}blance="{{.PointAccount.Balance}}"{{end}} name="spec_id"> + + {{.i18n.Tr "custom.resource_description"}} {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} @@ -334,6 +338,8 @@ placeholder="{{.i18n.Tr "cloudbrain.select_specification"}}" style='width:385px' ovalue="{{.spec_id}}" name="spec_id"> + + {{.i18n.Tr "custom.resource_description"}}
diff --git a/templates/repo/cloudbrain/inference/new.tmpl b/templates/repo/cloudbrain/inference/new.tmpl index 630df7a2e..fc57e3a6b 100644 --- a/templates/repo/cloudbrain/inference/new.tmpl +++ b/templates/repo/cloudbrain/inference/new.tmpl @@ -256,6 +256,8 @@ + + {{.i18n.Tr "custom.resource_description"}} {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index fb7ccbed1..0c545c7c1 100755 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -149,6 +149,8 @@ {{if .CloudBrainPaySwitch}}blance="{{.PointAccount.Balance}}"{{end}} name="spec_id"> + + {{.i18n.Tr "custom.resource_description"}} {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} diff --git a/templates/repo/cloudbrain/trainjob/new.tmpl b/templates/repo/cloudbrain/trainjob/new.tmpl index 607af5f07..9c778f3f8 100755 --- a/templates/repo/cloudbrain/trainjob/new.tmpl +++ b/templates/repo/cloudbrain/trainjob/new.tmpl @@ -216,6 +216,8 @@ {{if .CloudBrainPaySwitch}}blance="{{.PointAccount.Balance}}"{{end}} name="spec_id"> + + {{.i18n.Tr "custom.resource_description"}} {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} diff --git a/templates/repo/grampus/trainjob/gpu/new.tmpl b/templates/repo/grampus/trainjob/gpu/new.tmpl index e97ccfe59..97a7cedcf 100755 --- a/templates/repo/grampus/trainjob/gpu/new.tmpl +++ b/templates/repo/grampus/trainjob/gpu/new.tmpl @@ -189,6 +189,8 @@
+ + {{.i18n.Tr "custom.resource_description"}} {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} diff --git a/templates/repo/grampus/trainjob/npu/new.tmpl b/templates/repo/grampus/trainjob/npu/new.tmpl index 51a561d3d..3528cc661 100755 --- a/templates/repo/grampus/trainjob/npu/new.tmpl +++ b/templates/repo/grampus/trainjob/npu/new.tmpl @@ -199,6 +199,8 @@
+ + {{.i18n.Tr "custom.resource_description"}} {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} diff --git a/templates/repo/modelarts/inferencejob/new.tmpl b/templates/repo/modelarts/inferencejob/new.tmpl index d2f6a8194..855f5b1b6 100644 --- a/templates/repo/modelarts/inferencejob/new.tmpl +++ b/templates/repo/modelarts/inferencejob/new.tmpl @@ -276,6 +276,8 @@
+ + {{.i18n.Tr "custom.resource_description"}} {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} diff --git a/templates/repo/modelarts/notebook/new.tmpl b/templates/repo/modelarts/notebook/new.tmpl index f9c4670a5..da29a4e46 100755 --- a/templates/repo/modelarts/notebook/new.tmpl +++ b/templates/repo/modelarts/notebook/new.tmpl @@ -82,6 +82,8 @@ + + {{.i18n.Tr "custom.resource_description"}} {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} diff --git a/templates/repo/modelarts/trainjob/new.tmpl b/templates/repo/modelarts/trainjob/new.tmpl index 0e1e8eedb..1250b8805 100755 --- a/templates/repo/modelarts/trainjob/new.tmpl +++ b/templates/repo/modelarts/trainjob/new.tmpl @@ -247,6 +247,8 @@
+ + {{.i18n.Tr "custom.resource_description"}} {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} diff --git a/templates/repo/modelarts/trainjob/version_new.tmpl b/templates/repo/modelarts/trainjob/version_new.tmpl index 89bf6fc08..e1107ce84 100644 --- a/templates/repo/modelarts/trainjob/version_new.tmpl +++ b/templates/repo/modelarts/trainjob/version_new.tmpl @@ -236,6 +236,8 @@
+ + {{.i18n.Tr "custom.resource_description"}} {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} diff --git a/templates/repo/modelsafety/new.tmpl b/templates/repo/modelsafety/new.tmpl index 07bcd311d..1b97207b7 100644 --- a/templates/repo/modelsafety/new.tmpl +++ b/templates/repo/modelsafety/new.tmpl @@ -299,6 +299,8 @@ {{if .CloudBrainPaySwitch}}blance="{{.PointAccount.Balance}}"{{end}} name="spec_id"> + + {{.i18n.Tr "custom.resource_description"}} {{if .CloudBrainPaySwitch}}
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} From f81aefdec22014e3a065b423e69591d6df5b4df0 Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 25 Oct 2022 15:22:04 +0800 Subject: [PATCH 43/82] update --- routers/api/v1/repo/cloudbrain_dashboard.go | 3 +++ routers/routes/routes.go | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 6da731f5e..9d410f1c2 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1553,6 +1553,9 @@ func GetDurationRateStatistic(ctx *context.Context) { func CloudbrainDurationStatisticForTest(ctx *context.Context) { repo.CloudbrainDurationStatisticHour() + ctx.JSON(http.StatusOK, map[string]interface{}{ + "message": 0, + }) } func getBeginAndEndTime(ctx *context.Context) (time.Time, time.Time) { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 899918339..4fe7c6622 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -389,7 +389,6 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/^:type(issues|pulls)$", reqSignIn, user.Issues) m.Get("/milestones", reqSignIn, reqMilestonesDashboardPageEnabled, user.Milestones) m.Get("/cloudbrains", reqSignIn, user.Cloudbrains) - m.Get("/duration", repo.CloudbrainDurationStatisticHour) // ***** START: User ***** m.Group("/user", func() { From 31261936608e110d64347bd55b6377277ebd447e Mon Sep 17 00:00:00 2001 From: Gitea Date: Tue, 25 Oct 2022 16:11:34 +0800 Subject: [PATCH 44/82] modified --- options/locale/locale_en-US.ini | 2 +- options/locale/locale_zh-CN.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 773a338c1..ecd2bb038 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3216,7 +3216,7 @@ view_sample = View sample inference_output_path_rule = The inference output path is stored in the run parameter result_url. model_file_path_rule=The model file location is stored in the run parameter ckpt_url model_file_postfix_rule = The supported format of the model file is [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt] -model_convert_postfix_rule = The supported format of the model file is [.pth, .pkl, .onnx, .mindir, .ckpt, .pb] +model_convert_postfix_rule = The supported format of the model file is [.pth, .pkl, .onnx, .mindir, .ckpt, .pb, .pdmodel, .pdiparams, .params, .json] delete_task = Delete task task_delete_confirm = Are you sure you want to delete this task? Once this task is deleted, it cannot be recovered. operate_confirm = confirm diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 8ba4d252d..74e46c6ea 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -3234,7 +3234,7 @@ view_sample = 查看样例 inference_output_path_rule = 推理输出路径存储在运行参数 result_url 中。 model_file_path_rule = 模型文件位置存储在运行参数 ckpt_url 中。 model_file_postfix_rule = 模型文件支持的格式为 [ckpt, pb, h5, json, pkl, pth, t7, pdparams, onnx, pbtxt, keras, mlmodel, cfg, pt] -model_convert_postfix_rule = 模型文件支持的格式为 [.pth, .pkl, .onnx, .mindir, .ckpt, .pb] +model_convert_postfix_rule = 模型文件支持的格式为 [.pth, .pkl, .onnx, .mindir, .ckpt, .pb, .pdmodel, .pdiparams, .params, .json] delete_task = 删除任务 task_delete_confirm = 你确认删除该任务么？此任务一旦删除不可恢复。 operate_confirm = 确定操作 From 3dc3273f53afd2b6c03a7adebd60477ea73a3035 Mon Sep 17 00:00:00 2001 From: Gitea Date: Tue, 25 Oct 2022 16:16:54 +0800 Subject: [PATCH 45/82] modified --- templates/repo/modelmanage/convertIndex.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/modelmanage/convertIndex.tmpl b/templates/repo/modelmanage/convertIndex.tmpl index 1802b3fdd..05b5306c8 100644 --- a/templates/repo/modelmanage/convertIndex.tmpl +++ b/templates/repo/modelmanage/convertIndex.tmpl @@ -532,7 +532,7 @@ } } function isModel(filename){ - var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel","pdiparams",".params",".json"]; + var postfix=[".pth",".pkl",".onnx",".mindir",".ckpt",".pb",".pdmodel",".pdiparams",".params",".json"]; for(var i =0; i Date: Wed, 26 Oct 2022 09:20:18 +0800 Subject: [PATCH 46/82] =?UTF-8?q?#3063,=E9=A6=96=E9=A1=B5=E6=96=87?= =?UTF-8?q?=E5=AD=97=E6=8F=8F=E8=BF=B0=E6=9B=B4=E6=94=B9=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- options/locale/locale_en-US.ini | 2 +- options/locale/locale_zh-CN.ini | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 081db13ab..75314f6ec 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -266,7 +266,7 @@ page_dev_yunlao_desc4=Developers can freely select the corresponding computing r page_dev_yunlao_desc5=If your model requires more computing resources, you can also apply for it separately. page_dev_yunlao_apply=Apply Separately c2net_title=China Computing Network -c2net_desc=The artificial intelligence computing power network promotion alliance has access to 11 intelligent computing centers, with a total scale of 1924p. +c2net_desc=Extensive access to intelligent computing centers and supercomputing centers across the country to provide users with free computing resources. c2net_center=Center search=Search search_repo=Repository diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index ae8b6adc4..a6a841b76 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -267,8 +267,8 @@ page_dev_yunlao_desc3=中国算力网（C²NET）一期可实现不同人工智 page_dev_yunlao_desc4=开发者可以根据使用需求，自由选择相应计算资源，可以测试模型在不同硬件环境下的适配能力、性能、稳定性等。 page_dev_yunlao_desc5=如果您的模型需要更多的计算资源，也可以单独申请。 page_dev_yunlao_apply=单独申请 -c2net_title=智算网络 -c2net_desc=人工智能算力网络推进联盟已接入11家智算中心，算力总规模1924P +c2net_title=中国算力网 +c2net_desc=广泛接入全国各地智算中心、超算中心，为用户提供免费算力资源 c2net_center=中心 search=搜索 search_repo=项目 From 37b3cfebe1bee747ad3b09a653894fe63135e06c Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 26 Oct 2022 15:41:01 +0800 Subject: [PATCH 47/82] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E5=88=B7=E6=96=B0GPU=E8=A7=84=E6=A0=BC=E6=98=BE?= =?UTF-8?q?=E7=A4=BA=E6=A0=BC=E5=BC=8F=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 5b358b83b..b0bc0a62c 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/services/cloudbrain/resource" uuid "github.com/satori/go.uuid" ) @@ -69,14 +70,16 @@ func saveModelByParameters(jobId string, versionName string, name string, versio cloudType = models.TypeCloudBrainTwo } else if aiTask.ComputeResource == models.GPUResource { cloudType = models.TypeCloudBrainOne - var ResourceSpecs *models.ResourceSpecs - json.Unmarshal([]byte(setting.ResourceSpecs), &ResourceSpecs) - for _, tmp := range ResourceSpecs.ResourceSpec { - if tmp.Id == aiTask.ResourceSpecId { - flaverName := ctx.Tr("cloudbrain.gpu_num") + ": " + fmt.Sprint(tmp.GpuNum) + " " + ctx.Tr("cloudbrain.cpu_num") + ": " + fmt.Sprint(tmp.CpuNum) + " " + ctx.Tr("cloudbrain.memory") + "(MB): " + fmt.Sprint(tmp.MemMiB) + " " + ctx.Tr("cloudbrain.shared_memory") + "(MB): " + fmt.Sprint(tmp.ShareMemMiB) - aiTask.FlavorName = flaverName - } + spec, err := resource.GetAndCheckSpec(ctx.User.ID, int64(aiTask.ResourceSpecId), models.FindSpecsOptions{ + JobType: models.JobType(aiTask.JobType), + ComputeResource: models.GPU, + Cluster: models.OpenICluster, + AiCenterCode: models.AICenterOfCloudBrainOne}) + if err == nil { + flaverName := "GPU: " + fmt.Sprint(spec.AccCardsNum) + "*" + spec.AccCardType + ",CPU: " + fmt.Sprint(spec.CpuCores) + "," + ctx.Tr("cloudbrain.memory") + ": " + fmt.Sprint(spec.MemGiB) + "GB," + ctx.Tr("cloudbrain.shared_memory") + ": " + fmt.Sprint(spec.ShareMemGiB) + "GB" + aiTask.FlavorName = flaverName } + } accuracy := make(map[string]string) From e1fe38c0a3177dc2a60ab76ac37e63606518339b Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 26 Oct 2022 16:07:00 +0800 Subject: [PATCH 48/82] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=A7=84=E6=A0=BC?= =?UTF-8?q?=E6=98=BE=E7=A4=BA=E4=BF=A1=E6=81=AF=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/ai_model_manage.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index b0bc0a62c..f2b0fc6d1 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -70,16 +70,11 @@ func saveModelByParameters(jobId string, versionName string, name string, versio cloudType = models.TypeCloudBrainTwo } else if aiTask.ComputeResource == models.GPUResource { cloudType = models.TypeCloudBrainOne - spec, err := resource.GetAndCheckSpec(ctx.User.ID, int64(aiTask.ResourceSpecId), models.FindSpecsOptions{ - JobType: models.JobType(aiTask.JobType), - ComputeResource: models.GPU, - Cluster: models.OpenICluster, - AiCenterCode: models.AICenterOfCloudBrainOne}) + spec, err := resource.GetCloudbrainSpec(aiTask.ID) if err == nil { flaverName := "GPU: " + fmt.Sprint(spec.AccCardsNum) + "*" + spec.AccCardType + ",CPU: " + fmt.Sprint(spec.CpuCores) + "," + ctx.Tr("cloudbrain.memory") + ": " + fmt.Sprint(spec.MemGiB) + "GB," + ctx.Tr("cloudbrain.shared_memory") + ": " + fmt.Sprint(spec.ShareMemGiB) + "GB" aiTask.FlavorName = flaverName } - } accuracy := make(map[string]string) From 97a710c4e9e6f4f134a177e472be84694a3b73e6 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 27 Oct 2022 08:51:38 +0800 Subject: [PATCH 49/82] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E4=BF=AE=E6=94=B9=E4=B8=80=E4=B8=8B=E6=9C=80=E7=BB=88?= =?UTF-8?q?=E7=BB=93=E6=9D=9F=E6=97=B6=E9=97=B4=E5=80=BC=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/aisafety.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/routers/repo/aisafety.go b/routers/repo/aisafety.go index 554147d51..21a3b4d95 100644 --- a/routers/repo/aisafety.go +++ b/routers/repo/aisafety.go @@ -307,6 +307,9 @@ func queryTaskStatusFromModelSafetyTestServer(job *models.Cloudbrain) { if result.Data.Status == 1 { log.Info("The task is running....") } else { + job.EndTime = timeutil.TimeStampNow() + job.Duration = (int64(job.EndTime) - int64(job.StartTime)) / 1000 + job.TrainJobDuration = models.ConvertDurationToStr(job.Duration) if result.Data.Code == 0 { job.ResultJson = result.Data.StandardJson job.Status = string(models.JobSucceeded) From 50726948898a1c9d9ba5b9d03ebfee004a2ede6e Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 27 Oct 2022 08:56:47 +0800 Subject: [PATCH 50/82] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E6=9B=B4=E6=96=B0=E6=9C=80=E5=90=8E=E7=BB=93=E6=9D=9F?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/aisafety.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/routers/repo/aisafety.go b/routers/repo/aisafety.go index 21a3b4d95..650848d11 100644 --- a/routers/repo/aisafety.go +++ b/routers/repo/aisafety.go @@ -291,6 +291,7 @@ func queryTaskStatusFromCloudbrain(job *models.Cloudbrain) { } else { // job.Status = string(models.ModelSafetyTesting) + job.EndTime = 0 err = models.UpdateJob(job) if err != nil { log.Error("UpdateJob failed:", err) @@ -443,6 +444,9 @@ func updateJobFailed(job *models.Cloudbrain, msg string) { //update task failed. job.Status = string(models.ModelArtsTrainJobFailed) job.ResultJson = msg + job.EndTime = timeutil.TimeStampNow() + job.Duration = (int64(job.EndTime) - int64(job.StartTime)) / 1000 + job.TrainJobDuration = models.ConvertDurationToStr(job.Duration) err := models.UpdateJob(job) if err != nil { log.Error("UpdateJob failed:", err) From 5cdfc4c0a09d1b9cee259759082e76b1431ed9f4 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 27 Oct 2022 09:05:09 +0800 Subject: [PATCH 51/82] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E4=BF=AE=E6=AD=A3=E6=9C=80=E5=90=8E=E6=9B=B4=E6=96=B0?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- routers/repo/aisafety.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/repo/aisafety.go b/routers/repo/aisafety.go index 650848d11..e274f808e 100644 --- a/routers/repo/aisafety.go +++ b/routers/repo/aisafety.go @@ -309,7 +309,7 @@ func queryTaskStatusFromModelSafetyTestServer(job *models.Cloudbrain) { log.Info("The task is running....") } else { job.EndTime = timeutil.TimeStampNow() - job.Duration = (int64(job.EndTime) - int64(job.StartTime)) / 1000 + job.Duration = (job.EndTime.AsTime().Unix() - job.StartTime.AsTime().Unix()) / 1000 job.TrainJobDuration = models.ConvertDurationToStr(job.Duration) if result.Data.Code == 0 { job.ResultJson = result.Data.StandardJson @@ -445,7 +445,7 @@ func updateJobFailed(job *models.Cloudbrain, msg string) { job.Status = string(models.ModelArtsTrainJobFailed) job.ResultJson = msg job.EndTime = timeutil.TimeStampNow() - job.Duration = (int64(job.EndTime) - int64(job.StartTime)) / 1000 + job.Duration = (job.EndTime.AsTime().Unix() - job.StartTime.AsTime().Unix()) / 1000 job.TrainJobDuration = models.ConvertDurationToStr(job.Duration) err := models.UpdateJob(job) if err != nil { From 416004d08921f625edb398f4d49c224ae2c08504 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 27 Oct 2022 11:02:14 +0800 Subject: [PATCH 52/82] fix-2957 --- modules/setting/setting.go | 5 +++++ routers/repo/grampus.go | 6 ++---- services/cloudbrain/util.go | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 services/cloudbrain/util.go diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 2f468c850..2da970c75 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -597,6 +597,7 @@ var ( }{} C2NetInfos *C2NetSqInfos + C2NetMapInfo map[string]*C2NetSequenceInfo //elk config ElkUrl string @@ -1627,6 +1628,10 @@ func getGrampusConfig() { if err := json.Unmarshal([]byte(Grampus.C2NetSequence), &C2NetInfos); err != nil { log.Error("Unmarshal(C2NetSequence) failed:%v", err) } + C2NetMapInfo=make(map[string]*C2NetSequenceInfo) + for _,value :=range C2NetInfos.C2NetSqInfo{ + C2NetMapInfo[value.Name]=value + } } Grampus.SyncScriptProject = sec.Key("SYNC_SCRIPT_PROJECT").MustString("script_for_grampus") diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index d901298b7..e29bf0abc 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -37,6 +37,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" ) const ( @@ -915,10 +916,7 @@ func GrampusTrainJobShow(ctx *context.Context) { ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task) ctx.Data["displayJobName"] = task.DisplayJobName - aiCenterInfo := strings.Split(task.AiCenter, "+") - if len(aiCenterInfo) == 2 { - ctx.Data["ai_center"] = aiCenterInfo[1] - } + ctx.Data["ai_center"] = cloudbrainService.GetAiCenterShow(task.AiCenter,ctx) ctx.HTML(http.StatusOK, tplGrampusTrainJobShow) } diff --git a/services/cloudbrain/util.go b/services/cloudbrain/util.go new file mode 100644 index 000000000..ab738927e --- /dev/null +++ b/services/cloudbrain/util.go @@ -0,0 +1,33 @@ +package cloudbrain + +import ( + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/setting" + "strings" +) + +func GetAiCenterShow(aiCenter string,ctx *context.Context) string{ + aiCenterInfo := strings.Split(aiCenter, "+") + + if len(aiCenterInfo) == 2{ + if setting.C2NetMapInfo!=nil { + if info,ok:=setting.C2NetMapInfo[aiCenterInfo[0]];ok { + if ctx.Language() == "zh-CN" { + return info.Content + } else { + return info.ContentEN + } + }else{ + return aiCenterInfo[1] + } + + }else{ + return aiCenterInfo[1] + } + + } + + return "" + + +} From e9690883c3e05fcd1c092f5de655075735453350 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Thu, 27 Oct 2022 11:48:55 +0800 Subject: [PATCH 53/82] fix issue --- templates/repo/modelarts/trainjob/show.tmpl | 53 +++- web_src/js/features/cloudbrainShow.js | 263 +++++++++++++++----- 2 files changed, 254 insertions(+), 62 deletions(-) diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index e95a1233d..462319168 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -548,14 +548,17 @@
-
+
{{$.i18n.Tr "repo.modelarts.download_log"}} - +
+ + 全屏查看 +
@@ -577,7 +580,11 @@
+ + + + 

                             

 
@@ -609,6 +616,41 @@
                 

             

         

+
+        

+            

+                

+                    
日志文件

+                    

+                        

+                            
+                            退出全屏
+                        

+                    

+                

+                

+                    
+                        
+                    
+                    
+                        
+                    
+                    
+                    

+                        

+                            

+                        

+                        

+                    

+                    
+                

+            

+        

         {{end}} {{template "base/paginate" .}}
     

     
@@ -715,12 +757,17 @@
 
         

     

+    
 

 {{template "base/footer" .}}
 
 
 
 
\ No newline at end of file
diff --git a/templates/repo/cloudbrain/benchmark/show.tmpl b/templates/repo/cloudbrain/benchmark/show.tmpl
index 84d16d00c..37dc9c83a 100755
--- a/templates/repo/cloudbrain/benchmark/show.tmpl
+++ b/templates/repo/cloudbrain/benchmark/show.tmpl
@@ -524,14 +524,17 @@
                         

                     

                     

-                        
                          

@@ -549,12 +552,16 @@
                              

                              

-                                 

-                                     

-                                 

-                                 
-                                 
-                                 

+                                

+                                    

+                                

+                                
+                                
+                                
+                                
+                                	                                
+                                
+                                

                              

  
                          

@@ -565,6 +572,7 @@
                 

             

         

+        {{template "custom/max_log" .}}
         {{end}} {{template "base/paginate" .}}
     

     
diff --git a/templates/repo/cloudbrain/inference/show.tmpl b/templates/repo/cloudbrain/inference/show.tmpl
index 546f16848..79312ed1e 100644
--- a/templates/repo/cloudbrain/inference/show.tmpl
+++ b/templates/repo/cloudbrain/inference/show.tmpl
@@ -535,14 +535,17 @@
                     

 
                     

-                        

+                        

                              
                                  
                                  {{$.i18n.Tr "repo.modelarts.download_log"}}
                              
- 
+                             

+                                
+                                全屏查看
+                            

                          

                          

@@ -559,13 +562,17 @@
                                  
                              

                              

-                                 

+                                style="height: 300px !important; overflow: auto;">
+                                

                                      

-                                 

-                                 
-                                 
-                                 

+                                

+                                
+                                
+                                
+                                
+                                	                                
+                                
+                                

                              

  
                          

@@ -589,7 +596,7 @@
                 

             

         

-        
+        {{template "custom/max_log" .}}
         {{end}}
     
 
diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl
index a54989df4..315a84f45 100644
--- a/templates/repo/cloudbrain/trainjob/show.tmpl
+++ b/templates/repo/cloudbrain/trainjob/show.tmpl
@@ -524,14 +524,17 @@
                     
 
                     

-                       

+                       

                             
                                 
                                 {{$.i18n.Tr "repo.modelarts.download_log"}}
                             
-
+                            

+                                
+                                全屏查看
+                            

                         

                         

                             
@@ -552,7 +555,11 @@
                                     

                                 

                                 
-                                
+                                
+                                
+                                
+                                	                                
+                                
                                 

                             

 
@@ -581,6 +588,7 @@
                 

             
         
+        {{template "custom/max_log" .}}
         {{end}} {{template "base/paginate" .}}
     
     
diff --git a/templates/repo/grampus/trainjob/show.tmpl b/templates/repo/grampus/trainjob/show.tmpl
index 67c415488..a22cb1ab7 100755
--- a/templates/repo/grampus/trainjob/show.tmpl
+++ b/templates/repo/grampus/trainjob/show.tmpl
@@ -520,14 +520,17 @@
                         
                     
                     

-                        

+                        

                             
                                 
                                 {{$.i18n.Tr "repo.modelarts.download_log"}}
                             
-
+                            

+                                
+                                全屏查看
+                            

                         

                         

                             
@@ -548,7 +551,11 @@
                                     

                                 

                                 
-                                
+                                
+                                
+                                
+                                                           
+                                
                                 

                             

 
@@ -576,6 +583,7 @@
                 

             
         
+        {{template "custom/max_log" .}}
         {{end}} {{template "base/paginate" .}}
     
     
diff --git a/templates/repo/modelarts/inferencejob/show.tmpl b/templates/repo/modelarts/inferencejob/show.tmpl
index 7d671ae4c..c8ff73b2d 100644
--- a/templates/repo/modelarts/inferencejob/show.tmpl
+++ b/templates/repo/modelarts/inferencejob/show.tmpl
@@ -460,14 +460,17 @@ td, th {
                         
 
                         

-                            

+                            

                                 
                                     
                                     {{$.i18n.Tr "repo.modelarts.download_log"}}
                                 
-
+                                

+                                    
+                                    全屏查看
+                                

                             

                             

                                     
@@ -486,8 +489,12 @@ td, th {
                                     

                                         

                                     

-                                    
-                                    
+                                    
+                                    
+                                    
+                                    
+                                    	                                
+                                    
                                     

                                 

 
@@ -510,12 +517,8 @@ td, th {
                 

             

         
+        {{template "custom/max_log" .}}
         {{end}}
-
-
-
-
-
     
     
 
diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl
index 462319168..e86848cc9 100755
--- a/templates/repo/modelarts/trainjob/show.tmpl
+++ b/templates/repo/modelarts/trainjob/show.tmpl
@@ -616,41 +616,7 @@
                 
             
         
-
-        

-            

-                

-                    
日志文件

-                    

-                        

-                            
-                            退出全屏
-                        

-                    

-                

-                

-                    
-                        
-                    
-                    
-                        
-                    
-                    
-                    

-                        

-                            

-                        

-                        

-                    

-                    
-                

-            

-        

+        {{template "custom/max_log" .}}
         {{end}} {{template "base/paginate" .}}
     
     
@@ -764,10 +730,6 @@
 
 
 
\ No newline at end of file

From 3a2888d3ea8e060314be6e1b340bcf844a6f2ce4 Mon Sep 17 00:00:00 2001
From: liuzx 
Date: Thu, 27 Oct 2022 16:36:15 +0800
Subject: [PATCH 67/82] fix-2524

---
 models/cloudbrain.go           | 23 +++++++++++-
 modules/modelarts/modelarts.go | 65 ++++++++++++++++++++++++----------
 modules/modelarts/resty.go     | 60 +++++++++++++++++++++++++++++++
 routers/repo/modelarts.go      | 34 ++++++++++++++++++
 4 files changed, 162 insertions(+), 20 deletions(-)

diff --git a/models/cloudbrain.go b/models/cloudbrain.go
index 1eaeffcd3..d3e93a437 100755
--- a/models/cloudbrain.go
+++ b/models/cloudbrain.go
@@ -1070,6 +1070,12 @@ type CreateInferenceJobParams struct {
 	InfConfig   InfConfig `json:"config"`
 	WorkspaceID string    `json:"workspace_id"`
 }
+type CreateInfUserImageParams struct {
+	JobName     string             `json:"job_name"`
+	Description string             `json:"job_desc"`
+	Config      InfUserImageConfig `json:"config"`
+	WorkspaceID string             `json:"workspace_id"`
+}
 
 type InfConfig struct {
 	WorkServerNum int         `json:"worker_server_num"`
@@ -1084,6 +1090,21 @@ type InfConfig struct {
 	PoolID        string      `json:"pool_id"`
 }
 
+type InfUserImageConfig struct {
+	WorkServerNum int         `json:"worker_server_num"`
+	AppUrl        string      `json:"app_url"`       //训练作业的代码目录
+	BootFileUrl   string      `json:"boot_file_url"` //训练作业的代码启动文件，需要在代码目录下
+	Parameter     []Parameter `json:"parameter"`
+	DataUrl       string      `json:"data_url"` //训练作业需要的数据集OBS路径URL
+	EngineID      int64       `json:"engine_id"`
+	LogUrl        string      `json:"log_url"`
+	CreateVersion bool        `json:"create_version"`
+	Flavor        Flavor      `json:"flavor"`
+	PoolID        string      `json:"pool_id"`
+	UserImageUrl  string      `json:"user_image_url"`
+	UserCommand   string      `json:"user_command"`
+}
+
 type CreateTrainJobVersionParams struct {
 	Description string                `json:"job_desc"`
 	Config      TrainJobVersionConfig `json:"config"`
@@ -2024,7 +2045,7 @@ func GetCloudbrainRunCountByRepoID(repoID int64) (int, error) {
 }
 
 func GetModelSafetyCountByUserID(userID int64) (int, error) {
-	count, err := x.In("status", JobWaiting, JobRunning,ModelArtsTrainJobInit,ModelArtsTrainJobImageCreating,ModelArtsTrainJobSubmitTrying,ModelArtsTrainJobScaling,ModelArtsTrainJobCheckInit,ModelArtsTrainJobCheckRunning,ModelArtsTrainJobCheckRunningCompleted).And("job_type = ? and user_id = ?", string(JobTypeModelSafety), userID).Count(new(Cloudbrain))
+	count, err := x.In("status", JobWaiting, JobRunning, ModelArtsTrainJobInit, ModelArtsTrainJobImageCreating, ModelArtsTrainJobSubmitTrying, ModelArtsTrainJobScaling, ModelArtsTrainJobCheckInit, ModelArtsTrainJobCheckRunning, ModelArtsTrainJobCheckRunningCompleted).And("job_type = ? and user_id = ?", string(JobTypeModelSafety), userID).Count(new(Cloudbrain))
 	return int(count), err
 }
 
diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go
index 06521993e..567f6d620 100755
--- a/modules/modelarts/modelarts.go
+++ b/modules/modelarts/modelarts.go
@@ -143,6 +143,8 @@ type GenerateInferenceJobReq struct {
 	Spec              *models.Specification
 	DatasetName       string
 	JobType           string
+	UserImageUrl      string
+	UserCommand       string
 }
 
 type VersionInfo struct {
@@ -682,26 +684,51 @@ func GetOutputPathByCount(TotalVersionCount int) (VersionOutputPath string) {
 
 func GenerateInferenceJob(ctx *context.Context, req *GenerateInferenceJobReq) (err error) {
 	createTime := timeutil.TimeStampNow()
-	jobResult, err := createInferenceJob(models.CreateInferenceJobParams{
-		JobName:     req.JobName,
-		Description: req.Description,
-		InfConfig: models.InfConfig{
-			WorkServerNum: req.WorkServerNumber,
-			AppUrl:        req.CodeObsPath,
-			BootFileUrl:   req.BootFileUrl,
-			DataUrl:       req.DataUrl,
-			EngineID:      req.EngineID,
-			// TrainUrl:      req.TrainUrl,
-			LogUrl:        req.LogUrl,
-			PoolID:        req.PoolID,
-			CreateVersion: true,
-			Flavor: models.Flavor{
-				Code: req.Spec.SourceSpecId,
+	var jobResult *models.CreateTrainJobResult
+	var createErr error
+	if req.EngineID < 0 {
+		jobResult, createErr = createInferenceJobUserImage(models.CreateInfUserImageParams{
+			JobName:     req.JobName,
+			Description: req.Description,
+			Config: models.InfUserImageConfig{
+				WorkServerNum: req.WorkServerNumber,
+				AppUrl:        req.CodeObsPath,
+				BootFileUrl:   req.BootFileUrl,
+				DataUrl:       req.DataUrl,
+				// TrainUrl:      req.TrainUrl,
+				LogUrl:        req.LogUrl,
+				PoolID:        req.PoolID,
+				CreateVersion: true,
+				Flavor: models.Flavor{
+					Code: req.Spec.SourceSpecId,
+				},
+				Parameter:    req.Parameters,
+				UserImageUrl: req.UserImageUrl,
+				UserCommand:  req.UserCommand,
 			},
-			Parameter: req.Parameters,
-		},
-	})
-	if err != nil {
+		})
+	} else {
+		jobResult, createErr = createInferenceJob(models.CreateInferenceJobParams{
+			JobName:     req.JobName,
+			Description: req.Description,
+			InfConfig: models.InfConfig{
+				WorkServerNum: req.WorkServerNumber,
+				AppUrl:        req.CodeObsPath,
+				BootFileUrl:   req.BootFileUrl,
+				DataUrl:       req.DataUrl,
+				EngineID:      req.EngineID,
+				// TrainUrl:      req.TrainUrl,
+				LogUrl:        req.LogUrl,
+				PoolID:        req.PoolID,
+				CreateVersion: true,
+				Flavor: models.Flavor{
+					Code: req.Spec.SourceSpecId,
+				},
+				Parameter: req.Parameters,
+			},
+		})
+	}
+	if createErr != nil {
 		log.Error("createInferenceJob failed: %v", err.Error())
 		if strings.HasPrefix(err.Error(), UnknownErrorPrefix) {
 			log.Info("(%s)unknown error, set temp status", req.DisplayJobName)
diff --git a/modules/modelarts/resty.go b/modules/modelarts/resty.go
index fd1c467f3..c38300606 100755
--- a/modules/modelarts/resty.go
+++ b/modules/modelarts/resty.go
@@ -1197,6 +1197,66 @@ sendjob:
 	return &result, nil
 }
 
+func createInferenceJobUserImage(createJobParams models.CreateInfUserImageParams) (*models.CreateTrainJobResult, error) {
+	checkSetting()
+	client := getRestyClient()
+	var result models.CreateTrainJobResult
+
+	retry := 0
+
+sendjob:
+	res, err := client.R().
+		SetHeader("Content-Type", "application/json").
+		SetAuthToken(TOKEN).
+		SetBody(createJobParams).
+		SetResult(&result).
+		Post(HOST + "/v1/" + setting.ProjectID + urlTrainJob)
+
+	if err != nil {
+		return nil, fmt.Errorf("resty create train-job: %s", err)
+	}
+
+	req, _ := json.Marshal(createJobParams)
+	log.Info("%s", req)
+
+	if res.StatusCode() == http.StatusUnauthorized && retry < 1 {
+		retry++
+		_ = getToken()
+		goto sendjob
+	}
+
+	if res.StatusCode() != http.StatusOK {
+		var temp models.ErrorResult
+		if err = json.Unmarshal([]byte(res.String()), &temp); err != nil {
+			log.Error("json.Unmarshal failed(%s): %v", res.String(), err.Error())
+			return &result, fmt.Errorf("json.Unmarshal failed(%s): %v", res.String(), err.Error())
+		}
+		log.Error("createInferenceJobUserImage failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
+		bootFileErrorMsg := "Invalid OBS path '" + createJobParams.Config.BootFileUrl + "'."
+		dataSetErrorMsg := "Invalid OBS path '" + createJobParams.Config.DataUrl + "'."
+		if temp.ErrorMsg == bootFileErrorMsg {
+			log.Error("启动文件错误！createInferenceJobUserImage failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
+			return &result, fmt.Errorf("启动文件错误！")
+		}
+		if temp.ErrorMsg == dataSetErrorMsg {
+			log.Error("数据集错误！createInferenceJobUserImage failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
+			return &result, fmt.Errorf("数据集错误！")
+		}
+		if res.StatusCode() == http.StatusBadGateway {
+			return &result, fmt.Errorf(UnknownErrorPrefix+"createInferenceJobUserImage failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
+		} else {
+			return &result, fmt.Errorf("createInferenceJobUserImage failed(%d):%s(%s)", res.StatusCode(), temp.ErrorCode, temp.ErrorMsg)
+		}
+	}
+
+	if !result.IsSuccess {
+		log.Error("createInferenceJobUserImage failed(%s): %s", result.ErrorCode, result.ErrorMsg)
+		return &result, fmt.Errorf("createInferenceJobUserImage failed(%s): %s", result.ErrorCode, result.ErrorMsg)
+	}
+
+	return &result, nil
+}
+
 func createNotebook2(createJobParams models.CreateNotebook2Params) (*models.CreateNotebookResult, error) {
 	checkSetting()
 	client := getRestyClient()
diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go
index 07c1fcd3e..be59b0f3f 100755
--- a/routers/repo/modelarts.go
+++ b/routers/repo/modelarts.go
@@ -1312,6 +1312,36 @@ func getUserCommand(engineId int, req *modelarts.GenerateTrainJobReq) (string, s
 	return userCommand, userImageUrl
 }
 
+func getInfJobUserCommand(engineId int, req *modelarts.GenerateInferenceJobReq) (string, string) {
+	userImageUrl := ""
+	userCommand := ""
+	if engineId < 0 {
+		tmpCodeObsPath := strings.Trim(req.CodeObsPath, "/")
+		tmpCodeObsPaths := strings.Split(tmpCodeObsPath, "/")
+		lastCodeDir := "code"
+		if len(tmpCodeObsPaths) > 0 {
+			lastCodeDir = tmpCodeObsPaths[len(tmpCodeObsPaths)-1]
+		}
+		userCommand = "/bin/bash /home/work/run_train.sh 's3://" + req.CodeObsPath + "' '" + lastCodeDir + "/" + req.BootFile + "' '/tmp/log/train.log' --'data_url'='s3://" + req.DataUrl + "' --'train_url'='s3://" + req.TrainUrl + "'"
+		var versionInfos modelarts.VersionInfo
+		if err := json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil {
+			log.Info("json parse err." + err.Error())
+		} else {
+			for _, engine := range versionInfos.Version {
+				if engine.ID == engineId {
+					userImageUrl = engine.Url
+					break
+				}
+			}
+		}
+		for _, param := range req.Parameters {
+			userCommand += " --'" + param.Label + "'='" + param.Value + "'"
+		}
+		return userCommand, userImageUrl
+	}
+	return userCommand, userImageUrl
+}
+
 func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) {
 	ctx.Data["PageIsTrainJob"] = true
 	var jobID = ctx.Params(":jobid")
@@ -2171,6 +2201,10 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference
 		JobType:           string(models.JobTypeInference),
 	}
 
+	userCommand, userImageUrl := getInfJobUserCommand(engineID, req)
+	req.UserCommand = userCommand
+	req.UserImageUrl = userImageUrl
+
 	err = modelarts.GenerateInferenceJob(ctx, req)
 	if err != nil {
 		log.Error("GenerateTrainJob failed:%v", err.Error())

From 4905f4b0ec31950e13307fd1b93ab29ed454d04f Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Thu, 27 Oct 2022 16:38:30 +0800
Subject: [PATCH 68/82] fix issue

---
 routers/api/v1/repo/cloudbrain.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index ba46ab58c..f165836bd 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -573,6 +573,12 @@ func CloudbrainGetLog(ctx *context.APIContext) {
 				}
 			}
 		}
+		else {
+			if startLine > 0 {
+				startLine += 1
+				endLine += 1
+			}
+		}
 		result = getLogFromModelDir(job.JobName, startLine, endLine, resultPath)
 		if result == nil {
 			log.Error("GetJobLog failed: %v", err, ctx.Data["MsgID"])

From 276d9ca11cbdcd47ec80f7ce07b9aac2039dcb52 Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Thu, 27 Oct 2022 16:40:30 +0800
Subject: [PATCH 69/82] fix issue

---
 routers/api/v1/repo/cloudbrain.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index f165836bd..052fb2504 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -573,7 +573,7 @@ func CloudbrainGetLog(ctx *context.APIContext) {
 				}
 			}
 		}
-		else {
+		else{
 			if startLine > 0 {
 				startLine += 1
 				endLine += 1

From facf5478e49785a235ff3a7ff2c0e116383f4f6c Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Thu, 27 Oct 2022 16:42:40 +0800
Subject: [PATCH 70/82] fix issue

---
 routers/api/v1/repo/cloudbrain.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index 052fb2504..7fb30c687 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -573,10 +573,10 @@ func CloudbrainGetLog(ctx *context.APIContext) {
 				}
 			}
 		}
-		else{
-			if startLine > 0 {
+		else {
+			if startLine>0 {
 				startLine += 1
-				endLine += 1
+				endLine += 1 
 			}
 		}
 		result = getLogFromModelDir(job.JobName, startLine, endLine, resultPath)

From fd812c28216e4b76f32540d149ff60d09b55bd70 Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Thu, 27 Oct 2022 16:44:28 +0800
Subject: [PATCH 71/82] fix issue

---
 routers/api/v1/repo/cloudbrain.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index 7fb30c687..a15683e16 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -572,13 +572,13 @@ func CloudbrainGetLog(ctx *context.APIContext) {
 					startLine = 0
 				}
 			}
-		}
-		else {
-			if startLine>0 {
+		}else{
+			if startLine >0 {
 				startLine += 1
-				endLine += 1 
+				endLine += 1
 			}
 		}
+		
 		result = getLogFromModelDir(job.JobName, startLine, endLine, resultPath)
 		if result == nil {
 			log.Error("GetJobLog failed: %v", err, ctx.Data["MsgID"])

From a24371a79175459ef3cfdcd5ad69f08501dadfe3 Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Thu, 27 Oct 2022 16:50:47 +0800
Subject: [PATCH 72/82] fix issue

---
 routers/api/v1/repo/cloudbrain.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index a15683e16..17de648e5 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -743,8 +743,9 @@ func getLogFromModelDir(jobName string, startLine int, endLine int, resultPath s
 							count++
 						}
 					}
+					fileEndLine = fileEndLine + 1
 				}
-				fileEndLine = fileEndLine + 1
+				
 			} else {
 				log.Info("error:" + err.Error())
 			}

From c6c8a73be6616bc8b5096c4aa2c2aaabac5f0789 Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Thu, 27 Oct 2022 17:10:40 +0800
Subject: [PATCH 73/82] fix issue

---
 routers/api/v1/repo/cloudbrain.go | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index 17de648e5..7cfa15540 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -725,10 +725,10 @@ func getLogFromModelDir(jobName string, startLine int, endLine int, resultPath s
 					line, error := r.ReadString('\n')
 					if error == io.EOF {
 						if i >= startLine {
-							fileEndLine = i
 							re = re + line
 							count++
 						}
+						fileEndLine = i+1
 						log.Info("read file completed.")
 						break
 					}
@@ -738,12 +738,11 @@ func getLogFromModelDir(jobName string, startLine int, endLine int, resultPath s
 					}
 					if error == nil {
 						if i >= startLine {
-							fileEndLine = i
+							fileEndLine = i+1
 							re = re + line
 							count++
 						}
 					}
-					fileEndLine = fileEndLine + 1
 				}
 				
 			} else {

From d74ac930bb66bbeca7c70f752b057e145a5ccf8e Mon Sep 17 00:00:00 2001
From: zouap 
Date: Thu, 27 Oct 2022 17:17:17 +0800
Subject: [PATCH 74/82] =?UTF-8?q?=E6=97=A5=E5=BF=97=E6=98=BE=E7=A4=BA?=
 =?UTF-8?q?=E8=BF=94=E5=9B=9E=E8=A1=8C=E6=95=B0=E4=BF=AE=E6=94=B9=E3=80=82?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: zouap 
---
 routers/api/v1/repo/cloudbrain.go | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index ba46ab58c..37a9e0147 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -572,6 +572,11 @@ func CloudbrainGetLog(ctx *context.APIContext) {
 					startLine = 0
 				}
 			}
+		} else {
+			if startLine > 0 {
+				startLine += 1
+				endLine += 1
+			}
 		}
 		result = getLogFromModelDir(job.JobName, startLine, endLine, resultPath)
 		if result == nil {
@@ -723,6 +728,7 @@ func getLogFromModelDir(jobName string, startLine int, endLine int, resultPath s
 							re = re + line
 							count++
 						}
+						fileEndLine = i + 1
 						log.Info("read file completed.")
 						break
 					}
@@ -732,13 +738,12 @@ func getLogFromModelDir(jobName string, startLine int, endLine int, resultPath s
 					}
 					if error == nil {
 						if i >= startLine {
-							fileEndLine = i
+							fileEndLine = i + 1
 							re = re + line
 							count++
 						}
 					}
 				}
-				fileEndLine = fileEndLine + 1
 			} else {
 				log.Info("error:" + err.Error())
 			}

From b1623e00b805e995e8074eed3e279f51010e21ed Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Fri, 28 Oct 2022 09:43:15 +0800
Subject: [PATCH 75/82] fix issue

---
 web_src/js/features/cloudbrainShow.js | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/web_src/js/features/cloudbrainShow.js b/web_src/js/features/cloudbrainShow.js
index c392da4a1..282a6b706 100644
--- a/web_src/js/features/cloudbrainShow.js
+++ b/web_src/js/features/cloudbrainShow.js
@@ -143,6 +143,7 @@ export default async function initCloudrainSow() {
 
   function logTop(e) {
     let max = e.currentTarget.getAttribute("data-max") || "";
+    let lines = !!max ? 100 : 50;
     let version_name = $(this).data("version");
     let logContentDom = document.querySelector(`#log${max}${version_name}`);
     let ID = $(`#accordion${version_name}`).data("jobid");
@@ -153,7 +154,7 @@ export default async function initCloudrainSow() {
       display: "block",
     });
     $.get(
-      `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=&lines=50&order=asc`,
+      `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=&lines=${lines}&order=asc`,
       (data) => {
         $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css(
           "display",
@@ -190,6 +191,7 @@ export default async function initCloudrainSow() {
   }
   function logBottom(e) {
     let max = e.currentTarget.getAttribute("data-max") || "";
+    let lines = !!max ? 100 : 50;
     let version_name = $(this).data("version");
     let logContentDom = document.querySelector(`#log${max}${version_name}`);
     let ID = $(`#accordion${version_name}`).data("jobid");
@@ -201,7 +203,7 @@ export default async function initCloudrainSow() {
       display: "block",
     });
     $.get(
-      `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=&lines=50&order=desc`,
+      `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=&lines=${lines}&order=desc`,
       (data) => {
         $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css(
           "display",
@@ -231,7 +233,7 @@ export default async function initCloudrainSow() {
         );
         $(`#log${max}${version_name}`).append("
" + data.Content);
         $.get(
-          `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=${data.EndLine}&lines=50&order=desc`,
+          `/api/v1/repos/${repoPath}/${ID}/log?version_name=${version_name}&base_line=${data.EndLine}&lines=${lines}&order=desc`,
           (data) => {
             $(`#log${max}${version_name} .ui.inverted.active.dimmer`).css(
               "display",

From 5c0bc2fba57d6fd163276f55034d5d69c47a34c4 Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Fri, 28 Oct 2022 10:05:19 +0800
Subject: [PATCH 76/82] fix issue

---
 web_src/js/features/cloudbrainShow.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/web_src/js/features/cloudbrainShow.js b/web_src/js/features/cloudbrainShow.js
index 282a6b706..a6a2578c6 100644
--- a/web_src/js/features/cloudbrainShow.js
+++ b/web_src/js/features/cloudbrainShow.js
@@ -14,7 +14,7 @@ export default async function initCloudrainSow() {
     };
   }
 
-  function logScroll(version_name, repoPath, ID, max = "", lines = 50) {
+  function logScroll(version_name, repoPath, ID, max = "", lines = 60) {
     console.log("----------");
     let container = document.querySelector(`#log${max}${version_name}`);
     let scrollTop = container.scrollTop;
@@ -143,7 +143,7 @@ export default async function initCloudrainSow() {
 
   function logTop(e) {
     let max = e.currentTarget.getAttribute("data-max") || "";
-    let lines = !!max ? 100 : 50;
+    let lines = !!max ? 100 : 60;
     let version_name = $(this).data("version");
     let logContentDom = document.querySelector(`#log${max}${version_name}`);
     let ID = $(`#accordion${version_name}`).data("jobid");
@@ -191,7 +191,7 @@ export default async function initCloudrainSow() {
   }
   function logBottom(e) {
     let max = e.currentTarget.getAttribute("data-max") || "";
-    let lines = !!max ? 100 : 50;
+    let lines = !!max ? 100 : 60;
     let version_name = $(this).data("version");
     let logContentDom = document.querySelector(`#log${max}${version_name}`);
     let ID = $(`#accordion${version_name}`).data("jobid");
@@ -339,7 +339,7 @@ export default async function initCloudrainSow() {
           let startLine = $(
             `#log${version_name} input[name=start_line-max-copy]`
           ).val();
-          $(`#log-file-max${version_name}`).siblings("pre").remove();
+          $(`#log_file-max${version_name}`).siblings("pre").remove();
           $(`#log${version_name} input[name=start_line-max]`).val(startLine);
 
           $(".log-scroll-max").unbind("scroll");

From 4b77e3422f4cbaeeee666f7751c9719044127656 Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Fri, 28 Oct 2022 10:11:21 +0800
Subject: [PATCH 77/82] =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=A2=9E=E5=BC=BA?=
 =?UTF-8?q?=E6=94=BE=E5=A4=A7=E6=9F=A5=E7=9C=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 web_src/js/features/cloudbrainShow.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/web_src/js/features/cloudbrainShow.js b/web_src/js/features/cloudbrainShow.js
index a6a2578c6..2dc47884a 100644
--- a/web_src/js/features/cloudbrainShow.js
+++ b/web_src/js/features/cloudbrainShow.js
@@ -15,7 +15,6 @@ export default async function initCloudrainSow() {
   }
 
   function logScroll(version_name, repoPath, ID, max = "", lines = 60) {
-    console.log("----------");
     let container = document.querySelector(`#log${max}${version_name}`);
     let scrollTop = container.scrollTop;
     let scrollHeight = container.scrollHeight;

From b8e42dde5e2c0e60efbcdfea25b27a14a177d858 Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Fri, 28 Oct 2022 10:52:22 +0800
Subject: [PATCH 78/82] =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=A2=9E=E5=BC=BA?=
 =?UTF-8?q?=E6=94=BE=E5=A4=A7=E6=9F=A5=E7=9C=8B=E4=B8=AD=E8=8B=B1=E6=96=87?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 options/locale/locale_en-US.ini                 | 3 +++
 options/locale/locale_zh-CN.ini                 | 3 +++
 templates/custom/max_log.tmpl                   | 4 ++--
 templates/repo/cloudbrain/inference/show.tmpl   | 4 ++--
 templates/repo/cloudbrain/trainjob/show.tmpl    | 4 ++--
 templates/repo/grampus/trainjob/show.tmpl       | 4 ++--
 templates/repo/modelarts/inferencejob/show.tmpl | 4 ++--
 templates/repo/modelarts/trainjob/show.tmpl     | 4 ++--
 templates/repo/modelsafety/show.tmpl            | 6 +++---
 web_src/js/features/cloudbrainShow.js           | 6 ++++++
 10 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini
index 081db13ab..64d6657e4 100755
--- a/options/locale/locale_en-US.ini
+++ b/options/locale/locale_en-US.ini
@@ -1226,6 +1226,9 @@ modelarts.infer_job.select_model = Select Model
 modelarts.infer_job.boot_file_helper=The startup file is the entry file for your program execution and must end in.py.Such as  inference.py, main.py, example/inference.py, case/main.py.
 modelarts.infer_job.tooltip = The model has been deleted and cannot be viewed.
 modelarts.download_log=Download log file
+modelarts.log_file = Log file
+modelarts.fullscreen_log_file = View in full screen
+modelarts.exit_full_screen = Exit fullscreen
 modelarts.no_node_right = The value of 'Amount of Compute Node' is wrong, you have no right to use the current value of 'Amount of Compute Node'.
 
 
diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini
index ae8b6adc4..7345d2044 100755
--- a/options/locale/locale_zh-CN.ini
+++ b/options/locale/locale_zh-CN.ini
@@ -1239,6 +1239,9 @@ modelarts.infer_job.select_model = 选择模型
 modelarts.infer_job.boot_file_helper=启动文件是您程序执行的入口文件，必须是以.py结尾的文件。比如inference.py、main.py、example/inference.py、case/main.py。
 modelarts.infer_job.tooltip = 该模型已删除，无法查看。
 modelarts.download_log=下载日志文件
+modelarts.log_file=日志文件
+modelarts.fullscreen_log_file=全屏查看
+modelarts.exit_full_screen=退出全屏
 modelarts.no_node_right = 计算节点数的值配置错误，您没有权限使用当前配置的计算节点数。
 
 
diff --git a/templates/custom/max_log.tmpl b/templates/custom/max_log.tmpl
index b909fd913..978274f4e 100644
--- a/templates/custom/max_log.tmpl
+++ b/templates/custom/max_log.tmpl
@@ -1,11 +1,11 @@
 

     

         

-            
日志文件

+            

             

                 

                     
-                    退出全屏
+                    
                 

             

         

diff --git a/templates/repo/cloudbrain/inference/show.tmpl b/templates/repo/cloudbrain/inference/show.tmpl
index 79312ed1e..fe127b89d 100644
--- a/templates/repo/cloudbrain/inference/show.tmpl
+++ b/templates/repo/cloudbrain/inference/show.tmpl
@@ -542,9 +542,9 @@
                                  
                                  {{$.i18n.Tr "repo.modelarts.download_log"}}
                              
-                             

+                             

                                 
-                                全屏查看
+                                {{$.i18n.Tr "repo.modelarts.fullscreen_log_file"}}
                             

                          

                          

                                 {{$.i18n.Tr "repo.modelarts.download_log"}}
                             
-                            

+                            

                                 
-                                全屏查看
+                                {{$.i18n.Tr "repo.modelarts.fullscreen_log_file"}}
                             

                         

                         

diff --git a/templates/repo/grampus/trainjob/show.tmpl b/templates/repo/grampus/trainjob/show.tmpl
index a22cb1ab7..0813cf477 100755
--- a/templates/repo/grampus/trainjob/show.tmpl
+++ b/templates/repo/grampus/trainjob/show.tmpl
@@ -527,9 +527,9 @@
                                 
                                 {{$.i18n.Tr "repo.modelarts.download_log"}}
                             
-                            

+                            

                                 
-                                全屏查看
+                                {{$.i18n.Tr "repo.modelarts.fullscreen_log_file"}}
                             

                         

                         

diff --git a/templates/repo/modelarts/inferencejob/show.tmpl b/templates/repo/modelarts/inferencejob/show.tmpl
index c8ff73b2d..98240fbd4 100644
--- a/templates/repo/modelarts/inferencejob/show.tmpl
+++ b/templates/repo/modelarts/inferencejob/show.tmpl
@@ -467,9 +467,9 @@ td, th {
                                     
                                     {{$.i18n.Tr "repo.modelarts.download_log"}}
                                 
-                                

+                                

                                     
-                                    全屏查看
+                                    {{$.i18n.Tr "repo.modelarts.fullscreen_log_file"}}
                                 

                             

                             

diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl
index e86848cc9..74c124e76 100755
--- a/templates/repo/modelarts/trainjob/show.tmpl
+++ b/templates/repo/modelarts/trainjob/show.tmpl
@@ -555,9 +555,9 @@
                                 
                                 {{$.i18n.Tr "repo.modelarts.download_log"}}
                             
-                            

+                            

                                 
-                                全屏查看
+                                {{$.i18n.Tr "repo.modelarts.fullscreen_log_file"}}
                             

                         

                         

                               {{$.i18n.Tr "repo.modelarts.download_log"}}
                           
-                          

+                          

                               
-                              全屏查看
+                              {{$.i18n.Tr "repo.modelarts.fullscreen_log_file"}}
                           

                         

                         

                                 
                                 
-                                	                                
+                                
                                 
                                 

                              

diff --git a/web_src/js/features/cloudbrainShow.js b/web_src/js/features/cloudbrainShow.js
index 2dc47884a..d3338523d 100644
--- a/web_src/js/features/cloudbrainShow.js
+++ b/web_src/js/features/cloudbrainShow.js
@@ -312,6 +312,12 @@ export default async function initCloudrainSow() {
             "display",
             "none"
           );
+          $(".file-info #log-file-title").text(
+            $(".full-log-dialog").data("log")
+          );
+          $(".file-info #log-file-exit").text(
+            $(".full-log-dialog").data("exit")
+          );
         },
         onVisible: function () {
           $(`#log-max${version_name}`).append(

From b1a7c87bda08350e335f48c11faf0628437db6c3 Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Fri, 28 Oct 2022 10:54:11 +0800
Subject: [PATCH 79/82] =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=A2=9E=E5=BC=BA?=
 =?UTF-8?q?=E6=94=BE=E5=A4=A7=E6=9F=A5=E7=9C=8B=E4=B8=AD=E8=8B=B1=E6=96=87?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 templates/repo/cloudbrain/benchmark/show.tmpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/repo/cloudbrain/benchmark/show.tmpl b/templates/repo/cloudbrain/benchmark/show.tmpl
index 0048dd66f..34dcbcadd 100755
--- a/templates/repo/cloudbrain/benchmark/show.tmpl
+++ b/templates/repo/cloudbrain/benchmark/show.tmpl
@@ -533,7 +533,7 @@
                              
                              

                                 
-                                全屏查看
+                                {{$.i18n.Tr "repo.modelarts.download_log"}}
                             

                          

                          

Date: Fri, 28 Oct 2022 10:57:15 +0800
Subject: [PATCH 80/82] =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=A2=9E=E5=BC=BA?=
 =?UTF-8?q?=E6=94=BE=E5=A4=A7=E6=9F=A5=E7=9C=8B=E4=B8=AD=E8=8B=B1=E6=96=87?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 templates/repo/cloudbrain/benchmark/show.tmpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/repo/cloudbrain/benchmark/show.tmpl b/templates/repo/cloudbrain/benchmark/show.tmpl
index 34dcbcadd..de68a3241 100755
--- a/templates/repo/cloudbrain/benchmark/show.tmpl
+++ b/templates/repo/cloudbrain/benchmark/show.tmpl
@@ -531,7 +531,7 @@
                                  
                                  {{$.i18n.Tr "repo.modelarts.download_log"}}
                              
-                             

+                             

                                 
                                 {{$.i18n.Tr "repo.modelarts.download_log"}}
                             


From 13427eb8f7b0651a3bb7ccf12cf9119d78f3618a Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Fri, 28 Oct 2022 17:29:21 +0800
Subject: [PATCH 81/82] fix issue

---
 options/locale/locale_en-US.ini | 2 +-
 options/locale/locale_zh-CN.ini | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini
index 58fb5d976..e7d730027 100755
--- a/options/locale/locale_en-US.ini
+++ b/options/locale/locale_en-US.ini
@@ -1218,7 +1218,7 @@ cloudbrain.benchmark.evaluate_test=Test Script
 cloudbrain.benchmark.types={"type":[{"id":1,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=detection","first":"Target detection","second":[{"id":1,"value":"None","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"yangzhx","repo_name":"detection_benchmark_script"}]},{"id":2,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=reid","first":"Target re-identification","second":[{"id":1,"value":"Vehicle re-identification","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"JiahongXu","repo_name":"benchmark_reID_script"},{"id":2,"value":"Image-based person re-identification","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"JiahongXu","repo_name":"benchmark_reID_script"}]},{"id":3,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=tracking","first":"Multi-target tracking","second":[{"id":1,"value":"None","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"lix07","repo_name":"MOT_benchmark_script"}]}]}
 cloudbrain.morethanonejob=You already have a running or waiting task, create it after that task is over.
 cloudbrain.morethanonejob1=You have created a  similar task  that is waiting or running, please wait for the task to finish before creating it.
-cloudbrain.morethanonejob2=You can view all your Cloud Brain tasks in  Personal Center > Cloud Brain Tasks . 
+cloudbrain.morethanonejob2=You can view all your Cloud Brain tasks in  Personal Center > Cloud Brain Tasks . 
 
 modelarts.infer_job_model = Model
 modelarts.infer_job_model_file = Model File
diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini
index 15ce8ac6c..972865484 100755
--- a/options/locale/locale_zh-CN.ini
+++ b/options/locale/locale_zh-CN.ini
@@ -1232,7 +1232,7 @@ cloudbrain.benchmark.types={"type":[{"id":1,"rank_link":"https://git.openi.org.c
 cloudbrain.benchmark.model.types={"type":[{"id":1,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=detection","first":"目标检测","second":[{"id":1,"value":"无","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"yangzhx","repo_name":"detection_benchmark_script"}]},{"id":2,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=reid","first":"目标重识别","second":[{"id":1,"value":"车辆重识别","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"JiahongXu","repo_name":"benchmark_reID_script"},{"id":2,"value":"基于图像的行人重识别","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"JiahongXu","repo_name":"benchmark_reID_script"}]},{"id":3,"rank_link":"https://git.openi.org.cn/benchmark/?username=admin&algType=tracking","first":"多目标跟踪","second":[{"id":1,"value":"无","attachment":"84cf39c4-d8bc-41aa-aaa3-182ce289b105","owner":"lix07","repo_name":"MOT_benchmark_script"}]}]}
 cloudbrain.morethanonejob=您已经创建了一个正在等待或运行中的同类任务，请等待任务结束再创建。
 cloudbrain.morethanonejob1=您已经有 同类任务 正在等待或运行中，请等待任务结束再创建；
-cloudbrain.morethanonejob2=可以在 “个人中心 > 云脑任务” 查看您所有的云脑任务。
+cloudbrain.morethanonejob2=可以在 “个人中心 > 云脑任务” 查看您所有的云脑任务。
 
 modelarts.infer_job_model = 模型名称
 modelarts.infer_job_model_file = 模型文件

From 90e37e2f9a29d5881ed3edec9c064cfb8e1034dd Mon Sep 17 00:00:00 2001
From: zouap 
Date: Mon, 31 Oct 2022 09:55:05 +0800
Subject: [PATCH 82/82] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?=
 =?UTF-8?q?=EF=BC=8C=E8=A7=A3=E5=86=B3#3063?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: zouap 
---
 options/locale/locale_en-US.ini | 2 +-
 options/locale/locale_zh-CN.ini | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini
index b3b6cb58c..ba75009b1 100755
--- a/options/locale/locale_en-US.ini
+++ b/options/locale/locale_en-US.ini
@@ -266,7 +266,7 @@ page_dev_yunlao_desc4=Developers can freely select the corresponding computing r
 page_dev_yunlao_desc5=If your model requires more computing resources, you can also apply for it separately.
 page_dev_yunlao_apply=Apply Separately
 c2net_title=China Computing NET(C²NET)
-c2net_desc=Extensive access to intelligent computing centers and supercomputing centers across the country to provide users with free computing resources.
+c2net_desc=Extensive access to intelligent computing centers, supercomputing centers and big data centers across the country to provide users with free computing resources.
 c2net_center=Center
 search=Search
 search_repo=Repository
diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini
index a2664b820..5d5c54a1c 100755
--- a/options/locale/locale_zh-CN.ini
+++ b/options/locale/locale_zh-CN.ini
@@ -268,7 +268,7 @@ page_dev_yunlao_desc4=开发者可以根据使用需求，自由选择相应计
 page_dev_yunlao_desc5=如果您的模型需要更多的计算资源，也可以单独申请。
 page_dev_yunlao_apply=单独申请
 c2net_title=中国算力网(C²NET)
-c2net_desc=广泛接入全国各地智算中心、超算中心，为用户提供免费算力资源
+c2net_desc=广泛接入全国各地智算中心、超算中心与大数据中心等，为用户提供免费算力资源
 c2net_center=中心
 search=搜索
 search_repo=项目