From 94ed16350e48b5989c08a591dab3ec8cb18d211a Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 4 Nov 2022 18:20:01 +0800 Subject: [PATCH 001/253] fix-kanban --- models/cloudbrain.go | 2 +- modules/setting/setting.go | 42 +++++++---- routers/admin/cloudbrains.go | 2 + routers/api/v1/api.go | 1 + routers/api/v1/repo/cloudbrain_dashboard.go | 45 +++++++----- routers/api/v1/repo/modelarts.go | 1 - routers/repo/cloudbrain_statistic.go | 80 ++++++++++++--------- routers/user/home.go | 2 + services/cloudbrain/util.go | 53 ++++++++++++-- 9 files changed, 153 insertions(+), 75 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index dacb1b03a..a4fbf8e32 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1968,7 +1968,7 @@ func UpdateTrainJobVersion(job *Cloudbrain) error { func updateJobTrainVersion(e Engine, job *Cloudbrain) error { var sess *xorm.Session sess = e.Where("job_id = ? AND version_name=?", job.JobID, job.VersionName) - _, err := sess.Cols("status", "train_job_duration", "duration", "start_time", "end_time", "created_unix").Update(job) + _, err := sess.Cols("status", "train_job_duration", "duration", "start_time", "end_time", "created_unix", "ai_center").Update(job) return err } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 88557ee60..c16050500 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -598,20 +598,22 @@ var ( //grampus config Grampus = struct { - Env string - Host string - UserName string - Password string - SpecialPools string - C2NetSequence string - SyncScriptProject string - LocalCenterID string - AiCenterInfo string + Env string + Host string + UserName string + Password string + SpecialPools string + C2NetSequence string + SyncScriptProject string + LocalCenterID string + AiCenterInfo string + AiCenterCodeAndNameInfo string }{} - C2NetInfos *C2NetSqInfos - CenterInfos *AiCenterInfos - C2NetMapInfo map[string]*C2NetSequenceInfo + C2NetInfos *C2NetSqInfos + CenterInfos *AiCenterInfos + C2NetMapInfo map[string]*C2NetSequenceInfo + AiCenterCodeAndNameMapInfo map[string]*C2NetSequenceInfo //elk config ElkUrl string @@ -1651,13 +1653,23 @@ func getGrampusConfig() { Grampus.Password = sec.Key("PASSWORD").MustString("") Grampus.SpecialPools = sec.Key("SPECIAL_POOL").MustString("") Grampus.C2NetSequence = sec.Key("C2NET_SEQUENCE").MustString("{\"sequence\":[{\"id\":1,\"name\":\"cloudbrain_one\",\"content\":\"鹏城云脑一号\",\"content_en\":\"Pencheng Cloudbrain Ⅰ\"},{\"id\":2,\"name\":\"cloudbrain_two\",\"content\":\"鹏城云脑二号\",\"content_en\":\"Pencheng Cloudbrain Ⅱ\"},{\"id\":3,\"name\":\"beida\",\"content\":\"北大人工智能集群系统\",\"content_en\":\"Peking University AI Center\"},{\"id\":4,\"name\":\"hefei\",\"content\":\"合肥类脑智能开放平台\",\"content_en\":\"Hefei AI Center\"},{\"id\":5,\"name\":\"wuhan\",\"content\":\"武汉人工智能计算中心\",\"content_en\":\"Wuhan AI Center\"},{\"id\":6,\"name\":\"xian\",\"content\":\"西安未来人工智能计算中心\",\"content_en\":\"Xi'an AI Center\"},{\"id\":7,\"pclcci\":\"more\",\"content\":\"鹏城云计算所\",\"content_en\":\"Pengcheng Cloud Computing Institute\"},{\"id\":8,\"name\":\"xuchang\",\"content\":\"中原人工智能计算中心\",\"content_en\":\"Zhongyuan AI Center\"},{\"id\":9,\"name\":\"chengdu\",\"content\":\"成都人工智能计算中心\",\"content_en\":\"Chengdu AI Center\"},{\"id\":10,\"name\":\"more\",\"content\":\"横琴先进智能计算中心\",\"content_en\":\"Hengqin AI Center\"},{\"id\":11,\"name\":\"more\",\"content\":\"国家超级计算济南中心\",\"content_en\":\"HPC & AI Center\"}]}") + Grampus.AiCenterCodeAndNameInfo = sec.Key("AiCenterCodeAndName").MustString("{\"sequence\":[{\"id\":1,\"name\":\"cloudbrain_one\",\"content\":\"鹏城云脑一号\",\"content_en\":\"Pencheng Cloudbrain Ⅰ\"},{\"id\":2,\"name\":\"cloudbrain_two\",\"content\":\"鹏城云脑二号\",\"content_en\":\"Pencheng Cloudbrain Ⅱ\"},{\"id\":3,\"name\":\"beida\",\"content\":\"北大人工智能集群系统\",\"content_en\":\"Peking University AI Center\"},{\"id\":4,\"name\":\"hefei\",\"content\":\"合肥类脑智能开放平台\",\"content_en\":\"Hefei AI Center\"},{\"id\":5,\"name\":\"wuhan\",\"content\":\"武汉人工智能计算中心\",\"content_en\":\"Wuhan AI Center\"},{\"id\":6,\"name\":\"xian\",\"content\":\"西安未来人工智能计算中心\",\"content_en\":\"Xi'an AI Center\"},{\"id\":7,\"pclcci\":\"more\",\"content\":\"鹏城云计算所\",\"content_en\":\"Pengcheng Cloud Computing Institute\"},{\"id\":8,\"name\":\"xuchang\",\"content\":\"中原人工智能计算中心\",\"content_en\":\"Zhongyuan AI Center\"},{\"id\":9,\"name\":\"chengdu\",\"content\":\"成都人工智能计算中心\",\"content_en\":\"Chengdu AI Center\"},{\"id\":10,\"name\":\"more\",\"content\":\"横琴先进智能计算中心\",\"content_en\":\"Hengqin AI Center\"},{\"id\":11,\"name\":\"more\",\"content\":\"国家超级计算济南中心\",\"content_en\":\"HPC & AI Center\"}]}") if Grampus.C2NetSequence != "" { if err := json.Unmarshal([]byte(Grampus.C2NetSequence), &C2NetInfos); err != nil { log.Error("Unmarshal(C2NetSequence) failed:%v", err) } - C2NetMapInfo=make(map[string]*C2NetSequenceInfo) - for _,value :=range C2NetInfos.C2NetSqInfo{ - C2NetMapInfo[value.Name]=value + C2NetMapInfo = make(map[string]*C2NetSequenceInfo) + for _, value := range C2NetInfos.C2NetSqInfo { + C2NetMapInfo[value.Name] = value + } + } + if Grampus.AiCenterCodeAndNameInfo != "" { + if err := json.Unmarshal([]byte(Grampus.AiCenterCodeAndNameInfo), &C2NetInfos); err != nil { + log.Error("Unmarshal(AiCenterCodeAndNameInfo) failed:%v", err) + } + AiCenterCodeAndNameMapInfo = make(map[string]*C2NetSequenceInfo) + for _, value := range C2NetInfos.C2NetSqInfo { + AiCenterCodeAndNameMapInfo[value.Name] = value } } Grampus.SyncScriptProject = sec.Key("SYNC_SCRIPT_PROJECT").MustString("script_for_grampus") diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index cbf6782ed..d4a00fc34 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -17,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" ) const ( @@ -95,6 +96,7 @@ func CloudBrains(ctx *context.Context) { models.LoadSpecs4CloudbrainInfo(ciTasks) for i, task := range ciTasks { + ciTasks[i] = cloudbrainService.UpdateCloudbrainAiCenter(ciTasks[i]) ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 69de79c10..674c9e01d 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -604,6 +604,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/resource_usage_statistic", repo.GetDurationRateStatistic) m.Get("/resource_usage_rate_detail", repo.GetCloudbrainResourceUsageDetail) m.Get("/apitest_for_statistic", repo.CloudbrainDurationStatisticForTest) + m.Get("/apipost_for_cloudbrain_aicenter", repo.CloudbrainUpdateAiCenter) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index d1ccf1bf5..46f225a2e 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1659,28 +1659,29 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati return OpenIDurationRate, C2NetDurationRate, 0 } for _, cloudbrainStatistic := range cardDurationStatistics { + aiCenterName := repo.GetAiCenterNameByCode(cloudbrainStatistic.AiCenterCode) if cloudbrainStatistic.Cluster == models.OpenICluster { - if _, ok := OpenITotalDuration[cloudbrainStatistic.AiCenterName]; !ok { - OpenITotalDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration + if _, ok := OpenITotalDuration[aiCenterName]; !ok { + OpenITotalDuration[aiCenterName] = cloudbrainStatistic.CardsTotalDuration } else { - OpenITotalDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration + OpenITotalDuration[aiCenterName] += cloudbrainStatistic.CardsTotalDuration } - if _, ok := OpenIUsageDuration[cloudbrainStatistic.AiCenterName]; !ok { - OpenIUsageDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsUseDuration + if _, ok := OpenIUsageDuration[aiCenterName]; !ok { + OpenIUsageDuration[aiCenterName] = cloudbrainStatistic.CardsUseDuration } else { - OpenIUsageDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsUseDuration + OpenIUsageDuration[aiCenterName] += cloudbrainStatistic.CardsUseDuration } } if cloudbrainStatistic.Cluster == models.C2NetCluster { - if _, ok := C2NetTotalDuration[cloudbrainStatistic.AiCenterName]; !ok { - C2NetTotalDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsTotalDuration + if _, ok := C2NetTotalDuration[aiCenterName]; !ok { + C2NetTotalDuration[aiCenterName] = cloudbrainStatistic.CardsTotalDuration } else { - C2NetTotalDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsTotalDuration + C2NetTotalDuration[aiCenterName] += cloudbrainStatistic.CardsTotalDuration } - if _, ok := C2NetUsageDuration[cloudbrainStatistic.AiCenterName]; !ok { - C2NetUsageDuration[cloudbrainStatistic.AiCenterName] = cloudbrainStatistic.CardsUseDuration + if _, ok := C2NetUsageDuration[aiCenterName]; !ok { + C2NetUsageDuration[aiCenterName] = cloudbrainStatistic.CardsUseDuration } else { - C2NetUsageDuration[cloudbrainStatistic.AiCenterName] += cloudbrainStatistic.CardsUseDuration + C2NetUsageDuration[aiCenterName] += cloudbrainStatistic.CardsUseDuration } } } @@ -1690,16 +1691,17 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati return OpenIDurationRate, C2NetDurationRate, 0 } for _, v := range ResourceAiCenterRes { + aiCenterName := repo.GetAiCenterNameByCode(v.AiCenterCode) if cutString(v.AiCenterCode, 4) == cutString(models.AICenterOfCloudBrainOne, 4) { - if _, ok := OpenIUsageDuration[v.AiCenterName]; !ok { - OpenIUsageDuration[v.AiCenterName] = 0 + if _, ok := OpenIUsageDuration[aiCenterName]; !ok { + OpenIUsageDuration[aiCenterName] = 0 } - if _, ok := OpenITotalDuration[v.AiCenterName]; !ok { - OpenITotalDuration[v.AiCenterName] = 0 + if _, ok := OpenITotalDuration[aiCenterName]; !ok { + OpenITotalDuration[aiCenterName] = 0 } } else { - if _, ok := C2NetUsageDuration[v.AiCenterName]; !ok { - C2NetUsageDuration[v.AiCenterName] = 0 + if _, ok := C2NetUsageDuration[aiCenterName]; !ok { + C2NetUsageDuration[aiCenterName] = 0 } } } @@ -1831,3 +1833,10 @@ func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterC hourTimeStatistic.HourTimeUsageRate = hourTimeUsageRate return hourTimeStatistic, nil } + +func CloudbrainUpdateAiCenter(ctx *context.Context) { + // repo.CloudbrainDurationStatisticHour() + ctx.JSON(http.StatusOK, map[string]interface{}{ + "message": 0, + }) +} diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 5a0e21ed8..5b815b878 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -173,7 +173,6 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) { if len(result.JobInfo.Tasks) > 0 { if len(result.JobInfo.Tasks[0].CenterID) > 0 && len(result.JobInfo.Tasks[0].CenterName) > 0 { job.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0] - // aiCenterName = result.JobInfo.Tasks[0].CenterName[0] aiCenterName = cloudbrainService.GetAiCenterShow(job.AiCenter, ctx.Context) } } diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 3814c2daf..07bc97a32 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -7,6 +7,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" ) func CloudbrainDurationStatisticHour() { @@ -27,12 +28,13 @@ func CloudbrainDurationStatisticHour() { log.Info("GetCloudbrainByTime err: %v", err) return } - specMap := make(map[string]*models.Specification) + cloudbrainMap := make(map[string]*models.Cloudbrain) models.LoadSpecs4CloudbrainInfo(ciTasks) for _, cloudbrain := range ciTasks { - if _, ok := specMap[cloudbrain.Cloudbrain.Spec.AiCenterCode+"/"+cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { + cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain) + if _, ok := cloudbrainMap[cloudbrain.Cloudbrain.AiCenter+"/"+cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { if cloudbrain.Cloudbrain.Spec != nil { - specMap[cloudbrain.Cloudbrain.Spec.AiCenterCode+"/"+cloudbrain.Cloudbrain.Spec.AccCardType] = cloudbrain.Cloudbrain.Spec + cloudbrainMap[cloudbrain.Cloudbrain.AiCenter+"/"+cloudbrain.Cloudbrain.Spec.AccCardType] = &cloudbrain.Cloudbrain } } } @@ -46,43 +48,43 @@ func CloudbrainDurationStatisticHour() { } cardsTotalDurationMap := make(map[string]int) for _, resourceQueue := range resourceQueues { - cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterName+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource] = resourceQueue.CardsTotalNum * 1 * 60 * 60 + cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource] = resourceQueue.CardsTotalNum * 1 * 60 * 60 } - for centerCode, CardTypeInfo := range cloudBrainCenterCodeAndCardTypeInfo { - for cardType, cardDuration := range CardTypeInfo { - spec := specMap[centerCode+"/"+cardType] - if spec != nil { + for centerCode, CardTypes := range cloudBrainCenterCodeAndCardTypeInfo { + for cardType, cardDuration := range CardTypes { + cloudbrainTable := cloudbrainMap[centerCode+"/"+cardType] + if cloudbrainTable != nil { if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType); err != nil { log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) return } - if _, ok := cardsTotalDurationMap[spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource]; !ok { - cardsTotalDurationMap[spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource] = 0 + if _, ok := cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType+"/"+cloudbrainTable.ComputeResource]; !ok { + cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType+"/"+cloudbrainTable.ComputeResource] = 0 } cloudbrainDurationStat := models.CloudbrainDurationStatistic{ DateTime: dateTime, DayTime: dayTime, HourTime: hourTime, - Cluster: spec.Cluster, - AiCenterName: spec.AiCenterName, + Cluster: cloudbrainTable.Cluster, + AiCenterName: GetAiCenterNameByCode(centerCode), AiCenterCode: centerCode, AccCardType: cardType, - ComputeResource: spec.ComputeResource, + ComputeResource: cloudbrainTable.ComputeResource, CardsUseDuration: cardDuration, - CardsTotalDuration: cardsTotalDurationMap[spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource], + CardsTotalDuration: cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType+"/"+cloudbrainTable.ComputeResource], CreatedUnix: timeutil.TimeStampNow(), } if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) } - delete(cardsTotalDurationMap, spec.Cluster+"/"+spec.AiCenterName+"/"+centerCode+"/"+cardType+"/"+spec.ComputeResource) + delete(cardsTotalDurationMap, cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType+"/"+cloudbrainTable.ComputeResource) } } } for key, cardsTotalDuration := range cardsTotalDurationMap { - if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, strings.Split(key, "/")[2], strings.Split(key, "/")[3]); err != nil { + if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, strings.Split(key, "/")[1], strings.Split(key, "/")[2]); err != nil { log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) return } @@ -91,10 +93,10 @@ func CloudbrainDurationStatisticHour() { DayTime: dayTime, HourTime: hourTime, Cluster: strings.Split(key, "/")[0], - AiCenterName: strings.Split(key, "/")[1], - AiCenterCode: strings.Split(key, "/")[2], - AccCardType: strings.Split(key, "/")[3], - ComputeResource: strings.Split(key, "/")[4], + AiCenterName: GetAiCenterNameByCode(strings.Split(key, "/")[1]), + AiCenterCode: strings.Split(key, "/")[1], + AccCardType: strings.Split(key, "/")[2], + ComputeResource: strings.Split(key, "/")[3], CardsUseDuration: 0, CardsTotalDuration: cardsTotalDuration, CreatedUnix: timeutil.TimeStampNow(), @@ -107,12 +109,24 @@ func CloudbrainDurationStatisticHour() { log.Info("finish summary cloudbrainDurationStat") } +func GetAiCenterNameByCode(centerCode string) string { + var aiCenterName string + c2NetSequenceInfo := cloudbrainService.C2NetSequenceInfoByCode(centerCode) + if c2NetSequenceInfo != nil { + aiCenterName = c2NetSequenceInfo.Content + } else { + aiCenterName = "" + } + return aiCenterName + +} + func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) map[string]map[string]int { var WorkServerNumber int var AccCardsNum int cloudBrainCenterCodeAndCardType := make(map[string]map[string]int) for _, cloudbrain := range ciTasks { - + cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain) if cloudbrain.Cloudbrain.StartTime == 0 { cloudbrain.Cloudbrain.StartTime = cloudbrain.Cloudbrain.CreatedUnix } @@ -129,36 +143,36 @@ func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, be } else { AccCardsNum = cloudbrain.Cloudbrain.Spec.AccCardsNum } - if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode]; !ok { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode] = make(map[string]int) + if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter]; !ok { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter] = make(map[string]int) } if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { - if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { + if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) } else { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) } } else { if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) } else { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) } } } else { - if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { + if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) } else { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) } } else { if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) } else { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.Spec.AiCenterCode][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) } } diff --git a/routers/user/home.go b/routers/user/home.go index b6ab28f95..348a295ed 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -23,6 +23,7 @@ import ( "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" issue_service "code.gitea.io/gitea/services/issue" pull_service "code.gitea.io/gitea/services/pull" @@ -837,6 +838,7 @@ func Cloudbrains(ctx *context.Context) { } models.LoadSpecs4CloudbrainInfo(ciTasks) for i, _ := range ciTasks { + ciTasks[i] = cloudbrainService.UpdateCloudbrainAiCenter(ciTasks[i]) ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = ciTasks[i].ComputeResource diff --git a/services/cloudbrain/util.go b/services/cloudbrain/util.go index ab738927e..141807246 100644 --- a/services/cloudbrain/util.go +++ b/services/cloudbrain/util.go @@ -1,27 +1,29 @@ package cloudbrain import ( + "strings" + + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/setting" - "strings" ) -func GetAiCenterShow(aiCenter string,ctx *context.Context) string{ +func GetAiCenterShow(aiCenter string, ctx *context.Context) string { aiCenterInfo := strings.Split(aiCenter, "+") - if len(aiCenterInfo) == 2{ - if setting.C2NetMapInfo!=nil { - if info,ok:=setting.C2NetMapInfo[aiCenterInfo[0]];ok { + if len(aiCenterInfo) == 2 { + if setting.C2NetMapInfo != nil { + if info, ok := setting.C2NetMapInfo[aiCenterInfo[0]]; ok { if ctx.Language() == "zh-CN" { return info.Content } else { return info.ContentEN } - }else{ + } else { return aiCenterInfo[1] } - }else{ + } else { return aiCenterInfo[1] } @@ -29,5 +31,42 @@ func GetAiCenterShow(aiCenter string,ctx *context.Context) string{ return "" +} + +func C2NetSequenceInfoByCode(aiCenterCode string) *setting.C2NetSequenceInfo { + if setting.AiCenterCodeAndNameMapInfo != nil { + if info, ok := setting.AiCenterCodeAndNameMapInfo[aiCenterCode]; ok { + return info + } else { + return nil + } + } else { + return nil + } +} + +func getAiCenterCode(aiCenter string) string { + aiCenterInfo := strings.Split(aiCenter, "+") + return aiCenterInfo[0] +} + +func UpdateCloudbrainAiCenter(cloudbrain *models.CloudbrainInfo) *models.CloudbrainInfo { + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { + cloudbrain.Cloudbrain.AiCenter = models.AICenterOfCloudBrainOne + cloudbrain.Cloudbrain.Cluster = models.OpenICluster + } + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + cloudbrain.Cloudbrain.AiCenter = models.AICenterOfCloudBrainTwo + cloudbrain.Cloudbrain.Cluster = models.OpenICluster + } + if cloudbrain.Cloudbrain.Type == models.TypeCDCenter { + cloudbrain.Cloudbrain.AiCenter = models.AICenterOfChengdu + cloudbrain.Cloudbrain.Cluster = models.OpenICluster + } + if cloudbrain.Cloudbrain.Type == models.TypeC2Net && cloudbrain.Cloudbrain.AiCenter != "" { + cloudbrain.Cloudbrain.AiCenter = getAiCenterCode(cloudbrain.Cloudbrain.AiCenter) + cloudbrain.Cloudbrain.Cluster = models.C2NetCluster + } + return cloudbrain } From 517e1d517b6bbb00cb2a0a696a5012b452ab8a03 Mon Sep 17 00:00:00 2001 From: liuzx Date: Mon, 7 Nov 2022 18:29:30 +0800 Subject: [PATCH 002/253] update kanban --- models/cloudbrain_static.go | 23 +++++---- routers/api/v1/api.go | 4 +- routers/api/v1/repo/cloudbrain_dashboard.go | 38 +++++++++++++-- routers/repo/cloudbrain_statistic.go | 54 +++++++++++++++------ 4 files changed, 88 insertions(+), 31 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 19e55fb6d..92d9842a9 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -42,14 +42,14 @@ type TaskDetail struct { type CloudbrainDurationStatistic struct { ID int64 `xorm:"pk autoincr"` Cluster string - AiCenterCode string + AiCenterCode string `xorm:"INDEX"` AiCenterName string ComputeResource string - AccCardType string + AccCardType string `xorm:"INDEX"` - DateTime string - DayTime string - HourTime int + DateTime timeutil.TimeStamp `xorm:"INDEX"` + DayTime string `xorm:"INDEX"` + HourTime int `xorm:"INDEX"` CardsUseDuration int CardsTotalDuration int CardsTotalNum int @@ -275,8 +275,11 @@ func GetCloudbrainByTime(beginTime int64, endTime int64) ([]*CloudbrainInfo, err sess := x.NewSession() defer sess.Close() var cond = builder.NewCond() - cond = cond.And( - builder.And(builder.Gte{"cloudbrain.end_time": beginTime}, builder.Lte{"cloudbrain.end_time": endTime}), + cond = cond.Or( + builder.And(builder.Lte{"cloudbrain.start_time": beginTime}, builder.Gte{"cloudbrain.end_time": beginTime}), + ) + cond = cond.Or( + builder.And(builder.Gte{"cloudbrain.start_time": beginTime}, builder.Lte{"cloudbrain.start_time": endTime}), ) cond = cond.Or( builder.Eq{"cloudbrain.status": string(JobRunning)}, @@ -332,7 +335,7 @@ func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode s func GetCanUseCardInfo() ([]*ResourceQueue, error) { sess := x.NewSession() defer sess.Close() - sess.OrderBy("resource_queue.id ASC") + sess.OrderBy("resource_queue.cluster DESC, resource_queue.ai_center_code ASC") ResourceQueues := make([]*ResourceQueue, 0, 10) if err := sess.Table(&ResourceQueue{}).Find(&ResourceQueues); err != nil { log.Info("find error.") @@ -346,7 +349,7 @@ func GetCardDurationStatistics(opts *DurationStatisticOptions) ([]*CloudbrainDur var cond = builder.NewCond() if opts.BeginTime.Unix() > 0 && opts.EndTime.Unix() > 0 { cond = cond.And( - builder.And(builder.Gte{"cloudbrain_duration_statistic.created_unix": opts.BeginTime.Unix()}, builder.Lte{"cloudbrain_duration_statistic.created_unix": opts.EndTime.Unix()}), + builder.And(builder.Gte{"cloudbrain_duration_statistic.date_time": opts.BeginTime.Unix()}, builder.Lte{"cloudbrain_duration_statistic.date_time": opts.EndTime.Unix()}), ) } if opts.AiCenterCode != "" { @@ -365,7 +368,7 @@ func GetCardDurationStatistics(opts *DurationStatisticOptions) ([]*CloudbrainDur func GetDurationRecordBeginTime() ([]*CloudbrainDurationStatistic, error) { sess := xStatistic.NewSession() defer sess.Close() - sess.OrderBy("cloudbrain_duration_statistic.id ASC limit 1") + sess.OrderBy("cloudbrain_duration_statistic.date_time ASC limit 1") CloudbrainDurationStatistics := make([]*CloudbrainDurationStatistic, 0) if err := sess.Table(&CloudbrainDurationStatistic{}).Find(&CloudbrainDurationStatistics); err != nil { log.Info("find error.") diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 674c9e01d..8864e2e31 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -603,8 +603,10 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/overview_resource", repo.GetCloudbrainResourceOverview) m.Get("/resource_usage_statistic", repo.GetDurationRateStatistic) m.Get("/resource_usage_rate_detail", repo.GetCloudbrainResourceUsageDetail) + + m.Get("/resource_queues", repo.GetResourceQueues) m.Get("/apitest_for_statistic", repo.CloudbrainDurationStatisticForTest) - m.Get("/apipost_for_cloudbrain_aicenter", repo.CloudbrainUpdateAiCenter) + m.Get("/apipost_for_cloudbrain_data", repo.CloudbrainUpdateHistoryData) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 46f225a2e..e29468671 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1417,7 +1417,7 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { log.Error("Can not get GetDurationRecordBeginTime", err) return } - recordBeginTime := recordCloudbrainDuration[0].CreatedUnix + recordBeginTime := recordCloudbrainDuration[0].DateTime - 1*60*60 recordUpdateTime := time.Now().Unix() resourceQueues, err := models.GetCanUseCardInfo() if err != nil { @@ -1428,11 +1428,12 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { C2NetResourceDetail := []models.ResourceDetail{} for _, resourceQueue := range resourceQueues { if resourceQueue.Cluster == models.OpenICluster { + aiCenterName := repo.GetAiCenterNameByCode(resourceQueue.AiCenterCode) var resourceDetail models.ResourceDetail resourceDetail.QueueCode = resourceQueue.QueueCode resourceDetail.Cluster = resourceQueue.Cluster resourceDetail.AiCenterCode = resourceQueue.AiCenterCode - resourceDetail.AiCenterName = resourceQueue.AiCenterName + "/" + resourceQueue.AiCenterCode + resourceDetail.AiCenterName = aiCenterName + "/" + resourceQueue.AiCenterCode resourceDetail.ComputeResource = resourceQueue.ComputeResource resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")" resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum @@ -1440,11 +1441,12 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { OpenIResourceDetail = append(OpenIResourceDetail, resourceDetail) } if resourceQueue.Cluster == models.C2NetCluster { + aiCenterName := repo.GetAiCenterNameByCode(resourceQueue.AiCenterCode) var resourceDetail models.ResourceDetail resourceDetail.QueueCode = resourceQueue.QueueCode resourceDetail.Cluster = resourceQueue.Cluster resourceDetail.AiCenterCode = resourceQueue.AiCenterCode - resourceDetail.AiCenterName = resourceQueue.AiCenterName + "/" + resourceQueue.AiCenterCode + resourceDetail.AiCenterName = aiCenterName + "/" + resourceQueue.AiCenterCode resourceDetail.ComputeResource = resourceQueue.ComputeResource resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")" resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum @@ -1627,7 +1629,7 @@ func getAiCenterUsageDuration(beginTime time.Time, endTime time.Time, cloudbrain usageRate := float64(0) for _, cloudbrainStatistic := range cloudbrainStatistics { - if int64(cloudbrainStatistic.CreatedUnix) >= beginTime.Unix() && int64(cloudbrainStatistic.CreatedUnix) < endTime.Unix() { + if int64(cloudbrainStatistic.DateTime) >= beginTime.Unix() && int64(cloudbrainStatistic.DateTime) < endTime.Unix() { totalDuration += cloudbrainStatistic.CardsTotalDuration usageDuration += cloudbrainStatistic.CardsUseDuration } @@ -1835,8 +1837,34 @@ func getHourCloudbrainDuration(beginTime time.Time, endTime time.Time, aiCenterC } func CloudbrainUpdateAiCenter(ctx *context.Context) { - // repo.CloudbrainDurationStatisticHour() + repo.CloudbrainDurationStatisticHour() ctx.JSON(http.StatusOK, map[string]interface{}{ "message": 0, }) } + +func CloudbrainUpdateHistoryData(ctx *context.Context) { + repo.UpdateDurationStatisticHistoryData() + ctx.JSON(http.StatusOK, map[string]interface{}{ + "message": 0, + }) +} + +func GetResourceQueues(ctx *context.Context) { + resourceQueues, err := models.GetCanUseCardInfo() + if err != nil { + log.Info("GetCanUseCardInfo err: %v", err) + return + } + Resource := make([]*models.ResourceQueue, 0) + aiCenterCodeMap := make(map[string]string) + for _, resourceQueue := range resourceQueues { + if _, ok := aiCenterCodeMap[resourceQueue.AiCenterCode]; !ok { + resourceQueue.AiCenterName = repo.GetAiCenterNameByCode(resourceQueue.AiCenterCode) + Resource = append(Resource, resourceQueue) + } + } + ctx.JSON(http.StatusOK, map[string]interface{}{ + "resourceQueues": Resource, + }) +} diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 07bc97a32..5414c184b 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -11,17 +11,29 @@ import ( ) func CloudbrainDurationStatisticHour() { - - dateTime := time.Now().Format("2006-01-02 15:04:05") - dayTime := time.Now().Format("2006-01-02") now := time.Now() + currentTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location()) + summaryDurationStat(currentTime) +} +func UpdateDurationStatisticHistoryData() { + now := time.Now() currentTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location()) + stamp := time.Date(2022, 10, 1, 0, 0, 0, 0, now.Location()) + statisticTime := stamp.Add(+1 * time.Hour) + for statisticTime.Before(currentTime) || statisticTime.Equal(currentTime) { + summaryDurationStat(statisticTime) + statisticTime = statisticTime.Add(+1 * time.Hour) + } +} + +func summaryDurationStat(statisticTime time.Time) { + dateTime := timeutil.TimeStamp(statisticTime.Unix()) + dayTime := statisticTime.Format("2006-01-02") - m, _ := time.ParseDuration("-1h") - beginTime := currentTime.Add(m).Unix() - endTime := currentTime.Unix() - hourTime := currentTime.Add(m).Hour() + beginTime := statisticTime.Add(-1 * time.Hour).Unix() + hourTime := statisticTime.Add(-1 * time.Hour).Hour() + endTime := statisticTime.Unix() ciTasks, err := models.GetCloudbrainByTime(beginTime, endTime) if err != nil { @@ -99,6 +111,7 @@ func CloudbrainDurationStatisticHour() { ComputeResource: strings.Split(key, "/")[3], CardsUseDuration: 0, CardsTotalDuration: cardsTotalDuration, + CardsTotalNum: cardsTotalDuration / 1 / 60 / 60, CreatedUnix: timeutil.TimeStampNow(), } if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { @@ -115,7 +128,7 @@ func GetAiCenterNameByCode(centerCode string) string { if c2NetSequenceInfo != nil { aiCenterName = c2NetSequenceInfo.Content } else { - aiCenterName = "" + aiCenterName = centerCode } return aiCenterName @@ -151,31 +164,42 @@ func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, be if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) - } else { + } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) < endTime { cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + } else if int64(cloudbrain.Cloudbrain.StartTime) >= endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = 0 } } else { if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) - } else { + } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) < endTime { cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + } else if int64(cloudbrain.Cloudbrain.StartTime) >= endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += 0 } } } else { if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { - if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) - } else { + } else if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) } } else { - if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) - } else { + } else if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) } } - } } From b761e08da650254f1fce088775ccd3396848ccb3 Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 8 Nov 2022 10:55:18 +0800 Subject: [PATCH 003/253] fix-kanban --- models/cloudbrain_static.go | 2 +- routers/api/v1/repo/cloudbrain_dashboard.go | 1 + routers/repo/cloudbrain_statistic.go | 6 +++--- services/cloudbrain/util.go | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 92d9842a9..23cdebb9d 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -335,7 +335,7 @@ func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode s func GetCanUseCardInfo() ([]*ResourceQueue, error) { sess := x.NewSession() defer sess.Close() - sess.OrderBy("resource_queue.cluster DESC, resource_queue.ai_center_code ASC") + sess.OrderBy("resource_queue.cluster DESC, resource_queue.ai_center_name ASC") ResourceQueues := make([]*ResourceQueue, 0, 10) if err := sess.Table(&ResourceQueue{}).Find(&ResourceQueues); err != nil { log.Info("find error.") diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index e29468671..2737e299c 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1861,6 +1861,7 @@ func GetResourceQueues(ctx *context.Context) { for _, resourceQueue := range resourceQueues { if _, ok := aiCenterCodeMap[resourceQueue.AiCenterCode]; !ok { resourceQueue.AiCenterName = repo.GetAiCenterNameByCode(resourceQueue.AiCenterCode) + aiCenterCodeMap[resourceQueue.AiCenterCode] = resourceQueue.AiCenterCode Resource = append(Resource, resourceQueue) } } diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 5414c184b..fba2a9e07 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -124,9 +124,9 @@ func summaryDurationStat(statisticTime time.Time) { func GetAiCenterNameByCode(centerCode string) string { var aiCenterName string - c2NetSequenceInfo := cloudbrainService.C2NetSequenceInfoByCode(centerCode) - if c2NetSequenceInfo != nil { - aiCenterName = c2NetSequenceInfo.Content + aiCenterInfo := cloudbrainService.GetAiCenterInfoByCenterCode(centerCode) + if aiCenterInfo != nil { + aiCenterName = aiCenterInfo.Content } else { aiCenterName = centerCode } diff --git a/services/cloudbrain/util.go b/services/cloudbrain/util.go index 141807246..1d25ff8b1 100644 --- a/services/cloudbrain/util.go +++ b/services/cloudbrain/util.go @@ -33,7 +33,7 @@ func GetAiCenterShow(aiCenter string, ctx *context.Context) string { } -func C2NetSequenceInfoByCode(aiCenterCode string) *setting.C2NetSequenceInfo { +func GetAiCenterInfoByCenterCode(aiCenterCode string) *setting.C2NetSequenceInfo { if setting.AiCenterCodeAndNameMapInfo != nil { if info, ok := setting.AiCenterCodeAndNameMapInfo[aiCenterCode]; ok { return info From 5616a5a6ff1e9cd02aea2cef0f5ebaa342bf5074 Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 8 Nov 2022 16:42:34 +0800 Subject: [PATCH 004/253] fix-kanban --- models/cloudbrain.go | 22 +++++- modules/setting/setting.go | 4 +- routers/admin/cloudbrains.go | 1 + routers/api/v1/api.go | 2 - routers/api/v1/repo/cloudbrain_dashboard.go | 81 ++++++++++----------- routers/private/internal.go | 4 +- routers/repo/cloudbrain.go | 3 +- routers/repo/cloudbrain_statistic.go | 37 +++++++--- routers/repo/grampus.go | 3 +- routers/user/home.go | 2 + services/cloudbrain/util.go | 2 +- 11 files changed, 96 insertions(+), 65 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index a4fbf8e32..b8cd0bb8a 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1589,9 +1589,23 @@ func Cloudbrains(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { } } if (opts.AiCenter) != "" { - cond = cond.And( - builder.Like{"cloudbrain.ai_center", opts.AiCenter}, - ) + if opts.AiCenter == AICenterOfCloudBrainOne { + cond = cond.And( + builder.Eq{"cloudbrain.type": TypeCloudBrainOne}, + ) + } else if opts.AiCenter == AICenterOfCloudBrainTwo { + cond = cond.And( + builder.Eq{"cloudbrain.type": TypeCloudBrainTwo}, + ) + } else if opts.AiCenter == AICenterOfChengdu { + cond = cond.And( + builder.Eq{"cloudbrain.type": TypeCDCenter}, + ) + } else { + cond = cond.And( + builder.Like{"cloudbrain.ai_center", opts.AiCenter}, + ) + } } if (opts.Cluster) != "" { if opts.Cluster == "resource_cluster_openi" { @@ -2285,7 +2299,7 @@ func CloudbrainAllStatic(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, er } // sess.OrderBy("cloudbrain.created_unix DESC") cloudbrains := make([]*CloudbrainInfo, 0, setting.UI.IssuePagingNum) - if err := sess.Cols("status", "type", "job_type", "train_job_duration", "duration", "compute_resource", "created_unix", "start_time", "end_time", "work_server_number").Table(&Cloudbrain{}).Unscoped().Where(cond). + if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). Find(&cloudbrains); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index c16050500..bbc3dc167 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -608,6 +608,7 @@ var ( LocalCenterID string AiCenterInfo string AiCenterCodeAndNameInfo string + UsageRateBeginTime string }{} C2NetInfos *C2NetSqInfos @@ -1653,7 +1654,8 @@ func getGrampusConfig() { Grampus.Password = sec.Key("PASSWORD").MustString("") Grampus.SpecialPools = sec.Key("SPECIAL_POOL").MustString("") Grampus.C2NetSequence = sec.Key("C2NET_SEQUENCE").MustString("{\"sequence\":[{\"id\":1,\"name\":\"cloudbrain_one\",\"content\":\"鹏城云脑一号\",\"content_en\":\"Pencheng Cloudbrain Ⅰ\"},{\"id\":2,\"name\":\"cloudbrain_two\",\"content\":\"鹏城云脑二号\",\"content_en\":\"Pencheng Cloudbrain Ⅱ\"},{\"id\":3,\"name\":\"beida\",\"content\":\"北大人工智能集群系统\",\"content_en\":\"Peking University AI Center\"},{\"id\":4,\"name\":\"hefei\",\"content\":\"合肥类脑智能开放平台\",\"content_en\":\"Hefei AI Center\"},{\"id\":5,\"name\":\"wuhan\",\"content\":\"武汉人工智能计算中心\",\"content_en\":\"Wuhan AI Center\"},{\"id\":6,\"name\":\"xian\",\"content\":\"西安未来人工智能计算中心\",\"content_en\":\"Xi'an AI Center\"},{\"id\":7,\"pclcci\":\"more\",\"content\":\"鹏城云计算所\",\"content_en\":\"Pengcheng Cloud Computing Institute\"},{\"id\":8,\"name\":\"xuchang\",\"content\":\"中原人工智能计算中心\",\"content_en\":\"Zhongyuan AI Center\"},{\"id\":9,\"name\":\"chengdu\",\"content\":\"成都人工智能计算中心\",\"content_en\":\"Chengdu AI Center\"},{\"id\":10,\"name\":\"more\",\"content\":\"横琴先进智能计算中心\",\"content_en\":\"Hengqin AI Center\"},{\"id\":11,\"name\":\"more\",\"content\":\"国家超级计算济南中心\",\"content_en\":\"HPC & AI Center\"}]}") - Grampus.AiCenterCodeAndNameInfo = sec.Key("AiCenterCodeAndName").MustString("{\"sequence\":[{\"id\":1,\"name\":\"cloudbrain_one\",\"content\":\"鹏城云脑一号\",\"content_en\":\"Pencheng Cloudbrain Ⅰ\"},{\"id\":2,\"name\":\"cloudbrain_two\",\"content\":\"鹏城云脑二号\",\"content_en\":\"Pencheng Cloudbrain Ⅱ\"},{\"id\":3,\"name\":\"beida\",\"content\":\"北大人工智能集群系统\",\"content_en\":\"Peking University AI Center\"},{\"id\":4,\"name\":\"hefei\",\"content\":\"合肥类脑智能开放平台\",\"content_en\":\"Hefei AI Center\"},{\"id\":5,\"name\":\"wuhan\",\"content\":\"武汉人工智能计算中心\",\"content_en\":\"Wuhan AI Center\"},{\"id\":6,\"name\":\"xian\",\"content\":\"西安未来人工智能计算中心\",\"content_en\":\"Xi'an AI Center\"},{\"id\":7,\"pclcci\":\"more\",\"content\":\"鹏城云计算所\",\"content_en\":\"Pengcheng Cloud Computing Institute\"},{\"id\":8,\"name\":\"xuchang\",\"content\":\"中原人工智能计算中心\",\"content_en\":\"Zhongyuan AI Center\"},{\"id\":9,\"name\":\"chengdu\",\"content\":\"成都人工智能计算中心\",\"content_en\":\"Chengdu AI Center\"},{\"id\":10,\"name\":\"more\",\"content\":\"横琴先进智能计算中心\",\"content_en\":\"Hengqin AI Center\"},{\"id\":11,\"name\":\"more\",\"content\":\"国家超级计算济南中心\",\"content_en\":\"HPC & AI Center\"}]}") + Grampus.AiCenterCodeAndNameInfo = sec.Key("AI_CENTER_CODE_AND_NAME").MustString("{\"sequence\":[{\"id\":1,\"name\":\"cloudbrain_one\",\"content\":\"鹏城云脑一号\",\"content_en\":\"Pencheng Cloudbrain Ⅰ\"},{\"id\":2,\"name\":\"cloudbrain_two\",\"content\":\"鹏城云脑二号\",\"content_en\":\"Pencheng Cloudbrain Ⅱ\"},{\"id\":3,\"name\":\"beida\",\"content\":\"北大人工智能集群系统\",\"content_en\":\"Peking University AI Center\"},{\"id\":4,\"name\":\"hefei\",\"content\":\"合肥类脑智能开放平台\",\"content_en\":\"Hefei AI Center\"},{\"id\":5,\"name\":\"wuhan\",\"content\":\"武汉人工智能计算中心\",\"content_en\":\"Wuhan AI Center\"},{\"id\":6,\"name\":\"xian\",\"content\":\"西安未来人工智能计算中心\",\"content_en\":\"Xi'an AI Center\"},{\"id\":7,\"pclcci\":\"more\",\"content\":\"鹏城云计算所\",\"content_en\":\"Pengcheng Cloud Computing Institute\"},{\"id\":8,\"name\":\"xuchang\",\"content\":\"中原人工智能计算中心\",\"content_en\":\"Zhongyuan AI Center\"},{\"id\":9,\"name\":\"chengdu\",\"content\":\"成都人工智能计算中心\",\"content_en\":\"Chengdu AI Center\"},{\"id\":10,\"name\":\"more\",\"content\":\"横琴先进智能计算中心\",\"content_en\":\"Hengqin AI Center\"},{\"id\":11,\"name\":\"more\",\"content\":\"国家超级计算济南中心\",\"content_en\":\"HPC & AI Center\"}]}") + Grampus.UsageRateBeginTime = sec.Key("USAGE_RATE_BEGIN_TIME").MustString("2021-01-01 00:00:00") if Grampus.C2NetSequence != "" { if err := json.Unmarshal([]byte(Grampus.C2NetSequence), &C2NetInfos); err != nil { log.Error("Unmarshal(C2NetSequence) failed:%v", err) diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index d4a00fc34..d56c3ff62 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -97,6 +97,7 @@ func CloudBrains(ctx *context.Context) { for i, task := range ciTasks { ciTasks[i] = cloudbrainService.UpdateCloudbrainAiCenter(ciTasks[i]) + ciTasks[i].Cloudbrain.AiCenter = repo.GetAiCenterNameByCode(ciTasks[i].Cloudbrain.AiCenter) ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 8864e2e31..65d136870 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -603,10 +603,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/overview_resource", repo.GetCloudbrainResourceOverview) m.Get("/resource_usage_statistic", repo.GetDurationRateStatistic) m.Get("/resource_usage_rate_detail", repo.GetCloudbrainResourceUsageDetail) - m.Get("/resource_queues", repo.GetResourceQueues) m.Get("/apitest_for_statistic", repo.CloudbrainDurationStatisticForTest) - m.Get("/apipost_for_cloudbrain_data", repo.CloudbrainUpdateHistoryData) }) }, operationReq) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 2737e299c..86d8151fe 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/routers/repo" + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" "github.com/360EntSecGroup-Skylar/excelize/v2" ) @@ -747,8 +748,8 @@ func GetCloudbrainsDetailData(ctx *context.Context) { NeedRepoInfo: true, BeginTimeUnix: int64(recordBeginTime), EndTimeUnix: endTime.Unix(), - // AiCenter: aiCenter, - NeedDeleteInfo: needDeleteInfo, + AiCenter: aiCenter, + NeedDeleteInfo: needDeleteInfo, }) if err != nil { ctx.ServerError("Get job failed:", err) @@ -758,43 +759,42 @@ func GetCloudbrainsDetailData(ctx *context.Context) { nilTime := time.Time{} tasks := []models.TaskDetail{} for i, task := range ciTasks { - if aiCenter == "" || aiCenter == task.Cloudbrain.Spec.AiCenterCode { - ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource - var taskDetail models.TaskDetail - taskDetail.ID = ciTasks[i].Cloudbrain.ID - taskDetail.JobID = ciTasks[i].Cloudbrain.JobID - taskDetail.JobName = ciTasks[i].JobName - taskDetail.DisplayJobName = ciTasks[i].DisplayJobName - taskDetail.Status = ciTasks[i].Status - taskDetail.JobType = ciTasks[i].JobType - taskDetail.CreatedUnix = ciTasks[i].Cloudbrain.CreatedUnix - taskDetail.RunTime = ciTasks[i].Cloudbrain.TrainJobDuration - taskDetail.StartTime = ciTasks[i].StartTime - taskDetail.EndTime = ciTasks[i].EndTime - taskDetail.ComputeResource = ciTasks[i].ComputeResource - taskDetail.Type = ciTasks[i].Cloudbrain.Type - taskDetail.UserName = ciTasks[i].User.Name - taskDetail.RepoID = ciTasks[i].RepoID - if ciTasks[i].Repo != nil { - taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name - taskDetail.RepoAlias = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Alias - } - if ciTasks[i].Cloudbrain.WorkServerNumber >= 1 { - taskDetail.WorkServerNum = int64(ciTasks[i].Cloudbrain.WorkServerNumber) - } else { - taskDetail.WorkServerNum = 1 - } - taskDetail.CardDuration = repo.GetCloudbrainCardDuration(ciTasks[i].Cloudbrain) - taskDetail.WaitTime = repo.GetCloudbrainWaitTime(ciTasks[i].Cloudbrain) + task = cloudbrainService.UpdateCloudbrainAiCenter(task) + task.Cloudbrain.AiCenter = repo.GetAiCenterNameByCode(task.Cloudbrain.AiCenter) + var taskDetail models.TaskDetail + taskDetail.ID = ciTasks[i].Cloudbrain.ID + taskDetail.JobID = ciTasks[i].Cloudbrain.JobID + taskDetail.JobName = ciTasks[i].JobName + taskDetail.DisplayJobName = ciTasks[i].DisplayJobName + taskDetail.Status = ciTasks[i].Status + taskDetail.JobType = ciTasks[i].JobType + taskDetail.CreatedUnix = ciTasks[i].Cloudbrain.CreatedUnix + taskDetail.RunTime = ciTasks[i].Cloudbrain.TrainJobDuration + taskDetail.StartTime = ciTasks[i].StartTime + taskDetail.EndTime = ciTasks[i].EndTime + taskDetail.ComputeResource = ciTasks[i].ComputeResource + taskDetail.Type = ciTasks[i].Cloudbrain.Type + taskDetail.UserName = ciTasks[i].User.Name + taskDetail.RepoID = ciTasks[i].RepoID + if ciTasks[i].Repo != nil { + taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name + taskDetail.RepoAlias = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Alias + } + if ciTasks[i].Cloudbrain.WorkServerNumber >= 1 { + taskDetail.WorkServerNum = int64(ciTasks[i].Cloudbrain.WorkServerNumber) + } else { + taskDetail.WorkServerNum = 1 + } + taskDetail.CardDuration = repo.GetCloudbrainCardDuration(ciTasks[i].Cloudbrain) + taskDetail.WaitTime = repo.GetCloudbrainWaitTime(ciTasks[i].Cloudbrain) - if ciTasks[i].Cloudbrain.DeletedAt != nilTime || ciTasks[i].Repo == nil { - taskDetail.IsDelete = true - } else { - taskDetail.IsDelete = false - } - taskDetail.Spec = ciTasks[i].Spec - tasks = append(tasks, taskDetail) + if ciTasks[i].Cloudbrain.DeletedAt != nilTime || ciTasks[i].Repo == nil { + taskDetail.IsDelete = true + } else { + taskDetail.IsDelete = false } + taskDetail.Spec = ciTasks[i].Spec + tasks = append(tasks, taskDetail) } count := int64(len(tasks)) pager := context.NewPagination(int(count), pageSize, page, getTotalPage(count, pageSize)) @@ -1843,13 +1843,6 @@ func CloudbrainUpdateAiCenter(ctx *context.Context) { }) } -func CloudbrainUpdateHistoryData(ctx *context.Context) { - repo.UpdateDurationStatisticHistoryData() - ctx.JSON(http.StatusOK, map[string]interface{}{ - "message": 0, - }) -} - func GetResourceQueues(ctx *context.Context) { resourceQueues, err := models.GetCanUseCardInfo() if err != nil { diff --git a/routers/private/internal.go b/routers/private/internal.go index 3e2eeab31..ab5106290 100755 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -6,9 +6,10 @@ package private import ( - "code.gitea.io/gitea/routers/admin" "strings" + "code.gitea.io/gitea/routers/admin" + "code.gitea.io/gitea/routers/repo" "code.gitea.io/gitea/modules/log" @@ -53,6 +54,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/tool/update_repo_visit/:date", UpdateRepoVisit) m.Post("/task/history_handle/duration", repo.HandleTaskWithNoDuration) m.Post("/resources/specification/handle_historical_task", admin.RefreshHistorySpec) + m.Post("/duration_statisctic/history_handle", repo.CloudbrainUpdateHistoryData) }, CheckInternalToken) } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index de0e0e8bf..9fba84797 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2,7 +2,6 @@ package repo import ( "bufio" - "code.gitea.io/gitea/modules/urfs_client/urchin" "encoding/json" "errors" "fmt" @@ -16,6 +15,8 @@ import ( "time" "unicode/utf8" + "code.gitea.io/gitea/modules/urfs_client/urchin" + "code.gitea.io/gitea/modules/dataset" "code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index fba2a9e07..067920d56 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -1,11 +1,14 @@ package repo import ( + "net/http" "strings" "time" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/timeutil" cloudbrainService "code.gitea.io/gitea/services/cloudbrain" ) @@ -16,18 +19,22 @@ func CloudbrainDurationStatisticHour() { summaryDurationStat(currentTime) } -func UpdateDurationStatisticHistoryData() { +func UpdateDurationStatisticHistoryData() int64 { + var count int64 + recordBeginTime, _ := time.ParseInLocation("2006-01-02 15:04:05", setting.Grampus.UsageRateBeginTime, time.Local) now := time.Now() currentTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location()) - stamp := time.Date(2022, 10, 1, 0, 0, 0, 0, now.Location()) - statisticTime := stamp.Add(+1 * time.Hour) + statisticTime := recordBeginTime.Add(+1 * time.Hour) for statisticTime.Before(currentTime) || statisticTime.Equal(currentTime) { - summaryDurationStat(statisticTime) + countEach := summaryDurationStat(statisticTime) + count += countEach statisticTime = statisticTime.Add(+1 * time.Hour) } + return count } -func summaryDurationStat(statisticTime time.Time) { +func summaryDurationStat(statisticTime time.Time) int64 { + var count int64 dateTime := timeutil.TimeStamp(statisticTime.Unix()) dayTime := statisticTime.Format("2006-01-02") @@ -38,7 +45,7 @@ func summaryDurationStat(statisticTime time.Time) { ciTasks, err := models.GetCloudbrainByTime(beginTime, endTime) if err != nil { log.Info("GetCloudbrainByTime err: %v", err) - return + return 0 } cloudbrainMap := make(map[string]*models.Cloudbrain) models.LoadSpecs4CloudbrainInfo(ciTasks) @@ -56,7 +63,7 @@ func summaryDurationStat(statisticTime time.Time) { resourceQueues, err := models.GetCanUseCardInfo() if err != nil { log.Info("GetCanUseCardInfo err: %v", err) - return + return 0 } cardsTotalDurationMap := make(map[string]int) for _, resourceQueue := range resourceQueues { @@ -69,7 +76,7 @@ func summaryDurationStat(statisticTime time.Time) { if cloudbrainTable != nil { if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, centerCode, cardType); err != nil { log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) - return + return 0 } if _, ok := cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType+"/"+cloudbrainTable.ComputeResource]; !ok { cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType+"/"+cloudbrainTable.ComputeResource] = 0 @@ -90,6 +97,7 @@ func summaryDurationStat(statisticTime time.Time) { if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) } + count++ delete(cardsTotalDurationMap, cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType+"/"+cloudbrainTable.ComputeResource) } } @@ -98,7 +106,7 @@ func summaryDurationStat(statisticTime time.Time) { for key, cardsTotalDuration := range cardsTotalDurationMap { if err := models.DeleteCloudbrainDurationStatisticHour(dayTime, hourTime, strings.Split(key, "/")[1], strings.Split(key, "/")[2]); err != nil { log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error()) - return + return 0 } cloudbrainDurationStat := models.CloudbrainDurationStatistic{ DateTime: dateTime, @@ -117,9 +125,11 @@ func summaryDurationStat(statisticTime time.Time) { if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil { log.Error("Insert cloudbrainDurationStat failed: %v", err.Error()) } + count++ } log.Info("finish summary cloudbrainDurationStat") + return count } func GetAiCenterNameByCode(centerCode string) string { @@ -131,7 +141,6 @@ func GetAiCenterNameByCode(centerCode string) string { aiCenterName = centerCode } return aiCenterName - } func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, beginTime int64, endTime int64) map[string]map[string]int { @@ -205,3 +214,11 @@ func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, be return cloudBrainCenterCodeAndCardType } + +func CloudbrainUpdateHistoryData(ctx *context.Context) { + count := UpdateDurationStatisticHistoryData() + ctx.JSON(http.StatusOK, map[string]interface{}{ + "message": 0, + "count": count, + }) +} diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 4718fe04f..bae4956c2 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -1,7 +1,6 @@ package repo import ( - "code.gitea.io/gitea/modules/urfs_client/urchin" "encoding/json" "errors" "fmt" @@ -13,6 +12,8 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/urfs_client/urchin" + "code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" "code.gitea.io/gitea/modules/dataset" diff --git a/routers/user/home.go b/routers/user/home.go index 348a295ed..4d6a96204 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -23,6 +23,7 @@ import ( "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/routers/repo" cloudbrainService "code.gitea.io/gitea/services/cloudbrain" issue_service "code.gitea.io/gitea/services/issue" pull_service "code.gitea.io/gitea/services/pull" @@ -839,6 +840,7 @@ func Cloudbrains(ctx *context.Context) { models.LoadSpecs4CloudbrainInfo(ciTasks) for i, _ := range ciTasks { ciTasks[i] = cloudbrainService.UpdateCloudbrainAiCenter(ciTasks[i]) + ciTasks[i].Cloudbrain.AiCenter = repo.GetAiCenterNameByCode(ciTasks[i].Cloudbrain.AiCenter) ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = ciTasks[i].ComputeResource diff --git a/services/cloudbrain/util.go b/services/cloudbrain/util.go index 1d25ff8b1..dc9177ecf 100644 --- a/services/cloudbrain/util.go +++ b/services/cloudbrain/util.go @@ -63,7 +63,7 @@ func UpdateCloudbrainAiCenter(cloudbrain *models.CloudbrainInfo) *models.Cloudbr cloudbrain.Cloudbrain.AiCenter = models.AICenterOfChengdu cloudbrain.Cloudbrain.Cluster = models.OpenICluster } - if cloudbrain.Cloudbrain.Type == models.TypeC2Net && cloudbrain.Cloudbrain.AiCenter != "" { + if cloudbrain.Cloudbrain.Type == models.TypeC2Net { cloudbrain.Cloudbrain.AiCenter = getAiCenterCode(cloudbrain.Cloudbrain.AiCenter) cloudbrain.Cloudbrain.Cluster = models.C2NetCluster } From 72d3ebb2d4d8379629fdb3f5e2814f9e62cc7fca Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 8 Nov 2022 17:09:47 +0800 Subject: [PATCH 005/253] fix-kanban --- routers/admin/cloudbrains.go | 5 +++-- routers/api/v1/repo/cloudbrain_dashboard.go | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index d56c3ff62..8ca8344a8 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -189,7 +189,8 @@ func DownloadCloudBrains(ctx *context.Context) { } models.LoadSpecs4CloudbrainInfo(pageRecords) for _, record := range pageRecords { - + record = cloudbrainService.UpdateCloudbrainAiCenter(record) + record.Cloudbrain.AiCenter = repo.GetAiCenterNameByCode(record.Cloudbrain.AiCenter) for k, v := range allValues(row, record, ctx) { f.SetCellValue(cloudBrain, k, v) } @@ -211,7 +212,7 @@ func allValues(row int, rs *models.CloudbrainInfo, ctx *context.Context) map[str return map[string]string{getCellName("A", row): rs.DisplayJobName, getCellName("B", row): repo.GetCloudbrainCluster(rs.Cloudbrain, ctx), getCellName("C", row): rs.JobType, getCellName("D", row): rs.Status, getCellName("E", row): time.Unix(int64(rs.Cloudbrain.CreatedUnix), 0).Format(CREATE_TIME_FORMAT), getCellName("F", row): getDurationTime(rs), getCellName("G", row): rs.ComputeResource, - getCellName("H", row): repo.GetCloudbrainAiCenter(rs.Cloudbrain, ctx), getCellName("I", row): getCloudbrainCardType(rs), + getCellName("H", row): rs.Cloudbrain.AiCenter, getCellName("I", row): getCloudbrainCardType(rs), getCellName("J", row): rs.Name, getCellName("K", row): getRepoPathName(rs), getCellName("L", row): rs.JobName, } } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 86d8151fe..7b466e23c 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1225,7 +1225,8 @@ func DownloadCloudBrainBoard(ctx *context.Context) { } models.LoadSpecs4CloudbrainInfo(pageRecords) for _, record := range pageRecords { - + record = cloudbrainService.UpdateCloudbrainAiCenter(record) + record.Cloudbrain.AiCenter = repo.GetAiCenterNameByCode(record.Cloudbrain.AiCenter) for k, v := range allCloudbrainValues(row, record, ctx) { f.SetCellValue(cloudBrain, k, v) } @@ -1264,7 +1265,7 @@ func allCloudbrainValues(row int, rs *models.CloudbrainInfo, ctx *context.Contex getCellName("G", row): rs.TrainJobDuration, getCellName("H", row): repo.GetCloudbrainCardDuration(rs.Cloudbrain), getCellName("I", row): getBrainStartTime(rs), getCellName("J", row): getBrainEndTime(rs), getCellName("K", row): rs.ComputeResource, getCellName("L", row): getCloudbrainCardType(rs), - getCellName("M", row): getWorkServerNum(rs), getCellName("N", row): repo.GetCloudbrainAiCenter(rs.Cloudbrain, ctx), + getCellName("M", row): getWorkServerNum(rs), getCellName("N", row): rs.Cloudbrain.AiCenter, getCellName("O", row): getCloudbrainFlavorName(rs), getCellName("P", row): rs.Name, getCellName("Q", row): getBrainRepo(rs), getCellName("R", row): rs.JobName, getCellName("S", row): getBrainDeleteTime(rs), } From 853a19f3a939a6305a801726c6718c5fd47309c7 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 9 Nov 2022 10:16:54 +0800 Subject: [PATCH 006/253] fix-kanban --- routers/api/v1/repo/cloudbrain_dashboard.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 7b466e23c..455457865 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1557,7 +1557,7 @@ func getBeginAndEndTime(ctx *context.Context) (time.Time, time.Time) { ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) return beginTime, endTime } - brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() + brainRecordBeginTime := recordCloudbrainDuration[0].DateTime.AsTime() beginTime = brainRecordBeginTime endTime = now } else if queryType == "today" { @@ -1599,7 +1599,7 @@ func getBeginAndEndTime(ctx *context.Context) (time.Time, time.Time) { ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) return beginTime, endTime } - brainRecordBeginTime := recordCloudbrainDuration[0].CreatedUnix.AsTime() + brainRecordBeginTime := recordCloudbrainDuration[0].DateTime.AsTime() beginTime = brainRecordBeginTime endTime = now } else { From 5459a847b54f3a86ca214f578852cf515daeeebf Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 9 Nov 2022 16:43:10 +0800 Subject: [PATCH 007/253] fix-kanban --- routers/api/v1/repo/cloudbrain_dashboard.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 455457865..9d0d58f81 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -760,7 +760,6 @@ func GetCloudbrainsDetailData(ctx *context.Context) { tasks := []models.TaskDetail{} for i, task := range ciTasks { task = cloudbrainService.UpdateCloudbrainAiCenter(task) - task.Cloudbrain.AiCenter = repo.GetAiCenterNameByCode(task.Cloudbrain.AiCenter) var taskDetail models.TaskDetail taskDetail.ID = ciTasks[i].Cloudbrain.ID taskDetail.JobID = ciTasks[i].Cloudbrain.JobID @@ -776,6 +775,7 @@ func GetCloudbrainsDetailData(ctx *context.Context) { taskDetail.Type = ciTasks[i].Cloudbrain.Type taskDetail.UserName = ciTasks[i].User.Name taskDetail.RepoID = ciTasks[i].RepoID + taskDetail.AiCenter = repo.GetAiCenterNameByCode(task.Cloudbrain.AiCenter) if ciTasks[i].Repo != nil { taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name taskDetail.RepoAlias = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Alias From 6364b5fd88cdc64fbafe7868c3ed5df450e53232 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 9 Nov 2022 17:16:30 +0800 Subject: [PATCH 008/253] fix-kanban --- models/cloudbrain_static.go | 2 +- routers/api/v1/repo/cloudbrain_dashboard.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 23cdebb9d..92d9842a9 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -335,7 +335,7 @@ func DeleteCloudbrainDurationStatisticHour(date string, hour int, aiCenterCode s func GetCanUseCardInfo() ([]*ResourceQueue, error) { sess := x.NewSession() defer sess.Close() - sess.OrderBy("resource_queue.cluster DESC, resource_queue.ai_center_name ASC") + sess.OrderBy("resource_queue.cluster DESC, resource_queue.ai_center_code ASC") ResourceQueues := make([]*ResourceQueue, 0, 10) if err := sess.Table(&ResourceQueue{}).Find(&ResourceQueues); err != nil { log.Info("find error.") diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 9d0d58f81..2dd836aa8 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1434,7 +1434,7 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { resourceDetail.QueueCode = resourceQueue.QueueCode resourceDetail.Cluster = resourceQueue.Cluster resourceDetail.AiCenterCode = resourceQueue.AiCenterCode - resourceDetail.AiCenterName = aiCenterName + "/" + resourceQueue.AiCenterCode + resourceDetail.AiCenterName = resourceQueue.AiCenterCode + "/" + aiCenterName resourceDetail.ComputeResource = resourceQueue.ComputeResource resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")" resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum @@ -1447,7 +1447,7 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { resourceDetail.QueueCode = resourceQueue.QueueCode resourceDetail.Cluster = resourceQueue.Cluster resourceDetail.AiCenterCode = resourceQueue.AiCenterCode - resourceDetail.AiCenterName = aiCenterName + "/" + resourceQueue.AiCenterCode + resourceDetail.AiCenterName = resourceQueue.AiCenterCode + "/" + aiCenterName resourceDetail.ComputeResource = resourceQueue.ComputeResource resourceDetail.AccCardType = resourceQueue.AccCardType + "(" + resourceQueue.ComputeResource + ")" resourceDetail.CardsTotalNum = resourceQueue.CardsTotalNum From 18dd52769232071966a77c892eef1691ad8a858a Mon Sep 17 00:00:00 2001 From: chenshihai Date: Wed, 9 Nov 2022 17:39:37 +0800 Subject: [PATCH 009/253] =?UTF-8?q?=E4=BA=91=E8=84=91=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=88=97=E8=A1=A8=E9=A1=B5=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- templates/admin/cloudbrain/list.tmpl | 12 ++++++------ templates/admin/cloudbrain/search.tmpl | 10 ++++------ templates/admin/cloudbrain/search_dashboard.tmpl | 10 ++++------ templates/user/dashboard/cloudbrains.tmpl | 14 +++++++------- 4 files changed, 21 insertions(+), 25 deletions(-) diff --git a/templates/admin/cloudbrain/list.tmpl b/templates/admin/cloudbrain/list.tmpl index b6dfec77d..20e704a4d 100755 --- a/templates/admin/cloudbrain/list.tmpl +++ b/templates/admin/cloudbrain/list.tmpl @@ -170,7 +170,7 @@
- {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}} + {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}
@@ -184,16 +184,16 @@ spanEl.setAttribute('title', cardType); spanEl.innerText = cardType; - var cluster = spec.Cluster || '--'; + var cluster = {{.Cluster}} || '--'; var clusterName = document.querySelector('.cloudbrain_debug').dataset['cluster' + cluster[0] + cluster.toLocaleLowerCase().slice(1)] || '--'; spanEl = document.querySelector('.cluster_{{.DisplayJobName}}_{{$JobID}}'); spanEl.setAttribute('title', cluster); spanEl.innerText = clusterName; - var aiCenter = spec.AiCenterName || '--'; - spanEl = document.querySelector('.aicenter_{{.DisplayJobName}}_{{$JobID}}'); - spanEl.setAttribute('title', aiCenter); - spanEl.innerText = aiCenter; + // var aiCenter = spec.AiCenterName || '--'; + // spanEl = document.querySelector('.aicenter_{{.DisplayJobName}}_{{$JobID}}'); + // spanEl.setAttribute('title', aiCenter); + // spanEl.innerText = aiCenter; })(); diff --git a/templates/admin/cloudbrain/search.tmpl b/templates/admin/cloudbrain/search.tmpl index 12e8a7515..09e5f865e 100644 --- a/templates/admin/cloudbrain/search.tmpl +++ b/templates/admin/cloudbrain/search.tmpl @@ -71,22 +71,20 @@ document.addEventListener('DOMContentLoaded', function() { $.ajax({ type: "GET", - url: "/api/v1/cloudbrain/get_center_info", + url: "/api/v1/cloudbrainboard/cloudbrain/resource_queues", dataType: "json", data: {}, success: function (res) { - var data = res || []; + var data = res.resourceQueues || []; var aiCenterSelEl = $('#aiCenter-sel'); var itemEl = aiCenterSelEl.find('.menu .item').eq(0); var selectAiCenterCode = aiCenterSelEl.find('.default').attr('aicenter'); var selectAiCenterName = ''; var lang = document.querySelector('html').getAttribute('lang') || 'en-US'; - var except = ['', 'more']; for (var i = 0, iLen = data.length; i < iLen; i++) { var dataI = data[i]; - var aiCenterCode = dataI.name; - if (except.indexOf(aiCenterCode) >= 0) continue; - var aiCenterName = lang === 'en-US' ? dataI.content_en : dataI.content; + var aiCenterCode = dataI.AiCenterCode; + var aiCenterName = dataI.AiCenterName; var itemClone = itemEl.clone(); var oHref = itemClone.attr('href'); var oId = itemClone.attr('id'); diff --git a/templates/admin/cloudbrain/search_dashboard.tmpl b/templates/admin/cloudbrain/search_dashboard.tmpl index 2bf738dc9..7c4c1527d 100644 --- a/templates/admin/cloudbrain/search_dashboard.tmpl +++ b/templates/admin/cloudbrain/search_dashboard.tmpl @@ -85,22 +85,20 @@ document.addEventListener('DOMContentLoaded', function() { $.ajax({ type: "GET", - url: "/api/v1/cloudbrain/get_center_info", + url: "/api/v1/cloudbrainboard/cloudbrain/resource_queues", dataType: "json", data: {}, success: function (res) { - var data = res || []; + var data = res.resourceQueues || []; var aiCenterSelEl = $('#aiCenter-sel'); var itemEl = aiCenterSelEl.find('.menu .item').eq(0); var selectAiCenterCode = aiCenterSelEl.find('.default').attr('aicenter'); var selectAiCenterName = ''; var lang = document.querySelector('html').getAttribute('lang') || 'en-US'; - var except = ['', 'more']; for (var i = 0, iLen = data.length; i < iLen; i++) { var dataI = data[i]; - var aiCenterCode = dataI.name; - if (except.indexOf(aiCenterCode) >= 0) continue; - var aiCenterName = lang === 'en-US' ? dataI.content_en : dataI.content; + var aiCenterCode = dataI.AiCenterCode; + var aiCenterName = dataI.AiCenterName; var itemClone = itemEl.clone(); var oHref = itemClone.attr('href'); var oId = itemClone.attr('id'); diff --git a/templates/user/dashboard/cloudbrains.tmpl b/templates/user/dashboard/cloudbrains.tmpl index e3527b25e..a42dbdda8 100755 --- a/templates/user/dashboard/cloudbrains.tmpl +++ b/templates/user/dashboard/cloudbrains.tmpl @@ -65,7 +65,7 @@
- {{range .Tasks}} + {{range .Tasks}} {{if .Repo}}
@@ -154,7 +154,7 @@
- {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}} + {{if .AiCenter}}{{.AiCenter}}{{else}}--{{end}}
@@ -168,16 +168,16 @@ spanEl.setAttribute('title', cardType); spanEl.innerText = cardType; - var cluster = spec.Cluster || '--'; + var cluster = {{.Cluster}} || '--'; var clusterName = document.querySelector('.cloudbrain_debug').dataset['cluster' + cluster[0] + cluster.toLocaleLowerCase().slice(1)] || '--'; spanEl = document.querySelector('.cluster_{{.DisplayJobName}}_{{$JobID}}'); spanEl.setAttribute('title', cluster); spanEl.innerText = clusterName; - var aiCenter = spec.AiCenterName || '--'; - spanEl = document.querySelector('.aicenter_{{.DisplayJobName}}_{{$JobID}}'); - spanEl.setAttribute('title', aiCenter); - spanEl.innerText = aiCenter; + // var aiCenter = spec.AiCenterName || '--'; + // spanEl = document.querySelector('.aicenter_{{.DisplayJobName}}_{{$JobID}}'); + // spanEl.setAttribute('title', aiCenter); + // spanEl.innerText = aiCenter; })(); From fad494cc17051483f601afc4a1da23ef6c06eb4a Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 9 Nov 2022 17:40:05 +0800 Subject: [PATCH 010/253] fix-kanban --- routers/api/v1/repo/cloudbrain_dashboard.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 2dd836aa8..0cb883214 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -704,6 +704,16 @@ func GetCloudbrainsDetailData(ctx *context.Context) { aiCenter := ctx.Query("aiCenter") needDeleteInfo := ctx.Query("needDeleteInfo") + if cloudBrainType == models.TypeCloudBrainOne && aiCenter == models.AICenterOfCloudBrainOne { + aiCenter = "" + } + if cloudBrainType == models.TypeCloudBrainTwo && aiCenter == models.AICenterOfCloudBrainTwo { + aiCenter = "" + } + if cloudBrainType == models.TypeCDCenter && aiCenter == models.AICenterOfChengdu { + aiCenter = "" + } + page := ctx.QueryInt("page") pageSize := ctx.QueryInt("pagesize") if page <= 0 { @@ -733,7 +743,7 @@ func GetCloudbrainsDetailData(ctx *context.Context) { keyword := strings.Trim(ctx.Query("q"), " ") - ciTasks, _, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + ciTasks, count, err := models.CloudbrainAll(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, PageSize: pageSize, @@ -796,7 +806,6 @@ func GetCloudbrainsDetailData(ctx *context.Context) { taskDetail.Spec = ciTasks[i].Spec tasks = append(tasks, taskDetail) } - count := int64(len(tasks)) pager := context.NewPagination(int(count), pageSize, page, getTotalPage(count, pageSize)) pager.SetDefaultParams(ctx) pager.AddParam(ctx, "listType", "ListType") From da78669bc72419cd3bfe5ce7a5d4bb2453bbb219 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Wed, 9 Nov 2022 17:46:06 +0800 Subject: [PATCH 011/253] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/modelarts/modelarts.go | 69 +++- modules/modelarts_cd/modelarts.go | 51 +-- modules/setting/setting.go | 22 +- modules/structs/cloudbrain.go | 8 + options/locale/locale_en-US.ini | 4 + options/locale/locale_zh-CN.ini | 3 + routers/api/v1/api.go | 6 + routers/api/v1/repo/cloudbrain.go | 3 + routers/repo/aisafety.go | 9 +- routers/repo/cloudbrain.go | 26 +- routers/repo/grampus.go | 4 +- routers/repo/modelarts.go | 15 +- .../cloudbrain/cloudbrainTask/notebook.go | 330 ++++++++++++++++++ services/cloudbrain/cloudbrainTask/train.go | 12 + services/cloudbrain/util.go | 39 ++- 15 files changed, 501 insertions(+), 100 deletions(-) create mode 100644 services/cloudbrain/cloudbrainTask/notebook.go diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index dd502dfd0..65984d95f 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -1,13 +1,19 @@ package modelarts import ( + "encoding/base64" "encoding/json" "errors" "fmt" + "io/ioutil" + "net/http" + "os" "path" "strconv" "strings" + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/modules/modelarts_cd" "code.gitea.io/gitea/models" @@ -276,7 +282,7 @@ func GenerateTask(ctx *context.Context, jobName, uuid, description, flavor strin return nil } -func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, description, imageId string, spec *models.Specification) error { +func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, description, imageId string, spec *models.Specification, bootFile string) (string, error) { if poolInfos == nil { json.Unmarshal([]byte(setting.PoolInfos), &poolInfos) } @@ -284,7 +290,7 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc imageName, err := GetNotebookImageName(imageId) if err != nil { log.Error("GetNotebookImageName failed: %v", err.Error()) - return err + return "", err } createTime := timeutil.TimeStampNow() jobResult, err := createNotebook2(models.CreateNotebook2Params{ @@ -316,10 +322,10 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc }) if errTemp != nil { log.Error("InsertCloudbrainTemp failed: %v", errTemp.Error()) - return errTemp + return "", errTemp } } - return err + return "", err } task := &models.Cloudbrain{ Status: jobResult.Status, @@ -334,6 +340,7 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc Uuid: uuid, ComputeResource: models.NPUResource, Image: imageName, + BootFile: bootFile, Description: description, CreatedUnix: createTime, UpdatedUnix: createTime, @@ -342,12 +349,12 @@ func GenerateNotebook2(ctx *context.Context, displayJobName, jobName, uuid, desc err = models.CreateCloudbrain(task) if err != nil { - return err + return "", err } stringId := strconv.FormatInt(task.ID, 10) notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, stringId, displayJobName, models.ActionCreateDebugNPUTask) - return nil + return jobResult.ID, nil } func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (jobId string, err error) { @@ -907,6 +914,11 @@ func HandleNotebookInfo(task *models.Cloudbrain) error { if task.FlavorCode == "" { task.FlavorCode = result.Flavor } + + if oldStatus != task.Status && task.Status == string(models.ModelArtsRunning) && task.BootFile != "" { + uploadNoteBookFile(task, result) + + } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err) @@ -917,6 +929,51 @@ func HandleNotebookInfo(task *models.Cloudbrain) error { return nil } +func uploadNoteBookFile(task *models.Cloudbrain, result *models.GetNotebook2Result) { + jupyterUrl := result.Url + "?token=" + result.Token + client := getRestyClient() + res, err := client.R().Get(jupyterUrl) + codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath + if err != nil { + log.Error("browser jupyterUrl failed:%v", task.DisplayJobName, err) + + } else { + cookies := res.Cookies() + xsrf := "" + for _, cookie := range cookies { + if cookie.Name == "_xsrf" { + xsrf = cookie.Value + } + } + + fileContents, err := ioutil.ReadFile(codePath + "/" + task.BootFile) + if err != nil { + log.Error("read jupyter file failed:%v", task.DisplayJobName, err) + } + + base64Content := base64.StdEncoding.EncodeToString(fileContents) + + uploadUrl := result.Url + "/api/contents/" + path.Base(task.BootFile) + res, err = client.R(). + SetCookies(cookies). + SetHeader("X-XSRFToken", xsrf). + SetBody(map[string]interface{}{ + "type": "file", + "format": "base64", + "name": path.Base(task.BootFile), + "path": path.Base(task.BootFile), + "content": base64Content}). + Put(uploadUrl) + if err != nil { + log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) + } else if res.StatusCode() != http.StatusCreated { + log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) + } + + } + go os.RemoveAll(codePath) +} + func SyncTempStatusJob() { jobs, err := models.GetCloudBrainTempJobs() if err != nil { diff --git a/modules/modelarts_cd/modelarts.go b/modules/modelarts_cd/modelarts.go index 330b048ca..712cdc89b 100755 --- a/modules/modelarts_cd/modelarts.go +++ b/modules/modelarts_cd/modelarts.go @@ -88,11 +88,11 @@ type Parameters struct { } `json:"parameter"` } -func GenerateNotebook(ctx *context.Context, displayJobName, jobName, uuid, description, imageId string, spec *models.Specification) error { +func GenerateNotebook(ctx *context.Context, displayJobName, jobName, uuid, description, imageId string, spec *models.Specification, bootFile string) (string, error) { imageName, err := GetNotebookImageName(imageId) if err != nil { log.Error("GetNotebookImageName failed: %v", err.Error()) - return err + return "", err } createTime := timeutil.TimeStampNow() jobResult, err := createNotebook(models.CreateNotebookWithoutPoolParams{ @@ -123,10 +123,10 @@ func GenerateNotebook(ctx *context.Context, displayJobName, jobName, uuid, descr }) if errTemp != nil { log.Error("InsertCloudbrainTemp failed: %v", errTemp.Error()) - return errTemp + return "", errTemp } } - return err + return "", err } task := &models.Cloudbrain{ Status: jobResult.Status, @@ -145,16 +145,17 @@ func GenerateNotebook(ctx *context.Context, displayJobName, jobName, uuid, descr CreatedUnix: createTime, UpdatedUnix: createTime, Spec: spec, + BootFile: bootFile, } err = models.CreateCloudbrain(task) if err != nil { - return err + return "", err } stringId := strconv.FormatInt(task.ID, 10) notification.NotifyOtherTask(ctx.User, ctx.Repo.Repository, stringId, displayJobName, models.ActionCreateDebugNPUTask) - return nil + return jobResult.ID, nil } func GetNotebookImageName(imageId string) (string, error) { @@ -175,41 +176,3 @@ func GetNotebookImageName(imageId string) (string, error) { return imageName, nil } - -/* -func HandleNotebookInfo(task *models.Cloudbrain) error { - - result, err := GetNotebook(task.JobID) - if err != nil { - log.Error("GetNotebook2(%s) failed:%v", task.DisplayJobName, err) - return err - } - - if result != nil { - oldStatus := task.Status - task.Status = result.Status - if task.StartTime == 0 && result.Lease.UpdateTime > 0 { - task.StartTime = timeutil.TimeStamp(result.Lease.UpdateTime / 1000) - } - if task.EndTime == 0 && models.IsModelArtsDebugJobTerminal(task.Status) { - task.EndTime = timeutil.TimeStampNow() - } - task.CorrectCreateUnix() - task.ComputeAndSetDuration() - if oldStatus != task.Status { - notification.NotifyChangeCloudbrainStatus(task, oldStatus) - } - if task.FlavorCode == "" { - task.FlavorCode = result.Flavor - } - err = models.UpdateJob(task) - if err != nil { - log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err) - return err - } - } - - return nil -} - -*/ diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 88557ee60..a358a85d4 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -609,9 +609,9 @@ var ( AiCenterInfo string }{} - C2NetInfos *C2NetSqInfos - CenterInfos *AiCenterInfos - C2NetMapInfo map[string]*C2NetSequenceInfo + C2NetInfos *C2NetSqInfos + CenterInfos *AiCenterInfos + C2NetMapInfo map[string]*C2NetSequenceInfo //elk config ElkUrl string @@ -712,6 +712,16 @@ var ( TeamName string }{} + FileNoteBook = struct { + ProjectName string + ImageGPU string + SpecIdGPU int64 + SpecIdCPU int64 + ImageIdNPU string + ImageNPU string + SpecIdNPU int64 + }{} + ModelConvert = struct { GPU_PYTORCH_IMAGE string GpuQueue string @@ -1655,9 +1665,9 @@ func getGrampusConfig() { if err := json.Unmarshal([]byte(Grampus.C2NetSequence), &C2NetInfos); err != nil { log.Error("Unmarshal(C2NetSequence) failed:%v", err) } - C2NetMapInfo=make(map[string]*C2NetSequenceInfo) - for _,value :=range C2NetInfos.C2NetSqInfo{ - C2NetMapInfo[value.Name]=value + C2NetMapInfo = make(map[string]*C2NetSequenceInfo) + for _, value := range C2NetInfos.C2NetSqInfo { + C2NetMapInfo[value.Name] = value } } Grampus.SyncScriptProject = sec.Key("SYNC_SCRIPT_PROJECT").MustString("script_for_grampus") diff --git a/modules/structs/cloudbrain.go b/modules/structs/cloudbrain.go index 866c85dad..9ea5601c9 100644 --- a/modules/structs/cloudbrain.go +++ b/modules/structs/cloudbrain.go @@ -41,6 +41,14 @@ type CreateTrainJobOption struct { SpecId int64 `json:"spec_id" binding:"Required"` } +type CreateFileNotebookJobOption struct { + Type int `json:"type"` //0 CPU 1 GPU 2 NPU + File string `json:"file" binding:"Required"` + BranchName string `json:"branch_name" binding:"Required"` + OwnerName string `json:"owner_name" binding:"Required"` + ProjectName string `json:"project_name" binding:"Required"` +} + type Cloudbrain struct { ID int64 `json:"id"` JobID string `json:"job_id"` diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index e8e8722e2..3f0c0a010 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1007,6 +1007,7 @@ readme = README readme_helper = Select a README file template. auto_init = Initialize Repository (Adds .gitignore, License and README) create_repo = Create Repository +failed_to_create_repo=Failed to create repository, please try again later. create_course = Publish Course failed_to_create_course=Failed to publish course, please try again later. default_branch = Default Branch @@ -1041,6 +1042,9 @@ model_experience = Model Experience model_noright=You have no right to do the operation. model_rename=Duplicate model name, please modify model name. +notebook_file_not_exist=Notebook file does not exist. +notebook_select_wrong=Please select a Notebook(.ipynb) file first. + date=Date repo_add=Project Increment repo_total=Project Total diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index d13c99443..8b635112b 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1013,6 +1013,7 @@ readme=自述 readme_helper=选择自述文件模板。 auto_init=初始化存储库 (添加. gitignore、许可证和自述文件) create_repo=创建项目 +failed_to_create_repo=创建项目失败，请稍后再试。 create_course=发布课程 failed_to_create_course=发布课程失败，请稍后再试。 default_branch=默认分支 @@ -1041,6 +1042,8 @@ model_experience = 模型体验 model_noright=您没有操作权限。 model_rename=模型名称重复，请修改模型名称 +notebook_file_not_exist=Notebook文件不存在。 +notebook_select_wrong=请先选择Notebook（.ipynb）文件。 date=日期 repo_add=新增项目 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 813b77c8c..b7e3c6415 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -736,6 +736,12 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/my_favorite", repo.MyFavoriteDatasetMultiple) }, reqToken(), repoAssignment()) + m.Group("/file_notebook", func() { + + m.Post("/create", reqToken(), reqWeChat(), bind(api.CreateFileNotebookJobOption{}), repo.CreateFileNoteBook) + + }) + m.Group("/repos", func() { m.Get("/search", repo.Search) diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 2e25fdefe..5ea797e97 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -79,6 +79,9 @@ func CloudBrainShow(ctx *context.APIContext) { ctx.JSON(http.StatusOK, models.BaseMessageWithDataApi{Code: 0, Message: "", Data: convert.ToCloudBrain(task)}) } +func CreateFileNoteBook(ctx *context.APIContext, option api.CreateFileNotebookJobOption) { + cloudbrainTask.FileNotebookCreate(ctx.Context, option) +} func CreateCloudBrain(ctx *context.APIContext, option api.CreateTrainJobOption) { if option.Type == cloudbrainTask.TaskTypeCloudbrainOne { diff --git a/routers/repo/aisafety.go b/routers/repo/aisafety.go index b638a486b..6176fcda5 100644 --- a/routers/repo/aisafety.go +++ b/routers/repo/aisafety.go @@ -11,7 +11,8 @@ import ( "os" "strconv" "strings" - "time" + + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/aisafety" @@ -483,7 +484,6 @@ func isTaskNotFinished(status string) bool { } func AiSafetyCreateForGetGPU(ctx *context.Context) { - t := time.Now() ctx.Data["PageIsCloudBrain"] = true ctx.Data["IsCreate"] = true ctx.Data["type"] = models.TypeCloudBrainOne @@ -497,7 +497,7 @@ func AiSafetyCreateForGetGPU(ctx *context.Context) { log.Info("GPUBaseDataSetUUID=" + setting.ModelSafetyTest.GPUBaseDataSetUUID) log.Info("GPUCombatDataSetName=" + setting.ModelSafetyTest.GPUCombatDataSetName) log.Info("GPUCombatDataSetUUID=" + setting.ModelSafetyTest.GPUCombatDataSetUUID) - var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name) ctx.Data["display_job_name"] = displayJobName prepareCloudbrainOneSpecs(ctx) queuesDetail, _ := cloudbrain.GetQueuesDetail() @@ -514,12 +514,11 @@ func AiSafetyCreateForGetGPU(ctx *context.Context) { } func AiSafetyCreateForGetNPU(ctx *context.Context) { - t := time.Now() ctx.Data["PageIsCloudBrain"] = true ctx.Data["IsCreate"] = true ctx.Data["type"] = models.TypeCloudBrainTwo ctx.Data["compute_resource"] = models.NPUResource - var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name) ctx.Data["display_job_name"] = displayJobName ctx.Data["datasetType"] = models.TypeCloudBrainTwo ctx.Data["BaseDataSetName"] = setting.ModelSafetyTest.NPUBaseDataSetName diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 61c2925b8..607d05d3c 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -2,7 +2,6 @@ package repo import ( "bufio" - "code.gitea.io/gitea/modules/urfs_client/urchin" "encoding/json" "errors" "fmt" @@ -16,6 +15,10 @@ import ( "time" "unicode/utf8" + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" + + "code.gitea.io/gitea/modules/urfs_client/urchin" + "code.gitea.io/gitea/modules/dataset" "code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" @@ -91,28 +94,9 @@ func MustEnableCloudbrain(ctx *context.Context) { } } -func cutString(str string, lens int) string { - if len(str) < lens { - return str - } - return str[:lens] -} - -func jobNamePrefixValid(s string) string { - lowStr := strings.ToLower(s) - re := regexp.MustCompile(`[^a-z0-9_\\-]+`) - - removeSpecial := re.ReplaceAllString(lowStr, "") - - re = regexp.MustCompile(`^[_\\-]+`) - return re.ReplaceAllString(removeSpecial, "") - -} - func cloudBrainNewDataPrepare(ctx *context.Context, jobType string) error { ctx.Data["PageIsCloudBrain"] = true - t := time.Now() - var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name) ctx.Data["display_job_name"] = displayJobName ctx.Data["command"] = cloudbrain.GetCloudbrainDebugCommand() diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 0620350f6..c94bc2a5b 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -11,7 +11,6 @@ import ( "path" "strconv" "strings" - "time" "code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" @@ -75,8 +74,7 @@ func GrampusTrainJobNPUNew(ctx *context.Context) { func grampusTrainJobNewDataPrepare(ctx *context.Context, processType string) error { ctx.Data["PageIsCloudBrain"] = true - t := time.Now() - var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name) ctx.Data["display_job_name"] = displayJobName //get valid images diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index fabf7e555..214d4b0a1 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -15,6 +15,8 @@ import ( "time" "unicode/utf8" + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" + "code.gitea.io/gitea/services/cloudbrain/cloudbrainTask" "code.gitea.io/gitea/modules/dataset" @@ -128,8 +130,7 @@ func NotebookNew(ctx *context.Context) { func notebookNewDataPrepare(ctx *context.Context) error { ctx.Data["PageIsCloudBrain"] = true - t := time.Now() - var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name) ctx.Data["display_job_name"] = displayJobName attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID) @@ -239,9 +240,9 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm } if setting.ModelartsCD.Enabled { - err = modelarts_cd.GenerateNotebook(ctx, displayJobName, jobName, uuid, description, imageId, spec) + _, err = modelarts_cd.GenerateNotebook(ctx, displayJobName, jobName, uuid, description, imageId, spec, "") } else { - err = modelarts.GenerateNotebook2(ctx, displayJobName, jobName, uuid, description, imageId, spec) + _, err = modelarts.GenerateNotebook2(ctx, displayJobName, jobName, uuid, description, imageId, spec, "") } if err != nil { @@ -714,8 +715,7 @@ func trainJobNewDataPrepare(ctx *context.Context) error { // return //} - t := time.Now() - var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name) ctx.Data["display_job_name"] = displayJobName attachs, err := models.GetModelArtsTrainAttachments(ctx.User.ID) @@ -2351,8 +2351,7 @@ func inferenceJobNewDataPrepare(ctx *context.Context) error { ctx.Data["PageIsCloudBrain"] = true ctx.Data["newInference"] = true - t := time.Now() - var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + var displayJobName = cloudbrainService.GetDisplayJobName(ctx.User.Name) ctx.Data["display_job_name"] = displayJobName attachs, err := models.GetModelArtsTrainAttachments(ctx.User.ID) diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go new file mode 100644 index 000000000..ea1f8b081 --- /dev/null +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -0,0 +1,330 @@ +package cloudbrainTask + +import ( + "fmt" + "net/http" + "os" + + "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/modules/modelarts_cd" + + "code.gitea.io/gitea/modules/git" + + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/redis/redis_key" + "code.gitea.io/gitea/modules/redis/redis_lock" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/services/cloudbrain/resource" + "code.gitea.io/gitea/services/reward/point/account" + + "code.gitea.io/gitea/modules/setting" + cloudbrainService "code.gitea.io/gitea/services/cloudbrain" + repo_service "code.gitea.io/gitea/services/repository" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" +) + +func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption) { + + if ctx.Written() { + return + } + //create repo if not exist + repo, err := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) + if repo == nil { + repo, err = repo_service.CreateRepository(ctx.User, ctx.User, models.CreateRepoOptions{ + Name: setting.FileNoteBook.ProjectName, + Alias: "", + Description: "", + IssueLabels: "", + Gitignores: "", + License: "", + Readme: "Default", + IsPrivate: false, + AutoInit: true, + DefaultBranch: "master", + }) + } + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("repo.failed_to_create_repo")) + } + +} + +func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository) { + + displayJobName := cloudbrainService.GetDisplayJobName(ctx.User.Name) + jobName := util.ConvertDisplayJobNameToJobName(displayJobName) + jobType := string(models.JobTypeDebug) + + codePath := getCodePath(jobName) + + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), jobType, displayJobName)) + defer lock.UnLock() + isOk, err := lock.Lock(models.CloudbrainKeyDuration) + if !isOk { + log.Error("lock processed failed:%v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.cloudbrain_samejob_err"))) + return + } + + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) + if err == nil { + if len(tasks) != 0 { + log.Error("the job name did already exist", ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.cloudbrain_samejob_err"))) + return + } + } else { + if !models.IsErrJobNotExist(err) { + log.Error("system error, %v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("system error.")) + return + } + } + + count, err := GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainOne, jobType) + if err != nil { + log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("system error.")) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.cloudbrain.morethanonejob"))) + return + } + } + repoPath := models.RepoPath(repo.OwnerName, repo.Name) + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + ctx.Error(500, "RepoRef Invalid repo "+repoPath, err.Error()) + return + } + // We opened it, we should close it + defer func() { + // If it's been set to nil then assume someone else has closed it. + if ctx.Repo.GitRepo != nil { + ctx.Repo.GitRepo.Close() + } + }() + + ctx.Repo = &context.Repository{ + Repository: repo, + GitRepo: gitRepo, + } + + fileExist, err := ctx.Repo.FileExists(option.File, option.BranchName) + if err != nil || !fileExist { + log.Error("Get file error:", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + return + } + + command := cloudbrain.GetCloudbrainDebugCommand() + + errStr := uploadCodeFile(repo, codePath, option.BranchName, option.File, jobName) + if errStr != "" { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + return + } + + commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(option.BranchName) + specId := setting.FileNoteBook.SpecIdGPU + if option.Type == 0 { + specId = setting.FileNoteBook.SpecIdCPU + } + spec, err := resource.GetAndCheckSpec(ctx.User.ID, specId, models.FindSpecsOptions{ + JobType: models.JobType(jobType), + ComputeResource: models.GPU, + Cluster: models.OpenICluster, + AiCenterCode: models.AICenterOfCloudBrainOne}) + if err != nil || spec == nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.wrong_specification"))) + return + } + + if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { + log.Error("point balance is not enough,userId=%d specId=%d", ctx.User.ID, spec.ID) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("points.insufficient_points_balance"))) + return + } + + req := cloudbrain.GenerateCloudBrainTaskReq{ + Ctx: ctx, + DisplayJobName: displayJobName, + JobName: jobName, + Image: setting.FileNoteBook.ImageGPU, + Command: command, + Uuids: "", + DatasetNames: "", + DatasetInfos: nil, + CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), + ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), + BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), + Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), + BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), + JobType: jobType, + Description: getDescription(option), + BranchName: option.BranchName, + BootFile: option.File, + Params: "{\"parameter\":[]}", + CommitID: commitID, + BenchmarkTypeID: 0, + BenchmarkChildTypeID: 0, + ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), + Spec: spec, + } + + jobId, err := cloudbrain.GenerateTask(req) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 0, + Message: jobId, + }) + +} + +func getCodePath(jobName string) string { + return setting.JobPath + jobName + cloudbrain.CodeMountPath +} + +func getDescription(option api.CreateFileNotebookJobOption) string { + return option.OwnerName + "/" + option.ProjectName + "/" + option.File +} + +func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository) { + displayJobName := cloudbrainService.GetDisplayJobName(ctx.User.Name) + jobName := util.ConvertDisplayJobNameToJobName(displayJobName) + + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeDebug), displayJobName)) + isOk, err := lock.Lock(models.CloudbrainKeyDuration) + if !isOk { + log.Error("lock processed failed:%v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.cloudbrain_samejob_err"))) + return + } + defer lock.UnLock() + + count, err := GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeDebug)) + + if err != nil { + log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) + + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("system error.")) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.cloudbrain.morethanonejob"))) + return + } + } + + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeDebug), displayJobName) + if err == nil { + if len(tasks) != 0 { + log.Error("the job name did already exist", ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.cloudbrain_samejob_err"))) + return + } + } else { + if !models.IsErrJobNotExist(err) { + log.Error("system error, %v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("system error.")) + return + } + } + repoPath := models.RepoPath(repo.OwnerName, repo.Name) + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + ctx.Error(500, "RepoRef Invalid repo "+repoPath, err.Error()) + return + } + // We opened it, we should close it + defer func() { + // If it's been set to nil then assume someone else has closed it. + if ctx.Repo.GitRepo != nil { + ctx.Repo.GitRepo.Close() + } + }() + + ctx.Repo = &context.Repository{ + Repository: repo, + GitRepo: gitRepo, + } + + fileExist, err := ctx.Repo.FileExists(option.File, option.BranchName) + if err != nil || !fileExist { + log.Error("Get file error:", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + return + } + + err = downloadCode(repo, getCodePath(jobName), option.BranchName) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) + return + } + + var aiCenterCode = models.AICenterOfCloudBrainTwo + if setting.ModelartsCD.Enabled { + aiCenterCode = models.AICenterOfChengdu + } + spec, err := resource.GetAndCheckSpec(ctx.User.ID, setting.FileNoteBook.SpecIdNPU, models.FindSpecsOptions{ + JobType: models.JobTypeDebug, + ComputeResource: models.NPU, + Cluster: models.OpenICluster, + AiCenterCode: aiCenterCode}) + if err != nil || spec == nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.wrong_specification"))) + return + } + if !account.IsPointBalanceEnough(ctx.User.ID, spec.UnitPrice) { + log.Error("point balance is not enough,userId=%d specId=%d ", ctx.User.ID, spec.ID) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("points.insufficient_points_balance"))) + return + } + + var jobId string + if setting.ModelartsCD.Enabled { + jobId, err = modelarts_cd.GenerateNotebook(ctx, displayJobName, jobName, "", getDescription(option), setting.FileNoteBook.ImageIdNPU, spec, option.File) + } else { + jobId, err = modelarts.GenerateNotebook2(ctx, displayJobName, jobName, "", getDescription(option), setting.FileNoteBook.ImageIdNPU, spec, option.File) + } + + if err != nil { + log.Error("GenerateNotebook2 failed, %v", err, ctx.Data["MsgID"]) + + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + + return + } + + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 0, + Message: jobId, + }) + +} + +func uploadCodeFile(repo *models.Repository, codePath string, branchName string, filePath string, jobName string) string { + err := downloadCode(repo, codePath, branchName) + if err != nil { + return "cloudbrain.load_code_failed" + } + + err = uploadOneFileToMinio(codePath, filePath, jobName, cloudbrain.CodeMountPath+"/") + if err != nil { + return "cloudbrain.load_code_failed" + } + go os.RemoveAll(codePath) + return "" +} diff --git a/services/cloudbrain/cloudbrainTask/train.go b/services/cloudbrain/cloudbrainTask/train.go index 8e4673d66..5de69335a 100644 --- a/services/cloudbrain/cloudbrainTask/train.go +++ b/services/cloudbrain/cloudbrainTask/train.go @@ -810,6 +810,18 @@ func uploadCodeToMinio(codePath, jobName, parentDir string) error { return nil } +func uploadOneFileToMinio(codePath, filePath, jobName, parentDir string) error { + destObject := setting.CBCodePathPrefix + jobName + parentDir + path.Base(filePath) + sourceFile := codePath + filePath + err := storage.Attachments.UploadObject(destObject, sourceFile) + if err != nil { + log.Error("UploadObject(%s) failed: %s", filePath, err.Error()) + return err + } + return nil + +} + func readDir(dirname string) ([]os.FileInfo, error) { f, err := os.Open(dirname) if err != nil { diff --git a/services/cloudbrain/util.go b/services/cloudbrain/util.go index ab738927e..d9f0510be 100644 --- a/services/cloudbrain/util.go +++ b/services/cloudbrain/util.go @@ -1,27 +1,31 @@ package cloudbrain import ( + "regexp" + "strconv" + "strings" + "time" + "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/setting" - "strings" ) -func GetAiCenterShow(aiCenter string,ctx *context.Context) string{ +func GetAiCenterShow(aiCenter string, ctx *context.Context) string { aiCenterInfo := strings.Split(aiCenter, "+") - if len(aiCenterInfo) == 2{ - if setting.C2NetMapInfo!=nil { - if info,ok:=setting.C2NetMapInfo[aiCenterInfo[0]];ok { + if len(aiCenterInfo) == 2 { + if setting.C2NetMapInfo != nil { + if info, ok := setting.C2NetMapInfo[aiCenterInfo[0]]; ok { if ctx.Language() == "zh-CN" { return info.Content } else { return info.ContentEN } - }else{ + } else { return aiCenterInfo[1] } - }else{ + } else { return aiCenterInfo[1] } @@ -29,5 +33,26 @@ func GetAiCenterShow(aiCenter string,ctx *context.Context) string{ return "" +} +func GetDisplayJobName(username string) string { + t := time.Now() + return jobNamePrefixValid(cutString(username, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] +} + +func cutString(str string, lens int) string { + if len(str) < lens { + return str + } + return str[:lens] +} + +func jobNamePrefixValid(s string) string { + lowStr := strings.ToLower(s) + re := regexp.MustCompile(`[^a-z0-9_\\-]+`) + + removeSpecial := re.ReplaceAllString(lowStr, "") + + re = regexp.MustCompile(`^[_\\-]+`) + return re.ReplaceAllString(removeSpecial, "") } From fbf371069bd8343f03091439302479d47ad60447 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 9 Nov 2022 18:06:06 +0800 Subject: [PATCH 012/253] fix-kanban --- routers/api/v1/repo/cloudbrain_dashboard.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 0cb883214..5ba694e31 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -713,6 +713,20 @@ func GetCloudbrainsDetailData(ctx *context.Context) { if cloudBrainType == models.TypeCDCenter && aiCenter == models.AICenterOfChengdu { aiCenter = "" } + if cloudBrainType == models.TypeCloudBrainAll { + if aiCenter == models.AICenterOfCloudBrainOne { + cloudBrainType = models.TypeCloudBrainOne + aiCenter = "" + } + if aiCenter == models.AICenterOfCloudBrainTwo { + cloudBrainType = models.TypeCloudBrainTwo + aiCenter = "" + } + if aiCenter == models.AICenterOfChengdu { + cloudBrainType = models.TypeCDCenter + aiCenter = "" + } + } page := ctx.QueryInt("page") pageSize := ctx.QueryInt("pagesize") From 893ad1a2d75df34b47ae488938d7101e9af535aa Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 9 Nov 2022 18:25:48 +0800 Subject: [PATCH 013/253] fix-kanban --- routers/api/v1/repo/cloudbrain_dashboard.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 5ba694e31..1eb772b42 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1685,7 +1685,7 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati return OpenIDurationRate, C2NetDurationRate, 0 } for _, cloudbrainStatistic := range cardDurationStatistics { - aiCenterName := repo.GetAiCenterNameByCode(cloudbrainStatistic.AiCenterCode) + aiCenterName := cloudbrainStatistic.AiCenterCode + "/" + repo.GetAiCenterNameByCode(cloudbrainStatistic.AiCenterCode) if cloudbrainStatistic.Cluster == models.OpenICluster { if _, ok := OpenITotalDuration[aiCenterName]; !ok { OpenITotalDuration[aiCenterName] = cloudbrainStatistic.CardsTotalDuration @@ -1717,7 +1717,7 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati return OpenIDurationRate, C2NetDurationRate, 0 } for _, v := range ResourceAiCenterRes { - aiCenterName := repo.GetAiCenterNameByCode(v.AiCenterCode) + aiCenterName := v.AiCenterCode + "/" + repo.GetAiCenterNameByCode(v.AiCenterCode) if cutString(v.AiCenterCode, 4) == cutString(models.AICenterOfCloudBrainOne, 4) { if _, ok := OpenIUsageDuration[aiCenterName]; !ok { OpenIUsageDuration[aiCenterName] = 0 From e69ac40f14064c2e791a3c8baa4b8069a59a25f1 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 10 Nov 2022 16:08:34 +0800 Subject: [PATCH 014/253] fix-3108 --- routers/api/v1/repo/cloudbrain.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 2e25fdefe..5e533762a 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -596,12 +596,24 @@ func CloudbrainGetLog(ctx *context.APIContext) { existStr = taskRes.TaskStatuses[0].ExitDiagnostics } ctx.Data["existStr"] = existStr - log.Info("existStr=" + existStr) } else { ModelSafetyGetLog(ctx) return } + } + if job.JobType == string(models.JobTypeTrain) { + if job.Type == models.TypeCloudBrainOne { + result, err := cloudbrain.GetJob(job.JobID) + existStr := "" + if err == nil && result != nil { + jobRes, _ := models.ConvertToJobResultPayload(result.Payload) + taskRoles := jobRes.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + existStr = taskRes.TaskStatuses[0].ExitDiagnostics + } + ctx.Data["existStr"] = existStr + } } lines := ctx.QueryInt("lines") @@ -634,7 +646,7 @@ func CloudbrainGetLog(ctx *context.APIContext) { endLine += 1 } } - + result = getLogFromModelDir(job.JobName, startLine, endLine, resultPath) if result == nil { log.Error("GetJobLog failed: %v", err, ctx.Data["MsgID"]) From 5e4217242af8ea8cf9d72156196f79436246d4eb Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 10 Nov 2022 16:29:15 +0800 Subject: [PATCH 015/253] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/modelarts/modelarts.go | 58 ++++++--- modules/setting/setting.go | 9 ++ routers/api/v1/repo/modelarts.go | 7 +- .../cloudbrain/cloudbrainTask/notebook.go | 116 ++++++++++-------- 4 files changed, 123 insertions(+), 67 deletions(-) diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 65984d95f..1b4d093a1 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -7,11 +7,12 @@ import ( "fmt" "io/ioutil" "net/http" - "os" "path" "strconv" "strings" + "github.com/go-resty/resty/v2" + "code.gitea.io/gitea/modules/cloudbrain" "code.gitea.io/gitea/modules/modelarts_cd" @@ -932,20 +933,12 @@ func HandleNotebookInfo(task *models.Cloudbrain) error { func uploadNoteBookFile(task *models.Cloudbrain, result *models.GetNotebook2Result) { jupyterUrl := result.Url + "?token=" + result.Token client := getRestyClient() - res, err := client.R().Get(jupyterUrl) - codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath - if err != nil { - log.Error("browser jupyterUrl failed:%v", task.DisplayJobName, err) - + cookies, xsrf := getCookiesAndCsrf(client, jupyterUrl) + if xsrf == "" { + log.Error("browser jupyterUrl failed:%v", task.DisplayJobName) } else { - cookies := res.Cookies() - xsrf := "" - for _, cookie := range cookies { - if cookie.Name == "_xsrf" { - xsrf = cookie.Value - } - } + codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath fileContents, err := ioutil.ReadFile(codePath + "/" + task.BootFile) if err != nil { log.Error("read jupyter file failed:%v", task.DisplayJobName, err) @@ -953,8 +946,8 @@ func uploadNoteBookFile(task *models.Cloudbrain, result *models.GetNotebook2Resu base64Content := base64.StdEncoding.EncodeToString(fileContents) - uploadUrl := result.Url + "/api/contents/" + path.Base(task.BootFile) - res, err = client.R(). + uploadUrl := getJupyterBaseUrl(result.Url) + "api/contents/" + path.Base(task.BootFile) + res, err := client.R(). SetCookies(cookies). SetHeader("X-XSRFToken", xsrf). SetBody(map[string]interface{}{ @@ -971,7 +964,40 @@ func uploadNoteBookFile(task *models.Cloudbrain, result *models.GetNotebook2Resu } } - go os.RemoveAll(codePath) + +} + +func getJupyterBaseUrl(url string) string { + jupyterUrlLength := len(url) + baseUrl := url[0 : jupyterUrlLength-len(path.Base(url))] + return baseUrl +} + +func getCookiesAndCsrf(client *resty.Client, jupyterUrl string) ([]*http.Cookie, string) { + + var cookies []*http.Cookie + for i := 0; i < 4; i++ { + res, err := client.R().Get(jupyterUrl) + if err != nil { + log.Error("browser jupyterUrl failed.") + return cookies, "" + } else { + cookies = res.Cookies() + xsrf := "" + for _, cookie := range cookies { + if cookie.Name == "_xsrf" { + xsrf = cookie.Value + } + + } + if xsrf != "" { + return cookies, xsrf + } + + } + } + return cookies, "" + } func SyncTempStatusJob() { diff --git a/modules/setting/setting.go b/modules/setting/setting.go index a358a85d4..c9dc02fb6 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1587,6 +1587,15 @@ func NewContext() { Course.OrgName = sec.Key("org_name").MustString("") Course.TeamName = sec.Key("team_name").MustString("") + sec = Cfg.Section("file_notebook") + FileNoteBook.ProjectName = sec.Key("project_name").MustString("openi-notebook") + FileNoteBook.ImageIdNPU = sec.Key("imageid_npu").MustString("") + FileNoteBook.ImageNPU = sec.Key("image_npu").MustString("") + FileNoteBook.ImageGPU = sec.Key("image_gpu").MustString("") + FileNoteBook.SpecIdCPU = sec.Key("specid_cpu").MustInt64(-1) + FileNoteBook.SpecIdGPU = sec.Key("specid_gpu").MustInt64(-1) + FileNoteBook.SpecIdNPU = sec.Key("specid_npu").MustInt64(-1) + getGrampusConfig() getModelartsCDConfig() getModelConvertConfig() diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index cd129dd5f..d9426749f 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -39,8 +39,11 @@ func GetModelArtsNotebook2(ctx *context.APIContext) { ID := ctx.Params(":id") job, err := models.GetCloudbrainByID(ID) if err != nil { - ctx.NotFound(err) - return + job, err = models.GetCloudbrainByJobID(ID) + if err != nil { + ctx.NotFound(err) + return + } } err = modelarts.HandleNotebookInfo(job) if err != nil { diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index ea1f8b081..8180a2b0f 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -3,7 +3,7 @@ package cloudbrainTask import ( "fmt" "net/http" - "os" + "path" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/modelarts_cd" @@ -28,11 +28,25 @@ import ( "code.gitea.io/gitea/modules/util" ) +const NoteBookExtension = ".ipynb" + func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption) { if ctx.Written() { return } + + if path.Ext(option.File) != NoteBookExtension { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_select_wrong"))) + return + } + + isNotebookFileExist, _ := isNoteBookFileExist(ctx, option) + if !isNotebookFileExist { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + return + } + //create repo if not exist repo, err := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) if repo == nil { @@ -51,6 +65,12 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp } if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("repo.failed_to_create_repo")) + return + } + if option.Type <= 1 { + cloudBrainFileNoteBookCreate(ctx, option, repo) + } else { + modelartsFileNoteBookCreate(ctx, option, repo) } } @@ -99,40 +119,19 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot return } } - repoPath := models.RepoPath(repo.OwnerName, repo.Name) - gitRepo, err := git.OpenRepository(repoPath) - if err != nil { - ctx.Error(500, "RepoRef Invalid repo "+repoPath, err.Error()) - return - } - // We opened it, we should close it - defer func() { - // If it's been set to nil then assume someone else has closed it. - if ctx.Repo.GitRepo != nil { - ctx.Repo.GitRepo.Close() - } - }() - ctx.Repo = &context.Repository{ - Repository: repo, - GitRepo: gitRepo, - } - - fileExist, err := ctx.Repo.FileExists(option.File, option.BranchName) - if err != nil || !fileExist { - log.Error("Get file error:", err, ctx.Data["MsgID"]) + sourceRepo, err := models.GetRepositoryByOwnerAndName(option.OwnerName, option.ProjectName) + if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) return } - command := cloudbrain.GetCloudbrainDebugCommand() - - errStr := uploadCodeFile(repo, codePath, option.BranchName, option.File, jobName) + errStr := uploadCodeFile(sourceRepo, codePath, option.BranchName, option.File, jobName) if errStr != "" { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) return } - + command := cloudbrain.GetCloudbrainDebugCommand() commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(option.BranchName) specId := setting.FileNoteBook.SpecIdGPU if option.Type == 0 { @@ -242,33 +241,14 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote return } } - repoPath := models.RepoPath(repo.OwnerName, repo.Name) - gitRepo, err := git.OpenRepository(repoPath) - if err != nil { - ctx.Error(500, "RepoRef Invalid repo "+repoPath, err.Error()) - return - } - // We opened it, we should close it - defer func() { - // If it's been set to nil then assume someone else has closed it. - if ctx.Repo.GitRepo != nil { - ctx.Repo.GitRepo.Close() - } - }() - ctx.Repo = &context.Repository{ - Repository: repo, - GitRepo: gitRepo, - } - - fileExist, err := ctx.Repo.FileExists(option.File, option.BranchName) - if err != nil || !fileExist { - log.Error("Get file error:", err, ctx.Data["MsgID"]) + sourceRepo, err := models.GetRepositoryByOwnerAndName(option.OwnerName, option.ProjectName) + if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) return } - err = downloadCode(repo, getCodePath(jobName), option.BranchName) + err = downloadCode(sourceRepo, getCodePath(jobName), option.BranchName) if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) return @@ -292,6 +272,9 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("points.insufficient_points_balance"))) return } + ctx.Repo = &context.Repository{ + Repository: repo, + } var jobId string if setting.ModelartsCD.Enabled { @@ -315,6 +298,30 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote } +func isNoteBookFileExist(ctx *context.Context, option api.CreateFileNotebookJobOption) (bool, error) { + repoPathOfNoteBook := models.RepoPath(option.OwnerName, option.ProjectName) + + gitRepoOfNoteBook, err := git.OpenRepository(repoPathOfNoteBook) + if err != nil { + log.Error("RepoRef Invalid repo "+repoPathOfNoteBook, err.Error()) + return false, err + } + // We opened it, we should close it + defer func() { + // If it's been set to nil then assume someone else has closed it. + if gitRepoOfNoteBook != nil { + gitRepoOfNoteBook.Close() + } + }() + fileExist, err := fileExists(gitRepoOfNoteBook, option.File, option.BranchName) + if err != nil || !fileExist { + log.Error("Get file error:", err, ctx.Data["MsgID"]) + + return false, err + } + return true, nil +} + func uploadCodeFile(repo *models.Repository, codePath string, branchName string, filePath string, jobName string) string { err := downloadCode(repo, codePath, branchName) if err != nil { @@ -325,6 +332,17 @@ func uploadCodeFile(repo *models.Repository, codePath string, branchName string, if err != nil { return "cloudbrain.load_code_failed" } - go os.RemoveAll(codePath) return "" } + +func fileExists(gitRepo *git.Repository, path string, branch string) (bool, error) { + + commit, err := gitRepo.GetBranchCommit(branch) + if err != nil { + return false, err + } + if _, err := commit.GetTreeEntryByPath(path); err != nil { + return false, err + } + return true, nil +} From 34f15799a11d860ee69a84792f3ce30815bba6ce Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 10 Nov 2022 17:01:17 +0800 Subject: [PATCH 016/253] fix-kanban --- routers/admin/cloudbrains.go | 4 ++-- routers/api/v1/repo/cloudbrain_dashboard.go | 14 +++++++------- routers/repo/cloudbrain_statistic.go | 12 ++++++++---- routers/repo/grampus.go | 2 +- routers/user/home.go | 2 +- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index 8ca8344a8..91685251b 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -97,7 +97,7 @@ func CloudBrains(ctx *context.Context) { for i, task := range ciTasks { ciTasks[i] = cloudbrainService.UpdateCloudbrainAiCenter(ciTasks[i]) - ciTasks[i].Cloudbrain.AiCenter = repo.GetAiCenterNameByCode(ciTasks[i].Cloudbrain.AiCenter) + ciTasks[i].Cloudbrain.AiCenter = repo.GetAiCenterNameByCode(ciTasks[i].Cloudbrain.AiCenter, ctx.Language()) ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource @@ -190,7 +190,7 @@ func DownloadCloudBrains(ctx *context.Context) { models.LoadSpecs4CloudbrainInfo(pageRecords) for _, record := range pageRecords { record = cloudbrainService.UpdateCloudbrainAiCenter(record) - record.Cloudbrain.AiCenter = repo.GetAiCenterNameByCode(record.Cloudbrain.AiCenter) + record.Cloudbrain.AiCenter = repo.GetAiCenterNameByCode(record.Cloudbrain.AiCenter, ctx.Language()) for k, v := range allValues(row, record, ctx) { f.SetCellValue(cloudBrain, k, v) } diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 1eb772b42..f1ae2b9ee 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -799,7 +799,7 @@ func GetCloudbrainsDetailData(ctx *context.Context) { taskDetail.Type = ciTasks[i].Cloudbrain.Type taskDetail.UserName = ciTasks[i].User.Name taskDetail.RepoID = ciTasks[i].RepoID - taskDetail.AiCenter = repo.GetAiCenterNameByCode(task.Cloudbrain.AiCenter) + taskDetail.AiCenter = repo.GetAiCenterNameByCode(task.Cloudbrain.AiCenter, ctx.Language()) if ciTasks[i].Repo != nil { taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name taskDetail.RepoAlias = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Alias @@ -1249,7 +1249,7 @@ func DownloadCloudBrainBoard(ctx *context.Context) { models.LoadSpecs4CloudbrainInfo(pageRecords) for _, record := range pageRecords { record = cloudbrainService.UpdateCloudbrainAiCenter(record) - record.Cloudbrain.AiCenter = repo.GetAiCenterNameByCode(record.Cloudbrain.AiCenter) + record.Cloudbrain.AiCenter = repo.GetAiCenterNameByCode(record.Cloudbrain.AiCenter, ctx.Language()) for k, v := range allCloudbrainValues(row, record, ctx) { f.SetCellValue(cloudBrain, k, v) } @@ -1452,7 +1452,7 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { C2NetResourceDetail := []models.ResourceDetail{} for _, resourceQueue := range resourceQueues { if resourceQueue.Cluster == models.OpenICluster { - aiCenterName := repo.GetAiCenterNameByCode(resourceQueue.AiCenterCode) + aiCenterName := repo.GetAiCenterNameByCode(resourceQueue.AiCenterCode, ctx.Language()) var resourceDetail models.ResourceDetail resourceDetail.QueueCode = resourceQueue.QueueCode resourceDetail.Cluster = resourceQueue.Cluster @@ -1465,7 +1465,7 @@ func GetCloudbrainResourceOverview(ctx *context.Context) { OpenIResourceDetail = append(OpenIResourceDetail, resourceDetail) } if resourceQueue.Cluster == models.C2NetCluster { - aiCenterName := repo.GetAiCenterNameByCode(resourceQueue.AiCenterCode) + aiCenterName := repo.GetAiCenterNameByCode(resourceQueue.AiCenterCode, ctx.Language()) var resourceDetail models.ResourceDetail resourceDetail.QueueCode = resourceQueue.QueueCode resourceDetail.Cluster = resourceQueue.Cluster @@ -1685,7 +1685,7 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati return OpenIDurationRate, C2NetDurationRate, 0 } for _, cloudbrainStatistic := range cardDurationStatistics { - aiCenterName := cloudbrainStatistic.AiCenterCode + "/" + repo.GetAiCenterNameByCode(cloudbrainStatistic.AiCenterCode) + aiCenterName := cloudbrainStatistic.AiCenterCode + "/" + repo.GetAiCenterNameByCode(cloudbrainStatistic.AiCenterCode, "zh-CN") if cloudbrainStatistic.Cluster == models.OpenICluster { if _, ok := OpenITotalDuration[aiCenterName]; !ok { OpenITotalDuration[aiCenterName] = cloudbrainStatistic.CardsTotalDuration @@ -1717,7 +1717,7 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati return OpenIDurationRate, C2NetDurationRate, 0 } for _, v := range ResourceAiCenterRes { - aiCenterName := v.AiCenterCode + "/" + repo.GetAiCenterNameByCode(v.AiCenterCode) + aiCenterName := v.AiCenterCode + "/" + repo.GetAiCenterNameByCode(v.AiCenterCode, "zh-CN") if cutString(v.AiCenterCode, 4) == cutString(models.AICenterOfCloudBrainOne, 4) { if _, ok := OpenIUsageDuration[aiCenterName]; !ok { OpenIUsageDuration[aiCenterName] = 0 @@ -1877,7 +1877,7 @@ func GetResourceQueues(ctx *context.Context) { aiCenterCodeMap := make(map[string]string) for _, resourceQueue := range resourceQueues { if _, ok := aiCenterCodeMap[resourceQueue.AiCenterCode]; !ok { - resourceQueue.AiCenterName = repo.GetAiCenterNameByCode(resourceQueue.AiCenterCode) + resourceQueue.AiCenterName = repo.GetAiCenterNameByCode(resourceQueue.AiCenterCode, ctx.Language()) aiCenterCodeMap[resourceQueue.AiCenterCode] = resourceQueue.AiCenterCode Resource = append(Resource, resourceQueue) } diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 067920d56..547b78f06 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -86,7 +86,7 @@ func summaryDurationStat(statisticTime time.Time) int64 { DayTime: dayTime, HourTime: hourTime, Cluster: cloudbrainTable.Cluster, - AiCenterName: GetAiCenterNameByCode(centerCode), + AiCenterName: GetAiCenterNameByCode(centerCode, "zh-CN"), AiCenterCode: centerCode, AccCardType: cardType, ComputeResource: cloudbrainTable.ComputeResource, @@ -113,7 +113,7 @@ func summaryDurationStat(statisticTime time.Time) int64 { DayTime: dayTime, HourTime: hourTime, Cluster: strings.Split(key, "/")[0], - AiCenterName: GetAiCenterNameByCode(strings.Split(key, "/")[1]), + AiCenterName: GetAiCenterNameByCode(strings.Split(key, "/")[1], "zh-CN"), AiCenterCode: strings.Split(key, "/")[1], AccCardType: strings.Split(key, "/")[2], ComputeResource: strings.Split(key, "/")[3], @@ -132,11 +132,15 @@ func summaryDurationStat(statisticTime time.Time) int64 { return count } -func GetAiCenterNameByCode(centerCode string) string { +func GetAiCenterNameByCode(centerCode string, language string) string { var aiCenterName string aiCenterInfo := cloudbrainService.GetAiCenterInfoByCenterCode(centerCode) if aiCenterInfo != nil { - aiCenterName = aiCenterInfo.Content + if language == "zh-CN" { + aiCenterName = aiCenterInfo.Content + } else { + aiCenterName = aiCenterInfo.ContentEN + } } else { aiCenterName = centerCode } diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index ae4ad8a76..268830dc8 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -1123,7 +1123,7 @@ func HandleTaskWithAiCenter(ctx *context.Context) { log.Error("GetJob failed:" + err.Error()) continue } - if result != nil { + if len(result.JobInfo.Tasks) != 0 { if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 { task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0] } diff --git a/routers/user/home.go b/routers/user/home.go index 4d6a96204..62b0357ad 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -840,7 +840,7 @@ func Cloudbrains(ctx *context.Context) { models.LoadSpecs4CloudbrainInfo(ciTasks) for i, _ := range ciTasks { ciTasks[i] = cloudbrainService.UpdateCloudbrainAiCenter(ciTasks[i]) - ciTasks[i].Cloudbrain.AiCenter = repo.GetAiCenterNameByCode(ciTasks[i].Cloudbrain.AiCenter) + ciTasks[i].Cloudbrain.AiCenter = repo.GetAiCenterNameByCode(ciTasks[i].Cloudbrain.AiCenter, ctx.Language()) ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = ciTasks[i].ComputeResource From 3129a31307a4337b068eaf17fbba229cde83cd7c Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 10 Nov 2022 17:11:26 +0800 Subject: [PATCH 017/253] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/setting/setting.go | 18 ++++++++++-------- services/cloudbrain/cloudbrainTask/notebook.go | 6 ++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index c9dc02fb6..0131f9062 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -713,13 +713,14 @@ var ( }{} FileNoteBook = struct { - ProjectName string - ImageGPU string - SpecIdGPU int64 - SpecIdCPU int64 - ImageIdNPU string - ImageNPU string - SpecIdNPU int64 + ProjectName string + ImageGPU string + SpecIdGPU int64 + SpecIdCPU int64 + ImageIdNPU string + SpecIdNPU int64 + ImageIdNPUCD string + SpecIdNPUCD int64 }{} ModelConvert = struct { @@ -1590,11 +1591,12 @@ func NewContext() { sec = Cfg.Section("file_notebook") FileNoteBook.ProjectName = sec.Key("project_name").MustString("openi-notebook") FileNoteBook.ImageIdNPU = sec.Key("imageid_npu").MustString("") - FileNoteBook.ImageNPU = sec.Key("image_npu").MustString("") FileNoteBook.ImageGPU = sec.Key("image_gpu").MustString("") FileNoteBook.SpecIdCPU = sec.Key("specid_cpu").MustInt64(-1) FileNoteBook.SpecIdGPU = sec.Key("specid_gpu").MustInt64(-1) FileNoteBook.SpecIdNPU = sec.Key("specid_npu").MustInt64(-1) + FileNoteBook.ImageIdNPUCD = sec.Key("imageid_npu_cd").MustString("") + FileNoteBook.SpecIdNPUCD = sec.Key("specid_npu_cd").MustInt64(-1) getGrampusConfig() getModelartsCDConfig() diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index 8180a2b0f..85e582b56 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -255,10 +255,12 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote } var aiCenterCode = models.AICenterOfCloudBrainTwo + var specId = setting.FileNoteBook.SpecIdNPU if setting.ModelartsCD.Enabled { aiCenterCode = models.AICenterOfChengdu + specId = setting.FileNoteBook.SpecIdNPUCD } - spec, err := resource.GetAndCheckSpec(ctx.User.ID, setting.FileNoteBook.SpecIdNPU, models.FindSpecsOptions{ + spec, err := resource.GetAndCheckSpec(ctx.User.ID, specId, models.FindSpecsOptions{ JobType: models.JobTypeDebug, ComputeResource: models.NPU, Cluster: models.OpenICluster, @@ -278,7 +280,7 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote var jobId string if setting.ModelartsCD.Enabled { - jobId, err = modelarts_cd.GenerateNotebook(ctx, displayJobName, jobName, "", getDescription(option), setting.FileNoteBook.ImageIdNPU, spec, option.File) + jobId, err = modelarts_cd.GenerateNotebook(ctx, displayJobName, jobName, "", getDescription(option), setting.FileNoteBook.ImageIdNPUCD, spec, option.File) } else { jobId, err = modelarts.GenerateNotebook2(ctx, displayJobName, jobName, "", getDescription(option), setting.FileNoteBook.ImageIdNPU, spec, option.File) } From 8bab337c9fc606bec28925efb6e2864f5da3c15b Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 10 Nov 2022 17:38:40 +0800 Subject: [PATCH 018/253] fix-kanban --- models/cloudbrain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 6021a0326..6a084b682 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -2047,7 +2047,7 @@ func GetStoppedJobWithNoStartTimeEndTime() ([]*Cloudbrain, error) { func GetC2NetWithAiCenterWrongJob() ([]*Cloudbrain, error) { cloudbrains := make([]*Cloudbrain, 0) return cloudbrains, x. - In("status", ModelArtsTrainJobFailed, ModelArtsTrainJobKilled, ModelArtsStopped, JobStopped, JobFailed). + In("status", ModelArtsTrainJobCompleted, ModelArtsTrainJobFailed, ModelArtsTrainJobKilled, ModelArtsStopped, JobStopped, JobFailed, JobSucceeded). Where("type = ?", TypeC2Net). Find(&cloudbrains) } From 5088e8da18dbe9f082405d0db37337a75390180d Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 11 Nov 2022 08:51:20 +0800 Subject: [PATCH 019/253] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/cloudbrain/cloudbrainTask/notebook.go | 4 +--- services/cloudbrain/cloudbrainTask/train.go | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index 85e582b56..e5e5f3596 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -81,8 +81,6 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot jobName := util.ConvertDisplayJobNameToJobName(displayJobName) jobType := string(models.JobTypeDebug) - codePath := getCodePath(jobName) - lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), jobType, displayJobName)) defer lock.UnLock() isOk, err := lock.Lock(models.CloudbrainKeyDuration) @@ -126,7 +124,7 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot return } - errStr := uploadCodeFile(sourceRepo, codePath, option.BranchName, option.File, jobName) + errStr := uploadCodeFile(sourceRepo, getCodePath(jobName), option.BranchName, option.File, jobName) if errStr != "" { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) return diff --git a/services/cloudbrain/cloudbrainTask/train.go b/services/cloudbrain/cloudbrainTask/train.go index 5de69335a..00d01a7ce 100644 --- a/services/cloudbrain/cloudbrainTask/train.go +++ b/services/cloudbrain/cloudbrainTask/train.go @@ -812,7 +812,7 @@ func uploadCodeToMinio(codePath, jobName, parentDir string) error { func uploadOneFileToMinio(codePath, filePath, jobName, parentDir string) error { destObject := setting.CBCodePathPrefix + jobName + parentDir + path.Base(filePath) - sourceFile := codePath + filePath + sourceFile := codePath + "/" + filePath err := storage.Attachments.UploadObject(destObject, sourceFile) if err != nil { log.Error("UploadObject(%s) failed: %s", filePath, err.Error()) From 84c4666a31fa6d0c3cb08ac6d792c4eb09f2237c Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 11 Nov 2022 09:07:25 +0800 Subject: [PATCH 020/253] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/cloudbrain/cloudbrainTask/notebook.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index e5e5f3596..71aa71770 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -130,7 +130,6 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot return } command := cloudbrain.GetCloudbrainDebugCommand() - commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(option.BranchName) specId := setting.FileNoteBook.SpecIdGPU if option.Type == 0 { specId = setting.FileNoteBook.SpecIdCPU @@ -150,6 +149,9 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("points.insufficient_points_balance"))) return } + ctx.Repo = &context.Repository{ + Repository: repo, + } req := cloudbrain.GenerateCloudBrainTaskReq{ Ctx: ctx, @@ -170,7 +172,7 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot BranchName: option.BranchName, BootFile: option.File, Params: "{\"parameter\":[]}", - CommitID: commitID, + CommitID: "", BenchmarkTypeID: 0, BenchmarkChildTypeID: 0, ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), From 72086cb5b15a8fdbefe25492723459e80ed47583 Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 11 Nov 2022 09:55:33 +0800 Subject: [PATCH 021/253] fix-kanban --- routers/repo/cloudbrain_statistic.go | 83 ++++++++++++++-------------- 1 file changed, 43 insertions(+), 40 deletions(-) diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go index 547b78f06..a55c126b3 100644 --- a/routers/repo/cloudbrain_statistic.go +++ b/routers/repo/cloudbrain_statistic.go @@ -51,9 +51,11 @@ func summaryDurationStat(statisticTime time.Time) int64 { models.LoadSpecs4CloudbrainInfo(ciTasks) for _, cloudbrain := range ciTasks { cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain) - if _, ok := cloudbrainMap[cloudbrain.Cloudbrain.AiCenter+"/"+cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { - if cloudbrain.Cloudbrain.Spec != nil { - cloudbrainMap[cloudbrain.Cloudbrain.AiCenter+"/"+cloudbrain.Cloudbrain.Spec.AccCardType] = &cloudbrain.Cloudbrain + if cloudbrain.Cloudbrain.Spec != nil { + if _, ok := cloudbrainMap[cloudbrain.Cloudbrain.AiCenter+"/"+cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { + if cloudbrain.Cloudbrain.Spec != nil { + cloudbrainMap[cloudbrain.Cloudbrain.AiCenter+"/"+cloudbrain.Cloudbrain.Spec.AccCardType] = &cloudbrain.Cloudbrain + } } } } @@ -172,45 +174,46 @@ func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, be if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter]; !ok { cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter] = make(map[string]int) } - - if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { - if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { - if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) - } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) < endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) - } else if int64(cloudbrain.Cloudbrain.StartTime) >= endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = 0 - } - } else { - if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) - } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) < endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) - } else if int64(cloudbrain.Cloudbrain.StartTime) >= endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += 0 - } - } - } else { - if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { - if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) - } else if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) - } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) - } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + if cloudbrain.Cloudbrain.Spec != nil { + if cloudbrain.Cloudbrain.Status == string(models.ModelArtsRunning) { + if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { + if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) < endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + } else if int64(cloudbrain.Cloudbrain.StartTime) >= endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = 0 + } + } else { + if int64(cloudbrain.Cloudbrain.StartTime) < beginTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) < endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + } else if int64(cloudbrain.Cloudbrain.StartTime) >= endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += 0 + } } } else { - if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) - } else if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) - } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) - } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { - cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + if _, ok := cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType]; !ok { + if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + } else if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] = AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + } + } else { + if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(beginTime)) + } else if int64(cloudbrain.Cloudbrain.StartTime) <= beginTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(beginTime)) + } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) <= endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(cloudbrain.Cloudbrain.EndTime) - int(cloudbrain.Cloudbrain.StartTime)) + } else if beginTime <= int64(cloudbrain.Cloudbrain.StartTime) && int64(cloudbrain.Cloudbrain.StartTime) <= endTime && int64(cloudbrain.Cloudbrain.EndTime) > endTime { + cloudBrainCenterCodeAndCardType[cloudbrain.Cloudbrain.AiCenter][cloudbrain.Cloudbrain.Spec.AccCardType] += AccCardsNum * WorkServerNumber * (int(endTime) - int(cloudbrain.Cloudbrain.StartTime)) + } } } } From 9aefd14b9a78abf7a632cd38240bf430a9c06379 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 11 Nov 2022 09:59:12 +0800 Subject: [PATCH 022/253] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + .../cloudbrain/cloudbrainTask/notebook.go | 38 +++++++++++-------- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 3f0c0a010..c14955dff 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1044,6 +1044,7 @@ model_rename=Duplicate model name, please modify model name. notebook_file_not_exist=Notebook file does not exist. notebook_select_wrong=Please select a Notebook(.ipynb) file first. +notebook_file_no_right=You have no right to access the Notebook(.ipynb) file. date=Date repo_add=Project Increment diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 8b635112b..fc0743c59 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1044,6 +1044,7 @@ model_rename=模型名称重复，请修改模型名称 notebook_file_not_exist=Notebook文件不存在。 notebook_select_wrong=请先选择Notebook（.ipynb）文件。 +notebook_file_no_right=您没有这个Notebook文件的读权限。 date=日期 repo_add=新增项目 diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index 71aa71770..a598c47df 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -47,6 +47,24 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp return } + sourceRepo, err := models.GetRepositoryByOwnerAndName(option.OwnerName, option.ProjectName) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + return + } + + permission, err := models.GetUserRepoPermission(sourceRepo, ctx.User) + if err != nil { + log.Error("Get permission failed", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_no_right"))) + return + } + + if !permission.CanRead(models.UnitTypeCode) { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_no_right"))) + return + } + //create repo if not exist repo, err := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) if repo == nil { @@ -68,14 +86,14 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp return } if option.Type <= 1 { - cloudBrainFileNoteBookCreate(ctx, option, repo) + cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo) } else { - modelartsFileNoteBookCreate(ctx, option, repo) + modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo) } } -func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository) { +func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) { displayJobName := cloudbrainService.GetDisplayJobName(ctx.User.Name) jobName := util.ConvertDisplayJobNameToJobName(displayJobName) @@ -118,12 +136,6 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot } } - sourceRepo, err := models.GetRepositoryByOwnerAndName(option.OwnerName, option.ProjectName) - if err != nil { - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) - return - } - errStr := uploadCodeFile(sourceRepo, getCodePath(jobName), option.BranchName, option.File, jobName) if errStr != "" { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) @@ -199,7 +211,7 @@ func getDescription(option api.CreateFileNotebookJobOption) string { return option.OwnerName + "/" + option.ProjectName + "/" + option.File } -func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository) { +func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) { displayJobName := cloudbrainService.GetDisplayJobName(ctx.User.Name) jobName := util.ConvertDisplayJobNameToJobName(displayJobName) @@ -242,12 +254,6 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote } } - sourceRepo, err := models.GetRepositoryByOwnerAndName(option.OwnerName, option.ProjectName) - if err != nil { - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) - return - } - err = downloadCode(sourceRepo, getCodePath(jobName), option.BranchName) if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) From 97a5cfd3c239ff6436351feb8926673a283b7ec6 Mon Sep 17 00:00:00 2001 From: zhoupzh Date: Fri, 11 Nov 2022 17:43:17 +0800 Subject: [PATCH 023/253] fix issue --- templates/repo/cloudbrain/trainjob/show.tmpl | 84 +------------------- 1 file changed, 1 insertion(+), 83 deletions(-) diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl index 7bc3f2c82..f2cf2fabd 100644 --- a/templates/repo/cloudbrain/trainjob/show.tmpl +++ b/templates/repo/cloudbrain/trainjob/show.tmpl @@ -285,14 +285,8 @@
{{$.i18n.Tr "repo.modelarts.train_job.config"}} - - {{$.i18n.Tr "repo.cloudbrain.runinfo"}} - {{$.i18n.Tr "repo.modelarts.log"}} - - + data-version="{{.VersionName}}">{{$.i18n.Tr "repo.modelarts.log"}} {{$.i18n.Tr "repo.model_download"}}
@@ -430,9 +424,6 @@
- - - {{$.i18n.Tr "repo.modelarts.train_job.run_parameter"}} @@ -504,25 +495,6 @@
- -
-
- -
- - - - - -
- -
- -
- 

                             

-
                         

-                        
-
                     

-
                     

                         
                         
                         
                         

-
                         

                         

                             
                             {{$.i18n.Tr "repo.file_limit_100"}}
                         

                     

-
                 
             
         
@@ -700,11 +665,8 @@
                     
                 
             
-
-
         
     
-
 
 {{template "base/footer" .}}
 
@@ -982,50 +944,6 @@
         });
         stopBubbling(arguments.callee.caller.arguments[0])
     }
-
-    function parseInfo() {
-        let jsonValue = document.getElementById("json_value").value;
-        let jsonObj = JSON.parse(jsonValue);
-        let podRoleName = jsonObj["podRoleName"];
-        let html = "";
-        if (podRoleName != null) {
-            let task0 = podRoleName["task1-0"];
-            let podEvents = jsonObj["podEvents"];
-            let podEventArray = podEvents[task0];
-            if (podEventArray != null) {
-                for (var i = 0; i < podEventArray.length; i++) {
-                    if (podEventArray[i]["reason"] != "") {
-                        html += "
[" + podEventArray[i]["reason"] + "]
";
-                        html += "
" + podEventArray[i]["message"] + "
";
-                        html += "
" + podEventArray[i]["action"] + "
";
-                    }
-                }
-            }
-            let extras = jsonObj["extras"];
-            if (extras != null) {
-                for (var i = 0; i < extras.length; i++) {
-                    if (extras[i]["reason"] != "") {
-                        html += "
[" + extras[i]["reason"] + "]
";
-                        html += "
" + extras[i]["message"] + "
";
-                        html += "
" + extras[i]["action"] + "
";
-                    }
-                }
-            }
-        }
-
-        let string = document.getElementById("ExitDiagnostics").value;
-        string = string.replace(/\r\n/g, "
")
-        string = string.replace(/\n/g, "
");
-        string = string.replace(/(\r\n)|(\n)/g, '
');
-
-        if (string != "") {
-            html += "
[ExitDiagnostics]
";
-            html += "
" + string + "
";
-        }
-
-        document.getElementById("info_display").innerHTML = html;
-    }
-
     ;(function() {
         var SPEC = {{ .Spec }};
         var showPoint = false;

From b7c5c1538b4b4111e1a1fbcb7c77989f90ef1991 Mon Sep 17 00:00:00 2001
From: liuzx 
Date: Fri, 11 Nov 2022 18:29:56 +0800
Subject: [PATCH 024/253] fix-kanban

---
 models/cloudbrain_static.go          | 2 +-
 routers/repo/cloudbrain_statistic.go | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go
index 92d9842a9..21b9ceeb9 100644
--- a/models/cloudbrain_static.go
+++ b/models/cloudbrain_static.go
@@ -349,7 +349,7 @@ func GetCardDurationStatistics(opts *DurationStatisticOptions) ([]*CloudbrainDur
 	var cond = builder.NewCond()
 	if opts.BeginTime.Unix() > 0 && opts.EndTime.Unix() > 0 {
 		cond = cond.And(
-			builder.And(builder.Gte{"cloudbrain_duration_statistic.date_time": opts.BeginTime.Unix()}, builder.Lte{"cloudbrain_duration_statistic.date_time": opts.EndTime.Unix()}),
+			builder.And(builder.Gte{"cloudbrain_duration_statistic.date_time": opts.BeginTime.Unix()}, builder.Lt{"cloudbrain_duration_statistic.date_time": opts.EndTime.Unix()}),
 		)
 	}
 	if opts.AiCenterCode != "" {
diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go
index a55c126b3..d7969e29a 100644
--- a/routers/repo/cloudbrain_statistic.go
+++ b/routers/repo/cloudbrain_statistic.go
@@ -25,6 +25,7 @@ func UpdateDurationStatisticHistoryData() int64 {
 	now := time.Now()
 	currentTime := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), 0, 0, 0, now.Location())
 	statisticTime := recordBeginTime.Add(+1 * time.Hour)
+
 	for statisticTime.Before(currentTime) || statisticTime.Equal(currentTime) {
 		countEach := summaryDurationStat(statisticTime)
 		count += countEach
@@ -33,12 +34,12 @@ func UpdateDurationStatisticHistoryData() int64 {
 	return count
 }
 
+//statisticTime是当前的时辰，比如当前是2019-01-01 12:01:01，那么statisticTime就是2019-01-01 12:00:00
 func summaryDurationStat(statisticTime time.Time) int64 {
 	var count int64
-	dateTime := timeutil.TimeStamp(statisticTime.Unix())
-	dayTime := statisticTime.Format("2006-01-02")
-
+	dateTime := timeutil.TimeStamp(statisticTime.Add(-1 * time.Hour).Unix())
 	beginTime := statisticTime.Add(-1 * time.Hour).Unix()
+	dayTime := statisticTime.Add(-1 * time.Hour).Format("2006-01-02")
 	hourTime := statisticTime.Add(-1 * time.Hour).Hour()
 	endTime := statisticTime.Unix()
 

From 7f08370774f5e64c1b7117ed6fdfc37b6067b981 Mon Sep 17 00:00:00 2001
From: liuzx 
Date: Mon, 14 Nov 2022 14:58:44 +0800
Subject: [PATCH 025/253] fix-kanban

---
 models/cloudbrain_static.go                 | 9 +++++++++
 routers/api/v1/repo/cloudbrain_dashboard.go | 2 +-
 routers/repo/cloudbrain_statistic.go        | 8 +++++++-
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go
index 21b9ceeb9..e58e03e84 100644
--- a/models/cloudbrain_static.go
+++ b/models/cloudbrain_static.go
@@ -375,3 +375,12 @@ func GetDurationRecordBeginTime() ([]*CloudbrainDurationStatistic, error) {
 	}
 	return CloudbrainDurationStatistics, nil
 }
+func DeleteCloudbrainDurationStatistic() error {
+	sess := xStatistic.NewSession()
+	defer sess.Close()
+	if _, err := sess.Exec("TRUNCATE TABLE cloudbrain_duration_statistic"); err != nil {
+		log.Info("TRUNCATE cloudbrain_duration_statistic error.")
+		return err
+	}
+	return nil
+}
\ No newline at end of file
diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go
index f1ae2b9ee..900b4da12 100755
--- a/routers/api/v1/repo/cloudbrain_dashboard.go
+++ b/routers/api/v1/repo/cloudbrain_dashboard.go
@@ -1744,7 +1744,7 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati
 	for _, v := range OpenITotalDuration {
 		totalCanUse += float64(v)
 	}
-	for _, v := range OpenIUsageRate {
+	for _, v := range OpenIUsageDuration {
 		totalUse += float64(v)
 	}
 	if totalCanUse == 0 || totalUse == 0 {
diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go
index d7969e29a..edf9bda37 100644
--- a/routers/repo/cloudbrain_statistic.go
+++ b/routers/repo/cloudbrain_statistic.go
@@ -70,7 +70,11 @@ func summaryDurationStat(statisticTime time.Time) int64 {
 	}
 	cardsTotalDurationMap := make(map[string]int)
 	for _, resourceQueue := range resourceQueues {
-		cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource] = resourceQueue.CardsTotalNum * 1 * 60 * 60
+		if _, ok := cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource]; !ok {
+			cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource] = resourceQueue.CardsTotalNum * 1 * 60 * 60
+		} else {
+			cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource] += resourceQueue.CardsTotalNum * 1 * 60 * 60
+		}
 	}
 
 	for centerCode, CardTypes := range cloudBrainCenterCodeAndCardTypeInfo {
@@ -224,9 +228,11 @@ func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, be
 }
 
 func CloudbrainUpdateHistoryData(ctx *context.Context) {
+	err := models.DeleteCloudbrainDurationStatistic()
 	count := UpdateDurationStatisticHistoryData()
 	ctx.JSON(http.StatusOK, map[string]interface{}{
 		"message": 0,
 		"count":   count,
+		"err":     err,
 	})
 }

From 9d23ea7a4b97303c2fd3fb0409f10cd7e05b698b Mon Sep 17 00:00:00 2001
From: ychao_1983 
Date: Mon, 14 Nov 2022 15:28:59 +0800
Subject: [PATCH 026/253] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 routers/api/v1/api.go             |  2 +-
 routers/api/v1/repo/cloudbrain.go | 59 +++++++++++++++++++++++++++++--
 2 files changed, 57 insertions(+), 4 deletions(-)

diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go
index b7e3c6415..96de00c55 100755
--- a/routers/api/v1/api.go
+++ b/routers/api/v1/api.go
@@ -737,7 +737,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 		}, reqToken(), repoAssignment())
 
 		m.Group("/file_notebook", func() {
-
+			m.Get("", reqToken(), repo.GetFileNoteBookInfo)
 			m.Post("/create", reqToken(), reqWeChat(), bind(api.CreateFileNotebookJobOption{}), repo.CreateFileNoteBook)
 
 		})
diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index 5ea797e97..22b533baa 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -83,6 +83,56 @@ func CreateFileNoteBook(ctx *context.APIContext, option api.CreateFileNotebookJo
 	cloudbrainTask.FileNotebookCreate(ctx.Context, option)
 }
 
+func GetFileNoteBookInfo(ctx *context.APIContext) {
+	//image description  spec description  waiting count
+
+	specs, err := models.GetResourceSpecificationByIds([]int64{setting.FileNoteBook.SpecIdCPU, setting.FileNoteBook.SpecIdGPU, setting.FileNoteBook.SpecIdNPU, setting.FileNoteBook.SpecIdNPUCD})
+	if err != nil {
+		log.Error("Fail to query specifications", err)
+		ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_query_fail")))
+		return
+	}
+
+	var specCPU, specGpu, specNPU, specNPUCD *api.SpecificationShow
+	var specGpuQueueCode string
+	for _, spec := range specs {
+		if spec.ID == setting.FileNoteBook.SpecIdCPU {
+			specCPU = convert.ToSpecification(spec)
+		} else if spec.ID == setting.FileNoteBook.SpecIdGPU {
+			specGpu = convert.ToSpecification(spec)
+			specGpuQueueCode = spec.QueueCode
+		} else if spec.ID == setting.FileNoteBook.SpecIdNPU {
+			specNPU = convert.ToSpecification(spec)
+		} else if spec.ID == setting.FileNoteBook.SpecIdNPUCD {
+			specNPUCD = convert.ToSpecification(spec)
+		}
+	}
+
+	waitCountNpu := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "")
+
+	queuesMap, err := cloudbrain.GetQueuesDetail()
+	if err != nil {
+		log.Error("Fail to query gpu queues waiting count", err)
+		ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_query_fail")))
+		return
+	}
+	waitCountGPU := (*queuesMap)[specGpuQueueCode]
+	ctx.JSON(http.StatusOK, map[string]interface{}{
+		"code":                  0,
+		"specCpu":               specCPU,
+		"specGpu":               specGpu,
+		"specNpu":               specNPU,
+		"specNpuCd":             specNPUCD,
+		"waitCountGpu":          waitCountGPU,
+		"waitCountNpu":          waitCountNpu,
+		"imageCpuDescription":   setting.FileNoteBook.ImageCPUDescription,
+		"imageGpuDescription":   setting.FileNoteBook.ImageGPUDescription,
+		"imageNpuDescription":   setting.FileNoteBook.ImageNPUDescription,
+		"imageNpuCDDescription": setting.FileNoteBook.ImageNPUCDDescription,
+	})
+
+}
+
 func CreateCloudBrain(ctx *context.APIContext, option api.CreateTrainJobOption) {
 	if option.Type == cloudbrainTask.TaskTypeCloudbrainOne {
 		cloudbrainTask.CloudbrainOneTrainJobCreate(ctx.Context, option)
@@ -146,9 +196,12 @@ func GetCloudbrainTask(ctx *context.APIContext) {
 	ID := ctx.Params(":id")
 	job, err := models.GetCloudbrainByID(ID)
 	if err != nil {
-		ctx.NotFound(err)
-		log.Error("GetCloudbrainByID failed:", err)
-		return
+		job, err = models.GetCloudbrainByJobID(ID)
+		if err != nil {
+			ctx.NotFound(err)
+			log.Error("GetCloudbrainByID failed:", err)
+			return
+		}
 	}
 	if job.JobType == string(models.JobTypeModelSafety) {
 		routerRepo.GetAiSafetyTaskByJob(job)

From dc3baf3c2c6a8211e8797e89d58850befa3315a9 Mon Sep 17 00:00:00 2001
From: liuzx 
Date: Mon, 14 Nov 2022 15:41:52 +0800
Subject: [PATCH 027/253] fix-kanban

---
 models/cloudbrain_static.go                 | 10 ++++++++++
 routers/api/v1/repo/cloudbrain_dashboard.go |  6 +++++-
 routers/repo/cloudbrain_statistic.go        |  8 +++++++-
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go
index 21b9ceeb9..c0f6857ef 100644
--- a/models/cloudbrain_static.go
+++ b/models/cloudbrain_static.go
@@ -375,3 +375,13 @@ func GetDurationRecordBeginTime() ([]*CloudbrainDurationStatistic, error) {
 	}
 	return CloudbrainDurationStatistics, nil
 }
+
+func DeleteCloudbrainDurationStatistic() error {
+	sess := xStatistic.NewSession()
+	defer sess.Close()
+	if _, err := sess.Exec("TRUNCATE TABLE cloudbrain_duration_statistic"); err != nil {
+		log.Info("TRUNCATE cloudbrain_duration_statistic error.")
+		return err
+	}
+	return nil
+}
diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go
index f1ae2b9ee..d33c6c1ce 100755
--- a/routers/api/v1/repo/cloudbrain_dashboard.go
+++ b/routers/api/v1/repo/cloudbrain_dashboard.go
@@ -1676,6 +1676,8 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati
 	C2NetUsageDuration := make(map[string]int)
 	OpenIDurationRate := models.DurationRateStatistic{}
 	C2NetDurationRate := models.DurationRateStatistic{}
+	log.Info("beginTimeis:", beginTime)
+	log.Info("endTimeis:", endTime)
 	cardDurationStatistics, err := models.GetCardDurationStatistics(&models.DurationStatisticOptions{
 		BeginTime: beginTime,
 		EndTime:   endTime,
@@ -1711,6 +1713,8 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati
 			}
 		}
 	}
+	log.Info("OpenIUsageDurationis:", OpenIUsageDuration)
+	log.Info("OpenITotalDurationis:", OpenITotalDuration)
 	ResourceAiCenterRes, err := models.GetResourceAiCenters()
 	if err != nil {
 		log.Error("Can not get ResourceAiCenterRes.", err)
@@ -1744,7 +1748,7 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati
 	for _, v := range OpenITotalDuration {
 		totalCanUse += float64(v)
 	}
-	for _, v := range OpenIUsageRate {
+	for _, v := range OpenIUsageDuration {
 		totalUse += float64(v)
 	}
 	if totalCanUse == 0 || totalUse == 0 {
diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go
index d7969e29a..edf9bda37 100644
--- a/routers/repo/cloudbrain_statistic.go
+++ b/routers/repo/cloudbrain_statistic.go
@@ -70,7 +70,11 @@ func summaryDurationStat(statisticTime time.Time) int64 {
 	}
 	cardsTotalDurationMap := make(map[string]int)
 	for _, resourceQueue := range resourceQueues {
-		cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource] = resourceQueue.CardsTotalNum * 1 * 60 * 60
+		if _, ok := cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource]; !ok {
+			cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource] = resourceQueue.CardsTotalNum * 1 * 60 * 60
+		} else {
+			cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource] += resourceQueue.CardsTotalNum * 1 * 60 * 60
+		}
 	}
 
 	for centerCode, CardTypes := range cloudBrainCenterCodeAndCardTypeInfo {
@@ -224,9 +228,11 @@ func getcloudBrainCenterCodeAndCardTypeInfo(ciTasks []*models.CloudbrainInfo, be
 }
 
 func CloudbrainUpdateHistoryData(ctx *context.Context) {
+	err := models.DeleteCloudbrainDurationStatistic()
 	count := UpdateDurationStatisticHistoryData()
 	ctx.JSON(http.StatusOK, map[string]interface{}{
 		"message": 0,
 		"count":   count,
+		"err":     err,
 	})
 }

From 4f2cbc6baf7460bb525b2147bbe8d24a44ccfbf6 Mon Sep 17 00:00:00 2001
From: ychao_1983 
Date: Mon, 14 Nov 2022 15:42:58 +0800
Subject: [PATCH 028/253] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 models/resource_specification.go |  9 +++++++++
 modules/setting/setting.go       | 24 ++++++++++++++++--------
 2 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/models/resource_specification.go b/models/resource_specification.go
index 2da8d015d..3f88fe29f 100644
--- a/models/resource_specification.go
+++ b/models/resource_specification.go
@@ -285,6 +285,15 @@ func ResourceSpecOffShelf(id int64) (int64, error) {
 	return n, err
 }
 
+func GetResourceSpecificationByIds(ids []int64) ([]*Specification, error) {
+	r := make([]*Specification, 0)
+	err := x.In("id", ids).
+		Join("INNER", "resource_queue", "resource_queue.id = resource_specification.queue_id").
+		Find(&r)
+	return r, err
+
+}
+
 func GetResourceSpecification(r *ResourceSpecification) (*ResourceSpecification, error) {
 	has, err := x.Get(r)
 	if err != nil {
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 0131f9062..90b86eb4a 100755
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -713,14 +713,18 @@ var (
 	}{}
 
 	FileNoteBook = struct {
-		ProjectName  string
-		ImageGPU     string
-		SpecIdGPU    int64
-		SpecIdCPU    int64
-		ImageIdNPU   string
-		SpecIdNPU    int64
-		ImageIdNPUCD string
-		SpecIdNPUCD  int64
+		ProjectName           string
+		ImageGPU              string
+		SpecIdGPU             int64
+		SpecIdCPU             int64
+		ImageIdNPU            string
+		SpecIdNPU             int64
+		ImageIdNPUCD          string
+		SpecIdNPUCD           int64
+		ImageCPUDescription   string
+		ImageGPUDescription   string
+		ImageNPUDescription   string
+		ImageNPUCDDescription string
 	}{}
 
 	ModelConvert = struct {
@@ -1597,6 +1601,10 @@ func NewContext() {
 	FileNoteBook.SpecIdNPU = sec.Key("specid_npu").MustInt64(-1)
 	FileNoteBook.ImageIdNPUCD = sec.Key("imageid_npu_cd").MustString("")
 	FileNoteBook.SpecIdNPUCD = sec.Key("specid_npu_cd").MustInt64(-1)
+	FileNoteBook.ImageCPUDescription = sec.Key("image_cpu_desc").MustString("")
+	FileNoteBook.ImageGPUDescription = sec.Key("image_gpu_desc").MustString("")
+	FileNoteBook.ImageNPUDescription = sec.Key("image_npu_desc").MustString("")
+	FileNoteBook.ImageNPUCDDescription = sec.Key("image_npu_cd_desc").MustString("")
 
 	getGrampusConfig()
 	getModelartsCDConfig()

From ab34d3f14446b8a3c8452f333ed7b64685703e49 Mon Sep 17 00:00:00 2001
From: chenshihai 
Date: Mon, 14 Nov 2022 16:40:12 +0800
Subject: [PATCH 029/253] =?UTF-8?q?#3180=20=E8=AE=AD=E7=BB=83=E4=BB=BB?=
 =?UTF-8?q?=E5=8A=A1=E8=AF=A6=E6=83=85=E9=A1=B5=E6=B2=A1=E6=9C=89=E5=B1=95?=
 =?UTF-8?q?=E7=A4=BA=E4=BB=A3=E7=A0=81=E4=BB=93=E5=AF=B9=E5=BA=94=E7=9A=84?=
 =?UTF-8?q?=E7=89=88=E6=9C=AC=EF=BC=8C=E5=8F=AA=E6=9C=89=E5=88=86=E6=94=AF?=
 =?UTF-8?q?=E4=BF=A1=E6=81=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 templates/repo/cloudbrain/inference/show.tmpl   | 1 +
 templates/repo/cloudbrain/show.tmpl             | 1 +
 templates/repo/cloudbrain/trainjob/show.tmpl    | 1 +
 templates/repo/grampus/trainjob/show.tmpl       | 1 +
 templates/repo/modelarts/inferencejob/show.tmpl | 1 +
 templates/repo/modelarts/trainjob/show.tmpl     | 1 +
 templates/repo/modelsafety/show.tmpl            | 1 +
 7 files changed, 7 insertions(+)

diff --git a/templates/repo/cloudbrain/inference/show.tmpl b/templates/repo/cloudbrain/inference/show.tmpl
index 3154b8ac6..11ed2f0c5 100644
--- a/templates/repo/cloudbrain/inference/show.tmpl
+++ b/templates/repo/cloudbrain/inference/show.tmpl
@@ -446,6 +446,7 @@
                                                         
                                                             

                                                                 {{.BranchName}}
+                                                                {{SubStr .CommitID 0 10}}
                                                             

                                                         
                                                     
diff --git a/templates/repo/cloudbrain/show.tmpl b/templates/repo/cloudbrain/show.tmpl
index cca85fb32..27c3cc556 100755
--- a/templates/repo/cloudbrain/show.tmpl
+++ b/templates/repo/cloudbrain/show.tmpl
@@ -337,6 +337,7 @@
                                                     
                                                         

                                                             {{.BranchName}}
+                                                            {{SubStr .CommitID 0 10}}
                                                         

                                                     
                                                 
diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl
index ab6c40ead..c06df57c2 100644
--- a/templates/repo/cloudbrain/trainjob/show.tmpl
+++ b/templates/repo/cloudbrain/trainjob/show.tmpl
@@ -412,6 +412,7 @@
                                                     
                                                         

                                                             {{.BranchName}}
+                                                            {{SubStr .CommitID 0 10}}
                                                         

                                                     
                                                 
diff --git a/templates/repo/grampus/trainjob/show.tmpl b/templates/repo/grampus/trainjob/show.tmpl
index e8d4082f4..9d46a5256 100755
--- a/templates/repo/grampus/trainjob/show.tmpl
+++ b/templates/repo/grampus/trainjob/show.tmpl
@@ -422,6 +422,7 @@
                                                     
                                                         

                                                             {{.BranchName}}
+                                                            {{SubStr .CommitID 0 10}}
                                                         

                                                     
                                                 
diff --git a/templates/repo/modelarts/inferencejob/show.tmpl b/templates/repo/modelarts/inferencejob/show.tmpl
index 29bd1aaf3..5fa07f067 100644
--- a/templates/repo/modelarts/inferencejob/show.tmpl
+++ b/templates/repo/modelarts/inferencejob/show.tmpl
@@ -388,6 +388,7 @@ td, th {
                                                             
                                                                 

                                                                     {{.BranchName}}
+                                                                    {{SubStr .CommitID 0 10}}
                                                                 

                                                             
                                                         
diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl
index d615f90a8..bbe2d52e7 100755
--- a/templates/repo/modelarts/trainjob/show.tmpl
+++ b/templates/repo/modelarts/trainjob/show.tmpl
@@ -442,6 +442,7 @@
                                                     
                                                         

                                                             {{.BranchName}}
+                                                            {{SubStr .CommitID 0 10}}
                                                         

                                                     
                                                 
diff --git a/templates/repo/modelsafety/show.tmpl b/templates/repo/modelsafety/show.tmpl
index a46077443..2090441c9 100644
--- a/templates/repo/modelsafety/show.tmpl
+++ b/templates/repo/modelsafety/show.tmpl
@@ -833,6 +833,7 @@
           
           $(`[vfield="Description"]`).text(res['Description'] || '--');
           $(`[vfield="Parameters"]`).text(res['Parameters'] || '--');
+          $(`[vfield="BranchName"]`).html(res['BranchName'] + '' + res['CommitID'].slice(0, 10) + '');
 
           var imageName = res['Image'] || res['EngineName'];
           $(`[vimagetitle="Image"] span`).hide();

From e5fcb86c80d5ec90dd1d82b0c896e91600fc5b41 Mon Sep 17 00:00:00 2001
From: liuzx 
Date: Mon, 14 Nov 2022 18:56:30 +0800
Subject: [PATCH 030/253] fix-kanban

---
 models/cloudbrain_static.go                 |  7 +++---
 routers/api/v1/repo/cloudbrain_dashboard.go |  3 +--
 routers/repo/cloudbrain_statistic.go        | 24 +++++++++++++--------
 3 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go
index c0f6857ef..48dd477be 100644
--- a/models/cloudbrain_static.go
+++ b/models/cloudbrain_static.go
@@ -275,14 +275,15 @@ func GetCloudbrainByTime(beginTime int64, endTime int64) ([]*CloudbrainInfo, err
 	sess := x.NewSession()
 	defer sess.Close()
 	var cond = builder.NewCond()
+	sess.Exec("if ")
 	cond = cond.Or(
-		builder.And(builder.Lte{"cloudbrain.start_time": beginTime}, builder.Gte{"cloudbrain.end_time": beginTime}),
+		builder.And(builder.Gte{"cloudbrain.end_time": beginTime}, builder.Lte{"cloudbrain.start_time": beginTime}, builder.Gt{"cloudbrain.start_time": 0}),
 	)
 	cond = cond.Or(
-		builder.And(builder.Gte{"cloudbrain.start_time": beginTime}, builder.Lte{"cloudbrain.start_time": endTime}),
+		builder.And(builder.Gte{"cloudbrain.start_time": beginTime}, builder.Lte{"cloudbrain.start_time": endTime}, builder.Gt{"cloudbrain.start_time": 0}),
 	)
 	cond = cond.Or(
-		builder.Eq{"cloudbrain.status": string(JobRunning)},
+		builder.And(builder.Eq{"cloudbrain.status": string(JobRunning)}),
 	)
 	sess.OrderBy("cloudbrain.created_unix ASC")
 	cloudbrains := make([]*CloudbrainInfo, 0, 10)
diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go
index d33c6c1ce..ab5be5b82 100755
--- a/routers/api/v1/repo/cloudbrain_dashboard.go
+++ b/routers/api/v1/repo/cloudbrain_dashboard.go
@@ -1713,8 +1713,6 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati
 			}
 		}
 	}
-	log.Info("OpenIUsageDurationis:", OpenIUsageDuration)
-	log.Info("OpenITotalDurationis:", OpenITotalDuration)
 	ResourceAiCenterRes, err := models.GetResourceAiCenters()
 	if err != nil {
 		log.Error("Can not get ResourceAiCenterRes.", err)
@@ -1756,6 +1754,7 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati
 	} else {
 		totalUsageRate = totalUse / totalCanUse
 	}
+	delete(OpenIUsageDuration, "/")
 
 	OpenIDurationRate.AiCenterTotalDurationStat = OpenITotalDuration
 	OpenIDurationRate.AiCenterUsageDurationStat = OpenIUsageDuration
diff --git a/routers/repo/cloudbrain_statistic.go b/routers/repo/cloudbrain_statistic.go
index edf9bda37..43f5a3b19 100644
--- a/routers/repo/cloudbrain_statistic.go
+++ b/routers/repo/cloudbrain_statistic.go
@@ -50,7 +50,14 @@ func summaryDurationStat(statisticTime time.Time) int64 {
 	}
 	cloudbrainMap := make(map[string]*models.Cloudbrain)
 	models.LoadSpecs4CloudbrainInfo(ciTasks)
+
 	for _, cloudbrain := range ciTasks {
+		if cloudbrain.Cloudbrain.StartTime == 0 {
+			cloudbrain.Cloudbrain.StartTime = cloudbrain.Cloudbrain.CreatedUnix
+		}
+		if cloudbrain.Cloudbrain.EndTime == 0 {
+			cloudbrain.Cloudbrain.EndTime = cloudbrain.Cloudbrain.UpdatedUnix
+		}
 		cloudbrain = cloudbrainService.UpdateCloudbrainAiCenter(cloudbrain)
 		if cloudbrain.Cloudbrain.Spec != nil {
 			if _, ok := cloudbrainMap[cloudbrain.Cloudbrain.AiCenter+"/"+cloudbrain.Cloudbrain.Spec.AccCardType]; !ok {
@@ -68,12 +75,13 @@ func summaryDurationStat(statisticTime time.Time) int64 {
 		log.Info("GetCanUseCardInfo err: %v", err)
 		return 0
 	}
+
 	cardsTotalDurationMap := make(map[string]int)
 	for _, resourceQueue := range resourceQueues {
-		if _, ok := cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource]; !ok {
-			cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource] = resourceQueue.CardsTotalNum * 1 * 60 * 60
+		if _, ok := cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType]; !ok {
+			cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType] = resourceQueue.CardsTotalNum * 1 * 60 * 60
 		} else {
-			cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType+"/"+resourceQueue.ComputeResource] += resourceQueue.CardsTotalNum * 1 * 60 * 60
+			cardsTotalDurationMap[resourceQueue.Cluster+"/"+resourceQueue.AiCenterCode+"/"+resourceQueue.AccCardType] += resourceQueue.CardsTotalNum * 1 * 60 * 60
 		}
 	}
 
@@ -85,8 +93,8 @@ func summaryDurationStat(statisticTime time.Time) int64 {
 					log.Error("DeleteCloudbrainDurationStatisticHour failed: %v", err.Error())
 					return 0
 				}
-				if _, ok := cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType+"/"+cloudbrainTable.ComputeResource]; !ok {
-					cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType+"/"+cloudbrainTable.ComputeResource] = 0
+				if _, ok := cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType]; !ok {
+					cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType] = 0
 				}
 				cloudbrainDurationStat := models.CloudbrainDurationStatistic{
 					DateTime:           dateTime,
@@ -96,16 +104,15 @@ func summaryDurationStat(statisticTime time.Time) int64 {
 					AiCenterName:       GetAiCenterNameByCode(centerCode, "zh-CN"),
 					AiCenterCode:       centerCode,
 					AccCardType:        cardType,
-					ComputeResource:    cloudbrainTable.ComputeResource,
 					CardsUseDuration:   cardDuration,
-					CardsTotalDuration: cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType+"/"+cloudbrainTable.ComputeResource],
+					CardsTotalDuration: cardsTotalDurationMap[cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType],
 					CreatedUnix:        timeutil.TimeStampNow(),
 				}
 				if _, err = models.InsertCloudbrainDurationStatistic(&cloudbrainDurationStat); err != nil {
 					log.Error("Insert cloudbrainDurationStat failed: %v", err.Error())
 				}
 				count++
-				delete(cardsTotalDurationMap, cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType+"/"+cloudbrainTable.ComputeResource)
+				delete(cardsTotalDurationMap, cloudbrainTable.Cluster+"/"+centerCode+"/"+cardType)
 			}
 		}
 	}
@@ -123,7 +130,6 @@ func summaryDurationStat(statisticTime time.Time) int64 {
 			AiCenterName:       GetAiCenterNameByCode(strings.Split(key, "/")[1], "zh-CN"),
 			AiCenterCode:       strings.Split(key, "/")[1],
 			AccCardType:        strings.Split(key, "/")[2],
-			ComputeResource:    strings.Split(key, "/")[3],
 			CardsUseDuration:   0,
 			CardsTotalDuration: cardsTotalDuration,
 			CardsTotalNum:      cardsTotalDuration / 1 / 60 / 60,

From 108dc80b9eaba9f40ec0c62be9111624da2fc638 Mon Sep 17 00:00:00 2001
From: liuzx 
Date: Mon, 14 Nov 2022 19:03:58 +0800
Subject: [PATCH 031/253] fix-kanban

---
 routers/api/v1/repo/cloudbrain_dashboard.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go
index ab5be5b82..ad8890622 100755
--- a/routers/api/v1/repo/cloudbrain_dashboard.go
+++ b/routers/api/v1/repo/cloudbrain_dashboard.go
@@ -1754,7 +1754,7 @@ func getDurationStatistic(beginTime time.Time, endTime time.Time) (models.Durati
 	} else {
 		totalUsageRate = totalUse / totalCanUse
 	}
-	delete(OpenIUsageDuration, "/")
+	delete(C2NetUsageDuration, "/")
 
 	OpenIDurationRate.AiCenterTotalDurationStat = OpenITotalDuration
 	OpenIDurationRate.AiCenterUsageDurationStat = OpenIUsageDuration

From 0c78e3bc787a941ae208273f67eecafb53d8af9c Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Tue, 15 Nov 2022 10:37:50 +0800
Subject: [PATCH 032/253] fix issue

---
 templates/repo/home.tmpl | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/templates/repo/home.tmpl b/templates/repo/home.tmpl
index 7c44aa147..c938f0200 100644
--- a/templates/repo/home.tmpl
+++ b/templates/repo/home.tmpl
@@ -200,6 +200,12 @@
 						class="section">{{EllipsisString $v 30}}{{end}}{{end}}
 			{{end}}
+			

+				
+			

 			

 				

 					{{if .Repository.CanEnableEditor}}
@@ -223,6 +229,7 @@
 					
 					{{end}}
 				

+				
 
 			

 			


From b8d2a318dc22be37f63b47eb516a15c6a37f4a35 Mon Sep 17 00:00:00 2001
From: zouap 
Date: Tue, 15 Nov 2022 10:42:28 +0800
Subject: [PATCH 033/253] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E4=BF=AE=E6=94=B9?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: zouap 
---
 routers/repo/ai_model_convert.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/routers/repo/ai_model_convert.go b/routers/repo/ai_model_convert.go
index bd6a01072..ae3581b86 100644
--- a/routers/repo/ai_model_convert.go
+++ b/routers/repo/ai_model_convert.go
@@ -807,5 +807,4 @@ func ModelConvertDownloadModel(ctx *context.Context) {
 			http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect)
 		}
 	}
-
 }

From 464de2a6afd966dcf8005ce2c977a03281fd4bd7 Mon Sep 17 00:00:00 2001
From: ychao_1983 
Date: Tue, 15 Nov 2022 17:44:25 +0800
Subject: [PATCH 034/253] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 routers/api/v1/repo/cloudbrain.go | 42 ++++++++++++++++++++-----------
 1 file changed, 28 insertions(+), 14 deletions(-)

diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index 22b533baa..53eaf28c6 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -117,19 +117,33 @@ func GetFileNoteBookInfo(ctx *context.APIContext) {
 		return
 	}
 	waitCountGPU := (*queuesMap)[specGpuQueueCode]
-	ctx.JSON(http.StatusOK, map[string]interface{}{
-		"code":                  0,
-		"specCpu":               specCPU,
-		"specGpu":               specGpu,
-		"specNpu":               specNPU,
-		"specNpuCd":             specNPUCD,
-		"waitCountGpu":          waitCountGPU,
-		"waitCountNpu":          waitCountNpu,
-		"imageCpuDescription":   setting.FileNoteBook.ImageCPUDescription,
-		"imageGpuDescription":   setting.FileNoteBook.ImageGPUDescription,
-		"imageNpuDescription":   setting.FileNoteBook.ImageNPUDescription,
-		"imageNpuCDDescription": setting.FileNoteBook.ImageNPUCDDescription,
-	})
+	if !setting.ModelartsCD.Enabled{
+		ctx.JSON(http.StatusOK, map[string]interface{}{
+			"code":                  0,
+			"specCpu":               specCPU,
+			"specGpu":               specGpu,
+			"specNpu":               specNPU,
+			"waitCountGpu":          waitCountGPU,
+			"waitCountNpu":          waitCountNpu,
+			"imageCpuDescription":   setting.FileNoteBook.ImageCPUDescription,
+			"imageGpuDescription":   setting.FileNoteBook.ImageGPUDescription,
+			"imageNpuDescription":   setting.FileNoteBook.ImageNPUDescription,
+
+		})
+	} else{
+		ctx.JSON(http.StatusOK, map[string]interface{}{
+			"code":                  0,
+			"specCpu":               specCPU,
+			"specGpu":               specGpu,
+			"specNpu":               specNPUCD,
+			"waitCountGpu":          waitCountGPU,
+			"waitCountNpu":          waitCountNpu,
+			"imageCpuDescription":   setting.FileNoteBook.ImageCPUDescription,
+			"imageGpuDescription":   setting.FileNoteBook.ImageGPUDescription,
+			"imageNpuDescription":   setting.FileNoteBook.ImageNPUCDDescription,
+		})
+
+	}
 
 }
 
@@ -690,7 +704,7 @@ func CloudbrainGetLog(ctx *context.APIContext) {
 				endLine += 1
 			}
 		}
-		
+
 		result = getLogFromModelDir(job.JobName, startLine, endLine, resultPath)
 		if result == nil {
 			log.Error("GetJobLog failed: %v", err, ctx.Data["MsgID"])

From 7452b22d5af9876041f816aff7fd2dc95e35571e Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Tue, 15 Nov 2022 18:22:05 +0800
Subject: [PATCH 035/253] fix issue

---
 templates/repo/home.tmpl                      |   5 +-
 web_src/vuepages/apis/modules/notobook.js     |  41 ++
 .../vuepages/pages/notebook/debug/index.vue   | 368 ++++++++++++++++++
 .../pages/notebook/debug/vp-notebook-debug.js |  11 +
 4 files changed, 424 insertions(+), 1 deletion(-)
 create mode 100644 web_src/vuepages/apis/modules/notobook.js
 create mode 100644 web_src/vuepages/pages/notebook/debug/index.vue
 create mode 100644 web_src/vuepages/pages/notebook/debug/vp-notebook-debug.js

diff --git a/templates/repo/home.tmpl b/templates/repo/home.tmpl
index c938f0200..23e380e37 100644
--- a/templates/repo/home.tmpl
+++ b/templates/repo/home.tmpl
@@ -99,6 +99,7 @@
 		animation-fill-mode: both;
 	}
 
+
 

 	{{template "repo/header" .}}
 	

@@ -201,7 +202,7 @@
 							title="{{$v}}">{{EllipsisString $v 30}}{{end}}{{end}}

 			{{end}}
 			

-				
@@ -400,6 +401,7 @@
 		

 
 	

+	

 

 
 
+
 {{template "base/footer" .}}
\ No newline at end of file
diff --git a/web_src/vuepages/apis/modules/notobook.js b/web_src/vuepages/apis/modules/notobook.js
new file mode 100644
index 000000000..a13b46558
--- /dev/null
+++ b/web_src/vuepages/apis/modules/notobook.js
@@ -0,0 +1,41 @@
+import service from '../service';
+
+// Notebook新建页面需要的信息
+export const getFileNotebook = () => {
+  return service({
+    url: '/api/v1/file_notebook',
+    method: 'get',
+    params: {},
+  });
+}
+
+// 算力积分获取、消耗明细
+// operate-INCREASE 表示获取明细  DECREASE表示消耗明细, page-当前页, pageSize-每页条数
+export const getPointList = (params) => {
+  return service({
+    url: '/reward/point/record/list',
+    method: 'get',
+    params,
+  });
+}
+
+// 管理员充值、扣减用户积分
+// TargetUserId, OperateType-INCREASE,DECREASE, Amount, Remark, RewardType-POINT
+export const setPointOperate = (data) => {
+  return service({
+    url: '/operation/reward/point/account/operate',
+    method: 'post',
+    data,
+    params: {}
+  });
+}
+
+// 算力积分页面
+export const getPoint = () => {
+  return service({
+    url: '/reward/point',
+    method: 'get',
+    params: {},
+    data: {},
+  });
+}
diff --git a/web_src/vuepages/pages/notebook/debug/index.vue b/web_src/vuepages/pages/notebook/debug/index.vue
new file mode 100644
index 000000000..a4b95f18b
--- /dev/null
+++ b/web_src/vuepages/pages/notebook/debug/index.vue
@@ -0,0 +1,368 @@
+
+
+
diff --git a/web_src/vuepages/pages/notebook/debug/vp-notebook-debug.js b/web_src/vuepages/pages/notebook/debug/vp-notebook-debug.js
new file mode 100644
index 000000000..b92aef211
--- /dev/null
+++ b/web_src/vuepages/pages/notebook/debug/vp-notebook-debug.js
@@ -0,0 +1,11 @@
+import Vue from 'vue';
+import {Dialog} from 'element-ui';
+import 'element-ui/lib/theme-chalk/index.css';
+import { i18n, lang } from '~/langs';
+import App from './index.vue';
+
+Vue.use(Dialog)
+new Vue({
+  i18n,
+  render: (h) => h(App),
+}).$mount('#__vue-root');

From 082ba2af303b24d2584b906f0c455b8a03175ace Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Wed, 16 Nov 2022 09:50:52 +0800
Subject: [PATCH 036/253] fix issue

---
 templates/repo/cloudbrain/inference/show.tmpl | 52 -------------------
 1 file changed, 52 deletions(-)

diff --git a/templates/repo/cloudbrain/inference/show.tmpl b/templates/repo/cloudbrain/inference/show.tmpl
index 3154b8ac6..beeccebf4 100644
--- a/templates/repo/cloudbrain/inference/show.tmpl
+++ b/templates/repo/cloudbrain/inference/show.tmpl
@@ -262,8 +262,6 @@
                     

                         {{$.i18n.Tr "repo.modelarts.train_job.config"}}
-                        {{$.i18n.Tr "repo.cloudbrain.runinfo"}}
                         {{$.i18n.Tr "repo.modelarts.log"}}
                         
";
-                        html += "
" + podEventArray[i]["message"] + "
";
-                        html += "
" + podEventArray[i]["action"] + "
";
-                    }
-                }
-            }
-            let extras = jsonObj["extras"];
-            if (extras != null) {
-                for (var i = 0; i < extras.length; i++) {
-                    if (extras[i]["reason"] != "") {
-                        html += "
[" + extras[i]["reason"] + "]
";
-                        html += "
" + extras[i]["message"] + "
";
-                        html += "
" + extras[i]["action"] + "
";
-                    }
-                }
-            }
-        }
-
-        let string = document.getElementById("ExitDiagnostics").value;
-        string = string.replace(/\r\n/g, "
")
-        string = string.replace(/\n/g, "
");
-        string = string.replace(/(\r\n)|(\n)/g, '
');
-
-        if (string != "") {
-            html += "
[ExitDiagnostics]
";
-            html += "
" + string + "
";
-        }
-
-        document.getElementById("info_display").innerHTML = html;
-    }
-
     ;(function() {
         var SPEC = {{ .Spec }};
         var showPoint = false;

From 00230e114a3d16fcbbc8c4c825b86b272af3cd5f Mon Sep 17 00:00:00 2001
From: liuzx 
Date: Wed, 16 Nov 2022 10:02:00 +0800
Subject: [PATCH 037/253] fix-3108

---
 routers/api/v1/repo/cloudbrain.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index 437a63b3c..45e9ca2cb 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -602,7 +602,7 @@ func CloudbrainGetLog(ctx *context.APIContext) {
 		}
 	}
 
-	if job.JobType == string(models.JobTypeTrain) {
+	if job.JobType == string(models.JobTypeTrain) || job.JobType == string(models.JobTypeInference) {
 		if job.Type == models.TypeCloudBrainOne {
 			result, err := cloudbrain.GetJob(job.JobID)
 			existStr := ""

From f333dc6edaaa9c82ce5ccac9431d67f5dff5a387 Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Wed, 16 Nov 2022 10:04:39 +0800
Subject: [PATCH 038/253] fix issue

---
 templates/repo/cloudbrain/inference/show.tmpl | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/templates/repo/cloudbrain/inference/show.tmpl b/templates/repo/cloudbrain/inference/show.tmpl
index beeccebf4..aee08d659 100644
--- a/templates/repo/cloudbrain/inference/show.tmpl
+++ b/templates/repo/cloudbrain/inference/show.tmpl
@@ -513,25 +513,6 @@
 
                         

                     
-
-                    

-                        

-                            
-                            

-                                
-                                
-                                
-
-                                
-                            

-
-                        

-
-                    

-
                     

                         

                              
Date: Wed, 16 Nov 2022 10:23:40 +0800
Subject: [PATCH 039/253] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 models/resource_specification.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/models/resource_specification.go b/models/resource_specification.go
index 3f88fe29f..e04cc6ed9 100644
--- a/models/resource_specification.go
+++ b/models/resource_specification.go
@@ -287,7 +287,7 @@ func ResourceSpecOffShelf(id int64) (int64, error) {
 
 func GetResourceSpecificationByIds(ids []int64) ([]*Specification, error) {
 	r := make([]*Specification, 0)
-	err := x.In("id", ids).
+	err := x.In("resource_specification.id", ids).
 		Join("INNER", "resource_queue", "resource_queue.id = resource_specification.queue_id").
 		Find(&r)
 	return r, err

From 4be102021bfd556d1ce0218ad7e9d3babce3ef6b Mon Sep 17 00:00:00 2001
From: chenyifan01 
Date: Wed, 16 Nov 2022 16:03:32 +0800
Subject: [PATCH 040/253] #3027 add repo watch notify type

---
 models/repo.go          |  2 +-
 models/repo_watch.go    | 33 ++++++++++++++++++++++++++++++---
 modules/context/repo.go |  1 +
 routers/repo/repo.go    |  4 +++-
 4 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/models/repo.go b/models/repo.go
index f0760108b..832e3fc37 100755
--- a/models/repo.go
+++ b/models/repo.go
@@ -1279,7 +1279,7 @@ func CreateRepository(ctx DBContext, doer, u *User, repo *Repository, opts ...Cr
 	}
 
 	if setting.Service.AutoWatchNewRepos {
-		if err = watchRepo(ctx.e, doer.ID, repo.ID, true); err != nil {
+		if err = watchRepo(ctx.e, doer.ID, repo.ID, true, ReceiveAllNotification); err != nil {
 			return fmt.Errorf("watchRepo: %v", err)
 		}
 	}
diff --git a/models/repo_watch.go b/models/repo_watch.go
index 573a2d78a..7c43ee352 100644
--- a/models/repo_watch.go
+++ b/models/repo_watch.go
@@ -24,6 +24,14 @@ const (
 	RepoWatchModeAuto // 3
 )
 
+// NotifyType specifies what kind of watch the user has on a repository
+type NotifyType int8
+
+const (
+	RejectAllNotification  NotifyType = 0
+	ReceiveAllNotification NotifyType = 9
+)
+
 var ActionChan = make(chan *Action, 200)
 var ActionChan4Task = make(chan Action, 200)
 
@@ -34,6 +42,7 @@ type Watch struct {
 	RepoID      int64         `xorm:"UNIQUE(watch)"`
 	Mode        RepoWatchMode `xorm:"SMALLINT NOT NULL DEFAULT 1"`
 	CreatedUnix int64         `xorm:"created"`
+	NotifyType  NotifyType    `xorm:"SMALLINT NOT NULL DEFAULT 0"`
 }
 
 // getWatch gets what kind of subscription a user has on a given repository; returns dummy record if none found
@@ -60,8 +69,20 @@ func IsWatching(userID, repoID int64) bool {
 	return err == nil && isWatchMode(watch.Mode)
 }
 
+// GetWatchNotifyType
+func GetWatchNotifyType(userID, repoID int64) NotifyType {
+	watch, err := getWatch(x, userID, repoID)
+	if err != nil {
+		return RejectAllNotification
+	}
+	return watch.NotifyType
+}
+
 func watchRepoMode(e Engine, watch Watch, mode RepoWatchMode) (err error) {
 	if watch.Mode == mode {
+		if _, err := e.ID(watch.ID).Cols("notify_type").Update(watch); err != nil {
+			return err
+		}
 		return nil
 	}
 	if mode == RepoWatchModeAuto && (watch.Mode == RepoWatchModeDont || isWatchMode(watch.Mode)) {
@@ -109,7 +130,7 @@ func WatchRepoMode(userID, repoID int64, mode RepoWatchMode) (err error) {
 	return watchRepoMode(x, watch, mode)
 }
 
-func watchRepo(e Engine, userID, repoID int64, doWatch bool) (err error) {
+func watchRepo(e Engine, userID, repoID int64, doWatch bool, notifyTypes ...NotifyType) (err error) {
 	var watch Watch
 	if watch, err = getWatch(e, userID, repoID); err != nil {
 		return err
@@ -119,14 +140,19 @@ func watchRepo(e Engine, userID, repoID int64, doWatch bool) (err error) {
 	} else if !doWatch {
 		err = watchRepoMode(e, watch, RepoWatchModeNone)
 	} else {
+		notifyType := RejectAllNotification
+		if len(notifyTypes) > 0 {
+			notifyType = notifyTypes[0]
+		}
+		watch.NotifyType = notifyType
 		err = watchRepoMode(e, watch, RepoWatchModeNormal)
 	}
 	return err
 }
 
 // WatchRepo watch or unwatch repository.
-func WatchRepo(userID, repoID int64, watch bool) (err error) {
-	return watchRepo(x, userID, repoID, watch)
+func WatchRepo(userID, repoID int64, watch bool, notifyType ...NotifyType) (err error) {
+	return watchRepo(x, userID, repoID, watch, notifyType...)
 }
 
 func getWatchers(e Engine, repoID int64) ([]*Watch, error) {
@@ -156,6 +182,7 @@ func getRepoWatchersIDs(e Engine, repoID int64) ([]int64, error) {
 	return ids, e.Table("watch").
 		Where("watch.repo_id=?", repoID).
 		And("watch.mode<>?", RepoWatchModeDont).
+		And("watch.notify_type > ?", RejectAllNotification).
 		Select("user_id").
 		Find(&ids)
 }
diff --git a/modules/context/repo.go b/modules/context/repo.go
index 7c425c8c0..3bdc34f0d 100755
--- a/modules/context/repo.go
+++ b/modules/context/repo.go
@@ -474,6 +474,7 @@ func RepoAssignment() macaron.Handler {
 
 		if ctx.IsSigned {
 			ctx.Data["IsWatchingRepo"] = models.IsWatching(ctx.User.ID, repo.ID)
+			ctx.Data["WatchNotifyType"] = models.GetWatchNotifyType(ctx.User.ID, repo.ID)
 			ctx.Data["IsStaringRepo"] = models.IsStaring(ctx.User.ID, repo.ID)
 
 			ctx.Data["IsStaringDataset"] = models.IsDatasetStaringByRepoId(ctx.User.ID, repo.ID)
diff --git a/routers/repo/repo.go b/routers/repo/repo.go
index 2c8c2f45b..4919b2487 100644
--- a/routers/repo/repo.go
+++ b/routers/repo/repo.go
@@ -414,7 +414,9 @@ func Action(ctx *context.Context) {
 	var err error
 	switch ctx.Params(":action") {
 	case "watch":
-		err = models.WatchRepo(ctx.User.ID, ctx.Repo.Repository.ID, true)
+		err = models.WatchRepo(ctx.User.ID, ctx.Repo.Repository.ID, true, models.ReceiveAllNotification)
+	case "watch_but_reject":
+		err = models.WatchRepo(ctx.User.ID, ctx.Repo.Repository.ID, true, models.RejectAllNotification)
 	case "unwatch":
 		err = models.WatchRepo(ctx.User.ID, ctx.Repo.Repository.ID, false)
 	case "star":

From c76daae5fef9a7ba2523b1debed12601cdb9b63a Mon Sep 17 00:00:00 2001
From: liuzx 
Date: Wed, 16 Nov 2022 16:17:37 +0800
Subject: [PATCH 041/253] fix-3108

---
 routers/api/v1/repo/cloudbrain.go | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index 45e9ca2cb..066453e72 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -658,9 +658,16 @@ func CloudbrainGetLog(ctx *context.APIContext) {
 	if result["Content"] != nil {
 		content = result["Content"].(string)
 	}
-	if ctx.Data["existStr"] != nil && result["Lines"].(int) < 50 {
-		content = content + ctx.Data["existStr"].(string)
+	if job.JobType == string(models.JobTypeModelSafety) {
+		if ctx.Data["existStr"] != nil && result["Lines"].(int) < 50 {
+			content = content + ctx.Data["existStr"].(string)
+		}
+	} else {
+		if ctx.Data["existStr"] != nil {
+			content = content + ctx.Data["existStr"].(string)
+		}
 	}
+
 	logFileName := result["FileName"]
 
 	//Logs can only be downloaded if the file exists

From 951a785bcefd47f9160c65ac93e64b66930247a5 Mon Sep 17 00:00:00 2001
From: liuzx 
Date: Wed, 16 Nov 2022 17:20:17 +0800
Subject: [PATCH 042/253] fix-3108

---
 routers/api/v1/repo/cloudbrain.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index 066453e72..8d87954b3 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -662,7 +662,8 @@ func CloudbrainGetLog(ctx *context.APIContext) {
 		if ctx.Data["existStr"] != nil && result["Lines"].(int) < 50 {
 			content = content + ctx.Data["existStr"].(string)
 		}
-	} else {
+	}
+	if job.JobType == string(models.JobTypeTrain) || job.JobType == string(models.JobTypeInference) {
 		if ctx.Data["existStr"] != nil {
 			content = content + ctx.Data["existStr"].(string)
 		}

From 1ed8e113ce84494ea0aec329915f28dbdc3799ff Mon Sep 17 00:00:00 2001
From: liuzx 
Date: Wed, 16 Nov 2022 18:01:46 +0800
Subject: [PATCH 043/253] fix-3108

---
 templates/repo/cloudbrain/trainjob/show.tmpl | 61 +-------------------
 1 file changed, 1 insertion(+), 60 deletions(-)

diff --git a/templates/repo/cloudbrain/trainjob/show.tmpl b/templates/repo/cloudbrain/trainjob/show.tmpl
index 3af318286..75cad03b4 100644
--- a/templates/repo/cloudbrain/trainjob/show.tmpl
+++ b/templates/repo/cloudbrain/trainjob/show.tmpl
@@ -893,66 +893,7 @@
         $('.secondary.menu .item').tab();
     });
 
-    let userName
-    let repoPath
-    let jobID
-    let downlaodFlag = {{ $.canDownload }}
-    let taskID = {{ $.task.ID }}
-    let realJobName = {{ $.task.JobName }}
-    $(document).ready(function () {
-        let url = window.location.href;
-        let urlArr = url.split('/')
-        userName = urlArr.slice(-5)[0]
-        repoPath = urlArr.slice(-4)[0]
-        jobID = urlArr.slice(-1)[0]
-    })
-    function stopBubbling(e) {
-        e = window.event || e;
-        if (e.stopPropagation) {
-            e.stopPropagation();      //阻止事件 冒泡传播
-        } else {
-            e.cancelBubble = true;   //ie兼容
-        }
-    }
-
-    function loadLog(version_name) {
-        document.getElementById("mask").style.display = "block"
-        let startLine =  $('input[name=end_line]').val();
-        if(startLine==""){
-           startLine=0;
-        }
-        let endLine = $('input[name=end_line]').val();
-        if(endLine==""){
-            endLine = 50;
-        }
-        $.get(`/${userName}/${repoPath}/cloudbrain/train-job/${jobID}/get_log?endLine=${endLine}&startLine=${startLine}`, (data) => {
-            $('input[name=end_line]').val(data.EndLine)
-            $('input[name=start_line]').val(data.StartLine)
-            $(`#log_file${version_name}`).text(data.Content)
-            document.getElementById("mask").style.display = "none"
-        }).fail(function (err) {
-            console.log(err);
-            document.getElementById("mask").style.display = "none"
-        });
-    }
-
-    function refreshStatus(version_name) {
-        $.get(`/api/v1/repos/${userName}/${repoPath}/cloudbrain/${taskID}?version_name=${versionname}`, (data) => {
-            // header status and duration
-            //$(`#${version_name}-duration-span`).text(data.JobDuration)
-            $(`#${version_name}-status-span span`).text(data.JobStatus)
-            $(`#${version_name}-status-span i`).attr("class", data.JobStatus)
-            // detail status and duration
-            //$('#'+version_name+'-duration').text(data.JobDuration)
-            $('#' + version_name + '-status').text(data.JobStatus)
-            loadLog(version_name)
-
-
-        }).fail(function (err) {
-            console.log(err);
-        });
-        stopBubbling(arguments.callee.caller.arguments[0])
-    }
+   
     ;(function() {
         var SPEC = {{ .Spec }};
         var showPoint = false;

From eab954856da74928402dcbc7502bec7ff4c936a1 Mon Sep 17 00:00:00 2001
From: liuzx 
Date: Thu, 17 Nov 2022 09:39:30 +0800
Subject: [PATCH 044/253] fix-3108

---
 routers/api/v1/repo/cloudbrain.go | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go
index 8d87954b3..e57e59b5b 100755
--- a/routers/api/v1/repo/cloudbrain.go
+++ b/routers/api/v1/repo/cloudbrain.go
@@ -658,16 +658,16 @@ func CloudbrainGetLog(ctx *context.APIContext) {
 	if result["Content"] != nil {
 		content = result["Content"].(string)
 	}
-	if job.JobType == string(models.JobTypeModelSafety) {
-		if ctx.Data["existStr"] != nil && result["Lines"].(int) < 50 {
-			content = content + ctx.Data["existStr"].(string)
-		}
-	}
-	if job.JobType == string(models.JobTypeTrain) || job.JobType == string(models.JobTypeInference) {
-		if ctx.Data["existStr"] != nil {
-			content = content + ctx.Data["existStr"].(string)
-		}
-	}
+	// if job.JobType == string(models.JobTypeModelSafety) {
+	if ctx.Data["existStr"] != nil && result["Lines"].(int) < 50 {
+		content = content + ctx.Data["existStr"].(string)
+	}
+	// }
+	// if job.JobType == string(models.JobTypeTrain) || job.JobType == string(models.JobTypeInference) {
+	// 	if ctx.Data["existStr"] != nil {
+	// 		content = content + ctx.Data["existStr"].(string)
+	// 	}
+	// }
 
 	logFileName := result["FileName"]
 

From 9c6a1b6e2e639ac58aea5afd49eb093f1f11aec9 Mon Sep 17 00:00:00 2001
From: zhoupzh 
Date: Thu, 17 Nov 2022 09:54:56 +0800
Subject: [PATCH 045/253] fix issue

---
 templates/repo/home.tmpl                      |   5 +
 web_src/vuepages/apis/modules/notobook.js     |  48 +++---
 web_src/vuepages/apis/service.js              |   1 -
 .../vuepages/pages/notebook/debug/index.vue   | 163 +++++++++++++++---
 4 files changed, 163 insertions(+), 54 deletions(-)

diff --git a/templates/repo/home.tmpl b/templates/repo/home.tmpl
index 23e380e37..e88f6f52d 100644
--- a/templates/repo/home.tmpl
+++ b/templates/repo/home.tmpl
@@ -402,6 +402,7 @@
 
 	

 	

+	

 

 
 
 
 {{template "base/footer" .}}
\ No newline at end of file
diff --git a/web_src/vuepages/apis/modules/notobook.js b/web_src/vuepages/apis/modules/notobook.js
index a13b46558..31e2209c7 100644
--- a/web_src/vuepages/apis/modules/notobook.js
+++ b/web_src/vuepages/apis/modules/notobook.js
@@ -1,41 +1,39 @@
-import service from '../service';
+import service from "../service";
 
 // Notebook新建页面需要的信息
 export const getFileNotebook = () => {
   return service({
-    url: '/api/v1/file_notebook',
-    method: 'get',
+    url: "/api/v1/file_notebook",
+    method: "get",
     params: {},
   });
-}
+};
 
-// 算力积分获取、消耗明细
-// operate-INCREASE 表示获取明细  DECREASE表示消耗明细, page-当前页, pageSize-每页条数
-export const getPointList = (params) => {
+// Notebook新建调试任务
+// type, file, branch_name, owner_name, project_name
+export const createNotebook = (data) => {
   return service({
-    url: '/reward/point/record/list',
-    method: 'get',
-    params,
+    url: "/api/v1/file_notebook/create",
+    method: "post",
+    data,
+    params: {},
   });
-}
+};
 
-// 管理员充值、扣减用户积分
-// TargetUserId, OperateType-INCREASE,DECREASE, Amount, Remark, RewardType-POINT
-export const setPointOperate = (data) => {
+// Notebook新建页面需要的信息
+export const getCb1Notebook = (jobid) => {
   return service({
-    url: '/operation/reward/point/account/operate',
-    method: 'post',
-    data,
-    params: {}
+    url: `/api/v1/cloudbrain/${jobid}`,
+    method: "get",
+    params: {},
   });
-}
+};
 
-// 算力积分页面
-export const getPoint = () => {
+// Notebook新建页面需要的信息
+export const getCb2Notebook = (jobid) => {
   return service({
-    url: '/reward/point',
-    method: 'get',
+    url: `/api/v1/modelarts/notebook/${jobid}`,
+    method: "get",
     params: {},
-    data: {},
   });
-}
+};
diff --git a/web_src/vuepages/apis/service.js b/web_src/vuepages/apis/service.js
index 292b9ef78..c676a7d18 100644
--- a/web_src/vuepages/apis/service.js
+++ b/web_src/vuepages/apis/service.js
@@ -2,7 +2,6 @@ import axios from 'axios';
 
 const service = axios.create({
   baseURL: '/',
-  timeout: 20000,
 });
 
 service.interceptors.request.use((config) => {
diff --git a/web_src/vuepages/pages/notebook/debug/index.vue b/web_src/vuepages/pages/notebook/debug/index.vue
index a4b95f18b..9b375b5b8 100644
--- a/web_src/vuepages/pages/notebook/debug/index.vue
+++ b/web_src/vuepages/pages/notebook/debug/index.vue
@@ -5,12 +5,13 @@
       :visible.sync="dialogVisible"
       width="50%"
       :close-on-click-modal="false"
+      @closed="handleClose"
     >
       

         

           

             

@@ -19,14 +20,14 @@
             

               
CPU 环境

               

-                规格：CPU: 4，内存: 32GB，共享内存: 16GB
+                {{cpuSpec}}
               

               

-                镜像： ubuntuxx. xx-py37-torch1.9.1
+                镜像：{{notebookInfo.imageCpuDescription}}
               

             

             

-              
+              
               
           

-            

-              N
+            

+              G
             

             

-              
CPU 环境

+              
GPU 环境

               

-                规格：CPU: 4，内存: 32GB，共享内存: 16GB
+                {{gpuSpec}}
               

               

-                镜像： ubuntuxx. xx-py37-torch1.9.1
+                镜像：{{notebookInfo.imageGpuDescription}}
               

             

             

-                
+                
+                
               
           

-            

-              G
+            

+              N
             

             

-              
CPU 环境

+              
NPU 环境

               

-                规格：CPU: 4，内存: 32GB，共享内存: 16GB
+                {{npuSpec}}
               

               

-                镜像： ubuntuxx. xx-py37-torch1.9.1
+                镜像：{{notebookInfo.imageCpuDescription}}
               

             

             

-                
+                
               
         

         

-          

-            
+          

+            
+            
+            
+            当前有个{{waitCount}}任务在排队
+          

+          

+            
+            
+            
+            当前有个{{waitCount}}任务在排队
+          

+          

+            
+            
+            

+                
+                
+            

+            当前有个{{waitCount}}任务在排队
           

         

       

@@ -98,24 +118,35 @@
   

 
 
+{{if and (.IsViewFile) (IpynbBool .TreePath)}}
 
+{{end}}
 {{template "base/footer" .}}
\ No newline at end of file
diff --git a/web_src/vuepages/langs/config/en-US.js b/web_src/vuepages/langs/config/en-US.js
index bcd1cfe43..3ad30b5e8 100644
--- a/web_src/vuepages/langs/config/en-US.js
+++ b/web_src/vuepages/langs/config/en-US.js
@@ -192,7 +192,34 @@ const en = {
   dataDesensitizationModelExperience:'Data desensitization model experience',
   dataDesensitizationModelDesc:'Use AI technology to desensitize the face and license plate number in the picture. For more information about this model, please visit the project',
   limitFilesUpload:'Only jpg/jpeg/png files can be uploaded',
-  limitSizeUpload:'The size of the uploaded file cannot exceed 20M!',
+  limitSizeUpload: 'The size of the uploaded file cannot exceed 20M!',
+  notebook: {
+    createNewNotebook: "Create new notebook debug task",
+    sameTaskTips1: "You have created an",
+    sameTaskTips2: "equivalent task",
+    sameTaskTips3: "that is waiting or running, please wait for the task to finish before creating it.",
+    sameTaskTips4: "You can view all your Cloudbrain tasks in",
+    sameTaskTips5: "Home",
+    sameTaskTips6: "Cloudbrain Task",
+    sameTaskTips7: "",
+    cpuEnv: "CPU Environment",
+    gpuEnv: "GPU Environment",
+    npuEnv: "NPU environment",
+    newTask: "Create new task",
+    noQuene: "The current resources are sufficient and no tasks are queued",
+    queneTips1: "There are currently",
+    queneTips2: "tasks queued",
+    watiResource: "Waiting for resources to be allocated, please be patient",
+    debug: "Debug",
+    stop: "Stop",
+    stopping: "Stopping",
+    notebookRunning: "The debugging environment has been started, and it will automatically close after a single connection for 4 hours or no operation for half an hour",
+    stopSuccess: "Stop task succeeded",
+    specification: "Specification",
+    graphicMemory: "Graphic Memory",
+    memory: "Memory",
+    sharedMemory: "Shared Memory",
+  },
 }
 
 export default en;
diff --git a/web_src/vuepages/langs/config/zh-CN.js b/web_src/vuepages/langs/config/zh-CN.js
index 5b8e800d8..835beb3e0 100644
--- a/web_src/vuepages/langs/config/zh-CN.js
+++ b/web_src/vuepages/langs/config/zh-CN.js
@@ -1,198 +1,226 @@
 const zh = {
-  loading: '加载中...',
-  noData: '暂无数据',
-  date: '日期',
+  loading: "加载中...",
+  noData: "暂无数据",
+  date: "日期",
 
-  confirm: '确定',
-  cancel: '取消',
-  confirm1: '确认',
-  pleaseCompleteTheInformationFirst: '请先完善信息！',
-  submittedSuccessfully: '提交成功！',
-  submittedFailed: '提交失败！',
-  operation: '操作',
-  edit: '修改',
-  delete: '删除',
-  tips: '提示',
+  confirm: "确定",
+  cancel: "取消",
+  confirm1: "确认",
+  pleaseCompleteTheInformationFirst: "请先完善信息！",
+  submittedSuccessfully: "提交成功！",
+  submittedFailed: "提交失败！",
+  operation: "操作",
+  edit: "修改",
+  delete: "删除",
+  tips: "提示",
 
-  accomplishTask: '积分任务',
-  adminOperate: '管理员操作',
-  runCloudBrainTask: '运行云脑任务',
-  operating: '消耗中',
-  succeeded: '已完成',
-  debugTask: '调试任务',
-  trainTask: '训练任务',
-  inferenceTask: '推理任务',
-  benchmarkTask: '评测任务',
-  createPublicProject: '创建公开项目',
-  dailyPutforwardTasks: '每日提出任务',
-  dailyPR: '每日提出PR',
-  comment: '发表评论',
-  uploadDatasetFile: '上传数据集文件',
-  importNewModel: '导入新模型',
-  completeWechatCodeScanningVerification: '完成微信扫码验证',
-  dailyRunCloudbrainTasks: '每日运行云脑任务',
-  datasetRecommendedByThePlatform: '数据集被平台推荐',
-  submitNewPublicImage: '提交新公开镜像',
-  imageRecommendedByThePlatform: '镜像被平台推荐',
-  firstChangeofAvatar: '首次更换头像',
-  dailyCommit: '每日commit',
-  calcPointDetails: '算力积分明细',
-  calcPointAcquisitionInstructions: '积分获取说明',
-  CurrAvailableCalcPoints: '当前可用算力积分(分)',
-  totalGainCalcPoints: '总获取算力积分(分)',
-  totalConsumeCalcPoints: '总消耗算力积分(分)',
-  gainDetail: '获取明细',
-  consumeDetail: '消耗明细',
-  serialNumber: '流水号',
-  time: '时间',
-  scene: '场景',
-  behaviorOfPoint: '积分行为',
-  explanation: '说明',
-  points: '积分',
-  status: '状态',
-  runTime: '运行时长',
-  taskName: '任务名称',
+  accomplishTask: "积分任务",
+  adminOperate: "管理员操作",
+  runCloudBrainTask: "运行云脑任务",
+  operating: "消耗中",
+  succeeded: "已完成",
+  debugTask: "调试任务",
+  trainTask: "训练任务",
+  inferenceTask: "推理任务",
+  benchmarkTask: "评测任务",
+  createPublicProject: "创建公开项目",
+  dailyPutforwardTasks: "每日提出任务",
+  dailyPR: "每日提出PR",
+  comment: "发表评论",
+  uploadDatasetFile: "上传数据集文件",
+  importNewModel: "导入新模型",
+  completeWechatCodeScanningVerification: "完成微信扫码验证",
+  dailyRunCloudbrainTasks: "每日运行云脑任务",
+  datasetRecommendedByThePlatform: "数据集被平台推荐",
+  submitNewPublicImage: "提交新公开镜像",
+  imageRecommendedByThePlatform: "镜像被平台推荐",
+  firstChangeofAvatar: "首次更换头像",
+  dailyCommit: "每日commit",
+  calcPointDetails: "算力积分明细",
+  calcPointAcquisitionInstructions: "积分获取说明",
+  CurrAvailableCalcPoints: "当前可用算力积分(分)",
+  totalGainCalcPoints: "总获取算力积分(分)",
+  totalConsumeCalcPoints: "总消耗算力积分(分)",
+  gainDetail: "获取明细",
+  consumeDetail: "消耗明细",
+  serialNumber: "流水号",
+  time: "时间",
+  scene: "场景",
+  behaviorOfPoint: "积分行为",
+  explanation: "说明",
+  points: "积分",
+  status: "状态",
+  runTime: "运行时长",
+  taskName: "任务名称",
 
-  createdRepository: '创建了项目',
-  repositoryWasDel: '项目已删除',
-  openedIssue: '创建了任务',
-  createdPullRequest: '创建了合并请求',
-  commentedOnIssue: '评论了任务',
-  uploadDataset: '上传了数据集文件',
-  createdNewModel: '导入了新模型',
-  firstBindingWechatRewards: '首次绑定微信奖励',
-  created: '创建了',
-  type: '类型',
-  dataset: '数据集',
-  setAsRecommendedDataset: '被设置为推荐数据集',
-  committedImage: '提交了镜像',
-  image: '镜像',
-  setAsRecommendedImage: '被设置为推荐镜像',
-  updatedAvatar: '更新了头像',
-  pushedBranch: '推送了{branch}分支代码到',
-  deleteBranch: '从{repo}删除分支{branch}',
-  pushedTag: '推送了标签{tag}到',
-  deleteTag: '从{repo}删除了标签{tag}',
-  dailyMaxTips: '达到每日上限积分，不能拿满分',
-  memory: '内存',
-  sharedMemory: '共享内存',
-  ';': '；',
+  createdRepository: "创建了项目",
+  repositoryWasDel: "项目已删除",
+  openedIssue: "创建了任务",
+  createdPullRequest: "创建了合并请求",
+  commentedOnIssue: "评论了任务",
+  uploadDataset: "上传了数据集文件",
+  createdNewModel: "导入了新模型",
+  firstBindingWechatRewards: "首次绑定微信奖励",
+  created: "创建了",
+  type: "类型",
+  dataset: "数据集",
+  setAsRecommendedDataset: "被设置为推荐数据集",
+  committedImage: "提交了镜像",
+  image: "镜像",
+  setAsRecommendedImage: "被设置为推荐镜像",
+  updatedAvatar: "更新了头像",
+  pushedBranch: "推送了{branch}分支代码到",
+  deleteBranch: "从{repo}删除分支{branch}",
+  pushedTag: "推送了标签{tag}到",
+  deleteTag: "从{repo}删除了标签{tag}",
+  dailyMaxTips: "达到每日上限积分，不能拿满分",
+  memory: "内存",
+  sharedMemory: "共享内存",
+  ";": "；",
 
-  noPointGainRecord: '还没有积分获取记录',
-  noPointConsumeRecord: '还没有积分消耗记录',
+  noPointGainRecord: "还没有积分获取记录",
+  noPointConsumeRecord: "还没有积分消耗记录",
 
   resourcesManagement: {
-    OpenI: '启智集群',
-    C2Net: '智算集群',
-    OpenIOne: '云脑一',
-    OpenITwo: '云脑二',
-    OpenIChengdu: '启智成都智算',
-    chengduCenter: '成都智算',
-    pclcci: '鹏城云计算所',
-    hefeiCenter: '合肥类脑类脑智能开放平台',
-    xuchangCenter: '中原人工智能计算中心',
-    willOnShelf: '待上架',
-    onShelf: '已上架',
-    offShelf: '已下架',
-    toOnShelf: '上架',
-    toOffShelf: '下架',
-    toSetPriceAndOnShelf: '定价上架',
-    status: '状态',
-    allStatus: '全部状态',
-    syncAiNetwork: '同步智算网络',
-    resQueue: '资源池（队列）',
-    allResQueue: '全部资源池（队列）',
-    addResQueue: '新建资源池（队列）',
-    addResQueueBtn: '新增资源池',
-    editResQueue: '修改资源池（队列）',
-    resQueueName: '资源池（队列）名称',
-    whichCluster: '所属集群',
-    allCluster: '全部集群',
-    aiCenter: '智算中心',
-    aiCenterID: '智算中心ID',
-    allAiCenter: '全部智算中心',
-    computeResource: '计算资源',
-    allComputeResource: '全部计算资源',
-    accCardType: '卡类型',
-    allAccCardType: '全部卡类型',
-    cardsTotalNum: '卡数',
-    accCardsNum: '卡数',
-    remark: '备注',
-    pleaseEnterRemark: '请输入备注(最大长度不超过255)',
-    pleaseEnterPositiveIntegerCardsTotalNum: '请输入正整数的卡数！',
-    addResSpecificationAndPriceInfo: '新增资源规格和单价信息',
-    addResSpecificationBtn: '新增资源规格',
-    editResSpecificationAndPriceInfo: '修改资源规格和单价信息',
-    resSpecificationAndPriceManagement: '资源规格单价管理',
-    sourceSpecCode: '对应资源编码',
-    sourceSpecCodeTips: '云脑II需要填写对应的资源编码',
-    sourceSpecId: '智算网络资源规格ID',
-    cpuNum: 'CPU数',
-    gpuMem: '显存',
-    mem: '内存',
-    shareMem: '共享内存',
-    unitPrice: '单价',
-    point_hr: '积分/时',
-    node: '节点',
-    free: '免费',
-    onShelfConfirm: '请确认上架该规格？',
-    offShelfConfirm: '请确认下架该规格？',
-    onShelfCode1001: '上架失败，资源池（队列）不可用。',
-    onShelfCode1003: '上架失败，资源规格不可用。',
-    offShelfDlgTip1: '当前资源规格已在以下场景中使用：',
-    offShelfDlgTip2: '请确认进行下架操作？',
-    resSceneManagement: '算力资源应用场景管理',
-    addResScene: '新建算力资源应用场景',
-    addResSceneBtn: '新增应用场景',
-    editResScene: '修改算力资源应用场景',
-    resSceneName: '应用场景名称',
-    jobType: '任务类型',
-    allJobType: '全部任务类型',
-    isExclusive: '是否专属',
-    allExclusiveAndCommonUse: '全部专属和通用',
-    exclusive: '专属',
-    commonUse: '通用',
-    exclusiveOrg: '专属组织',
-    exclusiveOrgTips: '多个组织名之间用英文分号隔开',
-    computeCluster: '算力集群',
-    resourceSpecification: '资源规格',
-    lastUpdateTime: '最后更新时间',
-    resSceneDeleteConfirm: '是否确认删除当前应用场景？',
-    resourceSpecificationIsAvailable: '资源规格是否可用',
-    resourceSpecificationIsAvailableAll: '资源规格是否可用（全部）',
-    available: '可用',
-    notAvailable: '不可用',
+    OpenI: "启智集群",
+    C2Net: "智算集群",
+    OpenIOne: "云脑一",
+    OpenITwo: "云脑二",
+    OpenIChengdu: "启智成都智算",
+    chengduCenter: "成都智算",
+    pclcci: "鹏城云计算所",
+    hefeiCenter: "合肥类脑类脑智能开放平台",
+    xuchangCenter: "中原人工智能计算中心",
+    willOnShelf: "待上架",
+    onShelf: "已上架",
+    offShelf: "已下架",
+    toOnShelf: "上架",
+    toOffShelf: "下架",
+    toSetPriceAndOnShelf: "定价上架",
+    status: "状态",
+    allStatus: "全部状态",
+    syncAiNetwork: "同步智算网络",
+    resQueue: "资源池（队列）",
+    allResQueue: "全部资源池（队列）",
+    addResQueue: "新建资源池（队列）",
+    addResQueueBtn: "新增资源池",
+    editResQueue: "修改资源池（队列）",
+    resQueueName: "资源池（队列）名称",
+    whichCluster: "所属集群",
+    allCluster: "全部集群",
+    aiCenter: "智算中心",
+    aiCenterID: "智算中心ID",
+    allAiCenter: "全部智算中心",
+    computeResource: "计算资源",
+    allComputeResource: "全部计算资源",
+    accCardType: "卡类型",
+    allAccCardType: "全部卡类型",
+    cardsTotalNum: "卡数",
+    accCardsNum: "卡数",
+    remark: "备注",
+    pleaseEnterRemark: "请输入备注(最大长度不超过255)",
+    pleaseEnterPositiveIntegerCardsTotalNum: "请输入正整数的卡数！",
+    addResSpecificationAndPriceInfo: "新增资源规格和单价信息",
+    addResSpecificationBtn: "新增资源规格",
+    editResSpecificationAndPriceInfo: "修改资源规格和单价信息",
+    resSpecificationAndPriceManagement: "资源规格单价管理",
+    sourceSpecCode: "对应资源编码",
+    sourceSpecCodeTips: "云脑II需要填写对应的资源编码",
+    sourceSpecId: "智算网络资源规格ID",
+    cpuNum: "CPU数",
+    gpuMem: "显存",
+    mem: "内存",
+    shareMem: "共享内存",
+    unitPrice: "单价",
+    point_hr: "积分/时",
+    node: "节点",
+    free: "免费",
+    onShelfConfirm: "请确认上架该规格？",
+    offShelfConfirm: "请确认下架该规格？",
+    onShelfCode1001: "上架失败，资源池（队列）不可用。",
+    onShelfCode1003: "上架失败，资源规格不可用。",
+    offShelfDlgTip1: "当前资源规格已在以下场景中使用：",
+    offShelfDlgTip2: "请确认进行下架操作？",
+    resSceneManagement: "算力资源应用场景管理",
+    addResScene: "新建算力资源应用场景",
+    addResSceneBtn: "新增应用场景",
+    editResScene: "修改算力资源应用场景",
+    resSceneName: "应用场景名称",
+    jobType: "任务类型",
+    allJobType: "全部任务类型",
+    isExclusive: "是否专属",
+    allExclusiveAndCommonUse: "全部专属和通用",
+    exclusive: "专属",
+    commonUse: "通用",
+    exclusiveOrg: "专属组织",
+    exclusiveOrgTips: "多个组织名之间用英文分号隔开",
+    computeCluster: "算力集群",
+    resourceSpecification: "资源规格",
+    lastUpdateTime: "最后更新时间",
+    resSceneDeleteConfirm: "是否确认删除当前应用场景？",
+    resourceSpecificationIsAvailable: "资源规格是否可用",
+    resourceSpecificationIsAvailableAll: "资源规格是否可用（全部）",
+    available: "可用",
+    notAvailable: "不可用",
   },
   user: {
-    inviteFriends: '邀请好友',
-    inviteFriendsTips: '复制二维码或者注册邀请链接分享给好友',
-    clickToViewTheEventDetails: '点击查看活动详情',
-    copyRegistrationInvitationLink: '复制注册邀请链接',
-    registrationAdress: '注册地址：',
-    recommender: '推荐人：',
-    invitedFriends: '已邀请好友',
-    registrationTime: '注册时间',
-    theSharedContentHasBeenCopiedToTheClipboard: '分享内容已复制到剪切板',
-    copyError: '复制错误',
-    Activated: '已激活',
-    notActive: '未激活',
+    inviteFriends: "邀请好友",
+    inviteFriendsTips: "复制二维码或者注册邀请链接分享给好友",
+    clickToViewTheEventDetails: "点击查看活动详情",
+    copyRegistrationInvitationLink: "复制注册邀请链接",
+    registrationAdress: "注册地址：",
+    recommender: "推荐人：",
+    invitedFriends: "已邀请好友",
+    registrationTime: "注册时间",
+    theSharedContentHasBeenCopiedToTheClipboard: "分享内容已复制到剪切板",
+    copyError: "复制错误",
+    Activated: "已激活",
+    notActive: "未激活",
   },
-  tranformImageFailed:'图片脱敏失败',
-  originPicture:'原始图片',
-  desensitizationPicture:'脱敏图片',
-  desensitizationObject:'脱敏对象',
-  example:'示例',
-  startDesensitization:'开始处理',
-  all:'全部',
-  onlyFace:'仅人脸',
-  onlyLicensePlate:'仅车牌',
-  dragThePictureHere:'拖动图片到这里',
-  or:'或',
-  clickUpload:'点击上传',
-  dataDesensitizationModelExperience:'数据脱敏模型体验',
-  dataDesensitizationModelDesc:'利用人工智能AI技术，把图片中的人脸、车牌号码进行脱敏处理。该模型更多信息请访问项目',
-  limitFilesUpload:'只能上传 jpg/jpeg/png 格式的文件',
-  limitSizeUpload:'上传文件大小不能超过 20M !',
-}
+  tranformImageFailed: "图片脱敏失败",
+  originPicture: "原始图片",
+  desensitizationPicture: "脱敏图片",
+  desensitizationObject: "脱敏对象",
+  example: "示例",
+  startDesensitization: "开始处理",
+  all: "全部",
+  onlyFace: "仅人脸",
+  onlyLicensePlate: "仅车牌",
+  dragThePictureHere: "拖动图片到这里",
+  or: "或",
+  clickUpload: "点击上传",
+  dataDesensitizationModelExperience: "数据脱敏模型体验",
+  dataDesensitizationModelDesc:
+    "利用人工智能AI技术，把图片中的人脸、车牌号码进行脱敏处理。该模型更多信息请访问项目",
+  limitFilesUpload: "只能上传 jpg/jpeg/png 格式的文件",
+  limitSizeUpload: "上传文件大小不能超过 20M !",
+  notebook: {
+    createNewNotebook: "新建Notebook调试任务",
+    sameTaskTips1: "您已经有",
+    sameTaskTips2: "同类任务",
+    sameTaskTips3: "正在等待或运行中，请等待任务结束再创建",
+    sameTaskTips4: "可以在",
+    sameTaskTips5: "个人中心",
+    sameTaskTips6: "云脑任务",
+    sameTaskTips7: "查看您所有的云脑任务。",
+    cpuEnv: "CPU 环境",
+    gpuEnv: "GPU 环境",
+    npuEnv: "NPU 环境",
+    newTask: "新建任务",
+    noQuene: "当前资源充足，没有任务排队",
+    queneTips1: "当前有",
+    queneTips2: "个任务正在排队",
+    watiResource: "正在等待分配资源，请耐心等待",
+    debug: "调试",
+    stop: "停止",
+    stopping: "停止中",
+    notebookRunning: "调试环境已启动，单次连接 4 小时或无操作半小时后自动关闭",
+    stopSuccess: "停止任务成功",
+    specification: "规格",
+    graphicMemory: "显存",
+    memory: "内存",
+    sharedMemory: "共享内存",
+  },
+};
 
 export default zh;
diff --git a/web_src/vuepages/pages/notebook/debug/index.vue b/web_src/vuepages/pages/notebook/debug/index.vue
index 5faee62a4..b305816cb 100644
--- a/web_src/vuepages/pages/notebook/debug/index.vue
+++ b/web_src/vuepages/pages/notebook/debug/index.vue
@@ -1,7 +1,7 @@