From 292236988b38327f9c7527462cc97307b1f6efa2 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 17 Mar 2022 10:21:46 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/user_business_analysis.go | 365 +++++++++++++++++-------------- models/user_business_struct.go | 72 ++++++ 2 files changed, 278 insertions(+), 159 deletions(-) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 1c87f5d08..e059ef3e4 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -80,6 +80,19 @@ type UserBusinessAnalysisAll struct { Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + //cloudbraintask + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` } type UserBusinessAnalysis struct { @@ -145,6 +158,18 @@ type UserBusinessAnalysis struct { Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` } type UserBusinessAnalysisQueryOptions struct { @@ -333,6 +358,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize + resultMap[userRecord.ID].CommitDatasetNum += userRecord.CommitDatasetNum resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount @@ -379,13 +405,14 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS StarRepoCountMap := queryStar(start_unix, end_unix) WatchedCountMap := queryFollow(start_unix, end_unix) CommitCodeSizeMap := queryCommitCodeSize(start_unix, end_unix) - CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix) + CommitDatasetSizeMap, CommitDatasetNumMap := queryDatasetSize(start_unix, end_unix) SolveIssueCountMap := querySolveIssue(start_unix, end_unix) CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) LoginCountMap := queryLoginCount(start_unix, end_unix) OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix) - + CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) + AiModelManageMap := queryUserModel(start_unix, end_unix) DataDate := currentTimeNow.Format("2006-01-02") + " 00:01" cond := "type != 1 and is_active=true" @@ -411,84 +438,22 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS dateRecordAll.Name = userRecord.Name dateRecordAll.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) dateRecordAll.DataDate = DataDate - - if _, ok := CodeMergeCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.CodeMergeCount = 0 - } else { - dateRecordAll.CodeMergeCount = CodeMergeCountMap[dateRecordAll.ID] - } - - if _, ok := CommitCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.CommitCount = 0 - } else { - dateRecordAll.CommitCount = CommitCountMap[dateRecordAll.ID] - } - - if _, ok := IssueCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.IssueCount = 0 - } else { - dateRecordAll.IssueCount = IssueCountMap[dateRecordAll.ID] - } - - if _, ok := CommentCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.CommentCount = 0 - } else { - dateRecordAll.CommentCount = CommentCountMap[dateRecordAll.ID] - } - - if _, ok := FocusRepoCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.FocusRepoCount = 0 - } else { - dateRecordAll.FocusRepoCount = FocusRepoCountMap[dateRecordAll.ID] - } - - if _, ok := StarRepoCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.StarRepoCount = 0 - } else { - dateRecordAll.StarRepoCount = StarRepoCountMap[dateRecordAll.ID] - } - - if _, ok := WatchedCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.WatchedCount = 0 - } else { - dateRecordAll.WatchedCount = WatchedCountMap[dateRecordAll.ID] - } - - if _, ok := CommitCodeSizeMap[dateRecordAll.ID]; !ok { - dateRecordAll.CommitCodeSize = 0 - } else { - dateRecordAll.CommitCodeSize = CommitCodeSizeMap[dateRecordAll.ID] - } - - if _, ok := CommitDatasetSizeMap[dateRecordAll.ID]; !ok { - dateRecordAll.CommitDatasetSize = 0 - } else { - dateRecordAll.CommitDatasetSize = CommitDatasetSizeMap[dateRecordAll.ID] - } - - if _, ok := SolveIssueCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.SolveIssueCount = 0 - } else { - dateRecordAll.SolveIssueCount = SolveIssueCountMap[dateRecordAll.ID] - } - - if _, ok := wikiCountMap[dateRecordAll.Name]; !ok { - dateRecordAll.EncyclopediasCount = 0 - } else { - dateRecordAll.EncyclopediasCount = wikiCountMap[dateRecordAll.Name] - } - - if _, ok := CreateRepoCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.CreateRepoCount = 0 - } else { - dateRecordAll.CreateRepoCount = CreateRepoCountMap[dateRecordAll.ID] - } - - if _, ok := LoginCountMap[dateRecordAll.ID]; !ok { - dateRecordAll.LoginCount = 0 - } else { - dateRecordAll.LoginCount = LoginCountMap[dateRecordAll.ID] - } + dateRecordAll.UserLocation = userRecord.Location + + dateRecordAll.CodeMergeCount = getMapValue(dateRecordAll.ID, CodeMergeCountMap) + dateRecordAll.CommitCount = getMapValue(dateRecordAll.ID, CommitCountMap) + dateRecordAll.IssueCount = getMapValue(dateRecordAll.ID, IssueCountMap) + dateRecordAll.CommentCount = getMapValue(dateRecordAll.ID, CommentCountMap) + dateRecordAll.FocusRepoCount = getMapValue(dateRecordAll.ID, FocusRepoCountMap) + dateRecordAll.StarRepoCount = getMapValue(dateRecordAll.ID, StarRepoCountMap) + dateRecordAll.WatchedCount = getMapValue(dateRecordAll.ID, WatchedCountMap) + dateRecordAll.CommitCodeSize = getMapValue(dateRecordAll.ID, CommitCodeSizeMap) + dateRecordAll.CommitDatasetSize = getMapValue(dateRecordAll.ID, CommitDatasetSizeMap) + dateRecordAll.CommitDatasetNum = getMapValue(dateRecordAll.ID, CommitDatasetNumMap) + dateRecordAll.SolveIssueCount = getMapValue(dateRecordAll.ID, SolveIssueCountMap) + dateRecordAll.EncyclopediasCount = getMapKeyStringValue(dateRecordAll.Name, wikiCountMap) + dateRecordAll.CreateRepoCount = getMapValue(dateRecordAll.ID, CreateRepoCountMap) + dateRecordAll.LoginCount = getMapValue(dateRecordAll.ID, LoginCountMap) if _, ok := OpenIIndexMap[dateRecordAll.ID]; !ok { dateRecordAll.OpenIIndex = 0 @@ -496,7 +461,15 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS dateRecordAll.OpenIIndex = OpenIIndexMap[dateRecordAll.ID] } - dateRecordAll.CommitModelCount = 0 + dateRecordAll.CloudBrainTaskNum = getMapValue(dateRecordAll.ID, CloudBrainTaskMap) + dateRecordAll.GpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuDebugJob", CloudBrainTaskItemMap) + dateRecordAll.NpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuDebugJob", CloudBrainTaskItemMap) + dateRecordAll.GpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuTrainJob", CloudBrainTaskItemMap) + dateRecordAll.NpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuTrainJob", CloudBrainTaskItemMap) + dateRecordAll.NpuInferenceJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuInferenceJob", CloudBrainTaskItemMap) + dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) + dateRecordAll.CommitModelCount = getMapValue(dateRecordAll.ID, AiModelManageMap) + //dateRecordAll.UserIndex = getUserIndex(dateRecordAll) dateRecordBatch = append(dateRecordBatch, dateRecordAll) if len(dateRecordBatch) >= BATCH_INSERT_SIZE { @@ -528,7 +501,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static insertBatchSql := "INSERT INTO public." + tableName + "(id, count_date, code_merge_count, commit_count, issue_count, comment_count, focus_repo_count, star_repo_count, watched_count, gitea_age_month, commit_code_size, commit_dataset_size, " + - "commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date) " + + "commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location) " + "VALUES" for i, record := range dateRecords { @@ -536,7 +509,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static ", " + fmt.Sprint(record.IssueCount) + ", " + fmt.Sprint(record.CommentCount) + ", " + fmt.Sprint(record.FocusRepoCount) + ", " + fmt.Sprint(record.StarRepoCount) + ", " + fmt.Sprint(record.WatchedCount) + ", " + fmt.Sprint(record.GiteaAgeMonth) + ", " + fmt.Sprint(record.CommitCodeSize) + ", " + fmt.Sprint(record.CommitDatasetSize) + ", " + fmt.Sprint(record.CommitModelCount) + ", " + fmt.Sprint(record.SolveIssueCount) + ", " + fmt.Sprint(record.EncyclopediasCount) + ", " + fmt.Sprint(record.RegistDate) + - ", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "')" + ", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "'," + fmt.Sprint(record.CloudBrainTaskNum) + "," + fmt.Sprint(record.GpuDebugJob) + "," + fmt.Sprint(record.NpuDebugJob) + "," + fmt.Sprint(record.GpuTrainJob) + "," + fmt.Sprint(record.NpuTrainJob) + "," + fmt.Sprint(record.NpuInferenceJob) + "," + fmt.Sprint(record.GpuBenchMarkJob) + "," + fmt.Sprint(record.CloudBrainRunTime) + "," + fmt.Sprint(record.CommitDatasetNum) + "," + fmt.Sprint(record.UserIndex) + ",'" + record.UserLocation + "')" if i < (len(dateRecords) - 1) { insertBatchSql += "," } @@ -612,12 +585,13 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, } else { log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap))) } - CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix) + CommitDatasetSizeMap, CommitDatasetNumMap := queryDatasetSize(start_unix, end_unix) SolveIssueCountMap := querySolveIssue(start_unix, end_unix) CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) LoginCountMap := queryLoginCount(start_unix, end_unix) OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix) - + CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix) + AiModelManageMap := queryUserModel(start_unix, end_unix) statictisSess := xStatistic.NewSession() defer statictisSess.Close() @@ -647,47 +621,14 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, dateRecord.Name = userRecord.Name dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) dateRecord.DataDate = DataDate - if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { - dateRecord.CodeMergeCount = 0 - } else { - dateRecord.CodeMergeCount = CodeMergeCountMap[dateRecord.ID] - } - if _, ok := CommitCountMap[dateRecord.ID]; !ok { - dateRecord.CommitCount = 0 - } else { - dateRecord.CommitCount = CommitCountMap[dateRecord.ID] - } - - if _, ok := IssueCountMap[dateRecord.ID]; !ok { - dateRecord.IssueCount = 0 - } else { - dateRecord.IssueCount = IssueCountMap[dateRecord.ID] - } - - if _, ok := CommentCountMap[dateRecord.ID]; !ok { - dateRecord.CommentCount = 0 - } else { - dateRecord.CommentCount = CommentCountMap[dateRecord.ID] - } - - if _, ok := FocusRepoCountMap[dateRecord.ID]; !ok { - dateRecord.FocusRepoCount = 0 - } else { - dateRecord.FocusRepoCount = FocusRepoCountMap[dateRecord.ID] - } - - if _, ok := StarRepoCountMap[dateRecord.ID]; !ok { - dateRecord.StarRepoCount = 0 - } else { - dateRecord.StarRepoCount = StarRepoCountMap[dateRecord.ID] - } - - if _, ok := WatchedCountMap[dateRecord.ID]; !ok { - dateRecord.WatchedCount = 0 - } else { - dateRecord.WatchedCount = WatchedCountMap[dateRecord.ID] - } + dateRecord.CodeMergeCount = getMapValue(dateRecord.ID, CodeMergeCountMap) + dateRecord.CommitCount = getMapValue(dateRecord.ID, CommitCountMap) + dateRecord.IssueCount = getMapValue(dateRecord.ID, IssueCountMap) + dateRecord.CommentCount = getMapValue(dateRecord.ID, CommentCountMap) + dateRecord.FocusRepoCount = getMapValue(dateRecord.ID, FocusRepoCountMap) + dateRecord.StarRepoCount = getMapValue(dateRecord.ID, StarRepoCountMap) + dateRecord.WatchedCount = getMapValue(dateRecord.ID, WatchedCountMap) if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { dateRecord.CommitCodeSize = 0 @@ -695,35 +636,15 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) } - if _, ok := CommitDatasetSizeMap[dateRecord.ID]; !ok { - dateRecord.CommitDatasetSize = 0 - } else { - dateRecord.CommitDatasetSize = CommitDatasetSizeMap[dateRecord.ID] - } - - if _, ok := SolveIssueCountMap[dateRecord.ID]; !ok { - dateRecord.SolveIssueCount = 0 - } else { - dateRecord.SolveIssueCount = SolveIssueCountMap[dateRecord.ID] - } + dateRecord.CommitDatasetSize = getMapValue(dateRecord.ID, CommitDatasetSizeMap) + dateRecord.CommitDatasetNum = getMapValue(dateRecord.ID, CommitDatasetNumMap) + dateRecord.SolveIssueCount = getMapValue(dateRecord.ID, SolveIssueCountMap) - if _, ok := wikiCountMap[dateRecord.Name]; !ok { - dateRecord.EncyclopediasCount = 0 - } else { - dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name] - } + dateRecord.EncyclopediasCount = getMapKeyStringValue(dateRecord.Name, wikiCountMap) - if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok { - dateRecord.CreateRepoCount = 0 - } else { - dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] - } + dateRecord.CreateRepoCount = getMapValue(dateRecord.ID, CreateRepoCountMap) - if _, ok := LoginCountMap[dateRecord.ID]; !ok { - dateRecord.LoginCount = 0 - } else { - dateRecord.LoginCount = LoginCountMap[dateRecord.ID] - } + dateRecord.LoginCount = getMapValue(dateRecord.ID, LoginCountMap) if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { dateRecord.OpenIIndex = 0 @@ -731,7 +652,16 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] } - dateRecord.CommitModelCount = 0 + dateRecord.CloudBrainTaskNum = getMapValue(dateRecord.ID, CloudBrainTaskMap) + dateRecord.GpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuDebugJob", CloudBrainTaskItemMap) + dateRecord.NpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_NpuDebugJob", CloudBrainTaskItemMap) + dateRecord.GpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuTrainJob", CloudBrainTaskItemMap) + dateRecord.NpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_NpuTrainJob", CloudBrainTaskItemMap) + dateRecord.NpuInferenceJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_NpuInferenceJob", CloudBrainTaskItemMap) + dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap) + + dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap) + dateRecord.UserIndex = getUserIndex(dateRecord) _, err = statictisSess.Insert(&dateRecord) if err != nil { @@ -751,6 +681,27 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, return nil } +func getUserIndex(dateRecord UserBusinessAnalysis) float64 { + + return 0 +} + +func getMapKeyStringValue(key string, valueMap map[string]int) int { + if _, ok := valueMap[key]; !ok { + return 0 + } else { + return valueMap[key] + } +} + +func getMapValue(userId int64, valueMap map[int64]int) int { + if _, ok := valueMap[userId]; !ok { + return 0 + } else { + return valueMap[userId] + } +} + func getInt(str string) int { re, err := strconv.ParseInt(str, 10, 32) if err != nil { @@ -1051,16 +1002,17 @@ func queryFollow(start_unix int64, end_unix int64) map[int64]int { return resultMap } -func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { +func queryDatasetSize(start_unix int64, end_unix int64) (map[int64]int, map[int64]int) { sess := x.NewSession() defer sess.Close() - resultMap := make(map[int64]int) + resultSizeMap := make(map[int64]int) + resultNumMap := make(map[int64]int) cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) count, err := sess.Where(cond).Count(new(Attachment)) if err != nil { log.Info("query attachment error. return.") - return resultMap + return resultSizeMap, resultNumMap } var indexTotal int64 indexTotal = 0 @@ -1071,10 +1023,12 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList))) for _, attachRecord := range attachmentList { - if _, ok := resultMap[attachRecord.UploaderID]; !ok { - resultMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB + if _, ok := resultSizeMap[attachRecord.UploaderID]; !ok { + resultSizeMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB + resultNumMap[attachRecord.UploaderID] = 1 } else { - resultMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB + resultSizeMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB + resultNumMap[attachRecord.UploaderID] += 1 } } @@ -1084,7 +1038,7 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { } } - return resultMap + return resultSizeMap, resultNumMap } func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { @@ -1245,6 +1199,99 @@ func queryCommitCodeSize(start_unix int64, end_unix int64) map[int64]int { return resultMap } +func queryUserModel(start_unix int64, end_unix int64) map[int64]int { + sess := x.NewSession() + defer sess.Close() + resultMap := make(map[int64]int) + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Where(cond).Count(new(AiModelManage)) + if err != nil { + log.Info("query AiModelManage error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id").Table("ai_model_manage").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + aiModelList := make([]*AiModelManage, 0) + sess.Find(&aiModelList) + log.Info("query AiModelManage size=" + fmt.Sprint(len(aiModelList))) + for _, aiModelRecord := range aiModelList { + if _, ok := resultMap[aiModelRecord.UserId]; !ok { + resultMap[aiModelRecord.UserId] = 1 + } else { + resultMap[aiModelRecord.UserId] += 1 + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + return resultMap +} + +func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[string]int) { + sess := x.NewSession() + defer sess.Close() + resultMap := make(map[int64]int) + resultItemMap := make(map[string]int) + + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Where(cond).Count(new(Cloudbrain)) + if err != nil { + log.Info("query cloudbrain error. return.") + return resultMap, resultItemMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,job_type,user_id,duration,train_job_duration,type").Table("cloudbrain").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + cloudTaskList := make([]*Cloudbrain, 0) + sess.Find(&cloudTaskList) + log.Info("query cloudbrain size=" + fmt.Sprint(len(cloudTaskList))) + for _, cloudTaskRecord := range cloudTaskList { + if _, ok := resultMap[cloudTaskRecord.UserID]; !ok { + resultMap[cloudTaskRecord.UserID] = 1 + } else { + resultMap[cloudTaskRecord.UserID] += 1 + } + setMapKey("time", cloudTaskRecord.UserID, int(cloudTaskRecord.Duration), resultItemMap) + if cloudTaskRecord.Type == 1 { //npu + if cloudTaskRecord.JobType == "TRAIN" { + setMapKey("NpuTrainJob", cloudTaskRecord.UserID, 1, resultItemMap) + } else if cloudTaskRecord.JobType == "INFERENCE" { + setMapKey("NpuInferenceJob", cloudTaskRecord.UserID, 1, resultItemMap) + } else { + setMapKey("NpuDebugJob", cloudTaskRecord.UserID, 1, resultItemMap) + } + } else { //type=0 gpu + if cloudTaskRecord.JobType == "TRAIN" { + setMapKey("GpuTrainJob", cloudTaskRecord.UserID, 1, resultItemMap) + } else if cloudTaskRecord.JobType == "BENCHMARK" { + setMapKey("GpuBenchMarkJob", cloudTaskRecord.UserID, 1, resultItemMap) + } else { + setMapKey("GpuDebugJob", cloudTaskRecord.UserID, 1, resultItemMap) + } + } + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + + return resultMap, resultItemMap +} +func setMapKey(key string, userId int64, value int, resultItemMap map[string]int) { + newKey := fmt.Sprint(userId) + "_" + key + if _, ok := resultItemMap[newKey]; !ok { + resultItemMap[newKey] = value + } else { + resultItemMap[newKey] += value + } +} + func subMonth(t1, t2 time.Time) (month int) { y1 := t1.Year() y2 := t2.Year() diff --git a/models/user_business_struct.go b/models/user_business_struct.go index c435c0b07..066e11314 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -44,6 +44,18 @@ type UserBusinessAnalysisCurrentYear struct { //user Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` } type UserBusinessAnalysisLast30Day struct { @@ -88,6 +100,18 @@ type UserBusinessAnalysisLast30Day struct { //user Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` } type UserBusinessAnalysisLastMonth struct { @@ -132,6 +156,18 @@ type UserBusinessAnalysisLastMonth struct { //user Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` } type UserBusinessAnalysisCurrentMonth struct { @@ -176,6 +212,18 @@ type UserBusinessAnalysisCurrentMonth struct { //user Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` } type UserBusinessAnalysisCurrentWeek struct { @@ -220,6 +268,18 @@ type UserBusinessAnalysisCurrentWeek struct { //user Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` } type UserBusinessAnalysisYesterday struct { @@ -264,4 +324,16 @@ type UserBusinessAnalysisYesterday struct { //user Name string `xorm:"NOT NULL"` DataDate string `xorm:"NULL"` + + CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"` + GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"` + GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"` + NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"` + GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"` + CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"` + CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"` + UserIndex float64 `xorm:"NOT NULL DEFAULT 0"` + UserLocation string `xorm:"NULL"` }