Browse Source

提交代码。

Signed-off-by: zouap <zouap@pcl.ac.cn>
tags/v1.22.4.1^2
zouap 3 years ago
parent
commit
292236988b
2 changed files with 278 additions and 159 deletions
  1. +206
    -159
      models/user_business_analysis.go
  2. +72
    -0
      models/user_business_struct.go

+ 206
- 159
models/user_business_analysis.go View File

@@ -80,6 +80,19 @@ type UserBusinessAnalysisAll struct {
Name string `xorm:"NOT NULL"`

DataDate string `xorm:"NULL"`

//cloudbraintask
CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"`
GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"`
GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"`
CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"`
CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"`
UserIndex float64 `xorm:"NOT NULL DEFAULT 0"`
UserLocation string `xorm:"NULL"`
}

type UserBusinessAnalysis struct {
@@ -145,6 +158,18 @@ type UserBusinessAnalysis struct {
Name string `xorm:"NOT NULL"`

DataDate string `xorm:"NULL"`

CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"`
GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"`
GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"`
CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"`
CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"`
UserIndex float64 `xorm:"NOT NULL DEFAULT 0"`
UserLocation string `xorm:"NULL"`
}

type UserBusinessAnalysisQueryOptions struct {
@@ -333,6 +358,7 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus
resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount
resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize
resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize
resultMap[userRecord.ID].CommitDatasetNum += userRecord.CommitDatasetNum
resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount
resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount
resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount
@@ -379,13 +405,14 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
StarRepoCountMap := queryStar(start_unix, end_unix)
WatchedCountMap := queryFollow(start_unix, end_unix)
CommitCodeSizeMap := queryCommitCodeSize(start_unix, end_unix)
CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix)
CommitDatasetSizeMap, CommitDatasetNumMap := queryDatasetSize(start_unix, end_unix)
SolveIssueCountMap := querySolveIssue(start_unix, end_unix)
CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix)
LoginCountMap := queryLoginCount(start_unix, end_unix)

OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix)

CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix)
AiModelManageMap := queryUserModel(start_unix, end_unix)
DataDate := currentTimeNow.Format("2006-01-02") + " 00:01"

cond := "type != 1 and is_active=true"
@@ -411,84 +438,22 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
dateRecordAll.Name = userRecord.Name
dateRecordAll.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime())
dateRecordAll.DataDate = DataDate

if _, ok := CodeMergeCountMap[dateRecordAll.ID]; !ok {
dateRecordAll.CodeMergeCount = 0
} else {
dateRecordAll.CodeMergeCount = CodeMergeCountMap[dateRecordAll.ID]
}

if _, ok := CommitCountMap[dateRecordAll.ID]; !ok {
dateRecordAll.CommitCount = 0
} else {
dateRecordAll.CommitCount = CommitCountMap[dateRecordAll.ID]
}

if _, ok := IssueCountMap[dateRecordAll.ID]; !ok {
dateRecordAll.IssueCount = 0
} else {
dateRecordAll.IssueCount = IssueCountMap[dateRecordAll.ID]
}

if _, ok := CommentCountMap[dateRecordAll.ID]; !ok {
dateRecordAll.CommentCount = 0
} else {
dateRecordAll.CommentCount = CommentCountMap[dateRecordAll.ID]
}

if _, ok := FocusRepoCountMap[dateRecordAll.ID]; !ok {
dateRecordAll.FocusRepoCount = 0
} else {
dateRecordAll.FocusRepoCount = FocusRepoCountMap[dateRecordAll.ID]
}

if _, ok := StarRepoCountMap[dateRecordAll.ID]; !ok {
dateRecordAll.StarRepoCount = 0
} else {
dateRecordAll.StarRepoCount = StarRepoCountMap[dateRecordAll.ID]
}

if _, ok := WatchedCountMap[dateRecordAll.ID]; !ok {
dateRecordAll.WatchedCount = 0
} else {
dateRecordAll.WatchedCount = WatchedCountMap[dateRecordAll.ID]
}

if _, ok := CommitCodeSizeMap[dateRecordAll.ID]; !ok {
dateRecordAll.CommitCodeSize = 0
} else {
dateRecordAll.CommitCodeSize = CommitCodeSizeMap[dateRecordAll.ID]
}

if _, ok := CommitDatasetSizeMap[dateRecordAll.ID]; !ok {
dateRecordAll.CommitDatasetSize = 0
} else {
dateRecordAll.CommitDatasetSize = CommitDatasetSizeMap[dateRecordAll.ID]
}

if _, ok := SolveIssueCountMap[dateRecordAll.ID]; !ok {
dateRecordAll.SolveIssueCount = 0
} else {
dateRecordAll.SolveIssueCount = SolveIssueCountMap[dateRecordAll.ID]
}

if _, ok := wikiCountMap[dateRecordAll.Name]; !ok {
dateRecordAll.EncyclopediasCount = 0
} else {
dateRecordAll.EncyclopediasCount = wikiCountMap[dateRecordAll.Name]
}

if _, ok := CreateRepoCountMap[dateRecordAll.ID]; !ok {
dateRecordAll.CreateRepoCount = 0
} else {
dateRecordAll.CreateRepoCount = CreateRepoCountMap[dateRecordAll.ID]
}

if _, ok := LoginCountMap[dateRecordAll.ID]; !ok {
dateRecordAll.LoginCount = 0
} else {
dateRecordAll.LoginCount = LoginCountMap[dateRecordAll.ID]
}
dateRecordAll.UserLocation = userRecord.Location

dateRecordAll.CodeMergeCount = getMapValue(dateRecordAll.ID, CodeMergeCountMap)
dateRecordAll.CommitCount = getMapValue(dateRecordAll.ID, CommitCountMap)
dateRecordAll.IssueCount = getMapValue(dateRecordAll.ID, IssueCountMap)
dateRecordAll.CommentCount = getMapValue(dateRecordAll.ID, CommentCountMap)
dateRecordAll.FocusRepoCount = getMapValue(dateRecordAll.ID, FocusRepoCountMap)
dateRecordAll.StarRepoCount = getMapValue(dateRecordAll.ID, StarRepoCountMap)
dateRecordAll.WatchedCount = getMapValue(dateRecordAll.ID, WatchedCountMap)
dateRecordAll.CommitCodeSize = getMapValue(dateRecordAll.ID, CommitCodeSizeMap)
dateRecordAll.CommitDatasetSize = getMapValue(dateRecordAll.ID, CommitDatasetSizeMap)
dateRecordAll.CommitDatasetNum = getMapValue(dateRecordAll.ID, CommitDatasetNumMap)
dateRecordAll.SolveIssueCount = getMapValue(dateRecordAll.ID, SolveIssueCountMap)
dateRecordAll.EncyclopediasCount = getMapKeyStringValue(dateRecordAll.Name, wikiCountMap)
dateRecordAll.CreateRepoCount = getMapValue(dateRecordAll.ID, CreateRepoCountMap)
dateRecordAll.LoginCount = getMapValue(dateRecordAll.ID, LoginCountMap)

if _, ok := OpenIIndexMap[dateRecordAll.ID]; !ok {
dateRecordAll.OpenIIndex = 0
@@ -496,7 +461,15 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS
dateRecordAll.OpenIIndex = OpenIIndexMap[dateRecordAll.ID]
}

dateRecordAll.CommitModelCount = 0
dateRecordAll.CloudBrainTaskNum = getMapValue(dateRecordAll.ID, CloudBrainTaskMap)
dateRecordAll.GpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuDebugJob", CloudBrainTaskItemMap)
dateRecordAll.NpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuDebugJob", CloudBrainTaskItemMap)
dateRecordAll.GpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuTrainJob", CloudBrainTaskItemMap)
dateRecordAll.NpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuTrainJob", CloudBrainTaskItemMap)
dateRecordAll.NpuInferenceJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_NpuInferenceJob", CloudBrainTaskItemMap)
dateRecordAll.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecordAll.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap)
dateRecordAll.CommitModelCount = getMapValue(dateRecordAll.ID, AiModelManageMap)
//dateRecordAll.UserIndex = getUserIndex(dateRecordAll)

dateRecordBatch = append(dateRecordBatch, dateRecordAll)
if len(dateRecordBatch) >= BATCH_INSERT_SIZE {
@@ -528,7 +501,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static

insertBatchSql := "INSERT INTO public." + tableName +
"(id, count_date, code_merge_count, commit_count, issue_count, comment_count, focus_repo_count, star_repo_count, watched_count, gitea_age_month, commit_code_size, commit_dataset_size, " +
"commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date) " +
"commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date,cloud_brain_task_num,gpu_debug_job,npu_debug_job,gpu_train_job,npu_train_job,npu_inference_job,gpu_bench_mark_job,cloud_brain_run_time,commit_dataset_num,user_index,user_location) " +
"VALUES"

for i, record := range dateRecords {
@@ -536,7 +509,7 @@ func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, static
", " + fmt.Sprint(record.IssueCount) + ", " + fmt.Sprint(record.CommentCount) + ", " + fmt.Sprint(record.FocusRepoCount) + ", " + fmt.Sprint(record.StarRepoCount) +
", " + fmt.Sprint(record.WatchedCount) + ", " + fmt.Sprint(record.GiteaAgeMonth) + ", " + fmt.Sprint(record.CommitCodeSize) + ", " + fmt.Sprint(record.CommitDatasetSize) +
", " + fmt.Sprint(record.CommitModelCount) + ", " + fmt.Sprint(record.SolveIssueCount) + ", " + fmt.Sprint(record.EncyclopediasCount) + ", " + fmt.Sprint(record.RegistDate) +
", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "')"
", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "'," + fmt.Sprint(record.CloudBrainTaskNum) + "," + fmt.Sprint(record.GpuDebugJob) + "," + fmt.Sprint(record.NpuDebugJob) + "," + fmt.Sprint(record.GpuTrainJob) + "," + fmt.Sprint(record.NpuTrainJob) + "," + fmt.Sprint(record.NpuInferenceJob) + "," + fmt.Sprint(record.GpuBenchMarkJob) + "," + fmt.Sprint(record.CloudBrainRunTime) + "," + fmt.Sprint(record.CommitDatasetNum) + "," + fmt.Sprint(record.UserIndex) + ",'" + record.UserLocation + "')"
if i < (len(dateRecords) - 1) {
insertBatchSql += ","
}
@@ -612,12 +585,13 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
} else {
log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap)))
}
CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix)
CommitDatasetSizeMap, CommitDatasetNumMap := queryDatasetSize(start_unix, end_unix)
SolveIssueCountMap := querySolveIssue(start_unix, end_unix)
CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix)
LoginCountMap := queryLoginCount(start_unix, end_unix)
OpenIIndexMap := queryUserRepoOpenIIndex(start_unix, end_unix)

CloudBrainTaskMap, CloudBrainTaskItemMap := queryCloudBrainTask(start_unix, end_unix)
AiModelManageMap := queryUserModel(start_unix, end_unix)
statictisSess := xStatistic.NewSession()
defer statictisSess.Close()

@@ -647,47 +621,14 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
dateRecord.Name = userRecord.Name
dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime())
dateRecord.DataDate = DataDate
if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok {
dateRecord.CodeMergeCount = 0
} else {
dateRecord.CodeMergeCount = CodeMergeCountMap[dateRecord.ID]
}

if _, ok := CommitCountMap[dateRecord.ID]; !ok {
dateRecord.CommitCount = 0
} else {
dateRecord.CommitCount = CommitCountMap[dateRecord.ID]
}

if _, ok := IssueCountMap[dateRecord.ID]; !ok {
dateRecord.IssueCount = 0
} else {
dateRecord.IssueCount = IssueCountMap[dateRecord.ID]
}

if _, ok := CommentCountMap[dateRecord.ID]; !ok {
dateRecord.CommentCount = 0
} else {
dateRecord.CommentCount = CommentCountMap[dateRecord.ID]
}

if _, ok := FocusRepoCountMap[dateRecord.ID]; !ok {
dateRecord.FocusRepoCount = 0
} else {
dateRecord.FocusRepoCount = FocusRepoCountMap[dateRecord.ID]
}

if _, ok := StarRepoCountMap[dateRecord.ID]; !ok {
dateRecord.StarRepoCount = 0
} else {
dateRecord.StarRepoCount = StarRepoCountMap[dateRecord.ID]
}

if _, ok := WatchedCountMap[dateRecord.ID]; !ok {
dateRecord.WatchedCount = 0
} else {
dateRecord.WatchedCount = WatchedCountMap[dateRecord.ID]
}
dateRecord.CodeMergeCount = getMapValue(dateRecord.ID, CodeMergeCountMap)
dateRecord.CommitCount = getMapValue(dateRecord.ID, CommitCountMap)
dateRecord.IssueCount = getMapValue(dateRecord.ID, IssueCountMap)
dateRecord.CommentCount = getMapValue(dateRecord.ID, CommentCountMap)
dateRecord.FocusRepoCount = getMapValue(dateRecord.ID, FocusRepoCountMap)
dateRecord.StarRepoCount = getMapValue(dateRecord.ID, StarRepoCountMap)
dateRecord.WatchedCount = getMapValue(dateRecord.ID, WatchedCountMap)

if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok {
dateRecord.CommitCodeSize = 0
@@ -695,35 +636,15 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines)
}

if _, ok := CommitDatasetSizeMap[dateRecord.ID]; !ok {
dateRecord.CommitDatasetSize = 0
} else {
dateRecord.CommitDatasetSize = CommitDatasetSizeMap[dateRecord.ID]
}

if _, ok := SolveIssueCountMap[dateRecord.ID]; !ok {
dateRecord.SolveIssueCount = 0
} else {
dateRecord.SolveIssueCount = SolveIssueCountMap[dateRecord.ID]
}
dateRecord.CommitDatasetSize = getMapValue(dateRecord.ID, CommitDatasetSizeMap)
dateRecord.CommitDatasetNum = getMapValue(dateRecord.ID, CommitDatasetNumMap)
dateRecord.SolveIssueCount = getMapValue(dateRecord.ID, SolveIssueCountMap)

if _, ok := wikiCountMap[dateRecord.Name]; !ok {
dateRecord.EncyclopediasCount = 0
} else {
dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name]
}
dateRecord.EncyclopediasCount = getMapKeyStringValue(dateRecord.Name, wikiCountMap)

if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok {
dateRecord.CreateRepoCount = 0
} else {
dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID]
}
dateRecord.CreateRepoCount = getMapValue(dateRecord.ID, CreateRepoCountMap)

if _, ok := LoginCountMap[dateRecord.ID]; !ok {
dateRecord.LoginCount = 0
} else {
dateRecord.LoginCount = LoginCountMap[dateRecord.ID]
}
dateRecord.LoginCount = getMapValue(dateRecord.ID, LoginCountMap)

if _, ok := OpenIIndexMap[dateRecord.ID]; !ok {
dateRecord.OpenIIndex = 0
@@ -731,7 +652,16 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID]
}

dateRecord.CommitModelCount = 0
dateRecord.CloudBrainTaskNum = getMapValue(dateRecord.ID, CloudBrainTaskMap)
dateRecord.GpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuDebugJob", CloudBrainTaskItemMap)
dateRecord.NpuDebugJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_NpuDebugJob", CloudBrainTaskItemMap)
dateRecord.GpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuTrainJob", CloudBrainTaskItemMap)
dateRecord.NpuTrainJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_NpuTrainJob", CloudBrainTaskItemMap)
dateRecord.NpuInferenceJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_NpuInferenceJob", CloudBrainTaskItemMap)
dateRecord.GpuBenchMarkJob = getMapKeyStringValue(fmt.Sprint(dateRecord.ID)+"_GpuBenchMarkJob", CloudBrainTaskItemMap)

dateRecord.CommitModelCount = getMapValue(dateRecord.ID, AiModelManageMap)
dateRecord.UserIndex = getUserIndex(dateRecord)

_, err = statictisSess.Insert(&dateRecord)
if err != nil {
@@ -751,6 +681,27 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time,
return nil
}

func getUserIndex(dateRecord UserBusinessAnalysis) float64 {

return 0
}

func getMapKeyStringValue(key string, valueMap map[string]int) int {
if _, ok := valueMap[key]; !ok {
return 0
} else {
return valueMap[key]
}
}

func getMapValue(userId int64, valueMap map[int64]int) int {
if _, ok := valueMap[userId]; !ok {
return 0
} else {
return valueMap[userId]
}
}

func getInt(str string) int {
re, err := strconv.ParseInt(str, 10, 32)
if err != nil {
@@ -1051,16 +1002,17 @@ func queryFollow(start_unix int64, end_unix int64) map[int64]int {
return resultMap
}

func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int {
func queryDatasetSize(start_unix int64, end_unix int64) (map[int64]int, map[int64]int) {
sess := x.NewSession()
defer sess.Close()
resultMap := make(map[int64]int)
resultSizeMap := make(map[int64]int)
resultNumMap := make(map[int64]int)
cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)

count, err := sess.Where(cond).Count(new(Attachment))
if err != nil {
log.Info("query attachment error. return.")
return resultMap
return resultSizeMap, resultNumMap
}
var indexTotal int64
indexTotal = 0
@@ -1071,10 +1023,12 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int {

log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList)))
for _, attachRecord := range attachmentList {
if _, ok := resultMap[attachRecord.UploaderID]; !ok {
resultMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB
if _, ok := resultSizeMap[attachRecord.UploaderID]; !ok {
resultSizeMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB
resultNumMap[attachRecord.UploaderID] = 1
} else {
resultMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB
resultSizeMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB
resultNumMap[attachRecord.UploaderID] += 1
}
}

@@ -1084,7 +1038,7 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int {
}
}

return resultMap
return resultSizeMap, resultNumMap
}

func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int {
@@ -1245,6 +1199,99 @@ func queryCommitCodeSize(start_unix int64, end_unix int64) map[int64]int {
return resultMap
}

func queryUserModel(start_unix int64, end_unix int64) map[int64]int {
sess := x.NewSession()
defer sess.Close()
resultMap := make(map[int64]int)
cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)
count, err := sess.Where(cond).Count(new(AiModelManage))
if err != nil {
log.Info("query AiModelManage error. return.")
return resultMap
}
var indexTotal int64
indexTotal = 0
for {
sess.Select("id,user_id").Table("ai_model_manage").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
aiModelList := make([]*AiModelManage, 0)
sess.Find(&aiModelList)
log.Info("query AiModelManage size=" + fmt.Sprint(len(aiModelList)))
for _, aiModelRecord := range aiModelList {
if _, ok := resultMap[aiModelRecord.UserId]; !ok {
resultMap[aiModelRecord.UserId] = 1
} else {
resultMap[aiModelRecord.UserId] += 1
}
}
indexTotal += PAGE_SIZE
if indexTotal >= count {
break
}
}
return resultMap
}

func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[string]int) {
sess := x.NewSession()
defer sess.Close()
resultMap := make(map[int64]int)
resultItemMap := make(map[string]int)

cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)
count, err := sess.Where(cond).Count(new(Cloudbrain))
if err != nil {
log.Info("query cloudbrain error. return.")
return resultMap, resultItemMap
}
var indexTotal int64
indexTotal = 0
for {
sess.Select("id,job_type,user_id,duration,train_job_duration,type").Table("cloudbrain").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal))
cloudTaskList := make([]*Cloudbrain, 0)
sess.Find(&cloudTaskList)
log.Info("query cloudbrain size=" + fmt.Sprint(len(cloudTaskList)))
for _, cloudTaskRecord := range cloudTaskList {
if _, ok := resultMap[cloudTaskRecord.UserID]; !ok {
resultMap[cloudTaskRecord.UserID] = 1
} else {
resultMap[cloudTaskRecord.UserID] += 1
}
setMapKey("time", cloudTaskRecord.UserID, int(cloudTaskRecord.Duration), resultItemMap)
if cloudTaskRecord.Type == 1 { //npu
if cloudTaskRecord.JobType == "TRAIN" {
setMapKey("NpuTrainJob", cloudTaskRecord.UserID, 1, resultItemMap)
} else if cloudTaskRecord.JobType == "INFERENCE" {
setMapKey("NpuInferenceJob", cloudTaskRecord.UserID, 1, resultItemMap)
} else {
setMapKey("NpuDebugJob", cloudTaskRecord.UserID, 1, resultItemMap)
}
} else { //type=0 gpu
if cloudTaskRecord.JobType == "TRAIN" {
setMapKey("GpuTrainJob", cloudTaskRecord.UserID, 1, resultItemMap)
} else if cloudTaskRecord.JobType == "BENCHMARK" {
setMapKey("GpuBenchMarkJob", cloudTaskRecord.UserID, 1, resultItemMap)
} else {
setMapKey("GpuDebugJob", cloudTaskRecord.UserID, 1, resultItemMap)
}
}
}
indexTotal += PAGE_SIZE
if indexTotal >= count {
break
}
}

return resultMap, resultItemMap
}
func setMapKey(key string, userId int64, value int, resultItemMap map[string]int) {
newKey := fmt.Sprint(userId) + "_" + key
if _, ok := resultItemMap[newKey]; !ok {
resultItemMap[newKey] = value
} else {
resultItemMap[newKey] += value
}
}

func subMonth(t1, t2 time.Time) (month int) {
y1 := t1.Year()
y2 := t2.Year()


+ 72
- 0
models/user_business_struct.go View File

@@ -44,6 +44,18 @@ type UserBusinessAnalysisCurrentYear struct {
//user
Name string `xorm:"NOT NULL"`
DataDate string `xorm:"NULL"`

CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"`
GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"`
GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"`
CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"`
CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"`
UserIndex float64 `xorm:"NOT NULL DEFAULT 0"`
UserLocation string `xorm:"NULL"`
}

type UserBusinessAnalysisLast30Day struct {
@@ -88,6 +100,18 @@ type UserBusinessAnalysisLast30Day struct {
//user
Name string `xorm:"NOT NULL"`
DataDate string `xorm:"NULL"`

CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"`
GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"`
GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"`
CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"`
CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"`
UserIndex float64 `xorm:"NOT NULL DEFAULT 0"`
UserLocation string `xorm:"NULL"`
}

type UserBusinessAnalysisLastMonth struct {
@@ -132,6 +156,18 @@ type UserBusinessAnalysisLastMonth struct {
//user
Name string `xorm:"NOT NULL"`
DataDate string `xorm:"NULL"`

CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"`
GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"`
GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"`
CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"`
CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"`
UserIndex float64 `xorm:"NOT NULL DEFAULT 0"`
UserLocation string `xorm:"NULL"`
}

type UserBusinessAnalysisCurrentMonth struct {
@@ -176,6 +212,18 @@ type UserBusinessAnalysisCurrentMonth struct {
//user
Name string `xorm:"NOT NULL"`
DataDate string `xorm:"NULL"`

CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"`
GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"`
GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"`
CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"`
CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"`
UserIndex float64 `xorm:"NOT NULL DEFAULT 0"`
UserLocation string `xorm:"NULL"`
}

type UserBusinessAnalysisCurrentWeek struct {
@@ -220,6 +268,18 @@ type UserBusinessAnalysisCurrentWeek struct {
//user
Name string `xorm:"NOT NULL"`
DataDate string `xorm:"NULL"`

CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"`
GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"`
GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"`
CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"`
CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"`
UserIndex float64 `xorm:"NOT NULL DEFAULT 0"`
UserLocation string `xorm:"NULL"`
}

type UserBusinessAnalysisYesterday struct {
@@ -264,4 +324,16 @@ type UserBusinessAnalysisYesterday struct {
//user
Name string `xorm:"NOT NULL"`
DataDate string `xorm:"NULL"`

CloudBrainTaskNum int `xorm:"NOT NULL DEFAULT 0"`
GpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
NpuDebugJob int `xorm:"NOT NULL DEFAULT 0"`
GpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuTrainJob int `xorm:"NOT NULL DEFAULT 0"`
NpuInferenceJob int `xorm:"NOT NULL DEFAULT 0"`
GpuBenchMarkJob int `xorm:"NOT NULL DEFAULT 0"`
CloudBrainRunTime int `xorm:"NOT NULL DEFAULT 0"`
CommitDatasetNum int `xorm:"NOT NULL DEFAULT 0"`
UserIndex float64 `xorm:"NOT NULL DEFAULT 0"`
UserLocation string `xorm:"NULL"`
}

Loading…
Cancel
Save