From 26dcb2a1f246744ce28a9afd753801efafa8ccfd Mon Sep 17 00:00:00 2001 From: zouap Date: Wed, 24 Nov 2021 15:25:51 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- models/models.go | 1 + models/user_business_analysis.go | 189 +++++++++++++++++-------------- 2 files changed, 103 insertions(+), 87 deletions(-) diff --git a/models/models.go b/models/models.go index 4858ec104..e8a71bbd8 100755 --- a/models/models.go +++ b/models/models.go @@ -140,6 +140,7 @@ func init() { new(RepoStatistic), new(SummaryStatistic), new(UserBusinessAnalysis), + new(UserBusinessAnalysisAll), new(UserLoginLog), ) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 08be81241..608d5fcaa 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -4,13 +4,19 @@ import ( "encoding/json" "fmt" "sort" + "strconv" "time" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" + "xorm.io/xorm" ) +type UserBusinessAnalysisAll struct { + UserBusinessAnalysis +} + type UserBusinessAnalysis struct { ID int64 `xorm:"pk"` @@ -93,46 +99,12 @@ func (ulist UserBusinessAnalysisList) Less(i, j int) bool { return ulist[i].ID > ulist[j].ID } -func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis { - log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime)) - statictisSess := xStatistic.NewSession() - defer statictisSess.Close() - - statictisSess.Select("*").Table("user_business_analysis").Where(" count_date>=" + fmt.Sprint(startTime) + " and count_date<=" + fmt.Sprint(endTime)).OrderBy("count_date desc") - - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - statictisSess.Find(&userBusinessAnalysisList) +type UserBusinessAnalysisAllList []*UserBusinessAnalysisAll - resultMap := make(map[int64]*UserBusinessAnalysis) - log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - if _, ok := resultMap[userRecord.ID]; !ok { - resultMap[userRecord.ID] = userRecord - } else { - resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount - resultMap[userRecord.ID].CommitCount += userRecord.CommitCount - resultMap[userRecord.ID].IssueCount += userRecord.IssueCount - resultMap[userRecord.ID].CommentCount += userRecord.CommentCount - resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount - resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount - resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount - resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize - resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize - resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount - resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount - resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount - resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount - resultMap[userRecord.ID].LoginCount += userRecord.LoginCount - } - } - - userBusinessAnalysisReturnList := UserBusinessAnalysisList{} - for _, v := range resultMap { - userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) - } - sort.Sort(userBusinessAnalysisReturnList) - log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) - return userBusinessAnalysisReturnList +func (ulist UserBusinessAnalysisAllList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] } +func (ulist UserBusinessAnalysisAllList) Len() int { return len(ulist) } +func (ulist UserBusinessAnalysisAllList) Less(i, j int) bool { + return ulist[i].ID > ulist[j].ID } func getLastCountDate() int64 { @@ -153,71 +125,33 @@ func getLastCountDate() int64 { return pageStartTime.Unix() } -func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) { +func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysisAll, int64) { log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) statictisSess := xStatistic.NewSession() defer statictisSess.Close() - resultMap := make(map[int64]*UserBusinessAnalysis) - - var newAndCond = builder.NewCond() - if len(opts.UserName) > 0 { - newAndCond = newAndCond.And( - builder.Like{"name", opts.UserName}, - ) - } - if !opts.IsAll { - newAndCond = newAndCond.And( - builder.Gte{"count_date": opts.StartTime}, - ) - newAndCond = newAndCond.And( - builder.Lte{"count_date": opts.EndTime}, - ) - } - - allCount, err := statictisSess.Where(newAndCond).Count(new(UserBusinessAnalysis)) + allCount, err := statictisSess.Count(new(UserBusinessAnalysisAll)) if err != nil { log.Info("query error." + err.Error()) return nil, 0 } log.Info("query return total:" + fmt.Sprint(allCount)) - pageSize := 200 + pageSize := 1000 totalPage := int(allCount) / pageSize - + userBusinessAnalysisReturnList := UserBusinessAnalysisAllList{} for i := 0; i <= int(totalPage); i++ { - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(newAndCond).OrderBy("count_date desc").Limit(pageSize, i*pageSize). - Find(&userBusinessAnalysisList); err != nil { + userBusinessAnalysisAllList := make([]*UserBusinessAnalysisAll, 0) + if err := statictisSess.Table("user_business_analysis_all").OrderBy("id desc").Limit(pageSize, i*pageSize). + Find(&userBusinessAnalysisAllList); err != nil { return nil, 0 } - log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - if _, ok := resultMap[userRecord.ID]; !ok { - resultMap[userRecord.ID] = userRecord - } else { - resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount - resultMap[userRecord.ID].CommitCount += userRecord.CommitCount - resultMap[userRecord.ID].IssueCount += userRecord.IssueCount - resultMap[userRecord.ID].CommentCount += userRecord.CommentCount - resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount - resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount - resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount - resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize - resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize - resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount - resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount - resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount - resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount - resultMap[userRecord.ID].LoginCount += userRecord.LoginCount - } + log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisAllList))) + for _, userRecord := range userBusinessAnalysisAllList { + userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, userRecord) } } - userBusinessAnalysisReturnList := UserBusinessAnalysisList{} - for _, v := range resultMap { - userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v) - } sort.Sort(userBusinessAnalysisReturnList) log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) return userBusinessAnalysisReturnList, allCount @@ -491,10 +425,91 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, log.Info("insert daterecord failed." + err.Error()) return err } + + if isExistUserInAllTable(dateRecord.ID, statictisSess) { + updateCurrentData(dateRecord.ID, statictisSess, dateRecord) + } else { + log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(dateRecord.ID)) + allData := getAllData(dateRecord.ID, statictisSess) + allData.ID = dateRecord.ID + allData.CountDate = 0 + allData.DataDate = dateRecord.DataDate + allData.Email = dateRecord.Email + allData.OpenIIndex = dateRecord.OpenIIndex + allData.GiteaAgeMonth = dateRecord.GiteaAgeMonth + allData.Name = dateRecord.Name + allData.RegistDate = dateRecord.RegistDate + + _, err = statictisSess.Insert(&allData) + if err != nil { + log.Info("insert all data failed." + err.Error()) + return err + } + } + } return nil } +func updateCurrentData(userId int64, statictisSess *xorm.Session, currentData UserBusinessAnalysis) { + + _, err := statictisSess.Update("update user_business_analysis_all set code_merge_count+=" + fmt.Sprint(currentData.CodeMergeCount) + + ",commit_count+=" + fmt.Sprint(currentData.CommitCount) + + ",issue_count+=" + fmt.Sprint(currentData.IssueCount) + + ",comment_count+=" + fmt.Sprint(currentData.CommentCount) + + ",focus_repo_count+=" + fmt.Sprint(currentData.FocusRepoCount) + + ",star_repo_count+=" + fmt.Sprint(currentData.StarRepoCount) + + ",watched_count+=" + fmt.Sprint(currentData.WatchedCount) + + ",commit_code_size+=" + fmt.Sprint(currentData.CommitCodeSize) + + ",commit_dataset_size+=" + fmt.Sprint(currentData.CommitDatasetSize) + + ",commit_model_count+=" + fmt.Sprint(currentData.CommitModelCount) + + ",solve_issue_count+=" + fmt.Sprint(currentData.SolveIssueCount) + + ",encyclopedias_count+=" + fmt.Sprint(currentData.EncyclopediasCount) + + ",create_repo_count+=" + fmt.Sprint(currentData.CreateRepoCount) + + ",login_count+=" + fmt.Sprint(currentData.LoginCount) + + " where id=" + fmt.Sprint(userId)) + + if err != nil { + log.Info("update table failed." + err.Error()) + } + +} + +func isExistUserInAllTable(userId int64, statictisSess *xorm.Session) bool { + + allCount, err := statictisSess.Where("id=" + fmt.Sprint(userId)).Count(new(UserBusinessAnalysisAll)) + if err != nil { + return false + } + return allCount > 0 +} + +func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisAll { + var dateRecord UserBusinessAnalysisAll + + rows, err := statictisSess.Query("select sum(code_merge_count) as code_merge_count,sum(commit_count) as commit_count,sum(issue_count) as issue_count,sum(issue_count) as issue_count,sum(comment_count) as comment_count,sum(focus_repo_count) as focus_repo_count,sum(star_repo_count) as star_repo_count,sum(watched_count) as watched_count,sum(commit_code_size) as commit_code_size,sum(commit_dataset_size) as commit_dataset_size, sum(commit_model_count) as commit_model_count,sum(solve_issue_count) as solve_issue_count,sum(encyclopedias_count) as encyclopedias_count, sum(create_repo_count) as create_repo_count,sum(login_count) as login_count from public.user_business_analysis where id=" + fmt.Sprint(userId) + " group by id") + if err != nil { + for i, row := range rows { + log.Info("query user info, i=" + fmt.Sprint(i) + " code_merge_count=" + string(row["code_merge_count"])) + dateRecord.CodeMergeCount = strconv.FormatInt(string(row["code_merge_count"]), 10) + dateRecord.CommitCount = strconv.FormatInt(string(row["commit_count"]), 10) + dateRecord.IssueCount = strconv.FormatInt(string(row["issue_count"]), 10) + dateRecord.CommentCount = strconv.FormatInt(string(row["comment_count"]), 10) + dateRecord.FocusRepoCount = strconv.FormatInt(string(row["focus_repo_count"]), 10) + dateRecord.StarRepoCount = strconv.FormatInt(string(row["star_repo_count"]), 10) + dateRecord.WatchedCount = strconv.FormatInt(string(row["watched_count"]), 10) + dateRecord.CommitCodeSize = strconv.FormatInt(string(row["commit_code_size"]), 10) + dateRecord.CommitDatasetSize = strconv.FormatInt(string(row["commit_dataset_size"]), 10) + dateRecord.CommitModelCount = strconv.FormatInt(string(row["commit_model_count"]), 10) + dateRecord.SolveIssueCount = strconv.FormatInt(string(row["solve_issue_count"]), 10) + dateRecord.EncyclopediasCount = strconv.FormatInt(string(row["encyclopedias_count"]), 10) + dateRecord.CreateRepoCount = strconv.FormatInt(string(row["create_repo_count"]), 10) + dateRecord.LoginCount = strconv.FormatInt(string(row["login_count"]), 10) + } + } + return dateRecord +} + func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) { CounDataByDateAndReCount(wikiCountMap, startTime, endTime, false) }