diff --git a/models/models.go b/models/models.go index 2086fb7a4..11f445830 100755 --- a/models/models.go +++ b/models/models.go @@ -143,7 +143,7 @@ func init() { new(SummaryStatistic), new(UserBusinessAnalysis), new(UserBusinessAnalysisAll), - new(UserBusinessAnalysisYear), + new(UserBusinessAnalysisCurrentYear), new(UserBusinessAnalysisLast30Day), new(UserBusinessAnalysisLastMonth), new(UserBusinessAnalysisCurrentMonth), diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index a15b9db5f..5068961f7 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -10,10 +10,12 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" + "xorm.io/xorm" ) const ( - Page_SIZE = 2000 + PAGE_SIZE = 2000 + BATCH_INSERT_SIZE = 50 ) type UserBusinessAnalysisAll struct { @@ -163,14 +165,6 @@ func (ulist UserBusinessAnalysisList) Less(i, j int) bool { return ulist[i].ID > ulist[j].ID } -type UserBusinessAnalysisAllList []*UserBusinessAnalysisAll - -func (ulist UserBusinessAnalysisAllList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] } -func (ulist UserBusinessAnalysisAllList) Len() int { return len(ulist) } -func (ulist UserBusinessAnalysisAllList) Less(i, j int) bool { - return ulist[i].ID > ulist[j].ID -} - func getLastCountDate() int64 { statictisSess := xStatistic.NewSession() defer statictisSess.Close() @@ -202,9 +196,9 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi } log.Info("query return total:" + fmt.Sprint(allCount)) - pageSize := 1000 + pageSize := PAGE_SIZE totalPage := int(allCount) / pageSize - userBusinessAnalysisReturnList := UserBusinessAnalysisAllList{} + userBusinessAnalysisReturnList := make([]*UserBusinessAnalysisAll, 0) for i := 0; i <= int(totalPage); i++ { userBusinessAnalysisAllList := make([]*UserBusinessAnalysisAll, 0) if err := statictisSess.Table("user_business_analysis_all").OrderBy("id desc").Limit(pageSize, i*pageSize). @@ -217,7 +211,6 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi } } - sort.Sort(userBusinessAnalysisReturnList) log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) return userBusinessAnalysisReturnList, allCount } @@ -337,28 +330,24 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus return userBusinessAnalysisReturnList, count } -func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap map[string]*git.UserKPIStats) { - +func refreshUserStaticTable(wikiCountMap map[string]int, CommitCodeSizeMap map[string]*git.UserKPIStats, tableName string, pageStartTime time.Time, pageEndTime time.Time) { sess := x.NewSession() defer sess.Close() statictisSess := xStatistic.NewSession() defer statictisSess.Close() - log.Info("truncate all data from table: user_business_analysis_all") - statictisSess.Exec("TRUNCATE TABLE user_business_analysis_all") - - currentTimeNow := time.Now() - - startTime := currentTimeNow.AddDate(0, 0, -1) + log.Info("truncate all data from table: " + tableName) + statictisSess.Exec("TRUNCATE TABLE " + tableName) - pageStartTime := time.Date(2021, 11, 5, 0, 0, 0, 0, currentTimeNow.Location()) log.Info("pageStartTime:" + pageStartTime.Format("2006-01-02 15:04:05")) - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) log.Info("pageEndTime time:" + pageEndTime.Format("2006-01-02 15:04:05")) start_unix := pageStartTime.Unix() end_unix := pageEndTime.Unix() + currentTimeNow := time.Now() + startTime := currentTimeNow.AddDate(0, 0, -1) + CodeMergeCountMap := queryPullRequest(start_unix, end_unix) CommitCountMap := queryCommitAction(start_unix, end_unix, 5) IssueCountMap := queryCreateIssue(start_unix, end_unix) @@ -385,12 +374,14 @@ func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap ma } var indexTotal int64 indexTotal = 0 + insertCount := 0 + dateRecordBatch := make([]UserBusinessAnalysisAll, 0) for { - sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) userList := make([]*User, 0) sess.Find(&userList) - for i, userRecord := range userList { - log.Info("insert all static, i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) + + for _, userRecord := range userList { var dateRecordAll UserBusinessAnalysisAll dateRecordAll.ID = userRecord.ID dateRecordAll.Email = userRecord.Email @@ -484,18 +475,81 @@ func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap ma } dateRecordAll.CommitModelCount = 0 - _, err = statictisSess.Insert(&dateRecordAll) - if err != nil { - log.Info("insert all data failed." + err.Error()) + + dateRecordBatch = append(dateRecordBatch, dateRecordAll) + if len(dateRecordBatch) >= BATCH_INSERT_SIZE { + insertTable(dateRecordBatch, tableName, statictisSess) + insertCount += BATCH_INSERT_SIZE + if err != nil { + log.Info("insert all data failed." + err.Error()) + } + dateRecordBatch = make([]UserBusinessAnalysisAll, 0) } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } } + if len(dateRecordBatch) > 0 { + insertTable(dateRecordBatch, tableName, statictisSess) + insertCount += len(dateRecordBatch) + if err != nil { + log.Info("insert all data failed." + err.Error()) + } + } + + log.Info("refresh data finished.tableName=" + tableName + " total record:" + fmt.Sprint(insertCount)) +} + +func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, statictisSess *xorm.Session) { + + insertBatchSql := "INSERT INTO public." + tableName + + "(id, count_date, code_merge_count, commit_count, issue_count, comment_count, focus_repo_count, star_repo_count, watched_count, gitea_age_month, commit_code_size, commit_dataset_size, " + + "commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date) " + + "VALUES" + for _, record := range dateRecords { + insertBatchSql += "(" + fmt.Sprint(record.ID) + ", " + fmt.Sprint(record.CountDate) + ", " + fmt.Sprint(record.CodeMergeCount) + ", " + fmt.Sprint(record.CommitCount) + + ", " + fmt.Sprint(record.IssueCount) + ", " + fmt.Sprint(record.CommentCount) + ", " + fmt.Sprint(record.FocusRepoCount) + ", " + fmt.Sprint(record.StarRepoCount) + + ", " + fmt.Sprint(record.WatchedCount) + ", " + fmt.Sprint(record.GiteaAgeMonth) + ", " + fmt.Sprint(record.CommitCodeSize) + ", " + fmt.Sprint(record.CommitDatasetSize) + + ", " + fmt.Sprint(record.CommitModelCount) + ", " + fmt.Sprint(record.SolveIssueCount) + ", " + fmt.Sprint(record.EncyclopediasCount) + ", " + fmt.Sprint(record.RegistDate) + + ", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "')" + } + + statictisSess.Insert(insertBatchSql) +} + +func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap map[string]*git.UserKPIStats) { + currentTimeNow := time.Now() + pageStartTime := time.Date(2021, 11, 5, 0, 0, 0, 0, currentTimeNow.Location()) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_all", pageStartTime, pageEndTime) log.Info("refresh all data finished.") + + pageStartTime = time.Date(currentTimeNow.Year(), 0, 1, 0, 0, 0, 0, currentTimeNow.Location()) + refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_current_year", pageStartTime, pageEndTime) + + thisMonth := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), 1, 0, 0, 0, 0, currentTimeNow.Location()) + refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_current_month", thisMonth, pageEndTime) + + offset := int(time.Monday - currentTimeNow.Weekday()) + if offset > 0 { + offset = -6 + } + pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, offset) + refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_current_week", pageStartTime, pageEndTime) + + pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -30) + refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_last30_day", pageStartTime, pageEndTime) + + pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -1) + refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_yesterday", pageStartTime, pageEndTime) + + pageStartTime = thisMonth.AddDate(0, -1, 0) + pageEndTime = thisMonth.AddDate(0, 0, -1) + refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_last_month", pageStartTime, pageEndTime) + } func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) error { @@ -550,7 +604,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, var indexTotal int64 indexTotal = 0 for { - sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) userList := make([]*User, 0) sess.Find(&userList) @@ -660,7 +714,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -700,7 +754,7 @@ func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { issueAssigneesList := make([]*IssueAssignees, 0) sess.Select("issue_assignees.*").Table("issue_assignees"). Join("inner", "issue", "issue.id=issue_assignees.issue_id"). - Where(cond).OrderBy("issue_assignees.id asc").Limit(Page_SIZE, int(indexTotal)) + Where(cond).OrderBy("issue_assignees.id asc").Limit(PAGE_SIZE, int(indexTotal)) sess.Find(&issueAssigneesList) @@ -712,7 +766,7 @@ func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { resultMap[issueAssigneesRecord.AssigneeID] += 1 } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -735,7 +789,7 @@ func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { indexTotal = 0 for { issueList := make([]*Issue, 0) - sess.Select("issue.*").Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).OrderBy("issue.id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("issue.*").Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).OrderBy("issue.id asc").Limit(PAGE_SIZE, int(indexTotal)) sess.Find(&issueList) log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) for _, issueRecord := range issueList { @@ -745,7 +799,7 @@ func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { resultMap[issueRecord.PosterID] += 1 } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -768,7 +822,7 @@ func queryCommitAction(start_unix int64, end_unix int64, actionType int64) map[i var indexTotal int64 indexTotal = 0 for { - sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) actionList := make([]*Action, 0) sess.Find(&actionList) @@ -781,7 +835,7 @@ func queryCommitAction(start_unix int64, end_unix int64, actionType int64) map[i } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -805,7 +859,7 @@ func queryCreateIssue(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,poster_id").Table("issue").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,poster_id").Table("issue").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) issueList := make([]*Issue, 0) sess.Find(&issueList) log.Info("query issue size=" + fmt.Sprint(len(issueList))) @@ -816,7 +870,7 @@ func queryCreateIssue(start_unix int64, end_unix int64) map[int64]int { resultMap[issueRecord.PosterID] += 1 } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -839,7 +893,7 @@ func queryComment(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,type,poster_id").Table("comment").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,type,poster_id").Table("comment").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) commentList := make([]*Comment, 0) sess.Find(&commentList) log.Info("query Comment size=" + fmt.Sprint(len(commentList))) @@ -850,7 +904,7 @@ func queryComment(start_unix int64, end_unix int64) map[int64]int { resultMap[commentRecord.PosterID] += 1 } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -875,7 +929,7 @@ func queryWatch(start_unix int64, end_unix int64) map[int64]int { indexTotal = 0 for { watchList := make([]*Watch, 0) - sess.Select("id,user_id,repo_id").Table("watch").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,user_id,repo_id").Table("watch").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) sess.Find(&watchList) log.Info("query Watch size=" + fmt.Sprint(len(watchList))) @@ -887,7 +941,7 @@ func queryWatch(start_unix int64, end_unix int64) map[int64]int { } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -913,7 +967,7 @@ func queryStar(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,uid,repo_id").Table("star").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,uid,repo_id").Table("star").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) starList := make([]*Star, 0) sess.Find(&starList) @@ -926,7 +980,7 @@ func queryStar(start_unix int64, end_unix int64) map[int64]int { } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -949,7 +1003,7 @@ func queryFollow(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,user_id,follow_id").Table("follow").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,user_id,follow_id").Table("follow").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) followList := make([]*Follow, 0) sess.Find(&followList) @@ -962,7 +1016,7 @@ func queryFollow(start_unix int64, end_unix int64) map[int64]int { } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -985,7 +1039,7 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,uploader_id,size").Table("attachment").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,uploader_id,size").Table("attachment").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) attachmentList := make([]*Attachment, 0) sess.Find(&attachmentList) @@ -998,7 +1052,7 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -1021,7 +1075,7 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,owner_id,name").Table("repository").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,owner_id,name").Table("repository").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) repoList := make([]*Repository, 0) sess.Find(&repoList) log.Info("query Repository size=" + fmt.Sprint(len(repoList))) @@ -1032,7 +1086,7 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { resultMap[repoRecord.OwnerID] += 1 } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -1111,7 +1165,7 @@ func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - statictisSess.Select("id,u_id").Table("user_login_log").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + statictisSess.Select("id,u_id").Table("user_login_log").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) userLoginLogList := make([]*UserLoginLog, 0) statictisSess.Find(&userLoginLogList) log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) @@ -1122,7 +1176,7 @@ func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { resultMap[loginRecord.UId] += 1 } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } diff --git a/models/user_business_struct.go b/models/user_business_struct.go index b23bdfdca..c435c0b07 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -2,7 +2,7 @@ package models import "code.gitea.io/gitea/modules/timeutil" -type UserBusinessAnalysisYear struct { +type UserBusinessAnalysisCurrentYear struct { ID int64 `xorm:"pk"` CountDate int64 `xorm:"pk"` //action :ActionMergePullRequest // 11