diff --git a/models/models.go b/models/models.go index a72ebe5db..11f445830 100755 --- a/models/models.go +++ b/models/models.go @@ -143,6 +143,12 @@ func init() { new(SummaryStatistic), new(UserBusinessAnalysis), new(UserBusinessAnalysisAll), + new(UserBusinessAnalysisCurrentYear), + new(UserBusinessAnalysisLast30Day), + new(UserBusinessAnalysisLastMonth), + new(UserBusinessAnalysisCurrentMonth), + new(UserBusinessAnalysisCurrentWeek), + new(UserBusinessAnalysisYesterday), new(UserLoginLog), ) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index a15b9db5f..d04e350c2 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -10,10 +10,12 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" + "xorm.io/xorm" ) const ( - Page_SIZE = 2000 + PAGE_SIZE = 2000 + BATCH_INSERT_SIZE = 50 ) type UserBusinessAnalysisAll struct { @@ -163,14 +165,6 @@ func (ulist UserBusinessAnalysisList) Less(i, j int) bool { return ulist[i].ID > ulist[j].ID } -type UserBusinessAnalysisAllList []*UserBusinessAnalysisAll - -func (ulist UserBusinessAnalysisAllList) Swap(i, j int) { ulist[i], ulist[j] = ulist[j], ulist[i] } -func (ulist UserBusinessAnalysisAllList) Len() int { return len(ulist) } -func (ulist UserBusinessAnalysisAllList) Less(i, j int) bool { - return ulist[i].ID > ulist[j].ID -} - func getLastCountDate() int64 { statictisSess := xStatistic.NewSession() defer statictisSess.Close() @@ -189,6 +183,29 @@ func getLastCountDate() int64 { return pageStartTime.Unix() } +func QueryUserStaticDataByTableName(start int, pageSize int, tableName string, queryObj interface{}, userName string) ([]*UserBusinessAnalysisAll, int64) { + statictisSess := xStatistic.NewSession() + defer statictisSess.Close() + var cond = builder.NewCond() + if len(userName) > 0 { + cond = cond.And( + builder.Like{"name", userName}, + ) + } + allCount, err := statictisSess.Where(cond).Count(queryObj) + if err != nil { + log.Info("query error." + err.Error()) + return nil, 0 + } + log.Info("query return total:" + fmt.Sprint(allCount)) + userBusinessAnalysisAllList := make([]*UserBusinessAnalysisAll, 0) + if err := statictisSess.Table(tableName).Where(cond).OrderBy("commit_count desc,id desc").Limit(pageSize, start). + Find(&userBusinessAnalysisAllList); err != nil { + return nil, 0 + } + return userBusinessAnalysisAllList, allCount +} + func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysisAll, int64) { log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll)) @@ -202,9 +219,9 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi } log.Info("query return total:" + fmt.Sprint(allCount)) - pageSize := 1000 + pageSize := PAGE_SIZE totalPage := int(allCount) / pageSize - userBusinessAnalysisReturnList := UserBusinessAnalysisAllList{} + userBusinessAnalysisReturnList := make([]*UserBusinessAnalysisAll, 0) for i := 0; i <= int(totalPage); i++ { userBusinessAnalysisAllList := make([]*UserBusinessAnalysisAll, 0) if err := statictisSess.Table("user_business_analysis_all").OrderBy("id desc").Limit(pageSize, i*pageSize). @@ -217,7 +234,6 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi } } - sort.Sort(userBusinessAnalysisReturnList) log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList))) return userBusinessAnalysisReturnList, allCount } @@ -337,28 +353,24 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus return userBusinessAnalysisReturnList, count } -func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap map[string]*git.UserKPIStats) { - +func refreshUserStaticTable(wikiCountMap map[string]int, CommitCodeSizeMap map[string]*git.UserKPIStats, tableName string, pageStartTime time.Time, pageEndTime time.Time) { sess := x.NewSession() defer sess.Close() statictisSess := xStatistic.NewSession() defer statictisSess.Close() - log.Info("truncate all data from table: user_business_analysis_all") - statictisSess.Exec("TRUNCATE TABLE user_business_analysis_all") + log.Info("truncate all data from table: " + tableName) + statictisSess.Exec("TRUNCATE TABLE " + tableName) - currentTimeNow := time.Now() - - startTime := currentTimeNow.AddDate(0, 0, -1) - - pageStartTime := time.Date(2021, 11, 5, 0, 0, 0, 0, currentTimeNow.Location()) log.Info("pageStartTime:" + pageStartTime.Format("2006-01-02 15:04:05")) - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) log.Info("pageEndTime time:" + pageEndTime.Format("2006-01-02 15:04:05")) start_unix := pageStartTime.Unix() end_unix := pageEndTime.Unix() + currentTimeNow := time.Now() + startTime := currentTimeNow.AddDate(0, 0, -1) + CodeMergeCountMap := queryPullRequest(start_unix, end_unix) CommitCountMap := queryCommitAction(start_unix, end_unix, 5) IssueCountMap := queryCreateIssue(start_unix, end_unix) @@ -385,12 +397,14 @@ func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap ma } var indexTotal int64 indexTotal = 0 + insertCount := 0 + dateRecordBatch := make([]UserBusinessAnalysisAll, 0) for { - sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) userList := make([]*User, 0) sess.Find(&userList) - for i, userRecord := range userList { - log.Info("insert all static, i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) + + for _, userRecord := range userList { var dateRecordAll UserBusinessAnalysisAll dateRecordAll.ID = userRecord.ID dateRecordAll.Email = userRecord.Email @@ -484,18 +498,85 @@ func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap ma } dateRecordAll.CommitModelCount = 0 - _, err = statictisSess.Insert(&dateRecordAll) - if err != nil { - log.Info("insert all data failed." + err.Error()) + + dateRecordBatch = append(dateRecordBatch, dateRecordAll) + if len(dateRecordBatch) >= BATCH_INSERT_SIZE { + insertTable(dateRecordBatch, tableName, statictisSess) + insertCount += BATCH_INSERT_SIZE + if err != nil { + log.Info("insert all data failed." + err.Error()) + } + dateRecordBatch = make([]UserBusinessAnalysisAll, 0) } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } } + if len(dateRecordBatch) > 0 { + insertTable(dateRecordBatch, tableName, statictisSess) + insertCount += len(dateRecordBatch) + if err != nil { + log.Info("insert all data failed." + err.Error()) + } + } + + log.Info("refresh data finished.tableName=" + tableName + " total record:" + fmt.Sprint(insertCount)) +} + +func insertTable(dateRecords []UserBusinessAnalysisAll, tableName string, statictisSess *xorm.Session) { + + insertBatchSql := "INSERT INTO public." + tableName + + "(id, count_date, code_merge_count, commit_count, issue_count, comment_count, focus_repo_count, star_repo_count, watched_count, gitea_age_month, commit_code_size, commit_dataset_size, " + + "commit_model_count, solve_issue_count, encyclopedias_count, regist_date, create_repo_count, login_count, open_i_index, email, name, data_date) " + + "VALUES" + + for i, record := range dateRecords { + insertBatchSql += "(" + fmt.Sprint(record.ID) + ", " + fmt.Sprint(record.CountDate) + ", " + fmt.Sprint(record.CodeMergeCount) + ", " + fmt.Sprint(record.CommitCount) + + ", " + fmt.Sprint(record.IssueCount) + ", " + fmt.Sprint(record.CommentCount) + ", " + fmt.Sprint(record.FocusRepoCount) + ", " + fmt.Sprint(record.StarRepoCount) + + ", " + fmt.Sprint(record.WatchedCount) + ", " + fmt.Sprint(record.GiteaAgeMonth) + ", " + fmt.Sprint(record.CommitCodeSize) + ", " + fmt.Sprint(record.CommitDatasetSize) + + ", " + fmt.Sprint(record.CommitModelCount) + ", " + fmt.Sprint(record.SolveIssueCount) + ", " + fmt.Sprint(record.EncyclopediasCount) + ", " + fmt.Sprint(record.RegistDate) + + ", " + fmt.Sprint(record.CreateRepoCount) + ", " + fmt.Sprint(record.LoginCount) + ", " + fmt.Sprint(record.OpenIIndex) + ", '" + record.Email + "', '" + record.Name + "', '" + record.DataDate + "')" + if i < (len(dateRecords) - 1) { + insertBatchSql += "," + } + } + statictisSess.Exec(insertBatchSql) +} + +func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap map[string]*git.UserKPIStats) { + currentTimeNow := time.Now() + pageStartTime := time.Date(2021, 11, 5, 0, 0, 0, 0, currentTimeNow.Location()) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_all", pageStartTime, pageEndTime) log.Info("refresh all data finished.") + + pageStartTime = time.Date(currentTimeNow.Year(), 1, 1, 0, 0, 0, 0, currentTimeNow.Location()) + refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_current_year", pageStartTime, pageEndTime) + + thisMonth := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), 1, 0, 0, 0, 0, currentTimeNow.Location()) + refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_current_month", thisMonth, pageEndTime) + + offset := int(time.Monday - currentTimeNow.Weekday()) + if offset > 0 { + offset = -6 + } + pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, offset) + refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_current_week", pageStartTime, pageEndTime) + + pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -30) + refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_last30_day", pageStartTime, pageEndTime) + + pageStartTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -1) + pageEndTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).AddDate(0, 0, -1) + refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_yesterday", pageStartTime, pageEndTime) + + pageStartTime = thisMonth.AddDate(0, -1, 0) + pageEndTime = time.Date(currentTimeNow.Year(), currentTimeNow.Month(), 1, 23, 59, 59, 0, currentTimeNow.Location()).AddDate(0, 0, -1) + refreshUserStaticTable(wikiCountMap, CommitCodeSizeMap, "user_business_analysis_last_month", pageStartTime, pageEndTime) + } func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, endTime time.Time, isReCount bool) error { @@ -550,7 +631,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, var indexTotal int64 indexTotal = 0 for { - sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("`user`.*").Table("user").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) userList := make([]*User, 0) sess.Find(&userList) @@ -660,7 +741,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -700,7 +781,7 @@ func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { issueAssigneesList := make([]*IssueAssignees, 0) sess.Select("issue_assignees.*").Table("issue_assignees"). Join("inner", "issue", "issue.id=issue_assignees.issue_id"). - Where(cond).OrderBy("issue_assignees.id asc").Limit(Page_SIZE, int(indexTotal)) + Where(cond).OrderBy("issue_assignees.id asc").Limit(PAGE_SIZE, int(indexTotal)) sess.Find(&issueAssigneesList) @@ -712,7 +793,7 @@ func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { resultMap[issueAssigneesRecord.AssigneeID] += 1 } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -735,7 +816,7 @@ func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { indexTotal = 0 for { issueList := make([]*Issue, 0) - sess.Select("issue.*").Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).OrderBy("issue.id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("issue.*").Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).OrderBy("issue.id asc").Limit(PAGE_SIZE, int(indexTotal)) sess.Find(&issueList) log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) for _, issueRecord := range issueList { @@ -745,7 +826,7 @@ func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { resultMap[issueRecord.PosterID] += 1 } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -768,7 +849,7 @@ func queryCommitAction(start_unix int64, end_unix int64, actionType int64) map[i var indexTotal int64 indexTotal = 0 for { - sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) actionList := make([]*Action, 0) sess.Find(&actionList) @@ -781,7 +862,7 @@ func queryCommitAction(start_unix int64, end_unix int64, actionType int64) map[i } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -805,7 +886,7 @@ func queryCreateIssue(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,poster_id").Table("issue").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,poster_id").Table("issue").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) issueList := make([]*Issue, 0) sess.Find(&issueList) log.Info("query issue size=" + fmt.Sprint(len(issueList))) @@ -816,7 +897,7 @@ func queryCreateIssue(start_unix int64, end_unix int64) map[int64]int { resultMap[issueRecord.PosterID] += 1 } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -839,7 +920,7 @@ func queryComment(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,type,poster_id").Table("comment").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,type,poster_id").Table("comment").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) commentList := make([]*Comment, 0) sess.Find(&commentList) log.Info("query Comment size=" + fmt.Sprint(len(commentList))) @@ -850,7 +931,7 @@ func queryComment(start_unix int64, end_unix int64) map[int64]int { resultMap[commentRecord.PosterID] += 1 } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -875,7 +956,7 @@ func queryWatch(start_unix int64, end_unix int64) map[int64]int { indexTotal = 0 for { watchList := make([]*Watch, 0) - sess.Select("id,user_id,repo_id").Table("watch").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,user_id,repo_id").Table("watch").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) sess.Find(&watchList) log.Info("query Watch size=" + fmt.Sprint(len(watchList))) @@ -887,7 +968,7 @@ func queryWatch(start_unix int64, end_unix int64) map[int64]int { } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -913,7 +994,7 @@ func queryStar(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,uid,repo_id").Table("star").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,uid,repo_id").Table("star").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) starList := make([]*Star, 0) sess.Find(&starList) @@ -926,7 +1007,7 @@ func queryStar(start_unix int64, end_unix int64) map[int64]int { } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -949,7 +1030,7 @@ func queryFollow(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,user_id,follow_id").Table("follow").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,user_id,follow_id").Table("follow").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) followList := make([]*Follow, 0) sess.Find(&followList) @@ -962,7 +1043,7 @@ func queryFollow(start_unix int64, end_unix int64) map[int64]int { } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -985,7 +1066,7 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,uploader_id,size").Table("attachment").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,uploader_id,size").Table("attachment").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) attachmentList := make([]*Attachment, 0) sess.Find(&attachmentList) @@ -998,7 +1079,7 @@ func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -1021,7 +1102,7 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - sess.Select("id,owner_id,name").Table("repository").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + sess.Select("id,owner_id,name").Table("repository").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) repoList := make([]*Repository, 0) sess.Find(&repoList) log.Info("query Repository size=" + fmt.Sprint(len(repoList))) @@ -1032,7 +1113,7 @@ func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { resultMap[repoRecord.OwnerID] += 1 } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } @@ -1111,7 +1192,7 @@ func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { var indexTotal int64 indexTotal = 0 for { - statictisSess.Select("id,u_id").Table("user_login_log").Where(cond).OrderBy("id asc").Limit(Page_SIZE, int(indexTotal)) + statictisSess.Select("id,u_id").Table("user_login_log").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) userLoginLogList := make([]*UserLoginLog, 0) statictisSess.Find(&userLoginLogList) log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) @@ -1122,7 +1203,7 @@ func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { resultMap[loginRecord.UId] += 1 } } - indexTotal += Page_SIZE + indexTotal += PAGE_SIZE if indexTotal >= count { break } diff --git a/models/user_business_struct.go b/models/user_business_struct.go new file mode 100644 index 000000000..c435c0b07 --- /dev/null +++ b/models/user_business_struct.go @@ -0,0 +1,267 @@ +package models + +import "code.gitea.io/gitea/modules/timeutil" + +type UserBusinessAnalysisCurrentYear struct { + ID int64 `xorm:"pk"` + CountDate int64 `xorm:"pk"` + //action :ActionMergePullRequest // 11 + CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"` + //action :ActionCommitRepo + CommitCount int `xorm:"NOT NULL DEFAULT 0"` + //issue // 10 + IssueCount int `xorm:"NOT NULL DEFAULT 0"` + //comment table current date + CommentCount int `xorm:"NOT NULL DEFAULT 0"` + //watch table current date + FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //star table current date + StarRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //follow table + WatchedCount int `xorm:"NOT NULL DEFAULT 0"` + // user table + GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"` + // + CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"` + //attachement table + CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"` + //0 + CommitModelCount int `xorm:"NOT NULL DEFAULT 0"` + //issue, issueassignees + SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"` + //baike + EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"` + //user + RegistDate timeutil.TimeStamp `xorm:"NOT NULL"` + //repo + CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //login count, from elk + LoginCount int `xorm:"NOT NULL DEFAULT 0"` + //openi index + OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` + //user + Email string `xorm:"NOT NULL"` + //user + Name string `xorm:"NOT NULL"` + DataDate string `xorm:"NULL"` +} + +type UserBusinessAnalysisLast30Day struct { + ID int64 `xorm:"pk"` + CountDate int64 `xorm:"pk"` + //action :ActionMergePullRequest // 11 + CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"` + //action :ActionCommitRepo + CommitCount int `xorm:"NOT NULL DEFAULT 0"` + //issue // 10 + IssueCount int `xorm:"NOT NULL DEFAULT 0"` + //comment table current date + CommentCount int `xorm:"NOT NULL DEFAULT 0"` + //watch table current date + FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //star table current date + StarRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //follow table + WatchedCount int `xorm:"NOT NULL DEFAULT 0"` + // user table + GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"` + // + CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"` + //attachement table + CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"` + //0 + CommitModelCount int `xorm:"NOT NULL DEFAULT 0"` + //issue, issueassignees + SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"` + //baike + EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"` + //user + RegistDate timeutil.TimeStamp `xorm:"NOT NULL"` + //repo + CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //login count, from elk + LoginCount int `xorm:"NOT NULL DEFAULT 0"` + //openi index + OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` + //user + Email string `xorm:"NOT NULL"` + //user + Name string `xorm:"NOT NULL"` + DataDate string `xorm:"NULL"` +} + +type UserBusinessAnalysisLastMonth struct { + ID int64 `xorm:"pk"` + CountDate int64 `xorm:"pk"` + //action :ActionMergePullRequest // 11 + CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"` + //action :ActionCommitRepo + CommitCount int `xorm:"NOT NULL DEFAULT 0"` + //issue // 10 + IssueCount int `xorm:"NOT NULL DEFAULT 0"` + //comment table current date + CommentCount int `xorm:"NOT NULL DEFAULT 0"` + //watch table current date + FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //star table current date + StarRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //follow table + WatchedCount int `xorm:"NOT NULL DEFAULT 0"` + // user table + GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"` + // + CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"` + //attachement table + CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"` + //0 + CommitModelCount int `xorm:"NOT NULL DEFAULT 0"` + //issue, issueassignees + SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"` + //baike + EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"` + //user + RegistDate timeutil.TimeStamp `xorm:"NOT NULL"` + //repo + CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //login count, from elk + LoginCount int `xorm:"NOT NULL DEFAULT 0"` + //openi index + OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` + //user + Email string `xorm:"NOT NULL"` + //user + Name string `xorm:"NOT NULL"` + DataDate string `xorm:"NULL"` +} + +type UserBusinessAnalysisCurrentMonth struct { + ID int64 `xorm:"pk"` + CountDate int64 `xorm:"pk"` + //action :ActionMergePullRequest // 11 + CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"` + //action :ActionCommitRepo + CommitCount int `xorm:"NOT NULL DEFAULT 0"` + //issue // 10 + IssueCount int `xorm:"NOT NULL DEFAULT 0"` + //comment table current date + CommentCount int `xorm:"NOT NULL DEFAULT 0"` + //watch table current date + FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //star table current date + StarRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //follow table + WatchedCount int `xorm:"NOT NULL DEFAULT 0"` + // user table + GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"` + // + CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"` + //attachement table + CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"` + //0 + CommitModelCount int `xorm:"NOT NULL DEFAULT 0"` + //issue, issueassignees + SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"` + //baike + EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"` + //user + RegistDate timeutil.TimeStamp `xorm:"NOT NULL"` + //repo + CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //login count, from elk + LoginCount int `xorm:"NOT NULL DEFAULT 0"` + //openi index + OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` + //user + Email string `xorm:"NOT NULL"` + //user + Name string `xorm:"NOT NULL"` + DataDate string `xorm:"NULL"` +} + +type UserBusinessAnalysisCurrentWeek struct { + ID int64 `xorm:"pk"` + CountDate int64 `xorm:"pk"` + //action :ActionMergePullRequest // 11 + CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"` + //action :ActionCommitRepo + CommitCount int `xorm:"NOT NULL DEFAULT 0"` + //issue // 10 + IssueCount int `xorm:"NOT NULL DEFAULT 0"` + //comment table current date + CommentCount int `xorm:"NOT NULL DEFAULT 0"` + //watch table current date + FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //star table current date + StarRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //follow table + WatchedCount int `xorm:"NOT NULL DEFAULT 0"` + // user table + GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"` + // + CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"` + //attachement table + CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"` + //0 + CommitModelCount int `xorm:"NOT NULL DEFAULT 0"` + //issue, issueassignees + SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"` + //baike + EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"` + //user + RegistDate timeutil.TimeStamp `xorm:"NOT NULL"` + //repo + CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //login count, from elk + LoginCount int `xorm:"NOT NULL DEFAULT 0"` + //openi index + OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` + //user + Email string `xorm:"NOT NULL"` + //user + Name string `xorm:"NOT NULL"` + DataDate string `xorm:"NULL"` +} + +type UserBusinessAnalysisYesterday struct { + ID int64 `xorm:"pk"` + CountDate int64 `xorm:"pk"` + //action :ActionMergePullRequest // 11 + CodeMergeCount int `xorm:"NOT NULL DEFAULT 0"` + //action :ActionCommitRepo + CommitCount int `xorm:"NOT NULL DEFAULT 0"` + //issue // 10 + IssueCount int `xorm:"NOT NULL DEFAULT 0"` + //comment table current date + CommentCount int `xorm:"NOT NULL DEFAULT 0"` + //watch table current date + FocusRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //star table current date + StarRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //follow table + WatchedCount int `xorm:"NOT NULL DEFAULT 0"` + // user table + GiteaAgeMonth int `xorm:"NOT NULL DEFAULT 0"` + // + CommitCodeSize int `xorm:"NOT NULL DEFAULT 0"` + //attachement table + CommitDatasetSize int `xorm:"NOT NULL DEFAULT 0"` + //0 + CommitModelCount int `xorm:"NOT NULL DEFAULT 0"` + //issue, issueassignees + SolveIssueCount int `xorm:"NOT NULL DEFAULT 0"` + //baike + EncyclopediasCount int `xorm:"NOT NULL DEFAULT 0"` + //user + RegistDate timeutil.TimeStamp `xorm:"NOT NULL"` + //repo + CreateRepoCount int `xorm:"NOT NULL DEFAULT 0"` + //login count, from elk + LoginCount int `xorm:"NOT NULL DEFAULT 0"` + //openi index + OpenIIndex float64 `xorm:"NOT NULL DEFAULT 0"` + //user + Email string `xorm:"NOT NULL"` + //user + Name string `xorm:"NOT NULL"` + DataDate string `xorm:"NULL"` +} diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 1dceb5c4a..8704bc507 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -424,7 +424,13 @@ static.openiindex=OpenI Index static.registdate=Regist Date static.countdate=Count Date static.all=All - +static.public.user_business_analysis_current_month=Current_Month +static.public.user_business_analysis_current_week=Current_Week +static.public.user_business_analysis_current_year=Current_Year +static.public.user_business_analysis_last30_day=Last_30_day +static.public.user_business_analysis_last_month=Last_Month +static.public.user_business_analysis_yesterday=Yesterday +static.public.user_business_analysis_all=All [settings] profile = Profile account = Account diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 2e7f4d73c..b83099f78 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -428,6 +428,13 @@ static.openiindex=OpenI指数 static.registdate=用户注册时间 static.countdate=系统统计时间 static.all=所有 +static.public.user_business_analysis_current_month=本月 +static.public.user_business_analysis_current_week=本周 +static.public.user_business_analysis_current_year=今年 +static.public.user_business_analysis_last30_day=近30天 +static.public.user_business_analysis_last_month=上月 +static.public.user_business_analysis_yesterday=昨天 +static.public.user_business_analysis_all=所有 [settings] profile=个人信息 account=账号 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 518c63e4f..dcea46ed6 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -524,7 +524,7 @@ func RegisterRoutes(m *macaron.Macaron) { Get(notify.GetThread). Patch(notify.ReadThread) }, reqToken()) - + operationReq := context.Toggle(&context.ToggleOptions{SignInRequired: true, OperationRequired: true}) //Project board m.Group("/projectboard", func() { @@ -544,7 +544,13 @@ func RegisterRoutes(m *macaron.Macaron) { }, operationReq) m.Get("/query_user_static_page", operationReq, repo_ext.QueryUserStaticDataPage) - + m.Get("/query_user_current_month", operationReq, repo_ext.QueryUserStaticCurrentMonth) + m.Get("/query_user_current_week", operationReq, repo_ext.QueryUserStaticCurrentWeek) + m.Get("/query_user_current_year", operationReq, repo_ext.QueryUserStaticCurrentYear) + m.Get("/query_user_last30_day", operationReq, repo_ext.QueryUserStaticLast30Day) + m.Get("/query_user_last_month", operationReq, repo_ext.QueryUserStaticLastMonth) + m.Get("/query_user_yesterday", operationReq, repo_ext.QueryUserStaticYesterday) + m.Get("/query_user_all", operationReq, repo_ext.QueryUserStaticAll) // Users m.Group("/users", func() { m.Get("/search", user.Search) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 42189c57f..7df384cc4 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -15,6 +15,132 @@ import ( "github.com/360EntSecGroup-Skylar/excelize/v2" ) +const ( + PAGE_SIZE = 2000 +) + +func queryUserDataPage(ctx *context.Context, tableName string, queryObj interface{}) { + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := ctx.QueryInt("pageSize") + if pageSize <= 0 { + pageSize = setting.UI.IssuePagingNum + } + userName := ctx.Query("userName") + IsReturnFile := ctx.QueryBool("IsReturnFile") + + if IsReturnFile { + //writer exec file. + xlsx := excelize.NewFile() + sheetName := ctx.Tr("user.static.sheetname") + index := xlsx.NewSheet(sheetName) + xlsx.DeleteSheet("Sheet1") + dataHeader := map[string]string{ + "A1": ctx.Tr("user.static.id"), + "B1": ctx.Tr("user.static.name"), + "C1": ctx.Tr("user.static.codemergecount"), + "D1": ctx.Tr("user.static.commitcount"), + "E1": ctx.Tr("user.static.issuecount"), + "F1": ctx.Tr("user.static.commentcount"), + "G1": ctx.Tr("user.static.focusrepocount"), + "H1": ctx.Tr("user.static.starrepocount"), + "I1": ctx.Tr("user.static.logincount"), + "J1": ctx.Tr("user.static.watchedcount"), + "K1": ctx.Tr("user.static.commitcodesize"), + "L1": ctx.Tr("user.static.solveissuecount"), + "M1": ctx.Tr("user.static.encyclopediascount"), + "N1": ctx.Tr("user.static.createrepocount"), + "O1": ctx.Tr("user.static.openiindex"), + "P1": ctx.Tr("user.static.registdate"), + "Q1": ctx.Tr("user.static.countdate"), + } + for k, v := range dataHeader { + //设置单元格的值 + xlsx.SetCellValue(sheetName, k, v) + } + _, count := models.QueryUserStaticDataByTableName(1, 1, tableName, queryObj, userName) + var indexTotal int64 + indexTotal = 0 + for { + re, _ := models.QueryUserStaticDataByTableName(int(indexTotal), PAGE_SIZE, tableName, queryObj, "") + log.Info("return count=" + fmt.Sprint(count)) + for i, userRecord := range re { + rows := fmt.Sprint(i + 2) + xlsx.SetCellValue(sheetName, "A"+rows, userRecord.ID) + xlsx.SetCellValue(sheetName, "B"+rows, userRecord.Name) + xlsx.SetCellValue(sheetName, "C"+rows, userRecord.CodeMergeCount) + xlsx.SetCellValue(sheetName, "D"+rows, userRecord.CommitCount) + xlsx.SetCellValue(sheetName, "E"+rows, userRecord.IssueCount) + xlsx.SetCellValue(sheetName, "F"+rows, userRecord.CommentCount) + xlsx.SetCellValue(sheetName, "G"+rows, userRecord.FocusRepoCount) + xlsx.SetCellValue(sheetName, "H"+rows, userRecord.StarRepoCount) + xlsx.SetCellValue(sheetName, "I"+rows, userRecord.LoginCount) + xlsx.SetCellValue(sheetName, "J"+rows, userRecord.WatchedCount) + xlsx.SetCellValue(sheetName, "K"+rows, userRecord.CommitCodeSize) + xlsx.SetCellValue(sheetName, "L"+rows, userRecord.SolveIssueCount) + xlsx.SetCellValue(sheetName, "M"+rows, userRecord.EncyclopediasCount) + xlsx.SetCellValue(sheetName, "N"+rows, userRecord.CreateRepoCount) + xlsx.SetCellValue(sheetName, "O"+rows, fmt.Sprintf("%.2f", userRecord.OpenIIndex)) + + formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05") + xlsx.SetCellValue(sheetName, "P"+rows, formatTime[0:len(formatTime)-3]) + + formatTime = userRecord.DataDate + xlsx.SetCellValue(sheetName, "Q"+rows, formatTime+" 00:01") + } + + //设置默认打开的表单 + xlsx.SetActiveSheet(index) + filename := sheetName + "_" + ctx.Tr("user.static."+tableName) + ".xlsx" + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + if _, err := xlsx.WriteTo(ctx.Resp); err != nil { + log.Info("writer exel error." + err.Error()) + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + } else { + re, count := models.QueryUserStaticDataByTableName((page-1)*pageSize, pageSize, tableName, queryObj, userName) + mapInterface := make(map[string]interface{}) + mapInterface["data"] = re + mapInterface["count"] = count + ctx.JSON(http.StatusOK, mapInterface) + } +} + +func QueryUserStaticCurrentMonth(ctx *context.Context) { + queryUserDataPage(ctx, "public.user_business_analysis_current_month", new(models.UserBusinessAnalysisCurrentMonth)) +} + +func QueryUserStaticCurrentWeek(ctx *context.Context) { + queryUserDataPage(ctx, "public.user_business_analysis_current_week", new(models.UserBusinessAnalysisCurrentWeek)) +} + +func QueryUserStaticCurrentYear(ctx *context.Context) { + queryUserDataPage(ctx, "public.user_business_analysis_current_year", new(models.UserBusinessAnalysisCurrentYear)) +} + +func QueryUserStaticLast30Day(ctx *context.Context) { + queryUserDataPage(ctx, "public.user_business_analysis_last30_day", new(models.UserBusinessAnalysisLast30Day)) +} + +func QueryUserStaticLastMonth(ctx *context.Context) { + queryUserDataPage(ctx, "public.user_business_analysis_last_month", new(models.UserBusinessAnalysisLastMonth)) +} + +func QueryUserStaticYesterday(ctx *context.Context) { + queryUserDataPage(ctx, "public.user_business_analysis_yesterday", new(models.UserBusinessAnalysisYesterday)) +} + +func QueryUserStaticAll(ctx *context.Context) { + queryUserDataPage(ctx, "public.user_business_analysis_all", new(models.UserBusinessAnalysisAll)) +} + func QueryUserStaticDataPage(ctx *context.Context) { startDate := ctx.Query("startDate") endDate := ctx.Query("endDate") diff --git a/web_src/js/components/UserAnalysis.vue b/web_src/js/components/UserAnalysis.vue index beb4de8e9..c4c5608f1 100755 --- a/web_src/js/components/UserAnalysis.vue +++ b/web_src/js/components/UserAnalysis.vue @@ -27,10 +27,10 @@ - + - 下载报告 + 下载报告 下载报告 @@ -175,6 +175,7 @@ params:{startDate:'',endDate:'',page:1,pageSize:10,userName:''}, tableData: [], totalNum:0, + dataUrl:'../api/v1/query_user_static_page', pickerOptions: { }, value_time: '', @@ -274,68 +275,52 @@ let lastYear = lastMonthDate.getYear(); let lastMonth = lastMonthDate.getMonth(); + this.dataUrl = '../api/v1/query_user_static_page'; + if (typeof type_val=="undefined" || type_val=="null" || type_val==""){ this.params.startDate= this.formatDate(this.value_time[0].getFullYear(),this.value_time[0].getMonth() + 1,this.value_time[0].getDate()); this.params.endDate = this.formatDate(this.value_time[1].getFullYear(),this.value_time[1].getMonth() + 1,this.value_time[1].getDate()); }else{ switch(type_val){ case "yesterday_usr":{ - var now = new Date(); - var tmp = new Date(now.setTime(now.getTime()-24*60*60*1000)); - var yesterday = this.formatDate(tmp.getFullYear(),tmp.getMonth()+1,tmp.getDate()); - this.params.startDate = yesterday - this.params.endDate = yesterday this.value_time=[] - // document.getElementById("yesterday_usr").style.backgroundColor="409effd6" - // document.getElementById("current_week_usr") + this.dataUrl = '../api/v1/query_user_yesterday'; break } case "current_week_usr":{ - var now = new Date(); // 当前日期 - var nowDayOfWeek = now.getDay(); // 今天本周的第几天 - var day = nowDayOfWeek || 7; - this.params.startDate = this.formatDate(now.getFullYear(), nowMonth+1, nowDay + 1 - day); - this.params.endDate = today this.value_time=[] + this.dataUrl = '../api/v1/query_user_current_week'; break } case "current_month_usr":{ - this.params.startDate = this.formatDate(nowYear,nowMonth+1,1); - this.params.endDate = today this.value_time=[] + this.dataUrl = '../api/v1/query_user_current_month'; break } case "last_month_usr":{ - this.params.startDate=this.formatDate(nowYear, lastMonth+1, 1); - this.params.endDate=this.formatDate(nowYear, lastMonth+1, this.getMonthDays(nowYear,lastMonth)); this.value_time=[] + this.dataUrl = '../api/v1/query_user_last_month'; break - } case "monthly_usr":{ - var temp=new Date(now - 1000 * 60 * 60 * 24 * 30) - this.params.startDate = this.formatDate(temp.getFullYear(),temp.getMonth()+1,temp.getDate()); - this.params.endDate = today this.value_time=[] + this.dataUrl = '../api/v1/query_user_last30_day'; break } case "current_year_usr":{ - this.params.startDate = this.formatDate(now.getFullYear(), 1, 1); - this.params.endDate = today this.value_time=[] + this.dataUrl = '../api/v1/query_user_current_year'; break } case "all_usr":{ - console.log("e:"+today) - this.params.startDate = 'all'//this.formatDate(2000, 1, 1); //this.recordBeginTime// - this.params.endDate = today this.value_time=[] + this.dataUrl = '../api/v1/query_user_all'; break } } }; - this.$axios.get('../api/v1/query_user_static_page',{ + this.$axios.get(this.dataUrl,{ params:this.params }).then((res)=>{ this.tableData = res.data.data @@ -345,45 +330,17 @@ console.log("res.count:"+res.data.count) }) - // this.$axios.get('../tool/query_user_static',{ - // params:this.params - // }).then((res)=>{ - // this.currentPage = 1 - // this.tableData = res.data - // console.log(" this.tableData:", this.tableData.length) - // for(var i=0;i !search || data.Name.toLowerCase().includes(search.toLowerCase())) - + this.params.userName = this.search this.params.page = 1 this.page=1 this.getUserList(this.type_val, this.dynamic) }, - // goToDetailPage(pro_id,pro_name){ - // sessionStorage.setItem("pro_id",pro_id); - // sessionStorage.setItem("pro_name",pro_name); - // document.getElementById("pro_main").style.display="none"; - // document.getElementById("pro_detail").style.display="block"; - - // }, + tableHeaderStyle({row,column,rowIndex,columnIndex}){ if(rowIndex===0){ @@ -415,12 +372,6 @@ console.log('dateString', dateString); // > dateString 2021-07-06 14:23 return dateString; }, - - // transformTimestamp(timestamp){ - // var dateString= new Date(timestamp); - - // return dateString.toLocaleDateString().replace(/\//g, "-") + " " + dateString.toTimeString().substr(0, 8); - // }, }, mounted() { @@ -434,9 +385,6 @@ }, watch:{ search(val){ - // if(!val){ - // this.getUserList("all_usr",7) - // } if(!val){ this.params.userName = this.search this.params.page = 1