Merge branch 'V20211115' into wangjr_pro

go.mod

@@ -17,6 +17,7 @@ require (
 	gitea.com/macaron/macaron v1.4.0
 	gitea.com/macaron/session v0.0.0-20191207215012-613cebf0674d
 	gitea.com/macaron/toolbox v0.0.0-20190822013122-05ff0fc766b7
+	github.com/360EntSecGroup-Skylar/excelize/v2 v2.0.2
 	github.com/BurntSushi/toml v0.3.1
 	github.com/PuerkitoBio/goquery v1.5.0
 	github.com/RichardKnop/machinery v1.6.9
@@ -107,7 +108,6 @@ require (
 	github.com/unknwon/paginater v0.0.0-20151104151617-7748a72e0141
 	github.com/urfave/cli v1.22.1
 	github.com/xanzy/go-gitlab v0.31.0
-	github.com/360EntSecGroup-Skylar/excelize/v2 v2.0.2
 	github.com/yohcop/openid-go v1.0.0
 	github.com/yuin/goldmark v1.1.27
 	github.com/yuin/goldmark-meta v0.0.0-20191126180153-f0638e958b60

models/custom_migrations.go

@@ -22,7 +22,6 @@ var customMigrations = []CustomMigration{
 }
 
 var customMigrationsStatic = []CustomMigrationStatic{
-	{"Alter user static table field type  ", alterUserStaticTable},
 	{"Delete organization user history data ", deleteNotDisplayUser},
 	{"update issue_fixed_rate to 1 if num_issues is 0 ", updateIssueFixedRate},
 }
@@ -59,14 +58,6 @@ func syncTopicStruct(x *xorm.Engine) error {
 	return err
 }
 
-func alterUserStaticTable(x *xorm.Engine, static *xorm.Engine) error {
-	alterSql := "alter table public.user_business_analysis alter column open_i_index type double precision"
-
-	_, err := static.Exec(alterSql)
-	return err
-
-}
-
 func deleteNotDisplayUser(x *xorm.Engine, static *xorm.Engine) error {
 
 	querySQL := "select id,name from public.user where type=1"

models/repo_statistic.go

@@ -172,3 +172,10 @@ func UpdateRepoStat(repoStat *RepoStatistic) error {
 	_, err := xStatistic.Exec(sql, repoStat.Impact, repoStat.Completeness, repoStat.Liveness, repoStat.ProjectHealth, repoStat.TeamHealth, repoStat.Growth, repoStat.RadarTotal, repoStat.RepoID, repoStat.Date)
 	return err
 }
+
+func UpdateRepoStatVisits(repoStat *RepoStatistic) error {
+	sql := "update repo_statistic set num_visits=? where repo_id=? and date=?"
+
+	_, err := xStatistic.Exec(sql, repoStat.NumVisits, repoStat.RepoID, repoStat.Date)
+	return err
+}

models/user_business_analysis.go

@@ -153,6 +153,76 @@ func getLastCountDate() int64 {
 	return pageStartTime.Unix()
 }
 
+func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) {
+	log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll))
+
+	statictisSess := xStatistic.NewSession()
+	defer statictisSess.Close()
+
+	resultMap := make(map[int64]*UserBusinessAnalysis)
+
+	var newAndCond = builder.NewCond()
+	if len(opts.UserName) > 0 {
+		newAndCond = newAndCond.And(
+			builder.Like{"name", opts.UserName},
+		)
+	}
+	if !opts.IsAll {
+		newAndCond = newAndCond.And(
+			builder.Gte{"count_date": opts.StartTime},
+		)
+		newAndCond = newAndCond.And(
+			builder.Lte{"count_date": opts.EndTime},
+		)
+	}
+
+	allCount, err := statictisSess.Where(newAndCond).Count(new(UserBusinessAnalysis))
+	if err != nil {
+		log.Info("query error." + err.Error())
+		return nil, 0
+	}
+	log.Info("query return total:" + fmt.Sprint(allCount))
+	pageSize := 200
+	totalPage := int(allCount) / pageSize
+
+	for i := 0; i <= int(totalPage); i++ {
+		userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0)
+		if err := statictisSess.Table("user_business_analysis").Where(newAndCond).OrderBy("count_date desc").Limit(pageSize, i*pageSize).
+			Find(&userBusinessAnalysisList); err != nil {
+			return nil, 0
+		}
+		log.Info("query " + fmt.Sprint(i+1) + " result size=" + fmt.Sprint(len(userBusinessAnalysisList)))
+		for _, userRecord := range userBusinessAnalysisList {
+			if _, ok := resultMap[userRecord.ID]; !ok {
+				resultMap[userRecord.ID] = userRecord
+			} else {
+				resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount
+				resultMap[userRecord.ID].CommitCount += userRecord.CommitCount
+				resultMap[userRecord.ID].IssueCount += userRecord.IssueCount
+				resultMap[userRecord.ID].CommentCount += userRecord.CommentCount
+				resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount
+				resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount
+				resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount
+				resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize
+				resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize
+				resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount
+				resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount
+				resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount
+				resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount
+				resultMap[userRecord.ID].LoginCount += userRecord.LoginCount
+			}
+		}
+	}
+
+	userBusinessAnalysisReturnList := UserBusinessAnalysisList{}
+	for _, v := range resultMap {
+		userBusinessAnalysisReturnList = append(userBusinessAnalysisReturnList, v)
+	}
+	sort.Sort(userBusinessAnalysisReturnList)
+	log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList)))
+	return userBusinessAnalysisReturnList, allCount
+}
+
 func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) {
 
 	log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime) + " isAll=" + fmt.Sprint(opts.IsAll))

routers/private/internal.go

@@ -43,7 +43,8 @@ func RegisterRoutes(m *macaron.Macaron) {
 		m.Post("/manager/restart", Restart)
 		m.Post("/manager/flush-queues", bind(private.FlushOptions{}), FlushQueues)
 		m.Post("/tool/update_all_repo_commit_cnt", UpdateAllRepoCommitCnt)
-		m.Post("/tool/repo_stat", RepoStatisticManually)
+		m.Post("/tool/repo_stat/:date", RepoStatisticManually)
+		m.Post("/tool/update_repo_visit/:date", UpdateRepoVisit)
 
 	}, CheckInternalToken)
 }

routers/private/tool.go

@@ -39,8 +39,35 @@ func UpdateAllRepoCommitCnt(ctx *macaron.Context) {
 }
 
 func RepoStatisticManually(ctx *macaron.Context) {
-	date := ctx.Query("date")
+	date := ctx.Params("date")
 	repo.RepoStatisticDaily(date)
 	repo.SummaryStatisticDaily(date)
 	repo.TimingCountDataByDate(date)
 }
+
+func UpdateRepoVisit(ctx *macaron.Context) {
+	date := ctx.Params("date")
+	log.Info("date(%s)", date)
+
+	repos, err := models.GetAllRepositories()
+	if err != nil {
+		log.Error("GetAllRepositories failed:%v", err.Error(), ctx.Data["MsgID"])
+		ctx.JSON(http.StatusInternalServerError, map[string]string{
+			"error_msg": "GetAllRepositories failed",
+		})
+		return
+	}
+
+	for i, repoStat := range repos {
+		log.Info("%d:begin UpdateRepoVisits(id = %d, name = %s)", i, repoStat.ID, repoStat.Name)
+		if err = repo.UpdateRepoVisits(ctx, repoStat, date); err != nil {
+			log.Error("UpdateRepoVisits(id = %d, name = %s) failed:%v", repoStat.ID, repoStat.Name, err.Error())
+			continue
+		}
+		log.Info("%d:finish UpdateRepoVisits(id = %d, name = %s)", i, repoStat.ID, repoStat.Name)
+	}
+
+	ctx.JSON(http.StatusOK, map[string]string{
+		"error_msg": "",
+	})
+}

routers/repo/repo_statistic.go

@@ -1,15 +1,17 @@
 package repo
 
 import (
+	"errors"
 	"time"
 
-	"code.gitea.io/gitea/services/mailer"
-
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/normalization"
 	"code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/services/mailer"
+
+	"gitea.com/macaron/macaron"
 )
 
 func StatisticAuto() {
@@ -48,7 +50,7 @@ func RepoStatisticDaily(date string) {
 	var maxRepoRadar models.RepoStatistic
 
 	for i, repo := range repos {
-		log.Info("start statistic: %s", repo.Name)
+		log.Info("start statistic: %s", getDistinctProjectName(repo))
 		var numDevMonths, numWikiViews, numContributor, numKeyContributor, numCommitsGrowth, numCommitLinesGrowth, numContributorsGrowth int64
 		repoGitStat, err := models.GetRepoKPIStats(repo)
 		if err != nil {
@@ -230,7 +232,7 @@ func RepoStatisticDaily(date string) {
 
 		}
 
-		log.Info("finish statistic: %s", repo.Name)
+		log.Info("finish statistic: %s", getDistinctProjectName(repo))
 	}
 
 	//radar  map
@@ -274,3 +276,29 @@ func getStatTime(timeStr string) (string, string) {
 
 	return beginTime, endTime
 }
+
+func UpdateRepoVisits(ctx *macaron.Context, repo *models.Repository, date string) error {
+	beginTime, endTime := getStatTime(date)
+	var numVisits int
+	numVisits, err := repository.AppointProjectView(repo.OwnerName, repo.Name, beginTime, endTime)
+	if err != nil {
+		log.Error("AppointProjectView failed(%s): %v", getDistinctProjectName(repo), err)
+		return err
+	}
+
+	repoStat, err := models.GetRepoStatisticByDate(date, repo.ID)
+	if err != nil {
+		log.Error("GetRepoStatisticByDate failed(%s): %v", getDistinctProjectName(repo), err)
+		return err
+	} else if len(repoStat) != 1 {
+		log.Error("GetRepoStatisticByDate failed(%s): %v", getDistinctProjectName(repo), err)
+		return errors.New("not find repo")
+	}
+	repoStat[0].NumVisits = int64(numVisits)
+
+	if err = models.UpdateRepoStatVisits(repoStat[0]); err != nil {
+		log.Error("UpdateRepoStatVisits failed(%s): %v", getDistinctProjectName(repo), err)
+		return err
+	}
+	return nil
+}

routers/repo/user_data_analysis.go

@@ -76,11 +76,10 @@ func QueryUserStaticDataPage(ctx *context.Context) {
 		EndTime:   endTime.Unix(),
 		IsAll:     isAll,
 	}
-	mapInterface := make(map[string]interface{})
-	re, count := models.QueryUserStaticDataPage(pageOpts)
-	mapInterface["data"] = re
-	mapInterface["count"] = count
+
 	if IsReturnFile {
+		re, count := models.QueryUserStaticDataAll(pageOpts)
+		log.Info("return count=" + fmt.Sprint(count))
 		//writer exec file.
 		xlsx := excelize.NewFile()
 		sheetName := ctx.Tr("user.static.sheetname")
@@ -163,6 +162,10 @@ func QueryUserStaticDataPage(ctx *context.Context) {
 		}
 
 	} else {
+		mapInterface := make(map[string]interface{})
+		re, count := models.QueryUserStaticDataPage(pageOpts)
+		mapInterface["data"] = re
+		mapInterface["count"] = count
 		ctx.JSON(http.StatusOK, mapInterface)
 	}
 }