Merge branch 'V20211115' into wangjr_pro

models/custom_migrations.go

@@ -14,6 +14,8 @@ var customMigrations = []CustomMigration{
 	{"Custom v1 Topic struct change to support chinese", syncTopicStruct},
 }
 
+var customMigrationsStatic = []CustomMigration{}
+
 func MigrateCustom(x *xorm.Engine) {
 
 	for _, m := range customMigrations {
@@ -27,6 +29,17 @@ func MigrateCustom(x *xorm.Engine) {
 
 }
 
+func MigrateCustomStatic(x *xorm.Engine) {
+	for _, m := range customMigrationsStatic {
+		log.Info("Migration: %s", m.Description)
+		if err := m.Migrate(x); err != nil {
+
+			log.Error("Migration: %v", err)
+
+		}
+	}
+}
+
 func syncTopicStruct(x *xorm.Engine) error {
 
 	query := "ALTER TABLE topic ALTER COLUMN name TYPE varchar(105);"

models/models.go

@@ -190,7 +190,7 @@ func setEngine(engine *xorm.Engine, table []interface{}, database *setting.DBInf
 	engine.SetMaxIdleConns(setting.Database.MaxIdleConns)
 	engine.SetConnMaxLifetime(setting.Database.ConnMaxLifetime)
 	engine.Sync2(table...)
-	MigrateCustom(engine)
+
 	return nil
 }
 
@@ -222,7 +222,7 @@ func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err e
 	if err = newEngine(ctx, migrateFunc, x, tables, setting.Database); err != nil {
 		return fmt.Errorf("newEngine failed: %v", err)
 	}
-
+	MigrateCustom(x)
 	xStatistic, err = getEngine(setting.DatabaseStatistic)
 	if err != nil {
 		return fmt.Errorf("Failed to connect to database: %v", err)
@@ -230,6 +230,7 @@ func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err e
 	if err = newEngine(ctx, migrateFunc, xStatistic, tablesStatistic, setting.DatabaseStatistic); err != nil {
 		return fmt.Errorf("newEngine statistic failed: %v", err)
 	}
+	MigrateCustomStatic(xStatistic)
 
 	HasEngine = true
 

models/user_business_analysis.go

@@ -5,7 +5,9 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
+	"xorm.io/builder"
 )
 
 type UserBusinessAnalysis struct {
@@ -71,6 +73,14 @@ type UserBusinessAnalysis struct {
 	Name string `xorm:"NOT NULL"`
 }
 
+type UserBusinessAnalysisQueryOptions struct {
+	ListOptions
+	UserName  string
+	SortType  string
+	StartTime int64
+	EndTime   int64
+}
+
 func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis {
 	log.Info("query startTime =" + fmt.Sprint(startTime) + " endTime=" + fmt.Sprint(endTime))
 	statictisSess := xStatistic.NewSession()
@@ -114,11 +124,114 @@ func QueryUserStaticData(startTime int64, endTime int64) []*UserBusinessAnalysis
 	return userBusinessAnalysisReturnList
 }
 
+func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusinessAnalysis, int64) {
+
+	log.Info("query startTime =" + fmt.Sprint(opts.StartTime) + " endTime=" + fmt.Sprint(opts.EndTime))
+	statictisSess := xStatistic.NewSession()
+	defer statictisSess.Close()
+
+	currentTimeNow := time.Now()
+	pageStartTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location())
+	pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location())
+
+	var cond = builder.NewCond()
+	if len(opts.UserName) > 0 {
+		cond = cond.And(
+			builder.Eq{"name": opts.UserName},
+		)
+	}
+	cond = cond.And(
+		builder.Gte{"count_date": fmt.Sprint(pageStartTime)},
+	)
+	cond = cond.And(
+		builder.Lte{"count_date": fmt.Sprint(pageEndTime)},
+	)
+
+	count, err := statictisSess.Where(cond).Count(new(UserBusinessAnalysis))
+	if err != nil {
+		log.Info("query error." + err.Error())
+		return nil, 0
+	}
+
+	if opts.Page >= 0 && opts.PageSize > 0 {
+		var start int
+		if opts.Page == 0 {
+			start = 0
+		} else {
+			start = (opts.Page - 1) * opts.PageSize
+		}
+		statictisSess.Limit(opts.PageSize, start)
+	}
+	statictisSess.OrderBy("count_date desc")
+
+	userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0, setting.UI.IssuePagingNum)
+	if err := statictisSess.Table("user_business_analysis").Where(cond).
+		Find(&userBusinessAnalysisList); err != nil {
+		return nil, 0
+	}
+
+	resultMap := make(map[int64]*UserBusinessAnalysis)
+
+	var newAndCond = builder.NewCond()
+	var newOrCond = builder.NewCond()
+	for _, userRecord := range userBusinessAnalysisList {
+		newOrCond.Or(
+			builder.Eq{"id": userRecord.ID},
+		)
+	}
+	newAndCond = newAndCond.And(
+		newOrCond,
+	)
+	newAndCond = newAndCond.And(
+		builder.Gte{"count_date": fmt.Sprint(opts.StartTime)},
+	)
+	newAndCond = newAndCond.And(
+		builder.Lte{"count_date": fmt.Sprint(opts.EndTime)},
+	)
+
+	userBusinessAnalysisList = make([]*UserBusinessAnalysis, 0)
+	if err := statictisSess.Table("user_business_analysis").Where(newAndCond).
+		Find(&userBusinessAnalysisList); err != nil {
+		return nil, 0
+	}
+
+	log.Info("query result size=" + fmt.Sprint(len(userBusinessAnalysisList)))
+	for _, userRecord := range userBusinessAnalysisList {
+		if _, ok := resultMap[userRecord.ID]; !ok {
+			resultMap[userRecord.ID] = userRecord
+		} else {
+			resultMap[userRecord.ID].CodeMergeCount += userRecord.CodeMergeCount
+			resultMap[userRecord.ID].CommitCount += userRecord.CommitCount
+			resultMap[userRecord.ID].IssueCount += userRecord.IssueCount
+			resultMap[userRecord.ID].CommentCount += userRecord.CommentCount
+			resultMap[userRecord.ID].FocusRepoCount += userRecord.FocusRepoCount
+			resultMap[userRecord.ID].StarRepoCount += userRecord.StarRepoCount
+			resultMap[userRecord.ID].WatchedCount += userRecord.WatchedCount
+			resultMap[userRecord.ID].CommitCodeSize += userRecord.CommitCodeSize
+			resultMap[userRecord.ID].CommitDatasetSize += userRecord.CommitDatasetSize
+			resultMap[userRecord.ID].CommitModelCount += userRecord.CommitModelCount
+			resultMap[userRecord.ID].SolveIssueCount += userRecord.SolveIssueCount
+			resultMap[userRecord.ID].EncyclopediasCount += userRecord.EncyclopediasCount
+			resultMap[userRecord.ID].CreateRepoCount += userRecord.CreateRepoCount
+			resultMap[userRecord.ID].LoginCount += userRecord.LoginCount
+		}
+	}
+
+	userBusinessAnalysisReturnList := make([]*UserBusinessAnalysis, len(resultMap))
+	index := 0
+	for _, v := range resultMap {
+		userBusinessAnalysisReturnList[index] = v
+		index += 1
+	}
+	log.Info("return size=" + fmt.Sprint(len(userBusinessAnalysisReturnList)))
+	return userBusinessAnalysisReturnList, count
+}
+
 func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime time.Time) {
 	log.Info("start to count other user info data")
 	sess := x.NewSession()
 	defer sess.Close()
-	sess.Select("`user`.*").Table("user")
+	sess.Select("`user`.*").Table("user").Where("type != 1 and is_active=true")
 	userList := make([]*User, 0)
 	sess.Find(&userList)
 

modules/storage/obs.go

@@ -6,6 +6,7 @@ package storage
 
 import (
 	"io"
+	"net/url"
 	"path"
 	"strconv"
 	"strings"
@@ -262,6 +263,7 @@ func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error)
 	input.Method = obs.HttpMethodGet
 
 	reqParams := make(map[string]string)
+	fileName = url.QueryEscape(fileName)
 	reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\""
 	input.QueryParams = reqParams
 	output, err := ObsCli.CreateSignedUrl(input)

routers/api/v1/repo/repo_dashbord.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"strconv"
@@ -107,6 +108,7 @@ func GetAllProjectsPeriodStatistics(ctx *context.Context) {
 		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error"))
 		return
 	}
+	sql := generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, page, pageSize)
 
 	projectsPeriodData := ProjectsPeriodData{
 		RecordBeginTime: recordBeginTime.Format(DATE_FORMAT),
@@ -114,13 +116,23 @@ func GetAllProjectsPeriodStatistics(ctx *context.Context) {
 		TotalPage:       getTotalPage(total, pageSize),
 		TotalCount:      total,
 		LastUpdatedTime: latestUpdatedTime,
-		PageRecords:     models.GetRepoStatisticByRawSql(generatePageSql(beginTime, endTime, latestDate, q, orderBy, page, pageSize)),
+		PageRecords:     models.GetRepoStatisticByRawSql(sql),
 	}
 
 	ctx.JSON(http.StatusOK, projectsPeriodData)
 
 }
 
+func generateSqlByType(ctx *context.Context, beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string {
+	sql := ""
+	if ctx.QueryTrim("type") == "all" {
+		sql = generateTypeAllSql(beginTime, endTime, latestDate, q, orderBy, page, pageSize)
+	} else {
+		sql = generatePageSql(beginTime, endTime, latestDate, q, orderBy, page, pageSize)
+	}
+	return sql
+}
+
 func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) {
 
 	recordBeginTime, err := getRecordBeginTime()
@@ -158,7 +170,7 @@ func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) {
 		return
 	}
 
-	fileName := getFileName(ctx, beginTime, endTime)
+	fileName, frontName := getFileName(ctx, beginTime, endTime)
 
 	if err := os.MkdirAll(setting.RadarMap.Path, os.ModePerm); err != nil {
 		ctx.Error(http.StatusBadRequest, fmt.Errorf("Failed to create dir %s: %v", setting.AvatarUploadPath, err).Error())
@@ -175,7 +187,7 @@ func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) {
 	writer.Write(allProjectsPeroidHeader(ctx))
 	for i := 0; i <= totalPage; i++ {
 
-		pageRecords := models.GetRepoStatisticByRawSql(generatePageSql(beginTime, endTime, latestDate, q, orderBy, i+1, pageSize))
+		pageRecords := models.GetRepoStatisticByRawSql(generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, i+1, pageSize))
 		for _, record := range pageRecords {
 			e = writer.Write(allProjectsPeroidValues(record, ctx))
 			if e != nil {
@@ -186,20 +198,24 @@ func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) {
 
 	}
 
-	ctx.ServeFile(fileName)
+	ctx.ServeFile(fileName, url.QueryEscape(frontName))
 
 }
 
-func getFileName(ctx *context.Context, beginTime time.Time, endTime time.Time) string {
-	baseName := setting.RadarMap.Path + "/"
-	if ctx.QueryTrim("type") != "" {
-		baseName = baseName + ctx.QueryTrim("type") + "_"
-	}
+func getFileName(ctx *context.Context, beginTime time.Time, endTime time.Time) (string, string) {
+	baseName := setting.RadarMap.Path + "/项目分析_"
+
 	if ctx.QueryTrim("q") != "" {
 		baseName = baseName + ctx.QueryTrim("q") + "_"
 	}
-	baseName = baseName + beginTime.AddDate(0, 0, -1).Format(DATE_FORMAT) + "_to_" + endTime.AddDate(0, 0, -1).Format(DATE_FORMAT) + "_" + strconv.FormatInt(time.Now().Unix(), 10) + ".csv"
-	return baseName
+	if ctx.QueryTrim("type") == "all" {
+		baseName = baseName + "所有"
+	} else {
+		baseName = baseName + beginTime.AddDate(0, 0, -1).Format(DATE_FORMAT) + "_" + endTime.AddDate(0, 0, -1).Format(DATE_FORMAT)
+	}
+	frontName := baseName + ".csv"
+	localName := baseName + "_" + strconv.FormatInt(time.Now().Unix(), 10) + ".csv"
+	return localName, path.Base(frontName)
 }
 
 func ClearUnusedStatisticsFile() {
@@ -219,8 +235,8 @@ func ClearUnusedStatisticsFile() {
 
 func allProjectsPeroidHeader(ctx *context.Context) []string {
 
-	return []string{ctx.Tr("repos.id"), ctx.Tr("repos.projectName"), ctx.Tr("repos.isPrivate"), ctx.Tr("repos.openi"), ctx.Tr("repos.visit"), ctx.Tr("repos.download"), ctx.Tr("repos.pr"), ctx.Tr("repos.commit"),
-		ctx.Tr("repos.watches"), ctx.Tr("repos.stars"), ctx.Tr("repos.forks"), ctx.Tr("repos.issues"), ctx.Tr("repos.closedIssues"), ctx.Tr("repos.contributor")}
+	return []string{ctx.Tr("admin.repos.id"), ctx.Tr("admin.repos.projectName"), ctx.Tr("admin.repos.isPrivate"), ctx.Tr("admin.repos.openi"), ctx.Tr("admin.repos.visit"), ctx.Tr("admin.repos.download"), ctx.Tr("admin.repos.pr"), ctx.Tr("admin.repos.commit"),
+		ctx.Tr("admin.repos.watches"), ctx.Tr("admin.repos.stars"), ctx.Tr("admin.repos.forks"), ctx.Tr("admin.repos.issues"), ctx.Tr("admin.repos.closedIssues"), ctx.Tr("admin.repos.contributor")}
 
 }
 
@@ -234,9 +250,9 @@ func allProjectsPeroidValues(rs *models.RepoStatistic, ctx *context.Context) []s
 
 func getIsPrivateDisplay(private bool, ctx *context.Context) string {
 	if private {
-		return ctx.Tr("repos.yes")
+		return ctx.Tr("admin.repos.yes")
 	} else {
-		return ctx.Tr("repos.no")
+		return ctx.Tr("admin.repos.no")
 	}
 }
 
@@ -358,11 +374,11 @@ func generateTargetSql(beginTime time.Time, endTime time.Time, repoId int64) str
 	return sql
 }
 
-func generateCountSql(beginTime time.Time, endTime time.Time, yesterday string, q string) string {
+func generateCountSql(beginTime time.Time, endTime time.Time, latestDate string, q string) string {
 	countSql := "SELECT count(*) FROM  " +
 		"(SELECT repo_id FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
 		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," +
-		"(SELECT repo_id,name,is_private,radar_total from public.repo_statistic where date='" + yesterday + "') B" +
+		"(SELECT repo_id,name,is_private,radar_total from public.repo_statistic where date='" + latestDate + "') B" +
 		" where A.repo_id=B.repo_id"
 	if q != "" {
 		countSql = countSql + " and B.name like '%" + q + "%'"
@@ -370,18 +386,34 @@ func generateCountSql(beginTime time.Time, endTime time.Time, yesterday string,
 	return countSql
 }
 
-func generatePageSql(beginTime time.Time, endTime time.Time, yesterday string, q string, orderBy string, page int, pageSize int) string {
-	countSql := "SELECT A.repo_id,name,is_private,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM  " +
+func generateTypeAllSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string {
+	sql := "SELECT A.repo_id,name,is_private,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM  " +
+		"(SELECT repo_id,sum(num_visits) as num_visits " +
+		" FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
+		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," +
+		"(SELECT repo_id,name,is_private,radar_total,num_watches,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor from public.repo_statistic where date='" + latestDate + "') B" +
+		" where A.repo_id=B.repo_id"
+
+	if q != "" {
+		sql = sql + " and name like '%" + q + "%'"
+	}
+	sql = sql + " order by " + orderBy + " desc,repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize)
+	return sql
+}
+
+func generatePageSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string {
+
+	sql := "SELECT A.repo_id,name,is_private,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM  " +
 		"(SELECT repo_id,sum(num_watches_added) as num_watches,sum(num_visits) as num_visits, sum(num_downloads_added) as num_downloads,sum(num_pulls_added) as num_pulls,sum(num_commits_added) as num_commits,sum(num_stars_added) as num_stars,sum(num_forks_added) num_forks,sum(num_issues_added) as num_issues,sum(num_closed_issues_added) as num_closed_issues,sum(num_contributor_added) as num_contributor " +
 		" FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
 		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," +
-		"(SELECT repo_id,name,is_private,radar_total from public.repo_statistic where date='" + yesterday + "') B" +
+		"(SELECT repo_id,name,is_private,radar_total from public.repo_statistic where date='" + latestDate + "') B" +
 		" where A.repo_id=B.repo_id"
 	if q != "" {
-		countSql = countSql + " and B.name like '%" + q + "%'"
+		sql = sql + " and B.name like '%" + q + "%'"
 	}
-	countSql = countSql + " order by " + orderBy + " desc,A.repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize)
-	return countSql
+	sql = sql + " order by " + orderBy + " desc,A.repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize)
+	return sql
 }
 
 func getOrderBy(ctx *context.Context) string {

routers/repo/user_data_analysis.go

@@ -9,6 +9,7 @@ import (
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
 )
 
 func QueryUserStaticData(ctx *context.Context) {
@@ -19,7 +20,36 @@ func QueryUserStaticData(ctx *context.Context) {
 	endTime, _ := time.Parse("2006-01-02", endDate)
 	log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + "  endDate=" + fmt.Sprint(endTime.Unix()))
 	ctx.JSON(http.StatusOK, models.QueryUserStaticData(startTime.Unix(), endTime.Unix()))
+}
+
+func QueryUserStaticDataPage(ctx *context.Context) {
+	startDate := ctx.Query("startDate")
+	endDate := ctx.Query("endDate")
+	page := ctx.QueryInt("page")
+	if page <= 0 {
+		page = 1
+	}
+	userName := ctx.Query("userName")
 
+	log.Info("startDate=" + startDate + "  endDate=" + endDate + " userName=" + userName + " page=" + fmt.Sprint(page))
+	startTime, _ := time.Parse("2006-01-02", startDate)
+	endTime, _ := time.Parse("2006-01-02", endDate)
+	log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + "  endDate=" + fmt.Sprint(endTime.Unix()))
+
+	pageOpts := &models.UserBusinessAnalysisQueryOptions{
+		ListOptions: models.ListOptions{
+			Page:     page,
+			PageSize: setting.UI.IssuePagingNum,
+		},
+		UserName:  userName,
+		StartTime: startTime.Unix(),
+		EndTime:   endTime.Unix(),
+	}
+	mapInterface := make(map[string]interface{})
+	re, count := models.QueryUserStaticDataPage(pageOpts)
+	mapInterface["data"] = re
+	mapInterface["count"] = count
+	ctx.JSON(http.StatusOK, mapInterface)
 }
 
 func TimingCountDataByDate(date string) {

routers/repo/view.go

@@ -919,6 +919,7 @@ func Forks(ctx *context.Context) {
 }
 
 func Contributors(ctx *context.Context) {
+	ctx.Data["PageIsViewCode"] = true
 	ctx.HTML(http.StatusOK, tplContributors)
 }
 

routers/routes/routes.go

@@ -793,6 +793,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 
 	m.Post("/:username/:reponame/action/:action", reqSignIn, context.RepoAssignment(), context.UnitTypes(), repo.Action)
 	m.Get("/tool/query_user_static", repo.QueryUserStaticData)
+	m.Get("/tool/query_user_static_page", repo.QueryUserStaticDataPage)
 	// Grouping for those endpoints not requiring authentication
 	m.Group("/:username/:reponame", func() {
 		m.Get("/contributors", repo.Contributors)

templates/repo/header.tmpl

@@ -93,7 +93,7 @@
 			<div class="ui tabular stackable menu navbar">
 				{{if .Permission.CanRead $.UnitTypeCode}}
 				<div class="dropdown-menu">
-					<a class="{{if or .PageIsViewCode .PageIsReleaseList .PageIsWiki .PageIsActivity}}active{{end}} item" href="{{.RepoLink}}{{if (ne .BranchName .Repository.DefaultBranch)}}/src/{{.BranchNameSubURL | EscapePound}}{{end}}">
+					<a class="{{if or .PageIsViewCode .PageIsReleaseList .PageIsWiki .PageIsActivity .PageIsViewCode}}active{{end}} item" href="{{.RepoLink}}{{if (ne .BranchName .Repository.DefaultBranch)}}/src/{{.BranchNameSubURL | EscapePound}}{{end}}">
 							<span>{{svg "octicon-code" 16}} {{.i18n.Tr "repo.code"}} <i class="dropdown icon"></i></span>
 					</a>
 					<div class="dropdown-content">