aiforge/routers/repo/user_data_analysis.go

package repo

import (
	"fmt"
	"net/http"
	"net/url"
	"time"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/context"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/setting"
	"code.gitea.io/gitea/services/mailer"
	"github.com/360EntSecGroup-Skylar/excelize/v2"
)

const (
	PAGE_SIZE = 2000
)

func getExcelHeader(ctx *context.Context) map[string]string {
	excelHeader := make([]string, 0)
	excelHeader = append(excelHeader, ctx.Tr("user.static.id"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.name"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.UserIndex"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.UserIndexPrimitive"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.codemergecount"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.commitcount"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.issuecount"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.commentcount"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.focusrepocount"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.starrepocount"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.logincount"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.watchedcount"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.commitcodesize"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.solveissuecount"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.encyclopediascount"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.createrepocount"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.openiindex"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.CloudBrainTaskNum"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.CloudBrainRunTime"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.CommitDatasetNum"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.CommitModelCount"))

	excelHeader = append(excelHeader, ctx.Tr("user.static.FocusOtherUser"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.CollectDataset"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.CollectedDataset"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.RecommendDataset"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.CollectImage"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.CollectedImage"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.RecommendImage"))

	excelHeader = append(excelHeader, ctx.Tr("user.static.registdate"))
	excelHeader = append(excelHeader, ctx.Tr("user.static.countdate"))

	excelHeaderMap := make(map[string]string, 0)
	var i, tmp byte
	tmp = 'A'
	i = 0
	for _, value := range excelHeader {
		excelColumn := string(tmp+i) + fmt.Sprint(1)
		//log.Info("excelColumn=" + excelColumn)
		excelHeaderMap[excelColumn] = value
		i++
	}
	return excelHeaderMap
}

func writeExcel(row int, xlsx *excelize.File, sheetName string, userRecord *models.UserBusinessAnalysisAll) {
	rows := fmt.Sprint(row)
	var tmp byte
	tmp = 'A'
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.ID)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.Name)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, fmt.Sprintf("%.2f", userRecord.UserIndex))
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, fmt.Sprintf("%.2f", userRecord.UserIndexPrimitive))
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.CodeMergeCount)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.CommitCount)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.IssueCount)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.CommentCount)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.FocusRepoCount)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.StarRepoCount)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.LoginCount)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.WatchedCount)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.CommitCodeSize)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.SolveIssueCount)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.EncyclopediasCount)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.CreateRepoCount)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, fmt.Sprintf("%.2f", userRecord.OpenIIndex))
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.CloudBrainTaskNum)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, fmt.Sprintf("%.2f", float64(userRecord.CloudBrainRunTime)/3600))
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.CommitDatasetNum)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.CommitModelCount)
	tmp = tmp + 1

	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.FocusOtherUser)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.CollectDataset)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.CollectedDataset)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.RecommendDataset)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.CollectImage)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.CollectedImage)
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, userRecord.RecommendImage)
	tmp = tmp + 1

	formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05")
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, formatTime[0:len(formatTime)-3])
	formatTime = userRecord.DataDate
	tmp = tmp + 1
	xlsx.SetCellValue(sheetName, string(tmp)+rows, formatTime)

}

func queryUserDataPage(ctx *context.Context, tableName string, queryObj interface{}) {
	page := ctx.QueryInt("page")
	if page <= 0 {
		page = 1
	}
	pageSize := ctx.QueryInt("pageSize")
	if pageSize <= 0 {
		pageSize = setting.UI.IssuePagingNum
	}
	userName := ctx.Query("userName")
	IsReturnFile := ctx.QueryBool("IsReturnFile")

	if IsReturnFile {
		//writer exec file.
		xlsx := excelize.NewFile()
		sheetName := ctx.Tr("user.static.sheetname")
		index := xlsx.NewSheet(sheetName)
		xlsx.DeleteSheet("Sheet1")
		dataHeader := getExcelHeader(ctx)
		for k, v := range dataHeader {
			//设置单元格的值
			xlsx.SetCellValue(sheetName, k, v)
		}
		_, count := models.QueryUserStaticDataByTableName(1, 1, tableName, queryObj, userName)
		var indexTotal int64
		indexTotal = 0
		row := 1
		for {
			re, _ := models.QueryUserStaticDataByTableName(int(indexTotal), PAGE_SIZE, tableName, queryObj, "")
			log.Info("return count=" + fmt.Sprint(count))
			for _, userRecord := range re {
				row++
				writeExcel(row, xlsx, sheetName, userRecord)
				// rows := fmt.Sprint(row)
				// xlsx.SetCellValue(sheetName, "A"+rows, userRecord.ID)
				// xlsx.SetCellValue(sheetName, "B"+rows, userRecord.Name)
				// xlsx.SetCellValue(sheetName, "C"+rows, fmt.Sprintf("%.2f", userRecord.UserIndex))
				// xlsx.SetCellValue(sheetName, "D"+rows, userRecord.CodeMergeCount)
				// xlsx.SetCellValue(sheetName, "E"+rows, userRecord.CommitCount)
				// xlsx.SetCellValue(sheetName, "F"+rows, userRecord.IssueCount)
				// xlsx.SetCellValue(sheetName, "G"+rows, userRecord.CommentCount)
				// xlsx.SetCellValue(sheetName, "H"+rows, userRecord.FocusRepoCount)
				// xlsx.SetCellValue(sheetName, "I"+rows, userRecord.StarRepoCount)
				// xlsx.SetCellValue(sheetName, "J"+rows, userRecord.LoginCount)
				// xlsx.SetCellValue(sheetName, "K"+rows, userRecord.WatchedCount)
				// xlsx.SetCellValue(sheetName, "L"+rows, userRecord.CommitCodeSize)
				// xlsx.SetCellValue(sheetName, "M"+rows, userRecord.SolveIssueCount)
				// xlsx.SetCellValue(sheetName, "N"+rows, userRecord.EncyclopediasCount)
				// xlsx.SetCellValue(sheetName, "O"+rows, userRecord.CreateRepoCount)
				// xlsx.SetCellValue(sheetName, "P"+rows, fmt.Sprintf("%.2f", userRecord.OpenIIndex))
				// xlsx.SetCellValue(sheetName, "Q"+rows, userRecord.CloudBrainTaskNum)
				// xlsx.SetCellValue(sheetName, "R"+rows, fmt.Sprintf("%.2f", float64(userRecord.CloudBrainRunTime)/3600))
				// xlsx.SetCellValue(sheetName, "S"+rows, userRecord.CommitDatasetNum)
				// xlsx.SetCellValue(sheetName, "T"+rows, userRecord.CommitModelCount)
				// formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05")
				// xlsx.SetCellValue(sheetName, "U"+rows, formatTime[0:len(formatTime)-3])
				// formatTime = userRecord.DataDate
				// xlsx.SetCellValue(sheetName, "V"+rows, formatTime)
			}

			indexTotal += PAGE_SIZE
			if indexTotal >= count {
				break
			}
		}
		//设置默认打开的表单
		xlsx.SetActiveSheet(index)
		filename := sheetName + "_" + ctx.Tr("user.static."+tableName) + ".xlsx"
		ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename))
		ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
		if _, err := xlsx.WriteTo(ctx.Resp); err != nil {
			log.Info("writer exel error." + err.Error())
		}
	} else {
		re, count := models.QueryUserStaticDataByTableName((page-1)*pageSize, pageSize, tableName, queryObj, userName)
		mapInterface := make(map[string]interface{})
		mapInterface["data"] = re
		mapInterface["count"] = count
		ctx.JSON(http.StatusOK, mapInterface)
	}
}

func QueryMetrics(ctx *context.Context) {
	startDate := ctx.Query("startDate")
	endDate := ctx.Query("endDate")
	startTime, _ := time.ParseInLocation("2006-01-02", startDate, time.Local)
	endTime, _ := time.ParseInLocation("2006-01-02", endDate, time.Local)
	result, count := models.QueryMetrics(startTime.Unix(), endTime.Unix())
	mapInterface := make(map[string]interface{})
	mapInterface["data"] = result
	mapInterface["count"] = count
	ctx.JSON(http.StatusOK, mapInterface)
}

func QueryRankingList(ctx *context.Context) {
	key := ctx.Query("key")
	tableName := ctx.Query("tableName")
	limit := ctx.QueryInt("limit")

	result, count := models.QueryRankList(key, tableName, limit)
	mapInterface := make(map[string]interface{})
	mapInterface["data"] = result
	mapInterface["count"] = count
	ctx.JSON(http.StatusOK, mapInterface)
}

func QueryUserStaticCurrentMonth(ctx *context.Context) {
	queryUserDataPage(ctx, "public.user_business_analysis_current_month", new(models.UserBusinessAnalysisCurrentMonth))
}

func QueryUserStaticCurrentWeek(ctx *context.Context) {
	queryUserDataPage(ctx, "public.user_business_analysis_current_week", new(models.UserBusinessAnalysisCurrentWeek))
}

func QueryUserStaticCurrentYear(ctx *context.Context) {
	queryUserDataPage(ctx, "public.user_business_analysis_current_year", new(models.UserBusinessAnalysisCurrentYear))
}

func QueryUserStaticLast30Day(ctx *context.Context) {
	queryUserDataPage(ctx, "public.user_business_analysis_last30_day", new(models.UserBusinessAnalysisLast30Day))
}

func QueryUserStaticLastMonth(ctx *context.Context) {
	queryUserDataPage(ctx, "public.user_business_analysis_last_month", new(models.UserBusinessAnalysisLastMonth))
}

func QueryUserStaticYesterday(ctx *context.Context) {
	queryUserDataPage(ctx, "public.user_business_analysis_yesterday", new(models.UserBusinessAnalysisYesterday))
}

func QueryUserStaticAll(ctx *context.Context) {
	queryUserDataPage(ctx, "public.user_business_analysis_all", new(models.UserBusinessAnalysisAll))
}

func QueryUserStaticDataPage(ctx *context.Context) {
	startDate := ctx.Query("startDate")
	endDate := ctx.Query("endDate")
	page := ctx.QueryInt("page")
	if page <= 0 {
		page = 1
	}
	pageSize := ctx.QueryInt("pageSize")
	if pageSize <= 0 {
		pageSize = setting.UI.IssuePagingNum
	}
	userName := ctx.Query("userName")
	IsReturnFile := ctx.QueryBool("IsReturnFile")
	log.Info("startDate=" + startDate + "  endDate=" + endDate + " userName=" + userName + " page=" + fmt.Sprint(page))
	var startTime time.Time
	var endTime time.Time
	var isAll bool
	if startDate == "all" {
		isAll = true
		startTime = time.Now()
		endTime = time.Now()
	} else {
		startTime, _ = time.ParseInLocation("2006-01-02", startDate, time.Local)
		startTime = time.Date(startTime.Year(), startTime.Month(), startTime.Day(), 12, 0, 0, 0, startTime.Location())
		settingStartTime, _ := time.Parse("2006-01-02", setting.RadarMap.RecordBeginTime)
		if startTime.Unix() < settingStartTime.Unix() {
			startTime = settingStartTime
			startDate = settingStartTime.Format("2006-01-02")
		}
		endTime, _ = time.ParseInLocation("2006-01-02", endDate, time.Local)
		endTime = endTime.AddDate(0, 0, 1)
		endTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 23, 59, 59, 0, startTime.Location())

		isAll = false
		log.Info("startTime=" + fmt.Sprint(startTime.Unix()) + "  endDate=" + fmt.Sprint(endTime.Unix()))
	}

	if IsReturnFile {
		page = -1
		pageSize = -1
	}

	pageOpts := &models.UserBusinessAnalysisQueryOptions{
		ListOptions: models.ListOptions{
			Page:     page,
			PageSize: pageSize,
		},
		UserName:  userName,
		StartTime: startTime.Unix(),
		EndTime:   endTime.Unix(),
		IsAll:     isAll,
	}

	if IsReturnFile {
		re, count := models.QueryUserStaticDataAll(pageOpts)
		log.Info("return count=" + fmt.Sprint(count))
		//writer exec file.
		xlsx := excelize.NewFile()
		sheetName := ctx.Tr("user.static.sheetname")
		index := xlsx.NewSheet(sheetName)
		xlsx.DeleteSheet("Sheet1")

		dataHeader := getExcelHeader(ctx)
		for k, v := range dataHeader {
			//设置单元格的值
			xlsx.SetCellValue(sheetName, k, v)
		}

		for i, userRecord := range re {
			row := i + 2
			writeExcel(row, xlsx, sheetName, userRecord)
			// xlsx.SetCellValue(sheetName, "A"+rows, userRecord.ID)
			// xlsx.SetCellValue(sheetName, "B"+rows, userRecord.Name)
			// xlsx.SetCellValue(sheetName, "C"+rows, fmt.Sprintf("%.2f", userRecord.UserIndex))
			// xlsx.SetCellValue(sheetName, "D"+rows, userRecord.CodeMergeCount)
			// xlsx.SetCellValue(sheetName, "E"+rows, userRecord.CommitCount)
			// xlsx.SetCellValue(sheetName, "F"+rows, userRecord.IssueCount)
			// xlsx.SetCellValue(sheetName, "G"+rows, userRecord.CommentCount)
			// xlsx.SetCellValue(sheetName, "H"+rows, userRecord.FocusRepoCount)
			// xlsx.SetCellValue(sheetName, "I"+rows, userRecord.StarRepoCount)
			// xlsx.SetCellValue(sheetName, "J"+rows, userRecord.LoginCount)
			// xlsx.SetCellValue(sheetName, "K"+rows, userRecord.WatchedCount)
			// xlsx.SetCellValue(sheetName, "L"+rows, userRecord.CommitCodeSize)
			// xlsx.SetCellValue(sheetName, "M"+rows, userRecord.SolveIssueCount)
			// xlsx.SetCellValue(sheetName, "N"+rows, userRecord.EncyclopediasCount)
			// xlsx.SetCellValue(sheetName, "O"+rows, userRecord.CreateRepoCount)
			// xlsx.SetCellValue(sheetName, "P"+rows, fmt.Sprintf("%.2f", userRecord.OpenIIndex))
			// xlsx.SetCellValue(sheetName, "Q"+rows, userRecord.CloudBrainTaskNum)
			// xlsx.SetCellValue(sheetName, "R"+rows, fmt.Sprintf("%.2f", float64(userRecord.CloudBrainRunTime)/3600))
			// xlsx.SetCellValue(sheetName, "S"+rows, userRecord.CommitDatasetNum)
			// xlsx.SetCellValue(sheetName, "T"+rows, userRecord.CommitModelCount)
			// formatTime := userRecord.RegistDate.Format("2006-01-02 15:04:05")
			// xlsx.SetCellValue(sheetName, "U"+rows, formatTime[0:len(formatTime)-3])
			// formatTime = userRecord.DataDate
			// xlsx.SetCellValue(sheetName, "V"+rows, formatTime)
		}

		//设置默认打开的表单
		xlsx.SetActiveSheet(index)

		filename := sheetName + "_" + ctx.Tr("user.static.all") + ".xlsx"

		ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(filename))
		ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
		if _, err := xlsx.WriteTo(ctx.Resp); err != nil {
			log.Info("writer exel error." + err.Error())
		}

	} else {
		mapInterface := make(map[string]interface{})
		re, count := models.QueryUserStaticDataPage(pageOpts)
		mapInterface["data"] = re
		mapInterface["count"] = count
		ctx.JSON(http.StatusOK, mapInterface)
	}
}

func TimingCountDataByDateAndReCount(date string, isReCount bool) {

	t, _ := time.Parse("2006-01-02", date)
	startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location())

	endTime := time.Date(t.Year(), t.Month(), t.Day(), 23, 59, 59, 0, t.Location())

	//query wiki data
	log.Info("start to time count data")
	wikiMap := make(map[string]int)
	warnEmailMessage := "用户统计信息入库失败，请尽快定位。"
	repoList, err := models.GetAllRepositories()
	if err != nil {
		log.Error("query repo error." + err.Error())
		mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage)
		return
	}
	log.Info("start to query wiki data")
	for _, repoRecord := range repoList {
		wikiPath := models.WikiPath(repoRecord.OwnerName, repoRecord.Name)
		time, err := git.GetLatestCommitTime(wikiPath)
		if err == nil {
			log.Info("last commit time:" + time.Format("2006-01-02 15:04:05") + " wikiPath=" + wikiPath)
			if time.After(startTime) {
				wikiRepo, _, err := FindWikiRepoCommitByWikiPath(wikiPath)
				if err != nil {
					log.Error("wiki not exist. wikiPath=" + wikiPath)
				} else {
					log.Info("wiki exist, wikiPath=" + wikiPath)
					list, err := wikiRepo.GetCommitByPathAndDays(wikiPath, 1)
					if err != nil {
						log.Info("err,err=v%", err)
					} else {
						for logEntry := list.Front(); logEntry != nil; logEntry = logEntry.Next() {
							commit := logEntry.Value.(*git.Commit)
							log.Info("commit msg=" + commit.CommitMessage + " time=" + commit.Committer.When.Format("2006-01-02 15:04:05") + " user=" + commit.Committer.Name)
							if _, ok := wikiMap[commit.Committer.Name]; !ok {
								wikiMap[commit.Committer.Name] = 1
							} else {
								wikiMap[commit.Committer.Name] += 1
							}
						}
					}

				}
			}
		}
	}
	//other user info data
	err = models.CounDataByDateAndReCount(wikiMap, startTime, endTime, isReCount)
	if err != nil {
		log.Error("count user info error." + err.Error())
		mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage)
	}
	log.Info("start to count all user info data")
	//models.RefreshUserStaticAllTabel(wikiMap)
	log.Info("end to count all user info data")
}

func TimingCountDataByDate(date string) {
	TimingCountDataByDateAndReCount(date, true)
}

func TimingCountData() {
	log.Info("start to time count data")
	currentTimeNow := time.Now()
	log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05"))
	startTime := currentTimeNow.AddDate(0, 0, -1).Format("2006-01-02")
	TimingCountDataByDateAndReCount(startTime, false)
}