aiforge/routers/api/v1/repo/repo_dashbord.go

package repo

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"time"

	"github.com/360EntSecGroup-Skylar/excelize/v2"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/log"

	"code.gitea.io/gitea/modules/context"
	"code.gitea.io/gitea/modules/setting"
)

const DEFAULT_PAGE_SIZE = 10
const DATE_FORMAT = "2006-01-02"
const EXCEL_DATE_FORMAT = "20060102"

type ProjectsPeriodData struct {
	RecordBeginTime string                  `json:"recordBeginTime"`
	LastUpdatedTime string                  `json:"lastUpdatedTime"`
	PageSize        int                     `json:"pageSize"`
	TotalPage       int                     `json:"totalPage"`
	TotalCount      int64                   `json:"totalCount"`
	PageRecords     []*models.RepoStatistic `json:"pageRecords"`
}

type UserInfo struct {
	User          string `json:"user"`
	Mode          int    `json:"mode"`
	PR            int64  `json:"pr"`
	Commit        int    `json:"commit"`
	RelAvatarLink string `json:"relAvatarLink"`
	Email         string `json:"email"`
}

type ProjectLatestData struct {
	RecordBeginTime  string     `json:"recordBeginTime"`
	LastUpdatedTime  string     `json:"lastUpdatedTime"`
	CreatTime        string     `json:"creatTime"`
	OpenI            float64    `json:"openi"`
	Comment          int64      `json:"comment"`
	View             int64      `json:"view"`
	Download         int64      `json:"download"`
	IssueClosedRatio float32    `json:"issueClosedRatio"`
	Impact           float64    `json:"impact"`
	Completeness     float64    `json:"completeness"`
	Liveness         float64    `json:"liveness"`
	ProjectHealth    float64    `json:"projectHealth"`
	TeamHealth       float64    `json:"teamHealth"`
	Growth           float64    `json:"growth"`
	Description      string     `json:"description"`
	Top10            []UserInfo `json:"top10"`
}

func RestoreForkNumber(ctx *context.Context) {
	repos, err := models.GetAllRepositories()
	if err != nil {
		log.Error("GetAllRepositories failed: %v", err.Error())
		return
	}
	for _, repo := range repos {
		models.RestoreRepoStatFork(int64(repo.NumForks), repo.ID)
	}

	ctx.JSON(http.StatusOK, struct{}{})
}

func GetAllProjectsPeriodStatistics(ctx *context.Context) {

	recordBeginTime, err := getRecordBeginTime()
	if err != nil {
		log.Error("Can not get record begin time", err)
		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err"))
		return
	}
	beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime)
	if err != nil {
		log.Error("Parameter is wrong", err)
		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong"))
		return
	}
	q := ctx.QueryTrim("q")
	page := ctx.QueryInt("page")
	if page <= 0 {
		page = 1
	}
	pageSize := ctx.QueryInt("pagesize")
	if pageSize <= 0 {
		pageSize = DEFAULT_PAGE_SIZE
	}
	orderBy := getOrderBy(ctx)

	latestUpdatedTime, latestDate, err := models.GetRepoStatLastUpdatedTime()
	if err != nil {
		log.Error("Can not query the last updated time.", err)
		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error"))
		return
	}

	countSql := generateCountSql(beginTime, endTime, latestDate, q)
	total, err := models.CountRepoStatByRawSql(countSql)
	if err != nil {
		log.Error("Can not query total count.", err)
		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error"))
		return
	}
	sql := generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, page, pageSize)

	projectsPeriodData := ProjectsPeriodData{
		RecordBeginTime: recordBeginTime.Format(DATE_FORMAT),
		PageSize:        pageSize,
		TotalPage:       getTotalPage(total, pageSize),
		TotalCount:      total,
		LastUpdatedTime: latestUpdatedTime,
		PageRecords:     models.GetRepoStatisticByRawSql(sql),
	}

	ctx.JSON(http.StatusOK, projectsPeriodData)

}

func generateSqlByType(ctx *context.Context, beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string {
	sql := ""
	if ctx.QueryTrim("type") == "all" {
		sql = generateTypeAllSql(beginTime, endTime, latestDate, q, orderBy, page, pageSize)
	} else {
		sql = generatePageSql(beginTime, endTime, latestDate, q, orderBy, page, pageSize)
	}
	return sql
}

func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) {

	recordBeginTime, err := getRecordBeginTime()
	if err != nil {
		log.Error("Can not get record begin time", err)
		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err"))
		return
	}
	beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime)
	if err != nil {
		log.Error("Parameter is wrong", err)
		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong"))
		return
	}
	q := ctx.QueryTrim("q")
	page := ctx.QueryInt("page")
	if page <= 0 {
		page = 1
	}
	pageSize := 1000
	orderBy := getOrderBy(ctx)

	_, latestDate, err := models.GetRepoStatLastUpdatedTime()
	if err != nil {
		log.Error("Can not query the last updated time.", err)
		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error"))
		return
	}

	countSql := generateCountSql(beginTime, endTime, latestDate, q)
	total, err := models.CountRepoStatByRawSql(countSql)
	if err != nil {
		log.Error("Can not query total count.", err)
		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error"))
		return
	}
	var projectAnalysis = ctx.Tr("repo.repo_stat_inspect")
	fileName, frontName := getFileName(ctx, beginTime, endTime, projectAnalysis)

	if err := os.MkdirAll(setting.RadarMap.Path, os.ModePerm); err != nil {
		ctx.Error(http.StatusBadRequest, fmt.Errorf("Failed to create dir %s: %v", setting.AvatarUploadPath, err).Error())
	}

	totalPage := getTotalPage(total, pageSize)

	f := excelize.NewFile()

	index := f.NewSheet(projectAnalysis)

	for k, v := range allProjectsPeroidHeader(ctx) {
		f.SetCellValue(projectAnalysis, k, v)
	}

	var row = 2
	for i := 0; i <= totalPage; i++ {

		pageRecords := models.GetRepoStatisticByRawSql(generateSqlByType(ctx, beginTime, endTime, latestDate, q, orderBy, i+1, pageSize))
		for _, record := range pageRecords {

			for k, v := range allProjectsPeroidValues(row, record, ctx) {
				f.SetCellValue(projectAnalysis, k, v)
			}
			row++

		}

	}
	f.SetActiveSheet(index)

	if err := f.SaveAs(fileName); err != nil {
		log.Error("Can not generate file.", err)
		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.generate_statistic_file_error"))
		return
	}

	ctx.ServeFile(fileName, url.QueryEscape(frontName))

}

func getFileName(ctx *context.Context, beginTime time.Time, endTime time.Time, projectAnalysis string) (string, string) {
	baseName := setting.RadarMap.Path + "/" + projectAnalysis + "_"

	if ctx.QueryTrim("q") != "" {
		baseName = baseName + ctx.QueryTrim("q") + "_"
	}
	if ctx.QueryTrim("type") == "all" {
		baseName = baseName + ctx.Tr("repo.all")
	} else {
		baseName = baseName + beginTime.AddDate(0, 0, -1).Format(EXCEL_DATE_FORMAT) + "_" + endTime.AddDate(0, 0, -1).Format(EXCEL_DATE_FORMAT)
	}
	frontName := baseName + ".xlsx"
	localName := baseName + "_" + strconv.FormatInt(time.Now().Unix(), 10) + ".xlsx"
	return localName, path.Base(frontName)
}

func ClearUnusedStatisticsFile() {
	fileInfos, err := ioutil.ReadDir(setting.RadarMap.Path)
	if err != nil {
		log.Warn("can not read dir: "+setting.RadarMap.Path, err)
		return
	}

	for _, fileInfo := range fileInfos {
		if !fileInfo.IsDir() && fileInfo.ModTime().Before(time.Now().AddDate(0, 0, -1)) {
			os.Remove(path.Join(setting.RadarMap.Path, fileInfo.Name()))
		}
	}

}

func allProjectsPeroidHeader(ctx *context.Context) map[string]string {

	return map[string]string{"A1": ctx.Tr("admin.repos.id"), "B1": ctx.Tr("admin.repos.projectName"), "C1": ctx.Tr("admin.repos.isPrivate"), "D1": ctx.Tr("admin.repos.openi"), "E1": ctx.Tr("admin.repos.visit"), "F1": ctx.Tr("admin.repos.download"), "G1": ctx.Tr("admin.repos.pr"), "H1": ctx.Tr("admin.repos.commit"),
		"I1": ctx.Tr("admin.repos.watches"), "J1": ctx.Tr("admin.repos.stars"), "K1": ctx.Tr("admin.repos.forks"), "L1": ctx.Tr("admin.repos.issues"), "M1": ctx.Tr("admin.repos.closedIssues"), "N1": ctx.Tr("admin.repos.contributor")}

}

func allProjectsPeroidValues(row int, rs *models.RepoStatistic, ctx *context.Context) map[string]string {
	return map[string]string{getCellName("A", row): strconv.FormatInt(rs.RepoID, 10), getCellName("B", row): constructDistinctName(rs), getCellName("C", row): getIsPrivateDisplay(rs.IsPrivate, ctx), getCellName("D", row): strconv.FormatFloat(rs.RadarTotal, 'f', 2, 64),
		getCellName("E", row): strconv.FormatInt(rs.NumVisits, 10), getCellName("F", row): strconv.FormatInt(rs.NumDownloads, 10), getCellName("G", row): strconv.FormatInt(rs.NumPulls, 10), getCellName("H", row): strconv.FormatInt(rs.NumCommits, 10),
		getCellName("I", row): strconv.FormatInt(rs.NumWatches, 10), getCellName("J", row): strconv.FormatInt(rs.NumStars, 10), getCellName("K", row): strconv.FormatInt(rs.NumForks, 10), getCellName("L", row): strconv.FormatInt(rs.NumIssues, 10),
		getCellName("M", row): strconv.FormatInt(rs.NumClosedIssues, 10), getCellName("N", row): strconv.FormatInt(rs.NumContributor, 10),
	}
}

func getCellName(col string, row int) string {
	return col + strconv.Itoa(row)
}

func constructDistinctName(rs *models.RepoStatistic) string {
	if rs.OwnerName == "" {
		return rs.Name
	}
	return rs.OwnerName + "/" + rs.Name
}

func getIsPrivateDisplay(private bool, ctx *context.Context) string {
	if private {
		return ctx.Tr("admin.repos.yes")
	} else {
		return ctx.Tr("admin.repos.no")
	}
}

func GetProjectLatestStatistics(ctx *context.Context) {
	repoId := ctx.Params(":id")
	recordBeginTime, err := getRecordBeginTime()
	if err != nil {
		log.Error("Can not get record begin time", err)
		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err"))
		return
	}
	latestUpdatedTime, latestDate, err := models.GetRepoStatLastUpdatedTime(repoId)
	repoIdInt, _ := strconv.ParseInt(repoId, 10, 64)
	repoStat, err := models.GetRepoStatisticByDateAndRepoId(latestDate, repoIdInt)
	if err != nil {
		log.Error("Can not get the repo statistics "+repoId, err)
		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.get_repo_stat_error"))
		return
	}

	repository, err := models.GetRepositoryByID(repoIdInt)
	if err != nil {
		log.Error("Can not get the repo info "+repoId, err)
		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.get_repo_info_error"))
		return
	}
	projectLatestData := ProjectLatestData{
		RecordBeginTime:  recordBeginTime.Format(DATE_FORMAT),
		CreatTime:        time.Unix(int64(repository.CreatedUnix), 0).Format(DATE_FORMAT),
		LastUpdatedTime:  latestUpdatedTime,
		OpenI:            repoStat.RadarTotal,
		Comment:          repoStat.NumComments,
		View:             repoStat.NumVisits,
		Download:         repoStat.NumDownloads,
		IssueClosedRatio: repoStat.IssueFixedRate,
		Impact:           repoStat.Impact,
		Completeness:     repoStat.Completeness,
		Liveness:         repoStat.Liveness,
		ProjectHealth:    repoStat.ProjectHealth,
		TeamHealth:       repoStat.TeamHealth,
		Growth:           repoStat.Growth,
		Description:      repository.Description,
	}

	contributors, err := models.GetTop10Contributor(repository.RepoPath())
	if err != nil {
		log.Error("can not get contributors", err)
	}
	users := make([]UserInfo, 0)

	for _, contributor := range contributors {
		mode := repository.GetCollaboratorMode(contributor.UserId)
		if mode == -1 {
			if contributor.IsAdmin {
				mode = int(models.AccessModeAdmin)
			}
			if contributor.UserId == repository.OwnerID {
				mode = int(models.AccessModeOwner)
			}
		}

		pr := models.GetPullCountByUserAndRepoId(repoIdInt, contributor.UserId)
		userInfo := UserInfo{
			User:          contributor.Committer,
			Commit:        contributor.CommitCnt,
			Mode:          mode,
			PR:            pr,
			RelAvatarLink: contributor.RelAvatarLink,
		}
		users = append(users, userInfo)

	}

	projectLatestData.Top10 = users

	ctx.JSON(http.StatusOK, projectLatestData)

}

func GetProjectPeriodStatistics(ctx *context.Context) {
	repoId := ctx.Params(":id")
	recordBeginTime, err := getRecordBeginTime()
	if err != nil {
		log.Error("Can not get record begin time", err)
		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err"))
		return
	}

	repoIdInt, _ := strconv.ParseInt(repoId, 10, 64)

	if err != nil {
		log.Error("Can not get record begin time", err)
		ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err"))
		return
	}
	beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime)
	isOpenI := ctx.QueryBool("openi")
	var repositorys []*models.RepoStatistic
	if isOpenI {
		repositorys = models.GetRepoStatisticByRawSql(generateRadarSql(beginTime, endTime, repoIdInt))
	} else {
		repositorys = models.GetRepoStatisticByRawSql(generateTargetSql(beginTime, endTime, repoIdInt))
	}
	ctx.JSON(http.StatusOK, repositorys)
}

func generateRadarSql(beginTime time.Time, endTime time.Time, repoId int64) string {
	sql := "SELECT date, impact, completeness, liveness, project_health, team_health, growth, radar_total FROM repo_statistic" +
		" where repo_id=" + strconv.FormatInt(repoId, 10) + " and created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10)

	return sql
}

func generateTargetSql(beginTime time.Time, endTime time.Time, repoId int64) string {
	sql := "SELECT date, num_visits,num_downloads,num_commits FROM repo_statistic" +
		" where repo_id=" + strconv.FormatInt(repoId, 10) + " and created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10)

	return sql
}

func generateCountSql(beginTime time.Time, endTime time.Time, latestDate string, q string) string {
	countSql := "SELECT count(*) FROM  " +
		"(SELECT repo_id FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," +
		"(SELECT repo_id,name,is_private,radar_total from public.repo_statistic where date='" + latestDate + "') B" +
		" where A.repo_id=B.repo_id"
	if q != "" {
		countSql = countSql + " and B.name like '%" + q + "%'"
	}
	return countSql
}

func generateTypeAllSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string {
	sql := "SELECT A.repo_id,name,owner_name,is_private,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM  " +
		"(SELECT repo_id,sum(num_visits) as num_visits " +
		" FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," +
		"(SELECT repo_id,name,owner_name,is_private,radar_total,num_watches,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor from public.repo_statistic where date='" + latestDate + "') B" +
		" where A.repo_id=B.repo_id"

	if q != "" {
		sql = sql + " and name like '%" + q + "%'"
	}
	sql = sql + " order by " + orderBy + " desc,repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize)
	return sql
}

func generatePageSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string {

	sql := "SELECT A.repo_id,name,owner_name,is_private,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM  " +
		"(SELECT repo_id,sum(num_watches_added) as num_watches,sum(num_visits) as num_visits, sum(num_downloads_added) as num_downloads,sum(num_pulls_added) as num_pulls,sum(num_commits_added) as num_commits,sum(num_stars_added) as num_stars,sum(num_forks_added) num_forks,sum(num_issues_added) as num_issues,sum(num_closed_issues_added) as num_closed_issues,sum(num_contributor_added) as num_contributor " +
		" FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
		" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," +
		"(SELECT repo_id,name,owner_name,is_private,radar_total from public.repo_statistic where date='" + latestDate + "') B" +
		" where A.repo_id=B.repo_id"
	if q != "" {
		sql = sql + " and B.name like '%" + q + "%'"
	}
	sql = sql + " order by " + orderBy + " desc,A.repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize)
	return sql
}

func getOrderBy(ctx *context.Context) string {
	orderBy := ""
	switch ctx.Query("sort") {
	case "openi":
		orderBy = "B.radar_total"
	case "view":
		orderBy = "A.num_visits"
	case "download":
		orderBy = "A.num_downloads"
	case "pr":
		orderBy = "A.num_pulls"
	case "commit":
		orderBy = "A.num_commits"
	case "watch":
		orderBy = "A.num_watches"
	case "star":
		orderBy = "A.num_stars"
	case "fork":
		orderBy = "A.num_forks"
	case "issue":
		orderBy = "A.num_issues"
	case "issue_closed":
		orderBy = "A.num_closed_issues"
	case "contributor":
		orderBy = "A.num_contributor"
	default:
		orderBy = "B.radar_total"
	}
	return orderBy
}

func getTimePeroid(ctx *context.Context, recordBeginTime time.Time) (time.Time, time.Time, error) {
	queryType := ctx.QueryTrim("type")
	now := time.Now()
	recordBeginTimeTemp := recordBeginTime.AddDate(0, 0, 1)

	beginTimeStr := ctx.QueryTrim("beginTime")
	endTimeStr := ctx.QueryTrim("endTime")
	var beginTime time.Time
	var endTime time.Time
	var err error
	if queryType != "" {

		if queryType == "all" {
			beginTime = recordBeginTimeTemp
			endTime = now
		} else if queryType == "yesterday" {
			endTime = now
			beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location())

		} else if queryType == "current_week" {
			beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+2) //begin from monday
			beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
			endTime = now
		} else if queryType == "current_month" {
			endTime = now
			beginTime = time.Date(endTime.Year(), endTime.Month(), 2, 0, 0, 0, 0, now.Location())
		} else if queryType == "monthly" {
			endTime = now
			beginTime = now.AddDate(0, -1, 1)
			beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())

		} else if queryType == "current_year" {
			endTime = now
			beginTime = time.Date(endTime.Year(), 1, 2, 0, 0, 0, 0, now.Location())

		} else if queryType == "last_month" {

			lastMonthTime := now.AddDate(0, -1, 0)
			beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 2, 0, 0, 0, 0, now.Location())
			endTime = time.Date(now.Year(), now.Month(), 2, 0, 0, 0, 0, now.Location())

		} else {
			return now, now, fmt.Errorf("The value of type parameter is wrong.")

		}

	} else {
		if beginTimeStr == "" || endTimeStr == "" {
			//如果查询类型和开始时间结束时间都未设置，按queryType=all处理
			beginTime = recordBeginTimeTemp
			endTime = now

		} else {

			beginTime, err = time.Parse("2006-01-02", beginTimeStr)
			if err != nil {
				return now, now, err
			}

			endTime, err = time.Parse("2006-01-02", endTimeStr)
			if err != nil {
				return now, now, err
			}

			beginTime = beginTime.AddDate(0, 0, 1)
			endTime = endTime.AddDate(0, 0, 1)
		}

	}

	if beginTime.Before(recordBeginTimeTemp) {
		beginTime = recordBeginTimeTemp
	}

	return beginTime, endTime, nil

}

func getRecordBeginTime() (time.Time, error) {

	return time.Parse(DATE_FORMAT, setting.RadarMap.RecordBeginTime)
}

func getTotalPage(total int64, pageSize int) int {

	another := 0
	if int(total)%pageSize != 0 {
		another = 1
	}
	return int(total)/pageSize + another

}