From 4eb86a5d9cd367f3dabfac6ed51320a8864185a7 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Fri, 22 Apr 2022 14:22:47 +0800 Subject: [PATCH] =?UTF-8?q?fix-1937=E5=90=8E=E7=AB=AF=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- models/repo.go | 5 + models/summary_statistic.go | 34 +++ options/locale/locale_en-US.ini | 11 + options/locale/locale_zh-CN.ini | 12 + routers/api/v1/api.go | 3 + routers/api/v1/repo/repo_dashbord.go | 313 +++++++++++++++++++++++++ routers/repo/repo_summary_statistic.go | 7 + 7 files changed, 385 insertions(+) diff --git a/models/repo.go b/models/repo.go index 25bfb4a74..db2694617 100755 --- a/models/repo.go +++ b/models/repo.go @@ -1554,6 +1554,11 @@ func GetAllMirrorRepositoriesCount() (int64, error) { return x.Where("is_mirror = ?", true).Count(repo) } +func GetAllOrgRepositoriesCount() (int64, error) { + repo := new(Repository) + return x.Table("repository").Join("INNER", []string{"\"user\"", "u"}, "repository.owner_id = u.id and u.type=1").Count(repo) +} + func GetAllForkRepositoriesCount() (int64, error) { repo := new(Repository) return x.Where("is_fork = ?", true).Count(repo) diff --git a/models/summary_statistic.go b/models/summary_statistic.go index e5cf54b75..7787aa292 100644 --- a/models/summary_statistic.go +++ b/models/summary_statistic.go @@ -2,6 +2,8 @@ package models import ( "fmt" + "strconv" + "time" "code.gitea.io/gitea/modules/timeutil" ) @@ -45,6 +47,7 @@ type SummaryStatistic struct { NumRepoFork int64 `xorm:"NOT NULL DEFAULT 0"` NumRepoMirror int64 `xorm:"NOT NULL DEFAULT 0"` NumRepoSelf int64 `xorm:"NOT NULL DEFAULT 0"` + NumRepoOrg int64 `xorm:"NOT NULL DEFAULT 0"` CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` } @@ -69,6 +72,37 @@ func DeleteSummaryStatisticDaily(date string) error { return nil } +func GetLatest2SummaryStatistic() ([]*SummaryStatistic, error) { + summaryStatistics := make([]*SummaryStatistic, 0) + err := xStatistic.Desc("created_unix").Limit(2).Find(&summaryStatistics) + return summaryStatistics, err +} + +func GetSummaryStatisticByTimeCount(beginTime time.Time, endTime time.Time) (int64, error) { + summaryStatistics := new(SummaryStatistic) + total, err := xStatistic.Asc("created_unix").Where("created_unix>=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10)).Count(summaryStatistics) + return total, err +} + +func GetSummaryStatisticByDateCount(dates []string) (int64, error) { + summaryStatistics := new(SummaryStatistic) + total, err := xStatistic.Asc("created_unix").In("date", dates).Count(summaryStatistics) + return total, err +} + +func GetSummaryStatisticByTime(beginTime time.Time, endTime time.Time, page int, pageSize int) ([]*SummaryStatistic, error) { + summaryStatistics := make([]*SummaryStatistic, 0) + err := xStatistic.Asc("created_unix").Limit(pageSize+1, (page-1)*pageSize).Where("created_unix>=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10)).Find(&summaryStatistics) + + return summaryStatistics, err +} + +func GetSummaryStatisticByDates(dates []string, page int, pageSize int) ([]*SummaryStatistic, error) { + summaryStatistics := make([]*SummaryStatistic, 0) + err := xStatistic.Asc("created_unix").In("date", dates).Limit(pageSize+1, (page-1)*pageSize).Find(&summaryStatistics) + return summaryStatistics, err +} + func InsertSummaryStatistic(summaryStatistic *SummaryStatistic) (int64, error) { return xStatistic.Insert(summaryStatistic) } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index f53bea7b1..6d1ac24c7 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -937,6 +937,15 @@ model_manager = Model model_noright=No right model_rename=Duplicate model name, please modify model name. +date=Date +repo_add=Project Increment +repo_total=Project Total +repo_public_add=Public Project Increment +repo_private_add=Private Project Increment +repo_fork_add=Fork Project Increment +repo_mirror_add=Mirror Project Increment +repo_self_add=Custom Project Increment + debug=Debug debug_again=Restart stop=Stop @@ -1001,7 +1010,9 @@ get_repo_stat_error=Can not get the statistics of the repository. get_repo_info_error=Can not get the information of the repository. generate_statistic_file_error=Failed to generate file. repo_stat_inspect=ProjectAnalysis +repo_stat_develop=ProjectGrowthAnalysis all=All +current_year=Current_Year computing.all = All computing.Introduction=Introduction diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index c82347d5e..2232c36a1 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -938,6 +938,16 @@ model_manager = 模型 model_noright=无权限操作 model_rename=模型名称重复，请修改模型名称 + +date=日期 +repo_add=新增项目 +repo_total=累计项目 +repo_public_add=新增公开项目 +repo_private_add=新增私有项目 +repo_fork_add=新增派生项目 +repo_mirror_add=新增镜像项目 +repo_self_add=新增自建项目 + debug=调试 debug_again=再次调试 stop=停止 @@ -1009,7 +1019,9 @@ get_repo_stat_error=查询当前仓库的统计信息失败。 get_repo_info_error=查询当前仓库信息失败。 generate_statistic_file_error=生成文件失败。 repo_stat_inspect=项目分析 +repo_stat_develop=项目增长趋势 all=所有 +current_year=今年 computing.all=全部 computing.Introduction=简介 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 2b070a4b8..3cc1a6693 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -535,6 +535,9 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/restoreFork", repo.RestoreForkNumber) m.Get("/downloadAll", repo.ServeAllProjectsPeriodStatisticsFile) m.Get("/downloadAllOpenI", repo.ServeAllProjectsOpenIStatisticsFile) + m.Get("/summary", repo.GetLatestProjectsSummaryData) + m.Get("/summary/period", repo.GetProjectsSummaryData) + m.Get("/summary/download", repo.GetProjectsSummaryDataFile) m.Group("/project", func() { m.Get("", repo.GetAllProjectsPeriodStatistics) m.Get("/numVisit", repo.ProjectNumVisit) diff --git a/routers/api/v1/repo/repo_dashbord.go b/routers/api/v1/repo/repo_dashbord.go index 2c42f8a16..7ec315fd9 100644 --- a/routers/api/v1/repo/repo_dashbord.go +++ b/routers/api/v1/repo/repo_dashbord.go @@ -20,8 +20,10 @@ import ( const DEFAULT_PAGE_SIZE = 10 const DATE_FORMAT = "2006-01-02" +const MONTH_FORMAT = "2006-01" const EXCEL_DATE_FORMAT = "20060102" const CREATE_TIME_FORMAT = "2006/01/02 15:04:05" +const UPDATE_TIME_FORMAT = "2006-01-02 15:04:05" type ProjectsPeriodData struct { RecordBeginTime string `json:"recordBeginTime"` @@ -60,6 +62,40 @@ type ProjectLatestData struct { Top10 []UserInfo `json:"top10"` } +type ProjectSummaryBaseData struct { + NumReposAdd int64 `json:"numReposAdd"` + NumRepoPublicAdd int64 `json:"numRepoPublicAdd"` + NumRepoPrivateAdd int64 `json:"numRepoPrivateAdd"` + NumRepoForkAdd int64 `json:"numRepoForkAdd"` + NumRepoMirrorAdd int64 `json:"numRepoMirrorAdd"` + NumRepoSelfAdd int64 `json:"numRepoSelfAdd"` + NumRepos int64 `json:"numRepos"` + CreatTime string `json:"creatTime"` +} + +type ProjectSummaryData struct { + ProjectSummaryBaseData + NumRepoPublic int64 `json:"numRepoPublic"` + NumRepoPrivate int64 `json:"numRepoPrivate"` + NumRepoFork int64 `json:"numRepoFork"` + NumRepoMirror int64 `json:"numRepoMirror"` + NumRepoSelf int64 `json:"numRepoSelf"` + + NumRepoOrgAdd int64 `json:"numRepoOrgAdd"` + NumRepoNotOrgAdd int64 `json:"numRepoNotOrgAdd"` + + NumRepoOrg int64 `json:"numRepoOrg"` + NumRepoNotOrg int64 `json:"numRepoNotOrg"` +} + +type ProjectSummaryPeriodData struct { + RecordBeginTime string `json:"recordBeginTime"` + PageSize int `json:"pageSize"` + TotalPage int `json:"totalPage"` + TotalCount int64 `json:"totalCount"` + PageRecords []*ProjectSummaryBaseData `json:"pageRecords"` +} + func RestoreForkNumber(ctx *context.Context) { repos, err := models.GetAllRepositories() if err != nil { @@ -73,6 +109,144 @@ func RestoreForkNumber(ctx *context.Context) { ctx.JSON(http.StatusOK, struct{}{}) } +func GetLatestProjectsSummaryData(ctx *context.Context) { + stat, err := models.GetLatest2SummaryStatistic() + data := ProjectSummaryData{} + if err == nil && len(stat) > 0 { + data.NumRepos = stat[0].NumRepos + data.NumRepoOrg = stat[0].NumRepoOrg + data.NumRepoNotOrg = stat[0].NumRepos - stat[0].NumRepoOrg + data.NumRepoFork = stat[0].NumRepoFork + data.NumRepoMirror = stat[0].NumRepoMirror + data.NumRepoSelf = stat[0].NumRepoSelf + data.NumRepoPrivate = stat[0].NumRepoPrivate + data.NumRepoPublic = stat[0].NumRepoPublic + data.CreatTime = stat[0].CreatedUnix.Format(UPDATE_TIME_FORMAT) + if len(stat) == 2 { + data.NumReposAdd = stat[0].NumRepos - stat[1].NumRepos + data.NumRepoOrgAdd = stat[0].NumRepoOrg - stat[1].NumRepoOrg + data.NumRepoNotOrgAdd = (stat[0].NumRepos - stat[0].NumRepoOrg) - (stat[1].NumRepos - stat[1].NumRepoOrg) + data.NumRepoForkAdd = stat[0].NumRepoFork - stat[1].NumRepoFork + data.NumRepoMirrorAdd = stat[0].NumRepoMirror - stat[1].NumRepoMirror + data.NumRepoSelfAdd = stat[0].NumRepoSelf - stat[1].NumRepoSelf + data.NumRepoPrivateAdd = stat[0].NumRepoPrivate - stat[1].NumRepoPrivate + data.NumRepoPublicAdd = stat[0].NumRepoPublic - stat[1].NumRepoPublic + } + } + ctx.JSON(200, data) +} + +func GetProjectsSummaryData(ctx *context.Context) { + + var datas = make([]*ProjectSummaryBaseData, 0) + + recordBeginTime, err := getRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) + + beginTime = beginTime.AddDate(0, 0, -1) + + queryType := ctx.QueryTrim("type") + + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := ctx.QueryInt("pagesize") + if pageSize <= 0 { + pageSize = DEFAULT_PAGE_SIZE + } + var count int64 + + if queryType == "all" || queryType == "current_year" { + dates := getEndOfMonthDates(beginTime, endTime) + count, _ = models.GetSummaryStatisticByDateCount(dates) + stats, err := models.GetSummaryStatisticByDates(dates, page, pageSize) + if err != nil { + log.Warn("can not get summary data", err) + } else { + + for i, v := range stats { + if i == 0 { + continue + } + data := ProjectSummaryBaseData{} + setStatisticsData(&data, v, stats[i-1]) + createTime, _ := time.Parse(DATE_FORMAT, v.Date) + data.CreatTime = createTime.Format(MONTH_FORMAT) + datas = append(datas, &data) + } + } + + } else { + count, _ = models.GetSummaryStatisticByTimeCount(beginTime, endTime) + stats, err := models.GetSummaryStatisticByTime(beginTime, endTime, page, pageSize) + if err != nil { + log.Warn("can not get summary data", err) + } else { + + for i, v := range stats { + if i == 0 { + continue + } + data := ProjectSummaryBaseData{} + setStatisticsData(&data, v, stats[i-1]) + data.CreatTime = v.Date + datas = append(datas, &data) + } + } + + } + projectSummaryPeriodData := ProjectSummaryPeriodData{ + TotalCount: count - 1, + TotalPage: getTotalPage(count-1, pageSize), + RecordBeginTime: recordBeginTime.Format(DATE_FORMAT), + PageSize: pageSize, + PageRecords: datas, + } + + ctx.JSON(200, projectSummaryPeriodData) + +} + +func setStatisticsData(data *ProjectSummaryBaseData, v *models.SummaryStatistic, stats *models.SummaryStatistic) { + data.NumReposAdd = v.NumRepos - stats.NumRepos + data.NumRepoPublicAdd = v.NumRepoPublic - stats.NumRepoPublic + data.NumRepoPrivateAdd = v.NumRepoPrivate - stats.NumRepoPrivate + data.NumRepoMirrorAdd = v.NumRepoMirror - stats.NumRepoMirror + data.NumRepoForkAdd = v.NumRepoFork - stats.NumRepoFork + data.NumRepoSelfAdd = v.NumRepoSelf - stats.NumRepoSelf + + data.NumRepos = v.NumRepos +} + +func getEndOfMonthDates(beginTime time.Time, endTime time.Time) []string { + var dates = []string{} + date := endOfMonth(beginTime.AddDate(0, -1, 0)) + dates = append(dates, date.Format(DATE_FORMAT)) + + tempDate := endOfMonth(beginTime) + + for { + if tempDate.Before(endTime) { + dates = append(dates, tempDate.Format(DATE_FORMAT)) + tempDate = endOfMonth(tempDate.AddDate(0, 0, 1)) + } else { + break + } + } + + return dates +} + +func endOfMonth(date time.Time) time.Time { + return date.AddDate(0, 1, -date.Day()) +} + func GetAllProjectsPeriodStatistics(ctx *context.Context) { recordBeginTime, err := getRecordBeginTime() @@ -210,6 +384,118 @@ func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) { } +func GetProjectsSummaryDataFile(ctx *context.Context) { + + recordBeginTime, err := getRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) + beginTime = beginTime.AddDate(0, 0, -1) + if err != nil { + log.Error("Parameter is wrong", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) + return + } + + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := 100 + + if err != nil { + log.Error("Can not query the last updated time.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error")) + return + } + + var projectAnalysis = ctx.Tr("repo.repo_stat_develop") + fileName := getSummaryFileName(ctx, beginTime, endTime, projectAnalysis) + + f := excelize.NewFile() + + index := f.NewSheet(projectAnalysis) + f.DeleteSheet("Sheet1") + + for k, v := range allProjectsPeriodSummaryHeader(ctx) { + f.SetCellValue(projectAnalysis, k, v) + } + + var total int64 + queryType := ctx.QueryTrim("type") + row := 2 + + if queryType == "all" || queryType == "current_year" { + dates := getEndOfMonthDates(beginTime, endTime) + total, _ = models.GetSummaryStatisticByDateCount(dates) + totalPage := getTotalPage(total, pageSize) + + for i := 0; i < totalPage; i++ { + + stats, err := models.GetSummaryStatisticByDates(dates, i+1, pageSize) + if err != nil { + log.Warn("can not get summary data", err) + } else { + for j, v := range stats { + if j == 0 { + continue + } + data := ProjectSummaryBaseData{} + setStatisticsData(&data, v, stats[j-1]) + createTime, _ := time.Parse(DATE_FORMAT, v.Date) + data.CreatTime = createTime.Format(MONTH_FORMAT) + + for k, v := range allProjectsPeriodSummaryValues(row, &data, ctx) { + f.SetCellValue(projectAnalysis, k, v) + } + row++ + } + + } + + } + + } else { + total, _ = models.GetSummaryStatisticByTimeCount(beginTime, endTime) + totalPage := getTotalPage(total, pageSize) + + for i := 0; i < totalPage; i++ { + + stats, err := models.GetSummaryStatisticByTime(beginTime, endTime, i+1, pageSize) + if err != nil { + log.Warn("can not get summary data", err) + } else { + for j, v := range stats { + if j == 0 { + continue + } + data := ProjectSummaryBaseData{} + setStatisticsData(&data, v, stats[j-1]) + data.CreatTime = v.Date + + for k, v := range allProjectsPeriodSummaryValues(row, &data, ctx) { + f.SetCellValue(projectAnalysis, k, v) + } + row++ + } + + } + + } + } + + f.SetActiveSheet(index) + + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+url.QueryEscape(fileName)) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + + f.WriteTo(ctx.Resp) + +} + func ServeAllProjectsOpenIStatisticsFile(ctx *context.Context) { page := ctx.QueryInt("page") @@ -290,6 +576,20 @@ func getFileName(ctx *context.Context, beginTime time.Time, endTime time.Time, p return frontName } +func getSummaryFileName(ctx *context.Context, beginTime time.Time, endTime time.Time, projectAnalysis string) string { + baseName := projectAnalysis + "_" + + if ctx.QueryTrim("type") == "all" { + baseName = baseName + ctx.Tr("repo.all") + } else if ctx.QueryTrim("type") == "current_year" { + baseName = baseName + ctx.Tr("repo.current_year") + } else { + baseName = baseName + beginTime.Format(EXCEL_DATE_FORMAT) + "_" + endTime.AddDate(0, 0, -1).Format(EXCEL_DATE_FORMAT) + } + frontName := baseName + ".xlsx" + return frontName +} + func allProjectsPeroidHeader(ctx *context.Context) map[string]string { return map[string]string{"A1": ctx.Tr("admin.repos.id"), "B1": ctx.Tr("admin.repos.projectName"), "C1": ctx.Tr("repo.owner"), "D1": ctx.Tr("admin.repos.isPrivate"), "E1": ctx.Tr("admin.repos.openi"), "F1": ctx.Tr("admin.repos.visit"), "G1": ctx.Tr("admin.repos.download"), "H1": ctx.Tr("admin.repos.pr"), "I1": ctx.Tr("admin.repos.commit"), @@ -297,6 +597,19 @@ func allProjectsPeroidHeader(ctx *context.Context) map[string]string { } +func allProjectsPeriodSummaryHeader(ctx *context.Context) map[string]string { + + return map[string]string{"A1": ctx.Tr("repo.date"), "B1": ctx.Tr("repo.repo_add"), "C1": ctx.Tr("repo.repo_total"), "D1": ctx.Tr("repo.repo_public_add"), "E1": ctx.Tr("repo.repo_private_add"), "F1": ctx.Tr("repo.repo_self_add"), "G1": ctx.Tr("repo.repo_fork_add"), "H1": ctx.Tr("repo.repo_mirror_add")} + +} + +func allProjectsPeriodSummaryValues(row int, rs *ProjectSummaryBaseData, ctx *context.Context) map[string]string { + + return map[string]string{getCellName("A", row): rs.CreatTime, getCellName("B", row): strconv.FormatInt(rs.NumReposAdd, 10), getCellName("C", row): strconv.FormatInt(rs.NumRepos, 10), getCellName("D", row): strconv.FormatInt(rs.NumRepoPublicAdd, 10), getCellName("E", row): strconv.FormatInt(rs.NumRepoPrivateAdd, 10), + getCellName("F", row): strconv.FormatInt(rs.NumRepoSelfAdd, 10), getCellName("G", row): strconv.FormatInt(rs.NumRepoForkAdd, 10), getCellName("H", row): strconv.FormatInt(rs.NumRepoMirrorAdd, 10), + } +} + func allProjectsPeroidValues(row int, rs *models.RepoStatistic, ctx *context.Context) map[string]string { return map[string]string{getCellName("A", row): strconv.FormatInt(rs.RepoID, 10), getCellName("B", row): rs.DisplayName(), getCellName("C", row): rs.OwnerName, getCellName("D", row): getBoolDisplay(rs.IsPrivate, ctx), getCellName("E", row): strconv.FormatFloat(rs.RadarTotal, 'f', 2, 64), getCellName("F", row): strconv.FormatInt(rs.NumVisits, 10), getCellName("G", row): strconv.FormatInt(rs.NumDownloads, 10), getCellName("H", row): strconv.FormatInt(rs.NumPulls, 10), getCellName("I", row): strconv.FormatInt(rs.NumCommits, 10), diff --git a/routers/repo/repo_summary_statistic.go b/routers/repo/repo_summary_statistic.go index 3af31737c..65ba2cf0b 100644 --- a/routers/repo/repo_summary_statistic.go +++ b/routers/repo/repo_summary_statistic.go @@ -60,6 +60,12 @@ func SummaryStatisticDaily(date string) { } selfRepositoryNumber := repositoryNumer - mirrorRepositoryNumber - forkRepositoryNumber + organizationRepoNumber, err := models.GetAllOrgRepositoriesCount() + if err != nil { + log.Error("can not get org repository number", err) + organizationRepoNumber = 0 + } + //repository size repositorySize, err := models.GetAllRepositoriesSize() if err != nil { @@ -99,6 +105,7 @@ func SummaryStatisticDaily(date string) { NumRepoPrivate: privateRepositoryNumer, NumRepoPublic: publicRepositoryNumer, NumRepoSelf: selfRepositoryNumber, + NumRepoOrg: organizationRepoNumber, NumRepoBigModel: topicsCount[0], NumRepoAI: topicsCount[1], NumRepoVision: topicsCount[2],