diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index ed9829cef..eac081a8f 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/migrations" repository_service "code.gitea.io/gitea/modules/repository" + api_repo "code.gitea.io/gitea/routers/api/v1/repo" "code.gitea.io/gitea/routers/repo" mirror_service "code.gitea.io/gitea/services/mirror" ) @@ -195,6 +196,17 @@ func registerHandleUserStatistic() { }) } +func registerHandleClearRepoStatisticFile() { + RegisterTaskFatal("handle_repo_clear_statistic_file", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@daily", + }, func(ctx context.Context, _ *models.User, _ Config) error { + api_repo.ClearUnusedStatisticsFile() + return nil + }) +} + func initBasicTasks() { registerUpdateMirrorTask() registerRepoHealthCheck() diff --git a/modules/setting/setting.go b/modules/setting/setting.go index c64a96ac6..9f31612b6 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -547,6 +547,7 @@ var ( GrowthCommit float64 GrowthComments float64 RecordBeginTime string + Path string }{} ) @@ -1328,6 +1329,7 @@ func SetRadarMapConfig() { RadarMap.GrowthCommit = sec.Key("growth_commit").MustFloat64(0.2) RadarMap.GrowthComments = sec.Key("growth_comments").MustFloat64(0.2) RadarMap.RecordBeginTime = sec.Key("record_beigin_time").MustString("2021-11-05") + RadarMap.Path = sec.Key("PATH").MustString("data/projectborad") } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 5444bafd1..bc71693b8 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -2159,6 +2159,19 @@ repos.stars = Stars repos.forks = Forks repos.issues = Issues repos.size = Size +repos.id=ID +repos.projectName=Project Name +repos.isPrivate=Private +repos.openi=OpenI +repos.visit=Visit +repos.download=Code Download +repos.pr=PR +repos.commit=Commit +repos.closedIssues=Closed Issue +repos.contributor=Contributor +repos.yes=Yes +repos.no=No + datasets.dataset_manage_panel= Dataset Manage datasets.owner=Owner diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index ef23f5b69..a6d40355b 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -2162,6 +2162,19 @@ repos.stars=点赞数 repos.forks=派生数 repos.issues=任务数 repos.size=大小 +repos.id=ID +repos.projectName=项目名称 +repos.isPrivate=私有 +repos.openi=OpenI指数 +repos.visit=浏览量 +repos.download=代码下载量 +repos.pr=PR数 +repos.commit=Commit数 +repos.closedIssues=已解决任务数 +repos.contributor=贡献者数 +repos.yes=是 +repos.no=否 + datasets.dataset_manage_panel=数据集管理 datasets.owner=所有者 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 8b068d612..b2e701998 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -531,6 +531,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/project", func() { m.Get("", adminReq, repo.GetAllProjectsPeriodStatistics) + m.Get("/download", adminReq, repo.ServeAllProjectsPeriodStatisticsFile) m.Group("/:id", func() { m.Get("", adminReq, repo.GetProjectLatestStatistics) m.Get("/period", adminReq, repo.GetProjectPeriodStatistics) diff --git a/routers/api/v1/repo/repo_dashbord.go b/routers/api/v1/repo/repo_dashbord.go index 743f366e3..4555d8a5c 100644 --- a/routers/api/v1/repo/repo_dashbord.go +++ b/routers/api/v1/repo/repo_dashbord.go @@ -1,8 +1,12 @@ package repo import ( + "encoding/csv" "fmt" + "io/ioutil" "net/http" + "os" + "path" "strconv" "time" @@ -117,6 +121,125 @@ func GetAllProjectsPeriodStatistics(ctx *context.Context) { } +func ServeAllProjectsPeriodStatisticsFile(ctx *context.Context) { + + recordBeginTime, err := getRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + beginTime, endTime, err := getTimePeroid(ctx, recordBeginTime) + if err != nil { + log.Error("Parameter is wrong", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) + return + } + q := ctx.QueryTrim("q") + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := 1000 + orderBy := getOrderBy(ctx) + + _, latestDate, err := models.GetRepoStatLastUpdatedTime() + if err != nil { + log.Error("Can not query the last updated time.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.last_update_time_error")) + return + } + + countSql := generateCountSql(beginTime, endTime, latestDate, q) + total, err := models.CountRepoStatByRawSql(countSql) + if err != nil { + log.Error("Can not query total count.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.total_count_get_error")) + return + } + + fileName := getFileName(ctx, beginTime, endTime) + + if err := os.MkdirAll(setting.RadarMap.Path, os.ModePerm); err != nil { + ctx.Error(http.StatusBadRequest, fmt.Errorf("Failed to create dir %s: %v", setting.AvatarUploadPath, err).Error()) + } + + totalPage := getTotalPage(total, pageSize) + + f, e := os.Create(fileName) + defer f.Close() + if e != nil { + log.Warn("Failed to create file", e) + } + writer := csv.NewWriter(f) + writer.Write(allProjectsPeroidHeader(ctx)) + for i := 0; i <= totalPage; i++ { + + pageRecords := models.GetRepoStatisticByRawSql(generatePageSql(beginTime, endTime, latestDate, q, orderBy, i+1, pageSize)) + for _, record := range pageRecords { + e = writer.Write(allProjectsPeroidValues(record, ctx)) + if e != nil { + log.Warn("Failed to write record", e) + } + } + writer.Flush() + + } + + ctx.ServeFile(fileName) + +} + +func getFileName(ctx *context.Context, beginTime time.Time, endTime time.Time) string { + baseName := setting.RadarMap.Path + "/" + if ctx.QueryTrim("type") != "" { + baseName = baseName + ctx.QueryTrim("type") + "_" + } + if ctx.QueryTrim("q") != "" { + baseName = baseName + ctx.QueryTrim("q") + "_" + } + baseName = baseName + beginTime.Format(DATE_FORMAT) + "_to_" + endTime.Format(DATE_FORMAT) + "_" + strconv.FormatInt(time.Now().Unix(), 10) + ".csv" + return baseName +} + +func ClearUnusedStatisticsFile() { + fileInfos, err := ioutil.ReadDir(setting.RadarMap.Path) + if err != nil { + log.Warn("can not read dir: "+setting.RadarMap.Path, err) + return + } + + for _, fileInfo := range fileInfos { + if !fileInfo.IsDir() && fileInfo.ModTime().Before(time.Now().AddDate(0, 0, -1)) { + os.Remove(path.Join(setting.RadarMap.Path, fileInfo.Name())) + } + } + +} + +func allProjectsPeroidHeader(ctx *context.Context) []string { + + return []string{ctx.Tr("repos.id"), ctx.Tr("repos.projectName"), ctx.Tr("repos.isPrivate"), ctx.Tr("repos.openi"), ctx.Tr("repos.visit"), ctx.Tr("repos.download"), ctx.Tr("repos.pr"), ctx.Tr("repos.commit"), + ctx.Tr("repos.watches"), ctx.Tr("repos.stars"), ctx.Tr("repos.forks"), ctx.Tr("repos.issues"), ctx.Tr("repos.closedIssues"), ctx.Tr("repos.contributor")} + +} + +func allProjectsPeroidValues(rs *models.RepoStatistic, ctx *context.Context) []string { + return []string{strconv.FormatInt(rs.RepoID, 10), rs.Name, getIsPrivateDisplay(rs.IsPrivate, ctx), strconv.FormatFloat(rs.RadarTotal, 'f', 2, 64), + strconv.FormatInt(rs.NumVisits, 10), strconv.FormatInt(rs.NumDownloads, 10), strconv.FormatInt(rs.NumPulls, 10), strconv.FormatInt(rs.NumCommits, 10), + strconv.FormatInt(rs.NumWatches, 10), strconv.FormatInt(rs.NumStars, 10), strconv.FormatInt(rs.NumForks, 10), strconv.FormatInt(rs.NumIssues, 10), + strconv.FormatInt(rs.NumClosedIssues, 10), strconv.FormatInt(rs.NumContributor, 10), + } +} + +func getIsPrivateDisplay(private bool, ctx *context.Context) string { + if private { + return ctx.Tr("repos.yes") + } else { + return ctx.Tr("repos.no") + } +} + func GetProjectLatestStatistics(ctx *context.Context) { repoId := ctx.Params(":id") recordBeginTime, err := getRecordBeginTime()