Merge branch 'V20211018' into web

custom/conf/app.ini.sample

@@ -385,6 +385,17 @@ CONN_MAX_LIFETIME = 3s
 ; Database maximum number of open connections, default is 0 meaning no maximum
 MAX_OPEN_CONNS = 0
 
+[database_statistic]
+DB_TYPE  = postgres
+HOST     = 127.0.0.1:5432
+NAME     = statistic
+USER     = 
+PASSWD   = 
+SCHEMA   =
+SSL_MODE = disable
+CHARSET  = utf8
+PATH     = 
+
 [indexer]
 ; Issue indexer type, currently support: bleve, db or elasticsearch, default is bleve
 ISSUE_INDEXER_TYPE = bleve

models/attachment.go

@@ -464,3 +464,12 @@ func CanDelAttachment(isSigned bool, user *User, attach *Attachment) bool {
 	}
 	return false
 }
+
+func GetAttachmentSizeByDatasetID(datasetID int64) (int64, error) {
+	total, err := x.Where("dataset_id = ?", datasetID).SumInt(&Attachment{}, "size")
+	if err != nil {
+		return 0, err
+	}
+
+	return total, nil
+}

models/issue_comment.go

@@ -1016,3 +1016,19 @@ func UpdateCommentsMigrationsByType(tp structs.GitServiceType, originalAuthorID
 		})
 	return err
 }
+
+func GetCommentCountByRepoID(repoID int64) (int64, error) {
+	//sql := fmt.Sprintf("select count(1) from comment where issue_id in (select id from issue where repo_id = %d) and type = %d;", repoID, CommentTypeComment)
+	//res, err := x.Query(sql)
+	//if err != nil {
+	//	return 0, err
+	//}
+	//return int64(binary.BigEndian.Uint64(res[0]["count"])), nil
+
+	total, err := x.Where("issue_id in (select id from issue where repo_id = ?) and type = ?", repoID, CommentTypeComment).Count(&Comment{})
+	if err != nil {
+		return 0, err
+	}
+
+	return total, nil
+}

models/models.go

@@ -136,7 +136,7 @@ func init() {
 	)
 
 	tablesStatistic = append(tablesStatistic,
-		new(FileChunk),
+		new(RepoStatistic),
 		new(UserBusinessAnalysis),
 	)
 

models/repo_statistic.go

@@ -0,0 +1,60 @@
+package models
+
+import (
+	"code.gitea.io/gitea/modules/timeutil"
+	"fmt"
+)
+
+// RepoStatistic statistic info of all repository
+type RepoStatistic struct {
+	ID              int64  `xorm:"pk autoincr"`
+	RepoID          int64  `xorm:"unique(s) NOT NULL"`
+	Date            string `xorm:"unique(s) NOT NULL"`
+	NumWatches      int64  `xorm:"NOT NULL DEFAULT 0"`
+	NumStars        int64  `xorm:"NOT NULL DEFAULT 0"`
+	NumForks        int64  `xorm:"NOT NULL DEFAULT 0"`
+	NumDownloads    int64  `xorm:"NOT NULL DEFAULT 0"`
+	NumComments     int64  `xorm:"NOT NULL DEFAULT 0"`
+	NumVisits       int64  `xorm:"NOT NULL DEFAULT 0"`
+	NumClosedIssues int64  `xorm:"NOT NULL DEFAULT 0"`
+	NumVersions     int64  `xorm:"NOT NULL DEFAULT 0"`
+	//develop months
+	NumDevMonths      int64   `xorm:"NOT NULL DEFAULT 0"`
+	RepoSize          int64   `xorm:"NOT NULL DEFAULT 0"`
+	DatasetSize       int64   `xorm:"NOT NULL DEFAULT 0"`
+	NumModels         int64   `xorm:"NOT NULL DEFAULT 0"`
+	NumWikiViews      int64   `xorm:"NOT NULL DEFAULT 0"`
+	NumCommits        int64   `xorm:"NOT NULL DEFAULT 0"`
+	NumIssues         int64   `xorm:"NOT NULL DEFAULT 0"`
+	NumPulls          int64   `xorm:"NOT NULL DEFAULT 0"`
+	IssueFixedRate    float32 `xorm:"NOT NULL"`
+	NumContributor    int64   `xorm:"NOT NULL DEFAULT 0"`
+	NumKeyContributor int64   `xorm:"NOT NULL DEFAULT 0"`
+
+	CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
+	UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
+}
+
+func DeleteRepoStatDaily(date string) error {
+	sess := xStatistic.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return fmt.Errorf("Begin: %v", err)
+	}
+
+	if _, err := sess.Where("date = ?", date).Delete(&RepoStatistic{}); err != nil {
+		return fmt.Errorf("Delete: %v", err)
+	}
+
+	if err := sess.Commit(); err != nil {
+		sess.Close()
+		return fmt.Errorf("Commit: %v", err)
+	}
+
+	sess.Close()
+	return nil
+}
+
+func InsertRepoStat(repoStat *RepoStatistic) (int64, error) {
+	return xStatistic.Insert(repoStat)
+}

modules/cron/tasks_basic.go

@@ -163,6 +163,28 @@ func registerHandleBlockChainUnSuccessCommits() {
 	})
 }
 
+func registerHandleRepoStatistic() {
+	RegisterTaskFatal("handle_repo_statistic", &BaseConfig{
+		Enabled:    true,
+		RunAtStart: false,
+		Schedule:   "@daily",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		repo.RepoStatisticAuto()
+		return nil
+	})
+}
+
+func registerHandleUserStatistic() {
+	RegisterTaskFatal("handle_user_statistic", &BaseConfig{
+		Enabled:    true,
+		RunAtStart: false,
+		Schedule:   "@daily",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		repo.TimingCountData()
+		return nil
+	})
+}
+
 func initBasicTasks() {
 	registerUpdateMirrorTask()
 	registerRepoHealthCheck()
@@ -177,4 +199,7 @@ func initBasicTasks() {
 	registerHandleBlockChainUnSuccessRepos()
 	registerHandleBlockChainMergedPulls()
 	registerHandleBlockChainUnSuccessCommits()
+
+	registerHandleRepoStatistic()
+	registerHandleUserStatistic()
 }

routers/private/internal.go

@@ -42,7 +42,8 @@ func RegisterRoutes(m *macaron.Macaron) {
 		m.Post("/manager/shutdown", Shutdown)
 		m.Post("/manager/restart", Restart)
 		m.Post("/manager/flush-queues", bind(private.FlushOptions{}), FlushQueues)
-		m.Post("/cmd/update_all_repo_commit_cnt", UpdateAllRepoCommitCnt)
+		m.Post("/tool/update_all_repo_commit_cnt", UpdateAllRepoCommitCnt)
+		m.Post("/tool/repo_stat", RepoStatisticManually)
 
 	}, CheckInternalToken)
 }

routers/private/cmd.go → routers/private/tool.go

@@ -5,11 +5,13 @@
 package private
 
 import (
-	"gitea.com/macaron/macaron"
 	"net/http"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/routers/repo"
+
+	"gitea.com/macaron/macaron"
 )
 
 func UpdateAllRepoCommitCnt(ctx *macaron.Context) {
@@ -35,3 +37,8 @@ func UpdateAllRepoCommitCnt(ctx *macaron.Context) {
 		"error_msg": "",
 	})
 }
+
+func RepoStatisticManually(ctx *macaron.Context) {
+	date := ctx.Query("date")
+	repo.RepoStatisticDaily(date)
+}

routers/repo/repo_statistic.go

@@ -0,0 +1,122 @@
+package repo
+
+import (
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+)
+
+//auto daily or manually
+func RepoStatisticAuto() {
+	log.Info("", time.Now())
+	yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
+	RepoStatisticDaily(yesterday)
+}
+
+func RepoStatisticDaily(date string) {
+	log.Info("%s", date)
+	if err := models.DeleteRepoStatDaily(date); err != nil {
+		log.Error("DeleteRepoStatDaily failed: %v", err.Error())
+		return
+	}
+
+	repos, err := models.GetAllRepositories()
+	if err != nil {
+		log.Error("GetAllRepositories failed: %v", err.Error())
+		return
+	}
+
+	for _, repo := range repos {
+		log.Info("start statistic: %s", repo.Name)
+		repoGitStat, err := models.GetRepoKPIStats(repo)
+		if err != nil {
+			log.Error("GetRepoKPIStats failed: %s", repo.Name)
+			log.Error("failed statistic: %s", repo.Name)
+			continue
+		}
+
+		var issueFixedRate float32
+		if repo.NumIssues != 0 {
+			issueFixedRate = float32(repo.NumClosedIssues) / float32(repo.NumIssues)
+		}
+
+		numVersions, err := models.GetReleaseCountByRepoID(repo.ID, models.FindReleasesOptions{})
+		if err != nil {
+			log.Error("GetReleaseCountByRepoID failed: %s", repo.Name)
+			log.Error("failed statistic: %s", repo.Name)
+			continue
+		}
+
+		datasetSize, err := getDatasetSize(repo)
+		if err != nil {
+			log.Error("getDatasetSize failed: %s", repo.Name)
+			log.Error("failed statistic: %s", repo.Name)
+			continue
+		}
+
+		numComments, err := models.GetCommentCountByRepoID(repo.ID)
+		if err != nil {
+			log.Error("GetCommentCountByRepoID failed: %s", repo.Name)
+			log.Error("failed statistic: %s", repo.Name)
+			continue
+		}
+
+		//beginTime, endTime := getStatTime(date)
+		//numVisits := repository.AppointProjectView(repo.OwnerName, repo.Name, beginTime, endTime)
+		numVisits := 0
+
+		repoStat := models.RepoStatistic{
+			RepoID:            repo.ID,
+			Date:              date,
+			NumWatches:        int64(repo.NumWatches),
+			NumStars:          int64(repo.NumStars),
+			NumDownloads:      repo.CloneCnt,
+			NumComments:       numComments,
+			NumVisits:         int64(numVisits),
+			NumClosedIssues:   int64(repo.NumClosedIssues),
+			NumVersions:       numVersions,
+			NumDevMonths:      repoGitStat.DevelopAge,
+			RepoSize:          repo.Size,
+			DatasetSize:       datasetSize,
+			NumModels:         0,
+			NumWikiViews:      repoGitStat.WikiPages,
+			NumCommits:        repo.NumCommit,
+			NumIssues:         int64(repo.NumIssues),
+			NumPulls:          int64(repo.NumPulls),
+			IssueFixedRate:    issueFixedRate,
+			NumContributor:    repoGitStat.Contributors,
+			NumKeyContributor: repoGitStat.KeyContributors,
+		}
+
+		if _, err = models.InsertRepoStat(&repoStat); err != nil {
+			log.Error("InsertRepoStat failed: %s", repo.Name)
+			log.Error("failed statistic: %s", repo.Name)
+			continue
+		}
+
+		log.Info("finish statistic: %s", repo.Name)
+	}
+
+}
+
+func getDatasetSize(repo *models.Repository) (int64, error) {
+	dataset, err := models.GetDatasetByRepo(repo)
+	if err != nil {
+		return 0, err
+	}
+
+	return models.GetAttachmentSizeByDatasetID(dataset.ID)
+}
+
+func getStatTime(timeStr string) (string, string) {
+	t, _ := time.Parse("2006-01-02", timeStr)
+	timeNumber := t.Unix()
+	beginTimeNumber := timeNumber - 8*60*60
+	endTimeNumber := timeNumber + 16*60*60
+	beginTime := time.Unix(beginTimeNumber, 0).Format(time.RFC3339)
+	endTime := time.Unix(endTimeNumber, 0).Format(time.RFC3339)
+	log.Info("%s, %s", beginTime, endTime)
+
+	return beginTime, endTime
+}

routers/repo/user_data_analysis.go

@@ -8,7 +8,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 )
 
-func TimeingCountData() {
+func TimingCountData() {
 	//query wiki data
 	log.Info("start to time count data")
 	wikiMap := make(map[string]int)