Git statistics in Activity tab (#4724)

* Initial implementation for git statistics in Activity tab

* Create top user by commit count endpoint

* Add UI and update src-d/go-git dependency

* Add coloring

* Fix typo

* Move git activity stats data extraction to git module

* Fix message

* Add git code stats test

models/repo_activity.go

@@ -6,11 +6,22 @@ package models
 
 import (
 	"fmt"
+	"sort"
 	"time"
 
+	"code.gitea.io/gitea/modules/git"
+
 	"github.com/go-xorm/xorm"
 )
 
+// ActivityAuthorData represents statistical git commit count data
+type ActivityAuthorData struct {
+	Name       string `json:"name"`
+	Login      string `json:"login"`
+	AvatarLink string `json:"avatar_link"`
+	Commits    int64  `json:"commits"`
+}
+
 // ActivityStats represets issue and pull request information.
 type ActivityStats struct {
 	OpenedPRs                   PullRequestList
@@ -24,32 +35,97 @@ type ActivityStats struct {
 	UnresolvedIssues            IssueList
 	PublishedReleases           []*Release
 	PublishedReleaseAuthorCount int64
+	Code                        *git.CodeActivityStats
 }
 
 // GetActivityStats return stats for repository at given time range
-func GetActivityStats(repoID int64, timeFrom time.Time, releases, issues, prs bool) (*ActivityStats, error) {
-	stats := &ActivityStats{}
+func GetActivityStats(repo *Repository, timeFrom time.Time, releases, issues, prs, code bool) (*ActivityStats, error) {
+	stats := &ActivityStats{Code: &git.CodeActivityStats{}}
 	if releases {
-		if err := stats.FillReleases(repoID, timeFrom); err != nil {
+		if err := stats.FillReleases(repo.ID, timeFrom); err != nil {
 			return nil, fmt.Errorf("FillReleases: %v", err)
 		}
 	}
 	if prs {
-		if err := stats.FillPullRequests(repoID, timeFrom); err != nil {
+		if err := stats.FillPullRequests(repo.ID, timeFrom); err != nil {
 			return nil, fmt.Errorf("FillPullRequests: %v", err)
 		}
 	}
 	if issues {
-		if err := stats.FillIssues(repoID, timeFrom); err != nil {
+		if err := stats.FillIssues(repo.ID, timeFrom); err != nil {
 			return nil, fmt.Errorf("FillIssues: %v", err)
 		}
 	}
-	if err := stats.FillUnresolvedIssues(repoID, timeFrom, issues, prs); err != nil {
+	if err := stats.FillUnresolvedIssues(repo.ID, timeFrom, issues, prs); err != nil {
 		return nil, fmt.Errorf("FillUnresolvedIssues: %v", err)
 	}
+	if code {
+		gitRepo, err := git.OpenRepository(repo.RepoPath())
+		if err != nil {
+			return nil, fmt.Errorf("OpenRepository: %v", err)
+		}
+		code, err := gitRepo.GetCodeActivityStats(timeFrom, repo.DefaultBranch)
+		if err != nil {
+			return nil, fmt.Errorf("FillFromGit: %v", err)
+		}
+		stats.Code = code
+	}
 	return stats, nil
 }
 
+// GetActivityStatsTopAuthors returns top author stats for git commits for all branches
+func GetActivityStatsTopAuthors(repo *Repository, timeFrom time.Time, count int) ([]*ActivityAuthorData, error) {
+	gitRepo, err := git.OpenRepository(repo.RepoPath())
+	if err != nil {
+		return nil, fmt.Errorf("OpenRepository: %v", err)
+	}
+	code, err := gitRepo.GetCodeActivityStats(timeFrom, "")
+	if err != nil {
+		return nil, fmt.Errorf("FillFromGit: %v", err)
+	}
+	if code.Authors == nil {
+		return nil, nil
+	}
+	users := make(map[int64]*ActivityAuthorData)
+	for k, v := range code.Authors {
+		if len(k) == 0 {
+			continue
+		}
+		u, err := GetUserByEmail(k)
+		if u == nil || IsErrUserNotExist(err) {
+			continue
+		}
+		if err != nil {
+			return nil, err
+		}
+		if user, ok := users[u.ID]; !ok {
+			users[u.ID] = &ActivityAuthorData{
+				Name:       u.DisplayName(),
+				Login:      u.LowerName,
+				AvatarLink: u.AvatarLink(),
+				Commits:    v,
+			}
+		} else {
+			user.Commits += v
+		}
+	}
+	v := make([]*ActivityAuthorData, 0)
+	for _, u := range users {
+		v = append(v, u)
+	}
+
+	sort.Slice(v[:], func(i, j int) bool {
+		return v[i].Commits < v[j].Commits
+	})
+
+	cnt := count
+	if cnt > len(v) {
+		cnt = len(v)
+	}
+
+	return v[:cnt], nil
+}
+
 // ActivePRCount returns total active pull request count
 func (stats *ActivityStats) ActivePRCount() int {
 	return stats.OpenedPRCount() + stats.MergedPRCount()

modules/git/repo_stats.go

@@ -0,0 +1,108 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package git
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// CodeActivityStats represents git statistics data
+type CodeActivityStats struct {
+	AuthorCount              int64
+	CommitCount              int64
+	ChangedFiles             int64
+	Additions                int64
+	Deletions                int64
+	CommitCountInAllBranches int64
+	Authors                  map[string]int64
+}
+
+// GetCodeActivityStats returns code statistics for acitivity page
+func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) (*CodeActivityStats, error) {
+	stats := &CodeActivityStats{}
+
+	since := fromTime.Format(time.RFC3339)
+
+	stdout, err := NewCommand("rev-list", "--count", "--no-merges", "--branches=*", "--date=iso", fmt.Sprintf("--since='%s'", since)).RunInDirBytes(repo.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	c, err := strconv.ParseInt(strings.TrimSpace(string(stdout)), 10, 64)
+	if err != nil {
+		return nil, err
+	}
+	stats.CommitCountInAllBranches = c
+
+	args := []string{"log", "--numstat", "--no-merges", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--since='%s'", since)}
+	if len(branch) == 0 {
+		args = append(args, "--branches=*")
+	} else {
+		args = append(args, "--first-parent", branch)
+	}
+
+	stdout, err = NewCommand(args...).RunInDirBytes(repo.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	scanner := bufio.NewScanner(bytes.NewReader(stdout))
+	scanner.Split(bufio.ScanLines)
+	stats.CommitCount = 0
+	stats.Additions = 0
+	stats.Deletions = 0
+	authors := make(map[string]int64)
+	files := make(map[string]bool)
+	p := 0
+	for scanner.Scan() {
+		l := strings.TrimSpace(scanner.Text())
+		if l == "---" {
+			p = 1
+		} else if p == 0 {
+			continue
+		} else {
+			p++
+		}
+		if p > 4 && len(l) == 0 {
+			continue
+		}
+		switch p {
+		case 1: // Separator
+		case 2: // Commit sha-1
+			stats.CommitCount++
+		case 3: // Author
+		case 4: // E-mail
+			email := strings.ToLower(l)
+			i := authors[email]
+			authors[email] = i + 1
+		default: // Changed file
+			if parts := strings.Fields(l); len(parts) >= 3 {
+				if parts[0] != "-" {
+					if c, err := strconv.ParseInt(strings.TrimSpace(parts[0]), 10, 64); err == nil {
+						stats.Additions += c
+					}
+				}
+				if parts[1] != "-" {
+					if c, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64); err == nil {
+						stats.Deletions += c
+					}
+				}
+				if _, ok := files[parts[2]]; !ok {
+					files[parts[2]] = true
+				}
+			}
+		}
+	}
+	stats.AuthorCount = int64(len(authors))
+	stats.ChangedFiles = int64(len(files))
+	stats.Authors = authors
+
+	return stats, nil
+}

modules/git/repo_stats_test.go

@@ -0,0 +1,35 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package git
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRepository_GetCodeActivityStats(t *testing.T) {
+	bareRepo1Path := filepath.Join(testReposDir, "repo1_bare")
+	bareRepo1, err := OpenRepository(bareRepo1Path)
+	assert.NoError(t, err)
+
+	timeFrom, err := time.Parse(time.RFC3339, "2016-01-01T00:00:00+00:00")
+
+	code, err := bareRepo1.GetCodeActivityStats(timeFrom, "")
+	assert.NoError(t, err)
+	assert.NotNil(t, code)
+
+	assert.EqualValues(t, 8, code.CommitCount)
+	assert.EqualValues(t, 2, code.AuthorCount)
+	assert.EqualValues(t, 8, code.CommitCountInAllBranches)
+	assert.EqualValues(t, 10, code.Additions)
+	assert.EqualValues(t, 1, code.Deletions)
+	assert.Len(t, code.Authors, 2)
+	assert.Contains(t, code.Authors, "tris.git@shoddynet.org")
+	assert.EqualValues(t, 3, code.Authors["tris.git@shoddynet.org"])
+	assert.EqualValues(t, 5, code.Authors[""])
+}

options/locale/locale_en-US.ini

@@ -1061,6 +1061,24 @@ activity.title.releases_1 = %d Release
 activity.title.releases_n = %d Releases
 activity.title.releases_published_by = %s published by %s
 activity.published_release_label = Published
+activity.no_git_activity = There has not been any commit activity in this period.
+activity.git_stats_exclude_merges = Excluding merges,
+activity.git_stats_author_1 = %d author
+activity.git_stats_author_n = %d authors
+activity.git_stats_pushed = has pushed
+activity.git_stats_commit_1 = %d commit
+activity.git_stats_commit_n = %d commits
+activity.git_stats_push_to_branch = to %s and
+activity.git_stats_push_to_all_branches = to all branches.
+activity.git_stats_on_default_branch = On %s,
+activity.git_stats_file_1 = %d file
+activity.git_stats_file_n = %d files
+activity.git_stats_files_changed = have changed and there have been
+activity.git_stats_addition_1 = %d addition
+activity.git_stats_addition_n = %d additions
+activity.git_stats_and_deletions = and
+activity.git_stats_deletion_1 = %d deletion
+activity.git_stats_deletion_n = %d deletions
 
 search = Search
 search.search_repo = Search repository

routers/repo/activity.go

@@ -44,13 +44,42 @@ func Activity(ctx *context.Context) {
 	ctx.Data["PeriodText"] = ctx.Tr("repo.activity.period." + ctx.Data["Period"].(string))
 
 	var err error
-	if ctx.Data["Activity"], err = models.GetActivityStats(ctx.Repo.Repository.ID, timeFrom,
+	if ctx.Data["Activity"], err = models.GetActivityStats(ctx.Repo.Repository, timeFrom,
 		ctx.Repo.CanRead(models.UnitTypeReleases),
 		ctx.Repo.CanRead(models.UnitTypeIssues),
-		ctx.Repo.CanRead(models.UnitTypePullRequests)); err != nil {
+		ctx.Repo.CanRead(models.UnitTypePullRequests),
+		ctx.Repo.CanRead(models.UnitTypeCode)); err != nil {
 		ctx.ServerError("GetActivityStats", err)
 		return
 	}
 
 	ctx.HTML(200, tplActivity)
 }
+
+// ActivityAuthors renders JSON with top commit authors for given time period over all branches
+func ActivityAuthors(ctx *context.Context) {
+	timeUntil := time.Now()
+	var timeFrom time.Time
+
+	switch ctx.Params("period") {
+	case "daily":
+		timeFrom = timeUntil.Add(-time.Hour * 24)
+	case "halfweekly":
+		timeFrom = timeUntil.Add(-time.Hour * 72)
+	case "weekly":
+		timeFrom = timeUntil.Add(-time.Hour * 168)
+	case "monthly":
+		timeFrom = timeUntil.AddDate(0, -1, 0)
+	default:
+		timeFrom = timeUntil.Add(-time.Hour * 168)
+	}
+
+	var err error
+	authors, err := models.GetActivityStatsTopAuthors(ctx.Repo.Repository, timeFrom, 10)
+	if err != nil {
+		ctx.ServerError("GetActivityStatsTopAuthors", err)
+		return
+	}
+
+	ctx.JSON(200, authors)
+}

routers/routes/routes.go

@@ -802,6 +802,11 @@ func RegisterRoutes(m *macaron.Macaron) {
 			m.Get("/:period", repo.Activity)
 		}, context.RepoRef(), repo.MustBeNotEmpty, context.RequireRepoReaderOr(models.UnitTypePullRequests, models.UnitTypeIssues, models.UnitTypeReleases))
 
+		m.Group("/activity_author_data", func() {
+			m.Get("", repo.ActivityAuthors)
+			m.Get("/:period", repo.ActivityAuthors)
+		}, context.RepoRef(), repo.MustBeNotEmpty, context.RequireRepoReaderOr(models.UnitTypeCode))
+
 		m.Get("/archive/*", repo.MustBeNotEmpty, reqRepoCodeReader, repo.Download)
 
 		m.Group("/branches", func() {

templates/repo/activity.tmpl

@@ -81,6 +81,33 @@
 		</div>
 		{{end}}
 
+		{{if .Permission.CanRead $.UnitTypeCode}}
+			{{if eq .Activity.Code.CommitCountInAllBranches 0}}
+				<div class="ui center aligned segment">
+				<h4 class="ui header">{{.i18n.Tr "repo.activity.no_git_activity" }}</h4>
+				</div>
+			{{end}}
+			{{if gt .Activity.Code.CommitCountInAllBranches 0}}
+				<div class="ui attached segment horizontal segments">
+					<div class="ui attached segment text">
+					    {{.i18n.Tr "repo.activity.git_stats_exclude_merges" }}
+						<strong>{{.i18n.Tr (TrN .i18n.Lang .Activity.Code.AuthorCount "repo.activity.git_stats_author_1" "repo.activity.git_stats_author_n") .Activity.Code.AuthorCount }}</strong>
+						{{.i18n.Tr "repo.activity.git_stats_pushed" }}
+						<strong>{{.i18n.Tr (TrN .i18n.Lang .Activity.Code.CommitCount "repo.activity.git_stats_commit_1" "repo.activity.git_stats_commit_n") .Activity.Code.CommitCount }}</strong>
+						{{.i18n.Tr "repo.activity.git_stats_push_to_branch" .Repository.DefaultBranch }}
+						<strong>{{.i18n.Tr (TrN .i18n.Lang .Activity.Code.CommitCountInAllBranches "repo.activity.git_stats_commit_1" "repo.activity.git_stats_commit_n") .Activity.Code.CommitCountInAllBranches }}</strong>
+						{{.i18n.Tr "repo.activity.git_stats_push_to_all_branches" }}
+						{{.i18n.Tr "repo.activity.git_stats_on_default_branch" .Repository.DefaultBranch }}
+						<strong>{{.i18n.Tr (TrN .i18n.Lang .Activity.Code.ChangedFiles "repo.activity.git_stats_file_1" "repo.activity.git_stats_file_n") .Activity.Code.ChangedFiles }}</strong>
+						{{.i18n.Tr "repo.activity.git_stats_files_changed" }}
+						<strong class="text green">{{.i18n.Tr (TrN .i18n.Lang .Activity.Code.Additions "repo.activity.git_stats_addition_1" "repo.activity.git_stats_addition_n") .Activity.Code.Additions }}</strong>
+						{{.i18n.Tr "repo.activity.git_stats_and_deletions" }}
+						<strong class="text red">{{.i18n.Tr (TrN .i18n.Lang .Activity.Code.Deletions "repo.activity.git_stats_deletion_1" "repo.activity.git_stats_deletion_n") .Activity.Code.Deletions }}</strong>.
+					</div>
+				</div>
+			{{end}}
+		{{end}}
+
 		{{if gt .Activity.PublishedReleaseCount 0}}
 			<h4 class="ui horizontal divider header" id="published-releases">
 				<i class="text octicon octicon-tag"></i>