Merge pull request 'fix-359 平台总体指标数据入库' (#581) from fix-473 into V20211101

Reviewed-on: https://git.openi.org.cn/OpenI/aiforge/pulls/581
Reviewed-by: lewis <747342561@qq.com>

models/attachment.go

@@ -473,3 +473,7 @@ func GetAttachmentSizeByDatasetID(datasetID int64) (int64, error) {
 
 	return total, nil
 }
+
+func GetAllAttachmentSize() (int64, error) {
+	return x.SumInt(&Attachment{}, "size")
+}

models/models.go

@@ -137,6 +137,7 @@ func init() {
 
 	tablesStatistic = append(tablesStatistic,
 		new(RepoStatistic),
+		new(SummaryStatistic),
 		new(UserBusinessAnalysis),
 	)
 

models/repo.go

@@ -1430,6 +1430,15 @@ func GetAllRepositoriesByFilterCols(columns ...string) ([]*Repository, error) {
 
 }
 
+func GetAllRepositoriesCount() (int64, error) {
+	repo := new(Repository)
+	return x.Count(repo)
+}
+
+func GetAllRepositoriesSize() (int64, error) {
+	return x.SumInt(&Repository{}, "size")
+}
+
 func updateRepository(e Engine, repo *Repository, visibilityChanged bool) (err error) {
 	repo.LowerName = strings.ToLower(repo.Name)
 

models/summary_statistic.go

@@ -0,0 +1,69 @@
+package models
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/modules/timeutil"
+)
+
+var DomainMap = map[string]int{
+	"大模型":      0,
+	"ai开发工具":   1,
+	"计算机视觉":    2,
+	"自然语言处理":   3,
+	"机器学习":     4,
+	"神经网络":     5,
+	"自动驾驶":     6,
+	"机器人":      7,
+	"联邦学习":     8,
+	"数据挖掘":     9,
+	"risc-v开发": 10,
+}
+
+type SummaryStatistic struct {
+	ID                 int64              `xorm:"pk autoincr"`
+	Date               string             `xorm:"unique(s) NOT NULL"`
+	NumUsers           int64              `xorm:"NOT NULL DEFAULT 0"`
+	RepoSize           int64              `xorm:"NOT NULL DEFAULT 0"`
+	DatasetSize        int64              `xorm:"NOT NULL DEFAULT 0"`
+	NumOrganizations   int64              `xorm:"NOT NULL DEFAULT 0"`
+	NumModels          int64              `xorm:"NOT NULL DEFAULT 0"`
+	NumRepos           int64              `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoBigModel    int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoAI          int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoVision      int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoNLP         int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoML          int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoNN          int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoAutoDrive   int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoRobot       int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoLeagueLearn int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoDataMining  int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoRISC        int                `xorm:"NOT NULL DEFAULT 0"`
+	CreatedUnix        timeutil.TimeStamp `xorm:"INDEX created"`
+	UpdatedUnix        timeutil.TimeStamp `xorm:"INDEX updated"`
+}
+
+func DeleteSummaryStatisticDaily(date string) error {
+	sess := xStatistic.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return fmt.Errorf("Begin: %v", err)
+	}
+
+	if _, err := sess.Where("date = ?", date).Delete(&SummaryStatistic{}); err != nil {
+		return fmt.Errorf("Delete: %v", err)
+	}
+
+	if err := sess.Commit(); err != nil {
+		sess.Close()
+		return fmt.Errorf("Commit: %v", err)
+	}
+
+	sess.Close()
+	return nil
+}
+
+func InsertSummaryStatistic(summaryStatistic *SummaryStatistic) (int64, error) {
+	return xStatistic.Insert(summaryStatistic)
+}

models/topic.go

@@ -98,6 +98,13 @@ func GetTopicByName(name string) (*Topic, error) {
 	return &topic, nil
 }
 
+func GetAllUsedTopics() ([]*Topic, error) {
+	topics := make([]*Topic, 0)
+	err := x.Where("repo_count > ?", 0).Find(&topics)
+	return topics, err
+
+}
+
 // addTopicByNameToRepo adds a topic name to a repo and increments the topic count.
 // Returns topic after the addition
 func addTopicByNameToRepo(e Engine, repoID int64, topicName string) (*Topic, error) {

models/user.go

@@ -2071,6 +2071,18 @@ func SyncExternalUsers(ctx context.Context, updateExisting bool) error {
 	return nil
 }
 
+func GetUsersCount() (int64, error) {
+	user := new(User)
+	return x.Where("type=0").Count(user)
+
+}
+
+func GetOrganizationsCount() (int64, error) {
+	user := new(User)
+	return x.Where("type=1").Count(user)
+
+}
+
 func GetBlockChainUnSuccessUsers() ([]*User, error) {
 	users := make([]*User, 0, 10)
 	err := x.Where("public_key = ''").

modules/cron/tasks_basic.go

@@ -174,6 +174,16 @@ func registerHandleRepoStatistic() {
 	})
 }
 
+func registerHandleSummaryStatistic() {
+	RegisterTaskFatal("handle_summary_statistic", &BaseConfig{
+		Enabled:    true,
+		RunAtStart: false,
+		Schedule:   "@daily",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		repo.SummaryStatistic()
+		return nil
+	})
+}
 func registerHandleUserStatistic() {
 	RegisterTaskFatal("handle_user_statistic", &BaseConfig{
 		Enabled:    true,
@@ -202,4 +212,5 @@ func initBasicTasks() {
 
 	registerHandleRepoStatistic()
 	registerHandleUserStatistic()
+	registerHandleSummaryStatistic()
 }

routers/repo/repo_summary_statistic.go

@@ -0,0 +1,94 @@
+package repo
+
+import (
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+)
+
+func SummaryStatistic() {
+	log.Info("Generate summary statistic begin")
+	yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
+	SummaryStatisticDaily(yesterday)
+	log.Info("Generate summary statistic end")
+}
+
+func SummaryStatisticDaily(date string) {
+	log.Info("%s", date)
+	if err := models.DeleteSummaryStatisticDaily(date); err != nil {
+		log.Error("DeleteRepoStatDaily failed: %v", err.Error())
+		return
+	}
+
+	//user number
+	userNumber, err := models.GetUsersCount()
+	if err != nil {
+		log.Error("can not get user number", err)
+		userNumber = 0
+	}
+	//organization number
+	organizationNumber, err := models.GetOrganizationsCount()
+	if err != nil {
+		log.Error("can not get orgnazition number", err)
+		organizationNumber = 0
+	}
+	// repository number
+	repositoryNumer, err := models.GetAllRepositoriesCount()
+	if err != nil {
+		log.Error("can not get repository number", err)
+		repositoryNumer = 0
+	}
+	//repository size
+	repositorySize, err := models.GetAllRepositoriesSize()
+	if err != nil {
+		log.Error("can not get repository size", err)
+		repositorySize = 0
+	}
+	// dataset size
+	allDatasetSize, err := models.GetAllAttachmentSize()
+	if err != nil {
+		log.Error("can not get dataset size", err)
+		allDatasetSize = 0
+	}
+	//topic repo number
+	topics, err := models.GetAllUsedTopics()
+	if err != nil {
+		log.Error("can not get topics", err)
+	}
+	var topicsCount [11]int
+	for _, topic := range topics {
+
+		index, exists := models.DomainMap[topic.Name]
+		if exists {
+			topicsCount[index] = topic.RepoCount
+		}
+
+	}
+
+	summaryStat := models.SummaryStatistic{
+		Date:               date,
+		NumUsers:           userNumber,
+		RepoSize:           repositorySize,
+		DatasetSize:        allDatasetSize,
+		NumOrganizations:   organizationNumber,
+		NumRepos:           repositoryNumer,
+		NumRepoBigModel:    topicsCount[0],
+		NumRepoAI:          topicsCount[1],
+		NumRepoVision:      topicsCount[2],
+		NumRepoNLP:         topicsCount[3],
+		NumRepoML:          topicsCount[4],
+		NumRepoNN:          topicsCount[5],
+		NumRepoAutoDrive:   topicsCount[6],
+		NumRepoRobot:       topicsCount[7],
+		NumRepoLeagueLearn: topicsCount[8],
+		NumRepoDataMining:  topicsCount[9],
+		NumRepoRISC:        topicsCount[10],
+	}
+
+	if _, err = models.InsertSummaryStatistic(&summaryStat); err != nil {
+		log.Error("Insert summary Stat failed: %v", err.Error())
+	}
+
+	log.Info("finish summary statistic")
+}