Merge branch 'V20211101' of git.openi.org.cn:OpenI/aiforge into sync_failed_status

models/attachment.go

@@ -473,3 +473,7 @@ func GetAttachmentSizeByDatasetID(datasetID int64) (int64, error) {
 
 	return total, nil
 }
+
+func GetAllAttachmentSize() (int64, error) {
+	return x.SumInt(&Attachment{}, "size")
+}

models/cloudbrain.go

@@ -698,7 +698,7 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) {
 }
 
 func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool {
-	if !isSigned || job.Status != string(JobStopped) {
+	if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)){
 		return false
 	}
 	repo, err := GetRepositoryByID(job.RepoID)

models/models.go

@@ -137,6 +137,7 @@ func init() {
 
 	tablesStatistic = append(tablesStatistic,
 		new(RepoStatistic),
+		new(SummaryStatistic),
 		new(UserBusinessAnalysis),
 	)
 

models/repo.go

@@ -1430,6 +1430,15 @@ func GetAllRepositoriesByFilterCols(columns ...string) ([]*Repository, error) {
 
 }
 
+func GetAllRepositoriesCount() (int64, error) {
+	repo := new(Repository)
+	return x.Count(repo)
+}
+
+func GetAllRepositoriesSize() (int64, error) {
+	return x.SumInt(&Repository{}, "size")
+}
+
 func updateRepository(e Engine, repo *Repository, visibilityChanged bool) (err error) {
 	repo.LowerName = strings.ToLower(repo.Name)
 

models/summary_statistic.go

@@ -0,0 +1,69 @@
+package models
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/modules/timeutil"
+)
+
+var DomainMap = map[string]int{
+	"大模型":      0,
+	"ai开发工具":   1,
+	"计算机视觉":    2,
+	"自然语言处理":   3,
+	"机器学习":     4,
+	"神经网络":     5,
+	"自动驾驶":     6,
+	"机器人":      7,
+	"联邦学习":     8,
+	"数据挖掘":     9,
+	"risc-v开发": 10,
+}
+
+type SummaryStatistic struct {
+	ID                 int64              `xorm:"pk autoincr"`
+	Date               string             `xorm:"unique(s) NOT NULL"`
+	NumUsers           int64              `xorm:"NOT NULL DEFAULT 0"`
+	RepoSize           int64              `xorm:"NOT NULL DEFAULT 0"`
+	DatasetSize        int64              `xorm:"NOT NULL DEFAULT 0"`
+	NumOrganizations   int64              `xorm:"NOT NULL DEFAULT 0"`
+	NumModels          int64              `xorm:"NOT NULL DEFAULT 0"`
+	NumRepos           int64              `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoBigModel    int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoAI          int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoVision      int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoNLP         int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoML          int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoNN          int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoAutoDrive   int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoRobot       int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoLeagueLearn int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoDataMining  int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoRISC        int                `xorm:"NOT NULL DEFAULT 0"`
+	CreatedUnix        timeutil.TimeStamp `xorm:"INDEX created"`
+	UpdatedUnix        timeutil.TimeStamp `xorm:"INDEX updated"`
+}
+
+func DeleteSummaryStatisticDaily(date string) error {
+	sess := xStatistic.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return fmt.Errorf("Begin: %v", err)
+	}
+
+	if _, err := sess.Where("date = ?", date).Delete(&SummaryStatistic{}); err != nil {
+		return fmt.Errorf("Delete: %v", err)
+	}
+
+	if err := sess.Commit(); err != nil {
+		sess.Close()
+		return fmt.Errorf("Commit: %v", err)
+	}
+
+	sess.Close()
+	return nil
+}
+
+func InsertSummaryStatistic(summaryStatistic *SummaryStatistic) (int64, error) {
+	return xStatistic.Insert(summaryStatistic)
+}

models/topic.go

@@ -98,6 +98,13 @@ func GetTopicByName(name string) (*Topic, error) {
 	return &topic, nil
 }
 
+func GetAllUsedTopics() ([]*Topic, error) {
+	topics := make([]*Topic, 0)
+	err := x.Where("repo_count > ?", 0).Find(&topics)
+	return topics, err
+
+}
+
 // addTopicByNameToRepo adds a topic name to a repo and increments the topic count.
 // Returns topic after the addition
 func addTopicByNameToRepo(e Engine, repoID int64, topicName string) (*Topic, error) {

models/user.go

@@ -2071,6 +2071,18 @@ func SyncExternalUsers(ctx context.Context, updateExisting bool) error {
 	return nil
 }
 
+func GetUsersCount() (int64, error) {
+	user := new(User)
+	return x.Where("type=0").Count(user)
+
+}
+
+func GetOrganizationsCount() (int64, error) {
+	user := new(User)
+	return x.Where("type=1").Count(user)
+
+}
+
 func GetBlockChainUnSuccessUsers() ([]*User, error) {
 	users := make([]*User, 0, 10)
 	err := x.Where("public_key = ''").

modules/cron/tasks_basic.go

@@ -174,6 +174,16 @@ func registerHandleRepoStatistic() {
 	})
 }
 
+func registerHandleSummaryStatistic() {
+	RegisterTaskFatal("handle_summary_statistic", &BaseConfig{
+		Enabled:    true,
+		RunAtStart: false,
+		Schedule:   "@daily",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		repo.SummaryStatistic()
+		return nil
+	})
+}
 func registerHandleUserStatistic() {
 	RegisterTaskFatal("handle_user_statistic", &BaseConfig{
 		Enabled:    true,
@@ -202,4 +212,5 @@ func initBasicTasks() {
 
 	registerHandleRepoStatistic()
 	registerHandleUserStatistic()
+	registerHandleSummaryStatistic()
 }

options/locale/locale_zh-CN.ini

@@ -776,6 +776,7 @@ cloudbrain_creator=创建者
 cloudbrain_task=任务名称
 cloudbrain_operate=操作
 cloudbrain_status_createtime=状态/创建时间
+cloudbrain_jobname_err=只能以小写字母或数字开头且只包含小写字母、数字、_和-，不能以_结尾，最长36个字符。
 
 template.items=模板选项
 template.git_content=Git数据(默认分支)

routers/repo/cloudbrain.go

@@ -40,6 +40,8 @@ var (
 	categories *models.Categories
 )
 
+var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`)
+
 // MustEnableDataset check if repository enable internal cb
 func MustEnableCloudbrain(ctx *context.Context) {
 	if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) {
@@ -200,6 +202,11 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
 	gpuQueue := setting.JobType
 	codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
 	resourceSpecId := form.ResourceSpecId
+	
+	if !jobNamePattern.MatchString(jobName) {
+		ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplModelArtsNew, &form)
+		return
+	}
 
 	if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) {
 		log.Error("jobtype error:", jobType, ctx.Data["MsgID"])

routers/repo/modelarts.go

@@ -100,7 +100,10 @@ func ModelArtsCreate(ctx *context.Context, form auth.CreateModelArtsForm) {
 	uuid := form.Attachment
 	description := form.Description
 	//repo := ctx.Repo.Repository
-
+	if !jobNamePattern.MatchString(jobName) {
+		ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplModelArtsNew, &form)
+		return
+	}
 	err := modelarts.GenerateTask(ctx, jobName, uuid, description)
 	if err != nil {
 		ctx.RenderWithErr(err.Error(), tplModelArtsNew, &form)

routers/repo/repo_summary_statistic.go

@@ -0,0 +1,94 @@
+package repo
+
+import (
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+)
+
+func SummaryStatistic() {
+	log.Info("Generate summary statistic begin")
+	yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
+	SummaryStatisticDaily(yesterday)
+	log.Info("Generate summary statistic end")
+}
+
+func SummaryStatisticDaily(date string) {
+	log.Info("%s", date)
+	if err := models.DeleteSummaryStatisticDaily(date); err != nil {
+		log.Error("DeleteRepoStatDaily failed: %v", err.Error())
+		return
+	}
+
+	//user number
+	userNumber, err := models.GetUsersCount()
+	if err != nil {
+		log.Error("can not get user number", err)
+		userNumber = 0
+	}
+	//organization number
+	organizationNumber, err := models.GetOrganizationsCount()
+	if err != nil {
+		log.Error("can not get orgnazition number", err)
+		organizationNumber = 0
+	}
+	// repository number
+	repositoryNumer, err := models.GetAllRepositoriesCount()
+	if err != nil {
+		log.Error("can not get repository number", err)
+		repositoryNumer = 0
+	}
+	//repository size
+	repositorySize, err := models.GetAllRepositoriesSize()
+	if err != nil {
+		log.Error("can not get repository size", err)
+		repositorySize = 0
+	}
+	// dataset size
+	allDatasetSize, err := models.GetAllAttachmentSize()
+	if err != nil {
+		log.Error("can not get dataset size", err)
+		allDatasetSize = 0
+	}
+	//topic repo number
+	topics, err := models.GetAllUsedTopics()
+	if err != nil {
+		log.Error("can not get topics", err)
+	}
+	var topicsCount [11]int
+	for _, topic := range topics {
+
+		index, exists := models.DomainMap[topic.Name]
+		if exists {
+			topicsCount[index] = topic.RepoCount
+		}
+
+	}
+
+	summaryStat := models.SummaryStatistic{
+		Date:               date,
+		NumUsers:           userNumber,
+		RepoSize:           repositorySize,
+		DatasetSize:        allDatasetSize,
+		NumOrganizations:   organizationNumber,
+		NumRepos:           repositoryNumer,
+		NumRepoBigModel:    topicsCount[0],
+		NumRepoAI:          topicsCount[1],
+		NumRepoVision:      topicsCount[2],
+		NumRepoNLP:         topicsCount[3],
+		NumRepoML:          topicsCount[4],
+		NumRepoNN:          topicsCount[5],
+		NumRepoAutoDrive:   topicsCount[6],
+		NumRepoRobot:       topicsCount[7],
+		NumRepoLeagueLearn: topicsCount[8],
+		NumRepoDataMining:  topicsCount[9],
+		NumRepoRISC:        topicsCount[10],
+	}
+
+	if _, err = models.InsertSummaryStatistic(&summaryStat); err != nil {
+		log.Error("Insert summary Stat failed: %v", err.Error())
+	}
+
+	log.Info("finish summary statistic")
+}

templates/repo/cloudbrain/index.tmpl

@@ -337,9 +337,9 @@
                                             调试
                                         </a>
                                         
-                                        <form id="stopForm-{{.JobID}}" action="{{if ((eq .Status "STOPPED") or (eq .Status "FAILED"))}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/stop{{end}}" method="post" style="margin-left:-1px;">
+                                        <form id="stopForm-{{.JobID}}" action="{{if or (eq .Status "STOPPED") (eq .Status "FAILED")}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/stop{{end}}" method="post" style="margin-left:-1px;">
                                             {{$.CsrfTokenHtml}}
-                                            <a class="ui basic {{if ((eq .Status "STOPPED") or (eq .Status "FAILED"))}}disabled {{else}}blue {{end}}button" onclick="document.getElementById('stopForm-{{.JobID}}').submit();">
+                                            <a class="ui basic {{if or (eq .Status "STOPPED") (eq .Status "FAILED")}}disabled {{else}}blue {{end}}button" onclick="document.getElementById('stopForm-{{.JobID}}').submit();">
                                                 停止
                                             </a>
                                         </form>