Merge branch 'V20211101' into fix-472

models/attachment.go

@@ -473,3 +473,7 @@ func GetAttachmentSizeByDatasetID(datasetID int64) (int64, error) {
 
 	return total, nil
 }
+
+func GetAllAttachmentSize() (int64, error) {
+	return x.SumInt(&Attachment{}, "size")
+}

models/cloudbrain.go

@@ -151,23 +151,42 @@ type TaskPod struct {
 	TaskRoleStatus struct {
 		Name string `json:"name"`
 	} `json:"taskRoleStatus"`
-	TaskStatuses []struct {
-		TaskIndex       int       `json:"taskIndex"`
-		PodUID          string    `json:"podUid"`
-		PodIP           string    `json:"podIp"`
-		PodName         string    `json:"podName"`
-		ContainerID     string    `json:"containerId"`
-		ContainerIP     string    `json:"containerIp"`
-		ContainerGpus   string    `json:"containerGpus"`
-		State           string    `json:"state"`
-		StartAt         time.Time `json:"startAt"`
-		FinishedAt      time.Time `json:"finishedAt"`
-		ExitCode        int       `json:"exitCode"`
-		ExitDiagnostics string    `json:"exitDiagnostics"`
-		RetriedCount    int       `json:"retriedCount"`
-		StartTime       string
-		FinishedTime    string
-	} `json:"taskStatuses"`
+	//TaskStatuses []struct {
+	//	TaskIndex       int       `json:"taskIndex"`
+	//	PodUID          string    `json:"podUid"`
+	//	PodIP           string    `json:"podIp"`
+	//	PodName         string    `json:"podName"`
+	//	ContainerID     string    `json:"containerId"`
+	//	ContainerIP     string    `json:"containerIp"`
+	//	ContainerGpus   string    `json:"containerGpus"`
+	//	State           string    `json:"state"`
+	//	StartAt         time.Time `json:"startAt"`
+	//	FinishedAt      time.Time `json:"finishedAt"`
+	//	ExitCode        int       `json:"exitCode"`
+	//	ExitDiagnostics string    `json:"exitDiagnostics"`
+	//	RetriedCount    int       `json:"retriedCount"`
+	//	StartTime       string
+	//	FinishedTime    string
+	//} `json:"taskStatuses"`
+	TaskStatuses []TaskStatuses `json:"taskStatuses"`
+}
+
+type TaskStatuses struct {
+	TaskIndex       int       `json:"taskIndex"`
+	PodUID          string    `json:"podUid"`
+	PodIP           string    `json:"podIp"`
+	PodName         string    `json:"podName"`
+	ContainerID     string    `json:"containerId"`
+	ContainerIP     string    `json:"containerIp"`
+	ContainerGpus   string    `json:"containerGpus"`
+	State           string    `json:"state"`
+	StartAt         time.Time `json:"startAt"`
+	FinishedAt      time.Time `json:"finishedAt"`
+	ExitCode        int       `json:"exitCode"`
+	ExitDiagnostics string    `json:"exitDiagnostics"`
+	RetriedCount    int       `json:"retriedCount"`
+	StartTime       string
+	FinishedTime    string
 }
 
 type TaskInfo struct {
@@ -679,7 +698,7 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) {
 }
 
 func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool {
-	if !isSigned || job.Status != string(JobStopped) {
+	if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)){
 		return false
 	}
 	repo, err := GetRepositoryByID(job.RepoID)

models/models.go

@@ -137,6 +137,7 @@ func init() {
 
 	tablesStatistic = append(tablesStatistic,
 		new(RepoStatistic),
+		new(SummaryStatistic),
 		new(UserBusinessAnalysis),
 	)
 

models/repo.go

@@ -1430,6 +1430,15 @@ func GetAllRepositoriesByFilterCols(columns ...string) ([]*Repository, error) {
 
 }
 
+func GetAllRepositoriesCount() (int64, error) {
+	repo := new(Repository)
+	return x.Count(repo)
+}
+
+func GetAllRepositoriesSize() (int64, error) {
+	return x.SumInt(&Repository{}, "size")
+}
+
 func updateRepository(e Engine, repo *Repository, visibilityChanged bool) (err error) {
 	repo.LowerName = strings.ToLower(repo.Name)
 

models/summary_statistic.go

@@ -0,0 +1,69 @@
+package models
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/modules/timeutil"
+)
+
+var DomainMap = map[string]int{
+	"大模型":      0,
+	"ai开发工具":   1,
+	"计算机视觉":    2,
+	"自然语言处理":   3,
+	"机器学习":     4,
+	"神经网络":     5,
+	"自动驾驶":     6,
+	"机器人":      7,
+	"联邦学习":     8,
+	"数据挖掘":     9,
+	"risc-v开发": 10,
+}
+
+type SummaryStatistic struct {
+	ID                 int64              `xorm:"pk autoincr"`
+	Date               string             `xorm:"unique(s) NOT NULL"`
+	NumUsers           int64              `xorm:"NOT NULL DEFAULT 0"`
+	RepoSize           int64              `xorm:"NOT NULL DEFAULT 0"`
+	DatasetSize        int64              `xorm:"NOT NULL DEFAULT 0"`
+	NumOrganizations   int64              `xorm:"NOT NULL DEFAULT 0"`
+	NumModels          int64              `xorm:"NOT NULL DEFAULT 0"`
+	NumRepos           int64              `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoBigModel    int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoAI          int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoVision      int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoNLP         int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoML          int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoNN          int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoAutoDrive   int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoRobot       int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoLeagueLearn int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoDataMining  int                `xorm:"NOT NULL DEFAULT 0"`
+	NumRepoRISC        int                `xorm:"NOT NULL DEFAULT 0"`
+	CreatedUnix        timeutil.TimeStamp `xorm:"INDEX created"`
+	UpdatedUnix        timeutil.TimeStamp `xorm:"INDEX updated"`
+}
+
+func DeleteSummaryStatisticDaily(date string) error {
+	sess := xStatistic.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return fmt.Errorf("Begin: %v", err)
+	}
+
+	if _, err := sess.Where("date = ?", date).Delete(&SummaryStatistic{}); err != nil {
+		return fmt.Errorf("Delete: %v", err)
+	}
+
+	if err := sess.Commit(); err != nil {
+		sess.Close()
+		return fmt.Errorf("Commit: %v", err)
+	}
+
+	sess.Close()
+	return nil
+}
+
+func InsertSummaryStatistic(summaryStatistic *SummaryStatistic) (int64, error) {
+	return xStatistic.Insert(summaryStatistic)
+}

models/topic.go

@@ -98,6 +98,13 @@ func GetTopicByName(name string) (*Topic, error) {
 	return &topic, nil
 }
 
+func GetAllUsedTopics() ([]*Topic, error) {
+	topics := make([]*Topic, 0)
+	err := x.Where("repo_count > ?", 0).Find(&topics)
+	return topics, err
+
+}
+
 // addTopicByNameToRepo adds a topic name to a repo and increments the topic count.
 // Returns topic after the addition
 func addTopicByNameToRepo(e Engine, repoID int64, topicName string) (*Topic, error) {

models/user.go

@@ -2071,6 +2071,18 @@ func SyncExternalUsers(ctx context.Context, updateExisting bool) error {
 	return nil
 }
 
+func GetUsersCount() (int64, error) {
+	user := new(User)
+	return x.Where("type=0").Count(user)
+
+}
+
+func GetOrganizationsCount() (int64, error) {
+	user := new(User)
+	return x.Where("type=1").Count(user)
+
+}
+
 func GetBlockChainUnSuccessUsers() ([]*User, error) {
 	users := make([]*User, 0, 10)
 	err := x.Where("public_key = ''").

modules/cron/tasks_basic.go

@@ -174,6 +174,16 @@ func registerHandleRepoStatistic() {
 	})
 }
 
+func registerHandleSummaryStatistic() {
+	RegisterTaskFatal("handle_summary_statistic", &BaseConfig{
+		Enabled:    true,
+		RunAtStart: false,
+		Schedule:   "@daily",
+	}, func(ctx context.Context, _ *models.User, _ Config) error {
+		repo.SummaryStatistic()
+		return nil
+	})
+}
 func registerHandleUserStatistic() {
 	RegisterTaskFatal("handle_user_statistic", &BaseConfig{
 		Enabled:    true,
@@ -202,4 +212,5 @@ func initBasicTasks() {
 
 	registerHandleRepoStatistic()
 	registerHandleUserStatistic()
+	registerHandleSummaryStatistic()
 }

options/locale/locale_zh-CN.ini

@@ -776,6 +776,7 @@ cloudbrain_creator=创建者
 cloudbrain_task=任务名称
 cloudbrain_operate=操作
 cloudbrain_status_createtime=状态/创建时间
+cloudbrain_jobname_err=只能以小写字母或数字开头且只包含小写字母、数字、_和-，不能以_结尾，最长36个字符。
 
 template.items=模板选项
 template.git_content=Git数据(默认分支)

routers/repo/cloudbrain.go

@@ -40,6 +40,8 @@ var (
 	categories *models.Categories
 )
 
+var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`)
+
 // MustEnableDataset check if repository enable internal cb
 func MustEnableCloudbrain(ctx *context.Context) {
 	if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) {
@@ -200,6 +202,11 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
 	gpuQueue := setting.JobType
 	codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
 	resourceSpecId := form.ResourceSpecId
+	
+	if !jobNamePattern.MatchString(jobName) {
+		ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplModelArtsNew, &form)
+		return
+	}
 
 	if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) {
 		log.Error("jobtype error:", jobType, ctx.Data["MsgID"])
@@ -281,17 +288,30 @@ func CloudBrainShow(ctx *context.Context) {
 	if result != nil {
 		jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
 		jobRes.Resource.Memory = strings.ReplaceAll(jobRes.Resource.Memory, "Mi", "MB")
-		ctx.Data["result"] = jobRes
 		taskRoles := jobRes.TaskRoles
-		taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
-		ctx.Data["taskRes"] = taskRes
-		task.Status = taskRes.TaskStatuses[0].State
-		task.ContainerID = taskRes.TaskStatuses[0].ContainerID
-		task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
-		err = models.UpdateJob(task)
-		if err != nil {
-			ctx.Data["error"] = err.Error()
+		if jobRes.JobStatus.State != string(models.JobFailed) {
+			taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
+			ctx.Data["taskRes"] = taskRes
+			task.Status = taskRes.TaskStatuses[0].State
+			task.ContainerID = taskRes.TaskStatuses[0].ContainerID
+			task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
+			err = models.UpdateJob(task)
+			if err != nil {
+				ctx.Data["error"] = err.Error()
+			}
+		} else {
+			task.Status = jobRes.JobStatus.State
+			taskRes := models.TaskPod{TaskStatuses: []models.TaskStatuses{
+				{
+					State: jobRes.JobStatus.State,
+				},
+			}}
+			ctx.Data["taskRes"] = taskRes
+			jobRes.JobStatus.StartTime = time.Unix(int64(task.CreatedUnix), 0).Format("2006-01-02 15:04:05")
+			jobRes.JobStatus.EndTime = time.Unix(int64(task.UpdatedUnix), 0).Format("2006-01-02 15:04:05")
 		}
+
+		ctx.Data["result"] = jobRes
 	}
 
 	ctx.Data["task"] = task
@@ -351,7 +371,7 @@ func CloudBrainStop(ctx *context.Context) {
 		return
 	}
 
-	if task.Status == string(models.JobStopped) {
+	if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) {
 		log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"])
 		ctx.ServerError("the job has been stopped", errors.New("the job has been stopped"))
 		return
@@ -454,7 +474,7 @@ func CloudBrainDel(ctx *context.Context) {
 		return
 	}
 
-	if task.Status != string(models.JobStopped) {
+	if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed){
 		log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"])
 		ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped"))
 		return

routers/repo/modelarts.go

@@ -100,7 +100,10 @@ func ModelArtsCreate(ctx *context.Context, form auth.CreateModelArtsForm) {
 	uuid := form.Attachment
 	description := form.Description
 	//repo := ctx.Repo.Repository
-
+	if !jobNamePattern.MatchString(jobName) {
+		ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplModelArtsNew, &form)
+		return
+	}
 	err := modelarts.GenerateTask(ctx, jobName, uuid, description)
 	if err != nil {
 		ctx.RenderWithErr(err.Error(), tplModelArtsNew, &form)

routers/repo/repo_summary_statistic.go

@@ -0,0 +1,94 @@
+package repo
+
+import (
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+)
+
+func SummaryStatistic() {
+	log.Info("Generate summary statistic begin")
+	yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
+	SummaryStatisticDaily(yesterday)
+	log.Info("Generate summary statistic end")
+}
+
+func SummaryStatisticDaily(date string) {
+	log.Info("%s", date)
+	if err := models.DeleteSummaryStatisticDaily(date); err != nil {
+		log.Error("DeleteRepoStatDaily failed: %v", err.Error())
+		return
+	}
+
+	//user number
+	userNumber, err := models.GetUsersCount()
+	if err != nil {
+		log.Error("can not get user number", err)
+		userNumber = 0
+	}
+	//organization number
+	organizationNumber, err := models.GetOrganizationsCount()
+	if err != nil {
+		log.Error("can not get orgnazition number", err)
+		organizationNumber = 0
+	}
+	// repository number
+	repositoryNumer, err := models.GetAllRepositoriesCount()
+	if err != nil {
+		log.Error("can not get repository number", err)
+		repositoryNumer = 0
+	}
+	//repository size
+	repositorySize, err := models.GetAllRepositoriesSize()
+	if err != nil {
+		log.Error("can not get repository size", err)
+		repositorySize = 0
+	}
+	// dataset size
+	allDatasetSize, err := models.GetAllAttachmentSize()
+	if err != nil {
+		log.Error("can not get dataset size", err)
+		allDatasetSize = 0
+	}
+	//topic repo number
+	topics, err := models.GetAllUsedTopics()
+	if err != nil {
+		log.Error("can not get topics", err)
+	}
+	var topicsCount [11]int
+	for _, topic := range topics {
+
+		index, exists := models.DomainMap[topic.Name]
+		if exists {
+			topicsCount[index] = topic.RepoCount
+		}
+
+	}
+
+	summaryStat := models.SummaryStatistic{
+		Date:               date,
+		NumUsers:           userNumber,
+		RepoSize:           repositorySize,
+		DatasetSize:        allDatasetSize,
+		NumOrganizations:   organizationNumber,
+		NumRepos:           repositoryNumer,
+		NumRepoBigModel:    topicsCount[0],
+		NumRepoAI:          topicsCount[1],
+		NumRepoVision:      topicsCount[2],
+		NumRepoNLP:         topicsCount[3],
+		NumRepoML:          topicsCount[4],
+		NumRepoNN:          topicsCount[5],
+		NumRepoAutoDrive:   topicsCount[6],
+		NumRepoRobot:       topicsCount[7],
+		NumRepoLeagueLearn: topicsCount[8],
+		NumRepoDataMining:  topicsCount[9],
+		NumRepoRISC:        topicsCount[10],
+	}
+
+	if _, err = models.InsertSummaryStatistic(&summaryStat); err != nil {
+		log.Error("Insert summary Stat failed: %v", err.Error())
+	}
+
+	log.Info("finish summary statistic")
+}

templates/explore/dataset_list.tmpl

@@ -29,8 +29,12 @@
             {{.Repo.OwnerName}} / {{.Title}}
           </a>
           <div class="ui right metas">
-            <span class="text grey">{{svg "octicon-tasklist" 16}} {{$.i18n.Tr (printf "dataset.task.%s" .Task)}}</span>
-            <span class="text grey">{{svg "octicon-tag" 16}}{{$.i18n.Tr (printf "dataset.category.%s" .Category)}}</span>
+            {{if .Task}}
+              <span class="text grey">{{svg "octicon-tasklist" 16}} {{$.i18n.Tr (printf "dataset.task.%s" .Task)}}</span>
+            {{end}}
+            {{if .Category}}
+              <span class="text grey">{{svg "octicon-tag" 16}}{{$.i18n.Tr (printf "dataset.category.%s" .Category)}}</span>
+            {{end}}
             <span class="text grey">{{svg "octicon-flame" 16}} {{.DownloadTimes}}</span>
           </div>
         </div>

templates/repo/cloudbrain/index.tmpl

@@ -337,9 +337,9 @@
                                             调试
                                         </a>
                                         
-                                        <form id="stopForm-{{.JobID}}" action="{{if eq .Status "STOPPED"}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/stop{{end}}" method="post" style="margin-left:-1px;">
+                                        <form id="stopForm-{{.JobID}}" action="{{if or (eq .Status "STOPPED") (eq .Status "FAILED")}}javascript:void(0){{else}}{{$.Link}}/{{.JobID}}/stop{{end}}" method="post" style="margin-left:-1px;">
                                             {{$.CsrfTokenHtml}}
-                                            <a class="ui basic {{if eq .Status "STOPPED"}}disabled {{else}}blue {{end}}button" onclick="document.getElementById('stopForm-{{.JobID}}').submit();">
+                                            <a class="ui basic {{if or (eq .Status "STOPPED") (eq .Status "FAILED")}}disabled {{else}}blue {{end}}button" onclick="document.getElementById('stopForm-{{.JobID}}').submit();">
                                                 停止
                                             </a>
                                         </form>