diff --git a/models/attachment.go b/models/attachment.go index 684a38b21..96594bbb3 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -473,3 +473,7 @@ func GetAttachmentSizeByDatasetID(datasetID int64) (int64, error) { return total, nil } + +func GetAllAttachmentSize() (int64, error) { + return x.SumInt(&Attachment{}, "size") +} diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 4b2bec8e6..11defbb21 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -27,6 +27,7 @@ const ( JobTypeDebug JobType = "DEBUG" JobTypeBenchmark JobType = "BENCHMARK" JobTypeSnn4imagenet JobType = "SNN4IMAGENET" + JobTypeBrainScore JobType = "BRAINSCORE" ModelArtsCreateQueue ModelArtsJobStatus = "CREATE_QUEUING" //免费资源创建排队中 ModelArtsCreating ModelArtsJobStatus = "CREATING" //创建中 @@ -150,23 +151,42 @@ type TaskPod struct { TaskRoleStatus struct { Name string `json:"name"` } `json:"taskRoleStatus"` - TaskStatuses []struct { - TaskIndex int `json:"taskIndex"` - PodUID string `json:"podUid"` - PodIP string `json:"podIp"` - PodName string `json:"podName"` - ContainerID string `json:"containerId"` - ContainerIP string `json:"containerIp"` - ContainerGpus string `json:"containerGpus"` - State string `json:"state"` - StartAt time.Time `json:"startAt"` - FinishedAt time.Time `json:"finishedAt"` - ExitCode int `json:"exitCode"` - ExitDiagnostics string `json:"exitDiagnostics"` - RetriedCount int `json:"retriedCount"` - StartTime string - FinishedTime string - } `json:"taskStatuses"` + //TaskStatuses []struct { + // TaskIndex int `json:"taskIndex"` + // PodUID string `json:"podUid"` + // PodIP string `json:"podIp"` + // PodName string `json:"podName"` + // ContainerID string `json:"containerId"` + // ContainerIP string `json:"containerIp"` + // ContainerGpus string `json:"containerGpus"` + // State string `json:"state"` + // StartAt time.Time `json:"startAt"` + // FinishedAt time.Time `json:"finishedAt"` + // ExitCode int `json:"exitCode"` + // ExitDiagnostics string `json:"exitDiagnostics"` + // RetriedCount int `json:"retriedCount"` + // StartTime string + // FinishedTime string + //} `json:"taskStatuses"` + TaskStatuses []TaskStatuses `json:"taskStatuses"` +} + +type TaskStatuses struct { + TaskIndex int `json:"taskIndex"` + PodUID string `json:"podUid"` + PodIP string `json:"podIp"` + PodName string `json:"podName"` + ContainerID string `json:"containerId"` + ContainerIP string `json:"containerIp"` + ContainerGpus string `json:"containerGpus"` + State string `json:"state"` + StartAt time.Time `json:"startAt"` + FinishedAt time.Time `json:"finishedAt"` + ExitCode int `json:"exitCode"` + ExitDiagnostics string `json:"exitDiagnostics"` + RetriedCount int `json:"retriedCount"` + StartTime string + FinishedTime string } type TaskInfo struct { @@ -254,6 +274,11 @@ func ConvertToJobResultPayload(input map[string]interface{}) (JobResultPayload, err := json.Unmarshal(data, &jobResultPayload) jobResultPayload.JobStatus.StartTime = time.Unix(jobResultPayload.JobStatus.CreatedTime/1000, 0).Format("2006-01-02 15:04:05") jobResultPayload.JobStatus.EndTime = time.Unix(jobResultPayload.JobStatus.CompletedTime/1000, 0).Format("2006-01-02 15:04:05") + + if jobResultPayload.JobStatus.State == string(JobWaiting) { + jobResultPayload.JobStatus.StartTime = "-" + jobResultPayload.JobStatus.EndTime = "-" + } return jobResultPayload, err } @@ -673,7 +698,7 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) { } func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool { - if !isSigned || job.Status != string(JobStopped) { + if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)){ return false } repo, err := GetRepositoryByID(job.RepoID) diff --git a/models/dataset.go b/models/dataset.go index e7160006d..402a548ef 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -139,7 +139,14 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond { if opts.IncludePublic { cond = cond.And(builder.Eq{"dataset.status": DatasetStatusPublic}) if opts.OwnerID > 0 { - cond = cond.Or(builder.Eq{"repository.owner_id": opts.OwnerID}) + if len(opts.Keyword) == 0 { + cond = cond.Or(builder.Eq{"repository.owner_id": opts.OwnerID}) + } else { + subCon := builder.NewCond() + subCon = subCon.And(builder.Eq{"repository.owner_id": opts.OwnerID}, builder.Like{"dataset.title", opts.Keyword}) + cond = cond.Or(subCon) + + } } } else if opts.OwnerID > 0 { cond = cond.And(builder.Eq{"repository.owner_id": opts.OwnerID}) diff --git a/models/models.go b/models/models.go index 412148235..0cdcee4fd 100755 --- a/models/models.go +++ b/models/models.go @@ -137,6 +137,7 @@ func init() { tablesStatistic = append(tablesStatistic, new(RepoStatistic), + new(SummaryStatistic), new(UserBusinessAnalysis), ) diff --git a/models/repo.go b/models/repo.go index 7f4bfebba..dd36bf71d 100755 --- a/models/repo.go +++ b/models/repo.go @@ -1430,6 +1430,15 @@ func GetAllRepositoriesByFilterCols(columns ...string) ([]*Repository, error) { } +func GetAllRepositoriesCount() (int64, error) { + repo := new(Repository) + return x.Count(repo) +} + +func GetAllRepositoriesSize() (int64, error) { + return x.SumInt(&Repository{}, "size") +} + func updateRepository(e Engine, repo *Repository, visibilityChanged bool) (err error) { repo.LowerName = strings.ToLower(repo.Name) diff --git a/models/summary_statistic.go b/models/summary_statistic.go new file mode 100644 index 000000000..0addd472b --- /dev/null +++ b/models/summary_statistic.go @@ -0,0 +1,69 @@ +package models + +import ( + "fmt" + + "code.gitea.io/gitea/modules/timeutil" +) + +var DomainMap = map[string]int{ + "大模型": 0, + "ai开发工具": 1, + "计算机视觉": 2, + "自然语言处理": 3, + "机器学习": 4, + "神经网络": 5, + "自动驾驶": 6, + "机器人": 7, + "联邦学习": 8, + "数据挖掘": 9, + "risc-v开发": 10, +} + +type SummaryStatistic struct { + ID int64 `xorm:"pk autoincr"` + Date string `xorm:"unique(s) NOT NULL"` + NumUsers int64 `xorm:"NOT NULL DEFAULT 0"` + RepoSize int64 `xorm:"NOT NULL DEFAULT 0"` + DatasetSize int64 `xorm:"NOT NULL DEFAULT 0"` + NumOrganizations int64 `xorm:"NOT NULL DEFAULT 0"` + NumModels int64 `xorm:"NOT NULL DEFAULT 0"` + NumRepos int64 `xorm:"NOT NULL DEFAULT 0"` + NumRepoBigModel int `xorm:"NOT NULL DEFAULT 0"` + NumRepoAI int `xorm:"NOT NULL DEFAULT 0"` + NumRepoVision int `xorm:"NOT NULL DEFAULT 0"` + NumRepoNLP int `xorm:"NOT NULL DEFAULT 0"` + NumRepoML int `xorm:"NOT NULL DEFAULT 0"` + NumRepoNN int `xorm:"NOT NULL DEFAULT 0"` + NumRepoAutoDrive int `xorm:"NOT NULL DEFAULT 0"` + NumRepoRobot int `xorm:"NOT NULL DEFAULT 0"` + NumRepoLeagueLearn int `xorm:"NOT NULL DEFAULT 0"` + NumRepoDataMining int `xorm:"NOT NULL DEFAULT 0"` + NumRepoRISC int `xorm:"NOT NULL DEFAULT 0"` + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` +} + +func DeleteSummaryStatisticDaily(date string) error { + sess := xStatistic.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return fmt.Errorf("Begin: %v", err) + } + + if _, err := sess.Where("date = ?", date).Delete(&SummaryStatistic{}); err != nil { + return fmt.Errorf("Delete: %v", err) + } + + if err := sess.Commit(); err != nil { + sess.Close() + return fmt.Errorf("Commit: %v", err) + } + + sess.Close() + return nil +} + +func InsertSummaryStatistic(summaryStatistic *SummaryStatistic) (int64, error) { + return xStatistic.Insert(summaryStatistic) +} diff --git a/models/topic.go b/models/topic.go index b8d3d9d85..5533da7bc 100644 --- a/models/topic.go +++ b/models/topic.go @@ -98,6 +98,13 @@ func GetTopicByName(name string) (*Topic, error) { return &topic, nil } +func GetAllUsedTopics() ([]*Topic, error) { + topics := make([]*Topic, 0) + err := x.Where("repo_count > ?", 0).Find(&topics) + return topics, err + +} + // addTopicByNameToRepo adds a topic name to a repo and increments the topic count. // Returns topic after the addition func addTopicByNameToRepo(e Engine, repoID int64, topicName string) (*Topic, error) { @@ -178,7 +185,7 @@ func (opts *FindTopicOptions) toConds() builder.Cond { } if opts.Keyword != "" { - cond = cond.And(builder.Like{"topic.name", opts.Keyword}) + cond = cond.And(builder.Like{"topic.name", strings.ToLower(opts.Keyword)}) } return cond diff --git a/models/user.go b/models/user.go index 78ab4627a..1ee20d74c 100755 --- a/models/user.go +++ b/models/user.go @@ -2071,6 +2071,18 @@ func SyncExternalUsers(ctx context.Context, updateExisting bool) error { return nil } +func GetUsersCount() (int64, error) { + user := new(User) + return x.Where("type=0").Count(user) + +} + +func GetOrganizationsCount() (int64, error) { + user := new(User) + return x.Where("type=1").Count(user) + +} + func GetBlockChainUnSuccessUsers() ([]*User, error) { users := make([]*User, 0, 10) err := x.Where("public_key = ''"). diff --git a/modules/base/tool.go b/modules/base/tool.go index 8145522e2..cf2972990 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -224,7 +224,7 @@ func SizedAvatarLinkWithDomain(email string, size int) string { // FileSize calculates the file size and generate user-friendly string. func FileSize(s int64) string { - return humanize.IBytes(uint64(s)) + return humanize.Bytes(uint64(s)) } // PrettyNumber produces a string form of the given number in base 10 with diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 0de1db9a6..e09aaf931 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -16,6 +16,7 @@ const ( ModelMountPath = "/model" BenchMarkMountPath = "/benchmark" Snn4imagenetMountPath = "/snn4imagenet" + BrainScoreMountPath = "/brainscore" TaskInfoName = "/taskInfo" SubTaskName = "task1" diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 26cd16778..ed9829cef 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -174,6 +174,16 @@ func registerHandleRepoStatistic() { }) } +func registerHandleSummaryStatistic() { + RegisterTaskFatal("handle_summary_statistic", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@daily", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.SummaryStatistic() + return nil + }) +} func registerHandleUserStatistic() { RegisterTaskFatal("handle_user_statistic", &BaseConfig{ Enabled: true, @@ -202,4 +212,5 @@ func initBasicTasks() { registerHandleRepoStatistic() registerHandleUserStatistic() + registerHandleSummaryStatistic() } diff --git a/modules/repository/elk_pagedata.go b/modules/repository/elk_pagedata.go index bb027726d..03a61be1f 100644 --- a/modules/repository/elk_pagedata.go +++ b/modules/repository/elk_pagedata.go @@ -4,10 +4,10 @@ import ( "bytes" "encoding/base64" "encoding/json" - "fmt" "io/ioutil" "net/http" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" ) @@ -99,8 +99,8 @@ type ResultInfo struct { //elk输出的json结构end -//发送post请求到elk -func SendReqToElk(jsonStr []byte) (content string) { +//处理返回的elk数据，只保留totalView，即访问量；loaded是分片载入次数，用来判断返回的数据是否准确 +func GetResultFromElk(resultInfo ResultInfo, jsonStr []byte) (loaded int, totalView int, err error) { ElkBase64Init := setting.ElkUser + ":" + setting.ElkPassword ElkBase64 := base64.StdEncoding.EncodeToString([]byte(ElkBase64Init)) BasicElkBase64 := "Basic" + " " + ElkBase64 @@ -117,15 +117,11 @@ func SendReqToElk(jsonStr []byte) (content string) { } defer resp.Body.Close() body, _ := ioutil.ReadAll(resp.Body) - return string(body) -} -//处理返回的elk数据，只保留totalView，即访问量；loaded是分片载入次数，用来判断返回的数据是否准确 -func GetResultFromElk(resultinfo ResultInfo, jobResult string) (loaded int, totalView int) { - var resultTest ResultInfo - errs := json.Unmarshal([]byte(jobResult), &resultTest) - fmt.Println(errs) - return resultTest.Result.Loaded, resultTest.Result.RawResponse.Hits.Total + errs := json.Unmarshal([]byte(string(body)), &resultInfo) + log.Info("Get resultJson failed", errs) + + return resultInfo.Result.Loaded, resultInfo.Result.RawResponse.Hits.Total, err } //初始化传给elk的数据结构，给定用户名和项目名，查询的起止时间，返回初始化后的结构 @@ -211,23 +207,23 @@ func TagNameInit(MessageInfo string, Tagname string, Gte string, Lte string) (pr } //向elk发送请求，将获取的结果只保留访问量，输入是初始化后的数据结构，返回访问量 -func ViewInfo(viewInfo InputInfo) (totalView int) { +func ViewInfo(viewInfo InputInfo) (totalView int, err error) { jsons, errs := json.Marshal(viewInfo) if errs != nil { - fmt.Println("errs:", errs.Error()) + log.Info("errs:", errs) } - // fmt.Println("viewInfoInit:",string(jsons)) var jsonStr = []byte(jsons) var resultInfo ResultInfo - loaded, totalView := GetResultFromElk(resultInfo, SendReqToElk(jsonStr)) + loaded, totalView, err := GetResultFromElk(resultInfo, jsonStr) + time := 0 for { if loaded == 0 { - loaded_next, totalView := GetResultFromElk(resultInfo, SendReqToElk(jsonStr)) + loaded_next, totalView, err := GetResultFromElk(resultInfo, jsonStr) time++ + log.Info("time:", time) if loaded_next != 0 && time < 100 { - fmt.Println("totalView:", totalView) - return totalView + return totalView, err } if time > 100 { break @@ -236,20 +232,20 @@ func ViewInfo(viewInfo InputInfo) (totalView int) { break } } - fmt.Println("loaded:", loaded) - return totalView + return totalView, err } -// @title ProjectView +// @title AppointProjectView // @description 获取指定用户和项目的访问量 // @param User string "用户名" // @param Project string "项目名" // @param Gte string "起始时间" 如time.Now().AddDate(0, 0, -1).Format(time.RFC3339) // @param Lte string "结束时间" 如time.Now().Format(time.RFC3339) // @return totalView int "访问量" -func AppointProjectView(User string, Project string, Gte string, Lte string) (totalView int) { - InitInfo := ProjectViewInit(User, Project, Gte, Lte) - return ViewInfo(InitInfo) +func AppointProjectView(User string, Project string, Gte string, Lte string) (totalView int, err error) { + ProjectViewInitInfo := ProjectViewInit(User, Project, Gte, Lte) + ProjectTotalView, err := ViewInfo(ProjectViewInitInfo) + return ProjectTotalView, err } //统计项目相关页面的访问量 @@ -287,26 +283,44 @@ type ProjectInfo struct { Project_forks int } +type ErrorInfo struct { + Project_dataset_type_0 error + Project_dataset_type_1 error + Project_issues error + Project_labels error + Project_milestones error + Project_pulls error + Project_release error + Project_wiki error + Project_activity error + Project_cloudbrain error + Project_modelarts error + Project_blockchain error + Project_watchers error + Project_stars error + Project_forks error +} + // @title AllProjectView // @description 获取指定用户和项目的访问量 // @param Gte string "起始时间" 如time.Now().AddDate(0, 0, -1).Format(time.RFC3339) // @param Lte string "结束时间" // @return projectInfo ProjectInfo "统计所有项目中页面的浏览情况，不需要区分项目" -func AllProjectView(Gte string, Lte string) (projectInfo ProjectInfo) { - projectInfo.Project_dataset_type_0 = ViewInfo(AllProjectViewInit("/datasets?type=0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_dataset_type_1 = ViewInfo(AllProjectViewInit("/datasets?type=1", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_issues = ViewInfo(AllProjectViewInit("/issues HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_labels = ViewInfo(TagNameInit("/labels HTTP/2.0", "labels", Gte, Lte)) - projectInfo.Project_milestones = ViewInfo(AllProjectViewInit("/milestones HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_pulls = ViewInfo(AllProjectViewInit("/pulls HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_release = ViewInfo(AllProjectViewInit("/release HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_wiki = ViewInfo(AllProjectViewInit("/wiki HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_activity = ViewInfo(AllProjectViewInit("/activity HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_cloudbrain = ViewInfo(AllProjectViewInit("/cloudbrain HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_modelarts = ViewInfo(AllProjectViewInit("/modelarts HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_blockchain = ViewInfo(AllProjectViewInit("/blockchain HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_watchers = ViewInfo(AllProjectViewInit("/watchers HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_stars = ViewInfo(AllProjectViewInit("/stars HTTP/2.0", "%{[request][2]}", Gte, Lte)) - projectInfo.Project_forks = ViewInfo(AllProjectViewInit("/forks HTTP/2.0", "%{[request][2]}", Gte, Lte)) - return projectInfo +func AllProjectView(Gte string, Lte string) (projectViewInfo ProjectInfo, errorInfo ErrorInfo) { + projectViewInfo.Project_dataset_type_0, errorInfo.Project_dataset_type_0 = ViewInfo(AllProjectViewInit("/datasets?type=0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_dataset_type_1, errorInfo.Project_dataset_type_1 = ViewInfo(AllProjectViewInit("/datasets?type=1", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_issues, errorInfo.Project_issues = ViewInfo(AllProjectViewInit("/issues HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_labels, errorInfo.Project_labels = ViewInfo(TagNameInit("/labels HTTP/2.0", "labels", Gte, Lte)) + projectViewInfo.Project_milestones, errorInfo.Project_milestones = ViewInfo(AllProjectViewInit("/milestones HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_pulls, errorInfo.Project_pulls = ViewInfo(AllProjectViewInit("/pulls HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_release, errorInfo.Project_release = ViewInfo(AllProjectViewInit("/release HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_wiki, errorInfo.Project_wiki = ViewInfo(AllProjectViewInit("/wiki HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_activity, errorInfo.Project_activity = ViewInfo(AllProjectViewInit("/activity HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_cloudbrain, errorInfo.Project_cloudbrain = ViewInfo(AllProjectViewInit("/cloudbrain HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_modelarts, errorInfo.Project_modelarts = ViewInfo(AllProjectViewInit("/modelarts HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_blockchain, errorInfo.Project_blockchain = ViewInfo(AllProjectViewInit("/blockchain HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_watchers, errorInfo.Project_watchers = ViewInfo(AllProjectViewInit("/watchers HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_stars, errorInfo.Project_stars = ViewInfo(AllProjectViewInit("/stars HTTP/2.0", "%{[request][2]}", Gte, Lte)) + projectViewInfo.Project_forks, errorInfo.Project_forks = ViewInfo(AllProjectViewInit("/forks HTTP/2.0", "%{[request][2]}", Gte, Lte)) + return projectViewInfo, errorInfo } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 509483223..f9674ddc2 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -457,6 +457,11 @@ var ( Snn4imagenetCode string Snn4imagenetServerHost string + //snn4imagenet config + IsBrainScoreEnabled bool + BrainScoreCode string + BrainScoreServerHost string + //blockchain config BlockChainHost string CommitValidDate string @@ -1184,8 +1189,13 @@ func NewContext() { sec = Cfg.Section("snn4imagenet") IsSnn4imagenetEnabled = sec.Key("ENABLED").MustBool(false) - Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("https://yangzhx:justfortest123@git.openi.org.cn/yangzhx/detection_benchmark_script.git") - Snn4imagenetServerHost = sec.Key("HOST").MustString("http://192.168.202.90:3366/") + Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("https://yult:19910821ylt@git.openi.org.cn/yult/snn4imagenet_script.git") + Snn4imagenetServerHost = sec.Key("HOST").MustString("http://192.168.207.76:8080/") + + sec = Cfg.Section("brainscore") + IsBrainScoreEnabled = sec.Key("ENABLED").MustBool(false) + BrainScoreCode = sec.Key("BRAINSCORECODE").MustString("https://yult:19910821ylt@git.openi.org.cn/yult/brainscore_script.git") + BrainScoreServerHost = sec.Key("HOST").MustString("http://192.168.207.76:8080/") sec = Cfg.Section("blockchain") BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/") diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 6dc0d410c..9ccf9c5c6 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -776,6 +776,7 @@ cloudbrain_creator=创建者 cloudbrain_task=任务名称 cloudbrain_operate=操作 cloudbrain_status_createtime=状态/创建时间 +cloudbrain_jobname_err=只能以小写字母或数字开头且只包含小写字母、数字、_和-，不能以_结尾，最长36个字符。 template.items=模板选项 template.git_content=Git数据(默认分支) diff --git a/public/img/org-jd@2x-80.jpg b/public/img/org-jd@2x-80.jpg new file mode 100644 index 000000000..52a0a5d50 Binary files /dev/null and b/public/img/org-jd@2x-80.jpg differ diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index f4364f1e4..bfba5236b 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -65,12 +65,15 @@ func GetCloudbrainTask(ctx *context.APIContext) { return } - taskRoles := result.TaskRoles - taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + job.Status = result.JobStatus.State + if result.JobStatus.State != string(models.JobWaiting) && result.JobStatus.State != string(models.JobFailed) { + taskRoles := result.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) - job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP - job.ContainerID = taskRes.TaskStatuses[0].ContainerID - job.Status = taskRes.TaskStatuses[0].State + job.ContainerIp = taskRes.TaskStatuses[0].ContainerIP + job.ContainerID = taskRes.TaskStatuses[0].ContainerID + job.Status = taskRes.TaskStatuses[0].State + } if result.JobStatus.State != string(models.JobWaiting) { err = models.UpdateJob(job) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 648061171..0c83c99bd 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -40,6 +40,8 @@ var ( categories *models.Categories ) +var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`) + // MustEnableDataset check if repository enable internal cb func MustEnableCloudbrain(ctx *context.Context) { if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) { @@ -175,6 +177,9 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error{ ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled + ctx.Data["brainscore_path"] = cloudbrain.BrainScoreMountPath + ctx.Data["is_brainscore_enabled"] = setting.IsBrainScoreEnabled + return nil } @@ -197,8 +202,13 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { gpuQueue := setting.JobType codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath resourceSpecId := form.ResourceSpecId + + if !jobNamePattern.MatchString(jobName) { + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplModelArtsNew, &form) + return + } - if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) { + if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("jobtype error", tplCloudBrainNew, &form) @@ -247,6 +257,11 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "") } + brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath + if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { + downloadRateCode(repo, jobName, setting.BrainScoreCode, brainScorePath, "", "") + } + err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, jobType, gpuQueue, resourceSpecId) if err != nil { cloudBrainNewDataPrepare(ctx) @@ -273,17 +288,30 @@ func CloudBrainShow(ctx *context.Context) { if result != nil { jobRes, _ := models.ConvertToJobResultPayload(result.Payload) jobRes.Resource.Memory = strings.ReplaceAll(jobRes.Resource.Memory, "Mi", "MB") - ctx.Data["result"] = jobRes taskRoles := jobRes.TaskRoles - taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) - ctx.Data["taskRes"] = taskRes - task.Status = taskRes.TaskStatuses[0].State - task.ContainerID = taskRes.TaskStatuses[0].ContainerID - task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP - err = models.UpdateJob(task) - if err != nil { - ctx.Data["error"] = err.Error() + if jobRes.JobStatus.State != string(models.JobFailed) { + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + ctx.Data["taskRes"] = taskRes + task.Status = taskRes.TaskStatuses[0].State + task.ContainerID = taskRes.TaskStatuses[0].ContainerID + task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP + err = models.UpdateJob(task) + if err != nil { + ctx.Data["error"] = err.Error() + } + } else { + task.Status = jobRes.JobStatus.State + taskRes := models.TaskPod{TaskStatuses: []models.TaskStatuses{ + { + State: jobRes.JobStatus.State, + }, + }} + ctx.Data["taskRes"] = taskRes + jobRes.JobStatus.StartTime = time.Unix(int64(task.CreatedUnix), 0).Format("2006-01-02 15:04:05") + jobRes.JobStatus.EndTime = time.Unix(int64(task.UpdatedUnix), 0).Format("2006-01-02 15:04:05") } + + ctx.Data["result"] = jobRes } ctx.Data["task"] = task @@ -343,7 +371,7 @@ func CloudBrainStop(ctx *context.Context) { return } - if task.Status == string(models.JobStopped) { + if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) { log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"]) ctx.ServerError("the job has been stopped", errors.New("the job has been stopped")) return @@ -568,6 +596,8 @@ func GetRate(ctx *context.Context) { ctx.Redirect(setting.BenchmarkServerHost + "?username=" + ctx.User.Name) } else if job.JobType == string(models.JobTypeSnn4imagenet) { ctx.Redirect(setting.Snn4imagenetServerHost) + } else if job.JobType == string(models.JobTypeBrainScore) { + ctx.Redirect(setting.BrainScoreServerHost) } else { log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"]) } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 080c36377..ce6cfb4d5 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -100,7 +100,10 @@ func ModelArtsCreate(ctx *context.Context, form auth.CreateModelArtsForm) { uuid := form.Attachment description := form.Description //repo := ctx.Repo.Repository - + if !jobNamePattern.MatchString(jobName) { + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplModelArtsNew, &form) + return + } err := modelarts.GenerateTask(ctx, jobName, uuid, description) if err != nil { ctx.RenderWithErr(err.Error(), tplModelArtsNew, &form) diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index ceb410958..6ee5c9a59 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -5,6 +5,7 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/repository" ) //auto daily or manually @@ -62,9 +63,12 @@ func RepoStatisticDaily(date string) { continue } - //beginTime, endTime := getStatTime(date) - //numVisits := repository.AppointProjectView(repo.OwnerName, repo.Name, beginTime, endTime) - numVisits := 0 + beginTime, endTime := getStatTime(date) + numVisits, err := repository.AppointProjectView(repo.OwnerName, repo.Name, beginTime, endTime) + if err != nil { + log.Error("Get numVisits failed", err) + numVisits = 0 + } repoStat := models.RepoStatistic{ RepoID: repo.ID, diff --git a/routers/repo/repo_summary_statistic.go b/routers/repo/repo_summary_statistic.go new file mode 100644 index 000000000..53270664c --- /dev/null +++ b/routers/repo/repo_summary_statistic.go @@ -0,0 +1,94 @@ +package repo + +import ( + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" +) + +func SummaryStatistic() { + log.Info("Generate summary statistic begin") + yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + SummaryStatisticDaily(yesterday) + log.Info("Generate summary statistic end") +} + +func SummaryStatisticDaily(date string) { + log.Info("%s", date) + if err := models.DeleteSummaryStatisticDaily(date); err != nil { + log.Error("DeleteRepoStatDaily failed: %v", err.Error()) + return + } + + //user number + userNumber, err := models.GetUsersCount() + if err != nil { + log.Error("can not get user number", err) + userNumber = 0 + } + //organization number + organizationNumber, err := models.GetOrganizationsCount() + if err != nil { + log.Error("can not get orgnazition number", err) + organizationNumber = 0 + } + // repository number + repositoryNumer, err := models.GetAllRepositoriesCount() + if err != nil { + log.Error("can not get repository number", err) + repositoryNumer = 0 + } + //repository size + repositorySize, err := models.GetAllRepositoriesSize() + if err != nil { + log.Error("can not get repository size", err) + repositorySize = 0 + } + // dataset size + allDatasetSize, err := models.GetAllAttachmentSize() + if err != nil { + log.Error("can not get dataset size", err) + allDatasetSize = 0 + } + //topic repo number + topics, err := models.GetAllUsedTopics() + if err != nil { + log.Error("can not get topics", err) + } + var topicsCount [11]int + for _, topic := range topics { + + index, exists := models.DomainMap[topic.Name] + if exists { + topicsCount[index] = topic.RepoCount + } + + } + + summaryStat := models.SummaryStatistic{ + Date: date, + NumUsers: userNumber, + RepoSize: repositorySize, + DatasetSize: allDatasetSize, + NumOrganizations: organizationNumber, + NumRepos: repositoryNumer, + NumRepoBigModel: topicsCount[0], + NumRepoAI: topicsCount[1], + NumRepoVision: topicsCount[2], + NumRepoNLP: topicsCount[3], + NumRepoML: topicsCount[4], + NumRepoNN: topicsCount[5], + NumRepoAutoDrive: topicsCount[6], + NumRepoRobot: topicsCount[7], + NumRepoLeagueLearn: topicsCount[8], + NumRepoDataMining: topicsCount[9], + NumRepoRISC: topicsCount[10], + } + + if _, err = models.InsertSummaryStatistic(&summaryStat); err != nil { + log.Error("Insert summary Stat failed: %v", err.Error()) + } + + log.Info("finish summary statistic") +} diff --git a/routers/user/auth.go b/routers/user/auth.go index 126d0a4c8..13e338565 100755 --- a/routers/user/auth.go +++ b/routers/user/auth.go @@ -11,8 +11,6 @@ import ( "net/http" "strings" - "code.gitea.io/gitea/routers/repo" - "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/auth/oauth2" @@ -1058,7 +1056,6 @@ func SignOut(ctx *context.Context) { }) } HandleSignOut(ctx) - go repo.StopJobsByUserID(ctx.User.ID) ctx.Redirect(setting.AppSubURL + "/") } diff --git a/routers/user/setting/account.go b/routers/user/setting/account.go index 6165bfc5b..0a0fc558c 100644 --- a/routers/user/setting/account.go +++ b/routers/user/setting/account.go @@ -8,6 +8,8 @@ package setting import ( "errors" + "code.gitea.io/gitea/routers/repo" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" @@ -240,6 +242,7 @@ func DeleteAccount(ctx *context.Context) { ctx.ServerError("DeleteUser", err) } } else { + go repo.StopJobsByUserID(ctx.User.ID) log.Trace("Account deleted: %s", ctx.User.Name) ctx.Redirect(setting.AppSubURL + "/") } diff --git a/templates/explore/repo_orgtop.tmpl b/templates/explore/repo_orgtop.tmpl index f0e6d9118..df132e811 100755 --- a/templates/explore/repo_orgtop.tmpl +++ b/templates/explore/repo_orgtop.tmpl @@ -50,6 +50,13 @@ +
+
+ + 京东 + +
+
diff --git a/templates/repo/cloudbrain/index.tmpl b/templates/repo/cloudbrain/index.tmpl index 3af96998c..3d9c080db 100755 --- a/templates/repo/cloudbrain/index.tmpl +++ b/templates/repo/cloudbrain/index.tmpl @@ -337,9 +337,9 @@ 调试 -
+ {{$.CsrfTokenHtml}} - + 停止
@@ -491,7 +491,7 @@ console.log("---------",index,job) const jobID = job.dataset.jobid; const repoPath = job.dataset.repopath; - if (job.textContent.trim() == 'STOPPED') { + if (job.textContent.trim() == 'STOPPED' || job.textContent.trim() == 'FAILED') { return } @@ -499,7 +499,6 @@ $.get(`/api/v1/repos/${repoPath}/cloudbrain/${jobID}`, (data) => { const jobID = data.JobID const status = data.JobStatus - console.log("status",status) if (status != job.textContent.trim()) { //$('#' + jobID).text(status) //if (status == 'STOPPED') { diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index c45776b89..42a0e55ad 100755 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -89,6 +89,10 @@ display: none; } + .inline.required.field.cloudbrain_brainscore { + display: none; + } + .select2-container .select2-selection--single{ height:38px !important; } @@ -130,12 +134,13 @@
-
+
@@ -215,6 +220,10 @@
+
+ + +
- 点击或回车添加{{input}}标签 + 点击或回车添加{{input.toLowerCase()}}标签
-
{{input}}
+
{{input.toLowerCase()}}
@@ -134,7 +134,7 @@ export default { this.showSearchTopic = true } - else if(this.arrayTopics.indexOf(this.input)>-1){ + else if(this.arrayTopics.indexOf(this.input.toLowerCase())>-1){ this.showInputValue = false this.showSearchTopic = false @@ -142,7 +142,7 @@ export default { this.showInitTopic = [] let timestamp=new Date().getTime() - this.params.q = this.input + this.params.q = this.input.toLowerCase() this.params._ = timestamp this.$axios.get('/api/v1/topics/search',{ params:this.params @@ -224,11 +224,11 @@ export default { return }else{ let topic = this.input - if(this.arrayTopics.includes(topic)){ + if(this.arrayTopics.includes(topic.toLowerCase())){ return } else{ - this.arrayTopics.push(topic) + this.arrayTopics.push(topic.toLowerCase()) let topics = this.arrayTopics let strTopics = topics.join(',') @@ -268,7 +268,7 @@ export default { } else if(!this.showAddFlage){ let topic = this.input - this.arrayTopics.push(topic) + this.arrayTopics.push(topic.toLowerCase()) let topics = this.arrayTopics let strTopics = topics.join(',')