diff --git a/models/attachment.go b/models/attachment.go index bb99ba467..6fb98a07d 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -82,6 +82,7 @@ type AttachmentsOptions struct { NeedRepoInfo bool Keyword string RecommendOnly bool + UserId int64 } func (a *Attachment) AfterUpdate() { diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 1629acfeb..4840b78c0 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -24,6 +24,7 @@ type ModelArtsJobStatus string const ( TypeCloudBrainOne int = iota TypeCloudBrainTwo + TypeIntelligentNet TypeCloudBrainAll = -1 ) @@ -328,6 +329,11 @@ type CloudbrainsOptions struct { JobTypeNot bool NeedRepoInfo bool RepoIDList []int64 + BeginTime time.Time + EndTime time.Time + ComputeResource string + BeginTimeUnix int64 + EndTimeUnix int64 } type TaskPod struct { @@ -1183,6 +1189,11 @@ func Cloudbrains(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { builder.Eq{"cloudbrain.job_id": opts.JobID}, ) } + if (opts.ComputeResource) != "" { + cond = cond.And( + builder.Eq{"cloudbrain.compute_resource": opts.ComputeResource}, + ) + } if (opts.Type) >= 0 { cond = cond.And( @@ -1504,6 +1515,11 @@ func UpdateJob(job *Cloudbrain) error { return updateJob(x, job) } +func UpdateJobDurationWithDeleted(job *Cloudbrain) error { + _, err := x.Exec("update cloudbrain set start_time=?, end_time=?,train_job_duration=?,duration=? where id=?", job.StartTime, job.EndTime, job.TrainJobDuration, job.Duration, job.ID) + return err +} + func updateJob(e Engine, job *Cloudbrain) error { _, err := e.ID(job.ID).AllCols().Update(job) return err @@ -1574,6 +1590,10 @@ func GetStoppedJobWithNoDurationJob() ([]*Cloudbrain, error) { Limit(100). Find(&cloudbrains) } +func GetStoppedJobWithNoStartTimeEndTime() ([]*Cloudbrain, error) { + cloudbrains := make([]*Cloudbrain, 0) + return cloudbrains, x.SQL("select * from cloudbrain where status in (?,?,?,?,?,?,?) and (start_time is null or end_time is null) limit 100", ModelArtsTrainJobCompleted, ModelArtsTrainJobFailed, ModelArtsTrainJobKilled, ModelArtsStopped, JobStopped, JobFailed, JobSucceeded).Find(&cloudbrains) +} func GetCloudbrainCountByUserID(userID int64, jobType string) (int, error) { count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ? and type = ?", jobType, userID, TypeCloudBrainOne).Count(new(Cloudbrain)) @@ -1650,13 +1670,80 @@ func RestartCloudbrain(old *Cloudbrain, new *Cloudbrain) (err error) { func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { sess := x.NewSession() defer sess.Close() + var cond = builder.NewCond() + if opts.RepoID > 0 { + cond = cond.And( + builder.Eq{"cloudbrain.repo_id": opts.RepoID}, + ) + } + + if opts.UserID > 0 { + cond = cond.And( + builder.Eq{"cloudbrain.user_id": opts.UserID}, + ) + } + + if (opts.JobID) != "" { + cond = cond.And( + builder.Eq{"cloudbrain.job_id": opts.JobID}, + ) + } + if (opts.ComputeResource) != "" { + cond = cond.And( + builder.Eq{"cloudbrain.compute_resource": opts.ComputeResource}, + ) + } + if (opts.Type) >= 0 { cond = cond.And( builder.Eq{"cloudbrain.type": opts.Type}, ) } + if len(opts.JobTypes) > 0 { + if opts.JobTypeNot { + cond = cond.And( + builder.NotIn("cloudbrain.job_type", opts.JobTypes), + ) + } else { + cond = cond.And( + builder.In("cloudbrain.job_type", opts.JobTypes), + ) + } + } + + if (opts.IsLatestVersion) != "" { + cond = cond.And(builder.Or(builder.And(builder.Eq{"cloudbrain.is_latest_version": opts.IsLatestVersion}, builder.Eq{"cloudbrain.job_type": "TRAIN"}), builder.Neq{"cloudbrain.job_type": "TRAIN"})) + } + + if len(opts.CloudbrainIDs) > 0 { + cond = cond.And(builder.In("cloudbrain.id", opts.CloudbrainIDs)) + } + + if len(opts.JobStatus) > 0 { + if opts.JobStatusNot { + cond = cond.And( + builder.NotIn("cloudbrain.status", opts.JobStatus), + ) + } else { + cond = cond.And( + builder.In("cloudbrain.status", opts.JobStatus), + ) + } + } + if len(opts.RepoIDList) > 0 { + cond = cond.And( + builder.In("cloudbrain.repo_id", opts.RepoIDList), + ) + + } + if opts.BeginTimeUnix > 0 && opts.EndTimeUnix > 0 { + cond = cond.And( + builder.And(builder.Gte{"cloudbrain.created_unix": opts.BeginTimeUnix}, builder.Lte{"cloudbrain.created_unix": opts.EndTimeUnix}), + ) + } + var count int64 var err error condition := "cloudbrain.user_id = `user`.id" diff --git a/models/cloudbrain_static.go b/models/cloudbrain_static.go index 31f66d4fc..27cccf415 100644 --- a/models/cloudbrain_static.go +++ b/models/cloudbrain_static.go @@ -1,6 +1,191 @@ package models -import "code.gitea.io/gitea/modules/log" +import ( + "strconv" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" + "xorm.io/builder" +) + +type TaskDetail struct { + ID int64 `json:"ID"` + JobID string `json:"JobID"` + JobName string `json:"JobName"` + DisplayJobName string `json:"DisplayJobName"` + Status string `json:"Status"` + JobType string `json:"JobType"` + CreatedUnix timeutil.TimeStamp `json:"CreatedUnix"` + WaitTime string `json:"WaitTime"` + RunTime string `json:"RunTime"` + StartTime timeutil.TimeStamp `json:"StartTime"` + EndTime timeutil.TimeStamp `json:"EndTime"` + ComputeResource string `json:"ComputeResource"` + Type int `json:"Type"` + UserName string `json:"UserName"` + RepoName string `json:"RepoName"` + RepoAlias string `json:"RepoAlias"` + RepoID int64 `json:"RepoID"` + IsDelete bool `json:"IsDelete"` +} + +func GetDebugOnePeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type ='" + string(JobTypeDebug) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" + + return x.SQL(countSql).Count() +} +func GetDebugOnePeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeDebug, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + + return total, nil +} + +func GetTrainOnePeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type ='" + string(JobTypeTrain) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" + + return x.SQL(countSql).Count() +} +func GetTrainOnePeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeTrain, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + + return total, nil +} + +func GetBenchmarkOnePeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type ='" + string(JobTypeBenchmark) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" + return x.SQL(countSql).Count() +} +func GetBenchmarkOnePeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeBenchmark, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + + return total, nil +} +func GetDebugTwoPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type ='" + string(JobTypeDebug) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" + return x.SQL(countSql).Count() +} +func GetDebugTwoPeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeDebug, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + return total, nil +} +func GetTrainTwoPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type ='" + string(JobTypeTrain) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" + return x.SQL(countSql).Count() +} +func GetTrainTwoPeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeTrain, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + return total, nil +} +func GetInferenceTwoPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and job_type ='" + string(JobTypeInference) + "'" + + " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" + return x.SQL(countSql).Count() +} +func GetInferenceTwoPeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeInference, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + return total, nil +} + +func GetCloudBrainOnePeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" + return x.SQL(countSql).Count() +} +func GetCloudBrainOnePeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + return total, nil +} +func GetCloudBrainTwoPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + + " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" + return x.SQL(countSql).Count() +} +func GetCloudBrainTwoPeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { + total, err := x.Where("created_unix >= ? And created_unix < ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration") + if err != nil { + return 0, err + } + return total, nil +} + +func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count(distinct user_id) FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + return x.SQL(countSql).Count() +} +func GetTodayCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, error) { + countSql := "SELECT count FROM " + + "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + + " and created_unix<=" + strconv.FormatInt(endTime.Unix(), 10) + return x.SQL(countSql).Count() +} +func GetCreatorCount() (int64, error) { + countSql := "SELECT count(distinct user_id) FROM public.cloudbrain" + return x.SQL(countSql).Count() +} + +func GetRecordBeginTime() ([]*CloudbrainInfo, error) { + sess := x.NewSession() + defer sess.Close() + sess.OrderBy("cloudbrain.id ASC limit 1") + cloudbrains := make([]*CloudbrainInfo, 0) + if err := sess.Table(&Cloudbrain{}).Unscoped(). + Find(&cloudbrains); err != nil { + log.Info("find error.") + } + return cloudbrains, nil +} func GetAllStatusCloudBrain() map[string]int { sess := x.NewSession() @@ -20,3 +205,87 @@ func GetAllStatusCloudBrain() map[string]int { } return cloudBrainStatusResult } + +func GetWaittingTop() ([]*CloudbrainInfo, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.Eq{"cloudbrain.status": string(JobWaiting)}, + ) + sess.OrderBy("(cloudbrain.start_time-cloudbrain.created_unix) DESC limit 10") + cloudbrains := make([]*CloudbrainInfo, 0, 10) + if err := sess.Table(&Cloudbrain{}).Where(cond). + Find(&cloudbrains); err != nil { + log.Info("find error.") + } + return cloudbrains, nil +} +func GetRunningTop() ([]*CloudbrainInfo, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.Eq{"cloudbrain.status": string(JobRunning)}, + ) + sess.OrderBy("(cloudbrain.end_time-cloudbrain.start_time) DESC limit 10") + cloudbrains := make([]*CloudbrainInfo, 0, 10) + if err := sess.Table(&Cloudbrain{}).Where(cond). + Find(&cloudbrains); err != nil { + log.Info("find error.") + } + return cloudbrains, nil +} + +func getCreatePeriodCount(dateBeginTime string, dateEndTime string, hourBeginTime string, hourEndTime string) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where to_char(to_timestamp(created_unix), 'YYYY-MM-DD') >= '" + dateBeginTime + + "' and to_char(to_timestamp(created_unix), 'YYYY-MM-DD') < '" + dateEndTime + + "' and to_char(to_timestamp(created_unix), 'HH24:MI:SS') >= '" + hourBeginTime + + "' and to_char(to_timestamp(created_unix), 'HH24:MI:SS') < '" + hourEndTime + "'" + return x.SQL(countSql).Count() +} + +//SELECT * FROM xxx WHERE NOT ((endTime < hourBeginTime) OR (startTime > hourEndTime)) +func getRunPeriodCount(dateBeginTime string, dateEndTime string, hourBeginTime string, hourEndTime string) (int64, error) { + countSql := "SELECT count(*) FROM " + + "public.cloudbrain where not ((to_char(to_timestamp(start_time), ' HH24:MI:SS') > '" + hourEndTime + + "') or (to_char(to_timestamp(end_time), 'HH24:MI:SS') < '" + hourBeginTime + "'))" + + " and (to_char(to_timestamp(start_time), 'YYYY-MM-DD') >= '" + dateBeginTime + + "' and to_char(to_timestamp(start_time), 'YYYY-MM-DD') < '" + dateEndTime + "')" + return x.SQL(countSql).Count() + +} + +func GetCreateHourPeriodCount(dateBeginTime string, dateEndTime string) (map[string]interface{}, error) { + //0 to 23 for each hour, + dateHourMap := make(map[string]interface{}) + var slice = []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23} + for key, value := range slice { + hourBeginHour := util.AddZero(value) + ":00:00" + hourEndHour := util.AddZero(value+1) + ":00:00" + cout, err := getCreatePeriodCount(dateBeginTime, dateEndTime, hourBeginHour, hourEndHour) + if err != nil { + log.Error("Can not query getCreatePeriodCount.", err) + return nil, nil + } + dateHourMap[strconv.Itoa(key)] = cout + } + return dateHourMap, nil +} + +func GetRunHourPeriodCount(dateBeginTime string, dateEndTime string) (map[string]interface{}, error) { + dateHourMap := make(map[string]interface{}) + var slice = []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23} + for key, value := range slice { + hourBeginHour := util.AddZero(value) + ":00:00" + hourEndHour := util.AddZero(value+1) + ":00:00" + cout, err := getRunPeriodCount(dateBeginTime, dateEndTime, hourBeginHour, hourEndHour) + if err != nil { + log.Error("Can not query getRunPeriodCount.", err) + return nil, nil + } + dateHourMap[strconv.Itoa(key)] = cout + } + return dateHourMap, nil +} diff --git a/models/dataset.go b/models/dataset.go index 726a5010f..b7186ac0b 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -107,6 +107,7 @@ type SearchDatasetOptions struct { Category string Task string License string + DatasetIDs []int64 ListOptions SearchOrderBy IsOwner bool @@ -177,6 +178,12 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond { } } + if len(opts.DatasetIDs) > 0 { + subCon := builder.NewCond() + subCon = subCon.And(builder.In("dataset.id", opts.DatasetIDs)) + cond = cond.Or(subCon) + } + return cond } @@ -447,3 +454,11 @@ func IncreaseDownloadCount(datasetID int64) error { return nil } + +func GetCollaboratorDatasetIdsByUserID(userID int64) []int64 { + var datasets []int64 + _ = x.Table("dataset").Join("INNER", "collaboration", "dataset.repo_id = collaboration.repo_id and collaboration.mode>0 and collaboration.user_id=?", userID). + Cols("dataset.id").Find(&datasets) + return datasets + +} diff --git a/modules/modelarts/resty.go b/modules/modelarts/resty.go index 961e02538..6a2803cb1 100755 --- a/modules/modelarts/resty.go +++ b/modules/modelarts/resty.go @@ -1131,7 +1131,7 @@ sendjob: res, err := client.R(). SetAuthToken(TOKEN). SetResult(&result). - Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/pod/" + podName + "/metric-statistic") + Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/pod/" + podName + "/metric-statistic?statistic_type=each") if err != nil { return nil, fmt.Errorf("resty GetTrainJobMetricStatistic: %v", err) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index ec5b17cb1..6c305d296 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1020,13 +1020,16 @@ balance.total_view=余额总览 balance.available=可用余额： cloudbrain1=云脑1 cloudbrain2=云脑2 +intelligent_net=智算网络 cloudbrain_selection=云脑选择 cloudbrain_platform_selection=选择您准备使用的云脑平台： confirm_choice=确定 cloudbran1_tips=只有zip格式的数据集才能发起云脑任务 cloudbrain_creator=创建者 +cloudbrain_type=支撑算力 +cloudbrain_untype=未知支撑算力 cloudbrain_task=任务名称 -cloudbrain_task_type=任务类型 +cloudbrain_task_type=云脑任务类型 cloudbrain_task_name=云脑侧任务名称 cloudbrain_operate=操作 cloudbrain_status_createtime=状态/创建时间 @@ -1060,6 +1063,7 @@ computing.success=加入成功 modelarts.status=状态 modelarts.createtime=创建时间 +modelarts.deletetime=删除时间 modelarts.version_nums=版本数 modelarts.version=版本 modelarts.computing_resources=计算资源 @@ -1090,8 +1094,8 @@ modelarts.train_job.job_status=任务状态 modelarts.train_job.job_name=任务名称 modelarts.train_job.version=任务版本 modelarts.train_job.start_time=开始运行时间 -modelarts.train_job.end_time=运行结束时间 -modelarts.train_job.wait_time=等待时间 +modelarts.train_job.end_time=结束运行时间 +modelarts.train_job.wait_time=等待时长 modelarts.train_job.dura_time=运行时长 modelarts.train_job.description=任务描述 modelarts.train_job.parameter_setting=参数设置 diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 35089f0e7..0c280b0cb 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -574,9 +574,18 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/cloudbrainboard", func() { m.Get("/downloadAll", repo.DownloadCloudBrainBoard) m.Group("/cloudbrain", func() { + m.Get("/overview", repo.GetAllCloudbrainsOverview) + m.Get("/distribution", repo.GetAllCloudbrainsPeriodDistribution) + m.Get("/trend", repo.GetAllCloudbrainsTrend) + m.Get("/trend_detail_data", repo.GetAllCloudbrainsTrendDetail) m.Get("/status_analysis", repo.GetCloudbrainsStatusAnalysis) + m.Get("/detail_data", repo.GetCloudbrainsDetailData) + m.Get("/hours_data", repo.GetCloudbrainsCreateHoursData) + m.Get("/waitting_top_data", repo.GetWaittingTop) + m.Get("/running_top_data", repo.GetRunningTop) }) }, operationReq) + // Users m.Group("/users", func() { m.Get("/search", user.Search) diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index 0710a21e4..07298ffdc 100644 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -1,8 +1,10 @@ package repo import ( + "fmt" "net/http" "net/url" + "strings" "time" "code.gitea.io/gitea/models" @@ -11,6 +13,19 @@ import ( "github.com/360EntSecGroup-Skylar/excelize/v2" ) +type DateCloudbrainNum struct { + Date string `json:"date"` + CloudOneJobTypeRes map[string]int `json:"cloudOneJobTypeRes"` + CloudTwoJobTypeRes map[string]int `json:"cloudTwoJobTypeRes"` + IntelligentNetJobTypeRes map[string]int `json:"intelligentNetJobTypeRes"` + CloudBrainPeriodNum map[int]int `json:"cloudBrainPeriodNum"` + CloudBrainComputeResource map[string]int `json:"cloudBrainComputeResource"` +} +type DateCloudbrainInfo struct { + Date string `json:"date"` + CloudBrainPeriodNum map[int]int `json:"cloudBrainPeriodNum"` + CloudBrainComputeResource map[string]int `json:"cloudBrainComputeResource"` +} type CloudbrainsStatusAnalysis struct { JobWaitingCount int64 `json:"jobWaitingCount"` JobRunningCount int64 `json:"jobRunningCount"` @@ -21,6 +36,1119 @@ type CloudbrainsStatusAnalysis struct { JobInitCount int64 `json:"jobInitCount"` } +func GetAllCloudbrainsOverview(ctx *context.Context) { + recordCloudbrain, err := models.GetRecordBeginTime() + if err != nil { + log.Error("Can not get recordCloudbrain", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix + now := time.Now() + beginTime := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + endTime := now + todayCreatorCount, err := models.GetTodayCreatorCount(beginTime, endTime) + if err != nil { + log.Error("Can not query todayCreatorCount.", err) + return + } + creatorCount, err := models.GetCreatorCount() + if err != nil { + log.Error("Can not query creatorCount.", err) + return + } + todayStatusResult := make(map[string]int) + cloudBrainNum := make(map[int]int) + cloudBrainOneDuration := int64(0) + cloudBrainTwoDuration := int64(0) + intelligentNetDuration := int64(0) + todayNewJobCount := 0 + page := 1 + pagesize := 1000 + count := pagesize + for count == pagesize && count != 0 { + cloudbrains, _, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: pagesize, + }, + Type: models.TypeCloudBrainAll, + BeginTimeUnix: int64(recordBeginTime), + EndTimeUnix: endTime.Unix(), + }) + if err != nil { + ctx.ServerError("Get cloudbrains failed:", err) + return + } + + for _, cloudbrain := range cloudbrains { + if int64(cloudbrain.Cloudbrain.CreatedUnix) >= beginTime.Unix() && int64(cloudbrain.Cloudbrain.CreatedUnix) < endTime.Unix() { + todayNewJobCount += 1 + if _, ok := todayStatusResult[cloudbrain.Status]; !ok { + todayStatusResult[cloudbrain.Status] = 1 + } else { + todayStatusResult[cloudbrain.Status] += 1 + } + } + + if _, ok := cloudBrainNum[cloudbrain.Cloudbrain.Type]; !ok { + cloudBrainNum[cloudbrain.Cloudbrain.Type] = 1 + } else { + cloudBrainNum[cloudbrain.Cloudbrain.Type] += 1 + } + + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { + cloudBrainOneDuration = cloudBrainOneDuration + cloudbrain.Cloudbrain.Duration + } + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + cloudBrainTwoDuration = cloudBrainTwoDuration + cloudbrain.Cloudbrain.Duration + } + if cloudbrain.Cloudbrain.Type == models.TypeIntelligentNet { + intelligentNetDuration = intelligentNetDuration + cloudbrain.Cloudbrain.Duration + } + + } + count = len(cloudbrains) + page += 1 + } + + statusNameList := []string{string(models.ModelArtsTrainJobCompleted), string(models.JobFailed), string(models.ModelArtsTrainJobInit), + string(models.JobRunning), string(models.ModelArtsStartFailed), string(models.JobStopped), string(models.JobSucceeded), + string(models.JobWaiting), string(models.ModelArtsTrainJobKilled)} + for _, v := range statusNameList { + if _, ok := todayStatusResult[v]; !ok { + todayStatusResult[v] = 0 + } + } + + todayRunningCount := todayStatusResult[string(models.JobRunning)] + todayCompletedCount := todayStatusResult[string(models.ModelArtsTrainJobCompleted)] + todayStatusResult[string(models.JobFailed)] + + todayStatusResult[string(models.ModelArtsStartFailed)] + todayStatusResult[string(models.JobStopped)] + todayStatusResult[string(models.JobSucceeded)] + todayStatusResult[string(models.ModelArtsTrainJobKilled)] + todayWaitingCount := todayStatusResult[string(models.ModelArtsTrainJobInit)] + todayStatusResult[string(models.JobWaiting)] + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "recordBeginTime": recordBeginTime, + "updateTime": now.Unix(), + "cloudBrainNum": cloudBrainNum, + "cloudBrainOneDuration": cloudBrainOneDuration, + "cloudBrainTwoDuration": cloudBrainTwoDuration, + "intelligentNetDuration": intelligentNetDuration, + "todayCreatorCount": todayCreatorCount, + "creatorCount": creatorCount, + "todayRunningCount": todayRunningCount, + "todayCompletedCount": todayCompletedCount, + "todayWaitingCount": todayWaitingCount, + "todayNewJobCount": todayNewJobCount, + }) +} + +func GetAllCloudbrainsTrend(ctx *context.Context) { + + queryType := ctx.QueryTrim("type") + now := time.Now() + + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + var beginTime time.Time + var endTime time.Time + dateCloudbrainNum := make([]DateCloudbrainNum, 0) + var err error + if queryType != "" { + if queryType == "all" { + recordCloudbrain, err := models.GetRecordBeginTime() + if err != nil { + log.Error("Can not get recordCloudbrain", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + brainRecordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix.AsTime() + beginTime = brainRecordBeginTime + endTime = now + dateCloudbrainNum, err = getMonthCloudbrainNum(beginTime, endTime) + if err != nil { + log.Error("Can not query getMonthCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getMonthCloudbrainNum_get_error")) + return + } + } else if queryType == "today" { + beginTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + // endTimeTemp = now + endTime = beginTime.AddDate(0, 0, 1) + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + } else if queryType == "yesterday" { + beginTime = now.AddDate(0, 0, -1) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + } else if queryType == "last_7day" { + beginTime = now.AddDate(0, 0, -6) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + } else if queryType == "last_30day" { + beginTime = now.AddDate(0, 0, -29) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + } else if queryType == "current_month" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + } else if queryType == "current_year" { + endTime = now + beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location()) + // endTimeTemp = beginTime.AddDate(0, 1, 0) + dateCloudbrainNum, err = getMonthCloudbrainNum(beginTime, endTime) + if err != nil { + log.Error("Can not query getMonthCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getMonthCloudbrainNum_get_error")) + return + } + } else if queryType == "last_month" { + + lastMonthTime := now.AddDate(0, -1, 0) + beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + } + + } else { + if beginTimeStr == "" || endTimeStr == "" { + //如果查询类型和开始时间结束时间都未设置，按queryType=all处理 + recordCloudbrain, err := models.GetRecordBeginTime() + if err != nil { + log.Error("Can not get recordCloudbrain", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + brainRecordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix.AsTime() + beginTime = brainRecordBeginTime + endTime = now + dateCloudbrainNum, err = getMonthCloudbrainNum(beginTime, endTime) + if err != nil { + log.Error("Can not query getMonthCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getMonthCloudbrainNum_get_error")) + return + } + } else { + beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + if endTime.After(time.Now()) { + endTime = time.Now() + } + dateCloudbrainNum, err = getDayCloudbrainNum(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainNum.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainNum_get_error")) + return + } + } + + } + ctx.JSON(http.StatusOK, map[string]interface{}{ + "timeCloudbrainNum": dateCloudbrainNum, + }) + +} +func GetAllCloudbrainsTrendDetail(ctx *context.Context) { + queryType := ctx.QueryTrim("type") + now := time.Now() + + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + var beginTime time.Time + var endTime time.Time + var endTimeTemp time.Time + dayCloudbrainInfo := make([]DateCloudbrainInfo, 0) + var err error + var count int + if queryType != "" { + if queryType == "all" { + recordCloudbrain, err := models.GetRecordBeginTime() + if err != nil { + log.Error("Can not get recordCloudbrain", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + brainRecordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix.AsTime() + beginTime = brainRecordBeginTime + endTime = now + dayCloudbrainInfo, count, err = getMonthCloudbrainInfo(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainInfo.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + } else if queryType == "today" { + beginTime = now.AddDate(0, 0, 0) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + dayCloudbrainInfo, count, err = getDayCloudbrainInfo(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainInfo.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + + } else if queryType == "yesterday" { + beginTime = now.AddDate(0, 0, -1) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + dayCloudbrainInfo, count, err = getDayCloudbrainInfo(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainInfo.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + + } else if queryType == "last_7day" { + beginTime = now.AddDate(0, 0, -6) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + endTimeTemp = time.Date(endTimeTemp.Year(), endTimeTemp.Month(), endTimeTemp.Day(), 0, 0, 0, 0, now.Location()) + dayCloudbrainInfo, count, err = getDayCloudbrainInfo(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainInfo.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + } else if queryType == "last_30day" { + beginTime = now.AddDate(0, 0, -29) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + dayCloudbrainInfo, count, err = getDayCloudbrainInfo(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainInfo.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + } else if queryType == "current_month" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) + dayCloudbrainInfo, count, err = getDayCloudbrainInfo(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainInfo.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + + } else if queryType == "current_year" { + endTime = now + beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location()) + dayCloudbrainInfo, count, err = getMonthCloudbrainInfo(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainInfo.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + + } else if queryType == "last_month" { + + lastMonthTime := now.AddDate(0, -1, 0) + beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) + dayCloudbrainInfo, count, err = getDayCloudbrainInfo(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainInfo.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + + } + + } else { + if beginTimeStr == "" || endTimeStr == "" { + //如果查询类型和开始时间结束时间都未设置，按queryType=all处理 + recordCloudbrain, err := models.GetRecordBeginTime() + if err != nil { + log.Error("Can not get recordCloudbrain", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + brainRecordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix.AsTime() + beginTime = brainRecordBeginTime + endTime = now + dayCloudbrainInfo, count, err = getMonthCloudbrainInfo(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainInfo.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + } else { + beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + if endTime.After(time.Now()) { + endTime = time.Now() + } + endTimeTemp = beginTime.AddDate(0, 0, 1) + dayCloudbrainInfo, count, err = getDayCloudbrainInfo(beginTime, endTime) + if err != nil { + log.Error("Can not query getDayCloudbrainInfo.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("getDayCloudbrainInfo_get_error")) + return + } + } + + } + + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pagesize := ctx.QueryInt("pagesize") + if pagesize <= 0 { + pagesize = 5 + } + pageDateCloudbrainInfo := getPageDateCloudbrainInfo(dayCloudbrainInfo, page, pagesize) + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "totalCount": count, + "timeCloudbrainInfo": pageDateCloudbrainInfo, + }) + +} +func getPageDateCloudbrainInfo(dateCloudbrainInfo []DateCloudbrainInfo, page int, pagesize int) []DateCloudbrainInfo { + begin := (page - 1) * pagesize + end := (page) * pagesize + + if begin > len(dateCloudbrainInfo)-1 { + return nil + } + if end > len(dateCloudbrainInfo)-1 { + return dateCloudbrainInfo[begin:] + } else { + return dateCloudbrainInfo[begin:end] + } + +} + +func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { + queryType := ctx.QueryTrim("type") + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + recordCloudbrain, err := models.GetRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + recordBeginTime := time.Unix(int64(recordCloudbrain[0].Cloudbrain.CreatedUnix), 0) + beginTime, endTime, err := getCloudbrainTimePeroid(ctx, recordBeginTime) + if err != nil { + log.Error("Parameter is wrong", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.parameter_is_wrong")) + return + } + cloudOneJobTypeRes := make(map[string]int) + cloudTwoJobTypeRes := make(map[string]int) + intelligentNetJobTypeRes := make(map[string]int) + cloudBrainPeriodNum := make(map[int]int) + cloudBrainComputeResource := make(map[string]int) + beginTimeTemp := beginTime.Unix() + if queryType == "all" || (queryType == "" && (beginTimeStr == "" || endTimeStr == "")) { + beginTimeTemp = int64(recordCloudbrain[0].Cloudbrain.CreatedUnix) + } + + page := 1 + pagesize := 1000 + count := pagesize + //Each time a maximum of 1000 pieces of data are detected to the memory, batch processing + for count == pagesize && count != 0 { + cloudbrains, _, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: pagesize, + }, + Type: models.TypeCloudBrainAll, + BeginTimeUnix: beginTimeTemp, + EndTimeUnix: endTime.Unix(), + }) + if err != nil { + ctx.ServerError("Get cloudbrains failed:", err) + return + } + + for _, cloudbrain := range cloudbrains { + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { + if _, ok := cloudOneJobTypeRes[cloudbrain.JobType]; !ok { + cloudOneJobTypeRes[cloudbrain.JobType] = 1 + } else { + cloudOneJobTypeRes[cloudbrain.JobType] += 1 + } + } + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + if _, ok := cloudTwoJobTypeRes[cloudbrain.JobType]; !ok { + cloudTwoJobTypeRes[cloudbrain.JobType] = 1 + } else { + cloudTwoJobTypeRes[cloudbrain.JobType] += 1 + } + } + if cloudbrain.Cloudbrain.Type == models.TypeIntelligentNet { + if _, ok := intelligentNetJobTypeRes[cloudbrain.JobType]; !ok { + intelligentNetJobTypeRes[cloudbrain.JobType] = 1 + } else { + intelligentNetJobTypeRes[cloudbrain.JobType] += 1 + } + } + + if _, ok := cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type]; !ok { + cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] = 1 + } else { + cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] += 1 + } + + if _, ok := cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource]; !ok { + cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource] = 1 + } else { + cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource] += 1 + } + } + count = len(cloudbrains) + page += 1 + } + + jobTypeList := []string{string(models.JobTypeDebug), string(models.JobTypeTrain), string(models.JobTypeInference), string(models.JobTypeBenchmark), + string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)} + for _, v := range jobTypeList { + if _, ok := cloudOneJobTypeRes[v]; !ok { + cloudOneJobTypeRes[v] = 0 + } + if _, ok := cloudTwoJobTypeRes[v]; !ok { + cloudTwoJobTypeRes[v] = 0 + } + if _, ok := intelligentNetJobTypeRes[v]; !ok { + intelligentNetJobTypeRes[v] = 0 + } + } + cloudBrainTypeList := []int{0, 1, 2} + for _, v := range cloudBrainTypeList { + if _, ok := cloudBrainPeriodNum[v]; !ok { + cloudBrainPeriodNum[v] = 0 + } + } + + ComputeResourceList := []string{"CPU/GPU", "NPU"} + for _, v := range ComputeResourceList { + if _, ok := cloudBrainComputeResource[v]; !ok { + cloudBrainComputeResource[v] = 0 + } + } + + cloudOneJobTypeRes["EVALUATION"] = cloudOneJobTypeRes[string(models.JobTypeBenchmark)] + cloudOneJobTypeRes[string(models.JobTypeSnn4imagenet)] + cloudOneJobTypeRes[string(models.JobTypeBrainScore)] + cloudTwoJobTypeRes["EVALUATION"] = cloudTwoJobTypeRes[string(models.JobTypeBenchmark)] + cloudTwoJobTypeRes[string(models.JobTypeSnn4imagenet)] + cloudTwoJobTypeRes[string(models.JobTypeBrainScore)] + intelligentNetJobTypeRes["EVALUATION"] = intelligentNetJobTypeRes[string(models.JobTypeBenchmark)] + intelligentNetJobTypeRes[string(models.JobTypeSnn4imagenet)] + intelligentNetJobTypeRes[string(models.JobTypeBrainScore)] + ctx.JSON(http.StatusOK, map[string]interface{}{ + "cloudOneJobTypeRes": cloudOneJobTypeRes, + "cloudTwoJobTypeRes": cloudTwoJobTypeRes, + "intelligentNetJobTypeRes": intelligentNetJobTypeRes, + "cloudBrainPeriodNum": cloudBrainPeriodNum, + "cloudBrainComputeResource": cloudBrainComputeResource, + }) +} + +func GetCloudbrainsStatusAnalysis(ctx *context.Context) { + recordCloudbrain, err := models.GetRecordBeginTime() + if err != nil { + log.Error("Can not get recordCloudbrain", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix + endTime := time.Now() + cloudBrainStatusResult := make(map[string]int) + cloudBrainStatusAnalysis := make(map[string]int) + totalCount := 0 + + page := 1 + pagesize := 1000 + count := pagesize + for count == pagesize && count != 0 { + cloudbrains, _, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: pagesize, + }, + Type: models.TypeCloudBrainAll, + BeginTimeUnix: int64(recordBeginTime), + EndTimeUnix: endTime.Unix(), + }) + if err != nil { + ctx.ServerError("Get cloudbrains failed:", err) + return + } + for _, cloudbrain := range cloudbrains { + if _, ok := cloudBrainStatusResult[cloudbrain.Status]; !ok { + cloudBrainStatusResult[cloudbrain.Status] = 1 + } else { + cloudBrainStatusResult[cloudbrain.Status] += 1 + } + } + count = len(cloudbrains) + totalCount = totalCount + count + page += 1 + } + + var jobStatuses []string + jobStatuses = append(jobStatuses, string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobFailed), string(models.ModelArtsRunning), string(models.ModelArtsTrainJobCompleted), + string(models.ModelArtsStarting), string(models.ModelArtsRestarting), string(models.ModelArtsStartFailed), + string(models.ModelArtsStopping), string(models.ModelArtsStopped), string(models.JobSucceeded)) + jobStatusesCount := 0 + for _, v := range jobStatuses { + if _, ok := cloudBrainStatusResult[v]; !ok { + cloudBrainStatusAnalysis[v] = 0 + } else { + cloudBrainStatusAnalysis[v] = cloudBrainStatusResult[v] + } + jobStatusesCount = jobStatusesCount + cloudBrainStatusResult[v] + } + cloudBrainStatusAnalysis["OTHER"] = totalCount - jobStatusesCount + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "cloudBrainStatusResult": cloudBrainStatusAnalysis, + }) +} + +func GetCloudbrainsDetailData(ctx *context.Context) { + recordCloudbrain, err := models.GetRecordBeginTime() + if err != nil { + log.Error("Can not get recordCloudbrain", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix + endTime := time.Now() + listType := ctx.Query("listType") + jobType := ctx.Query("jobType") + jobStatus := ctx.Query("jobStatus") + cloudBrainType := ctx.QueryInt("Type") + + page := ctx.QueryInt("page") + pageSize := ctx.QueryInt("pagesize") + if page <= 0 { + page = 1 + } + if pageSize <= 0 { + pageSize = 10 + } + var jobTypes []string + jobTypeNot := false + if jobType == string(models.JobTypeBenchmark) { + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + } else if jobType != "all" && jobType != "" { + jobTypes = append(jobTypes, jobType) + } + + var jobStatuses []string + jobStatusNot := false + if jobStatus == "other" { + jobStatusNot = true + jobStatuses = append(jobStatuses, string(models.ModelArtsTrainJobWaiting), string(models.ModelArtsTrainJobFailed), string(models.ModelArtsRunning), string(models.ModelArtsTrainJobCompleted), + string(models.ModelArtsStarting), string(models.ModelArtsRestarting), string(models.ModelArtsStartFailed), + string(models.ModelArtsStopping), string(models.ModelArtsStopped), string(models.JobSucceeded)) + } else if jobStatus != "all" && jobStatus != "" { + jobStatuses = append(jobStatuses, jobStatus) + } + + keyword := strings.Trim(ctx.Query("q"), " ") + + ciTasks, count, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: pageSize, + }, + Keyword: keyword, + Type: cloudBrainType, + ComputeResource: listType, + JobTypeNot: jobTypeNot, + JobStatusNot: jobStatusNot, + JobStatus: jobStatuses, + JobTypes: jobTypes, + NeedRepoInfo: true, + BeginTimeUnix: int64(recordBeginTime), + EndTimeUnix: endTime.Unix(), + }) + if err != nil { + ctx.ServerError("Get job failed:", err) + return + } + nilTime := time.Time{} + tasks := []models.TaskDetail{} + for i, task := range ciTasks { + ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource + + var taskDetail models.TaskDetail + taskDetail.ID = ciTasks[i].Cloudbrain.ID + taskDetail.JobName = ciTasks[i].JobName + taskDetail.DisplayJobName = ciTasks[i].DisplayJobName + taskDetail.Status = ciTasks[i].Status + taskDetail.JobType = ciTasks[i].JobType + taskDetail.CreatedUnix = ciTasks[i].Cloudbrain.CreatedUnix + taskDetail.RunTime = ciTasks[i].Cloudbrain.TrainJobDuration + taskDetail.StartTime = ciTasks[i].StartTime + taskDetail.EndTime = ciTasks[i].EndTime + taskDetail.ComputeResource = ciTasks[i].ComputeResource + taskDetail.Type = ciTasks[i].Cloudbrain.Type + taskDetail.UserName = ciTasks[i].User.Name + taskDetail.RepoID = ciTasks[i].RepoID + if ciTasks[i].Repo != nil { + taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name + taskDetail.RepoAlias = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Alias + } + if ciTasks[i].Cloudbrain.Status == string(models.JobWaiting) { + WaitTimeInt := time.Now().Unix() - ciTasks[i].Cloudbrain.CreatedUnix.AsTime().Unix() + taskDetail.WaitTime = models.ConvertDurationToStr(WaitTimeInt) + if WaitTimeInt < 0 { + taskDetail.WaitTime = "00:00:00" + } + } else if ciTasks[i].Cloudbrain.Status == string(models.JobStopped) && ciTasks[i].Cloudbrain.StartTime.AsTime().Unix() == 0 { + WaitTimeInt := ciTasks[i].Cloudbrain.EndTime.AsTime().Unix() - ciTasks[i].Cloudbrain.CreatedUnix.AsTime().Unix() + taskDetail.WaitTime = models.ConvertDurationToStr(WaitTimeInt) + if WaitTimeInt < 0 { + taskDetail.WaitTime = "00:00:00" + + } + } else { + WaitTimeInt := ciTasks[i].Cloudbrain.StartTime.AsTime().Unix() - ciTasks[i].Cloudbrain.CreatedUnix.AsTime().Unix() + taskDetail.WaitTime = models.ConvertDurationToStr(WaitTimeInt) + if WaitTimeInt < 0 { + taskDetail.WaitTime = "00:00:00" + } + } + + if ciTasks[i].Cloudbrain.Type == models.TypeCloudBrainTwo || (ciTasks[i].Cloudbrain.Type == models.TypeCloudBrainOne && ciTasks[i].Cloudbrain.JobType == "TRAIN") { + taskDetail.JobID = ciTasks[i].Cloudbrain.JobID + } + + if ciTasks[i].Cloudbrain.DeletedAt != nilTime { + taskDetail.IsDelete = true + } else { + taskDetail.IsDelete = false + } + tasks = append(tasks, taskDetail) + } + + pager := context.NewPagination(int(count), pageSize, page, getTotalPage(count, pageSize)) + pager.SetDefaultParams(ctx) + pager.AddParam(ctx, "listType", "ListType") + + ctx.JSON(http.StatusOK, map[string]interface{}{ + "Title": ctx.Tr("kanban.cloudBrains"), + "Tasks": tasks, + "Keyword": keyword, + "pager": pager, + "count": count, + }) +} + +func GetCloudbrainsCreateHoursData(ctx *context.Context) { + recordCloudbrain, err := models.GetRecordBeginTime() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + recordBeginTime := time.Unix(int64(recordCloudbrain[0].Cloudbrain.CreatedUnix), 0) + // recordBeginTime, _ := time.Parse("2006-01-02 15:04:05", recordTime) + now := time.Now() + queryType := ctx.QueryTrim("type") + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + var beginTime time.Time + var endTime time.Time + createHourPeriodCount := make(map[string]interface{}) + if queryType != "" { + if queryType == "all" { + beginTime = recordBeginTime + endTime = now.AddDate(0, 0, 1) + endTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + } else if queryType == "today" { + beginTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + endTime = now.AddDate(0, 0, 1) + } else if queryType == "yesterday" { + beginTime = now.AddDate(0, 0, -1) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + } else if queryType == "current_week" { + beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+1) //begin from monday + endTime = now + } else if queryType == "current_month" { + beginTime = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) + endTime = now.AddDate(0, 0, 1) + endTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + } else if queryType == "monthly" { + endTime = now + beginTime = now.AddDate(0, -1, 0) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + } else if queryType == "current_year" { + beginTime = time.Date(now.Year(), 1, 1, 0, 0, 0, 0, now.Location()) + endTime = now.AddDate(0, 0, 1) + endTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + } else if queryType == "last_month" { + lastMonthTime := now.AddDate(0, -1, 0) + beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) + } else if queryType == "last_7day" { + beginTime = now.AddDate(0, 0, -6) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now.AddDate(0, 0, 1) + endTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + } else if queryType == "last_30day" { + beginTime = now.AddDate(0, 0, -29) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now.AddDate(0, 0, 1) + endTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + } + } else { + if beginTimeStr == "" || endTimeStr == "" { + //如果查询类型和开始时间结束时间都未设置，按queryType=all处理 + beginTime = recordBeginTime + endTime = now.AddDate(0, 0, 1) + endTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + } else { + beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + endTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + } + } + dateBeginTime := beginTime.Format("2006-01-02") + dateEndTime := endTime.Format("2006-01-02") + createHourPeriodCount, err = models.GetCreateHourPeriodCount(dateBeginTime, dateEndTime) + if err != nil { + log.Error("Can not query hourPeriodCount.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("hourPeriodCount_get_error")) + return + } + + runHourPeriodCount, err := models.GetRunHourPeriodCount(dateBeginTime, dateEndTime) + if err != nil { + log.Error("Can not query runHourPeriodCount.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("runHourPeriodCount_get_error")) + return + } + ctx.JSON(http.StatusOK, map[string]interface{}{ + "recordBeginTime": recordCloudbrain[0].Cloudbrain.CreatedUnix, + "updateTime": now.Unix(), + "createHourPeriodCount": createHourPeriodCount, + "runHourPeriodCount": runHourPeriodCount, + }) + +} +func GetWaittingTop(ctx *context.Context) { + ciTasks, err := models.GetWaittingTop() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + tasks := []models.TaskDetail{} + for i, task := range ciTasks { + ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource + + var taskDetail models.TaskDetail + taskDetail.DisplayJobName = ciTasks[i].DisplayJobName + taskDetail.UserName = ciTasks[i].User.Name + taskDetail.RepoID = ciTasks[i].RepoID + if ciTasks[i].Repo != nil { + taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name + } + WaitTimeInt := time.Now().Unix() - ciTasks[i].Cloudbrain.CreatedUnix.AsTime().Unix() + taskDetail.WaitTime = models.ConvertDurationToStr(WaitTimeInt) + + if WaitTimeInt < 0 { + taskDetail.WaitTime = "00:00:00" + } + tasks = append(tasks, taskDetail) + } + ctx.JSON(http.StatusOK, map[string]interface{}{ + "tasks": tasks, + }) +} +func GetRunningTop(ctx *context.Context) { + ciTasks, err := models.GetRunningTop() + if err != nil { + log.Error("Can not get record begin time", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("repo.record_begintime_get_err")) + return + } + tasks := []models.TaskDetail{} + for i, task := range ciTasks { + ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource + + var taskDetail models.TaskDetail + taskDetail.DisplayJobName = ciTasks[i].DisplayJobName + taskDetail.RunTime = ciTasks[i].Cloudbrain.TrainJobDuration + taskDetail.UserName = ciTasks[i].User.Name + taskDetail.RepoID = ciTasks[i].RepoID + if ciTasks[i].Repo != nil { + taskDetail.RepoName = ciTasks[i].Repo.OwnerName + "/" + ciTasks[i].Repo.Name + } + + tasks = append(tasks, taskDetail) + } + ctx.JSON(http.StatusOK, map[string]interface{}{ + "tasks": tasks, + }) +} + +func getCloudbrainCount(beginTime time.Time, endTime time.Time, cloudbrains []*models.CloudbrainInfo) (map[string]int, map[string]int, map[string]int, map[int]int, map[string]int) { + + cloudOneJobTypeRes := make(map[string]int) + cloudTwoJobTypeRes := make(map[string]int) + intelligentNetJobTypeRes := make(map[string]int) + cloudBrainPeriodNum := make(map[int]int) + cloudBrainComputeResource := make(map[string]int) + for _, cloudbrain := range cloudbrains { + if int64(cloudbrain.Cloudbrain.CreatedUnix) >= beginTime.Unix() && int64(cloudbrain.Cloudbrain.CreatedUnix) < endTime.Unix() { + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainOne { + if _, ok := cloudOneJobTypeRes[cloudbrain.Cloudbrain.JobType]; !ok { + cloudOneJobTypeRes[cloudbrain.Cloudbrain.JobType] = 1 + } else { + cloudOneJobTypeRes[cloudbrain.Cloudbrain.JobType] += 1 + } + } + if cloudbrain.Cloudbrain.Type == models.TypeCloudBrainTwo { + if _, ok := cloudTwoJobTypeRes[cloudbrain.JobType]; !ok { + cloudTwoJobTypeRes[cloudbrain.Cloudbrain.JobType] = 1 + } else { + cloudTwoJobTypeRes[cloudbrain.Cloudbrain.JobType] += 1 + } + } + + if _, ok := cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type]; !ok { + cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] = 1 + } else { + cloudBrainPeriodNum[cloudbrain.Cloudbrain.Type] += 1 + } + + if _, ok := cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource]; !ok { + cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource] = 1 + } else { + cloudBrainComputeResource[cloudbrain.Cloudbrain.ComputeResource] += 1 + } + + } + } + + ComputeResourceList := []string{"CPU/GPU", "NPU"} + for _, v := range ComputeResourceList { + if _, ok := cloudBrainComputeResource[v]; !ok { + cloudBrainComputeResource[v] = 0 + } + } + + jobTypeList := []string{"DEBUG", "BENCHMARK", "INFERENCE", "TRAIN", "SNN4IMAGENET", "BRAINSCORE"} + cloudBrainTypeList := []int{0, 1, 2} + for _, v := range jobTypeList { + if _, ok := cloudOneJobTypeRes[v]; !ok { + cloudOneJobTypeRes[v] = 0 + } + if _, ok := cloudTwoJobTypeRes[v]; !ok { + cloudTwoJobTypeRes[v] = 0 + } + if _, ok := intelligentNetJobTypeRes[v]; !ok { + intelligentNetJobTypeRes[v] = 0 + } + } + for _, v := range cloudBrainTypeList { + if _, ok := cloudBrainPeriodNum[v]; !ok { + cloudBrainPeriodNum[v] = 0 + } + } + cloudBrainPeriodNum[-1] = cloudBrainPeriodNum[0] + cloudBrainPeriodNum[1] + cloudBrainPeriodNum[2] + return cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource +} + +func getDayCloudbrainNum(beginTime time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { + var endTimeTemp time.Time + endTimeTemp = beginTime.AddDate(0, 0, 1) + cloudbrains, _, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + Type: models.TypeCloudBrainAll, + BeginTimeUnix: beginTime.Unix(), + EndTimeUnix: endTime.Unix(), + }) + if err != nil { + log.Error("Get cloudbrains failed:", err) + return nil, err + } + dayCloudbrainNum := make([]DateCloudbrainNum, 0) + for endTimeTemp.Before(endTime) || endTimeTemp.Equal(endTime) { + cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(beginTime, endTimeTemp, cloudbrains) + dayCloudbrainNum = append(dayCloudbrainNum, DateCloudbrainNum{ + Date: beginTime.Format("2006/01/02"), + CloudOneJobTypeRes: cloudOneJobTypeRes, + CloudTwoJobTypeRes: cloudTwoJobTypeRes, + IntelligentNetJobTypeRes: intelligentNetJobTypeRes, + CloudBrainPeriodNum: cloudBrainPeriodNum, + CloudBrainComputeResource: cloudBrainComputeResource, + }) + if endTime.Before(endTimeTemp.AddDate(0, 0, 1)) && endTimeTemp.Before(endTime) { + beginTime = endTimeTemp + endTimeTemp = endTime + } else { + beginTime = endTimeTemp + endTimeTemp = beginTime.AddDate(0, 0, 1) + } + } + return dayCloudbrainNum, nil +} +func getMonthCloudbrainNum(beginTime time.Time, endTime time.Time) ([]DateCloudbrainNum, error) { + var endTimeTemp time.Time + now := time.Now() + endTimeTemp = beginTime.AddDate(0, 1, 0) + endTimeTemp = time.Date(endTimeTemp.Year(), endTimeTemp.Month(), 1, 0, 0, 0, 0, now.Location()) + monthCloudbrainNum := make([]DateCloudbrainNum, 0) + cloudbrains, _, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + Type: models.TypeCloudBrainAll, + BeginTimeUnix: beginTime.Unix(), + EndTimeUnix: endTime.Unix(), + }) + if err != nil { + log.Error("Getcloudbrains failed:%v", err) + return nil, err + } + for endTimeTemp.Before(endTime) || endTimeTemp.Equal(endTime) { + cloudOneJobTypeRes, cloudTwoJobTypeRes, intelligentNetJobTypeRes, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(beginTime, endTimeTemp, cloudbrains) + monthCloudbrainNum = append(monthCloudbrainNum, DateCloudbrainNum{ + Date: beginTime.Format("2006/01"), + CloudOneJobTypeRes: cloudOneJobTypeRes, + CloudTwoJobTypeRes: cloudTwoJobTypeRes, + IntelligentNetJobTypeRes: intelligentNetJobTypeRes, + CloudBrainPeriodNum: cloudBrainPeriodNum, + CloudBrainComputeResource: cloudBrainComputeResource, + }) + if endTime.Before(endTimeTemp.AddDate(0, 1, 0)) && endTimeTemp.Before(endTime) { + beginTime = endTimeTemp + endTimeTemp = endTime + } else { + beginTime = endTimeTemp + endTimeTemp = beginTime.AddDate(0, 1, 0) + } + } + return monthCloudbrainNum, nil +} + +func getDayCloudbrainInfo(beginTime time.Time, endTime time.Time) ([]DateCloudbrainInfo, int, error) { + now := time.Now() + endTimeTemp := time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + if endTimeTemp.Equal(endTime) { + endTimeTemp = endTimeTemp.AddDate(0, 0, -1) + } + cloudbrains, _, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + Type: models.TypeCloudBrainAll, + BeginTimeUnix: beginTime.Unix(), + EndTimeUnix: endTime.Unix(), + }) + if err != nil { + log.Error("Getcloudbrains failed:%v", err) + return nil, 0, err + } + dayCloudbrainInfo := make([]DateCloudbrainInfo, 0) + count := 0 + for beginTime.Before(endTimeTemp) || beginTime.Equal(endTimeTemp) { + _, _, _, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(endTimeTemp, endTime, cloudbrains) + dayCloudbrainInfo = append(dayCloudbrainInfo, DateCloudbrainInfo{ + Date: endTimeTemp.Format("2006/01/02"), + CloudBrainPeriodNum: cloudBrainPeriodNum, + CloudBrainComputeResource: cloudBrainComputeResource, + }) + endTime = endTimeTemp + endTimeTemp = endTimeTemp.AddDate(0, 0, -1) + count += 1 + } + return dayCloudbrainInfo, count, nil +} + +func getMonthCloudbrainInfo(beginTime time.Time, endTime time.Time) ([]DateCloudbrainInfo, int, error) { + now := time.Now() + endTimeTemp := time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) + if endTimeTemp.Equal(endTime) { + endTimeTemp = endTimeTemp.AddDate(0, -1, 0) + } + cloudbrains, _, err := models.CloudbrainAll(&models.CloudbrainsOptions{ + Type: models.TypeCloudBrainAll, + BeginTimeUnix: beginTime.Unix(), + EndTimeUnix: endTime.Unix(), + }) + if err != nil { + log.Error("Getcloudbrains failed:%v", err) + return nil, 0, err + } + dayCloudbrainInfo := make([]DateCloudbrainInfo, 0) + count := 0 + for beginTime.Before(endTimeTemp) || beginTime.Equal(endTimeTemp) || (endTimeTemp.Before(beginTime) && beginTime.Before(endTime)) { + _, _, _, cloudBrainPeriodNum, cloudBrainComputeResource := getCloudbrainCount(endTimeTemp, endTime, cloudbrains) + dayCloudbrainInfo = append(dayCloudbrainInfo, DateCloudbrainInfo{ + Date: endTimeTemp.Format("2006/01"), + CloudBrainPeriodNum: cloudBrainPeriodNum, + CloudBrainComputeResource: cloudBrainComputeResource, + }) + endTime = endTimeTemp + endTimeTemp = endTimeTemp.AddDate(0, -1, 0) + count += 1 + } + return dayCloudbrainInfo, count, nil +} + func DownloadCloudBrainBoard(ctx *context.Context) { page := 1 @@ -44,9 +1172,7 @@ func DownloadCloudBrainBoard(ctx *context.Context) { ctx.Error(http.StatusBadRequest, ctx.Tr("repo.cloudbrain_query_fail")) return } - totalPage := getTotalPage(total, pageSize) - f := excelize.NewFile() index := f.NewSheet(cloudBrain) @@ -89,26 +1215,27 @@ func DownloadCloudBrainBoard(ctx *context.Context) { f.WriteTo(ctx.Resp) } + func getCloudbrainFileName(baseName string) string { return baseName + "_" + time.Now().Format(EXCEL_DATE_FORMAT) + ".xlsx" } func allCloudbrainHeader(ctx *context.Context) map[string]string { - return map[string]string{"A1": ctx.Tr("repo.cloudbrain_task"), "B1": ctx.Tr("repo.cloudbrain_task_type"), "C1": ctx.Tr("repo.modelarts.status"), - "D1": ctx.Tr("repo.modelarts.createtime"), "E1": ctx.Tr("repo.modelarts.train_job.wait_time"), "F1": ctx.Tr("repo.modelarts.train_job.dura_time"), - "G1": ctx.Tr("repo.modelarts.train_job.start_time"), - "H1": ctx.Tr("repo.modelarts.train_job.end_time"), "I1": ctx.Tr("repo.modelarts.computing_resources"), - "J1": ctx.Tr("repo.cloudbrain_creator"), "K1": ctx.Tr("repo.repo_name"), "L1": ctx.Tr("repo.cloudbrain_task_name")} + return map[string]string{"A1": ctx.Tr("repo.cloudbrain_task"), "B1": ctx.Tr("repo.cloudbrain_type"), "C1": ctx.Tr("repo.modelarts.status"), "D1": ctx.Tr("repo.cloudbrain_task_type"), + "E1": ctx.Tr("repo.modelarts.createtime"), "F1": ctx.Tr("repo.modelarts.train_job.wait_time"), "G1": ctx.Tr("repo.modelarts.train_job.dura_time"), + "H1": ctx.Tr("repo.modelarts.train_job.start_time"), + "I1": ctx.Tr("repo.modelarts.train_job.end_time"), "J1": ctx.Tr("repo.modelarts.computing_resources"), + "K1": ctx.Tr("repo.cloudbrain_creator"), "L1": ctx.Tr("repo.repo_name"), "M1": ctx.Tr("repo.cloudbrain_task_name"), "N1": ctx.Tr("repo.modelarts.deletetime")} } func allCloudbrainValues(row int, rs *models.CloudbrainInfo, ctx *context.Context) map[string]string { - return map[string]string{getCellName("A", row): rs.DisplayJobName, getCellName("B", row): rs.JobType, getCellName("C", row): rs.Status, - getCellName("D", row): time.Unix(int64(rs.Cloudbrain.CreatedUnix), 0).Format(CREATE_TIME_FORMAT), getCellName("E", row): getBrainWaitTime(rs), - getCellName("F", row): rs.TrainJobDuration, getCellName("G", row): getBrainStartTime(rs), - getCellName("H", row): getBrainEndTime(rs), - getCellName("I", row): rs.ComputeResource, getCellName("J", row): rs.Name, getCellName("K", row): getBrainRepo(rs), - getCellName("L", row): rs.JobName, + return map[string]string{getCellName("A", row): rs.DisplayJobName, getCellName("B", row): getCloudbrainType(rs, ctx), getCellName("C", row): rs.Status, getCellName("D", row): rs.JobType, + getCellName("E", row): time.Unix(int64(rs.Cloudbrain.CreatedUnix), 0).Format(CREATE_TIME_FORMAT), getCellName("F", row): getBrainWaitTime(rs), + getCellName("G", row): rs.TrainJobDuration, getCellName("H", row): getBrainStartTime(rs), + getCellName("I", row): getBrainEndTime(rs), + getCellName("J", row): rs.ComputeResource, getCellName("K", row): rs.Name, getCellName("L", row): getBrainRepo(rs), + getCellName("M", row): rs.JobName, getCellName("N", row): getBrainDeleteTime(rs), } } func getBrainRepo(rs *models.CloudbrainInfo) string { @@ -136,16 +1263,114 @@ func getBrainEndTime(rs *models.CloudbrainInfo) string { } func getBrainWaitTime(rs *models.CloudbrainInfo) string { - waitTime := rs.Cloudbrain.StartTime - rs.Cloudbrain.CreatedUnix + var waitTime int64 + if rs.Cloudbrain.Status == string(models.JobWaiting) { + waitTime = time.Now().Unix() - rs.Cloudbrain.CreatedUnix.AsTime().Unix() + } else { + waitTime = int64(rs.Cloudbrain.StartTime - rs.Cloudbrain.CreatedUnix) + } if waitTime <= 0 { - return "0" + return "00:00:00" } else { - return models.ConvertDurationToStr(int64(waitTime)) + return models.ConvertDurationToStr(waitTime) } } -func GetCloudbrainsStatusAnalysis(ctx *context.Context) { - cloudBrainStatusResult := models.GetAllStatusCloudBrain() - ctx.JSON(http.StatusOK, map[string]interface{}{ - "cloudBrainStatusResult": cloudBrainStatusResult, - }) +func getCloudbrainType(rs *models.CloudbrainInfo, ctx *context.Context) string { + if rs.Cloudbrain.Type == models.TypeCloudBrainOne { + return ctx.Tr("repo.cloudbrain1") + } else if rs.Cloudbrain.Type == models.TypeCloudBrainTwo { + return ctx.Tr("repo.cloudbrain2") + } else if rs.Cloudbrain.Type == models.TypeIntelligentNet { + return ctx.Tr("repo.intelligent_net") + } else { + return ctx.Tr("repo.cloudbrain_untype") + } +} + +func getBrainDeleteTime(rs *models.CloudbrainInfo) string { + nilTime := time.Time{} + if rs.Cloudbrain.DeletedAt != nilTime { + return rs.Cloudbrain.DeletedAt.Format("2006-01-02 15:04:05") + } else { + return "" + } +} +func getCloudbrainTimePeroid(ctx *context.Context, recordBeginTime time.Time) (time.Time, time.Time, error) { + queryType := ctx.QueryTrim("type") + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + now := time.Now() + + var beginTime time.Time + var endTime time.Time + var err error + if queryType != "" { + + if queryType == "all" { + beginTime = recordBeginTime + endTime = now + } else if queryType == "today" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + } else if queryType == "yesterday" { + endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + beginTime = endTime.AddDate(0, 0, -1) + + } else if queryType == "last_7day" { + beginTime = now.AddDate(0, 0, -6) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + } else if queryType == "last_30day" { + beginTime = now.AddDate(0, 0, -29) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + } else if queryType == "current_month" { + endTime = now + beginTime = time.Date(endTime.Year(), endTime.Month(), 1, 0, 0, 0, 0, now.Location()) + } else if queryType == "monthly" { + endTime = now + beginTime = now.AddDate(0, -1, 1) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + + } else if queryType == "current_year" { + endTime = now + beginTime = time.Date(endTime.Year(), 1, 1, 0, 0, 0, 0, now.Location()) + + } else if queryType == "last_month" { + + lastMonthTime := now.AddDate(0, -1, 0) + beginTime = time.Date(lastMonthTime.Year(), lastMonthTime.Month(), 1, 0, 0, 0, 0, now.Location()) + endTime = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, now.Location()) + + } else { + return now, now, fmt.Errorf("The value of type parameter is wrong.") + + } + + } else { + if beginTimeStr == "" || endTimeStr == "" { + //如果查询类型和开始时间结束时间都未设置，按queryType=all处理 + beginTime = recordBeginTime + endTime = now + + } else { + + beginTime, err = time.ParseInLocation("2006-01-02", beginTimeStr, time.Local) + if err != nil { + return now, now, err + } + + endTime, err = time.ParseInLocation("2006-01-02", endTimeStr, time.Local) + if err != nil { + return now, now, err + } + } + + } + + if beginTime.Before(recordBeginTime) { + beginTime = recordBeginTime + } + + return beginTime, endTime, nil } diff --git a/routers/api/v1/repo/repo_dashbord.go b/routers/api/v1/repo/repo_dashbord.go index 6d7360053..b19c93371 100644 --- a/routers/api/v1/repo/repo_dashbord.go +++ b/routers/api/v1/repo/repo_dashbord.go @@ -192,7 +192,6 @@ func GetProjectsSummaryData(ctx *context.Context) { } - projectSummaryPeriodData := ProjectSummaryPeriodData{ TotalCount: count - 1, RecordBeginTime: recordBeginTime.Format(DATE_FORMAT), @@ -203,7 +202,7 @@ func GetProjectsSummaryData(ctx *context.Context) { } -func reverse(datas []*ProjectSummaryBaseData ) []*ProjectSummaryBaseData { +func reverse(datas []*ProjectSummaryBaseData) []*ProjectSummaryBaseData { for i := 0; i < len(datas)/2; i++ { j := len(datas) - i - 1 datas[i], datas[j] = datas[j], datas[i] @@ -211,8 +210,6 @@ func reverse(datas []*ProjectSummaryBaseData ) []*ProjectSummaryBaseData { return datas } - - func setStatisticsData(data *ProjectSummaryBaseData, v *models.SummaryStatistic, stats *models.SummaryStatistic) { data.NumReposAdd = v.NumRepos - stats.NumRepos data.NumRepoPublicAdd = v.NumRepoPublic - stats.NumRepoPublic @@ -890,12 +887,19 @@ func getTimePeroid(ctx *context.Context, recordBeginTime time.Time) (time.Time, if queryType == "all" { beginTime = recordBeginTimeTemp endTime = now - } else if queryType == "yesterday" { + } else if queryType == "today" { endTime = now beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) + } else if queryType == "yesterday" { + endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) + beginTime = endTime.AddDate(0, 0, -1) - } else if queryType == "current_week" { - beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+2) //begin from monday + } else if queryType == "last_7day" { + beginTime = now.AddDate(0, 0, -7) + beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) + endTime = now + } else if queryType == "last_30day" { + beginTime = now.AddDate(0, 0, -30) beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) endTime = now } else if queryType == "current_month" { diff --git a/routers/home.go b/routers/home.go index e31bed4ff..ae184a72a 100755 --- a/routers/home.go +++ b/routers/home.go @@ -343,6 +343,12 @@ func ExploreDatasets(ctx *context.Context) { if ctx.User != nil && !ctx.User.IsAdmin { ownerID = ctx.User.ID } + var datasetsIds []int64 + if ownerID > 0 { + + datasetsIds = models.GetCollaboratorDatasetIdsByUserID(ownerID) + } + opts := &models.SearchDatasetOptions{ Keyword: keyword, IncludePublic: true, @@ -351,6 +357,7 @@ func ExploreDatasets(ctx *context.Context) { Task: task, License: license, OwnerID: ownerID, + DatasetIDs: datasetsIds, RecommendOnly: ctx.QueryBool("recommend"), ListOptions: models.ListOptions{ Page: page, diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index b034d3d12..a93ff018c 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -1479,11 +1479,19 @@ func SyncCloudbrainStatus() { } func HandleTaskWithNoDuration(ctx *context.Context) { + mode := ctx.Query("mode") log.Info("HandleTaskWithNoDuration start") count := 0 start := time.Now().Unix() for { - cloudBrains, err := models.GetStoppedJobWithNoDurationJob() + var cloudBrains []*models.Cloudbrain + var err error + if mode == "1" { + cloudBrains, err = models.GetStoppedJobWithNoStartTimeEndTime() + } else { + cloudBrains, err = models.GetStoppedJobWithNoDurationJob() + } + if err != nil { log.Error("HandleTaskWithNoTrainJobDuration failed:", err.Error()) break @@ -1558,7 +1566,7 @@ func handleNoDurationTask(cloudBrains []*models.Cloudbrain) { } task.Duration = task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix() task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) - err = models.UpdateJob(task) + err = models.UpdateJobDurationWithDeleted(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) } @@ -1583,7 +1591,7 @@ func handleNoDurationTask(cloudBrains []*models.Cloudbrain) { } task.CorrectCreateUnix() task.ComputeAndSetDuration() - err = models.UpdateJob(task) + err = models.UpdateJobDurationWithDeleted(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) continue @@ -1604,7 +1612,7 @@ func handleNoDurationTask(cloudBrains []*models.Cloudbrain) { task.EndTime = task.StartTime.Add(result.Duration / 1000) } task.ComputeAndSetDuration() - err = models.UpdateJob(task) + err = models.UpdateJobDurationWithDeleted(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) continue @@ -1625,7 +1633,7 @@ func updateDefaultDuration(task *models.Cloudbrain) { task.StartTime = task.CreatedUnix task.EndTime = task.UpdatedUnix task.ComputeAndSetDuration() - err := models.UpdateJob(task) + err := models.UpdateJobDurationWithDeleted(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.JobName, err) } diff --git a/routers/repo/dataset.go b/routers/repo/dataset.go index 05a89964f..d3ae1624b 100755 --- a/routers/repo/dataset.go +++ b/routers/repo/dataset.go @@ -452,10 +452,11 @@ func PublicDataset(ctx *context.Context) { } func MyFavoriteDataset(ctx *context.Context) { - page := ctx.QueryInt("page") + UserId := ctx.User.ID cloudbrainType := ctx.QueryInt("type") keyword := strings.Trim(ctx.Query("q"), " ") - var datasetIDs []int64 + var NotColDatasetIDs []int64 + var IsColDatasetIDs []int64 datasetStars, err := models.GetDatasetStarByUser(ctx.User) if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("GetDatasetStarByUser failed", err))) @@ -467,31 +468,84 @@ func MyFavoriteDataset(ctx *context.Context) { }) return } - for i, _ := range datasetStars { - datasetIDs = append(datasetIDs, datasetStars[i].DatasetID) + for _, datasetStar := range datasetStars { + DatasetIsCollaborator := DatasetIsCollaborator(ctx, datasetStar.DatasetID) + dataset, err := models.GetDatasetByID(datasetStar.DatasetID) + if err != nil { + return + } + repo, err := models.GetRepositoryByID(dataset.RepoID) + if err != nil { + return + } + if repo.OwnerID == ctx.User.ID || DatasetIsCollaborator { + IsColDatasetIDs = append(IsColDatasetIDs, datasetStar.DatasetID) + } else { + NotColDatasetIDs = append(NotColDatasetIDs, datasetStar.DatasetID) + } } - datasets, count, err := models.Attachments(&models.AttachmentsOptions{ - ListOptions: models.ListOptions{ - Page: page, - PageSize: setting.UI.DatasetPagingNum, - }, + NotColDatasets, NotColcount, err := models.Attachments(&models.AttachmentsOptions{ Keyword: keyword, NeedDatasetIDs: true, - DatasetIDs: datasetIDs, + DatasetIDs: NotColDatasetIDs, NeedIsPrivate: true, IsPrivate: false, Type: cloudbrainType, JustNeedZipFile: true, NeedRepoInfo: true, RecommendOnly: ctx.QueryBool("recommend"), + UserId: UserId, + }) + if err != nil { + ctx.ServerError("datasets", err) + return + } + //If is collaborator, there is no need to determine whether the dataset is private or public + IsColDatasets, IsColcount, err := models.Attachments(&models.AttachmentsOptions{ + Keyword: keyword, + NeedDatasetIDs: true, + DatasetIDs: IsColDatasetIDs, + NeedIsPrivate: false, + Type: cloudbrainType, + JustNeedZipFile: true, + NeedRepoInfo: true, + RecommendOnly: ctx.QueryBool("recommend"), + UserId: UserId, }) if err != nil { ctx.ServerError("datasets", err) return } + for _, NotColDataset := range NotColDatasets { + IsColDatasets = append(IsColDatasets, NotColDataset) + } + datasets := IsColDatasets + count := NotColcount + IsColcount + //排序 + sort.Slice(datasets, func(i, j int) bool { + return datasets[i].Attachment.CreatedUnix > datasets[j].Attachment.CreatedUnix + }) - data, err := json.Marshal(datasets) + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pagesize := ctx.QueryInt("pagesize") + if pagesize <= 0 { + pagesize = 5 + } + pageDatasetsInfo := getPageDatasets(datasets, page, pagesize) + if pageDatasetsInfo == nil { + ctx.JSON(200, map[string]string{ + "result_code": "0", + "data": "[]", + "count": strconv.FormatInt(count, 10), + }) + return + } + data, err := json.Marshal(pageDatasetsInfo) + log.Info("data:", data) if err != nil { log.Error("json.Marshal failed:", err.Error()) ctx.JSON(200, map[string]string{ @@ -507,6 +561,29 @@ func MyFavoriteDataset(ctx *context.Context) { "count": strconv.FormatInt(count, 10), }) +} +func getPageDatasets(AttachmentInfos []*models.AttachmentInfo, page int, pagesize int) []*models.AttachmentInfo { + begin := (page - 1) * pagesize + end := (page) * pagesize + + if begin > len(AttachmentInfos)-1 { + return nil + } + if end > len(AttachmentInfos)-1 { + return AttachmentInfos[begin:] + } else { + return AttachmentInfos[begin:end] + } + +} +func getTotalPage(total int64, pageSize int) int { + + another := 0 + if int(total)%pageSize != 0 { + another = 1 + } + return int(total)/pageSize + another + } func GetDatasetStatus(ctx *context.Context) { @@ -533,3 +610,30 @@ func GetDatasetStatus(ctx *context.Context) { "AttachmentStatus": fmt.Sprint(attachment.DecompressState), }) } +func DatasetIsCollaborator(ctx *context.Context, DatasetID int64) bool { + dataset, err := models.GetDatasetByID(DatasetID) + if err != nil { + log.Info("query dataset error") + } else { + repo, err := models.GetRepositoryByID(dataset.RepoID) + if err != nil { + log.Info("query repo error.") + } else { + repo.GetOwner() + if ctx.User != nil { + if repo.Owner.IsOrganization() { + if repo.Owner.IsUserPartOfOrg(ctx.User.ID) { + log.Info("org user may visit the attach.") + return true + } + } + isCollaborator, _ := repo.IsCollaborator(ctx.User.ID) + if isCollaborator { + log.Info("Collaborator user may visit the attach.") + return true + } + } + } + } + return false +} diff --git a/routers/routes/routes.go b/routers/routes/routes.go index dd0605841..7732ff217 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -331,6 +331,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/all/search/", routers.Search) m.Get("/all/search/", routers.EmptySearch) m.Get("/all/dosearch/", routers.SearchApi) + m.Post("/user/login/kanban", user.SignInPostAPI) m.Get("/home/term", routers.HomeTerm) m.Group("/explore", func() { m.Get("", func(ctx *context.Context) { @@ -365,6 +366,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/user", func() { m.Get("/login", user.SignIn) m.Get("/login/cloud_brain", user.SignInCloudBrain) + m.Post("/login", bindIgnErr(auth.SignInForm{}), user.SignInPost) m.Group("", func() { m.Combo("/login/openid"). diff --git a/routers/user/auth.go b/routers/user/auth.go index a02cf24dc..1edc5a0dd 100755 --- a/routers/user/auth.go +++ b/routers/user/auth.go @@ -176,6 +176,41 @@ func SignInCloudBrain(ctx *context.Context) { ctx.HTML(200, tplSignInCloudBrain) } +func SignInPostAPI(ctx *context.Context) { + ctx.Data["Title"] = ctx.Tr("sign_in") + UserName := ctx.Query("UserName") + Password := ctx.Query("Password") + log.Info("0000000") + orderedOAuth2Names, oauth2Providers, err := models.GetActiveOAuth2Providers() + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + ctx.Data["OrderedOAuth2Names"] = orderedOAuth2Names + ctx.Data["OAuth2Providers"] = oauth2Providers + ctx.Data["Title"] = ctx.Tr("sign_in") + ctx.Data["SignInLink"] = setting.AppSubURL + "/user/login" + ctx.Data["PageIsSignIn"] = true + ctx.Data["PageIsLogin"] = true + ctx.Data["IsCourse"] = ctx.QueryBool("course") + ctx.Data["EnableSSPI"] = models.IsSSPIEnabled() + + if ctx.HasError() { + ctx.HTML(200, tplSignIn) + return + } + u, err := models.UserSignIn(UserName, Password) + if err != nil { + ctx.ServerError("UserSignIn", err) + return + } + models.SaveLoginInfoToDb(ctx.Req.Request, u) + // If this user is enrolled in 2FA, we can't sign the user in just yet. + // Instead, redirect them to the 2FA authentication page. + //handleSignInFull(ctx, u, form.Remember, false) + handleSignInFullNotRedirect(ctx, u, true, false) +} + // SignInPost response for sign in request func SignInPost(ctx *context.Context, form auth.SignInForm) { ctx.Data["Title"] = ctx.Tr("sign_in") @@ -518,6 +553,68 @@ func handleSignIn(ctx *context.Context, u *models.User, remember bool) { handleSignInFull(ctx, u, remember, true) } +func handleSignInFullNotRedirect(ctx *context.Context, u *models.User, remember bool, obeyRedirect bool) string { + + log.Info("enter here.") + if remember { + days := 86400 * setting.LogInRememberDays + ctx.SetCookie(setting.CookieUserName, u.Name, days, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true) + ctx.SetSuperSecureCookie(base.EncodeMD5(u.Rands+u.Passwd), + setting.CookieRememberName, u.Name, days, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true) + } + + _ = ctx.Session.Delete("openid_verified_uri") + _ = ctx.Session.Delete("openid_signin_remember") + _ = ctx.Session.Delete("openid_determined_email") + _ = ctx.Session.Delete("openid_determined_username") + _ = ctx.Session.Delete("twofaUid") + _ = ctx.Session.Delete("twofaRemember") + _ = ctx.Session.Delete("u2fChallenge") + _ = ctx.Session.Delete("linkAccount") + if err := ctx.Session.Set("uid", u.ID); err != nil { + log.Error("Error setting uid %d in session: %v", u.ID, err) + } + if err := ctx.Session.Set("uname", u.Name); err != nil { + log.Error("Error setting uname %s session: %v", u.Name, err) + } + if err := ctx.Session.Release(); err != nil { + log.Error("Unable to store session: %v", err) + } + + // If the user does not have a locale set, we save the current one. + if len(u.Language) == 0 { + if len(ctx.GetCookie("lang")) != 0 { + u.Language = ctx.GetCookie("lang") + } else { + u.Language = ctx.Locale.Language() + } + + if err := models.UpdateUserCols(u, "language"); err != nil { + log.Error(fmt.Sprintf("Error updating user language [user: %d, locale: %s]", u.ID, u.Language)) + return setting.AppSubURL + "/dashboard" + } + } else { + // Language setting of the user use the one previously set + if len(ctx.GetCookie("lang")) != 0 { + u.Language = ctx.GetCookie("lang") + } + } + + ctx.SetCookie("lang", u.Language, nil, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true) + + // Clear whatever CSRF has right now, force to generate a new one + ctx.SetCookie(setting.CSRFCookieName, "", -1, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true) + + // Register last login + u.SetLastLogin() + if err := models.UpdateUserCols(u, "last_login_unix"); err != nil { + ctx.ServerError("UpdateUserCols", err) + return setting.AppSubURL + "/dashboard" + } + + return setting.AppSubURL + "/dashboard" +} + func handleSignInFull(ctx *context.Context, u *models.User, remember bool, obeyRedirect bool) string { if remember { days := 86400 * setting.LogInRememberDays diff --git a/templates/explore/datasets.tmpl b/templates/explore/datasets.tmpl index 0cfe1fd18..327c4fb98 100644 --- a/templates/explore/datasets.tmpl +++ b/templates/explore/datasets.tmpl @@ -199,7 +199,7 @@ {{else}} + style="display: flex;align-items: center;justify-content: flex-end;cursor: pointer;font-size: 12px;font-weight: normal;flex: 1;margin-left: 1.5rem;">
arr.concat().sort((a, b) => (a[k] > b[k] ? 1 : a[k] < b[k] ? -1 : 0)); $('.metric_chart').click(function (e) { let versionName = $(this).data('version') let myCharts = echarts.init(document.getElementById(`metric-${versionName}`)) - $.get(`${window.config.AppSubUrl}/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/metric_statistics?version_name=${versionName}&statistic_type=each`, (res) => { + $.get(`${window.config.AppSubUrl}/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/metric_statistics?version_name=${versionName}&statistic_type=each&metrics=`, (res) => { let filterDta = res.MetricsInfo.filter((item) => { return !(['recvBytesRate', 'diskWriteRate', 'sendBytesRate', 'diskReadRate'].includes(item.metric)) }) + filterDta = sortBy(filterDta, "metric") let legenData = filterDta.map((item) => { return item.metric })