| @@ -82,6 +82,7 @@ type AttachmentsOptions struct { | |||||
| NeedRepoInfo bool | NeedRepoInfo bool | ||||
| Keyword string | Keyword string | ||||
| RecommendOnly bool | RecommendOnly bool | ||||
| UserId int64 | |||||
| } | } | ||||
| func (a *Attachment) AfterUpdate() { | func (a *Attachment) AfterUpdate() { | ||||
| @@ -24,7 +24,7 @@ type ModelArtsJobStatus string | |||||
| const ( | const ( | ||||
| TypeCloudBrainOne int = iota | TypeCloudBrainOne int = iota | ||||
| TypeCloudBrainTwo | TypeCloudBrainTwo | ||||
| TypeCloudBrainGrampus | |||||
| TypeC2Net //智算网络 | |||||
| TypeCloudBrainAll = -1 | TypeCloudBrainAll = -1 | ||||
| ) | ) | ||||
| @@ -339,7 +339,11 @@ type CloudbrainsOptions struct { | |||||
| JobTypeNot bool | JobTypeNot bool | ||||
| NeedRepoInfo bool | NeedRepoInfo bool | ||||
| RepoIDList []int64 | RepoIDList []int64 | ||||
| BeginTime time.Time | |||||
| EndTime time.Time | |||||
| ComputeResource string | ComputeResource string | ||||
| BeginTimeUnix int64 | |||||
| EndTimeUnix int64 | |||||
| } | } | ||||
| type TaskPod struct { | type TaskPod struct { | ||||
| @@ -1273,6 +1277,11 @@ func Cloudbrains(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { | |||||
| builder.Eq{"cloudbrain.job_id": opts.JobID}, | builder.Eq{"cloudbrain.job_id": opts.JobID}, | ||||
| ) | ) | ||||
| } | } | ||||
| if (opts.ComputeResource) != "" { | |||||
| cond = cond.And( | |||||
| builder.Eq{"cloudbrain.compute_resource": opts.ComputeResource}, | |||||
| ) | |||||
| } | |||||
| if (opts.Type) >= 0 { | if (opts.Type) >= 0 { | ||||
| cond = cond.And( | cond = cond.And( | ||||
| @@ -1605,6 +1614,11 @@ func UpdateJob(job *Cloudbrain) error { | |||||
| return updateJob(x, job) | return updateJob(x, job) | ||||
| } | } | ||||
| func UpdateJobDurationWithDeleted(job *Cloudbrain) error { | |||||
| _, err := x.Exec("update cloudbrain set start_time=?, end_time=?,train_job_duration=?,duration=? where id=?", job.StartTime, job.EndTime, job.TrainJobDuration, job.Duration, job.ID) | |||||
| return err | |||||
| } | |||||
| func updateJob(e Engine, job *Cloudbrain) error { | func updateJob(e Engine, job *Cloudbrain) error { | ||||
| _, err := e.ID(job.ID).AllCols().Update(job) | _, err := e.ID(job.ID).AllCols().Update(job) | ||||
| return err | return err | ||||
| @@ -1675,6 +1689,10 @@ func GetStoppedJobWithNoDurationJob() ([]*Cloudbrain, error) { | |||||
| Limit(100). | Limit(100). | ||||
| Find(&cloudbrains) | Find(&cloudbrains) | ||||
| } | } | ||||
| func GetStoppedJobWithNoStartTimeEndTime() ([]*Cloudbrain, error) { | |||||
| cloudbrains := make([]*Cloudbrain, 0) | |||||
| return cloudbrains, x.SQL("select * from cloudbrain where status in (?,?,?,?,?,?,?) and (start_time is null or end_time is null) limit 100", ModelArtsTrainJobCompleted, ModelArtsTrainJobFailed, ModelArtsTrainJobKilled, ModelArtsStopped, JobStopped, JobFailed, JobSucceeded).Find(&cloudbrains) | |||||
| } | |||||
| func GetCloudbrainCountByUserID(userID int64, jobType string) (int, error) { | func GetCloudbrainCountByUserID(userID int64, jobType string) (int, error) { | ||||
| count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ? and type = ?", jobType, userID, TypeCloudBrainOne).Count(new(Cloudbrain)) | count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ? and type = ?", jobType, userID, TypeCloudBrainOne).Count(new(Cloudbrain)) | ||||
| @@ -1713,7 +1731,7 @@ func GetCloudbrainInferenceJobCountByUserID(userID int64) (int, error) { | |||||
| } | } | ||||
| func GetGrampusCountByUserID(userID int64, jobType, computeResource string) (int, error) { | func GetGrampusCountByUserID(userID int64, jobType, computeResource string) (int, error) { | ||||
| count, err := x.In("status", GrampusStatusPending, GrampusStatusRunning).And("job_type = ? and user_id = ? and type = ?", jobType, userID, TypeCloudBrainGrampus).And("compute_resource = ?", computeResource).Count(new(Cloudbrain)) | |||||
| count, err := x.In("status", GrampusStatusPending, GrampusStatusRunning).And("job_type = ? and user_id = ? and type = ?", jobType, userID, TypeC2Net).And("compute_resource = ?", computeResource).Count(new(Cloudbrain)) | |||||
| return int(count), err | return int(count), err | ||||
| } | } | ||||
| @@ -1756,13 +1774,80 @@ func RestartCloudbrain(old *Cloudbrain, new *Cloudbrain) (err error) { | |||||
| func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { | func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { | ||||
| sess := x.NewSession() | sess := x.NewSession() | ||||
| defer sess.Close() | defer sess.Close() | ||||
| var cond = builder.NewCond() | var cond = builder.NewCond() | ||||
| if opts.RepoID > 0 { | |||||
| cond = cond.And( | |||||
| builder.Eq{"cloudbrain.repo_id": opts.RepoID}, | |||||
| ) | |||||
| } | |||||
| if opts.UserID > 0 { | |||||
| cond = cond.And( | |||||
| builder.Eq{"cloudbrain.user_id": opts.UserID}, | |||||
| ) | |||||
| } | |||||
| if (opts.JobID) != "" { | |||||
| cond = cond.And( | |||||
| builder.Eq{"cloudbrain.job_id": opts.JobID}, | |||||
| ) | |||||
| } | |||||
| if (opts.ComputeResource) != "" { | |||||
| cond = cond.And( | |||||
| builder.Eq{"cloudbrain.compute_resource": opts.ComputeResource}, | |||||
| ) | |||||
| } | |||||
| if (opts.Type) >= 0 { | if (opts.Type) >= 0 { | ||||
| cond = cond.And( | cond = cond.And( | ||||
| builder.Eq{"cloudbrain.type": opts.Type}, | builder.Eq{"cloudbrain.type": opts.Type}, | ||||
| ) | ) | ||||
| } | } | ||||
| if len(opts.JobTypes) > 0 { | |||||
| if opts.JobTypeNot { | |||||
| cond = cond.And( | |||||
| builder.NotIn("cloudbrain.job_type", opts.JobTypes), | |||||
| ) | |||||
| } else { | |||||
| cond = cond.And( | |||||
| builder.In("cloudbrain.job_type", opts.JobTypes), | |||||
| ) | |||||
| } | |||||
| } | |||||
| if (opts.IsLatestVersion) != "" { | |||||
| cond = cond.And(builder.Or(builder.And(builder.Eq{"cloudbrain.is_latest_version": opts.IsLatestVersion}, builder.Eq{"cloudbrain.job_type": "TRAIN"}), builder.Neq{"cloudbrain.job_type": "TRAIN"})) | |||||
| } | |||||
| if len(opts.CloudbrainIDs) > 0 { | |||||
| cond = cond.And(builder.In("cloudbrain.id", opts.CloudbrainIDs)) | |||||
| } | |||||
| if len(opts.JobStatus) > 0 { | |||||
| if opts.JobStatusNot { | |||||
| cond = cond.And( | |||||
| builder.NotIn("cloudbrain.status", opts.JobStatus), | |||||
| ) | |||||
| } else { | |||||
| cond = cond.And( | |||||
| builder.In("cloudbrain.status", opts.JobStatus), | |||||
| ) | |||||
| } | |||||
| } | |||||
| if len(opts.RepoIDList) > 0 { | |||||
| cond = cond.And( | |||||
| builder.In("cloudbrain.repo_id", opts.RepoIDList), | |||||
| ) | |||||
| } | |||||
| if opts.BeginTimeUnix > 0 && opts.EndTimeUnix > 0 { | |||||
| cond = cond.And( | |||||
| builder.And(builder.Gte{"cloudbrain.created_unix": opts.BeginTimeUnix}, builder.Lte{"cloudbrain.created_unix": opts.EndTimeUnix}), | |||||
| ) | |||||
| } | |||||
| var count int64 | var count int64 | ||||
| var err error | var err error | ||||
| condition := "cloudbrain.user_id = `user`.id" | condition := "cloudbrain.user_id = `user`.id" | ||||
| @@ -1,6 +1,191 @@ | |||||
| package models | package models | ||||
| import "code.gitea.io/gitea/modules/log" | |||||
| import ( | |||||
| "strconv" | |||||
| "time" | |||||
| "code.gitea.io/gitea/modules/log" | |||||
| "code.gitea.io/gitea/modules/timeutil" | |||||
| "code.gitea.io/gitea/modules/util" | |||||
| "xorm.io/builder" | |||||
| ) | |||||
| type TaskDetail struct { | |||||
| ID int64 `json:"ID"` | |||||
| JobID string `json:"JobID"` | |||||
| JobName string `json:"JobName"` | |||||
| DisplayJobName string `json:"DisplayJobName"` | |||||
| Status string `json:"Status"` | |||||
| JobType string `json:"JobType"` | |||||
| CreatedUnix timeutil.TimeStamp `json:"CreatedUnix"` | |||||
| WaitTime string `json:"WaitTime"` | |||||
| RunTime string `json:"RunTime"` | |||||
| StartTime timeutil.TimeStamp `json:"StartTime"` | |||||
| EndTime timeutil.TimeStamp `json:"EndTime"` | |||||
| ComputeResource string `json:"ComputeResource"` | |||||
| Type int `json:"Type"` | |||||
| UserName string `json:"UserName"` | |||||
| RepoName string `json:"RepoName"` | |||||
| RepoAlias string `json:"RepoAlias"` | |||||
| RepoID int64 `json:"RepoID"` | |||||
| IsDelete bool `json:"IsDelete"` | |||||
| } | |||||
| func GetDebugOnePeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| countSql := "SELECT count(*) FROM " + | |||||
| "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + | |||||
| " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + | |||||
| " and job_type ='" + string(JobTypeDebug) + "'" + | |||||
| " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" | |||||
| return x.SQL(countSql).Count() | |||||
| } | |||||
| func GetDebugOnePeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeDebug, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration") | |||||
| if err != nil { | |||||
| return 0, err | |||||
| } | |||||
| return total, nil | |||||
| } | |||||
| func GetTrainOnePeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| countSql := "SELECT count(*) FROM " + | |||||
| "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + | |||||
| " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + | |||||
| " and job_type ='" + string(JobTypeTrain) + "'" + | |||||
| " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" | |||||
| return x.SQL(countSql).Count() | |||||
| } | |||||
| func GetTrainOnePeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeTrain, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration") | |||||
| if err != nil { | |||||
| return 0, err | |||||
| } | |||||
| return total, nil | |||||
| } | |||||
| func GetBenchmarkOnePeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| countSql := "SELECT count(*) FROM " + | |||||
| "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + | |||||
| " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + | |||||
| " and job_type ='" + string(JobTypeBenchmark) + "'" + | |||||
| " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" | |||||
| return x.SQL(countSql).Count() | |||||
| } | |||||
| func GetBenchmarkOnePeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeBenchmark, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration") | |||||
| if err != nil { | |||||
| return 0, err | |||||
| } | |||||
| return total, nil | |||||
| } | |||||
| func GetDebugTwoPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| countSql := "SELECT count(*) FROM " + | |||||
| "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + | |||||
| " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + | |||||
| " and job_type ='" + string(JobTypeDebug) + "'" + | |||||
| " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" | |||||
| return x.SQL(countSql).Count() | |||||
| } | |||||
| func GetDebugTwoPeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeDebug, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration") | |||||
| if err != nil { | |||||
| return 0, err | |||||
| } | |||||
| return total, nil | |||||
| } | |||||
| func GetTrainTwoPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| countSql := "SELECT count(*) FROM " + | |||||
| "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + | |||||
| " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + | |||||
| " and job_type ='" + string(JobTypeTrain) + "'" + | |||||
| " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" | |||||
| return x.SQL(countSql).Count() | |||||
| } | |||||
| func GetTrainTwoPeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeTrain, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration") | |||||
| if err != nil { | |||||
| return 0, err | |||||
| } | |||||
| return total, nil | |||||
| } | |||||
| func GetInferenceTwoPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| countSql := "SELECT count(*) FROM " + | |||||
| "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + | |||||
| " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + | |||||
| " and job_type ='" + string(JobTypeInference) + "'" + | |||||
| " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" | |||||
| return x.SQL(countSql).Count() | |||||
| } | |||||
| func GetInferenceTwoPeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeInference, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration") | |||||
| if err != nil { | |||||
| return 0, err | |||||
| } | |||||
| return total, nil | |||||
| } | |||||
| func GetCloudBrainOnePeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| countSql := "SELECT count(*) FROM " + | |||||
| "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + | |||||
| " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + | |||||
| " and type='" + strconv.Itoa(TypeCloudBrainOne) + "'" | |||||
| return x.SQL(countSql).Count() | |||||
| } | |||||
| func GetCloudBrainOnePeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| total, err := x.Where("created_unix >= ? And created_unix < ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration") | |||||
| if err != nil { | |||||
| return 0, err | |||||
| } | |||||
| return total, nil | |||||
| } | |||||
| func GetCloudBrainTwoPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| countSql := "SELECT count(*) FROM " + | |||||
| "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + | |||||
| " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + | |||||
| " and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'" | |||||
| return x.SQL(countSql).Count() | |||||
| } | |||||
| func GetCloudBrainTwoPeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| total, err := x.Where("created_unix >= ? And created_unix < ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration") | |||||
| if err != nil { | |||||
| return 0, err | |||||
| } | |||||
| return total, nil | |||||
| } | |||||
| func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| countSql := "SELECT count(distinct user_id) FROM " + | |||||
| "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + | |||||
| " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) | |||||
| return x.SQL(countSql).Count() | |||||
| } | |||||
| func GetTodayCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, error) { | |||||
| countSql := "SELECT count FROM " + | |||||
| "public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + | |||||
| " and created_unix<=" + strconv.FormatInt(endTime.Unix(), 10) | |||||
| return x.SQL(countSql).Count() | |||||
| } | |||||
| func GetCreatorCount() (int64, error) { | |||||
| countSql := "SELECT count(distinct user_id) FROM public.cloudbrain" | |||||
| return x.SQL(countSql).Count() | |||||
| } | |||||
| func GetRecordBeginTime() ([]*CloudbrainInfo, error) { | |||||
| sess := x.NewSession() | |||||
| defer sess.Close() | |||||
| sess.OrderBy("cloudbrain.id ASC limit 1") | |||||
| cloudbrains := make([]*CloudbrainInfo, 0) | |||||
| if err := sess.Table(&Cloudbrain{}).Unscoped(). | |||||
| Find(&cloudbrains); err != nil { | |||||
| log.Info("find error.") | |||||
| } | |||||
| return cloudbrains, nil | |||||
| } | |||||
| func GetAllStatusCloudBrain() map[string]int { | func GetAllStatusCloudBrain() map[string]int { | ||||
| sess := x.NewSession() | sess := x.NewSession() | ||||
| @@ -20,3 +205,87 @@ func GetAllStatusCloudBrain() map[string]int { | |||||
| } | } | ||||
| return cloudBrainStatusResult | return cloudBrainStatusResult | ||||
| } | } | ||||
| func GetWaittingTop() ([]*CloudbrainInfo, error) { | |||||
| sess := x.NewSession() | |||||
| defer sess.Close() | |||||
| var cond = builder.NewCond() | |||||
| cond = cond.And( | |||||
| builder.Eq{"cloudbrain.status": string(JobWaiting)}, | |||||
| ) | |||||
| sess.OrderBy("(cloudbrain.start_time-cloudbrain.created_unix) DESC limit 10") | |||||
| cloudbrains := make([]*CloudbrainInfo, 0, 10) | |||||
| if err := sess.Table(&Cloudbrain{}).Where(cond). | |||||
| Find(&cloudbrains); err != nil { | |||||
| log.Info("find error.") | |||||
| } | |||||
| return cloudbrains, nil | |||||
| } | |||||
| func GetRunningTop() ([]*CloudbrainInfo, error) { | |||||
| sess := x.NewSession() | |||||
| defer sess.Close() | |||||
| var cond = builder.NewCond() | |||||
| cond = cond.And( | |||||
| builder.Eq{"cloudbrain.status": string(JobRunning)}, | |||||
| ) | |||||
| sess.OrderBy("(cloudbrain.end_time-cloudbrain.start_time) DESC limit 10") | |||||
| cloudbrains := make([]*CloudbrainInfo, 0, 10) | |||||
| if err := sess.Table(&Cloudbrain{}).Where(cond). | |||||
| Find(&cloudbrains); err != nil { | |||||
| log.Info("find error.") | |||||
| } | |||||
| return cloudbrains, nil | |||||
| } | |||||
| func getCreatePeriodCount(dateBeginTime string, dateEndTime string, hourBeginTime string, hourEndTime string) (int64, error) { | |||||
| countSql := "SELECT count(*) FROM " + | |||||
| "public.cloudbrain where to_char(to_timestamp(created_unix), 'YYYY-MM-DD') >= '" + dateBeginTime + | |||||
| "' and to_char(to_timestamp(created_unix), 'YYYY-MM-DD') < '" + dateEndTime + | |||||
| "' and to_char(to_timestamp(created_unix), 'HH24:MI:SS') >= '" + hourBeginTime + | |||||
| "' and to_char(to_timestamp(created_unix), 'HH24:MI:SS') < '" + hourEndTime + "'" | |||||
| return x.SQL(countSql).Count() | |||||
| } | |||||
| //SELECT * FROM xxx WHERE NOT ((endTime < hourBeginTime) OR (startTime > hourEndTime)) | |||||
| func getRunPeriodCount(dateBeginTime string, dateEndTime string, hourBeginTime string, hourEndTime string) (int64, error) { | |||||
| countSql := "SELECT count(*) FROM " + | |||||
| "public.cloudbrain where not ((to_char(to_timestamp(start_time), ' HH24:MI:SS') > '" + hourEndTime + | |||||
| "') or (to_char(to_timestamp(end_time), 'HH24:MI:SS') < '" + hourBeginTime + "'))" + | |||||
| " and (to_char(to_timestamp(start_time), 'YYYY-MM-DD') >= '" + dateBeginTime + | |||||
| "' and to_char(to_timestamp(start_time), 'YYYY-MM-DD') < '" + dateEndTime + "')" | |||||
| return x.SQL(countSql).Count() | |||||
| } | |||||
| func GetCreateHourPeriodCount(dateBeginTime string, dateEndTime string) (map[string]interface{}, error) { | |||||
| //0 to 23 for each hour, | |||||
| dateHourMap := make(map[string]interface{}) | |||||
| var slice = []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23} | |||||
| for key, value := range slice { | |||||
| hourBeginHour := util.AddZero(value) + ":00:00" | |||||
| hourEndHour := util.AddZero(value+1) + ":00:00" | |||||
| cout, err := getCreatePeriodCount(dateBeginTime, dateEndTime, hourBeginHour, hourEndHour) | |||||
| if err != nil { | |||||
| log.Error("Can not query getCreatePeriodCount.", err) | |||||
| return nil, nil | |||||
| } | |||||
| dateHourMap[strconv.Itoa(key)] = cout | |||||
| } | |||||
| return dateHourMap, nil | |||||
| } | |||||
| func GetRunHourPeriodCount(dateBeginTime string, dateEndTime string) (map[string]interface{}, error) { | |||||
| dateHourMap := make(map[string]interface{}) | |||||
| var slice = []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23} | |||||
| for key, value := range slice { | |||||
| hourBeginHour := util.AddZero(value) + ":00:00" | |||||
| hourEndHour := util.AddZero(value+1) + ":00:00" | |||||
| cout, err := getRunPeriodCount(dateBeginTime, dateEndTime, hourBeginHour, hourEndHour) | |||||
| if err != nil { | |||||
| log.Error("Can not query getRunPeriodCount.", err) | |||||
| return nil, nil | |||||
| } | |||||
| dateHourMap[strconv.Itoa(key)] = cout | |||||
| } | |||||
| return dateHourMap, nil | |||||
| } | |||||
| @@ -90,7 +90,7 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error | |||||
| JobName: req.JobName, | JobName: req.JobName, | ||||
| DisplayJobName: req.DisplayJobName, | DisplayJobName: req.DisplayJobName, | ||||
| JobType: string(models.JobTypeTrain), | JobType: string(models.JobTypeTrain), | ||||
| Type: models.TypeCloudBrainGrampus, | |||||
| Type: models.TypeC2Net, | |||||
| Uuid: req.Uuid, | Uuid: req.Uuid, | ||||
| DatasetName: req.DatasetName, | DatasetName: req.DatasetName, | ||||
| CommitID: req.CommitID, | CommitID: req.CommitID, | ||||
| @@ -81,7 +81,6 @@ func getToken() error { | |||||
| } | } | ||||
| TOKEN = result.Token | TOKEN = result.Token | ||||
| log.Info(TOKEN) | |||||
| return nil | return nil | ||||
| } | } | ||||
| @@ -124,7 +123,6 @@ func GetJob(jobID string) (*models.GetGrampusJobResponse, error) { | |||||
| client := getRestyClient() | client := getRestyClient() | ||||
| var result models.GetGrampusJobResponse | var result models.GetGrampusJobResponse | ||||
| log.Info(jobID, TOKEN) | |||||
| retry := 0 | retry := 0 | ||||
| sendjob: | sendjob: | ||||
| @@ -146,7 +144,7 @@ sendjob: | |||||
| if result.ErrorCode != 0 { | if result.ErrorCode != 0 { | ||||
| log.Error("GetJob failed(%d): %s", result.ErrorCode, result.ErrorMsg) | log.Error("GetJob failed(%d): %s", result.ErrorCode, result.ErrorMsg) | ||||
| return &result, fmt.Errorf("GetJob failed(%d): %s", result.ErrorCode, result.ErrorMsg) | |||||
| return nil, fmt.Errorf("GetJob failed(%d): %s", result.ErrorCode, result.ErrorMsg) | |||||
| } | } | ||||
| return &result, nil | return &result, nil | ||||
| @@ -1131,7 +1131,7 @@ sendjob: | |||||
| res, err := client.R(). | res, err := client.R(). | ||||
| SetAuthToken(TOKEN). | SetAuthToken(TOKEN). | ||||
| SetResult(&result). | SetResult(&result). | ||||
| Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/pod/" + podName + "/metric-statistic") | |||||
| Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/pod/" + podName + "/metric-statistic?statistic_type=each") | |||||
| if err != nil { | if err != nil { | ||||
| return nil, fmt.Errorf("resty GetTrainJobMetricStatistic: %v", err) | return nil, fmt.Errorf("resty GetTrainJobMetricStatistic: %v", err) | ||||
| @@ -1020,13 +1020,16 @@ balance.total_view=余额总览 | |||||
| balance.available=可用余额: | balance.available=可用余额: | ||||
| cloudbrain1=云脑1 | cloudbrain1=云脑1 | ||||
| cloudbrain2=云脑2 | cloudbrain2=云脑2 | ||||
| intelligent_net=智算网络 | |||||
| cloudbrain_selection=云脑选择 | cloudbrain_selection=云脑选择 | ||||
| cloudbrain_platform_selection=选择您准备使用的云脑平台: | cloudbrain_platform_selection=选择您准备使用的云脑平台: | ||||
| confirm_choice=确定 | confirm_choice=确定 | ||||
| cloudbran1_tips=只有zip格式的数据集才能发起云脑任务 | cloudbran1_tips=只有zip格式的数据集才能发起云脑任务 | ||||
| cloudbrain_creator=创建者 | cloudbrain_creator=创建者 | ||||
| cloudbrain_type=支撑算力 | |||||
| cloudbrain_untype=未知支撑算力 | |||||
| cloudbrain_task=任务名称 | cloudbrain_task=任务名称 | ||||
| cloudbrain_task_type=任务类型 | |||||
| cloudbrain_task_type=云脑任务类型 | |||||
| cloudbrain_task_name=云脑侧任务名称 | cloudbrain_task_name=云脑侧任务名称 | ||||
| cloudbrain_operate=操作 | cloudbrain_operate=操作 | ||||
| cloudbrain_status_createtime=状态/创建时间 | cloudbrain_status_createtime=状态/创建时间 | ||||
| @@ -1060,6 +1063,7 @@ computing.success=加入成功 | |||||
| modelarts.status=状态 | modelarts.status=状态 | ||||
| modelarts.createtime=创建时间 | modelarts.createtime=创建时间 | ||||
| modelarts.deletetime=删除时间 | |||||
| modelarts.version_nums=版本数 | modelarts.version_nums=版本数 | ||||
| modelarts.version=版本 | modelarts.version=版本 | ||||
| modelarts.computing_resources=计算资源 | modelarts.computing_resources=计算资源 | ||||
| @@ -1090,8 +1094,8 @@ modelarts.train_job.job_status=任务状态 | |||||
| modelarts.train_job.job_name=任务名称 | modelarts.train_job.job_name=任务名称 | ||||
| modelarts.train_job.version=任务版本 | modelarts.train_job.version=任务版本 | ||||
| modelarts.train_job.start_time=开始运行时间 | modelarts.train_job.start_time=开始运行时间 | ||||
| modelarts.train_job.end_time=运行结束时间 | |||||
| modelarts.train_job.wait_time=等待时间 | |||||
| modelarts.train_job.end_time=结束运行时间 | |||||
| modelarts.train_job.wait_time=等待时长 | |||||
| modelarts.train_job.dura_time=运行时长 | modelarts.train_job.dura_time=运行时长 | ||||
| modelarts.train_job.description=任务描述 | modelarts.train_job.description=任务描述 | ||||
| modelarts.train_job.parameter_setting=参数设置 | modelarts.train_job.parameter_setting=参数设置 | ||||
| @@ -574,9 +574,18 @@ func RegisterRoutes(m *macaron.Macaron) { | |||||
| m.Group("/cloudbrainboard", func() { | m.Group("/cloudbrainboard", func() { | ||||
| m.Get("/downloadAll", repo.DownloadCloudBrainBoard) | m.Get("/downloadAll", repo.DownloadCloudBrainBoard) | ||||
| m.Group("/cloudbrain", func() { | m.Group("/cloudbrain", func() { | ||||
| m.Get("/overview", repo.GetAllCloudbrainsOverview) | |||||
| m.Get("/distribution", repo.GetAllCloudbrainsPeriodDistribution) | |||||
| m.Get("/trend", repo.GetAllCloudbrainsTrend) | |||||
| m.Get("/trend_detail_data", repo.GetAllCloudbrainsTrendDetail) | |||||
| m.Get("/status_analysis", repo.GetCloudbrainsStatusAnalysis) | m.Get("/status_analysis", repo.GetCloudbrainsStatusAnalysis) | ||||
| m.Get("/detail_data", repo.GetCloudbrainsDetailData) | |||||
| m.Get("/hours_data", repo.GetCloudbrainsCreateHoursData) | |||||
| m.Get("/waitting_top_data", repo.GetWaittingTop) | |||||
| m.Get("/running_top_data", repo.GetRunningTop) | |||||
| }) | }) | ||||
| }, operationReq) | }, operationReq) | ||||
| // Users | // Users | ||||
| m.Group("/users", func() { | m.Group("/users", func() { | ||||
| m.Get("/search", user.Search) | m.Get("/search", user.Search) | ||||
| @@ -191,7 +191,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) { | |||||
| if err != nil { | if err != nil { | ||||
| log.Error("UpdateJob failed:", err) | log.Error("UpdateJob failed:", err) | ||||
| } | } | ||||
| } else if job.Type == models.TypeCloudBrainGrampus { | |||||
| } else if job.Type == models.TypeC2Net { | |||||
| result, err := grampus.GetJob(jobID) | result, err := grampus.GetJob(jobID) | ||||
| if err != nil { | if err != nil { | ||||
| log.Error("GetJob(%s) failed:%v", job.JobName, err) | log.Error("GetJob(%s) failed:%v", job.JobName, err) | ||||
| @@ -192,7 +192,6 @@ func GetProjectsSummaryData(ctx *context.Context) { | |||||
| } | } | ||||
| projectSummaryPeriodData := ProjectSummaryPeriodData{ | projectSummaryPeriodData := ProjectSummaryPeriodData{ | ||||
| TotalCount: count - 1, | TotalCount: count - 1, | ||||
| RecordBeginTime: recordBeginTime.Format(DATE_FORMAT), | RecordBeginTime: recordBeginTime.Format(DATE_FORMAT), | ||||
| @@ -203,7 +202,7 @@ func GetProjectsSummaryData(ctx *context.Context) { | |||||
| } | } | ||||
| func reverse(datas []*ProjectSummaryBaseData ) []*ProjectSummaryBaseData { | |||||
| func reverse(datas []*ProjectSummaryBaseData) []*ProjectSummaryBaseData { | |||||
| for i := 0; i < len(datas)/2; i++ { | for i := 0; i < len(datas)/2; i++ { | ||||
| j := len(datas) - i - 1 | j := len(datas) - i - 1 | ||||
| datas[i], datas[j] = datas[j], datas[i] | datas[i], datas[j] = datas[j], datas[i] | ||||
| @@ -211,8 +210,6 @@ func reverse(datas []*ProjectSummaryBaseData ) []*ProjectSummaryBaseData { | |||||
| return datas | return datas | ||||
| } | } | ||||
| func setStatisticsData(data *ProjectSummaryBaseData, v *models.SummaryStatistic, stats *models.SummaryStatistic) { | func setStatisticsData(data *ProjectSummaryBaseData, v *models.SummaryStatistic, stats *models.SummaryStatistic) { | ||||
| data.NumReposAdd = v.NumRepos - stats.NumRepos | data.NumReposAdd = v.NumRepos - stats.NumRepos | ||||
| data.NumRepoPublicAdd = v.NumRepoPublic - stats.NumRepoPublic | data.NumRepoPublicAdd = v.NumRepoPublic - stats.NumRepoPublic | ||||
| @@ -890,12 +887,19 @@ func getTimePeroid(ctx *context.Context, recordBeginTime time.Time) (time.Time, | |||||
| if queryType == "all" { | if queryType == "all" { | ||||
| beginTime = recordBeginTimeTemp | beginTime = recordBeginTimeTemp | ||||
| endTime = now | endTime = now | ||||
| } else if queryType == "yesterday" { | |||||
| } else if queryType == "today" { | |||||
| endTime = now | endTime = now | ||||
| beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) | beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location()) | ||||
| } else if queryType == "yesterday" { | |||||
| endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) | |||||
| beginTime = endTime.AddDate(0, 0, -1) | |||||
| } else if queryType == "current_week" { | |||||
| beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+2) //begin from monday | |||||
| } else if queryType == "last_7day" { | |||||
| beginTime = now.AddDate(0, 0, -7) | |||||
| beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) | |||||
| endTime = now | |||||
| } else if queryType == "last_30day" { | |||||
| beginTime = now.AddDate(0, 0, -30) | |||||
| beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) | beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location()) | ||||
| endTime = now | endTime = now | ||||
| } else if queryType == "current_month" { | } else if queryType == "current_month" { | ||||
| @@ -1470,7 +1470,7 @@ func SyncCloudbrainStatus() { | |||||
| } else { | } else { | ||||
| log.Error("task.JobType(%s) is error:%s", task.JobName, task.JobType) | log.Error("task.JobType(%s) is error:%s", task.JobName, task.JobType) | ||||
| } | } | ||||
| } else if task.Type == models.TypeCloudBrainGrampus { | |||||
| } else if task.Type == models.TypeC2Net { | |||||
| result, err := grampus.GetJob(task.JobID) | result, err := grampus.GetJob(task.JobID) | ||||
| if err != nil { | if err != nil { | ||||
| log.Error("GetTrainJob(%s) failed:%v", task.JobName, err) | log.Error("GetTrainJob(%s) failed:%v", task.JobName, err) | ||||
| @@ -1507,11 +1507,19 @@ func SyncCloudbrainStatus() { | |||||
| } | } | ||||
| func HandleTaskWithNoDuration(ctx *context.Context) { | func HandleTaskWithNoDuration(ctx *context.Context) { | ||||
| mode := ctx.Query("mode") | |||||
| log.Info("HandleTaskWithNoDuration start") | log.Info("HandleTaskWithNoDuration start") | ||||
| count := 0 | count := 0 | ||||
| start := time.Now().Unix() | start := time.Now().Unix() | ||||
| for { | for { | ||||
| cloudBrains, err := models.GetStoppedJobWithNoDurationJob() | |||||
| var cloudBrains []*models.Cloudbrain | |||||
| var err error | |||||
| if mode == "1" { | |||||
| cloudBrains, err = models.GetStoppedJobWithNoStartTimeEndTime() | |||||
| } else { | |||||
| cloudBrains, err = models.GetStoppedJobWithNoDurationJob() | |||||
| } | |||||
| if err != nil { | if err != nil { | ||||
| log.Error("HandleTaskWithNoTrainJobDuration failed:", err.Error()) | log.Error("HandleTaskWithNoTrainJobDuration failed:", err.Error()) | ||||
| break | break | ||||
| @@ -1586,7 +1594,7 @@ func handleNoDurationTask(cloudBrains []*models.Cloudbrain) { | |||||
| } | } | ||||
| task.Duration = task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix() | task.Duration = task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix() | ||||
| task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) | task.TrainJobDuration = models.ConvertDurationToStr(task.Duration) | ||||
| err = models.UpdateJob(task) | |||||
| err = models.UpdateJobDurationWithDeleted(task) | |||||
| if err != nil { | if err != nil { | ||||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | ||||
| } | } | ||||
| @@ -1611,7 +1619,7 @@ func handleNoDurationTask(cloudBrains []*models.Cloudbrain) { | |||||
| } | } | ||||
| task.CorrectCreateUnix() | task.CorrectCreateUnix() | ||||
| task.ComputeAndSetDuration() | task.ComputeAndSetDuration() | ||||
| err = models.UpdateJob(task) | |||||
| err = models.UpdateJobDurationWithDeleted(task) | |||||
| if err != nil { | if err != nil { | ||||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | ||||
| continue | continue | ||||
| @@ -1632,7 +1640,7 @@ func handleNoDurationTask(cloudBrains []*models.Cloudbrain) { | |||||
| task.EndTime = task.StartTime.Add(result.Duration / 1000) | task.EndTime = task.StartTime.Add(result.Duration / 1000) | ||||
| } | } | ||||
| task.ComputeAndSetDuration() | task.ComputeAndSetDuration() | ||||
| err = models.UpdateJob(task) | |||||
| err = models.UpdateJobDurationWithDeleted(task) | |||||
| if err != nil { | if err != nil { | ||||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | ||||
| continue | continue | ||||
| @@ -1653,7 +1661,7 @@ func updateDefaultDuration(task *models.Cloudbrain) { | |||||
| task.StartTime = task.CreatedUnix | task.StartTime = task.CreatedUnix | ||||
| task.EndTime = task.UpdatedUnix | task.EndTime = task.UpdatedUnix | ||||
| task.ComputeAndSetDuration() | task.ComputeAndSetDuration() | ||||
| err := models.UpdateJob(task) | |||||
| err := models.UpdateJobDurationWithDeleted(task) | |||||
| if err != nil { | if err != nil { | ||||
| log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | log.Error("UpdateJob(%s) failed:%v", task.JobName, err) | ||||
| } | } | ||||
| @@ -452,10 +452,11 @@ func PublicDataset(ctx *context.Context) { | |||||
| } | } | ||||
| func MyFavoriteDataset(ctx *context.Context) { | func MyFavoriteDataset(ctx *context.Context) { | ||||
| page := ctx.QueryInt("page") | |||||
| UserId := ctx.User.ID | |||||
| cloudbrainType := ctx.QueryInt("type") | cloudbrainType := ctx.QueryInt("type") | ||||
| keyword := strings.Trim(ctx.Query("q"), " ") | keyword := strings.Trim(ctx.Query("q"), " ") | ||||
| var datasetIDs []int64 | |||||
| var NotColDatasetIDs []int64 | |||||
| var IsColDatasetIDs []int64 | |||||
| datasetStars, err := models.GetDatasetStarByUser(ctx.User) | datasetStars, err := models.GetDatasetStarByUser(ctx.User) | ||||
| if err != nil { | if err != nil { | ||||
| ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("GetDatasetStarByUser failed", err))) | ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("GetDatasetStarByUser failed", err))) | ||||
| @@ -467,31 +468,84 @@ func MyFavoriteDataset(ctx *context.Context) { | |||||
| }) | }) | ||||
| return | return | ||||
| } | } | ||||
| for i, _ := range datasetStars { | |||||
| datasetIDs = append(datasetIDs, datasetStars[i].DatasetID) | |||||
| for _, datasetStar := range datasetStars { | |||||
| DatasetIsCollaborator := DatasetIsCollaborator(ctx, datasetStar.DatasetID) | |||||
| dataset, err := models.GetDatasetByID(datasetStar.DatasetID) | |||||
| if err != nil { | |||||
| return | |||||
| } | |||||
| repo, err := models.GetRepositoryByID(dataset.RepoID) | |||||
| if err != nil { | |||||
| return | |||||
| } | |||||
| if repo.OwnerID == ctx.User.ID || DatasetIsCollaborator { | |||||
| IsColDatasetIDs = append(IsColDatasetIDs, datasetStar.DatasetID) | |||||
| } else { | |||||
| NotColDatasetIDs = append(NotColDatasetIDs, datasetStar.DatasetID) | |||||
| } | |||||
| } | } | ||||
| datasets, count, err := models.Attachments(&models.AttachmentsOptions{ | |||||
| ListOptions: models.ListOptions{ | |||||
| Page: page, | |||||
| PageSize: setting.UI.DatasetPagingNum, | |||||
| }, | |||||
| NotColDatasets, NotColcount, err := models.Attachments(&models.AttachmentsOptions{ | |||||
| Keyword: keyword, | Keyword: keyword, | ||||
| NeedDatasetIDs: true, | NeedDatasetIDs: true, | ||||
| DatasetIDs: datasetIDs, | |||||
| DatasetIDs: NotColDatasetIDs, | |||||
| NeedIsPrivate: true, | NeedIsPrivate: true, | ||||
| IsPrivate: false, | IsPrivate: false, | ||||
| Type: cloudbrainType, | Type: cloudbrainType, | ||||
| JustNeedZipFile: true, | JustNeedZipFile: true, | ||||
| NeedRepoInfo: true, | NeedRepoInfo: true, | ||||
| RecommendOnly: ctx.QueryBool("recommend"), | RecommendOnly: ctx.QueryBool("recommend"), | ||||
| UserId: UserId, | |||||
| }) | |||||
| if err != nil { | |||||
| ctx.ServerError("datasets", err) | |||||
| return | |||||
| } | |||||
| //If is collaborator, there is no need to determine whether the dataset is private or public | |||||
| IsColDatasets, IsColcount, err := models.Attachments(&models.AttachmentsOptions{ | |||||
| Keyword: keyword, | |||||
| NeedDatasetIDs: true, | |||||
| DatasetIDs: IsColDatasetIDs, | |||||
| NeedIsPrivate: false, | |||||
| Type: cloudbrainType, | |||||
| JustNeedZipFile: true, | |||||
| NeedRepoInfo: true, | |||||
| RecommendOnly: ctx.QueryBool("recommend"), | |||||
| UserId: UserId, | |||||
| }) | }) | ||||
| if err != nil { | if err != nil { | ||||
| ctx.ServerError("datasets", err) | ctx.ServerError("datasets", err) | ||||
| return | return | ||||
| } | } | ||||
| for _, NotColDataset := range NotColDatasets { | |||||
| IsColDatasets = append(IsColDatasets, NotColDataset) | |||||
| } | |||||
| datasets := IsColDatasets | |||||
| count := NotColcount + IsColcount | |||||
| //排序 | |||||
| sort.Slice(datasets, func(i, j int) bool { | |||||
| return datasets[i].Attachment.CreatedUnix > datasets[j].Attachment.CreatedUnix | |||||
| }) | |||||
| data, err := json.Marshal(datasets) | |||||
| page := ctx.QueryInt("page") | |||||
| if page <= 0 { | |||||
| page = 1 | |||||
| } | |||||
| pagesize := ctx.QueryInt("pagesize") | |||||
| if pagesize <= 0 { | |||||
| pagesize = 5 | |||||
| } | |||||
| pageDatasetsInfo := getPageDatasets(datasets, page, pagesize) | |||||
| if pageDatasetsInfo == nil { | |||||
| ctx.JSON(200, map[string]string{ | |||||
| "result_code": "0", | |||||
| "data": "[]", | |||||
| "count": strconv.FormatInt(count, 10), | |||||
| }) | |||||
| return | |||||
| } | |||||
| data, err := json.Marshal(pageDatasetsInfo) | |||||
| log.Info("data:", data) | |||||
| if err != nil { | if err != nil { | ||||
| log.Error("json.Marshal failed:", err.Error()) | log.Error("json.Marshal failed:", err.Error()) | ||||
| ctx.JSON(200, map[string]string{ | ctx.JSON(200, map[string]string{ | ||||
| @@ -507,6 +561,29 @@ func MyFavoriteDataset(ctx *context.Context) { | |||||
| "count": strconv.FormatInt(count, 10), | "count": strconv.FormatInt(count, 10), | ||||
| }) | }) | ||||
| } | |||||
| func getPageDatasets(AttachmentInfos []*models.AttachmentInfo, page int, pagesize int) []*models.AttachmentInfo { | |||||
| begin := (page - 1) * pagesize | |||||
| end := (page) * pagesize | |||||
| if begin > len(AttachmentInfos)-1 { | |||||
| return nil | |||||
| } | |||||
| if end > len(AttachmentInfos)-1 { | |||||
| return AttachmentInfos[begin:] | |||||
| } else { | |||||
| return AttachmentInfos[begin:end] | |||||
| } | |||||
| } | |||||
| func getTotalPage(total int64, pageSize int) int { | |||||
| another := 0 | |||||
| if int(total)%pageSize != 0 { | |||||
| another = 1 | |||||
| } | |||||
| return int(total)/pageSize + another | |||||
| } | } | ||||
| func GetDatasetStatus(ctx *context.Context) { | func GetDatasetStatus(ctx *context.Context) { | ||||
| @@ -533,3 +610,30 @@ func GetDatasetStatus(ctx *context.Context) { | |||||
| "AttachmentStatus": fmt.Sprint(attachment.DecompressState), | "AttachmentStatus": fmt.Sprint(attachment.DecompressState), | ||||
| }) | }) | ||||
| } | } | ||||
| func DatasetIsCollaborator(ctx *context.Context, DatasetID int64) bool { | |||||
| dataset, err := models.GetDatasetByID(DatasetID) | |||||
| if err != nil { | |||||
| log.Info("query dataset error") | |||||
| } else { | |||||
| repo, err := models.GetRepositoryByID(dataset.RepoID) | |||||
| if err != nil { | |||||
| log.Info("query repo error.") | |||||
| } else { | |||||
| repo.GetOwner() | |||||
| if ctx.User != nil { | |||||
| if repo.Owner.IsOrganization() { | |||||
| if repo.Owner.IsUserPartOfOrg(ctx.User.ID) { | |||||
| log.Info("org user may visit the attach.") | |||||
| return true | |||||
| } | |||||
| } | |||||
| isCollaborator, _ := repo.IsCollaborator(ctx.User.ID) | |||||
| if isCollaborator { | |||||
| log.Info("Collaborator user may visit the attach.") | |||||
| return true | |||||
| } | |||||
| } | |||||
| } | |||||
| } | |||||
| return false | |||||
| } | |||||
| @@ -108,6 +108,7 @@ func grampusParamCheckCreateTrainJob(form auth.CreateGrampusTrainJobForm) error | |||||
| func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrainJobForm) { | func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrainJobForm) { | ||||
| displayJobName := form.DisplayJobName | displayJobName := form.DisplayJobName | ||||
| //todo: mod jobName | |||||
| jobName := util.ConvertDisplayJobNameToJobName(displayJobName) | jobName := util.ConvertDisplayJobNameToJobName(displayJobName) | ||||
| jobName = displayJobName | jobName = displayJobName | ||||
| uuid := form.Attachment | uuid := form.Attachment | ||||
| @@ -120,7 +121,6 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain | |||||
| dataMinioPath := setting.Attachment.Minio.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid | dataMinioPath := setting.Attachment.Minio.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid | ||||
| branchName := form.BranchName | branchName := form.BranchName | ||||
| flavorName := form.FlavorName | flavorName := form.FlavorName | ||||
| engineName := form.EngineName | |||||
| image := strings.TrimSpace(form.Image) | image := strings.TrimSpace(form.Image) | ||||
| if !jobNamePattern.MatchString(displayJobName) { | if !jobNamePattern.MatchString(displayJobName) { | ||||
| @@ -229,23 +229,24 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain | |||||
| commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName) | commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName) | ||||
| req := &grampus.GenerateTrainJobReq{ | req := &grampus.GenerateTrainJobReq{ | ||||
| JobName: jobName, | |||||
| DisplayJobName: displayJobName, | |||||
| ComputeResource: models.GPUResource, | |||||
| Command: command, | |||||
| ResourceSpecId: form.FlavorID, | |||||
| ImageUrl: image, | |||||
| Description: description, | |||||
| BootFile: bootFile, | |||||
| Uuid: uuid, | |||||
| CommitID: commitID, | |||||
| BranchName: branchName, | |||||
| Params: form.Params, | |||||
| FlavorName: flavorName, | |||||
| EngineName: engineName, | |||||
| DatasetName: attachment.Name, | |||||
| IsLatestVersion: modelarts.IsLatestVersion, | |||||
| VersionCount: modelarts.VersionCount, | |||||
| JobName: jobName, | |||||
| DisplayJobName: displayJobName, | |||||
| ComputeResource: models.GPUResource, | |||||
| Command: command, | |||||
| ResourceSpecId: form.FlavorID, | |||||
| ImageUrl: image, | |||||
| Description: description, | |||||
| BootFile: bootFile, | |||||
| Uuid: uuid, | |||||
| CommitID: commitID, | |||||
| BranchName: branchName, | |||||
| Params: form.Params, | |||||
| FlavorName: flavorName, | |||||
| EngineName: image, | |||||
| DatasetName: attachment.Name, | |||||
| IsLatestVersion: modelarts.IsLatestVersion, | |||||
| VersionCount: modelarts.VersionCount, | |||||
| WorkServerNumber: 1, | |||||
| } | } | ||||
| err = grampus.GenerateTrainJob(ctx, req) | err = grampus.GenerateTrainJob(ctx, req) | ||||
| @@ -260,6 +261,7 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain | |||||
| func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrainJobForm) { | func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrainJobForm) { | ||||
| displayJobName := form.DisplayJobName | displayJobName := form.DisplayJobName | ||||
| //todo: mod jobName | |||||
| jobName := util.ConvertDisplayJobNameToJobName(displayJobName) | jobName := util.ConvertDisplayJobNameToJobName(displayJobName) | ||||
| uuid := form.Attachment | uuid := form.Attachment | ||||
| description := form.Description | description := form.Description | ||||
| @@ -331,6 +331,7 @@ func RegisterRoutes(m *macaron.Macaron) { | |||||
| m.Post("/all/search/", routers.Search) | m.Post("/all/search/", routers.Search) | ||||
| m.Get("/all/search/", routers.EmptySearch) | m.Get("/all/search/", routers.EmptySearch) | ||||
| m.Get("/all/dosearch/", routers.SearchApi) | m.Get("/all/dosearch/", routers.SearchApi) | ||||
| m.Post("/user/login/kanban", user.SignInPostAPI) | |||||
| m.Get("/home/term", routers.HomeTerm) | m.Get("/home/term", routers.HomeTerm) | ||||
| m.Group("/explore", func() { | m.Group("/explore", func() { | ||||
| m.Get("", func(ctx *context.Context) { | m.Get("", func(ctx *context.Context) { | ||||
| @@ -365,6 +366,7 @@ func RegisterRoutes(m *macaron.Macaron) { | |||||
| m.Group("/user", func() { | m.Group("/user", func() { | ||||
| m.Get("/login", user.SignIn) | m.Get("/login", user.SignIn) | ||||
| m.Get("/login/cloud_brain", user.SignInCloudBrain) | m.Get("/login/cloud_brain", user.SignInCloudBrain) | ||||
| m.Post("/login", bindIgnErr(auth.SignInForm{}), user.SignInPost) | m.Post("/login", bindIgnErr(auth.SignInForm{}), user.SignInPost) | ||||
| m.Group("", func() { | m.Group("", func() { | ||||
| m.Combo("/login/openid"). | m.Combo("/login/openid"). | ||||
| @@ -176,6 +176,41 @@ func SignInCloudBrain(ctx *context.Context) { | |||||
| ctx.HTML(200, tplSignInCloudBrain) | ctx.HTML(200, tplSignInCloudBrain) | ||||
| } | } | ||||
| func SignInPostAPI(ctx *context.Context) { | |||||
| ctx.Data["Title"] = ctx.Tr("sign_in") | |||||
| UserName := ctx.Query("UserName") | |||||
| Password := ctx.Query("Password") | |||||
| log.Info("0000000") | |||||
| orderedOAuth2Names, oauth2Providers, err := models.GetActiveOAuth2Providers() | |||||
| if err != nil { | |||||
| ctx.ServerError("UserSignIn", err) | |||||
| return | |||||
| } | |||||
| ctx.Data["OrderedOAuth2Names"] = orderedOAuth2Names | |||||
| ctx.Data["OAuth2Providers"] = oauth2Providers | |||||
| ctx.Data["Title"] = ctx.Tr("sign_in") | |||||
| ctx.Data["SignInLink"] = setting.AppSubURL + "/user/login" | |||||
| ctx.Data["PageIsSignIn"] = true | |||||
| ctx.Data["PageIsLogin"] = true | |||||
| ctx.Data["IsCourse"] = ctx.QueryBool("course") | |||||
| ctx.Data["EnableSSPI"] = models.IsSSPIEnabled() | |||||
| if ctx.HasError() { | |||||
| ctx.HTML(200, tplSignIn) | |||||
| return | |||||
| } | |||||
| u, err := models.UserSignIn(UserName, Password) | |||||
| if err != nil { | |||||
| ctx.ServerError("UserSignIn", err) | |||||
| return | |||||
| } | |||||
| models.SaveLoginInfoToDb(ctx.Req.Request, u) | |||||
| // If this user is enrolled in 2FA, we can't sign the user in just yet. | |||||
| // Instead, redirect them to the 2FA authentication page. | |||||
| //handleSignInFull(ctx, u, form.Remember, false) | |||||
| handleSignInFullNotRedirect(ctx, u, true, false) | |||||
| } | |||||
| // SignInPost response for sign in request | // SignInPost response for sign in request | ||||
| func SignInPost(ctx *context.Context, form auth.SignInForm) { | func SignInPost(ctx *context.Context, form auth.SignInForm) { | ||||
| ctx.Data["Title"] = ctx.Tr("sign_in") | ctx.Data["Title"] = ctx.Tr("sign_in") | ||||
| @@ -518,6 +553,68 @@ func handleSignIn(ctx *context.Context, u *models.User, remember bool) { | |||||
| handleSignInFull(ctx, u, remember, true) | handleSignInFull(ctx, u, remember, true) | ||||
| } | } | ||||
| func handleSignInFullNotRedirect(ctx *context.Context, u *models.User, remember bool, obeyRedirect bool) string { | |||||
| log.Info("enter here.") | |||||
| if remember { | |||||
| days := 86400 * setting.LogInRememberDays | |||||
| ctx.SetCookie(setting.CookieUserName, u.Name, days, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true) | |||||
| ctx.SetSuperSecureCookie(base.EncodeMD5(u.Rands+u.Passwd), | |||||
| setting.CookieRememberName, u.Name, days, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true) | |||||
| } | |||||
| _ = ctx.Session.Delete("openid_verified_uri") | |||||
| _ = ctx.Session.Delete("openid_signin_remember") | |||||
| _ = ctx.Session.Delete("openid_determined_email") | |||||
| _ = ctx.Session.Delete("openid_determined_username") | |||||
| _ = ctx.Session.Delete("twofaUid") | |||||
| _ = ctx.Session.Delete("twofaRemember") | |||||
| _ = ctx.Session.Delete("u2fChallenge") | |||||
| _ = ctx.Session.Delete("linkAccount") | |||||
| if err := ctx.Session.Set("uid", u.ID); err != nil { | |||||
| log.Error("Error setting uid %d in session: %v", u.ID, err) | |||||
| } | |||||
| if err := ctx.Session.Set("uname", u.Name); err != nil { | |||||
| log.Error("Error setting uname %s session: %v", u.Name, err) | |||||
| } | |||||
| if err := ctx.Session.Release(); err != nil { | |||||
| log.Error("Unable to store session: %v", err) | |||||
| } | |||||
| // If the user does not have a locale set, we save the current one. | |||||
| if len(u.Language) == 0 { | |||||
| if len(ctx.GetCookie("lang")) != 0 { | |||||
| u.Language = ctx.GetCookie("lang") | |||||
| } else { | |||||
| u.Language = ctx.Locale.Language() | |||||
| } | |||||
| if err := models.UpdateUserCols(u, "language"); err != nil { | |||||
| log.Error(fmt.Sprintf("Error updating user language [user: %d, locale: %s]", u.ID, u.Language)) | |||||
| return setting.AppSubURL + "/dashboard" | |||||
| } | |||||
| } else { | |||||
| // Language setting of the user use the one previously set | |||||
| if len(ctx.GetCookie("lang")) != 0 { | |||||
| u.Language = ctx.GetCookie("lang") | |||||
| } | |||||
| } | |||||
| ctx.SetCookie("lang", u.Language, nil, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true) | |||||
| // Clear whatever CSRF has right now, force to generate a new one | |||||
| ctx.SetCookie(setting.CSRFCookieName, "", -1, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true) | |||||
| // Register last login | |||||
| u.SetLastLogin() | |||||
| if err := models.UpdateUserCols(u, "last_login_unix"); err != nil { | |||||
| ctx.ServerError("UpdateUserCols", err) | |||||
| return setting.AppSubURL + "/dashboard" | |||||
| } | |||||
| return setting.AppSubURL + "/dashboard" | |||||
| } | |||||
| func handleSignInFull(ctx *context.Context, u *models.User, remember bool, obeyRedirect bool) string { | func handleSignInFull(ctx *context.Context, u *models.User, remember bool, obeyRedirect bool) string { | ||||
| if remember { | if remember { | ||||
| days := 86400 * setting.LogInRememberDays | days := 86400 * setting.LogInRememberDays | ||||
| @@ -80,7 +80,7 @@ | |||||
| {{.CsrfTokenHtml}} | {{.CsrfTokenHtml}} | ||||
| <input type="hidden" name="action" value="update"> | <input type="hidden" name="action" value="update"> | ||||
| <input type="hidden" id="ai_engine_name" name="engine_names" value=""> | <input type="hidden" id="ai_engine_name" name="engine_names" value=""> | ||||
| <input type="hidden" id="ai_flaver_name" name="flaver_names" value=""> | |||||
| <input type="hidden" id="ai_flavor_name" name="flavor_name" value=""> | |||||
| <h4 class="unite title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4> | <h4 class="unite title ui header ">{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:</h4> | ||||
| <div class="required unite min_title inline field"> | <div class="required unite min_title inline field"> | ||||
| <label style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label> | <label style="font-weight: normal;">{{.i18n.Tr "cloudbrain.resource_cluster"}}</label> | ||||
| @@ -162,7 +162,7 @@ | |||||
| <span> | <span> | ||||
| <i class="question circle icon link" data-content={{.i18n.Tr "repo.modelarts.train_job.boot_file_helper"}} data-position="right center" data-variation="mini"></i> | <i class="question circle icon link" data-content={{.i18n.Tr "repo.modelarts.train_job.boot_file_helper"}} data-position="right center" data-variation="mini"></i> | ||||
| </span> | </span> | ||||
| <a href="https://git.openi.org.cn/OpenIOSSG/MNIST_PytorchExample_GPU" target="_blank">查看样例</a> | |||||
| <a href="https://git.openi.org.cn/OpenIOSSG/MNIST_PytorchExample_GPU/src/branch/master/train_for_c2net.py" target="_blank">查看样例</a> | |||||
| </div> | </div> | ||||
| @@ -194,7 +194,7 @@ | |||||
| </div> | </div> | ||||
| </div> | </div> | ||||
| <div class="required unite min_title inline field" id="flaver_name"> | |||||
| <div class="required unite min_title inline field" id="flavor_name"> | |||||
| <label style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.standard"}}</label> | <label style="font-weight: normal;">{{.i18n.Tr "repo.modelarts.train_job.standard"}}</label> | ||||
| <select class="ui dropdown width81" id="trainjob-flavor" style='width:385px' name="flavor"> | <select class="ui dropdown width81" id="trainjob-flavor" style='width:385px' name="flavor"> | ||||
| {{range .flavor_infos}} | {{range .flavor_infos}} | ||||
| @@ -416,9 +416,9 @@ | |||||
| } | } | ||||
| function get_name(){ | function get_name(){ | ||||
| let name1=$("#engine_name .text").text() | let name1=$("#engine_name .text").text() | ||||
| let name2=$("#flaver_name .text").text() | |||||
| let name2=$("#flavor_name .text").text() | |||||
| $("input#ai_engine_name").val(name1) | $("input#ai_engine_name").val(name1) | ||||
| $("input#ai_flaver_name").val(name2) | |||||
| $("input#ai_flavor_name").val(name2) | |||||
| } | } | ||||
| $('.ui.create_train_job.green.button').click(function(e) { | $('.ui.create_train_job.green.button').click(function(e) { | ||||
| @@ -5094,8 +5094,9 @@ function initChartsNpu() { | |||||
| data: [] | data: [] | ||||
| }, | }, | ||||
| grid: { | grid: { | ||||
| top: '30%', | |||||
| top: '35%', | |||||
| bottom: '2%', | bottom: '2%', | ||||
| x: '2%', | |||||
| containLabel: true | containLabel: true | ||||
| }, | }, | ||||
| tooltip: { | tooltip: { | ||||
| @@ -5131,14 +5132,16 @@ function initChartsNpu() { | |||||
| series: [] | series: [] | ||||
| }; | }; | ||||
| const sortBy = (arr, k) => arr.concat().sort((a, b) => (a[k] > b[k] ? 1 : a[k] < b[k] ? -1 : 0)); | |||||
| $('.metric_chart').click(function (e) { | $('.metric_chart').click(function (e) { | ||||
| let versionName = $(this).data('version') | let versionName = $(this).data('version') | ||||
| let myCharts = echarts.init(document.getElementById(`metric-${versionName}`)) | let myCharts = echarts.init(document.getElementById(`metric-${versionName}`)) | ||||
| $.get(`${window.config.AppSubUrl}/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/metric_statistics?version_name=${versionName}&statistic_type=each`, (res) => { | |||||
| $.get(`${window.config.AppSubUrl}/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/metric_statistics?version_name=${versionName}&statistic_type=each&metrics=`, (res) => { | |||||
| let filterDta = res.MetricsInfo.filter((item) => { | let filterDta = res.MetricsInfo.filter((item) => { | ||||
| return !(['recvBytesRate', 'diskWriteRate', 'sendBytesRate', 'diskReadRate'].includes(item.metric)) | return !(['recvBytesRate', 'diskWriteRate', 'sendBytesRate', 'diskReadRate'].includes(item.metric)) | ||||
| }) | }) | ||||
| filterDta = sortBy(filterDta, "metric") | |||||
| let legenData = filterDta.map((item) => { | let legenData = filterDta.map((item) => { | ||||
| return item.metric | return item.metric | ||||
| }) | }) | ||||