diff --git a/.gitignore b/.gitignore index ceeeaa92c..aca4040d2 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,7 @@ coverage.all /custom/conf/app.ini !/custom/conf/app.ini.sample /custom/public/kanban +/custom/public/annual-report /data /indexers /log diff --git a/custom/public/css/placeholder-home.css b/custom/public/css/placeholder-home.css new file mode 100644 index 000000000..9f0431def --- /dev/null +++ b/custom/public/css/placeholder-home.css @@ -0,0 +1 @@ +/* placeholder-home.css */ diff --git a/custom/public/css/placeholder.css b/custom/public/css/placeholder.css new file mode 100644 index 000000000..1ae0f997e --- /dev/null +++ b/custom/public/css/placeholder.css @@ -0,0 +1 @@ +/* placeholder.css */ diff --git a/custom/public/img/home-banner-02-1.jpg b/custom/public/img/home-banner-02-1.jpg new file mode 100644 index 000000000..13a090eac Binary files /dev/null and b/custom/public/img/home-banner-02-1.jpg differ diff --git a/custom/public/img/home-banner-02-2.png b/custom/public/img/home-banner-02-2.png new file mode 100644 index 000000000..c48c347de Binary files /dev/null and b/custom/public/img/home-banner-02-2.png differ diff --git a/custom/public/img/logo-w-origin.svg b/custom/public/img/logo-w-origin.svg new file mode 100644 index 000000000..133f63d23 --- /dev/null +++ b/custom/public/img/logo-w-origin.svg @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + diff --git a/custom/public/img/logo-w.svg b/custom/public/img/logo-w.svg index 133f63d23..48308634c 100644 --- a/custom/public/img/logo-w.svg +++ b/custom/public/img/logo-w.svg @@ -1,45 +1 @@ - - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/custom/public/js/placeholder-home.js b/custom/public/js/placeholder-home.js new file mode 100644 index 000000000..f967c5fcb --- /dev/null +++ b/custom/public/js/placeholder-home.js @@ -0,0 +1 @@ +/* placeholder-home.js */ diff --git a/custom/public/js/placeholder.js b/custom/public/js/placeholder.js new file mode 100644 index 000000000..e2c4bcff7 --- /dev/null +++ b/custom/public/js/placeholder.js @@ -0,0 +1 @@ +/* placeholder.js */ diff --git a/models/action.go b/models/action.go index 869acb762..6d93fbcd3 100755 --- a/models/action.go +++ b/models/action.go @@ -67,6 +67,7 @@ const ( ActionChangeUserAvatar //38 ActionCreateGrampusNPUDebugTask //39 ActionCreateGrampusGPUDebugTask //40 + ActionCreateGrampusGCUDebugTask //41 ) // Action represents user operation type and other information to @@ -380,7 +381,8 @@ func (a *Action) IsCloudbrainAction() bool { ActionCreateGrampusGPUDebugTask, ActionCreateGrampusNPUDebugTask, ActionCreateGrampusNPUTrainTask, - ActionCreateGrampusGPUTrainTask: + ActionCreateGrampusGPUTrainTask, + ActionCreateGrampusGCUDebugTask: return true } return false diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go index 866088a35..d55370ea1 100644 --- a/models/ai_model_manage.go +++ b/models/ai_model_manage.go @@ -485,6 +485,12 @@ func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) { return aiModelManages, count, nil } +func QueryModelConvertCountByRepoID(repoId int64) int64 { + convert := new(AiModelConvert) + total, _ := x.Where("repo_id =?", repoId).Count(convert) + return total +} + func QueryModelConvertByRepoID(repoId int64) ([]*AiModelConvert, error) { sess := x.NewSession() defer sess.Close() diff --git a/models/attachment.go b/models/attachment.go index 2b747db21..2788ef559 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -259,6 +259,17 @@ func GetAttachmentsByCommentID(commentID int64) ([]*Attachment, error) { return getAttachmentsByCommentID(x, commentID) } +func GetAttachmentByDatasetIdFileName(fileName string, datasetId int64) (*Attachment, error) { + attach := &Attachment{DatasetID: datasetId, Name: fileName} + has, err := x.Get(attach) + if err != nil { + return nil, err + } else if !has { + return nil, err + } + return attach, nil +} + func getAttachmentsByCommentID(e Engine, commentID int64) ([]*Attachment, error) { attachments := make([]*Attachment, 0, 10) return attachments, e.Where("comment_id=?", commentID).Find(&attachments) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 37965c73a..68841544e 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -30,11 +30,13 @@ const ( TypeCDCenter //成都智算中心 TypeCloudBrainAll = -1 + AccCardsNumAll = -1 ) const ( NPUResource = "NPU" GPUResource = "CPU/GPU" + GCUResource = "GCU" AllResource = "all" //notebook storage category @@ -60,6 +62,7 @@ const ( JobTypeModelSafety JobType = "MODELSAFETY" JobTypeSnn4imagenet JobType = "SNN4IMAGENET" JobTypeBrainScore JobType = "BRAINSCORE" + JobTypeSnn4Ecoset JobType = "SNN4ECOSET" JobTypeTrain JobType = "TRAIN" JobTypeInference JobType = "INFERENCE" @@ -134,6 +137,11 @@ const ( //ComputeResource GPU = "GPU" NPU = "NPU" + GCU = "GCU" +) + +const ( + AIModelPath = "aimodels/" ) type Cloudbrain struct { @@ -179,14 +187,14 @@ type Cloudbrain struct { AiCenter string //grampus ai center: center_id+center_name TrainUrl string //输出模型的obs路径 - BranchName string //分支名称 + BranchName string `xorm:"varchar(2550)"` //分支名称 Parameters string //传给modelarts的param参数 - BootFile string //启动文件 + BootFile string `xorm:"varchar(2550)"` //启动文件 DataUrl string `xorm:"varchar(3500)"` //数据集的obs路径 LogUrl string //日志输出的obs路径 PreVersionId int64 //父版本的版本id FlavorCode string //modelarts上的规格id - Description string `xorm:"varchar(256)"` //描述 + Description string `xorm:"varchar(2550)"` //描述 WorkServerNumber int //节点数 FlavorName string //规格名称 EngineName string //引擎名称 @@ -205,7 +213,7 @@ type Cloudbrain struct { BenchmarkTypeRankLink string `xorm:"-"` StartTime timeutil.TimeStamp EndTime timeutil.TimeStamp - Cleared bool `xorm:"DEFAULT false"` + Cleared bool `xorm:"DEFAULT false"` Spec *Specification `xorm:"-"` } @@ -302,6 +310,9 @@ func (task *Cloudbrain) IsUserHasRight(user *User) bool { func (task *Cloudbrain) IsGPUTask() bool { return task.ComputeResource == GPUResource } +func (task *Cloudbrain) IsGCUTask() bool { + return task.ComputeResource == GCUResource +} func (task *Cloudbrain) IsNPUTask() bool { return task.ComputeResource == NPUResource } @@ -335,6 +346,9 @@ func IsModelArtsDebugJobTerminal(status string) bool { func IsCloudBrainOneDebugJobTerminal(status string) bool { return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded) } +func IsModelBenchMarkJobType(jobType string) bool { + return jobType == string(JobTypeSnn4imagenet) || jobType == string(JobTypeBrainScore) || jobType == string(JobTypeSnn4Ecoset) +} func ParseAndSetDurationFromCloudBrainOne(result JobResultPayload, task *Cloudbrain) { isActivated := result.JobStatus.CreatedTime > 0 @@ -449,29 +463,32 @@ type GetImagesPayload struct { type CloudbrainsOptions struct { ListOptions - RepoID int64 // include all repos if empty - UserID int64 - JobID string - SortType string - CloudbrainIDs []int64 - JobStatus []string - JobStatusNot bool - Keyword string - Type int - JobTypes []string - VersionName string - IsLatestVersion string - JobTypeNot bool - NeedRepoInfo bool - RepoIDList []int64 - BeginTime time.Time - EndTime time.Time - ComputeResource string - BeginTimeUnix int64 - EndTimeUnix int64 - AiCenter string - NeedDeleteInfo string - Cluster string + RepoID int64 // include all repos if empty + UserID int64 + JobID string + SortType string + CloudbrainIDs []int64 + JobStatus []string + JobStatusNot bool + Keyword string + Type int + JobTypes []string + VersionName string + IsLatestVersion string + JobTypeNot bool + NeedRepoInfo bool + RepoIDList []int64 + BeginTime time.Time + EndTime time.Time + ComputeResource string + BeginTimeUnix int64 + EndTimeUnix int64 + AiCenter string + NeedDeleteInfo string + Cluster string + AccCardType string + AccCardsNum int + WorkServerNumber int } type TaskPod struct { @@ -1453,7 +1470,7 @@ type GetNotebookListResult struct { NotebookList []NotebookList `json:"data"` } -//Grampus +// Grampus type GrampusResult struct { ErrorCode int `json:"errorCode"` ErrorMsg string `json:"errorMsg"` @@ -1559,7 +1576,8 @@ type CreateGrampusJobResponse struct { type GetGrampusJobResponse struct { GrampusResult - JobInfo GrampusJobInfo `json:"otJob"` + JobInfo GrampusJobInfo `json:"otJob"` + ExitDiagnostics string `json:"exitDiagnostics"` } type GrampusNotebookResponse struct { @@ -1823,7 +1841,7 @@ func QueryModelTrainJobVersionList(jobId string) ([]*Cloudbrain, int, error) { return cloudbrains, int(len(cloudbrains)), nil } -func QueryModelTrainJobList(repoId int64) ([]*CloudbrainInfo, int, error) { +func QueryModelTrainJobList(repoId int64) ([]*Cloudbrain, int, error) { sess := x.NewSession() defer sess.Close() var cond = builder.NewCond() @@ -1840,14 +1858,14 @@ func QueryModelTrainJobList(repoId int64) ([]*CloudbrainInfo, int, error) { // builder.In("type", 0, 1), // ) - cloudbrains := make([]*CloudbrainInfo, 0) - if err := sess.Select("job_id,display_job_name").Table(&Cloudbrain{}).Where(cond).OrderBy("created_unix DESC"). + cloudbrains := make([]*Cloudbrain, 0) + if err := sess.Select("*").Table(&Cloudbrain{}).Where(cond).OrderBy("created_unix DESC"). Find(&cloudbrains); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } keys := make(map[string]string) - uniqueElements := make([]*CloudbrainInfo, 0) + uniqueElements := make([]*Cloudbrain, 0) for _, entry := range cloudbrains { if _, value := keys[entry.JobID]; !value { keys[entry.JobID] = entry.DisplayJobName @@ -1988,7 +2006,7 @@ func GetCloudbrainByID(id string) (*Cloudbrain, error) { return getRepoCloudBrain(cb) } -func IsCloudbrainExistByJobName(jobName string)(bool,error){ +func IsCloudbrainExistByJobName(jobName string) (bool, error) { return x.Unscoped().Exist(&Cloudbrain{ JobName: jobName, }) @@ -2126,6 +2144,15 @@ func GetCloudbrainByName(jobName string) (*Cloudbrain, error) { cb := &Cloudbrain{JobName: jobName} return getRepoCloudBrain(cb) } +func GetWaitOrRunFileNotebookByRepo(repoId int64, cloudbrainType int) (*Cloudbrain, error) { + cloudBrain := new(Cloudbrain) + has, err := x.In("status", JobWaiting, JobRunning, ModelArtsCreateQueue, ModelArtsCreating, ModelArtsStarting, + ModelArtsReadyToStart, ModelArtsResizing, ModelArtsStartQueuing, ModelArtsRunning, ModelArtsDeleting, ModelArtsRestarting).Where("repo_id=? and type=? and boot_file!=''", repoId, cloudbrainType).Get(cloudBrain) + if has { + return cloudBrain, err + } + return nil, err +} func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool { if !isSigned || (job.Status != string(JobStopped) && job.Status != string(JobFailed) && job.Status != string(ModelArtsStartFailed) && job.Status != string(ModelArtsCreateFailed)) { @@ -2159,7 +2186,7 @@ func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) { Find(&cloudbrains) } -func GetCloudBrainOneStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) { +func GetGPUStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) { cloudbrains := make([]*Cloudbrain, 0, 10) endTimeBefore := time.Now().Unix() - int64(days)*24*3600 missEndTimeBefore := endTimeBefore - 24*3600 @@ -2168,29 +2195,31 @@ func GetCloudBrainOneStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbra JobStopped, JobSucceeded, JobFailed, ModelArtsCreateFailed, ModelArtsStartFailed, ModelArtsUnavailable, ModelArtsResizFailed, ModelArtsDeleted, ModelArtsStopped, ModelArtsTrainJobCanceled, ModelArtsTrainJobCheckFailed, ModelArtsTrainJobCompleted, ModelArtsTrainJobDeleteFailed, ModelArtsTrainJobDeployServiceFailed, ModelArtsTrainJobFailed, ModelArtsTrainJobImageFailed, ModelArtsTrainJobKilled, ModelArtsTrainJobLost, ModelArtsTrainJobSubmitFailed, ModelArtsTrainJobSubmitModelFailed). - Where("(((end_time is null or end_time=0) and updated_unix 0 { + if opts.WorkServerNumber == 1 { + cond = cond.And(builder.Or( + builder.Eq{"cloudbrain.work_server_number": 0}, + builder.Eq{"cloudbrain.work_server_number": 1}, + builder.IsNull{"cloudbrain.work_server_number"}, + )) + } else { + cond = cond.And( + builder.Eq{"cloudbrain.work_server_number": opts.WorkServerNumber}, + ) + } + } + + if opts.AccCardType != "" { + cond = cond.And(builder.Eq{"cloudbrain_spec.acc_card_type": opts.AccCardType}) + } + if opts.AccCardsNum >= 0 { + cond = cond.And(builder.Eq{"cloudbrain_spec.acc_cards_num": opts.AccCardsNum}) + } + var count int64 var err error condition := "cloudbrain.user_id = `user`.id" if len(opts.Keyword) == 0 { - count, err = sess.Unscoped().Where(cond).Count(new(Cloudbrain)) + count, err = sess.Table(&Cloudbrain{}).Unscoped().Where(cond). + Join("left", "`user`", condition). + Join("left", "cloudbrain_spec", "cloudbrain.id = cloudbrain_spec.cloudbrain_id"). + Count(new(CloudbrainInfo)) } else { lowerKeyWord := strings.ToLower(opts.Keyword) cond = cond.And(builder.Or(builder.Like{"LOWER(cloudbrain.job_name)", lowerKeyWord}, builder.Like{"LOWER(cloudbrain.display_job_name)", lowerKeyWord}, builder.Like{"`user`.lower_name", lowerKeyWord})) count, err = sess.Table(&Cloudbrain{}).Unscoped().Where(cond). - Join("left", "`user`", condition).Count(new(CloudbrainInfo)) + Join("left", "`user`", condition). + Join("left", "cloudbrain_spec", "cloudbrain.id = cloudbrain_spec.cloudbrain_id"). + Count(new(CloudbrainInfo)) } @@ -2455,6 +2510,7 @@ func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { cloudbrains := make([]*CloudbrainInfo, 0, setting.UI.IssuePagingNum) if err := sess.Table(&Cloudbrain{}).Unscoped().Where(cond). Join("left", "`user`", condition). + Join("left", "cloudbrain_spec", "cloudbrain.id = cloudbrain_spec.cloudbrain_id"). Find(&cloudbrains); err != nil { return nil, 0, fmt.Errorf("Find: %v", err) } @@ -2609,6 +2665,8 @@ type DatasetInfo struct { DataLocalPath string Name string FullName string + Type int + Size int64 } func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetInfo, string, error) { @@ -2648,8 +2706,14 @@ func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetIn if len(grampusType) > 0 { if grampusType[0] == GPU { dataLocalPath = setting.Attachment.Minio.BasePath + path.Join(attach.UUID[0:1], attach.UUID[1:2]) + "/" + attach.UUID - } else { + } else if grampusType[0] == NPU { dataLocalPath = setting.BasePath + path.Join(attach.UUID[0:1], attach.UUID[1:2]) + "/" + attach.UUID + "/" + } else if grampusType[0] == GCU { + if attach.Type == TypeCloudBrainOne { + dataLocalPath = setting.Attachment.Minio.BasePath + path.Join(attach.UUID[0:1], attach.UUID[1:2]) + "/" + attach.UUID + } else { + dataLocalPath = setting.BasePath + path.Join(attach.UUID[0:1], attach.UUID[1:2]) + "/" + attach.UUID + "/" + } } } else { @@ -2664,6 +2728,8 @@ func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetIn DataLocalPath: dataLocalPath, Name: fileName, FullName: attach.Name, + Type: attach.Type, + Size: attach.Size, } if i == 0 { datasetNames = attach.Name diff --git a/models/repo_statistic.go b/models/repo_statistic.go index ecdd77e57..b99b7c259 100755 --- a/models/repo_statistic.go +++ b/models/repo_statistic.go @@ -36,7 +36,7 @@ type RepoStatistic struct { NumDevMonths int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` RepoSize int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` DatasetSize int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` - NumModels int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` + NumModels int64 `xorm:"NOT NULL DEFAULT 0" json:"model"` NumWikiViews int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` NumCommits int64 `xorm:"NOT NULL DEFAULT 0" json:"commit"` NumCommitsAdded int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` @@ -55,6 +55,15 @@ type RepoStatistic struct { NumIssuesGrowth int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` NumCommentsGrowth int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` + NumDatasetFile int64 `xorm:"NOT NULL DEFAULT 0" json:"datasetFiles"` + NumCloudbrain int64 `xorm:"NOT NULL DEFAULT 0" json:"cloudbrains"` + NumModelConvert int64 `xorm:"NOT NULL DEFAULT 0" json:"modelConverts"` + + NumDatasetFileAdded int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` + NumCloudbrainAdded int64 `xorm:"NOT NULL DEFAULT 0" json:"-"` + NumModelConvertAdded int64 `xorm:"NOT NULL DEFAULT 0" json:"- "` + NumModelsAdded int64 `xorm:"NOT NULL DEFAULT 0" json:"- "` + Impact float64 `xorm:"NOT NULL DEFAULT 0" json:"impact"` Completeness float64 `xorm:"NOT NULL DEFAULT 0" json:"completeness"` Liveness float64 `xorm:"NOT NULL DEFAULT 0" json:"liveness"` diff --git a/models/task_config.go b/models/task_config.go index f86032fc9..14ca6b223 100644 --- a/models/task_config.go +++ b/models/task_config.go @@ -39,6 +39,7 @@ func GetTaskTypeFromAction(a ActionType) TaskType { ActionCreateGrampusGPUDebugTask, ActionCreateGrampusNPUDebugTask, ActionCreateGrampusNPUTrainTask, + ActionCreateGrampusGCUDebugTask, ActionCreateGrampusGPUTrainTask: return TaskCreateCloudbrainTask case ActionCreateRepo: diff --git a/models/user.go b/models/user.go index dad252d92..d34860f2d 100755 --- a/models/user.go +++ b/models/user.go @@ -16,6 +16,7 @@ import ( "fmt" _ "image/jpeg" // Needed for jpeg support "image/png" + "math/rand" "os" "path/filepath" "regexp" @@ -495,7 +496,11 @@ func (u *User) RealSizedAvatarLink(size int) string { // may either be a sub-URL to this site, or a full URL to an external avatar // service. func (u *User) RelAvatarLink() string { - return u.SizedRelAvatarLink(base.DefaultAvatarSize) + append := "" + if u.UseCustomAvatar { + append = "?" + fmt.Sprint(rand.Intn(100)) + } + return u.SizedRelAvatarLink(base.DefaultAvatarSize) + append } // AvatarLink returns user avatar absolute link. diff --git a/models/user_analysis_for_activity.go b/models/user_analysis_for_activity.go index 99ff990ce..d872669e6 100644 --- a/models/user_analysis_for_activity.go +++ b/models/user_analysis_for_activity.go @@ -1,6 +1,7 @@ package models import ( + "encoding/json" "fmt" "time" @@ -450,19 +451,42 @@ func QueryUserLoginInfo(userIds []int64) []*UserLoginLog { return loginList } +var WeekBonusData = make(map[int64][]int) + func QueryUserAnnualReport(userId int64) *UserSummaryCurrentYear { statictisSess := xStatistic.NewSession() defer statictisSess.Close() log.Info("userId=" + fmt.Sprint(userId)) + if len(WeekBonusData) == 0 { + WeekBonusData = getBonusWeekDataMap() + } reList := make([]*UserSummaryCurrentYear, 0) err := statictisSess.Select("*").Table(new(UserSummaryCurrentYear)).Where("id=" + fmt.Sprint(userId)).Find(&reList) if err == nil { if len(reList) > 0 { + record, ok := WeekBonusData[userId] + if ok { + bonusInfo := make(map[string]int) + bonusInfo["order"] = record[0] + bonusInfo["money"] = record[1] + bonusInfo["week"] = record[2] + bonusInfo["num"] = record[3] + bonusInfoJson, _ := json.Marshal(bonusInfo) + reList[0].WeekBonusData = string(bonusInfoJson) + } return reList[0] } } else { log.Info("error:=" + err.Error()) } + dbuser, err := GetUserByID(userId) + if err == nil { + return &UserSummaryCurrentYear{ + ID: dbuser.ID, + Name: dbuser.Name, + RegistDate: dbuser.CreatedUnix, + } + } return nil } diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index d5ab871ce..11f6d9d01 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -330,7 +330,7 @@ func QueryUserStaticDataForUserDefine(opts *UserBusinessAnalysisQueryOptions, wi DataDate := currentTimeNow.Format("2006-01-02 15:04") CodeMergeCountMap := queryPullRequest(start_unix, end_unix) - CommitCountMap, _ := queryCommitAction(start_unix, end_unix, 5) + CommitCountMap := queryCommitAction(start_unix, end_unix, 5) IssueCountMap := queryCreateIssue(start_unix, end_unix) CommentCountMap := queryComment(start_unix, end_unix) @@ -586,7 +586,7 @@ func refreshUserStaticTable(wikiCountMap map[string]int, tableName string, pageS startTime := currentTimeNow.AddDate(0, 0, -1) CodeMergeCountMap := queryPullRequest(start_unix, end_unix) - CommitCountMap, _ := queryCommitAction(start_unix, end_unix, 5) + CommitCountMap := queryCommitAction(start_unix, end_unix, 5) IssueCountMap := queryCreateIssue(start_unix, end_unix) CommentCountMap := queryComment(start_unix, end_unix) @@ -762,7 +762,8 @@ func RefreshUserYearTable(pageStartTime time.Time, pageEndTime time.Time) { end_unix := pageEndTime.Unix() CodeMergeCountMap := queryPullRequest(start_unix, end_unix) - CommitCountMap, mostActiveMap := queryCommitAction(start_unix, end_unix, 5) + CommitCountMap := queryCommitAction(start_unix, end_unix, 5) + mostActiveMap := queryMostActiveCommitAction(start_unix, end_unix) IssueCountMap := queryCreateIssue(start_unix, end_unix) CommentCountMap := queryComment(start_unix, end_unix) @@ -841,9 +842,9 @@ func RefreshUserYearTable(pageStartTime time.Time, pageEndTime time.Time) { repoInfo := getRepoDetailInfo(DetailInfoMap, dateRecordAll.ID, MostDownloadMap) dataSetInfo, datasetscore := getDataSetInfo(dateRecordAll.ID, CreatedDataset, dataSetDownloadMap, CommitDatasetNumMap, CollectedDataset) scoreMap["datasetscore"] = datasetscore - codeInfo, codescore := getCodeInfo(dateRecordAll) + codeInfo, codescore := getCodeInfo(&dateRecordAll) scoreMap["codescore"] = codescore - cloudBrainInfo := getCloudBrainInfo(dateRecordAll, CloudBrainTaskItemMap, scoreMap) + cloudBrainInfo := getCloudBrainInfo(&dateRecordAll, CloudBrainTaskItemMap, scoreMap) playARoll := getPlayARoll(bonusMap, dateRecordAll.Name, scoreMap) re := &UserSummaryCurrentYear{ ID: dateRecordAll.ID, @@ -880,6 +881,76 @@ func isUserYearData(tableName string) bool { return false } +func getBonusWeekDataMap() map[int64][]int { + bonusMap := make(map[int64][]int) + url := setting.RecommentRepoAddr + "bonus/weekdata/record.txt" + content, err := GetContentFromPromote(url) + if err == nil { + filenames := strings.Split(content, "\n") + for i := 0; i < len(filenames); i++ { + if strings.HasSuffix(filenames[i], "\r") { + filenames[i] = filenames[i][0 : len(filenames[i])-len("\r")] + } + url = setting.RecommentRepoAddr + "bonus/weekdata/" + filenames[i] + csvContent, err1 := GetContentFromPromote(url) + if err1 == nil { + //read csv + lines := strings.Split(csvContent, "\n") + for j := 1; j < len(lines); j++ { + if strings.HasSuffix(lines[j], "\r") { + lines[j] = lines[j][0 : len(lines[j])-len("\r")] + } + log.Info("aLine=" + lines[j]) + aLine := strings.Split(lines[j], ",") + if len(aLine) < 4 { + continue + } + userId := getInt64Value(aLine[0]) + order := getIntValue(aLine[2]) + money := getIntValue(aLine[3]) + week, num := getWeekAndNum(filenames[i]) + //log.Info("userId=" + fmt.Sprint(userId) + " order=" + fmt.Sprint(order) + " money=" + fmt.Sprint(money) + " week=" + fmt.Sprint(week) + " num=" + fmt.Sprint(num)) + //email := lines[2] + record, ok := bonusMap[userId] + if !ok { + record = make([]int, 4) + record[0] = order + record[1] = money + record[2] = week + record[3] = num + bonusMap[userId] = record + } else { + if record[0] > order { + record[0] = order + record[1] = money + record[2] = week + record[3] = num + } else { + if record[0] == order && record[1] < money { + record[1] = money + record[2] = week + record[3] = num + } + } + } + } + } + } + } + return bonusMap +} + +func getWeekAndNum(name string) (int, int) { + name = name[0 : len(name)-4] + tmp := strings.Split(name, "_") + if len(tmp) == 2 { + week := getIntValue(tmp[0]) + num := getIntValue(tmp[1]) + return week, num + } + return 0, 0 +} + func getBonusMap() map[string]map[string]int { bonusMap := make(map[string]map[string]int) url := setting.RecommentRepoAddr + "bonus/record.txt" @@ -887,12 +958,18 @@ func getBonusMap() map[string]map[string]int { if err == nil { filenames := strings.Split(content, "\n") for i := 0; i < len(filenames); i++ { + if strings.HasSuffix(filenames[i], "\r") { + filenames[i] = filenames[i][0 : len(filenames[i])-len("\r")] + } url = setting.RecommentRepoAddr + "bonus/" + filenames[i] csvContent, err1 := GetContentFromPromote(url) if err1 == nil { //read csv lines := strings.Split(csvContent, "\n") for j := 1; j < len(lines); j++ { + if strings.HasSuffix(lines[j], "\r") { + lines[j] = lines[j][0 : len(lines[j])-len("\r")] + } aLine := strings.Split(lines[j], ",") if len(aLine) < 7 { continue @@ -923,6 +1000,14 @@ func getIntValue(val string) int { return 0 } +func getInt64Value(val string) int64 { + i, err := strconv.ParseInt(val, 10, 64) + if err == nil { + return i + } + return 0 +} + func getPlayARoll(bonusMap map[string]map[string]int, userName string, scoreMap map[string]float64) string { bonusInfo := make(map[string]string) record, ok := bonusMap[userName] @@ -943,7 +1028,7 @@ func getPlayARoll(bonusMap map[string]map[string]int, userName string, scoreMap } } -func getCloudBrainInfo(dateRecordAll UserBusinessAnalysisAll, CloudBrainTaskItemMap map[string]int, scoreMap map[string]float64) string { +func getCloudBrainInfo(dateRecordAll *UserBusinessAnalysisAll, CloudBrainTaskItemMap map[string]int, scoreMap map[string]float64) string { trainscore := 0.0 debugscore := 0.0 runtime := 0.0 @@ -959,19 +1044,16 @@ func getCloudBrainInfo(dateRecordAll UserBusinessAnalysisAll, CloudBrainTaskItem trainscore = float64(dateRecordAll.GpuTrainJob+dateRecordAll.NpuTrainJob) / float64(50) } cloudBrainInfo["inference_task_num"] = fmt.Sprint(dateRecordAll.NpuInferenceJob + CloudBrainTaskItemMap[fmt.Sprint(dateRecordAll.ID)+"_GpuInferenceJob"]) + cloudBrainInfo["benchmark_task_num"] = fmt.Sprint(dateRecordAll.GpuBenchMarkJob) cloudBrainInfo["card_runtime"] = fmt.Sprint(dateRecordAll.CloudBrainRunTime) - if dateRecordAll.CloudBrainRunTime >= 100 { - runtime = float64(dateRecordAll.CloudBrainRunTime) / float64(100) - } cloudBrainInfo["card_runtime_money"] = fmt.Sprint(dateRecordAll.CloudBrainRunTime * 5) cloudBrainInfo["CloudBrainOne"] = fmt.Sprint(CloudBrainTaskItemMap[fmt.Sprint(dateRecordAll.ID)+"_CloudBrainOne"]) cloudBrainInfo["CloudBrainTwo"] = fmt.Sprint(CloudBrainTaskItemMap[fmt.Sprint(dateRecordAll.ID)+"_CloudBrainTwo"]) cloudBrainInfo["C2Net"] = fmt.Sprint(CloudBrainTaskItemMap[fmt.Sprint(dateRecordAll.ID)+"_C2Net"]) - cloudBrainInfoJson, _ := json.Marshal(cloudBrainInfo) scoreMap["trainscore"] = trainscore scoreMap["debugscore"] = debugscore - scoreMap["runtime"] = runtime + return string(cloudBrainInfoJson) } else { scoreMap["trainscore"] = trainscore @@ -981,7 +1063,7 @@ func getCloudBrainInfo(dateRecordAll UserBusinessAnalysisAll, CloudBrainTaskItem } } -func getCodeInfo(dateRecordAll UserBusinessAnalysisAll) (string, float64) { +func getCodeInfo(dateRecordAll *UserBusinessAnalysisAll) (string, float64) { if dateRecordAll.CommitCount > 0 { codeInfo := make(map[string]string) codeInfo["commit_count"] = fmt.Sprint(dateRecordAll.CommitCount) @@ -1165,7 +1247,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, DataDate := CountDate.Format("2006-01-02") CodeMergeCountMap := queryPullRequest(start_unix, end_unix) - CommitCountMap, _ := queryCommitAction(start_unix, end_unix, 5) + CommitCountMap := queryCommitAction(start_unix, end_unix, 5) IssueCountMap := queryCreateIssue(start_unix, end_unix) CommentCountMap := queryComment(start_unix, end_unix) @@ -1311,7 +1393,7 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, useMetrics.TotalActivateRegistUser = getMapKeyStringValue("TotalActivateRegistUser", userMetrics) useMetrics.TotalHasActivityUser = getMapKeyStringValue("TotalHasActivityUser", userMetrics) useMetrics.CurrentDayRegistUser = getMapKeyStringValue("CurrentDayRegistUser", userMetrics) - count, err = sess.Where("type=0").Count(new(User)) + count, err = sess.Where("type=0 and created_unix<=" + fmt.Sprint(end_unix)).Count(new(User)) if err != nil { log.Info("query user error. return.") } @@ -1659,52 +1741,48 @@ func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { return resultMap } -func queryCommitAction(start_unix int64, end_unix int64, actionType int64) (map[int64]int, map[int64]map[string]int) { +func queryMostActiveCommitAction(start_unix int64, end_unix int64) map[int64]map[string]int { sess := x.NewSession() defer sess.Close() - resultMap := make(map[int64]int) + mostActiveMap := make(map[int64]map[string]int) cond := "user_id=act_user_id and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) count, err := sess.Where(cond).Count(new(Action)) if err != nil { log.Info("query action error. return.") - return resultMap, mostActiveMap + return mostActiveMap } var indexTotal int64 indexTotal = 0 for { - sess.Select("id,user_id,op_type,act_user_id,created_unix").Table("action").Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) - actionList := make([]*Action, 0) - sess.Find(&actionList) - - log.Info("query action size=" + fmt.Sprint(len(actionList))) + actionList, err := sess.QueryInterface("select id,user_id,op_type,act_user_id,created_unix from public.action where " + cond + " order by id asc limit " + fmt.Sprint(PAGE_SIZE) + " offset " + fmt.Sprint(indexTotal)) + if err != nil { + log.Info("error:" + err.Error()) + continue + } + log.Info("query mostactive action size=" + fmt.Sprint(len(actionList))) for _, actionRecord := range actionList { - if int64(actionRecord.OpType) == actionType { - if _, ok := resultMap[actionRecord.UserID]; !ok { - resultMap[actionRecord.UserID] = 1 - } else { - resultMap[actionRecord.UserID] += 1 - } - } - key := getDate(actionRecord.CreatedUnix) - if _, ok := mostActiveMap[actionRecord.UserID]; !ok { + userId := convertInterfaceToInt64(actionRecord["user_id"]) + created_unix := timeutil.TimeStamp(convertInterfaceToInt64(actionRecord["created_unix"])) + key := getDate(created_unix) + if _, ok := mostActiveMap[userId]; !ok { tmpMap := make(map[string]int) tmpMap[key] = 1 - mostActiveMap[actionRecord.UserID] = tmpMap + mostActiveMap[userId] = tmpMap } else { - mostActiveMap[actionRecord.UserID][key] = getMapKeyStringValue(key, mostActiveMap[actionRecord.UserID]) + 1 + mostActiveMap[userId][key] = getMapKeyStringValue(key, mostActiveMap[userId]) + 1 } - utcTime := actionRecord.CreatedUnix.AsTime() + utcTime := created_unix.AsTime() hour := utcTime.Hour() if hour >= 0 && hour <= 5 { key = "hour_hour" - if getMapKeyStringValue(key, mostActiveMap[actionRecord.UserID]) < hour { - mostActiveMap[actionRecord.UserID][key] = hour - mostActiveMap[actionRecord.UserID]["hour_day"] = utcTime.Day() - mostActiveMap[actionRecord.UserID]["hour_month"] = int(utcTime.Month()) - mostActiveMap[actionRecord.UserID]["hour_year"] = utcTime.Year() + if getMapKeyStringValue(key, mostActiveMap[userId]) < hour { + mostActiveMap[userId][key] = hour + mostActiveMap[userId]["hour_day"] = utcTime.Day() + mostActiveMap[userId]["hour_month"] = int(utcTime.Month()) + mostActiveMap[userId]["hour_year"] = utcTime.Year() } } } @@ -1713,9 +1791,60 @@ func queryCommitAction(start_unix int64, end_unix int64, actionType int64) (map[ break } } + return mostActiveMap +} - return resultMap, mostActiveMap +func queryCommitAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { + sess := x.NewSession() + defer sess.Close() + resultMap := make(map[int64]int) + cond := "op_type=" + fmt.Sprint(actionType) + " and user_id=act_user_id and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Where(cond).Count(new(Action)) + if err != nil { + log.Info("query action error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + actionList, err := sess.QueryInterface("select id,user_id,op_type,act_user_id,created_unix from public.action where " + cond + " order by id asc limit " + fmt.Sprint(PAGE_SIZE) + " offset " + fmt.Sprint(indexTotal)) + if err != nil { + log.Info("error:" + err.Error()) + continue + } + log.Info("query action size=" + fmt.Sprint(len(actionList))) + for _, actionRecord := range actionList { + userId := convertInterfaceToInt64(actionRecord["user_id"]) + + if _, ok := resultMap[userId]; !ok { + resultMap[userId] = 1 + } else { + resultMap[userId] += 1 + } + + } + indexTotal += PAGE_SIZE + if indexTotal >= count { + break + } + } + return resultMap +} + +func convertInterfaceToInt64(obj interface{}) int64 { + switch obj.(type) { + case int8: + return int64(obj.(int8)) + case int16: + return int64(obj.(int16)) + case int32: + return int64(obj.(int32)) + case int64: + return obj.(int64) + } + return 0 } + func getDate(createTime timeutil.TimeStamp) string { return createTime.Format("2006-01-02") } @@ -2408,15 +2537,16 @@ func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[s resultItemMap := make(map[string]int) cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) - count, err := sess.Where(cond).Count(new(Cloudbrain)) + count, err := sess.Where(cond).Unscoped().Count(new(Cloudbrain)) if err != nil { log.Info("query cloudbrain error. return.") return resultMap, resultItemMap } + log.Info("cloudbrain count=" + fmt.Sprint(count)) var indexTotal int64 indexTotal = 0 for { - sess.Select("id,job_type,user_id,duration,train_job_duration,type").Table("cloudbrain").Unscoped().Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) + sess.Select("id,job_type,user_id,duration,train_job_duration,type,compute_resource").Table("cloudbrain").Unscoped().Where(cond).OrderBy("id asc").Limit(PAGE_SIZE, int(indexTotal)) cloudTaskList := make([]*Cloudbrain, 0) sess.Find(&cloudTaskList) log.Info("query cloudbrain size=" + fmt.Sprint(len(cloudTaskList))) @@ -2435,6 +2565,8 @@ func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[s setMapKey("NpuTrainJob", cloudTaskRecord.UserID, 1, resultItemMap) } else if cloudTaskRecord.JobType == "INFERENCE" { setMapKey("NpuInferenceJob", cloudTaskRecord.UserID, 1, resultItemMap) + } else if cloudTaskRecord.JobType == "BENCHMARK" || cloudTaskRecord.JobType == "MODELSAFETY" { + setMapKey("GpuBenchMarkJob", cloudTaskRecord.UserID, 1, resultItemMap) } else { setMapKey("NpuDebugJob", cloudTaskRecord.UserID, 1, resultItemMap) } @@ -2444,12 +2576,12 @@ func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[s setMapKey("GpuTrainJob", cloudTaskRecord.UserID, 1, resultItemMap) } else if cloudTaskRecord.JobType == "INFERENCE" { setMapKey("GpuInferenceJob", cloudTaskRecord.UserID, 1, resultItemMap) - } else if cloudTaskRecord.JobType == "BENCHMARK" { + } else if cloudTaskRecord.JobType == "BENCHMARK" || cloudTaskRecord.JobType == "MODELSAFETY" { setMapKey("GpuBenchMarkJob", cloudTaskRecord.UserID, 1, resultItemMap) } else { setMapKey("GpuDebugJob", cloudTaskRecord.UserID, 1, resultItemMap) } - } else if cloudTaskRecord.Type == 2 { + } else if cloudTaskRecord.Type == 2 || cloudTaskRecord.Type == 3 { setMapKey("C2Net", cloudTaskRecord.UserID, 1, resultItemMap) if cloudTaskRecord.ComputeResource == NPUResource { if cloudTaskRecord.JobType == "TRAIN" { @@ -2471,7 +2603,6 @@ func queryCloudBrainTask(start_unix int64, end_unix int64) (map[int64]int, map[s break } } - return resultMap, resultItemMap } diff --git a/models/user_business_struct.go b/models/user_business_struct.go index 00c7f6176..61f499732 100644 --- a/models/user_business_struct.go +++ b/models/user_business_struct.go @@ -18,9 +18,9 @@ type UserSummaryCurrentYear struct { CodeInfo string `xorm:"varchar(500)"` //代码提交次数，提交总代码行数，最晚的提交时间 CloudBrainInfo string `xorm:"varchar(1000)"` //，创建了XX 个云脑任务，调试任务XX 个，训练任务XX 个，推理任务XX 个，累计运行了XXXX 卡时，累计节省xxxxx 元 //这些免费的算力资源分别有，XX% 来自鹏城云脑1，XX% 来自鹏城云脑2，XX% 来自智算网络 - PlayARoll string `xorm:"varchar(500)"` //你参加了XX 次“我为开源打榜狂”活动，累计上榜XX 次，总共获得了社区XXX 元的激励 - - Label string `xorm:"varchar(500)"` + PlayARoll string `xorm:"varchar(500)"` //你参加了XX 次“我为开源打榜狂”活动，累计上榜XX 次，总共获得了社区XXX 元的激励 + WeekBonusData string `xorm:"-"` + Label string `xorm:"varchar(500)"` } type UserBusinessAnalysisCurrentYear struct { diff --git a/modules/auth/modelarts.go b/modules/auth/modelarts.go index 0061648ce..874bbb0a4 100755 --- a/modules/auth/modelarts.go +++ b/modules/auth/modelarts.go @@ -28,6 +28,7 @@ type CreateModelArtsNotebookForm struct { LabelName string `form:"label_names"` PreTrainModelUrl string `form:"pre_train_model_url"` SpecId int64 `form:"spec_id" binding:"Required"` + DatasetName string `form:"dataset_name"` } func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index c85f4b8cd..1929c58a6 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -32,10 +32,10 @@ const ( Snn4imagenetMountPath = "/snn4imagenet" BrainScoreMountPath = "/brainscore" TaskInfoName = "/taskInfo" - Snn4imagenetCommand = `/opt/conda/bin/python /snn4imagenet/testSNN_script.py --modelname '%s' --modelpath '/dataset' --modeldescription '%s' >/model/benchmark-log.txt` - BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/dataset' -d '%s' >/model/benchmark-log.txt` - - SubTaskName = "task1" + Snn4imagenetCommand = `/opt/conda/bin/python /benchmark/testSNN_script.py --modelname '%s' --modelpath '/pretrainmodel/%s' --modeldescription '%s' >/model/benchmark-log.txt` + BrainScoreCommand = `bash /benchmark/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/pretrainmodel/%s' -d '%s' >/model/benchmark-log.txt` + Snn4EcosetCommand = `/opt/conda/bin/python /benchmark/testSNN_script.py --datapath '/dataset' --modelname '%s' --modelpath '/pretrainmodel/%s' --modeldescription '%s' >/model/benchmark-log.txt` + SubTaskName = "task1" Success = "S000" @@ -257,20 +257,6 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) { ReadOnly: true, }, }, - { - HostPath: models.StHostPath{ - Path: req.Snn4ImageNetPath, - MountPath: Snn4imagenetMountPath, - ReadOnly: true, - }, - }, - { - HostPath: models.StHostPath{ - Path: req.BrainScorePath, - MountPath: BrainScoreMountPath, - ReadOnly: true, - }, - }, { HostPath: models.StHostPath{ Path: req.ResultPath, @@ -406,7 +392,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) { } func IsBenchmarkJob(jobType string) bool { - return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType + return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType || string(models.JobTypeSnn4Ecoset) == jobType } func GetWaitingCloudbrainCount(cloudbrainType int, computeResource string, jobTypes ...models.JobType) int64 { @@ -675,6 +661,7 @@ func IsElementExist(s []string, str string) bool { return false } + func GetCloudBrainByIdOrJobId(id string, initialQuery string) (*models.Cloudbrain, error) { _, err := strconv.ParseInt(id, 10, 64) var job *models.Cloudbrain @@ -709,6 +696,7 @@ type GenerateModelArtsNotebookReq struct { ImageId string AutoStopDurationMs int64 + BranchName string Spec *models.Specification ModelName string diff --git a/modules/context/auth.go b/modules/context/auth.go index efde3bc2e..c9af1456d 100755 --- a/modules/context/auth.go +++ b/modules/context/auth.go @@ -81,7 +81,12 @@ func Toggle(options *ToggleOptions) macaron.Handler { // Redirect to dashboard if user tries to visit any non-login page. if options.SignOutRequired && ctx.IsSigned && ctx.Req.URL.RequestURI() != "/" { - ctx.Redirect(setting.AppSubURL + "/") + redirectTo := ctx.Query("redirect_to") + if len(redirectTo) > 0 { + ctx.Redirect(redirectTo) + } else { + ctx.Redirect(setting.AppSubURL + "/") + } return } diff --git a/modules/convert/cloudbrain.go b/modules/convert/cloudbrain.go index 599da4800..73e37b1ea 100644 --- a/modules/convert/cloudbrain.go +++ b/modules/convert/cloudbrain.go @@ -28,14 +28,13 @@ func ToCloudBrain(task *models.Cloudbrain) *api.Cloudbrain { BootFile: task.BootFile, Description: task.Description, ModelName: task.ModelName, - - ModelVersion: task.ModelVersion, - CkptName: task.CkptName, + VersionName: task.VersionName, + ModelVersion: task.ModelVersion, + CkptName: task.CkptName, StartTime: int64(task.StartTime), EndTime: int64(task.EndTime), - - Spec: ToSpecification(task.Spec), + Spec: ToSpecification(task.Spec), } } func ToAttachment(attachment *models.Attachment) *api.AttachmentShow { @@ -89,6 +88,9 @@ func ToDataset(dataset *models.Dataset) *api.Dataset { } func ToSpecification(s *models.Specification) *api.SpecificationShow { + if s == nil { + return nil + } return &api.SpecificationShow{ ID: s.ID, AccCardsNum: s.AccCardsNum, diff --git a/modules/grampus/grampus.go b/modules/grampus/grampus.go index 37e6fc1bf..b65b42bdd 100755 --- a/modules/grampus/grampus.go +++ b/modules/grampus/grampus.go @@ -19,6 +19,7 @@ const ( ProcessorTypeNPU = "npu.huawei.com/NPU" ProcessorTypeGPU = "nvidia.com/gpu" + ProcessorTypeGCU = "enflame-tech.com/gcu" GpuWorkDir = "/tmp/" NpuWorkDir = "/cache/" @@ -108,6 +109,7 @@ type GenerateNotebookJobReq struct { Spec *models.Specification CodeName string ModelPath string //参考启智GPU调试， 挂载/model目录用户的模型可以输出到这个目录 + ModelStorageType int } func getEndPoint() string { @@ -148,6 +150,36 @@ func getDatasetGPUGrampus(datasetInfos map[string]models.DatasetInfo) ([]models. } return datasetGrampus, command } +func getDatasetGCUGrampus(datasetInfos map[string]models.DatasetInfo) ([]models.GrampusDataset, string) { + var datasetGrampus []models.GrampusDataset + var command = "" + obsEndPoint := getEndPoint() + for uuid, datasetInfo := range datasetInfos { + if datasetInfo.Type == models.TypeCloudBrainOne { + datasetGrampus = append(datasetGrampus, models.GrampusDataset{ + Name: datasetInfo.FullName, + Bucket: setting.Attachment.Minio.Bucket, + EndPoint: setting.Attachment.Minio.Endpoint, + ObjectKey: datasetInfo.DataLocalPath, + ReadOnly: true, + ContainerPath: "/dataset1/" + datasetInfo.Name, + }) + + command += "cp /dataset1/'" + datasetInfo.Name + "'/" + uuid + " /dataset/'" + datasetInfo.FullName + "';" + + } else { + datasetGrampus = append(datasetGrampus, models.GrampusDataset{ + Name: datasetInfo.FullName, + Bucket: setting.Bucket, + EndPoint: obsEndPoint, + ObjectKey: datasetInfo.DataLocalPath + datasetInfo.FullName, + ContainerPath: "/dataset/" + datasetInfo.Name, + }) + } + + } + return datasetGrampus, command +} func GenerateNotebookJob(ctx *context.Context, req *GenerateNotebookJobReq) (jobId string, err error) { createTime := timeutil.TimeStampNow() @@ -178,25 +210,45 @@ func GenerateNotebookJob(ctx *context.Context, req *GenerateNotebookJobReq) (job imageUrl = "" req.Command = "" } else { - datasetGrampus, cpCommand = getDatasetGPUGrampus(req.DatasetInfos) + if ProcessorTypeGCU == req.ProcessType { + datasetGrampus, cpCommand = getDatasetGCUGrampus(req.DatasetInfos) + } else { + datasetGrampus, cpCommand = getDatasetGPUGrampus(req.DatasetInfos) + } if len(req.ModelName) != 0 { - datasetGrampus = append(datasetGrampus, models.GrampusDataset{ - Name: req.ModelName, - Bucket: setting.Attachment.Minio.Bucket, - EndPoint: setting.Attachment.Minio.Endpoint, - ObjectKey: req.PreTrainModelPath, - ReadOnly: true, - ContainerPath: cloudbrain.PretrainModelMountPath, - }) + if req.ModelStorageType == models.TypeCloudBrainOne { + datasetGrampus = append(datasetGrampus, models.GrampusDataset{ + Name: req.ModelName, + Bucket: setting.Attachment.Minio.Bucket, + EndPoint: setting.Attachment.Minio.Endpoint, + ObjectKey: req.PreTrainModelPath, + ReadOnly: true, + ContainerPath: cloudbrain.PretrainModelMountPath, + }) + } else { + datasetGrampus = append(datasetGrampus, models.GrampusDataset{ + Name: req.ModelName, + Bucket: setting.Bucket, + EndPoint: getEndPoint(), + ReadOnly: true, + ObjectKey: req.PreTrainModelPath, + ContainerPath: cloudbrain.PretrainModelMountPath, + }) + } + } + codeArchiveName := cloudbrain.DefaultBranchName + ".zip" codeGrampus = models.GrampusDataset{ Name: req.CodeName, Bucket: setting.Attachment.Minio.Bucket, EndPoint: setting.Attachment.Minio.Endpoint, - ObjectKey: req.CodeStoragePath + cloudbrain.DefaultBranchName + ".zip", + ObjectKey: req.CodeStoragePath + codeArchiveName, ReadOnly: false, ContainerPath: cloudbrain.CodeMountPath, } + if ProcessorTypeGCU == req.ProcessType { + imageUrl = "" + } req.Command = fmt.Sprintf(CommandGpuDebug, cpCommand, setting.CullIdleTimeout, setting.CullIdleTimeout, setting.CullInterval, setting.CullIdleTimeout, setting.CullInterval) log.Info("debug command:" + req.Command) @@ -215,6 +267,7 @@ func GenerateNotebookJob(ctx *context.Context, req *GenerateNotebookJobReq) (job AutoStopDuration: autoStopDurationMs, Capacity: setting.Capacity, Command: req.Command, + CenterID: req.Spec.GetAvailableCenterIds(ctx.User.ID), }, }, }) @@ -263,6 +316,8 @@ func GenerateNotebookJob(ctx *context.Context, req *GenerateNotebookJobReq) (job actionType = models.ActionCreateGrampusNPUDebugTask } else if req.ComputeResource == models.GPUResource { actionType = models.ActionCreateGrampusGPUDebugTask + } else if req.ComputeResource == models.GCUResource { + actionType = models.ActionCreateGrampusGCUDebugTask } task, err := models.GetCloudbrainByJobID(jobID) if err != nil { diff --git a/modules/grampus/resty.go b/modules/grampus/resty.go index a0d5384e2..3611240b9 100755 --- a/modules/grampus/resty.go +++ b/modules/grampus/resty.go @@ -198,7 +198,6 @@ sendjob: SetAuthToken(TOKEN). SetResult(&result). Get(HOST + urlTrainJob + "/" + jobID) - if err != nil { return nil, fmt.Errorf("resty GetJob: %v", err) } diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index dcad1eb00..7f67e36f4 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -1,13 +1,9 @@ package modelarts import ( - "encoding/base64" "encoding/json" "errors" "fmt" - "io/ioutil" - "net/http" - "path" "strconv" "strings" @@ -26,9 +22,9 @@ import ( const ( //notebook - storageTypeOBS = "obs" - autoStopDuration = 4 * 60 * 60 - AutoStopDurationMs = 4 * 60 * 60 * 1000 + storageTypeOBS = "obs" + autoStopDuration = 4 * 60 * 60 + AutoStopDurationMs = 4 * 60 * 60 * 1000 CodePath = "/code/" OutputPath = "/output/" @@ -172,7 +168,6 @@ type OrgMultiNode struct { Node []int `json:"node"` } - type Parameters struct { Parameter []struct { Label string `json:"label"` @@ -239,6 +234,7 @@ func GenerateNotebook2(ctx *context.Context, req cloudbrain.GenerateModelArtsNot ComputeResource: models.NPUResource, Image: imageName, BootFile: req.BootFile, + BranchName: req.BranchName, Description: req.Description, CreatedUnix: createTime, UpdatedUnix: createTime, @@ -830,10 +826,6 @@ func HandleNotebookInfo(task *models.Cloudbrain) error { task.FlavorCode = result.Flavor } - if oldStatus != task.Status && task.Status == string(models.ModelArtsRunning) && task.BootFile != "" { - uploadNoteBookFile(task, result) - - } err = models.UpdateJob(task) if err != nil { log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err) @@ -844,81 +836,6 @@ func HandleNotebookInfo(task *models.Cloudbrain) error { return nil } -func uploadNoteBookFile(task *models.Cloudbrain, result *models.GetNotebook2Result) { - jupyterUrl := result.Url + "?token=" + result.Token - - cookies, xsrf := getCookiesAndCsrf(jupyterUrl) - if xsrf == "" { - log.Error("browser jupyterUrl failed:%v", task.DisplayJobName) - } else { - - codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath - fileContents, err := ioutil.ReadFile(codePath + "/" + task.BootFile) - if err != nil { - log.Error("read jupyter file failed:%v", task.DisplayJobName, err) - } - - base64Content := base64.StdEncoding.EncodeToString(fileContents) - client := getRestyClient() - uploadUrl := getJupyterBaseUrl(result.Url) + "api/contents/" + path.Base(task.BootFile) - res, err := client.R(). - SetCookies(cookies). - SetHeader("X-XSRFToken", xsrf). - SetBody(map[string]interface{}{ - "type": "file", - "format": "base64", - "name": path.Base(task.BootFile), - "path": path.Base(task.BootFile), - "content": base64Content}). - Put(uploadUrl) - if err != nil { - log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) - } else if res.StatusCode() != http.StatusCreated { - log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) - } - - } - -} - -func getJupyterBaseUrl(url string) string { - jupyterUrlLength := len(url) - baseUrl := url[0 : jupyterUrlLength-len(path.Base(url))] - return baseUrl -} - -func getCookiesAndCsrf(jupyterUrl string) ([]*http.Cookie, string) { - log.Info("jupyter url:" + jupyterUrl) - var cookies []*http.Cookie - const retryTimes = 10 - for i := 0; i < retryTimes; i++ { - res, err := http.Get(jupyterUrl) - if err != nil { - log.Error("browser jupyterUrl failed.", err) - if i == retryTimes-1 { - return cookies, "" - } - - } else { - cookies = res.Cookies() - xsrf := "" - for _, cookie := range cookies { - if cookie.Name == "_xsrf" { - xsrf = cookie.Value - break - } - - } - if xsrf != "" { - return cookies, xsrf - } - - } - } - return cookies, "" - -} - func SyncTempStatusJob() { jobs, err := models.GetCloudBrainTempJobs() if err != nil { diff --git a/modules/modelarts_cd/modelarts.go b/modules/modelarts_cd/modelarts.go index bdc42002a..9d74c0919 100755 --- a/modules/modelarts_cd/modelarts.go +++ b/modules/modelarts_cd/modelarts.go @@ -148,6 +148,7 @@ func GenerateNotebook(ctx *context.Context, req cloudbrain.GenerateModelArtsNote UpdatedUnix: createTime, Spec: req.Spec, BootFile: req.BootFile, + BranchName: req.BranchName, ModelName: req.ModelName, ModelVersion: req.ModelVersion, LabelName: req.LabelName, diff --git a/modules/notebook/contentManager.go b/modules/notebook/contentManager.go new file mode 100644 index 000000000..823e59665 --- /dev/null +++ b/modules/notebook/contentManager.go @@ -0,0 +1,198 @@ +package notebook + +import ( + "crypto/tls" + "encoding/base64" + "fmt" + "io/ioutil" + "net/http" + "path" + "strings" + + "github.com/go-resty/resty/v2" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/modules/setting" + + "code.gitea.io/gitea/modules/log" +) + +var restyClient *resty.Client + +type NotebookApiResponse struct { + Name string `json:"name"` + Path string `json:"path"` +} + +type NotebookContent struct { + Url string + Path string + Cookies []*http.Cookie + Xsrf string + PathType string //file directory + Token string +} + +func (c *NotebookContent) IsNotebookFileCanBrowser() bool { + if c.Xsrf == "" { + c.SetCookiesAndCsrf() + } + if c.Xsrf == "" { + log.Warn("xsrf is empty, can not broswer url:" + c.Url) + return false + } + return c.IsNoteBookContentsExist() + +} + +func (c *NotebookContent) SetCookiesAndCsrf() { + log.Info("jupyter url:" + c.Url) + var cookies []*http.Cookie + const retryTimes = 10 + url := c.Url + if c.Token != "" { + url = c.Url + "?token=" + c.Token + } + for i := 0; i < retryTimes; i++ { + res, err := http.Get(url) + if err != nil { + log.Error("browser jupyterUrl failed.", err) + if i == retryTimes-1 { + c.Cookies = cookies + } + + } else { + cookies = res.Cookies() + xsrf := "" + for _, cookie := range cookies { + if cookie.Name == "_xsrf" { + xsrf = cookie.Value + if len(cookies) > 1 { + break + } + + } + + } + if xsrf != "" { + c.Cookies = cookies + c.Xsrf = xsrf + } + + } + } + c.Cookies = cookies + +} + +func (c *NotebookContent) IsNoteBookContentsExist() bool { + client := getRestyClient() + uploadUrl := getJupyterBaseUrl(c.Url) + "api/contents/" + c.Path + "?type=" + c.PathType + res, err := client.R(). + SetCookies(c.Cookies). + SetHeader("X-XSRFToken", c.Xsrf). + Get(uploadUrl) + if err != nil { + log.Warn("browser url error:"+uploadUrl, err) + return false + } + return res.StatusCode() == http.StatusOK +} + +func (c *NotebookContent) UploadNoteBookFile(task *models.Cloudbrain) error { + + err := c.MakeNoteBookDir() + if err != nil { + return err + } + + codePath := setting.JobPath + task.JobName + cloudbrain.CodeMountPath + fileContents, err := ioutil.ReadFile(codePath + "/" + c.Path) + if err != nil { + log.Error("read jupyter file failed:%v", task.DisplayJobName, err) + } + + base64Content := base64.StdEncoding.EncodeToString(fileContents) + client := getRestyClient() + uploadUrl := getJupyterBaseUrl(c.Url) + "api/contents/" + c.Path + res, err := client.R(). + SetCookies(c.Cookies). + SetHeader("X-XSRFToken", c.Xsrf). + SetBody(map[string]interface{}{ + "type": "file", + "format": "base64", + "name": path.Base(c.Path), + "path": c.Path, + "content": base64Content}). + Put(uploadUrl) + if err != nil { + log.Error("upload jupyter file failed:%v", task.DisplayJobName, err) + return err + } else if res.StatusCode() != http.StatusCreated { + log.Error("upload jupyter file failed:%v, status is %s", task.DisplayJobName, res.Status(), err) + return fmt.Errorf("status:", res.StatusCode()) + } + return nil +} + +/** + if c.Path is a/b/c.txt + makedir a/b + if c.Path is a/b/c + makedir a/b +*/ +func (c *NotebookContent) MakeNoteBookDir() error { + filePaths := strings.Split(c.Path, "/") + + for i := 0; i < len(filePaths)-1; i++ { + cTemp := &NotebookContent{ + Url: c.Url, + Cookies: c.Cookies, + Path: path.Join(filePaths[0 : i+1]...), + PathType: "directory", + Xsrf: c.Xsrf, + } + if !cTemp.IsNoteBookContentsExist() { + + createTempDirUrl := getJupyterBaseUrl(cTemp.Url) + "api/contents/" + cTemp.Path + client := getRestyClient() + var jobResult NotebookApiResponse + res, err := client.R(). + SetCookies(c.Cookies). + SetHeader("X-XSRFToken", c.Xsrf). + SetBody(map[string]interface{}{ + "type": cTemp.PathType, + "path": cTemp.Path, + }).SetResult(&jobResult). + Put(createTempDirUrl) + if err != nil { + return err + } + if res.StatusCode() != http.StatusCreated { + return fmt.Errorf("status code:" + res.Status()) + } + + } + + } + return nil +} + +func getJupyterBaseUrl(url string) string { + jupyterUrlLength := len(url) + baseUrl := url + if strings.HasSuffix(url, "lab") { + baseUrl = url[0 : jupyterUrlLength-len(path.Base(url))] + } + + return baseUrl +} + +func getRestyClient() *resty.Client { + if restyClient == nil { + restyClient = resty.New() + restyClient.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) + } + return restyClient +} diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 3268cce60..43427893f 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -139,6 +139,9 @@ func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opt } repo.IsMirror = true + if repo.Description == "" { + repo.Description = opts.Description + } err = models.UpdateRepository(repo, false) } else { repo, err = CleanUpMigrateInfo(repo) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index d4e776062..09e7259f2 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -518,7 +518,7 @@ var ( MaxDatasetNum int CullIdleTimeout string CullInterval string - + DebugAttachSize int //benchmark config IsBenchmarkEnabled bool @@ -544,6 +544,12 @@ var ( BrainScoreName string BrainScoreServerHost string + IsSnn4EcosetEnabled bool + Snn4EcosetOwner string + Snn4EcosetName string + Snn4EcosetServerHost string + Snn4AttachmentName string + //blockchain config BlockChainHost string CommitValidDate string @@ -617,14 +623,14 @@ var ( UsageRateBeginTime string }{} - ClearStrategy= struct { - Enabled bool - ResultSaveDays int - BatchSize int - DebugJobSize int - TrashSaveDays int - Cron string - RunAtStart bool + ClearStrategy = struct { + Enabled bool + ResultSaveDays int + BatchSize int + DebugJobSize int + TrashSaveDays int + Cron string + RunAtStart bool }{} C2NetInfos *C2NetSqInfos @@ -711,6 +717,7 @@ var ( ProjectHealth float64 ProjectHealthIssueCompleteRatio float64 + ProjectHealth0IssueCloseRatio float64 TeamHealth float64 TeamHealthContributors float64 @@ -1497,6 +1504,7 @@ func NewContext() { MaxDatasetNum = sec.Key("MAX_DATASET_NUM").MustInt(5) CullIdleTimeout = sec.Key("CULL_IDLE_TIMEOUT").MustString("900") CullInterval = sec.Key("CULL_INTERVAL").MustString("60") + DebugAttachSize = sec.Key("DEBUG_ATTACH_SIZE").MustInt(20) sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) @@ -1522,6 +1530,13 @@ func NewContext() { BrainScoreName = sec.Key("NAME").MustString("") BrainScoreServerHost = sec.Key("HOST").MustString("") + sec = Cfg.Section("snn4ecoset") + IsSnn4EcosetEnabled = sec.Key("ENABLED").MustBool(false) + Snn4EcosetOwner = sec.Key("OWNER").MustString("") + Snn4EcosetName = sec.Key("NAME").MustString("") + Snn4EcosetServerHost = sec.Key("HOST").MustString("") + Snn4AttachmentName = sec.Key("DATASET").MustString("") + sec = Cfg.Section("blockchain") BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/") CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15") @@ -1705,16 +1720,16 @@ func getModelartsCDConfig() { getNotebookFlavorInfos() } -func getClearStrategy(){ +func getClearStrategy() { sec := Cfg.Section("clear_strategy") - ClearStrategy.Enabled=sec.Key("ENABLED").MustBool(false) - ClearStrategy.ResultSaveDays=sec.Key("RESULT_SAVE_DAYS").MustInt(30) - ClearStrategy.BatchSize=sec.Key("BATCH_SIZE").MustInt(500) - ClearStrategy.DebugJobSize=sec.Key("DEBUG_BATCH_SIZE").MustInt(100) - ClearStrategy.TrashSaveDays=sec.Key("TRASH_SAVE_DAYS").MustInt(90) - ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ?") - ClearStrategy.RunAtStart=sec.Key("RUN_AT_START").MustBool(false) + ClearStrategy.Enabled = sec.Key("ENABLED").MustBool(false) + ClearStrategy.ResultSaveDays = sec.Key("RESULT_SAVE_DAYS").MustInt(30) + ClearStrategy.BatchSize = sec.Key("BATCH_SIZE").MustInt(500) + ClearStrategy.DebugJobSize = sec.Key("DEBUG_BATCH_SIZE").MustInt(100) + ClearStrategy.TrashSaveDays = sec.Key("TRASH_SAVE_DAYS").MustInt(90) + ClearStrategy.Cron = sec.Key("CRON").MustString("* 0,30 2-8 * * ?") + ClearStrategy.RunAtStart = sec.Key("RUN_AT_START").MustBool(false) } func getGrampusConfig() { @@ -1781,6 +1796,7 @@ func SetRadarMapConfig() { RadarMap.LivenessRelease = sec.Key("liveness_release").MustFloat64(0.4) RadarMap.ProjectHealth = sec.Key("project_health").MustFloat64(0.1) RadarMap.ProjectHealthIssueCompleteRatio = sec.Key("project_health_issue_complete_ratio").MustFloat64(100) + RadarMap.ProjectHealth0IssueCloseRatio = sec.Key("project_health_0_issue_close_ratio").MustFloat64(0.0) RadarMap.TeamHealth = sec.Key("team_health").MustFloat64(0.1) RadarMap.TeamHealthContributors = sec.Key("team_health_contributors").MustFloat64(0.2) RadarMap.TeamHealthKeyContributors = sec.Key("team_health_key_contributors").MustFloat64(0.6) diff --git a/modules/storage/minio_ext.go b/modules/storage/minio_ext.go index d4a8abba5..6aeb89aab 100755 --- a/modules/storage/minio_ext.go +++ b/modules/storage/minio_ext.go @@ -391,3 +391,19 @@ func GetPartInfos(objectName string, uploadID string) (string, error) { return chunks, nil } + +func IsObjectExist4Minio(bucket, objectName string) (bool, error) { + _, core, err := getClients() + if err != nil { + log.Error("getClients failed:", err.Error()) + return false, err + } + + _, err = core.StatObject(bucket, objectName, miniov6.StatObjectOptions{}) + if err != nil { + log.Error("GetObjectMetadata error.%v", err) + return false, err + } + + return true, nil +} diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 83b03ed44..d00d000b5 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -614,7 +614,7 @@ func ObsCreateObject(path string) error { return nil } -func GetObsLogFileName(prefix string) (string, error) { +func GetObsLogFileName(prefix string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = setting.Bucket input.Prefix = prefix @@ -622,10 +622,39 @@ func GetObsLogFileName(prefix string) (string, error) { output, err := ObsCli.ListObjects(input) if err != nil { log.Error("PutObject failed:", err.Error()) - return "", err + return nil, err } if output == nil || len(output.Contents) == 0 { - return "", errors.New("obs log files not exist") + return nil, errors.New("obs log files not exist") + } + fileInfos := make([]FileInfo, 0) + for _, val := range output.Contents { + //result[num] = c.Key + if strings.HasSuffix(val.Key, ".log") { + log.Info("log fileName=" + val.Key) + fileInfo := FileInfo{ + ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), + FileName: val.Key[len(prefix)-3:], //加上 job + Size: val.Size, + IsDir: false, + ParenDir: prefix[0 : len(prefix)-3], + } + fileInfos = append(fileInfos, fileInfo) + } + + } + return fileInfos, nil +} + +func IsObjectExist4Obs(bucket, key string) (bool, error) { + + _, err := ObsCli.GetObjectMetadata(&obs.GetObjectMetadataInput{ + Bucket: bucket, + Key: key, + }) + if err != nil { + log.Error("GetObjectMetadata error.%v", err) + return false, err } - return output.Contents[0].Key, nil + return true, nil } diff --git a/modules/structs/cloudbrain.go b/modules/structs/cloudbrain.go index 9ea5601c9..cba297dc0 100644 --- a/modules/structs/cloudbrain.go +++ b/modules/structs/cloudbrain.go @@ -47,36 +47,37 @@ type CreateFileNotebookJobOption struct { BranchName string `json:"branch_name" binding:"Required"` OwnerName string `json:"owner_name" binding:"Required"` ProjectName string `json:"project_name" binding:"Required"` + JobId string `json:"job_id"` } type Cloudbrain struct { - ID int64 `json:"id"` - JobID string `json:"job_id"` - JobType string `json:"job_type"` - Type int `json:"type"` - DisplayJobName string `json:"display_job_name"` - Status string `json:"status"` - CreatedUnix int64 `json:"created_unix"` - RepoID int64 `json:"repo_id"` - Duration int64 `json:"duration"` //运行时长 单位秒 - TrainJobDuration string `json:"train_job_duration"` - ImageID string `json:"image_id"` //grampus image_id - Image string `json:"image"` - Uuid string `json:"uuid"` //数据集id - DatasetName string `json:"dataset_name"` - ComputeResource string `json:"compute_resource"` //计算资源，例如npu - AiCenter string `json:"ai_center"` //grampus ai center: center_id+center_name - BranchName string `json:"branch_name"` //分支名称 - Parameters string `json:"parameters"` //传给modelarts的param参数 - BootFile string `json:"boot_file"` //启动文件 - Description string `json:"description"` //描述 - ModelName string `json:"model_name"` //模型名称 - ModelVersion string `json:"model_version"` //模型版本 - CkptName string `json:"ckpt_name"` //权重文件名称 - StartTime int64 `json:"start_time"` - EndTime int64 `json:"end_time"` - - Spec *SpecificationShow `json:"spec"` + ID int64 `json:"id"` + JobID string `json:"job_id"` + JobType string `json:"job_type"` + Type int `json:"type"` + DisplayJobName string `json:"display_job_name"` + Status string `json:"status"` + CreatedUnix int64 `json:"created_unix"` + RepoID int64 `json:"repo_id"` + Duration int64 `json:"duration"` //运行时长 单位秒 + TrainJobDuration string `json:"train_job_duration"` + ImageID string `json:"image_id"` //grampus image_id + Image string `json:"image"` + Uuid string `json:"uuid"` //数据集id + DatasetName string `json:"dataset_name"` + ComputeResource string `json:"compute_resource"` //计算资源，例如npu + AiCenter string `json:"ai_center"` //grampus ai center: center_id+center_name + BranchName string `json:"branch_name"` //分支名称 + Parameters string `json:"parameters"` //传给modelarts的param参数 + BootFile string `json:"boot_file"` //启动文件 + Description string `json:"description"` //描述 + ModelName string `json:"model_name"` //模型名称 + ModelVersion string `json:"model_version"` //模型版本 + CkptName string `json:"ckpt_name"` //权重文件名称 + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` + VersionName string `json:"version_name"` + Spec *SpecificationShow `json:"spec"` } type SpecificationShow struct { diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 9d31952d6..ce8c5dfe7 100755 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -98,11 +98,16 @@ func NewFuncMap() []template.FuncMap { "AllowedReactions": func() []string { return setting.UI.Reactions }, + "DebugAttachSize": func() int { + return setting.DebugAttachSize * 1000 * 1000 * 1000 + }, "AvatarLink": models.AvatarLink, "Safe": Safe, "SafeJS": SafeJS, "Str2html": Str2html, "subOne": subOne, + "addOne": addOne, + "TimeStampNow": timeutil.TimeStampNow, "TimeSince": timeutil.TimeSince, "TimeSinceUnix": timeutil.TimeSinceUnix, "TimeSinceUnix1": timeutil.TimeSinceUnix1, @@ -153,7 +158,7 @@ func NewFuncMap() []template.FuncMap { "EscapePound": func(str string) string { return strings.NewReplacer("%", "%25", "#", "%23", " ", "%20", "?", "%3F").Replace(str) }, - "IpynbBool":func(str string) bool{ + "IpynbBool": func(str string) bool { return strings.Contains(str, ".ipynb") }, "nl2br": func(text string) template.HTML { @@ -363,6 +368,7 @@ func NewTextFuncMap() []texttmpl.FuncMap { "AppDomain": func() string { return setting.Domain }, + "TimeStampNow": timeutil.TimeStampNow, "TimeSince": timeutil.TimeSince, "TimeSinceUnix": timeutil.TimeSinceUnix, "TimeSinceUnix1": timeutil.TimeSinceUnix1, @@ -470,7 +476,9 @@ func Str2html(raw string) template.HTML { func subOne(length int) int { return length - 1 } - +func addOne(length int64) int64 { + return length + 1 +} // Escape escapes a HTML string func Escape(raw string) string { return html.EscapeString(raw) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 035b323e0..ecd2598b7 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -406,6 +406,8 @@ sspi_auth_failed = SSPI authentication failed change_email = Change email change_email_address = Change email address new_email_address = New email address +openi_community_really_awesome = OpenI, Really Awesome! + [phone] format_err=The format of phone number is wrong. query_err=Fail to query phone number, please try again later. @@ -577,6 +579,7 @@ static.CloudBrainTaskNum=CloudBrain Task Count static.CloudBrainRunTime=CloudBrain Run Time static.CommitDatasetNum=Commit Dataset Count static.CommitModelCount=Commit Model Count +static.ModelConvertCount=Model Convert Count static.UserIndex=Normalized user index static.UserIndexPrimitive=User Index static.countdate=Count Date @@ -1060,8 +1063,12 @@ model_rename=Duplicate model name, please modify model name. notebook_file_not_exist=Notebook file does not exist. notebook_select_wrong=Please select a Notebook(.ipynb) file first. +notebook_path_too_long=The total length of selected file or files path exceed 255 characters, please select a shorter path file or change the file path. +notebook_branch_name_too_long=The total length of branch or branches name exceed 255 characters, please select a file in other branch. notebook_file_no_right=You have no right to access the Notebook(.ipynb) file. +notebook_repo_conflict=The files in different branches of the same repository can not run together. debug_again_fail=Fail to restart debug task, please try again later. +debug_again_fail_forever=The task was scheduled failed last time, can not restart. date=Date repo_add=Project Increment @@ -1079,6 +1086,7 @@ delete=Delete more=More gpu_type_all=All model_download=Model Download +all_result_download=All result download submit_image=Submit Image modify_image=Modify Image image_exist=Image name has been used, please use a new one. @@ -1091,8 +1099,8 @@ image_delete_fail=Failed to delete image, please try again later. image_overwrite=You had submitted the same name image before, are you sure to overwrite the original image? download=Download score=Score -wait_count_start = There are currently -wait_count_end = tasks queued +wait_count_start = Your current queue position is +wait_count_end = file_limit_100 = Display up to 100 files or folders in a single directory images.name = Image Tag images.name_placerholder = Please enter the image name @@ -1257,6 +1265,13 @@ modelarts.fullscreen_log_file = View in full screen modelarts.exit_full_screen = Exit fullscreen modelarts.no_node_right = The value of 'Amount of Compute Node' is wrong, you have no right to use the current value of 'Amount of Compute Node'. +scrolled_logs_top = You have scrolled to the top of the log +scrolled_logs_top_pls_retry = You have scrolled to the top of the log, please try again later! +scrolled_logs_bottom = You have scrolled to the bottom of the log +scrolled_logs_bottom_pls_retry = You have scrolled to the bottom of the log, please try again later! + +canceled_operation = You have canceled the operation +successfully_deleted = Successfully deleted debug_task_not_created = Debug task has not been created train_task_not_created = Train task has not been created @@ -1345,6 +1360,7 @@ modelconvert.inputshapeerror=Format input error, please input such as: 1,1,32,32 modelconvert.manage.create_error1=A model transformation task with the same name already exists. modelconvert.manage.create_error2=Only one running model transformation task can be created. modelconvert.manage.model_not_exist=The model in the task does not exist or has been deleted. +modelconvert.manage.model_file_not_exist=The model file in the task does not exist or has been deleted. modelconvert.manage.no_operate_right=You have no right to do the operation. debug.manage.model_not_exist=The model in the task does not exist or has been deleted, please create a new debug job. @@ -1470,7 +1486,7 @@ blame = Blame normal_view = Normal View line = line lines = lines -notebook_open = Open in Notebook +notebook_open = Run Online editor.new_file = New File editor.upload_file = Upload File @@ -2755,6 +2771,10 @@ repos.pr=PR repos.commit=Commit repos.closedIssues=Closed Issue repos.contributor=Contributor +repos.numDataset=Dataset File +repos.numCloudbrain=Cloudbrain Task +repos.numModel=Model +repos.numModelConvert=Model Convert Task repos.yes=Yes repos.no=No @@ -3127,6 +3147,7 @@ task_gpudebugjob=`created CPU/GPU type debugging task %s` task_c2net_gpudebugjob=`created CPU/GPU type debugging task %s` task_c2net_npudebugjob=`created NPU type debugging task %s` +task_c2ent_gcudebugjob=`created GCU type debugging task %s` task_nputrainjob=`created NPU training task %s` task_inferencejob=`created reasoning task %s` task_benchmark=`created profiling task %s` @@ -3287,6 +3308,7 @@ point_hr = Point/hr DEBUG = DEBUG SNN4IMAGENET = BENCHMARK BRAINSCORE = BENCHMARK +SNN4ECOSET = BENCHMARK MODELSAFETY = BENCHMARK TRAIN = TRAIN INFERENCE = INFERENCE @@ -3300,13 +3322,15 @@ Stopped_failed=Fail to stop the job, please try again later. Stopped_success_update_status_fail=Succeed in stopping th job, but failed to update the job status and duration time. load_code_failed=Fail to load code, please check if the right branch is selected. +error.debug_datasetsize = The size of dataset exceeds limitation (%dGB) error.dataset_select = dataset select error：the count exceed the limit or has same name +error.partial_datasets_not_available = There are non-existent or deleted files in the selected dataset file, please select again new_train_gpu_tooltips = The code is storaged in %s, the dataset is storaged in %s, the pre-trained model is storaged in the run parameter %s, and please put your model into %s then you can download it online new_debug_gpu_tooltips = The code is storaged in %s, the dataset is storaged in %s, the pre-trained model is storaged in the %s, and please put your model into %s then you can download it online new_debug_gpu_tooltips1 = The code is storaged in %s, the dataset is storaged in %s, the pre-trained model is storaged in the %s. new_train_npu_tooltips = The code is storaged in %s, the pre-trained model is storaged in the run parameter %s, and please put your model into %s then you can download it online new_infer_gpu_tooltips = The dataset is stored in %s, the model file is stored in %s, please store the inference output in %s for subsequent downloads. -code_obs_address = Code OBS address +code_obs_address = Code OBS address [points] points = points diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index d8e37a8d0..5cef94087 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -73,7 +73,7 @@ your_starred=已点赞 your_settings=设置 invite_friends=邀请好友 your_friend=您的好友 -invite_you_to_join_the_OpenI_AI_Collaboration_Platform_and_enjoy_abundant_free_computing_resources=邀请您加入启智社区AI协作平台，畅享充沛的免费算力资源！ +invite_you_to_join_the_OpenI_AI_Collaboration_Platform_and_enjoy_abundant_free_computing_resources=邀请您加入启智社区AI协作平台，畅享充沛的普惠算力资源！ recommender=推荐人 all=所有 @@ -409,6 +409,8 @@ sspi_auth_failed=SSPI 认证失败 change_email=修改邮箱 change_email_address=修改邮箱地址 new_email_address=新邮箱地址 +openi_community_really_awesome=启智社区 确实给力 + [phone] format_err=手机号格式错误。 query_err=查询手机号失败，请稍后再试。 @@ -581,6 +583,7 @@ static.CloudBrainTaskNum=云脑任务数 static.CloudBrainRunTime=云脑运行时间（小时） static.CommitDatasetNum=上传（提交）数据集文件数 static.CommitModelCount=提交模型数 +static.ModelConvertCount=模型转换数 static.UserIndex=归一化用户指数 static.UserIndexPrimitive=用户指数 static.countdate=系统统计时间 @@ -1059,8 +1062,12 @@ model_rename=模型名称重复，请修改模型名称 notebook_file_not_exist=Notebook文件不存在。 notebook_select_wrong=请先选择Notebook（.ipynb）文件。 +notebook_path_too_long=选择的一个或多个Notebook文件路径总长度超过255个字符，请选择路径较短的文件或调整文件路径。 +notebook_branch_name_too_long=选择的一个或多个Notebook文件分支名总长度超过255个字符，请选择其他分支的文件。 notebook_file_no_right=您没有这个Notebook文件的读权限。 +notebook_repo_conflict=同一个仓库的不同分支文件不能同时运行。 debug_again_fail=再次调试失败，请稍后再试。 +debug_again_fail_forever=这个任务之前没有调度成功，不能再次调试。 date=日期 repo_add=新增项目 @@ -1078,6 +1085,7 @@ delete=删除 more=更多 gpu_type_all=全部 model_download=结果下载 +all_result_download=全部结果下载 submit_image=提交镜像 modify_image=修改镜像 image_exist=镜像Tag已被使用，请修改镜像Tag。 @@ -1090,8 +1098,8 @@ image_delete_fail=删除镜像失败，请稍后再试。 image_overwrite=您已经提交过相同名称的镜像，您确定要覆盖原来提交的镜像吗？ download=模型下载 score=评分 -wait_count_start = 当前有 -wait_count_end = 个任务正在排队 +wait_count_start = 您当前排队位置是第 +wait_count_end = 位 file_limit_100 = 单目录下最多显示100个文件或文件夹 images.name = 镜像Tag images.name_placerholder = 请输入镜像Tag @@ -1269,6 +1277,13 @@ modelarts.fullscreen_log_file=全屏查看 modelarts.exit_full_screen=退出全屏 modelarts.no_node_right = 计算节点数的值配置错误，您没有权限使用当前配置的计算节点数。 +scrolled_logs_top = 您已翻阅至日志顶部 +scrolled_logs_top_pls_retry = 您已翻阅至日志顶部，请稍后再试！ +scrolled_logs_bottom = 您已翻阅至日志底部 +scrolled_logs_bottom_pls_retry = 您已翻阅至日志底部，请稍后再试！ + +canceled_operation = 您已取消操作 +successfully_deleted = 删除成功 debug_task_not_created = 未创建过调试任务 train_task_not_created = 未创建过训练任务 @@ -1359,6 +1374,7 @@ modelconvert.modelfileempty=请选择模型文件。 modelconvert.manage.create_error1=相同的名称模型转换任务已经存在。 modelconvert.manage.create_error2=只能创建一个正在运行的模型转换任务。 modelconvert.manage.model_not_exist=任务中选择的模型不存在或者已被删除。 +modelconvert.manage.model_file_not_exist=任务中选择的模型文件不存在或者已被删除。 modelconvert.manage.no_operate_right=您没有操作权限。 @@ -1488,7 +1504,7 @@ normal_view=普通视图 line=行 lines=行 -notebook_open = 在Notebook中打开 +notebook_open = 在线运行 editor.new_file=新建文件 editor.upload_file=上传文件 @@ -1672,7 +1688,7 @@ issues.action_assignee_no_select=未指派 issues.opened_by=由 %[3]s 于 %[1]s创建 pulls.merged_by=由 %[3]s 于 %[1]s 合并 pulls.merged_by_fake=由 %[2]s 于 %[1]s 合并 -issues.closed_by=按 %[3]s 关闭%[1]s +issues.closed_by=由 %[3]s 创建，被关闭于 %[1]s issues.opened_by_fake=由 %[2]s 于 %[1]s创建 issues.closed_by_fake=通过 %[2]s 关闭 %[1]s issues.previous=上一页 @@ -2773,6 +2789,11 @@ repos.pr=PR数 repos.commit=Commit数 repos.closedIssues=已解决任务数 repos.contributor=贡献者数 +repos.numDataset=数据集文件数 +repos.numCloudbrain=云脑任务数 +repos.numModel=模型数 +repos.numModelConvert=转换任务数 + repos.yes=是 repos.no=否 @@ -3145,6 +3166,7 @@ task_gpudebugjob=`创建了CPU/GPU类型调试任务 task_npudebugjob=`创建了NPU类型调试任务 %s` task_c2net_gpudebugjob=`创建了CPU/GPU类型调试任务 %s` task_c2net_npudebugjob=`创建了NPU类型调试任务 %s` +task_c2ent_gcudebugjob=`创建了GCU类型调试任务 %s` task_nputrainjob=`创建了NPU类型训练任务 %s` task_inferencejob=`创建了推理任务 %s` task_benchmark=`创建了评测任务 %s` @@ -3307,6 +3329,7 @@ point_hr = 积分/时 DEBUG = 调试任务 SNN4IMAGENET = 评测任务 BRAINSCORE = 评测任务 +SNN4ECOSET = 评测任务 MODELSAFETY = 评测任务 TRAIN = 训练任务 INFERENCE = 推理任务 @@ -3320,8 +3343,9 @@ Stopped_failed=任务停止失败，请稍后再试。 Stopped_success_update_status_fail=任务停止成功，状态及运行时间更新失败。 load_code_failed=代码加载失败，请确认选择了正确的分支。 - +error.debug_datasetsize = 数据集大小超过限制（%dGB） error.dataset_select = 数据集选择错误：数量超过限制或者有同名数据集 +error.partial_datasets_not_available = 选择的数据集文件中有不存在或已删除的文件，请重新选择 new_train_gpu_tooltips = 训练脚本存储在 %s 中，数据集存储在 %s 中，预训练模型存放在运行参数 %s 中，训练输出请存储在 %s 中以供后续下载。 new_debug_gpu_tooltips = 项目代码存储在 %s 中，数据集存储在 %s 中，选择的模型存储在 %s 中，调试输出请存储在 %s 中以供后续下载。 new_debug_gpu_tooltips1 = 项目代码存储在 %s 中，数据集存储在 %s 中，选择的模型存储在 %s 中。 diff --git a/package-lock.json b/package-lock.json index 7dc42d04c..ee922450a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1982,28 +1982,6 @@ "object.assign": "^4.1.0" } }, - "babel-polyfill": { - "version": "6.26.0", - "resolved": "https://registry.npm.taobao.org/babel-polyfill/download/babel-polyfill-6.26.0.tgz", - "integrity": "sha1-N5k3q8Z9eJWXCtxiHyhM2WbPIVM=", - "requires": { - "babel-runtime": "^6.26.0", - "core-js": "^2.5.0", - "regenerator-runtime": "^0.10.5" - }, - "dependencies": { - "core-js": { - "version": "2.6.12", - "resolved": "https://registry.npm.taobao.org/core-js/download/core-js-2.6.12.tgz?cache=0&sync_timestamp=1611040749668&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fcore-js%2Fdownload%2Fcore-js-2.6.12.tgz", - "integrity": "sha1-2TM9+nsGXjR8xWgiGdb2kIWcwuw=" - }, - "regenerator-runtime": { - "version": "0.10.5", - "resolved": "https://registry.npm.taobao.org/regenerator-runtime/download/regenerator-runtime-0.10.5.tgz", - "integrity": "sha1-M2w+/BIgrc7dosn6tntaeVWjNlg=" - } - } - }, "babel-runtime": { "version": "6.26.0", "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", @@ -2194,9 +2172,9 @@ "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" }, "blueimp-md5": { - "version": "2.18.0", - "resolved": "https://registry.npm.taobao.org/blueimp-md5/download/blueimp-md5-2.18.0.tgz", - "integrity": "sha1-EVK+EzXwxrORHtnjbbVPPmrFKTU=" + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/blueimp-md5/-/blueimp-md5-2.19.0.tgz", + "integrity": "sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==" }, "bn.js": { "version": "5.1.1", @@ -3536,6 +3514,11 @@ "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-2.2.0.tgz", "integrity": "sha1-QGXiATz5+5Ft39gu+1Bq1MZ2kGI=" }, + "dayjs": { + "version": "1.10.7", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.7.tgz", + "integrity": "sha512-P6twpd70BcPK34K26uJ1KT3wlhpuOAPoMwJzpsIWUxHZ7wpmbdZL/hQqBDfz7hGurYSa5PhzdhDHtt319hL3ig==" + }, "de-indent": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/de-indent/-/de-indent-1.0.2.tgz", @@ -4152,25 +4135,29 @@ "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, "esdk-obs-browserjs": { - "version": "3.20.7", - "resolved": "https://registry.npm.taobao.org/esdk-obs-browserjs/download/esdk-obs-browserjs-3.20.7.tgz", - "integrity": "sha1-vhziRlKEhW3PgZPl0DyX68bJI0s=", + "version": "3.22.3", + "resolved": "https://registry.npmjs.org/esdk-obs-browserjs/-/esdk-obs-browserjs-3.22.3.tgz", + "integrity": "sha512-MATZXp0FwjPtKG9tpdfURa3koUarR/ev+tbO0oUKgj0GRt0798ZxmfCvYvRpgNst4w1ht4E79ikD4H40UYLgPA==", "requires": { - "axios": "^0.19.0", - "babel-polyfill": "^6.26.0", - "blueimp-md5": "^2.10.0", - "js-base64": "^2.3.2", - "jssha": "^2.3.1", - "urijs": "^1.19.1" + "axios": "^0.26.1", + "blueimp-md5": "^2.18.0", + "js-base64": "^3.7.1", + "jssha": "^3.2.0", + "urijs": "^1.19.7" }, "dependencies": { "axios": { - "version": "0.19.2", - "resolved": "https://registry.npm.taobao.org/axios/download/axios-0.19.2.tgz?cache=0&sync_timestamp=1608609215811&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Faxios%2Fdownload%2Faxios-0.19.2.tgz", - "integrity": "sha1-PqNsXYgY0NX4qKl6bTa4bNwAyyc=", + "version": "0.26.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", + "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==", "requires": { - "follow-redirects": "1.5.10" + "follow-redirects": "^1.14.8" } + }, + "js-base64": { + "version": "3.7.3", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.3.tgz", + "integrity": "sha512-PAr6Xg2jvd7MCR6Ld9Jg3BmTcjYsHEBx1VlwEwULb/qowPf5VD9kEMagj23Gm7JRnSvE/Da/57nChZjnvL8v6A==" } } }, @@ -5382,27 +5369,9 @@ } }, "follow-redirects": { - "version": "1.5.10", - "resolved": "https://registry.npm.taobao.org/follow-redirects/download/follow-redirects-1.5.10.tgz?cache=0&sync_timestamp=1611606737937&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Ffollow-redirects%2Fdownload%2Ffollow-redirects-1.5.10.tgz", - "integrity": "sha1-e3qfmuov3/NnhqlP9kPtB/T/Xio=", - "requires": { - "debug": "=3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npm.taobao.org/debug/download/debug-3.1.0.tgz?cache=0&sync_timestamp=1607566533140&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fdebug%2Fdownload%2Fdebug-3.1.0.tgz", - "integrity": "sha1-W7WgZyYotkFJVmuhaBnmFRjGcmE=", - "requires": { - "ms": "2.0.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - } - } + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" }, "fomantic-ui": { "version": "2.8.4", @@ -7884,9 +7853,9 @@ } }, "jssha": { - "version": "2.4.2", - "resolved": "https://registry.npm.taobao.org/jssha/download/jssha-2.4.2.tgz", - "integrity": "sha1-2VCwlWNJKL1rK9odQtqaOnYtZek=" + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/jssha/-/jssha-3.3.0.tgz", + "integrity": "sha512-w9OtT4ALL+fbbwG3gw7erAO0jvS5nfvrukGPMWIAoea359B26ALXGpzy4YJSp9yGnpUvuvOw1nSjSoHDfWSr1w==" }, "just-debounce": { "version": "1.0.0", @@ -14400,9 +14369,9 @@ } }, "urijs": { - "version": "1.19.6", - "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.6.tgz", - "integrity": "sha512-eSXsXZ2jLvGWeLYlQA3Gh36BcjF+0amo92+wHPyN1mdR8Nxf75fuEuYTd9c0a+m/vhCjRK0ESlE9YNLW+E1VEw==" + "version": "1.19.11", + "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.11.tgz", + "integrity": "sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ==" }, "urix": { "version": "0.1.0", diff --git a/public/home/home.js b/public/home/home.js index fe843161e..626a1302f 100755 --- a/public/home/home.js +++ b/public/home/home.js @@ -243,11 +243,12 @@ document.onreadystatechange = function () { html += recordPrefix + actionName; html += " " + getRepotext(record) + "" } - else if(record.OpType == "24" || record.OpType == "26" || record.OpType == "27" || record.OpType == "28" || record.OpType == "30" || record.OpType == "31" || record.OpType == "32" || record.OpType == "33"){ + else if(record.OpType == "24" || record.OpType == "26" || record.OpType == "27" || record.OpType == "28" || record.OpType == "30" + || record.OpType == "31" || record.OpType == "32" || record.OpType == "33"){ html += recordPrefix + actionName; html += " " + record.RefName + "" } - else if(record.OpType == "25" || record.OpType == "29" || record.OpType == "39" || record.OpType == "40"){ + else if(record.OpType == "25" || record.OpType == "29" || record.OpType == "39" || record.OpType == "40" || record.OpType == "41"){ html += recordPrefix + actionName; html += " " + record.RefName + "" } @@ -294,7 +295,7 @@ function getTaskLink(record){ re = re + "/cloudbrain/train-job/" + record.Content; }else if(record.OpType == 32 || record.OpType == 33){ re = re + "/grampus/train-job/" + record.Content; - }else if(record.OpType == 39 || record.OpType == 40){ + }else if(record.OpType == 39 || record.OpType == 40 || record.OpType == 41){ re = re + "/grampus/notebook/" + record.Content; } @@ -453,9 +454,10 @@ var actionNameZH={ "33":"创建了CPU/GPU类型训练任务", "35":"创建的数据集 {dataset} 被设置为推荐数据集", "36":"提交了镜像 {image}", - "37": "提交的镜像 {image} 被设置为推荐镜像", + "37":"提交的镜像 {image} 被设置为推荐镜像", "39":"创建了CPU/GPU类型调试任务", "40":"创建了NPU类型调试任务", + "41":"创建了GCU类型调试任务", }; var actionNameEN={ @@ -486,9 +488,10 @@ var actionNameEN={ "33":" created CPU/GPU type training task", "35":" created dataset {dataset} was set as recommended dataset", "36":"committed image {image}", - "37": "committed image {image} was set as recommended image", + "37":"committed image {image} was set as recommended image", "39":" created CPU/GPU type debugging task ", "40":" created NPU type debugging task ", + "41":" created GCU type debugging task ", }; var repoAndOrgZH={ diff --git a/public/img/login_bg_default.png b/public/img/login_bg_default.png new file mode 100644 index 000000000..0c80142c9 Binary files /dev/null and b/public/img/login_bg_default.png differ diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index 91685251b..2002e69f4 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -53,7 +53,7 @@ func CloudBrains(ctx *context.Context) { var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeBenchmark) { - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 4936c2362..ba7346481 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -549,6 +549,8 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqToken()) + m.Get("/compute-nodes", reqToken(), user.GetComputeNodes) + // Notifications m.Group("/notifications", func() { m.Combo(""). @@ -745,7 +747,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/file_notebook", func() { m.Get("", repo.GetFileNoteBookInfo) m.Post("/create", reqToken(), reqWeChat(), bind(api.CreateFileNotebookJobOption{}), repo.CreateFileNoteBook) - + m.Post("/status", reqToken(), bind(api.CreateFileNotebookJobOption{}), repo.FileNoteBookStatus) }) m.Group("/repos", func() { @@ -1024,6 +1026,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/query_model_byName", repo.QueryModelByName) m.Get("/query_model_for_predict", repo.QueryModelListForPredict) m.Get("/query_modelfile_for_predict", repo.QueryModelFileForPredict) + m.Get("/query_train_job", repo.QueryTrainJobList) + m.Get("/query_train_job_version", repo.QueryTrainJobVersionList) m.Get("/query_train_model", repo.QueryTrainModelList) m.Post("/create_model_convert", repo.CreateModelConvert) m.Post("/convert_stop", repo.StopModelConvert) diff --git a/routers/api/v1/repo/attachments.go b/routers/api/v1/repo/attachments.go index cb36ba2ee..7db8c9067 100644 --- a/routers/api/v1/repo/attachments.go +++ b/routers/api/v1/repo/attachments.go @@ -1,22 +1,67 @@ package repo import ( + "net/http" + + "code.gitea.io/gitea/modules/log" + + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" routeRepo "code.gitea.io/gitea/routers/repo" ) func GetSuccessChunks(ctx *context.APIContext) { + if errStr := checkDatasetPermission(ctx); errStr != "" { + ctx.JSON(http.StatusForbidden, ctx.Tr(errStr)) + } + routeRepo.GetSuccessChunks(ctx.Context) } +func checkDatasetPermission(ctx *context.APIContext) string { + datasetId := ctx.QueryInt64("dataset_id") + + dataset, err := models.GetDatasetByID(datasetId) + if err != nil { + log.Warn("can not find dataset", err) + + return "dataset.query_dataset_fail" + } + repo, err := models.GetRepositoryByID(dataset.RepoID) + if err != nil { + log.Warn("can not find repo", err) + return "dataset.query_dataset_fail" + } + + permission, err := models.GetUserRepoPermission(repo, ctx.User) + if err != nil { + log.Warn("can not find repo permission for user", err) + return "dataset.query_dataset_fail" + } + if !permission.CanWrite(models.UnitTypeDatasets) { + + return "error.no_right" + } + return "" +} + func NewMultipart(ctx *context.APIContext) { + if errStr := checkDatasetPermission(ctx); errStr != "" { + ctx.JSON(http.StatusForbidden, ctx.Tr(errStr)) + } routeRepo.NewMultipart(ctx.Context) } func GetMultipartUploadUrl(ctx *context.APIContext) { + if errStr := checkDatasetPermission(ctx); errStr != "" { + ctx.JSON(http.StatusForbidden, ctx.Tr(errStr)) + } routeRepo.GetMultipartUploadUrl(ctx.Context) } func CompleteMultipart(ctx *context.APIContext) { + if errStr := checkDatasetPermission(ctx); errStr != "" { + ctx.JSON(http.StatusForbidden, ctx.Tr(errStr)) + } routeRepo.CompleteMultipart(ctx.Context) } diff --git a/routers/api/v1/repo/cloudbrain.go b/routers/api/v1/repo/cloudbrain.go index 805443788..250a08423 100755 --- a/routers/api/v1/repo/cloudbrain.go +++ b/routers/api/v1/repo/cloudbrain.go @@ -110,6 +110,9 @@ func GeneralCloudBrainJobStop(ctx *context.APIContext) { func CreateFileNoteBook(ctx *context.APIContext, option api.CreateFileNotebookJobOption) { cloudbrainTask.FileNotebookCreate(ctx.Context, option) } +func FileNoteBookStatus(ctx *context.APIContext, option api.CreateFileNotebookJobOption) { + cloudbrainTask.FileNotebookStatus(ctx.Context, option) +} func GetFileNoteBookInfo(ctx *context.APIContext) { //image description spec description waiting count diff --git a/routers/api/v1/repo/cloudbrain_dashboard.go b/routers/api/v1/repo/cloudbrain_dashboard.go index bb04038b9..b980c70b3 100755 --- a/routers/api/v1/repo/cloudbrain_dashboard.go +++ b/routers/api/v1/repo/cloudbrain_dashboard.go @@ -623,7 +623,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { } jobTypeList := []string{string(models.JobTypeDebug), string(models.JobTypeTrain), string(models.JobTypeInference), string(models.JobTypeBenchmark), - string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)} + string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)} for _, v := range jobTypeList { if _, ok := cloudOneJobTypeRes[v]; !ok { cloudOneJobTypeRes[v] = 0 @@ -645,7 +645,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { } } - ComputeResourceList := []string{"CPU/GPU", "NPU"} + ComputeResourceList := []string{"CPU/GPU", "NPU", "GCU"} for _, v := range ComputeResourceList { if _, ok := cloudBrainComputeResource[v]; !ok { cloudBrainComputeResource[v] = 0 @@ -687,7 +687,6 @@ func GetCloudbrainsDetailData(ctx *context.Context) { return } recordBeginTime := recordCloudbrain[0].Cloudbrain.CreatedUnix - endTime := time.Now() listType := ctx.Query("listType") jobType := ctx.Query("jobType") jobStatus := ctx.Query("jobStatus") @@ -695,6 +694,33 @@ func GetCloudbrainsDetailData(ctx *context.Context) { aiCenter := ctx.Query("aiCenter") needDeleteInfo := ctx.Query("needDeleteInfo") + accCardType := ctx.Query("accCardType") + accCardsNum := ctx.QueryInt("accCardsNum") + workServerNumber := ctx.QueryInt("workServerNumber") + beginTimeStr := ctx.QueryTrim("beginTime") + endTimeStr := ctx.QueryTrim("endTime") + var beginTimeUnix int64 + var endTimeUnix int64 + if beginTimeStr == "" || endTimeStr == "" { + beginTimeUnix = int64(recordBeginTime) + endTimeUnix = time.Now().Unix() + } else { + beginTime, err := time.ParseInLocation("2006-01-02T15:04:05", beginTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + beginTimeUnix = beginTime.Unix() + endTime, err := time.ParseInLocation("2006-01-02T15:04:05", endTimeStr, time.Local) + if err != nil { + log.Error("Can not ParseInLocation.", err) + ctx.Error(http.StatusBadRequest, ctx.Tr("ParseInLocation_get_error")) + return + } + endTimeUnix = endTime.Unix() + } + if cloudBrainType == models.TypeCloudBrainOne && aiCenter == models.AICenterOfCloudBrainOne { aiCenter = "" } @@ -730,7 +756,7 @@ func GetCloudbrainsDetailData(ctx *context.Context) { var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeBenchmark) { - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } @@ -753,18 +779,21 @@ func GetCloudbrainsDetailData(ctx *context.Context) { Page: page, PageSize: pageSize, }, - Keyword: keyword, - Type: cloudBrainType, - ComputeResource: listType, - JobTypeNot: jobTypeNot, - JobStatusNot: jobStatusNot, - JobStatus: jobStatuses, - JobTypes: jobTypes, - NeedRepoInfo: true, - BeginTimeUnix: int64(recordBeginTime), - EndTimeUnix: endTime.Unix(), - AiCenter: aiCenter, - NeedDeleteInfo: needDeleteInfo, + Keyword: keyword, + Type: cloudBrainType, + ComputeResource: listType, + JobTypeNot: jobTypeNot, + JobStatusNot: jobStatusNot, + JobStatus: jobStatuses, + JobTypes: jobTypes, + NeedRepoInfo: true, + BeginTimeUnix: beginTimeUnix, + EndTimeUnix: endTimeUnix, + AiCenter: aiCenter, + NeedDeleteInfo: needDeleteInfo, + AccCardType: accCardType, + AccCardsNum: accCardsNum, + WorkServerNumber: workServerNumber, }) if err != nil { ctx.ServerError("Get job failed:", err) @@ -1039,7 +1068,7 @@ func getCloudbrainCount(beginTime time.Time, endTime time.Time, cloudbrains []*m } } - ComputeResourceList := []string{"CPU/GPU", "NPU"} + ComputeResourceList := []string{"CPU/GPU", "NPU", "GCU"} for _, v := range ComputeResourceList { if _, ok := cloudBrainComputeResource[v]; !ok { cloudBrainComputeResource[v] = 0 @@ -1231,8 +1260,8 @@ func DownloadCloudBrainBoard(ctx *context.Context) { Type: models.TypeCloudBrainAll, BeginTimeUnix: int64(recordBeginTime), EndTimeUnix: endTime.Unix(), + AccCardsNum: models.AccCardsNumAll, }) - log.Info("totalcountisis:", total) if err != nil { log.Warn("Can not get cloud brain info", err) @@ -1261,6 +1290,7 @@ func DownloadCloudBrainBoard(ctx *context.Context) { BeginTimeUnix: int64(recordBeginTime), EndTimeUnix: endTime.Unix(), NeedRepoInfo: true, + AccCardsNum: models.AccCardsNumAll, }) if err != nil { log.Warn("Can not get cloud brain info", err) diff --git a/routers/api/v1/repo/modelmanage.go b/routers/api/v1/repo/modelmanage.go index 3b0aed0d5..696231fea 100644 --- a/routers/api/v1/repo/modelmanage.go +++ b/routers/api/v1/repo/modelmanage.go @@ -4,8 +4,10 @@ import ( "net/http" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/convert" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/storage" + api "code.gitea.io/gitea/modules/structs" routerRepo "code.gitea.io/gitea/routers/repo" ) @@ -54,6 +56,21 @@ func QueryModelListForPredict(ctx *context.APIContext) { routerRepo.QueryModelListForPredict(ctx.Context) } +func QueryTrainJobList(ctx *context.APIContext) { + result, err := routerRepo.QueryTrainJobListApi(ctx.Context) + if err != nil { + log.Info("query error." + err.Error()) + ctx.JSON(http.StatusOK, nil) + } else { + re := make([]*api.Cloudbrain, 0) + for _, task := range result { + conRe := convert.ToCloudBrain(task) + re = append(re, conRe) + } + ctx.JSON(http.StatusOK, re) + } +} + func QueryTrainModelList(ctx *context.APIContext) { result, err := routerRepo.QueryTrainModelFileById(ctx.Context) if err != nil { @@ -63,6 +80,21 @@ func QueryTrainModelList(ctx *context.APIContext) { ctx.JSON(http.StatusOK, re) } +func QueryTrainJobVersionList(ctx *context.APIContext) { + result, err := routerRepo.QueryTrainJobVersionListApi(ctx.Context) + if err != nil { + log.Info("query error." + err.Error()) + ctx.JSON(http.StatusOK, nil) + } else { + re := make([]*api.Cloudbrain, 0) + for _, task := range result { + conRe := convert.ToCloudBrain(task) + re = append(re, conRe) + } + ctx.JSON(http.StatusOK, re) + } +} + func convertFileFormat(result []storage.FileInfo) []FileInfo { re := make([]FileInfo, 0) if result != nil { diff --git a/routers/api/v1/repo/repo_dashbord.go b/routers/api/v1/repo/repo_dashbord.go index b3a01cff1..e3a54b4fa 100644 --- a/routers/api/v1/repo/repo_dashbord.go +++ b/routers/api/v1/repo/repo_dashbord.go @@ -601,7 +601,7 @@ func getSummaryFileName(ctx *context.Context, beginTime time.Time, endTime time. func allProjectsPeroidHeader(ctx *context.Context) map[string]string { return map[string]string{"A1": ctx.Tr("admin.repos.id"), "B1": ctx.Tr("admin.repos.projectName"), "C1": ctx.Tr("repo.owner"), "D1": ctx.Tr("admin.repos.isPrivate"), "E1": ctx.Tr("admin.repos.openi"), "F1": ctx.Tr("admin.repos.visit"), "G1": ctx.Tr("admin.repos.download"), "H1": ctx.Tr("admin.repos.pr"), "I1": ctx.Tr("admin.repos.commit"), - "J1": ctx.Tr("admin.repos.watches"), "K1": ctx.Tr("admin.repos.stars"), "L1": ctx.Tr("admin.repos.forks"), "M1": ctx.Tr("admin.repos.issues"), "N1": ctx.Tr("admin.repos.closedIssues"), "O1": ctx.Tr("admin.repos.contributor"), "P1": ctx.Tr("admin.repos.isFork"), "Q1": ctx.Tr("admin.repos.isMirror"), "R1": ctx.Tr("admin.repos.create")} + "J1": ctx.Tr("admin.repos.watches"), "K1": ctx.Tr("admin.repos.stars"), "L1": ctx.Tr("admin.repos.forks"), "M1": ctx.Tr("admin.repos.issues"), "N1": ctx.Tr("admin.repos.closedIssues"), "O1": ctx.Tr("admin.repos.contributor"), "P1": ctx.Tr("admin.repos.numDataset"), "Q1": ctx.Tr("admin.repos.numCloudbrain"), "R1": ctx.Tr("admin.repos.numModel"), "S1": ctx.Tr("admin.repos.numModelConvert"), "T1": ctx.Tr("admin.repos.isFork"), "U1": ctx.Tr("admin.repos.isMirror"), "V1": ctx.Tr("admin.repos.create")} } @@ -619,11 +619,13 @@ func allProjectsPeriodSummaryValues(row int, rs *ProjectSummaryBaseData, ctx *co } func allProjectsPeroidValues(row int, rs *models.RepoStatistic, ctx *context.Context) map[string]string { + return map[string]string{getCellName("A", row): strconv.FormatInt(rs.RepoID, 10), getCellName("B", row): rs.DisplayName(), getCellName("C", row): rs.OwnerName, getCellName("D", row): getBoolDisplay(rs.IsPrivate, ctx), getCellName("E", row): strconv.FormatFloat(rs.RadarTotal, 'f', 2, 64), getCellName("F", row): strconv.FormatInt(rs.NumVisits, 10), getCellName("G", row): strconv.FormatInt(rs.NumDownloads, 10), getCellName("H", row): strconv.FormatInt(rs.NumPulls, 10), getCellName("I", row): strconv.FormatInt(rs.NumCommits, 10), getCellName("J", row): strconv.FormatInt(rs.NumWatches, 10), getCellName("K", row): strconv.FormatInt(rs.NumStars, 10), getCellName("L", row): strconv.FormatInt(rs.NumForks, 10), getCellName("M", row): strconv.FormatInt(rs.NumIssues, 10), - getCellName("N", row): strconv.FormatInt(rs.NumClosedIssues, 10), getCellName("O", row): strconv.FormatInt(rs.NumContributor, 10), getCellName("P", row): getBoolDisplay(rs.IsFork, ctx), getCellName("Q", row): getBoolDisplay(rs.IsMirror, ctx), getCellName("R", row): time.Unix(int64(rs.RepoCreatedUnix), 0).Format(CREATE_TIME_FORMAT), + getCellName("N", row): strconv.FormatInt(rs.NumClosedIssues, 10), getCellName("O", row): strconv.FormatInt(rs.NumContributor, 10), getCellName("P", row): strconv.FormatInt(rs.NumDatasetFile, 10), getCellName("Q", row): strconv.FormatInt(rs.NumCloudbrain, 10), getCellName("R", row): strconv.FormatInt(rs.NumModels, 10), getCellName("S", row): strconv.FormatInt(rs.NumModelConvert, 10), getCellName("T", row): getBoolDisplay(rs.IsFork, ctx), getCellName("U", row): getBoolDisplay(rs.IsMirror, ctx), getCellName("V", row): time.Unix(int64(rs.RepoCreatedUnix), 0).Format(CREATE_TIME_FORMAT), } + } func allProjectsOpenIHeader() map[string]string { @@ -804,11 +806,11 @@ func generateOpenICountSql(latestDate string) string { } func generateTypeAllSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string { - sql := "SELECT A.repo_id,name,alias,owner_name,is_private,is_mirror,is_fork,repo_created_unix,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM " + + sql := "SELECT A.repo_id,name,alias,owner_name,is_private,is_mirror,is_fork,repo_created_unix,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor,num_models,num_model_convert,num_cloudbrain,num_dataset_file FROM " + "(SELECT repo_id,sum(num_visits) as num_visits " + " FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," + - "(SELECT repo_id,name,alias,owner_name,is_private,is_mirror,is_fork,repo_created_unix,radar_total,num_watches,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor from public.repo_statistic where date='" + latestDate + "') B" + + "(SELECT repo_id,name,alias,owner_name,is_private,is_mirror,is_fork,repo_created_unix,radar_total,num_watches,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor,num_models,num_model_convert,num_cloudbrain,num_dataset_file from public.repo_statistic where date='" + latestDate + "') B" + " where A.repo_id=B.repo_id" if q != "" { @@ -828,8 +830,8 @@ func generateTypeAllOpenISql(latestDate string, page int, pageSize int) string { func generatePageSql(beginTime time.Time, endTime time.Time, latestDate string, q string, orderBy string, page int, pageSize int) string { - sql := "SELECT A.repo_id,name,alias,owner_name,is_private,is_mirror,is_fork,repo_created_unix,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor FROM " + - "(SELECT repo_id,sum(num_watches_added) as num_watches,sum(num_visits) as num_visits, sum(num_downloads_added) as num_downloads,sum(num_pulls_added) as num_pulls,sum(num_commits_added) as num_commits,sum(num_stars_added) as num_stars,sum(num_forks_added) num_forks,sum(num_issues_added) as num_issues,sum(num_closed_issues_added) as num_closed_issues,sum(num_contributor_added) as num_contributor " + + sql := "SELECT A.repo_id,name,alias,owner_name,is_private,is_mirror,is_fork,repo_created_unix,radar_total,num_watches,num_visits,num_downloads,num_pulls,num_commits,num_stars,num_forks,num_issues,num_closed_issues,num_contributor,num_models,num_model_convert,num_cloudbrain,num_dataset_file FROM " + + "(SELECT repo_id,sum(num_watches_added) as num_watches,sum(num_visits) as num_visits, sum(num_downloads_added) as num_downloads,sum(num_pulls_added) as num_pulls,sum(num_commits_added) as num_commits,sum(num_stars_added) as num_stars,sum(num_forks_added) num_forks,sum(num_issues_added) as num_issues,sum(num_closed_issues_added) as num_closed_issues,sum(num_contributor_added) as num_contributor,sum(num_models_added) as num_models,sum(num_model_convert_added) as num_model_convert,sum(num_dataset_file_added) as num_dataset_file, sum(num_cloudbrain_added) as num_cloudbrain " + " FROM repo_statistic where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) + " and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) + " group by repo_id) A," + "(SELECT repo_id,name,alias,owner_name,is_private,is_mirror,is_fork,repo_created_unix,radar_total from public.repo_statistic where date='" + latestDate + "') B" + diff --git a/routers/api/v1/user/repo.go b/routers/api/v1/user/repo.go index 4024bf96c..e9b8c6675 100644 --- a/routers/api/v1/user/repo.go +++ b/routers/api/v1/user/repo.go @@ -5,6 +5,7 @@ package user import ( + "code.gitea.io/gitea/modules/modelarts" "net/http" "code.gitea.io/gitea/models" @@ -146,3 +147,22 @@ func ListOrgRepos(ctx *context.APIContext) { listUserRepos(ctx, ctx.Org.Organization, ctx.IsSigned) } + +func GetComputeNodes(ctx *context.APIContext) { + taskeType := ctx.QueryInt("type") + if taskeType == 2 { + ctx.JSON(http.StatusOK, []int{1}) + } else { + modelarts.InitMultiNode() + if modelarts.MultiNodeConfig != nil { + for _, info := range modelarts.MultiNodeConfig.Info { + if isInOrg, _ := models.IsOrganizationMemberByOrgName(info.Org, ctx.User.ID); isInOrg { + ctx.JSON(http.StatusOK, info.Node) + return + } + } + } + ctx.JSON(http.StatusOK, []int{1}) + } + +} diff --git a/routers/home.go b/routers/home.go index 40a41bd68..21b1db946 100755 --- a/routers/home.go +++ b/routers/home.go @@ -42,6 +42,7 @@ const ( tplExploreImages base.TplName = "explore/images" tplExploreExploreDataAnalysis base.TplName = "explore/data_analysis" tplHomeTerm base.TplName = "terms" + tplHomeAnnual base.TplName = "annual_privacy" tplHomePrivacy base.TplName = "privacy" tplResoruceDesc base.TplName = "resource_desc" tplRepoSquare base.TplName = "explore/repos/square" @@ -966,6 +967,9 @@ func RecommendHomeInfo(ctx *context.Context) { func HomeTerm(ctx *context.Context) { ctx.HTML(200, tplHomeTerm) } +func HomeAnnual(ctx *context.Context) { + ctx.HTML(200, tplHomeAnnual) +} func HomePrivacy(ctx *context.Context) { ctx.HTML(200, tplHomePrivacy) diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index 592194371..e67568394 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -2,7 +2,6 @@ package repo import ( "archive/zip" - "code.gitea.io/gitea/services/repository" "encoding/json" "errors" "fmt" @@ -12,6 +11,8 @@ import ( "regexp" "strings" + "code.gitea.io/gitea/services/repository" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" @@ -75,7 +76,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio cloudType := aiTask.Type modelSelectedFile := ctx.Query("modelSelectedFile") //download model zip //train type - if aiTask.ComputeResource == models.NPUResource { + if aiTask.ComputeResource == models.NPUResource || aiTask.ComputeResource == models.GCUResource { cloudType = models.TypeCloudBrainTwo } else if aiTask.ComputeResource == models.GPUResource { cloudType = models.TypeCloudBrainOne @@ -710,36 +711,42 @@ func downloadFromCloudBrainTwo(path string, task *models.AiModelManage, ctx *con } func QueryTrainJobVersionList(ctx *context.Context) { + + VersionListTasks, err := QueryTrainJobVersionListApi(ctx) + if err != nil { + ctx.JSON(200, nil) + } else { + ctx.JSON(200, VersionListTasks) + } +} + +func QueryTrainJobVersionListApi(ctx *context.Context) ([]*models.Cloudbrain, error) { log.Info("query train job version list. start.") JobID := ctx.Query("jobId") if JobID == "" { JobID = ctx.Query("JobId") } - VersionListTasks, count, err := models.QueryModelTrainJobVersionList(JobID) - log.Info("query return count=" + fmt.Sprint(count)) + return VersionListTasks, err +} + +func QueryTrainJobList(ctx *context.Context) { + VersionListTasks, err := QueryTrainJobListApi(ctx) if err != nil { - ctx.ServerError("QueryTrainJobList:", err) + ctx.JSON(200, nil) } else { ctx.JSON(200, VersionListTasks) } } -func QueryTrainJobList(ctx *context.Context) { - log.Info("query train job list. start.") +func QueryTrainJobListApi(ctx *context.Context) ([]*models.Cloudbrain, error) { repoId := ctx.QueryInt64("repoId") - VersionListTasks, count, err := models.QueryModelTrainJobList(repoId) log.Info("query return count=" + fmt.Sprint(count)) - if err != nil { - ctx.ServerError("QueryTrainJobList:", err) - } else { - ctx.JSON(200, VersionListTasks) - } - + return VersionListTasks, err } func QueryTrainModelFileById(ctx *context.Context) ([]storage.FileInfo, error) { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 905c25a64..6d25511cc 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -285,7 +285,7 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } var datasetInfos map[string]models.DatasetInfo var datasetNames string - //var + var attachSize int64 if uuids != "" { datasetInfos, datasetNames, err = models.GetDatasetInfo(uuids) if err != nil { @@ -294,6 +294,18 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) return } + + if jobType == string(models.JobTypeDebug) { + for _, infos := range datasetInfos { + attachSize += infos.Size + } + if attachSize > int64(setting.DebugAttachSize*1000*1000*1000) { + log.Error("The DatasetSize exceeds the limit (%dGB)", setting.DebugAttachSize) // GB + cloudBrainNewDataPrepare(ctx, jobType) + ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize", setting.DebugAttachSize), tpl, &form) + return + } + } } command := cloudbrain.GetCloudbrainDebugCommand() @@ -387,7 +399,6 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { req.ModelVersion = form.ModelVersion req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl req.PreTrainModelUrl = form.PreTrainModelUrl - } _, err = cloudbrain.GenerateTask(req) @@ -592,8 +603,10 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra } -/** - 检查用户传输的参数是否符合专属资源池 +/* +* + + 检查用户传输的参数是否符合专属资源池 */ func checkCloudBrainSpecialPool(ctx *context.Context, jobType string, queue string, resourceSpecId int) string { if cloudbrain.SpecialPools != nil { @@ -843,7 +856,7 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo if task.JobType == string(models.JobTypeBenchmark) { task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") - } else if task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) { + } else if models.IsModelBenchMarkJobType(task.JobType) { task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") task.BenchmarkTypeName = task.JobType ctx.Data["BenchmarkTypeName"] = task.JobType @@ -911,10 +924,13 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo func CloudBrainDebug(ctx *context.Context) { task := ctx.Cloudbrain debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName - if task.BootFile != "" { - ctx.Redirect(getFileUrl(debugUrl, task.BootFile)) + if ctx.QueryTrim("file") != "" { + ctx.Redirect(getFileUrl(debugUrl, ctx.QueryTrim("file"))) } else { + if task.BootFile != "" { + go cloudbrainTask.UploadNotebookFiles(task) + } ctx.Redirect(debugUrl) } @@ -1306,8 +1322,8 @@ func DeleteJobsByRepoID(repoID int64) { DeleteJobs(cloudBrains) } -/** - +/* +* */ func StopJobs(cloudBrains []*models.Cloudbrain) { @@ -1638,6 +1654,21 @@ func CloudBrainDownloadModel(ctx *context.Context) { ctx.Resp.Header().Set("Cache-Control", "max-age=0") http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) } + +func CloudBrainDownloadMultiModel(ctx *context.Context) { + parentDir := ctx.Query("parentDir") + jobName := ctx.Query("jobName") + filePath := "jobs/" + jobName + "/model/" + parentDir + allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, filePath) + if err == nil { + returnFileName := jobName + ".zip" + MinioDownloadManyFile(filePath, ctx, returnFileName, allFile) + } else { + log.Info("error,msg=" + err.Error()) + ctx.ServerError("no file to download.", err) + } +} + func CloudBrainDownloadInferenceResult(ctx *context.Context) { parentDir := ctx.Query("parentDir") fileName := ctx.Query("fileName") @@ -1674,6 +1705,8 @@ func GetRate(ctx *context.Context) { ctx.Redirect(setting.Snn4imagenetServerHost) } else if job.JobType == string(models.JobTypeBrainScore) { ctx.Redirect(setting.BrainScoreServerHost) + } else if job.JobType == string(models.JobTypeSnn4Ecoset) { + ctx.Redirect(setting.Snn4EcosetServerHost) } else { log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"]) } @@ -2146,7 +2179,7 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { } var jobTypes []string - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeModelSafety)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset), string(models.JobTypeModelSafety)) ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, @@ -2179,14 +2212,16 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { ciTasks[i].BenchmarkTypeName = "" if ciTasks[i].JobType == string(models.JobTypeBenchmark) { ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") - } else if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) || ciTasks[i].JobType == string(models.JobTypeBrainScore) { + } else if models.IsModelBenchMarkJobType(ciTasks[i].JobType) { ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) { ciTasks[i].BenchmarkTypeRankLink = setting.Snn4imagenetServerHost - } else { + } else if ciTasks[i].JobType == string(models.JobTypeBrainScore) { ciTasks[i].BenchmarkTypeRankLink = setting.BrainScoreServerHost + } else { + ciTasks[i].BenchmarkTypeRankLink = setting.Snn4EcosetServerHost } } @@ -2536,7 +2571,6 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) displayJobName := form.DisplayJobName jobName := util.ConvertDisplayJobNameToJobName(displayJobName) image := form.Image - uuid := form.Attachment jobType := form.JobType codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath branchName := cloudbrain.DefaultBranchName @@ -2578,7 +2612,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) return } - if jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) { + if !models.IsModelBenchMarkJobType(jobType) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx, jobType) ctx.RenderWithErr("jobtype error", tpl, &form) @@ -2607,29 +2641,41 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) mkModelPath(modelPath) uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/") - snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath + benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { - downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "", ctx.User.Name) - uploadCodeToMinio(snn4imagenetPath+"/", jobName, cloudbrain.Snn4imagenetMountPath+"/") - command = fmt.Sprintf(cloudbrain.Snn4imagenetCommand, displayJobName, trimSpaceNewlineInString(form.Description)) + downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, benchmarkPath, "", "", ctx.User.Name) + uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/") + command = fmt.Sprintf(cloudbrain.Snn4imagenetCommand, displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description)) } benchmarkChildTypeID := 0 - brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath + if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { - downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "", ctx.User.Name) - uploadCodeToMinio(brainScorePath+"/", jobName, cloudbrain.BrainScoreMountPath+"/") + downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, benchmarkPath, "", "", ctx.User.Name) + uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/") benchmarkChildTypeID = form.BenchmarkChildTypeID - command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description)) + command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description)) } + var uuid string + var datasetInfos map[string]models.DatasetInfo + var datasetNames string + if setting.IsSnn4EcosetEnabled && jobType == string(models.JobTypeSnn4Ecoset) { + downloadRateCode(repo, jobName, setting.Snn4EcosetOwner, setting.Snn4EcosetName, benchmarkPath, "", "", ctx.User.Name) + uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/") + command = fmt.Sprintf(cloudbrain.Snn4EcosetCommand, displayJobName, form.CkptName, trimSpaceNewlineInString(form.Description)) + + attachment, err := getEcosetAttachment() + if err != nil { + log.Error("load benchmark code failed", err) + cloudBrainNewDataPrepare(ctx, jobType) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) + return + } + uuid = attachment.UUID + datasetInfos, datasetNames, _ = models.GetDatasetInfo(uuid) - datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid) - if err != nil { - log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx, jobType) - ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) - return } + spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{ JobType: models.JobTypeBenchmark, ComputeResource: models.GPU, @@ -2661,8 +2707,6 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) CodePath: storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"), ModelPath: storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"), BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), - Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), - BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), JobType: jobType, Description: form.Description, BranchName: branchName, @@ -2674,6 +2718,14 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), Spec: spec, } + if form.ModelName != "" { + req.ModelName = form.ModelName + req.LabelName = form.LabelName + req.CkptName = form.CkptName + req.ModelVersion = form.ModelVersion + req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl + req.PreTrainModelUrl = form.PreTrainModelUrl + } _, err = cloudbrain.GenerateTask(req) if err != nil { @@ -2685,6 +2737,21 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") } +func getEcosetAttachment() (*models.Attachment, error) { + ecosetRepo, err := models.GetRepositoryByOwnerAndName(setting.Snn4EcosetOwner, setting.Snn4EcosetName) + if err != nil { + return nil, err + } + + datasetInfo, err := models.GetDatasetByRepo(ecosetRepo) + if err != nil { + return nil, err + } + + return models.GetAttachmentByDatasetIdFileName(setting.Snn4AttachmentName, datasetInfo.ID) + +} + func getBrainRegion(benchmarkChildTypeID int) string { values := []string{"V1", "V2", "V4", "IT"} return values[benchmarkChildTypeID] @@ -2745,18 +2812,24 @@ func InferenceCloudBrainJobShow(ctx *context.Context) { cloudBrainShow(ctx, tplCloudBrainInferenceJobShow, models.JobTypeInference) } -func DownloadInferenceResultFile(ctx *context.Context) { +func DownloadGPUInferenceResultFile(ctx *context.Context) { var jobID = ctx.Params(":jobid") - var versionName = ctx.Query("version_name") - task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + task, err := models.GetCloudbrainByJobID(jobID) if err != nil { log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error()) return } - - allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, task.ResultUrl) - returnFileName := task.DisplayJobName + ".zip" - MinioDownloadManyFile(task.ResultUrl, ctx, returnFileName, allFile) + parentDir := ctx.Query("parentDir") + filePath := "jobs/" + task.JobName + "/result/" + parentDir + log.Info("prefix=" + filePath) + allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, filePath) + if err == nil { + returnFileName := task.DisplayJobName + ".zip" + MinioDownloadManyFile(filePath, ctx, returnFileName, allFile) + } else { + log.Info("error,msg=" + err.Error()) + ctx.ServerError("no file to download.", err) + } } func getInferenceJobCommand(form auth.CreateCloudBrainInferencForm) (string, error) { diff --git a/routers/repo/dataset.go b/routers/repo/dataset.go index 1f445492d..32c884b5e 100755 --- a/routers/repo/dataset.go +++ b/routers/repo/dataset.go @@ -373,6 +373,7 @@ func datasetMultiple(ctx *context.Context, opts *models.SearchDatasetOptions) { } data, err := json.Marshal(datasets) + log.Info("datakey", string(data)) if err != nil { log.Error("json.Marshal failed:", err.Error()) ctx.JSON(200, map[string]string{ diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 14db1a50d..74e2f06d6 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -53,6 +53,8 @@ const ( //NPU tplGrampusNotebookNPUNew base.TplName = "repo/grampus/notebook/npu/new" tplGrampusTrainJobNPUNew base.TplName = "repo/grampus/trainjob/npu/new" + //GCU + tplGrampusNotebookGCUNew base.TplName = "repo/grampus/notebook/gcu/new" ) func GrampusNotebookNew(ctx *context.Context) { @@ -61,6 +63,8 @@ func GrampusNotebookNew(ctx *context.Context) { processType := grampus.ProcessorTypeGPU if notebookType == 1 { processType = grampus.ProcessorTypeNPU + } else if notebookType == 2 { + processType = grampus.ProcessorTypeGCU } err := grampusNotebookNewDataPrepare(ctx, processType) if err != nil { @@ -69,8 +73,10 @@ func GrampusNotebookNew(ctx *context.Context) { } if processType == grampus.ProcessorTypeGPU { ctx.HTML(http.StatusOK, tplGrampusNotebookGPUNew) - } else { + } else if processType == grampus.ProcessorTypeNPU { ctx.HTML(http.StatusOK, tplGrampusNotebookNPUNew) + } else if processType == grampus.ProcessorTypeGCU { + ctx.HTML(http.StatusOK, tplGrampusNotebookGCUNew) } } @@ -117,6 +123,12 @@ func GrampusNotebookCreate(ctx *context.Context, form auth.CreateGrampusNotebook computeSource = models.NPUResource computeSourceSimple = models.NPU codeStoragePath = grampus.JobPath + jobName + modelarts.CodePath + } else if form.Type == 2 { + tpl = tplGrampusNotebookGCUNew + processType = grampus.ProcessorTypeGCU + computeSource = models.GCUResource + computeSourceSimple = models.GCU + codeStoragePath = setting.CBCodePathPrefix + jobName + cloudbrain.CodeMountPath + "/" } lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeDebug), displayJobName)) @@ -190,6 +202,7 @@ func GrampusNotebookCreate(ctx *context.Context, form auth.CreateGrampusNotebook var datasetInfos map[string]models.DatasetInfo var datasetNames string + var attachSize int64 //var if uuid != "" { datasetInfos, datasetNames, err = models.GetDatasetInfo(uuid, computeSourceSimple) @@ -199,6 +212,21 @@ func GrampusNotebookCreate(ctx *context.Context, form auth.CreateGrampusNotebook ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) return } + uuidArray := strings.Split(uuid, ";") + if datasetInfos == nil || len(datasetInfos) < len(uuidArray) { + grampusNotebookNewDataPrepare(ctx, processType) + ctx.RenderWithErr(ctx.Tr("cloudbrain.error.partial_datasets_not_available"), tpl, &form) + return + } + for _, infos := range datasetInfos { + attachSize += infos.Size + } + if attachSize > int64(setting.DebugAttachSize*1000*1000*1000) { + log.Error("The DatasetSize exceeds the limit (%dGB)", setting.DebugAttachSize) // GB + grampusNotebookNewDataPrepare(ctx, processType) + ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize", setting.DebugAttachSize), tpl, &form) + return + } } //prepare code and out path @@ -215,7 +243,7 @@ func GrampusNotebookCreate(ctx *context.Context, form auth.CreateGrampusNotebook return } - if processType == grampus.ProcessorTypeGPU { + if processType == grampus.ProcessorTypeGPU || processType == grampus.ProcessorTypeGCU { if err := uploadCodeToMinio(codeLocalPath+"/", jobName, cloudbrain.CodeMountPath+"/"); err != nil { log.Error("Failed to uploadCodeToMinio: %s (%v)", repo.FullName(), err, ctx.Data["MsgID"]) grampusNotebookNewDataPrepare(ctx, processType) @@ -255,20 +283,26 @@ func GrampusNotebookCreate(ctx *context.Context, form auth.CreateGrampusNotebook if form.ModelName != "" { //使用预训练模型训练 - _, err := models.QueryModelByPath(form.PreTrainModelUrl) + m, err := models.QueryModelByPath(form.PreTrainModelUrl) if err != nil { log.Error("Can not find model", err) grampusNotebookNewDataPrepare(ctx, processType) ctx.RenderWithErr(ctx.Tr("repo.modelconvert.manage.model_not_exist"), tpl, &form) return } + if !cloudbrainTask.IsModelFileExists(m, form.CkptName) { + log.Error("model file not exist.name = %s", form.CkptName) + grampusNotebookNewDataPrepare(ctx, processType) + ctx.RenderWithErr(ctx.Tr("repo.modelconvert.manage.model_file_not_exist"), tpl, &form) + return + } req.ModelName = form.ModelName req.LabelName = form.LabelName req.CkptName = form.CkptName req.ModelVersion = form.ModelVersion req.PreTrainModelUrl = form.PreTrainModelUrl req.PreTrainModelPath = getPreTrainModelPath(form.PreTrainModelUrl, form.CkptName) - + req.ModelStorageType = m.Type } _, err = grampus.GenerateNotebookJob(ctx, req) @@ -287,7 +321,7 @@ func grampusNotebookNewDataPrepare(ctx *context.Context, processType string) err ctx.Data["display_job_name"] = displayJobName //get valid images - if processType == grampus.ProcessorTypeNPU { + if processType == grampus.ProcessorTypeNPU || processType == grampus.ProcessorTypeGCU { images, err := grampus.GetImages(processType, string(models.JobTypeDebug)) if err != nil { log.Error("GetImages failed:", err.Error()) @@ -303,6 +337,10 @@ func grampusNotebookNewDataPrepare(ctx *context.Context, processType string) err computeResourceSimple = models.NPU datasetType = models.TypeCloudBrainTwo computeResource = models.NPUResource + } else if processType == grampus.ProcessorTypeGCU { + computeResourceSimple = models.GCU + datasetType = models.TypeCloudBrainAll + computeResource = models.GCUResource } prepareGrampusSpecs(ctx, computeResourceSimple, models.JobTypeDebug) @@ -1239,7 +1277,7 @@ func GrampusTrainJobShow(ctx *context.Context) { return } task.ContainerIp = "" - + task.User, _ = models.GetUserByID(task.UserID) if task.DeletedAt.IsZero() { //normal record result, err := grampus.GetJob(task.JobID) if err != nil { @@ -1308,6 +1346,7 @@ func GrampusTrainJobShow(ctx *context.Context) { taskList := make([]*models.Cloudbrain, 0) taskList = append(taskList, task) prepareSpec4Show(ctx, task) + ctx.Data["version_list_task"] = taskList ctx.Data["datasetDownload"] = GetCloudBrainDataSetInfo(task.Uuid, task.DatasetName, false) ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task) @@ -1348,16 +1387,39 @@ func GrampusGetLog(ctx *context.Context) { return } + result, err := grampus.GetJob(jobID) + if err != nil { + log.Error("GetJob(%s) failed:%v", job.JobName, err) + ctx.JSON(http.StatusOK, map[string]interface{}{ + "JobName": job.JobName, + "Content": "", + "CanLogDownload": false, + }) + return + } + exitDiagnostics := "" + if result != nil { + exitDiagnostics = result.ExitDiagnostics + } + content, err := grampus.GetTrainJobLog(job.JobID) if err != nil { log.Error("GetTrainJobLog failed: %v", err, ctx.Data["MsgID"]) ctx.JSON(http.StatusOK, map[string]interface{}{ "JobName": job.JobName, - "Content": "", + "Content": exitDiagnostics, "CanLogDownload": false, }) return } + + if result != nil { + job.Status = grampus.TransTrainJobStatus(result.JobInfo.Status) + if job.Status == models.GrampusStatusFailed { + content = content + "\n" + exitDiagnostics + } + } + canLogDownload := err == nil && job.IsUserHasRight(ctx.User) ctx.JSON(http.StatusOK, map[string]interface{}{ "JobName": job.JobName, @@ -1636,7 +1698,11 @@ func GrampusNotebookRestart(ctx *context.Context) { if task.ComputeResource == models.NPUResource { computeSourceSimple = models.NPU action = models.ActionCreateGrampusNPUDebugTask + } else if task.ComputeResource == models.GCUResource { + computeSourceSimple = models.GCU + action = models.ActionCreateGrampusGCUDebugTask } + spec, err = resource.GetAndCheckSpec(ctx.User.ID, oldSpec.ID, models.FindSpecsOptions{ JobType: models.JobType(task.JobType), ComputeResource: computeSourceSimple, @@ -1652,7 +1718,7 @@ func GrampusNotebookRestart(ctx *context.Context) { errorMsg = ctx.Tr("points.insufficient_points_balance") break } - if task.IsGPUTask() { + if task.IsGPUTask() || task.IsGCUTask() { if _, err := os.Stat(getOldJobPath(task)); err != nil { log.Error("Can not find job minio path", err) resultCode = "-1" @@ -1682,6 +1748,10 @@ func GrampusNotebookRestart(ctx *context.Context) { if res.GrampusResult.ErrorCode != 0 || res.NewId == "" { log.Error("ManageNotebook2 failed:" + res.GrampusResult.ErrorMsg) errorMsg = ctx.Tr("repo.debug_again_fail") + if res.GrampusResult.ErrorCode == 5005 { + errorMsg = ctx.Tr("repo.debug_again_fail_forever") + } + break } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 3dbe101a8..86fbd5bdb 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -218,6 +218,22 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm return } } + + var datasetInfos map[string]models.DatasetInfo + var attachSize int64 + if uuid != "" { + datasetInfos, _, err = models.GetDatasetInfo(uuid) + for _, infos := range datasetInfos { + attachSize += infos.Size + } + if attachSize > int64(setting.DebugAttachSize*1000*1000*1000) { + log.Error("The DatasetSize exceeds the limit (%dGB)", setting.DebugAttachSize) //GB + notebookNewDataPrepare(ctx) + ctx.RenderWithErr(ctx.Tr("cloudbrain.error.debug_datasetsize", setting.DebugAttachSize), tplModelArtsNotebookNew, &form) + return + } + } + var aiCenterCode = models.AICenterOfCloudBrainTwo if setting.ModelartsCD.Enabled { aiCenterCode = models.AICenterOfChengdu @@ -439,9 +455,13 @@ func NotebookDebug2(ctx *context.Context) { ctx.RenderWithErr(err.Error(), tplModelArtsNotebookIndex, nil) return } - if task.BootFile != "" { - ctx.Redirect(getFileUrl(result.Url, task.BootFile) + "?token=" + result.Token) + + if ctx.QueryTrim("file") != "" { + ctx.Redirect(getFileUrl(result.Url, ctx.QueryTrim("file")) + "&token=" + result.Token) } else { + if task.BootFile != "" { + go cloudbrainTask.UploadNotebookFiles(task) + } ctx.Redirect(result.Url + "?token=" + result.Token) } @@ -463,7 +483,7 @@ func getFileUrl(url string, filename string) string { } } - return url + middle + path.Base(filename) + return url + middle + filename + "?reset" } func NotebookRestart(ctx *context.Context) { @@ -630,7 +650,7 @@ func NotebookStop(ctx *context.Context) { if task.Status != string(models.ModelArtsRunning) { log.Error("the job(%s) is not running", task.JobName, ctx.Data["MsgID"]) resultCode = "-1" - errorMsg = "the job is not running" + errorMsg = ctx.Tr("cloudbrain.Already_stopped") break } @@ -2589,7 +2609,8 @@ func inferenceJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModel ctx.Data["datasetType"] = models.TypeCloudBrainTwo waitCount := cloudbrain.GetWaitingCloudbrainCount(models.TypeCloudBrainTwo, "") ctx.Data["WaitCount"] = waitCount - + NotStopTaskCount, _ := cloudbrainTask.GetNotFinalStatusTaskCount(ctx.User.ID, models.TypeCloudBrainTwo, string(models.JobTypeInference)) + ctx.Data["NotStopTaskCount"] = NotStopTaskCount return nil } func InferenceJobShow(ctx *context.Context) { @@ -2653,6 +2674,46 @@ func InferenceJobShow(ctx *context.Context) { ctx.HTML(http.StatusOK, tplModelArtsInferenceJobShow) } +func MultiModelDownload(ctx *context.Context) { + var ( + err error + ) + jobID := ctx.Params(":jobid") + versionName := ctx.Query("version_name") + parentDir := ctx.Query("parent_dir") + + task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName) + if err != nil { + log.Error("GetCloudbrainByJobIDAndVersionName(%s) failed:%v", task.JobName, err.Error()) + return + } + + if task.ComputeResource == models.NPUResource { + path := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName, parentDir), "/") + path = strings.TrimSuffix(path, "/") + path += "/" + allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path) + if err == nil { + returnFileName := task.DisplayJobName + ".zip" + ObsDownloadManyFile(path, ctx, returnFileName, allFile) + } else { + log.Info("error,msg=" + err.Error()) + ctx.ServerError("no file to download.", err) + } + } else if task.ComputeResource == models.GPUResource { + filePath := setting.CBCodePathPrefix + task.JobName + cloudbrain.ModelMountPath + "/" + parentDir + allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, filePath) + if err == nil { + returnFileName := task.DisplayJobName + ".zip" + MinioDownloadManyFile(filePath, ctx, returnFileName, allFile) + } else { + log.Info("error,msg=" + err.Error()) + ctx.ServerError("no file to download.", err) + } + } + +} + func ModelDownload(ctx *context.Context) { var ( err error @@ -2832,15 +2893,19 @@ func TrainJobDownloadLogFile(ctx *context.Context) { ctx.ServerError("GetObsLogFileName", err) return } - - url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, key) - if err != nil { - log.Error("GetObsCreateSignedUrlByBucketAndKey failed: %v", err.Error(), ctx.Data["msgID"]) - ctx.ServerError("GetObsCreateSignedUrlByBucketAndKey", err) - return + if len(key) > 1 { + ObsDownloadManyFile(prefix[0:len(prefix)-3], ctx, task.DisplayJobName+".zip", key) + } else { + url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, key[0].ParenDir+key[0].FileName) + if err != nil { + log.Error("GetObsCreateSignedUrlByBucketAndKey failed: %v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetObsCreateSignedUrlByBucketAndKey", err) + return + } + ctx.Resp.Header().Set("Cache-Control", "max-age=0") + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect) } - ctx.Resp.Header().Set("Cache-Control", "max-age=0") - http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusTemporaryRedirect) + } func getDatasUrlListByUUIDS(uuidStr string) ([]models.Datasurl, string, string, bool, error) { var isMultiDataset bool diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index c1a7954a7..c889046e3 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -75,7 +75,7 @@ func RepoStatisticDaily(date string) { if repo.NumIssues != 0 { issueFixedRate = float32(repo.NumClosedIssues) / float32(repo.NumIssues) } else { - issueFixedRate = 1.0 + issueFixedRate = float32(setting.RadarMap.ProjectHealth0IssueCloseRatio) } var numVersions int64 @@ -124,7 +124,7 @@ func RepoStatisticDaily(date string) { NumDevMonths: numDevMonths, RepoSize: repo.Size, DatasetSize: datasetSize, - NumModels: 0, + NumModels: repo.ModelCnt, NumWikiViews: numWikiViews, NumCommits: numCommits, NumIssues: int64(repo.NumIssues), @@ -135,6 +135,9 @@ func RepoStatisticDaily(date string) { NumCommitsGrowth: numCommitsGrowth, NumCommitLinesGrowth: numCommitLinesGrowth, NumContributorsGrowth: numContributorsGrowth, + NumCloudbrain: repo.AiTaskCnt, + NumDatasetFile: repo.DatasetCnt, + NumModelConvert: models.QueryModelConvertCountByRepoID(repo.ID), } dayBeforeDate := t.AddDate(0, 0, -1).Format("2006-01-02") @@ -155,6 +158,10 @@ func RepoStatisticDaily(date string) { repoStat.NumIssuesAdded = repoStat.NumIssues - repoStatisticBefore.NumIssues repoStat.NumPullsAdded = repoStat.NumPulls - repoStatisticBefore.NumPulls repoStat.NumContributorAdded = repoStat.NumContributor - repoStatisticBefore.NumContributor + repoStat.NumModelsAdded = repoStat.NumModels - repoStatisticBefore.NumModels + repoStat.NumCloudbrainAdded = repoStat.NumCloudbrain - repoStatisticBefore.NumCloudbrain + repoStat.NumModelConvertAdded = repoStat.NumModelConvert - repoStatisticBefore.NumModelConvert + repoStat.NumDatasetFileAdded = repoStat.NumDatasetFile - repoStatisticBefore.NumDatasetFile } } day4MonthsAgo := t.AddDate(0, -4, 0) diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 063a20999..2def76ff4 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -359,6 +359,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/all/dosearch/", routers.SearchApi) m.Post("/user/login/kanban", user.SignInPostAPI) m.Get("/home/term", routers.HomeTerm) + m.Get("/home/annual_privacy", routers.HomeAnnual) m.Get("/home/notice", routers.HomeNoticeTmpl) m.Get("/home/privacy", routers.HomePrivacy) m.Get("/extension/tuomin/upload", modelapp.ProcessImageUI) @@ -1186,6 +1187,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate) m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) m.Get("/download_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadModel) + m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRight, repo.CloudBrainDownloadMultiModel) }) m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.CloudBrainNew) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) @@ -1209,6 +1211,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainTrainJobDel) //m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainDownloadModel) + m.Get("/download_multi_model", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.CloudBrainDownloadMultiModel) //m.Get("/get_log", cloudbrain.AdminOrJobCreaterRightForTrain, repo.GetLogFromModelDir) //m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainTrainJobVersionNew) @@ -1221,8 +1224,8 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/:jobid", func() { m.Get("", reqRepoCloudBrainReader, repo.InferenceCloudBrainJobShow) m.Get("/result_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainDownloadInferenceResult) - - m.Get("/downloadall", repo.DownloadInferenceResultFile) + m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRightForTrain, repo.CloudBrainDownloadMultiModel) + m.Get("/downloadall", cloudbrain.AdminOrJobCreaterRightForTrain, repo.DownloadGPUInferenceResultFile) }) m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.InferenceCloudBrainJobNew) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainInferencForm{}), repo.CloudBrainInferenceJobCreate) @@ -1248,6 +1251,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.GrampusStopJob) m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.GrampusTrainJobDel) m.Get("/model_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ModelDownload) + m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRightForTrain, repo.MultiModelDownload) m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, repo.GrampusTrainJobVersionNew) m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateGrampusTrainJobForm{}), repo.GrampusTrainJobVersionCreate) }) @@ -1333,6 +1337,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.TrainJobStop) m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.TrainJobDel) m.Get("/model_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ModelDownload) + m.Get("/download_multi_model", cloudbrain.AdminOrJobCreaterRightForTrain, repo.MultiModelDownload) m.Get("/download_log_file", cloudbrain.AdminOrJobCreaterRightForTrain, repo.TrainJobDownloadLogFile) m.Get("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, context.PointAccount(), repo.TrainJobNewVersion) m.Post("/create_version", reqWechatBind, cloudbrain.AdminOrJobCreaterRightForTrain, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) @@ -1348,7 +1353,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/:jobid", func() { m.Get("", reqRepoCloudBrainReader, repo.InferenceJobShow) m.Get("/result_download", cloudbrain.AdminOrJobCreaterRightForTrain, repo.ResultDownload) - m.Get("/downloadall", repo.DownloadMultiResultFile) + m.Get("/downloadall", cloudbrain.AdminOrJobCreaterRightForTrain, repo.DownloadMultiResultFile) }) m.Get("/create", reqWechatBind, reqRepoCloudBrainWriter, context.PointAccount(), repo.InferenceJobNew) m.Post("/create", reqWechatBind, reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsInferenceJobForm{}), repo.InferenceJobCreate) diff --git a/routers/user/auth.go b/routers/user/auth.go index 5314571d2..61f53e200 100755 --- a/routers/user/auth.go +++ b/routers/user/auth.go @@ -36,6 +36,7 @@ import ( "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/externalaccount" "code.gitea.io/gitea/services/mailer" + "code.gitea.io/gitea/services/repository" "gitea.com/macaron/captcha" "github.com/markbates/goth" @@ -120,7 +121,6 @@ func checkAutoLogin(ctx *context.Context) bool { ctx.ServerError("AutoSignIn", err) return true } - redirectTo := ctx.Query("redirect_to") if len(redirectTo) > 0 { ctx.SetCookie("redirect_to", redirectTo, 0, setting.AppSubURL, "", setting.SessionConfig.Secure, true) @@ -129,7 +129,6 @@ func checkAutoLogin(ctx *context.Context) bool { } if isSucceed { - isCourse := ctx.QueryBool("course") ctx.SetCookie("redirect_to", "", -1, setting.AppSubURL, "", setting.SessionConfig.Secure, true) if redirectTo == "" && isCourse { @@ -137,7 +136,6 @@ func checkAutoLogin(ctx *context.Context) bool { ctx.RedirectToFirst(redirectToCourse) } else { ctx.RedirectToFirst(redirectTo, setting.AppSubURL+string(setting.LandingPageURL)) - } return true } @@ -145,10 +143,14 @@ func checkAutoLogin(ctx *context.Context) bool { return false } +func getActivityTpl() string { + result, _ := repository.RecommendContentFromPromote(setting.RecommentRepoAddr + "/signin/activity_tpl") + return result +} + // SignIn render sign in page func SignIn(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("sign_in") - // Check auto-login. if checkAutoLogin(ctx) { return @@ -168,6 +170,7 @@ func SignIn(ctx *context.Context) { ctx.Data["PageIsLogin"] = true ctx.Data["EnableSSPI"] = models.IsSSPIEnabled() ctx.Data["EnableCloudBrain"] = true + ctx.Data["ActivityTpl"] = getActivityTpl() ctx.HTML(200, tplSignIn) } @@ -185,6 +188,7 @@ func SignInCloudBrain(ctx *context.Context) { ctx.Data["PageIsSignIn"] = true ctx.Data["PageIsCloudBrainLogin"] = true ctx.Data["EnableCloudBrain"] = true + ctx.Data["ActivityTpl"] = getActivityTpl() ctx.HTML(200, tplSignInCloudBrain) } @@ -197,6 +201,7 @@ func SignInPhone(ctx *context.Context) { } ctx.Data["PageIsPhoneLogin"] = true + ctx.Data["ActivityTpl"] = getActivityTpl() ctx.HTML(200, tplSignInPhone) } @@ -206,6 +211,7 @@ func SignInPhonePost(ctx *context.Context, form auth.PhoneNumberCodeForm) { ctx.Data["PageIsPhoneLogin"] = true ctx.Data["IsCourse"] = ctx.QueryBool("course") ctx.Data["EnableCloudBrain"] = true + ctx.Data["ActivityTpl"] = getActivityTpl() if ctx.HasError() { ctx.HTML(200, tplSignInPhone) @@ -270,7 +276,6 @@ func SignInPostAPI(ctx *context.Context) { func SignInPostCommon(ctx *context.Context, form auth.SignInForm) { ctx.Data["Title"] = ctx.Tr("sign_in") - orderedOAuth2Names, oauth2Providers, err := models.GetActiveOAuth2Providers() if err != nil { ctx.ServerError("UserSignIn", err) @@ -285,7 +290,6 @@ func SignInPostCommon(ctx *context.Context, form auth.SignInForm) { ctx.Data["IsCourse"] = ctx.QueryBool("course") ctx.Data["EnableSSPI"] = models.IsSSPIEnabled() ctx.Data["EnableCloudBrain"] = true - if ctx.HasError() { ctx.HTML(200, tplSignIn) return @@ -356,6 +360,7 @@ func SignInPostCommon(ctx *context.Context, form auth.SignInForm) { func SignInCloudBrainPost(ctx *context.Context, form auth.SignInForm) { ctx.Data["PageIsCloudBrainLogin"] = true ctx.Data["SignInLink"] = setting.AppSubURL + "/user/login/cloud_brain" + ctx.Data["ActivityTpl"] = getActivityTpl() SignInPostCommon(ctx, form) } @@ -363,6 +368,7 @@ func SignInCloudBrainPost(ctx *context.Context, form auth.SignInForm) { func SignInPost(ctx *context.Context, form auth.SignInForm) { ctx.Data["PageIsLogin"] = true ctx.Data["SignInLink"] = setting.AppSubURL + "/user/login" + ctx.Data["ActivityTpl"] = getActivityTpl() SignInPostCommon(ctx, form) } @@ -757,7 +763,6 @@ func handleSignInFull(ctx *context.Context, u *models.User, remember bool, obeyR } return redirectTo } - if obeyRedirect { ctx.Redirect(setting.AppSubURL + "/dashboard") } @@ -1257,6 +1262,7 @@ func SignUp(ctx *context.Context) { //Show Disabled Registration message if DisableRegistration or AllowOnlyExternalRegistration options are true ctx.Data["DisableRegistration"] = setting.Service.DisableRegistration || setting.Service.AllowOnlyExternalRegistration + ctx.Data["ActivityTpl"] = getActivityTpl() ctx.HTML(200, tplSignUp) } @@ -1272,6 +1278,7 @@ func SignUpPost(ctx *context.Context, cpt *captcha.Captcha, form auth.RegisterFo ctx.Data["CaptchaType"] = setting.Service.CaptchaType ctx.Data["RecaptchaSitekey"] = setting.Service.RecaptchaSitekey ctx.Data["PageIsSignUp"] = true + ctx.Data["ActivityTpl"] = getActivityTpl() //Permission denied if DisableRegistration or AllowOnlyExternalRegistration options are true if setting.Service.DisableRegistration || setting.Service.AllowOnlyExternalRegistration { diff --git a/routers/user/home.go b/routers/user/home.go index 62b0357ad..e4ff50a45 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -779,7 +779,7 @@ func Cloudbrains(ctx *context.Context) { var jobTypes []string jobTypeNot := false if jobType == string(models.JobTypeBenchmark) { - jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) + jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)) } else if jobType != "all" && jobType != "" { jobTypes = append(jobTypes, jobType) } diff --git a/services/cloudbrain/clear.go b/services/cloudbrain/clear.go index 44613ee3c..e14414c74 100644 --- a/services/cloudbrain/clear.go +++ b/services/cloudbrain/clear.go @@ -14,21 +14,21 @@ import ( func ClearCloudbrainResultSpace() { log.Info("clear cloudbrain one result space begin.") - if !setting.ClearStrategy.Enabled{ + if !setting.ClearStrategy.Enabled { return } - tasks, err := models.GetCloudBrainOneStoppedNotDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.BatchSize) + tasks, err := models.GetGPUStoppedNotDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.BatchSize) if err != nil { log.Warn("Failed to get cloudbrain, clear result failed.", err) return } - debugTasks, err := models.GetCloudBrainOneStoppedDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.DebugJobSize) + debugTasks, err := models.GetGPUStoppedDebugJobDaysAgo(setting.ClearStrategy.ResultSaveDays, setting.ClearStrategy.DebugJobSize) if err != nil { log.Warn("Failed to get debug cloudbrain.", err) } - tasks=append(tasks,debugTasks...) + tasks = append(tasks, debugTasks...) if err != nil { log.Warn("Failed to get cloudbrain, clear result failed.", err) @@ -38,7 +38,7 @@ func ClearCloudbrainResultSpace() { for _, task := range tasks { err := DeleteCloudbrainOneJobStorage(task.JobName) if err == nil { - log.Info("clear job in cloudbrain table:"+task.JobName) + log.Info("clear job in cloudbrain table:" + task.JobName) ids = append(ids, task.ID) } } @@ -69,10 +69,10 @@ func clearMinioHistoryTrashFile() { SortModTimeAscend(miniofiles) for _, file := range miniofiles { - if file.Name()!="" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { + if file.Name() != "" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { - has,err:=models.IsCloudbrainExistByJobName(file.Name()) - if err==nil && !has { + has, err := models.IsCloudbrainExistByJobName(file.Name()) + if err == nil && !has { dirPath := setting.CBCodePathPrefix + file.Name() + "/" log.Info("clear job in minio trash:" + file.Name()) storage.Attachments.DeleteDir(dirPath) @@ -90,7 +90,7 @@ func clearMinioHistoryTrashFile() { } } -func clearLocalHistoryTrashFile() { +func clearLocalHistoryTrashFile() { files, err := ioutil.ReadDir(setting.JobPath) processCount := 0 if err != nil { @@ -99,11 +99,11 @@ func clearLocalHistoryTrashFile() { SortModTimeAscend(files) for _, file := range files { //清理n天前的历史垃圾数据，清理job目录 - if file.Name()!="" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { - has,err:=models.IsCloudbrainExistByJobName(file.Name()) - if err==nil && !has{ + if file.Name() != "" && file.ModTime().Before(time.Now().AddDate(0, 0, -setting.ClearStrategy.TrashSaveDays)) { + has, err := models.IsCloudbrainExistByJobName(file.Name()) + if err == nil && !has { os.RemoveAll(setting.JobPath + file.Name()) - log.Info("clear job in local trash:"+file.Name()) + log.Info("clear job in local trash:" + file.Name()) processCount++ } if processCount == setting.ClearStrategy.BatchSize { @@ -127,7 +127,7 @@ func SortModTimeAscend(files []os.FileInfo) { func DeleteCloudbrainOneJobStorage(jobName string) error { - if jobName==""{ + if jobName == "" { return nil } //delete local diff --git a/services/cloudbrain/cloudbrainTask/ai_model.go b/services/cloudbrain/cloudbrainTask/ai_model.go new file mode 100644 index 000000000..02cc392be --- /dev/null +++ b/services/cloudbrain/cloudbrainTask/ai_model.go @@ -0,0 +1,30 @@ +package cloudbrainTask + +import ( + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" +) + +func IsModelFileExists(model *models.AiModelManage, fileName string) bool { + if model.Type == models.TypeCloudBrainTwo { + key := models.AIModelPath + models.AttachmentRelativePath(model.ID) + "/" + fileName + log.Info("IsModelFileExists TypeCloudBrainTwo key=%s", key) + isExist, err := storage.IsObjectExist4Obs(setting.Bucket, key) + if err != nil { + return false + } + return isExist + } else if model.Type == models.TypeCloudBrainOne { + prefix := models.AIModelPath + models.AttachmentRelativePath(model.ID) + "/" + objectName := prefix + fileName + log.Info("IsModelFileExists TypeCloudBrainOne objectName=%s", objectName) + isExist, err := storage.IsObjectExist4Minio(setting.Attachment.Minio.Bucket, objectName) + if err != nil { + return false + } + return isExist + } + return false +} diff --git a/services/cloudbrain/cloudbrainTask/count.go b/services/cloudbrain/cloudbrainTask/count.go index 4ae742c3a..0010164ae 100644 --- a/services/cloudbrain/cloudbrainTask/count.go +++ b/services/cloudbrain/cloudbrainTask/count.go @@ -34,7 +34,7 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s ComputeResource: models.GPUResource, }, string(models.JobTypeBenchmark) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { CloudBrainTypes: []int{models.TypeCloudBrainOne}, - JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet}, + JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet, models.JobTypeSnn4Ecoset}, NotFinalStatuses: CloudbrainOneNotFinalStatuses, ComputeResource: models.GPUResource, }, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { @@ -72,11 +72,16 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s JobType: []models.JobType{models.JobTypeDebug}, NotFinalStatuses: GrampusNotFinalStatuses, ComputeResource: models.NPUResource, +}, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeC2Net) + "-" + models.GCUResource: { + CloudBrainTypes: []int{models.TypeC2Net}, + JobType: []models.JobType{models.JobTypeDebug}, + NotFinalStatuses: GrampusNotFinalStatuses, + ComputeResource: models.GCUResource, }} func GetNotFinalStatusTaskCount(uid int64, cloudbrainType int, jobType string, computeResource ...string) (int, error) { jobNewType := jobType - if jobType == string(models.JobTypeSnn4imagenet) || jobType == string(models.JobTypeBrainScore) { + if models.IsModelBenchMarkJobType(jobType) { jobNewType = string(models.JobTypeBenchmark) } diff --git a/services/cloudbrain/cloudbrainTask/notebook.go b/services/cloudbrain/cloudbrainTask/notebook.go index cc9563520..ff1a2100a 100644 --- a/services/cloudbrain/cloudbrainTask/notebook.go +++ b/services/cloudbrain/cloudbrainTask/notebook.go @@ -4,6 +4,9 @@ import ( "fmt" "net/http" "path" + "strings" + + "code.gitea.io/gitea/modules/notebook" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/modelarts_cd" @@ -29,6 +32,10 @@ import ( ) const NoteBookExtension = ".ipynb" +const CPUType = 0 +const GPUType = 1 +const NPUType = 2 +const CharacterLength = 2550 func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption) { @@ -40,6 +47,14 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_select_wrong"))) return } + if len(getBootFile(option.File, option.OwnerName, option.ProjectName)) > CharacterLength { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_path_too_long"))) + return + } + if len(option.BranchName) > CharacterLength { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_branch_name_too_long"))) + return + } isNotebookFileExist, _ := isNoteBookFileExist(ctx, option) if !isNotebookFileExist { @@ -66,7 +81,7 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp } //create repo if not exist - repo, err := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) + repo, _ := models.GetRepositoryByName(ctx.User.ID, setting.FileNoteBook.ProjectName) if repo == nil { repo, err = repo_service.CreateRepository(ctx.User, ctx.User, models.CreateRepoOptions{ Name: setting.FileNoteBook.ProjectName, @@ -80,19 +95,237 @@ func FileNotebookCreate(ctx *context.Context, option api.CreateFileNotebookJobOp AutoInit: true, DefaultBranch: "master", }) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.failed_to_create_notebook_repo", setting.FileNoteBook.ProjectName))) + return + } + } else { + + noteBook, _ := models.GetWaitOrRunFileNotebookByRepo(repo.ID, getCloudbrainType(option.Type)) + if noteBook != nil { + + if isRepoConfilcts(option, noteBook) { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_repo_conflict"))) + return + } + + if isNotebookSpecMath(option, noteBook) { + if !isRepoMatch(option, noteBook) { + err = downloadCode(sourceRepo, getCodePath(noteBook.JobName, sourceRepo), option.BranchName) + if err != nil { + log.Error("download code failed", err) + if !strings.Contains(err.Error(), "already exists and is not an empty directory") { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) + return + } + } + } + if !isRepoFileMatch(option, noteBook) { + if len(noteBook.BootFile)+len(getBootFile(option.File, option.OwnerName, option.ProjectName))+1 <= CharacterLength { + noteBook.BootFile += ";" + getBootFile(option.File, option.OwnerName, option.ProjectName) + } else { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_path_too_long"))) + return + } + if len(noteBook.BranchName)+len(option.BranchName)+1 <= CharacterLength { + noteBook.BranchName += ";" + option.BranchName + } else { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_branch_name_too_long"))) + return + } + + if len(noteBook.Description)+len(getDescription(option))+1 <= CharacterLength { + noteBook.Description += ";" + getDescription(option) + } + + err := models.UpdateJob(noteBook) + if err != nil { + log.Error("GenerateNotebook2 failed, %v", err, ctx.Data["MsgID"]) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + } + + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 0, + Message: noteBook.JobID, + }) + return + } + + } + } + + if option.Type <= GPUType { + cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo) + } else { + modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo) + } + +} +func FileNotebookStatus(ctx *context.Context, option api.CreateFileNotebookJobOption) { + if ctx.Written() { + return } + + if path.Ext(option.File) != NoteBookExtension { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_select_wrong"))) + return + } + + isNotebookFileExist, _ := isNoteBookFileExist(ctx, option) + if !isNotebookFileExist { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + return + } + + task, err := models.GetCloudbrainByJobID(option.JobId) if err != nil { - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.failed_to_create_notebook_repo", setting.FileNoteBook.ProjectName))) + log.Error("job not found:"+option.JobId, err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Job id may not be right. can not find job.")) return } - if option.Type <= 1 { - cloudBrainFileNoteBookCreate(ctx, option, repo, sourceRepo) + if task.BootFile == "" || task.Status != string(models.ModelArtsRunning) { + log.Warn("Boot file is empty or status is running. ") + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("Boot file is empty or status is running.")) + return + } + if !isRepoFileMatch(option, task) { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("can not math repo file.")) + return + } + debugBaseUrl, token, err := getBaseUrlAndToken(task) + if err != nil { + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(err.Error())) + return + } + + if uploadNotebookFileIfCannotBroswer(debugBaseUrl, getBootFile(option.File, option.OwnerName, option.ProjectName), task, token) { + ctx.JSON(http.StatusOK, models.BaseOKMessageApi) } else { - modelartsFileNoteBookCreate(ctx, option, repo, sourceRepo) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi("upload failed.")) + } } +func getBaseUrlAndToken(task *models.Cloudbrain) (string, string, error) { + var debugBaseUrl string + var token string + if task.Type == models.TypeCloudBrainOne { + debugBaseUrl = setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName + "/lab" + + } else { + var result *models.GetNotebook2Result + var err error + if task.Type == models.TypeCloudBrainTwo { + result, err = modelarts.GetNotebook2(task.JobID) + } else if task.Type == models.TypeCDCenter { + result, err = modelarts_cd.GetNotebook(task.JobID) + } + if err != nil || result == nil || result.Status != string(models.ModelArtsRunning) || result.Url == "" { + log.Error("notebook job not found:"+task.JobID, err) + return "", "", fmt.Errorf("can not get job or job is invalid.") + } + + debugBaseUrl = result.Url + token = result.Token + + } + return debugBaseUrl, token, nil +} + +func uploadNotebookFileIfCannotBroswer(debugBaseUrl string, bootFile string, task *models.Cloudbrain, token string) bool { + c := ¬ebook.NotebookContent{ + Url: debugBaseUrl, + Path: bootFile, + PathType: "file", + Token: token, + } + if c.IsNotebookFileCanBrowser() { + return true + } else { + c.SetCookiesAndCsrf() + c.UploadNoteBookFile(task) + return c.IsNotebookFileCanBrowser() + } + +} + +func isNotebookSpecMath(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + if option.Type == NPUType || option.Type == CPUType { + return true + } + spec, err := models.GetCloudbrainSpecByID(book.ID) + if err != nil { + log.Warn("can not get spec ", err) + return false + } + return spec.AccCardsNum > 0 +} + +func isRepoConfilcts(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + bootFiles := strings.Split(book.BootFile, ";") + branches := strings.Split(book.BranchName, ";") + + for i, bootFile := range bootFiles { + splits := strings.Split(bootFile, "/") + if len(splits) >= 3 { + if splits[0] == option.OwnerName && splits[1] == option.ProjectName && branches[i] != option.BranchName { + return true + } + } + } + + return false + +} + +func isRepoMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + bootFiles := strings.Split(book.BootFile, ";") + + for _, bootFile := range bootFiles { + splits := strings.Split(bootFile, "/") + if len(splits) >= 3 { + if splits[0] == option.OwnerName && splits[1] == option.ProjectName { + return true + } + } + } + return false + +} + +func isRepoFileMatch(option api.CreateFileNotebookJobOption, book *models.Cloudbrain) bool { + bootFiles := strings.Split(book.BootFile, ";") + branches := strings.Split(book.BranchName, ";") + + for i, bootFile := range bootFiles { + if branches[i] == option.BranchName && getBootFile(option.File, option.OwnerName, option.ProjectName) == bootFile { + return true + } + } + + return false + +} +func UploadNotebookFiles(task *models.Cloudbrain) { + if task.Status == string(models.JobRunning) && task.BootFile != "" { + + debugBaseUrl, token, err := getBaseUrlAndToken(task) + if err != nil { + log.Error("can not get base url:", err) + return + } + bootFiles := strings.Split(task.BootFile, ";") + + for _, bootFile := range bootFiles { + uploadNotebookFileIfCannotBroswer(debugBaseUrl, bootFile, task, token) + } + + } +} + func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) { displayJobName := cloudbrainService.GetDisplayJobName(ctx.User.Name) @@ -131,17 +364,18 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - ctx.JSON(http.StatusOK,models.BaseMessageApi{ - Code: 2, + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 2, Message: ctx.Tr("repo.cloudbrain.morethanonejob"), }) return } } - errStr := uploadCodeFile(sourceRepo, getCodePath(jobName), option.BranchName, option.File, jobName) - if errStr != "" { - ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("repo.notebook_file_not_exist"))) + err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName) + if err != nil { + log.Error("download code failed", err) + ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) return } command := cloudbrain.GetCloudbrainDebugCommand() @@ -185,7 +419,7 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot JobType: jobType, Description: getDescription(option), BranchName: option.BranchName, - BootFile: option.File, + BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName), Params: "{\"parameter\":[]}", CommitID: "", BenchmarkTypeID: 0, @@ -206,12 +440,26 @@ func cloudBrainFileNoteBookCreate(ctx *context.Context, option api.CreateFileNot } -func getCodePath(jobName string) string { - return setting.JobPath + jobName + cloudbrain.CodeMountPath +func getCloudbrainType(optionType int) int { + if optionType <= GPUType { + return models.TypeCloudBrainOne + } + if setting.ModelartsCD.Enabled { + return models.TypeCDCenter + } + return models.TypeCloudBrainTwo +} + +func getCodePath(jobName string, repo *models.Repository) string { + return setting.JobPath + jobName + cloudbrain.CodeMountPath + "/" + repo.OwnerName + "/" + repo.Name } func getDescription(option api.CreateFileNotebookJobOption) string { - return option.OwnerName + "/" + option.ProjectName + "/" + option.File + des := option.OwnerName + "/" + option.ProjectName + "/" + option.File + if len(des) <= CharacterLength { + return des + } + return "" } func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNotebookJobOption, repo *models.Repository, sourceRepo *models.Repository) { @@ -237,8 +485,8 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - ctx.JSON(http.StatusOK,models.BaseMessageApi{ - Code: 2, + ctx.JSON(http.StatusOK, models.BaseMessageApi{ + Code: 2, Message: ctx.Tr("repo.cloudbrain.morethanonejob"), }) return @@ -260,7 +508,7 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote } } - err = downloadCode(sourceRepo, getCodePath(jobName), option.BranchName) + err = downloadCode(sourceRepo, getCodePath(jobName, sourceRepo), option.BranchName) if err != nil { ctx.JSON(http.StatusOK, models.BaseErrorMessageApi(ctx.Tr("cloudbrain.load_code_failed"))) return @@ -297,8 +545,9 @@ func modelartsFileNoteBookCreate(ctx *context.Context, option api.CreateFileNote Description: getDescription(option), ImageId: setting.FileNoteBook.ImageIdNPU, Spec: spec, - BootFile: "", + BootFile: getBootFile(option.File, option.OwnerName, option.ProjectName), AutoStopDurationMs: modelarts.AutoStopDurationMs / 4, + BranchName: option.BranchName, } if setting.ModelartsCD.Enabled { @@ -347,17 +596,8 @@ func isNoteBookFileExist(ctx *context.Context, option api.CreateFileNotebookJobO return true, nil } -func uploadCodeFile(repo *models.Repository, codePath string, branchName string, filePath string, jobName string) string { - err := downloadCode(repo, codePath, branchName) - if err != nil { - return "cloudbrain.load_code_failed" - } - - err = uploadOneFileToMinio(codePath, filePath, jobName, cloudbrain.CodeMountPath+"/") - if err != nil { - return "cloudbrain.load_code_failed" - } - return "" +func getBootFile(filePath string, ownerName string, projectName string) string { + return ownerName + "/" + projectName + "/" + filePath } func fileExists(gitRepo *git.Repository, path string, branch string) (bool, error) { diff --git a/services/socketwrap/clientManager.go b/services/socketwrap/clientManager.go index 7bac92ab8..2a752acf5 100755 --- a/services/socketwrap/clientManager.go +++ b/services/socketwrap/clientManager.go @@ -10,7 +10,7 @@ import ( "github.com/elliotchance/orderedmap" ) -var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35, 39, 40} +var opTypes = []int{1, 2, 5, 6, 7, 9, 10, 11, 12, 13, 14, 15, 17, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 35, 39, 40, 41} type ClientsManager struct { Clients *orderedmap.OrderedMap diff --git a/templates/admin/cloudbrain/list.tmpl b/templates/admin/cloudbrain/list.tmpl index f6d20216a..91b0e913a 100755 --- a/templates/admin/cloudbrain/list.tmpl +++ b/templates/admin/cloudbrain/list.tmpl @@ -89,21 +89,21 @@
{{$JobID := '0'}} - {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK" "MODELSAFETY"}} + {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK" "MODELSAFETY" "SNN4ECOSET"}} {{$JobID = .Cloudbrain.ID}} {{else}} {{$JobID = .JobID}} {{end}}
- {{if eq .JobType "DEBUG"}} + {{if eq .JobType "DEBUG"}} {{.DisplayJobName}} - {{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE")}} + {{else if or (eq .JobType "SNN4IMAGENET") (eq .JobType "BRAINSCORE") (eq .JobType "SNN4ECOSET")}} @@ -233,7 +233,7 @@ {{if eq .Status "RUNNING" "WAITING" "CREATING" "STARTING"}} {{$.i18n.Tr "repo.debug"}} @@ -263,7 +263,7 @@ {{else}} - {{if eq .JobType "DEBUG" "BENCHMARK" "SNN4IMAGENET" "BRAINSCORE"}} + {{if eq .JobType "DEBUG" "BENCHMARK" "SNN4IMAGENET" "BRAINSCORE" "SNN4ECOSET"}}
{{$.CsrfTokenHtml}} {{else}} {{$.CsrfTokenHtml}} {{$.i18n.Tr "repo.delete"}} @@ -321,7 +321,7 @@
{{else}} {{$JobID := '0'}} - {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK"}} + {{if eq .JobType "DEBUG" "SNN4IMAGENET" "BRAINSCORE" "BENCHMARK" "SNN4ECOSET"}} {{$JobID = .Cloudbrain.ID}} {{else}} {{$JobID = .JobID}} diff --git a/templates/admin/cloudbrain/search.tmpl b/templates/admin/cloudbrain/search.tmpl index 09e5f865e..95332ad0f 100644 --- a/templates/admin/cloudbrain/search.tmpl +++ b/templates/admin/cloudbrain/search.tmpl @@ -35,6 +35,7 @@ SNN4IMAGENET BRAINSCORE MODELSAFETY + SNN4ECOSET
@@ -44,6 +45,7 @@ {{.i18n.Tr "admin.cloudbrain.all_computing_resources"}} CPU/GPU NPU + GCU
diff --git a/templates/admin/cloudbrain/search_dashboard.tmpl b/templates/admin/cloudbrain/search_dashboard.tmpl index 7c4c1527d..d7140f556 100644 --- a/templates/admin/cloudbrain/search_dashboard.tmpl +++ b/templates/admin/cloudbrain/search_dashboard.tmpl @@ -55,6 +55,7 @@ {{.i18n.Tr "admin.cloudbrain.all_computing_resources"}} CPU/GPU NPU + GCU
diff --git a/templates/annual_privacy.tmpl b/templates/annual_privacy.tmpl new file mode 100644 index 000000000..2dd583338 --- /dev/null +++ b/templates/annual_privacy.tmpl @@ -0,0 +1,35 @@ +{{template "base/head_home" .}} +
+

OpenI启智社区2022年度报告授权协议

+
+

+ 感谢您阅读《OpenI启智社区2022年度报告授权协议》!在您正式使用OpenI启智社区2022年度报告之前应仔细阅读并充分理解本协议中的全部内容，如您不同意本协议中的任何条款，请立即停止使用OpenI启智社区2022年度报告。您使用OpenI启智社区2022年度报告的行为将被视为已经仔细阅读、充分理解并毫无保留地接受本协议所有条款。 +

+ +

+ 1. 制作年度报告 +

+

+ OpenI启智社区2022年度报告将根据您在平台的历史信息，帮助您生成一份专属年度报告。为此，我们将使用2022自然年度您在OpenI启智社区产生的行为信息，包括但不限于用户名、注册时间、创建项目数、项目下载次数、commit次数、代码行数、创建数据集、上传数据集文件、数据集被收藏数、数据集下载数、云脑任务所有相关数据（ GPU/NPU 调试任务、训练任务、运行卡时）。您理解并同意，上述信息是OpenI启智社区生成年度报告的必备信息，如您拒绝授权使用，OpenI启智社区将无法为您制作并提供专属年度报告。未经您的书面同意，我们保证不以超越本协议约定范围使用您的个人信息。 +

+

+ 2. 使用年度报告 +

+

+ OpenI启智社区提供的年度报告仅限您个人使用，您可自行留存欣赏或无偿分享、公开。您理解并同意，如因您分享、公开年度报告而产生的任何损失(包括但不限于个人信息泄露等)应由您自行承担，请您在分享、公开年度报告前审慎考虑。 +

+

+ 3. 知识产权 +

+

+ 年度报告及其内容(包括但不限于软件、技术、程序、网页、文字、图片、音频、视频、页面设计、商标等)的知识产权由OpenI启智社区享有，您理解并同意，您不得超越本协议目的使用年度报告中的内容素材，如您希望以任何形式将年度报告中的内容素材用于本协议约定范围之外，应当经过所有实际权利人的书面许可。 +

+

+ 4. 隐私政策 +

+

+ 其他本隐私保护指引未有涉及的，将适用《OpenI启智社区AI协作平台使用协议》和《OpenI启智社区AI协作平台隐私协议》。 +

+ +
+{{template "base/footer" .}} \ No newline at end of file diff --git a/templates/base/footer.tmpl b/templates/base/footer.tmpl index 738578a8a..d652f3ab1 100755 --- a/templates/base/footer.tmpl +++ b/templates/base/footer.tmpl @@ -17,9 +17,9 @@ {{if .RequireSimpleMDE}} - - - + + + @@ -27,10 +27,10 @@ {{if .RequireMinicolors}} - + {{end}} {{if .RequireU2F}} - + {{end}} {{if .EnableCaptcha}} {{if eq .CaptchaType "recaptcha"}} @@ -38,17 +38,17 @@ {{end}} {{end}} {{if .RequireTribute}} - + {{end}} {{if .PageIsHome}} - + {{end}} {{template "custom/footer" .}} {{if .PageIsHome}} - - + + {{if .RequireSimpleMDE}} - - - + + + @@ -24,10 +24,10 @@ {{if .RequireMinicolors}} - + {{end}} {{if .RequireU2F}} - + {{end}} {{if .EnableCaptcha}} {{if eq .CaptchaType "recaptcha"}} @@ -35,10 +35,10 @@ {{end}} {{end}} {{if .RequireTribute}} - + {{end}} {{if .PageIsHome}} - + {{end}} diff --git a/templates/base/head.tmpl b/templates/base/head.tmpl index f4766bc9d..113813c09 100755 --- a/templates/base/head.tmpl +++ b/templates/base/head.tmpl @@ -5,7 +5,7 @@ {{if .Title}}{{.Title}} - {{end}} {{if .Repository.Name}}{{.Repository.Name}} - {{end}}{{AppName}} - + {{if UseServiceWorker}} - - - - + + + + {{if .RequireSimpleMDE}} - + {{end}} {{if .RequireTribute}} - + {{end}} @@ -137,7 +137,7 @@ {{if .RequireMinicolors}} - + {{end}} {{if .PageIsUserProfile}} @@ -179,7 +179,7 @@ {{else if ne DefaultTheme "gitea"}} {{end}} - + {{template "custom/header" .}} - + {{template "custom/body_outer_pre" .}} diff --git a/templates/base/head_course.tmpl b/templates/base/head_course.tmpl index fe21a0e05..ac5a849d2 100644 --- a/templates/base/head_course.tmpl +++ b/templates/base/head_course.tmpl @@ -5,7 +5,7 @@ {{if .Title}}{{.Title}} - {{end}} {{if .Repository.Name}}{{.Repository.Name}} - {{end}}{{AppName}} - + {{if UseServiceWorker}} - - - - + + + + {{if .RequireSimpleMDE}} - + {{end}} {{if .RequireTribute}} - + {{end}} @@ -137,7 +137,7 @@ {{if .RequireMinicolors}} - + {{end}} {{if .PageIsUserProfile}} @@ -179,7 +179,7 @@ {{else if ne DefaultTheme "gitea"}} {{end}} - + {{template "custom/header" .}} - + {{template "custom/body_outer_pre" .}} diff --git a/templates/base/head_fluid.tmpl b/templates/base/head_fluid.tmpl index 1c16e7140..851509861 100644 --- a/templates/base/head_fluid.tmpl +++ b/templates/base/head_fluid.tmpl @@ -5,7 +5,7 @@ {{if .Title}}{{.Title}} - {{end}} {{if .Repository.Name}}{{.Repository.Name}} - {{end}}{{AppName}} - + {{if UseServiceWorker}} - - - - + + + + {{if .RequireSimpleMDE}} - + {{end}} {{if .RequireTribute}} - + {{end}} @@ -137,7 +137,7 @@ {{if .RequireMinicolors}} - + {{end}} {{if .PageIsUserProfile}} @@ -179,7 +179,7 @@ {{else if ne DefaultTheme "gitea"}} {{end}} - + {{template "custom/header" .}} - + diff --git a/templates/base/head_home.tmpl b/templates/base/head_home.tmpl index 6456e2a4a..77a0fcc55 100644 --- a/templates/base/head_home.tmpl +++ b/templates/base/head_home.tmpl @@ -5,7 +5,7 @@ {{if .Title}}{{.Title}} - {{end}} {{if .Repository.Name}}{{.Repository.Name}} - {{end}}{{AppName}} - + {{if UseServiceWorker}} - - - - + + + + - + {{if .RequireSimpleMDE}} - + {{end}} {{if .RequireTribute}} - + {{end}} @@ -138,7 +138,7 @@ {{if .RequireMinicolors}} - + {{end}} {{if .PageIsUserProfile}} @@ -191,14 +191,14 @@ var _hmt = _hmt || []; s.parentNode.insertBefore(hm, s); })(); - + - + - - + + - + {{template "custom/body_outer_pre" .}} diff --git a/templates/base/head_pro.tmpl b/templates/base/head_pro.tmpl index c643ea873..b1f7501f8 100644 --- a/templates/base/head_pro.tmpl +++ b/templates/base/head_pro.tmpl @@ -5,7 +5,7 @@ {{if .Title}}{{.Title}} - {{end}} {{if .Repository.Name}}{{.Repository.Name}} - {{end}}{{AppName}} - + {{if UseServiceWorker}} - - - - + + + + {{if .RequireSimpleMDE}} - + {{end}} {{if .RequireTribute}} - + {{end}} @@ -137,7 +137,7 @@ {{if .RequireMinicolors}} - + {{end}} {{if .PageIsUserProfile}} @@ -179,7 +179,7 @@ {{else if ne DefaultTheme "gitea"}} {{end}} - + {{template "custom/header" .}} - + diff --git a/templates/custom/header.tmpl b/templates/custom/header.tmpl index e69de29bb..cbc5f7e8f 100644 --- a/templates/custom/header.tmpl +++ b/templates/custom/header.tmpl @@ -0,0 +1,2 @@ + + diff --git a/templates/custom/home/home_top.tmpl b/templates/custom/home/home_top.tmpl index 195c7feee..a4010f0d6 100644 --- a/templates/custom/home/home_top.tmpl +++ b/templates/custom/home/home_top.tmpl @@ -1786,8 +1786,40 @@ background-repeat: no-repeat; background-size: cover; padding: 45% 0; - } + } } + + ._hm-pg-bg-4 { + background: url(/img/home-banner-02-1.jpg); + background-repeat: no-repeat; + background-position: center center; + background-size: cover; + } + ._hm-pg-bg-4-content { + position: absolute; + margin: auto; + left: 0; + top: -10%; + right: 0; + bottom: 0; + display: block; + width: 790px; + height: 315px; + background: url(/img/home-banner-02-2.png); + background-repeat: no-repeat; + background-position: center center; + background-size: cover; + } + @media only screen and (max-width: 767px) { + ._hm-pg-bg-4 { + background-size: 200% 100%; + } + ._hm-pg-bg-4-content { + top: -15%; + width: 90%; + height: 150px; + } + }
@@ -2029,12 +2061,18 @@
+
+ +
+ +
+
diff --git a/templates/custom/task_wait_count.tmpl b/templates/custom/task_wait_count.tmpl index fb8ee71fb..997a0f1c3 100644 --- a/templates/custom/task_wait_count.tmpl +++ b/templates/custom/task_wait_count.tmpl @@ -1,7 +1,7 @@
- {{.i18n.Tr "repo.wait_count_start"}} {{.WaitCount}} {{.i18n.Tr "repo.wait_count_end"}} + {{.i18n.Tr "repo.wait_count_start"}} {{if not .WaitCount}}1{{else}}{{addOne .WaitCount}}{{end}} {{.i18n.Tr "repo.wait_count_end"}}
- + + + + {{template "base/head" .}}
diff --git a/templates/explore/repo_orgtop.tmpl b/templates/explore/repo_orgtop.tmpl index bba16dd47..17d02189f 100755 --- a/templates/explore/repo_orgtop.tmpl +++ b/templates/explore/repo_orgtop.tmpl @@ -1,5 +1,5 @@ - - + + {{template "custom/global_mask" .}}
{{template "repo/header" .}}
+ {{if eq .NotStopTaskCount 0}} {{template "base/alert" .}} {{end}} @@ -54,7 +26,7 @@ {{.i18n.Tr "repo.cloudbrain.benchmark.model"}} 模型安全评测 + href="{{.RepoLink}}/modelsafety/create_gpu">{{.i18n.Tr "modelsafety.model_security_evaluation"}}
@@ -78,21 +50,12 @@ onkeydown="this.value=this.value.substring(0, 255)" onkeyup="this.value=this.value.substring(0, 255)">{{.description}}
- - -
+
  - @@ -116,22 +79,10 @@ {{.i18n.Tr "cloudbrain.view_sample"}}
- + {{template "custom/select_model" .}}
- {{template "custom/select_dataset_train" .}} - +
{{range .benchmark_types}} {{if eq .Id $.benchmarkTypeID}} @@ -226,26 +168,13 @@
-
- -
- -
- -
- {{.i18n.Tr "repo.cloudbrain.benchmark.algorithm"}} - {{.i18n.Tr "repo.cloudbrain.benchmark.model"}} - 模型安全评测 -
-
- -
-
- - {{template "custom/task_wait_count" .}} -
-
- - - {{.i18n.Tr "repo.cloudbrain_jobname_err"}} -
-
- - -
- - -
-
- -
- - - - {{.i18n.Tr "custom.resource_description"}} -
- -
- - - {{.i18n.Tr "cloudbrain.view_sample"}} -
- - -
+
- {{.i18n.Tr "repo.cloudbrain.cancel"}} + {{.i18n.Tr "repo.cloudbrain.cancel"}}
- {{end}}
{{template "base/footer" .}} + - + + + - + {{end}} diff --git a/templates/repo/datasets/label/index.tmpl b/templates/repo/datasets/label/index.tmpl index 9a0863974..24816bdec 100644 --- a/templates/repo/datasets/label/index.tmpl +++ b/templates/repo/datasets/label/index.tmpl @@ -215,7 +215,7 @@ - + {{template "base/footer" .}} diff --git a/templates/repo/debugjob/index.tmpl b/templates/repo/debugjob/index.tmpl index 255bde108..953a52c16 100755 --- a/templates/repo/debugjob/index.tmpl +++ b/templates/repo/debugjob/index.tmpl @@ -152,6 +152,7 @@
{{$.i18n.Tr "repo.gpu_type_all"}}
CPU/GPU
NPU
+
GCU
{{if .Permission.CanWrite $.UnitTypeCloudBrain}} diff --git a/templates/repo/grampus/notebook/gcu/new.tmpl b/templates/repo/grampus/notebook/gcu/new.tmpl new file mode 100644 index 000000000..8698447fb --- /dev/null +++ b/templates/repo/grampus/notebook/gcu/new.tmpl @@ -0,0 +1,161 @@ +{{template "base/head" .}} +{{template "custom/global_mask" .}} +
+ {{template "repo/header" .}} +
+ + {{if eq .NotStopTaskCount 0}} + {{template "base/alert" .}} + {{end}} + +

+ {{.i18n.Tr "repo.modelarts.train_job.new_debug"}} +

+ {{template "custom/alert_cb" .}} +
+
+ + + {{.CsrfTokenHtml}} +

{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:

+
+ +
+ + + {{.i18n.Tr "cloudbrain.resource_cluster_openi"}} + + + + {{.i18n.Tr "cloudbrain.resource_cluster_c2net"}}(Beta) + +
+
+
+ +
+ + + + + + CPU/GPU + + + + + + Ascend NPU + + + + + + GCU +
+
+
+ + {{template "custom/task_wait_count" .}} +
+ + {{.i18n.Tr "cloudbrain.new_debug_gpu_tooltips1" "/code" "/dataset" "/pretrainmodel" | Safe}} +
+
+
+ + +
+ {{.i18n.Tr "repo.cloudbrain_jobname_err"}} +
+ + {{if .description}} + + {{else}} + + {{end}} +
+
+
+ + +
+ {{template "custom/select_model" .}} +
+ + +
+ +
+ +
+ +
+ + + + {{.i18n.Tr "custom.resource_description"}} + {{if .CloudBrainPaySwitch}} +
+ {{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} + + + {{$.i18n.Tr "points.points_acquisition_instructions"}} + +
+ {{end}} +
+
+ + + {{.i18n.Tr "repo.cloudbrain.cancel"}} +
+
+
+ + +
+
+{{template "base/footer" .}} + + + diff --git a/templates/repo/grampus/notebook/gpu/new.tmpl b/templates/repo/grampus/notebook/gpu/new.tmpl index 4a752ec26..d08486bd5 100644 --- a/templates/repo/grampus/notebook/gpu/new.tmpl +++ b/templates/repo/grampus/notebook/gpu/new.tmpl @@ -4,8 +4,7 @@
{{template "repo/header" .}}
- - + {{if eq .NotStopTaskCount 0}} {{template "base/alert" .}} {{end}} @@ -20,8 +19,8 @@
{{.CsrfTokenHtml}} +

{{.i18n.Tr "repo.modelarts.train_job.basic_info"}}:

-
@@ -55,6 +54,14 @@ d="M3 2.992C3 2.444 3.445 2 3.993 2h16.014a1 1 0 0 1 .993.992v18.016a.993.993 0 0 1-.993.992H3.993A1 1 0 0 1 3 21.008V2.992zM19 11V4H5v7h14zm0 2H5v7h14v-7zM9 6h6v2H9V6zm0 9h6v2H9v-2z" /> Ascend NPU + + + + + + GCU
@@ -120,7 +127,7 @@ {{.i18n.Tr "custom.resource_description"}} {{if .CloudBrainPaySwitch}} -
+
{{$.i18n.Tr "points.balance_of_points"}}{{.PointAccount.Balance}}{{$.i18n.Tr "points.points"}}{{$.i18n.Tr "points.expected_time"}}{{$.i18n.Tr "points.hours"}} @@ -148,7 +155,7 @@ {{template "base/footer" .}} - - - + + - - - - - - - - - - - + + + + + + + + + + + diff --git a/templates/repo/home.tmpl b/templates/repo/home.tmpl index 298d564e3..da5dbfd3b 100644 --- a/templates/repo/home.tmpl +++ b/templates/repo/home.tmpl @@ -311,7 +311,7 @@
-

简介

+

{{if .Repository.DescriptionHTML}} @@ -363,15 +363,15 @@

{{$lenCon := len .ContributorInfo}} {{if lt $lenCon 25 }} - 贡献者 ({{len .ContributorInfo}}) + {{.i18n.Tr "home.contributors"}} ({{len .ContributorInfo}}) {{else}} - 贡献者 ({{len .ContributorInfo}}+) + {{.i18n.Tr "home.contributors"}} ({{len .ContributorInfo}}+) {{end}}
- + 全部 + href="{{.RepoLink}}/contributors?type={{if .IsViewBranch}}branch{{else}}tag{{end}}&name={{.BranchName}}">{{.i18n.Tr "repo.computing.all"}} {{svg "octicon-chevron-right" 16}}

diff --git a/templates/repo/issue/view_content/pull.tmpl b/templates/repo/issue/view_content/pull.tmpl index 06f838322..761c2a609 100755 --- a/templates/repo/issue/view_content/pull.tmpl +++ b/templates/repo/issue/view_content/pull.tmpl @@ -448,7 +448,7 @@
- + @@ -266,8 +266,8 @@ {{template "base/footer" .}} - - + +
- {{template "user/auth/signin_navbar" .}} -
-
- {{template "user/auth/signin_inner" .}} +
+
+
+
+ {{ if .ActivityTpl }} + {{ .ActivityTpl | Safe}} + {{ else }} + {{ template "user/auth/signin_up_img_default" .}} + {{ end }} +
+
+
+
+ {{template "user/auth/signin_navbar" .}} + {{template "user/auth/signin_inner" .}} +
+
diff --git a/templates/user/auth/signin_cloud_brain.tmpl b/templates/user/auth/signin_cloud_brain.tmpl index 6b0b9c38f..9412d83e6 100755 --- a/templates/user/auth/signin_cloud_brain.tmpl +++ b/templates/user/auth/signin_cloud_brain.tmpl @@ -1,9 +1,48 @@ {{template "base/head" .}} +