diff --git a/models/cloudbrain.go b/models/cloudbrain.go index bb1241247..8ecacb4fe 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -31,6 +31,7 @@ const ( JobTypeBrainScore JobType = "BRAINSCORE" JobTypeTrain JobType = "TRAIN" + //notebook ModelArtsCreateQueue ModelArtsJobStatus = "CREATE_QUEUING" //免费资源创建排队中 ModelArtsCreating ModelArtsJobStatus = "CREATING" //创建中 ModelArtsCreateFailed ModelArtsJobStatus = "CREATE_FAILED" //创建失败 @@ -46,6 +47,30 @@ const ( ModelArtsDeleted ModelArtsJobStatus = "DELETED" //已删除 ModelArtsResizing ModelArtsJobStatus = "RESIZING" //规格变更中 ModelArtsResizFailed ModelArtsJobStatus = "RESIZE_FAILED" //规格变更失败 + + //trainjob + ModelArtsTrainJobUnknown ModelArtsJobStatus = "UNKNOWN" //作业状态未知 + ModelArtsTrainJobInit ModelArtsJobStatus = "INIT" //作业初始化状态 + ModelArtsTrainJobImageCreating ModelArtsJobStatus = "IMAGE_CREATING" //作业镜像正在创建 + ModelArtsTrainJobImageFailed ModelArtsJobStatus = "IMAGE_FAILED" //作业镜像创建失败 + ModelArtsTrainJobSubmitTrying ModelArtsJobStatus = "SUBMIT_TRYING" //作业正在提交 + ModelArtsTrainJobSubmitFailed ModelArtsJobStatus = "SUBMIT_FAILED" //作业提交失败 + ModelArtsTrainJobDeleteFailed ModelArtsJobStatus = "DELETE_FAILED" //作业删除失败 + ModelArtsTrainJobWaiting ModelArtsJobStatus = "WAITING" //作业正在排队中 + ModelArtsTrainJobRunning ModelArtsJobStatus = "RUNNING" //作业正在运行中 + ModelArtsTrainJobKilling ModelArtsJobStatus = "KILLING" //作业正在取消 + ModelArtsTrainJobCompleted ModelArtsJobStatus = "COMPLETED" //作业已经完成 + ModelArtsTrainJobFailed ModelArtsJobStatus = "FAILED" //作业运行失败 + ModelArtsTrainJobKilled ModelArtsJobStatus = "KILLED" //作业取消成功 + ModelArtsTrainJobCanceled ModelArtsJobStatus = "CANCELED" //作业取消 + ModelArtsTrainJobLost ModelArtsJobStatus = "LOST" //作业丢失 + ModelArtsTrainJobScaling ModelArtsJobStatus = "SCALING" //作业正在扩容 + ModelArtsTrainJobSubmitModelFailed ModelArtsJobStatus = "SUBMIT_MODEL_FAILED" //提交模型失败 + ModelArtsTrainJobDeployServiceFailed ModelArtsJobStatus = "DEPLOY_SERVICE_FAILED" //部署服务失败 + ModelArtsTrainJobCheckInit ModelArtsJobStatus = "CHECK_INIT" //审核作业初始化 + ModelArtsTrainJobCheckRunning ModelArtsJobStatus = "CHECK_RUNNING" //审核作业正在运行中 + ModelArtsTrainJobCheckRunningCompleted ModelArtsJobStatus = "CHECK_RUNNING_COMPLETED" //审核作业已经完成 + ModelArtsTrainJobCheckFailed ModelArtsJobStatus = "CHECK_FAILED" //审核作业失败 ) type Cloudbrain struct { @@ -66,6 +91,7 @@ type Cloudbrain struct { DeletedAt time.Time `xorm:"deleted"` CanDebug bool `xorm:"-"` CanDel bool `xorm:"-"` + CanModify bool `xorm:"-"` Type int VersionID int64 //版本id @@ -1007,13 +1033,13 @@ func GetCloudbrainByJobIDAndIsLatestVersion(jobID string, isLatestVersion string func GetCloudbrainsNeededStopByUserID(userID int64) ([]*Cloudbrain, error) { cloudBrains := make([]*Cloudbrain, 0) - err := x.Cols("job_id", "status", "type").Where("user_id=? AND status !=?", userID, string(JobStopped)).Find(&cloudBrains) + err := x.Cols("job_id", "status", "type", "job_type", "version_id").Where("user_id=? AND status !=?", userID, string(JobStopped)).Find(&cloudBrains) return cloudBrains, err } func GetCloudbrainsNeededStopByRepoID(repoID int64) ([]*Cloudbrain, error) { cloudBrains := make([]*Cloudbrain, 0) - err := x.Cols("job_id", "status", "type").Where("repo_id=? AND status !=?", repoID, string(JobStopped)).Find(&cloudBrains) + err := x.Cols("job_id", "status", "type", "job_type", "version_id").Where("repo_id=? AND status !=?", repoID, string(JobStopped)).Find(&cloudBrains) return cloudBrains, err } @@ -1091,3 +1117,31 @@ func CanDelJob(isSigned bool, user *User, job *CloudbrainInfo) bool { } return false } + +func GetCloudBrainUnStoppedJob() ([]*Cloudbrain, error) { + cloudbrains := make([]*Cloudbrain, 0, 10) + return cloudbrains, x. + NotIn("status", + JobStopped, JobSucceeded, JobFailed, ModelArtsCreateFailed, ModelArtsStartFailed, ModelArtsUnavailable, ModelArtsResizFailed, ModelArtsDeleted, + ModelArtsStopped, ModelArtsTrainJobCanceled, ModelArtsTrainJobCheckFailed, ModelArtsTrainJobCompleted, ModelArtsTrainJobDeleteFailed, ModelArtsTrainJobDeployServiceFailed, + ModelArtsTrainJobFailed, ModelArtsTrainJobImageFailed, ModelArtsTrainJobKilled, ModelArtsTrainJobLost, ModelArtsTrainJobSubmitFailed, ModelArtsTrainJobSubmitModelFailed). + Limit(100). + Find(&cloudbrains) +} + +func GetCloudbrainCountByUserID(userID int64) (int, error) { + count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ? and type = ?", JobTypeDebug, userID, TypeCloudBrainOne).Count(new(Cloudbrain)) + return int(count), err +} + +func GetCloudbrainNotebookCountByUserID(userID int64) (int, error) { + count, err := x.In("status", ModelArtsCreateQueue, ModelArtsCreating, ModelArtsStarting, ModelArtsReadyToStart, ModelArtsResizing, ModelArtsStartQueuing, ModelArtsRunning, ModelArtsRestarting). + And("job_type = ? and user_id = ? and type = ?", JobTypeDebug, userID, TypeCloudBrainTwo).Count(new(Cloudbrain)) + return int(count), err +} + +func GetCloudbrainTrainJobCountByUserID(userID int64) (int, error) { + count, err := x.In("status", ModelArtsTrainJobInit, ModelArtsTrainJobImageCreating, ModelArtsTrainJobSubmitTrying, ModelArtsTrainJobWaiting, ModelArtsTrainJobRunning, ModelArtsTrainJobScaling, ModelArtsTrainJobCheckInit, ModelArtsTrainJobCheckRunning, ModelArtsTrainJobCheckRunningCompleted). + And("job_type = ? and user_id = ? and type = ?", JobTypeTrain, userID, TypeCloudBrainTwo).Count(new(Cloudbrain)) + return int(count), err +} diff --git a/models/repo.go b/models/repo.go index 1a5cf122c..6d73aa28a 100755 --- a/models/repo.go +++ b/models/repo.go @@ -667,15 +667,11 @@ func (repo *Repository) getReviewersPublic(e Engine, doerID, posterID int64) (_ users := make([]*User, 0) const SQLCmd = "SELECT * FROM `user` WHERE id IN ( " + - "SELECT user_id FROM `access` WHERE repo_id = ? AND mode >= ? AND user_id NOT IN ( ?, ?) " + - "UNION " + - "SELECT user_id FROM `watch` WHERE repo_id = ? AND user_id NOT IN ( ?, ?) AND mode IN (?, ?) " + - ") ORDER BY name" + "SELECT user_id FROM `access` WHERE repo_id = ? AND mode >= ? AND user_id NOT IN ( ?, ?) ) ORDER BY name " if err = e. SQL(SQLCmd, - repo.ID, AccessModeRead, doerID, posterID, - repo.ID, doerID, posterID, RepoWatchModeNormal, RepoWatchModeAuto). + repo.ID, AccessModeWrite, doerID, posterID). Find(&users); err != nil { return nil, err } diff --git a/models/user.go b/models/user.go index 1ee20d74c..ec96420dc 100755 --- a/models/user.go +++ b/models/user.go @@ -145,6 +145,7 @@ type User struct { AllowImportLocal bool // Allow migrate repository by local path AllowCreateOrganization bool `xorm:"DEFAULT true"` ProhibitLogin bool `xorm:"NOT NULL DEFAULT false"` + IsOperator bool `xorm:"NOT NULL DEFAULT false"` //运营人员 // Avatar Avatar string `xorm:"VARCHAR(2048) NOT NULL"` @@ -928,8 +929,17 @@ var ( "template", "user", "vendor", - } - reservedUserPatterns = []string{"*.keys", "*.gpg"} + "dashbord", + "operation", + "blockchain", + "avatar", + "swagger.v1.json", + "secure", + "serviceworker.js", + "self", + "repo-avatars", + } + reservedUserPatterns = []string{"*.keys", "*.gpg", "*.png"} ) // isUsableName checks if name is reserved or pattern of name is not allowed @@ -1551,11 +1561,11 @@ func GetUserByActivateEmail(email string) (*User, error) { if err := ctx.e.Join("INNER", "email_address", "email_address.uid = \"user\".id"). Where("email_address.email= ?", email). Find(&users); err != nil { - return nil,err + return nil, err } if len(users) >= 1 { - return &users[0],nil - }else { + return &users[0], nil + } else { // Finally, if email address is the protected email address:用户邮件地址设置为隐藏电子邮件地址 if strings.HasSuffix(email, fmt.Sprintf("@%s", setting.Service.NoReplyAddress)) { username := strings.TrimSuffix(email, fmt.Sprintf("@%s", setting.Service.NoReplyAddress)) @@ -1571,6 +1581,7 @@ func GetUserByActivateEmail(email string) (*User, error) { return nil, errors.New("cannot find user by email") } } + // GetUserByEmail returns the user object by given e-mail if exists. func GetUserByEmail(email string) (*User, error) { return GetUserByEmailContext(DefaultDBContext(), email) diff --git a/modules/auth/auth.go b/modules/auth/auth.go index 16ea9f15e..352e50ca0 100644 --- a/modules/auth/auth.go +++ b/modules/auth/auth.go @@ -9,6 +9,9 @@ import ( "reflect" "strings" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth/sso" "code.gitea.io/gitea/modules/validation" @@ -31,6 +34,8 @@ func SignedInUser(ctx *macaron.Context, sess session.Store) (*models.User, bool) return nil, false } + checkAutoLogin(ctx, sess) + // Try to sign in with each of the enabled plugins for _, ssoMethod := range sso.Methods() { if !ssoMethod.IsEnabled() { @@ -46,6 +51,23 @@ func SignedInUser(ctx *macaron.Context, sess session.Store) (*models.User, bool) return nil, false } +func checkAutoLogin(ctx *macaron.Context, sess session.Store) { + uid := sess.Get("uid") + if uid == nil { + uname := ctx.GetCookie(setting.CookieUserName) + + u, err := models.GetUserByName(uname) + if err == nil { + + if val, ok := ctx.GetSuperSecureCookie( + base.EncodeMD5(u.Rands+u.Passwd), setting.CookieRememberName); ok && val == u.Name { + sess.Set("uid", u.ID) + } + } + } + +} + // Form form binding interface type Form interface { binding.Validator diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 8f6bf4e17..299133290 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -1,9 +1,10 @@ package cloudbrain import ( - "code.gitea.io/gitea/modules/setting" "errors" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" @@ -16,7 +17,7 @@ const ( ModelMountPath = "/model" BenchMarkMountPath = "/benchmark" Snn4imagenetMountPath = "/snn4imagenet" - BrainScoreMountPath = "/brainscore" + BrainScoreMountPath = "/brainscore" TaskInfoName = "/taskInfo" SubTaskName = "task1" @@ -28,6 +29,72 @@ var ( ResourceSpecs *models.ResourceSpecs ) +func isAdminOrOwnerOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool { + + if err != nil { + return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() + } else { + return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID + } + +} + +func CanDeleteDebugJob(ctx *context.Context, job *models.Cloudbrain) bool { + + if job.Status != string(models.JobStopped) && job.Status != string(models.JobFailed) && job.Status != string(models.ModelArtsStartFailed) && job.Status != string(models.ModelArtsCreateFailed) { + return false + } + return isAdminOrOwnerOrJobCreater(ctx, job, nil) +} + +func CanDeleteTrainJob(ctx *context.Context, job *models.Cloudbrain) bool { + + return isAdminOrOwnerOrJobCreater(ctx, job, nil) +} + +func CanCreateOrDebugJob(ctx *context.Context) bool { + return ctx.Repo.CanWrite(models.UnitTypeCloudBrain) +} + +func CanModifyJob(ctx *context.Context, job *models.Cloudbrain) bool { + + return isAdminOrJobCreater(ctx, job, nil) +} + +func isAdminOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool { + + if err != nil { + return ctx.IsUserSiteAdmin() + } else { + return ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID + } + +} + +func AdminOrOwnerOrJobCreaterRight(ctx *context.Context) { + + var jobID = ctx.Params(":jobid") + + job, err := models.GetCloudbrainByJobID(jobID) + + if !isAdminOrOwnerOrJobCreater(ctx, job, err) { + + ctx.NotFound(ctx.Req.URL.RequestURI(), nil) + } + +} + +func AdminOrJobCreaterRight(ctx *context.Context) { + + var jobID = ctx.Params(":jobid") + job, err := models.GetCloudbrainByJobID(jobID) + if !isAdminOrJobCreater(ctx, job, err) { + + ctx.NotFound(ctx.Req.URL.RequestURI(), nil) + } + +} + func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue string, resourceSpecId int) error { dataActualPath := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + diff --git a/modules/context/auth.go b/modules/context/auth.go index 3f53e6fce..9877657eb 100755 --- a/modules/context/auth.go +++ b/modules/context/auth.go @@ -145,8 +145,7 @@ func Toggle(options *ToggleOptions) macaron.Handler { } if options.OperationRequired { - //todo: add isOperator judgement - if !ctx.User.IsAdmin { + if !ctx.User.IsOperator { ctx.Error(403) return } diff --git a/modules/context/context.go b/modules/context/context.go index 6877780e3..5f09e190d 100755 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -310,6 +310,7 @@ func Contexter() macaron.Handler { ctx.Data["SignedUserID"] = ctx.User.ID ctx.Data["SignedUserName"] = ctx.User.Name ctx.Data["IsAdmin"] = ctx.User.IsAdmin + ctx.Data["IsOperator"] = ctx.User.IsOperator c.Data["SignedUserName"] = ctx.User.Name } else { ctx.Data["SignedUserID"] = int64(0) diff --git a/modules/context/repo.go b/modules/context/repo.go index c6e5e8edd..9f8a178fc 100755 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -524,7 +524,7 @@ func RepoAssignment() macaron.Handler { } ctx.Data["Tags"] = tags - brs, err := ctx.Repo.GitRepo.GetBranches() + brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 0) if err != nil { ctx.ServerError("GetBranches", err) return @@ -712,7 +712,7 @@ func RepoRefByType(refType RepoRefType) macaron.Handler { refName = ctx.Repo.Repository.DefaultBranch ctx.Repo.BranchName = refName if !ctx.Repo.GitRepo.IsBranchExist(refName) { - brs, err := ctx.Repo.GitRepo.GetBranches() + brs, _, err := ctx.Repo.GitRepo.GetBranches(0, 0) if err != nil { ctx.ServerError("GetBranches", err) return diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 207018c20..294690d45 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -185,6 +185,17 @@ func registerHandleSummaryStatistic() { }) } +func registerSyncCloudbrainStatus() { + RegisterTaskFatal("sync_cloudbrain_status", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@every 10m", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.SyncCloudbrainStatus() + return nil + }) +} + func initBasicTasks() { registerUpdateMirrorTask() registerRepoHealthCheck() @@ -202,4 +213,6 @@ func initBasicTasks() { registerHandleRepoAndUserStatistic() registerHandleSummaryStatistic() + + registerSyncCloudbrainStatus() } diff --git a/modules/git/repo_branch.go b/modules/git/repo_branch.go old mode 100644 new mode 100755 index 8f9c802e0..ad5acfdbf --- a/modules/git/repo_branch.go +++ b/modules/git/repo_branch.go @@ -6,7 +6,9 @@ package git import ( + "bufio" "fmt" + "io" "strings" "github.com/go-git/go-git/v5/plumbing" @@ -74,25 +76,6 @@ func (repo *Repository) SetDefaultBranch(name string) error { return err } -// GetBranches returns all branches of the repository. -func (repo *Repository) GetBranches() ([]string, error) { - var branchNames []string - - branches, err := repo.gogitRepo.Branches() - if err != nil { - return nil, err - } - - _ = branches.ForEach(func(branch *plumbing.Reference) error { - branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix)) - return nil - }) - - // TODO: Sort? - - return branchNames, nil -} - // GetBranch returns a branch by it's name func (repo *Repository) GetBranch(branch string) (*Branch, error) { if !repo.IsBranchExist(branch) { @@ -106,16 +89,16 @@ func (repo *Repository) GetBranch(branch string) (*Branch, error) { } // GetBranchesByPath returns a branch by it's path -func GetBranchesByPath(path string) ([]*Branch, error) { +func GetBranchesByPath(path string, skip, limit int) ([]*Branch, int, error) { gitRepo, err := OpenRepository(path) if err != nil { - return nil, err + return nil, 0, err } defer gitRepo.Close() - brs, err := gitRepo.GetBranches() + brs, countAll, err := gitRepo.GetBranches(skip, limit) if err != nil { - return nil, err + return nil, 0, err } branches := make([]*Branch, len(brs)) @@ -127,7 +110,7 @@ func GetBranchesByPath(path string) ([]*Branch, error) { } } - return branches, nil + return branches, countAll, nil } // DeleteBranchOptions Option(s) for delete branch @@ -183,3 +166,91 @@ func (repo *Repository) RemoveRemote(name string) error { func (branch *Branch) GetCommit() (*Commit, error) { return branch.gitRepo.GetBranchCommit(branch.Name) } + +// GetBranches returns branches from the repository, skipping skip initial branches and +// returning at most limit branches, or all branches if limit is 0. +func (repo *Repository) GetBranches(skip, limit int) ([]string, int, error) { + return callShowRef(repo.Path, BranchPrefix, "--heads", skip, limit) +} + +// callShowRef return refs, if limit = 0 it will not limit +func callShowRef(repoPath, prefix, arg string, skip, limit int) (branchNames []string, countAll int, err error) { + stdoutReader, stdoutWriter := io.Pipe() + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + + go func() { + stderrBuilder := &strings.Builder{} + err := NewCommand("show-ref", arg).RunInDirPipeline(repoPath, stdoutWriter, stderrBuilder) + if err != nil { + if stderrBuilder.Len() == 0 { + _ = stdoutWriter.Close() + return + } + _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String())) + } else { + _ = stdoutWriter.Close() + } + }() + + i := 0 + bufReader := bufio.NewReader(stdoutReader) + for i < skip { + _, isPrefix, err := bufReader.ReadLine() + if err == io.EOF { + return branchNames, i, nil + } + if err != nil { + return nil, 0, err + } + if !isPrefix { + i++ + } + } + for limit == 0 || i < skip+limit { + // The output of show-ref is simply a list: + // SP LF + _, err := bufReader.ReadSlice(' ') + for err == bufio.ErrBufferFull { + // This shouldn't happen but we'll tolerate it for the sake of peace + _, err = bufReader.ReadSlice(' ') + } + if err == io.EOF { + return branchNames, i, nil + } + if err != nil { + return nil, 0, err + } + + branchName, err := bufReader.ReadString('\n') + if err == io.EOF { + // This shouldn't happen... but we'll tolerate it for the sake of peace + return branchNames, i, nil + } + if err != nil { + return nil, i, err + } + branchName = strings.TrimPrefix(branchName, prefix) + if len(branchName) > 0 { + branchName = branchName[:len(branchName)-1] + } + branchNames = append(branchNames, branchName) + i++ + } + // count all refs + for limit != 0 { + _, isPrefix, err := bufReader.ReadLine() + if err == io.EOF { + return branchNames, i, nil + } + if err != nil { + return nil, 0, err + } + if !isPrefix { + i++ + } + } + return branchNames, i, nil +} diff --git a/modules/git/repo_tag.go b/modules/git/repo_tag.go old mode 100644 new mode 100755 index 7780e3477..08f6f5ec1 --- a/modules/git/repo_tag.go +++ b/modules/git/repo_tag.go @@ -10,7 +10,6 @@ import ( "strings" "github.com/go-git/go-git/v5/plumbing" - "github.com/mcuadros/go-version" ) // TagPrefix tags prefix path on the repository @@ -225,29 +224,35 @@ func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, error) { return tags, nil } -// GetTags returns all tags of the repository. -func (repo *Repository) GetTags() ([]string, error) { - var tagNames []string - - tags, err := repo.gogitRepo.Tags() - if err != nil { - return nil, err - } +//// GetTags returns all tags of the repository. +//func (repo *Repository) GetTags() ([]string, error) { +// var tagNames []string +// +// tags, err := repo.gogitRepo.Tags() +// if err != nil { +// return nil, err +// } +// +// _ = tags.ForEach(func(tag *plumbing.Reference) error { +// tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix)) +// return nil +// }) +// +// version.Sort(tagNames) +// +// // Reverse order +// for i := 0; i < len(tagNames)/2; i++ { +// j := len(tagNames) - i - 1 +// tagNames[i], tagNames[j] = tagNames[j], tagNames[i] +// } +// +// return tagNames, nil +//} - _ = tags.ForEach(func(tag *plumbing.Reference) error { - tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix)) - return nil - }) - - version.Sort(tagNames) - - // Reverse order - for i := 0; i < len(tagNames)/2; i++ { - j := len(tagNames) - i - 1 - tagNames[i], tagNames[j] = tagNames[j], tagNames[i] - } - - return tagNames, nil +// GetTags returns all tags of the repository. +func (repo *Repository) GetTags() (tags []string, err error) { + tags, _, err = callShowRef(repo.Path, TagPrefix, "--tags", 0, 0) + return } // GetTagType gets the type of the tag, either commit (simple) or tag (annotated) diff --git a/modules/git/utils.go b/modules/git/utils.go old mode 100644 new mode 100755 index 83209924c..0d044feda --- a/modules/git/utils.go +++ b/modules/git/utils.go @@ -140,3 +140,11 @@ func ParseBool(value string) (result bool, valid bool) { } return intValue != 0, true } + +// ConcatenateError concatenats an error with stderr string +func ConcatenateError(err error, stderr string) error { + if len(stderr) == 0 { + return err + } + return fmt.Errorf("%w - %s", err, stderr) +} diff --git a/modules/migrations/github.go b/modules/migrations/github.go index b6bdab09f..bee21fb54 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -13,6 +13,9 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/setting" + "golang.org/x/net/proxy" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/structs" @@ -98,13 +101,41 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith ) client = oauth2.NewClient(downloader.ctx, ts) } else { - client = &http.Client{ - Transport: &http.Transport{ - Proxy: func(req *http.Request) (*url.URL, error) { - req.SetBasicAuth(userName, password) - return nil, nil + if setting.Migrations.Proxy != "" { + contextDialer, err := getProxyDialContext() + if err != nil { + log.Warn("Failed to use proxy for Github.", err) + client = &http.Client{ + Transport: &http.Transport{ + + Proxy: func(req *http.Request) (*url.URL, error) { + req.SetBasicAuth(userName, password) + return nil, nil + }, + }, + } + } else { + client = &http.Client{ + Transport: &http.Transport{ + + Proxy: func(req *http.Request) (*url.URL, error) { + req.SetBasicAuth(userName, password) + return nil, nil + }, + DialContext: contextDialer.DialContext, + }, + } + } + } else { + client = &http.Client{ + Transport: &http.Transport{ + + Proxy: func(req *http.Request) (*url.URL, error) { + req.SetBasicAuth(userName, password) + return nil, nil + }, }, - }, + } } } } @@ -112,6 +143,25 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith return &downloader } +func getProxyDialContext() (proxy.ContextDialer, error) { + authInfo := &proxy.Auth{ + setting.Migrations.Username, + setting.Migrations.Password, + } + dialSocksProxy, err := proxy.SOCKS5("tcp", setting.Migrations.Proxy, authInfo, proxy.Direct) + + if err != nil { + return nil, err + } + if contextDialer, ok := dialSocksProxy.(proxy.ContextDialer); ok { + return contextDialer, nil + + } else { + return nil, fmt.Errorf("It is not a valiad dialer.") + } + +} + // SetContext set context func (g *GithubDownloaderV3) SetContext(ctx context.Context) { g.ctx = ctx diff --git a/modules/repository/branch.go b/modules/repository/branch.go old mode 100644 new mode 100755 index 418ba25c8..7b203dd91 --- a/modules/repository/branch.go +++ b/modules/repository/branch.go @@ -23,9 +23,10 @@ func GetBranch(repo *models.Repository, branch string) (*git.Branch, error) { return gitRepo.GetBranch(branch) } -// GetBranches returns all the branches of a repository -func GetBranches(repo *models.Repository) ([]*git.Branch, error) { - return git.GetBranchesByPath(repo.RepoPath()) +// GetBranches returns branches from the repository, skipping skip initial branches and +// returning at most limit branches, or all branches if limit is 0. +func GetBranches(repo *models.Repository, skip, limit int) ([]*git.Branch, int, error) { + return git.GetBranchesByPath(repo.RepoPath(), skip, limit) } // checkBranchName validates branch name with existing repository branches @@ -36,7 +37,7 @@ func checkBranchName(repo *models.Repository, name string) error { } defer gitRepo.Close() - branches, err := GetBranches(repo) + branches, _, err := GetBranches(repo, 0, 0) if err != nil { return err } diff --git a/modules/repository/repo.go b/modules/repository/repo.go index d57b16c91..3268cce60 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -6,6 +6,7 @@ package repository import ( "fmt" + "net/url" "os" "path" "strings" @@ -54,29 +55,34 @@ func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opt repo.NumWatches = 1 } - migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second + migrateTimeout := getMigrateTimeout(opts.CloneAddr) var err error if err = os.RemoveAll(repoPath); err != nil { return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err) } - + log.Info("clone begin:" + opts.CloneAddr) if err = git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, }); err != nil { + log.Warn("clone err") return repo, fmt.Errorf("Clone: %v", err) } + log.Info("clone end:" + opts.CloneAddr) + if opts.Wiki { + log.Info("test wiki path begin") wikiPath := models.WikiPath(u.Name, opts.RepoName) wikiRemotePath := wikiRemoteURL(opts.CloneAddr) + log.Info("test wiki path end") if len(wikiRemotePath) > 0 { if err := os.RemoveAll(wikiPath); err != nil { return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) } - + log.Info("wiki clone begin") if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, @@ -88,6 +94,7 @@ func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opt return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) } } + log.Info("wiki clone end") } } @@ -137,9 +144,20 @@ func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opt repo, err = CleanUpMigrateInfo(repo) } + log.Info("clone all end:" + opts.CloneAddr) + return repo, err } +func getMigrateTimeout(urlClone string) time.Duration { + u, err := url.Parse(urlClone) + if err == nil && strings.EqualFold(u.Host, "github.com") { + return time.Duration(setting.Git.Timeout.GitHubMigrate) * time.Second + } + return time.Duration(setting.Git.Timeout.Migrate) * time.Second + +} + // cleanUpMigrateGitConfig removes mirror info which prevents "push --all". // This also removes possible user credentials. func cleanUpMigrateGitConfig(configPath string) error { diff --git a/modules/setting/git.go b/modules/setting/git.go index f83758f38..306febc78 100644 --- a/modules/setting/git.go +++ b/modules/setting/git.go @@ -27,12 +27,13 @@ var ( EnableAutoGitWireProtocol bool PullRequestPushMessage bool Timeout struct { - Default int - Migrate int - Mirror int - Clone int - Pull int - GC int `ini:"GC"` + Default int + Migrate int + GitHubMigrate int + Mirror int + Clone int + Pull int + GC int `ini:"GC"` } `ini:"git.timeout"` }{ DisableDiffHighlight: false, @@ -45,19 +46,21 @@ var ( EnableAutoGitWireProtocol: true, PullRequestPushMessage: true, Timeout: struct { - Default int - Migrate int - Mirror int - Clone int - Pull int - GC int `ini:"GC"` + Default int + Migrate int + GitHubMigrate int + Mirror int + Clone int + Pull int + GC int `ini:"GC"` }{ - Default: int(git.DefaultCommandExecutionTimeout / time.Second), - Migrate: 600, - Mirror: 300, - Clone: 300, - Pull: 300, - GC: 60, + Default: int(git.DefaultCommandExecutionTimeout / time.Second), + Migrate: 900, + GitHubMigrate: 1800, + Mirror: 1200, + Clone: 300, + Pull: 300, + GC: 60, }, } ) diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 4d4df6201..abca79b7d 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -83,6 +83,7 @@ func newIndexerService() { Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20) Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024) Indexer.StartupTimeout = sec.Key("STARTUP_TIMEOUT").MustDuration(30 * time.Second) + log.Info("New IndexerService Inited.") } // IndexerGlobFromString parses a comma separated list of patterns and returns a glob.Glob slice suited for repo indexing diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go index 51d6bbcf1..0a948cd1f 100644 --- a/modules/setting/migrations.go +++ b/modules/setting/migrations.go @@ -9,6 +9,9 @@ var ( Migrations = struct { MaxAttempts int RetryBackoff int + Proxy string + Username string + Password string }{ MaxAttempts: 3, RetryBackoff: 3, @@ -19,4 +22,7 @@ func newMigrationsService() { sec := Cfg.Section("migrations") Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts) Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff) + Migrations.Proxy = sec.Key("Proxy").MustString("") + Migrations.Username = sec.Key("Username").MustString("") + Migrations.Password = sec.Key("Password").MustString("") } diff --git a/modules/setting/queue.go b/modules/setting/queue.go index 8bdca1017..999d1afe0 100644 --- a/modules/setting/queue.go +++ b/modules/setting/queue.go @@ -160,6 +160,7 @@ func NewQueueService() { if _, ok := sectionMap["LENGTH"]; !ok { _, _ = section.NewKey("LENGTH", fmt.Sprintf("%d", Repository.PullRequestQueueLength)) } + log.Info("New QueueService Inited.") } // ParseQueueConnStr parses a queue connection string diff --git a/modules/setting/webhook.go b/modules/setting/webhook.go index 4a0c593c8..34cf8a62d 100644 --- a/modules/setting/webhook.go +++ b/modules/setting/webhook.go @@ -48,4 +48,5 @@ func newWebhookService() { } } Webhook.ProxyHosts = sec.Key("PROXY_HOSTS").Strings(",") + log.Info("New WebhookService Inited.") } diff --git a/modules/storage/storage.go b/modules/storage/storage.go index e0e875650..191871d93 100755 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -51,6 +51,7 @@ func Init() error { switch setting.Attachment.StoreType { case LocalStorageType: Attachments, err = NewLocalStorage(setting.Attachment.Path) + log.Info("local storage inited.") case MinioStorageType: minio := setting.Attachment.Minio Attachments, err = NewMinioStorage( @@ -62,6 +63,7 @@ func Init() error { minio.BasePath, minio.UseSSL, ) + log.Info("minio storage inited.") default: return fmt.Errorf("Unsupported attachment store type: %s", setting.Attachment.StoreType) } @@ -71,6 +73,7 @@ func Init() error { log.Error("obs.New failed:", err) return err } + log.Info("obs cli inited.") if err != nil { return err diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index e26e07903..54e362a0a 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -228,6 +228,7 @@ users=用户 organizations=组织 images = 云脑镜像 search=搜索 +search_pro=搜项目 code=代码 data_analysis=数字看板(内测) repo_no_results=未找到匹配的项目。 diff --git a/public/img/overview_rgb.svg b/public/img/overview_rgb.svg new file mode 100644 index 000000000..1a1257e23 --- /dev/null +++ b/public/img/overview_rgb.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/pro_rgb.svg b/public/img/pro_rgb.svg new file mode 100644 index 000000000..2110c2115 --- /dev/null +++ b/public/img/pro_rgb.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/search.svg b/public/img/search.svg new file mode 100644 index 000000000..ec91b07dd --- /dev/null +++ b/public/img/search.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/user_rgb.svg b/public/img/user_rgb.svg new file mode 100644 index 000000000..e5cf24cf0 --- /dev/null +++ b/public/img/user_rgb.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index b7ef8d48f..518c63e4f 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -75,6 +75,7 @@ import ( "code.gitea.io/gitea/routers/api/v1/repo" _ "code.gitea.io/gitea/routers/api/v1/swagger" // for swagger generation "code.gitea.io/gitea/routers/api/v1/user" + repo_ext "code.gitea.io/gitea/routers/repo" "gitea.com/macaron/binding" "gitea.com/macaron/macaron" @@ -523,23 +524,26 @@ func RegisterRoutes(m *macaron.Macaron) { Get(notify.GetThread). Patch(notify.ReadThread) }, reqToken()) - adminReq := context.Toggle(&context.ToggleOptions{SignInRequired: true, AdminRequired: true}) + + operationReq := context.Toggle(&context.ToggleOptions{SignInRequired: true, OperationRequired: true}) //Project board m.Group("/projectboard", func() { - m.Get("/restoreFork", adminReq, repo.RestoreForkNumber) - m.Get("/downloadAll", adminReq, repo.ServeAllProjectsPeriodStatisticsFile) - m.Get("/downloadAllOpenI", adminReq, repo.ServeAllProjectsOpenIStatisticsFile) + m.Get("/restoreFork", repo.RestoreForkNumber) + m.Get("/downloadAll", repo.ServeAllProjectsPeriodStatisticsFile) + m.Get("/downloadAllOpenI", repo.ServeAllProjectsOpenIStatisticsFile) m.Group("/project", func() { - m.Get("", adminReq, repo.GetAllProjectsPeriodStatistics) + m.Get("", repo.GetAllProjectsPeriodStatistics) m.Group("/:id", func() { - m.Get("", adminReq, repo.GetProjectLatestStatistics) - m.Get("/period", adminReq, repo.GetProjectPeriodStatistics) + m.Get("", repo.GetProjectLatestStatistics) + m.Get("/period", repo.GetProjectPeriodStatistics) }) }) - }) + }, operationReq) + + m.Get("/query_user_static_page", operationReq, repo_ext.QueryUserStaticDataPage) // Users m.Group("/users", func() { diff --git a/routers/api/v1/repo/branch.go b/routers/api/v1/repo/branch.go old mode 100644 new mode 100755 index 57c74d7da..f4d1c924c --- a/routers/api/v1/repo/branch.go +++ b/routers/api/v1/repo/branch.go @@ -204,7 +204,7 @@ func ListBranches(ctx *context.APIContext) { // "200": // "$ref": "#/responses/BranchList" - branches, err := repo_module.GetBranches(ctx.Repo.Repository) + branches, _, err := repo_module.GetBranches(ctx.Repo.Repository,0,0) if err != nil { ctx.Error(http.StatusInternalServerError, "GetBranches", err) return diff --git a/routers/api/v1/repo/repo_dashbord.go b/routers/api/v1/repo/repo_dashbord.go index 2926fc4f2..a8887a744 100644 --- a/routers/api/v1/repo/repo_dashbord.go +++ b/routers/api/v1/repo/repo_dashbord.go @@ -5,6 +5,7 @@ import ( "net/http" "net/url" "strconv" + "strings" "time" "github.com/360EntSecGroup-Skylar/excelize/v2" @@ -467,7 +468,7 @@ func generateCountSql(beginTime time.Time, endTime time.Time, latestDate string, "(SELECT repo_id,name,is_private,radar_total from public.repo_statistic where date='" + latestDate + "') B" + " where A.repo_id=B.repo_id" if q != "" { - countSql = countSql + " and B.name like '%" + q + "%'" + countSql = countSql + " and LOWER(B.name) like '%" + strings.ToLower(q) + "%'" } return countSql } @@ -488,7 +489,7 @@ func generateTypeAllSql(beginTime time.Time, endTime time.Time, latestDate strin " where A.repo_id=B.repo_id" if q != "" { - sql = sql + " and name like '%" + q + "%'" + sql = sql + " and LOWER(name) like '%" + strings.ToLower(q) + "%'" } sql = sql + " order by " + orderBy + " desc,repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize) return sql @@ -511,7 +512,7 @@ func generatePageSql(beginTime time.Time, endTime time.Time, latestDate string, "(SELECT repo_id,name,owner_name,is_private,radar_total from public.repo_statistic where date='" + latestDate + "') B" + " where A.repo_id=B.repo_id" if q != "" { - sql = sql + " and B.name like '%" + q + "%'" + sql = sql + " and LOWER(B.name) like '%" + strings.ToLower(q) + "%'" } sql = sql + " order by " + orderBy + " desc,A.repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize) return sql diff --git a/routers/init.go b/routers/init.go index 26829d228..8b93b64d8 100755 --- a/routers/init.go +++ b/routers/init.go @@ -60,11 +60,17 @@ func NewServices() { if err := storage.Init(); err != nil { log.Fatal("storage init failed: %v", err) } + log.Info("storage init succeed.") mailer.NewContext() + log.Info("mailer.NewContext() succeed.") _ = cache.NewContext() + log.Info("cache.NewContext() succeed.") notification.NewContext() + log.Info("notification.NewContext() succeed.") decompression.NewContext() + log.Info("decompression.NewContext() succeed.") labelmsg.Init() + log.Info("labelmsg.Init() succeed.") } // In case of problems connecting to DB, retry connection. Eg, PGSQL in Docker Container on Synology diff --git a/routers/repo/branch.go b/routers/repo/branch.go old mode 100644 new mode 100755 index e7eac04bc..c8e492373 --- a/routers/repo/branch.go +++ b/routers/repo/branch.go @@ -181,7 +181,7 @@ func deleteBranch(ctx *context.Context, branchName string) error { } func loadBranches(ctx *context.Context) []*Branch { - rawBranches, err := repo_module.GetBranches(ctx.Repo.Repository) + rawBranches, _, err := repo_module.GetBranches(ctx.Repo.Repository, 0, 0) if err != nil { ctx.ServerError("GetBranches", err) return nil diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 28f3a0184..b2ded0752 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -74,12 +74,13 @@ func CloudBrainIndex(ctx *context.Context) { timestamp := time.Now().Unix() for i, task := range ciTasks { if task.Status == string(models.JobRunning) && (timestamp-int64(task.Cloudbrain.CreatedUnix) > 10) { - ciTasks[i].CanDebug = true + ciTasks[i].CanDebug = cloudbrain.CanCreateOrDebugJob(ctx) } else { ciTasks[i].CanDebug = false } - ciTasks[i].CanDel = models.CanDelJob(ctx.IsSigned, ctx.User, task) + ciTasks[i].CanDel = cloudbrain.CanDeleteDebugJob(ctx, &task.Cloudbrain) + } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) @@ -88,6 +89,7 @@ func CloudBrainIndex(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true ctx.Data["Tasks"] = ciTasks + ctx.Data["CanCreate"] = cloudbrain.CanCreateOrDebugJob(ctx) ctx.HTML(200, tplCloudBrainIndex) } @@ -216,7 +218,22 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { return } - _, err := models.GetCloudbrainByName(jobName) + count, err := models.GetCloudbrainCountByUserID(ctx.User.ID) + if err != nil { + log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tplCloudBrainNew, &form) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplCloudBrainNew, &form) + return + } + } + + _, err = models.GetCloudbrainByName(jobName) if err == nil { log.Error("the job name did already exist", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) @@ -437,14 +454,22 @@ func StopJobs(cloudBrains []*models.Cloudbrain) { logErrorAndUpdateJobStatus(err, taskInfo) } else { - param := models.NotebookAction{ - Action: models.ActionStop, + if taskInfo.JobType == string(models.JobTypeTrain) { + err := retry(3, time.Second*30, func() error { + _, err := modelarts.StopTrainJob(taskInfo.JobID, strconv.FormatInt(taskInfo.VersionID, 10)) + return err + }) + logErrorAndUpdateJobStatus(err, taskInfo) + } else { + param := models.NotebookAction{ + Action: models.ActionStop, + } + err := retry(3, time.Second*30, func() error { + _, err := modelarts.StopJob(taskInfo.JobID, param) + return err + }) + logErrorAndUpdateJobStatus(err, taskInfo) } - err := retry(3, time.Second*30, func() error { - _, err := modelarts.StopJob(taskInfo.JobID, param) - return err - }) - logErrorAndUpdateJobStatus(err, taskInfo) } } @@ -715,3 +740,78 @@ func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benc return nil } + +func SyncCloudbrainStatus() { + cloudBrains, err := models.GetCloudBrainUnStoppedJob() + if err != nil { + log.Error("GetCloudBrainUnStoppedJob failed:", err.Error()) + return + } + + for _, task := range cloudBrains { + if task.Type == models.TypeCloudBrainOne { + result, err := cloudbrain.GetJob(task.JobID) + if err != nil { + log.Error("GetJob(%s) failed:%v", task.JobName, err) + continue + } + + if result != nil { + jobRes, _ := models.ConvertToJobResultPayload(result.Payload) + taskRoles := jobRes.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + task.Status = taskRes.TaskStatuses[0].State + if task.Status != string(models.JobWaiting) { + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + continue + } + } + } + } else if task.Type == models.TypeCloudBrainTwo { + if task.JobType == string(models.JobTypeDebug) { + result, err := modelarts.GetJob(task.JobID) + if err != nil { + log.Error("GetJob(%s) failed:%v", task.JobName, err) + continue + } + + if result != nil { + task.Status = result.Status + + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + continue + } + } + } else if task.JobType == string(models.JobTypeTrain) { + result, err := modelarts.GetTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10)) + if err != nil { + log.Error("GetTrainJob(%s) failed:%v", task.JobName, err) + continue + } + + if result != nil { + task.Status = modelarts.TransTrainJobStatus(result.IntStatus) + task.Duration = result.Duration + task.TrainJobDuration = result.TrainJobDuration + + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + continue + } + } + } else { + log.Error("task.JobType(%s) is error:%s", task.JobName, task.JobType) + } + + } else { + log.Error("task.Type(%s) is error:%d", task.JobName, task.Type) + } + } + + return +} diff --git a/routers/repo/compare.go b/routers/repo/compare.go old mode 100644 new mode 100755 index 97bb5e6b1..babe416a7 --- a/routers/repo/compare.go +++ b/routers/repo/compare.go @@ -507,7 +507,7 @@ func getBranchesForRepo(user *models.User, repo *models.Repository) (bool, []str } defer gitRepo.Close() - branches, err := gitRepo.GetBranches() + branches, _, err := gitRepo.GetBranches(0, 0) if err != nil { return false, nil, err } @@ -528,7 +528,7 @@ func CompareDiff(ctx *context.Context) { } if ctx.Data["PageIsComparePull"] == true { - headBranches, err := headGitRepo.GetBranches() + headBranches, _, err := headGitRepo.GetBranches(0,0) if err != nil { ctx.ServerError("GetBranches", err) return diff --git a/routers/repo/issue.go b/routers/repo/issue.go index 555cd065f..77ed0251d 100755 --- a/routers/repo/issue.go +++ b/routers/repo/issue.go @@ -424,7 +424,7 @@ func RetrieveRepoMetas(ctx *context.Context, repo *models.Repository, isPull boo return nil } - brs, err := ctx.Repo.GitRepo.GetBranches() + brs, _, err := ctx.Repo.GitRepo.GetBranches(0,0) if err != nil { ctx.ServerError("GetBranches", err) return nil @@ -1390,7 +1390,7 @@ func isLegalReviewRequest(reviewer, doer *models.User, isAdd bool, issue *models var pemResult bool if isAdd { - pemResult = permReviewer.CanAccessAny(models.AccessModeRead, models.UnitTypePullRequests) + pemResult = permReviewer.CanAccessAny(models.AccessModeWrite, models.UnitTypePullRequests) if !pemResult { return fmt.Errorf("Reviewer can't read [user_id: %d, repo_name: %s]", reviewer.ID, issue.Repo.Name) } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 0bd15d921..ca3d1cfac 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -11,6 +11,8 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" @@ -124,10 +126,11 @@ func NotebookIndex(ctx *context.Context) { for i, task := range ciTasks { if task.Status == string(models.JobRunning) { - ciTasks[i].CanDebug = true + ciTasks[i].CanDebug = cloudbrain.CanCreateOrDebugJob(ctx) } else { ciTasks[i].CanDebug = false } + ciTasks[i].CanDel = cloudbrain.CanDeleteDebugJob(ctx, &task.Cloudbrain) } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) @@ -136,6 +139,7 @@ func NotebookIndex(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true ctx.Data["Tasks"] = ciTasks + ctx.Data["CanCreate"] = cloudbrain.CanCreateOrDebugJob(ctx) ctx.HTML(200, tplModelArtsNotebookIndex) } @@ -171,7 +175,22 @@ func NotebookCreate(ctx *context.Context, form auth.CreateModelArtsNotebookForm) description := form.Description flavor := form.Flavor - err := modelarts.GenerateTask(ctx, jobName, uuid, description, flavor) + count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID) + if err != nil { + log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system error", tplModelArtsNotebookNew, &form) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsNotebookNew, &form) + return + } + } + + err = modelarts.GenerateTask(ctx, jobName, uuid, description, flavor) if err != nil { ctx.RenderWithErr(err.Error(), tplModelArtsNotebookNew, &form) return @@ -342,12 +361,18 @@ func TrainJobIndex(ctx *context.Context) { return } + for i, task := range tasks { + tasks[i].CanDel = cloudbrain.CanDeleteTrainJob(ctx, &task.Cloudbrain) + tasks[i].CanModify = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain) + } + pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) pager.SetDefaultParams(ctx) ctx.Data["Page"] = pager ctx.Data["PageIsCloudBrain"] = true ctx.Data["Tasks"] = tasks + ctx.Data["CanCreate"] = cloudbrain.CanCreateOrDebugJob(ctx) ctx.HTML(200, tplModelArtsTrainJobIndex) } @@ -416,16 +441,8 @@ func trainJobNewDataPrepare(ctx *context.Context) error { outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath ctx.Data["train_url"] = outputObsPath - - Branches, err := ctx.Repo.GitRepo.GetBranches() - if err != nil { - ctx.ServerError("GetBranches error:", err) - return err - } - ctx.Data["Branches"] = Branches - ctx.Data["BranchesCount"] = len(Branches) ctx.Data["params"] = "" - ctx.Data["BranchName"] = ctx.Repo.BranchName + ctx.Data["branchName"] = ctx.Repo.BranchName configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom) if err != nil { @@ -494,14 +511,6 @@ func trainJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArts outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath ctx.Data["train_url"] = outputObsPath - Branches, err := ctx.Repo.GitRepo.GetBranches() - if err != nil { - ctx.ServerError("GetBranches error:", err) - return err - } - ctx.Data["Branches"] = Branches - ctx.Data["BranchesCount"] = len(Branches) - configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom) if err != nil { ctx.ServerError("getConfigList failed:", err) @@ -597,13 +606,13 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error { outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath ctx.Data["train_url"] = outputObsPath - Branches, err := ctx.Repo.GitRepo.GetBranches() + branches, _, err := ctx.Repo.GitRepo.GetBranches(0, 0) if err != nil { ctx.ServerError("GetBranches error:", err) return err } - ctx.Data["branches"] = Branches + ctx.Data["branches"] = branches ctx.Data["branch_name"] = task.BranchName ctx.Data["description"] = task.Description ctx.Data["boot_file"] = task.BootFile @@ -686,12 +695,12 @@ func versionErrorDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrai outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath ctx.Data["train_url"] = outputObsPath - Branches, err := ctx.Repo.GitRepo.GetBranches() + branches, _, err := ctx.Repo.GitRepo.GetBranches(0, 0) if err != nil { ctx.ServerError("GetBranches error:", err) return err } - ctx.Data["branches"] = Branches + ctx.Data["branches"] = branches ctx.Data["description"] = form.Description ctx.Data["dataset_name"] = task.DatasetName ctx.Data["work_server_number"] = form.WorkServerNumber @@ -739,6 +748,21 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) VersionCount := modelarts.VersionCount EngineName := form.EngineName + count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) + if err != nil { + log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) + trainJobErrorNewDataPrepare(ctx, form) + ctx.RenderWithErr("system error", tplModelArtsTrainJobNew, &form) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + trainJobErrorNewDataPrepare(ctx, form) + ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsTrainJobNew, &form) + return + } + } + if err := paramCheckCreateTrainJob(form); err != nil { log.Error("paramCheckCreateTrainJob failed:(%v)", err) trainJobErrorNewDataPrepare(ctx, form) @@ -891,7 +915,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) return } - err := modelarts.GenerateTrainJob(ctx, req) + err = modelarts.GenerateTrainJob(ctx, req) if err != nil { log.Error("GenerateTrainJob failed:%v", err.Error()) trainJobErrorNewDataPrepare(ctx, form) @@ -905,6 +929,21 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ ctx.Data["PageIsTrainJob"] = true var jobID = ctx.Params(":jobid") + count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) + if err != nil { + log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) + versionErrorDataPrepare(ctx, form) + ctx.RenderWithErr("system error", tplModelArtsTrainJobVersionNew, &form) + return + } else { + if count >= 1 { + log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) + versionErrorDataPrepare(ctx, form) + ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsTrainJobVersionNew, &form) + return + } + } + latestTask, err := models.GetCloudbrainByJobIDAndIsLatestVersion(jobID, modelarts.IsLatestVersion) if err != nil { ctx.ServerError("GetCloudbrainByJobIDAndIsLatestVersion faild:", err) diff --git a/routers/repo/repo_statistic.go b/routers/repo/repo_statistic.go index 92e29861d..11b421659 100755 --- a/routers/repo/repo_statistic.go +++ b/routers/repo/repo_statistic.go @@ -208,7 +208,7 @@ func RepoStatisticDaily(date string) { maxRepoRadar.Completeness = tempRepoStat.Completeness } - if tempRepoStat.Liveness < minRepoRadar.Completeness { + if tempRepoStat.Liveness < minRepoRadar.Liveness { minRepoRadar.Liveness = tempRepoStat.Liveness } diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 9aae774b6..9552b3139 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -12,6 +12,8 @@ import ( "text/template" "time" + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/routers/operation" "code.gitea.io/gitea/routers/private" @@ -792,7 +794,7 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqSignIn, context.RepoAssignment(), context.UnitTypes(), reqRepoAdmin, context.RepoRef()) m.Post("/:username/:reponame/action/:action", reqSignIn, context.RepoAssignment(), context.UnitTypes(), repo.Action) - m.Get("/tool/query_user_static_page", adminReq, repo.QueryUserStaticDataPage) + // Grouping for those endpoints not requiring authentication m.Group("/:username/:reponame", func() { m.Get("/contributors", repo.Contributors) @@ -957,15 +959,15 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("", reqRepoCloudBrainReader, repo.CloudBrainIndex) m.Group("/:jobid", func() { m.Get("", reqRepoCloudBrainReader, repo.CloudBrainShow) - m.Get("/debug", reqRepoCloudBrainReader, repo.CloudBrainDebug) - m.Post("/commit_image", reqRepoCloudBrainWriter, bindIgnErr(auth.CommitImageCloudBrainForm{}), repo.CloudBrainCommitImage) - m.Post("/stop", reqRepoCloudBrainWriter, repo.CloudBrainStop) - m.Post("/del", reqRepoCloudBrainWriter, repo.CloudBrainDel) + m.Get("/debug", reqRepoCloudBrainWriter, repo.CloudBrainDebug) + m.Post("/commit_image", cloudbrain.AdminOrOwnerOrJobCreaterRight, bindIgnErr(auth.CommitImageCloudBrainForm{}), repo.CloudBrainCommitImage) + m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainStop) + m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainDel) m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate) m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) - m.Get("/download_model", reqRepoCloudBrainReader, repo.CloudBrainDownloadModel) + m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainDownloadModel) }) - m.Get("/create", reqRepoCloudBrainReader, repo.CloudBrainNew) + m.Get("/create", reqRepoCloudBrainWriter, repo.CloudBrainNew) m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) }, context.RepoRef()) @@ -978,9 +980,9 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("", reqRepoCloudBrainReader, repo.NotebookIndex) m.Group("/:jobid", func() { m.Get("", reqRepoCloudBrainReader, repo.NotebookShow) - m.Get("/debug", reqRepoCloudBrainReader, repo.NotebookDebug) - m.Post("/stop", reqRepoCloudBrainWriter, repo.NotebookStop) - m.Post("/del", reqRepoCloudBrainWriter, repo.NotebookDel) + m.Get("/debug", reqRepoCloudBrainWriter, repo.NotebookDebug) + m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.NotebookStop) + m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.NotebookDel) }) m.Get("/create", reqRepoCloudBrainWriter, repo.NotebookNew) m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsNotebookForm{}), repo.NotebookCreate) @@ -990,13 +992,13 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("", reqRepoCloudBrainReader, repo.TrainJobIndex) m.Group("/:jobid", func() { m.Get("", reqRepoCloudBrainReader, repo.TrainJobShow) - m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop) - m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel) - m.Get("/model_download", reqRepoCloudBrainReader, repo.ModelDownload) - m.Get("/create_version", reqRepoCloudBrainReader, repo.TrainJobNewVersion) - m.Post("/create_version", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) + m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.TrainJobStop) + m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.TrainJobDel) + m.Get("/model_download", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.ModelDownload) + m.Get("/create_version", cloudbrain.AdminOrJobCreaterRight, repo.TrainJobNewVersion) + m.Post("/create_version", cloudbrain.AdminOrJobCreaterRight, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) }) - m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNew) + m.Get("/create", reqRepoCloudBrainWriter, repo.TrainJobNew) m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate) m.Get("/para-config-list", reqRepoCloudBrainReader, repo.TrainJobGetConfigList) diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go old mode 100644 new mode 100755 index 165e7cd35..924574471 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -252,7 +252,7 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { } } - branches, err := repo_module.GetBranches(m.Repo) + branches, _, err := repo_module.GetBranches(m.Repo,0,0) if err != nil { log.Error("GetBranches: %v", err) return nil, false diff --git a/services/pull/pull.go b/services/pull/pull.go old mode 100644 new mode 100755 index fb4af0637..230a9e389 --- a/services/pull/pull.go +++ b/services/pull/pull.go @@ -452,7 +452,7 @@ func CloseBranchPulls(doer *models.User, repoID int64, branch string) error { // CloseRepoBranchesPulls close all pull requests which head branches are in the given repository func CloseRepoBranchesPulls(doer *models.User, repo *models.Repository) error { - branches, err := git.GetBranchesByPath(repo.RepoPath()) + branches, _, err := git.GetBranchesByPath(repo.RepoPath(), 0, 0) if err != nil { return err } diff --git a/templates/base/head_navbar.tmpl b/templates/base/head_navbar.tmpl index 49b3181e7..299832f2d 100755 --- a/templates/base/head_navbar.tmpl +++ b/templates/base/head_navbar.tmpl @@ -17,47 +17,57 @@ {{if .IsSigned}} - {{.i18n.Tr "index"}} - {{.i18n.Tr "custom.head.openi"}} - {{if not .UnitIssuesGlobalDisabled}} - {{.i18n.Tr "issues"}} - {{end}} - {{if not .UnitPullsGlobalDisabled}} - {{.i18n.Tr "pull_requests"}} - {{end}} - {{if not (and .UnitIssuesGlobalDisabled .UnitPullsGlobalDisabled)}} - {{if .ShowMilestonesDashboardPage}}{{.i18n.Tr "milestones"}}{{end}} - {{end}} + + + {{.i18n.Tr "custom.head.project"}} + {{.i18n.Tr "custom.head.dataset"}} + {{else if .IsLandingPageHome}} - {{.i18n.Tr "home"}} - {{.i18n.Tr "custom.head.openi"}} + + + {{.i18n.Tr "custom.head.project"}} + {{.i18n.Tr "custom.head.dataset"}} {{else if .IsLandingPageExplore}} @@ -78,7 +88,18 @@ */}} {{if .IsSigned}} -