diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 8bc971f2c..d294c8823 100755 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -1059,50 +1059,50 @@ RESULT_BACKEND = redis://localhost:6379 [cloudbrain] USERNAME = PASSWORD = -REST_SERVER_HOST = http://192.168.202.73 -JOB_PATH = /datasets/minio/data/opendata/jobs/ -DEBUG_SERVER_HOST = http://192.168.202.73/ +REST_SERVER_HOST = +JOB_PATH = +DEBUG_SERVER_HOST = ; cloudbrain visit opendata -USER = cW4cMtH24eoWPE7X -PWD = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DA.C -GPU_TYPE_DEFAULT = openidebug -GPU_TYPES = {"gpu_type":[{"id":1,"queue":"openidebug","value":"T4"},{"id":2,"queue":"openidgx","value":"V100"}]} +USER = +PWD = +GPU_TYPE_DEFAULT = +GPU_TYPES = [benchmark] -ENABLED = true -BENCHMARKCODE = https://yangzhx:justfortest123@git.openi.org.cn/yangzhx/detection_benchmark_script.git -HOST = http://192.168.202.90:3366/ +ENABLED = +BENCHMARKCODE = +HOST = [snn4imagenet] -ENABLED = true -SNN4IMAGENETCODE = https://yult:eh2Ten4iLYjFkbj@git.openi.org.cn/ylt/snn4imagenet.git -HOST = http://192.168.202.90:3366/ +ENABLED = +SNN4IMAGENETCODE = +HOST = [decompress] -HOST = http://192.168.207.34:39987 -USER = cW4cMtH24eoWPE7X -PASSWORD = 4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC +HOST = +USER = +PASSWORD = [blockchain] -HOST = http://192.168.207.84:3002/ -COMMIT_VALID_DATE = 2021-01-15 +HOST = +COMMIT_VALID_DATE = [obs] -ENDPOINT = https://obs.cn-south-222.ai.pcl.cn -ACCESS_KEY_ID = FDP3LRMHLB9S77VWEHE3 -SECRET_ACCESS_KEY = LyM82Wk80pgjhs2z7AdDcsdpCWhbsJtSzQ7hkESN -BUCKET = testopendata -LOCATION = cn-south-222 -BASE_PATH = attachment/ +ENDPOINT = +ACCESS_KEY_ID = +SECRET_ACCESS_KEY = +BUCKET = +LOCATION = +BASE_PATH = [modelarts] -ORGANIZATION = modelarts -ENDPOINT = https://modelarts.cn-south-222.ai.pcl.cn -PROJECT_ID = edfccf24aace4e17a56da6bcbb55a5aa -PROJECT_NAME = cn-south-222_test -USERNAME = test1 -PASSWORD = Qizhi@test. -DOMAIN = cn-south-222 +ORGANIZATION = +ENDPOINT = +PROJECT_ID = +PROJECT_NAME = +USERNAME = +PASSWORD = +DOMAIN = [radar_map] impact=0.3 diff --git a/models/ai_model_manage.go b/models/ai_model_manage.go new file mode 100644 index 000000000..581b19a9c --- /dev/null +++ b/models/ai_model_manage.go @@ -0,0 +1,202 @@ +package models + +import ( + "fmt" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/timeutil" + "xorm.io/builder" + "xorm.io/xorm" +) + +type AiModelManage struct { + ID string `xorm:"pk"` + Name string `xorm:"INDEX NOT NULL"` + Version string `xorm:"NOT NULL"` + VersionCount int `xorm:"NOT NULL DEFAULT 0"` + New int `xorm:"NOT NULL"` + Type int `xorm:"NOT NULL"` + Size int64 `xorm:"NOT NULL"` + Description string `xorm:"varchar(2000)"` + Label string `xorm:"varchar(1000)"` + Path string `xorm:"varchar(400) NOT NULL"` + DownloadCount int `xorm:"NOT NULL DEFAULT 0"` + Engine int64 `xorm:"NOT NULL DEFAULT 0"` + Status int `xorm:"NOT NULL DEFAULT 0"` + Accuracy string `xorm:"varchar(1000)"` + AttachmentId string `xorm:"NULL"` + RepoId int64 `xorm:"INDEX NULL"` + CodeBranch string `xorm:"varchar(400) NULL"` + CodeCommitID string `xorm:"NULL"` + UserId int64 `xorm:"NOT NULL"` + UserName string + UserRelAvatarLink string + TrainTaskInfo string `xorm:"text NULL"` + CreatedUnix timeutil.TimeStamp `xorm:"created"` + UpdatedUnix timeutil.TimeStamp `xorm:"updated"` + IsCanOper bool +} + +type AiModelQueryOptions struct { + ListOptions + RepoID int64 // include all repos if empty + UserID int64 + ModelID string + SortType string + New int + // JobStatus CloudbrainStatus + Type int +} + +func SaveModelToDb(model *AiModelManage) error { + sess := x.NewSession() + defer sess.Close() + + re, err := sess.Insert(model) + if err != nil { + log.Info("insert error." + err.Error()) + return err + } + log.Info("success to save db.re=" + fmt.Sprint((re))) + return nil +} + +func QueryModelById(id string) (*AiModelManage, error) { + sess := x.NewSession() + defer sess.Close() + sess.Select("*").Table("ai_model_manage"). + Where("id='" + id + "'") + aiModelManageList := make([]*AiModelManage, 0) + err := sess.Find(&aiModelManageList) + if err == nil { + if len(aiModelManageList) == 1 { + return aiModelManageList[0], nil + } + } + return nil, err +} + +func DeleteModelById(id string) error { + sess := x.NewSession() + defer sess.Close() + + re, err := sess.Delete(&AiModelManage{ + ID: id, + }) + if err != nil { + return err + } + log.Info("success to delete from db.re=" + fmt.Sprint((re))) + return nil + +} + +func ModifyModelDescription(id string, description string) error { + var sess *xorm.Session + sess = x.ID(id) + defer sess.Close() + re, err := sess.Cols("description").Update(&AiModelManage{ + Description: description, + }) + if err != nil { + return err + } + log.Info("success to update description from db.re=" + fmt.Sprint((re))) + return nil +} + +func ModifyModelNewProperty(id string, new int, versioncount int) error { + var sess *xorm.Session + sess = x.ID(id) + defer sess.Close() + re, err := sess.Cols("new", "version_count").Update(&AiModelManage{ + New: new, + VersionCount: versioncount, + }) + if err != nil { + return err + } + log.Info("success to update new property from db.re=" + fmt.Sprint((re))) + return nil +} + +func ModifyModelDownloadCount(id string) error { + sess := x.NewSession() + defer sess.Close() + if _, err := sess.Exec("UPDATE `ai_model_manage` SET download_count = download_count + 1 WHERE id = ?", id); err != nil { + return err + } + + return nil +} + +func QueryModelByName(name string, repoId int64) []*AiModelManage { + sess := x.NewSession() + defer sess.Close() + sess.Select("*").Table("ai_model_manage"). + Where("name='" + name + "' and repo_id=" + fmt.Sprint(repoId)).OrderBy("created_unix desc") + aiModelManageList := make([]*AiModelManage, 0) + sess.Find(&aiModelManageList) + return aiModelManageList +} + +func QueryModel(opts *AiModelQueryOptions) ([]*AiModelManage, int64, error) { + sess := x.NewSession() + defer sess.Close() + + var cond = builder.NewCond() + if opts.RepoID > 0 { + cond = cond.And( + builder.Eq{"ai_model_manage.repo_id": opts.RepoID}, + ) + } + + if opts.UserID > 0 { + cond = cond.And( + builder.Eq{"ai_model_manage.user_id": opts.UserID}, + ) + } + + if opts.New >= 0 { + cond = cond.And( + builder.Eq{"ai_model_manage.new": opts.New}, + ) + } + + if len(opts.ModelID) > 0 { + cond = cond.And( + builder.Eq{"ai_model_manage.id": opts.ModelID}, + ) + } + + if (opts.Type) >= 0 { + cond = cond.And( + builder.Eq{"ai_model_manage.type": opts.Type}, + ) + } + + count, err := sess.Where(cond).Count(new(AiModelManage)) + if err != nil { + return nil, 0, fmt.Errorf("Count: %v", err) + } + + if opts.Page >= 0 && opts.PageSize > 0 { + var start int + if opts.Page == 0 { + start = 0 + } else { + start = (opts.Page - 1) * opts.PageSize + } + sess.Limit(opts.PageSize, start) + } + + sess.OrderBy("ai_model_manage.created_unix DESC") + aiModelManages := make([]*AiModelManage, 0, setting.UI.IssuePagingNum) + if err := sess.Table("ai_model_manage").Where(cond). + Find(&aiModelManages); err != nil { + return nil, 0, fmt.Errorf("Find: %v", err) + } + + return aiModelManages, count, nil +} diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 7b196a321..509f4a9ed 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -91,6 +91,7 @@ type Cloudbrain struct { DeletedAt time.Time `xorm:"deleted"` CanDebug bool `xorm:"-"` CanDel bool `xorm:"-"` + CanModify bool `xorm:"-"` Type int VersionID int64 //版本id @@ -928,6 +929,58 @@ func Cloudbrains(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) { return cloudbrains, count, nil } +func QueryModelTrainJobVersionList(jobId string) ([]*CloudbrainInfo, int, error) { + sess := x.NewSession() + defer sess.Close() + + var cond = builder.NewCond() + + cond = cond.And( + builder.Eq{"cloudbrain.job_id": jobId}, + ) + cond = cond.And( + builder.Eq{"cloudbrain.Status": "COMPLETED"}, + ) + + sess.OrderBy("cloudbrain.created_unix DESC") + cloudbrains := make([]*CloudbrainInfo, 0) + if err := sess.Table(&Cloudbrain{}).Where(cond). + Find(&cloudbrains); err != nil { + return nil, 0, fmt.Errorf("Find: %v", err) + } + + return cloudbrains, int(len(cloudbrains)), nil +} + +func QueryModelTrainJobList(repoId int64) ([]*CloudbrainInfo, int, error) { + sess := x.NewSession() + defer sess.Close() + var cond = builder.NewCond() + cond = cond.And( + builder.Eq{"repo_id": repoId}, + ) + cond = cond.And( + builder.Eq{"Status": "COMPLETED"}, + ) + + cloudbrains := make([]*CloudbrainInfo, 0) + if err := sess.Select("job_id,job_name").Table(&Cloudbrain{}).Where(cond).OrderBy("created_unix DESC"). + Find(&cloudbrains); err != nil { + return nil, 0, fmt.Errorf("Find: %v", err) + } + + keys := make(map[string]string) + uniqueElements := make([]*CloudbrainInfo, 0) + for _, entry := range cloudbrains { + if _, value := keys[entry.JobID]; !value { + keys[entry.JobID] = entry.JobName + uniqueElements = append(uniqueElements, entry) + } + } + + return uniqueElements, int(len(uniqueElements)), nil +} + func CloudbrainsVersionList(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int, error) { sess := x.NewSession() defer sess.Close() @@ -1032,13 +1085,13 @@ func GetCloudbrainByJobIDAndIsLatestVersion(jobID string, isLatestVersion string func GetCloudbrainsNeededStopByUserID(userID int64) ([]*Cloudbrain, error) { cloudBrains := make([]*Cloudbrain, 0) - err := x.Cols("job_id", "status", "type").Where("user_id=? AND status !=?", userID, string(JobStopped)).Find(&cloudBrains) + err := x.Cols("job_id", "status", "type", "job_type", "version_id").Where("user_id=? AND status !=?", userID, string(JobStopped)).Find(&cloudBrains) return cloudBrains, err } func GetCloudbrainsNeededStopByRepoID(repoID int64) ([]*Cloudbrain, error) { cloudBrains := make([]*Cloudbrain, 0) - err := x.Cols("job_id", "status", "type").Where("repo_id=? AND status !=?", repoID, string(JobStopped)).Find(&cloudBrains) + err := x.Cols("job_id", "status", "type", "job_type", "version_id").Where("repo_id=? AND status !=?", repoID, string(JobStopped)).Find(&cloudBrains) return cloudBrains, err } @@ -1065,9 +1118,7 @@ func UpdateJob(job *Cloudbrain) error { } func updateJob(e Engine, job *Cloudbrain) error { - var sess *xorm.Session - sess = e.Where("job_id = ?", job.JobID) - _, err := sess.Cols("status", "container_id", "container_ip").Update(job) + _, err := e.ID(job.ID).AllCols().Update(job) return err } diff --git a/models/dataset.go b/models/dataset.go index 402a548ef..2b3de752b 100755 --- a/models/dataset.go +++ b/models/dataset.go @@ -93,6 +93,7 @@ type SearchDatasetOptions struct { IncludePublic bool ListOptions SearchOrderBy + IsOwner bool } func CreateDataset(dataset *Dataset) (err error) { @@ -150,6 +151,9 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond { } } else if opts.OwnerID > 0 { cond = cond.And(builder.Eq{"repository.owner_id": opts.OwnerID}) + if !opts.IsOwner { + cond = cond.And(builder.Eq{"dataset.status": DatasetStatusPublic}) + } } return cond diff --git a/models/models.go b/models/models.go index 809f9b0ca..e8a71bbd8 100755 --- a/models/models.go +++ b/models/models.go @@ -133,6 +133,7 @@ func init() { new(FileChunk), new(BlockChain), new(RecommendOrg), + new(AiModelManage), ) tablesStatistic = append(tablesStatistic, diff --git a/models/repo.go b/models/repo.go index 1a5cf122c..a4417e4bd 100755 --- a/models/repo.go +++ b/models/repo.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "html/template" + "xorm.io/xorm" "code.gitea.io/gitea/modules/blockchain" @@ -648,53 +649,41 @@ func (repo *Repository) GetAssignees() (_ []*User, err error) { return repo.getAssignees(x) } -func (repo *Repository) getReviewersPrivate(e Engine, doerID, posterID int64) (users []*User, err error) { - users = make([]*User, 0, 20) - - if err = e. - SQL("SELECT * FROM `user` WHERE id in (SELECT user_id FROM `access` WHERE repo_id = ? AND mode >= ? AND user_id NOT IN ( ?, ?)) ORDER BY name", - repo.ID, AccessModeRead, - doerID, posterID). - Find(&users); err != nil { - return nil, err - } - - return users, nil -} - -func (repo *Repository) getReviewersPublic(e Engine, doerID, posterID int64) (_ []*User, err error) { - - users := make([]*User, 0) - - const SQLCmd = "SELECT * FROM `user` WHERE id IN ( " + - "SELECT user_id FROM `access` WHERE repo_id = ? AND mode >= ? AND user_id NOT IN ( ?, ?) " + - "UNION " + - "SELECT user_id FROM `watch` WHERE repo_id = ? AND user_id NOT IN ( ?, ?) AND mode IN (?, ?) " + - ") ORDER BY name" - - if err = e. - SQL(SQLCmd, - repo.ID, AccessModeRead, doerID, posterID, - repo.ID, doerID, posterID, RepoWatchModeNormal, RepoWatchModeAuto). - Find(&users); err != nil { - return nil, err - } - - return users, nil -} - func (repo *Repository) getReviewers(e Engine, doerID, posterID int64) (users []*User, err error) { if err = repo.getOwner(e); err != nil { return nil, err } - if repo.IsPrivate || - (repo.Owner.IsOrganization() && repo.Owner.Visibility == api.VisibleTypePrivate) { - users, err = repo.getReviewersPrivate(x, doerID, posterID) + if repo.Owner.IsOrganization() { + const SQLCmd = "SELECT * FROM `user` WHERE id IN (" + + "SELECT DISTINCT(t3.user_id) FROM ( " + + "SELECT user_id FROM `access` WHERE repo_id = ? AND mode >= ?" + + "UNION select t2.uid as user_id from team t1 left join team_user t2 on t1.id = t2.team_id where t1.org_id = ? and t1.authorize = 4) t3)" + + " AND id NOT IN ( ?, ?) ORDER BY name " + + if err = e. + SQL(SQLCmd, + repo.ID, AccessModeWrite, repo.ID, doerID, posterID). + Find(&users); err != nil { + return nil, err + } } else { - users, err = repo.getReviewersPublic(x, doerID, posterID) + const SQLCmd = "SELECT * FROM `user` WHERE id IN ( " + + "SELECT DISTINCT(t3.user_id) FROM ( " + + "SELECT user_id FROM `access` WHERE repo_id = ? AND mode >= ? " + + " UNION" + + " SELECT owner_id as user_id FROM `repository` WHERE id = ?) t3)" + + " AND id NOT IN ( ?, ?) ORDER BY name " + + if err = e. + SQL(SQLCmd, + repo.ID, AccessModeWrite, repo.ID, doerID, posterID). + Find(&users); err != nil { + return nil, err + } } - return + + return users, nil } // GetReviewers get all users can be requested to review @@ -1118,6 +1107,12 @@ func CreateRepository(ctx DBContext, doer, u *User, repo *Repository) (err error Type: tp, Config: &BlockChainConfig{EnableBlockChain: true}, }) + } else if tp == UnitTypeModelManage { + units = append(units, RepoUnit{ + RepoID: repo.ID, + Type: tp, + Config: &ModelManageConfig{EnableModelManage: true}, + }) } else { units = append(units, RepoUnit{ RepoID: repo.ID, @@ -1630,6 +1625,9 @@ func DeleteRepository(doer *User, uid, repoID int64) error { releaseAttachments = append(releaseAttachments, attachments[i].RelativePath()) } + // Delete dataset attachment record and remove related files + deleteDatasetAttachmentByRepoId(sess, repoID) + if err = deleteBeans(sess, &Access{RepoID: repo.ID}, &Action{RepoID: repo.ID}, @@ -1815,6 +1813,20 @@ func DeleteRepository(doer *User, uid, repoID int64) error { return nil } +func deleteDatasetAttachmentByRepoId(sess *xorm.Session, repoId int64) error { + attachments := make([]*Attachment, 0) + if err := sess.Join("INNER", "dataset", "dataset.id = attachment.dataset_id"). + Where("dataset.repo_id = ?", repoId). + Find(&attachments); err != nil { + return err + } + if len(attachments) == 0 { + return nil + } + _, err := DeleteAttachments(attachments, true) + return err +} + // GetRepositoryByOwnerAndName returns the repository by given ownername and reponame. func GetRepositoryByOwnerAndName(ownerName, repoName string) (*Repository, error) { return getRepositoryByOwnerAndName(x, ownerName, repoName) diff --git a/models/repo_unit.go b/models/repo_unit.go index 518c4b979..5f118029f 100755 --- a/models/repo_unit.go +++ b/models/repo_unit.go @@ -131,6 +131,20 @@ type CloudBrainConfig struct { EnableCloudBrain bool } +type ModelManageConfig struct { + EnableModelManage bool +} + +// FromDB fills up a CloudBrainConfig from serialized format. +func (cfg *ModelManageConfig) FromDB(bs []byte) error { + return json.Unmarshal(bs, &cfg) +} + +// ToDB exports a CloudBrainConfig to a serialized format. +func (cfg *ModelManageConfig) ToDB() ([]byte, error) { + return json.Marshal(cfg) +} + // FromDB fills up a CloudBrainConfig from serialized format. func (cfg *CloudBrainConfig) FromDB(bs []byte) error { return json.Unmarshal(bs, &cfg) @@ -176,6 +190,8 @@ func (r *RepoUnit) BeforeSet(colName string, val xorm.Cell) { r.Config = new(CloudBrainConfig) case UnitTypeBlockChain: r.Config = new(BlockChainConfig) + case UnitTypeModelManage: + r.Config = new(ModelManageConfig) default: panic("unrecognized repo unit type: " + com.ToStr(*val)) } diff --git a/models/unit.go b/models/unit.go index e2b73841a..381491388 100755 --- a/models/unit.go +++ b/models/unit.go @@ -27,6 +27,7 @@ const ( UnitTypeDatasets UnitType = 10 // 10 Dataset UnitTypeCloudBrain UnitType = 11 // 11 CloudBrain UnitTypeBlockChain UnitType = 12 // 12 BlockChain + UnitTypeModelManage UnitType = 13 // 13 ModelManage ) // Value returns integer value for unit type @@ -56,6 +57,8 @@ func (u UnitType) String() string { return "UnitTypeCloudBrain" case UnitTypeBlockChain: return "UnitTypeBlockChain" + case UnitTypeModelManage: + return "UnitTypeModelManage" } return fmt.Sprintf("Unknown UnitType %d", u) } @@ -80,6 +83,7 @@ var ( UnitTypeDatasets, UnitTypeCloudBrain, UnitTypeBlockChain, + UnitTypeModelManage, } // DefaultRepoUnits contains the default unit types @@ -92,6 +96,7 @@ var ( UnitTypeDatasets, UnitTypeCloudBrain, UnitTypeBlockChain, + UnitTypeModelManage, } // NotAllowedDefaultRepoUnits contains units that can't be default @@ -281,6 +286,14 @@ var ( 7, } + UnitModelManage = Unit{ + UnitTypeModelManage, + "repo.modelmanage", + "/modelmanage", + "repo.modelmanage.desc", + 8, + } + // Units contains all the units Units = map[UnitType]Unit{ UnitTypeCode: UnitCode, @@ -293,6 +306,7 @@ var ( UnitTypeDatasets: UnitDataset, UnitTypeCloudBrain: UnitCloudBrain, UnitTypeBlockChain: UnitBlockChain, + UnitTypeModelManage: UnitModelManage, } ) diff --git a/models/user.go b/models/user.go index 1ee20d74c..b362472e8 100755 --- a/models/user.go +++ b/models/user.go @@ -145,6 +145,7 @@ type User struct { AllowImportLocal bool // Allow migrate repository by local path AllowCreateOrganization bool `xorm:"DEFAULT true"` ProhibitLogin bool `xorm:"NOT NULL DEFAULT false"` + IsOperator bool `xorm:"NOT NULL DEFAULT false"` //运营人员 // Avatar Avatar string `xorm:"VARCHAR(2048) NOT NULL"` @@ -928,8 +929,17 @@ var ( "template", "user", "vendor", - } - reservedUserPatterns = []string{"*.keys", "*.gpg"} + "dashboard", + "operation", + "blockchain", + "avatar", + "swagger.v1.json", + "secure", + "serviceworker.js", + "self", + "repo-avatars", + } + reservedUserPatterns = []string{"*.keys", "*.gpg", "*.png"} ) // isUsableName checks if name is reserved or pattern of name is not allowed @@ -1551,11 +1561,11 @@ func GetUserByActivateEmail(email string) (*User, error) { if err := ctx.e.Join("INNER", "email_address", "email_address.uid = \"user\".id"). Where("email_address.email= ?", email). Find(&users); err != nil { - return nil,err + return nil, err } if len(users) >= 1 { - return &users[0],nil - }else { + return &users[0], nil + } else { // Finally, if email address is the protected email address:用户邮件地址设置为隐藏电子邮件地址 if strings.HasSuffix(email, fmt.Sprintf("@%s", setting.Service.NoReplyAddress)) { username := strings.TrimSuffix(email, fmt.Sprintf("@%s", setting.Service.NoReplyAddress)) @@ -1571,6 +1581,7 @@ func GetUserByActivateEmail(email string) (*User, error) { return nil, errors.New("cannot find user by email") } } + // GetUserByEmail returns the user object by given e-mail if exists. func GetUserByEmail(email string) (*User, error) { return GetUserByEmailContext(DefaultDBContext(), email) diff --git a/models/user_business_analysis.go b/models/user_business_analysis.go index 15f67e858..9b9d5e0ad 100644 --- a/models/user_business_analysis.go +++ b/models/user_business_analysis.go @@ -7,10 +7,14 @@ import ( "strconv" "time" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/timeutil" "xorm.io/builder" - "xorm.io/xorm" +) + +const ( + Page_SIZE = 2000 ) type UserBusinessAnalysisAll struct { @@ -199,7 +203,13 @@ func QueryUserStaticDataAll(opts *UserBusinessAnalysisQueryOptions) ([]*UserBusi } log.Info("query return total:" + fmt.Sprint(allCount)) if allCount == 0 { - RefreshUserStaticAllTabel() + CommitCodeSizeMap, err := GetAllUserKPIStats() + if err != nil { + log.Info("query commit code errr.") + } else { + log.Info("query commit code size, len=" + fmt.Sprint(len(CommitCodeSizeMap))) + } + RefreshUserStaticAllTabel(make(map[string]int), CommitCodeSizeMap) } pageSize := 1000 totalPage := int(allCount) / pageSize @@ -336,47 +346,164 @@ func QueryUserStaticDataPage(opts *UserBusinessAnalysisQueryOptions) ([]*UserBus return userBusinessAnalysisReturnList, count } -func RefreshUserStaticAllTabel() { +func RefreshUserStaticAllTabel(wikiCountMap map[string]int, CommitCodeSizeMap map[string]*git.UserKPIStats) { + + sess := x.NewSession() + defer sess.Close() statictisSess := xStatistic.NewSession() defer statictisSess.Close() - log.Info("delete all data from table: user_business_analysis_all") - statictisSess.Exec("delete from user_business_analysis_all") + log.Info("truncate all data from table: user_business_analysis_all") + statictisSess.Exec("TRUNCATE TABLE user_business_analysis_all") currentTimeNow := time.Now() - pageStartTime := getLastCountDate() - pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()).Unix() - var cond = builder.NewCond() - cond = cond.And( - builder.Gte{"count_date": pageStartTime}, - ) - cond = cond.And( - builder.Lte{"count_date": pageEndTime}, - ) - userBusinessAnalysisList := make([]*UserBusinessAnalysis, 0) - if err := statictisSess.Table("user_business_analysis").Where(cond).OrderBy("id desc"). - Find(&userBusinessAnalysisList); err != nil { + startTime := currentTimeNow.AddDate(0, 0, -1) + + pageStartTime := time.Date(2021, 11, 5, 0, 0, 0, 0, currentTimeNow.Location()) + log.Info("pageStartTime:" + pageStartTime.Format("2006-01-02 15:04:05")) + pageEndTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 23, 59, 59, 0, currentTimeNow.Location()) + log.Info("pageEndTime time:" + pageEndTime.Format("2006-01-02 15:04:05")) + + start_unix := pageStartTime.Unix() + end_unix := pageEndTime.Unix() + + CodeMergeCountMap := queryPullRequest(start_unix, end_unix) + CommitCountMap := queryCommitAction(start_unix, end_unix, 5) + IssueCountMap := queryAction(start_unix, end_unix, 6) + + CommentCountMap := queryComment(start_unix, end_unix) + FocusRepoCountMap := queryWatch(start_unix, end_unix) + StarRepoCountMap := queryStar(start_unix, end_unix) + WatchedCountMap := queryFollow(start_unix, end_unix) + + CommitDatasetSizeMap := queryDatasetSize(start_unix, end_unix) + SolveIssueCountMap := querySolveIssue(start_unix, end_unix) + CreateRepoCountMap := queryUserCreateRepo(start_unix, end_unix) + LoginCountMap := queryLoginCount(start_unix, end_unix) + + OpenIIndexMap := queryUserRepoOpenIIndex(startTime.Unix(), end_unix) + + DataDate := currentTimeNow.Format("2006-01-02") + + cond := "type != 1 and is_active=true" + count, err := sess.Where(cond).Count(new(User)) + if err != nil { + log.Info("query user error. return.") return } - log.Info("query all data from table: user_business_analysis,len=" + fmt.Sprint(len(userBusinessAnalysisList))) - for _, userRecord := range userBusinessAnalysisList { - log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(userRecord.ID)) - allData := getAllData(userRecord.ID, statictisSess) - allData.ID = userRecord.ID - allData.CountDate = 0 - allData.DataDate = userRecord.DataDate - allData.Email = userRecord.Email - allData.OpenIIndex = userRecord.OpenIIndex - allData.GiteaAgeMonth = userRecord.GiteaAgeMonth - allData.Name = userRecord.Name - allData.RegistDate = userRecord.RegistDate - - _, err := statictisSess.Insert(&allData) - if err != nil { - log.Info("insert all data failed." + err.Error()) + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("`user`.*").Table("user").Where(cond).Limit(Page_SIZE, int(indexTotal)) + userList := make([]*User, 0) + sess.Find(&userList) + for i, userRecord := range userList { + log.Info("insert all static, i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) + var dateRecordAll UserBusinessAnalysisAll + dateRecordAll.ID = userRecord.ID + dateRecordAll.Email = userRecord.Email + dateRecordAll.RegistDate = userRecord.CreatedUnix + dateRecordAll.Name = userRecord.Name + dateRecordAll.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) + dateRecordAll.DataDate = DataDate + + if _, ok := CodeMergeCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CodeMergeCount = 0 + } else { + dateRecordAll.CodeMergeCount = CodeMergeCountMap[dateRecordAll.ID] + } + + if _, ok := CommitCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CommitCount = 0 + } else { + dateRecordAll.CommitCount = CommitCountMap[dateRecordAll.ID] + } + + if _, ok := IssueCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.IssueCount = 0 + } else { + dateRecordAll.IssueCount = IssueCountMap[dateRecordAll.ID] + } + + if _, ok := CommentCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CommentCount = 0 + } else { + dateRecordAll.CommentCount = CommentCountMap[dateRecordAll.ID] + } + + if _, ok := FocusRepoCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.FocusRepoCount = 0 + } else { + dateRecordAll.FocusRepoCount = FocusRepoCountMap[dateRecordAll.ID] + } + + if _, ok := StarRepoCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.StarRepoCount = 0 + } else { + dateRecordAll.StarRepoCount = StarRepoCountMap[dateRecordAll.ID] + } + + if _, ok := WatchedCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.WatchedCount = 0 + } else { + dateRecordAll.WatchedCount = WatchedCountMap[dateRecordAll.ID] + } + + if _, ok := CommitCodeSizeMap[dateRecordAll.Email]; !ok { + dateRecordAll.CommitCodeSize = 0 + } else { + dateRecordAll.CommitCodeSize = int(CommitCodeSizeMap[dateRecordAll.Email].CommitLines) + } + + if _, ok := CommitDatasetSizeMap[dateRecordAll.ID]; !ok { + dateRecordAll.CommitDatasetSize = 0 + } else { + dateRecordAll.CommitDatasetSize = CommitDatasetSizeMap[dateRecordAll.ID] + } + + if _, ok := SolveIssueCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.SolveIssueCount = 0 + } else { + dateRecordAll.SolveIssueCount = SolveIssueCountMap[dateRecordAll.ID] + } + + if _, ok := wikiCountMap[dateRecordAll.Name]; !ok { + dateRecordAll.EncyclopediasCount = 0 + } else { + dateRecordAll.EncyclopediasCount = wikiCountMap[dateRecordAll.Name] + } + + if _, ok := CreateRepoCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.CreateRepoCount = 0 + } else { + dateRecordAll.CreateRepoCount = CreateRepoCountMap[dateRecordAll.ID] + } + + if _, ok := LoginCountMap[dateRecordAll.ID]; !ok { + dateRecordAll.LoginCount = 0 + } else { + dateRecordAll.LoginCount = LoginCountMap[dateRecordAll.ID] + } + + if _, ok := OpenIIndexMap[dateRecordAll.ID]; !ok { + dateRecordAll.OpenIIndex = 0 + } else { + dateRecordAll.OpenIIndex = OpenIIndexMap[dateRecordAll.ID] + } + + dateRecordAll.CommitModelCount = 0 + _, err = statictisSess.Insert(&dateRecordAll) + if err != nil { + log.Info("insert all data failed." + err.Error()) + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + log.Info("refresh all data finished.") } @@ -385,19 +512,13 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, log.Info("start to count other user info data") sess := x.NewSession() defer sess.Close() - sess.Select("`user`.*").Table("user").Where("type != 1 and is_active=true") - userList := make([]*User, 0) - sess.Find(&userList) currentTimeNow := time.Now() log.Info("current time:" + currentTimeNow.Format("2006-01-02 15:04:05")) - //yesterday := currentTimeNow.AddDate(0, 0, -1) - //startTime := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, yesterday.Location()) start_unix := startTime.Unix() log.Info("DB query time:" + startTime.Format("2006-01-02 15:04:05")) - //endTime := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 0, 0, 0, currentTimeNow.Location()) end_unix := endTime.Unix() CountDate := time.Date(currentTimeNow.Year(), currentTimeNow.Month(), currentTimeNow.Day(), 0, 1, 0, 0, currentTimeNow.Location()) if isReCount { @@ -429,193 +550,134 @@ func CounDataByDateAndReCount(wikiCountMap map[string]int, startTime time.Time, statictisSess := xStatistic.NewSession() defer statictisSess.Close() - for i, userRecord := range userList { - var dateRecord UserBusinessAnalysis - dateRecord.ID = userRecord.ID - log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) - dateRecord.CountDate = CountDate.Unix() - - statictisSess.Delete(&dateRecord) - - dateRecord.Email = userRecord.Email - dateRecord.RegistDate = userRecord.CreatedUnix - dateRecord.Name = userRecord.Name - dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) - dateRecord.DataDate = DataDate - if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { - dateRecord.CodeMergeCount = 0 - } else { - dateRecord.CodeMergeCount = CodeMergeCountMap[dateRecord.ID] - } - - if _, ok := CommitCountMap[dateRecord.ID]; !ok { - dateRecord.CommitCount = 0 - } else { - dateRecord.CommitCount = CommitCountMap[dateRecord.ID] - } + cond := "type != 1 and is_active=true" + count, err := sess.Where(cond).Count(new(User)) + if err != nil { + log.Info("query user error. return.") + return err + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("`user`.*").Table("user").Where(cond).Limit(Page_SIZE, int(indexTotal)) + userList := make([]*User, 0) + sess.Find(&userList) + + for i, userRecord := range userList { + var dateRecord UserBusinessAnalysis + dateRecord.ID = userRecord.ID + log.Info("i=" + fmt.Sprint(i) + " userName=" + userRecord.Name) + dateRecord.CountDate = CountDate.Unix() + + statictisSess.Delete(&dateRecord) + + dateRecord.Email = userRecord.Email + dateRecord.RegistDate = userRecord.CreatedUnix + dateRecord.Name = userRecord.Name + dateRecord.GiteaAgeMonth = subMonth(currentTimeNow, userRecord.CreatedUnix.AsTime()) + dateRecord.DataDate = DataDate + if _, ok := CodeMergeCountMap[dateRecord.ID]; !ok { + dateRecord.CodeMergeCount = 0 + } else { + dateRecord.CodeMergeCount = CodeMergeCountMap[dateRecord.ID] + } - if _, ok := IssueCountMap[dateRecord.ID]; !ok { - dateRecord.IssueCount = 0 - } else { - dateRecord.IssueCount = IssueCountMap[dateRecord.ID] - } + if _, ok := CommitCountMap[dateRecord.ID]; !ok { + dateRecord.CommitCount = 0 + } else { + dateRecord.CommitCount = CommitCountMap[dateRecord.ID] + } - if _, ok := CommentCountMap[dateRecord.ID]; !ok { - dateRecord.CommentCount = 0 - } else { - dateRecord.CommentCount = CommentCountMap[dateRecord.ID] - } + if _, ok := IssueCountMap[dateRecord.ID]; !ok { + dateRecord.IssueCount = 0 + } else { + dateRecord.IssueCount = IssueCountMap[dateRecord.ID] + } - if _, ok := FocusRepoCountMap[dateRecord.ID]; !ok { - dateRecord.FocusRepoCount = 0 - } else { - dateRecord.FocusRepoCount = FocusRepoCountMap[dateRecord.ID] - } + if _, ok := CommentCountMap[dateRecord.ID]; !ok { + dateRecord.CommentCount = 0 + } else { + dateRecord.CommentCount = CommentCountMap[dateRecord.ID] + } - if _, ok := StarRepoCountMap[dateRecord.ID]; !ok { - dateRecord.StarRepoCount = 0 - } else { - dateRecord.StarRepoCount = StarRepoCountMap[dateRecord.ID] - } + if _, ok := FocusRepoCountMap[dateRecord.ID]; !ok { + dateRecord.FocusRepoCount = 0 + } else { + dateRecord.FocusRepoCount = FocusRepoCountMap[dateRecord.ID] + } - if _, ok := WatchedCountMap[dateRecord.ID]; !ok { - dateRecord.WatchedCount = 0 - } else { - dateRecord.WatchedCount = WatchedCountMap[dateRecord.ID] - } + if _, ok := StarRepoCountMap[dateRecord.ID]; !ok { + dateRecord.StarRepoCount = 0 + } else { + dateRecord.StarRepoCount = StarRepoCountMap[dateRecord.ID] + } - if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { - dateRecord.CommitCodeSize = 0 - } else { - dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) - } + if _, ok := WatchedCountMap[dateRecord.ID]; !ok { + dateRecord.WatchedCount = 0 + } else { + dateRecord.WatchedCount = WatchedCountMap[dateRecord.ID] + } - if _, ok := CommitDatasetSizeMap[dateRecord.ID]; !ok { - dateRecord.CommitDatasetSize = 0 - } else { - dateRecord.CommitDatasetSize = CommitDatasetSizeMap[dateRecord.ID] - } + if _, ok := CommitCodeSizeMap[dateRecord.Email]; !ok { + dateRecord.CommitCodeSize = 0 + } else { + dateRecord.CommitCodeSize = int(CommitCodeSizeMap[dateRecord.Email].CommitLines) + } - if _, ok := SolveIssueCountMap[dateRecord.ID]; !ok { - dateRecord.SolveIssueCount = 0 - } else { - dateRecord.SolveIssueCount = SolveIssueCountMap[dateRecord.ID] - } + if _, ok := CommitDatasetSizeMap[dateRecord.ID]; !ok { + dateRecord.CommitDatasetSize = 0 + } else { + dateRecord.CommitDatasetSize = CommitDatasetSizeMap[dateRecord.ID] + } - if _, ok := wikiCountMap[dateRecord.Name]; !ok { - dateRecord.EncyclopediasCount = 0 - } else { - dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name] - } + if _, ok := SolveIssueCountMap[dateRecord.ID]; !ok { + dateRecord.SolveIssueCount = 0 + } else { + dateRecord.SolveIssueCount = SolveIssueCountMap[dateRecord.ID] + } - if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok { - dateRecord.CreateRepoCount = 0 - } else { - dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] - } + if _, ok := wikiCountMap[dateRecord.Name]; !ok { + dateRecord.EncyclopediasCount = 0 + } else { + dateRecord.EncyclopediasCount = wikiCountMap[dateRecord.Name] + } - if _, ok := LoginCountMap[dateRecord.ID]; !ok { - dateRecord.LoginCount = 0 - } else { - dateRecord.LoginCount = LoginCountMap[dateRecord.ID] - } + if _, ok := CreateRepoCountMap[dateRecord.ID]; !ok { + dateRecord.CreateRepoCount = 0 + } else { + dateRecord.CreateRepoCount = CreateRepoCountMap[dateRecord.ID] + } - if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { - dateRecord.OpenIIndex = 0 - } else { - dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] - } + if _, ok := LoginCountMap[dateRecord.ID]; !ok { + dateRecord.LoginCount = 0 + } else { + dateRecord.LoginCount = LoginCountMap[dateRecord.ID] + } - dateRecord.CommitModelCount = 0 + if _, ok := OpenIIndexMap[dateRecord.ID]; !ok { + dateRecord.OpenIIndex = 0 + } else { + dateRecord.OpenIIndex = OpenIIndexMap[dateRecord.ID] + } - _, err = statictisSess.Insert(&dateRecord) - if err != nil { - log.Info("insert daterecord failed." + err.Error()) - return err - } + dateRecord.CommitModelCount = 0 - if isExistUserInAllTable(dateRecord.ID, statictisSess) { - updateCurrentData(dateRecord.ID, statictisSess, dateRecord) - } else { - log.Info("insert to UserBusinessAnalysisAll table,user id=" + fmt.Sprint(dateRecord.ID)) - allData := getAllData(dateRecord.ID, statictisSess) - allData.ID = dateRecord.ID - allData.CountDate = 0 - allData.DataDate = dateRecord.DataDate - allData.Email = dateRecord.Email - allData.OpenIIndex = dateRecord.OpenIIndex - allData.GiteaAgeMonth = dateRecord.GiteaAgeMonth - allData.Name = dateRecord.Name - allData.RegistDate = dateRecord.RegistDate - - _, err = statictisSess.Insert(&allData) + _, err = statictisSess.Insert(&dateRecord) if err != nil { - log.Info("insert all data failed." + err.Error()) + log.Info("insert daterecord failed." + err.Error()) return err } } + indexTotal += Page_SIZE + if indexTotal >= count { + break + } } - return nil -} - -func updateCurrentData(userId int64, statictisSess *xorm.Session, currentData UserBusinessAnalysis) { - - _, err := statictisSess.Update("update user_business_analysis_all set code_merge_count+=" + fmt.Sprint(currentData.CodeMergeCount) + - ",commit_count+=" + fmt.Sprint(currentData.CommitCount) + - ",issue_count+=" + fmt.Sprint(currentData.IssueCount) + - ",comment_count+=" + fmt.Sprint(currentData.CommentCount) + - ",focus_repo_count+=" + fmt.Sprint(currentData.FocusRepoCount) + - ",star_repo_count+=" + fmt.Sprint(currentData.StarRepoCount) + - ",watched_count+=" + fmt.Sprint(currentData.WatchedCount) + - ",commit_code_size+=" + fmt.Sprint(currentData.CommitCodeSize) + - ",commit_dataset_size+=" + fmt.Sprint(currentData.CommitDatasetSize) + - ",commit_model_count+=" + fmt.Sprint(currentData.CommitModelCount) + - ",solve_issue_count+=" + fmt.Sprint(currentData.SolveIssueCount) + - ",encyclopedias_count+=" + fmt.Sprint(currentData.EncyclopediasCount) + - ",create_repo_count+=" + fmt.Sprint(currentData.CreateRepoCount) + - ",login_count+=" + fmt.Sprint(currentData.LoginCount) + - " where id=" + fmt.Sprint(userId)) - - if err != nil { - log.Info("update table failed." + err.Error()) - } - -} -func isExistUserInAllTable(userId int64, statictisSess *xorm.Session) bool { - - allCount, err := statictisSess.Where("id=" + fmt.Sprint(userId)).Count(new(UserBusinessAnalysisAll)) - if err != nil { - return false - } - return allCount > 0 -} + RefreshUserStaticAllTabel(wikiCountMap, CommitCodeSizeMap) -func getAllData(userId int64, statictisSess *xorm.Session) UserBusinessAnalysisAll { - var dateRecord UserBusinessAnalysisAll - - rows, err := statictisSess.Query("select sum(code_merge_count) as code_merge_count,sum(commit_count) as commit_count,sum(issue_count) as issue_count,sum(issue_count) as issue_count,sum(comment_count) as comment_count,sum(focus_repo_count) as focus_repo_count,sum(star_repo_count) as star_repo_count,sum(watched_count) as watched_count,sum(commit_code_size) as commit_code_size,sum(commit_dataset_size) as commit_dataset_size, sum(commit_model_count) as commit_model_count,sum(solve_issue_count) as solve_issue_count,sum(encyclopedias_count) as encyclopedias_count, sum(create_repo_count) as create_repo_count,sum(login_count) as login_count from public.user_business_analysis where id=" + fmt.Sprint(userId) + " group by id") - if err == nil { - for i, row := range rows { - log.Info("query user info, i=" + fmt.Sprint(i) + " code_merge_count=" + string(row["code_merge_count"])) - dateRecord.CodeMergeCount = getInt(string(row["code_merge_count"])) - dateRecord.CommitCount = getInt(string(row["commit_count"])) - dateRecord.IssueCount = getInt(string(row["issue_count"])) - dateRecord.CommentCount = getInt(string(row["comment_count"])) - dateRecord.FocusRepoCount = getInt(string(row["focus_repo_count"])) - dateRecord.StarRepoCount = getInt(string(row["star_repo_count"])) - dateRecord.WatchedCount = getInt(string(row["watched_count"])) - dateRecord.CommitCodeSize = getInt(string(row["commit_code_size"])) - dateRecord.CommitDatasetSize = getInt(string(row["commit_dataset_size"])) - dateRecord.CommitModelCount = getInt(string(row["commit_model_count"])) - dateRecord.SolveIssueCount = getInt(string(row["solve_issue_count"])) - dateRecord.EncyclopediasCount = getInt(string(row["encyclopedias_count"])) - dateRecord.CreateRepoCount = getInt(string(row["create_repo_count"])) - dateRecord.LoginCount = getInt(string(row["login_count"])) - } - } - return dateRecord + return nil } func getInt(str string) int { @@ -631,44 +693,70 @@ func CounDataByDate(wikiCountMap map[string]int, startTime time.Time, endTime ti } func querySolveIssue(start_unix int64, end_unix int64) map[int64]int { - //select issue_assignees.* from issue_assignees,issue where issue.is_closed=true and issue.id=issue_assignees.issue_id sess := x.NewSession() defer sess.Close() - sess.Select("issue_assignees.*").Table("issue_assignees"). - Join("inner", "issue", "issue.id=issue_assignees.issue_id"). - Where("issue.is_closed=true and issue.closed_unix>=" + fmt.Sprint(start_unix) + " and issue.closed_unix<=" + fmt.Sprint(end_unix)) - issueAssigneesList := make([]*IssueAssignees, 0) - sess.Find(&issueAssigneesList) resultMap := make(map[int64]int) - log.Info("query IssueAssignees size=" + fmt.Sprint(len(issueAssigneesList))) - for _, issueAssigneesRecord := range issueAssigneesList { - if _, ok := resultMap[issueAssigneesRecord.AssigneeID]; !ok { - resultMap[issueAssigneesRecord.AssigneeID] = 1 - } else { - resultMap[issueAssigneesRecord.AssigneeID] += 1 + cond := "issue.is_closed=true and issue.closed_unix>=" + fmt.Sprint(start_unix) + " and issue.closed_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Table("issue_assignees").Join("inner", "issue", "issue.id=issue_assignees.issue_id").Where(cond).Count(new(IssueAssignees)) + if err != nil { + log.Info("query issue error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + issueAssigneesList := make([]*IssueAssignees, 0) + sess.Select("issue_assignees.*").Table("issue_assignees"). + Join("inner", "issue", "issue.id=issue_assignees.issue_id"). + Where(cond).Limit(Page_SIZE, int(indexTotal)) + + sess.Find(&issueAssigneesList) + + log.Info("query IssueAssignees size=" + fmt.Sprint(len(issueAssigneesList))) + for _, issueAssigneesRecord := range issueAssigneesList { + if _, ok := resultMap[issueAssigneesRecord.AssigneeID]; !ok { + resultMap[issueAssigneesRecord.AssigneeID] = 1 + } else { + resultMap[issueAssigneesRecord.AssigneeID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } - return resultMap + return resultMap } func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - - sess.Select("issue.*").Table("issue"). - Join("inner", "pull_request", "issue.id=pull_request.issue_id"). - Where("pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix)) - - issueList := make([]*Issue, 0) - sess.Find(&issueList) resultMap := make(map[int64]int) - log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) - for _, issueRecord := range issueList { - if _, ok := resultMap[issueRecord.PosterID]; !ok { - resultMap[issueRecord.PosterID] = 1 - } else { - resultMap[issueRecord.PosterID] += 1 + cond := "pull_request.merged_unix>=" + fmt.Sprint(start_unix) + " and pull_request.merged_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).Count(new(Issue)) + if err != nil { + log.Info("query issue error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + issueList := make([]*Issue, 0) + sess.Select("issue.*").Table("issue").Join("inner", "pull_request", "issue.id=pull_request.issue_id").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Find(&issueList) + log.Info("query issue(PR) size=" + fmt.Sprint(len(issueList))) + for _, issueRecord := range issueList { + if _, ok := resultMap[issueRecord.PosterID]; !ok { + resultMap[issueRecord.PosterID] = 1 + } else { + resultMap[issueRecord.PosterID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap @@ -677,34 +765,68 @@ func queryPullRequest(start_unix int64, end_unix int64) map[int64]int { func queryCommitAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,op_type,act_user_id").Table("action").Where("user_id=act_user_id and op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - actionList := make([]*Action, 0) - sess.Find(&actionList) resultMap := make(map[int64]int) - log.Info("query action size=" + fmt.Sprint(len(actionList))) - for _, actionRecord := range actionList { - if _, ok := resultMap[actionRecord.UserID]; !ok { - resultMap[actionRecord.UserID] = 1 - } else { - resultMap[actionRecord.UserID] += 1 + + cond := "user_id=act_user_id and op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Action)) + if err != nil { + log.Info("query action error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).Limit(Page_SIZE, int(indexTotal)) + actionList := make([]*Action, 0) + sess.Find(&actionList) + + log.Info("query action size=" + fmt.Sprint(len(actionList))) + for _, actionRecord := range actionList { + if _, ok := resultMap[actionRecord.UserID]; !ok { + resultMap[actionRecord.UserID] = 1 + } else { + resultMap[actionRecord.UserID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } func queryAction(start_unix int64, end_unix int64, actionType int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,op_type,act_user_id").Table("action").Where("op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - actionList := make([]*Action, 0) - sess.Find(&actionList) resultMap := make(map[int64]int) - log.Info("query action size=" + fmt.Sprint(len(actionList))) - for _, actionRecord := range actionList { - if _, ok := resultMap[actionRecord.UserID]; !ok { - resultMap[actionRecord.UserID] = 1 - } else { - resultMap[actionRecord.UserID] += 1 + cond := "op_type=" + fmt.Sprint(actionType) + " and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Action)) + if err != nil { + log.Info("query Action error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id,op_type,act_user_id").Table("action").Where(cond).Limit(Page_SIZE, int(indexTotal)) + actionList := make([]*Action, 0) + sess.Find(&actionList) + log.Info("query action size=" + fmt.Sprint(len(actionList))) + for _, actionRecord := range actionList { + if _, ok := resultMap[actionRecord.UserID]; !ok { + resultMap[actionRecord.UserID] = 1 + } else { + resultMap[actionRecord.UserID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap @@ -714,16 +836,30 @@ func queryComment(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,type,poster_id").Table("comment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - commentList := make([]*Comment, 0) - sess.Find(&commentList) + cond := "created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) resultMap := make(map[int64]int) - log.Info("query Comment size=" + fmt.Sprint(len(commentList))) - for _, commentRecord := range commentList { - if _, ok := resultMap[commentRecord.PosterID]; !ok { - resultMap[commentRecord.PosterID] = 1 - } else { - resultMap[commentRecord.PosterID] += 1 + count, err := sess.Where(cond).Count(new(Comment)) + if err != nil { + log.Info("query Comment error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,type,poster_id").Table("comment").Where(cond).Limit(Page_SIZE, int(indexTotal)) + commentList := make([]*Comment, 0) + sess.Find(&commentList) + log.Info("query Comment size=" + fmt.Sprint(len(commentList))) + for _, commentRecord := range commentList { + if _, ok := resultMap[commentRecord.PosterID]; !ok { + resultMap[commentRecord.PosterID] = 1 + } else { + resultMap[commentRecord.PosterID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap @@ -733,18 +869,37 @@ func queryWatch(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,repo_id").Table("watch").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - watchList := make([]*Watch, 0) - sess.Find(&watchList) + + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + resultMap := make(map[int64]int) - log.Info("query Watch size=" + fmt.Sprint(len(watchList))) - for _, watchRecord := range watchList { - if _, ok := resultMap[watchRecord.UserID]; !ok { - resultMap[watchRecord.UserID] = 1 - } else { - resultMap[watchRecord.UserID] += 1 + count, err := sess.Where(cond).Count(new(Watch)) + if err != nil { + log.Info("query issue error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + watchList := make([]*Watch, 0) + sess.Select("id,user_id,repo_id").Table("watch").Where(cond).Limit(Page_SIZE, int(indexTotal)) + sess.Find(&watchList) + + log.Info("query Watch size=" + fmt.Sprint(len(watchList))) + for _, watchRecord := range watchList { + if _, ok := resultMap[watchRecord.UserID]; !ok { + resultMap[watchRecord.UserID] = 1 + } else { + resultMap[watchRecord.UserID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } @@ -753,82 +908,151 @@ func queryStar(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,uid,repo_id").Table("star").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - starList := make([]*Star, 0) - sess.Find(&starList) + + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) resultMap := make(map[int64]int) - log.Info("query Star size=" + fmt.Sprint(len(starList))) - for _, starRecord := range starList { - if _, ok := resultMap[starRecord.UID]; !ok { - resultMap[starRecord.UID] = 1 - } else { - resultMap[starRecord.UID] += 1 + + count, err := sess.Where(cond).Count(new(Star)) + if err != nil { + log.Info("query star error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,uid,repo_id").Table("star").Where(cond).Limit(Page_SIZE, int(indexTotal)) + starList := make([]*Star, 0) + sess.Find(&starList) + + log.Info("query Star size=" + fmt.Sprint(len(starList))) + for _, starRecord := range starList { + if _, ok := resultMap[starRecord.UID]; !ok { + resultMap[starRecord.UID] = 1 + } else { + resultMap[starRecord.UID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } return resultMap - } func queryFollow(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,user_id,follow_id").Table("follow").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - followList := make([]*Follow, 0) - sess.Find(&followList) resultMap := make(map[int64]int) - log.Info("query Follow size=" + fmt.Sprint(len(followList))) - for _, followRecord := range followList { - if _, ok := resultMap[followRecord.FollowID]; !ok { - resultMap[followRecord.FollowID] = 1 - } else { - resultMap[followRecord.FollowID] += 1 + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Follow)) + if err != nil { + log.Info("query follow error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,user_id,follow_id").Table("follow").Where(cond).Limit(Page_SIZE, int(indexTotal)) + followList := make([]*Follow, 0) + sess.Find(&followList) + + log.Info("query Follow size=" + fmt.Sprint(len(followList))) + for _, followRecord := range followList { + if _, ok := resultMap[followRecord.FollowID]; !ok { + resultMap[followRecord.FollowID] = 1 + } else { + resultMap[followRecord.FollowID] += 1 + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } func queryDatasetSize(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,uploader_id,size").Table("attachment").Where(" created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - attachmentList := make([]*Attachment, 0) - sess.Find(&attachmentList) resultMap := make(map[int64]int) - log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList))) - for _, attachRecord := range attachmentList { - if _, ok := resultMap[attachRecord.UploaderID]; !ok { - resultMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB - } else { - resultMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB + cond := " created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + + count, err := sess.Where(cond).Count(new(Attachment)) + if err != nil { + log.Info("query attachment error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,uploader_id,size").Table("attachment").Where(cond).Limit(Page_SIZE, int(indexTotal)) + attachmentList := make([]*Attachment, 0) + sess.Find(&attachmentList) + + log.Info("query Attachment size=" + fmt.Sprint(len(attachmentList))) + for _, attachRecord := range attachmentList { + if _, ok := resultMap[attachRecord.UploaderID]; !ok { + resultMap[attachRecord.UploaderID] = int(attachRecord.Size / (1024 * 1024)) //MB + } else { + resultMap[attachRecord.UploaderID] += int(attachRecord.Size / (1024 * 1024)) //MB + } + } + + indexTotal += Page_SIZE + if indexTotal >= count { + break } } - return resultMap + return resultMap } func queryUserCreateRepo(start_unix int64, end_unix int64) map[int64]int { sess := x.NewSession() defer sess.Close() - sess.Select("id,owner_id,name").Table("repository").Where("is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - repoList := make([]*Repository, 0) - sess.Find(&repoList) resultMap := make(map[int64]int) - log.Info("query Repository size=" + fmt.Sprint(len(repoList))) - for _, repoRecord := range repoList { - if _, ok := resultMap[repoRecord.OwnerID]; !ok { - resultMap[repoRecord.OwnerID] = 1 - } else { - resultMap[repoRecord.OwnerID] += 1 + + cond := "is_fork=false and created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := sess.Where(cond).Count(new(Repository)) + if err != nil { + log.Info("query Repository error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + sess.Select("id,owner_id,name").Table("repository").Where(cond).Limit(Page_SIZE, int(indexTotal)) + repoList := make([]*Repository, 0) + sess.Find(&repoList) + log.Info("query Repository size=" + fmt.Sprint(len(repoList))) + for _, repoRecord := range repoList { + if _, ok := resultMap[repoRecord.OwnerID]; !ok { + resultMap[repoRecord.OwnerID] = 1 + } else { + resultMap[repoRecord.OwnerID] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 { statictisSess := xStatistic.NewSession() defer statictisSess.Close() - statictisSess.Select("repo_id,radar_total").Table("repo_statistic").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) + + statictisSess.Select("id,repo_id,radar_total").Table("repo_statistic").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)).OrderBy("id desc") repoStatisticList := make([]*RepoStatistic, 0) statictisSess.Find(&repoStatisticList) repoOpenIIndexMap := make(map[int64]float64) @@ -884,18 +1108,34 @@ func queryUserRepoOpenIIndex(start_unix int64, end_unix int64) map[int64]float64 func queryLoginCount(start_unix int64, end_unix int64) map[int64]int { statictisSess := xStatistic.NewSession() defer statictisSess.Close() - statictisSess.Select("id,u_id").Table("user_login_log").Where("created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix)) - userLoginLogList := make([]*UserLoginLog, 0) - statictisSess.Find(&userLoginLogList) + resultMap := make(map[int64]int) - log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) - for _, loginRecord := range userLoginLogList { - if _, ok := resultMap[loginRecord.UId]; !ok { - resultMap[loginRecord.UId] = 1 - } else { - resultMap[loginRecord.UId] += 1 + cond := "created_unix>=" + fmt.Sprint(start_unix) + " and created_unix<=" + fmt.Sprint(end_unix) + count, err := statictisSess.Where(cond).Count(new(UserLoginLog)) + if err != nil { + log.Info("query UserLoginLog error. return.") + return resultMap + } + var indexTotal int64 + indexTotal = 0 + for { + statictisSess.Select("id,u_id").Table("user_login_log").Where(cond).Limit(Page_SIZE, int(indexTotal)) + userLoginLogList := make([]*UserLoginLog, 0) + statictisSess.Find(&userLoginLogList) + log.Info("query user login size=" + fmt.Sprint(len(userLoginLogList))) + for _, loginRecord := range userLoginLogList { + if _, ok := resultMap[loginRecord.UId]; !ok { + resultMap[loginRecord.UId] = 1 + } else { + resultMap[loginRecord.UId] += 1 + } + } + indexTotal += Page_SIZE + if indexTotal >= count { + break } } + return resultMap } diff --git a/modules/auth/auth.go b/modules/auth/auth.go index 16ea9f15e..352e50ca0 100644 --- a/modules/auth/auth.go +++ b/modules/auth/auth.go @@ -9,6 +9,9 @@ import ( "reflect" "strings" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth/sso" "code.gitea.io/gitea/modules/validation" @@ -31,6 +34,8 @@ func SignedInUser(ctx *macaron.Context, sess session.Store) (*models.User, bool) return nil, false } + checkAutoLogin(ctx, sess) + // Try to sign in with each of the enabled plugins for _, ssoMethod := range sso.Methods() { if !ssoMethod.IsEnabled() { @@ -46,6 +51,23 @@ func SignedInUser(ctx *macaron.Context, sess session.Store) (*models.User, bool) return nil, false } +func checkAutoLogin(ctx *macaron.Context, sess session.Store) { + uid := sess.Get("uid") + if uid == nil { + uname := ctx.GetCookie(setting.CookieUserName) + + u, err := models.GetUserByName(uname) + if err == nil { + + if val, ok := ctx.GetSuperSecureCookie( + base.EncodeMD5(u.Rands+u.Passwd), setting.CookieRememberName); ok && val == u.Name { + sess.Set("uid", u.ID) + } + } + } + +} + // Form form binding interface type Form interface { binding.Validator diff --git a/modules/auth/repo_form.go b/modules/auth/repo_form.go index 8352026fe..8061c6469 100755 --- a/modules/auth/repo_form.go +++ b/modules/auth/repo_form.go @@ -122,6 +122,7 @@ type RepoSettingForm struct { // Advanced settings EnableDataset bool EnableCloudBrain bool + EnableModelManager bool EnableWiki bool EnableExternalWiki bool ExternalWikiURL string diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 8f6bf4e17..0f1c700d2 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -1,8 +1,10 @@ package cloudbrain import ( - "code.gitea.io/gitea/modules/setting" "errors" + "strconv" + + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/context" @@ -16,7 +18,7 @@ const ( ModelMountPath = "/model" BenchMarkMountPath = "/benchmark" Snn4imagenetMountPath = "/snn4imagenet" - BrainScoreMountPath = "/brainscore" + BrainScoreMountPath = "/brainscore" TaskInfoName = "/taskInfo" SubTaskName = "task1" @@ -28,6 +30,75 @@ var ( ResourceSpecs *models.ResourceSpecs ) +func isAdminOrOwnerOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool { + if !ctx.IsSigned { + return false + } + log.Info("is repo owner:" + strconv.FormatBool(ctx.IsUserRepoOwner())) + log.Info("is user admin:" + strconv.FormatBool(ctx.IsUserSiteAdmin())) + if err != nil { + + return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() + } else { + log.Info("is job creator:" + strconv.FormatBool(ctx.User.ID == job.UserID)) + return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID + } + +} + +func CanDeleteJob(ctx *context.Context, job *models.Cloudbrain) bool { + + return isAdminOrOwnerOrJobCreater(ctx, job, nil) +} + +func CanCreateOrDebugJob(ctx *context.Context) bool { + if !ctx.IsSigned { + return false + } + return ctx.Repo.CanWrite(models.UnitTypeCloudBrain) +} + +func CanModifyJob(ctx *context.Context, job *models.Cloudbrain) bool { + + return isAdminOrJobCreater(ctx, job, nil) +} + +func isAdminOrJobCreater(ctx *context.Context, job *models.Cloudbrain, err error) bool { + if !ctx.IsSigned { + return false + } + if err != nil { + return ctx.IsUserSiteAdmin() + } else { + return ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID + } + +} + +func AdminOrOwnerOrJobCreaterRight(ctx *context.Context) { + + var jobID = ctx.Params(":jobid") + + job, err := models.GetCloudbrainByJobID(jobID) + + if !isAdminOrOwnerOrJobCreater(ctx, job, err) { + + ctx.NotFound(ctx.Req.URL.RequestURI(), nil) + } + +} + +func AdminOrJobCreaterRight(ctx *context.Context) { + + var jobID = ctx.Params(":jobid") + job, err := models.GetCloudbrainByJobID(jobID) + if !isAdminOrJobCreater(ctx, job, err) { + + ctx.NotFound(ctx.Req.URL.RequestURI(), nil) + } + +} + func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue string, resourceSpecId int) error { dataActualPath := setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + @@ -46,7 +117,7 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, log.Error("no such resourceSpecId(%d)", resourceSpecId, ctx.Data["MsgID"]) return errors.New("no such resourceSpec") } - + jobResult, err := CreateJob(jobName, models.CreateJobParams{ JobName: jobName, RetryCount: 1, @@ -131,8 +202,8 @@ func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, JobName: jobName, SubTaskName: SubTaskName, JobType: jobType, - Type: models.TypeCloudBrainOne, - Uuid: uuid, + Type: models.TypeCloudBrainOne, + Uuid: uuid, }) if err != nil { diff --git a/modules/context/auth.go b/modules/context/auth.go index 3f53e6fce..9877657eb 100755 --- a/modules/context/auth.go +++ b/modules/context/auth.go @@ -145,8 +145,7 @@ func Toggle(options *ToggleOptions) macaron.Handler { } if options.OperationRequired { - //todo: add isOperator judgement - if !ctx.User.IsAdmin { + if !ctx.User.IsOperator { ctx.Error(403) return } diff --git a/modules/context/context.go b/modules/context/context.go index 6877780e3..5f09e190d 100755 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -310,6 +310,7 @@ func Contexter() macaron.Handler { ctx.Data["SignedUserID"] = ctx.User.ID ctx.Data["SignedUserName"] = ctx.User.Name ctx.Data["IsAdmin"] = ctx.User.IsAdmin + ctx.Data["IsOperator"] = ctx.User.IsOperator c.Data["SignedUserName"] = ctx.User.Name } else { ctx.Data["SignedUserID"] = int64(0) diff --git a/modules/context/repo.go b/modules/context/repo.go index 9f8a178fc..de494c1bc 100755 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -821,5 +821,6 @@ func UnitTypes() macaron.Handler { ctx.Data["UnitTypeExternalWiki"] = models.UnitTypeExternalWiki ctx.Data["UnitTypeExternalTracker"] = models.UnitTypeExternalTracker ctx.Data["UnitTypeBlockChain"] = models.UnitTypeBlockChain + ctx.Data["UnitTypeModelManage"] = models.UnitTypeModelManage } } diff --git a/modules/cron/tasks_basic.go b/modules/cron/tasks_basic.go index 207018c20..b9838e66f 100755 --- a/modules/cron/tasks_basic.go +++ b/modules/cron/tasks_basic.go @@ -134,7 +134,7 @@ func registerHandleBlockChainUnSuccessRepos() { RegisterTaskFatal("handle_blockchain_unsuccess_repos", &BaseConfig{ Enabled: true, RunAtStart: true, - Schedule: "@every 1m", + Schedule: "@every 10m", }, func(ctx context.Context, _ *models.User, _ Config) error { repo.HandleBlockChainUnSuccessRepos() return nil @@ -145,7 +145,7 @@ func registerHandleBlockChainMergedPulls() { RegisterTaskFatal("handle_blockchain_merged_pull", &BaseConfig{ Enabled: true, RunAtStart: true, - Schedule: "@every 1m", + Schedule: "@every 10m", }, func(ctx context.Context, _ *models.User, _ Config) error { repo.HandleBlockChainMergedPulls() return nil @@ -156,7 +156,7 @@ func registerHandleBlockChainUnSuccessCommits() { RegisterTaskFatal("handle_blockchain_unsuccess_commits", &BaseConfig{ Enabled: true, RunAtStart: true, - Schedule: "@every 3m", + Schedule: "@every 10m", }, func(ctx context.Context, _ *models.User, _ Config) error { repo.HandleBlockChainUnSuccessCommits() return nil @@ -185,6 +185,17 @@ func registerHandleSummaryStatistic() { }) } +func registerSyncCloudbrainStatus() { + RegisterTaskFatal("sync_cloudbrain_status", &BaseConfig{ + Enabled: true, + RunAtStart: false, + Schedule: "@every 10m", + }, func(ctx context.Context, _ *models.User, _ Config) error { + repo.SyncCloudbrainStatus() + return nil + }) +} + func initBasicTasks() { registerUpdateMirrorTask() registerRepoHealthCheck() @@ -202,4 +213,6 @@ func initBasicTasks() { registerHandleRepoAndUserStatistic() registerHandleSummaryStatistic() + + registerSyncCloudbrainStatus() } diff --git a/modules/git/commit.go b/modules/git/commit.go index 5e492e27e..8e6857f4a 100644 --- a/modules/git/commit.go +++ b/modules/git/commit.go @@ -271,6 +271,15 @@ func AllCommitsCount(repoPath string) (int64, error) { return strconv.ParseInt(strings.TrimSpace(stdout), 10, 64) } +func HEADCommitsCount(repoPath string) (int64, error) { + stdout, err := NewCommand("rev-list", "HEAD", "--count").RunInDir(repoPath) + if err != nil { + return 0, err + } + + return strconv.ParseInt(strings.TrimSpace(stdout), 10, 64) +} + func commitsCount(repoPath, revision, relpath string) (int64, error) { cmd := NewCommand("rev-list", "--count") cmd.AddArguments(revision) diff --git a/modules/git/repo.go b/modules/git/repo.go index a2a4d28af..772f1d149 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -52,6 +52,10 @@ func (repo *Repository) GetAllCommitsCount() (int64, error) { return AllCommitsCount(repo.Path) } +func (repo *Repository) GetHeadCommitsCount() (int64, error) { + return HEADCommitsCount(repo.Path) +} + func (repo *Repository) parsePrettyFormatLogToList(logs []byte) (*list.List, error) { l := list.New() if len(logs) == 0 { diff --git a/modules/git/repo_stats_custom.go b/modules/git/repo_stats_custom.go index 95bad6678..d70a17052 100644 --- a/modules/git/repo_stats_custom.go +++ b/modules/git/repo_stats_custom.go @@ -126,7 +126,7 @@ func GetUserKPIStats(repoPath string) (map[string]*UserKPIStats, error) { func SetRepoKPIStats(repoPath string, fromTime time.Time, stats *RepoKPIStats, newContributers map[string]struct{}) error { since := fromTime.Format(time.RFC3339) - args := []string{"log", "--numstat", "--no-merges", "--branches=*", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--since='%s'", since)} + args := []string{"log", "--numstat", "--no-merges", "HEAD", "--pretty=format:---%n%h%n%an%n%ae%n", "--date=iso", fmt.Sprintf("--since='%s'", since)} stdout, err := NewCommand(args...).RunInDirBytes(repoPath) if err != nil { @@ -212,7 +212,7 @@ func SetRepoKPIStats(repoPath string, fromTime time.Time, stats *RepoKPIStats, n func GetContributorsDetail(repoPath string, fromTime time.Time) ([]Contributor, error) { since := fromTime.Format(time.RFC3339) - cmd := NewCommand("shortlog", "-sne", "--all", fmt.Sprintf("--since='%s'", since)) + cmd := NewCommand("shortlog", "-sne", "HEAD", fmt.Sprintf("--since='%s'", since)) stdout, err := cmd.RunInDir(repoPath) if err != nil { return nil, err diff --git a/modules/migrations/github.go b/modules/migrations/github.go index b6bdab09f..bee21fb54 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -13,6 +13,9 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/setting" + "golang.org/x/net/proxy" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/structs" @@ -98,13 +101,41 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith ) client = oauth2.NewClient(downloader.ctx, ts) } else { - client = &http.Client{ - Transport: &http.Transport{ - Proxy: func(req *http.Request) (*url.URL, error) { - req.SetBasicAuth(userName, password) - return nil, nil + if setting.Migrations.Proxy != "" { + contextDialer, err := getProxyDialContext() + if err != nil { + log.Warn("Failed to use proxy for Github.", err) + client = &http.Client{ + Transport: &http.Transport{ + + Proxy: func(req *http.Request) (*url.URL, error) { + req.SetBasicAuth(userName, password) + return nil, nil + }, + }, + } + } else { + client = &http.Client{ + Transport: &http.Transport{ + + Proxy: func(req *http.Request) (*url.URL, error) { + req.SetBasicAuth(userName, password) + return nil, nil + }, + DialContext: contextDialer.DialContext, + }, + } + } + } else { + client = &http.Client{ + Transport: &http.Transport{ + + Proxy: func(req *http.Request) (*url.URL, error) { + req.SetBasicAuth(userName, password) + return nil, nil + }, }, - }, + } } } } @@ -112,6 +143,25 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith return &downloader } +func getProxyDialContext() (proxy.ContextDialer, error) { + authInfo := &proxy.Auth{ + setting.Migrations.Username, + setting.Migrations.Password, + } + dialSocksProxy, err := proxy.SOCKS5("tcp", setting.Migrations.Proxy, authInfo, proxy.Direct) + + if err != nil { + return nil, err + } + if contextDialer, ok := dialSocksProxy.(proxy.ContextDialer); ok { + return contextDialer, nil + + } else { + return nil, fmt.Errorf("It is not a valiad dialer.") + } + +} + // SetContext set context func (g *GithubDownloaderV3) SetContext(ctx context.Context) { g.ctx = ctx diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index a64c317a8..273d59012 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -48,8 +48,12 @@ const ( PerPage = 10 IsLatestVersion = "1" NotLatestVersion = "0" - ComputeResource = "NPU" - VersionCount = 1 + // ComputeResource = "NPU" + NPUResource = "NPU" + GPUResource = "CPU/GPU" + AllResource = "all" + DebugType = -1 + VersionCount = 1 SortByCreateTime = "create_time" ConfigTypeCustom = "custom" @@ -273,7 +277,7 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error DatasetName: attach.Name, CommitID: req.CommitID, IsLatestVersion: req.IsLatestVersion, - ComputeResource: ComputeResource, + ComputeResource: NPUResource, EngineID: req.EngineID, TrainUrl: req.TrainUrl, BranchName: req.BranchName, @@ -356,7 +360,7 @@ func GenerateTrainJobVersion(ctx *context.Context, req *GenerateTrainJobReq, job CommitID: req.CommitID, IsLatestVersion: req.IsLatestVersion, PreVersionName: req.PreVersionName, - ComputeResource: ComputeResource, + ComputeResource: NPUResource, EngineID: req.EngineID, TrainUrl: req.TrainUrl, BranchName: req.BranchName, diff --git a/modules/normalization/normalization.go b/modules/normalization/normalization.go index ce616d7f8..a258a13a7 100644 --- a/modules/normalization/normalization.go +++ b/modules/normalization/normalization.go @@ -4,6 +4,8 @@ import ( "code.gitea.io/gitea/modules/setting" ) +const MAX_LINES_RECORD = 100 + func Normalization(value float64, minValue float64, maxValue float64) float64 { min := int64(minValue * 100) @@ -72,9 +74,12 @@ func GetTeamHealthInitValue(contributors int64, keyContributors int64, newContri } -func GetRepoGrowthInitValue(codelinesGrowth int64, issueGrowth int64, commitsGrowth int64, newContributors int64, commentsGrowth int64) float64 { - - return setting.RadarMap.GrowthCodeLines*float64(codelinesGrowth) + +func GetRepoGrowthInitValue(codeLinesGrowth int64, issueGrowth int64, commitsGrowth int64, newContributors int64, commentsGrowth int64) float64 { + codeLinesKB := codeLinesGrowth / 1000 + if codeLinesKB > MAX_LINES_RECORD { + codeLinesKB = MAX_LINES_RECORD + } + return setting.RadarMap.GrowthCodeLines*float64(codeLinesKB) + setting.RadarMap.GrowthIssue*float64(issueGrowth) + setting.RadarMap.GrowthCommit*float64(commitsGrowth) + setting.RadarMap.GrowthContributors*float64(newContributors) + diff --git a/modules/repository/repo.go b/modules/repository/repo.go index d57b16c91..3268cce60 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -6,6 +6,7 @@ package repository import ( "fmt" + "net/url" "os" "path" "strings" @@ -54,29 +55,34 @@ func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opt repo.NumWatches = 1 } - migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second + migrateTimeout := getMigrateTimeout(opts.CloneAddr) var err error if err = os.RemoveAll(repoPath); err != nil { return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err) } - + log.Info("clone begin:" + opts.CloneAddr) if err = git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, Timeout: migrateTimeout, }); err != nil { + log.Warn("clone err") return repo, fmt.Errorf("Clone: %v", err) } + log.Info("clone end:" + opts.CloneAddr) + if opts.Wiki { + log.Info("test wiki path begin") wikiPath := models.WikiPath(u.Name, opts.RepoName) wikiRemotePath := wikiRemoteURL(opts.CloneAddr) + log.Info("test wiki path end") if len(wikiRemotePath) > 0 { if err := os.RemoveAll(wikiPath); err != nil { return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) } - + log.Info("wiki clone begin") if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{ Mirror: true, Quiet: true, @@ -88,6 +94,7 @@ func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opt return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err) } } + log.Info("wiki clone end") } } @@ -137,9 +144,20 @@ func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opt repo, err = CleanUpMigrateInfo(repo) } + log.Info("clone all end:" + opts.CloneAddr) + return repo, err } +func getMigrateTimeout(urlClone string) time.Duration { + u, err := url.Parse(urlClone) + if err == nil && strings.EqualFold(u.Host, "github.com") { + return time.Duration(setting.Git.Timeout.GitHubMigrate) * time.Second + } + return time.Duration(setting.Git.Timeout.Migrate) * time.Second + +} + // cleanUpMigrateGitConfig removes mirror info which prevents "push --all". // This also removes possible user credentials. func cleanUpMigrateGitConfig(configPath string) error { diff --git a/modules/setting/git.go b/modules/setting/git.go index f83758f38..306febc78 100644 --- a/modules/setting/git.go +++ b/modules/setting/git.go @@ -27,12 +27,13 @@ var ( EnableAutoGitWireProtocol bool PullRequestPushMessage bool Timeout struct { - Default int - Migrate int - Mirror int - Clone int - Pull int - GC int `ini:"GC"` + Default int + Migrate int + GitHubMigrate int + Mirror int + Clone int + Pull int + GC int `ini:"GC"` } `ini:"git.timeout"` }{ DisableDiffHighlight: false, @@ -45,19 +46,21 @@ var ( EnableAutoGitWireProtocol: true, PullRequestPushMessage: true, Timeout: struct { - Default int - Migrate int - Mirror int - Clone int - Pull int - GC int `ini:"GC"` + Default int + Migrate int + GitHubMigrate int + Mirror int + Clone int + Pull int + GC int `ini:"GC"` }{ - Default: int(git.DefaultCommandExecutionTimeout / time.Second), - Migrate: 600, - Mirror: 300, - Clone: 300, - Pull: 300, - GC: 60, + Default: int(git.DefaultCommandExecutionTimeout / time.Second), + Migrate: 900, + GitHubMigrate: 1800, + Mirror: 1200, + Clone: 300, + Pull: 300, + GC: 60, }, } ) diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 4d4df6201..abca79b7d 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -83,6 +83,7 @@ func newIndexerService() { Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20) Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(1024 * 1024) Indexer.StartupTimeout = sec.Key("STARTUP_TIMEOUT").MustDuration(30 * time.Second) + log.Info("New IndexerService Inited.") } // IndexerGlobFromString parses a comma separated list of patterns and returns a glob.Glob slice suited for repo indexing diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go index 51d6bbcf1..0a948cd1f 100644 --- a/modules/setting/migrations.go +++ b/modules/setting/migrations.go @@ -9,6 +9,9 @@ var ( Migrations = struct { MaxAttempts int RetryBackoff int + Proxy string + Username string + Password string }{ MaxAttempts: 3, RetryBackoff: 3, @@ -19,4 +22,7 @@ func newMigrationsService() { sec := Cfg.Section("migrations") Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts) Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff) + Migrations.Proxy = sec.Key("Proxy").MustString("") + Migrations.Username = sec.Key("Username").MustString("") + Migrations.Password = sec.Key("Password").MustString("") } diff --git a/modules/setting/queue.go b/modules/setting/queue.go index 8bdca1017..999d1afe0 100644 --- a/modules/setting/queue.go +++ b/modules/setting/queue.go @@ -160,6 +160,7 @@ func NewQueueService() { if _, ok := sectionMap["LENGTH"]; !ok { _, _ = section.NewKey("LENGTH", fmt.Sprintf("%d", Repository.PullRequestQueueLength)) } + log.Info("New QueueService Inited.") } // ParseQueueConnStr parses a queue connection string diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 92eae63b0..2d70e47b1 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -433,35 +433,43 @@ var ( AuthUser string AuthPassword string + //home page + RecommentRepoAddr string + //labelsystem config LabelTaskName string LabelDatasetDeleteQueue string DecompressOBSTaskName string //cloudbrain config - CBAuthUser string - CBAuthPassword string - RestServerHost string - JobPath string - JobType string - GpuTypes string - DebugServerHost string - ResourceSpecs string + CBAuthUser string + CBAuthPassword string + RestServerHost string + JobPath string + CBCodePathPrefix string + JobType string + GpuTypes string + DebugServerHost string + ResourceSpecs string + MaxDuration int64 //benchmark config IsBenchmarkEnabled bool - BenchmarkCode string + BenchmarkOwner string + BenchmarkName string BenchmarkServerHost string BenchmarkCategory string //snn4imagenet config IsSnn4imagenetEnabled bool - Snn4imagenetCode string + Snn4imagenetOwner string + Snn4imagenetName string Snn4imagenetServerHost string //snn4imagenet config IsBrainScoreEnabled bool - BrainScoreCode string + BrainScoreOwner string + BrainScoreName string BrainScoreServerHost string //blockchain config @@ -549,7 +557,7 @@ var ( RecordBeginTime string IgnoreMirrorRepo bool }{} - + Warn_Notify_Mails []string ) @@ -1216,51 +1224,59 @@ func NewContext() { sec = Cfg.Section("decompress") DecompressAddress = sec.Key("HOST").MustString("http://192.168.207.34:39987") - AuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X") - AuthPassword = sec.Key("PASSWORD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC") + AuthUser = sec.Key("USER").MustString("") + AuthPassword = sec.Key("PASSWORD").MustString("") sec = Cfg.Section("labelsystem") LabelTaskName = sec.Key("LabelTaskName").MustString("LabelRedisQueue") LabelDatasetDeleteQueue = sec.Key("LabelDatasetDeleteQueue").MustString("LabelDatasetDeleteQueue") DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue") + sec = Cfg.Section("homepage") + RecommentRepoAddr = sec.Key("Address").MustString("https://git.openi.org.cn/OpenIOSSG/promote/raw/branch/master/") + sec = Cfg.Section("cloudbrain") - CBAuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X") - CBAuthPassword = sec.Key("PWD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC") + CBAuthUser = sec.Key("USER").MustString("") + CBAuthPassword = sec.Key("PWD").MustString("") RestServerHost = sec.Key("REST_SERVER_HOST").MustString("http://192.168.202.73") JobPath = sec.Key("JOB_PATH").MustString("/datasets/minio/data/opendata/jobs/") + CBCodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("jobs/") DebugServerHost = sec.Key("DEBUG_SERVER_HOST").MustString("http://192.168.202.73") JobType = sec.Key("GPU_TYPE_DEFAULT").MustString("openidebug") GpuTypes = sec.Key("GPU_TYPES").MustString("") ResourceSpecs = sec.Key("RESOURCE_SPECS").MustString("") + MaxDuration = sec.Key("MAX_DURATION").MustInt64(14400) sec = Cfg.Section("benchmark") IsBenchmarkEnabled = sec.Key("ENABLED").MustBool(false) - BenchmarkCode = sec.Key("BENCHMARKCODE").MustString("https://yangzhx:justfortest123@git.openi.org.cn/yangzhx/detection_benchmark_script.git") - BenchmarkServerHost = sec.Key("HOST").MustString("http://192.168.202.90:3366/") + BenchmarkOwner = sec.Key("OWNER").MustString("") + BenchmarkName = sec.Key("NAME").MustString("") + BenchmarkServerHost = sec.Key("HOST").MustString("") BenchmarkCategory = sec.Key("CATEGORY").MustString("") sec = Cfg.Section("snn4imagenet") IsSnn4imagenetEnabled = sec.Key("ENABLED").MustBool(false) - Snn4imagenetCode = sec.Key("SNN4IMAGENETCODE").MustString("https://yult:19910821ylt@git.openi.org.cn/yult/snn4imagenet_script.git") - Snn4imagenetServerHost = sec.Key("HOST").MustString("http://192.168.207.76:8080/") + Snn4imagenetOwner = sec.Key("OWNER").MustString("") + Snn4imagenetName = sec.Key("NAME").MustString("") + Snn4imagenetServerHost = sec.Key("HOST").MustString("") sec = Cfg.Section("brainscore") IsBrainScoreEnabled = sec.Key("ENABLED").MustBool(false) - BrainScoreCode = sec.Key("BRAINSCORECODE").MustString("https://yult:19910821ylt@git.openi.org.cn/yult/brainscore_script.git") - BrainScoreServerHost = sec.Key("HOST").MustString("http://192.168.207.76:8080/") + BrainScoreOwner = sec.Key("OWNER").MustString("") + BrainScoreName = sec.Key("NAME").MustString("") + BrainScoreServerHost = sec.Key("HOST").MustString("") sec = Cfg.Section("blockchain") BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/") CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15") sec = Cfg.Section("obs") - Endpoint = sec.Key("ENDPOINT").MustString("112.95.163.82") + Endpoint = sec.Key("ENDPOINT").MustString("") AccessKeyID = sec.Key("ACCESS_KEY_ID").MustString("") SecretAccessKey = sec.Key("SECRET_ACCESS_KEY").MustString("") - Bucket = sec.Key("BUCKET").MustString("testopendata") - Location = sec.Key("LOCATION").MustString("cn-south-222") - BasePath = sec.Key("BASE_PATH").MustString("attachment/") + Bucket = sec.Key("BUCKET").MustString("") + Location = sec.Key("LOCATION").MustString("") + BasePath = sec.Key("BASE_PATH").MustString("") TrainJobModelPath = sec.Key("TrainJobModel_Path").MustString("job/") OutPutPath = sec.Key("Output_Path").MustString("output/") CodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("code/") @@ -1268,17 +1284,17 @@ func NewContext() { PROXYURL = sec.Key("PROXY_URL").MustString("") sec = Cfg.Section("modelarts") - ModelArtsHost = sec.Key("ENDPOINT").MustString("112.95.163.80") - IamHost = sec.Key("IAMHOST").MustString("112.95.163.80") + ModelArtsHost = sec.Key("ENDPOINT").MustString("") + IamHost = sec.Key("IAMHOST").MustString("") ProjectID = sec.Key("PROJECT_ID").MustString("") ProjectName = sec.Key("PROJECT_NAME").MustString("") ModelArtsUsername = sec.Key("USERNAME").MustString("") ModelArtsPassword = sec.Key("PASSWORD").MustString("") - ModelArtsDomain = sec.Key("DOMAIN").MustString("cn-south-222") + ModelArtsDomain = sec.Key("DOMAIN").MustString("") AllowedOrg = sec.Key("ORGANIZATION").MustString("") ProfileID = sec.Key("PROFILE_ID").MustString("") PoolInfos = sec.Key("POOL_INFOS").MustString("") - Flavor = sec.Key("FLAVOR").MustString("modelarts.bm.910.arm.public.2") + Flavor = sec.Key("FLAVOR").MustString("") ResourcePools = sec.Key("Resource_Pools").MustString("") Engines = sec.Key("Engines").MustString("") EngineVersions = sec.Key("Engine_Versions").MustString("") @@ -1286,10 +1302,10 @@ func NewContext() { TrainJobFLAVORINFOS = sec.Key("TrainJob_FLAVOR_INFOS").MustString("") sec = Cfg.Section("elk") - ElkUrl = sec.Key("ELKURL").MustString("http://192.168.207.35:5601/internal/bsearch") - ElkUser = sec.Key("ELKUSER").MustString("Qizhi") - ElkPassword = sec.Key("ELKPASSWORD").MustString("Pcl2020") - Index = sec.Key("INDEX").MustString("filebeat-7.3.2*") + ElkUrl = sec.Key("ELKURL").MustString("") + ElkUser = sec.Key("ELKUSER").MustString("") + ElkPassword = sec.Key("ELKPASSWORD").MustString("") + Index = sec.Key("INDEX").MustString("") TimeField = sec.Key("TIMEFIELD").MustString(" @timestamptest") ElkTimeFormat = sec.Key("ELKTIMEFORMAT").MustString("date_time") @@ -1313,7 +1329,7 @@ func SetRadarMapConfig() { RadarMap.CompletenessIssuesClosed = sec.Key("completeness_issues_closed").MustFloat64(0.2) RadarMap.CompletenessReleases = sec.Key("completeness_releases").MustFloat64(0.3) RadarMap.CompletenessDevelopAge = sec.Key("completeness_develop_age").MustFloat64(0.1) - RadarMap.CompletenessDataset = sec.Key("completeness_dataset").MustFloat64(0.1) + RadarMap.CompletenessDataset = sec.Key("completeness_dataset").MustFloat64(0) RadarMap.CompletenessModel = sec.Key("completeness_model").MustFloat64(0.1) RadarMap.CompletenessWiki = sec.Key("completeness_wiki").MustFloat64(0.1) RadarMap.Liveness = sec.Key("liveness").MustFloat64(0.3) diff --git a/modules/setting/webhook.go b/modules/setting/webhook.go index 4a0c593c8..34cf8a62d 100644 --- a/modules/setting/webhook.go +++ b/modules/setting/webhook.go @@ -48,4 +48,5 @@ func newWebhookService() { } } Webhook.ProxyHosts = sec.Key("PROXY_HOSTS").Strings(",") + log.Info("New WebhookService Inited.") } diff --git a/modules/storage/local.go b/modules/storage/local.go index c462dcd9e..d46a5528d 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -76,3 +76,7 @@ func (l *LocalStorage) PresignedPutURL(path string) (string, error) { func (l *LocalStorage) HasObject(path string) (bool, error) { return false, nil } + +func (l *LocalStorage) UploadObject(fileName, filePath string) error { + return nil +} diff --git a/modules/storage/minio.go b/modules/storage/minio.go index b14442d56..664e58d1b 100755 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -122,3 +122,9 @@ func (m *MinioStorage) HasObject(path string) (bool, error) { return hasObject, nil } + +//upload object +func (m *MinioStorage) UploadObject(fileName, filePath string) error { + _, err := m.client.FPutObject(m.bucket, fileName, filePath, minio.PutObjectOptions{}) + return err +} diff --git a/modules/storage/obs.go b/modules/storage/obs.go index a5c463bb0..367ffe1e8 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -5,6 +5,7 @@ package storage import ( + "errors" "io" "net/url" "path" @@ -140,11 +141,51 @@ func ObsMultiPartUpload(uuid string, uploadId string, partNumber int, fileName s } -func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { +//delete all file under the dir path +func ObsRemoveObject(bucket string, path string) error { + log.Info("Bucket=" + bucket + " path=" + path) + if len(path) == 0 { + return errors.New("path canot be null.") + } + input := &obs.ListObjectsInput{} + input.Bucket = bucket + // 设置每页100个对象 + input.MaxKeys = 100 + input.Prefix = path + index := 1 + log.Info("prefix=" + input.Prefix) + for { + output, err := ObsCli.ListObjects(input) + if err == nil { + log.Info("Page:%d\n", index) + index++ + for _, val := range output.Contents { + log.Info("delete obs file:" + val.Key) + delObj := &obs.DeleteObjectInput{} + delObj.Bucket = setting.Bucket + delObj.Key = val.Key + ObsCli.DeleteObject(delObj) + } + if output.IsTruncated { + input.Marker = output.NextMarker + } else { + break + } + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Info("Code:%s\n", obsError.Code) + log.Info("Message:%s\n", obsError.Message) + } + return err + } + } + return nil +} + +func ObsDownloadAFile(bucket string, key string) (io.ReadCloser, error) { input := &obs.GetObjectInput{} - input.Bucket = setting.Bucket - input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") - // input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") + input.Bucket = bucket + input.Key = key output, err := ObsCli.GetObject(input) if err == nil { log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n", @@ -158,6 +199,11 @@ func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { } } +func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) { + + return ObsDownloadAFile(setting.Bucket, strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")) +} + func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { input := &obs.GetObjectInput{} input.Bucket = setting.Bucket @@ -176,6 +222,160 @@ func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) { } } +func ObsCopyManyFile(srcBucket string, srcPath string, destBucket string, destPath string) (int64, error) { + input := &obs.ListObjectsInput{} + input.Bucket = srcBucket + // 设置每页100个对象 + input.MaxKeys = 100 + input.Prefix = srcPath + index := 1 + length := len(srcPath) + var fileTotalSize int64 + log.Info("prefix=" + input.Prefix) + for { + output, err := ObsCli.ListObjects(input) + if err == nil { + log.Info("Page:%d\n", index) + index++ + for _, val := range output.Contents { + destKey := destPath + val.Key[length:] + obsCopyFile(srcBucket, val.Key, destBucket, destKey) + fileTotalSize += val.Size + } + if output.IsTruncated { + input.Marker = output.NextMarker + } else { + break + } + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Info("Code:%s\n", obsError.Code) + log.Info("Message:%s\n", obsError.Message) + } + return 0, err + } + } + return fileTotalSize, nil +} + +func obsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKeyName string) error { + input := &obs.CopyObjectInput{} + input.Bucket = destBucket + input.Key = destKeyName + input.CopySourceBucket = srcBucket + input.CopySourceKey = srcKeyName + _, err := ObsCli.CopyObject(input) + if err == nil { + log.Info("copy success,destBuckName:%s, destkeyname:%s", destBucket, destKeyName) + } else { + log.Info("copy failed,,destBuckName:%s, destkeyname:%s", destBucket, destKeyName) + if obsError, ok := err.(obs.ObsError); ok { + log.Info(obsError.Code) + log.Info(obsError.Message) + } + return err + } + return nil +} + +func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relativePath string) ([]FileInfo, error) { + input := &obs.ListObjectsInput{} + input.Bucket = bucket + input.Prefix = prefixRootPath + relativePath + if !strings.HasSuffix(input.Prefix, "/") { + input.Prefix += "/" + } + output, err := ObsCli.ListObjects(input) + fileInfos := make([]FileInfo, 0) + prefixLen := len(input.Prefix) + if err == nil { + for _, val := range output.Contents { + log.Info("val key=" + val.Key) + var isDir bool + var fileName string + if val.Key == input.Prefix { + continue + } + if strings.Contains(val.Key[prefixLen:len(val.Key)-1], "/") { + continue + } + if strings.HasSuffix(val.Key, "/") { + isDir = true + fileName = val.Key[prefixLen : len(val.Key)-1] + relativePath += val.Key[prefixLen:] + } else { + isDir = false + fileName = val.Key[prefixLen:] + } + fileInfo := FileInfo{ + ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), + FileName: fileName, + Size: val.Size, + IsDir: isDir, + ParenDir: relativePath, + } + fileInfos = append(fileInfos, fileInfo) + } + return fileInfos, err + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) + } + return nil, err + } + +} + +func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) { + input := &obs.ListObjectsInput{} + input.Bucket = bucket + // 设置每页100个对象 + input.MaxKeys = 100 + input.Prefix = prefix + index := 1 + fileInfos := make([]FileInfo, 0) + prefixLen := len(prefix) + log.Info("prefix=" + input.Prefix) + for { + output, err := ObsCli.ListObjects(input) + if err == nil { + log.Info("Page:%d\n", index) + index++ + for _, val := range output.Contents { + var isDir bool + if prefixLen == len(val.Key) { + continue + } + if strings.HasSuffix(val.Key, "/") { + isDir = true + } else { + isDir = false + } + fileInfo := FileInfo{ + ModTime: val.LastModified.Format("2006-01-02 15:04:05"), + FileName: val.Key[prefixLen:], + Size: val.Size, + IsDir: isDir, + ParenDir: "", + } + fileInfos = append(fileInfos, fileInfo) + } + if output.IsTruncated { + input.Marker = output.NextMarker + } else { + break + } + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Info("Code:%s\n", obsError.Code) + log.Info("Message:%s\n", obsError.Message) + } + return nil, err + } + } + return fileInfos, nil +} + func GetObsListObject(jobName, parentDir, versionName string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = setting.Bucket @@ -258,27 +458,6 @@ func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, file return output.SignedUrl, nil } -func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) { - input := &obs.CreateSignedUrlInput{} - input.Bucket = setting.Bucket - input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/") - - input.Expires = 60 * 60 - input.Method = obs.HttpMethodGet - - reqParams := make(map[string]string) - fileName = url.QueryEscape(fileName) - reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\"" - input.QueryParams = reqParams - output, err := ObsCli.CreateSignedUrl(input) - if err != nil { - log.Error("CreateSignedUrl failed:", err.Error()) - return "", err - } - log.Info("SignedUrl:%s", output.SignedUrl) - return output.SignedUrl, nil -} - func GetObsCreateSignedUrlByBucketAndKey(bucket, key string) (string, error) { input := &obs.CreateSignedUrlInput{} input.Bucket = bucket @@ -302,7 +481,10 @@ func GetObsCreateSignedUrlByBucketAndKey(bucket, key string) (string, error) { } return output.SignedUrl, nil +} +func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) { + return GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/")) } func ObsGetPreSignedUrl(uuid, fileName string) (string, error) { diff --git a/modules/storage/storage.go b/modules/storage/storage.go index e0e875650..d364346f0 100755 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -26,6 +26,7 @@ type ObjectStorage interface { PresignedGetURL(path string, fileName string) (string, error) PresignedPutURL(path string) (string, error) HasObject(path string) (bool, error) + UploadObject(fileName, filePath string) error } // Copy copys a file from source ObjectStorage to dest ObjectStorage @@ -51,6 +52,7 @@ func Init() error { switch setting.Attachment.StoreType { case LocalStorageType: Attachments, err = NewLocalStorage(setting.Attachment.Path) + log.Info("local storage inited.") case MinioStorageType: minio := setting.Attachment.Minio Attachments, err = NewMinioStorage( @@ -62,6 +64,7 @@ func Init() error { minio.BasePath, minio.UseSSL, ) + log.Info("minio storage inited.") default: return fmt.Errorf("Unsupported attachment store type: %s", setting.Attachment.StoreType) } @@ -71,6 +74,7 @@ func Init() error { log.Error("obs.New failed:", err) return err } + log.Info("obs cli inited.") if err != nil { return err diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 440147d58..d58a61756 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -816,6 +816,11 @@ get_repo_info_error=Can not get the information of the repository. generate_statistic_file_error=Fail to generate file. repo_stat_inspect=ProjectAnalysis all=All +modelarts.status=Status +modelarts.createtime=CreateTime +modelarts.version_nums = Version Nums +modelarts.version = Version +modelarts.computing_resources=compute Resources modelarts.notebook=Debug Task modelarts.train_job=Train Task modelarts.train_job.new_debug= New Debug Task @@ -823,6 +828,10 @@ modelarts.train_job.new_train=New Train Task modelarts.train_job.config=Configuration information modelarts.train_job.new=New train Task modelarts.train_job.new_place=The description should not exceed 256 characters +modelarts.model_name=Model Name +modelarts.model_size=Model Size +modelarts.import_model=Import Model + modelarts.modify=Modify modelarts.current_version=Current version modelarts.parent_version=Parent Version @@ -874,6 +883,20 @@ modelarts.train_job_para_admin=train_job_para_admin modelarts.train_job_para.edit=train_job_para.edit modelarts.train_job_para.connfirm=train_job_para.connfirm +model.manage.import_new_model=Import New Model +model.manage.create_error=Equal Name and Version has existed. +model.manage.model_name = Model Name +model.manage.version = Version +model.manage.label = Label +model.manage.size = Size +model.manage.create_time = Create Time +model.manage.Description = Description +model.manage.Accuracy = Accuracy +model.manage.F1 = F1 +model.manage.Precision = Precision +model.manage.Recall = Recall +model.manage.sava_model = Sava Model + template.items = Template Items template.git_content = Git Content (Default Branch) template.git_hooks = Git Hooks @@ -1552,6 +1575,7 @@ settings.external_wiki_url_error = The external wiki URL is not a valid URL. settings.external_wiki_url_desc = Visitors are redirected to the external wiki URL when clicking the wiki tab. settings.dataset_desc = Enable Repository Dataset settings.cloudbrain_desc = Enable Cloudbarin +settings.model_desc = Enable Model Manage settings.issues_desc = Enable Repository Issue Tracker settings.use_internal_issue_tracker = Use Built-In Issue Tracker settings.use_external_issue_tracker = Use External Issue Tracker diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index e26e07903..45cabda8f 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -228,6 +228,7 @@ users=用户 organizations=组织 images = 云脑镜像 search=搜索 +search_pro=搜项目 code=代码 data_analysis=数字看板(内测) repo_no_results=未找到匹配的项目。 @@ -781,6 +782,9 @@ datasets=数据集 datasets.desc=数据集功能 cloudbrain_helper=使用GPU/NPU资源，开启Notebook、模型训练任务等 +model_manager = 模型 +model_noright=无权限操作 + debug=调试 stop=停止 delete=删除 @@ -823,6 +827,7 @@ all=所有 modelarts.status=状态 modelarts.createtime=创建时间 modelarts.version_nums=版本数 +modelarts.version=版本 modelarts.computing_resources=计算资源 modelarts.notebook=调试任务 modelarts.train_job=训练任务 @@ -830,7 +835,11 @@ modelarts.train_job.new_debug=新建调试任务 modelarts.train_job.new_train=新建训练任务 modelarts.train_job.config=配置信息 modelarts.train_job.new=新建训练任务 -modelarts.train_job.new_place=描述字数不超过255个字符 +modelarts.train_job.new_place=描述字数不超过256个字符 +modelarts.model_name=模型名称 +modelarts.model_size=模型大小 +modelarts.import_model=导入模型 +modelarts.train_job.label_place=输入标签，多个标签用空格区分 modelarts.modify=修改 modelarts.current_version=当前版本 modelarts.parent_version=父版本 @@ -848,7 +857,7 @@ modelarts.train_job.description=任务描述 modelarts.train_job.parameter_setting=参数设置 modelarts.train_job.parameter_setting_info=参数信息 modelarts.train_job.fast_parameter_setting=一键式参数配置 -modelarts.train_job.fast_parameter_setting_config=如您已保存过参数配置，可单击 +modelarts.train_job.fast_parameter_setting_config=如您已保存过参数配置，可单击 modelarts.train_job.fast_parameter_setting_config_link=这里 modelarts.train_job.frames=常用框架 modelarts.train_job.algorithm_origin=算法来源 @@ -886,6 +895,19 @@ modelarts.train_job_para_admin=任务参数管理 modelarts.train_job_para.edit=编辑 modelarts.train_job_para.connfirm=确定 +model.manage.import_new_model=导入新模型 +model.manage.create_error=相同的名称和版本的模型已经存在。 +model.manage.model_name = 模型名称 +model.manage.version = 版本 +model.manage.label = 标签 +model.manage.size = 大小 +model.manage.create_time = 创建时间 +model.manage.description = 描述 +model.manage.Accuracy = 准确率 +model.manage.F1 = F1值 +model.manage.Precision = 精确率 +model.manage.Recall = 召回率 +model.manage.sava_model = 保存模型 template.items=模板选项 template.git_content=Git数据(默认分支) @@ -1565,6 +1587,7 @@ settings.external_wiki_url_error=外部百科链接无效 settings.external_wiki_url_desc=当点击任务标签时，访问者将被重定向到外部任务系统的URL。 settings.dataset_desc=启用数据集 settings.cloudbrain_desc = 启用云脑 +settings.model_desc = 启用模型管理 settings.issues_desc=启用任务系统 settings.use_internal_issue_tracker=使用内置的轻量级任务管理系统 settings.use_external_issue_tracker=使用外部的任务管理系统 @@ -1965,7 +1988,7 @@ team_unit_desc=允许访问项目单元 team_unit_disabled=(已禁用) form.name_reserved=组织名称 '%s' 是被保留的。 -form.name_pattern_not_allowed=项目名称中不允许使用 "%s"。 +form.name_pattern_not_allowed=组织名称中不允许使用 "%s"。 form.create_org_not_allowed=此账号禁止创建组织 settings=组织设置 diff --git a/public/img/member.svg b/public/img/member.svg new file mode 100644 index 000000000..5e91d1c67 --- /dev/null +++ b/public/img/member.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/overview_rgb.svg b/public/img/overview_rgb.svg new file mode 100644 index 000000000..1a1257e23 --- /dev/null +++ b/public/img/overview_rgb.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/pro_num.svg b/public/img/pro_num.svg new file mode 100644 index 000000000..f6d5ae7d6 --- /dev/null +++ b/public/img/pro_num.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/pro_rgb.svg b/public/img/pro_rgb.svg new file mode 100644 index 000000000..2110c2115 --- /dev/null +++ b/public/img/pro_rgb.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/search.svg b/public/img/search.svg new file mode 100644 index 000000000..ec91b07dd --- /dev/null +++ b/public/img/search.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/img/user_rgb.svg b/public/img/user_rgb.svg new file mode 100644 index 000000000..e5cf24cf0 --- /dev/null +++ b/public/img/user_rgb.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index b7ef8d48f..518c63e4f 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -75,6 +75,7 @@ import ( "code.gitea.io/gitea/routers/api/v1/repo" _ "code.gitea.io/gitea/routers/api/v1/swagger" // for swagger generation "code.gitea.io/gitea/routers/api/v1/user" + repo_ext "code.gitea.io/gitea/routers/repo" "gitea.com/macaron/binding" "gitea.com/macaron/macaron" @@ -523,23 +524,26 @@ func RegisterRoutes(m *macaron.Macaron) { Get(notify.GetThread). Patch(notify.ReadThread) }, reqToken()) - adminReq := context.Toggle(&context.ToggleOptions{SignInRequired: true, AdminRequired: true}) + + operationReq := context.Toggle(&context.ToggleOptions{SignInRequired: true, OperationRequired: true}) //Project board m.Group("/projectboard", func() { - m.Get("/restoreFork", adminReq, repo.RestoreForkNumber) - m.Get("/downloadAll", adminReq, repo.ServeAllProjectsPeriodStatisticsFile) - m.Get("/downloadAllOpenI", adminReq, repo.ServeAllProjectsOpenIStatisticsFile) + m.Get("/restoreFork", repo.RestoreForkNumber) + m.Get("/downloadAll", repo.ServeAllProjectsPeriodStatisticsFile) + m.Get("/downloadAllOpenI", repo.ServeAllProjectsOpenIStatisticsFile) m.Group("/project", func() { - m.Get("", adminReq, repo.GetAllProjectsPeriodStatistics) + m.Get("", repo.GetAllProjectsPeriodStatistics) m.Group("/:id", func() { - m.Get("", adminReq, repo.GetProjectLatestStatistics) - m.Get("/period", adminReq, repo.GetProjectPeriodStatistics) + m.Get("", repo.GetProjectLatestStatistics) + m.Get("/period", repo.GetProjectPeriodStatistics) }) }) - }) + }, operationReq) + + m.Get("/query_user_static_page", operationReq, repo_ext.QueryUserStaticDataPage) // Users m.Group("/users", func() { diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index b4d5fc010..a18da3fe5 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -240,26 +240,28 @@ func DelTrainJobVersion(ctx *context.APIContext) { ctx.ServerError("get VersionListCount faild", err) return } - - // 判断当前删掉的任务是否是最新版本，若是，将排序后的TotalVersionCount置为删掉的最新版本的TotalVersionCount，若不是，按时间排序后的版本列表的第一个版本设置为最新版本，TotalVersionCount不变 - if task.IsLatestVersion == modelarts.IsLatestVersion { - err = models.SetVersionCountAndLatestVersion(jobID, VersionTaskList[0].Cloudbrain.VersionName, VersionListCount, modelarts.IsLatestVersion, task.TotalVersionCount) - if err != nil { - ctx.ServerError("UpdateJobVersionCount failed", err) - return - } - } else { - err = models.SetVersionCountAndLatestVersion(jobID, VersionTaskList[0].VersionName, VersionListCount, modelarts.IsLatestVersion, VersionTaskList[0].Cloudbrain.TotalVersionCount) - if err != nil { - ctx.ServerError("UpdateJobVersionCount failed", err) - return + if VersionListCount > 0 { + // 判断当前删掉的任务是否是最新版本，若是，将排序后的TotalVersionCount置为删掉的最新版本的TotalVersionCount，若不是，按时间排序后的版本列表的第一个版本设置为最新版本，TotalVersionCount不变 + if task.IsLatestVersion == modelarts.IsLatestVersion { + err = models.SetVersionCountAndLatestVersion(jobID, VersionTaskList[0].Cloudbrain.VersionName, VersionListCount, modelarts.IsLatestVersion, task.TotalVersionCount) + if err != nil { + ctx.ServerError("UpdateJobVersionCount failed", err) + return + } + } else { + err = models.SetVersionCountAndLatestVersion(jobID, VersionTaskList[0].VersionName, VersionListCount, modelarts.IsLatestVersion, VersionTaskList[0].Cloudbrain.TotalVersionCount) + if err != nil { + ctx.ServerError("UpdateJobVersionCount failed", err) + return + } } } ctx.JSON(http.StatusOK, map[string]interface{}{ - "JobID": jobID, - "VersionName": versionName, - "StatusOK": 0, + "JobID": jobID, + "VersionName": versionName, + "StatusOK": 0, + "VersionListCount": VersionListCount, }) } diff --git a/routers/api/v1/repo/repo_dashbord.go b/routers/api/v1/repo/repo_dashbord.go index 2926fc4f2..a8887a744 100644 --- a/routers/api/v1/repo/repo_dashbord.go +++ b/routers/api/v1/repo/repo_dashbord.go @@ -5,6 +5,7 @@ import ( "net/http" "net/url" "strconv" + "strings" "time" "github.com/360EntSecGroup-Skylar/excelize/v2" @@ -467,7 +468,7 @@ func generateCountSql(beginTime time.Time, endTime time.Time, latestDate string, "(SELECT repo_id,name,is_private,radar_total from public.repo_statistic where date='" + latestDate + "') B" + " where A.repo_id=B.repo_id" if q != "" { - countSql = countSql + " and B.name like '%" + q + "%'" + countSql = countSql + " and LOWER(B.name) like '%" + strings.ToLower(q) + "%'" } return countSql } @@ -488,7 +489,7 @@ func generateTypeAllSql(beginTime time.Time, endTime time.Time, latestDate strin " where A.repo_id=B.repo_id" if q != "" { - sql = sql + " and name like '%" + q + "%'" + sql = sql + " and LOWER(name) like '%" + strings.ToLower(q) + "%'" } sql = sql + " order by " + orderBy + " desc,repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize) return sql @@ -511,7 +512,7 @@ func generatePageSql(beginTime time.Time, endTime time.Time, latestDate string, "(SELECT repo_id,name,owner_name,is_private,radar_total from public.repo_statistic where date='" + latestDate + "') B" + " where A.repo_id=B.repo_id" if q != "" { - sql = sql + " and B.name like '%" + q + "%'" + sql = sql + " and LOWER(B.name) like '%" + strings.ToLower(q) + "%'" } sql = sql + " order by " + orderBy + " desc,A.repo_id" + " limit " + strconv.Itoa(pageSize) + " offset " + strconv.Itoa((page-1)*pageSize) return sql diff --git a/routers/home.go b/routers/home.go index d2c93c771..7cc353ed8 100755 --- a/routers/home.go +++ b/routers/home.go @@ -7,6 +7,8 @@ package routers import ( "bytes" + "fmt" + "io/ioutil" "net/http" "strings" @@ -511,3 +513,43 @@ func NotFound(ctx *context.Context) { ctx.Data["Title"] = "Page Not Found" ctx.NotFound("home.NotFound", nil) } +func RecommendOrgFromPromote(ctx *context.Context) { + url := setting.RecommentRepoAddr + "organizations" + recommendFromPromote(ctx, url) +} + +func recommendFromPromote(ctx *context.Context, url string) { + resp, err := http.Get(url) + if err != nil { + log.Info("Get organizations url error=" + err.Error()) + ctx.ServerError("QueryTrainJobList:", err) + return + } + bytes, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + log.Info("Get organizations url error=" + err.Error()) + ctx.ServerError("QueryTrainJobList:", err) + return + } + + allLineStr := string(bytes) + lines := strings.Split(allLineStr, "\n") + result := make([]string, len(lines)) + for i, line := range lines { + + tmpIndex := strings.Index(line, ".") + log.Info("i=" + fmt.Sprint(i) + " line=" + line + " tmpIndex=" + fmt.Sprint(tmpIndex)) + if tmpIndex == -1 { + result[i] = strings.Trim(line, " ") + } else { + result[i] = strings.Trim(line[tmpIndex+1:], " ") + } + } + ctx.JSON(http.StatusOK, result) +} + +func RecommendRepoFromPromote(ctx *context.Context) { + url := setting.RecommentRepoAddr + "projects" + recommendFromPromote(ctx, url) +} diff --git a/routers/init.go b/routers/init.go index 26829d228..8b93b64d8 100755 --- a/routers/init.go +++ b/routers/init.go @@ -60,11 +60,17 @@ func NewServices() { if err := storage.Init(); err != nil { log.Fatal("storage init failed: %v", err) } + log.Info("storage init succeed.") mailer.NewContext() + log.Info("mailer.NewContext() succeed.") _ = cache.NewContext() + log.Info("cache.NewContext() succeed.") notification.NewContext() + log.Info("notification.NewContext() succeed.") decompression.NewContext() + log.Info("decompression.NewContext() succeed.") labelmsg.Init() + log.Info("labelmsg.Init() succeed.") } // In case of problems connecting to DB, retry connection. Eg, PGSQL in Docker Container on Synology diff --git a/routers/private/hook.go b/routers/private/hook.go index a673362fa..79c8ecfc5 100755 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -539,7 +539,7 @@ func updateRepoCommitCnt(ctx *macaron.Context, repo *models.Repository) error { } defer gitRepo.Close() - count, err := gitRepo.GetAllCommitsCount() + count, err := gitRepo.GetHeadCommitsCount() if err != nil { log.Error("GetAllCommitsCount failed:%v", err.Error(), ctx.Data["MsgID"]) return err diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go new file mode 100644 index 000000000..ad68f5b99 --- /dev/null +++ b/routers/repo/ai_model_manage.go @@ -0,0 +1,571 @@ +package repo + +import ( + "archive/zip" + "encoding/json" + "errors" + "fmt" + "net/http" + "path" + "strings" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + uuid "github.com/satori/go.uuid" +) + +const ( + Model_prefix = "aimodels/" + tplModelManageIndex = "repo/modelmanage/index" + tplModelManageDownload = "repo/modelmanage/download" + tplModelInfo = "repo/modelmanage/showinfo" + MODEL_LATEST = 1 + MODEL_NOT_LATEST = 0 +) + +func saveModelByParameters(jobId string, versionName string, name string, version string, label string, description string, ctx *context.Context) error { + aiTask, err := models.GetCloudbrainByJobIDAndVersionName(jobId, versionName) + if err != nil { + log.Info("query task error." + err.Error()) + return err + } + + uuid := uuid.NewV4() + id := uuid.String() + modelPath := id + var lastNewModelId string + var modelSize int64 + cloudType := models.TypeCloudBrainTwo + + log.Info("find task name:" + aiTask.JobName) + aimodels := models.QueryModelByName(name, aiTask.RepoID) + if len(aimodels) > 0 { + for _, model := range aimodels { + if model.Version == version { + return errors.New(ctx.Tr("repo.model.manage.create_error")) + } + if model.New == MODEL_LATEST { + lastNewModelId = model.ID + } + } + } + cloudType = aiTask.Type + //download model zip //train type + if cloudType == models.TypeCloudBrainTwo { + modelPath, modelSize, err = downloadModelFromCloudBrainTwo(id, aiTask.JobName, "", aiTask.TrainUrl) + if err != nil { + log.Info("download model from CloudBrainTwo faild." + err.Error()) + return err + } + } + accuracy := make(map[string]string) + accuracy["F1"] = "" + accuracy["Recall"] = "" + accuracy["Accuracy"] = "" + accuracy["Precision"] = "" + accuracyJson, _ := json.Marshal(accuracy) + log.Info("accuracyJson=" + string(accuracyJson)) + aiTaskJson, _ := json.Marshal(aiTask) + + model := &models.AiModelManage{ + ID: id, + Version: version, + VersionCount: len(aimodels) + 1, + Label: label, + Name: name, + Description: description, + New: MODEL_LATEST, + Type: cloudType, + Path: modelPath, + Size: modelSize, + AttachmentId: aiTask.Uuid, + RepoId: aiTask.RepoID, + UserId: ctx.User.ID, + CodeBranch: aiTask.BranchName, + CodeCommitID: aiTask.CommitID, + Engine: aiTask.EngineID, + TrainTaskInfo: string(aiTaskJson), + Accuracy: string(accuracyJson), + } + + err = models.SaveModelToDb(model) + if err != nil { + return err + } + if len(lastNewModelId) > 0 { + //udpate status and version count + models.ModifyModelNewProperty(lastNewModelId, MODEL_NOT_LATEST, 0) + } + + log.Info("save model end.") + + return nil +} + +func SaveModel(ctx *context.Context) { + log.Info("save model start.") + JobId := ctx.Query("JobId") + VersionName := ctx.Query("VersionName") + name := ctx.Query("Name") + version := ctx.Query("Version") + label := ctx.Query("Label") + description := ctx.Query("Description") + + if !ctx.Repo.CanWrite(models.UnitTypeModelManage) { + ctx.ServerError("No right.", errors.New(ctx.Tr("repo.model_noright"))) + return + } + + if JobId == "" || VersionName == "" { + ctx.Error(500, fmt.Sprintf("JobId or VersionName is null.")) + return + } + + if name == "" || version == "" { + ctx.Error(500, fmt.Sprintf("name or version is null.")) + return + } + + err := saveModelByParameters(JobId, VersionName, name, version, label, description, ctx) + + if err != nil { + log.Info("save model error." + err.Error()) + ctx.Error(500, fmt.Sprintf("save model error. %v", err)) + return + } + + log.Info("save model end.") +} + +func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string, trainUrl string) (string, int64, error) { + objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/") + if trainUrl != "" { + objectkey = strings.Trim(trainUrl[len(setting.Bucket)+1:], "/") + } + + modelDbResult, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, objectkey, "") + log.Info("bucket=" + setting.Bucket + " objectkey=" + objectkey) + if err != nil { + log.Info("get TrainJobListModel failed:", err) + return "", 0, err + } + if len(modelDbResult) == 0 { + return "", 0, errors.New("cannot create model, as model is empty.") + } + + prefix := objectkey + "/" + destKeyNamePrefix := Model_prefix + models.AttachmentRelativePath(modelUUID) + "/" + + size, err := storage.ObsCopyManyFile(setting.Bucket, prefix, setting.Bucket, destKeyNamePrefix) + + dataActualPath := setting.Bucket + "/" + destKeyNamePrefix + return dataActualPath, size, nil +} + +func DeleteModel(ctx *context.Context) { + log.Info("delete model start.") + id := ctx.Query("ID") + err := deleteModelByID(ctx, id) + if err != nil { + ctx.JSON(500, err.Error()) + } else { + ctx.JSON(200, map[string]string{ + "result_code": "0", + }) + } +} +func isCanDeleteOrDownload(ctx *context.Context, model *models.AiModelManage) bool { + if ctx.User.IsAdmin || ctx.User.ID == model.UserId { + return true + } + if ctx.Repo.IsOwner() { + return true + } + return false +} + +func deleteModelByID(ctx *context.Context, id string) error { + log.Info("delete model start. id=" + id) + model, err := models.QueryModelById(id) + if !isCanDeleteOrDownload(ctx, model) { + return errors.New(ctx.Tr("repo.model_noright")) + } + if err == nil { + log.Info("bucket=" + setting.Bucket + " path=" + model.Path) + if strings.HasPrefix(model.Path, setting.Bucket+"/"+Model_prefix) { + err := storage.ObsRemoveObject(setting.Bucket, model.Path[len(setting.Bucket)+1:]) + if err != nil { + log.Info("Failed to delete model. id=" + id) + return err + } + } + err = models.DeleteModelById(id) + if err == nil { //find a model to change new + aimodels := models.QueryModelByName(model.Name, model.RepoId) + if model.New == MODEL_LATEST { + if len(aimodels) > 0 { + //udpate status and version count + models.ModifyModelNewProperty(aimodels[0].ID, MODEL_LATEST, len(aimodels)) + } + } else { + for _, tmpModel := range aimodels { + if tmpModel.New == MODEL_LATEST { + models.ModifyModelNewProperty(tmpModel.ID, MODEL_LATEST, len(aimodels)) + break + } + } + } + } + } + return err +} + +func QueryModelByParameters(repoId int64, page int) ([]*models.AiModelManage, int64, error) { + + return models.QueryModel(&models.AiModelQueryOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + RepoID: repoId, + Type: -1, + New: MODEL_LATEST, + }) +} + +func DownloadMultiModelFile(ctx *context.Context) { + log.Info("DownloadMultiModelFile start.") + id := ctx.Query("ID") + log.Info("id=" + id) + task, err := models.QueryModelById(id) + if err != nil { + log.Error("no such model!", err.Error()) + ctx.ServerError("no such model:", err) + return + } + if !isCanDeleteOrDownload(ctx, task) { + ctx.ServerError("no right.", errors.New(ctx.Tr("repo.model_noright"))) + return + } + + path := Model_prefix + models.AttachmentRelativePath(id) + "/" + + allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path) + if err == nil { + //count++ + models.ModifyModelDownloadCount(id) + + returnFileName := task.Name + "_" + task.Version + ".zip" + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+returnFileName) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + w := zip.NewWriter(ctx.Resp) + defer w.Close() + for _, oneFile := range allFile { + if oneFile.IsDir { + log.Info("zip dir name:" + oneFile.FileName) + } else { + log.Info("zip file name:" + oneFile.FileName) + fDest, err := w.Create(oneFile.FileName) + if err != nil { + log.Info("create zip entry error, download file failed: %s\n", err.Error()) + ctx.ServerError("download file failed:", err) + return + } + body, err := storage.ObsDownloadAFile(setting.Bucket, path+oneFile.FileName) + if err != nil { + log.Info("download file failed: %s\n", err.Error()) + ctx.ServerError("download file failed:", err) + return + } else { + defer body.Close() + p := make([]byte, 1024) + var readErr error + var readCount int + // 读取对象内容 + for { + readCount, readErr = body.Read(p) + if readCount > 0 { + fDest.Write(p[:readCount]) + } + if readErr != nil { + break + } + } + } + } + } + } else { + log.Info("error,msg=" + err.Error()) + ctx.ServerError("no file to download.", err) + } +} + +func QueryTrainJobVersionList(ctx *context.Context) { + log.Info("query train job version list. start.") + JobID := ctx.Query("JobID") + + VersionListTasks, count, err := models.QueryModelTrainJobVersionList(JobID) + + log.Info("query return count=" + fmt.Sprint(count)) + + if err != nil { + ctx.ServerError("QueryTrainJobList:", err) + } else { + ctx.JSON(200, VersionListTasks) + } +} + +func QueryTrainJobList(ctx *context.Context) { + log.Info("query train job list. start.") + repoId := ctx.QueryInt64("repoId") + + VersionListTasks, count, err := models.QueryModelTrainJobList(repoId) + log.Info("query return count=" + fmt.Sprint(count)) + + if err != nil { + ctx.ServerError("QueryTrainJobList:", err) + } else { + ctx.JSON(200, VersionListTasks) + } + +} + +func DownloadSingleModelFile(ctx *context.Context) { + log.Info("DownloadSingleModelFile start.") + id := ctx.Params(":ID") + parentDir := ctx.Query("parentDir") + fileName := ctx.Query("fileName") + path := Model_prefix + models.AttachmentRelativePath(id) + "/" + parentDir + fileName + + if setting.PROXYURL != "" { + body, err := storage.ObsDownloadAFile(setting.Bucket, path) + if err != nil { + log.Info("download error.") + } else { + //count++ + models.ModifyModelDownloadCount(id) + defer body.Close() + ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName) + ctx.Resp.Header().Set("Content-Type", "application/octet-stream") + p := make([]byte, 1024) + var readErr error + var readCount int + // 读取对象内容 + for { + readCount, readErr = body.Read(p) + if readCount > 0 { + ctx.Resp.Write(p[:readCount]) + //fmt.Printf("%s", p[:readCount]) + } + if readErr != nil { + break + } + } + } + } else { + url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path) + if err != nil { + log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetObsCreateSignedUrl", err) + return + } + //count++ + models.ModifyModelDownloadCount(id) + http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently) + } +} + +func ShowModelInfo(ctx *context.Context) { + ctx.Data["ID"] = ctx.Query("ID") + ctx.Data["name"] = ctx.Query("name") + ctx.Data["isModelManage"] = true + ctx.HTML(200, tplModelInfo) +} + +func ShowSingleModel(ctx *context.Context) { + name := ctx.Query("name") + + log.Info("Show single ModelInfo start.name=" + name) + models := models.QueryModelByName(name, ctx.Repo.Repository.ID) + + userIds := make([]int64, len(models)) + for i, model := range models { + model.IsCanOper = isOper(ctx, model.UserId) + userIds[i] = model.UserId + } + userNameMap := queryUserName(userIds) + + for _, model := range models { + value := userNameMap[model.UserId] + if value != nil { + model.UserName = value.Name + model.UserRelAvatarLink = value.RelAvatarLink() + } + } + ctx.JSON(http.StatusOK, models) +} + +func queryUserName(intSlice []int64) map[int64]*models.User { + keys := make(map[int64]string) + uniqueElements := []int64{} + for _, entry := range intSlice { + if _, value := keys[entry]; !value { + keys[entry] = "" + uniqueElements = append(uniqueElements, entry) + } + } + result := make(map[int64]*models.User) + userLists, err := models.GetUsersByIDs(uniqueElements) + if err == nil { + for _, user := range userLists { + result[user.ID] = user + } + } + + return result +} + +func ShowOneVersionOtherModel(ctx *context.Context) { + repoId := ctx.Repo.Repository.ID + name := ctx.Query("name") + aimodels := models.QueryModelByName(name, repoId) + + userIds := make([]int64, len(aimodels)) + for i, model := range aimodels { + model.IsCanOper = isOper(ctx, model.UserId) + userIds[i] = model.UserId + } + userNameMap := queryUserName(userIds) + + for _, model := range aimodels { + value := userNameMap[model.UserId] + if value != nil { + model.UserName = value.Name + model.UserRelAvatarLink = value.RelAvatarLink() + } + } + + if len(aimodels) > 0 { + ctx.JSON(200, aimodels[1:]) + } else { + ctx.JSON(200, aimodels) + } +} + +func ShowModelTemplate(ctx *context.Context) { + ctx.Data["isModelManage"] = true + ctx.HTML(200, tplModelManageIndex) +} + +func isQueryRight(ctx *context.Context) bool { + if ctx.Repo.Repository.IsPrivate { + if ctx.Repo.CanRead(models.UnitTypeModelManage) || ctx.User.IsAdmin || ctx.Repo.IsAdmin() || ctx.Repo.IsOwner() { + return true + } + return false + } else { + return true + } +} + +func isOper(ctx *context.Context, modelUserId int64) bool { + if ctx.User == nil { + return false + } + if ctx.User.IsAdmin || ctx.Repo.IsOwner() || ctx.User.ID == modelUserId { + return true + } + return false +} + +func ShowModelPageInfo(ctx *context.Context) { + log.Info("ShowModelInfo start.") + if !isQueryRight(ctx) { + ctx.ServerError("no right.", errors.New(ctx.Tr("repo.model_noright"))) + return + } + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + pageSize := ctx.QueryInt("pageSize") + if pageSize <= 0 { + pageSize = setting.UI.IssuePagingNum + } + repoId := ctx.Repo.Repository.ID + Type := -1 + modelResult, count, err := models.QueryModel(&models.AiModelQueryOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: pageSize, + }, + RepoID: repoId, + Type: Type, + New: MODEL_LATEST, + }) + if err != nil { + ctx.ServerError("Cloudbrain", err) + return + } + + userIds := make([]int64, len(modelResult)) + for i, model := range modelResult { + model.IsCanOper = isOper(ctx, model.UserId) + userIds[i] = model.UserId + } + + userNameMap := queryUserName(userIds) + + for _, model := range modelResult { + value := userNameMap[model.UserId] + if value != nil { + model.UserName = value.Name + model.UserRelAvatarLink = value.RelAvatarLink() + } + } + + mapInterface := make(map[string]interface{}) + mapInterface["data"] = modelResult + mapInterface["count"] = count + ctx.JSON(http.StatusOK, mapInterface) +} + +func ModifyModel(id string, description string) error { + err := models.ModifyModelDescription(id, description) + if err == nil { + log.Info("modify success.") + } else { + log.Info("Failed to modify.id=" + id + " desc=" + description + " error:" + err.Error()) + } + return err +} + +func ModifyModelInfo(ctx *context.Context) { + log.Info("modify model start.") + id := ctx.Query("ID") + description := ctx.Query("Description") + + task, err := models.QueryModelById(id) + if err != nil { + log.Error("no such model!", err.Error()) + ctx.ServerError("no such model:", err) + return + } + if !isCanDeleteOrDownload(ctx, task) { + ctx.ServerError("no right.", errors.New(ctx.Tr("repo.model_noright"))) + return + } + + err = ModifyModel(id, description) + + if err != nil { + log.Info("modify error," + err.Error()) + ctx.ServerError("error.", err) + } else { + ctx.JSON(200, "success") + } + +} diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index d704ee0d9..3f5fce013 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -8,7 +8,6 @@ import ( "io" "net/http" "os" - "os/exec" "regexp" "sort" "strconv" @@ -50,46 +49,6 @@ func MustEnableCloudbrain(ctx *context.Context) { return } } -func CloudBrainIndex(ctx *context.Context) { - MustEnableCloudbrain(ctx) - repo := ctx.Repo.Repository - page := ctx.QueryInt("page") - if page <= 0 { - page = 1 - } - - ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ - ListOptions: models.ListOptions{ - Page: page, - PageSize: setting.UI.IssuePagingNum, - }, - RepoID: repo.ID, - Type: models.TypeCloudBrainOne, - }) - if err != nil { - ctx.ServerError("Cloudbrain", err) - return - } - - timestamp := time.Now().Unix() - for i, task := range ciTasks { - if task.Status == string(models.JobRunning) && (timestamp-int64(task.Cloudbrain.CreatedUnix) > 10) { - ciTasks[i].CanDebug = true - } else { - ciTasks[i].CanDebug = false - } - - ciTasks[i].CanDel = models.CanDelJob(ctx.IsSigned, ctx.User, task) - } - - pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) - pager.SetDefaultParams(ctx) - ctx.Data["Page"] = pager - - ctx.Data["PageIsCloudBrain"] = true - ctx.Data["Tasks"] = ciTasks - ctx.HTML(200, tplCloudBrainIndex) -} func cutString(str string, lens int) string { if len(str) < lens { @@ -247,14 +206,11 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } repo := ctx.Repo.Repository downloadCode(repo, codePath) + uploadCodeToMinio(codePath+"/", jobName, "/code/") - modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath - err = os.MkdirAll(modelPath, os.ModePerm) - if err != nil { - cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) - return - } + modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/" + mkModelPath(modelPath) + uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/") benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath if setting.IsBenchmarkEnabled && jobType == string(models.JobTypeBenchmark) { @@ -264,26 +220,31 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { gpuType = gpuInfo.Value } } - downloadRateCode(repo, jobName, setting.BenchmarkCode, benchmarkPath, form.BenchmarkCategory, gpuType) + downloadRateCode(repo, jobName, setting.BenchmarkOwner, setting.BrainScoreName, benchmarkPath, form.BenchmarkCategory, gpuType) + uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/") } snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) { - downloadRateCode(repo, jobName, setting.Snn4imagenetCode, snn4imagenetPath, "", "") + downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "") + uploadCodeToMinio(snn4imagenetPath+"/", jobName, cloudbrain.Snn4imagenetMountPath+"/") } brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) { - downloadRateCode(repo, jobName, setting.BrainScoreCode, brainScorePath, "", "") + downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "") + uploadCodeToMinio(brainScorePath+"/", jobName, cloudbrain.BrainScoreMountPath+"/") } - err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, resourceSpecId) + err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, getMinioPath(jobName, cloudbrain.ModelMountPath+"/"), + getMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), getMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), + getMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), jobType, gpuQueue, resourceSpecId) if err != nil { cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) return } - ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain") + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob") } func CloudBrainShow(ctx *context.Context) { @@ -338,7 +299,7 @@ func CloudBrainDebug(ctx *context.Context) { var jobID = ctx.Params(":jobid") if !ctx.IsSigned { log.Error("the user has not signed in") - ctx.Error(http.StatusForbidden, "","the user has not signed in") + ctx.Error(http.StatusForbidden, "", "the user has not signed in") return } task, err := models.GetCloudbrainByJobID(jobID) @@ -355,7 +316,7 @@ func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrain var jobID = ctx.Params(":jobid") if !ctx.IsSigned { log.Error("the user has not signed in") - ctx.Error(http.StatusForbidden, "","the user has not signed in") + ctx.Error(http.StatusForbidden, "", "the user has not signed in") return } task, err := models.GetCloudbrainByJobID(jobID) @@ -415,8 +376,7 @@ func CloudBrainStop(ctx *context.Context) { ctx.ServerError("UpdateJob failed", err) return } - - ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain") + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob") } func StopJobsByUserID(userID int64) { @@ -452,14 +412,22 @@ func StopJobs(cloudBrains []*models.Cloudbrain) { logErrorAndUpdateJobStatus(err, taskInfo) } else { - param := models.NotebookAction{ - Action: models.ActionStop, + if taskInfo.JobType == string(models.JobTypeTrain) { + err := retry(3, time.Second*30, func() error { + _, err := modelarts.StopTrainJob(taskInfo.JobID, strconv.FormatInt(taskInfo.VersionID, 10)) + return err + }) + logErrorAndUpdateJobStatus(err, taskInfo) + } else { + param := models.NotebookAction{ + Action: models.ActionStop, + } + err := retry(3, time.Second*30, func() error { + _, err := modelarts.StopJob(taskInfo.JobID, param) + return err + }) + logErrorAndUpdateJobStatus(err, taskInfo) } - err := retry(3, time.Second*30, func() error { - _, err := modelarts.StopJob(taskInfo.JobID, param) - return err - }) - logErrorAndUpdateJobStatus(err, taskInfo) } } @@ -510,8 +478,7 @@ func CloudBrainDel(ctx *context.Context) { ctx.ServerError("DeleteJob failed", err) return } - - ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain") + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob") } func CloudBrainShowModels(ctx *context.Context) { @@ -528,10 +495,10 @@ func CloudBrainShowModels(ctx *context.Context) { } //get dirs - dirs, err := getModelDirs(task.JobName, parentDir) + dirs, err := GetModelDirs(task.JobName, parentDir) if err != nil { - log.Error("getModelDirs failed:%v", err.Error(), ctx.Data["msgID"]) - ctx.ServerError("getModelDirs failed:", err) + log.Error("GetModelDirs failed:%v", err.Error(), ctx.Data["msgID"]) + ctx.ServerError("GetModelDirs failed:", err) return } @@ -591,9 +558,9 @@ func getImages(ctx *context.Context, imageType string) { log.Info("Get images end") } -func getModelDirs(jobName string, parentDir string) (string, error) { +func GetModelDirs(jobName string, parentDir string) (string, error) { var req string - modelActualPath := setting.JobPath + jobName + "/model/" + modelActualPath := getMinioPath(jobName, cloudbrain.ModelMountPath+"/") if parentDir == "" { req = "baseDir=" + modelActualPath } else { @@ -603,6 +570,10 @@ func getModelDirs(jobName string, parentDir string) (string, error) { return getDirs(req) } +func getMinioPath(jobName, suffixPath string) string { + return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + jobName + suffixPath +} + func CloudBrainDownloadModel(ctx *context.Context) { parentDir := ctx.Query("parentDir") fileName := ctx.Query("fileName") @@ -684,19 +655,21 @@ func downloadCode(repo *models.Repository, codePath string) error { return nil } -func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benchmarkCategory, gpuType string) error { +func downloadRateCode(repo *models.Repository, taskName, rateOwnerName, rateRepoName, codePath, benchmarkCategory, gpuType string) error { err := os.MkdirAll(codePath, os.ModePerm) if err != nil { log.Error("mkdir codePath failed", err.Error()) return err } - command := "git clone " + gitPath + " " + codePath - cmd := exec.Command("/bin/bash", "-c", command) - output, err := cmd.Output() - log.Info(string(output)) + repoExt, err := models.GetRepositoryByOwnerAndName(rateOwnerName, rateRepoName) if err != nil { - log.Error("exec.Command(%s) failed:%v", command, err) + log.Error("GetRepositoryByOwnerAndName(%s) failed", rateRepoName, err.Error()) + return err + } + + if err := git.Clone(repoExt.RepoPath(), codePath, git.CloneRepoOptions{}); err != nil { + log.Error("Failed to clone repository: %s (%v)", repoExt.FullName(), err) return err } @@ -730,3 +703,144 @@ func downloadRateCode(repo *models.Repository, taskName, gitPath, codePath, benc return nil } + +func uploadCodeToMinio(codePath, jobName, parentDir string) error { + files, err := readDir(codePath) + if err != nil { + log.Error("readDir(%s) failed: %s", codePath, err.Error()) + return err + } + + for _, file := range files { + if file.IsDir() { + if err = uploadCodeToMinio(codePath+file.Name()+"/", jobName, parentDir+file.Name()+"/"); err != nil { + log.Error("uploadCodeToMinio(%s) failed: %s", file.Name(), err.Error()) + return err + } + } else { + destObject := setting.CBCodePathPrefix + jobName + parentDir + file.Name() + sourceFile := codePath + file.Name() + err = storage.Attachments.UploadObject(destObject, sourceFile) + if err != nil { + log.Error("UploadObject(%s) failed: %s", file.Name(), err.Error()) + return err + } + } + } + + return nil +} + +func mkModelPath(modelPath string) error { + err := os.MkdirAll(modelPath, os.ModePerm) + if err != nil { + log.Error("MkdirAll(%s) failed:%v", modelPath, err) + return err + } + + fileName := modelPath + "README" + f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) + if err != nil { + log.Error("OpenFile failed", err.Error()) + return err + } + + defer f.Close() + + _, err = f.WriteString("You can put the model file into this directory and download it by the web page.") + if err != nil { + log.Error("WriteString failed", err.Error()) + return err + } + + return nil +} + +func SyncCloudbrainStatus() { + cloudBrains, err := models.GetCloudBrainUnStoppedJob() + if err != nil { + log.Error("GetCloudBrainUnStoppedJob failed:", err.Error()) + return + } + + for _, task := range cloudBrains { + if task.Type == models.TypeCloudBrainOne { + result, err := cloudbrain.GetJob(task.JobID) + if err != nil { + log.Error("GetJob(%s) failed:%v", task.JobName, err) + continue + } + + if result != nil { + jobRes, _ := models.ConvertToJobResultPayload(result.Payload) + taskRoles := jobRes.TaskRoles + taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{})) + task.Status = taskRes.TaskStatuses[0].State + if task.Status != string(models.JobWaiting) { + task.Duration = time.Now().Unix() - taskRes.TaskStatuses[0].StartAt.Unix() + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + } + + if task.Duration >= setting.MaxDuration { + log.Info("begin to stop job(%s), because of the duration", task.JobName) + err = cloudbrain.StopJob(task.JobID) + if err != nil { + log.Error("StopJob(%s) failed:%v", task.JobName, err) + continue + } + task.Status = string(models.JobStopped) + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + } + } + } + } + } else if task.Type == models.TypeCloudBrainTwo { + if task.JobType == string(models.JobTypeDebug) { + result, err := modelarts.GetJob(task.JobID) + if err != nil { + log.Error("GetJob(%s) failed:%v", task.JobName, err) + continue + } + + if result != nil { + task.Status = result.Status + + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + continue + } + } + } else if task.JobType == string(models.JobTypeTrain) { + result, err := modelarts.GetTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10)) + if err != nil { + log.Error("GetTrainJob(%s) failed:%v", task.JobName, err) + continue + } + + if result != nil { + task.Status = modelarts.TransTrainJobStatus(result.IntStatus) + task.Duration = result.Duration + task.TrainJobDuration = result.TrainJobDuration + + err = models.UpdateJob(task) + if err != nil { + log.Error("UpdateJob(%s) failed:%v", task.JobName, err) + continue + } + } + } else { + log.Error("task.JobType(%s) is error:%s", task.JobName, task.JobType) + } + + } else { + log.Error("task.Type(%s) is error:%d", task.JobName, task.Type) + } + } + + return +} diff --git a/routers/repo/dir.go b/routers/repo/dir.go index 406f3dc73..81549e76a 100755 --- a/routers/repo/dir.go +++ b/routers/repo/dir.go @@ -12,7 +12,6 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/obs" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" ) @@ -70,40 +69,10 @@ func DeleteAllUnzipFile(attachment *models.Attachment, parentDir string) { } } if attachment.Type == models.TypeCloudBrainTwo { - - input := &obs.ListObjectsInput{} - input.Bucket = setting.Bucket - // 设置每页100个对象 - input.MaxKeys = 100 - input.Prefix = setting.BasePath + attachment.RelativePath() + attachment.UUID - index := 1 - log.Info("prefix=" + input.Prefix) - for { - output, err := storage.ObsCli.ListObjects(input) - if err == nil { - log.Info("Page:%d\n", index) - index++ - for _, val := range output.Contents { - log.Info("delete obs file:" + val.Key) - delObj := &obs.DeleteObjectInput{} - delObj.Bucket = setting.Bucket - delObj.Key = val.Key - storage.ObsCli.DeleteObject(delObj) - } - if output.IsTruncated { - input.Marker = output.NextMarker - } else { - break - } - } else { - if obsError, ok := err.(obs.ObsError); ok { - log.Info("Code:%s\n", obsError.Code) - log.Info("Message:%s\n", obsError.Message) - } - break - } + err := storage.ObsRemoveObject(setting.Bucket, setting.BasePath+attachment.RelativePath()+attachment.UUID) + if err != nil { + log.Info("delete file error.") } - } } diff --git a/routers/repo/http.go b/routers/repo/http.go index ad2abf567..87406a2c3 100644 --- a/routers/repo/http.go +++ b/routers/repo/http.go @@ -257,7 +257,6 @@ func HTTP(ctx *context.Context) { models.EnvPusherID + fmt.Sprintf("=%d", authUser.ID), models.EnvIsDeployKey + "=false", } - if !authUser.KeepEmailPrivate { environ = append(environ, models.EnvPusherEmail+"="+authUser.Email) } @@ -559,6 +558,7 @@ func serviceRPC(h serviceHandler, service string) { if service == "receive-pack" { cmd.Env = append(os.Environ(), h.environ...) } + cmd.Stdout = h.w cmd.Stdin = reqBody cmd.Stderr = &stderr diff --git a/routers/repo/issue.go b/routers/repo/issue.go index b9ac0ff22..77ed0251d 100755 --- a/routers/repo/issue.go +++ b/routers/repo/issue.go @@ -1390,7 +1390,7 @@ func isLegalReviewRequest(reviewer, doer *models.User, isAdd bool, issue *models var pemResult bool if isAdd { - pemResult = permReviewer.CanAccessAny(models.AccessModeRead, models.UnitTypePullRequests) + pemResult = permReviewer.CanAccessAny(models.AccessModeWrite, models.UnitTypePullRequests) if !pemResult { return fmt.Errorf("Reviewer can't read [user_id: %d, repo_name: %s]", reviewer.ID, issue.Repo.Name) } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 848a9bbae..d2a094bc8 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "io" - "io/ioutil" "net/http" "os" "path" @@ -12,6 +11,8 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" @@ -27,6 +28,8 @@ import ( ) const ( + tplDebugJobIndex base.TplName = "repo/debugjob/index" + tplModelArtsNotebookIndex base.TplName = "repo/modelarts/notebook/index" tplModelArtsNotebookNew base.TplName = "repo/modelarts/notebook/new" tplModelArtsNotebookShow base.TplName = "repo/modelarts/notebook/show" @@ -37,21 +40,23 @@ const ( tplModelArtsTrainJobVersionNew base.TplName = "repo/modelarts/trainjob/version_new" ) -// MustEnableDataset check if repository enable internal cb -func MustEnableModelArts(ctx *context.Context) { - if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) { - ctx.NotFound("MustEnableCloudbrain", nil) - return - } -} - -func NotebookIndex(ctx *context.Context) { - MustEnableModelArts(ctx) +func DebugJobIndex(ctx *context.Context) { + debugListType := ctx.Query("debugListType") + MustEnableCloudbrain(ctx) repo := ctx.Repo.Repository page := ctx.QueryInt("page") if page <= 0 { page = 1 } + debugType := modelarts.DebugType + jobType := string(models.JobTypeDebug) + if debugListType == modelarts.GPUResource { + debugType = models.TypeCloudBrainOne + jobType = "" + } + if debugListType == modelarts.NPUResource { + debugType = models.TypeCloudBrainTwo + } ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ @@ -59,29 +64,44 @@ func NotebookIndex(ctx *context.Context) { PageSize: setting.UI.IssuePagingNum, }, RepoID: repo.ID, - Type: models.TypeCloudBrainTwo, - JobType: string(models.JobTypeDebug), + Type: debugType, + JobType: jobType, }) if err != nil { - ctx.ServerError("Cloudbrain", err) + ctx.ServerError("Get debugjob faild:", err) return } for i, task := range ciTasks { - if task.Status == string(models.JobRunning) { - ciTasks[i].CanDebug = true - } else { - ciTasks[i].CanDebug = false + if task.Cloudbrain.Type == models.TypeCloudBrainOne { + ciTasks[i].CanDebug = cloudbrain.CanCreateOrDebugJob(ctx) + ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) + ciTasks[i].Cloudbrain.ComputeResource = modelarts.GPUResource + } + if task.Cloudbrain.Type == models.TypeCloudBrainTwo { + ciTasks[i].CanDebug = cloudbrain.CanCreateOrDebugJob(ctx) + ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) + ciTasks[i].Cloudbrain.ComputeResource = modelarts.NPUResource } + } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) pager.SetDefaultParams(ctx) ctx.Data["Page"] = pager - ctx.Data["PageIsCloudBrain"] = true ctx.Data["Tasks"] = ciTasks - ctx.HTML(200, tplModelArtsNotebookIndex) + ctx.Data["CanCreate"] = cloudbrain.CanCreateOrDebugJob(ctx) + ctx.Data["RepoIsEmpty"] = repo.IsEmpty + ctx.HTML(200, tplDebugJobIndex) +} + +// MustEnableDataset check if repository enable internal cb +func MustEnableModelArts(ctx *context.Context) { + if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) { + ctx.NotFound("MustEnableCloudbrain", nil) + return + } } func NotebookNew(ctx *context.Context) { @@ -136,8 +156,7 @@ func NotebookCreate(ctx *context.Context, form auth.CreateModelArtsNotebookForm) ctx.RenderWithErr(err.Error(), tplModelArtsNotebookNew, &form) return } - - ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook") + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob") } func NotebookShow(ctx *context.Context) { @@ -244,8 +263,7 @@ func NotebookStop(ctx *context.Context) { ctx.ServerError("UpdateJob failed", err) return } - - ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook") + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob") } func NotebookDel(ctx *context.Context) { @@ -275,7 +293,7 @@ func NotebookDel(ctx *context.Context) { return } - ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/notebook") + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob") } func TrainJobIndex(ctx *context.Context) { @@ -302,12 +320,19 @@ func TrainJobIndex(ctx *context.Context) { return } + for i, task := range tasks { + tasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) + tasks[i].CanModify = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain) + } + pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) pager.SetDefaultParams(ctx) ctx.Data["Page"] = pager ctx.Data["PageIsCloudBrain"] = true ctx.Data["Tasks"] = tasks + ctx.Data["CanCreate"] = cloudbrain.CanCreateOrDebugJob(ctx) + ctx.Data["RepoIsEmpty"] = repo.IsEmpty ctx.HTML(200, tplModelArtsTrainJobIndex) } @@ -376,9 +401,8 @@ func trainJobNewDataPrepare(ctx *context.Context) error { outputObsPath := "/" + setting.Bucket + modelarts.JobPath + jobName + modelarts.OutputPath ctx.Data["train_url"] = outputObsPath - ctx.Data["params"] = "" - ctx.Data["BranchName"] = ctx.Repo.BranchName + ctx.Data["branchName"] = ctx.Repo.BranchName configList, err := getConfigList(modelarts.PerPage, 1, modelarts.SortByCreateTime, "desc", "", modelarts.ConfigTypeCustom) if err != nil { @@ -390,7 +414,7 @@ func trainJobNewDataPrepare(ctx *context.Context) error { return nil } -func ErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error { +func trainJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error { ctx.Data["PageIsCloudBrain"] = true //can, err := canUserCreateTrainJob(ctx.User.ID) @@ -570,7 +594,7 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error { return nil } -func VersionErrorDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error { +func versionErrorDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error { ctx.Data["PageIsCloudBrain"] = true var jobID = ctx.Params(":jobid") // var versionName = ctx.Params(":version-name") @@ -687,13 +711,13 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) if err != nil { log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) - ErrorNewDataPrepare(ctx, form) + trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("system error", tplModelArtsTrainJobNew, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - ErrorNewDataPrepare(ctx, form) + trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsTrainJobNew, &form) return } @@ -701,22 +725,23 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) if err := paramCheckCreateTrainJob(form); err != nil { log.Error("paramCheckCreateTrainJob failed:(%v)", err) - ErrorNewDataPrepare(ctx, form) + trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobNew, &form) return } - attach, err := models.GetAttachmentByUUID(uuid) - if err != nil { - log.Error("GetAttachmentByUUID(%s) failed:%v", uuid, err.Error()) - return - } + // attach, err := models.GetAttachmentByUUID(uuid) + // if err != nil { + // log.Error("GetAttachmentByUUID(%s) failed:%v", uuid, err.Error()) + // return + // } //todo: del the codeLocalPath - _, err = ioutil.ReadDir(codeLocalPath) - if err == nil { - os.RemoveAll(codeLocalPath) - } + // _, err := ioutil.ReadDir(codeLocalPath) + // if err == nil { + // os.RemoveAll(codeLocalPath) + // } + os.RemoveAll(codeLocalPath) gitRepo, _ := git.OpenRepository(repo.RepoPath()) commitID, _ := gitRepo.GetBranchCommitID(branch_name) @@ -725,7 +750,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) Branch: branch_name, }); err != nil { log.Error("创建任务失败，服务器超时！: %s (%v)", repo.FullName(), err) - trainJobNewDataPrepare(ctx) + trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("创建任务失败，服务器超时！", tplModelArtsTrainJobNew, &form) return } @@ -733,14 +758,14 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) //todo: upload code (send to file_server todo this work?) if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err) - trainJobNewDataPrepare(ctx) + trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("Failed to obsMkdir_output", tplModelArtsTrainJobNew, &form) return } if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err) - trainJobNewDataPrepare(ctx) + trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("Failed to obsMkdir_log", tplModelArtsTrainJobNew, &form) return } @@ -749,7 +774,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { // if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil { log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) - trainJobNewDataPrepare(ctx) + trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("Failed to uploadCodeToObs", tplModelArtsTrainJobNew, &form) return } @@ -769,7 +794,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) err := json.Unmarshal([]byte(params), ¶meters) if err != nil { log.Error("Failed to Unmarshal params: %s (%v)", params, err) - trainJobNewDataPrepare(ctx) + trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("运行参数错误", tplModelArtsTrainJobNew, &form) return } @@ -812,7 +837,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) if err != nil { log.Error("Failed to CreateTrainJobConfig: %v", err) - trainJobNewDataPrepare(ctx) + trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("保存作业参数失败:"+err.Error(), tplModelArtsTrainJobNew, &form) return } @@ -853,12 +878,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) err = modelarts.GenerateTrainJob(ctx, req) if err != nil { log.Error("GenerateTrainJob failed:%v", err.Error()) - trainJobNewDataPrepare(ctx) - ctx.Data["bootFile"] = form.BootFile - ctx.Data["uuid"] = form.Attachment - ctx.Data["datasetName"] = attach.Name - ctx.Data["params"] = Parameters.Parameter - ctx.Data["branch_name"] = branch_name + trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobNew, &form) return } @@ -872,13 +892,13 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) if err != nil { log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) - VersionErrorDataPrepare(ctx, form) + versionErrorDataPrepare(ctx, form) ctx.RenderWithErr("system error", tplModelArtsTrainJobVersionNew, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) - VersionErrorDataPrepare(ctx, form) + versionErrorDataPrepare(ctx, form) ctx.RenderWithErr("you have already a running or waiting task, can not create more", tplModelArtsTrainJobVersionNew, &form) return } @@ -922,7 +942,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ if err := paramCheckCreateTrainJob(form); err != nil { log.Error("paramCheckCreateTrainJob failed:(%v)", err) - VersionErrorDataPrepare(ctx, form) + versionErrorDataPrepare(ctx, form) ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form) return } @@ -946,7 +966,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ Branch: branch_name, }); err != nil { log.Error("创建任务失败，任务名称已存在！: %s (%v)", repo.FullName(), err) - VersionErrorDataPrepare(ctx, form) + versionErrorDataPrepare(ctx, form) ctx.RenderWithErr("创建任务失败，任务名称已存在！", tplModelArtsTrainJobVersionNew, &form) return } @@ -954,14 +974,14 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ //todo: upload code (send to file_server todo this work?) if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.OutputPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_output: %s (%v)", repo.FullName(), err) - VersionErrorDataPrepare(ctx, form) + versionErrorDataPrepare(ctx, form) ctx.RenderWithErr("Failed to obsMkdir_output", tplModelArtsTrainJobVersionNew, &form) return } if err := obsMkdir(setting.CodePathPrefix + jobName + modelarts.LogPath + VersionOutputPath + "/"); err != nil { log.Error("Failed to obsMkdir_log: %s (%v)", repo.FullName(), err) - VersionErrorDataPrepare(ctx, form) + versionErrorDataPrepare(ctx, form) ctx.RenderWithErr("Failed to obsMkdir_log", tplModelArtsTrainJobVersionNew, &form) return } @@ -971,7 +991,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ // if err := uploadCodeToObs(codeLocalPath, jobName, ""); err != nil { if err := uploadCodeToObs(codeLocalPath, jobName, parentDir); err != nil { log.Error("Failed to uploadCodeToObs: %s (%v)", repo.FullName(), err) - VersionErrorDataPrepare(ctx, form) + versionErrorDataPrepare(ctx, form) ctx.RenderWithErr("Failed to uploadCodeToObs", tplModelArtsTrainJobVersionNew, &form) return } @@ -991,7 +1011,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ err := json.Unmarshal([]byte(params), ¶meters) if err != nil { log.Error("Failed to Unmarshal params: %s (%v)", params, err) - VersionErrorDataPrepare(ctx, form) + versionErrorDataPrepare(ctx, form) ctx.RenderWithErr("运行参数错误", tplModelArtsTrainJobVersionNew, &form) return } @@ -1010,7 +1030,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ if isSaveParam == "on" { if form.ParameterTemplateName == "" { log.Error("ParameterTemplateName is empty") - VersionErrorDataPrepare(ctx, form) + versionErrorDataPrepare(ctx, form) ctx.RenderWithErr("保存作业参数时，作业参数名称不能为空", tplModelArtsTrainJobVersionNew, &form) return } @@ -1034,7 +1054,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ if err != nil { log.Error("Failed to CreateTrainJobConfig: %v", err) - VersionErrorDataPrepare(ctx, form) + versionErrorDataPrepare(ctx, form) ctx.RenderWithErr("保存作业参数失败:"+err.Error(), tplModelArtsTrainJobVersionNew, &form) return } @@ -1081,7 +1101,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ err = modelarts.GenerateTrainJobVersion(ctx, req, jobID) if err != nil { log.Error("GenerateTrainJob failed:%v", err.Error()) - VersionErrorDataPrepare(ctx, form) + versionErrorDataPrepare(ctx, form) ctx.RenderWithErr(err.Error(), tplModelArtsTrainJobVersionNew, &form) return } @@ -1210,7 +1230,7 @@ func TrainJobShow(ctx *context.Context) { ctx.Data["canNewJob"] = canNewJob //将运行参数转化为epoch_size = 3, device_target = Ascend的格式 - for i, _ := range VersionListTasks { + for i, task := range VersionListTasks { var parameters models.Parameters @@ -1231,6 +1251,9 @@ func TrainJobShow(ctx *context.Context) { } else { VersionListTasks[i].Parameters = "" } + + VersionListTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain) + VersionListTasks[i].CanModify = cloudbrain.CanModifyJob(ctx, &task.Cloudbrain) } pager := context.NewPagination(VersionListCount, setting.UI.IssuePagingNum, page, 5) diff --git a/routers/repo/repo.go b/routers/repo/repo.go index 437521d5a..a182e9087 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -532,6 +532,7 @@ func Download(ctx *context.Context) { } ctx.Repo.Repository.IncreaseCloneCnt() + ctx.Repo.Repository.IncreaseGitCloneCnt() ctx.ServeFile(archivePath, ctx.Repo.Repository.Name+"-"+refName+ext) } diff --git a/routers/repo/setting.go b/routers/repo/setting.go index bf11f9e5a..f7da8f4a8 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -239,6 +239,18 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { deleteUnitTypes = append(deleteUnitTypes, models.UnitTypeCloudBrain) } + if form.EnableModelManager && !models.UnitTypeModelManage.UnitGlobalDisabled() { + units = append(units, models.RepoUnit{ + RepoID: repo.ID, + Type: models.UnitTypeModelManage, + Config: &models.ModelManageConfig{ + EnableModelManage: form.EnableModelManager, + }, + }) + } else if !models.UnitTypeModelManage.UnitGlobalDisabled() { + deleteUnitTypes = append(deleteUnitTypes, models.UnitTypeModelManage) + } + if form.EnableWiki && form.EnableExternalWiki && !models.UnitTypeExternalWiki.UnitGlobalDisabled() { if !validation.IsValidExternalURL(form.ExternalWikiURL) { ctx.Flash.Error(ctx.Tr("repo.settings.external_wiki_url_error")) diff --git a/routers/repo/user_data_analysis.go b/routers/repo/user_data_analysis.go index 2f37dc15e..42189c57f 100755 --- a/routers/repo/user_data_analysis.go +++ b/routers/repo/user_data_analysis.go @@ -148,11 +148,6 @@ func QueryUserStaticDataPage(ctx *context.Context) { func TimingCountDataByDateAndReCount(date string, isReCount bool) { - if date == "refreshAll" { - models.RefreshUserStaticAllTabel() - return - } - t, _ := time.Parse("2006-01-02", date) startTime := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, t.Location()) @@ -205,10 +200,9 @@ func TimingCountDataByDateAndReCount(date string, isReCount bool) { log.Error("count user info error." + err.Error()) mailer.SendWarnNotifyMail(setting.Warn_Notify_Mails, warnEmailMessage) } - - if isReCount { - models.RefreshUserStaticAllTabel() - } + log.Info("start to count all user info data") + //models.RefreshUserStaticAllTabel(wikiMap) + log.Info("end to count all user info data") } func TimingCountDataByDate(date string) { diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 3bacb7549..30e486b98 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -12,6 +12,8 @@ import ( "text/template" "time" + "code.gitea.io/gitea/modules/cloudbrain" + "code.gitea.io/gitea/routers/operation" "code.gitea.io/gitea/routers/private" @@ -313,6 +315,8 @@ func RegisterRoutes(m *macaron.Macaron) { }) m.Get("/", routers.Home) m.Get("/dashboard", routers.Dashboard) + m.Get("/recommend/org", routers.RecommendOrgFromPromote) + m.Get("/recommend/repo", routers.RecommendRepoFromPromote) m.Group("/explore", func() { m.Get("", func(ctx *context.Context) { ctx.Redirect(setting.AppSubURL + "/explore/repos") @@ -612,6 +616,8 @@ func RegisterRoutes(m *macaron.Macaron) { reqRepoDatasetWriter := context.RequireRepoWriter(models.UnitTypeDatasets) reqRepoCloudBrainReader := context.RequireRepoReader(models.UnitTypeCloudBrain) reqRepoCloudBrainWriter := context.RequireRepoWriter(models.UnitTypeCloudBrain) + reqRepoModelManageReader := context.RequireRepoReader(models.UnitTypeModelManage) + reqRepoModelManageWriter := context.RequireRepoWriter(models.UnitTypeModelManage) //reqRepoBlockChainReader := context.RequireRepoReader(models.UnitTypeBlockChain) //reqRepoBlockChainWriter := context.RequireRepoWriter(models.UnitTypeBlockChain) @@ -792,7 +798,7 @@ func RegisterRoutes(m *macaron.Macaron) { }, reqSignIn, context.RepoAssignment(), context.UnitTypes(), reqRepoAdmin, context.RepoRef()) m.Post("/:username/:reponame/action/:action", reqSignIn, context.RepoAssignment(), context.UnitTypes(), repo.Action) - m.Get("/tool/query_user_static_page", adminReq, repo.QueryUserStaticDataPage) + // Grouping for those endpoints not requiring authentication m.Group("/:username/:reponame", func() { m.Get("/contributors", repo.Contributors) @@ -954,29 +960,48 @@ func RegisterRoutes(m *macaron.Macaron) { }, context.RepoRef()) m.Group("/cloudbrain", func() { - m.Get("", reqRepoCloudBrainReader, repo.CloudBrainIndex) m.Group("/:jobid", func() { m.Get("", reqRepoCloudBrainReader, repo.CloudBrainShow) - m.Get("/debug", reqRepoCloudBrainReader, repo.CloudBrainDebug) - m.Post("/commit_image", reqRepoCloudBrainWriter, bindIgnErr(auth.CommitImageCloudBrainForm{}), repo.CloudBrainCommitImage) - m.Post("/stop", reqRepoCloudBrainWriter, repo.CloudBrainStop) - m.Post("/del", reqRepoCloudBrainWriter, repo.CloudBrainDel) + m.Get("/debug", reqRepoCloudBrainWriter, repo.CloudBrainDebug) + m.Post("/commit_image", cloudbrain.AdminOrOwnerOrJobCreaterRight, bindIgnErr(auth.CommitImageCloudBrainForm{}), repo.CloudBrainCommitImage) + m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainStop) + m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainDel) m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate) m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) - m.Get("/download_model", reqRepoCloudBrainReader, repo.CloudBrainDownloadModel) + m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainDownloadModel) }) - m.Get("/create", reqRepoCloudBrainReader, repo.CloudBrainNew) + m.Get("/create", reqRepoCloudBrainWriter, repo.CloudBrainNew) m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) }, context.RepoRef()) + m.Group("/modelmanage", func() { + m.Post("/create_model", reqRepoModelManageWriter, repo.SaveModel) + m.Delete("/delete_model", repo.DeleteModel) + m.Put("/modify_model", repo.ModifyModelInfo) + m.Get("/show_model", reqRepoModelManageReader, repo.ShowModelTemplate) + m.Get("/show_model_info", repo.ShowModelInfo) + m.Get("/show_model_info_api", repo.ShowSingleModel) + m.Get("/show_model_api", repo.ShowModelPageInfo) + m.Get("/show_model_child_api", repo.ShowOneVersionOtherModel) + m.Get("/query_train_job", reqRepoCloudBrainReader, repo.QueryTrainJobList) + m.Get("/query_train_job_version", reqRepoCloudBrainReader, repo.QueryTrainJobVersionList) + m.Group("/:ID", func() { + m.Get("", repo.ShowSingleModel) + m.Get("/downloadsingle", repo.DownloadSingleModelFile) + }) + m.Get("/downloadall", repo.DownloadMultiModelFile) + }, context.RepoRef()) + + m.Group("/debugjob", func() { + m.Get("", reqRepoCloudBrainReader, repo.DebugJobIndex) + }, context.RepoRef()) m.Group("/modelarts", func() { m.Group("/notebook", func() { - m.Get("", reqRepoCloudBrainReader, repo.NotebookIndex) m.Group("/:jobid", func() { m.Get("", reqRepoCloudBrainReader, repo.NotebookShow) - m.Get("/debug", reqRepoCloudBrainReader, repo.NotebookDebug) - m.Post("/stop", reqRepoCloudBrainWriter, repo.NotebookStop) - m.Post("/del", reqRepoCloudBrainWriter, repo.NotebookDel) + m.Get("/debug", reqRepoCloudBrainWriter, repo.NotebookDebug) + m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.NotebookStop) + m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.NotebookDel) }) m.Get("/create", reqRepoCloudBrainWriter, repo.NotebookNew) m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsNotebookForm{}), repo.NotebookCreate) @@ -986,13 +1011,13 @@ func RegisterRoutes(m *macaron.Macaron) { m.Get("", reqRepoCloudBrainReader, repo.TrainJobIndex) m.Group("/:jobid", func() { m.Get("", reqRepoCloudBrainReader, repo.TrainJobShow) - m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop) - m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel) - m.Get("/model_download", reqRepoCloudBrainReader, repo.ModelDownload) - m.Get("/create_version", reqRepoCloudBrainReader, repo.TrainJobNewVersion) - m.Post("/create_version", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) + m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.TrainJobStop) + m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.TrainJobDel) + m.Get("/model_download", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.ModelDownload) + m.Get("/create_version", cloudbrain.AdminOrJobCreaterRight, repo.TrainJobNewVersion) + m.Post("/create_version", cloudbrain.AdminOrJobCreaterRight, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) }) - m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNew) + m.Get("/create", reqRepoCloudBrainWriter, repo.TrainJobNew) m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate) m.Get("/para-config-list", reqRepoCloudBrainReader, repo.TrainJobGetConfigList) diff --git a/routers/user/profile.go b/routers/user/profile.go index 0e29b6117..41d8561d6 100755 --- a/routers/user/profile.go +++ b/routers/user/profile.go @@ -6,6 +6,7 @@ package user import ( + "errors" "fmt" "path" "strings" @@ -103,10 +104,47 @@ func Profile(ctx *context.Context) { return } + for _, org := range orgs { + _, repoCount, err := models.SearchRepository(&models.SearchRepoOptions{ + OwnerID: org.ID, + Private: ctx.IsSigned, + Actor: ctx.User, + }) + if err != nil { + ctx.ServerError("SearchRepository", err) + return + } + + var opts = models.FindOrgMembersOpts{ + OrgID: org.ID, + PublicOnly: true, + } + + if ctx.User != nil { + isMember, err := org.IsOrgMember(ctx.User.ID) + if err != nil { + ctx.Error(500, "IsOrgMember") + return + } + opts.PublicOnly = !isMember && !ctx.User.IsAdmin + } + + membersCount, err := models.CountOrgMembers(opts) + if err != nil { + ctx.ServerError("CountOrgMembers", err) + return + } + org.NumMembers = int(membersCount) + org.NumRepos = int(repoCount) + } + ctx.Data["Orgs"] = orgs ctx.Data["HasOrgsVisible"] = models.HasOrgsVisible(orgs, ctx.User) tab := ctx.Query("tab") + if tab == "" { + tab = "activity" + } ctx.Data["TabName"] = tab page := ctx.QueryInt("page") @@ -210,10 +248,15 @@ func Profile(ctx *context.Context) { total = int(count) case "datasets": + var isOwner = false + if ctx.User != nil && ctx.User.ID == ctxUser.ID { + isOwner = true + } datasetSearchOptions := &models.SearchDatasetOptions{ Keyword: keyword, OwnerID: ctxUser.ID, SearchOrderBy: orderBy, + IsOwner: isOwner, ListOptions: models.ListOptions{ Page: page, PageSize: setting.UI.ExplorePagingNum, @@ -230,7 +273,7 @@ func Profile(ctx *context.Context) { } total = int(count) ctx.Data["Datasets"] = datasets - default: + case "repository": repos, count, err = models.SearchRepository(&models.SearchRepoOptions{ ListOptions: models.ListOptions{ PageSize: setting.UI.User.RepoPagingNum, @@ -251,6 +294,9 @@ func Profile(ctx *context.Context) { } total = int(count) + default: + ctx.ServerError("tab error", errors.New("tab error")) + return } ctx.Data["Repos"] = repos ctx.Data["Total"] = total diff --git a/templates/base/head_navbar.tmpl b/templates/base/head_navbar.tmpl index 49b3181e7..53d0f8839 100755 --- a/templates/base/head_navbar.tmpl +++ b/templates/base/head_navbar.tmpl @@ -17,47 +17,63 @@ {{if .IsSigned}} - {{.i18n.Tr "index"}} - {{.i18n.Tr "custom.head.openi"}} - {{if not .UnitIssuesGlobalDisabled}} - {{.i18n.Tr "issues"}} - {{end}} - {{if not .UnitPullsGlobalDisabled}} - {{.i18n.Tr "pull_requests"}} - {{end}} - {{if not (and .UnitIssuesGlobalDisabled .UnitPullsGlobalDisabled)}} - {{if .ShowMilestonesDashboardPage}}{{.i18n.Tr "milestones"}}{{end}} - {{end}} -
+
+
+ + {{.i18n.Tr "index"}} + + +
+ {{.i18n.Tr "issues"}} + {{.i18n.Tr "pull_requests"}} + {{.i18n.Tr "milestones"}} +
+
+
+ + {{.i18n.Tr "custom.head.project"}} + {{.i18n.Tr "custom.head.dataset"}} + +
{{else if .IsLandingPageHome}} - {{.i18n.Tr "home"}} - {{.i18n.Tr "custom.head.openi"}} +
+
+ + {{.i18n.Tr "home"}} + + +
+ {{.i18n.Tr "issues"}} + {{.i18n.Tr "pull_requests"}} + {{.i18n.Tr "milestones"}} +
+
+
-
+ {{.i18n.Tr "custom.head.project"}} + {{.i18n.Tr "custom.head.dataset"}} +