| @@ -35,6 +35,7 @@ type AiModelManage struct { | |||
| TrainTaskInfo string `xorm:"text NULL"` | |||
| CreatedUnix timeutil.TimeStamp `xorm:"created"` | |||
| UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` | |||
| IsCanOper bool | |||
| } | |||
| type AiModelQueryOptions struct { | |||
| @@ -1,9 +1,10 @@ | |||
| package cloudbrain | |||
| import ( | |||
| "code.gitea.io/gitea/modules/setting" | |||
| "errors" | |||
| "code.gitea.io/gitea/modules/setting" | |||
| "code.gitea.io/gitea/models" | |||
| "code.gitea.io/gitea/modules/context" | |||
| "code.gitea.io/gitea/modules/log" | |||
| @@ -16,7 +17,7 @@ const ( | |||
| ModelMountPath = "/model" | |||
| BenchMarkMountPath = "/benchmark" | |||
| Snn4imagenetMountPath = "/snn4imagenet" | |||
| BrainScoreMountPath = "/brainscore" | |||
| BrainScoreMountPath = "/brainscore" | |||
| TaskInfoName = "/taskInfo" | |||
| SubTaskName = "task1" | |||
| @@ -28,6 +29,52 @@ var ( | |||
| ResourceSpecs *models.ResourceSpecs | |||
| ) | |||
| func isAdminOrOwnerOrJobCreater(ctx *context.Context, jobId string) bool { | |||
| job, err := models.GetCloudbrainByJobID(jobId) | |||
| if err != nil { | |||
| return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() | |||
| } else { | |||
| return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID | |||
| } | |||
| } | |||
| func isAdminOrJobCreater(ctx *context.Context, jobId string) bool { | |||
| job, err := models.GetCloudbrainByJobID(jobId) | |||
| if err != nil { | |||
| return ctx.IsUserSiteAdmin() | |||
| } else { | |||
| return ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID | |||
| } | |||
| } | |||
| func AdminOrOwnerOrJobCreaterRight(ctx *context.Context) { | |||
| var jobID = ctx.Params(":jobid") | |||
| if !isAdminOrOwnerOrJobCreater(ctx, jobID) { | |||
| ctx.NotFound(ctx.Req.URL.RequestURI(), nil) | |||
| } | |||
| } | |||
| func AdminOrJobCreaterRight(ctx *context.Context) { | |||
| var jobID = ctx.Params(":jobid") | |||
| if !isAdminOrJobCreater(ctx, jobID) { | |||
| ctx.NotFound(ctx.Req.URL.RequestURI(), nil) | |||
| } | |||
| } | |||
| func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue string, resourceSpecId int) error { | |||
| dataActualPath := setting.Attachment.Minio.RealPath + | |||
| setting.Attachment.Minio.Bucket + "/" + | |||
| @@ -433,6 +433,9 @@ var ( | |||
| AuthUser string | |||
| AuthPassword string | |||
| //home page | |||
| RecommentRepoAddr string | |||
| //labelsystem config | |||
| LabelTaskName string | |||
| LabelDatasetDeleteQueue string | |||
| @@ -549,7 +552,7 @@ var ( | |||
| RecordBeginTime string | |||
| IgnoreMirrorRepo bool | |||
| }{} | |||
| Warn_Notify_Mails []string | |||
| ) | |||
| @@ -1224,6 +1227,9 @@ func NewContext() { | |||
| LabelDatasetDeleteQueue = sec.Key("LabelDatasetDeleteQueue").MustString("LabelDatasetDeleteQueue") | |||
| DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue") | |||
| sec = Cfg.Section("homepage") | |||
| RecommentRepoAddr = sec.Key("Address").MustString("https://git.openi.org.cn/OpenIOSSG/promote/raw/branch/master/") | |||
| sec = Cfg.Section("cloudbrain") | |||
| CBAuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X") | |||
| CBAuthPassword = sec.Key("PWD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC") | |||
| @@ -783,7 +783,7 @@ datasets.desc=数据集功能 | |||
| cloudbrain_helper=使用GPU/NPU资源,开启Notebook、模型训练任务等 | |||
| model_manager = 模型管理 | |||
| model_noright=无权限操作 | |||
| debug=调试 | |||
| stop=停止 | |||
| @@ -7,6 +7,8 @@ package routers | |||
| import ( | |||
| "bytes" | |||
| "fmt" | |||
| "io/ioutil" | |||
| "net/http" | |||
| "strings" | |||
| @@ -511,3 +513,36 @@ func NotFound(ctx *context.Context) { | |||
| ctx.Data["Title"] = "Page Not Found" | |||
| ctx.NotFound("home.NotFound", nil) | |||
| } | |||
| func RecommendOrgFromPromote(ctx *context.Context) { | |||
| url := setting.RecommentRepoAddr + "organizations" | |||
| recommendFromPromote(ctx, url) | |||
| } | |||
| func recommendFromPromote(ctx *context.Context, url string) { | |||
| resp, err := http.Get(url) | |||
| if err != nil { | |||
| log.Info("Get organizations url error=" + err.Error()) | |||
| ctx.ServerError("QueryTrainJobList:", err) | |||
| return | |||
| } | |||
| bytes, err := ioutil.ReadAll(resp.Body) | |||
| resp.Body.Close() | |||
| if err != nil { | |||
| log.Info("Get organizations url error=" + err.Error()) | |||
| ctx.ServerError("QueryTrainJobList:", err) | |||
| return | |||
| } | |||
| allLineStr := string(bytes) | |||
| lines := strings.Split(allLineStr, "\n") | |||
| for i, line := range lines { | |||
| log.Info("i=" + fmt.Sprint(i) + " line=" + line) | |||
| } | |||
| ctx.JSON(http.StatusOK, lines) | |||
| } | |||
| func RecommendRepoFromPromote(ctx *context.Context) { | |||
| url := setting.RecommentRepoAddr + "projects" | |||
| recommendFromPromote(ctx, url) | |||
| } | |||
| @@ -164,6 +164,7 @@ func isErrUnverifiedCommit(err error) bool { | |||
| // HookPreReceive checks whether a individual commit is acceptable | |||
| func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) { | |||
| log.Info("Git pre start..................................") | |||
| ownerName := ctx.Params(":owner") | |||
| repoName := ctx.Params(":repo") | |||
| repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) | |||
| @@ -370,6 +371,9 @@ func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) { | |||
| // HookPostReceive updates services and users | |||
| func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { | |||
| log.Info("Git post start..................................") | |||
| ownerName := ctx.Params(":owner") | |||
| repoName := ctx.Params(":repo") | |||
| @@ -118,6 +118,11 @@ func SaveModel(ctx *context.Context) { | |||
| label := ctx.Query("Label") | |||
| description := ctx.Query("Description") | |||
| if !ctx.Repo.CanWrite(models.UnitTypeCloudBrain) { | |||
| ctx.ServerError("No right.", errors.New(ctx.Tr("repo.model_noright"))) | |||
| return | |||
| } | |||
| if JobId == "" || VersionName == "" { | |||
| ctx.Error(500, fmt.Sprintf("JobId or VersionName is null.")) | |||
| return | |||
| @@ -164,7 +169,7 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir | |||
| func DeleteModel(ctx *context.Context) { | |||
| log.Info("delete model start.") | |||
| id := ctx.Query("ID") | |||
| err := DeleteModelByID(id) | |||
| err := deleteModelByID(ctx, id) | |||
| if err != nil { | |||
| ctx.JSON(500, err.Error()) | |||
| } else { | |||
| @@ -173,11 +178,22 @@ func DeleteModel(ctx *context.Context) { | |||
| }) | |||
| } | |||
| } | |||
| func isCanDeleteOrDownload(ctx *context.Context, model *models.AiModelManage) bool { | |||
| if ctx.User.IsAdmin || ctx.User.ID == model.UserId { | |||
| return true | |||
| } | |||
| if ctx.Repo.IsOwner() { | |||
| return true | |||
| } | |||
| return false | |||
| } | |||
| func DeleteModelByID(id string) error { | |||
| func deleteModelByID(ctx *context.Context, id string) error { | |||
| log.Info("delete model start. id=" + id) | |||
| model, err := models.QueryModelById(id) | |||
| if !isCanDeleteOrDownload(ctx, model) { | |||
| return errors.New(ctx.Tr("repo.model_noright")) | |||
| } | |||
| if err == nil { | |||
| log.Info("bucket=" + setting.Bucket + " path=" + model.Path) | |||
| if strings.HasPrefix(model.Path, setting.Bucket+"/"+Model_prefix) { | |||
| @@ -224,6 +240,11 @@ func DownloadMultiModelFile(ctx *context.Context) { | |||
| ctx.ServerError("no such model:", err) | |||
| return | |||
| } | |||
| if !isCanDeleteOrDownload(ctx, task) { | |||
| ctx.ServerError("no right.", errors.New(ctx.Tr("repo.model_noright"))) | |||
| return | |||
| } | |||
| path := Model_prefix + models.AttachmentRelativePath(id) + "/" | |||
| allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path) | |||
| @@ -381,9 +402,30 @@ func ShowModelTemplate(ctx *context.Context) { | |||
| ctx.HTML(200, tplModelManageIndex) | |||
| } | |||
| func isQueryRight(ctx *context.Context) bool { | |||
| if ctx.Repo.Repository.IsPrivate { | |||
| if ctx.User.IsAdmin || ctx.Repo.IsAdmin() || ctx.Repo.IsOwner() || ctx.Repo.CanRead(models.UnitTypeCloudBrain) { | |||
| return true | |||
| } | |||
| return false | |||
| } else { | |||
| return true | |||
| } | |||
| } | |||
| func isOper(ctx *context.Context, modelUserId int64) bool { | |||
| if ctx.User.IsAdmin || ctx.Repo.IsAdmin() || ctx.Repo.IsOwner() || ctx.User.ID == modelUserId { | |||
| return true | |||
| } | |||
| return false | |||
| } | |||
| func ShowModelPageInfo(ctx *context.Context) { | |||
| log.Info("ShowModelInfo start.") | |||
| if !isQueryRight(ctx) { | |||
| ctx.ServerError("no right.", errors.New(ctx.Tr("repo.model_noright"))) | |||
| return | |||
| } | |||
| page := ctx.QueryInt("page") | |||
| if page <= 0 { | |||
| page = 1 | |||
| @@ -404,6 +446,10 @@ func ShowModelPageInfo(ctx *context.Context) { | |||
| return | |||
| } | |||
| for _, model := range modelResult { | |||
| model.IsCanOper = isOper(ctx, model.UserId) | |||
| } | |||
| mapInterface := make(map[string]interface{}) | |||
| mapInterface["data"] = modelResult | |||
| mapInterface["count"] = count | |||
| @@ -12,6 +12,8 @@ import ( | |||
| "text/template" | |||
| "time" | |||
| "code.gitea.io/gitea/modules/cloudbrain" | |||
| "code.gitea.io/gitea/routers/operation" | |||
| "code.gitea.io/gitea/routers/private" | |||
| @@ -313,6 +315,8 @@ func RegisterRoutes(m *macaron.Macaron) { | |||
| }) | |||
| m.Get("/", routers.Home) | |||
| m.Get("/dashboard", routers.Dashboard) | |||
| m.Get("/recommend/org", routers.RecommendOrgFromPromote) | |||
| m.Get("/recommend/repo", routers.RecommendRepoFromPromote) | |||
| m.Group("/explore", func() { | |||
| m.Get("", func(ctx *context.Context) { | |||
| ctx.Redirect(setting.AppSubURL + "/explore/repos") | |||
| @@ -957,15 +961,15 @@ func RegisterRoutes(m *macaron.Macaron) { | |||
| m.Get("", reqRepoCloudBrainReader, repo.CloudBrainIndex) | |||
| m.Group("/:jobid", func() { | |||
| m.Get("", reqRepoCloudBrainReader, repo.CloudBrainShow) | |||
| m.Get("/debug", reqRepoCloudBrainReader, repo.CloudBrainDebug) | |||
| m.Post("/commit_image", reqRepoCloudBrainWriter, bindIgnErr(auth.CommitImageCloudBrainForm{}), repo.CloudBrainCommitImage) | |||
| m.Post("/stop", reqRepoCloudBrainWriter, repo.CloudBrainStop) | |||
| m.Post("/del", reqRepoCloudBrainWriter, repo.CloudBrainDel) | |||
| m.Get("/debug", reqRepoCloudBrainWriter, repo.CloudBrainDebug) | |||
| m.Post("/commit_image", cloudbrain.AdminOrOwnerOrJobCreaterRight, bindIgnErr(auth.CommitImageCloudBrainForm{}), repo.CloudBrainCommitImage) | |||
| m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainStop) | |||
| m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainDel) | |||
| m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate) | |||
| m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) | |||
| m.Get("/download_model", reqRepoCloudBrainReader, repo.CloudBrainDownloadModel) | |||
| m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainDownloadModel) | |||
| }) | |||
| m.Get("/create", reqRepoCloudBrainReader, repo.CloudBrainNew) | |||
| m.Get("/create", reqRepoCloudBrainWriter, repo.CloudBrainNew) | |||
| m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) | |||
| }, context.RepoRef()) | |||
| m.Group("/modelmanage", func() { | |||
| @@ -991,9 +995,9 @@ func RegisterRoutes(m *macaron.Macaron) { | |||
| m.Get("", reqRepoCloudBrainReader, repo.NotebookIndex) | |||
| m.Group("/:jobid", func() { | |||
| m.Get("", reqRepoCloudBrainReader, repo.NotebookShow) | |||
| m.Get("/debug", reqRepoCloudBrainReader, repo.NotebookDebug) | |||
| m.Post("/stop", reqRepoCloudBrainWriter, repo.NotebookStop) | |||
| m.Post("/del", reqRepoCloudBrainWriter, repo.NotebookDel) | |||
| m.Get("/debug", reqRepoCloudBrainWriter, repo.NotebookDebug) | |||
| m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.NotebookStop) | |||
| m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.NotebookDel) | |||
| }) | |||
| m.Get("/create", reqRepoCloudBrainWriter, repo.NotebookNew) | |||
| m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsNotebookForm{}), repo.NotebookCreate) | |||
| @@ -1003,13 +1007,13 @@ func RegisterRoutes(m *macaron.Macaron) { | |||
| m.Get("", reqRepoCloudBrainReader, repo.TrainJobIndex) | |||
| m.Group("/:jobid", func() { | |||
| m.Get("", reqRepoCloudBrainReader, repo.TrainJobShow) | |||
| m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop) | |||
| m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel) | |||
| m.Get("/model_download", reqRepoCloudBrainReader, repo.ModelDownload) | |||
| m.Get("/create_version", reqRepoCloudBrainReader, repo.TrainJobNewVersion) | |||
| m.Post("/create_version", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) | |||
| m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.TrainJobStop) | |||
| m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.TrainJobDel) | |||
| m.Get("/model_download", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.ModelDownload) | |||
| m.Get("/create_version", cloudbrain.AdminOrJobCreaterRight, repo.TrainJobNewVersion) | |||
| m.Post("/create_version", cloudbrain.AdminOrJobCreaterRight, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) | |||
| }) | |||
| m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNew) | |||
| m.Get("/create", reqRepoCloudBrainWriter, repo.TrainJobNew) | |||
| m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate) | |||
| m.Get("/para-config-list", reqRepoCloudBrainReader, repo.TrainJobGetConfigList) | |||