| @@ -35,6 +35,7 @@ type AiModelManage struct { | |||||
| TrainTaskInfo string `xorm:"text NULL"` | TrainTaskInfo string `xorm:"text NULL"` | ||||
| CreatedUnix timeutil.TimeStamp `xorm:"created"` | CreatedUnix timeutil.TimeStamp `xorm:"created"` | ||||
| UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` | UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` | ||||
| IsCanOper bool | |||||
| } | } | ||||
| type AiModelQueryOptions struct { | type AiModelQueryOptions struct { | ||||
| @@ -1,9 +1,10 @@ | |||||
| package cloudbrain | package cloudbrain | ||||
| import ( | import ( | ||||
| "code.gitea.io/gitea/modules/setting" | |||||
| "errors" | "errors" | ||||
| "code.gitea.io/gitea/modules/setting" | |||||
| "code.gitea.io/gitea/models" | "code.gitea.io/gitea/models" | ||||
| "code.gitea.io/gitea/modules/context" | "code.gitea.io/gitea/modules/context" | ||||
| "code.gitea.io/gitea/modules/log" | "code.gitea.io/gitea/modules/log" | ||||
| @@ -16,7 +17,7 @@ const ( | |||||
| ModelMountPath = "/model" | ModelMountPath = "/model" | ||||
| BenchMarkMountPath = "/benchmark" | BenchMarkMountPath = "/benchmark" | ||||
| Snn4imagenetMountPath = "/snn4imagenet" | Snn4imagenetMountPath = "/snn4imagenet" | ||||
| BrainScoreMountPath = "/brainscore" | |||||
| BrainScoreMountPath = "/brainscore" | |||||
| TaskInfoName = "/taskInfo" | TaskInfoName = "/taskInfo" | ||||
| SubTaskName = "task1" | SubTaskName = "task1" | ||||
| @@ -28,6 +29,52 @@ var ( | |||||
| ResourceSpecs *models.ResourceSpecs | ResourceSpecs *models.ResourceSpecs | ||||
| ) | ) | ||||
| func isAdminOrOwnerOrJobCreater(ctx *context.Context, jobId string) bool { | |||||
| job, err := models.GetCloudbrainByJobID(jobId) | |||||
| if err != nil { | |||||
| return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() | |||||
| } else { | |||||
| return ctx.IsUserRepoOwner() || ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID | |||||
| } | |||||
| } | |||||
| func isAdminOrJobCreater(ctx *context.Context, jobId string) bool { | |||||
| job, err := models.GetCloudbrainByJobID(jobId) | |||||
| if err != nil { | |||||
| return ctx.IsUserSiteAdmin() | |||||
| } else { | |||||
| return ctx.IsUserSiteAdmin() || ctx.User.ID == job.UserID | |||||
| } | |||||
| } | |||||
| func AdminOrOwnerOrJobCreaterRight(ctx *context.Context) { | |||||
| var jobID = ctx.Params(":jobid") | |||||
| if !isAdminOrOwnerOrJobCreater(ctx, jobID) { | |||||
| ctx.NotFound(ctx.Req.URL.RequestURI(), nil) | |||||
| } | |||||
| } | |||||
| func AdminOrJobCreaterRight(ctx *context.Context) { | |||||
| var jobID = ctx.Params(":jobid") | |||||
| if !isAdminOrJobCreater(ctx, jobID) { | |||||
| ctx.NotFound(ctx.Req.URL.RequestURI(), nil) | |||||
| } | |||||
| } | |||||
| func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue string, resourceSpecId int) error { | func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue string, resourceSpecId int) error { | ||||
| dataActualPath := setting.Attachment.Minio.RealPath + | dataActualPath := setting.Attachment.Minio.RealPath + | ||||
| setting.Attachment.Minio.Bucket + "/" + | setting.Attachment.Minio.Bucket + "/" + | ||||
| @@ -433,6 +433,9 @@ var ( | |||||
| AuthUser string | AuthUser string | ||||
| AuthPassword string | AuthPassword string | ||||
| //home page | |||||
| RecommentRepoAddr string | |||||
| //labelsystem config | //labelsystem config | ||||
| LabelTaskName string | LabelTaskName string | ||||
| LabelDatasetDeleteQueue string | LabelDatasetDeleteQueue string | ||||
| @@ -549,7 +552,7 @@ var ( | |||||
| RecordBeginTime string | RecordBeginTime string | ||||
| IgnoreMirrorRepo bool | IgnoreMirrorRepo bool | ||||
| }{} | }{} | ||||
| Warn_Notify_Mails []string | Warn_Notify_Mails []string | ||||
| ) | ) | ||||
| @@ -1224,6 +1227,9 @@ func NewContext() { | |||||
| LabelDatasetDeleteQueue = sec.Key("LabelDatasetDeleteQueue").MustString("LabelDatasetDeleteQueue") | LabelDatasetDeleteQueue = sec.Key("LabelDatasetDeleteQueue").MustString("LabelDatasetDeleteQueue") | ||||
| DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue") | DecompressOBSTaskName = sec.Key("DecompressOBSTaskName").MustString("LabelDecompressOBSQueue") | ||||
| sec = Cfg.Section("homepage") | |||||
| RecommentRepoAddr = sec.Key("Address").MustString("https://git.openi.org.cn/OpenIOSSG/promote/raw/branch/master/") | |||||
| sec = Cfg.Section("cloudbrain") | sec = Cfg.Section("cloudbrain") | ||||
| CBAuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X") | CBAuthUser = sec.Key("USER").MustString("cW4cMtH24eoWPE7X") | ||||
| CBAuthPassword = sec.Key("PWD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC") | CBAuthPassword = sec.Key("PWD").MustString("4BPmgvK2hb2Eywwyp4YZRY4B7yQf4DAC") | ||||
| @@ -783,7 +783,7 @@ datasets.desc=数据集功能 | |||||
| cloudbrain_helper=使用GPU/NPU资源,开启Notebook、模型训练任务等 | cloudbrain_helper=使用GPU/NPU资源,开启Notebook、模型训练任务等 | ||||
| model_manager = 模型管理 | model_manager = 模型管理 | ||||
| model_noright=无权限操作 | |||||
| debug=调试 | debug=调试 | ||||
| stop=停止 | stop=停止 | ||||
| @@ -7,6 +7,8 @@ package routers | |||||
| import ( | import ( | ||||
| "bytes" | "bytes" | ||||
| "fmt" | |||||
| "io/ioutil" | |||||
| "net/http" | "net/http" | ||||
| "strings" | "strings" | ||||
| @@ -511,3 +513,36 @@ func NotFound(ctx *context.Context) { | |||||
| ctx.Data["Title"] = "Page Not Found" | ctx.Data["Title"] = "Page Not Found" | ||||
| ctx.NotFound("home.NotFound", nil) | ctx.NotFound("home.NotFound", nil) | ||||
| } | } | ||||
| func RecommendOrgFromPromote(ctx *context.Context) { | |||||
| url := setting.RecommentRepoAddr + "organizations" | |||||
| recommendFromPromote(ctx, url) | |||||
| } | |||||
| func recommendFromPromote(ctx *context.Context, url string) { | |||||
| resp, err := http.Get(url) | |||||
| if err != nil { | |||||
| log.Info("Get organizations url error=" + err.Error()) | |||||
| ctx.ServerError("QueryTrainJobList:", err) | |||||
| return | |||||
| } | |||||
| bytes, err := ioutil.ReadAll(resp.Body) | |||||
| resp.Body.Close() | |||||
| if err != nil { | |||||
| log.Info("Get organizations url error=" + err.Error()) | |||||
| ctx.ServerError("QueryTrainJobList:", err) | |||||
| return | |||||
| } | |||||
| allLineStr := string(bytes) | |||||
| lines := strings.Split(allLineStr, "\n") | |||||
| for i, line := range lines { | |||||
| log.Info("i=" + fmt.Sprint(i) + " line=" + line) | |||||
| } | |||||
| ctx.JSON(http.StatusOK, lines) | |||||
| } | |||||
| func RecommendRepoFromPromote(ctx *context.Context) { | |||||
| url := setting.RecommentRepoAddr + "projects" | |||||
| recommendFromPromote(ctx, url) | |||||
| } | |||||
| @@ -164,6 +164,7 @@ func isErrUnverifiedCommit(err error) bool { | |||||
| // HookPreReceive checks whether a individual commit is acceptable | // HookPreReceive checks whether a individual commit is acceptable | ||||
| func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) { | func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) { | ||||
| log.Info("Git pre start..................................") | |||||
| ownerName := ctx.Params(":owner") | ownerName := ctx.Params(":owner") | ||||
| repoName := ctx.Params(":repo") | repoName := ctx.Params(":repo") | ||||
| repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) | repo, err := models.GetRepositoryByOwnerAndName(ownerName, repoName) | ||||
| @@ -370,6 +371,9 @@ func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) { | |||||
| // HookPostReceive updates services and users | // HookPostReceive updates services and users | ||||
| func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { | func HookPostReceive(ctx *macaron.Context, opts private.HookOptions) { | ||||
| log.Info("Git post start..................................") | |||||
| ownerName := ctx.Params(":owner") | ownerName := ctx.Params(":owner") | ||||
| repoName := ctx.Params(":repo") | repoName := ctx.Params(":repo") | ||||
| @@ -118,6 +118,11 @@ func SaveModel(ctx *context.Context) { | |||||
| label := ctx.Query("Label") | label := ctx.Query("Label") | ||||
| description := ctx.Query("Description") | description := ctx.Query("Description") | ||||
| if !ctx.Repo.CanWrite(models.UnitTypeCloudBrain) { | |||||
| ctx.ServerError("No right.", errors.New(ctx.Tr("repo.model_noright"))) | |||||
| return | |||||
| } | |||||
| if JobId == "" || VersionName == "" { | if JobId == "" || VersionName == "" { | ||||
| ctx.Error(500, fmt.Sprintf("JobId or VersionName is null.")) | ctx.Error(500, fmt.Sprintf("JobId or VersionName is null.")) | ||||
| return | return | ||||
| @@ -164,7 +169,7 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir | |||||
| func DeleteModel(ctx *context.Context) { | func DeleteModel(ctx *context.Context) { | ||||
| log.Info("delete model start.") | log.Info("delete model start.") | ||||
| id := ctx.Query("ID") | id := ctx.Query("ID") | ||||
| err := DeleteModelByID(id) | |||||
| err := deleteModelByID(ctx, id) | |||||
| if err != nil { | if err != nil { | ||||
| ctx.JSON(500, err.Error()) | ctx.JSON(500, err.Error()) | ||||
| } else { | } else { | ||||
| @@ -173,11 +178,22 @@ func DeleteModel(ctx *context.Context) { | |||||
| }) | }) | ||||
| } | } | ||||
| } | } | ||||
| func isCanDeleteOrDownload(ctx *context.Context, model *models.AiModelManage) bool { | |||||
| if ctx.User.IsAdmin || ctx.User.ID == model.UserId { | |||||
| return true | |||||
| } | |||||
| if ctx.Repo.IsOwner() { | |||||
| return true | |||||
| } | |||||
| return false | |||||
| } | |||||
| func DeleteModelByID(id string) error { | |||||
| func deleteModelByID(ctx *context.Context, id string) error { | |||||
| log.Info("delete model start. id=" + id) | log.Info("delete model start. id=" + id) | ||||
| model, err := models.QueryModelById(id) | model, err := models.QueryModelById(id) | ||||
| if !isCanDeleteOrDownload(ctx, model) { | |||||
| return errors.New(ctx.Tr("repo.model_noright")) | |||||
| } | |||||
| if err == nil { | if err == nil { | ||||
| log.Info("bucket=" + setting.Bucket + " path=" + model.Path) | log.Info("bucket=" + setting.Bucket + " path=" + model.Path) | ||||
| if strings.HasPrefix(model.Path, setting.Bucket+"/"+Model_prefix) { | if strings.HasPrefix(model.Path, setting.Bucket+"/"+Model_prefix) { | ||||
| @@ -224,6 +240,11 @@ func DownloadMultiModelFile(ctx *context.Context) { | |||||
| ctx.ServerError("no such model:", err) | ctx.ServerError("no such model:", err) | ||||
| return | return | ||||
| } | } | ||||
| if !isCanDeleteOrDownload(ctx, task) { | |||||
| ctx.ServerError("no right.", errors.New(ctx.Tr("repo.model_noright"))) | |||||
| return | |||||
| } | |||||
| path := Model_prefix + models.AttachmentRelativePath(id) + "/" | path := Model_prefix + models.AttachmentRelativePath(id) + "/" | ||||
| allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path) | allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path) | ||||
| @@ -381,9 +402,30 @@ func ShowModelTemplate(ctx *context.Context) { | |||||
| ctx.HTML(200, tplModelManageIndex) | ctx.HTML(200, tplModelManageIndex) | ||||
| } | } | ||||
| func isQueryRight(ctx *context.Context) bool { | |||||
| if ctx.Repo.Repository.IsPrivate { | |||||
| if ctx.User.IsAdmin || ctx.Repo.IsAdmin() || ctx.Repo.IsOwner() || ctx.Repo.CanRead(models.UnitTypeCloudBrain) { | |||||
| return true | |||||
| } | |||||
| return false | |||||
| } else { | |||||
| return true | |||||
| } | |||||
| } | |||||
| func isOper(ctx *context.Context, modelUserId int64) bool { | |||||
| if ctx.User.IsAdmin || ctx.Repo.IsAdmin() || ctx.Repo.IsOwner() || ctx.User.ID == modelUserId { | |||||
| return true | |||||
| } | |||||
| return false | |||||
| } | |||||
| func ShowModelPageInfo(ctx *context.Context) { | func ShowModelPageInfo(ctx *context.Context) { | ||||
| log.Info("ShowModelInfo start.") | log.Info("ShowModelInfo start.") | ||||
| if !isQueryRight(ctx) { | |||||
| ctx.ServerError("no right.", errors.New(ctx.Tr("repo.model_noright"))) | |||||
| return | |||||
| } | |||||
| page := ctx.QueryInt("page") | page := ctx.QueryInt("page") | ||||
| if page <= 0 { | if page <= 0 { | ||||
| page = 1 | page = 1 | ||||
| @@ -404,6 +446,10 @@ func ShowModelPageInfo(ctx *context.Context) { | |||||
| return | return | ||||
| } | } | ||||
| for _, model := range modelResult { | |||||
| model.IsCanOper = isOper(ctx, model.UserId) | |||||
| } | |||||
| mapInterface := make(map[string]interface{}) | mapInterface := make(map[string]interface{}) | ||||
| mapInterface["data"] = modelResult | mapInterface["data"] = modelResult | ||||
| mapInterface["count"] = count | mapInterface["count"] = count | ||||
| @@ -12,6 +12,8 @@ import ( | |||||
| "text/template" | "text/template" | ||||
| "time" | "time" | ||||
| "code.gitea.io/gitea/modules/cloudbrain" | |||||
| "code.gitea.io/gitea/routers/operation" | "code.gitea.io/gitea/routers/operation" | ||||
| "code.gitea.io/gitea/routers/private" | "code.gitea.io/gitea/routers/private" | ||||
| @@ -313,6 +315,8 @@ func RegisterRoutes(m *macaron.Macaron) { | |||||
| }) | }) | ||||
| m.Get("/", routers.Home) | m.Get("/", routers.Home) | ||||
| m.Get("/dashboard", routers.Dashboard) | m.Get("/dashboard", routers.Dashboard) | ||||
| m.Get("/recommend/org", routers.RecommendOrgFromPromote) | |||||
| m.Get("/recommend/repo", routers.RecommendRepoFromPromote) | |||||
| m.Group("/explore", func() { | m.Group("/explore", func() { | ||||
| m.Get("", func(ctx *context.Context) { | m.Get("", func(ctx *context.Context) { | ||||
| ctx.Redirect(setting.AppSubURL + "/explore/repos") | ctx.Redirect(setting.AppSubURL + "/explore/repos") | ||||
| @@ -957,15 +961,15 @@ func RegisterRoutes(m *macaron.Macaron) { | |||||
| m.Get("", reqRepoCloudBrainReader, repo.CloudBrainIndex) | m.Get("", reqRepoCloudBrainReader, repo.CloudBrainIndex) | ||||
| m.Group("/:jobid", func() { | m.Group("/:jobid", func() { | ||||
| m.Get("", reqRepoCloudBrainReader, repo.CloudBrainShow) | m.Get("", reqRepoCloudBrainReader, repo.CloudBrainShow) | ||||
| m.Get("/debug", reqRepoCloudBrainReader, repo.CloudBrainDebug) | |||||
| m.Post("/commit_image", reqRepoCloudBrainWriter, bindIgnErr(auth.CommitImageCloudBrainForm{}), repo.CloudBrainCommitImage) | |||||
| m.Post("/stop", reqRepoCloudBrainWriter, repo.CloudBrainStop) | |||||
| m.Post("/del", reqRepoCloudBrainWriter, repo.CloudBrainDel) | |||||
| m.Get("/debug", reqRepoCloudBrainWriter, repo.CloudBrainDebug) | |||||
| m.Post("/commit_image", cloudbrain.AdminOrOwnerOrJobCreaterRight, bindIgnErr(auth.CommitImageCloudBrainForm{}), repo.CloudBrainCommitImage) | |||||
| m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainStop) | |||||
| m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainDel) | |||||
| m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate) | m.Get("/rate", reqRepoCloudBrainReader, repo.GetRate) | ||||
| m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) | m.Get("/models", reqRepoCloudBrainReader, repo.CloudBrainShowModels) | ||||
| m.Get("/download_model", reqRepoCloudBrainReader, repo.CloudBrainDownloadModel) | |||||
| m.Get("/download_model", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.CloudBrainDownloadModel) | |||||
| }) | }) | ||||
| m.Get("/create", reqRepoCloudBrainReader, repo.CloudBrainNew) | |||||
| m.Get("/create", reqRepoCloudBrainWriter, repo.CloudBrainNew) | |||||
| m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) | m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateCloudBrainForm{}), repo.CloudBrainCreate) | ||||
| }, context.RepoRef()) | }, context.RepoRef()) | ||||
| m.Group("/modelmanage", func() { | m.Group("/modelmanage", func() { | ||||
| @@ -991,9 +995,9 @@ func RegisterRoutes(m *macaron.Macaron) { | |||||
| m.Get("", reqRepoCloudBrainReader, repo.NotebookIndex) | m.Get("", reqRepoCloudBrainReader, repo.NotebookIndex) | ||||
| m.Group("/:jobid", func() { | m.Group("/:jobid", func() { | ||||
| m.Get("", reqRepoCloudBrainReader, repo.NotebookShow) | m.Get("", reqRepoCloudBrainReader, repo.NotebookShow) | ||||
| m.Get("/debug", reqRepoCloudBrainReader, repo.NotebookDebug) | |||||
| m.Post("/stop", reqRepoCloudBrainWriter, repo.NotebookStop) | |||||
| m.Post("/del", reqRepoCloudBrainWriter, repo.NotebookDel) | |||||
| m.Get("/debug", reqRepoCloudBrainWriter, repo.NotebookDebug) | |||||
| m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.NotebookStop) | |||||
| m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.NotebookDel) | |||||
| }) | }) | ||||
| m.Get("/create", reqRepoCloudBrainWriter, repo.NotebookNew) | m.Get("/create", reqRepoCloudBrainWriter, repo.NotebookNew) | ||||
| m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsNotebookForm{}), repo.NotebookCreate) | m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsNotebookForm{}), repo.NotebookCreate) | ||||
| @@ -1003,13 +1007,13 @@ func RegisterRoutes(m *macaron.Macaron) { | |||||
| m.Get("", reqRepoCloudBrainReader, repo.TrainJobIndex) | m.Get("", reqRepoCloudBrainReader, repo.TrainJobIndex) | ||||
| m.Group("/:jobid", func() { | m.Group("/:jobid", func() { | ||||
| m.Get("", reqRepoCloudBrainReader, repo.TrainJobShow) | m.Get("", reqRepoCloudBrainReader, repo.TrainJobShow) | ||||
| m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop) | |||||
| m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel) | |||||
| m.Get("/model_download", reqRepoCloudBrainReader, repo.ModelDownload) | |||||
| m.Get("/create_version", reqRepoCloudBrainReader, repo.TrainJobNewVersion) | |||||
| m.Post("/create_version", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) | |||||
| m.Post("/stop", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.TrainJobStop) | |||||
| m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.TrainJobDel) | |||||
| m.Get("/model_download", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.ModelDownload) | |||||
| m.Get("/create_version", cloudbrain.AdminOrJobCreaterRight, repo.TrainJobNewVersion) | |||||
| m.Post("/create_version", cloudbrain.AdminOrJobCreaterRight, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreateVersion) | |||||
| }) | }) | ||||
| m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNew) | |||||
| m.Get("/create", reqRepoCloudBrainWriter, repo.TrainJobNew) | |||||
| m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate) | m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate) | ||||
| m.Get("/para-config-list", reqRepoCloudBrainReader, repo.TrainJobGetConfigList) | m.Get("/para-config-list", reqRepoCloudBrainReader, repo.TrainJobGetConfigList) | ||||