From d982b29bf39905ccdd4124dc05f6b2694072699d Mon Sep 17 00:00:00 2001 From: Gitea Date: Wed, 20 Oct 2021 09:55:55 +0800 Subject: [PATCH] fix conflict --- models/file_chunk.go | 2 + models/repo.go | 19 +-- modules/auth/modelarts.go | 32 ++-- modules/modelarts/modelarts.go | 10 +- modules/modelarts/resty.go | 50 +++++- modules/setting/setting.go | 16 +- routers/repo/modelarts.go | 227 +++++++++++++++++++++++++++- routers/routes/routes.go | 13 +- templates/repo/modelarts/index.tmpl | 5 + 9 files changed, 326 insertions(+), 48 deletions(-) diff --git a/models/file_chunk.go b/models/file_chunk.go index 9888031b8..8decb7b44 100755 --- a/models/file_chunk.go +++ b/models/file_chunk.go @@ -17,6 +17,8 @@ const ( TypeCloudBrainOne = 0 TypeCloudBrainNotebook = 1 TypeCloudBrainTrainJob = 2 + + TypeCloudBrainTwo = 1 ) type FileChunk struct { diff --git a/models/repo.go b/models/repo.go index 2742c3e31..29da3788d 100755 --- a/models/repo.go +++ b/models/repo.go @@ -6,13 +6,14 @@ package models import ( - "code.gitea.io/gitea/modules/blockchain" "context" "crypto/md5" "errors" "fmt" "html/template" + "code.gitea.io/gitea/modules/blockchain" + // Needed for jpeg support _ "image/jpeg" "image/png" @@ -171,11 +172,11 @@ type Repository struct { NumOpenIssues int `xorm:"-"` NumPulls int NumClosedPulls int - NumOpenPulls int `xorm:"-"` - NumMilestones int `xorm:"NOT NULL DEFAULT 0"` - NumClosedMilestones int `xorm:"NOT NULL DEFAULT 0"` - NumOpenMilestones int `xorm:"-"` - NumCommit int64 `xorm:"NOT NULL DEFAULT 0"` + NumOpenPulls int `xorm:"-"` + NumMilestones int `xorm:"NOT NULL DEFAULT 0"` + NumClosedMilestones int `xorm:"NOT NULL DEFAULT 0"` + NumOpenMilestones int `xorm:"-"` + NumCommit int64 `xorm:"NOT NULL DEFAULT 0"` IsPrivate bool `xorm:"INDEX"` IsEmpty bool `xorm:"INDEX"` @@ -215,8 +216,8 @@ type Repository struct { CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` - Hot int64 `xorm:"-"` - Active int64 `xorm:"-"` + Hot int64 `xorm:"-"` + Active int64 `xorm:"-"` } // SanitizedOriginalURL returns a sanitized OriginalURL @@ -2458,7 +2459,7 @@ func (repo *Repository) IncreaseCloneCnt() { } func UpdateRepositoryCommitNum(repo *Repository) error { - if _,err := x.Exec("UPDATE `repository` SET num_commit = ? where id = ?", repo.NumCommit, repo.ID); err != nil { + if _, err := x.Exec("UPDATE `repository` SET num_commit = ? where id = ?", repo.NumCommit, repo.ID); err != nil { return err } diff --git a/modules/auth/modelarts.go b/modules/auth/modelarts.go index 8dc9f3493..f2e5aeed5 100755 --- a/modules/auth/modelarts.go +++ b/modules/auth/modelarts.go @@ -5,6 +5,16 @@ import ( "gitea.com/macaron/macaron" ) +type CreateModelArtsForm struct { + JobName string `form:"job_name" binding:"Required"` + Attachment string `form:"attachment"` + Description string `form:"description"` +} + +func (f *CreateModelArtsForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { + return validate(errs, ctx.Data, f, ctx.Locale) +} + type CreateModelArtsNotebookForm struct { JobName string `form:"job_name" binding:"Required"` Attachment string `form:"attachment"` @@ -16,18 +26,18 @@ func (f *CreateModelArtsNotebookForm) Validate(ctx *macaron.Context, errs bindin } type CreateModelArtsTrainJobForm struct { - JobName string `form:"job_name" binding:"Required"` - Attachment string `form:"attachment" binding:"Required"` - BootFile string `form:"boot_file" binding:"Required"` - WorkServerNumber int `form:"work_server_number" binding:"Required"` - EngineID int `form:"engine_id" binding:"Required"` - PoolID string `form:"pool_id" binding:"Required"` - Flavor string `form:"flavor" binding:"Required"` - Params string `form:"run_para_list" binding:"Required"` - Description string `form:"description"` - IsSaveParam string `form:"is_save_para"` + JobName string `form:"job_name" binding:"Required"` + Attachment string `form:"attachment" binding:"Required"` + BootFile string `form:"boot_file" binding:"Required"` + WorkServerNumber int `form:"work_server_number" binding:"Required"` + EngineID int `form:"engine_id" binding:"Required"` + PoolID string `form:"pool_id" binding:"Required"` + Flavor string `form:"flavor" binding:"Required"` + Params string `form:"run_para_list" binding:"Required"` + Description string `form:"description"` + IsSaveParam string `form:"is_save_para"` ParameterTemplateName string `form:"parameter_template_name"` - PrameterDescription string `form:"parameter_description"` + PrameterDescription string `form:"parameter_description"` } func (f *CreateModelArtsTrainJobForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 046c2b916..ed56f73ea 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -30,11 +30,11 @@ const ( "{\"id\":120,\"value\":\"MindSpore-1.1.1-c76-tr5-python3.7-euleros2.8-aarch64\"}," + "{\"id\":117,\"value\":\"TF-1.15-c75-python3.7-euleros2.8-aarch64\"}" + "]}" - FlavorInfos = "{\"flavor\":[{\"code\":\"modelarts.bm.910.arm.public.2\",\"value\":\"Ascend : 2 * Ascend 910 CPU：48 核 512GiB\"}," + - "{\"code\":\"modelarts.bm.910.arm.public.8\",\"value\":\"Ascend : 8 * Ascend 910 CPU：192 核 2048GiB\"}," + - "{\"code\":\"modelarts.bm.910.arm.public.4\",\"value\":\"Ascend : 4 * Ascend 910 CPU：96 核 1024GiB\"}," + - "{\"code\":\"modelarts.bm.910.arm.public.1\",\"value\":\"Ascend : 1 * Ascend 910 CPU：24 核 256GiB\"}" + - "]}" + // FlavorInfos = "{\"flavor\":[{\"code\":\"modelarts.bm.910.arm.public.2\",\"value\":\"Ascend : 2 * Ascend 910 CPU：48 核 512GiB\"}," + + // "{\"code\":\"modelarts.bm.910.arm.public.8\",\"value\":\"Ascend : 8 * Ascend 910 CPU：192 核 2048GiB\"}," + + // "{\"code\":\"modelarts.bm.910.arm.public.4\",\"value\":\"Ascend : 4 * Ascend 910 CPU：96 核 1024GiB\"}," + + // "{\"code\":\"modelarts.bm.910.arm.public.1\",\"value\":\"Ascend : 1 * Ascend 910 CPU：24 核 256GiB\"}" + + // "]}" CodePath = "/code/" OutputPath = "/output/" LogPath = "/log/" diff --git a/modules/modelarts/resty.go b/modules/modelarts/resty.go index a6cf816a4..d17478c94 100755 --- a/modules/modelarts/resty.go +++ b/modules/modelarts/resty.go @@ -1,15 +1,16 @@ package modelarts import ( - "code.gitea.io/gitea/models" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" "crypto/tls" "encoding/json" "fmt" - "github.com/go-resty/resty/v2" "net/http" "strconv" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "github.com/go-resty/resty/v2" ) var ( @@ -91,7 +92,7 @@ func getToken() error { return nil } -func createNotebook(createJobParams models.CreateNotebookParams) (*models.CreateNotebookResult, error) { +func CreateJob(createJobParams models.CreateNotebookParams) (*models.CreateNotebookResult, error) { checkSetting() client := getRestyClient() var result models.CreateNotebookResult @@ -252,6 +253,45 @@ sendjob: return &result, nil } +func DelJob(jobID string) (*models.NotebookDelResult, error) { + checkSetting() + client := getRestyClient() + var result models.NotebookDelResult + + retry := 0 + +sendjob: + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetAuthToken(TOKEN). + SetResult(&result). + Delete(HOST + "/v1/" + setting.ProjectID + urlNotebook + "/" + jobID) + + if err != nil { + return &result, fmt.Errorf("resty DelJob: %v", err) + } + + if res.StatusCode() == http.StatusUnauthorized && retry < 1 { + retry++ + _ = getToken() + goto sendjob + } + + var response models.NotebookResult + err = json.Unmarshal(res.Body(), &response) + if err != nil { + log.Error("json.Unmarshal failed: %s", err.Error()) + return &result, fmt.Errorf("son.Unmarshal failed: %s", err.Error()) + } + + if len(response.ErrorCode) != 0 { + log.Error("DelJob failed(%s): %s", response.ErrorCode, response.ErrorMsg) + return &result, fmt.Errorf("DelJob failed(%s): %s", response.ErrorCode, response.ErrorMsg) + } + + return &result, nil +} + func GetJobToken(jobID string) (*models.NotebookGetJobTokenResult, error) { checkSetting() client := getRestyClient() diff --git a/modules/setting/setting.go b/modules/setting/setting.go index b5f0cf72a..1ace323ad 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -468,11 +468,8 @@ var ( Bucket string Location string BasePath string -<<<<<<< HEAD - CodePathPrefix string -======= + CodePathPrefix string UserBasePath string ->>>>>>> origin/develop //modelarts config ModelArtsHost string @@ -482,14 +479,11 @@ var ( ModelArtsUsername string ModelArtsPassword string ModelArtsDomain string -<<<<<<< HEAD - AllowedOrg string -======= + AllowedOrg string ProfileID string PoolInfos string Flavor string FlavorInfos string ->>>>>>> origin/develop ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -1195,11 +1189,8 @@ func NewContext() { Bucket = sec.Key("BUCKET").MustString("testopendata") Location = sec.Key("LOCATION").MustString("cn-south-222") BasePath = sec.Key("BASE_PATH").MustString("attachment/") -<<<<<<< HEAD CodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("code/") -======= UserBasePath = sec.Key("BASE_PATH_USER").MustString("users/") ->>>>>>> origin/develop sec = Cfg.Section("modelarts") ModelArtsHost = sec.Key("ENDPOINT").MustString("112.95.163.80") @@ -1209,14 +1200,11 @@ func NewContext() { ModelArtsUsername = sec.Key("USERNAME").MustString("") ModelArtsPassword = sec.Key("PASSWORD").MustString("") ModelArtsDomain = sec.Key("DOMAIN").MustString("cn-south-222") -<<<<<<< HEAD AllowedOrg = sec.Key("ORGANIZATION").MustString("") -======= ProfileID = sec.Key("PROFILE_ID").MustString("") PoolInfos = sec.Key("POOL_INFOS").MustString("") Flavor = sec.Key("FLAVOR").MustString("") FlavorInfos = sec.Key("FLAVOR_INFOS").MustString("") ->>>>>>> origin/develop } func loadInternalToken(sec *ini.Section) string { diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index c017f31ff..796b51a41 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -12,8 +12,6 @@ import ( "strings" "time" - "code.gitea.io/gitea/modules/modelarts" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/obs" @@ -29,10 +27,15 @@ import ( ) const ( + // tplModelArtsNotebookIndex base.TplName = "repo/modelarts/notebook/index" tplModelArtsNotebookIndex base.TplName = "repo/modelarts/notebook/index" tplModelArtsNotebookNew base.TplName = "repo/modelarts/notebook/new" tplModelArtsNotebookShow base.TplName = "repo/modelarts/notebook/show" + tplModelArtsIndex base.TplName = "repo/modelarts/index" + tplModelArtsNew base.TplName = "repo/modelarts/new" + tplModelArtsShow base.TplName = "repo/modelarts/show" + tplModelArtsTrainJobIndex base.TplName = "repo/modelarts/trainjob/index" tplModelArtsTrainJobNew base.TplName = "repo/modelarts/trainjob/new" tplModelArtsTrainJobShow base.TplName = "repo/modelarts/trainjob/show" @@ -46,6 +49,224 @@ func MustEnableModelArts(ctx *context.Context) { } } +func ModelArtsIndex(ctx *context.Context) { + MustEnableModelArts(ctx) + repo := ctx.Repo.Repository + page := ctx.QueryInt("page") + if page <= 0 { + page = 1 + } + + ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ + ListOptions: models.ListOptions{ + Page: page, + PageSize: setting.UI.IssuePagingNum, + }, + RepoID: repo.ID, + Type: models.TypeCloudBrainTwo, + }) + if err != nil { + ctx.ServerError("Cloudbrain", err) + return + } + + for i, task := range ciTasks { + if task.Status == string(models.JobRunning) { + ciTasks[i].CanDebug = true + } else { + ciTasks[i].CanDebug = false + } + } + + pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5) + pager.SetDefaultParams(ctx) + ctx.Data["Page"] = pager + + ctx.Data["PageIsCloudBrain"] = true + ctx.Data["Tasks"] = ciTasks + ctx.HTML(200, tplModelArtsIndex) +} + +func ModelArtsNew(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + + t := time.Now() + var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] + ctx.Data["job_name"] = jobName + + attachs, err := models.GetModelArtsUserAttachments(ctx.User.ID) + if err != nil { + ctx.ServerError("GetAllUserAttachments failed:", err) + return + } + + ctx.Data["attachments"] = attachs + ctx.Data["dataset_path"] = modelarts.DataSetMountPath + ctx.Data["env"] = modelarts.NotebookEnv + ctx.Data["notebook_type"] = modelarts.NotebookType + if modelarts.FlavorInfos == nil { + json.Unmarshal([]byte(setting.FlavorInfos), &modelarts.FlavorInfos) + } + ctx.Data["flavors"] = modelarts.FlavorInfos.FlavorInfo + ctx.HTML(200, tplModelArtsNew) +} + +func ModelArtsCreate(ctx *context.Context, form auth.CreateModelArtsForm) { + ctx.Data["PageIsCloudBrain"] = true + jobName := form.JobName + uuid := form.Attachment + description := form.Description + //repo := ctx.Repo.Repository + + err := modelarts.GenerateTask(ctx, jobName, uuid, description) + if err != nil { + ctx.RenderWithErr(err.Error(), tplModelArtsNew, &form) + return + } + + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts") +} + +func ModelArtsShow(ctx *context.Context) { + ctx.Data["PageIsCloudBrain"] = true + + var jobID = ctx.Params(":jobid") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.Data["error"] = err.Error() + ctx.RenderWithErr(err.Error(), tplModelArtsIndex, nil) + return + } + + result, err := modelarts.GetJob(jobID) + if err != nil { + ctx.Data["error"] = err.Error() + ctx.RenderWithErr(err.Error(), tplModelArtsIndex, nil) + return + } + + if result != nil { + task.Status = result.Status + err = models.UpdateJob(task) + if err != nil { + ctx.Data["error"] = err.Error() + ctx.RenderWithErr(err.Error(), tplModelArtsIndex, nil) + return + } + + createTime, _ := com.StrTo(result.CreationTimestamp).Int64() + result.CreateTime = time.Unix(int64(createTime/1000), 0).Format("2006-01-02 15:04:05") + endTime, _ := com.StrTo(result.LatestUpdateTimestamp).Int64() + result.LatestUpdateTime = time.Unix(int64(endTime/1000), 0).Format("2006-01-02 15:04:05") + result.QueuingInfo.BeginTime = time.Unix(int64(result.QueuingInfo.BeginTimestamp/1000), 0).Format("2006-01-02 15:04:05") + result.QueuingInfo.EndTime = time.Unix(int64(result.QueuingInfo.EndTimestamp/1000), 0).Format("2006-01-02 15:04:05") + } + + ctx.Data["task"] = task + ctx.Data["jobID"] = jobID + ctx.Data["result"] = result + ctx.HTML(200, tplModelArtsShow) +} + +func ModelArtsDebug(ctx *context.Context) { + var jobID = ctx.Params(":jobid") + _, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.ServerError("GetCloudbrainByJobID failed", err) + return + } + + result, err := modelarts.GetJob(jobID) + if err != nil { + ctx.RenderWithErr(err.Error(), tplModelArtsIndex, nil) + return + } + + res, err := modelarts.GetJobToken(jobID) + if err != nil { + ctx.RenderWithErr(err.Error(), tplModelArtsIndex, nil) + return + } + + urls := strings.Split(result.Spec.Annotations.Url, "/") + urlPrefix := result.Spec.Annotations.TargetDomain + for i, url := range urls { + if i > 2 { + urlPrefix += "/" + url + } + } + + //urlPrefix := result.Spec.Annotations.TargetDomain + "/modelarts/internal/hub/notebook/user/" + task.JobID + log.Info(urlPrefix) + debugUrl := urlPrefix + "?token=" + res.Token + ctx.Redirect(debugUrl) +} + +func ModelArtsStop(ctx *context.Context) { + var jobID = ctx.Params(":jobid") + log.Info(jobID) + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.ServerError("GetCloudbrainByJobID failed", err) + return + } + + if task.Status != string(models.JobRunning) { + log.Error("the job(%s) is not running", task.JobName) + ctx.ServerError("the job is not running", errors.New("the job is not running")) + return + } + + param := models.NotebookAction{ + Action: models.ActionStop, + } + res, err := modelarts.StopJob(jobID, param) + if err != nil { + log.Error("StopJob(%s) failed:%v", task.JobName, err.Error()) + ctx.ServerError("StopJob failed", err) + return + } + + task.Status = res.CurrentStatus + err = models.UpdateJob(task) + if err != nil { + ctx.ServerError("UpdateJob failed", err) + return + } + + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts") +} + +func ModelArtsDel(ctx *context.Context) { + var jobID = ctx.Params(":jobid") + task, err := models.GetCloudbrainByJobID(jobID) + if err != nil { + ctx.ServerError("GetCloudbrainByJobID failed", err) + return + } + + if task.Status != string(models.JobStopped) { + log.Error("the job(%s) has not been stopped", task.JobName) + ctx.ServerError("the job has not been stopped", errors.New("the job has not been stopped")) + return + } + + _, err = modelarts.DelJob(jobID) + if err != nil { + log.Error("DelJob(%s) failed:%v", task.JobName, err.Error()) + ctx.ServerError("DelJob failed", err) + return + } + + err = models.DeleteJob(task) + if err != nil { + ctx.ServerError("DeleteJob failed", err) + return + } + + ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts") +} + func NotebookIndex(ctx *context.Context) { MustEnableModelArts(ctx) repo := ctx.Repo.Repository @@ -348,7 +569,7 @@ func TrainJobNew(ctx *context.Context) { ctx.Data["engine_versions"] = versionInfos.Version var flavorInfos modelarts.Flavor - if err = json.Unmarshal([]byte(modelarts.FlavorInfos), &flavorInfos); err != nil { + if err = json.Unmarshal([]byte(setting.FlavorInfos), &flavorInfos); err != nil { ctx.ServerError("json.Unmarshal failed:", err) return } diff --git a/routers/routes/routes.go b/routers/routes/routes.go index 7b8068b50..f7f99535e 100755 --- a/routers/routes/routes.go +++ b/routers/routes/routes.go @@ -6,13 +6,14 @@ package routes import ( "bytes" - "code.gitea.io/gitea/routers/operation" "encoding/gob" "net/http" "path" "text/template" "time" + "code.gitea.io/gitea/routers/operation" + "code.gitea.io/gitea/routers/secure" "code.gitea.io/gitea/models" @@ -958,6 +959,16 @@ func RegisterRoutes(m *macaron.Macaron) { }, context.RepoRef()) m.Group("/modelarts", func() { + m.Get("", reqRepoCloudBrainReader, repo.ModelArtsIndex) + m.Group("/:jobid", func() { + m.Get("", reqRepoCloudBrainReader, repo.ModelArtsShow) + m.Get("/debug", reqRepoCloudBrainReader, repo.ModelArtsDebug) + m.Post("/stop", reqRepoCloudBrainWriter, repo.ModelArtsStop) + m.Post("/del", reqRepoCloudBrainWriter, repo.ModelArtsDel) + }) + m.Get("/create", reqRepoCloudBrainWriter, repo.ModelArtsNew) + m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsForm{}), repo.ModelArtsCreate) + m.Group("/notebook", func() { m.Get("", reqRepoCloudBrainReader, repo.NotebookIndex) m.Group("/:jobid", func() { diff --git a/templates/repo/modelarts/index.tmpl b/templates/repo/modelarts/index.tmpl index dabbe0b59..85f2a0b2d 100755 --- a/templates/repo/modelarts/index.tmpl +++ b/templates/repo/modelarts/index.tmpl @@ -214,6 +214,11 @@ {{if .Permission.CanWrite $.UnitTypeCloudBrain}} {{.i18n.Tr "repo.cloudbrain.new"}} {{end}} + +
+ {{if .Permission.CanWrite $.UnitTypeCloudBrain}} + {{.i18n.Tr "repo.cloudbrain.new"}} {{end}} +

使用鹏城云脑计算资源进行调试，云脑1提供CPU / GPU资源，云脑2提供Ascend NPU资源；调试使用的数据集也需要上传到对应的环境。