From 739de0df5c8d5a97a2e3f07535a16bf5d33fd566 Mon Sep 17 00:00:00 2001 From: yuyuanshifu <747342561@qq.com> Date: Thu, 19 Nov 2020 18:02:36 +0800 Subject: [PATCH] add code --- models/attachment.go | 16 +++++++ models/cloudbrain.go | 9 ++-- models/dataset.go | 0 modules/auth/cloudbrain.go | 5 +- modules/cloudbrain/cloudbrain.go | 29 ++++++++---- modules/cloudbrain/resty.go | 13 +++-- modules/setting/setting.go | 4 ++ routers/repo/attachment.go | 2 +- routers/repo/cloudbrain.go | 76 +++++++++++++++++++++++++++--- templates/repo/cloudbrain/new.tmpl | 26 +++++----- 10 files changed, 137 insertions(+), 43 deletions(-) mode change 100644 => 100755 models/dataset.go mode change 100644 => 100755 modules/auth/cloudbrain.go diff --git a/models/attachment.go b/models/attachment.go index 3d4e4d136..20f94d1a1 100755 --- a/models/attachment.go +++ b/models/attachment.go @@ -371,3 +371,19 @@ func getPrivateAttachments(e Engine, userID int64) ([]*Attachment, error) { return attachments, e.Where("uploader_id = ? and decompress_state = ?", userID, DecompressStateDone).Find(&attachments) } +func GetAllUserAttachments(userID int64) ([]*Attachment, error) { + attachsPub, err := getAllPublicAttachments(x) + if err != nil { + log.Error("getAllPublicAttachments failed:%v", err) + return nil, err + } + + attachsPri, err := getPrivateAttachments(x, userID) + if err != nil { + log.Error("getPrivateAttachments failed:%v", err) + return nil, err + } + + return append(attachsPub, attachsPri...), nil +} + diff --git a/models/cloudbrain.go b/models/cloudbrain.go index dffb4dd0c..552ce74a7 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -77,9 +77,9 @@ type CreateJobParams struct { } type CreateJobResult struct { - Code string - Msg string - Payload map[string]interface{} + Code string `json:"code"` + Msg string `json:"msg"` + Payload map[string]interface{} `json:"payload"` } type GetJobResult struct { @@ -91,7 +91,7 @@ type GetJobResult struct { type GetImagesResult struct { Code string `json:"code"` Msg string `json:"msg"` - Payload map[string]ImageInfo `json:"payload"` + Payload map[string]*ImageInfo `json:"payload"` } type CloudbrainsOptions struct { @@ -212,6 +212,7 @@ type ImageInfo struct { Provider string `json:"provider"` Createtime string `json:"createtime"` Remark string `json:"remark"` + PlaceView string } diff --git a/models/dataset.go b/models/dataset.go old mode 100644 new mode 100755 diff --git a/modules/auth/cloudbrain.go b/modules/auth/cloudbrain.go old mode 100644 new mode 100755 index 938bf087e..ca3848a65 --- a/modules/auth/cloudbrain.go +++ b/modules/auth/cloudbrain.go @@ -8,8 +8,9 @@ import ( // CreateDatasetForm form for dataset page type CreateCloudBrainForm struct { JobName string `form:"job_name" binding:"Required"` - Image string `binding:"Required"` - Command string `binding:"Required"` + Image string `form:"image" binding:"Required"` + Command string `form:"command" binding:"Required"` + Attachment string `form:"attachment" binding:"Required"` } func (f *CreateCloudBrainForm) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors { diff --git a/modules/cloudbrain/cloudbrain.go b/modules/cloudbrain/cloudbrain.go index 6639797c0..494e2799f 100755 --- a/modules/cloudbrain/cloudbrain.go +++ b/modules/cloudbrain/cloudbrain.go @@ -1,6 +1,7 @@ package cloudbrain import ( + "code.gitea.io/gitea/modules/setting" "errors" "code.gitea.io/gitea/models" @@ -9,18 +10,25 @@ import ( ) const ( - Command = `pip3 install jupyterlab==1.1.4;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir=\"/userhome\" --port=80 --NotebookApp.token=\"\" --LabApp.allow_origin=\"self https://cloudbrain.pcl.ac.cn\"` + Command = `pip3 install jupyterlab==2.2.5 -i https://pypi.tuna.tsinghua.edu.cn/simple;service ssh stop;jupyter lab --no-browser --ip=0.0.0.0 --allow-root --notebook-dir="/code" --port=80 --LabApp.token="" --LabApp.allow_origin="self https://cloudbrain.pcl.ac.cn"` CodeMountPath = "/code" DataSetMountPath = "/dataset" ModelMountPath = "/model" SubTaskName = "task1" - DebugGPUType = "DEBUG" + DebugGPUType = "debug" + + Success = "S000" ) -func GenerateTask(ctx *context.Context, jobName, image, command string) error { +func GenerateTask(ctx *context.Context, jobName, image, command, uuid, codePath, modelPath string) error { + dataActualPath := setting.Attachment.Minio.RealPath + + setting.Attachment.Minio.Bucket + "/" + + setting.Attachment.Minio.BasePath + + models.AttachmentRelativePath(uuid) + + uuid jobResult, err := CreateJob(jobName, models.CreateJobParams{ JobName: jobName, RetryCount: 1, @@ -45,31 +53,32 @@ func GenerateTask(ctx *context.Context, jobName, image, command string) error { Volumes: []models.Volume{ { HostPath: models.StHostPath{ - Path: "", + Path: codePath, MountPath: CodeMountPath, - ReadOnly: true, + ReadOnly: false, }, }, { HostPath: models.StHostPath{ - Path: "", + Path: dataActualPath, MountPath: DataSetMountPath, - ReadOnly: false, + ReadOnly: true, }, }, { HostPath: models.StHostPath{ - Path: "", + Path: modelPath, MountPath: ModelMountPath, - ReadOnly: true, + ReadOnly: false, }, }, }, }) if err != nil { + log.Error("CreateJob failed:", err.Error()) return err } - if jobResult.Code != "S000" { + if jobResult.Code != Success { log.Error("CreateJob(%s) failed:%s", jobName, jobResult.Msg) return errors.New(jobResult.Msg) } diff --git a/modules/cloudbrain/resty.go b/modules/cloudbrain/resty.go index b49caedb6..09229b254 100755 --- a/modules/cloudbrain/resty.go +++ b/modules/cloudbrain/resty.go @@ -1,6 +1,7 @@ package cloudbrain import ( + "code.gitea.io/gitea/modules/log" "fmt" "code.gitea.io/gitea/models" @@ -47,7 +48,7 @@ func loginCloudbrain() error { return fmt.Errorf("resty loginCloudbrain: %s", err) } - if loginResult.Code != "S000" { + if loginResult.Code != Success { return fmt.Errorf("%s: %s", loginResult.Msg, res.String()) } @@ -62,13 +63,15 @@ func CreateJob(jobName string, createJobParams models.CreateJobParams) (*models. retry := 0 + log.Info("", createJobParams.Volumes) + sendjob: res, err := client.R(). SetHeader("Content-Type", "application/json"). SetAuthToken(TOKEN). SetBody(createJobParams). SetResult(&jobResult). - Put(HOST + "/rest-server/api/v1/jobs/" + jobName) + Post(HOST + "/rest-server/api/v1/jobs/") if err != nil { return nil, fmt.Errorf("resty create job: %s", err) @@ -80,7 +83,7 @@ sendjob: goto sendjob } - if jobResult.Code != "S000" { + if jobResult.Code != Success { return &jobResult, fmt.Errorf("jobResult err: %s", res.String()) } @@ -112,7 +115,7 @@ sendjob: goto sendjob } - if getJobResult.Code != "S000" { + if getJobResult.Code != Success { return &getJobResult, fmt.Errorf("jobResult GetJob err: %s", res.String()) } @@ -143,7 +146,7 @@ sendjob: goto sendjob } - if getImagesResult.Code != "S000" { + if getImagesResult.Code != Success { return &getImagesResult, fmt.Errorf("getImgesResult err: %s", res.String()) } diff --git a/modules/setting/setting.go b/modules/setting/setting.go index af902cb79..45056e262 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -438,6 +438,8 @@ var ( ClientSecret string UserCeterHost string RestServerHost string + JobPath string + DebugServerHost string ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -1113,6 +1115,8 @@ func NewContext() { ClientSecret = sec.Key("CLIENT_SECRET").MustString("J5ykfVl2kcxW0H9cawSL") UserCeterHost = sec.Key("USER_CENTER_HOST").MustString("http://192.168.202.73:31441") RestServerHost = sec.Key("REST_SERVER_HOST").MustString("http://192.168.202.73") + JobPath = sec.Key("JOB_PATH").MustString("/datasets/minio/data/opendata/jobs/") + DebugServerHost = sec.Key("DEBUG_SERVER_HOST").MustString("http://192.168.202.73") } func loadInternalToken(sec *ini.Section) string { diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index 3881ff733..6bb3933fa 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -661,7 +661,7 @@ func queryDatasets(ctx *context.Context, username string, attachs []*models.Atta continue } - datasets = append(datasets, CloudBrainDataset{attch.UUID, + datasets = append(datasets, CloudBrainDataset{string(attch.ID), attch.Name, setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 5b54f3791..898b49c5f 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -1,7 +1,10 @@ package repo import ( + "os" + "os/exec" "strconv" + "strings" "time" "code.gitea.io/gitea/models" @@ -9,6 +12,7 @@ import ( "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/cloudbrain" "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" ) @@ -66,7 +70,7 @@ func CloudBrainNew(ctx *context.Context) { ctx.Data["PageIsCloudBrain"] = true t := time.Now() - var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[:5] + var jobName = cutString(ctx.User.Name, 5) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] ctx.Data["job_name"] = jobName result, err := cloudbrain.GetImages() @@ -74,13 +78,24 @@ func CloudBrainNew(ctx *context.Context) { ctx.Data["error"] = err.Error() } - if result != nil { - //models.ConvertToImagesResultPayload(result.Payload) - ctx.Data["image"] = result.Payload - } else { - ctx.Data["image"] = "192.168.202.74:5000/user-images/deepo:v2.0" + for i,payload := range result.Payload { + if strings.HasPrefix(result.Payload[i].Place,"192.168") { + result.Payload[i].PlaceView = payload.Place[strings.Index(payload.Place, "/"): len(payload.Place)-1] + } else { + result.Payload[i].PlaceView = payload.Place + } + + } + + ctx.Data["images"] = result.Payload + + attachs, err := models.GetAllUserAttachments(ctx.User.ID) + if err != nil { + ctx.ServerError("GetAllUserAttachments failed:", err) + return } + ctx.Data["attachments"] = attachs ctx.Data["command"] = cloudbrain.Command ctx.Data["code_path"] = cloudbrain.CodeMountPath ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath @@ -93,7 +108,23 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { jobName := form.JobName image := form.Image command := form.Command - err := cloudbrain.GenerateTask(ctx, jobName, image, command) + uuid := form.Attachment + codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath + repo := ctx.Repo.Repository + err := downloadCode(repo, codePath) + if err != nil { + ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) + return + } + + modelPath := setting.JobPath + jobName + "/model" + err = os.MkdirAll(modelPath, os.ModePerm) + if err != nil { + ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) + return + } + + err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath) if err != nil { ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) return @@ -127,3 +158,34 @@ func CloudBrainShow(ctx *context.Context) { ctx.Data["jobID"] = jobID ctx.HTML(200, tplCloudBrainShow) } + +func downloadCode(repo *models.Repository, codePath string) error { + /* + if err := git.Clone(repo.RepoPath(), codePath, git.CloneRepoOptions{ + Bare: true, + Shared: true, + }); err != nil { + log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err) + return "", fmt.Errorf("Failed to clone repository: %s (%v)", repo.FullName(), err) + } + + */ + + err := os.MkdirAll(codePath, os.ModePerm) + if err != nil { + log.Error("MkdirAll failed:%v", err) + return err + } + + command := "git clone " + repo.CloneLink().HTTPS + " " + codePath + cmd := exec.Command("/bin/bash", "-c", command) + output, err := cmd.Output() + if err != nil { + log.Error("exec.Command failed:%v", err) + return err + } + + log.Info(string(output)) + + return nil +} diff --git a/templates/repo/cloudbrain/new.tmpl b/templates/repo/cloudbrain/new.tmpl index cfaead632..7d9e1bf12 100755 --- a/templates/repo/cloudbrain/new.tmpl +++ b/templates/repo/cloudbrain/new.tmpl @@ -16,39 +16,37 @@
-
- -
- <--> -
- - + {{range .images}} + {{end}}
- +
- +
- +
- +
- +