From bba7081a0d1dba41d1168c326f9782e4b34e50dc Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 16 Aug 2021 14:37:49 +0800 Subject: [PATCH 01/10] adjust modelarts resource --- modules/modelarts/modelarts.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 367fce061..2b2f014c7 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -15,14 +15,16 @@ const ( flavor = "modelarts.kat1.xlarge" //profileID = "Python3-ascend910-arm" profileID = "efa847c0-7359-11eb-b34f-0255ac100057" - poolID = "pool1328035d" - poolName = "train-private-1" + //poolID = "pool1328035d" + //poolName = "train-private-1" + poolID = "pool35eafa8f" + poolName = "train-private-0703" poolType = "USER_DEFINED" DataSetMountPath = "/home/ma-user/work" NotebookEnv = "Python3" NotebookType = "Ascend" - FlavorInfo = "Ascend: 1*Ascend 910 CPU: 24 核 96GiB (modelarts.kat1.xlarge)" + FlavorInfo = "Ascend: 1*Ascend 910 CPU: 24 核 256GiB (modelarts.kat1.xlarge)" ) func GenerateTask(ctx *context.Context, jobName, uuid, description string) error { From 05c5e568ce3f9d10f4417a0ae4e29edd96c9aa4d Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Mon, 16 Aug 2021 18:07:19 +0800 Subject: [PATCH 02/10] add modelarts config --- models/cloudbrain.go | 29 ++++++++++++++++++++++++----- modules/modelarts/modelarts.go | 19 ++++++++++++++----- modules/setting/setting.go | 8 ++++++++ routers/repo/modelarts.go | 6 +++++- templates/repo/modelarts/new.tmpl | 7 ++++++- 5 files changed, 57 insertions(+), 12 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 7a2966f74..1cef5e1be 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -72,11 +72,11 @@ type CloudBrainLoginResult struct { type TaskRole struct { Name string `json:"name"` - TaskNumber int `json:"taskNumber"` - MinSucceededTaskCount int `json:"minSucceededTaskCount"` - MinFailedTaskCount int `json:"minFailedTaskCount"` - CPUNumber int `json:"cpuNumber"` - GPUNumber int `json:"gpuNumber"` + TaskNumber int `json:"taskNumber"` + MinSucceededTaskCount int `json:"minSucceededTaskCount"` + MinFailedTaskCount int `json:"minFailedTaskCount"` + CPUNumber int `json:"cpuNumber"` + GPUNumber int `json:"gpuNumber"` MemoryMB int `json:"memoryMB"` ShmMB int `json:"shmMB"` Command string `json:"command"` @@ -298,6 +298,25 @@ type ResourceSpec struct { ShareMemMiB int `json:"shareMemMiB"` } +type FlavorInfos struct { + FlavorInfo []*FlavorInfo `json:"flavor_info"` +} + +type FlavorInfo struct { + Id int `json:"id"` + Value string `json:"value"` +} + +type PoolInfos struct { + PoolInfo []*PoolInfo `json:"pool_info"` +} + +type PoolInfo struct { + PoolId string `json:"pool_id"` + PoolName string `json:"pool_name"` + PoolType string `json:"pool_type"` +} + type CommitImageParams struct { Ip string `json:"ip"` TaskContainerId string `json:"taskContainerId"` diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 2b2f014c7..fc2183194 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -2,6 +2,7 @@ package modelarts import ( "code.gitea.io/gitea/modules/setting" + "encoding/json" "path" "code.gitea.io/gitea/models" @@ -27,17 +28,25 @@ const ( FlavorInfo = "Ascend: 1*Ascend 910 CPU: 24 核 256GiB (modelarts.kat1.xlarge)" ) +var ( + poolInfos *models.PoolInfos + FlavorInfos *models.FlavorInfos +) + func GenerateTask(ctx *context.Context, jobName, uuid, description string) error { dataActualPath := setting.Bucket + "/" + setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + "/" + if poolInfos == nil { + json.Unmarshal([]byte(setting.PoolInfos), &poolInfos) + } jobResult, err := CreateJob(models.CreateNotebookParams{ JobName: jobName, Description: description, - ProfileID: profileID, - Flavor: flavor, + ProfileID: setting.ProfileID, + Flavor: setting.Flavor, Pool: models.Pool{ - ID: poolID, - Name: poolName, - Type: poolType, + ID: poolInfos.PoolInfo[0].PoolId, + Name: poolInfos.PoolInfo[0].PoolName, + Type: poolInfos.PoolInfo[0].PoolType, }, Spec: models.Spec{ Storage: models.Storage{ diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 986a18313..142369f88 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -473,6 +473,10 @@ var ( ModelArtsUsername string ModelArtsPassword string ModelArtsDomain string + ProfileID string + PoolInfos string + Flavor string + FlavorInfos string ) // DateLang transforms standard language locale name to corresponding value in datetime plugin. @@ -1182,6 +1186,10 @@ func NewContext() { ModelArtsUsername = sec.Key("USERNAME").MustString("") ModelArtsPassword = sec.Key("PASSWORD").MustString("") ModelArtsDomain = sec.Key("DOMAIN").MustString("cn-south-222") + ProfileID = sec.Key("PROFILE_ID").MustString("") + PoolInfos = sec.Key("POOL_INFOS").MustString("") + Flavor = sec.Key("FLAVOR").MustString("") + FlavorInfos = sec.Key("FLAVOR_INFOS").MustString("") } func loadInternalToken(sec *ini.Section) string { diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index f147a1f41..2dc327d33 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -2,6 +2,7 @@ package repo import ( "code.gitea.io/gitea/modules/modelarts" + "encoding/json" "errors" "github.com/unknwon/com" "strconv" @@ -84,7 +85,10 @@ func ModelArtsNew(ctx *context.Context) { ctx.Data["dataset_path"] = modelarts.DataSetMountPath ctx.Data["env"] = modelarts.NotebookEnv ctx.Data["notebook_type"] = modelarts.NotebookType - ctx.Data["flavor"] = modelarts.FlavorInfo + if modelarts.FlavorInfos == nil { + json.Unmarshal([]byte(setting.FlavorInfos), &modelarts.FlavorInfos) + } + ctx.Data["flavors"] = modelarts.FlavorInfos.FlavorInfo ctx.HTML(200, tplModelArtsNew) } diff --git a/templates/repo/modelarts/new.tmpl b/templates/repo/modelarts/new.tmpl index bb52e3585..c77a1fe47 100755 --- a/templates/repo/modelarts/new.tmpl +++ b/templates/repo/modelarts/new.tmpl @@ -132,7 +132,12 @@
- +
From 9345c74f6738ef08983cfbc1e99b683d45cc228f Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 17 Aug 2021 10:05:17 +0800 Subject: [PATCH 03/10] unified active-mail format --- options/locale/locale_zh-CN.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 5ac54f6c8..9dd2d254b 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -2435,7 +2435,7 @@ future=将来 1y=1年 seconds=%d 秒 minutes=%d 分钟 -hours=%d 小时 +hours=%d hours days=%d 天 weeks=%d 周 months=%d 个月 From 0eae06188d76fd752340c8e65a8e1407b83e4381 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 17 Aug 2021 15:42:52 +0800 Subject: [PATCH 04/10] fix-50 --- models/file_chunk.go | 12 +++++++++++- routers/repo/attachment.go | 17 +++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/models/file_chunk.go b/models/file_chunk.go index 9f513bd34..fc2beef56 100755 --- a/models/file_chunk.go +++ b/models/file_chunk.go @@ -87,7 +87,7 @@ func InsertFileChunk(fileChunk *FileChunk) (_ *FileChunk, err error) { return fileChunk, nil } -// UpdateAttachment updates the given attachment in database +// UpdateFileChunk updates the given file_chunk in database func UpdateFileChunk(fileChunk *FileChunk) error { return updateFileChunk(x, fileChunk) } @@ -98,3 +98,13 @@ func updateFileChunk(e Engine, fileChunk *FileChunk) error { _, err := sess.Cols("is_uploaded").Update(fileChunk) return err } + +// DeleteFileChunk delete the given file_chunk in database +func DeleteFileChunk(fileChunk *FileChunk) error { + return deleteFileChunk(x, fileChunk) +} + +func deleteFileChunk(e Engine, fileChunk *FileChunk) error { + _, err := e.ID(fileChunk.ID).Delete(fileChunk) + return err +} diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index fea7a3384..49c72b1c3 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -483,16 +483,25 @@ func GetSuccessChunks(ctx *context.Context) { if typeCloudBrain == models.TypeCloudBrainOne { chunks, err = storage.GetPartInfos(fileChunk.UUID, fileChunk.UploadID) if err != nil { - ctx.ServerError("GetPartInfos failed", err) - return + log.Error("GetPartInfos failed:%v", err.Error()) } } else { chunks, err = storage.GetObsPartInfos(fileChunk.UUID, fileChunk.UploadID) if err != nil { - ctx.ServerError("GetObsPartInfos failed", err) - return + log.Error("GetObsPartInfos failed:%v", err.Error()) } } + + if err != nil { + models.DeleteFileChunk(fileChunk) + ctx.JSON(200, map[string]string{ + "uuid": "", + "uploaded": "0", + "uploadID": "", + "chunks": "", + }) + return + } } var attachID int64 From a684b4f65a8baa6eb3b16d0196299dc73cfb6d2f Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 17 Aug 2021 16:44:06 +0800 Subject: [PATCH 05/10] adjust-modelarts --- modules/modelarts/modelarts.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index fc2183194..a5cb8388b 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -13,19 +13,10 @@ import ( const ( storageTypeOBS = "obs" autoStopDuration = 4 * 60 * 60 - flavor = "modelarts.kat1.xlarge" - //profileID = "Python3-ascend910-arm" - profileID = "efa847c0-7359-11eb-b34f-0255ac100057" - //poolID = "pool1328035d" - //poolName = "train-private-1" - poolID = "pool35eafa8f" - poolName = "train-private-0703" - poolType = "USER_DEFINED" DataSetMountPath = "/home/ma-user/work" NotebookEnv = "Python3" NotebookType = "Ascend" - FlavorInfo = "Ascend: 1*Ascend 910 CPU: 24 核 256GiB (modelarts.kat1.xlarge)" ) var ( From 17594c6f50f1eba87c64c5034f531b52b07eee0d Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Tue, 17 Aug 2021 18:01:53 +0800 Subject: [PATCH 06/10] mod --- modules/timeutil/since.go | 0 options/locale/locale_zh-CN.ini | 2 +- services/mailer/mail.go | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 modules/timeutil/since.go mode change 100644 => 100755 services/mailer/mail.go diff --git a/modules/timeutil/since.go b/modules/timeutil/since.go old mode 100644 new mode 100755 diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 9dd2d254b..5ac54f6c8 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -2435,7 +2435,7 @@ future=将来 1y=1年 seconds=%d 秒 minutes=%d 分钟 -hours=%d hours +hours=%d 小时 days=%d 天 weeks=%d 周 months=%d 个月 diff --git a/services/mailer/mail.go b/services/mailer/mail.go old mode 100644 new mode 100755 index dd5af445b..e4a210fd3 --- a/services/mailer/mail.go +++ b/services/mailer/mail.go @@ -59,7 +59,7 @@ func SendTestMail(email string) error { func SendUserMail(language string, u *models.User, tpl base.TplName, code, subject, info string) { data := map[string]interface{}{ "DisplayName": u.DisplayName(), - "ActiveCodeLives": timeutil.MinutesToFriendly(setting.Service.ActiveCodeLives, language), + "ActiveCodeLives": timeutil.MinutesToFriendly(setting.Service.ActiveCodeLives, "en-US"), "ResetPwdCodeLives": timeutil.MinutesToFriendly(setting.Service.ResetPwdCodeLives, language), "Code": code, } From 4a876c63e6634b535feb55b606d977a18b051745 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 18 Aug 2021 17:47:42 +0800 Subject: [PATCH 07/10] get real error_msg --- modules/cloudbrain/resty.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/cloudbrain/resty.go b/modules/cloudbrain/resty.go index ef7d70ef3..4e30ea0e4 100755 --- a/modules/cloudbrain/resty.go +++ b/modules/cloudbrain/resty.go @@ -82,6 +82,12 @@ sendjob: Post(HOST + "/rest-server/api/v1/jobs/") if err != nil { + if res != nil { + var response models.CloudBrainResult + json.Unmarshal(res.Body(), &response) + log.Error("code(%s), msg(%s)", response.Code, response.Msg) + return nil, fmt.Errorf(response.Msg) + } return nil, fmt.Errorf("resty create job: %s", err) } From 99c5117ff9cb542337ed1980e49294d6c079dc5d Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 18 Aug 2021 17:53:04 +0800 Subject: [PATCH 08/10] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=82=AE=E4=BB=B6?= =?UTF-8?q?=E6=97=B6=EF=BC=8C=E9=82=AE=E4=BB=B6=E5=86=85=E5=AE=B9=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E7=BB=9F=E4=B8=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/mailer/mail.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/mailer/mail.go b/services/mailer/mail.go index e4a210fd3..e9395f130 100755 --- a/services/mailer/mail.go +++ b/services/mailer/mail.go @@ -97,7 +97,7 @@ func SendResetPasswordMail(locale Locale, u *models.User) { func SendActivateEmailMail(locale Locale, u *models.User, email *models.EmailAddress) { data := map[string]interface{}{ "DisplayName": u.DisplayName(), - "ActiveCodeLives": timeutil.MinutesToFriendly(setting.Service.ActiveCodeLives, locale.Language()), + "ActiveCodeLives": timeutil.MinutesToFriendly(setting.Service.ActiveCodeLives, "en-US"), "Code": u.GenerateEmailActivateCode(email.Email), "Email": email.Email, } From cd0a64a6ae235b2d78ecaac61e4205cebce6242b Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 19 Aug 2021 11:01:31 +0800 Subject: [PATCH 09/10] =?UTF-8?q?=E5=88=9B=E5=BB=BA=E4=BA=91=E8=84=91?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=8A=A5=E9=94=99=E6=97=B6=EF=BC=8C=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E5=8F=AF=E7=BB=A7=E7=BB=AD=E6=93=8D=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/cloudbrain.go | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 50396aedb..dc1ead481 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -94,9 +94,8 @@ func jobNamePrefixValid(s string) string { return re.ReplaceAllString(lowStr, "") } -func CloudBrainNew(ctx *context.Context) { +func cloudBrainNewDataPrepare(ctx *context.Context) error{ ctx.Data["PageIsCloudBrain"] = true - t := time.Now() var jobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] ctx.Data["job_name"] = jobName @@ -104,7 +103,7 @@ func CloudBrainNew(ctx *context.Context) { result, err := cloudbrain.GetImages() if err != nil { ctx.Data["error"] = err.Error() - log.Error("cloudbrain.GetImages failed:", err.Error(), ctx.Data["msgID"]) + log.Error("cloudbrain.GetImages failed:", err.Error(), ctx.Data["MsgID"]) } for i, payload := range result.Payload.ImageInfo { @@ -120,7 +119,7 @@ func CloudBrainNew(ctx *context.Context) { resultPublic, err := cloudbrain.GetPublicImages() if err != nil { ctx.Data["error"] = err.Error() - log.Error("cloudbrain.GetPublicImages failed:", err.Error(), ctx.Data["msgID"]) + log.Error("cloudbrain.GetPublicImages failed:", err.Error(), ctx.Data["MsgID"]) } for i, payload := range resultPublic.Payload.ImageInfo { @@ -135,8 +134,8 @@ func CloudBrainNew(ctx *context.Context) { attachs, err := models.GetAllUserAttachments(ctx.User.ID) if err != nil { - ctx.ServerError("GetAllUserAttachments failed:", err) - return + log.Error("GetAllUserAttachments failed: %v", err, ctx.Data["MsgID"]) + return err } ctx.Data["attachments"] = attachs @@ -163,6 +162,16 @@ func CloudBrainNew(ctx *context.Context) { ctx.Data["resource_specs"] = cloudbrain.ResourceSpecs.ResourceSpec ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled + + return nil +} + +func CloudBrainNew(ctx *context.Context) { + err := cloudBrainNewDataPrepare(ctx) + if err != nil { + ctx.ServerError("get new cloudbrain info failed", err) + return + } ctx.HTML(200, tplCloudBrainNew) } @@ -178,7 +187,8 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { resourceSpecId := form.ResourceSpecId if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeSnn4imagenet) { - log.Error("jobtype error:", jobType, ctx.Data["msgID"]) + log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("jobtype error", tplCloudBrainNew, &form) return } @@ -186,11 +196,13 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { _, err := models.GetCloudbrainByName(jobName) if err == nil { log.Error("the job name did already exist", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("the job name did already exist", tplCloudBrainNew, &form) return } else { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("system error", tplCloudBrainNew, &form) return } @@ -201,6 +213,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath err = os.MkdirAll(modelPath, os.ModePerm) if err != nil { + cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) return } @@ -224,6 +237,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { err = cloudbrain.GenerateTask(ctx, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, jobType, gpuQueue, resourceSpecId) if err != nil { + cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(err.Error(), tplCloudBrainNew, &form) return } From 5a8998c931208787a31bad8315dcf7ec7bfc57d5 Mon Sep 17 00:00:00 2001 From: ychao_1983 Date: Thu, 19 Aug 2021 11:38:05 +0800 Subject: [PATCH 10/10] =?UTF-8?q?fix-92=E5=90=8E=E7=AB=AF=E8=BF=94?= =?UTF-8?q?=E5=9B=9E=E4=BB=BB=E5=8A=A1=E5=90=8D=E7=A7=B0=E4=B8=8D=E5=90=AB?= =?UTF-8?q?.,=E5=8F=AF=E4=BB=A5=E5=90=AB=5F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routers/repo/cloudbrain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index dc1ead481..7a78f914a 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -90,7 +90,7 @@ func cutString(str string, lens int) string { func jobNamePrefixValid(s string) string { lowStr := strings.ToLower(s) - re := regexp.MustCompile(`[^a-z0-9\\.\\-]+`) + re := regexp.MustCompile(`[^a-z0-9_\\-]+`) return re.ReplaceAllString(lowStr, "") }