From 6896276b44dabccbb8f4b7fc677276eb16cf6c6f Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 17 Aug 2022 16:03:42 +0800 Subject: [PATCH 01/11] redis to lock tasks --- modules/auth/wechat/access_token.go | 5 +++-- modules/redis/redis_key/cloudbrain_redis_key.go | 7 +++++++ modules/redis/redis_lock/lock.go | 13 +++++++------ routers/repo/modelarts.go | 11 +++++++++++ 4 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 modules/redis/redis_key/cloudbrain_redis_key.go diff --git a/modules/auth/wechat/access_token.go b/modules/auth/wechat/access_token.go index f9516e3e1..79ca6fc06 100644 --- a/modules/auth/wechat/access_token.go +++ b/modules/auth/wechat/access_token.go @@ -1,10 +1,11 @@ package wechat import ( + "time" + "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" "code.gitea.io/gitea/modules/redis/redis_lock" - "time" ) const EMPTY_REDIS_VAL = "Nil" @@ -35,7 +36,7 @@ func refreshAccessToken() { } func refreshAndGetAccessToken() string { - if ok := accessTokenLock.LockWithWait(3*time.Second, 3*time.Second); ok { + if ok, _ := accessTokenLock.LockWithWait(3*time.Second, 3*time.Second); ok { defer accessTokenLock.UnLock() token, _ := redis_client.Get(redis_key.WechatAccessTokenKey()) if token != "" { diff --git a/modules/redis/redis_key/cloudbrain_redis_key.go b/modules/redis/redis_key/cloudbrain_redis_key.go new file mode 100644 index 000000000..4c5d05dfc --- /dev/null +++ b/modules/redis/redis_key/cloudbrain_redis_key.go @@ -0,0 +1,7 @@ +package redis_key + +const CLOUDBRAIN_PREFIX = "cloudbrain" + +func CloudbrainBindingJobNameKey(repoId string, jobType string, jobName string) string { + return KeyJoin(CLOUDBRAIN_PREFIX, repoId, jobType, jobName, "redis_key") +} diff --git a/modules/redis/redis_lock/lock.go b/modules/redis/redis_lock/lock.go index b8cd837f1..597f9467f 100644 --- a/modules/redis/redis_lock/lock.go +++ b/modules/redis/redis_lock/lock.go @@ -1,8 +1,9 @@ package redis_lock import ( - "code.gitea.io/gitea/modules/redis/redis_client" "time" + + "code.gitea.io/gitea/modules/redis/redis_client" ) type DistributeLock struct { @@ -18,21 +19,21 @@ func (lock *DistributeLock) Lock(expireTime time.Duration) bool { return isOk } -func (lock *DistributeLock) LockWithWait(expireTime time.Duration, waitTime time.Duration) bool { +func (lock *DistributeLock) LockWithWait(expireTime time.Duration, waitTime time.Duration) (bool, error) { start := time.Now().Unix() * 1000 duration := waitTime.Milliseconds() for { - isOk, _ := redis_client.Setnx(lock.lockKey, "", expireTime) + isOk, err := redis_client.Setnx(lock.lockKey, "", expireTime) if isOk { - return true + return true, nil } if time.Now().Unix()*1000-start > duration { - return false + return false, err } time.Sleep(50 * time.Millisecond) } - return false + return false, nil } func (lock *DistributeLock) UnLock() error { diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 763308930..a99a5716c 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -25,6 +25,8 @@ import ( "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/obs" + "code.gitea.io/gitea/modules/redis/redis_key" + "code.gitea.io/gitea/modules/redis/redis_lock" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" @@ -1090,6 +1092,15 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) VersionCount := modelarts.VersionCountOne EngineName := form.EngineName + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName)) + isOk, err := lock.LockWithWait(60*time.Second, 60*time.Second) + if !isOk { + log.Error("The Task have been process:%v", err, ctx.Data["MsgID"]) + trainJobErrorNewDataPrepare(ctx, form) + ctx.RenderWithErr("system error", tplModelArtsTrainJobNew, &form) + return + } + count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) if err != nil { log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) From b7d5d2f7508607d580e404c24deae2478d4a8bb7 Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 18 Aug 2022 10:33:47 +0800 Subject: [PATCH 02/11] fix two same task --- modules/auth/wechat/access_token.go | 4 ++-- modules/redis/redis_lock/lock.go | 17 ++++++++--------- routers/repo/cloudbrain.go | 24 +++++++++++++++++++++++- routers/repo/modelarts.go | 11 ----------- 4 files changed, 33 insertions(+), 23 deletions(-) diff --git a/modules/auth/wechat/access_token.go b/modules/auth/wechat/access_token.go index 79ca6fc06..a7f23c1ee 100644 --- a/modules/auth/wechat/access_token.go +++ b/modules/auth/wechat/access_token.go @@ -29,14 +29,14 @@ func GetWechatAccessToken() string { } func refreshAccessToken() { - if ok := accessTokenLock.Lock(3 * time.Second); ok { + if ok, _ := accessTokenLock.Lock(3 * time.Second); ok { defer accessTokenLock.UnLock() callAccessTokenAndUpdateCache() } } func refreshAndGetAccessToken() string { - if ok, _ := accessTokenLock.LockWithWait(3*time.Second, 3*time.Second); ok { + if ok := accessTokenLock.LockWithWait(3*time.Second, 3*time.Second); ok { defer accessTokenLock.UnLock() token, _ := redis_client.Get(redis_key.WechatAccessTokenKey()) if token != "" { diff --git a/modules/redis/redis_lock/lock.go b/modules/redis/redis_lock/lock.go index 597f9467f..5aee2ed0f 100644 --- a/modules/redis/redis_lock/lock.go +++ b/modules/redis/redis_lock/lock.go @@ -14,26 +14,25 @@ func NewDistributeLock(lockKey string) *DistributeLock { return &DistributeLock{lockKey: lockKey} } -func (lock *DistributeLock) Lock(expireTime time.Duration) bool { - isOk, _ := redis_client.Setnx(lock.lockKey, "", expireTime) - return isOk +func (lock *DistributeLock) Lock(expireTime time.Duration) (bool, error) { + isOk, err := redis_client.Setnx(lock.lockKey, "", expireTime) + return isOk, err } -func (lock *DistributeLock) LockWithWait(expireTime time.Duration, waitTime time.Duration) (bool, error) { +func (lock *DistributeLock) LockWithWait(expireTime time.Duration, waitTime time.Duration) bool { start := time.Now().Unix() * 1000 duration := waitTime.Milliseconds() for { - isOk, err := redis_client.Setnx(lock.lockKey, "", expireTime) + isOk, _ := redis_client.Setnx(lock.lockKey, "", expireTime) if isOk { - return true, nil + return true } if time.Now().Unix()*1000-start > duration { - return false, err + return false } time.Sleep(50 * time.Millisecond) } - - return false, nil + return false } func (lock *DistributeLock) UnLock() error { diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index c3c6c43cb..baddce401 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -30,6 +30,8 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/modules/redis/redis_key" + "code.gitea.io/gitea/modules/redis/redis_lock" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" @@ -247,11 +249,21 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { tpl = tplCloudBrainTrainJobNew } + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName)) + isOk, err := lock.Lock(5 * time.Second) + if !isOk { + log.Error("The Task have been process:%v", err, ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) + ctx.RenderWithErr("system process error", tpl, &form) + return + } + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) + defer lock.UnLock() ctx.RenderWithErr("the job name did already exist", tpl, &form) return } @@ -259,6 +271,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) + defer lock.UnLock() ctx.RenderWithErr("system error", tpl, &form) return } @@ -266,6 +279,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if !jobNamePattern.MatchString(displayJobName) { cloudBrainNewDataPrepare(ctx) + defer lock.UnLock() ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form) return } @@ -273,6 +287,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeTrain) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) + defer lock.UnLock() ctx.RenderWithErr("jobtype error", tpl, &form) return } @@ -281,12 +296,14 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) + defer lock.UnLock() ctx.RenderWithErr("system error", tpl, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) + defer lock.UnLock() ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) return } @@ -299,6 +316,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if err != nil { log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) + defer lock.UnLock() ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) return } @@ -310,6 +328,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if err != nil || !bootFileExist { log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) + defer lock.UnLock() ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form) return } @@ -318,6 +337,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if err != nil { log.Error("getTrainJobCommand failed: %v", err) cloudBrainNewDataPrepare(ctx) + defer lock.UnLock() ctx.RenderWithErr(err.Error(), tpl, &form) return } @@ -329,6 +349,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if errStr != "" { cloudBrainNewDataPrepare(ctx) + defer lock.UnLock() ctx.RenderWithErr(errStr, tpl, &form) return } @@ -339,6 +360,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { errStr = loadCodeAndMakeModelPath(repo, codePath, branchName, jobName, cloudbrain.ModelMountPath) if errStr != "" { cloudBrainNewDataPrepare(ctx) + defer lock.UnLock() ctx.RenderWithErr(ctx.Tr(errStr), tpl, &form) return } @@ -375,10 +397,10 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { err = cloudbrain.GenerateTask(req) if err != nil { cloudBrainNewDataPrepare(ctx) + defer lock.UnLock() ctx.RenderWithErr(err.Error(), tpl, &form) return } - if jobType == string(models.JobTypeTrain) { ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=all") } else { diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index a99a5716c..763308930 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -25,8 +25,6 @@ import ( "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/obs" - "code.gitea.io/gitea/modules/redis/redis_key" - "code.gitea.io/gitea/modules/redis/redis_lock" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" @@ -1092,15 +1090,6 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) VersionCount := modelarts.VersionCountOne EngineName := form.EngineName - lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName)) - isOk, err := lock.LockWithWait(60*time.Second, 60*time.Second) - if !isOk { - log.Error("The Task have been process:%v", err, ctx.Data["MsgID"]) - trainJobErrorNewDataPrepare(ctx, form) - ctx.RenderWithErr("system error", tplModelArtsTrainJobNew, &form) - return - } - count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) if err != nil { log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) From 5011da4ca1f3d6026fd4a5489b33ec7bc2434e4a Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 19 Aug 2022 12:13:39 +0800 Subject: [PATCH 03/11] add redis key --- modules/cloudbrain/resty.go | 13 ++++++------- routers/repo/cloudbrain.go | 14 +++++++------- routers/repo/modelarts.go | 14 ++++++++++++++ templates/repo/modelarts/trainjob/new.tmpl | 2 +- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/modules/cloudbrain/resty.go b/modules/cloudbrain/resty.go index 7b714c4b5..7a3ab25f2 100755 --- a/modules/cloudbrain/resty.go +++ b/modules/cloudbrain/resty.go @@ -24,10 +24,10 @@ var ( ) const ( - JobHasBeenStopped = "S410" - Public = "public" - Custom = "custom" - LogPageSize = 500 + JobHasBeenStopped = "S410" + Public = "public" + Custom = "custom" + LogPageSize = 500 LogPageTokenExpired = "5m" pageSize = 15 QueuesDetailUrl = "/rest-server/api/v2/queuesdetail" @@ -140,7 +140,6 @@ sendjob: if jobResult.Code != Success { return &jobResult, fmt.Errorf("jobResult err: %s", res.String()) } - return &jobResult, nil } @@ -497,7 +496,7 @@ func GetJobAllLog(scrollID string) (*models.GetJobLogResult, error) { client := getRestyClient() var result models.GetJobLogResult req := models.GetAllJobLogParams{ - Scroll: LogPageTokenExpired, + Scroll: LogPageTokenExpired, ScrollID: scrollID, } @@ -521,7 +520,7 @@ func GetJobAllLog(scrollID string) (*models.GetJobLogResult, error) { return &result, nil } -func DeleteJobLogToken(scrollID string) (error) { +func DeleteJobLogToken(scrollID string) error { checkSetting() client := getRestyClient() var result models.DeleteJobLogTokenResult diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index baddce401..119e77348 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -250,13 +250,13 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName)) - isOk, err := lock.Lock(5 * time.Second) - if !isOk { - log.Error("The Task have been process:%v", err, ctx.Data["MsgID"]) - cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("system process error", tpl, &form) - return - } + // isOk, err := lock.Lock(5 * time.Second) + // if !isOk { + // log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + // // cloudBrainNewDataPrepare(ctx) + // // ctx.RenderWithErr("The task have been processed, please wait a minute", tpl, &form) + // // return + // } tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) if err == nil { diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 763308930..b98772775 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -25,6 +25,8 @@ import ( "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/obs" + "code.gitea.io/gitea/modules/redis/redis_client" + "code.gitea.io/gitea/modules/redis/redis_key" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" @@ -784,6 +786,8 @@ func setSpecBySpecialPoolConfig(ctx *context.Context, jobType string) { } func trainJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error { + var taskJobnameKey = ctx.Query("taskJobnameKey") + redis_client.Del(taskJobnameKey) ctx.Data["PageIsCloudBrain"] = true //can, err := canUserCreateTrainJob(ctx.User.ID) @@ -1090,6 +1094,16 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) VersionCount := modelarts.VersionCountOne EngineName := form.EngineName + taskJobnameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName) + isOk, err := redis_client.Setnx(taskJobnameKey, "", 5*time.Second) + ctx.Data["taskJobnameKey"] = taskJobnameKey + if !isOk { + log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + // trainJobErrorNewDataPrepare(ctx, form) + ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsTrainJobIndex, &form) + return + } + count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) if err != nil { log.Error("GetCloudbrainTrainJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) diff --git a/templates/repo/modelarts/trainjob/new.tmpl b/templates/repo/modelarts/trainjob/new.tmpl index 51f447f46..002f9599d 100755 --- a/templates/repo/modelarts/trainjob/new.tmpl +++ b/templates/repo/modelarts/trainjob/new.tmpl @@ -485,7 +485,7 @@ }, onSuccess: function () { // $('.ui.page.dimmer').dimmer('show') - document.getElementById("mask").style.display = "block" + // document.getElementById("mask").style.display = "block" }, onFailure: function (e) { return false; From 54081e35579f91f930622f090c69973fc5725fe8 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 24 Aug 2022 11:31:21 +0800 Subject: [PATCH 04/11] add rediskey stop same task --- routers/repo/cloudbrain.go | 54 ++++++++++++++++++++++---------------- routers/repo/grampus.go | 23 ++++++++++++++++ routers/repo/modelarts.go | 47 ++++++++++++++++++++++++++++----- 3 files changed, 95 insertions(+), 29 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 119e77348..0226c1414 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -30,8 +30,8 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" + "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" - "code.gitea.io/gitea/modules/redis/redis_lock" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" @@ -100,6 +100,9 @@ func jobNamePrefixValid(s string) string { } func cloudBrainNewDataPrepare(ctx *context.Context) error { + var taskJobNameKey = ctx.Query("taskJobNameKey") + redis_client.Del(taskJobNameKey) + ctx.Data["PageIsCloudBrain"] = true t := time.Now() var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] @@ -249,21 +252,20 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { tpl = tplCloudBrainTrainJobNew } - lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName)) - // isOk, err := lock.Lock(5 * time.Second) - // if !isOk { - // log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) - // // cloudBrainNewDataPrepare(ctx) - // // ctx.RenderWithErr("The task have been processed, please wait a minute", tpl, &form) - // // return - // } + taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName) + isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) + ctx.Data["taskJobNameKey"] = taskJobNameKey + if !isOk { + log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + ctx.RenderWithErr("The task have been processed, please wait a minute", tpl, &form) + return + } tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) if err == nil { if len(tasks) != 0 { log.Error("the job name did already exist", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - defer lock.UnLock() ctx.RenderWithErr("the job name did already exist", tpl, &form) return } @@ -271,7 +273,6 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if !models.IsErrJobNotExist(err) { log.Error("system error, %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - defer lock.UnLock() ctx.RenderWithErr("system error", tpl, &form) return } @@ -279,7 +280,6 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if !jobNamePattern.MatchString(displayJobName) { cloudBrainNewDataPrepare(ctx) - defer lock.UnLock() ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form) return } @@ -287,7 +287,6 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeTrain) { log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - defer lock.UnLock() ctx.RenderWithErr("jobtype error", tpl, &form) return } @@ -296,14 +295,12 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if err != nil { log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - defer lock.UnLock() ctx.RenderWithErr("system error", tpl, &form) return } else { if count >= 1 { log.Error("the user already has running or waiting task", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - defer lock.UnLock() ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) return } @@ -316,7 +313,6 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if err != nil { log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - defer lock.UnLock() ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) return } @@ -328,7 +324,6 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if err != nil || !bootFileExist { log.Error("Get bootfile error:", err, ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - defer lock.UnLock() ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form) return } @@ -337,7 +332,6 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if err != nil { log.Error("getTrainJobCommand failed: %v", err) cloudBrainNewDataPrepare(ctx) - defer lock.UnLock() ctx.RenderWithErr(err.Error(), tpl, &form) return } @@ -349,7 +343,6 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { if errStr != "" { cloudBrainNewDataPrepare(ctx) - defer lock.UnLock() ctx.RenderWithErr(errStr, tpl, &form) return } @@ -360,7 +353,6 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { errStr = loadCodeAndMakeModelPath(repo, codePath, branchName, jobName, cloudbrain.ModelMountPath) if errStr != "" { cloudBrainNewDataPrepare(ctx) - defer lock.UnLock() ctx.RenderWithErr(ctx.Tr(errStr), tpl, &form) return } @@ -397,7 +389,6 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { err = cloudbrain.GenerateTask(req) if err != nil { cloudBrainNewDataPrepare(ctx) - defer lock.UnLock() ctx.RenderWithErr(err.Error(), tpl, &form) return } @@ -446,10 +437,19 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra bootFile := strings.TrimSpace(form.BootFile) labelName := form.LabelName repo := ctx.Repo.Repository + tpl := tplCloudBrainInferenceJobNew + + taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeInference), displayJobName) + isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) + ctx.Data["taskJobNameKey"] = taskJobNameKey + if !isOk { + log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + ctx.RenderWithErr("The task have been processed, please wait a minute", tpl, &form) + return + } ckptUrl := setting.Attachment.Minio.RealPath + form.TrainUrl + form.CkptName log.Info("ckpt url:" + ckptUrl) - tpl := tplCloudBrainInferenceJobNew command, err := getInferenceJobCommand(form) if err != nil { log.Error("getTrainJobCommand failed: %v", err) @@ -2274,12 +2274,20 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo resourceSpecId := cloudbrain.BenchMarkResourceID benchmarkTypeID := form.BenchmarkTypeID benchmarkChildTypeID := form.BenchmarkChildTypeID + repo := ctx.Repo.Repository ctx.Data["description"] = form.Description ctx.Data["benchmarkTypeID"] = benchmarkTypeID ctx.Data["benchmark_child_types_id_hidden"] = benchmarkChildTypeID - repo := ctx.Repo.Repository + taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeBenchmark), displayJobName) + isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) + ctx.Data["taskJobNameKey"] = taskJobNameKey + if !isOk { + log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + ctx.RenderWithErr("The task have been processed, please wait a minute", tplCloudBrainBenchmarkNew, &form) + return + } tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeBenchmark), displayJobName) if err == nil { diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index cdde7596c..3342f6ce1 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -17,6 +17,8 @@ import ( "code.gitea.io/gitea/modules/grampus" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/notification" + "code.gitea.io/gitea/modules/redis/redis_client" + "code.gitea.io/gitea/modules/redis/redis_key" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" "github.com/unknwon/com" @@ -61,6 +63,9 @@ func GrampusTrainJobNPUNew(ctx *context.Context) { } func grampusTrainJobNewDataPrepare(ctx *context.Context, processType string) error { + var taskJobNameKey = ctx.Query("taskJobNameKey") + redis_client.Del(taskJobNameKey) + ctx.Data["PageIsCloudBrain"] = true t := time.Now() @@ -209,6 +214,15 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain flavorName := form.FlavorName image := strings.TrimSpace(form.Image) + taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName) + isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) + ctx.Data["taskJobNameKey"] = taskJobNameKey + if !isOk { + log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + ctx.RenderWithErr("The task have been processed, please wait a minute", tplGrampusTrainJobGPUNew, &form) + return + } + if !jobNamePattern.MatchString(displayJobName) { grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplGrampusTrainJobGPUNew, &form) @@ -401,6 +415,15 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain versionCount := modelarts.VersionCountOne engineName := form.EngineName + taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName) + isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) + ctx.Data["taskJobNameKey"] = taskJobNameKey + if !isOk { + log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + ctx.RenderWithErr("The task have been processed, please wait a minute", tplGrampusTrainJobNPUNew, &form) + return + } + if !jobNamePattern.MatchString(displayJobName) { grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplGrampusTrainJobNPUNew, &form) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index b98772775..8c69851db 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -125,6 +125,9 @@ func NotebookNew(ctx *context.Context) { } func notebookNewDataPrepare(ctx *context.Context) error { + var taskJobNameKey = ctx.Query("taskJobNameKey") + redis_client.Del(taskJobNameKey) + ctx.Data["PageIsCloudBrain"] = true t := time.Now() var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] @@ -210,6 +213,15 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm imageId := form.ImageId repo := ctx.Repo.Repository + taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeDebug), displayJobName) + isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) + ctx.Data["taskJobNameKey"] = taskJobNameKey + if !isOk { + log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsNotebookNew, &form) + return + } + count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID) if err != nil { log.Error("GetCloudbrainNotebookCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -786,8 +798,8 @@ func setSpecBySpecialPoolConfig(ctx *context.Context, jobType string) { } func trainJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error { - var taskJobnameKey = ctx.Query("taskJobnameKey") - redis_client.Del(taskJobnameKey) + var taskJobNameKey = ctx.Query("taskJobNameKey") + redis_client.Del(taskJobNameKey) ctx.Data["PageIsCloudBrain"] = true //can, err := canUserCreateTrainJob(ctx.User.ID) @@ -976,6 +988,9 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error { } func versionErrorDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error { + var taskJobNameKey = ctx.Query("taskJobNameKey") + redis_client.Del(taskJobNameKey) + ctx.Data["PageIsCloudBrain"] = true var jobID = ctx.Params(":jobid") // var versionName = ctx.Params(":version-name") @@ -1094,12 +1109,11 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) VersionCount := modelarts.VersionCountOne EngineName := form.EngineName - taskJobnameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName) - isOk, err := redis_client.Setnx(taskJobnameKey, "", 5*time.Second) - ctx.Data["taskJobnameKey"] = taskJobnameKey + taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName) + isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) + ctx.Data["taskJobNameKey"] = taskJobNameKey if !isOk { log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) - // trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsTrainJobIndex, &form) return } @@ -1417,6 +1431,15 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ EngineName := form.EngineName isLatestVersion := modelarts.IsLatestVersion + taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName) + isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) + ctx.Data["taskJobNameKey"] = taskJobNameKey + if !isOk { + log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsTrainJobVersionNew, &form) + return + } + canNewJob, _ := canUserCreateTrainJobVersion(ctx, latestTask.UserID) if !canNewJob { versionErrorDataPrepare(ctx, form) @@ -2000,6 +2023,15 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference ckptUrl := "/" + form.TrainUrl + form.CkptName log.Info("ckpt url:" + ckptUrl) + taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeInference), displayJobName) + isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) + ctx.Data["taskJobNameKey"] = taskJobNameKey + if !isOk { + log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsInferenceJobNew, &form) + return + } + count, err := models.GetCloudbrainInferenceJobCountByUserID(ctx.User.ID) if err != nil { log.Error("GetCloudbrainInferenceJobCountByUserID failed:%v", err, ctx.Data["MsgID"]) @@ -2378,6 +2410,9 @@ func inferenceJobNewDataPrepare(ctx *context.Context) error { } func inferenceJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArtsInferenceJobForm) error { + var taskJobNameKey = ctx.Query("taskJobNameKey") + redis_client.Del(taskJobNameKey) + ctx.Data["PageIsCloudBrain"] = true t := time.Now() From 621f19507054e5096911fd40d466e4874254e664 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 24 Aug 2022 11:37:31 +0800 Subject: [PATCH 05/11] add rediskey stop same task --- modules/auth/wechat/access_token.go | 2 +- modules/redis/redis_lock/lock.go | 6 +++--- templates/repo/modelarts/trainjob/new.tmpl | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/auth/wechat/access_token.go b/modules/auth/wechat/access_token.go index a7f23c1ee..375322233 100644 --- a/modules/auth/wechat/access_token.go +++ b/modules/auth/wechat/access_token.go @@ -29,7 +29,7 @@ func GetWechatAccessToken() string { } func refreshAccessToken() { - if ok, _ := accessTokenLock.Lock(3 * time.Second); ok { + if ok := accessTokenLock.Lock(3 * time.Second); ok { defer accessTokenLock.UnLock() callAccessTokenAndUpdateCache() } diff --git a/modules/redis/redis_lock/lock.go b/modules/redis/redis_lock/lock.go index 5aee2ed0f..2d3776d90 100644 --- a/modules/redis/redis_lock/lock.go +++ b/modules/redis/redis_lock/lock.go @@ -14,9 +14,9 @@ func NewDistributeLock(lockKey string) *DistributeLock { return &DistributeLock{lockKey: lockKey} } -func (lock *DistributeLock) Lock(expireTime time.Duration) (bool, error) { - isOk, err := redis_client.Setnx(lock.lockKey, "", expireTime) - return isOk, err +func (lock *DistributeLock) Lock(expireTime time.Duration) bool { + isOk, _ := redis_client.Setnx(lock.lockKey, "", expireTime) + return isOk } func (lock *DistributeLock) LockWithWait(expireTime time.Duration, waitTime time.Duration) bool { diff --git a/templates/repo/modelarts/trainjob/new.tmpl b/templates/repo/modelarts/trainjob/new.tmpl index 002f9599d..51f447f46 100755 --- a/templates/repo/modelarts/trainjob/new.tmpl +++ b/templates/repo/modelarts/trainjob/new.tmpl @@ -485,7 +485,7 @@ }, onSuccess: function () { // $('.ui.page.dimmer').dimmer('show') - // document.getElementById("mask").style.display = "block" + document.getElementById("mask").style.display = "block" }, onFailure: function (e) { return false; From 7c6a21a6cfbdf8ed32a3079b00e7c2fa2a2e934c Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 1 Sep 2022 12:04:48 +0800 Subject: [PATCH 06/11] add lock to stop same task --- routers/repo/cloudbrain.go | 38 ++++++++++++++++++-------------- routers/repo/grampus.go | 21 ++++++++---------- routers/repo/modelarts.go | 45 ++++++++++++++------------------------ 3 files changed, 48 insertions(+), 56 deletions(-) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 28d6b4361..34917d2cd 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -30,8 +30,8 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/modelarts" - "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" + "code.gitea.io/gitea/modules/redis/redis_lock" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" @@ -100,9 +100,6 @@ func jobNamePrefixValid(s string) string { } func cloudBrainNewDataPrepare(ctx *context.Context) error { - var taskJobNameKey = ctx.Query("taskJobNameKey") - redis_client.Del(taskJobNameKey) - ctx.Data["PageIsCloudBrain"] = true t := time.Now() var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] @@ -252,14 +249,14 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { tpl = tplCloudBrainTrainJobNew } - taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName) - isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) - ctx.Data["taskJobNameKey"] = taskJobNameKey + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), jobType, displayJobName)) + isOk := lock.Lock(60 * time.Second) if !isOk { - log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + log.Error("The task have been processed", ctx.Data["MsgID"]) ctx.RenderWithErr("The task have been processed, please wait a minute", tpl, &form) return } + defer lock.UnLock() tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) if err == nil { @@ -439,14 +436,14 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra repo := ctx.Repo.Repository tpl := tplCloudBrainInferenceJobNew - taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeInference), displayJobName) - isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) - ctx.Data["taskJobNameKey"] = taskJobNameKey + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), jobType, displayJobName)) + isOk := lock.Lock(60 * time.Second) if !isOk { - log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + log.Error("The task have been processed", ctx.Data["MsgID"]) ctx.RenderWithErr("The task have been processed, please wait a minute", tpl, &form) return } + defer lock.UnLock() ckptUrl := setting.Attachment.Minio.RealPath + form.TrainUrl + form.CkptName log.Info("ckpt url:" + ckptUrl) @@ -2319,14 +2316,14 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo ctx.Data["benchmarkTypeID"] = benchmarkTypeID ctx.Data["benchmark_child_types_id_hidden"] = benchmarkChildTypeID - taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeBenchmark), displayJobName) - isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) - ctx.Data["taskJobNameKey"] = taskJobNameKey + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), form.JobType, displayJobName)) + isOk := lock.Lock(60 * time.Second) if !isOk { - log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + log.Error("The task have been processed", ctx.Data["MsgID"]) ctx.RenderWithErr("The task have been processed, please wait a minute", tplCloudBrainBenchmarkNew, &form) return } + defer lock.UnLock() tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeBenchmark), displayJobName) if err == nil { @@ -2515,6 +2512,15 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) tpl := tplCloudBrainBenchmarkNew command := cloudbrain.GetCloudbrainDebugCommand() + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), jobType, displayJobName)) + isOk := lock.Lock(60 * time.Second) + if !isOk { + log.Error("The task have been processed", ctx.Data["MsgID"]) + ctx.RenderWithErr("The task have been processed, please wait a minute", tpl, &form) + return + } + defer lock.UnLock() + tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName) if err == nil { if len(tasks) != 0 { diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index ab86f9e04..61d0156d1 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -17,8 +17,8 @@ import ( "code.gitea.io/gitea/modules/grampus" "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/notification" - "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" + "code.gitea.io/gitea/modules/redis/redis_lock" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" "github.com/unknwon/com" @@ -63,9 +63,6 @@ func GrampusTrainJobNPUNew(ctx *context.Context) { } func grampusTrainJobNewDataPrepare(ctx *context.Context, processType string) error { - var taskJobNameKey = ctx.Query("taskJobNameKey") - redis_client.Del(taskJobNameKey) - ctx.Data["PageIsCloudBrain"] = true t := time.Now() @@ -214,14 +211,14 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain flavorName := form.FlavorName image := strings.TrimSpace(form.Image) - taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName) - isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) - ctx.Data["taskJobNameKey"] = taskJobNameKey + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName)) + isOk := lock.Lock(60 * time.Second) if !isOk { - log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + log.Error("The task have been processed", ctx.Data["MsgID"]) ctx.RenderWithErr("The task have been processed, please wait a minute", tplGrampusTrainJobGPUNew, &form) return } + defer lock.UnLock() if !jobNamePattern.MatchString(displayJobName) { grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) @@ -415,14 +412,14 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain versionCount := modelarts.VersionCountOne engineName := form.EngineName - taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName) - isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) - ctx.Data["taskJobNameKey"] = taskJobNameKey + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName)) + isOk := lock.Lock(60 * time.Second) if !isOk { - log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + log.Error("The task have been processed", ctx.Data["MsgID"]) ctx.RenderWithErr("The task have been processed, please wait a minute", tplGrampusTrainJobNPUNew, &form) return } + defer lock.UnLock() if !jobNamePattern.MatchString(displayJobName) { grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index f50c6b132..9851b3842 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -25,8 +25,8 @@ import ( "code.gitea.io/gitea/modules/modelarts" "code.gitea.io/gitea/modules/notification" "code.gitea.io/gitea/modules/obs" - "code.gitea.io/gitea/modules/redis/redis_client" "code.gitea.io/gitea/modules/redis/redis_key" + "code.gitea.io/gitea/modules/redis/redis_lock" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/timeutil" @@ -125,9 +125,6 @@ func NotebookNew(ctx *context.Context) { } func notebookNewDataPrepare(ctx *context.Context) error { - var taskJobNameKey = ctx.Query("taskJobNameKey") - redis_client.Del(taskJobNameKey) - ctx.Data["PageIsCloudBrain"] = true t := time.Now() var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:] @@ -213,14 +210,14 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm imageId := form.ImageId repo := ctx.Repo.Repository - taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeDebug), displayJobName) - isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) - ctx.Data["taskJobNameKey"] = taskJobNameKey + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeDebug), displayJobName)) + isOk := lock.Lock(60 * time.Second) if !isOk { - log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + log.Error("The task have been processed", ctx.Data["MsgID"]) ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsNotebookNew, &form) return } + defer lock.UnLock() count, err := models.GetCloudbrainNotebookCountByUserID(ctx.User.ID) if err != nil { @@ -816,8 +813,6 @@ func setSpecBySpecialPoolConfig(ctx *context.Context, jobType string) { } func trainJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error { - var taskJobNameKey = ctx.Query("taskJobNameKey") - redis_client.Del(taskJobNameKey) ctx.Data["PageIsCloudBrain"] = true //can, err := canUserCreateTrainJob(ctx.User.ID) @@ -1013,9 +1008,6 @@ func trainJobNewVersionDataPrepare(ctx *context.Context) error { } func versionErrorDataPrepare(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) error { - var taskJobNameKey = ctx.Query("taskJobNameKey") - redis_client.Del(taskJobNameKey) - ctx.Data["PageIsCloudBrain"] = true var jobID = ctx.Params(":jobid") // var versionName = ctx.Params(":version-name") @@ -1134,14 +1126,14 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) VersionCount := modelarts.VersionCountOne EngineName := form.EngineName - taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName) - isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) - ctx.Data["taskJobNameKey"] = taskJobNameKey + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName)) + isOk := lock.Lock(60 * time.Second) if !isOk { - log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + log.Error("The task have been processed", ctx.Data["MsgID"]) ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsTrainJobIndex, &form) return } + defer lock.UnLock() count, err := models.GetCloudbrainTrainJobCountByUserID(ctx.User.ID) if err != nil { @@ -1456,14 +1448,14 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ EngineName := form.EngineName isLatestVersion := modelarts.IsLatestVersion - taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName) - isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) - ctx.Data["taskJobNameKey"] = taskJobNameKey + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName)) + isOk := lock.Lock(60 * time.Second) if !isOk { - log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + log.Error("The task have been processed", ctx.Data["MsgID"]) ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsTrainJobVersionNew, &form) return } + defer lock.UnLock() canNewJob, _ := canUserCreateTrainJobVersion(ctx, latestTask.UserID) if !canNewJob { @@ -2058,14 +2050,14 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference ckptUrl := "/" + form.TrainUrl + form.CkptName log.Info("ckpt url:" + ckptUrl) - taskJobNameKey := redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeInference), displayJobName) - isOk, err := redis_client.Setnx(taskJobNameKey, "", 5*time.Second) - ctx.Data["taskJobNameKey"] = taskJobNameKey + lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeInference), displayJobName)) + isOk := lock.Lock(60 * time.Second) if !isOk { - log.Error("The task have been processed:%v", err, ctx.Data["MsgID"]) + log.Error("The task have been processed", ctx.Data["MsgID"]) ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsInferenceJobNew, &form) return } + defer lock.UnLock() count, err := models.GetCloudbrainInferenceJobCountByUserID(ctx.User.ID) if err != nil { @@ -2468,9 +2460,6 @@ func inferenceJobNewDataPrepare(ctx *context.Context) error { } func inferenceJobErrorNewDataPrepare(ctx *context.Context, form auth.CreateModelArtsInferenceJobForm) error { - var taskJobNameKey = ctx.Query("taskJobNameKey") - redis_client.Del(taskJobNameKey) - ctx.Data["PageIsCloudBrain"] = true t := time.Now() From cb3d7947b94a9e66aedf5cee925ced607c2c823f Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 6 Sep 2022 09:30:53 +0800 Subject: [PATCH 07/11] fix-2859 --- routers/repo/cloudbrain.go | 4 ++++ routers/repo/grampus.go | 2 ++ routers/repo/modelarts.go | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 34917d2cd..790e210ad 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -253,6 +253,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { isOk := lock.Lock(60 * time.Second) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("The task have been processed, please wait a minute", tpl, &form) return } @@ -440,6 +441,7 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra isOk := lock.Lock(60 * time.Second) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("The task have been processed, please wait a minute", tpl, &form) return } @@ -2320,6 +2322,7 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo isOk := lock.Lock(60 * time.Second) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("The task have been processed, please wait a minute", tplCloudBrainBenchmarkNew, &form) return } @@ -2516,6 +2519,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) isOk := lock.Lock(60 * time.Second) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) + cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr("The task have been processed, please wait a minute", tpl, &form) return } diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index 61d0156d1..c305889b8 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -215,6 +215,7 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain isOk := lock.Lock(60 * time.Second) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) + grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) ctx.RenderWithErr("The task have been processed, please wait a minute", tplGrampusTrainJobGPUNew, &form) return } @@ -416,6 +417,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain isOk := lock.Lock(60 * time.Second) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) + grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) ctx.RenderWithErr("The task have been processed, please wait a minute", tplGrampusTrainJobNPUNew, &form) return } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 9851b3842..fddbb557c 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -214,6 +214,7 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm isOk := lock.Lock(60 * time.Second) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) + notebookNewDataPrepare(ctx) ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsNotebookNew, &form) return } @@ -1130,6 +1131,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) isOk := lock.Lock(60 * time.Second) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) + trainJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsTrainJobIndex, &form) return } @@ -1452,6 +1454,7 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ isOk := lock.Lock(60 * time.Second) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) + versionErrorDataPrepare(ctx, form) ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsTrainJobVersionNew, &form) return } @@ -2054,6 +2057,7 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference isOk := lock.Lock(60 * time.Second) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) + inferenceJobErrorNewDataPrepare(ctx, form) ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsInferenceJobNew, &form) return } From c358b54fa655fd3bb733561e1e63a03c64650ee0 Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 6 Sep 2022 16:39:42 +0800 Subject: [PATCH 08/11] fix-2859 --- routers/repo/modelarts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index fddbb557c..8821175b6 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1132,7 +1132,7 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) trainJobErrorNewDataPrepare(ctx, form) - ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsTrainJobIndex, &form) + ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsTrainJobNew, &form) return } defer lock.UnLock() From 86b05f642fabddd08ca3cf62d6e2c156dcc46285 Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 7 Sep 2022 17:18:37 +0800 Subject: [PATCH 09/11] fix-2069 --- models/cloudbrain.go | 3 ++- options/locale/locale_en-US.ini | 1 + options/locale/locale_zh-CN.ini | 1 + routers/repo/cloudbrain.go | 16 ++++++++-------- routers/repo/grampus.go | 8 ++++---- routers/repo/modelarts.go | 19 ++++++++++--------- 6 files changed, 26 insertions(+), 22 deletions(-) diff --git a/models/cloudbrain.go b/models/cloudbrain.go index 5be35ea0a..47d09d9ce 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -101,7 +101,8 @@ const ( ModelArtsTrainJobCheckRunningCompleted ModelArtsJobStatus = "CHECK_RUNNING_COMPLETED" //审核作业已经完成 ModelArtsTrainJobCheckFailed ModelArtsJobStatus = "CHECK_FAILED" //审核作业失败 - DURATION_STR_ZERO = "00:00:00" + DURATION_STR_ZERO = "00:00:00" + CloudbrainKeyDuration = 24 * time.Hour //grampus GrampusStatusPending = "pending" diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index f6ff5d96f..26a1538d2 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1092,6 +1092,7 @@ cloudbrain_operate = Operate cloudbrain_status_createtime = Status/Createtime cloudbrain_status_runtime = Running Time cloudbrain_jobname_err=Name must start with a lowercase letter or number,can include lowercase letter,number,_ and -,can not end with _, and can be up to 36 characters long. +cloudbrain_samejob_err=You have created a task with the same name, the system is processing it, please wait a minute. cloudbrain_bootfile_err=The bootfile does not exist in the repository cloudbrain_query_fail=Failed to query cloudbrain information. cloudbrain.mirror_tag = Mirror Tag diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 17ec29319..636697cb1 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1096,6 +1096,7 @@ cloudbrain_operate=操作 cloudbrain_status_createtime=状态/创建时间 cloudbrain_status_runtime = 运行时长 cloudbrain_jobname_err=只能以小写字母或数字开头且只包含小写字母、数字、_和-，不能以_结尾，最长36个字符。 +cloudbrain_samejob_err=您已经创建了同名任务，系统处理中，请您稍候。 cloudbrain_bootfile_err=仓库中不存在启动文件 cloudbrain_query_fail=查询云脑任务失败。 cloudbrain.mirror_tag = 镜像标签 diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 58aabd129..02a1f6a2a 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -250,11 +250,11 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), jobType, displayJobName)) - isOk := lock.Lock(60 * time.Second) + isOk := lock.Lock(models.CloudbrainKeyDuration) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("The task have been processed, please wait a minute", tpl, &form) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tpl, &form) return } defer lock.UnLock() @@ -438,11 +438,11 @@ func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBra tpl := tplCloudBrainInferenceJobNew lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), jobType, displayJobName)) - isOk := lock.Lock(60 * time.Second) + isOk := lock.Lock(models.CloudbrainKeyDuration) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("The task have been processed, please wait a minute", tpl, &form) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tpl, &form) return } defer lock.UnLock() @@ -2319,11 +2319,11 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo ctx.Data["benchmark_child_types_id_hidden"] = benchmarkChildTypeID lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), form.JobType, displayJobName)) - isOk := lock.Lock(60 * time.Second) + isOk := lock.Lock(models.CloudbrainKeyDuration) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("The task have been processed, please wait a minute", tplCloudBrainBenchmarkNew, &form) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tplCloudBrainBenchmarkNew, &form) return } defer lock.UnLock() @@ -2517,11 +2517,11 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) command := cloudbrain.GetCloudbrainDebugCommand() lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), jobType, displayJobName)) - isOk := lock.Lock(60 * time.Second) + isOk := lock.Lock(models.CloudbrainKeyDuration) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) cloudBrainNewDataPrepare(ctx) - ctx.RenderWithErr("The task have been processed, please wait a minute", tpl, &form) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tpl, &form) return } defer lock.UnLock() diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index c305889b8..05a89cc49 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -212,11 +212,11 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain image := strings.TrimSpace(form.Image) lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName)) - isOk := lock.Lock(60 * time.Second) + isOk := lock.Lock(models.CloudbrainKeyDuration) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU) - ctx.RenderWithErr("The task have been processed, please wait a minute", tplGrampusTrainJobGPUNew, &form) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tplGrampusTrainJobGPUNew, &form) return } defer lock.UnLock() @@ -414,11 +414,11 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain engineName := form.EngineName lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName)) - isOk := lock.Lock(60 * time.Second) + isOk := lock.Lock(models.CloudbrainKeyDuration) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU) - ctx.RenderWithErr("The task have been processed, please wait a minute", tplGrampusTrainJobNPUNew, &form) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tplGrampusTrainJobNPUNew, &form) return } defer lock.UnLock() diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index daac6b160..f7f464324 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -2,7 +2,6 @@ package repo import ( "archive/zip" - "code.gitea.io/gitea/modules/modelarts_cd" "encoding/json" "errors" "fmt" @@ -16,6 +15,8 @@ import ( "time" "unicode/utf8" + "code.gitea.io/gitea/modules/modelarts_cd" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/base" @@ -154,11 +155,11 @@ func Notebook2Create(ctx *context.Context, form auth.CreateModelArtsNotebookForm repo := ctx.Repo.Repository lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeDebug), displayJobName)) - isOk := lock.Lock(60 * time.Second) + isOk := lock.Lock(models.CloudbrainKeyDuration) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) notebookNewDataPrepare(ctx) - ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsNotebookNew, &form) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tplModelArtsNotebookNew, &form) return } defer lock.UnLock() @@ -1094,11 +1095,11 @@ func TrainJobCreate(ctx *context.Context, form auth.CreateModelArtsTrainJobForm) } lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName)) - isOk := lock.Lock(60 * time.Second) + isOk := lock.Lock(models.CloudbrainKeyDuration) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) trainJobErrorNewDataPrepare(ctx, form) - ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsTrainJobNew, &form) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tplModelArtsTrainJobNew, &form) return } defer lock.UnLock() @@ -1466,11 +1467,11 @@ func TrainJobCreateVersion(ctx *context.Context, form auth.CreateModelArtsTrainJ isLatestVersion := modelarts.IsLatestVersion lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeTrain), displayJobName)) - isOk := lock.Lock(60 * time.Second) + isOk := lock.Lock(models.CloudbrainKeyDuration) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) versionErrorDataPrepare(ctx, form) - ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsTrainJobVersionNew, &form) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tplModelArtsTrainJobVersionNew, &form) return } defer lock.UnLock() @@ -2072,11 +2073,11 @@ func InferenceJobCreate(ctx *context.Context, form auth.CreateModelArtsInference } lock := redis_lock.NewDistributeLock(redis_key.CloudbrainBindingJobNameKey(fmt.Sprint(repo.ID), string(models.JobTypeInference), displayJobName)) - isOk := lock.Lock(60 * time.Second) + isOk := lock.Lock(models.CloudbrainKeyDuration) if !isOk { log.Error("The task have been processed", ctx.Data["MsgID"]) inferenceJobErrorNewDataPrepare(ctx, form) - ctx.RenderWithErr("The task have been processed, please wait a minute", tplModelArtsInferenceJobNew, &form) + ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_samejob_err"), tplModelArtsInferenceJobNew, &form) return } defer lock.UnLock() From 98954ef658aa1b73fd38d8431abc011a79829032 Mon Sep 17 00:00:00 2001 From: liuzx Date: Tue, 13 Sep 2022 11:22:37 +0800 Subject: [PATCH 10/11] fix-bug --- routers/repo/modelarts.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 769f86115..10d1557a2 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -2,8 +2,6 @@ package repo import ( "archive/zip" - "code.gitea.io/gitea/modules/modelarts_cd" - "code.gitea.io/gitea/services/cloudbrain/resource" "encoding/json" "errors" "fmt" @@ -18,6 +16,7 @@ import ( "unicode/utf8" "code.gitea.io/gitea/modules/modelarts_cd" + "code.gitea.io/gitea/services/cloudbrain/resource" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/auth" From 708bd9a075d2782d8176fbdd8b4e12a02f66ce33 Mon Sep 17 00:00:00 2001 From: liuzx Date: Fri, 16 Sep 2022 09:18:46 +0800 Subject: [PATCH 11/11] update --- options/locale/locale_en-US.ini | 2 +- options/locale/locale_zh-CN.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index ac3f5cb9c..8d3868b73 100755 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1093,7 +1093,7 @@ cloudbrain_operate = Operate cloudbrain_status_createtime = Status/Createtime cloudbrain_status_runtime = Running Time cloudbrain_jobname_err=Name must start with a lowercase letter or number,can include lowercase letter,number,_ and -,can not end with _, and can be up to 36 characters long. -cloudbrain_samejob_err=You have created a task with the same name, the system is processing it, please wait a minute. +cloudbrain_samejob_err=A task with the same name has been created, the system is processing it, please wait a minute. cloudbrain_bootfile_err=The bootfile does not exist in the repository cloudbrain_query_fail=Failed to query cloudbrain information. cloudbrain.mirror_tag = Mirror Tag diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index b14882a14..b02600803 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1097,7 +1097,7 @@ cloudbrain_operate=操作 cloudbrain_status_createtime=状态/创建时间 cloudbrain_status_runtime = 运行时长 cloudbrain_jobname_err=只能以小写字母或数字开头且只包含小写字母、数字、_和-，不能以_结尾，最长36个字符。 -cloudbrain_samejob_err=您已经创建了同名任务，系统处理中，请您稍候。 +cloudbrain_samejob_err=同名任务已经被创建，系统处理中，请您稍候。 cloudbrain_bootfile_err=仓库中不存在启动文件 cloudbrain_query_fail=查询云脑任务失败。 cloudbrain.mirror_tag = 镜像标签