diff --git a/modules/modelarts/modelarts.go b/modules/modelarts/modelarts.go index 81a77f8b1..e5e3d4eec 100755 --- a/modules/modelarts/modelarts.go +++ b/modules/modelarts/modelarts.go @@ -963,13 +963,30 @@ func handleTempNotebook(temp *models.CloudbrainTemp) error { } if isExist { - log.Info("find the record(%s)", temp.JobName) - _, err := ManageNotebook2(temp.JobID, models.NotebookAction{Action: models.ActionStop}) - if err != nil { - log.Error("ManageNotebook2(%s) failed:%v", temp.JobName, err) - break + log.Info("find the record(%s), status(%s)", temp.JobName, temp.Status) + if temp.Status == string(models.ModelArtsCreateFailed) { + err = models.UpdateCloudbrainTemp(temp) + if err != nil { + log.Error("UpdateCloudbrainTemp failed:%v", err) + break + } + + _, err := DelNotebook2(temp.JobID) + if err != nil { + log.Error("DelNotebook2 failed:%v", err) + break + } + + temp.Status = string(models.ModelArtsDeleted) + } else { + _, err := ManageNotebook2(temp.JobID, models.NotebookAction{Action: models.ActionStop}) + if err != nil { + log.Error("ManageNotebook2(%s) failed:%v", temp.JobName, err) + break + } + temp.Status = string(models.ModelArtsStopping) } - temp.Status = string(models.ModelArtsStopping) + models.UpdateCloudbrainTemp(temp) } else { log.Error("can not find the record(%s) till now", temp.JobName) @@ -1100,12 +1117,12 @@ func handleTempTrainJobMultiVersion(temp *models.CloudbrainTemp) error { if result != nil { count, _ := models.GetCloudbrainCountByJobName(temp.JobName, temp.JobType, temp.Type) if result.VersionCount == int64(count+1) { - log.Info("find the record(%s)", temp.JobName) - isExist = true temp.Status = TransTrainJobStatus(result.JobVersionList[0].IntStatus) temp.VersionID = strconv.FormatInt(result.JobVersionList[0].VersionID, 10) + log.Info("find the record(%s), status(%s)", temp.JobName, temp.Status) + _, err := StopTrainJob(temp.JobID, temp.VersionID) if err != nil { log.Error("StopTrainJob failed:%v", err) @@ -1161,13 +1178,13 @@ func handleTempTrainJob(temp *models.CloudbrainTemp) error { if result != nil { for _, job := range result.JobList { if temp.JobName == job.JobName && TransTrainJobStatus(job.IntStatus) != string(models.ModelArtsTrainJobFailed) { - log.Info("find the record(%s)", temp.JobName) - isExist = true temp.Status = TransTrainJobStatus(job.IntStatus) temp.JobID = strconv.FormatInt(job.JobID, 10) temp.VersionID = strconv.FormatInt(job.VersionID, 10) + log.Info("find the record(%s), status(%s)", temp.JobName, temp.Status) + _, err = StopTrainJob(temp.JobID, temp.VersionID) if err != nil { log.Error("StopTrainJob(%s) failed:%v", temp.JobName, err) diff --git a/modules/setting/setting.go b/modules/setting/setting.go index e9251edae..a04eb4e72 100755 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -1419,7 +1419,7 @@ func NewContext() { Flavor = sec.Key("FLAVOR").MustString("") ImageInfos = sec.Key("IMAGE_INFOS").MustString("") Capacity = sec.Key("IMAGE_INFOS").MustInt(100) - MaxTempQueryTimes = sec.Key("MAX_TEMP_QUERY_TIMES").MustInt(10) + MaxTempQueryTimes = sec.Key("MAX_TEMP_QUERY_TIMES").MustInt(30) ResourcePools = sec.Key("Resource_Pools").MustString("") Engines = sec.Key("Engines").MustString("") EngineVersions = sec.Key("Engine_Versions").MustString("")