Browse Source

提交代码

tags/v1.22.12.1^2
ychao_1983 3 years ago
parent
commit
6761b77c23
4 changed files with 74 additions and 38 deletions
  1. +3
    -1
      modules/grampus/grampus.go
  2. +27
    -26
      routers/repo/cloudbrain.go
  3. +16
    -11
      routers/repo/modelarts.go
  4. +28
    -0
      services/cloudbrain/cloudbrainTask/sync_status.go

+ 3
- 1
modules/grampus/grampus.go View File

@@ -142,6 +142,7 @@ func GenerateNotebookJob(ctx *context.Context, req *GenerateNotebookJobReq) (job

var datasetGrampus []models.GrampusDataset
var codeGrampus models.GrampusDataset
imageUrl := req.ImageUrl
if ProcessorTypeNPU == req.ProcessType {
datasetGrampus = getDatasetGrampus(req.DatasetInfos)
codeGrampus = models.GrampusDataset{
@@ -150,6 +151,7 @@ func GenerateNotebookJob(ctx *context.Context, req *GenerateNotebookJobReq) (job
EndPoint: getEndPoint(),
ObjectKey: req.CodeStoragePath + cloudbrain.DefaultBranchName + ".zip",
}
imageUrl = ""
} else {
datasetGrampus = getDatasetGPUGrampus(req.DatasetInfos)
codeGrampus = models.GrampusDataset{
@@ -170,7 +172,7 @@ func GenerateNotebookJob(ctx *context.Context, req *GenerateNotebookJobReq) (job
Name: req.JobName,
ResourceSpecId: req.Spec.SourceSpecId,
ImageId: req.ImageId,
ImageUrl: req.ImageUrl,
ImageUrl: imageUrl,
Datasets: datasetGrampus,
Code: codeGrampus,
AutoStopDuration: autoStopDurationMs,


+ 27
- 26
routers/repo/cloudbrain.go View File

@@ -1845,6 +1845,7 @@ func SyncCloudbrainStatus() {
if task.JobType == string(models.JobTypeModelSafety) {
continue
}

if task.Type == models.TypeCloudBrainOne {

task, err = cloudbrainTask.SyncCloudBrainOneStatus(task)
@@ -1853,32 +1854,7 @@ func SyncCloudbrainStatus() {
continue
}

if task.Status != string(models.JobWaiting) {
if task.Duration >= setting.MaxDuration && task.JobType == string(models.JobTypeDebug) {
log.Info("begin to stop job(%s), because of the duration", task.DisplayJobName)
err = cloudbrain.StopJob(task.JobID)
if err != nil {
log.Error("StopJob(%s) failed:%v", task.DisplayJobName, err)
continue
}
oldStatus := task.Status
task.Status = string(models.JobStopped)
if task.EndTime == 0 {
task.EndTime = timeutil.TimeStampNow()
}
task.ComputeAndSetDuration()
if oldStatus != task.Status {
notification.NotifyChangeCloudbrainStatus(task, oldStatus)
}
err = models.UpdateJob(task)
if err != nil {
log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err)
continue
}
}

}
} else if task.Type == models.TypeCloudBrainTwo {
} else if task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeCDCenter {
if task.JobType == string(models.JobTypeDebug) {
err := modelarts.HandleNotebookInfo(task)
if err != nil {
@@ -1942,6 +1918,31 @@ func SyncCloudbrainStatus() {
} else {
log.Error("task.Type(%s) is error:%d", task.JobName, task.Type)
}
if task.Status != string(models.JobWaiting) {
if task.Duration >= setting.MaxDuration && task.JobType == string(models.JobTypeDebug) {
log.Info("begin to stop job(%s), because of the duration", task.DisplayJobName)
err = cloudbrainTask.StopDebugJob(task)
if err != nil {
log.Error("StopJob(%s) failed:%v", task.DisplayJobName, err)
continue
}
oldStatus := task.Status
task.Status = string(models.JobStopped)
if task.EndTime == 0 {
task.EndTime = timeutil.TimeStampNow()
}
task.ComputeAndSetDuration()
if oldStatus != task.Status {
notification.NotifyChangeCloudbrainStatus(task, oldStatus)
}
err = models.UpdateJob(task)
if err != nil {
log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err)
continue
}
}

}
}

return


+ 16
- 11
routers/repo/modelarts.go View File

@@ -543,17 +543,7 @@ func NotebookStop(ctx *context.Context) {
break
}

param := models.NotebookAction{
Action: models.ActionStop,
}

var err error
var res *models.NotebookActionResult
if task.Type == models.TypeCloudBrainTwo {
res, err = modelarts.ManageNotebook2(task.JobID, param)
} else if task.Type == models.TypeCDCenter {
res, err = modelarts_cd.ManageNotebook(task.JobID, param)
}
err, res := StopModelArtsNotebook(task)

if err != nil {
log.Error("ManageNotebook2(%s) failed:%v", task.JobName, err.Error(), ctx.Data["MsgID"])
@@ -594,6 +584,21 @@ func NotebookStop(ctx *context.Context) {
})
}

func StopModelArtsNotebook(task *models.Cloudbrain) (error, *models.NotebookActionResult) {
param := models.NotebookAction{
Action: models.ActionStop,
}

var err error
var res *models.NotebookActionResult
if task.Type == models.TypeCloudBrainTwo {
res, err = modelarts.ManageNotebook2(task.JobID, param)
} else if task.Type == models.TypeCDCenter {
res, err = modelarts_cd.ManageNotebook(task.JobID, param)
}
return err, res
}

func NotebookDel(ctx *context.Context) {
var listType = ctx.Query("debugListType")
task := ctx.Cloudbrain


+ 28
- 0
services/cloudbrain/cloudbrainTask/sync_status.go View File

@@ -3,6 +3,9 @@ package cloudbrainTask
import (
"net/http"

"code.gitea.io/gitea/modules/modelarts"
"code.gitea.io/gitea/modules/modelarts_cd"

"code.gitea.io/gitea/modules/grampus"
"code.gitea.io/gitea/modules/timeutil"

@@ -131,3 +134,28 @@ func isNoteBookReady(task *models.Cloudbrain) bool {
return false

}

func StopDebugJob(task *models.Cloudbrain) error {
param := models.NotebookAction{
Action: models.ActionStop,
}
var err error = nil

if task.JobType == string(models.JobTypeDebug) {
if task.Type == models.TypeCloudBrainOne {
return cloudbrain.StopJob(task.JobID)
} else if task.Type == models.TypeCloudBrainTwo {
_, err = modelarts.ManageNotebook2(task.JobID, param)

} else if task.Type == models.TypeCDCenter {
_, err = modelarts_cd.ManageNotebook(task.JobID, param)

} else if task.Type == models.TypeC2Net {
_, err = grampus.StopJob(task.JobID, task.JobType)

}

}
return err

}

Loading…
Cancel
Save