diff --git a/models/cloudbrain.go b/models/cloudbrain.go index b3320c49d..1e010859b 100755 --- a/models/cloudbrain.go +++ b/models/cloudbrain.go @@ -1858,7 +1858,7 @@ func GetBenchmarkCountByUserID(userID int64) (int, error) { func GetWaitingCloudbrainCount(cloudbrainType int, computeResource string, jobTypes ...JobType) (int64, error) { sess := x.Where("status=? and type=?", JobWaiting, cloudbrainType) if len(jobTypes) > 0 { - sess.In("JobType", jobTypes) + sess.In("job_type", jobTypes) } if computeResource != "" { sess.And("compute_resource=?", computeResource) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 9a4774fa3..3d9452f93 100755 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -925,7 +925,7 @@ func RegisterRoutes(m *macaron.Macaron) { m.Group("/inference-job", func() { m.Group("/:jobid", func() { m.Get("", repo.GetCloudBrainInferenceJob) - m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRight, repo.DelCloudBrainJob) + m.Post("/del", cloudbrain.AdminOrOwnerOrJobCreaterRightForTrain, repo.DelCloudBrainJob) m.Get("/result_list", repo.InferencJobResultList) }) }) diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index c4ed6385a..049e84e6c 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -131,10 +131,12 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error { } ctx.Data["train_gpu_types"] = trainGpuInfos.GpuInfo - if inferenceGpuInfos == nil { + if inferenceGpuInfos == nil && setting.InferenceGpuTypes != "" { json.Unmarshal([]byte(setting.InferenceGpuTypes), &inferenceGpuInfos) } - ctx.Data["inference_gpu_types"] = inferenceGpuInfos.GpuInfo + if inferenceGpuInfos != nil { + ctx.Data["inference_gpu_types"] = inferenceGpuInfos.GpuInfo + } if benchmarkGpuInfos == nil { json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos) @@ -156,10 +158,12 @@ func cloudBrainNewDataPrepare(ctx *context.Context) error { } ctx.Data["train_resource_specs"] = cloudbrain.TrainResourceSpecs.ResourceSpec - if cloudbrain.InferenceResourceSpecs == nil { + if cloudbrain.InferenceResourceSpecs == nil && setting.InferenceResourceSpecs != "" { json.Unmarshal([]byte(setting.InferenceResourceSpecs), &cloudbrain.InferenceResourceSpecs) } - ctx.Data["inference_resource_specs"] = cloudbrain.InferenceResourceSpecs.ResourceSpec + if cloudbrain.InferenceResourceSpecs != nil { + ctx.Data["inference_resource_specs"] = cloudbrain.InferenceResourceSpecs.ResourceSpec + } if cloudbrain.SpecialPools != nil { var debugGpuTypes []*models.GpuInfo @@ -260,6 +264,7 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { } if !jobNamePattern.MatchString(displayJobName) { + cloudBrainNewDataPrepare(ctx) ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form) return } diff --git a/routers/repo/modelarts.go b/routers/repo/modelarts.go index 509df334c..032ff806b 100755 --- a/routers/repo/modelarts.go +++ b/routers/repo/modelarts.go @@ -1238,7 +1238,13 @@ func getUserCommand(engineId int, req *modelarts.GenerateTrainJobReq) (string, s userImageUrl := "" userCommand := "" if engineId < 0 { - userCommand = "/bin/bash /home/work/run_train.sh 's3://" + req.CodeObsPath + "' 'code/" + req.BootFile + "' '/tmp/log/train.log' --'data_url'='s3://" + req.DataUrl + "' --'train_url'='s3://" + req.TrainUrl + "'" + tmpCodeObsPath := strings.Trim(req.CodeObsPath, "/") + tmpCodeObsPaths := strings.Split(tmpCodeObsPath, "/") + lastCodeDir := "code" + if len(tmpCodeObsPaths) > 0 { + lastCodeDir = tmpCodeObsPaths[len(tmpCodeObsPaths)-1] + } + userCommand = "/bin/bash /home/work/run_train.sh 's3://" + req.CodeObsPath + "' '" + lastCodeDir + "/" + req.BootFile + "' '/tmp/log/train.log' --'data_url'='s3://" + req.DataUrl + "' --'train_url'='s3://" + req.TrainUrl + "'" var versionInfos modelarts.VersionInfo if err := json.Unmarshal([]byte(setting.EngineVersions), &versionInfos); err != nil { log.Info("json parse err." + err.Error()) diff --git a/templates/custom/wait_count.tmpl b/templates/custom/wait_count.tmpl index 77c49712d..bef8f1327 100644 --- a/templates/custom/wait_count.tmpl +++ b/templates/custom/wait_count.tmpl @@ -3,13 +3,12 @@ {{$gpuQueue := 0}} {{range $k,$v :=.gpu_types}} {{if eq $k 0}} - {{ $queue := $v.Queue }} + {{ $queue = $v.Queue }} {{ end }} {{ end }} - {{ range $k,$v :=.QueuesDetail }} {{if eq $k $queue}} - {{$gpuQueue :=$v}} + {{$gpuQueue =$v}} {{ end }} {{ end }} {{$.duration}} - + @@ -588,33 +586,4 @@ } document.getElementById("info_display").innerHTML = html; } - function stopBubbling(e) { - e = window.event || e; - if (e.stopPropagation) { - e.stopPropagation(); //阻止事件 冒泡传播 - } else { - e.cancelBubble = true; //ie兼容 - } - } - - function refreshStatus(version_name) { - $(".ui.accordion.border-according").each((index, job) => { - const jobID = job.dataset.jobid; - const repoPath = job.dataset.repopath; - const versionname = job.dataset.version - $.get(`/api/v1/repos/${repoPath}/cloudbrain/${jobID}?version_name=${versionname}`, (data) => { - // header status and duration - //$(`#${version_name}-duration-span`).text(data.JobDuration) - $(`#${version_name}-status-span span`).text(data.JobStatus) - $(`#${version_name}-status-span i`).attr("class", data.JobStatus) - // detail status and duration - //$('#'+version_name+'-duration').text(data.JobDuration) - $('#' + version_name + '-status').text(data.JobStatus) - parseLog() - }).fail(function (err) { - console.log(err); - }); - stopBubbling(arguments.callee.caller.arguments[0]) - }) - } \ No newline at end of file diff --git a/templates/repo/debugjob/index.tmpl b/templates/repo/debugjob/index.tmpl index 14a74ef31..043816e9f 100755 --- a/templates/repo/debugjob/index.tmpl +++ b/templates/repo/debugjob/index.tmpl @@ -386,7 +386,7 @@ {{$.CsrfTokenHtml}} {{if .CanDel}} {{$.i18n.Tr "repo.stop"}} @@ -405,7 +405,7 @@ {{$.CsrfTokenHtml}} {{if .CanDel}} {{$.i18n.Tr "repo.delete"}} diff --git a/templates/repo/grampus/trainjob/show.tmpl b/templates/repo/grampus/trainjob/show.tmpl index 80c59c28a..53201a467 100755 --- a/templates/repo/grampus/trainjob/show.tmpl +++ b/templates/repo/grampus/trainjob/show.tmpl @@ -236,7 +236,7 @@ {{range $k ,$v := .version_list_task}}
@@ -270,9 +270,9 @@ class="cti-mgRight-sm">{{$.i18n.Tr "repo.modelarts.train_job.dura_time"}}： {{.TrainJobDuration}} - + + +
@@ -871,64 +871,7 @@ size = size.toFixed(0);//保留的小数位数 return size + unitArr[index]; } - function refreshStatus(version_name) { - $.get(`/api/v1/repos/${userName}/${repoPath}/grampus/train-job/${jobID}?version_name=${version_name}`, (data) => { - // header status and duration - $(`#${version_name}-duration-span`).text(data.JobDuration) - $(`#${version_name}-status-span span`).text(data.JobStatus) - $(`#${version_name}-status-span i`).attr("class", data.JobStatus) - // detail status and duration - $('#' + version_name + '-duration').text(data.JobDuration) - $('#' + version_name + '-status').text(data.JobStatus) - $('#' + version_name + '-ai_center').text(data.AiCenter) - loadLog(version_name) - - - }).fail(function (err) { - console.log(err); - }); - stopBubbling(arguments.callee.caller.arguments[0]) - } - function deleteVersion(version_name) { - stopBubbling(arguments.callee.caller.arguments[0]) - let flag = 1; - $('.ui.basic.modal').modal({ - onDeny: function () { - flag = false - }, - onApprove: function () { - $.post(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/del_version`, { version_name: version_name }, (data) => { - if (data.VersionListCount === 0) { - location.href = `/${userName}/${repoPath}/modelarts/train-job` - } else { - $('#accordion' + version_name).remove() - } - - }).fail(function (err) { - console.log(err); - }); - flag = true - }, - onHidden: function () { - if (flag == false) { - $('.alert').html('您已取消操作').removeClass('alert-success').addClass('alert-danger').show().delay(1500).fadeOut(); - } - } - }) - .modal('show') - } - function stopVersion(version_name) { - stopBubbling(arguments.callee.caller.arguments[0]) - $.post(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/stop_version`, { version_name: version_name }, (data) => { - if (data.StatusOK === 0) { - $('#' + version_name + '-stop').addClass('disabled') - refreshStatus(version_name) - } - }).fail(function (err) { - console.log(err); - }); - } function loadLog(version_name) { document.getElementById("mask").style.display = "block" $.get(`/api/v1/repos/${userName}/${repoPath}/grampus/train-job/${jobID}/log?version_name=${version_name}&lines=50&order=asc`, (data) => { diff --git a/templates/repo/modelarts/trainjob/show.tmpl b/templates/repo/modelarts/trainjob/show.tmpl index 3936aa620..794e42457 100755 --- a/templates/repo/modelarts/trainjob/show.tmpl +++ b/templates/repo/modelarts/trainjob/show.tmpl @@ -270,12 +270,14 @@ {{end}} {{if .CanDel}} - {{$.i18n.Tr "repo.stop"}} + data-jobid="{{.JobID}}" + data-repopath="{{$.RepoRelPath}}/modelarts/train-job" + data-version = "{{.VersionName}}" + >{{$.i18n.Tr "repo.stop"}} {{else}} - {{$.i18n.Tr "repo.stop"}} + {{$.i18n.Tr "repo.stop"}} {{end}} @@ -308,10 +310,9 @@ class="cti-mgRight-sm">{{$.i18n.Tr "repo.modelarts.train_job.dura_time"}}： {{.TrainJobDuration}} - - + + +
@@ -877,28 +878,6 @@ $('#name').val(modelName) $('#version').val("0.0.1") } - function refreshStatus(version_name) { - $.get(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}?version_name=${version_name}`, (data) => { - // header status and duration - $(`#${version_name}-duration-span`).text(data.JobDuration) - $(`#${version_name}-status-span span`).text(data.JobStatus) - $(`#${version_name}-status-span i`).attr("class", data.JobStatus) - // detail status and duration - $('#' + version_name + '-duration').text(data.JobDuration) - $('#' + version_name + '-status').text(data.JobStatus) - console.log(data) - if (["KILLED", "FAILED", "START_FAILED", "STOPPED", "COMPLETED"].includes(data.JobStatus)) { - $('#' + version_name + '-stop').addClass('disabled') - } - - loadLog(version_name) - - - }).fail(function (err) { - console.log(err); - }); - stopBubbling(arguments.callee.caller.arguments[0]) - } function deleteVersion(version_name) { stopBubbling(arguments.callee.caller.arguments[0]) let flag = 1; @@ -928,17 +907,7 @@ .modal('show') } - function stopVersion(version_name) { - stopBubbling(arguments.callee.caller.arguments[0]) - $.post(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/stop_version`, { version_name: version_name }, (data) => { - if (data.StatusOK === 0) { - $('#' + version_name + '-stop').addClass('disabled') - refreshStatus(version_name) - } - }).fail(function (err) { - console.log(err); - }); - } + function loadLog(version_name) { $.get(`/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/log?version_name=${version_name}&lines=50&order=asc`, (data) => { $('input[name=end_line]').val(data.EndLine) diff --git a/web_src/js/features/cloudrbanin.js b/web_src/js/features/cloudrbanin.js index 15cb47e97..b7a9e4eef 100644 --- a/web_src/js/features/cloudrbanin.js +++ b/web_src/js/features/cloudrbanin.js @@ -123,6 +123,7 @@ export default async function initCloudrain() { "KILLED", "COMPLETED", "SUCCEEDED", + "CREATE_FAILED" ].includes(status) ) { $("#ai-delete-" + ID) @@ -166,6 +167,7 @@ export default async function initCloudrain() { "START_FAILED", "SUCCEEDED", "STOPPED", + "CREATE_FAILED", ].includes(status) ) { return; @@ -177,6 +179,7 @@ export default async function initCloudrain() { "KILLING", "COMPLETED", "SUCCEEDED", + "CREATE_FAILED", "STOPPED", ]; $.get( @@ -327,6 +330,23 @@ export default async function initCloudrain() { }); } + $(".stop-show-version").click(function (e) { + const ID = this.dataset.jobid; + const repoPath = this.dataset.repopath; + const version_name = this.dataset.version; + const url = `/api/v1/repos/${repoPath}/${ID}/stop_version`; + $.post(url, { version_name: version_name }, (data) => { + if (data.StatusOK === 0) { + $(`#${version_name}-stop`).removeClass("blue"); + $(`#${version_name}-stop`).addClass("disabled"); + refreshStatusShow(version_name, ID, repoPath); + } + }).fail(function (err) { + console.log(err); + }); + e.stopPropagation(); + }); + $("#refresh-status").click(function (e) { let version_name = $(this).data("version"); let ID = $(`#accordion${version_name}`).data("jobid"); @@ -339,11 +359,16 @@ export default async function initCloudrain() { $.get( `/api/v1/repos/${repoPath}/${ID}?version_name=${version_name}`, (data) => { + //accroding下的状态 $(`#${version_name}-status-span span`).text(data.JobStatus); + //accroding下的状态图标 $(`#${version_name}-status-span i`).attr("class", data.JobStatus); + //accroding下的运行时长 $(`#${version_name}-duration-span`).text(data.JobDuration); - $("#" + versionname + "-duration").text(data.JobDuration); - $("#" + versionname + "-status").text(data.JobStatus); + //配置信息详情页的状态 + $(`#${version_name}-status`).text(data.JobStatus); + //配置信息详情页的状态 + $(`#${version_name}-duration`).text(data.JobDuration); } ).fail(function (err) { console.log(err); @@ -363,6 +388,7 @@ export default async function initCloudrain() { "KILLED", "COMPLETED", "SUCCEEDED", + "CREATE_FAILED", ].includes(data.JobStatus) ) { $("#ai-delete-" + ID) @@ -458,10 +484,15 @@ export default async function initCloudrain() { }); function setWaitNums() { - console.log($(".cloudbrain-type")); if ($(".cloudbrain-type").length === 0 && $(".gpu-type").length === 0) { return; } + if ( + $(".cloudbrain-type").length !== 0 && + !$(".cloudbrain-type").data("queue") + ) { + return; + } let waitNums = $(".cloudbrain-type").data("queue").split("map")[1]; let test = new Map(); let waitNumsArray = waitNums.split(" ");