From 3408c62c82f357188dfc883a5821adfeedc5a25d Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 2 Nov 2022 15:20:22 +0800 Subject: [PATCH 01/14] fix-3137 --- routers/admin/cloudbrains.go | 24 +++++++++++++----------- routers/user/home.go | 23 ++++++++++++----------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index 5f42c8a51..c4e0e827a 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -70,7 +70,7 @@ func CloudBrains(ctx *context.Context) { keyword := strings.Trim(ctx.Query("q"), " ") - ciTasks, _, err := models.Cloudbrains(&models.CloudbrainsOptions{ + ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, PageSize: setting.UI.IssuePagingNum, @@ -84,6 +84,7 @@ func CloudBrains(ctx *context.Context) { IsLatestVersion: modelarts.IsLatestVersion, ComputeResource: listType, Type: models.TypeCloudBrainAll, + AiCenter: aiCenter, Cluster: cluster, }) if err != nil { @@ -92,24 +93,25 @@ func CloudBrains(ctx *context.Context) { } models.LoadSpecs4CloudbrainInfo(ciTasks) - tasks := []*models.CloudbrainInfo{} - - for _, task := range ciTasks { - if aiCenter == "" || aiCenter == task.Cloudbrain.Spec.AiCenterCode { - task.CanDebug = true - task.CanDel = true - task.Cloudbrain.ComputeResource = task.ComputeResource - tasks = append(tasks, task) + + for i, task := range ciTasks { + ciTasks[i].CanDebug = true + ciTasks[i].CanDel = true + ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource + + if ciTasks[i].Cloudbrain.Type == models.TypeC2Net { + ciTasks[i].Cloudbrain.Spec.Cluster = models.C2NetCluster + } else { + ciTasks[i].Cloudbrain.Spec.Cluster = models.OpenICluster } } - count := int64(len(tasks)) pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, getTotalPage(count, setting.UI.IssuePagingNum)) pager.SetDefaultParams(ctx) pager.AddParam(ctx, "listType", "ListType") ctx.Data["Page"] = pager ctx.Data["PageIsCloudBrain"] = true - ctx.Data["Tasks"] = tasks + ctx.Data["Tasks"] = ciTasks ctx.Data["CanCreate"] = true ctx.Data["Keyword"] = keyword diff --git a/routers/user/home.go b/routers/user/home.go index 345b9d296..1ec6e4e3e 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -812,7 +812,7 @@ func Cloudbrains(ctx *context.Context) { for i, _ := range repos { repoIDList = append(repoIDList, repos[i].ID) } - ciTasks, _, err := models.Cloudbrains(&models.CloudbrainsOptions{ + ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{ ListOptions: models.ListOptions{ Page: page, PageSize: setting.UI.IssuePagingNum, @@ -828,29 +828,30 @@ func Cloudbrains(ctx *context.Context) { RepoIDList: repoIDList, ComputeResource: listType, Type: models.TypeCloudBrainAll, + AiCenter: aiCenter, Cluster: cluster, }) if err != nil { ctx.ServerError("Get job failed:", err) return } - models.LoadSpecs4CloudbrainInfo(ciTasks) - tasks := []*models.CloudbrainInfo{} - for _, task := range ciTasks { - if aiCenter == "" || aiCenter == task.Cloudbrain.Spec.AiCenterCode { - task.CanDebug = true - task.CanDel = true - task.Cloudbrain.ComputeResource = task.ComputeResource - tasks = append(tasks, task) + for i, _ := range ciTasks { + ciTasks[i].CanDebug = true + ciTasks[i].CanDel = true + ciTasks[i].Cloudbrain.ComputeResource = ciTasks[i].ComputeResource + + if ciTasks[i].Cloudbrain.Type == models.TypeC2Net { + ciTasks[i].Cloudbrain.Spec.Cluster = models.C2NetCluster + } else { + ciTasks[i].Cloudbrain.Spec.Cluster = models.OpenICluster } } - count := int64(len(tasks)) pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, getTotalPage(count, setting.UI.IssuePagingNum)) pager.SetDefaultParams(ctx) pager.AddParam(ctx, "listType", "ListType") ctx.Data["Page"] = pager ctx.Data["PageIsUserCloudBrain"] = true - ctx.Data["Tasks"] = tasks + ctx.Data["Tasks"] = ciTasks ctx.Data["CanCreate"] = true ctx.Data["Keyword"] = keyword From d987da9beabccdffcc18bf34d594de3d060fe2ba Mon Sep 17 00:00:00 2001 From: chenshihai Date: Wed, 2 Nov 2022 15:25:59 +0800 Subject: [PATCH 02/14] update aicenter list --- templates/admin/cloudbrain/search.tmpl | 47 +++++++++--------- .../admin/cloudbrain/search_dashboard.tmpl | 49 ++++++++++--------- 2 files changed, 51 insertions(+), 45 deletions(-) diff --git a/templates/admin/cloudbrain/search.tmpl b/templates/admin/cloudbrain/search.tmpl index 5d250db29..12e8a7515 100644 --- a/templates/admin/cloudbrain/search.tmpl +++ b/templates/admin/cloudbrain/search.tmpl @@ -71,33 +71,36 @@ document.addEventListener('DOMContentLoaded', function() { $.ajax({ type: "GET", - url: "/admin/resources/queue/centers", + url: "/api/v1/cloudbrain/get_center_info", dataType: "json", data: {}, success: function (res) { - if (res && res.Code === 0) { - var data = res.Data; - var aiCenterSelEl = $('#aiCenter-sel'); - var itemEl = aiCenterSelEl.find('.menu .item').eq(0); - var selectAiCenterCode = aiCenterSelEl.find('.default').attr('aicenter'); - var selectAiCenterName = ''; - for (var i = 0, iLen = data.length; i < iLen; i++) { - var dataI = data[i]; - var itemClone = itemEl.clone(); - var oHref = itemClone.attr('href'); - var oId = itemClone.attr('id'); - itemClone.attr('data-value', dataI.AiCenterCode); - itemClone.removeAttr('id'); - itemClone.attr('href', oHref + dataI.AiCenterCode); - itemClone.text(dataI.AiCenterName); - aiCenterSelEl.find('.menu').append(itemClone); - if (selectAiCenterCode === dataI.AiCenterCode) { - selectAiCenterName = dataI.AiCenterName; - } + var data = res || []; + var aiCenterSelEl = $('#aiCenter-sel'); + var itemEl = aiCenterSelEl.find('.menu .item').eq(0); + var selectAiCenterCode = aiCenterSelEl.find('.default').attr('aicenter'); + var selectAiCenterName = ''; + var lang = document.querySelector('html').getAttribute('lang') || 'en-US'; + var except = ['', 'more']; + for (var i = 0, iLen = data.length; i < iLen; i++) { + var dataI = data[i]; + var aiCenterCode = dataI.name; + if (except.indexOf(aiCenterCode) >= 0) continue; + var aiCenterName = lang === 'en-US' ? dataI.content_en : dataI.content; + var itemClone = itemEl.clone(); + var oHref = itemClone.attr('href'); + var oId = itemClone.attr('id'); + itemClone.attr('data-value', aiCenterCode); + itemClone.removeAttr('id'); + itemClone.attr('href', oHref + aiCenterCode); + itemClone.text(aiCenterName); + aiCenterSelEl.find('.menu').append(itemClone); + if (selectAiCenterCode === aiCenterCode) { + selectAiCenterName = aiCenterName; } - selectAiCenterCode && aiCenterSelEl.dropdown('set selected', selectAiCenterCode); - selectAiCenterName && aiCenterSelEl.dropdown('set text', selectAiCenterName); } + selectAiCenterCode && aiCenterSelEl.dropdown('set selected', selectAiCenterCode); + selectAiCenterName && aiCenterSelEl.dropdown('set text', selectAiCenterName); }, error: function (err) { console.log(err); diff --git a/templates/admin/cloudbrain/search_dashboard.tmpl b/templates/admin/cloudbrain/search_dashboard.tmpl index d6ec1d5e4..2bf738dc9 100644 --- a/templates/admin/cloudbrain/search_dashboard.tmpl +++ b/templates/admin/cloudbrain/search_dashboard.tmpl @@ -85,33 +85,36 @@ document.addEventListener('DOMContentLoaded', function() { $.ajax({ type: "GET", - url: "/admin/resources/queue/centers", + url: "/api/v1/cloudbrain/get_center_info", dataType: "json", data: {}, success: function (res) { - if (res && res.Code === 0) { - var data = res.Data; - var aiCenterSelEl = $('#aiCenter-sel'); - var itemEl = aiCenterSelEl.find('.menu .item').eq(0); - var selectAiCenterCode = aiCenterSelEl.find('.default').attr('aicenter'); - var selectAiCenterName = ''; - for (var i = 0, iLen = data.length; i < iLen; i++) { - var dataI = data[i]; - var itemClone = itemEl.clone(); - var oHref = itemClone.attr('href'); - var oId = itemClone.attr('id'); - itemClone.attr('data-value', dataI.AiCenterCode); - itemClone.removeAttr('id'); - itemClone.attr('href', oHref + dataI.AiCenterCode); - itemClone.text(dataI.AiCenterName); - aiCenterSelEl.find('.menu').append(itemClone); - if (selectAiCenterCode === dataI.AiCenterCode) { - selectAiCenterName = dataI.AiCenterName; - } + var data = res || []; + var aiCenterSelEl = $('#aiCenter-sel'); + var itemEl = aiCenterSelEl.find('.menu .item').eq(0); + var selectAiCenterCode = aiCenterSelEl.find('.default').attr('aicenter'); + var selectAiCenterName = ''; + var lang = document.querySelector('html').getAttribute('lang') || 'en-US'; + var except = ['', 'more']; + for (var i = 0, iLen = data.length; i < iLen; i++) { + var dataI = data[i]; + var aiCenterCode = dataI.name; + if (except.indexOf(aiCenterCode) >= 0) continue; + var aiCenterName = lang === 'en-US' ? dataI.content_en : dataI.content; + var itemClone = itemEl.clone(); + var oHref = itemClone.attr('href'); + var oId = itemClone.attr('id'); + itemClone.attr('data-value', aiCenterCode); + itemClone.removeAttr('id'); + itemClone.attr('href', oHref + aiCenterCode); + itemClone.text(aiCenterName); + aiCenterSelEl.find('.menu').append(itemClone); + if (selectAiCenterCode === aiCenterCode) { + selectAiCenterName = aiCenterName; } - selectAiCenterCode && aiCenterSelEl.dropdown('set selected', selectAiCenterCode); - selectAiCenterName && aiCenterSelEl.dropdown('set text', selectAiCenterName); } + selectAiCenterCode && aiCenterSelEl.dropdown('set selected', selectAiCenterCode); + selectAiCenterName && aiCenterSelEl.dropdown('set text', selectAiCenterName); }, error: function (err) { console.log(err); @@ -119,4 +122,4 @@ }); }); })(); - \ No newline at end of file + From 6c6be767001f7ef1ba37123b3c0f02bc75bc8eec Mon Sep 17 00:00:00 2001 From: liuzx Date: Wed, 2 Nov 2022 16:14:40 +0800 Subject: [PATCH 03/14] fix-bug --- routers/user/home.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/user/home.go b/routers/user/home.go index 1ec6e4e3e..d9c3b7add 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -835,6 +835,7 @@ func Cloudbrains(ctx *context.Context) { ctx.ServerError("Get job failed:", err) return } + models.LoadSpecs4CloudbrainInfo(ciTasks) for i, _ := range ciTasks { ciTasks[i].CanDebug = true ciTasks[i].CanDel = true From e81d2bcbcf912e5b1facedfbe25e864273ae59ed Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Wed, 2 Nov 2022 17:10:12 +0800 Subject: [PATCH 04/14] add npu --- routers/api/v1/repo/modelarts.go | 2 +- routers/repo/cloudbrain.go | 4 ++-- routers/repo/grampus.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 52d600942..8e311376c 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -182,7 +182,7 @@ func GetModelArtsTrainJobVersion(ctx *context.APIContext) { } if oldStatus != job.Status { notification.NotifyChangeCloudbrainStatus(job, oldStatus) - if models.IsTrainJobTerminal(job.Status) { + if models.IsTrainJobTerminal(job.Status) && job.ComputeResource == models.NPUResource { if len(result.JobInfo.Tasks[0].CenterID) == 1 { urchin.GetBackNpuModel(job.ID, grampus.GetRemoteEndPoint(result.JobInfo.Tasks[0].CenterID[0]), grampus.BucketRemote, grampus.GetNpuModelObjectKey(job.JobName), grampus.GetCenterProxy(setting.Grampus.LocalCenterID)) } diff --git a/routers/repo/cloudbrain.go b/routers/repo/cloudbrain.go index 300e8d426..de0e0e8bf 100755 --- a/routers/repo/cloudbrain.go +++ b/routers/repo/cloudbrain.go @@ -757,7 +757,7 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo if ctx.Written() { return } - if task.Status==string(models.JobWaiting) || task.Status==string(models.JobRunning) { + if task.Status == string(models.JobWaiting) || task.Status == string(models.JobRunning) { result, err := cloudbrain.GetJob(task.JobID) if err != nil { log.Info("error:" + err.Error()) @@ -1944,7 +1944,7 @@ func SyncCloudbrainStatus() { task.CorrectCreateUnix() if oldStatus != task.Status { notification.NotifyChangeCloudbrainStatus(task, oldStatus) - if models.IsTrainJobTerminal(task.Status) { + if models.IsTrainJobTerminal(task.Status) && task.ComputeResource == models.NPUResource { if len(result.JobInfo.Tasks[0].CenterID) == 1 { urchin.GetBackNpuModel(task.ID, grampus.GetRemoteEndPoint(result.JobInfo.Tasks[0].CenterID[0]), grampus.BucketRemote, grampus.GetNpuModelObjectKey(task.JobName), grampus.GetCenterProxy(setting.Grampus.LocalCenterID)) } diff --git a/routers/repo/grampus.go b/routers/repo/grampus.go index e776e1ae9..4718fe04f 100755 --- a/routers/repo/grampus.go +++ b/routers/repo/grampus.go @@ -879,7 +879,7 @@ func GrampusTrainJobShow(ctx *context.Context) { task.CorrectCreateUnix() if oldStatus != task.Status { notification.NotifyChangeCloudbrainStatus(task, oldStatus) - if models.IsTrainJobTerminal(task.Status) { + if models.IsTrainJobTerminal(task.Status) && task.ComputeResource == models.NPUResource { if len(result.JobInfo.Tasks[0].CenterID) == 1 { urchin.GetBackNpuModel(task.ID, grampus.GetRemoteEndPoint(result.JobInfo.Tasks[0].CenterID[0]), grampus.BucketRemote, grampus.GetNpuModelObjectKey(task.JobName), grampus.GetCenterProxy(setting.Grampus.LocalCenterID)) } From 7f8c8fb3ebc2804914f1687be4a61f9960fb4494 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 3 Nov 2022 10:13:03 +0800 Subject: [PATCH 05/14] fix bug --- routers/api/v1/repo/modelarts.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 8e311376c..a1e4e0a65 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -470,7 +470,12 @@ func ModelList(ctx *context.APIContext) { status := models.StorageScheduleSucceed var fileInfos []storage.FileInfo if task.ComputeResource == models.NPUResource { - fileInfos, err = storage.GetObsListObject(task.JobName, "output/", parentDir, versionName) + //fileInfos, err = storage.GetObsListObject(task.JobName, "output/", parentDir, versionName) + prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, "output/", versionName, parentDir), "/") + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + fileInfos, err = storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, "") if err != nil { log.Info("get TrainJobListModel failed:", err) ctx.ServerError("GetObsListObject:", err) @@ -484,6 +489,10 @@ func ModelList(ctx *context.APIContext) { if models.IsTrainJobTerminal(task.Status) { if task.Status == models.GrampusStatusStopped { status = models.StorageNoFile + } else if task.Status == models.GrampusStatusFailed { + if task.AiCenter == "" { + status = models.StorageNoFile + } } else { record, _ := models.GetScheduleRecordByCloudbrainID(task.ID) if record != nil { From 528e6bf4fe1c33a11c2c30000ba0cf842cc8a603 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 3 Nov 2022 10:58:24 +0800 Subject: [PATCH 06/14] debug --- routers/api/v1/repo/modelarts.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index a1e4e0a65..eb787dc0f 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -471,11 +471,8 @@ func ModelList(ctx *context.APIContext) { var fileInfos []storage.FileInfo if task.ComputeResource == models.NPUResource { //fileInfos, err = storage.GetObsListObject(task.JobName, "output/", parentDir, versionName) - prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, "output/", versionName, parentDir), "/") - if !strings.HasSuffix(prefix, "/") { - prefix += "/" - } - fileInfos, err = storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, "") + prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName), "/") + fileInfos, err = storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, parentDir) if err != nil { log.Info("get TrainJobListModel failed:", err) ctx.ServerError("GetObsListObject:", err) From 3ab49a622dae839f32c5d62872c636b297ba280d Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 3 Nov 2022 11:17:25 +0800 Subject: [PATCH 07/14] debug --- modules/storage/obs.go | 66 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 846020c20..fc52209b4 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -360,6 +360,7 @@ func obsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKey return nil } +/* func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relativePath string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = bucket @@ -414,6 +415,71 @@ func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relative } +*/ + +func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relativePath string) ([]FileInfo, error) { + input := &obs.ListObjectsInput{} + input.Bucket = bucket + input.Prefix = prefixRootPath + relativePath + if !strings.HasSuffix(input.Prefix, "/") { + input.Prefix += "/" + } + fileInfos := make([]FileInfo, 0) + prefixLen := len(input.Prefix) + fileMap := make(map[string]bool, 0) + index := 1 + for { + output, err := ObsCli.ListObjects(input) + if err == nil { + log.Info("Page:%d\n", index) + index++ + for _, val := range output.Contents { + log.Info("val key=" + val.Key) + var isDir bool + var fileName string + if val.Key == input.Prefix { + continue + } + fileName = val.Key[prefixLen:] + //log.Info("fileName =" + fileName) + files := strings.Split(fileName, "/") + if fileMap[files[0]] { + continue + } else { + fileMap[files[0]] = true + } + ParenDir := relativePath + fileName = files[0] + if len(files) > 1 { + isDir = true + ParenDir += fileName + "/" + } else { + isDir = false + } + fileInfo := FileInfo{ + ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), + FileName: fileName, + Size: val.Size, + IsDir: isDir, + ParenDir: ParenDir, + } + fileInfos = append(fileInfos, fileInfo) + } + if output.IsTruncated { + input.Marker = output.NextMarker + } else { + break + } + } else { + if obsError, ok := err.(obs.ObsError); ok { + log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) + } + return nil, err + } + } + return fileInfos, nil +} + func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = bucket From c9a131c823b5e56614b400be5a5a1f32152d4c44 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 3 Nov 2022 11:37:19 +0800 Subject: [PATCH 08/14] debug --- modules/storage/obs.go | 1 + routers/api/v1/repo/modelarts.go | 1 + 2 files changed, 2 insertions(+) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index fc52209b4..e90e0e9b7 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -421,6 +421,7 @@ func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relative input := &obs.ListObjectsInput{} input.Bucket = bucket input.Prefix = prefixRootPath + relativePath + log.Info(input.Prefix) if !strings.HasSuffix(input.Prefix, "/") { input.Prefix += "/" } diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index eb787dc0f..e420cf2cd 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -472,6 +472,7 @@ func ModelList(ctx *context.APIContext) { if task.ComputeResource == models.NPUResource { //fileInfos, err = storage.GetObsListObject(task.JobName, "output/", parentDir, versionName) prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName), "/") + log.Info(prefix) fileInfos, err = storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, parentDir) if err != nil { log.Info("get TrainJobListModel failed:", err) From e860bc355265e5b871e8068b68e4bb52ad21fb6d Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 3 Nov 2022 11:41:21 +0800 Subject: [PATCH 09/14] debug --- routers/api/v1/repo/modelarts.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index e420cf2cd..009b0c7a8 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -473,6 +473,9 @@ func ModelList(ctx *context.APIContext) { //fileInfos, err = storage.GetObsListObject(task.JobName, "output/", parentDir, versionName) prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName), "/") log.Info(prefix) + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } fileInfos, err = storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, parentDir) if err != nil { log.Info("get TrainJobListModel failed:", err) From 3de7bc82346130c3d6b9a82f7ad77eed5a73b202 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 3 Nov 2022 11:45:23 +0800 Subject: [PATCH 10/14] debug --- routers/api/v1/repo/modelarts.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 009b0c7a8..489b0fb57 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -476,6 +476,7 @@ func ModelList(ctx *context.APIContext) { if !strings.HasSuffix(prefix, "/") { prefix += "/" } + log.Info(prefix) fileInfos, err = storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, parentDir) if err != nil { log.Info("get TrainJobListModel failed:", err) From 7c2ba62918e36c00b7ee99b41ded450944d9169c Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 3 Nov 2022 11:47:50 +0800 Subject: [PATCH 11/14] show models --- modules/storage/obs.go | 3 --- routers/api/v1/repo/modelarts.go | 3 --- 2 files changed, 6 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index e90e0e9b7..52e05a60d 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -421,7 +421,6 @@ func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relative input := &obs.ListObjectsInput{} input.Bucket = bucket input.Prefix = prefixRootPath + relativePath - log.Info(input.Prefix) if !strings.HasSuffix(input.Prefix, "/") { input.Prefix += "/" } @@ -435,14 +434,12 @@ func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relative log.Info("Page:%d\n", index) index++ for _, val := range output.Contents { - log.Info("val key=" + val.Key) var isDir bool var fileName string if val.Key == input.Prefix { continue } fileName = val.Key[prefixLen:] - //log.Info("fileName =" + fileName) files := strings.Split(fileName, "/") if fileMap[files[0]] { continue diff --git a/routers/api/v1/repo/modelarts.go b/routers/api/v1/repo/modelarts.go index 489b0fb57..5a0e21ed8 100755 --- a/routers/api/v1/repo/modelarts.go +++ b/routers/api/v1/repo/modelarts.go @@ -470,13 +470,10 @@ func ModelList(ctx *context.APIContext) { status := models.StorageScheduleSucceed var fileInfos []storage.FileInfo if task.ComputeResource == models.NPUResource { - //fileInfos, err = storage.GetObsListObject(task.JobName, "output/", parentDir, versionName) prefix := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, task.JobName, setting.OutPutPath, versionName), "/") - log.Info(prefix) if !strings.HasSuffix(prefix, "/") { prefix += "/" } - log.Info(prefix) fileInfos, err = storage.GetOneLevelAllObjectUnderDir(setting.Bucket, prefix, parentDir) if err != nil { log.Info("get TrainJobListModel failed:", err) From 75807fd123025e4b4ae1c45ffe15cd725fe3ee65 Mon Sep 17 00:00:00 2001 From: lewis <747342561@qq.com> Date: Thu, 3 Nov 2022 14:58:17 +0800 Subject: [PATCH 12/14] del unused code --- modules/storage/obs.go | 57 ------------------------------------------ 1 file changed, 57 deletions(-) diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 52e05a60d..57ef63029 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -360,63 +360,6 @@ func obsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKey return nil } -/* -func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relativePath string) ([]FileInfo, error) { - input := &obs.ListObjectsInput{} - input.Bucket = bucket - input.Prefix = prefixRootPath + relativePath - if !strings.HasSuffix(input.Prefix, "/") { - input.Prefix += "/" - } - output, err := ObsCli.ListObjects(input) - fileInfos := make([]FileInfo, 0) - prefixLen := len(input.Prefix) - fileMap := make(map[string]bool, 0) - if err == nil { - for _, val := range output.Contents { - log.Info("val key=" + val.Key) - var isDir bool - var fileName string - if val.Key == input.Prefix { - continue - } - fileName = val.Key[prefixLen:] - log.Info("fileName =" + fileName) - files := strings.Split(fileName, "/") - if fileMap[files[0]] { - continue - } else { - fileMap[files[0]] = true - } - ParenDir := relativePath - fileName = files[0] - if len(files) > 1 { - isDir = true - ParenDir += fileName + "/" - } else { - isDir = false - } - fileInfo := FileInfo{ - ModTime: val.LastModified.Local().Format("2006-01-02 15:04:05"), - FileName: fileName, - Size: val.Size, - IsDir: isDir, - ParenDir: ParenDir, - } - fileInfos = append(fileInfos, fileInfo) - } - return fileInfos, err - } else { - if obsError, ok := err.(obs.ObsError); ok { - log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message) - } - return nil, err - } - -} - -*/ - func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relativePath string) ([]FileInfo, error) { input := &obs.ListObjectsInput{} input.Bucket = bucket From 2c45939e17e5a0d72f90ebe77b2093b363197a7d Mon Sep 17 00:00:00 2001 From: liuzx Date: Thu, 3 Nov 2022 15:43:52 +0800 Subject: [PATCH 13/14] fix-3153 --- routers/admin/cloudbrains.go | 11 ++++++----- routers/user/home.go | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/routers/admin/cloudbrains.go b/routers/admin/cloudbrains.go index c4e0e827a..cbf6782ed 100755 --- a/routers/admin/cloudbrains.go +++ b/routers/admin/cloudbrains.go @@ -98,11 +98,12 @@ func CloudBrains(ctx *context.Context) { ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource - - if ciTasks[i].Cloudbrain.Type == models.TypeC2Net { - ciTasks[i].Cloudbrain.Spec.Cluster = models.C2NetCluster - } else { - ciTasks[i].Cloudbrain.Spec.Cluster = models.OpenICluster + if ciTasks[i].Cloudbrain.Spec != nil { + if ciTasks[i].Cloudbrain.Type == models.TypeC2Net { + ciTasks[i].Cloudbrain.Spec.Cluster = models.C2NetCluster + } else { + ciTasks[i].Cloudbrain.Spec.Cluster = models.OpenICluster + } } } diff --git a/routers/user/home.go b/routers/user/home.go index d9c3b7add..b6ab28f95 100755 --- a/routers/user/home.go +++ b/routers/user/home.go @@ -840,11 +840,12 @@ func Cloudbrains(ctx *context.Context) { ciTasks[i].CanDebug = true ciTasks[i].CanDel = true ciTasks[i].Cloudbrain.ComputeResource = ciTasks[i].ComputeResource - - if ciTasks[i].Cloudbrain.Type == models.TypeC2Net { - ciTasks[i].Cloudbrain.Spec.Cluster = models.C2NetCluster - } else { - ciTasks[i].Cloudbrain.Spec.Cluster = models.OpenICluster + if ciTasks[i].Cloudbrain.Spec != nil { + if ciTasks[i].Cloudbrain.Type == models.TypeC2Net { + ciTasks[i].Cloudbrain.Spec.Cluster = models.C2NetCluster + } else { + ciTasks[i].Cloudbrain.Spec.Cluster = models.OpenICluster + } } } pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, getTotalPage(count, setting.UI.IssuePagingNum)) From 2ed3a2623cab5dccc1cf9bf605972b2aa647d0f5 Mon Sep 17 00:00:00 2001 From: zouap Date: Thu, 3 Nov 2022 16:54:54 +0800 Subject: [PATCH 14/14] =?UTF-8?q?=E5=85=8D=E8=B4=B9=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E6=88=90=E6=99=AE=E6=83=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zouap --- options/locale/locale_zh-CN.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 766bd3b85..63ff87345 100755 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -268,7 +268,7 @@ page_dev_yunlao_desc4=开发者可以根据使用需求，自由选择相应计 page_dev_yunlao_desc5=如果您的模型需要更多的计算资源，也可以单独申请。 page_dev_yunlao_apply=单独申请 c2net_title=中国算力网(C²NET) -c2net_desc=广泛接入全国各地智算中心、超算中心与大数据中心等，为用户提供免费算力资源 +c2net_desc=广泛接入全国各地智算中心、超算中心与大数据中心等，为用户提供普惠算力资源 c2net_center=中心 search=搜索 search_repo=项目 @@ -287,7 +287,7 @@ explore_AI = 探索更好的AI，来这里发现更有意思的 datasets = 数据集 repositories = 项目 use_plt__fuction = 使用本平台提供的AI协作功能，如：托管代码、共享数据、调试算法或训练模型，请先 -provide_resoure = 平台目前免费提供CPU、GPU、NPU的算力资源，可进行多种类型的AI任务。 +provide_resoure = 平台目前提供CPU、GPU、NPU的普惠算力资源，可进行多种类型的AI任务。 create_pro = 创建项目 activity = 活动 no_events = 还没有与您相关的活动