Browse Source

Merge branch 'fix-2208' of https://git.openi.org.cn/OpenI/aiforge into fix-2208

tags/v1.22.6.1^2
liuzx 3 years ago
parent
commit
59d212204a
20 changed files with 1809 additions and 50 deletions
  1. +87
    -0
      models/cloudbrain.go
  2. +270
    -1
      models/cloudbrain_static.go
  3. +15
    -0
      models/dataset.go
  4. +2
    -1
      modules/cloudbrain/cloudbrain.go
  5. +1
    -1
      modules/modelarts/resty.go
  6. +1
    -0
      options/locale/locale_en-US.ini
  7. +8
    -3
      options/locale/locale_zh-CN.ini
  8. +9
    -0
      routers/api/v1/api.go
  9. +1246
    -21
      routers/api/v1/repo/cloudbrain_dashboard.go
  10. +11
    -7
      routers/api/v1/repo/repo_dashbord.go
  11. +7
    -0
      routers/home.go
  12. +9
    -1
      routers/repo/ai_model_manage.go
  13. +18
    -8
      routers/repo/cloudbrain.go
  14. +2
    -0
      routers/routes/routes.go
  15. +97
    -0
      routers/user/auth.go
  16. +1
    -1
      templates/explore/datasets.tmpl
  17. +5
    -0
      templates/repo/modelmanage/index.tmpl
  18. +14
    -3
      templates/repo/modelmanage/showinfo.tmpl
  19. +1
    -1
      web_src/js/components/Model.vue
  20. +5
    -2
      web_src/js/index.js

+ 87
- 0
models/cloudbrain.go View File

@@ -24,6 +24,7 @@ type ModelArtsJobStatus string
const (
TypeCloudBrainOne int = iota
TypeCloudBrainTwo
TypeIntelligentNet

TypeCloudBrainAll = -1
)
@@ -328,6 +329,11 @@ type CloudbrainsOptions struct {
JobTypeNot bool
NeedRepoInfo bool
RepoIDList []int64
BeginTime time.Time
EndTime time.Time
ComputeResource string
BeginTimeUnix int64
EndTimeUnix int64
}

type TaskPod struct {
@@ -1183,6 +1189,11 @@ func Cloudbrains(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) {
builder.Eq{"cloudbrain.job_id": opts.JobID},
)
}
if (opts.ComputeResource) != "" {
cond = cond.And(
builder.Eq{"cloudbrain.compute_resource": opts.ComputeResource},
)
}

if (opts.Type) >= 0 {
cond = cond.And(
@@ -1504,6 +1515,11 @@ func UpdateJob(job *Cloudbrain) error {
return updateJob(x, job)
}

func UpdateJobDurationWithDeleted(job *Cloudbrain) error {
_, err := x.Exec("update cloudbrain set start_time=?, end_time=?,train_job_duration=?,duration=? where id=?", job.StartTime, job.EndTime, job.TrainJobDuration, job.Duration, job.ID)
return err
}

func updateJob(e Engine, job *Cloudbrain) error {
_, err := e.ID(job.ID).AllCols().Update(job)
return err
@@ -1574,6 +1590,10 @@ func GetStoppedJobWithNoDurationJob() ([]*Cloudbrain, error) {
Limit(100).
Find(&cloudbrains)
}
func GetStoppedJobWithNoStartTimeEndTime() ([]*Cloudbrain, error) {
cloudbrains := make([]*Cloudbrain, 0)
return cloudbrains, x.SQL("select * from cloudbrain where status in (?,?,?,?,?,?,?) and (start_time is null or end_time is null) limit 100", ModelArtsTrainJobCompleted, ModelArtsTrainJobFailed, ModelArtsTrainJobKilled, ModelArtsStopped, JobStopped, JobFailed, JobSucceeded).Find(&cloudbrains)
}

func GetCloudbrainCountByUserID(userID int64, jobType string) (int, error) {
count, err := x.In("status", JobWaiting, JobRunning).And("job_type = ? and user_id = ? and type = ?", jobType, userID, TypeCloudBrainOne).Count(new(Cloudbrain))
@@ -1650,13 +1670,80 @@ func RestartCloudbrain(old *Cloudbrain, new *Cloudbrain) (err error) {
func CloudbrainAll(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, error) {
sess := x.NewSession()
defer sess.Close()

var cond = builder.NewCond()
if opts.RepoID > 0 {
cond = cond.And(
builder.Eq{"cloudbrain.repo_id": opts.RepoID},
)
}

if opts.UserID > 0 {
cond = cond.And(
builder.Eq{"cloudbrain.user_id": opts.UserID},
)
}

if (opts.JobID) != "" {
cond = cond.And(
builder.Eq{"cloudbrain.job_id": opts.JobID},
)
}
if (opts.ComputeResource) != "" {
cond = cond.And(
builder.Eq{"cloudbrain.compute_resource": opts.ComputeResource},
)
}

if (opts.Type) >= 0 {
cond = cond.And(
builder.Eq{"cloudbrain.type": opts.Type},
)
}

if len(opts.JobTypes) > 0 {
if opts.JobTypeNot {
cond = cond.And(
builder.NotIn("cloudbrain.job_type", opts.JobTypes),
)
} else {
cond = cond.And(
builder.In("cloudbrain.job_type", opts.JobTypes),
)
}
}

if (opts.IsLatestVersion) != "" {
cond = cond.And(builder.Or(builder.And(builder.Eq{"cloudbrain.is_latest_version": opts.IsLatestVersion}, builder.Eq{"cloudbrain.job_type": "TRAIN"}), builder.Neq{"cloudbrain.job_type": "TRAIN"}))
}

if len(opts.CloudbrainIDs) > 0 {
cond = cond.And(builder.In("cloudbrain.id", opts.CloudbrainIDs))
}

if len(opts.JobStatus) > 0 {
if opts.JobStatusNot {
cond = cond.And(
builder.NotIn("cloudbrain.status", opts.JobStatus),
)
} else {
cond = cond.And(
builder.In("cloudbrain.status", opts.JobStatus),
)
}
}
if len(opts.RepoIDList) > 0 {
cond = cond.And(
builder.In("cloudbrain.repo_id", opts.RepoIDList),
)

}
if opts.BeginTimeUnix > 0 && opts.EndTimeUnix > 0 {
cond = cond.And(
builder.And(builder.Gte{"cloudbrain.created_unix": opts.BeginTimeUnix}, builder.Lte{"cloudbrain.created_unix": opts.EndTimeUnix}),
)
}

var count int64
var err error
condition := "cloudbrain.user_id = `user`.id"


+ 270
- 1
models/cloudbrain_static.go View File

@@ -1,6 +1,191 @@
package models

import "code.gitea.io/gitea/modules/log"
import (
"strconv"
"time"

"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
"xorm.io/builder"
)

type TaskDetail struct {
ID int64 `json:"ID"`
JobID string `json:"JobID"`
JobName string `json:"JobName"`
DisplayJobName string `json:"DisplayJobName"`
Status string `json:"Status"`
JobType string `json:"JobType"`
CreatedUnix timeutil.TimeStamp `json:"CreatedUnix"`
WaitTime string `json:"WaitTime"`
RunTime string `json:"RunTime"`
StartTime timeutil.TimeStamp `json:"StartTime"`
EndTime timeutil.TimeStamp `json:"EndTime"`
ComputeResource string `json:"ComputeResource"`
Type int `json:"Type"`
UserName string `json:"UserName"`
RepoName string `json:"RepoName"`
RepoAlias string `json:"RepoAlias"`
RepoID int64 `json:"RepoID"`
IsDelete bool `json:"IsDelete"`
}

func GetDebugOnePeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
" and job_type ='" + string(JobTypeDebug) + "'" +
" and type='" + strconv.Itoa(TypeCloudBrainOne) + "'"

return x.SQL(countSql).Count()
}
func GetDebugOnePeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) {
total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeDebug, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration")
if err != nil {
return 0, err
}

return total, nil
}

func GetTrainOnePeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
" and job_type ='" + string(JobTypeTrain) + "'" +
" and type='" + strconv.Itoa(TypeCloudBrainOne) + "'"

return x.SQL(countSql).Count()
}
func GetTrainOnePeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) {
total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeTrain, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration")
if err != nil {
return 0, err
}

return total, nil
}

func GetBenchmarkOnePeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
" and job_type ='" + string(JobTypeBenchmark) + "'" +
" and type='" + strconv.Itoa(TypeCloudBrainOne) + "'"
return x.SQL(countSql).Count()
}
func GetBenchmarkOnePeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) {
total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeBenchmark, TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration")
if err != nil {
return 0, err
}

return total, nil
}
func GetDebugTwoPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
" and job_type ='" + string(JobTypeDebug) + "'" +
" and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'"
return x.SQL(countSql).Count()
}
func GetDebugTwoPeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) {
total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeDebug, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration")
if err != nil {
return 0, err
}
return total, nil
}
func GetTrainTwoPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
" and job_type ='" + string(JobTypeTrain) + "'" +
" and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'"
return x.SQL(countSql).Count()
}
func GetTrainTwoPeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) {
total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeTrain, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration")
if err != nil {
return 0, err
}
return total, nil
}
func GetInferenceTwoPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
" and job_type ='" + string(JobTypeInference) + "'" +
" and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'"
return x.SQL(countSql).Count()
}
func GetInferenceTwoPeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) {
total, err := x.Where("created_unix >= ? And created_unix < ? And job_type = ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), JobTypeInference, TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration")
if err != nil {
return 0, err
}
return total, nil
}

func GetCloudBrainOnePeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
" and type='" + strconv.Itoa(TypeCloudBrainOne) + "'"
return x.SQL(countSql).Count()
}
func GetCloudBrainOnePeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) {
total, err := x.Where("created_unix >= ? And created_unix < ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), TypeCloudBrainOne).SumInt(&Cloudbrain{}, "duration")
if err != nil {
return 0, err
}
return total, nil
}
func GetCloudBrainTwoPeriodCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10) +
" and type='" + strconv.Itoa(TypeCloudBrainTwo) + "'"
return x.SQL(countSql).Count()
}
func GetCloudBrainTwoPeriodDuration(beginTime time.Time, endTime time.Time) (int64, error) {
total, err := x.Where("created_unix >= ? And created_unix < ? And type = ? ", strconv.FormatInt(beginTime.Unix(), 10), strconv.FormatInt(endTime.Unix(), 10), TypeCloudBrainTwo).SumInt(&Cloudbrain{}, "duration")
if err != nil {
return 0, err
}
return total, nil
}

func GetTodayCreatorCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count(distinct user_id) FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<" + strconv.FormatInt(endTime.Unix(), 10)
return x.SQL(countSql).Count()
}
func GetTodayCloudbrainCount(beginTime time.Time, endTime time.Time) (int64, error) {
countSql := "SELECT count FROM " +
"public.cloudbrain where created_unix >=" + strconv.FormatInt(beginTime.Unix(), 10) +
" and created_unix<=" + strconv.FormatInt(endTime.Unix(), 10)
return x.SQL(countSql).Count()
}
func GetCreatorCount() (int64, error) {
countSql := "SELECT count(distinct user_id) FROM public.cloudbrain"
return x.SQL(countSql).Count()
}

func GetRecordBeginTime() ([]*CloudbrainInfo, error) {
sess := x.NewSession()
defer sess.Close()
sess.OrderBy("cloudbrain.id ASC limit 1")
cloudbrains := make([]*CloudbrainInfo, 0)
if err := sess.Table(&Cloudbrain{}).Unscoped().
Find(&cloudbrains); err != nil {
log.Info("find error.")
}
return cloudbrains, nil
}

func GetAllStatusCloudBrain() map[string]int {
sess := x.NewSession()
@@ -20,3 +205,87 @@ func GetAllStatusCloudBrain() map[string]int {
}
return cloudBrainStatusResult
}

func GetWaittingTop() ([]*CloudbrainInfo, error) {
sess := x.NewSession()
defer sess.Close()
var cond = builder.NewCond()
cond = cond.And(
builder.Eq{"cloudbrain.status": string(JobWaiting)},
)
sess.OrderBy("(cloudbrain.start_time-cloudbrain.created_unix) DESC limit 10")
cloudbrains := make([]*CloudbrainInfo, 0, 10)
if err := sess.Table(&Cloudbrain{}).Where(cond).
Find(&cloudbrains); err != nil {
log.Info("find error.")
}
return cloudbrains, nil
}
func GetRunningTop() ([]*CloudbrainInfo, error) {
sess := x.NewSession()
defer sess.Close()
var cond = builder.NewCond()
cond = cond.And(
builder.Eq{"cloudbrain.status": string(JobRunning)},
)
sess.OrderBy("(cloudbrain.end_time-cloudbrain.start_time) DESC limit 10")
cloudbrains := make([]*CloudbrainInfo, 0, 10)
if err := sess.Table(&Cloudbrain{}).Where(cond).
Find(&cloudbrains); err != nil {
log.Info("find error.")
}
return cloudbrains, nil
}

func getCreatePeriodCount(dateBeginTime string, dateEndTime string, hourBeginTime string, hourEndTime string) (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where to_char(to_timestamp(created_unix), 'YYYY-MM-DD') >= '" + dateBeginTime +
"' and to_char(to_timestamp(created_unix), 'YYYY-MM-DD') < '" + dateEndTime +
"' and to_char(to_timestamp(created_unix), 'HH24:MI:SS') >= '" + hourBeginTime +
"' and to_char(to_timestamp(created_unix), 'HH24:MI:SS') < '" + hourEndTime + "'"
return x.SQL(countSql).Count()
}

//SELECT * FROM xxx WHERE NOT ((endTime < hourBeginTime) OR (startTime > hourEndTime))
func getRunPeriodCount(dateBeginTime string, dateEndTime string, hourBeginTime string, hourEndTime string) (int64, error) {
countSql := "SELECT count(*) FROM " +
"public.cloudbrain where not ((to_char(to_timestamp(start_time), ' HH24:MI:SS') > '" + hourEndTime +
"') or (to_char(to_timestamp(end_time), 'HH24:MI:SS') < '" + hourBeginTime + "'))" +
" and (to_char(to_timestamp(start_time), 'YYYY-MM-DD') >= '" + dateBeginTime +
"' and to_char(to_timestamp(start_time), 'YYYY-MM-DD') < '" + dateEndTime + "')"
return x.SQL(countSql).Count()

}

func GetCreateHourPeriodCount(dateBeginTime string, dateEndTime string) (map[string]interface{}, error) {
//0 to 23 for each hour,
dateHourMap := make(map[string]interface{})
var slice = []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23}
for key, value := range slice {
hourBeginHour := util.AddZero(value) + ":00:00"
hourEndHour := util.AddZero(value+1) + ":00:00"
cout, err := getCreatePeriodCount(dateBeginTime, dateEndTime, hourBeginHour, hourEndHour)
if err != nil {
log.Error("Can not query getCreatePeriodCount.", err)
return nil, nil
}
dateHourMap[strconv.Itoa(key)] = cout
}
return dateHourMap, nil
}

func GetRunHourPeriodCount(dateBeginTime string, dateEndTime string) (map[string]interface{}, error) {
dateHourMap := make(map[string]interface{})
var slice = []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23}
for key, value := range slice {
hourBeginHour := util.AddZero(value) + ":00:00"
hourEndHour := util.AddZero(value+1) + ":00:00"
cout, err := getRunPeriodCount(dateBeginTime, dateEndTime, hourBeginHour, hourEndHour)
if err != nil {
log.Error("Can not query getRunPeriodCount.", err)
return nil, nil
}
dateHourMap[strconv.Itoa(key)] = cout
}
return dateHourMap, nil
}

+ 15
- 0
models/dataset.go View File

@@ -107,6 +107,7 @@ type SearchDatasetOptions struct {
Category string
Task string
License string
DatasetIDs []int64
ListOptions
SearchOrderBy
IsOwner bool
@@ -177,6 +178,12 @@ func SearchDatasetCondition(opts *SearchDatasetOptions) builder.Cond {
}
}

if len(opts.DatasetIDs) > 0 {
subCon := builder.NewCond()
subCon = subCon.And(builder.In("dataset.id", opts.DatasetIDs))
cond = cond.Or(subCon)
}

return cond
}

@@ -447,3 +454,11 @@ func IncreaseDownloadCount(datasetID int64) error {

return nil
}

func GetCollaboratorDatasetIdsByUserID(userID int64) []int64 {
var datasets []int64
_ = x.Table("dataset").Join("INNER", "collaboration", "dataset.repo_id = collaboration.repo_id and collaboration.mode>0 and collaboration.user_id=?", userID).
Cols("dataset.id").Find(&datasets)
return datasets

}

+ 2
- 1
modules/cloudbrain/cloudbrain.go View File

@@ -187,7 +187,7 @@ func AdminOrImageCreaterRight(ctx *context.Context) {

}

func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, description, branchName, bootFile, params string, benchmarkTypeID, benchmarkChildTypeID, resourceSpecId int) error {
func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command, uuid, codePath, modelPath, benchmarkPath, snn4imagenetPath, brainScorePath, jobType, gpuQueue, description, branchName, bootFile, params, commitID string, benchmarkTypeID, benchmarkChildTypeID, resourceSpecId int) error {

dataActualPath := setting.Attachment.Minio.RealPath +
setting.Attachment.Minio.Bucket + "/" +
@@ -336,6 +336,7 @@ func GenerateTask(ctx *context.Context, displayJobName, jobName, image, command,
Parameters: params,
CreatedUnix: createTime,
UpdatedUnix: createTime,
CommitID: commitID,
})

if err != nil {


+ 1
- 1
modules/modelarts/resty.go View File

@@ -1131,7 +1131,7 @@ sendjob:
res, err := client.R().
SetAuthToken(TOKEN).
SetResult(&result).
Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/pod/" + podName + "/metric-statistic")
Get(HOST + "/v1/" + setting.ProjectID + urlTrainJob + "/" + jobID + "/versions/" + versionID + "/pod/" + podName + "/metric-statistic?statistic_type=each")

if err != nil {
return nil, fmt.Errorf("resty GetTrainJobMetricStatistic: %v", err)


+ 1
- 0
options/locale/locale_en-US.ini View File

@@ -1092,6 +1092,7 @@ modelarts.train_job.fast_parameter_setting_config_link=fast_parameter_setting_co
modelarts.train_job.frames=frames
modelarts.train_job.algorithm_origin=Algorithm Origin
modelarts.train_job.AI_driver=AI Engine
modelarts.train_job.AI_Engine=AI Engine
modelarts.train_job.start_file=Start File
modelarts.train_job.boot_file_helper=The startup file is the entry file that your program executes, and it must be a file ending in .py
modelarts.train_job.dataset=Dataset


+ 8
- 3
options/locale/locale_zh-CN.ini View File

@@ -1020,13 +1020,16 @@ balance.total_view=余额总览
balance.available=可用余额:
cloudbrain1=云脑1
cloudbrain2=云脑2
intelligent_net=智算网络
cloudbrain_selection=云脑选择
cloudbrain_platform_selection=选择您准备使用的云脑平台:
confirm_choice=确定
cloudbran1_tips=只有zip格式的数据集才能发起云脑任务
cloudbrain_creator=创建者
cloudbrain_type=支撑算力
cloudbrain_untype=未知支撑算力
cloudbrain_task=任务名称
cloudbrain_task_type=任务类型
cloudbrain_task_type=云脑任务类型
cloudbrain_task_name=云脑侧任务名称
cloudbrain_operate=操作
cloudbrain_status_createtime=状态/创建时间
@@ -1060,6 +1063,7 @@ computing.success=加入成功

modelarts.status=状态
modelarts.createtime=创建时间
modelarts.deletetime=删除时间
modelarts.version_nums=版本数
modelarts.version=版本
modelarts.computing_resources=计算资源
@@ -1090,8 +1094,8 @@ modelarts.train_job.job_status=任务状态
modelarts.train_job.job_name=任务名称
modelarts.train_job.version=任务版本
modelarts.train_job.start_time=开始运行时间
modelarts.train_job.end_time=运行结束时间
modelarts.train_job.wait_time=等待时
modelarts.train_job.end_time=结束运行时间
modelarts.train_job.wait_time=等待时
modelarts.train_job.dura_time=运行时长
modelarts.train_job.description=任务描述
modelarts.train_job.parameter_setting=参数设置
@@ -1102,6 +1106,7 @@ modelarts.train_job.fast_parameter_setting_config_link=这里
modelarts.train_job.frames=常用框架
modelarts.train_job.algorithm_origin=算法来源
modelarts.train_job.AI_driver=AI引擎
modelarts.train_job.AI_Engine=模型框架
modelarts.train_job.start_file=启动文件
modelarts.train_job.boot_file_helper=启动文件是您程序执行的入口文件,必须是以.py结尾的文件。比如train.py、main.py、example/train.py、case/main.py。
modelarts.train_job.boot_file_place=填写启动文件路径,默认为train.py


+ 9
- 0
routers/api/v1/api.go View File

@@ -574,9 +574,18 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Group("/cloudbrainboard", func() {
m.Get("/downloadAll", repo.DownloadCloudBrainBoard)
m.Group("/cloudbrain", func() {
m.Get("/overview", repo.GetAllCloudbrainsOverview)
m.Get("/distribution", repo.GetAllCloudbrainsPeriodDistribution)
m.Get("/trend", repo.GetAllCloudbrainsTrend)
m.Get("/trend_detail_data", repo.GetAllCloudbrainsTrendDetail)
m.Get("/status_analysis", repo.GetCloudbrainsStatusAnalysis)
m.Get("/detail_data", repo.GetCloudbrainsDetailData)
m.Get("/hours_data", repo.GetCloudbrainsCreateHoursData)
m.Get("/waitting_top_data", repo.GetWaittingTop)
m.Get("/running_top_data", repo.GetRunningTop)
})
}, operationReq)

// Users
m.Group("/users", func() {
m.Get("/search", user.Search)


+ 1246
- 21
routers/api/v1/repo/cloudbrain_dashboard.go
File diff suppressed because it is too large
View File


+ 11
- 7
routers/api/v1/repo/repo_dashbord.go View File

@@ -192,7 +192,6 @@ func GetProjectsSummaryData(ctx *context.Context) {

}


projectSummaryPeriodData := ProjectSummaryPeriodData{
TotalCount: count - 1,
RecordBeginTime: recordBeginTime.Format(DATE_FORMAT),
@@ -203,7 +202,7 @@ func GetProjectsSummaryData(ctx *context.Context) {

}

func reverse(datas []*ProjectSummaryBaseData ) []*ProjectSummaryBaseData {
func reverse(datas []*ProjectSummaryBaseData) []*ProjectSummaryBaseData {
for i := 0; i < len(datas)/2; i++ {
j := len(datas) - i - 1
datas[i], datas[j] = datas[j], datas[i]
@@ -211,8 +210,6 @@ func reverse(datas []*ProjectSummaryBaseData ) []*ProjectSummaryBaseData {
return datas
}



func setStatisticsData(data *ProjectSummaryBaseData, v *models.SummaryStatistic, stats *models.SummaryStatistic) {
data.NumReposAdd = v.NumRepos - stats.NumRepos
data.NumRepoPublicAdd = v.NumRepoPublic - stats.NumRepoPublic
@@ -890,12 +887,19 @@ func getTimePeroid(ctx *context.Context, recordBeginTime time.Time) (time.Time,
if queryType == "all" {
beginTime = recordBeginTimeTemp
endTime = now
} else if queryType == "yesterday" {
} else if queryType == "today" {
endTime = now
beginTime = time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, now.Location())
} else if queryType == "yesterday" {
endTime = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
beginTime = endTime.AddDate(0, 0, -1)

} else if queryType == "current_week" {
beginTime = now.AddDate(0, 0, -int(time.Now().Weekday())+2) //begin from monday
} else if queryType == "last_7day" {
beginTime = now.AddDate(0, 0, -7)
beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
endTime = now
} else if queryType == "last_30day" {
beginTime = now.AddDate(0, 0, -30)
beginTime = time.Date(beginTime.Year(), beginTime.Month(), beginTime.Day(), 0, 0, 0, 0, now.Location())
endTime = now
} else if queryType == "current_month" {


+ 7
- 0
routers/home.go View File

@@ -343,6 +343,12 @@ func ExploreDatasets(ctx *context.Context) {
if ctx.User != nil && !ctx.User.IsAdmin {
ownerID = ctx.User.ID
}
var datasetsIds []int64
if ownerID > 0 {

datasetsIds = models.GetCollaboratorDatasetIdsByUserID(ownerID)
}

opts := &models.SearchDatasetOptions{
Keyword: keyword,
IncludePublic: true,
@@ -351,6 +357,7 @@ func ExploreDatasets(ctx *context.Context) {
Task: task,
License: license,
OwnerID: ownerID,
DatasetIDs: datasetsIds,
RecommendOnly: ctx.QueryBool("recommend"),
ListOptions: models.ListOptions{
Page: page,


+ 9
- 1
routers/repo/ai_model_manage.go View File

@@ -66,6 +66,14 @@ func saveModelByParameters(jobId string, versionName string, name string, versio
return err
}
} else if cloudType == models.TypeCloudBrainOne {
var ResourceSpecs *models.ResourceSpecs
json.Unmarshal([]byte(setting.ResourceSpecs), &ResourceSpecs)
for _, tmp := range ResourceSpecs.ResourceSpec {
if tmp.Id == aiTask.ResourceSpecId {
flaverName := ctx.Tr("cloudbrain.gpu_num") + ": " + fmt.Sprint(tmp.GpuNum) + " " + ctx.Tr("cloudbrain.cpu_num") + ": " + fmt.Sprint(tmp.CpuNum) + " " + ctx.Tr("cloudbrain.memory") + "(MB): " + fmt.Sprint(tmp.MemMiB) + " " + ctx.Tr("cloudbrain.shared_memory") + "(MB): " + fmt.Sprint(tmp.ShareMemMiB)
aiTask.FlavorName = flaverName
}
}
modelPath, modelSize, err = downloadModelFromCloudBrainOne(id, aiTask.JobName, "", aiTask.TrainUrl)
if err != nil {
log.Info("download model from CloudBrainOne faild." + err.Error())
@@ -97,7 +105,7 @@ func saveModelByParameters(jobId string, versionName string, name string, versio
UserId: ctx.User.ID,
CodeBranch: aiTask.BranchName,
CodeCommitID: aiTask.CommitID,
Engine: aiTask.EngineID,
Engine: int64(engine),
TrainTaskInfo: string(aiTaskJson),
Accuracy: string(accuracyJson),
}


+ 18
- 8
routers/repo/cloudbrain.go View File

@@ -283,11 +283,13 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
mkModelPath(modelPath)
uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/")

commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName)

err = cloudbrain.GenerateTask(ctx, displayJobName, jobName, image, command, uuid, storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), jobType, gpuQueue, form.Description, branchName, form.BootFile, form.Params,
0, 0, resourceSpecId)
commitID, 0, 0, resourceSpecId)
if err != nil {
cloudBrainNewDataPrepare(ctx)
ctx.RenderWithErr(err.Error(), tpl, &form)
@@ -1479,11 +1481,19 @@ func SyncCloudbrainStatus() {
}

func HandleTaskWithNoDuration(ctx *context.Context) {
mode := ctx.Query("mode")
log.Info("HandleTaskWithNoDuration start")
count := 0
start := time.Now().Unix()
for {
cloudBrains, err := models.GetStoppedJobWithNoDurationJob()
var cloudBrains []*models.Cloudbrain
var err error
if mode == "1" {
cloudBrains, err = models.GetStoppedJobWithNoStartTimeEndTime()
} else {
cloudBrains, err = models.GetStoppedJobWithNoDurationJob()
}

if err != nil {
log.Error("HandleTaskWithNoTrainJobDuration failed:", err.Error())
break
@@ -1558,7 +1568,7 @@ func handleNoDurationTask(cloudBrains []*models.Cloudbrain) {
}
task.Duration = task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix()
task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
err = models.UpdateJob(task)
err = models.UpdateJobDurationWithDeleted(task)
if err != nil {
log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
}
@@ -1583,7 +1593,7 @@ func handleNoDurationTask(cloudBrains []*models.Cloudbrain) {
}
task.CorrectCreateUnix()
task.ComputeAndSetDuration()
err = models.UpdateJob(task)
err = models.UpdateJobDurationWithDeleted(task)
if err != nil {
log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
continue
@@ -1604,7 +1614,7 @@ func handleNoDurationTask(cloudBrains []*models.Cloudbrain) {
task.EndTime = task.StartTime.Add(result.Duration / 1000)
}
task.ComputeAndSetDuration()
err = models.UpdateJob(task)
err = models.UpdateJobDurationWithDeleted(task)
if err != nil {
log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
continue
@@ -1625,7 +1635,7 @@ func updateDefaultDuration(task *models.Cloudbrain) {
task.StartTime = task.CreatedUnix
task.EndTime = task.UpdatedUnix
task.ComputeAndSetDuration()
err := models.UpdateJob(task)
err := models.UpdateJobDurationWithDeleted(task)
if err != nil {
log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
}
@@ -1976,7 +1986,7 @@ func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainFo
storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), string(models.JobTypeBenchmark), gpuQueue, form.Description, cloudbrain.DefaultBranchName, "", "",
benchmarkTypeID, benchmarkChildTypeID, resourceSpecId)
"", benchmarkTypeID, benchmarkChildTypeID, resourceSpecId)
if err != nil {
cloudBrainNewDataPrepare(ctx)
ctx.RenderWithErr(err.Error(), tplCloudBrainBenchmarkNew, &form)
@@ -2074,7 +2084,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), jobType, gpuQueue, form.Description, branchName, form.BootFile, form.Params,
0, benchmarkChildTypeID, resourceSpecId)
"", 0, benchmarkChildTypeID, resourceSpecId)
if err != nil {
cloudBrainNewDataPrepare(ctx)
ctx.RenderWithErr(err.Error(), tpl, &form)


+ 2
- 0
routers/routes/routes.go View File

@@ -331,6 +331,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Post("/all/search/", routers.Search)
m.Get("/all/search/", routers.EmptySearch)
m.Get("/all/dosearch/", routers.SearchApi)
m.Post("/user/login/kanban", user.SignInPostAPI)
m.Get("/home/term", routers.HomeTerm)
m.Group("/explore", func() {
m.Get("", func(ctx *context.Context) {
@@ -365,6 +366,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Group("/user", func() {
m.Get("/login", user.SignIn)
m.Get("/login/cloud_brain", user.SignInCloudBrain)
m.Post("/login", bindIgnErr(auth.SignInForm{}), user.SignInPost)
m.Group("", func() {
m.Combo("/login/openid").


+ 97
- 0
routers/user/auth.go View File

@@ -176,6 +176,41 @@ func SignInCloudBrain(ctx *context.Context) {
ctx.HTML(200, tplSignInCloudBrain)
}

func SignInPostAPI(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("sign_in")
UserName := ctx.Query("UserName")
Password := ctx.Query("Password")
log.Info("0000000")
orderedOAuth2Names, oauth2Providers, err := models.GetActiveOAuth2Providers()
if err != nil {
ctx.ServerError("UserSignIn", err)
return
}
ctx.Data["OrderedOAuth2Names"] = orderedOAuth2Names
ctx.Data["OAuth2Providers"] = oauth2Providers
ctx.Data["Title"] = ctx.Tr("sign_in")
ctx.Data["SignInLink"] = setting.AppSubURL + "/user/login"
ctx.Data["PageIsSignIn"] = true
ctx.Data["PageIsLogin"] = true
ctx.Data["IsCourse"] = ctx.QueryBool("course")
ctx.Data["EnableSSPI"] = models.IsSSPIEnabled()

if ctx.HasError() {
ctx.HTML(200, tplSignIn)
return
}
u, err := models.UserSignIn(UserName, Password)
if err != nil {
ctx.ServerError("UserSignIn", err)
return
}
models.SaveLoginInfoToDb(ctx.Req.Request, u)
// If this user is enrolled in 2FA, we can't sign the user in just yet.
// Instead, redirect them to the 2FA authentication page.
//handleSignInFull(ctx, u, form.Remember, false)
handleSignInFullNotRedirect(ctx, u, true, false)
}

// SignInPost response for sign in request
func SignInPost(ctx *context.Context, form auth.SignInForm) {
ctx.Data["Title"] = ctx.Tr("sign_in")
@@ -518,6 +553,68 @@ func handleSignIn(ctx *context.Context, u *models.User, remember bool) {
handleSignInFull(ctx, u, remember, true)
}

func handleSignInFullNotRedirect(ctx *context.Context, u *models.User, remember bool, obeyRedirect bool) string {

log.Info("enter here.")
if remember {
days := 86400 * setting.LogInRememberDays
ctx.SetCookie(setting.CookieUserName, u.Name, days, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true)
ctx.SetSuperSecureCookie(base.EncodeMD5(u.Rands+u.Passwd),
setting.CookieRememberName, u.Name, days, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true)
}

_ = ctx.Session.Delete("openid_verified_uri")
_ = ctx.Session.Delete("openid_signin_remember")
_ = ctx.Session.Delete("openid_determined_email")
_ = ctx.Session.Delete("openid_determined_username")
_ = ctx.Session.Delete("twofaUid")
_ = ctx.Session.Delete("twofaRemember")
_ = ctx.Session.Delete("u2fChallenge")
_ = ctx.Session.Delete("linkAccount")
if err := ctx.Session.Set("uid", u.ID); err != nil {
log.Error("Error setting uid %d in session: %v", u.ID, err)
}
if err := ctx.Session.Set("uname", u.Name); err != nil {
log.Error("Error setting uname %s session: %v", u.Name, err)
}
if err := ctx.Session.Release(); err != nil {
log.Error("Unable to store session: %v", err)
}

// If the user does not have a locale set, we save the current one.
if len(u.Language) == 0 {
if len(ctx.GetCookie("lang")) != 0 {
u.Language = ctx.GetCookie("lang")
} else {
u.Language = ctx.Locale.Language()
}

if err := models.UpdateUserCols(u, "language"); err != nil {
log.Error(fmt.Sprintf("Error updating user language [user: %d, locale: %s]", u.ID, u.Language))
return setting.AppSubURL + "/dashboard"
}
} else {
// Language setting of the user use the one previously set
if len(ctx.GetCookie("lang")) != 0 {
u.Language = ctx.GetCookie("lang")
}
}

ctx.SetCookie("lang", u.Language, nil, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true)

// Clear whatever CSRF has right now, force to generate a new one
ctx.SetCookie(setting.CSRFCookieName, "", -1, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true)

// Register last login
u.SetLastLogin()
if err := models.UpdateUserCols(u, "last_login_unix"); err != nil {
ctx.ServerError("UpdateUserCols", err)
return setting.AppSubURL + "/dashboard"
}

return setting.AppSubURL + "/dashboard"
}

func handleSignInFull(ctx *context.Context, u *models.User, remember bool, obeyRedirect bool) string {
if remember {
days := 86400 * setting.LogInRememberDays


+ 1
- 1
templates/explore/datasets.tmpl View File

@@ -199,7 +199,7 @@
</span>
{{else}}
<span
style="display: flex;align-items: center;justify-content: flex-end;cursor: pointer;font-size: 12px;font-weight: normal;flex: 1;">
style="display: flex;align-items: center;justify-content: flex-end;cursor: pointer;font-size: 12px;font-weight: normal;flex: 1;margin-left: 1.5rem;">

<div style="line-height: 1;margin-right: 4px;margin-bottom: -2px;">
<svg width="1.4em" height="1.4em" viewBox="0 0 32 32"


+ 5
- 0
templates/repo/modelmanage/index.tmpl View File

@@ -1,5 +1,10 @@
<!-- 头部导航栏 -->
{{template "base/head" .}}
<style>
.text{
color: rgba(0,0,0,.87)!important
}
</style>
<!-- 弹窗 -->
<div id="mask">
<div id="loadingPage">


+ 14
- 3
templates/repo/modelmanage/showinfo.tmpl View File

@@ -141,7 +141,7 @@
<td class="ti-text-form-content word-elipsis"><span id="Parameters" title=""></span></td>
</tr>
<tr>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.train_job.AI_driver"}}</td>
<td class="ti-text-form-label text-width80">{{$.i18n.Tr "repo.modelarts.train_job.AI_Engine"}}</td>
<td class="ti-text-form-content word-elipsis"><span id="EngineName" title=""></span></td>
</tr>
<tr>
@@ -191,7 +191,6 @@
</div>
</div>
</div>
</div>
</div>
@@ -232,13 +231,25 @@ function loadInfo(){
renderInfo(initObj,initModelAcc,id)
loadModelFile(data[0].ID,data[0].Version,'','','init')
})
}
function getEngineName(model){
if(model.Engine == 0){
return "Pytorch";
}else if(model.Engine == 1 || model.Engine == 121){
return "TensorFlow";
}else if(model.Engine == 2 || model.Engine == 122){
return "MindSpore";
}else{
return "Other"
}
}
function transObj(data){
let {ID,Name,Version,Label,Size,Description,CreatedUnix,Accuracy,CodeBranch,CodeCommitID,TrainTaskInfo} = data[0]
let modelAcc = JSON.parse(Accuracy)
TrainTaskInfo = JSON.parse(TrainTaskInfo)
// Parameters = JSON.parse(Parameters)
let {Parameters,EngineName} = TrainTaskInfo
let {Parameters} = TrainTaskInfo
let EngineName = getEngineName(data[0])
Parameters = JSON.parse(Parameters)
Parameters = Parameters.parameter.length === 0 ? '--':Parameters.parameter
let size = tranSize(Size)


+ 1
- 1
web_src/js/components/Model.vue View File

@@ -60,7 +60,7 @@
</el-table-column>
<el-table-column
prop="EngineName"
label="AI引擎"
label="模型框架"
align="center"
min-width="8.5%"
>


+ 5
- 2
web_src/js/index.js View File

@@ -5093,8 +5093,9 @@ function initChartsNpu() {
data: []
},
grid: {
top: '30%',
top: '35%',
bottom: '2%',
x: '2%',
containLabel: true
},
tooltip: {
@@ -5130,14 +5131,16 @@ function initChartsNpu() {

series: []
};
const sortBy = (arr, k) => arr.concat().sort((a, b) => (a[k] > b[k] ? 1 : a[k] < b[k] ? -1 : 0));
$('.metric_chart').click(function (e) {
let versionName = $(this).data('version')
let myCharts = echarts.init(document.getElementById(`metric-${versionName}`))
$.get(`${window.config.AppSubUrl}/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/metric_statistics?version_name=${versionName}&statistic_type=each`, (res) => {
$.get(`${window.config.AppSubUrl}/api/v1/repos/${userName}/${repoPath}/modelarts/train-job/${jobID}/metric_statistics?version_name=${versionName}&statistic_type=each&metrics=`, (res) => {
let filterDta = res.MetricsInfo.filter((item) => {

return !(['recvBytesRate', 'diskWriteRate', 'sendBytesRate', 'diskReadRate'].includes(item.metric))
})
filterDta = sortBy(filterDta, "metric")
let legenData = filterDta.map((item) => {
return item.metric
})


Loading…
Cancel
Save