Browse Source

模型评测增加类型

tags/v1.22.12.2^2
ychao_1983 3 years ago
parent
commit
c35c8f2eb4
9 changed files with 132 additions and 51 deletions
  1. +11
    -0
      models/attachment.go
  2. +10
    -6
      models/cloudbrain.go
  3. +17
    -6
      modules/cloudbrain/cloudbrain.go
  4. +28
    -17
      modules/setting/setting.go
  5. +1
    -1
      routers/admin/cloudbrains.go
  6. +2
    -2
      routers/api/v1/repo/cloudbrain_dashboard.go
  7. +60
    -16
      routers/repo/cloudbrain.go
  8. +1
    -1
      routers/user/home.go
  9. +2
    -2
      services/cloudbrain/cloudbrainTask/count.go

+ 11
- 0
models/attachment.go View File

@@ -259,6 +259,17 @@ func GetAttachmentsByCommentID(commentID int64) ([]*Attachment, error) {
return getAttachmentsByCommentID(x, commentID) return getAttachmentsByCommentID(x, commentID)
} }


func GetAttachmentByDatasetIdFileName(fileName string, datasetId int64) (*Attachment, error) {
attach := &Attachment{DatasetID: datasetId, Name: fileName}
has, err := x.Get(attach)
if err != nil {
return nil, err
} else if !has {
return nil, err
}
return attach, nil
}

func getAttachmentsByCommentID(e Engine, commentID int64) ([]*Attachment, error) { func getAttachmentsByCommentID(e Engine, commentID int64) ([]*Attachment, error) {
attachments := make([]*Attachment, 0, 10) attachments := make([]*Attachment, 0, 10)
return attachments, e.Where("comment_id=?", commentID).Find(&attachments) return attachments, e.Where("comment_id=?", commentID).Find(&attachments)


+ 10
- 6
models/cloudbrain.go View File

@@ -60,6 +60,7 @@ const (
JobTypeModelSafety JobType = "MODELSAFETY" JobTypeModelSafety JobType = "MODELSAFETY"
JobTypeSnn4imagenet JobType = "SNN4IMAGENET" JobTypeSnn4imagenet JobType = "SNN4IMAGENET"
JobTypeBrainScore JobType = "BRAINSCORE" JobTypeBrainScore JobType = "BRAINSCORE"
JobTypeSnn4Ecoset JobType = "SNN4ECOSET"
JobTypeTrain JobType = "TRAIN" JobTypeTrain JobType = "TRAIN"
JobTypeInference JobType = "INFERENCE" JobTypeInference JobType = "INFERENCE"


@@ -204,7 +205,7 @@ type Cloudbrain struct {
BenchmarkTypeRankLink string `xorm:"-"` BenchmarkTypeRankLink string `xorm:"-"`
StartTime timeutil.TimeStamp StartTime timeutil.TimeStamp
EndTime timeutil.TimeStamp EndTime timeutil.TimeStamp
Cleared bool `xorm:"DEFAULT false"`
Cleared bool `xorm:"DEFAULT false"`
Spec *Specification `xorm:"-"` Spec *Specification `xorm:"-"`
} }


@@ -328,6 +329,9 @@ func IsModelArtsDebugJobTerminal(status string) bool {
func IsCloudBrainOneDebugJobTerminal(status string) bool { func IsCloudBrainOneDebugJobTerminal(status string) bool {
return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded) return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded)
} }
func IsBenchMarkJobType(jobType string) bool {
return jobType == string(JobTypeSnn4imagenet) || jobType == string(JobTypeBrainScore) || jobType == string(JobTypeSnn4Ecoset)
}


func ParseAndSetDurationFromCloudBrainOne(result JobResultPayload, task *Cloudbrain) { func ParseAndSetDurationFromCloudBrainOne(result JobResultPayload, task *Cloudbrain) {
isActivated := result.JobStatus.CreatedTime > 0 isActivated := result.JobStatus.CreatedTime > 0
@@ -1906,7 +1910,7 @@ func GetCloudbrainByID(id string) (*Cloudbrain, error) {
return getRepoCloudBrain(cb) return getRepoCloudBrain(cb)
} }


func IsCloudbrainExistByJobName(jobName string)(bool,error){
func IsCloudbrainExistByJobName(jobName string) (bool, error) {
return x.Unscoped().Exist(&Cloudbrain{ return x.Unscoped().Exist(&Cloudbrain{
JobName: jobName, JobName: jobName,
}) })
@@ -2070,25 +2074,25 @@ func GetCloudBrainOneStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbra
Limit(limit). Limit(limit).
Find(&cloudbrains) Find(&cloudbrains)
} }

/** /**
本方法考虑了再次调试的情况,多次调试取最后一次的任务的结束时间 本方法考虑了再次调试的情况,多次调试取最后一次的任务的结束时间
*/
*/
func GetCloudBrainOneStoppedDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) { func GetCloudBrainOneStoppedDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) {
cloudbrains := make([]*Cloudbrain, 0, 10) cloudbrains := make([]*Cloudbrain, 0, 10)
endTimeBefore := time.Now().Unix() - int64(days)*24*3600 endTimeBefore := time.Now().Unix() - int64(days)*24*3600
missEndTimeBefore := endTimeBefore - 24*3600 missEndTimeBefore := endTimeBefore - 24*3600
sql:=`SELECT id,job_name,job_id from (SELECT DISTINCT ON (job_name)
sql := `SELECT id,job_name,job_id from (SELECT DISTINCT ON (job_name)
id, job_name, job_id,status,end_time,updated_unix,cleared id, job_name, job_id,status,end_time,updated_unix,cleared
FROM cloudbrain FROM cloudbrain
where type=0 and job_type='DEBUG' where type=0 and job_type='DEBUG'
ORDER BY job_name, updated_unix DESC) a ORDER BY job_name, updated_unix DESC) a
where status in ('STOPPED','SUCCEEDED','FAILED') and (((end_time is null or end_time=0) and updated_unix<? and updated_unix != 0 ) or (end_time<? and end_time != 0)) and cleared=false` where status in ('STOPPED','SUCCEEDED','FAILED') and (((end_time is null or end_time=0) and updated_unix<? and updated_unix != 0 ) or (end_time<? and end_time != 0)) and cleared=false`


return cloudbrains, x.Unscoped().SQL(sql,missEndTimeBefore, endTimeBefore).Limit(limit).Find(&cloudbrains)
return cloudbrains, x.Unscoped().SQL(sql, missEndTimeBefore, endTimeBefore).Limit(limit).Find(&cloudbrains)


} }



func UpdateCloudBrainRecordsCleared(ids []int64) error { func UpdateCloudBrainRecordsCleared(ids []int64) error {
pageSize := 150 pageSize := 150
n := len(ids) / pageSize n := len(ids) / pageSize


+ 17
- 6
modules/cloudbrain/cloudbrain.go View File

@@ -30,9 +30,10 @@ const (
BenchMarkResourceID = 1 BenchMarkResourceID = 1
Snn4imagenetMountPath = "/snn4imagenet" Snn4imagenetMountPath = "/snn4imagenet"
BrainScoreMountPath = "/brainscore" BrainScoreMountPath = "/brainscore"
Snn4EcosetMountPath = "/snn4ecoset"
TaskInfoName = "/taskInfo" TaskInfoName = "/taskInfo"
Snn4imagenetCommand = `/opt/conda/bin/python /snn4imagenet/testSNN_script.py --modelname '%s' --modelpath '/dataset' --modeldescription '%s' >/model/benchmark-log.txt`
BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/dataset' -d '%s' >/model/benchmark-log.txt`
Snn4imagenetCommand = `/opt/conda/bin/python /snn4imagenet/testSNN_script.py --modelname '%s' --modelpath '/pretrainmodel' --modeldescription '%s' >/model/benchmark-log.txt`
BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/pretrainmodel' -d '%s' >/model/benchmark-log.txt`


SubTaskName = "task1" SubTaskName = "task1"


@@ -60,6 +61,7 @@ type GenerateCloudBrainTaskReq struct {
ModelPath string ModelPath string
BenchmarkPath string BenchmarkPath string
Snn4ImageNetPath string Snn4ImageNetPath string
Snn4EcosetPath string
BrainScorePath string BrainScorePath string
JobType string JobType string
Description string Description string
@@ -288,6 +290,15 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) {
}, },
}) })
} }
if req.Snn4EcosetPath != "" { //ecoset benchmark
volumes = append(volumes, models.Volume{
HostPath: models.StHostPath{
Path: req.Snn4EcosetPath,
MountPath: Snn4EcosetMountPath,
ReadOnly: true,
},
})
}


if len(req.DatasetInfos) == 1 { if len(req.DatasetInfos) == 1 {
volumes = append(volumes, models.Volume{ volumes = append(volumes, models.Volume{
@@ -405,7 +416,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) {
} }


func IsBenchmarkJob(jobType string) bool { func IsBenchmarkJob(jobType string) bool {
return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType
return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType || string(models.JobTypeSnn4Ecoset) == jobType
} }


func GetWaitingCloudbrainCount(cloudbrainType int, computeResource string, jobTypes ...models.JobType) int64 { func GetWaitingCloudbrainCount(cloudbrainType int, computeResource string, jobTypes ...models.JobType) int64 {
@@ -653,7 +664,7 @@ func IsElementExist(s []string, str string) bool {
return false return false
} }


func GetCloudBrainByIdOrJobId(id string) (*models.Cloudbrain,error) {
func GetCloudBrainByIdOrJobId(id string) (*models.Cloudbrain, error) {
_, err := strconv.ParseInt(id, 10, 64) _, err := strconv.ParseInt(id, 10, 64)
var job *models.Cloudbrain var job *models.Cloudbrain
if err != nil { if err != nil {
@@ -661,10 +672,10 @@ func GetCloudBrainByIdOrJobId(id string) (*models.Cloudbrain,error) {
job, err = models.GetCloudbrainByJobID(id) job, err = models.GetCloudbrainByJobID(id)
} else { } else {
job, err = models.GetCloudbrainByID(id) job, err = models.GetCloudbrainByID(id)
if err!=nil{
if err != nil {
job, err = models.GetCloudbrainByJobID(id) job, err = models.GetCloudbrainByJobID(id)
} }


} }
return job,err
return job, err
} }

+ 28
- 17
modules/setting/setting.go View File

@@ -519,7 +519,6 @@ var (
CullIdleTimeout string CullIdleTimeout string
CullInterval string CullInterval string



//benchmark config //benchmark config
IsBenchmarkEnabled bool IsBenchmarkEnabled bool
BenchmarkOwner string BenchmarkOwner string
@@ -544,6 +543,12 @@ var (
BrainScoreName string BrainScoreName string
BrainScoreServerHost string BrainScoreServerHost string


IsSnn4EcosetEnabled bool
Snn4EcosetOwner string
Snn4EcosetName string
Snn4EcosetServerHost string
Snn4AttachmentName string

//blockchain config //blockchain config
BlockChainHost string BlockChainHost string
CommitValidDate string CommitValidDate string
@@ -614,14 +619,14 @@ var (
UsageRateBeginTime string UsageRateBeginTime string
}{} }{}


ClearStrategy= struct {
Enabled bool
ResultSaveDays int
BatchSize int
DebugJobSize int
TrashSaveDays int
Cron string
RunAtStart bool
ClearStrategy = struct {
Enabled bool
ResultSaveDays int
BatchSize int
DebugJobSize int
TrashSaveDays int
Cron string
RunAtStart bool
}{} }{}


C2NetInfos *C2NetSqInfos C2NetInfos *C2NetSqInfos
@@ -1515,6 +1520,12 @@ func NewContext() {
BrainScoreName = sec.Key("NAME").MustString("") BrainScoreName = sec.Key("NAME").MustString("")
BrainScoreServerHost = sec.Key("HOST").MustString("") BrainScoreServerHost = sec.Key("HOST").MustString("")


sec = Cfg.Section("snn4ecoset")
IsSnn4EcosetEnabled = sec.Key("ENABLED").MustBool(false)
Snn4EcosetOwner = sec.Key("OWNER").MustString("")
Snn4EcosetName = sec.Key("NAME").MustString("")
Snn4imagenetServerHost = sec.Key("HOST").MustString("")

sec = Cfg.Section("blockchain") sec = Cfg.Section("blockchain")
BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/") BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/")
CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15") CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15")
@@ -1691,16 +1702,16 @@ func getModelartsCDConfig() {
getNotebookFlavorInfos() getNotebookFlavorInfos()
} }


func getClearStrategy(){
func getClearStrategy() {


sec := Cfg.Section("clear_strategy") sec := Cfg.Section("clear_strategy")
ClearStrategy.Enabled=sec.Key("ENABLED").MustBool(false)
ClearStrategy.ResultSaveDays=sec.Key("RESULT_SAVE_DAYS").MustInt(30)
ClearStrategy.BatchSize=sec.Key("BATCH_SIZE").MustInt(500)
ClearStrategy.DebugJobSize=sec.Key("DEBUG_BATCH_SIZE").MustInt(100)
ClearStrategy.TrashSaveDays=sec.Key("TRASH_SAVE_DAYS").MustInt(90)
ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ?")
ClearStrategy.RunAtStart=sec.Key("RUN_AT_START").MustBool(false)
ClearStrategy.Enabled = sec.Key("ENABLED").MustBool(false)
ClearStrategy.ResultSaveDays = sec.Key("RESULT_SAVE_DAYS").MustInt(30)
ClearStrategy.BatchSize = sec.Key("BATCH_SIZE").MustInt(500)
ClearStrategy.DebugJobSize = sec.Key("DEBUG_BATCH_SIZE").MustInt(100)
ClearStrategy.TrashSaveDays = sec.Key("TRASH_SAVE_DAYS").MustInt(90)
ClearStrategy.Cron = sec.Key("CRON").MustString("* 0,30 2-8 * * ?")
ClearStrategy.RunAtStart = sec.Key("RUN_AT_START").MustBool(false)
} }


func getGrampusConfig() { func getGrampusConfig() {


+ 1
- 1
routers/admin/cloudbrains.go View File

@@ -53,7 +53,7 @@ func CloudBrains(ctx *context.Context) {
var jobTypes []string var jobTypes []string
jobTypeNot := false jobTypeNot := false
if jobType == string(models.JobTypeBenchmark) { if jobType == string(models.JobTypeBenchmark) {
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet))
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset))
} else if jobType != "all" && jobType != "" { } else if jobType != "all" && jobType != "" {
jobTypes = append(jobTypes, jobType) jobTypes = append(jobTypes, jobType)
} }


+ 2
- 2
routers/api/v1/repo/cloudbrain_dashboard.go View File

@@ -647,7 +647,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) {
} }


jobTypeList := []string{string(models.JobTypeDebug), string(models.JobTypeTrain), string(models.JobTypeInference), string(models.JobTypeBenchmark), jobTypeList := []string{string(models.JobTypeDebug), string(models.JobTypeTrain), string(models.JobTypeInference), string(models.JobTypeBenchmark),
string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)}
string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)}
for _, v := range jobTypeList { for _, v := range jobTypeList {
if _, ok := cloudOneJobTypeRes[v]; !ok { if _, ok := cloudOneJobTypeRes[v]; !ok {
cloudOneJobTypeRes[v] = 0 cloudOneJobTypeRes[v] = 0
@@ -754,7 +754,7 @@ func GetCloudbrainsDetailData(ctx *context.Context) {
var jobTypes []string var jobTypes []string
jobTypeNot := false jobTypeNot := false
if jobType == string(models.JobTypeBenchmark) { if jobType == string(models.JobTypeBenchmark) {
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet))
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset))
} else if jobType != "all" && jobType != "" { } else if jobType != "all" && jobType != "" {
jobTypes = append(jobTypes, jobType) jobTypes = append(jobTypes, jobType)
} }


+ 60
- 16
routers/repo/cloudbrain.go View File

@@ -379,7 +379,6 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
req.ModelVersion = form.ModelVersion req.ModelVersion = form.ModelVersion
req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl
req.PreTrainModelUrl = form.PreTrainModelUrl req.PreTrainModelUrl = form.PreTrainModelUrl

} }


_, err = cloudbrain.GenerateTask(req) _, err = cloudbrain.GenerateTask(req)
@@ -711,7 +710,6 @@ func CloudBrainRestart(ctx *context.Context) {
}) })
} }



func getOldJobPath(task *models.Cloudbrain) string { func getOldJobPath(task *models.Cloudbrain) string {
return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + task.JobName return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + task.JobName
} }
@@ -786,7 +784,7 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo


if task.JobType == string(models.JobTypeBenchmark) { if task.JobType == string(models.JobTypeBenchmark) {
task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm")
} else if task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) {
} else if models.IsBenchMarkJobType(task.JobType) {
task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model")
task.BenchmarkTypeName = task.JobType task.BenchmarkTypeName = task.JobType
ctx.Data["BenchmarkTypeName"] = task.JobType ctx.Data["BenchmarkTypeName"] = task.JobType
@@ -854,10 +852,10 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo
func CloudBrainDebug(ctx *context.Context) { func CloudBrainDebug(ctx *context.Context) {
task := ctx.Cloudbrain task := ctx.Cloudbrain
debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName
if task.BootFile!=""{
ctx.Redirect(getFileUrl(debugUrl,task.BootFile))
if task.BootFile != "" {
ctx.Redirect(getFileUrl(debugUrl, task.BootFile))


}else{
} else {
ctx.Redirect(debugUrl) ctx.Redirect(debugUrl)
} }


@@ -1617,6 +1615,8 @@ func GetRate(ctx *context.Context) {
ctx.Redirect(setting.Snn4imagenetServerHost) ctx.Redirect(setting.Snn4imagenetServerHost)
} else if job.JobType == string(models.JobTypeBrainScore) { } else if job.JobType == string(models.JobTypeBrainScore) {
ctx.Redirect(setting.BrainScoreServerHost) ctx.Redirect(setting.BrainScoreServerHost)
} else if job.JobType == string(models.JobTypeSnn4Ecoset) {
ctx.Redirect(setting.Snn4EcosetServerHost)
} else { } else {
log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"]) log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"])
} }
@@ -2117,14 +2117,16 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) {
ciTasks[i].BenchmarkTypeName = "" ciTasks[i].BenchmarkTypeName = ""
if ciTasks[i].JobType == string(models.JobTypeBenchmark) { if ciTasks[i].JobType == string(models.JobTypeBenchmark) {
ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm")
} else if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) || ciTasks[i].JobType == string(models.JobTypeBrainScore) {
} else if models.IsBenchMarkJobType(ciTasks[i].JobType) {
ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model")
ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType


if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) { if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) {
ciTasks[i].BenchmarkTypeRankLink = setting.Snn4imagenetServerHost ciTasks[i].BenchmarkTypeRankLink = setting.Snn4imagenetServerHost
} else {
} else if ciTasks[i].JobType == string(models.JobTypeBrainScore) {
ciTasks[i].BenchmarkTypeRankLink = setting.BrainScoreServerHost ciTasks[i].BenchmarkTypeRankLink = setting.BrainScoreServerHost
} else {
ciTasks[i].BenchmarkTypeRankLink = setting.Snn4EcosetServerHost
} }


} }
@@ -2474,7 +2476,6 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
displayJobName := form.DisplayJobName displayJobName := form.DisplayJobName
jobName := util.ConvertDisplayJobNameToJobName(displayJobName) jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
image := form.Image image := form.Image
uuid := form.Attachment
jobType := form.JobType jobType := form.JobType
codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
branchName := cloudbrain.DefaultBranchName branchName := cloudbrain.DefaultBranchName
@@ -2516,7 +2517,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
return return
} }


if jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) {
if !models.IsBenchMarkJobType(jobType) {
log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) log.Error("jobtype error:", jobType, ctx.Data["MsgID"])
cloudBrainNewDataPrepare(ctx, jobType) cloudBrainNewDataPrepare(ctx, jobType)
ctx.RenderWithErr("jobtype error", tpl, &form) ctx.RenderWithErr("jobtype error", tpl, &form)
@@ -2560,14 +2561,33 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
benchmarkChildTypeID = form.BenchmarkChildTypeID benchmarkChildTypeID = form.BenchmarkChildTypeID
command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description)) command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description))
} }
snn4EcosetPath := setting.JobPath + jobName + cloudbrain.Snn4EcosetMountPath
var uuid string
var datasetInfos map[string]models.DatasetInfo
var datasetNames string
if setting.IsSnn4EcosetEnabled && jobType == string(models.JobTypeSnn4Ecoset) {
err = downloadRateCode(repo, jobName, setting.Snn4EcosetOwner, setting.Snn4EcosetName, snn4EcosetPath, "", "", ctx.User.Name)
if err != nil {
log.Error("load benchmark code failed", err)
cloudBrainNewDataPrepare(ctx, jobType)
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form)
return
}
err = uploadCodeToMinio(snn4EcosetPath+"/", jobName, cloudbrain.Snn4EcosetMountPath+"/")
command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description))

attachment, err := getEcosetAttachment()
if err != nil {
log.Error("load benchmark code failed", err)
cloudBrainNewDataPrepare(ctx, jobType)
ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form)
return
}
uuid = attachment.UUID
datasetInfos, datasetNames, _ = models.GetDatasetInfo(uuid)


datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid)
if err != nil {
log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
cloudBrainNewDataPrepare(ctx, jobType)
ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
return
} }

spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{ spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{
JobType: models.JobTypeBenchmark, JobType: models.JobTypeBenchmark,
ComputeResource: models.GPU, ComputeResource: models.GPU,
@@ -2601,6 +2621,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
Snn4EcosetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4EcosetMountPath+"/"),
JobType: jobType, JobType: jobType,
Description: form.Description, Description: form.Description,
BranchName: branchName, BranchName: branchName,
@@ -2612,6 +2633,14 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"),
Spec: spec, Spec: spec,
} }
if form.ModelName != "" {
req.ModelName = form.ModelName
req.LabelName = form.LabelName
req.CkptName = form.CkptName
req.ModelVersion = form.ModelVersion
req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl
req.PreTrainModelUrl = form.PreTrainModelUrl
}


_, err = cloudbrain.GenerateTask(req) _, err = cloudbrain.GenerateTask(req)
if err != nil { if err != nil {
@@ -2623,6 +2652,21 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm)
ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")
} }


func getEcosetAttachment() (*models.Attachment, error) {
ecosetRepo, err := models.GetRepositoryByOwnerAndName(setting.Snn4EcosetOwner, setting.Snn4EcosetName)
if err != nil {
return nil, err
}

datasetInfo, err := models.GetDatasetByRepo(ecosetRepo)
if err != nil {
return nil, err
}

return models.GetAttachmentByDatasetIdFileName(setting.Snn4AttachmentName, datasetInfo.ID)

}

func getBrainRegion(benchmarkChildTypeID int) string { func getBrainRegion(benchmarkChildTypeID int) string {
values := []string{"V1", "V2", "V4", "IT"} values := []string{"V1", "V2", "V4", "IT"}
return values[benchmarkChildTypeID] return values[benchmarkChildTypeID]


+ 1
- 1
routers/user/home.go View File

@@ -779,7 +779,7 @@ func Cloudbrains(ctx *context.Context) {
var jobTypes []string var jobTypes []string
jobTypeNot := false jobTypeNot := false
if jobType == string(models.JobTypeBenchmark) { if jobType == string(models.JobTypeBenchmark) {
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet))
jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset))
} else if jobType != "all" && jobType != "" { } else if jobType != "all" && jobType != "" {
jobTypes = append(jobTypes, jobType) jobTypes = append(jobTypes, jobType)
} }


+ 2
- 2
services/cloudbrain/cloudbrainTask/count.go View File

@@ -34,7 +34,7 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s
ComputeResource: models.GPUResource, ComputeResource: models.GPUResource,
}, string(models.JobTypeBenchmark) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { }, string(models.JobTypeBenchmark) + "-" + strconv.Itoa(models.TypeCloudBrainOne): {
CloudBrainTypes: []int{models.TypeCloudBrainOne}, CloudBrainTypes: []int{models.TypeCloudBrainOne},
JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet},
JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet, models.JobTypeSnn4Ecoset},
NotFinalStatuses: CloudbrainOneNotFinalStatuses, NotFinalStatuses: CloudbrainOneNotFinalStatuses,
ComputeResource: models.GPUResource, ComputeResource: models.GPUResource,
}, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { }, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): {
@@ -66,7 +66,7 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s


func GetNotFinalStatusTaskCount(uid int64, cloudbrainType int, jobType string, computeResource ...string) (int, error) { func GetNotFinalStatusTaskCount(uid int64, cloudbrainType int, jobType string, computeResource ...string) (int, error) {
jobNewType := jobType jobNewType := jobType
if jobType == string(models.JobTypeSnn4imagenet) || jobType == string(models.JobTypeBrainScore) {
if models.IsBenchMarkJobType(jobType) {
jobNewType = string(models.JobTypeBenchmark) jobNewType = string(models.JobTypeBenchmark)
} }




Loading…
Cancel
Save