| @@ -259,6 +259,17 @@ func GetAttachmentsByCommentID(commentID int64) ([]*Attachment, error) { | |||||
| return getAttachmentsByCommentID(x, commentID) | return getAttachmentsByCommentID(x, commentID) | ||||
| } | } | ||||
| func GetAttachmentByDatasetIdFileName(fileName string, datasetId int64) (*Attachment, error) { | |||||
| attach := &Attachment{DatasetID: datasetId, Name: fileName} | |||||
| has, err := x.Get(attach) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } else if !has { | |||||
| return nil, err | |||||
| } | |||||
| return attach, nil | |||||
| } | |||||
| func getAttachmentsByCommentID(e Engine, commentID int64) ([]*Attachment, error) { | func getAttachmentsByCommentID(e Engine, commentID int64) ([]*Attachment, error) { | ||||
| attachments := make([]*Attachment, 0, 10) | attachments := make([]*Attachment, 0, 10) | ||||
| return attachments, e.Where("comment_id=?", commentID).Find(&attachments) | return attachments, e.Where("comment_id=?", commentID).Find(&attachments) | ||||
| @@ -60,6 +60,7 @@ const ( | |||||
| JobTypeModelSafety JobType = "MODELSAFETY" | JobTypeModelSafety JobType = "MODELSAFETY" | ||||
| JobTypeSnn4imagenet JobType = "SNN4IMAGENET" | JobTypeSnn4imagenet JobType = "SNN4IMAGENET" | ||||
| JobTypeBrainScore JobType = "BRAINSCORE" | JobTypeBrainScore JobType = "BRAINSCORE" | ||||
| JobTypeSnn4Ecoset JobType = "SNN4ECOSET" | |||||
| JobTypeTrain JobType = "TRAIN" | JobTypeTrain JobType = "TRAIN" | ||||
| JobTypeInference JobType = "INFERENCE" | JobTypeInference JobType = "INFERENCE" | ||||
| @@ -204,7 +205,7 @@ type Cloudbrain struct { | |||||
| BenchmarkTypeRankLink string `xorm:"-"` | BenchmarkTypeRankLink string `xorm:"-"` | ||||
| StartTime timeutil.TimeStamp | StartTime timeutil.TimeStamp | ||||
| EndTime timeutil.TimeStamp | EndTime timeutil.TimeStamp | ||||
| Cleared bool `xorm:"DEFAULT false"` | |||||
| Cleared bool `xorm:"DEFAULT false"` | |||||
| Spec *Specification `xorm:"-"` | Spec *Specification `xorm:"-"` | ||||
| } | } | ||||
| @@ -328,6 +329,9 @@ func IsModelArtsDebugJobTerminal(status string) bool { | |||||
| func IsCloudBrainOneDebugJobTerminal(status string) bool { | func IsCloudBrainOneDebugJobTerminal(status string) bool { | ||||
| return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded) | return status == string(JobStopped) || status == string(JobFailed) || status == string(JobSucceeded) | ||||
| } | } | ||||
| func IsBenchMarkJobType(jobType string) bool { | |||||
| return jobType == string(JobTypeSnn4imagenet) || jobType == string(JobTypeBrainScore) || jobType == string(JobTypeSnn4Ecoset) | |||||
| } | |||||
| func ParseAndSetDurationFromCloudBrainOne(result JobResultPayload, task *Cloudbrain) { | func ParseAndSetDurationFromCloudBrainOne(result JobResultPayload, task *Cloudbrain) { | ||||
| isActivated := result.JobStatus.CreatedTime > 0 | isActivated := result.JobStatus.CreatedTime > 0 | ||||
| @@ -1906,7 +1910,7 @@ func GetCloudbrainByID(id string) (*Cloudbrain, error) { | |||||
| return getRepoCloudBrain(cb) | return getRepoCloudBrain(cb) | ||||
| } | } | ||||
| func IsCloudbrainExistByJobName(jobName string)(bool,error){ | |||||
| func IsCloudbrainExistByJobName(jobName string) (bool, error) { | |||||
| return x.Unscoped().Exist(&Cloudbrain{ | return x.Unscoped().Exist(&Cloudbrain{ | ||||
| JobName: jobName, | JobName: jobName, | ||||
| }) | }) | ||||
| @@ -2070,25 +2074,25 @@ func GetCloudBrainOneStoppedNotDebugJobDaysAgo(days int, limit int) ([]*Cloudbra | |||||
| Limit(limit). | Limit(limit). | ||||
| Find(&cloudbrains) | Find(&cloudbrains) | ||||
| } | } | ||||
| /** | /** | ||||
| 本方法考虑了再次调试的情况,多次调试取最后一次的任务的结束时间 | 本方法考虑了再次调试的情况,多次调试取最后一次的任务的结束时间 | ||||
| */ | |||||
| */ | |||||
| func GetCloudBrainOneStoppedDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) { | func GetCloudBrainOneStoppedDebugJobDaysAgo(days int, limit int) ([]*Cloudbrain, error) { | ||||
| cloudbrains := make([]*Cloudbrain, 0, 10) | cloudbrains := make([]*Cloudbrain, 0, 10) | ||||
| endTimeBefore := time.Now().Unix() - int64(days)*24*3600 | endTimeBefore := time.Now().Unix() - int64(days)*24*3600 | ||||
| missEndTimeBefore := endTimeBefore - 24*3600 | missEndTimeBefore := endTimeBefore - 24*3600 | ||||
| sql:=`SELECT id,job_name,job_id from (SELECT DISTINCT ON (job_name) | |||||
| sql := `SELECT id,job_name,job_id from (SELECT DISTINCT ON (job_name) | |||||
| id, job_name, job_id,status,end_time,updated_unix,cleared | id, job_name, job_id,status,end_time,updated_unix,cleared | ||||
| FROM cloudbrain | FROM cloudbrain | ||||
| where type=0 and job_type='DEBUG' | where type=0 and job_type='DEBUG' | ||||
| ORDER BY job_name, updated_unix DESC) a | ORDER BY job_name, updated_unix DESC) a | ||||
| where status in ('STOPPED','SUCCEEDED','FAILED') and (((end_time is null or end_time=0) and updated_unix<? and updated_unix != 0 ) or (end_time<? and end_time != 0)) and cleared=false` | where status in ('STOPPED','SUCCEEDED','FAILED') and (((end_time is null or end_time=0) and updated_unix<? and updated_unix != 0 ) or (end_time<? and end_time != 0)) and cleared=false` | ||||
| return cloudbrains, x.Unscoped().SQL(sql,missEndTimeBefore, endTimeBefore).Limit(limit).Find(&cloudbrains) | |||||
| return cloudbrains, x.Unscoped().SQL(sql, missEndTimeBefore, endTimeBefore).Limit(limit).Find(&cloudbrains) | |||||
| } | } | ||||
| func UpdateCloudBrainRecordsCleared(ids []int64) error { | func UpdateCloudBrainRecordsCleared(ids []int64) error { | ||||
| pageSize := 150 | pageSize := 150 | ||||
| n := len(ids) / pageSize | n := len(ids) / pageSize | ||||
| @@ -30,9 +30,10 @@ const ( | |||||
| BenchMarkResourceID = 1 | BenchMarkResourceID = 1 | ||||
| Snn4imagenetMountPath = "/snn4imagenet" | Snn4imagenetMountPath = "/snn4imagenet" | ||||
| BrainScoreMountPath = "/brainscore" | BrainScoreMountPath = "/brainscore" | ||||
| Snn4EcosetMountPath = "/snn4ecoset" | |||||
| TaskInfoName = "/taskInfo" | TaskInfoName = "/taskInfo" | ||||
| Snn4imagenetCommand = `/opt/conda/bin/python /snn4imagenet/testSNN_script.py --modelname '%s' --modelpath '/dataset' --modeldescription '%s' >/model/benchmark-log.txt` | |||||
| BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/dataset' -d '%s' >/model/benchmark-log.txt` | |||||
| Snn4imagenetCommand = `/opt/conda/bin/python /snn4imagenet/testSNN_script.py --modelname '%s' --modelpath '/pretrainmodel' --modeldescription '%s' >/model/benchmark-log.txt` | |||||
| BrainScoreCommand = `bash /brainscore/brainscore_test_par4shSrcipt.sh -b '%s' -n '%s' -p '/pretrainmodel' -d '%s' >/model/benchmark-log.txt` | |||||
| SubTaskName = "task1" | SubTaskName = "task1" | ||||
| @@ -60,6 +61,7 @@ type GenerateCloudBrainTaskReq struct { | |||||
| ModelPath string | ModelPath string | ||||
| BenchmarkPath string | BenchmarkPath string | ||||
| Snn4ImageNetPath string | Snn4ImageNetPath string | ||||
| Snn4EcosetPath string | |||||
| BrainScorePath string | BrainScorePath string | ||||
| JobType string | JobType string | ||||
| Description string | Description string | ||||
| @@ -288,6 +290,15 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) { | |||||
| }, | }, | ||||
| }) | }) | ||||
| } | } | ||||
| if req.Snn4EcosetPath != "" { //ecoset benchmark | |||||
| volumes = append(volumes, models.Volume{ | |||||
| HostPath: models.StHostPath{ | |||||
| Path: req.Snn4EcosetPath, | |||||
| MountPath: Snn4EcosetMountPath, | |||||
| ReadOnly: true, | |||||
| }, | |||||
| }) | |||||
| } | |||||
| if len(req.DatasetInfos) == 1 { | if len(req.DatasetInfos) == 1 { | ||||
| volumes = append(volumes, models.Volume{ | volumes = append(volumes, models.Volume{ | ||||
| @@ -405,7 +416,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) (string, error) { | |||||
| } | } | ||||
| func IsBenchmarkJob(jobType string) bool { | func IsBenchmarkJob(jobType string) bool { | ||||
| return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType | |||||
| return string(models.JobTypeModelSafety) == jobType || string(models.JobTypeBenchmark) == jobType || string(models.JobTypeBrainScore) == jobType || string(models.JobTypeSnn4imagenet) == jobType || string(models.JobTypeSnn4Ecoset) == jobType | |||||
| } | } | ||||
| func GetWaitingCloudbrainCount(cloudbrainType int, computeResource string, jobTypes ...models.JobType) int64 { | func GetWaitingCloudbrainCount(cloudbrainType int, computeResource string, jobTypes ...models.JobType) int64 { | ||||
| @@ -653,7 +664,7 @@ func IsElementExist(s []string, str string) bool { | |||||
| return false | return false | ||||
| } | } | ||||
| func GetCloudBrainByIdOrJobId(id string) (*models.Cloudbrain,error) { | |||||
| func GetCloudBrainByIdOrJobId(id string) (*models.Cloudbrain, error) { | |||||
| _, err := strconv.ParseInt(id, 10, 64) | _, err := strconv.ParseInt(id, 10, 64) | ||||
| var job *models.Cloudbrain | var job *models.Cloudbrain | ||||
| if err != nil { | if err != nil { | ||||
| @@ -661,10 +672,10 @@ func GetCloudBrainByIdOrJobId(id string) (*models.Cloudbrain,error) { | |||||
| job, err = models.GetCloudbrainByJobID(id) | job, err = models.GetCloudbrainByJobID(id) | ||||
| } else { | } else { | ||||
| job, err = models.GetCloudbrainByID(id) | job, err = models.GetCloudbrainByID(id) | ||||
| if err!=nil{ | |||||
| if err != nil { | |||||
| job, err = models.GetCloudbrainByJobID(id) | job, err = models.GetCloudbrainByJobID(id) | ||||
| } | } | ||||
| } | } | ||||
| return job,err | |||||
| return job, err | |||||
| } | } | ||||
| @@ -519,7 +519,6 @@ var ( | |||||
| CullIdleTimeout string | CullIdleTimeout string | ||||
| CullInterval string | CullInterval string | ||||
| //benchmark config | //benchmark config | ||||
| IsBenchmarkEnabled bool | IsBenchmarkEnabled bool | ||||
| BenchmarkOwner string | BenchmarkOwner string | ||||
| @@ -544,6 +543,12 @@ var ( | |||||
| BrainScoreName string | BrainScoreName string | ||||
| BrainScoreServerHost string | BrainScoreServerHost string | ||||
| IsSnn4EcosetEnabled bool | |||||
| Snn4EcosetOwner string | |||||
| Snn4EcosetName string | |||||
| Snn4EcosetServerHost string | |||||
| Snn4AttachmentName string | |||||
| //blockchain config | //blockchain config | ||||
| BlockChainHost string | BlockChainHost string | ||||
| CommitValidDate string | CommitValidDate string | ||||
| @@ -614,14 +619,14 @@ var ( | |||||
| UsageRateBeginTime string | UsageRateBeginTime string | ||||
| }{} | }{} | ||||
| ClearStrategy= struct { | |||||
| Enabled bool | |||||
| ResultSaveDays int | |||||
| BatchSize int | |||||
| DebugJobSize int | |||||
| TrashSaveDays int | |||||
| Cron string | |||||
| RunAtStart bool | |||||
| ClearStrategy = struct { | |||||
| Enabled bool | |||||
| ResultSaveDays int | |||||
| BatchSize int | |||||
| DebugJobSize int | |||||
| TrashSaveDays int | |||||
| Cron string | |||||
| RunAtStart bool | |||||
| }{} | }{} | ||||
| C2NetInfos *C2NetSqInfos | C2NetInfos *C2NetSqInfos | ||||
| @@ -1515,6 +1520,12 @@ func NewContext() { | |||||
| BrainScoreName = sec.Key("NAME").MustString("") | BrainScoreName = sec.Key("NAME").MustString("") | ||||
| BrainScoreServerHost = sec.Key("HOST").MustString("") | BrainScoreServerHost = sec.Key("HOST").MustString("") | ||||
| sec = Cfg.Section("snn4ecoset") | |||||
| IsSnn4EcosetEnabled = sec.Key("ENABLED").MustBool(false) | |||||
| Snn4EcosetOwner = sec.Key("OWNER").MustString("") | |||||
| Snn4EcosetName = sec.Key("NAME").MustString("") | |||||
| Snn4imagenetServerHost = sec.Key("HOST").MustString("") | |||||
| sec = Cfg.Section("blockchain") | sec = Cfg.Section("blockchain") | ||||
| BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/") | BlockChainHost = sec.Key("HOST").MustString("http://192.168.136.66:3302/") | ||||
| CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15") | CommitValidDate = sec.Key("COMMIT_VALID_DATE").MustString("2021-01-15") | ||||
| @@ -1691,16 +1702,16 @@ func getModelartsCDConfig() { | |||||
| getNotebookFlavorInfos() | getNotebookFlavorInfos() | ||||
| } | } | ||||
| func getClearStrategy(){ | |||||
| func getClearStrategy() { | |||||
| sec := Cfg.Section("clear_strategy") | sec := Cfg.Section("clear_strategy") | ||||
| ClearStrategy.Enabled=sec.Key("ENABLED").MustBool(false) | |||||
| ClearStrategy.ResultSaveDays=sec.Key("RESULT_SAVE_DAYS").MustInt(30) | |||||
| ClearStrategy.BatchSize=sec.Key("BATCH_SIZE").MustInt(500) | |||||
| ClearStrategy.DebugJobSize=sec.Key("DEBUG_BATCH_SIZE").MustInt(100) | |||||
| ClearStrategy.TrashSaveDays=sec.Key("TRASH_SAVE_DAYS").MustInt(90) | |||||
| ClearStrategy.Cron=sec.Key("CRON").MustString("* 0,30 2-8 * * ?") | |||||
| ClearStrategy.RunAtStart=sec.Key("RUN_AT_START").MustBool(false) | |||||
| ClearStrategy.Enabled = sec.Key("ENABLED").MustBool(false) | |||||
| ClearStrategy.ResultSaveDays = sec.Key("RESULT_SAVE_DAYS").MustInt(30) | |||||
| ClearStrategy.BatchSize = sec.Key("BATCH_SIZE").MustInt(500) | |||||
| ClearStrategy.DebugJobSize = sec.Key("DEBUG_BATCH_SIZE").MustInt(100) | |||||
| ClearStrategy.TrashSaveDays = sec.Key("TRASH_SAVE_DAYS").MustInt(90) | |||||
| ClearStrategy.Cron = sec.Key("CRON").MustString("* 0,30 2-8 * * ?") | |||||
| ClearStrategy.RunAtStart = sec.Key("RUN_AT_START").MustBool(false) | |||||
| } | } | ||||
| func getGrampusConfig() { | func getGrampusConfig() { | ||||
| @@ -53,7 +53,7 @@ func CloudBrains(ctx *context.Context) { | |||||
| var jobTypes []string | var jobTypes []string | ||||
| jobTypeNot := false | jobTypeNot := false | ||||
| if jobType == string(models.JobTypeBenchmark) { | if jobType == string(models.JobTypeBenchmark) { | ||||
| jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) | |||||
| jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)) | |||||
| } else if jobType != "all" && jobType != "" { | } else if jobType != "all" && jobType != "" { | ||||
| jobTypes = append(jobTypes, jobType) | jobTypes = append(jobTypes, jobType) | ||||
| } | } | ||||
| @@ -647,7 +647,7 @@ func GetAllCloudbrainsPeriodDistribution(ctx *context.Context) { | |||||
| } | } | ||||
| jobTypeList := []string{string(models.JobTypeDebug), string(models.JobTypeTrain), string(models.JobTypeInference), string(models.JobTypeBenchmark), | jobTypeList := []string{string(models.JobTypeDebug), string(models.JobTypeTrain), string(models.JobTypeInference), string(models.JobTypeBenchmark), | ||||
| string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)} | |||||
| string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)} | |||||
| for _, v := range jobTypeList { | for _, v := range jobTypeList { | ||||
| if _, ok := cloudOneJobTypeRes[v]; !ok { | if _, ok := cloudOneJobTypeRes[v]; !ok { | ||||
| cloudOneJobTypeRes[v] = 0 | cloudOneJobTypeRes[v] = 0 | ||||
| @@ -754,7 +754,7 @@ func GetCloudbrainsDetailData(ctx *context.Context) { | |||||
| var jobTypes []string | var jobTypes []string | ||||
| jobTypeNot := false | jobTypeNot := false | ||||
| if jobType == string(models.JobTypeBenchmark) { | if jobType == string(models.JobTypeBenchmark) { | ||||
| jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) | |||||
| jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)) | |||||
| } else if jobType != "all" && jobType != "" { | } else if jobType != "all" && jobType != "" { | ||||
| jobTypes = append(jobTypes, jobType) | jobTypes = append(jobTypes, jobType) | ||||
| } | } | ||||
| @@ -379,7 +379,6 @@ func cloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) { | |||||
| req.ModelVersion = form.ModelVersion | req.ModelVersion = form.ModelVersion | ||||
| req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl | req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl | ||||
| req.PreTrainModelUrl = form.PreTrainModelUrl | req.PreTrainModelUrl = form.PreTrainModelUrl | ||||
| } | } | ||||
| _, err = cloudbrain.GenerateTask(req) | _, err = cloudbrain.GenerateTask(req) | ||||
| @@ -711,7 +710,6 @@ func CloudBrainRestart(ctx *context.Context) { | |||||
| }) | }) | ||||
| } | } | ||||
| func getOldJobPath(task *models.Cloudbrain) string { | func getOldJobPath(task *models.Cloudbrain) string { | ||||
| return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + task.JobName | return setting.Attachment.Minio.RealPath + setting.Attachment.Minio.Bucket + "/" + setting.CBCodePathPrefix + task.JobName | ||||
| } | } | ||||
| @@ -786,7 +784,7 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo | |||||
| if task.JobType == string(models.JobTypeBenchmark) { | if task.JobType == string(models.JobTypeBenchmark) { | ||||
| task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") | task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") | ||||
| } else if task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) { | |||||
| } else if models.IsBenchMarkJobType(task.JobType) { | |||||
| task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") | task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") | ||||
| task.BenchmarkTypeName = task.JobType | task.BenchmarkTypeName = task.JobType | ||||
| ctx.Data["BenchmarkTypeName"] = task.JobType | ctx.Data["BenchmarkTypeName"] = task.JobType | ||||
| @@ -854,10 +852,10 @@ func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.Jo | |||||
| func CloudBrainDebug(ctx *context.Context) { | func CloudBrainDebug(ctx *context.Context) { | ||||
| task := ctx.Cloudbrain | task := ctx.Cloudbrain | ||||
| debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName | debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName | ||||
| if task.BootFile!=""{ | |||||
| ctx.Redirect(getFileUrl(debugUrl,task.BootFile)) | |||||
| if task.BootFile != "" { | |||||
| ctx.Redirect(getFileUrl(debugUrl, task.BootFile)) | |||||
| }else{ | |||||
| } else { | |||||
| ctx.Redirect(debugUrl) | ctx.Redirect(debugUrl) | ||||
| } | } | ||||
| @@ -1617,6 +1615,8 @@ func GetRate(ctx *context.Context) { | |||||
| ctx.Redirect(setting.Snn4imagenetServerHost) | ctx.Redirect(setting.Snn4imagenetServerHost) | ||||
| } else if job.JobType == string(models.JobTypeBrainScore) { | } else if job.JobType == string(models.JobTypeBrainScore) { | ||||
| ctx.Redirect(setting.BrainScoreServerHost) | ctx.Redirect(setting.BrainScoreServerHost) | ||||
| } else if job.JobType == string(models.JobTypeSnn4Ecoset) { | |||||
| ctx.Redirect(setting.Snn4EcosetServerHost) | |||||
| } else { | } else { | ||||
| log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"]) | log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"]) | ||||
| } | } | ||||
| @@ -2117,14 +2117,16 @@ func CloudBrainBenchmarkIndex(ctx *context.Context) { | |||||
| ciTasks[i].BenchmarkTypeName = "" | ciTasks[i].BenchmarkTypeName = "" | ||||
| if ciTasks[i].JobType == string(models.JobTypeBenchmark) { | if ciTasks[i].JobType == string(models.JobTypeBenchmark) { | ||||
| ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") | ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm") | ||||
| } else if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) || ciTasks[i].JobType == string(models.JobTypeBrainScore) { | |||||
| } else if models.IsBenchMarkJobType(ciTasks[i].JobType) { | |||||
| ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") | ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model") | ||||
| ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType | ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType | ||||
| if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) { | if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) { | ||||
| ciTasks[i].BenchmarkTypeRankLink = setting.Snn4imagenetServerHost | ciTasks[i].BenchmarkTypeRankLink = setting.Snn4imagenetServerHost | ||||
| } else { | |||||
| } else if ciTasks[i].JobType == string(models.JobTypeBrainScore) { | |||||
| ciTasks[i].BenchmarkTypeRankLink = setting.BrainScoreServerHost | ciTasks[i].BenchmarkTypeRankLink = setting.BrainScoreServerHost | ||||
| } else { | |||||
| ciTasks[i].BenchmarkTypeRankLink = setting.Snn4EcosetServerHost | |||||
| } | } | ||||
| } | } | ||||
| @@ -2474,7 +2476,6 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) | |||||
| displayJobName := form.DisplayJobName | displayJobName := form.DisplayJobName | ||||
| jobName := util.ConvertDisplayJobNameToJobName(displayJobName) | jobName := util.ConvertDisplayJobNameToJobName(displayJobName) | ||||
| image := form.Image | image := form.Image | ||||
| uuid := form.Attachment | |||||
| jobType := form.JobType | jobType := form.JobType | ||||
| codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath | codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath | ||||
| branchName := cloudbrain.DefaultBranchName | branchName := cloudbrain.DefaultBranchName | ||||
| @@ -2516,7 +2517,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) | |||||
| return | return | ||||
| } | } | ||||
| if jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) { | |||||
| if !models.IsBenchMarkJobType(jobType) { | |||||
| log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) | log.Error("jobtype error:", jobType, ctx.Data["MsgID"]) | ||||
| cloudBrainNewDataPrepare(ctx, jobType) | cloudBrainNewDataPrepare(ctx, jobType) | ||||
| ctx.RenderWithErr("jobtype error", tpl, &form) | ctx.RenderWithErr("jobtype error", tpl, &form) | ||||
| @@ -2560,14 +2561,33 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) | |||||
| benchmarkChildTypeID = form.BenchmarkChildTypeID | benchmarkChildTypeID = form.BenchmarkChildTypeID | ||||
| command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description)) | command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description)) | ||||
| } | } | ||||
| snn4EcosetPath := setting.JobPath + jobName + cloudbrain.Snn4EcosetMountPath | |||||
| var uuid string | |||||
| var datasetInfos map[string]models.DatasetInfo | |||||
| var datasetNames string | |||||
| if setting.IsSnn4EcosetEnabled && jobType == string(models.JobTypeSnn4Ecoset) { | |||||
| err = downloadRateCode(repo, jobName, setting.Snn4EcosetOwner, setting.Snn4EcosetName, snn4EcosetPath, "", "", ctx.User.Name) | |||||
| if err != nil { | |||||
| log.Error("load benchmark code failed", err) | |||||
| cloudBrainNewDataPrepare(ctx, jobType) | |||||
| ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) | |||||
| return | |||||
| } | |||||
| err = uploadCodeToMinio(snn4EcosetPath+"/", jobName, cloudbrain.Snn4EcosetMountPath+"/") | |||||
| command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description)) | |||||
| attachment, err := getEcosetAttachment() | |||||
| if err != nil { | |||||
| log.Error("load benchmark code failed", err) | |||||
| cloudBrainNewDataPrepare(ctx, jobType) | |||||
| ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form) | |||||
| return | |||||
| } | |||||
| uuid = attachment.UUID | |||||
| datasetInfos, datasetNames, _ = models.GetDatasetInfo(uuid) | |||||
| datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid) | |||||
| if err != nil { | |||||
| log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"]) | |||||
| cloudBrainNewDataPrepare(ctx, jobType) | |||||
| ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form) | |||||
| return | |||||
| } | } | ||||
| spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{ | spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{ | ||||
| JobType: models.JobTypeBenchmark, | JobType: models.JobTypeBenchmark, | ||||
| ComputeResource: models.GPU, | ComputeResource: models.GPU, | ||||
| @@ -2601,6 +2621,7 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) | |||||
| BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), | BenchmarkPath: storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"), | ||||
| Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), | Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"), | ||||
| BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), | BrainScorePath: storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"), | ||||
| Snn4EcosetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4EcosetMountPath+"/"), | |||||
| JobType: jobType, | JobType: jobType, | ||||
| Description: form.Description, | Description: form.Description, | ||||
| BranchName: branchName, | BranchName: branchName, | ||||
| @@ -2612,6 +2633,14 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) | |||||
| ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), | ResultPath: storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"), | ||||
| Spec: spec, | Spec: spec, | ||||
| } | } | ||||
| if form.ModelName != "" { | |||||
| req.ModelName = form.ModelName | |||||
| req.LabelName = form.LabelName | |||||
| req.CkptName = form.CkptName | |||||
| req.ModelVersion = form.ModelVersion | |||||
| req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl | |||||
| req.PreTrainModelUrl = form.PreTrainModelUrl | |||||
| } | |||||
| _, err = cloudbrain.GenerateTask(req) | _, err = cloudbrain.GenerateTask(req) | ||||
| if err != nil { | if err != nil { | ||||
| @@ -2623,6 +2652,21 @@ func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) | |||||
| ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") | ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark") | ||||
| } | } | ||||
| func getEcosetAttachment() (*models.Attachment, error) { | |||||
| ecosetRepo, err := models.GetRepositoryByOwnerAndName(setting.Snn4EcosetOwner, setting.Snn4EcosetName) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| datasetInfo, err := models.GetDatasetByRepo(ecosetRepo) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| return models.GetAttachmentByDatasetIdFileName(setting.Snn4AttachmentName, datasetInfo.ID) | |||||
| } | |||||
| func getBrainRegion(benchmarkChildTypeID int) string { | func getBrainRegion(benchmarkChildTypeID int) string { | ||||
| values := []string{"V1", "V2", "V4", "IT"} | values := []string{"V1", "V2", "V4", "IT"} | ||||
| return values[benchmarkChildTypeID] | return values[benchmarkChildTypeID] | ||||
| @@ -779,7 +779,7 @@ func Cloudbrains(ctx *context.Context) { | |||||
| var jobTypes []string | var jobTypes []string | ||||
| jobTypeNot := false | jobTypeNot := false | ||||
| if jobType == string(models.JobTypeBenchmark) { | if jobType == string(models.JobTypeBenchmark) { | ||||
| jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet)) | |||||
| jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeModelSafety), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet), string(models.JobTypeSnn4Ecoset)) | |||||
| } else if jobType != "all" && jobType != "" { | } else if jobType != "all" && jobType != "" { | ||||
| jobTypes = append(jobTypes, jobType) | jobTypes = append(jobTypes, jobType) | ||||
| } | } | ||||
| @@ -34,7 +34,7 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s | |||||
| ComputeResource: models.GPUResource, | ComputeResource: models.GPUResource, | ||||
| }, string(models.JobTypeBenchmark) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { | }, string(models.JobTypeBenchmark) + "-" + strconv.Itoa(models.TypeCloudBrainOne): { | ||||
| CloudBrainTypes: []int{models.TypeCloudBrainOne}, | CloudBrainTypes: []int{models.TypeCloudBrainOne}, | ||||
| JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet}, | |||||
| JobType: []models.JobType{models.JobTypeBenchmark, models.JobTypeBrainScore, models.JobTypeSnn4imagenet, models.JobTypeSnn4Ecoset}, | |||||
| NotFinalStatuses: CloudbrainOneNotFinalStatuses, | NotFinalStatuses: CloudbrainOneNotFinalStatuses, | ||||
| ComputeResource: models.GPUResource, | ComputeResource: models.GPUResource, | ||||
| }, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { | }, string(models.JobTypeDebug) + "-" + strconv.Itoa(models.TypeCloudBrainTwo): { | ||||
| @@ -66,7 +66,7 @@ var StatusInfoDict = map[string]StatusInfo{string(models.JobTypeDebug) + "-" + s | |||||
| func GetNotFinalStatusTaskCount(uid int64, cloudbrainType int, jobType string, computeResource ...string) (int, error) { | func GetNotFinalStatusTaskCount(uid int64, cloudbrainType int, jobType string, computeResource ...string) (int, error) { | ||||
| jobNewType := jobType | jobNewType := jobType | ||||
| if jobType == string(models.JobTypeSnn4imagenet) || jobType == string(models.JobTypeBrainScore) { | |||||
| if models.IsBenchMarkJobType(jobType) { | |||||
| jobNewType = string(models.JobTypeBenchmark) | jobNewType = string(models.JobTypeBenchmark) | ||||
| } | } | ||||