aiforge/routers/repo/cloudbrain.go

package repo

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"

	"code.gitea.io/gitea/services/cloudbrain/resource"

	"code.gitea.io/gitea/modules/notification"

	"code.gitea.io/gitea/modules/grampus"

	"code.gitea.io/gitea/modules/timeutil"
	"github.com/unknwon/i18n"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/auth"
	"code.gitea.io/gitea/modules/base"
	"code.gitea.io/gitea/modules/cloudbrain"
	"code.gitea.io/gitea/modules/context"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/modelarts"
	"code.gitea.io/gitea/modules/setting"
	"code.gitea.io/gitea/modules/storage"
	"code.gitea.io/gitea/modules/util"
)

const (
	tplCloudBrainNew        base.TplName = "repo/cloudbrain/new"
	tplCloudBrainShow       base.TplName = "repo/cloudbrain/show"
	tplCloudBrainShowModels base.TplName = "repo/cloudbrain/models/index"

	tplCloudBrainBenchmarkIndex base.TplName = "repo/cloudbrain/benchmark/index"
	tplCloudBrainBenchmarkNew   base.TplName = "repo/cloudbrain/benchmark/new"
	tplCloudBrainBenchmarkShow  base.TplName = "repo/cloudbrain/benchmark/show"

	tplCloudBrainImageSubmit base.TplName = "repo/cloudbrain/image/submit"
	tplCloudBrainImageEdit   base.TplName = "repo/cloudbrain/image/edit"

	tplCloudBrainTrainJobNew        base.TplName = "repo/cloudbrain/trainjob/new"
	tplCloudBrainTrainJobVersionNew base.TplName = "repo/cloudbrain/trainjob/versionnew"
	tplCloudBrainTrainJobShow       base.TplName = "repo/cloudbrain/trainjob/show"

	tplCloudBrainInferenceJobNew  base.TplName = "repo/cloudbrain/inference/new"
	tplCloudBrainInferenceJobShow base.TplName = "repo/cloudbrain/inference/show"
)

var (
	gpuInfos               *models.GpuInfos
	categories             *models.Categories
	benchmarkTypes         *models.BenchmarkTypes
	benchmarkGpuInfos      *models.GpuInfos
	benchmarkResourceSpecs *models.ResourceSpecs
	trainGpuInfos          *models.GpuInfos
	inferenceGpuInfos      *models.GpuInfos
)

const BENCHMARK_TYPE_CODE = "repo.cloudbrain.benchmark.types"
const CLONE_FILE_PREFIX = "file:///"

var benchmarkTypesMap = make(map[string]*models.BenchmarkTypes, 0)

var jobNamePattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-_]{1,34}[a-z0-9-]$`)

// MustEnableDataset check if repository enable internal cb
func MustEnableCloudbrain(ctx *context.Context) {
	if !ctx.Repo.CanRead(models.UnitTypeCloudBrain) {
		ctx.NotFound("MustEnableCloudbrain", nil)
		return
	}
}

func cutString(str string, lens int) string {
	if len(str) < lens {
		return str
	}
	return str[:lens]
}

func jobNamePrefixValid(s string) string {
	lowStr := strings.ToLower(s)
	re := regexp.MustCompile(`[^a-z0-9_\\-]+`)

	removeSpecial := re.ReplaceAllString(lowStr, "")

	re = regexp.MustCompile(`^[_\\-]+`)
	return re.ReplaceAllString(removeSpecial, "")

}

func cloudBrainNewDataPrepare(ctx *context.Context) error {
	ctx.Data["PageIsCloudBrain"] = true
	t := time.Now()
	var displayJobName = jobNamePrefixValid(cutString(ctx.User.Name, 5)) + t.Format("2006010215") + strconv.Itoa(int(t.Unix()))[5:]
	ctx.Data["display_job_name"] = displayJobName

	ctx.Data["command"] = cloudbrain.GetCloudbrainDebugCommand()
	ctx.Data["code_path"] = cloudbrain.CodeMountPath
	ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
	ctx.Data["model_path"] = cloudbrain.ModelMountPath
	ctx.Data["benchmark_path"] = cloudbrain.BenchMarkMountPath
	ctx.Data["is_benchmark_enabled"] = setting.IsBenchmarkEnabled

	if categories == nil {
		json.Unmarshal([]byte(setting.BenchmarkCategory), &categories)
	}
	ctx.Data["benchmark_categories"] = categories.Category

	ctx.Data["benchmark_types"] = GetBenchmarkTypes(ctx).BenchmarkType
	queuesDetail, _ := cloudbrain.GetQueuesDetail()
	if queuesDetail != nil {
		ctx.Data["QueuesDetail"] = queuesDetail
	}

	prepareCloudbrainOneSpecs(ctx)

	ctx.Data["params"] = ""
	ctx.Data["branchName"] = ctx.Repo.BranchName

	ctx.Data["snn4imagenet_path"] = cloudbrain.Snn4imagenetMountPath
	ctx.Data["is_snn4imagenet_enabled"] = setting.IsSnn4imagenetEnabled

	ctx.Data["brainscore_path"] = cloudbrain.BrainScoreMountPath
	ctx.Data["is_brainscore_enabled"] = setting.IsBrainScoreEnabled

	ctx.Data["datasetType"] = models.TypeCloudBrainOne

	ctx.Data["benchmarkMode"] = ctx.Query("benchmarkMode")

	if ctx.Cloudbrain != nil {
		ctx.Data["branch_name"] = ctx.Cloudbrain.BranchName
		ctx.Data["image_name"] = ctx.Cloudbrain.Image
		ctx.Data["image_id"] = ctx.Cloudbrain.ImageID
		ctx.Data["boot_file"] = ctx.Cloudbrain.BootFile
		ctx.Data["description"] = ctx.Cloudbrain.Description
		spec, _ := resource.GetCloudbrainSpec(ctx.Cloudbrain.ID)
		if spec != nil {
			ctx.Data["spec_id"] = spec.ID
		}
		ctx.Data["run_para_list"] = ctx.Cloudbrain.Parameters
		ctx.Data["model_name"] = ctx.Cloudbrain.ModelName
		ctx.Data["label_name"] = ctx.Cloudbrain.LabelName
		ctx.Data["ckpt_name"] = ctx.Cloudbrain.CkptName
		ctx.Data["model_version"] = ctx.Cloudbrain.ModelVersion
		ctx.Data["pre_train_model_url"] = ctx.Cloudbrain.PreTrainModelUrl
		ctx.Data["compute_resource"] = ctx.Cloudbrain.ComputeResource
		ctx.Data["attachment"] = ctx.Cloudbrain.Uuid
		ctx.Data["cluster_type"] = models.OpenICluster
		_, _, datasetNames, _, err := getDatasUrlListByUUIDS(ctx.Cloudbrain.Uuid)
		if err != nil {
			log.Info("query dataset error," + err.Error())
			ctx.Data["dataset_name"] = ""
		} else {
			ctx.Data["dataset_name"] = datasetNames
		}

	}

	return nil
}

func prepareCloudbrainOneSpecs(ctx *context.Context) {
	debugSpecs, _ := resource.FindAvailableSpecs(ctx.User.ID, models.FindSpecsOptions{
		JobType:         models.JobTypeDebug,
		ComputeResource: models.GPU,
		Cluster:         models.OpenICluster,
		AiCenterCode:    models.AICenterOfCloudBrainOne,
	})
	ctx.Data["debug_specs"] = debugSpecs

	trainSpecs, _ := resource.FindAvailableSpecs(ctx.User.ID, models.FindSpecsOptions{
		JobType:         models.JobTypeTrain,
		ComputeResource: models.GPU,
		Cluster:         models.OpenICluster,
		AiCenterCode:    models.AICenterOfCloudBrainOne,
	})
	ctx.Data["train_specs"] = trainSpecs

	inferenceSpecs, _ := resource.FindAvailableSpecs(ctx.User.ID, models.FindSpecsOptions{
		JobType:         models.JobTypeInference,
		ComputeResource: models.GPU,
		Cluster:         models.OpenICluster,
		AiCenterCode:    models.AICenterOfCloudBrainOne,
	})
	ctx.Data["inference_specs"] = inferenceSpecs

	benchmarkSpecs, _ := resource.FindAvailableSpecs(ctx.User.ID, models.FindSpecsOptions{
		JobType:         models.JobTypeBenchmark,
		ComputeResource: models.GPU,
		Cluster:         models.OpenICluster,
		AiCenterCode:    models.AICenterOfCloudBrainOne,
	})
	ctx.Data["benchmark_specs"] = benchmarkSpecs
}

func CloudBrainNew(ctx *context.Context) {
	err := cloudBrainNewDataPrepare(ctx)
	if err != nil {
		ctx.ServerError("get new cloudbrain info failed", err)
		return
	}
	ctx.Data["PageIsGPUDebug"] = true
	ctx.HTML(200, tplCloudBrainNew)
}

func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
	ctx.Data["PageIsCloudBrain"] = true
	displayJobName := form.DisplayJobName
	jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
	image := strings.TrimSpace(form.Image)
	uuids := form.Attachment
	jobType := form.JobType
	codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
	branchName := form.BranchName
	bootFile := strings.TrimSpace(form.BootFile)
	repo := ctx.Repo.Repository
	tpl := tplCloudBrainNew

	if jobType == string(models.JobTypeTrain) {
		tpl = tplCloudBrainTrainJobNew
	}

	var jobID = ctx.Params(":jobid")
	log.Info("jobIDTest= %s", jobID)
	// if jobID != "" {
	// 	tpl = tplCloudBrainTrainJobVersionNew
	// }

	tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName)
	if err == nil {
		if len(tasks) != 0 {
			log.Error("the job name did already exist", ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr("the job name did already exist", tpl, &form)
			return
		}
	} else {
		if !models.IsErrJobNotExist(err) {
			log.Error("system error, %v", err, ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr("system error", tpl, &form)
			return
		}
	}

	if !jobNamePattern.MatchString(displayJobName) {
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form)
		return
	}

	if jobType != string(models.JobTypeBenchmark) && jobType != string(models.JobTypeDebug) && jobType != string(models.JobTypeTrain) {
		log.Error("jobtype error:", jobType, ctx.Data["MsgID"])
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr("jobtype error", tpl, &form)
		return
	}

	count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType)
	if err != nil {
		log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr("system error", tpl, &form)
		return
	} else {
		if count >= 1 {
			log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form)
			return
		}
	}
	var datasetInfos map[string]models.DatasetInfo
	var datasetNames string
	//var
	if uuids != "" {
		datasetInfos, datasetNames, err = models.GetDatasetInfo(uuids)
		if err != nil {
			log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
			return
		}
	}

	command := cloudbrain.GetCloudbrainDebugCommand()
	if jobType == string(models.JobTypeTrain) {
		bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName)
		if err != nil || !bootFileExist {
			log.Error("Get bootfile error:", err, ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form)
			return
		}
		tpl = tplCloudBrainTrainJobNew
		commandTrain, err := getTrainJobCommand(form)
		if err != nil {
			log.Error("getTrainJobCommand failed: %v", err)
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr(err.Error(), tpl, &form)
			return
		}

		command = commandTrain
	}

	if branchName == "" {
		branchName = cloudbrain.DefaultBranchName
	}
	errStr := loadCodeAndMakeModelPath(repo, codePath, branchName, jobName, cloudbrain.ModelMountPath)
	if errStr != "" {
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr(ctx.Tr(errStr), tpl, &form)
		return
	}

	commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName)

	spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{
		JobType:         models.JobType(jobType),
		ComputeResource: models.GPU,
		Cluster:         models.OpenICluster,
		AiCenterCode:    models.AICenterOfCloudBrainOne})
	if err != nil || spec == nil {
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr("Resource specification not available", tpl, &form)
		return
	}

	req := cloudbrain.GenerateCloudBrainTaskReq{
		Ctx:                  ctx,
		DisplayJobName:       displayJobName,
		JobName:              jobName,
		Image:                image,
		Command:              command,
		Uuids:                uuids,
		DatasetNames:         datasetNames,
		DatasetInfos:         datasetInfos,
		CodePath:             storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
		ModelPath:            storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
		BenchmarkPath:        storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
		Snn4ImageNetPath:     storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
		BrainScorePath:       storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
		JobType:              jobType,
		Description:          form.Description,
		BranchName:           branchName,
		BootFile:             form.BootFile,
		Params:               form.Params,
		CommitID:             commitID,
		BenchmarkTypeID:      0,
		BenchmarkChildTypeID: 0,
		ResultPath:           storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"),
		Spec:                 spec,
	}

	if form.ModelName != "" { //使用预训练模型训练
		req.ModelName = form.ModelName
		req.LabelName = form.LabelName
		req.CkptName = form.CkptName
		req.ModelVersion = form.ModelVersion
		req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl
		req.PreTrainModelUrl = form.PreTrainModelUrl

	}

	err = cloudbrain.GenerateTask(req)
	if err != nil {
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr(err.Error(), tpl, &form)
		return
	}

	if jobType == string(models.JobTypeTrain) {
		ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=all")
	} else {
		ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=all")
	}
}

func CloudBrainTrainJobVersionCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
	CloudBrainCreate(ctx, form)
}

func loadCodeAndMakeModelPath(repo *models.Repository, codePath string, branchName string, jobName string, resultPath string) string {
	err := downloadCode(repo, codePath, branchName)
	if err != nil {
		return "cloudbrain.load_code_failed"
	}

	err = uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/")
	if err != nil {
		return "cloudbrain.load_code_failed"
	}

	modelPath := setting.JobPath + jobName + resultPath + "/"
	err = mkModelPath(modelPath)
	if err != nil {
		return "cloudbrain.load_code_failed"
	}
	err = uploadCodeToMinio(modelPath, jobName, resultPath+"/")
	if err != nil {
		return "cloudbrain.load_code_failed"
	}

	return ""
}

func CloudBrainInferenceJobCreate(ctx *context.Context, form auth.CreateCloudBrainInferencForm) {
	ctx.Data["PageIsCloudBrain"] = true
	displayJobName := form.DisplayJobName
	jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
	image := strings.TrimSpace(form.Image)
	uuid := form.Attachment
	jobType := string(models.JobTypeInference)
	codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
	branchName := form.BranchName
	bootFile := strings.TrimSpace(form.BootFile)
	labelName := form.LabelName
	repo := ctx.Repo.Repository

	ckptUrl := setting.Attachment.Minio.RealPath + form.TrainUrl + form.CkptName
	log.Info("ckpt url:" + ckptUrl)
	tpl := tplCloudBrainInferenceJobNew
	command, err := getInferenceJobCommand(form)
	if err != nil {
		log.Error("getTrainJobCommand failed: %v", err)
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr(err.Error(), tpl, &form)
		return
	}

	tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName)
	if err == nil {
		if len(tasks) != 0 {
			log.Error("the job name did already exist", ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr("the job name did already exist", tpl, &form)
			return
		}
	} else {
		if !models.IsErrJobNotExist(err) {
			log.Error("system error, %v", err, ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr("system error", tpl, &form)
			return
		}
	}

	if !jobNamePattern.MatchString(displayJobName) {
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form)
		return
	}

	bootFileExist, err := ctx.Repo.FileExists(bootFile, branchName)
	if err != nil || !bootFileExist {
		log.Error("Get bootfile error:", err, ctx.Data["MsgID"])
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_bootfile_err"), tpl, &form)
		return
	}

	count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, jobType)
	if err != nil {
		log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr("system error", tpl, &form)
		return
	} else {
		if count >= 1 {
			log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form)
			return
		}
	}

	if branchName == "" {
		branchName = cloudbrain.DefaultBranchName
	}
	errStr := loadCodeAndMakeModelPath(repo, codePath, branchName, jobName, cloudbrain.ResultPath)
	if errStr != "" {
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr(ctx.Tr(errStr), tpl, &form)
		return
	}

	commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName)

	datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid)
	if err != nil {
		log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
		return
	}
	spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{
		JobType:         models.JobTypeInference,
		ComputeResource: models.GPU,
		Cluster:         models.OpenICluster,
		AiCenterCode:    models.AICenterOfCloudBrainOne})
	if err != nil || spec == nil {
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr("Resource specification not available", tpl, &form)
		return
	}
	req := cloudbrain.GenerateCloudBrainTaskReq{
		Ctx:              ctx,
		DisplayJobName:   displayJobName,
		JobName:          jobName,
		Image:            image,
		Command:          command,
		Uuids:            uuid,
		DatasetNames:     datasetNames,
		DatasetInfos:     datasetInfos,
		CodePath:         storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
		ModelPath:        setting.Attachment.Minio.RealPath + form.TrainUrl,
		BenchmarkPath:    storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
		Snn4ImageNetPath: storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
		BrainScorePath:   storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
		JobType:          jobType,
		Description:      form.Description,
		BranchName:       branchName,
		BootFile:         form.BootFile,
		Params:           form.Params,
		CommitID:         commitID,
		ResultPath:       storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"),
		ModelName:        form.ModelName,
		ModelVersion:     form.ModelVersion,
		CkptName:         form.CkptName,
		TrainUrl:         form.TrainUrl,
		LabelName:        labelName,
		Spec:             spec,
	}

	err = cloudbrain.GenerateTask(req)
	if err != nil {
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr(err.Error(), tpl, &form)
		return
	}

	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/inference-job")

}

/**
  检查用户传输的参数是否符合专属资源池
*/
func checkCloudBrainSpecialPool(ctx *context.Context, jobType string, queue string, resourceSpecId int) string {
	if cloudbrain.SpecialPools != nil {

		var isInPoolOrg = false
		var matchSpecialPool = false

		for _, specialPool := range cloudbrain.SpecialPools.Pools {

			if cloudbrain.IsElementExist(specialPool.JobType, jobType) && cloudbrain.IsQueueInSpecialtPool(specialPool.Pool, queue) {
				if cloudbrain.IsResourceSpecInSpecialPool(specialPool.ResourceSpec, resourceSpecId) {
					matchSpecialPool = true
					org, _ := models.GetOrgByName(specialPool.Org)
					if org != nil {
						isInPoolOrg, _ = models.IsOrganizationMember(org.ID, ctx.User.ID)
						if isInPoolOrg {
							break //传入参数，和专属资源池匹配上了，检查通过
						}
					}
				}

			}

		}
		//资源池有匹配上，但是用户不在相应的组织中，返回错误信息。界面已经过滤了选择，界面操作不会到这个逻辑
		if matchSpecialPool && !isInPoolOrg {
			return ctx.Tr("repo.grampus.no_operate_right")
		}

	}
	//没有匹配到资源池或者没有设置专属资源池，检查通过; 获取和资源池完全匹配检查通过
	return ""
}

func CloudBrainRestart(ctx *context.Context) {
	var ID = ctx.Params(":id")
	var resultCode = "0"
	var errorMsg = ""
	var status = string(models.JobWaiting)
	task := ctx.Cloudbrain
	for {
		if task.Status != string(models.JobStopped) && task.Status != string(models.JobSucceeded) && task.Status != string(models.JobFailed) {
			log.Error("the job(%s) is not stopped", task.JobName, ctx.Data["MsgID"])
			resultCode = "-1"
			errorMsg = "the job is not stopped"
			break
		}

		if task.Image == "" || task.GpuQueue == "" || task.Type != models.TypeCloudBrainOne {
			log.Error("the job(%s) version is too old", task.JobName, ctx.Data["MsgID"])
			resultCode = "-1"
			errorMsg = "the job's version is too old and can not be restarted"
			break
		}

		if !ctx.IsSigned || (ctx.User.ID != task.UserID && !ctx.IsUserSiteAdmin()) {
			log.Error("the user has no right ro restart the job", task.JobName, ctx.Data["MsgID"])
			resultCode = "-1"
			errorMsg = "you have no right to restart the job"
			break
		}

		specOld, err := resource.GetCloudbrainSpec(task.ID)
		if err != nil || specOld == nil {
			log.Error("CloudBrainRestart GetCloudbrainSpec error.task.id = %d", task.ID)
			resultCode = "-1"
			errorMsg = "Resource specification not support any more"
			break
		}
		spec, err := resource.GetAndCheckSpec(ctx.User.ID, specOld.ID, models.FindSpecsOptions{
			JobType:         models.JobType(task.JobType),
			ComputeResource: models.GPU,
			Cluster:         models.OpenICluster,
			AiCenterCode:    models.AICenterOfCloudBrainOne})
		if err != nil || spec == nil {
			log.Error("CloudBrainRestart GetAndCheckSpec error.task.id = %d", task.ID)
			resultCode = "-1"
			errorMsg = "Resource specification not support any more"
			break
		}
		task.Spec = spec

		count, err := models.GetCloudbrainCountByUserID(ctx.User.ID, string(models.JobTypeDebug))
		if err != nil {
			log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
			resultCode = "-1"
			errorMsg = "system error"
			break
		} else {
			if count >= 1 {
				log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
				resultCode = "-1"
				errorMsg = ctx.Tr("repo.cloudbrain.morethanonejob")
				break
			}
		}

		err = cloudbrain.RestartTask(ctx, task, &ID)
		if err != nil {
			log.Error("RestartTask failed:%v", err.Error(), ctx.Data["MsgID"])
			resultCode = "-1"
			errorMsg = "system error"
			break
		}

		break
	}

	ctx.JSON(200, map[string]string{
		"result_code": resultCode,
		"error_msg":   errorMsg,
		"status":      status,
		"id":          ID,
	})
}

func CloudBrainBenchMarkShow(ctx *context.Context) {
	cloudBrainShow(ctx, tplCloudBrainBenchmarkShow, models.JobTypeBenchmark)
}

func CloudBrainShow(ctx *context.Context) {
	cloudBrainShow(ctx, tplCloudBrainShow, models.JobTypeDebug)
}

func CloudBrainTrainJobShow(ctx *context.Context) {
	cloudBrainShow(ctx, tplCloudBrainTrainJobShow, models.JobTypeTrain)
}

func cloudBrainShow(ctx *context.Context, tpName base.TplName, jobType models.JobType) {
	ctx.Data["PageIsCloudBrain"] = true
	debugListType := ctx.Query("debugListType")
	cloudbrain.InitSpecialPool()

	var task *models.Cloudbrain
	var err error
	if jobType == models.JobTypeTrain || jobType == models.JobTypeInference {
		task, err = models.GetCloudbrainByJobID(ctx.Params(":jobid"))
	} else {
		task, err = models.GetCloudbrainByIDWithDeleted(ctx.Params(":id"))
	}

	if err != nil {
		log.Info("error:" + err.Error())
		ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
		return
	}

	result, err := cloudbrain.GetJob(task.JobID)
	if err != nil {
		log.Info("error:" + err.Error())
		ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
		return
	}
	prepareSpec4Show(ctx, task)
	if ctx.Written() {
		return
	}

	if result != nil {
		jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
		taskRoles := jobRes.TaskRoles
		taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
		ctx.Data["taskRes"] = taskRes
		ctx.Data["ExitDiagnostics"] = taskRes.TaskStatuses[0].ExitDiagnostics
		oldStatus := task.Status
		task.Status = taskRes.TaskStatuses[0].State
		task.ContainerID = taskRes.TaskStatuses[0].ContainerID
		task.ContainerIp = taskRes.TaskStatuses[0].ContainerIP
		models.ParseAndSetDurationFromCloudBrainOne(jobRes, task)

		if task.DeletedAt.IsZero() { //normal record
			if oldStatus != task.Status {
				notification.NotifyChangeCloudbrainStatus(task, oldStatus)
			}
			err = models.UpdateJob(task)
			if err != nil {
				ctx.Data["error"] = err.Error()
				return
			}
		} else { //deleted record

		}

		ctx.Data["result"] = jobRes
	} else {
		log.Info("error:" + err.Error())
		return
	}

	user, err := models.GetUserByID(task.UserID)
	if err == nil {
		task.User = user
	}

	if task.BenchmarkTypeID > 0 {
		for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
			if task.BenchmarkTypeID == benchmarkType.Id {
				ctx.Data["BenchmarkTypeName"] = benchmarkType.First
				task.BenchmarkTypeName = benchmarkType.First
				for _, benchmarkChildType := range benchmarkType.Second {
					if task.BenchmarkChildTypeID == benchmarkChildType.Id {
						ctx.Data["BenchmarkChildTypeName"] = benchmarkChildType.Value
						break
					}
				}
				break
			}
		}
	}

	if task.JobType == string(models.JobTypeBenchmark) {
		task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm")
	} else if task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) {
		task.BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model")
		task.BenchmarkTypeName = task.JobType
		ctx.Data["BenchmarkTypeName"] = task.JobType
		if task.JobType == string(models.JobTypeBrainScore) {
			ctx.Data["BenchmarkChildTypeName"] = getBrainRegion(task.BenchmarkChildTypeID)
		}

	}

	if task.TrainJobDuration == "" {
		if task.Duration == 0 {
			var duration int64
			if task.Status == string(models.JobWaiting) {
				duration = 0
			} else if task.Status == string(models.JobRunning) {
				duration = time.Now().Unix() - int64(task.CreatedUnix)
			} else {
				duration = int64(task.UpdatedUnix) - int64(task.CreatedUnix)
			}
			task.Duration = duration
		}
		task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
	}
	ctx.Data["duration"] = task.TrainJobDuration

	if len(task.Parameters) > 0 {
		var parameters models.Parameters

		err := json.Unmarshal([]byte(task.Parameters), &parameters)
		if err != nil {
			log.Error("Failed to Unmarshal Parameters: %s (%v)", task.Parameters, err)
			task.Parameters = ""
		} else {
			if len(parameters.Parameter) > 0 {
				paramTemp := ""
				for _, Parameter := range parameters.Parameter {
					param := Parameter.Label + " = " + Parameter.Value + "; "
					paramTemp = paramTemp + param
				}
				task.Parameters = paramTemp[:len(paramTemp)-2]
			} else {
				task.Parameters = ""
			}
		}

	}
	ctx.Data["datasetDownload"] = GetCloudBrainDataSetInfo(task.Uuid, task.DatasetName, false)
	ctx.Data["task"] = task
	labelName := strings.Fields(task.LabelName)
	ctx.Data["LabelName"] = labelName
	ctx.Data["jobName"] = task.JobName
	ctx.Data["displayJobName"] = task.DisplayJobName
	version_list_task := make([]*models.Cloudbrain, 0)
	version_list_task = append(version_list_task, task)
	ctx.Data["version_list_task"] = version_list_task
	ctx.Data["debugListType"] = debugListType
	ctx.Data["code_path"] = cloudbrain.CodeMountPath
	ctx.Data["dataset_path"] = cloudbrain.DataSetMountPath
	ctx.Data["model_path"] = cloudbrain.ModelMountPath
	ctx.Data["canDownload"] = cloudbrain.CanModifyJob(ctx, task)
	ctx.Data["branchName"] = task.BranchName
	ctx.HTML(200, tpName)
}

func CloudBrainDebug(ctx *context.Context) {
	task := ctx.Cloudbrain
	debugUrl := setting.DebugServerHost + "jpylab_" + task.JobID + "_" + task.SubTaskName
	ctx.Redirect(debugUrl)
}

func prepareSpec4Show(ctx *context.Context, task *models.Cloudbrain) {
	s, err := resource.GetCloudbrainSpec(task.ID)
	if err != nil {
		log.Info("error:" + err.Error())
		ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
		return
	}
	ctx.Data["Spec"] = s
}

func oldPrepareSpec4Show(ctx *context.Context, task *models.Cloudbrain) {
	hasSpec := false
	if task.JobType == string(models.JobTypeTrain) {
		if cloudbrain.TrainResourceSpecs == nil {
			json.Unmarshal([]byte(setting.TrainResourceSpecs), &cloudbrain.TrainResourceSpecs)
		}

		for _, tmp := range cloudbrain.TrainResourceSpecs.ResourceSpec {
			if tmp.Id == task.ResourceSpecId {
				hasSpec = true
				ctx.Data["GpuNum"] = tmp.GpuNum
				ctx.Data["CpuNum"] = tmp.CpuNum
				ctx.Data["MemMiB"] = tmp.MemMiB
				ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB
				break
			}
		}

	} else if task.JobType == string(models.JobTypeInference) {
		if cloudbrain.InferenceResourceSpecs == nil {
			json.Unmarshal([]byte(setting.InferenceResourceSpecs), &cloudbrain.InferenceResourceSpecs)
		}
		for _, tmp := range cloudbrain.InferenceResourceSpecs.ResourceSpec {
			if tmp.Id == task.ResourceSpecId {
				hasSpec = true
				ctx.Data["GpuNum"] = tmp.GpuNum
				ctx.Data["CpuNum"] = tmp.CpuNum
				ctx.Data["MemMiB"] = tmp.MemMiB
				ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB
				break
			}
		}
	} else {
		if cloudbrain.ResourceSpecs == nil {
			json.Unmarshal([]byte(setting.ResourceSpecs), &cloudbrain.ResourceSpecs)
		}
		for _, tmp := range cloudbrain.ResourceSpecs.ResourceSpec {
			if tmp.Id == task.ResourceSpecId {
				hasSpec = true
				ctx.Data["GpuNum"] = tmp.GpuNum
				ctx.Data["CpuNum"] = tmp.CpuNum
				ctx.Data["MemMiB"] = tmp.MemMiB
				ctx.Data["ShareMemMiB"] = tmp.ShareMemMiB
				break

			}
		}
	}

	if !hasSpec && cloudbrain.SpecialPools != nil {

		for _, specialPool := range cloudbrain.SpecialPools.Pools {

			if specialPool.ResourceSpec != nil {

				for _, spec := range specialPool.ResourceSpec {
					if task.ResourceSpecId == spec.Id {
						ctx.Data["GpuNum"] = spec.GpuNum
						ctx.Data["CpuNum"] = spec.CpuNum
						ctx.Data["MemMiB"] = spec.MemMiB
						ctx.Data["ShareMemMiB"] = spec.ShareMemMiB
						break
					}
				}
			}
		}
	}
}

func CloudBrainCommitImageShow(ctx *context.Context) {
	ctx.Data["PageIsCloudBrain"] = true
	ctx.Data["Type"] = ctx.Cloudbrain.Type
	ctx.HTML(200, tplCloudBrainImageSubmit)
}

func GetImage(ctx *context.Context) {

	var ID = ctx.Params(":id")
	id, _ := strconv.ParseInt(ID, 10, 64)

	image, err := models.GetImageByID(id)
	if err != nil {
		log.Error("GetImageByID failed:%v", err.Error())
		ctx.JSON(http.StatusNotFound, nil)
	}
	ctx.JSON(http.StatusOK, image)

}

func CloudBrainImageEdit(ctx *context.Context) {
	ctx.Data["PageIsImageEdit"] = true
	ctx.Data["PageFrom"] = ctx.Params(":from")
	var ID = ctx.Params(":id")
	id, err := strconv.ParseInt(ID, 10, 64)
	if err != nil {
		log.Error("GetImageByID failed:%v", err.Error())
		ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
	}
	image, err := models.GetImageByID(id)
	if err != nil {
		log.Error("GetImageByID failed:%v", err.Error())
		ctx.NotFound(ctx.Req.URL.RequestURI(), nil)
	}
	ctx.Data["Image"] = image
	ctx.HTML(http.StatusOK, tplCloudBrainImageEdit)

}

func CloudBrainImageEditPost(ctx *context.Context, form auth.EditImageCloudBrainForm) {

	if utf8.RuneCountInString(form.Description) > 255 {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 255)))
		return
	}

	validTopics, errMessage := checkTopics(form.Topics)
	if errMessage != "" {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr(errMessage)))
		return
	}
	image, err := models.GetImageByID(form.ID)
	if err != nil {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_not_exist")))

	}

	image.IsPrivate = form.IsPrivate
	image.Description = form.Description

	err = models.WithTx(func(ctx models.DBContext) error {
		if err := models.UpdateLocalImage(image); err != nil {
			return err
		}
		if err := models.SaveImageTopics(image.ID, validTopics...); err != nil {
			return err
		}
		return nil

	})

	if err != nil {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_not_exist")))

	} else {
		ctx.JSON(http.StatusOK, models.BaseOKMessage)
	}

}

func CloudBrainImageDelete(ctx *context.Context) {
	var ID = ctx.Params(":id")
	id, err := strconv.ParseInt(ID, 10, 64)
	if err != nil {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_not_exist")))
		return
	}

	err = models.DeleteLocalImage(id)
	if err != nil {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_delete_fail")))
	} else {
		ctx.JSON(http.StatusOK, models.BaseOKMessage)
	}

}

func CloudBrainCommitImageCheck(ctx *context.Context, form auth.CommitImageCloudBrainForm) {
	isExist, _ := models.IsImageExistByUser(form.Tag, ctx.User.ID)
	if isExist {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("repo.image_overwrite")))
	} else {
		ctx.JSON(http.StatusOK, models.BaseOKMessage)
	}

}

func CloudBrainAdminCommitImage(ctx *context.Context, form auth.CommitAdminImageCloudBrainForm) {

	if !NamePattern.MatchString(form.Tag) {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err")))
		return
	}

	if utf8.RuneCountInString(form.Description) > 255 {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 255)))
		return
	}

	validTopics, errMessage := checkTopics(form.Topics)
	if errMessage != "" {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr(errMessage)))
		return
	}

	err := cloudbrain.CommitAdminImage(models.CommitImageParams{
		CommitImageCloudBrainParams: models.CommitImageCloudBrainParams{
			ImageDescription: form.Description,
			ImageTag:         form.Tag,
		},
		IsPrivate:      form.IsPrivate,
		CloudBrainType: form.Type,
		Topics:         validTopics,
		UID:            ctx.User.ID,
		Type:           models.GetRecommondType(form.IsRecommend),
		Place:          form.Place,
	})
	if err != nil {
		log.Error("CommitImagefailed")
		if models.IsErrImageTagExist(err) {
			ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_exist")))

		} else if models.IsErrorImageCommitting(err) {
			ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_committing")))
		} else {
			ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_commit_fail")))
		}

		return
	}

	ctx.JSON(200, models.BaseOKMessage)
}

func CloudBrainCommitImage(ctx *context.Context, form auth.CommitImageCloudBrainForm) {

	if !NamePattern.MatchString(form.Tag) {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.title_format_err")))
		return
	}

	if utf8.RuneCountInString(form.Description) > 255 {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr("dataset.description_format_err", 255)))
		return
	}

	validTopics, errMessage := checkTopics(form.Topics)
	if errMessage != "" {
		ctx.JSON(http.StatusOK, models.BaseErrorMessage(ctx.Tr(errMessage)))
		return
	}

	err := cloudbrain.CommitImage(ctx.Cloudbrain.JobID, models.CommitImageParams{
		CommitImageCloudBrainParams: models.CommitImageCloudBrainParams{
			Ip:               ctx.Cloudbrain.ContainerIp,
			TaskContainerId:  ctx.Cloudbrain.ContainerID,
			ImageDescription: form.Description,
			ImageTag:         form.Tag,
		},
		IsPrivate:      form.IsPrivate,
		CloudBrainType: form.Type,
		Topics:         validTopics,
		UID:            ctx.User.ID,
	})
	if err != nil {
		log.Error("CommitImage(%s) failed:%v", ctx.Cloudbrain.JobName, err.Error(), ctx.Data["msgID"])
		if models.IsErrImageTagExist(err) {
			ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_exist")))

		} else if models.IsErrorImageCommitting(err) {
			ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_committing")))
		} else {
			ctx.JSON(200, models.BaseErrorMessage(ctx.Tr("repo.image_commit_fail")))
		}

		return
	}

	ctx.JSON(200, models.BaseOKMessage)
}

func checkTopics(Topics string) ([]string, string) {
	var topics = make([]string, 0)
	var topicsStr = strings.TrimSpace(Topics)
	if len(topicsStr) > 0 {
		topics = strings.Split(topicsStr, ",")
	}

	validTopics, invalidTopics := models.SanitizeAndValidateImageTopics(topics)

	if len(validTopics) > 25 {
		return nil, "repo.topic.count_prompt"

	}

	if len(invalidTopics) > 0 {
		return nil, "repo.imagetopic.format_prompt"

	}
	return validTopics, ""
}

func CloudBrainStop(ctx *context.Context) {
	var ID = ctx.Params(":id")
	var resultCode = "0"
	var errorMsg = ""
	var status = ""

	task := ctx.Cloudbrain
	for {
		if task.Status == string(models.JobStopped) || task.Status == string(models.JobFailed) || task.Status == string(models.JobSucceeded) {
			log.Error("the job(%s) has been stopped", task.JobName, ctx.Data["msgID"])
			resultCode = "-1"
			errorMsg = "cloudbrain.Already_stopped"
			break
		}

		err := cloudbrain.StopJob(task.JobID)
		if err != nil {
			log.Error("StopJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
			resultCode = "-1"
			errorMsg = "cloudbrain.Stopped_failed"
			break
		}
		oldStatus := task.Status
		task.Status = string(models.JobStopped)
		if task.EndTime == 0 {
			task.EndTime = timeutil.TimeStampNow()
		}
		task.ComputeAndSetDuration()
		if oldStatus != task.Status {
			notification.NotifyChangeCloudbrainStatus(task, oldStatus)
		}
		err = models.UpdateJob(task)
		if err != nil {
			log.Error("UpdateJob(%s) failed:%v", task.JobName, err, ctx.Data["msgID"])
			resultCode = "-1"
			errorMsg = "cloudbrain.Stopped_success_update_status_fail"
			break
		}

		status = task.Status
		break
	}

	ctx.JSON(200, map[string]interface{}{
		"result_code": resultCode,
		"error_msg":   ctx.Tr(errorMsg),
		"status":      status,
		"id":          ID,
		"StatusOK":    0,
	})
}

func StopJobsByUserID(userID int64) {
	cloudBrains, err := models.GetCloudbrainsNeededStopByUserID(userID)
	if err != nil {
		log.Warn("Failed to get cloudBrain info", err)
		return
	}
	StopJobs(cloudBrains)

}

func StopJobsByRepoID(repoID int64) {
	cloudBrains, err := models.GetCloudbrainsNeededStopByRepoID(repoID)
	if err != nil {
		log.Warn("Failed to get cloudBrain info", err)
		return
	}
	StopJobs(cloudBrains)
}

func DeleteJobsByRepoID(repoID int64) {
	cloudBrains, err := models.GetCloudbrainsNeededDeleteByRepoID(repoID)
	if err != nil {
		log.Warn("Failed to get cloudBrain info", err)
		return
	}
	DeleteJobs(cloudBrains)
}

/**

 */
func StopJobs(cloudBrains []*models.Cloudbrain) {

	for _, taskInfo := range cloudBrains {

		if taskInfo.Type == models.TypeCloudBrainOne {
			err := retry(3, time.Second*30, func() error {
				return cloudbrain.StopJob(taskInfo.JobID)
			})

			logErrorAndUpdateJobStatus(err, taskInfo)
		} else {
			if taskInfo.JobType == string(models.JobTypeTrain) {
				err := retry(3, time.Second*30, func() error {
					_, err := modelarts.StopTrainJob(taskInfo.JobID, strconv.FormatInt(taskInfo.VersionID, 10))
					return err
				})
				logErrorAndUpdateJobStatus(err, taskInfo)
			} else {
				param := models.NotebookAction{
					Action: models.ActionStop,
				}
				err := retry(3, time.Second*30, func() error {
					_, err := modelarts.ManageNotebook(taskInfo.JobID, param)
					return err
				})
				logErrorAndUpdateJobStatus(err, taskInfo)
			}
		}

	}
}

func DeleteJobs(cloudBrains []*models.Cloudbrain) {
	for _, taskInfo := range cloudBrains {
		err := models.DeleteJob(taskInfo)
		if err != nil {
			log.Warn("Failed to DeleteJob:", err)
			return
		}
		if taskInfo.Type == models.TypeCloudBrainOne {
			cloudbrain.DelCloudBrainJob(taskInfo.JobName)
			DeleteCloudbrainJobStorage(taskInfo.JobName, models.TypeCloudBrainOne)
		}
		if taskInfo.Type == models.TypeCloudBrainTwo {
			if taskInfo.JobType == string(models.JobTypeTrain) || taskInfo.JobType == string(models.JobTypeInference) {

				modelarts.DelTrainJob(taskInfo.JobID)
				DeleteJobStorage(taskInfo.JobName)
			}
			if taskInfo.JobType == string(models.JobTypeDebug) {
				modelarts.DelNotebook2(taskInfo.JobID)
			}
		}
		if taskInfo.Type == models.TypeC2Net {
			if taskInfo.JobType == string(models.JobTypeTrain) {
				cloudbrain.DelCloudBrainJob(taskInfo.JobName)
				DeleteCloudbrainJobStorage(taskInfo.JobName, models.TypeCloudBrainOne)
			}
		}
	}
}

func retry(attempts int, sleep time.Duration, f func() error) (err error) {
	for i := 0; i < attempts; i++ {
		if i > 0 {
			log.Warn("retrying after error:", err)
			time.Sleep(sleep)
		}
		err = f()
		if err == nil {
			return nil
		}
	}
	return fmt.Errorf("after %d attempts, last error: %s", attempts, err)
}

func logErrorAndUpdateJobStatus(err error, taskInfo *models.Cloudbrain) {
	if err != nil {
		log.Warn("Failed to stop cloudBrain job:"+taskInfo.JobID, err)
	} else {
		oldStatus := taskInfo.Status
		taskInfo.Status = string(models.JobStopped)
		if taskInfo.EndTime == 0 {
			taskInfo.EndTime = timeutil.TimeStampNow()
		}
		taskInfo.ComputeAndSetDuration()
		if oldStatus != taskInfo.Status {
			notification.NotifyChangeCloudbrainStatus(taskInfo, oldStatus)
		}
		err = models.UpdateJob(taskInfo)
		if err != nil {
			log.Warn("UpdateJob failed", err)
		}
	}
}

func CloudBrainDel(ctx *context.Context) {
	var listType = ctx.Query("debugListType")
	if err := deleteCloudbrainJob(ctx); err != nil {
		log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"])
		ctx.ServerError(err.Error(), err)
		return
	}

	var isAdminPage = ctx.Query("isadminpage")
	var isHomePage = ctx.Query("ishomepage")
	if ctx.IsUserSiteAdmin() && isAdminPage == "true" {
		ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains")
	} else if isHomePage == "true" {
		ctx.Redirect(setting.AppSubURL + "/cloudbrains")
	} else {
		ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/debugjob?debugListType=" + listType)
	}
}

func deleteCloudbrainJob(ctx *context.Context) error {
	task := ctx.Cloudbrain

	if task.Status != string(models.JobStopped) && task.Status != string(models.JobFailed) && task.Status != string(models.JobSucceeded) {
		log.Error("the job(%s) has not been stopped", task.JobName, ctx.Data["msgID"])
		return errors.New("the job has not been stopped")
	}

	err := models.DeleteJob(task)
	if err != nil {
		log.Error("DeleteJob failed: %v", err, ctx.Data["msgID"])
		return err
	}

	DeleteCloudbrainJobStorage(task.JobName, models.TypeCloudBrainOne)

	return nil
}

func CloudBrainShowModels(ctx *context.Context) {
	ctx.Data["PageIsCloudBrain"] = true

	ID := ctx.Params(":id")
	parentDir := ctx.Query("parentDir")
	dirArray := strings.Split(parentDir, "/")
	task, err := models.GetCloudbrainByID(ID)
	if err != nil {
		log.Error("no such job!", ctx.Data["msgID"])
		ctx.ServerError("no such job:", err)
		return
	}

	//get dirs
	dirs, err := GetModelDirs(task.JobName, parentDir)
	if err != nil {
		log.Error("GetModelDirs failed:%v", err.Error(), ctx.Data["msgID"])
		ctx.ServerError("GetModelDirs failed:", err)
		return
	}

	var fileInfos []storage.FileInfo
	err = json.Unmarshal([]byte(dirs), &fileInfos)
	if err != nil {
		log.Error("json.Unmarshal failed:%v", err.Error(), ctx.Data["msgID"])
		ctx.ServerError("json.Unmarshal failed:", err)
		return
	}

	for i, fileInfo := range fileInfos {
		temp, _ := time.Parse("2006-01-02 15:04:05", fileInfo.ModTime)
		fileInfos[i].ModTime = temp.Local().Format("2006-01-02 15:04:05")
	}

	sort.Slice(fileInfos, func(i, j int) bool {
		return fileInfos[i].ModTime > fileInfos[j].ModTime
	})

	ctx.Data["Path"] = dirArray
	ctx.Data["Dirs"] = fileInfos
	ctx.Data["task"] = task
	ctx.Data["ID"] = ID
	ctx.HTML(200, tplCloudBrainShowModels)
}

func GetPublicImages(ctx *context.Context) {
	uid := getUID(ctx)
	opts := models.SearchImageOptions{
		IncludePublicOnly:   true,
		UID:                 uid,
		Keyword:             ctx.Query("q"),
		Topics:              ctx.Query("topic"),
		IncludeOfficialOnly: ctx.QueryBool("recommend"),
		SearchOrderBy:       "type desc, num_stars desc,id desc",
		Status:              models.IMAGE_STATUS_SUCCESS,
		CloudbrainType:      ctx.QueryInt("cloudbrainType"),
	}

	getImages(ctx, &opts)

}

func GetCustomImages(ctx *context.Context) {
	uid := getUID(ctx)
	opts := models.SearchImageOptions{
		UID:              uid,
		IncludeOwnerOnly: true,
		Keyword:          ctx.Query("q"),
		Topics:           ctx.Query("topic"),
		Status:           -1,
		SearchOrderBy:    "id desc",
	}
	getImages(ctx, &opts)

}
func GetStarImages(ctx *context.Context) {

	uid := getUID(ctx)
	opts := models.SearchImageOptions{
		UID:             uid,
		IncludeStarByMe: true,
		Keyword:         ctx.Query("q"),
		Topics:          ctx.Query("topic"),
		Status:          models.IMAGE_STATUS_SUCCESS,
		SearchOrderBy:   "id desc",
	}
	getImages(ctx, &opts)

}

func getUID(ctx *context.Context) int64 {
	var uid int64 = -1
	if ctx.IsSigned {
		uid = ctx.User.ID
	}
	return uid
}

func GetAllImages(ctx *context.Context) {
	uid := getUID(ctx)
	opts := models.SearchImageOptions{
		UID:                 uid,
		Keyword:             ctx.Query("q"),
		Topics:              ctx.Query("topic"),
		IncludeOfficialOnly: ctx.QueryBool("recommend"),
		SearchOrderBy:       "id desc",
		Status:              -1,
	}

	if ctx.Query("private") != "" {
		if ctx.QueryBool("private") {
			opts.IncludePrivateOnly = true
		} else {
			opts.IncludePublicOnly = true
		}
	}
	getImages(ctx, &opts)

}

func getImages(ctx *context.Context, opts *models.SearchImageOptions) {
	page := ctx.QueryInt("page")
	if page <= 0 {
		page = 1
	}

	pageSize := ctx.QueryInt("pageSize")
	if pageSize <= 0 {
		pageSize = 15
	}
	opts.ListOptions = models.ListOptions{
		Page:     page,
		PageSize: pageSize,
	}
	imageList, total, err := models.SearchImage(opts)
	if err != nil {
		log.Error("Can not get images:%v", err)
		ctx.JSON(http.StatusOK, models.ImagesPageResult{
			Count:  0,
			Images: []*models.Image{},
		})
	} else {
		ctx.JSON(http.StatusOK, models.ImagesPageResult{
			Count:  total,
			Images: imageList,
		})
	}
}

func GetModelDirs(jobName string, parentDir string) (string, error) {
	var req string
	modelActualPath := storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/")
	if parentDir == "" {
		req = "baseDir=" + modelActualPath
	} else {
		req = "baseDir=" + modelActualPath + "&parentDir=" + parentDir
	}

	return getDirs(req)
}

func GetResultDirs(jobName string, parentDir string) (string, error) {
	var req string
	modelActualPath := storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/")
	if parentDir == "" {
		req = "baseDir=" + modelActualPath
	} else {
		req = "baseDir=" + modelActualPath + "&parentDir=" + parentDir
	}

	return getDirs(req)
}

func CloudBrainDownloadModel(ctx *context.Context) {
	parentDir := ctx.Query("parentDir")
	fileName := ctx.Query("fileName")
	jobName := ctx.Query("jobName")
	filePath := "jobs/" + jobName + "/model/" + parentDir
	url, err := storage.Attachments.PresignedGetURL(filePath, fileName)
	if err != nil {
		log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"])
		ctx.ServerError("PresignedGetURL", err)
		return
	}
	ctx.Resp.Header().Set("Cache-Control", "max-age=0")
	http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
}
func CloudBrainDownloadInferenceResult(ctx *context.Context) {
	parentDir := ctx.Query("parentDir")
	fileName := ctx.Query("fileName")
	jobName := ctx.Query("jobName")
	filePath := "jobs/" + jobName + "/result/" + parentDir
	url, err := storage.Attachments.PresignedGetURL(filePath, fileName)
	if err != nil {
		log.Error("PresignedGetURL failed: %v", err.Error(), ctx.Data["msgID"])
		ctx.ServerError("PresignedGetURL", err)
		return
	}
	ctx.Resp.Header().Set("Cache-Control", "max-age=0")
	http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
}

func GetRate(ctx *context.Context) {
	isObjectDetcionAll := ctx.QueryBool("isObjectDetcionAll")
	if isObjectDetcionAll {
		ctx.Redirect(setting.BenchmarkServerHost + "?username=admin")
		return
	}

	var ID = ctx.Params(":id")
	job, err := models.GetCloudbrainByID(ID)
	if err != nil {
		ctx.ServerError("GetCloudbrainByJobID failed", err)
		return
	}

	if job.JobType == string(models.JobTypeBenchmark) {
		log.Info("url=" + setting.BenchmarkServerHost + "?username=" + ctx.User.Name)
		ctx.Redirect(setting.BenchmarkServerHost + "?username=" + ctx.User.Name)
	} else if job.JobType == string(models.JobTypeSnn4imagenet) {
		ctx.Redirect(setting.Snn4imagenetServerHost)
	} else if job.JobType == string(models.JobTypeBrainScore) {
		ctx.Redirect(setting.BrainScoreServerHost)
	} else {
		log.Error("JobType error:%s", job.JobType, ctx.Data["msgID"])
	}
}

func downloadCode(repo *models.Repository, codePath, branchName string) error {
	//add "file:///" prefix to make the depth valid
	if err := git.Clone(CLONE_FILE_PREFIX+repo.RepoPath(), codePath, git.CloneRepoOptions{Branch: branchName, Depth: 1}); err != nil {
		log.Error("Failed to clone repository: %s (%v)", repo.FullName(), err)
		return err
	}

	configFile, err := os.OpenFile(codePath+"/.git/config", os.O_RDWR, 0666)
	if err != nil {
		log.Error("open file(%s) failed:%v", codePath+"/,git/config", err)
		return err
	}

	defer configFile.Close()

	pos := int64(0)
	reader := bufio.NewReader(configFile)
	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				log.Error("not find the remote-url")
				return nil
			} else {
				log.Error("read error: %v", err)
				return err
			}
		}

		if strings.Contains(line, "url") && strings.Contains(line, ".git") {
			originUrl := "\turl = " + repo.CloneLink().HTTPS + "\n"
			if len(line) > len(originUrl) {
				originUrl += strings.Repeat(" ", len(line)-len(originUrl))
			}
			bytes := []byte(originUrl)
			_, err := configFile.WriteAt(bytes, pos)
			if err != nil {
				log.Error("WriteAt failed:%v", err)
				return err
			}
			break
		}

		pos += int64(len(line))
	}

	return nil
}

func downloadRateCode(repo *models.Repository, taskName, rateOwnerName, rateRepoName, codePath, benchmarkCategory, gpuType, userName string) error {
	err := os.MkdirAll(codePath, os.ModePerm)
	if err != nil {
		log.Error("mkdir codePath failed", err.Error())
		return err
	}

	repoExt, err := models.GetRepositoryByOwnerAndName(rateOwnerName, rateRepoName)
	if err != nil {
		log.Error("GetRepositoryByOwnerAndName(%s) failed", rateRepoName, err.Error())
		return err
	}

	if err := git.Clone(CLONE_FILE_PREFIX+repoExt.RepoPath(), codePath, git.CloneRepoOptions{Depth: 1}); err != nil {
		log.Error("Failed to clone repository: %s (%v)", repoExt.FullName(), err)
		return err
	}

	fileName := codePath + cloudbrain.TaskInfoName
	f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
	if err != nil {
		log.Error("OpenFile failed", err.Error())
		return err
	}

	defer f.Close()

	data, err := json.Marshal(models.TaskInfo{
		Username:          userName,
		TaskName:          taskName,
		CodeName:          repo.Name,
		BenchmarkCategory: strings.Split(benchmarkCategory, ","),
		CodeLink:          strings.TrimSuffix(repo.CloneLink().HTTPS, ".git"),
		GpuType:           gpuType,
	})
	if err != nil {
		log.Error("json.Marshal failed", err.Error())
		return err
	}

	_, err = f.Write(data)
	if err != nil {
		log.Error("WriteString failed", err.Error())
		return err
	}

	return nil
}

func uploadCodeToMinio(codePath, jobName, parentDir string) error {
	files, err := readDir(codePath)
	if err != nil {
		log.Error("readDir(%s) failed: %s", codePath, err.Error())
		return err
	}

	for _, file := range files {
		if file.IsDir() {
			if err = uploadCodeToMinio(codePath+file.Name()+"/", jobName, parentDir+file.Name()+"/"); err != nil {
				log.Error("uploadCodeToMinio(%s) failed: %s", file.Name(), err.Error())
				return err
			}
		} else {
			destObject := setting.CBCodePathPrefix + jobName + parentDir + file.Name()
			sourceFile := codePath + file.Name()
			err = storage.Attachments.UploadObject(destObject, sourceFile)
			if err != nil {
				log.Error("UploadObject(%s) failed: %s", file.Name(), err.Error())
				return err
			}
		}
	}

	return nil
}

func mkModelPath(modelPath string) error {
	return mkPathAndReadMeFile(modelPath, "You can put the files into this directory and download the files by the web page.")
}

func mkPathAndReadMeFile(path string, text string) error {
	err := os.MkdirAll(path, os.ModePerm)
	if err != nil {
		log.Error("MkdirAll(%s) failed:%v", path, err)
		return err
	}

	fileName := path + "README"
	f, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm)
	if err != nil {
		log.Error("OpenFile failed", err.Error())
		return err
	}

	defer f.Close()

	_, err = f.WriteString(text)
	if err != nil {
		log.Error("WriteString failed", err.Error())
		return err
	}

	return nil
}

func DeleteCloudbrainJobStorage(jobName string, cloudbrainType int) error {
	//delete local
	localJobPath := setting.JobPath + jobName
	err := os.RemoveAll(localJobPath)
	if err != nil {
		log.Error("RemoveAll(%s) failed:%v", localJobPath, err)
	}

	//delete oss
	if cloudbrainType == models.TypeCloudBrainOne {
		dirPath := setting.CBCodePathPrefix + jobName + "/"
		err = storage.Attachments.DeleteDir(dirPath)
		if err != nil {
			log.Error("DeleteDir(%s) failed:%v", localJobPath, err)
		}
	} else if cloudbrainType == models.TypeCloudBrainTwo {
		//dirPath := setting.CodePathPrefix + jobName + "/"
		//err = storage.ObsRemoveObject(setting.Bucket, dirPath)
		//if err != nil {
		//	log.Error("ObsRemoveObject(%s) failed:%v", localJobPath, err)
		//}
		log.Info("no need to delete")
	} else {
		log.Error("cloudbrainType(%d) error", cloudbrainType)
	}

	return nil
}

func SyncCloudbrainStatus() {
	cloudBrains, err := models.GetCloudBrainUnStoppedJob()
	if err != nil {
		log.Error("GetCloudBrainUnStoppedJob failed:", err.Error())
		return
	}

	for _, task := range cloudBrains {
		if task.Type == models.TypeCloudBrainOne {
			result, err := cloudbrain.GetJob(task.JobID)
			if err != nil {
				log.Error("GetJob(%s) failed:%v", task.JobName, err)
				continue
			}

			if result != nil {
				jobRes, _ := models.ConvertToJobResultPayload(result.Payload)
				taskRoles := jobRes.TaskRoles
				taskRes, _ := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
				oldStatus := task.Status
				task.Status = taskRes.TaskStatuses[0].State
				if task.Status != string(models.JobWaiting) {
					models.ParseAndSetDurationFromCloudBrainOne(jobRes, task)
					if oldStatus != task.Status {
						notification.NotifyChangeCloudbrainStatus(task, oldStatus)
					}
					err = models.UpdateJob(task)
					if err != nil {
						log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
					}

					var maxDuration int64
					if task.JobType == string(models.JobTypeBenchmark) {
						maxDuration = setting.BenchmarkMaxDuration
					} else if task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) {
						maxDuration = setting.ModelBenchmarkMaxDuration
					} else {
						maxDuration = setting.MaxDuration
					}

					if task.Duration >= maxDuration && task.JobType != string(models.JobTypeTrain) {
						log.Info("begin to stop job(%s), because of the duration", task.DisplayJobName)
						err = cloudbrain.StopJob(task.JobID)
						if err != nil {
							log.Error("StopJob(%s) failed:%v", task.DisplayJobName, err)
							continue
						}
						task.Status = string(models.JobStopped)
						if task.EndTime == 0 {
							task.EndTime = timeutil.TimeStampNow()
						}
						task.ComputeAndSetDuration()
						if oldStatus != task.Status {
							notification.NotifyChangeCloudbrainStatus(task, oldStatus)
						}
						err = models.UpdateJob(task)
						if err != nil {
							log.Error("UpdateJob(%s) failed:%v", task.DisplayJobName, err)
							continue
						}
					}
				}
			}
		} else if task.Type == models.TypeCloudBrainTwo {
			if task.JobType == string(models.JobTypeDebug) {
				err := modelarts.HandleNotebookInfo(task)
				if err != nil {
					log.Error("HandleNotebookInfo(%s) failed:%v", task.DisplayJobName, err)
					continue
				}
			} else if task.JobType == string(models.JobTypeTrain) || task.JobType == string(models.JobTypeInference) {
				err := modelarts.HandleTrainJobInfo(task)
				if err != nil {
					log.Error("HandleTrainJobInfo(%s) failed:%v", task.DisplayJobName, err)
					continue
				}
			} else {
				log.Error("task.JobType(%s) is error:%s", task.DisplayJobName, task.JobType)
			}
		} else if task.Type == models.TypeC2Net {
			result, err := grampus.GetJob(task.JobID)
			if err != nil {
				log.Error("GetTrainJob(%s) failed:%v", task.DisplayJobName, err)
				continue
			}

			if result != nil {
				if len(result.JobInfo.Tasks[0].CenterID) == 1 && len(result.JobInfo.Tasks[0].CenterName) == 1 {
					task.AiCenter = result.JobInfo.Tasks[0].CenterID[0] + "+" + result.JobInfo.Tasks[0].CenterName[0]
				}
				oldStatus := task.Status
				task.Status = grampus.TransTrainJobStatus(result.JobInfo.Status)
				task.Duration = result.JobInfo.RunSec
				task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)

				if task.StartTime == 0 && result.JobInfo.StartedAt > 0 {
					task.StartTime = timeutil.TimeStamp(result.JobInfo.StartedAt)
				}
				if task.EndTime == 0 && models.IsTrainJobTerminal(task.Status) && task.StartTime > 0 {
					task.EndTime = task.StartTime.Add(task.Duration)
				}
				task.CorrectCreateUnix()
				if oldStatus != task.Status {
					notification.NotifyChangeCloudbrainStatus(task, oldStatus)
				}
				err = models.UpdateJob(task)
				if err != nil {
					log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
					continue
				}
			}
		} else {
			log.Error("task.Type(%s) is error:%d", task.JobName, task.Type)
		}
	}

	return
}

func HandleTaskWithNoDuration(ctx *context.Context) {
	mode := ctx.Query("mode")
	log.Info("HandleTaskWithNoDuration start")
	count := 0
	start := time.Now().Unix()
	for {
		var cloudBrains []*models.Cloudbrain
		var err error
		if mode == "1" {
			cloudBrains, err = models.GetStoppedJobWithNoStartTimeEndTime()
		} else {
			cloudBrains, err = models.GetStoppedJobWithNoDurationJob()
		}

		if err != nil {
			log.Error("HandleTaskWithNoTrainJobDuration failed:", err.Error())
			break
		}
		if len(cloudBrains) == 0 {
			log.Info("HandleTaskWithNoTrainJobDuration:no task need handle")
			break
		}
		handleNoDurationTask(cloudBrains)
		count += len(cloudBrains)
		if len(cloudBrains) < 100 {
			log.Info("HandleTaskWithNoTrainJobDuration:task less than 100")
			break
		}
		if time.Now().Unix()-start > 600 {
			log.Info("HandleTaskWithNoDuration : time out")
			ctx.JSON(200, fmt.Sprintf("task stop for time out,count=%d", count))
			return
		}
	}
	log.Info("HandleTaskWithNoTrainJobDuration:count=%d", count)
	ctx.JSON(200, fmt.Sprintf("success,count=%d", count))
}

func handleNoDurationTask(cloudBrains []*models.Cloudbrain) {
	for _, task := range cloudBrains {
		time.Sleep(time.Millisecond * 100)
		log.Info("Handle job ,%+v", task)
		if task.Type == models.TypeCloudBrainOne {
			result, err := cloudbrain.GetJob(task.JobID)
			if err != nil {
				log.Error("GetJob(%s) failed:%v", task.JobName, err)
				updateDefaultDuration(task)
				continue
			}

			if result != nil {
				if result.Msg != "success" {
					updateDefaultDuration(task)
					continue
				}
				jobRes, err := models.ConvertToJobResultPayload(result.Payload)
				if err != nil || len(jobRes.TaskRoles) == 0 {
					updateDefaultDuration(task)
					continue
				}
				taskRoles := jobRes.TaskRoles
				taskRes, err := models.ConvertToTaskPod(taskRoles[cloudbrain.SubTaskName].(map[string]interface{}))
				if err != nil || len(taskRes.TaskStatuses) == 0 {
					updateDefaultDuration(task)
					continue
				}
				task.Status = taskRes.TaskStatuses[0].State
				log.Info("task startTime = %v  endTime= %v ,jobId=%d", jobRes.JobStatus.StartTime, jobRes.JobStatus.EndTime, task.ID)
				if jobRes.JobStatus.CreatedTime > 0 {
					task.StartTime = timeutil.TimeStamp(jobRes.JobStatus.CreatedTime / 1000)
					if jobRes.JobStatus.CompletedTime > 0 {
						task.EndTime = timeutil.TimeStamp(jobRes.JobStatus.CompletedTime / 1000)
					} else {
						task.EndTime = task.UpdatedUnix
					}
				} else {
					task.StartTime = 0
					task.EndTime = 0
				}

				if task.EndTime < task.StartTime {
					log.Info("endTime[%v] is less than starTime[%v],jobId=%d", task.EndTime, task.StartTime, task.ID)
					st := task.StartTime
					task.StartTime = task.EndTime
					task.EndTime = st
				}
				task.Duration = task.EndTime.AsTime().Unix() - task.StartTime.AsTime().Unix()
				task.TrainJobDuration = models.ConvertDurationToStr(task.Duration)
				err = models.UpdateJobDurationWithDeleted(task)
				if err != nil {
					log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
				}
			}
		} else if task.Type == models.TypeCloudBrainTwo {
			if task.JobType == string(models.JobTypeDebug) {
				//result, err := modelarts.GetJob(task.JobID)
				result, err := modelarts.GetNotebook2(task.JobID)
				if err != nil {
					log.Error("GetJob(%s) failed:%v", task.JobName, err)
					updateDefaultDuration(task)
					continue
				}

				if result != nil {
					task.Status = result.Status
					startTime := result.Lease.CreateTime
					duration := result.Lease.Duration / 1000
					if startTime > 0 {
						task.StartTime = timeutil.TimeStamp(startTime / 1000)
						task.EndTime = task.StartTime.Add(duration)
					}
					task.CorrectCreateUnix()
					task.ComputeAndSetDuration()
					err = models.UpdateJobDurationWithDeleted(task)
					if err != nil {
						log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
						continue
					}
				}
			} else if task.JobType == string(models.JobTypeTrain) || task.JobType == string(models.JobTypeInference) {
				result, err := modelarts.GetTrainJob(task.JobID, strconv.FormatInt(task.VersionID, 10))
				if err != nil {
					log.Error("GetTrainJob(%s) failed:%v", task.JobName, err)
					updateDefaultDuration(task)
					continue
				}

				if result != nil {
					startTime := result.StartTime / 1000
					if startTime > 0 {
						task.StartTime = timeutil.TimeStamp(startTime)
						task.EndTime = task.StartTime.Add(result.Duration / 1000)
					}
					task.ComputeAndSetDuration()
					err = models.UpdateJobDurationWithDeleted(task)
					if err != nil {
						log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
						continue
					}
				}
			} else {
				log.Error("task.JobType(%s) is error:%s", task.JobName, task.JobType)
			}

		} else {
			log.Error("task.Type(%s) is error:%d", task.JobName, task.Type)
		}
	}
}

func updateDefaultDuration(task *models.Cloudbrain) {
	log.Info("updateDefaultDuration: taskId=%d", task.ID)
	task.StartTime = task.CreatedUnix
	task.EndTime = task.UpdatedUnix
	task.ComputeAndSetDuration()
	err := models.UpdateJobDurationWithDeleted(task)
	if err != nil {
		log.Error("UpdateJob(%s) failed:%v", task.JobName, err)
	}
}

func CloudBrainBenchmarkIndex(ctx *context.Context) {
	MustEnableCloudbrain(ctx)
	repo := ctx.Repo.Repository
	page := ctx.QueryInt("page")
	if page <= 0 {
		page = 1
	}

	var jobTypes []string
	jobTypes = append(jobTypes, string(models.JobTypeBenchmark), string(models.JobTypeBrainScore), string(models.JobTypeSnn4imagenet))
	ciTasks, count, err := models.Cloudbrains(&models.CloudbrainsOptions{
		ListOptions: models.ListOptions{
			Page:     page,
			PageSize: setting.UI.IssuePagingNum,
		},
		RepoID:   repo.ID,
		Type:     models.TypeCloudBrainOne,
		JobTypes: jobTypes,
	})
	if err != nil {
		ctx.ServerError("Get debugjob faild:", err)
		return
	}

	for i, task := range ciTasks {
		ciTasks[i].CanDel = cloudbrain.CanDeleteJob(ctx, &task.Cloudbrain)
		ciTasks[i].Cloudbrain.ComputeResource = task.ComputeResource
		if ciTasks[i].TrainJobDuration == "" {
			if ciTasks[i].Duration == 0 {
				var duration int64
				if task.Status == string(models.JobRunning) {
					duration = time.Now().Unix() - int64(task.Cloudbrain.CreatedUnix)
				} else {
					duration = int64(task.Cloudbrain.UpdatedUnix) - int64(task.Cloudbrain.CreatedUnix)
				}
				ciTasks[i].Duration = duration
			}
			ciTasks[i].TrainJobDuration = models.ConvertDurationToStr(ciTasks[i].Duration)
		}
		ciTasks[i].BenchmarkTypeName = ""
		if ciTasks[i].JobType == string(models.JobTypeBenchmark) {
			ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.algorithm")
		} else if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) || ciTasks[i].JobType == string(models.JobTypeBrainScore) {
			ciTasks[i].BenchmarkType = ctx.Tr("repo.cloudbrain.benchmark.model")
			ciTasks[i].BenchmarkTypeName = ciTasks[i].JobType

			if ciTasks[i].JobType == string(models.JobTypeSnn4imagenet) {
				ciTasks[i].BenchmarkTypeRankLink = setting.Snn4imagenetServerHost
			} else {
				ciTasks[i].BenchmarkTypeRankLink = setting.BrainScoreServerHost
			}

		}

		if task.BenchmarkTypeID > 0 {
			for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
				if task.BenchmarkTypeID == benchmarkType.Id {
					ciTasks[i].BenchmarkTypeRankLink = benchmarkType.RankLink
					ciTasks[i].BenchmarkTypeName = benchmarkType.First
					break
				}
			}
		}
	}

	pager := context.NewPagination(int(count), setting.UI.IssuePagingNum, page, 5)

	ctx.Data["Page"] = pager
	ctx.Data["PageIsCloudBrain"] = true
	ctx.Data["Tasks"] = ciTasks
	ctx.Data["CanCreate"] = cloudbrain.CanCreateOrDebugJob(ctx)
	ctx.Data["RepoIsEmpty"] = repo.IsEmpty
	ctx.HTML(200, tplCloudBrainBenchmarkIndex)
}

func GetChildTypes(ctx *context.Context) {
	benchmarkTypeID := ctx.QueryInt("benchmark_type_id")
	re := make(map[string]interface{})
	for {
		var isExist bool
		for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
			if benchmarkTypeID == benchmarkType.Id {
				isExist = true
				re["child_types"] = benchmarkType.Second
				re["result_code"] = "0"
				break
			}
		}
		if !isExist {
			re["result_code"] = "1"
			log.Error("no such benchmark_type_id", ctx.Data["MsgID"])
			re["errMsg"] = "system error"
			break
		}
		break
	}
	ctx.JSON(200, re)
}

func CloudBrainBenchmarkNew(ctx *context.Context) {
	ctx.Data["description"] = ""
	ctx.Data["benchmarkTypeID"] = -1
	ctx.Data["benchmark_child_types_id_hidden"] = -1
	err := cloudBrainNewDataPrepare(ctx)
	if err != nil {
		ctx.ServerError("get new cloudbrain info failed", err)
		return
	}
	ctx.HTML(200, tplCloudBrainBenchmarkNew)
}

func getBenchmarkAttachment(benchmarkTypeID, benchmarkChildTypeID int, ctx *context.Context) (*models.BenchmarkDataset, error) {
	var childInfo *models.BenchmarkDataset

	var isExist bool
	for _, benchmarkType := range GetBenchmarkTypes(ctx).BenchmarkType {
		if benchmarkType.Id == benchmarkTypeID {
			for _, childType := range benchmarkType.Second {
				if childType.Id == benchmarkChildTypeID {
					childInfo = childType
					isExist = true
					break
				}
			}
			break
		}
	}

	if !isExist {
		log.Error("no such benchmark_type_id&benchmark_child_type_id")
		return childInfo, errors.New("no such benchmark_type_id&benchmark_child_type_id")
	}

	return childInfo, nil
}

func getBenchmarkGpuQueue(gpuQueue string) (string, error) {
	queue := ""
	if benchmarkGpuInfos == nil {
		if err := json.Unmarshal([]byte(setting.BenchmarkGpuTypes), &benchmarkGpuInfos); err != nil {
			log.Error("json.Unmarshal BenchmarkGpuTypes(%s) failed:%v", setting.BenchmarkGpuTypes, err)
			return queue, err
		}
	}

	var isExist bool
	for _, gpuInfo := range benchmarkGpuInfos.GpuInfo {
		if gpuQueue == gpuInfo.Queue {
			isExist = true
			queue = gpuQueue
			break
		}
	}

	if !isExist {
		log.Error("no such gpuQueue, %s", gpuQueue)
		return queue, errors.New("no such gpuQueue")
	}

	return queue, nil
}

func getBenchmarkResourceSpec(resourceSpecID int) (int, error) {
	var id int
	if benchmarkResourceSpecs == nil {
		if err := json.Unmarshal([]byte(setting.BenchmarkResourceSpecs), &benchmarkResourceSpecs); err != nil {
			log.Error("json.Unmarshal BenchmarkResourceSpecs(%s) failed:%v", setting.BenchmarkResourceSpecs, err)
			return id, err
		}
	}

	var isExist bool
	for _, resourceSpec := range benchmarkResourceSpecs.ResourceSpec {
		if resourceSpecID == resourceSpec.Id {
			isExist = true
			id = resourceSpecID
			break
		}
	}

	if !isExist {
		log.Error("no such resourceSpecID, %d", resourceSpecID)
		return id, errors.New("no such resourceSpec")
	}

	return id, nil
}

func CloudBrainBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
	jobType := form.JobType
	if jobType == string(models.JobTypeBenchmark) {
		BenchMarkAlgorithmCreate(ctx, form)
	} else {
		ModelBenchmarkCreate(ctx, form)
	}

}

func BenchMarkAlgorithmCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
	ctx.Data["PageIsCloudBrain"] = true
	displayJobName := form.DisplayJobName
	jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
	image := strings.TrimSpace(form.Image)
	command := cloudbrain.CommandBenchmark
	codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
	benchmarkTypeID := form.BenchmarkTypeID
	benchmarkChildTypeID := form.BenchmarkChildTypeID

	ctx.Data["description"] = form.Description
	ctx.Data["benchmarkTypeID"] = benchmarkTypeID
	ctx.Data["benchmark_child_types_id_hidden"] = benchmarkChildTypeID

	repo := ctx.Repo.Repository

	tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, string(models.JobTypeBenchmark), displayJobName)
	if err == nil {
		if len(tasks) != 0 {
			log.Error("the job name did already exist", ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr("the job name did already exist", tplCloudBrainBenchmarkNew, &form)
			return
		}
	} else {
		if !models.IsErrJobNotExist(err) {
			log.Error("system error, %v", err, ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
			return
		}
	}

	if !jobNamePattern.MatchString(jobName) {
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tplCloudBrainBenchmarkNew, &form)
		return
	}

	childInfo, err := getBenchmarkAttachment(benchmarkTypeID, benchmarkChildTypeID, ctx)
	if err != nil {
		log.Error("getBenchmarkAttachment failed:%v", err, ctx.Data["MsgID"])
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr("benchmark type error", tplCloudBrainBenchmarkNew, &form)
		return
	}

	spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{
		JobType:         models.JobTypeBenchmark,
		ComputeResource: models.GPU,
		Cluster:         models.OpenICluster,
		AiCenterCode:    models.AICenterOfCloudBrainOne})
	if err != nil || spec == nil {
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr("Resource specification not available", tplCloudBrainBenchmarkNew, &form)
		return
	}

	count, err := models.GetBenchmarkCountByUserID(ctx.User.ID)
	if err != nil {
		log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
		return
	} else {
		if count >= 1 {
			log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tplCloudBrainBenchmarkNew, &form)
			return
		}
	}

	os.RemoveAll(codePath)
	if err := downloadCode(repo, codePath, cloudbrain.DefaultBranchName); err != nil {
		log.Error("downloadCode failed, %v", err, ctx.Data["MsgID"])
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
		return
	}

	if _, err := os.Stat(codePath + "/train.py"); err != nil {
		if os.IsNotExist(err) {
			// file does not exist
			log.Error("train.py does not exist, %v", err, ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr("train.py does not exist", tplCloudBrainBenchmarkNew, &form)
		} else {
			log.Error("Stat failed, %v", err, ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
		}
		return
	} else if _, err := os.Stat(codePath + "/test.py"); err != nil {
		if os.IsNotExist(err) {
			// file does not exist
			log.Error("test.py does not exist, %v", err, ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr("test.py does not exist", tplCloudBrainBenchmarkNew, &form)
		} else {
			log.Error("Stat failed, %v", err, ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
		}
		return
	}

	if err := uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/"); err != nil {
		log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"])
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
		return
	}

	benchmarkPath := setting.JobPath + jobName + cloudbrain.BenchMarkMountPath

	if err := downloadRateCode(repo, jobName, childInfo.Owner, childInfo.RepoName, benchmarkPath, form.BenchmarkCategory, spec.AccCardType, ctx.User.Name); err != nil {
		log.Error("downloadRateCode failed, %v", err, ctx.Data["MsgID"])
		//cloudBrainNewDataPrepare(ctx)
		//ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
		//return
	}

	if err := uploadCodeToMinio(benchmarkPath+"/", jobName, cloudbrain.BenchMarkMountPath+"/"); err != nil {
		log.Error("uploadCodeToMinio failed, %v", err, ctx.Data["MsgID"])
		//cloudBrainNewDataPrepare(ctx)
		//ctx.RenderWithErr("system error", tplCloudBrainBenchmarkNew, &form)
		//return
	}

	uuid := childInfo.Attachment
	datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid)
	if err != nil {
		log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tplCloudBrainBenchmarkNew, &form)
		return
	}
	log.Info("Command=" + command)
	log.Info("ModelPath=" + storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"))
	req := cloudbrain.GenerateCloudBrainTaskReq{
		Ctx:                  ctx,
		DisplayJobName:       displayJobName,
		JobName:              jobName,
		Image:                image,
		Command:              command,
		Uuids:                uuid,
		DatasetNames:         datasetNames,
		DatasetInfos:         datasetInfos,
		CodePath:             storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
		ModelPath:            storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
		BenchmarkPath:        storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
		Snn4ImageNetPath:     storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
		BrainScorePath:       storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
		JobType:              string(models.JobTypeBenchmark),
		Description:          form.Description,
		BranchName:           cloudbrain.DefaultBranchName,
		BootFile:             "",
		Params:               "",
		CommitID:             "",
		BenchmarkTypeID:      benchmarkTypeID,
		BenchmarkChildTypeID: benchmarkChildTypeID,
		ResultPath:           storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"),
		Spec:                 spec,
	}

	err = cloudbrain.GenerateTask(req)
	if err != nil {
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr(err.Error(), tplCloudBrainBenchmarkNew, &form)
		return
	}
	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")

}

func ModelBenchmarkCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
	ctx.Data["PageIsCloudBrain"] = true
	displayJobName := form.DisplayJobName
	jobName := util.ConvertDisplayJobNameToJobName(displayJobName)
	image := form.Image
	uuid := form.Attachment
	jobType := form.JobType
	codePath := setting.JobPath + jobName + cloudbrain.CodeMountPath
	branchName := cloudbrain.DefaultBranchName
	repo := ctx.Repo.Repository

	tpl := tplCloudBrainBenchmarkNew
	command := cloudbrain.GetCloudbrainDebugCommand()

	tasks, err := models.GetCloudbrainsByDisplayJobName(repo.ID, jobType, displayJobName)
	if err == nil {
		if len(tasks) != 0 {
			log.Error("the job name did already exist", ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr("the job name did already exist", tpl, &form)
			return
		}
	} else {
		if !models.IsErrJobNotExist(err) {
			log.Error("system error, %v", err, ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr("system error", tpl, &form)
			return
		}
	}

	if !jobNamePattern.MatchString(displayJobName) {
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr(ctx.Tr("repo.cloudbrain_jobname_err"), tpl, &form)
		return
	}

	if jobType != string(models.JobTypeSnn4imagenet) && jobType != string(models.JobTypeBrainScore) {
		log.Error("jobtype error:", jobType, ctx.Data["MsgID"])
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr("jobtype error", tpl, &form)
		return
	}

	count, err := models.GetBenchmarkCountByUserID(ctx.User.ID)
	if err != nil {
		log.Error("GetCloudbrainCountByUserID failed:%v", err, ctx.Data["MsgID"])
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr("system error", tpl, &form)
		return
	} else {
		if count >= 1 {
			log.Error("the user already has running or waiting task", ctx.Data["MsgID"])
			cloudBrainNewDataPrepare(ctx)
			ctx.RenderWithErr(ctx.Tr("repo.cloudbrain.morethanonejob"), tpl, &form)
			return
		}
	}

	downloadCode(repo, codePath, branchName)
	uploadCodeToMinio(codePath+"/", jobName, cloudbrain.CodeMountPath+"/")

	modelPath := setting.JobPath + jobName + cloudbrain.ModelMountPath + "/"
	mkModelPath(modelPath)
	uploadCodeToMinio(modelPath, jobName, cloudbrain.ModelMountPath+"/")

	snn4imagenetPath := setting.JobPath + jobName + cloudbrain.Snn4imagenetMountPath
	if setting.IsSnn4imagenetEnabled && jobType == string(models.JobTypeSnn4imagenet) {
		downloadRateCode(repo, jobName, setting.Snn4imagenetOwner, setting.Snn4imagenetName, snn4imagenetPath, "", "", ctx.User.Name)
		uploadCodeToMinio(snn4imagenetPath+"/", jobName, cloudbrain.Snn4imagenetMountPath+"/")
		command = fmt.Sprintf(cloudbrain.Snn4imagenetCommand, displayJobName, trimSpaceNewlineInString(form.Description))

	}
	benchmarkChildTypeID := 0
	brainScorePath := setting.JobPath + jobName + cloudbrain.BrainScoreMountPath
	if setting.IsBrainScoreEnabled && jobType == string(models.JobTypeBrainScore) {
		downloadRateCode(repo, jobName, setting.BrainScoreOwner, setting.BrainScoreName, brainScorePath, "", "", ctx.User.Name)
		uploadCodeToMinio(brainScorePath+"/", jobName, cloudbrain.BrainScoreMountPath+"/")
		benchmarkChildTypeID = form.BenchmarkChildTypeID
		command = fmt.Sprintf(cloudbrain.BrainScoreCommand, getBrainRegion(benchmarkChildTypeID), displayJobName, trimSpaceNewlineInString(form.Description))
	}

	datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid)
	if err != nil {
		log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tpl, &form)
		return
	}
	spec, err := resource.GetAndCheckSpec(ctx.User.ID, form.SpecId, models.FindSpecsOptions{
		JobType:         models.JobTypeBenchmark,
		ComputeResource: models.GPU,
		Cluster:         models.OpenICluster,
		AiCenterCode:    models.AICenterOfCloudBrainOne})
	if err != nil || spec == nil {
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr("Resource specification not available", tpl, &form)
		return
	}
	log.Info("Command=" + command)
	log.Info("ModelPath=" + storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"))
	req := cloudbrain.GenerateCloudBrainTaskReq{
		Ctx:                  ctx,
		DisplayJobName:       displayJobName,
		JobName:              jobName,
		Image:                image,
		Command:              command,
		Uuids:                uuid,
		DatasetNames:         datasetNames,
		DatasetInfos:         datasetInfos,
		CodePath:             storage.GetMinioPath(jobName, cloudbrain.CodeMountPath+"/"),
		ModelPath:            storage.GetMinioPath(jobName, cloudbrain.ModelMountPath+"/"),
		BenchmarkPath:        storage.GetMinioPath(jobName, cloudbrain.BenchMarkMountPath+"/"),
		Snn4ImageNetPath:     storage.GetMinioPath(jobName, cloudbrain.Snn4imagenetMountPath+"/"),
		BrainScorePath:       storage.GetMinioPath(jobName, cloudbrain.BrainScoreMountPath+"/"),
		JobType:              jobType,
		Description:          form.Description,
		BranchName:           branchName,
		BootFile:             form.BootFile,
		Params:               form.Params,
		CommitID:             "",
		BenchmarkTypeID:      0,
		BenchmarkChildTypeID: benchmarkChildTypeID,
		ResultPath:           storage.GetMinioPath(jobName, cloudbrain.ResultPath+"/"),
		Spec:                 spec,
	}

	err = cloudbrain.GenerateTask(req)
	if err != nil {
		cloudBrainNewDataPrepare(ctx)
		ctx.RenderWithErr(err.Error(), tpl, &form)
		return
	}

	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")
}

func getBrainRegion(benchmarkChildTypeID int) string {
	values := []string{"V1", "V2", "V4", "IT"}
	return values[benchmarkChildTypeID]
}

func trimSpaceNewlineInString(s string) string {
	re := regexp.MustCompile(`\r?\n`)
	return re.ReplaceAllString(s, " ")

}

func BenchmarkDel(ctx *context.Context) {
	if err := deleteCloudbrainJob(ctx); err != nil {
		log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"])
		ctx.ServerError(err.Error(), err)
		return
	}

	var isAdminPage = ctx.Query("isadminpage")
	var isHomePage = ctx.Query("ishomepage")
	if ctx.IsUserSiteAdmin() && isAdminPage == "true" {
		ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains")
	} else if isHomePage == "true" {
		ctx.Redirect(setting.AppSubURL + "/cloudbrains")
	} else {
		ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/cloudbrain/benchmark")
	}
}

func CloudBrainTrainJobNew(ctx *context.Context) {
	err := cloudBrainNewDataPrepare(ctx)
	if err != nil {
		ctx.ServerError("get new train-job info failed", err)
		return
	}
	ctx.HTML(http.StatusOK, tplCloudBrainTrainJobNew)
}

func CloudBrainTrainJobVersionNew(ctx *context.Context) {
	err := cloudBrainNewDataPrepare(ctx)
	if err != nil {
		ctx.ServerError("get new train-job info failed", err)
		return
	}
	ctx.HTML(http.StatusOK, tplCloudBrainTrainJobVersionNew)
}

func InferenceCloudBrainJobNew(ctx *context.Context) {
	err := cloudBrainNewDataPrepare(ctx)
	if err != nil {
		ctx.ServerError("get new train-job info failed", err)
		return
	}
	ctx.HTML(http.StatusOK, tplCloudBrainInferenceJobNew)
}

func InferenceCloudBrainJobShow(ctx *context.Context) {
	cloudBrainShow(ctx, tplCloudBrainInferenceJobShow, models.JobTypeInference)
}

func DownloadInferenceResultFile(ctx *context.Context) {
	var jobID = ctx.Params(":jobid")
	var versionName = ctx.Query("version_name")
	task, err := models.GetCloudbrainByJobIDAndVersionName(jobID, versionName)
	if err != nil {
		log.Error("GetCloudbrainByJobID(%s) failed:%v", task.JobName, err.Error())
		return
	}

	allFile, err := storage.GetAllObjectByBucketAndPrefixMinio(setting.Attachment.Minio.Bucket, task.ResultUrl)
	returnFileName := task.DisplayJobName + ".zip"
	MinioDownloadManyFile(task.ResultUrl, ctx, returnFileName, allFile)
}

func getInferenceJobCommand(form auth.CreateCloudBrainInferencForm) (string, error) {
	var command string
	bootFile := strings.TrimSpace(form.BootFile)
	params := form.Params

	if !strings.HasSuffix(bootFile, ".py") {
		log.Error("bootFile(%s) format error", bootFile)
		return command, errors.New("bootFile format error")
	}

	var parameters models.Parameters
	var param string
	if len(params) != 0 {
		err := json.Unmarshal([]byte(params), &parameters)
		if err != nil {
			log.Error("Failed to Unmarshal params: %s (%v)", params, err)
			return command, err
		}

		for _, parameter := range parameters.Parameter {
			param += " --" + parameter.Label + "=" + parameter.Value
		}
	}

	param += " --modelname" + "=" + form.CkptName

	command += "python /code/" + bootFile + param + " > " + cloudbrain.ResultPath + "/" + form.DisplayJobName + "-" + cloudbrain.LogFile

	return command, nil
}

func getTrainJobCommand(form auth.CreateCloudBrainForm) (string, error) {
	var command string
	bootFile := strings.TrimSpace(form.BootFile)
	params := form.Params

	if !strings.HasSuffix(bootFile, ".py") {
		log.Error("bootFile(%s) format error", bootFile)
		return command, errors.New("bootFile format error")
	}

	var parameters models.Parameters
	var param string
	if len(params) != 0 {
		err := json.Unmarshal([]byte(params), &parameters)
		if err != nil {
			log.Error("Failed to Unmarshal params: %s (%v)", params, err)
			return command, err
		}

		for _, parameter := range parameters.Parameter {
			param += " --" + parameter.Label + "=" + parameter.Value
		}
	}
	if form.CkptName != "" {
		param += " --pretrainmodelname" + "=" + form.CkptName
	}

	command += "python /code/" + bootFile + param + " > " + cloudbrain.ModelMountPath + "/" + form.DisplayJobName + "-" + cloudbrain.LogFile

	return command, nil
}

func CloudBrainTrainJobDel(ctx *context.Context) {
	var listType = ctx.Query("listType")
	if err := deleteCloudbrainJob(ctx); err != nil {
		log.Error("deleteCloudbrainJob failed: %v", err, ctx.Data["msgID"])
		ctx.ServerError(err.Error(), err)
		return
	}

	var isAdminPage = ctx.Query("isadminpage")
	var isHomePage = ctx.Query("ishomepage")
	if ctx.IsUserSiteAdmin() && isAdminPage == "true" {
		ctx.Redirect(setting.AppSubURL + "/admin" + "/cloudbrains")
	} else if isHomePage == "true" {
		ctx.Redirect(setting.AppSubURL + "/cloudbrains")
	} else {
		ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job?listType=" + listType)
	}
}

func GetBenchmarkTypes(ctx *context.Context) *models.BenchmarkTypes {
	var lang = ctx.Locale.Language()
	if benchmarkTypesMap[lang] == nil {
		var val = i18n.Tr(lang, BENCHMARK_TYPE_CODE)
		//use config
		val = setting.BenchmarkTypes
		var tempType *models.BenchmarkTypes
		if err := json.Unmarshal([]byte(val), &tempType); err != nil {
			log.Error("json.Unmarshal BenchmarkTypes(%s) failed:%v", val, err, ctx.Data["MsgID"])
			return &models.BenchmarkTypes{}
		}
		benchmarkTypesMap[lang] = tempType
	}
	return benchmarkTypesMap[lang]
}

func GetCloudbrainAiCenter(task models.Cloudbrain, ctx *context.Context) string {
	if task.Type == models.TypeCloudBrainOne {
		return ctx.Tr("repo.cloudbrain1")
	} else if task.Type == models.TypeCloudBrainTwo {
		return ctx.Tr("repo.cloudbrain2")
	} else if task.Type == models.TypeCDCenter {
		return ctx.Tr("repo.cdCenter")
	} else if task.Type == models.TypeC2Net {
		return getCutStringAiCenterByAiCenter(task.AiCenter)
	}
	return ""
}
func getCutStringAiCenterByAiCenter(aiCenter string) string {
	if aiCenter == "" {
		return ""
	}
	index := strings.LastIndex(aiCenter, "+")
	return aiCenter[index+1:]

}
func GetCloudbrainCluster(task models.Cloudbrain, ctx *context.Context) string {
	if task.Type == models.TypeCloudBrainOne || task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeCDCenter {
		return ctx.Tr("cloudbrain.resource_cluster_openi")
	} else if task.Type == models.TypeC2Net {
		return ctx.Tr("cloudbrain.resource_cluster_c2net")
	}
	return ""
}
func GetCloudbrainCardDuration(task models.Cloudbrain) string {
	cardNum, _, _ := GetCloudbrainCardNumAndType(task)
	cardDuration := models.ConvertDurationToStr(int64(cardNum) * task.Duration)
	return cardDuration
}
func GetCloudbrainWaitTime(task models.Cloudbrain) string {
	var waitTime string
	if task.Status == string(models.JobWaiting) {
		waitTimeInt := time.Now().Unix() - task.CreatedUnix.AsTime().Unix()
		waitTime = models.ConvertDurationToStr(waitTimeInt)
		if waitTimeInt < 0 {
			waitTime = "00:00:00"
		}
	} else if task.Status == string(models.JobStopped) && task.StartTime.AsTime().Unix() == 0 {
		waitTimeInt := task.EndTime.AsTime().Unix() - task.CreatedUnix.AsTime().Unix()
		waitTime = models.ConvertDurationToStr(waitTimeInt)
		if waitTimeInt < 0 {
			waitTime = "00:00:00"

		}
	} else {
		waitTimeInt := task.StartTime.AsTime().Unix() - task.CreatedUnix.AsTime().Unix()
		waitTime = models.ConvertDurationToStr(waitTimeInt)
		if waitTimeInt < 0 {
			waitTime = "00:00:00"
		}
	}
	return waitTime
}

func GetCloudbrainCardNumAndType(task models.Cloudbrain) (int, string, error) {
	if !models.SpecsMapInitFlag {
		models.InitCloudbrainOneResourceSpecMap()
	}
	if !models.GpuInfosMapInitFlag {
		models.InitCloudbrainOneGpuInfoMap()
	}
	flavorName, err := GetCloudbrainFlavorName(task)
	if err != nil {
		return 0, "", nil
	}
	return getCardNumAndTypeByFlavorname(flavorName)
}

func getCardNumAndTypeByFlavorname(FlavorName string) (int, string, error) {
	if FlavorName == "" {
		return 0, "", nil
	} else {
		var beginIndex = strings.Index(FlavorName, ":")
		var lastIndex = strings.LastIndex(FlavorName, ":")
		var endIndex = strings.Index(FlavorName, "*")
		if endIndex >= (beginIndex+1) && lastIndex >= (endIndex+1) {
			cardNum, err := strconv.Atoi(strings.TrimSpace(FlavorName[beginIndex+1 : endIndex]))
			if err != nil {
				log.Error("strconv.Atoi failed: %v", err)
				return 0, "", err
			}
			cardType := strings.TrimSpace(FlavorName[endIndex+1 : lastIndex])
			return cardNum, cardType, err
		}
		return 0, "", nil
	}
}

func GetCloudbrainFlavorName(task models.Cloudbrain) (string, error) {
	if task.Type == models.TypeCloudBrainOne {
		resourceSpec, gpuInfo, err := getCloudBrainOneResourceSpec(task)
		if err != nil {
			log.Info("getCloudBrainOneResourceSpec err:", err)
			return "", err
		} else {
			if resourceSpec == nil || gpuInfo == nil {
				err := errors.New("resourceSpec or gpuInfo is nil")
				return "", err
			} else {
				CloudbrainOneFlavorName := "GPU:" + strconv.Itoa(resourceSpec.GpuNum) + "*Nvidia-" + gpuInfo.Value +
					" | CPU:" + strconv.Itoa(resourceSpec.CpuNum) + "核" + strconv.Itoa(resourceSpec.MemMiB) + "MB"
				return CloudbrainOneFlavorName, nil
			}
		}
	} else if (task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeC2Net || task.Type == models.TypeCDCenter) && task.FlavorName != "" {
		replaceFlavorName := strings.ReplaceAll(task.FlavorName, "：", ":")
		return replaceFlavorName, nil
	} else if (task.Type == models.TypeCloudBrainTwo || task.Type == models.TypeCDCenter) && task.FlavorName == "" && task.FlavorCode != "" {
		cloudbrainTwoFlavorName := getFlavorNameByFlavorCode(task.FlavorCode)
		return cloudbrainTwoFlavorName, nil
	} else if task.Type == models.TypeCloudBrainTwo && task.JobType == string(models.JobTypeDebug) && task.FlavorName == "" && task.FlavorCode == "" {
		tasks, err := models.GetModelartsReDebugTaskByJobId(task.JobID)
		if err != nil {
			return "", err
		}
		if len(tasks) >= 1 {
			return getFlavorNameByFlavorCode(tasks[0].FlavorCode), nil
		}
		return "", nil
	}
	return "", nil
}

func getCloudBrainOneResourceSpec(task models.Cloudbrain) (*models.ResourceSpec, *models.GpuInfo, error) {
	gpuQueueDefault := "openidebug"
	if task.GpuQueue != "" {
		gpuQueueDefault = task.GpuQueue
	}
	if task.ResourceSpecId >= 0 {
		if task.JobType == string(models.JobTypeTrain) {
			if models.CloudbrainTrainResourceSpecsMap[task.ResourceSpecId] != nil {
				return models.CloudbrainTrainResourceSpecsMap[task.ResourceSpecId], models.CloudbrainTrainGpuInfosMap[gpuQueueDefault], nil
			} else {
				return models.CloudbrainSpecialResourceSpecsMap[task.ResourceSpecId], models.CloudbrainSpecialGpuInfosMap[gpuQueueDefault], nil
			}
		} else if task.JobType == string(models.JobTypeDebug) {
			if models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId] != nil {
				return models.CloudbrainDebugResourceSpecsMap[task.ResourceSpecId], models.CloudbrainDebugGpuInfosMap[gpuQueueDefault], nil
			} else {
				return models.CloudbrainSpecialResourceSpecsMap[task.ResourceSpecId], models.CloudbrainSpecialGpuInfosMap[gpuQueueDefault], nil
			}
		} else if task.JobType == string(models.JobTypeInference) {
			return models.CloudbrainInferenceResourceSpecsMap[task.ResourceSpecId], models.CloudbrainInferenceGpuInfosMap[gpuQueueDefault], nil
		} else if task.JobType == string(models.JobTypeBenchmark) || task.JobType == string(models.JobTypeSnn4imagenet) || task.JobType == string(models.JobTypeBrainScore) {
			return models.CloudbrainBenchmarkResourceSpecsMap[task.ResourceSpecId], models.CloudbrainBenchmarkGpuInfosMap[gpuQueueDefault], nil
		}
	} else {
		err := errors.New("ResourceSpecId is null")
		return nil, nil, err
	}
	return nil, nil, nil
}
func getFlavorNameByFlavorCode(flavorCode string) string {
	index := strings.LastIndex(flavorCode, ".")
	cardNum, err := strconv.Atoi(strings.TrimSpace(flavorCode[index+1 : len(flavorCode)]))
	if err != nil {
		log.Error("strconv.Atoi failed: %v", err)
		return ""
	}
	cloudbrainTwoFlavorName := "Ascend:" + strings.TrimSpace(flavorCode[index+1:len(flavorCode)]) +
		"*Ascend-910(" + strconv.Itoa(cardNum*32) + "GB)|ARM:" + strconv.Itoa(cardNum*24) +
		"核" + strconv.Itoa(cardNum*256) + "GB"
	return cloudbrainTwoFlavorName
}