提交代码

models/cloudbrain.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -170,24 +171,25 @@ type Cloudbrain struct {
 	ImageID         string //grampus image_id
 	AiCenter        string //grampus ai center: center_id+center_name
 
-	TrainUrl          string //输出模型的obs路径
-	BranchName        string //分支名称
-	Parameters        string //传给modelarts的param参数
-	BootFile          string //启动文件
-	DataUrl           string //数据集的obs路径
-	LogUrl            string //日志输出的obs路径
-	PreVersionId      int64  //父版本的版本id
-	FlavorCode        string //modelarts上的规格id
-	Description       string `xorm:"varchar(256)"` //描述
-	WorkServerNumber  int    //节点数
-	FlavorName        string //规格名称
-	EngineName        string //引擎名称
-	TotalVersionCount int    //任务的所有版本数量，包括删除的
-	LabelName         string //标签名称
-	ModelName         string //模型名称
-	ModelVersion      string //模型版本
-	CkptName          string //权重文件名称
-	ResultUrl         string //推理结果的obs路径
+	TrainUrl            string //输出模型的obs路径
+	BranchName          string //分支名称
+	Parameters          string //传给modelarts的param参数
+	BootFile            string //启动文件
+	DataUrl             string //数据集的obs路径
+	LogUrl              string //日志输出的obs路径
+	PreVersionId        int64  //父版本的版本id
+	FlavorCode          string //modelarts上的规格id
+	Description         string `xorm:"varchar(256)"` //描述
+	WorkServerNumber    int    //节点数
+	FlavorName          string //规格名称
+	EngineName          string //引擎名称
+	TotalVersionCount   int    //任务的所有版本数量，包括删除的
+	LabelName           string //标签名称
+	ModelName           string //模型名称
+	ModelVersion        string //模型版本
+	CkptName            string //权重文件名称
+	PreTrainingModelUrl string //预训练模型地址
+	ResultUrl           string //推理结果的obs路径
 
 	User                  *User       `xorm:"-"`
 	Repo                  *Repository `xorm:"-"`
@@ -603,6 +605,16 @@ type ResourceSpec struct {
 	ShareMemMiB int `json:"shareMemMiB"`
 }
 
+type FlavorInfos struct {
+	FlavorInfo []*FlavorInfo `json:"flavor_info"`
+}
+
+type FlavorInfo struct {
+	Id    int    `json:"id"`
+	Value string `json:"value"`
+	Desc  string `json:"desc"`
+}
+
 type SpecialPools struct {
 	Pools []*SpecialPool `json:"pools"`
 }
@@ -2223,9 +2235,10 @@ func CloudbrainAllStatic(opts *CloudbrainsOptions) ([]*CloudbrainInfo, int64, er
 type DatasetInfo struct {
 	DataLocalPath string
 	Name          string
+	FullName      string
 }
 
-func GetDatasetInfo(uuidStr string) (map[string]DatasetInfo, string, error) {
+func GetDatasetInfo(uuidStr string, grampusType ...string) (map[string]DatasetInfo, string, error) {
 	var datasetNames string
 	uuids := strings.Split(uuidStr, ";")
 	if len(uuids) > setting.MaxDatasetNum {
@@ -2258,16 +2271,26 @@ func GetDatasetInfo(uuidStr string) (map[string]DatasetInfo, string, error) {
 				return nil, datasetNames, errors.New("the dataset name is same")
 			}
 		}
+		var dataLocalPath string
+		if len(grampusType) > 0 {
+			if grampusType[0] == GPU {
+				dataLocalPath = setting.Attachment.Minio.BasePath + path.Join(attach.UUID[0:1], attach.UUID[1:2]) + "/" + attach.UUID
+			} else {
+				dataLocalPath = setting.BasePath + path.Join(attach.UUID[0:1], attach.UUID[1:2]) + "/" + attach.UUID + "/"
+			}
 
-		dataLocalPath := setting.Attachment.Minio.RealPath +
-			setting.Attachment.Minio.Bucket + "/" +
-			setting.Attachment.Minio.BasePath +
-			AttachmentRelativePath(attach.UUID) +
-			attach.UUID
+		} else {
+			dataLocalPath = setting.Attachment.Minio.RealPath +
+				setting.Attachment.Minio.Bucket + "/" +
+				setting.Attachment.Minio.BasePath +
+				AttachmentRelativePath(attach.UUID) +
+				attach.UUID
+		}
 
 		datasetInfos[attach.UUID] = DatasetInfo{
 			DataLocalPath: dataLocalPath,
 			Name:          fileName,
+			FullName:      attach.Name,
 		}
 		if i == 0 {
 			datasetNames = attach.Name

modules/auth/cloudbrain.go

@@ -23,6 +23,11 @@ type CreateCloudBrainForm struct {
 	BootFile             string `form:"boot_file"`
 	Params               string `form:"run_para_list"`
 	BranchName           string `form:"branch_name"`
+	ModelName            string `form:"model_name"`
+	ModelVersion         string `form:"model_version"`
+	CkptName             string `form:"ckpt_name"`
+	LabelName            string `form:"label_names"`
+	PreTrainModelUrl     string `form:"pre_train_model_url"`
 	DatasetName          string `form:"dataset_name"`
 	SpecId               int64  `form:"spec_id"`
 }

modules/auth/grampus.go

@@ -18,6 +18,11 @@ type CreateGrampusTrainJobForm struct {
 	WorkServerNumber int    `form:"work_server_number" binding:"Required"`
 	Image            string `form:"image"`
 	DatasetName      string `form:"dataset_name"`
+	ModelName        string `form:"model_name"`
+	ModelVersion     string `form:"model_version"`
+	CkptName         string `form:"ckpt_name"`
+	LabelName        string `form:"label_names"`
+	PreTrainModelUrl string `form:"pre_train_model_url"`
 	SpecId           int64  `form:"spec_id"`
 }
 

modules/auth/modelarts.go

@@ -48,6 +48,11 @@ type CreateModelArtsTrainJobForm struct {
 	FlavorName            string `form:"flaver_names" binding:"Required"`
 	EngineName            string `form:"engine_names" binding:"Required"`
 	SpecId                int64  `form:"spec_id" binding:"Required"`
+	ModelName             string `form:"model_name"`
+	ModelVersion          string `form:"model_version"`
+	CkptName              string `form:"ckpt_name"`
+	LabelName             string `form:"label_names"`
+	PreTrainModelUrl      string `form:"pre_train_model_url"`
 }
 
 type CreateModelArtsInferenceJobForm struct {

modules/cloudbrain/cloudbrain.go

@@ -24,6 +24,7 @@ const (
 	CodeMountPath         = "/code"
 	DataSetMountPath      = "/dataset"
 	ModelMountPath        = "/model"
+	PretrainModelMountPath = "/pretrainmodel"
 	LogFile               = "log.txt"
 	BenchMarkMountPath    = "/benchmark"
 	BenchMarkResourceID   = 1
@@ -77,6 +78,8 @@ type GenerateCloudBrainTaskReq struct {
 	ModelVersion         string
 	CkptName             string
 	LabelName            string
+	PreTrainModelPath    string
+	PreTrainingModelUrl  string
 	Spec                 *models.Specification
 }
 
@@ -276,6 +279,16 @@ func GenerateTask(req GenerateCloudBrainTaskReq) error {
 		},
 	}
 
+	if req.PreTrainingModelUrl != "" { //预训练
+		volumes = append(volumes, models.Volume{
+			HostPath: models.StHostPath{
+				Path:      req.PreTrainModelPath,
+				MountPath: PretrainModelMountPath,
+				ReadOnly:  true,
+			},
+		})
+	}
+
 	if len(req.DatasetInfos) == 1 {
 		volumes = append(volumes, models.Volume{
 			HostPath: models.StHostPath{
@@ -359,6 +372,7 @@ func GenerateTask(req GenerateCloudBrainTaskReq) error {
 		CkptName:             req.CkptName,
 		ResultUrl:            req.ResultPath,
 		LabelName:            req.LabelName,
+		PreTrainingModelUrl:  req.PreTrainingModelUrl,
 		CreatedUnix:          createTime,
 		UpdatedUnix:          createTime,
 		CommitID:             req.CommitID,

modules/grampus/grampus.go

@@ -62,9 +62,17 @@ type GenerateTrainJobReq struct {
 	TotalVersionCount int
 	ComputeResource   string
 	ProcessType       string
-	DatasetName       string
-	Params            string
-	Spec              *models.Specification
+
+	DatasetNames        string
+	DatasetInfos        map[string]models.DatasetInfo
+	Params              string
+	ModelName           string
+	LabelName           string
+	CkptName            string
+	ModelVersion        string
+	PreTrainModelPath   string
+	PreTrainingModelUrl string
+	Spec                *models.Specification
 }
 
 func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error) {
@@ -94,33 +102,38 @@ func GenerateTrainJob(ctx *context.Context, req *GenerateTrainJobReq) (err error
 
 	jobID := jobResult.JobInfo.JobID
 	err = models.CreateCloudbrain(&models.Cloudbrain{
-		Status:            TransTrainJobStatus(jobResult.JobInfo.Status),
-		UserID:            ctx.User.ID,
-		RepoID:            ctx.Repo.Repository.ID,
-		JobID:             jobID,
-		JobName:           req.JobName,
-		DisplayJobName:    req.DisplayJobName,
-		JobType:           string(models.JobTypeTrain),
-		Type:              models.TypeC2Net,
-		Uuid:              req.Uuid,
-		DatasetName:       req.DatasetName,
-		CommitID:          req.CommitID,
-		IsLatestVersion:   req.IsLatestVersion,
-		ComputeResource:   req.ComputeResource,
-		ImageID:           req.ImageId,
-		TrainUrl:          req.TrainUrl,
-		BranchName:        req.BranchName,
-		Parameters:        req.Params,
-		BootFile:          req.BootFile,
-		DataUrl:           req.DataUrl,
-		Description:       req.Description,
-		WorkServerNumber:  req.WorkServerNumber,
-		EngineName:        req.EngineName,
-		VersionCount:      req.VersionCount,
-		TotalVersionCount: req.TotalVersionCount,
-		CreatedUnix:       createTime,
-		UpdatedUnix:       createTime,
-		Spec:              req.Spec,
+		Status:              TransTrainJobStatus(jobResult.JobInfo.Status),
+		UserID:              ctx.User.ID,
+		RepoID:              ctx.Repo.Repository.ID,
+		JobID:               jobID,
+		JobName:             req.JobName,
+		DisplayJobName:      req.DisplayJobName,
+		JobType:             string(models.JobTypeTrain),
+		Type:                models.TypeC2Net,
+		Uuid:                req.Uuid,
+		DatasetName:         req.DatasetNames,
+		CommitID:            req.CommitID,
+		IsLatestVersion:     req.IsLatestVersion,
+		ComputeResource:     req.ComputeResource,
+		ImageID:             req.ImageId,
+		TrainUrl:            req.TrainUrl,
+		BranchName:          req.BranchName,
+		Parameters:          req.Params,
+		BootFile:            req.BootFile,
+		DataUrl:             req.DataUrl,
+		Description:         req.Description,
+		WorkServerNumber:    req.WorkServerNumber,
+		EngineName:          req.EngineName,
+		VersionCount:        req.VersionCount,
+		TotalVersionCount:   req.TotalVersionCount,
+		CreatedUnix:         createTime,
+		UpdatedUnix:         createTime,
+		Spec:                req.Spec,
+		ModelName:           req.ModelName,
+		ModelVersion:        req.ModelVersion,
+		LabelName:           req.LabelName,
+		PreTrainingModelUrl: req.PreTrainingModelUrl,
+		CkptName:            req.CkptName,
 	})
 
 	if err != nil {

routers/repo/cloudbrain.go

@@ -328,6 +328,16 @@ func CloudBrainCreate(ctx *context.Context, form auth.CreateCloudBrainForm) {
 		Spec:                 spec,
 	}
 
+	if form.ModelName != "" { //使用预训练模型训练
+		req.ModelName = form.ModelName
+		req.LabelName = form.LabelName
+		req.CkptName = form.CkptName
+		req.ModelVersion = form.ModelVersion
+		req.PreTrainModelPath = setting.Attachment.Minio.RealPath + form.PreTrainModelUrl
+		req.PreTrainingModelUrl = form.PreTrainModelUrl
+
+	}
+
 	err = cloudbrain.GenerateTask(req)
 	if err != nil {
 		cloudBrainNewDataPrepare(ctx)
@@ -2629,6 +2639,9 @@ func getTrainJobCommand(form auth.CreateCloudBrainForm) (string, error) {
 			param += " --" + parameter.Label + "=" + parameter.Value
 		}
 	}
+	if form.CkptName != "" {
+		param += " --pretrainmodelname" + "=" + form.CkptName
+	}
 
 	command += "python /code/" + bootFile + param + " > " + cloudbrain.ModelMountPath + "/" + form.DisplayJobName + "-" + cloudbrain.LogFile
 

routers/repo/grampus.go

@@ -210,7 +210,6 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 	repo := ctx.Repo.Repository
 	codeLocalPath := setting.JobPath + jobName + cloudbrain.CodeMountPath + "/"
 	codeMinioPath := setting.CBCodePathPrefix + jobName + cloudbrain.CodeMountPath + "/"
-	dataMinioPath := setting.Attachment.Minio.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid
 	branchName := form.BranchName
 	image := strings.TrimSpace(form.Image)
 
@@ -290,11 +289,12 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 	}
 
 	//check dataset
-	attachment, err := models.GetAttachmentByUUID(uuid)
+
+	datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid, models.GPU)
 	if err != nil {
-		log.Error("GetAttachmentByUUID failed:", err.Error(), ctx.Data["MsgID"])
+		log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
 		grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
-		ctx.RenderWithErr("dataset is not exist", tplGrampusTrainJobGPUNew, &form)
+		ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tplGrampusTrainJobGPUNew, &form)
 		return
 	}
 
@@ -336,8 +336,22 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 		return
 	}
 
+	var datasetRemotePath, allFileName string
+	for _, datasetInfo := range datasetInfos {
+		if datasetRemotePath == "" {
+			datasetRemotePath = datasetInfo.DataLocalPath
+			allFileName = datasetInfo.FullName
+		} else {
+			datasetRemotePath = datasetRemotePath + ";" + datasetInfo.DataLocalPath
+			allFileName = allFileName + ";" + datasetInfo.FullName
+		}
+
+	}
+
 	//prepare command
-	command, err := generateCommand(repo.Name, grampus.ProcessorTypeGPU, codeMinioPath+cloudbrain.DefaultBranchName+".zip", dataMinioPath, bootFile, params, setting.CBCodePathPrefix+jobName+cloudbrain.ModelMountPath+"/", attachment.Name)
+	preTrainModelPath := getPreTrainModelPath(form.PreTrainModelUrl, form.CkptName)
+
+	command, err := generateCommand(repo.Name, grampus.ProcessorTypeGPU, codeMinioPath+cloudbrain.DefaultBranchName+".zip", datasetRemotePath, bootFile, params, setting.CBCodePathPrefix+jobName+cloudbrain.ModelMountPath+"/", allFileName, preTrainModelPath, form.CkptName)
 	if err != nil {
 		log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"])
 		grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeGPU)
@@ -348,26 +362,37 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 	commitID, _ := ctx.Repo.GitRepo.GetBranchCommitID(branchName)
 
 	req := &grampus.GenerateTrainJobReq{
-		JobName:          jobName,
-		DisplayJobName:   displayJobName,
-		ComputeResource:  models.GPUResource,
-		ProcessType:      grampus.ProcessorTypeGPU,
-		Command:          command,
-		ImageUrl:         image,
-		Description:      description,
-		BootFile:         bootFile,
-		Uuid:             uuid,
-		CommitID:         commitID,
-		BranchName:       branchName,
-		Params:           form.Params,
-		EngineName:       image,
-		DatasetName:      attachment.Name,
+		JobName:         jobName,
+		DisplayJobName:  displayJobName,
+		ComputeResource: models.GPUResource,
+		ProcessType:     grampus.ProcessorTypeGPU,
+		Command:         command,
+		ImageUrl:        image,
+		Description:     description,
+		BootFile:        bootFile,
+		Uuid:            uuid,
+		CommitID:        commitID,
+		BranchName:      branchName,
+		Params:          form.Params,
+		EngineName:      image,
+		DatasetNames:    datasetNames,
+		DatasetInfos:    datasetInfos,
+
 		IsLatestVersion:  modelarts.IsLatestVersion,
 		VersionCount:     modelarts.VersionCountOne,
 		WorkServerNumber: 1,
 		Spec:             spec,
 	}
 
+	if form.ModelName != "" { //使用预训练模型训练
+		req.ModelName = form.ModelName
+		req.LabelName = form.LabelName
+		req.CkptName = form.CkptName
+		req.ModelVersion = form.ModelVersion
+		req.PreTrainingModelUrl = form.PreTrainModelUrl
+
+	}
+
 	err = grampus.GenerateTrainJob(ctx, req)
 	if err != nil {
 		log.Error("GenerateTrainJob failed:%v", err.Error(), ctx.Data["MsgID"])
@@ -378,6 +403,17 @@ func GrampusTrainJobGpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 	ctx.Redirect(setting.AppSubURL + ctx.Repo.RepoLink + "/modelarts/train-job")
 }
 
+func getPreTrainModelPath(pretrainModelDir string, fileName string) string {
+	index := strings.Index(pretrainModelDir, "/")
+	if index > 0 {
+		filterBucket := pretrainModelDir[index+1:]
+		return filterBucket + fileName
+	} else {
+		return ""
+	}
+
+}
+
 func checkSpecialPool(ctx *context.Context, resourceType string) string {
 	grampus.InitSpecialPool()
 	if grampus.SpecialPools != nil {
@@ -410,7 +446,7 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 	repo := ctx.Repo.Repository
 	codeLocalPath := setting.JobPath + jobName + modelarts.CodePath
 	codeObsPath := grampus.JobPath + jobName + modelarts.CodePath
-	dataObsPath := setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + "/"
+	//dataObsPath := setting.BasePath + path.Join(uuid[0:1], uuid[1:2]) + "/" + uuid + "/"
 	branchName := form.BranchName
 	isLatestVersion := modelarts.IsLatestVersion
 	versionCount := modelarts.VersionCountOne
@@ -492,11 +528,11 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 	}
 
 	//check dataset
-	attachment, err := models.GetAttachmentByUUID(uuid)
+	datasetInfos, datasetNames, err := models.GetDatasetInfo(uuid, models.NPU)
 	if err != nil {
-		log.Error("GetAttachmentByUUID failed:", err.Error(), ctx.Data["MsgID"])
+		log.Error("GetDatasetInfo failed: %v", err, ctx.Data["MsgID"])
 		grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
-		ctx.RenderWithErr("dataset is not exist", tplGrampusTrainJobNPUNew, &form)
+		ctx.RenderWithErr(ctx.Tr("cloudbrain.error.dataset_select"), tplGrampusTrainJobNPUNew, &form)
 		return
 	}
 
@@ -528,8 +564,21 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 		return
 	}
 
+	var datasetRemotePath, allFileName string
+	for _, datasetInfo := range datasetInfos {
+		if datasetRemotePath == "" {
+			datasetRemotePath = datasetInfo.DataLocalPath + "'" + datasetInfo.FullName + "'"
+			allFileName = datasetInfo.FullName
+		} else {
+			datasetRemotePath = datasetRemotePath + ";" + datasetInfo.DataLocalPath + "'" + datasetInfo.FullName + "'"
+			allFileName = allFileName + ";" + datasetInfo.FullName
+		}
+
+	}
+
 	//prepare command
-	command, err := generateCommand(repo.Name, grampus.ProcessorTypeNPU, codeObsPath+cloudbrain.DefaultBranchName+".zip", dataObsPath+"'"+attachment.Name+"'", bootFile, params, setting.CodePathPrefix+jobName+modelarts.OutputPath, attachment.Name)
+	preTrainModelPath := getPreTrainModelPath(form.PreTrainModelUrl, form.CkptName)
+	command, err := generateCommand(repo.Name, grampus.ProcessorTypeNPU, codeObsPath+cloudbrain.DefaultBranchName+".zip", datasetRemotePath, bootFile, params, setting.CodePathPrefix+jobName+modelarts.OutputPath, allFileName, preTrainModelPath, form.CkptName)
 	if err != nil {
 		log.Error("Failed to generateCommand: %s (%v)", displayJobName, err, ctx.Data["MsgID"])
 		grampusTrainJobNewDataPrepare(ctx, grampus.ProcessorTypeNPU)
@@ -546,7 +595,6 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 		ProcessType:       grampus.ProcessorTypeNPU,
 		Command:           command,
 		ImageId:           form.ImageID,
-		DataUrl:           dataObsPath,
 		Description:       description,
 		CodeObsPath:       codeObsPath,
 		BootFileUrl:       codeObsPath + bootFile,
@@ -560,9 +608,18 @@ func GrampusTrainJobNpuCreate(ctx *context.Context, form auth.CreateGrampusTrain
 		EngineName:        engineName,
 		VersionCount:      versionCount,
 		TotalVersionCount: modelarts.TotalVersionCount,
-		DatasetName:       attachment.Name,
+		DatasetNames:      datasetNames,
+		DatasetInfos:      datasetInfos,
 		Spec:              spec,
 	}
+	if form.ModelName != "" { //使用预训练模型训练
+		req.ModelName = form.ModelName
+		req.LabelName = form.LabelName
+		req.CkptName = form.CkptName
+		req.ModelVersion = form.ModelVersion
+		req.PreTrainingModelUrl = form.PreTrainModelUrl
+
+	}
 
 	err = grampus.GenerateTrainJob(ctx, req)
 	if err != nil {
@@ -776,7 +833,7 @@ func GrampusGetLog(ctx *context.Context) {
 	return
 }
 
-func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bootFile, paramSrc, outputRemotePath, datasetName string) (string, error) {
+func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bootFile, paramSrc, outputRemotePath, datasetName, pretrainModelPath, pretrainModelFileName string) (string, error) {
 	var command string
 
 	workDir := grampus.NpuWorkDir
@@ -788,18 +845,18 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo
 	//download code & dataset
 	if processorType == grampus.ProcessorTypeNPU {
 		commandDownload := "./downloader_for_obs " + setting.Bucket + " " + codeRemotePath + " " + grampus.CodeArchiveName + " " + dataRemotePath + " '" + datasetName + "';"
+		commandDownload = processPretrainModelParameter(pretrainModelPath, pretrainModelFileName, commandDownload)
 		command += commandDownload
 	} else if processorType == grampus.ProcessorTypeGPU {
-		commandDownload := "./downloader_for_minio " + setting.Grampus.Env + " " + codeRemotePath + " " + grampus.CodeArchiveName + " " + dataRemotePath + " '" + datasetName + "';"
+		commandDownload := "./downloader_for_minio " + setting.Grampus.Env + " " + codeRemotePath + " " + grampus.CodeArchiveName + " " + dataRemotePath + " '" + datasetName + "'"
+		commandDownload = processPretrainModelParameter(pretrainModelPath, pretrainModelFileName, commandDownload)
 		command += commandDownload
 	}
 
 	//unzip code & dataset
-	toolUnzip := "unzip -q '"
-	if strings.HasSuffix(datasetName, ".tar.gz") {
-		toolUnzip = "tar -zxvf '"
-	}
-	commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + toolUnzip + datasetName + "';"
+	unZipDatasetCommand := generateDatasetUnzipCommand(datasetName)
+
+	commandUnzip := "cd " + workDir + "code;unzip -q master.zip;echo \"start to unzip dataset\";cd " + workDir + "dataset;" + unZipDatasetCommand
 	command += commandUnzip
 
 	command += "echo \"unzip finished;start to exec code;\";"
@@ -859,6 +916,38 @@ func generateCommand(repoName, processorType, codeRemotePath, dataRemotePath, bo
 	return command, nil
 }
 
+func processPretrainModelParameter(pretrainModelPath string, pretrainModelFileName string, commandDownload string) string {
+	commandDownloadTemp := commandDownload
+	if pretrainModelPath != "" {
+		commandDownloadTemp += " '" + pretrainModelPath + "' '" + pretrainModelFileName + "'"
+	}
+	commandDownloadTemp += ";"
+	return commandDownloadTemp
+}
+
+func generateDatasetUnzipCommand(datasetName string) string {
+	var unZipDatasetCommand string
+
+	datasetNameArray := strings.Split(datasetName, ";")
+	if len(datasetNameArray) == 1 { //单数据集
+		unZipDatasetCommand = "unzip -q '" + datasetName + "';"
+		if strings.HasSuffix(datasetName, ".tar.gz") {
+			unZipDatasetCommand = "tar --strip-components=1 -zxvf '" + datasetName + "';"
+		}
+
+	} else { //多数据集
+		for _, datasetNameTemp := range datasetNameArray {
+			if strings.HasSuffix(datasetName, ".tar.gz") {
+				unZipDatasetCommand = unZipDatasetCommand + "tar -zxvf '" + datasetName + "';"
+			} else {
+				unZipDatasetCommand = unZipDatasetCommand + "unzip -q '" + datasetNameTemp + "' -d './" + strings.TrimSuffix(datasetNameTemp, ".zip") + "';"
+			}
+		}
+
+	}
+	return unZipDatasetCommand
+}
+
 func downloadZipCode(ctx *context.Context, codePath, branchName string) error {
 	archiveType := git.ZIP
 	archivePath := codePath