download

modules/modelarts/modelarts.go

@@ -237,13 +237,13 @@ func TransTrainJobStatus(status int) string {
 	case 8:
 		return "RUNNING"
 	case 9:
-		return "STOPED"
+		return "STOPPED"
 	case 10:
 		return "COMPLETED"
 	case 11:
 		return "FAILED"
 	case 12:
-		return "STOPED"
+		return "STOPPED"
 	case 13:
 		return "CANCELED"
 	case 14:

modules/setting/setting.go

@@ -473,6 +473,8 @@ var (
 	Bucket          string
 	Location        string
 	BasePath        string
+	JobModelPath    string
+	OutPutPath      string
 	CodePathPrefix  string
 	UserBasePath    string
 
@@ -488,11 +490,12 @@ var (
 	ProfileID         string
 	PoolInfos         string
 	Flavor            string
+	FlavorInfos       string
 	//train-job
-	ResourcePools  string
-	Engines        string
-	EngineVersions string
-	FlavorInfos    string
+	ResourcePools       string
+	Engines             string
+	EngineVersions      string
+	TrainJobFlavorInfos string
 
 	//elk config
 	ElkUrl        string
@@ -1252,6 +1255,8 @@ func NewContext() {
 	Bucket = sec.Key("BUCKET").MustString("testopendata")
 	Location = sec.Key("LOCATION").MustString("cn-south-222")
 	BasePath = sec.Key("BASE_PATH").MustString("attachment/")
+	JobModelPath = sec.Key("Job_Model_PATH").MustString("job/")
+	OutPutPath = sec.Key("OutPut_PATH").MustString("output/")
 	CodePathPrefix = sec.Key("CODE_PATH_PREFIX").MustString("code/")
 	UserBasePath = sec.Key("BASE_PATH_USER").MustString("users/")
 	PROXYURL = sec.Key("PROXY_URL").MustString("")
@@ -1272,6 +1277,7 @@ func NewContext() {
 	Engines = sec.Key("Engines").MustString("")
 	EngineVersions = sec.Key("Engine_Versions").MustString("")
 	FlavorInfos = sec.Key("FLAVOR_INFOS").MustString("")
+	TrainJobFlavorInfos = sec.Key("TrainJob_FLAVOR_INFOS").MustString("")
 
 	sec = Cfg.Section("elk")
 	ElkUrl = sec.Key("ELKURL").MustString("http://192.168.207.35:5601/internal/bsearch")

modules/storage/obs.go

@@ -134,6 +134,7 @@ func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) {
 	input := &obs.GetObjectInput{}
 	input.Bucket = setting.Bucket
 	input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
+	// input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/")
 	output, err := ObsCli.GetObject(input)
 	if err == nil {
 		log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n",
@@ -148,6 +149,54 @@ func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) {
 	}
 }
 
+func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) {
+	input := &obs.GetObjectInput{}
+	input.Bucket = setting.Bucket
+	input.Key = strings.TrimPrefix(path.Join(setting.JobModelPath, JobName, setting.OutPutPath, fileName), "/")
+	// input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/")
+	output, err := ObsCli.GetObject(input)
+	if err == nil {
+		log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n",
+			output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified)
+		return output.Body, nil
+	} else if obsError, ok := err.(obs.ObsError); ok {
+		fmt.Printf("Code:%s\n", obsError.Code)
+		fmt.Printf("Message:%s\n", obsError.Message)
+		return nil, obsError
+	} else {
+		return nil, err
+	}
+}
+
+func GetObsListObject(jobName string) ([]string, error) {
+	// jobName = "liuzx202110271830856"
+	input := &obs.ListObjectsInput{}
+	input.Bucket = setting.Bucket
+	input.Prefix = strings.TrimPrefix(path.Join(setting.JobModelPath, jobName, setting.OutPutPath), "/")
+	log.Info("input.Prefix:", input.Prefix)
+	output, err := ObsCli.ListObjects(input)
+	ModelListArr := make([]string, 0)
+	if err == nil {
+		fmt.Printf("RequestId:%s\n", output.RequestId)
+		for index, val := range output.Contents {
+			fmt.Printf("Content[%d]-OwnerId:%s, ETag:%s, Key:%s, LastModified:%s, Size:%d\n",
+				index, val.Owner.ID, val.ETag, val.Key, val.LastModified, val.Size)
+			str1 := strings.Split(val.Key, "/")
+			ModelList := str1[len(str1)-1]
+			ModelListArr = append(ModelListArr, ModelList)
+		}
+		return ModelListArr, err
+	} else {
+		if obsError, ok := err.(obs.ObsError); ok {
+			fmt.Println(obsError.Code)
+			fmt.Println(obsError.Message)
+		} else {
+			fmt.Println(err)
+		}
+		return nil, err
+	}
+}
+
 func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) {
 
 	input := &obs.CreateSignedUrlInput{}

routers/repo/cloudbrain.go

@@ -580,7 +580,6 @@ func CloudBrainDownloadModel(ctx *context.Context) {
 		ctx.ServerError("PresignedGetURL", err)
 		return
 	}
-
 	http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
 }
 
@@ -599,11 +598,34 @@ func CloudBrainDownloadModel(ctx *context.Context) {
 // 	http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
 // }
 
-func TrainJobloadModel(ctx *context.Context) {
-	uuid := ctx.Query("uuid")
+func TrainJobListModel(ctx *context.Context) {
+	ctx.Data["PageIsCloudBrain"] = true
+	jobID := ctx.Params(":jobid")
+	task, err := models.GetCloudbrainByJobID(jobID)
+	if err != nil {
+		log.Error("no such job!", ctx.Data["err"])
+		ctx.ServerError("no such job:", err)
+		return
+	}
+	TrainJobListModel, err := storage.GetObsListObject(task.JobName)
+	if err != nil {
+		log.Info("get TrainJobListModel failed:", err)
+		return
+	}
+	ctx.Data["task"] = task
+	ctx.Data["JobID"] = jobID
+	ctx.Data["TrainJobListModel"] = TrainJobListModel
+	ctx.HTML(200, tplModelArtsTrainJobListModel)
+}
+
+func TrainJobDownloadModel(ctx *context.Context) {
+	JobName := ctx.Query("JobName")
 	fileName := ctx.Query("file_name")
 
-	body, err := storage.ObsDownload(uuid, fileName)
+	// JobName = "liuzx202110271830856"
+	// fileName = "Untitled.ipynb"
+
+	body, err := storage.ObsModelDownload(JobName, fileName)
 	if err != nil {
 		log.Info("download error.")
 	} else {

routers/repo/modelarts.go

@@ -36,9 +36,10 @@ const (
 	tplModelArtsNew   base.TplName = "repo/modelarts/new"
 	tplModelArtsShow  base.TplName = "repo/modelarts/show"
 
-	tplModelArtsTrainJobIndex base.TplName = "repo/modelarts/trainjob/index"
-	tplModelArtsTrainJobNew   base.TplName = "repo/modelarts/trainjob/new"
-	tplModelArtsTrainJobShow  base.TplName = "repo/modelarts/trainjob/show"
+	tplModelArtsTrainJobIndex     base.TplName = "repo/modelarts/trainjob/index"
+	tplModelArtsTrainJobNew       base.TplName = "repo/modelarts/trainjob/new"
+	tplModelArtsTrainJobShow      base.TplName = "repo/modelarts/trainjob/show"
+	tplModelArtsTrainJobListModel base.TplName = "repo/modelarts/trainjob/list_model"
 )
 
 // MustEnableDataset check if repository enable internal cb
@@ -331,6 +332,7 @@ func NotebookNew(ctx *context.Context) {
 		json.Unmarshal([]byte(setting.FlavorInfos), &modelarts.FlavorInfos)
 	}
 	ctx.Data["flavors"] = modelarts.FlavorInfos.FlavorInfo
+
 	ctx.HTML(200, tplModelArtsNotebookNew)
 }
 
@@ -610,7 +612,7 @@ func TrainJobNew(ctx *context.Context) {
 	ctx.Data["engine_versions"] = versionInfos.Version
 
 	var flavorInfos modelarts.Flavor
-	if err = json.Unmarshal([]byte(setting.FlavorInfos), &flavorInfos); err != nil {
+	if err = json.Unmarshal([]byte(setting.TrainJobFlavorInfos), &flavorInfos); err != nil {
 		ctx.ServerError("json.Unmarshal failed:", err)
 		return
 	}
@@ -899,6 +901,11 @@ func TrainJobShow(ctx *context.Context) {
 		} else {
 			result.TrainJobDuration = "00:00:00"
 		}
+		err = models.SetTrainJobStatusByJobID(jobID, modelarts.TransTrainJobStatus(result.IntStatus), int(result.Duration), string(result.TrainJobDuration))
+		if err != nil {
+			ctx.ServerError("UpdateJob failed", err)
+			return
+		}
 		result.Status = modelarts.TransTrainJobStatus(result.IntStatus)
 		result.DatasetName = attach.Name
 	}

routers/routes/routes.go

@@ -991,7 +991,8 @@ func RegisterRoutes(m *macaron.Macaron) {
 					m.Post("/stop", reqRepoCloudBrainWriter, repo.TrainJobStop)
 					m.Post("/del", reqRepoCloudBrainWriter, repo.TrainJobDel)
 					m.Get("/log", reqRepoCloudBrainReader, repo.TrainJobGetLog)
-					m.Get("/download_model", reqRepoCloudBrainReader, repo.TrainJobloadModel)
+					m.Get("/models", reqRepoCloudBrainReader, repo.TrainJobListModel)
+					m.Get("/download_model", reqRepoCloudBrainReader, repo.TrainJobDownloadModel)
 				})
 				m.Get("/create", reqRepoCloudBrainReader, repo.TrainJobNew)
 				m.Post("/create", reqRepoCloudBrainWriter, bindIgnErr(auth.CreateModelArtsTrainJobForm{}), repo.TrainJobCreate)

templates/repo/modelarts/trainjob/index.tmpl

@@ -360,7 +360,7 @@
                                     </div>
                                     <div class="ui compact buttons" style="margin-right:10px;">
                                             <!-- 模型下载 -->
-                                            <a class="ui basic blue button" href="{{$.Link}}/{{.JobID}}/download_model" target="_blank">
+                                            <a class="ui basic blue button" href="{{$.Link}}/{{.JobID}}/models" target="_blank">
                                                 模型下载
                                             </a>
                                             

templates/repo/modelarts/trainjob/list_model.tmpl

@@ -0,0 +1 @@
+shuchu