提交代码。

Signed-off-by: zouap <zouap@pcl.ac.cn>

modules/storage/obs.go

@@ -280,7 +280,7 @@ func ObsCopyFile(srcBucket string, srcKeyName string, destBucket string, destKey
 	return nil
 }
 
-func GetAllObsListObjectUnderDir(bucket string, prefixRootPath string, relativePath string) ([]FileInfo, error) {
+func GetOneLevelAllObjectUnderDir(bucket string, prefixRootPath string, relativePath string) ([]FileInfo, error) {
 	input := &obs.ListObjectsInput{}
 	input.Bucket = bucket
 	input.Prefix = prefixRootPath + relativePath
@@ -324,47 +324,56 @@ func GetAllObsListObjectUnderDir(bucket string, prefixRootPath string, relativeP
 
 }
 
-func GetObsListObjectByBucketAndPrefix(bucket string, prefix string, parentDir string) ([]FileInfo, error) {
+func GetAllObjectByBucketAndPrefix(bucket string, prefix string) ([]FileInfo, error) {
 	input := &obs.ListObjectsInput{}
 	input.Bucket = bucket
+	// 设置每页100个对象
+	input.MaxKeys = 100
 	input.Prefix = prefix
-	output, err := ObsCli.ListObjects(input)
+	index := 1
 	fileInfos := make([]FileInfo, 0)
-	if err == nil {
-		for _, val := range output.Contents {
-			str1 := strings.Split(val.Key, "/")
-			log.Info("val.Key=" + val.Key)
-			var isDir bool
-			var fileName, nextParentDir string
-			if strings.HasSuffix(val.Key, "/") {
-				fileName = str1[len(str1)-2]
-				isDir = true
-				nextParentDir = fileName
-				if fileName == parentDir || (fileName+"/") == setting.OutPutPath {
+	prefixLen := len(prefix)
+	log.Info("prefix=" + input.Prefix)
+	for {
+		output, err := ObsCli.ListObjects(input)
+		if err == nil {
+			log.Info("Page:%d\n", index)
+			index++
+			for _, val := range output.Contents {
+				var isDir bool
+				if val.Key == prefix {
 					continue
 				}
+				if strings.HasSuffix(val.Key, "/") {
+					isDir = true
+				} else {
+					isDir = false
+				}
+				fileInfo := FileInfo{
+					ModTime:  val.LastModified.Format("2006-01-02 15:04:05"),
+					FileName: val.Key[prefixLen:],
+					Size:     val.Size,
+					IsDir:    isDir,
+					ParenDir: "",
+				}
+				fileInfos = append(fileInfos, fileInfo)
+			}
+			if output.IsTruncated {
+				input.Marker = output.NextMarker
 			} else {
-				fileName = str1[len(str1)-1]
-				isDir = false
+				break
 			}
-			log.Info("fileName=" + fileName)
-			fileInfo := FileInfo{
-				ModTime:  val.LastModified.Format("2006-01-02 15:04:05"),
-				FileName: fileName,
-				Size:     val.Size,
-				IsDir:    isDir,
-				ParenDir: nextParentDir,
+		} else {
+			if obsError, ok := err.(obs.ObsError); ok {
+				log.Info("Code:%s\n", obsError.Code)
+				log.Info("Message:%s\n", obsError.Message)
 			}
-			fileInfos = append(fileInfos, fileInfo)
+			return nil, err
 		}
-		return fileInfos, err
-	} else {
-		if obsError, ok := err.(obs.ObsError); ok {
-			log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message)
-		}
-		return nil, err
 	}
 
+	return fileInfos, nil
+
 }
 
 func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) {

routers/repo/ai_model_manage.go

@@ -1,6 +1,7 @@
 package repo
 
 import (
+	"archive/zip"
 	"errors"
 	"fmt"
 	"net/http"
@@ -105,7 +106,7 @@ func SaveModel(ctx *context.Context) {
 func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir string) (string, int64, error) {
 
 	objectkey := strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/")
-	modelDbResult, err := storage.GetAllObsListObjectUnderDir(setting.Bucket, objectkey, "")
+	modelDbResult, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, objectkey, "")
 	if err != nil {
 		log.Info("get TrainJobListModel failed:", err)
 		return "", 0, err
@@ -119,21 +120,6 @@ func downloadModelFromCloudBrainTwo(modelUUID string, jobName string, parentDir
 
 	size, err := storage.ObsCopyManyFile(setting.Bucket, prefix, setting.Bucket, destKeyNamePrefix)
 
-	// for _, modelFile := range modelDbResult {
-	// 	if modelFile.IsDir {
-	// 		log.Info("copy dir, continue. dir=" + modelFile.FileName)
-	// 		continue
-	// 	}
-	// 	srcKeyName := prefix + modelFile.FileName
-	// 	log.Info("copy file, bucket=" + setting.Bucket + ", src keyname=" + srcKeyName)
-	// 	destKeyName := destKeyNamePrefix + modelFile.FileName
-	// 	log.Info("Dest key name=" + destKeyName)
-	// 	err := storage.ObsCopyFile(setting.Bucket, srcKeyName, setting.Bucket, destKeyName)
-	// 	if err != nil {
-	// 		log.Info("copy failed.")
-	// 	}
-	// 	size += modelFile.Size
-	// }
 	dataActualPath := setting.Bucket + "/" + destKeyNamePrefix
 	return dataActualPath, size, nil
 }
@@ -177,10 +163,6 @@ func DeleteModelByID(id string) error {
 	return err
 }
 
-func DownloadModel(ctx *context.Context) {
-	log.Info("download model start.")
-}
-
 func QueryModelByParameters(repoId int64, page int) ([]*models.AiModelManage, int64, error) {
 
 	return models.QueryModel(&models.AiModelQueryOptions{
@@ -198,6 +180,56 @@ func DownloadMultiModelFile(ctx *context.Context) {
 	log.Info("DownloadMultiModelFile start.")
 	id := ctx.Query("ID")
 	log.Info("id=" + id)
+	task, err := models.QueryModelById(id)
+	if err != nil {
+		log.Error("no such model!", err.Error())
+		ctx.ServerError("no such model:", err)
+		return
+	}
+	path := Model_prefix + models.AttachmentRelativePath(id) + "/"
+	returnFileName := task.Name + "_" + task.Version + ".zip"
+	ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+returnFileName)
+	ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
+	allFile, err := storage.GetAllObjectByBucketAndPrefix(setting.Bucket, path)
+	if err != nil {
+
+		w := zip.NewWriter(ctx.Resp)
+		defer w.Close()
+
+		for _, oneFile := range allFile {
+			if !oneFile.IsDir {
+				log.Info("zip file  name:" + oneFile.FileName)
+				fDest, err := w.Create(oneFile.FileName)
+				if err != nil {
+					log.Info("create zip entry error, download file failed: %s\n", err.Error())
+					ctx.ServerError("download file failed:", err)
+					return
+				}
+				body, err := storage.ObsDownloadAFile(setting.Bucket, path+oneFile.FileName)
+				if err != nil {
+					log.Info("download file failed: %s\n", err.Error())
+					ctx.ServerError("download file failed:", err)
+					return
+				} else {
+					defer body.Close()
+					p := make([]byte, 1024)
+					var readErr error
+					var readCount int
+					// 读取对象内容
+					for {
+						readCount, readErr = body.Read(p)
+						if readCount > 0 {
+							fDest.Write(p[:readCount])
+						}
+						if readErr != nil {
+							break
+						}
+					}
+				}
+
+			}
+		}
+	}
 }
 
 func DownloadSingleModelFile(ctx *context.Context) {
@@ -206,13 +238,39 @@ func DownloadSingleModelFile(ctx *context.Context) {
 	parentDir := ctx.Query("parentDir")
 	fileName := ctx.Query("fileName")
 	path := Model_prefix + models.AttachmentRelativePath(id) + "/" + parentDir + fileName
-	url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path)
-	if err != nil {
-		log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
-		ctx.ServerError("GetObsCreateSignedUrl", err)
-		return
+
+	if setting.PROXYURL != "" {
+		body, err := storage.ObsDownloadAFile(setting.Bucket, path)
+		if err != nil {
+			log.Info("download error.")
+		} else {
+			defer body.Close()
+			ctx.Resp.Header().Set("Content-Disposition", "attachment; filename="+fileName)
+			ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
+			p := make([]byte, 1024)
+			var readErr error
+			var readCount int
+			// 读取对象内容
+			for {
+				readCount, readErr = body.Read(p)
+				if readCount > 0 {
+					ctx.Resp.Write(p[:readCount])
+					//fmt.Printf("%s", p[:readCount])
+				}
+				if readErr != nil {
+					break
+				}
+			}
+		}
+	} else {
+		url, err := storage.GetObsCreateSignedUrlByBucketAndKey(setting.Bucket, path)
+		if err != nil {
+			log.Error("GetObsCreateSignedUrl failed: %v", err.Error(), ctx.Data["msgID"])
+			ctx.ServerError("GetObsCreateSignedUrl", err)
+			return
+		}
+		http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
 	}
-	http.Redirect(ctx.Resp, ctx.Req.Request, url, http.StatusMovedPermanently)
 }
 
 func ShowSingleModel(ctx *context.Context) {
@@ -226,7 +284,7 @@ func ShowSingleModel(ctx *context.Context) {
 		return
 	}
 	log.Info("bucket=" + setting.Bucket + " key=" + task.Path[len(setting.Bucket)+1:])
-	models, err := storage.GetAllObsListObjectUnderDir(setting.Bucket, task.Path[len(setting.Bucket)+1:], parentDir)
+	models, err := storage.GetOneLevelAllObjectUnderDir(setting.Bucket, task.Path[len(setting.Bucket)+1:], parentDir)
 	if err != nil {
 		log.Info("get model list failed:", err)
 		ctx.ServerError("GetObsListObject:", err)

routers/routes/routes.go

@@ -966,6 +966,7 @@ func RegisterRoutes(m *macaron.Macaron) {
 				m.Get("", reqRepoCloudBrainReader, repo.ShowSingleModel)
 				m.Get("/downloadsingle", reqRepoCloudBrainReader, repo.DownloadSingleModelFile)
 			})
+			m.Get("/downloadall", reqRepoCloudBrainReader, repo.DownloadMultiModelFile)
 		}, context.RepoRef())
 
 		m.Group("/modelarts", func() {

templates/repo/modelmanage/download.tmpl

@@ -16,7 +16,7 @@
       <div class="row">
         <div class="ui sixteen wide column">
           <div class="dir list">
-             <a class="title" href="{{$.RepoLink}}/modelmanage/downloadall?id={{.ID}}">
+             <a class="title" href="{{$.RepoLink}}/modelmanage/downloadall?ID={{.ID}}">
                   下载所有
              </a>
             {{if .Dirs}}