aiforge/modules/storage/obs.go

// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package storage

import (
	"io"
	"net/url"
	"path"
	"sort"
	"strconv"
	"strings"

	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/obs"
	"code.gitea.io/gitea/modules/setting"

	"github.com/unknwon/com"
)

type FileInfo struct {
	FileName string `json:"FileName"`
	ModTime  string `json:"ModTime"`
	IsDir    bool   `json:"IsDir"`
	Size     int64  `json:"Size"`
	ParenDir string `json:"ParenDir"`
	UUID     string `json:"UUID"`
}

//check if has the object
//todo：修改查询方式
func ObsHasObject(path string) (bool, error) {
	hasObject := false
	output, err := ObsCli.ListObjects(&obs.ListObjectsInput{Bucket: setting.Bucket})
	if err != nil {
		log.Error("ListObjects failed:%v", err)
		return hasObject, err
	}

	for _, obj := range output.Contents {
		//obj.Key:attachment/0/1/019fd24e-4ef7-41cc-9f85-4a7b8504d958
		if path == obj.Key {
			hasObject = true
			break
		}
	}

	return hasObject, nil
}

func GetObsPartInfos(uuid string, uploadID string) (string, error) {
	key := strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, uuid)), "/")

	output, err := ObsCli.ListParts(&obs.ListPartsInput{
		Bucket:   setting.Bucket,
		Key:      key,
		UploadId: uploadID,
	})
	if err != nil {
		log.Error("ListParts failed:", err.Error())
		return "", err
	}

	var chunks string
	for _, partInfo := range output.Parts {
		chunks += strconv.Itoa(partInfo.PartNumber) + "-" + partInfo.ETag + ","
	}

	return chunks, nil
}

func NewObsMultiPartUpload(uuid, fileName string) (string, error) {
	input := &obs.InitiateMultipartUploadInput{}
	input.Bucket = setting.Bucket
	input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")

	output, err := ObsCli.InitiateMultipartUpload(input)
	if err != nil {
		log.Error("InitiateMultipartUpload failed:", err.Error())
		return "", err
	}

	return output.UploadId, nil
}

func CompleteObsMultiPartUpload(uuid, uploadID, fileName string) error {
	input := &obs.CompleteMultipartUploadInput{}
	input.Bucket = setting.Bucket
	input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
	input.UploadId = uploadID
	output, err := ObsCli.ListParts(&obs.ListPartsInput{
		Bucket:   setting.Bucket,
		Key:      input.Key,
		UploadId: uploadID,
	})
	if err != nil {
		log.Error("ListParts failed:", err.Error())
		return err
	}

	for _, partInfo := range output.Parts {
		input.Parts = append(input.Parts, obs.Part{
			PartNumber: partInfo.PartNumber,
			ETag:       partInfo.ETag,
		})
	}

	_, err = ObsCli.CompleteMultipartUpload(input)
	if err != nil {
		log.Error("CompleteMultipartUpload failed:", err.Error())
		return err
	}

	return nil
}

func ObsMultiPartUpload(uuid string, uploadId string, partNumber int, fileName string, putBody io.ReadCloser) error {
	input := &obs.UploadPartInput{}
	input.Bucket = setting.Bucket
	input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
	input.UploadId = uploadId
	input.PartNumber = partNumber
	input.Body = putBody
	output, err := ObsCli.UploadPart(input)
	if err == nil {
		log.Info("RequestId:%s\n", output.RequestId)
		log.Info("ETag:%s\n", output.ETag)
		return nil
	} else {
		if obsError, ok := err.(obs.ObsError); ok {
			log.Info(obsError.Code)
			log.Info(obsError.Message)
			return obsError
		} else {
			log.Error("error:", err.Error())
			return err
		}
	}

}

func ObsDownload(uuid string, fileName string) (io.ReadCloser, error) {
	input := &obs.GetObjectInput{}
	input.Bucket = setting.Bucket
	input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
	// input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/")
	output, err := ObsCli.GetObject(input)
	if err == nil {
		log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n",
			output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified)
		return output.Body, nil
	} else if obsError, ok := err.(obs.ObsError); ok {
		log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message)
		return nil, obsError
	} else {
		return nil, err
	}
}

func ObsModelDownload(JobName string, fileName string) (io.ReadCloser, error) {
	input := &obs.GetObjectInput{}
	input.Bucket = setting.Bucket
	input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, JobName, setting.OutPutPath, fileName), "/")
	// input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/")
	output, err := ObsCli.GetObject(input)
	if err == nil {
		log.Info("StorageClass:%s, ETag:%s, ContentType:%s, ContentLength:%d, LastModified:%s\n",
			output.StorageClass, output.ETag, output.ContentType, output.ContentLength, output.LastModified)
		return output.Body, nil
	} else if obsError, ok := err.(obs.ObsError); ok {
		log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message)
		return nil, obsError
	} else {
		return nil, err
	}
}

func GetObsListObject(jobName, parentDir string) ([]FileInfo, error) {
	input := &obs.ListObjectsInput{}
	input.Bucket = setting.Bucket
	input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/")
	strPrefix := strings.Split(input.Prefix, "/")
	output, err := ObsCli.ListObjects(input)
	fileInfos := make([]FileInfo, 0)
	if err == nil {
		for _, val := range output.Contents {
			str1 := strings.Split(val.Key, "/")
			var isDir bool
			var fileName, nextParentDir string
			if strings.HasSuffix(val.Key, "/") {
				//dirs in next level dir
				if len(str1)-len(strPrefix) > 2 {
					continue
				}
				fileName = str1[len(str1)-2]
				isDir = true
				if parentDir == "" {
					nextParentDir = fileName
				} else {
					nextParentDir = parentDir + "/" + fileName
				}

				if fileName == strPrefix[len(strPrefix)-1] || (fileName+"/") == setting.OutPutPath {
					continue
				}
			} else {
				//files in next level dir
				if len(str1)-len(strPrefix) > 1 {
					continue
				}
				fileName = str1[len(str1)-1]
				isDir = false
				nextParentDir = parentDir
			}

			fileInfo := FileInfo{
				ModTime:  val.LastModified.Local().Format("2006-01-02 15:04:05"),
				FileName: fileName,
				Size:     val.Size,
				IsDir:    isDir,
				ParenDir: nextParentDir,
			}
			fileInfos = append(fileInfos, fileInfo)
		}
		sort.Slice(fileInfos, func(i, j int) bool {
			return fileInfos[i].ModTime > fileInfos[j].ModTime
		})
		return fileInfos, err
	} else {
		if obsError, ok := err.(obs.ObsError); ok {
			log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message)
		}
		return nil, err
	}
}

func GetObsListObjectVersion(jobName, parentDir string, VersionOutputPath string) ([]FileInfo, error) {
	input := &obs.ListObjectsInput{}
	input.Bucket = setting.Bucket
	input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, VersionOutputPath, parentDir), "/")
	strPrefix := strings.Split(input.Prefix, "/")
	output, err := ObsCli.ListObjects(input)
	fileInfos := make([]FileInfo, 0)
	if err == nil {
		for _, val := range output.Contents {
			str1 := strings.Split(val.Key, "/")
			var isDir bool
			var fileName, nextParentDir string
			if strings.HasSuffix(val.Key, "/") {
				//dirs in next level dir
				if len(str1)-len(strPrefix) > 2 {
					continue
				}
				fileName = str1[len(str1)-2]
				isDir = true
				if parentDir == "" {
					nextParentDir = fileName
				} else {
					nextParentDir = parentDir + "/" + fileName
				}

				if fileName == strPrefix[len(strPrefix)-1] || (fileName+"/") == setting.OutPutPath {
					continue
				}
			} else {
				//files in next level dir
				if len(str1)-len(strPrefix) > 1 {
					continue
				}
				fileName = str1[len(str1)-1]
				isDir = false
				nextParentDir = parentDir
			}

			fileInfo := FileInfo{
				ModTime:  val.LastModified.Local().Format("2006-01-02 15:04:05"),
				FileName: fileName,
				Size:     val.Size,
				IsDir:    isDir,
				ParenDir: nextParentDir,
			}
			fileInfos = append(fileInfos, fileInfo)
		}
		sort.Slice(fileInfos, func(i, j int) bool {
			return fileInfos[i].ModTime > fileInfos[j].ModTime
		})
		return fileInfos, err
	} else {
		if obsError, ok := err.(obs.ObsError); ok {
			log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message)
		}
		return nil, err
	}
}

func GetVersionObsListObject(jobName, parentDir string) ([]FileInfo, error) {
	input := &obs.ListObjectsInput{}
	input.Bucket = setting.Bucket
	input.Prefix = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir), "/")
	strPrefix := strings.Split(input.Prefix, "/")
	output, err := ObsCli.ListObjects(input)
	fileInfos := make([]FileInfo, 0)
	if err == nil {
		for _, val := range output.Contents {
			str1 := strings.Split(val.Key, "/")
			var isDir bool
			var fileName, nextParentDir string
			if strings.HasSuffix(val.Key, "/") {
				//dirs in next level dir
				if len(str1)-len(strPrefix) > 2 {
					continue
				}
				fileName = str1[len(str1)-2]
				isDir = true
				if parentDir == "" {
					nextParentDir = fileName
				} else {
					nextParentDir = parentDir + "/" + fileName
				}

				if fileName == strPrefix[len(strPrefix)-1] || (fileName+"/") == setting.OutPutPath {
					continue
				}
			} else {
				//files in next level dir
				if len(str1)-len(strPrefix) > 1 {
					continue
				}
				fileName = str1[len(str1)-1]
				isDir = false
				nextParentDir = parentDir
			}

			fileInfo := FileInfo{
				ModTime:  val.LastModified.Local().Format("2006-01-02 15:04:05"),
				FileName: fileName,
				Size:     val.Size,
				IsDir:    isDir,
				ParenDir: nextParentDir,
			}
			fileInfos = append(fileInfos, fileInfo)
		}
		return fileInfos, err
	} else {
		if obsError, ok := err.(obs.ObsError); ok {
			log.Error("Code:%s, Message:%s", obsError.Code, obsError.Message)
		}
		return nil, err
	}
}

func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) {

	input := &obs.CreateSignedUrlInput{}
	input.Bucket = setting.Bucket
	input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
	input.Expires = 60 * 60
	input.Method = obs.HttpMethodPut

	input.QueryParams = map[string]string{
		"partNumber": com.ToStr(partNumber, 10),
		"uploadId":   uploadId,
		//"partSize":	com.ToStr(partSize,10),
	}

	output, err := ObsCli.CreateSignedUrl(input)
	if err != nil {
		log.Error("CreateSignedUrl failed:", err.Error())
		return "", err
	}

	return output.SignedUrl, nil
}

func GetObsCreateSignedUrl(jobName, parentDir, fileName string) (string, error) {
	input := &obs.CreateSignedUrlInput{}
	input.Bucket = setting.Bucket
	input.Key = strings.TrimPrefix(path.Join(setting.TrainJobModelPath, jobName, setting.OutPutPath, parentDir, fileName), "/")

	input.Expires = 60 * 60
	input.Method = obs.HttpMethodGet

	reqParams := make(map[string]string)
	fileName = url.QueryEscape(fileName)
	reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\""
	input.QueryParams = reqParams
	output, err := ObsCli.CreateSignedUrl(input)
	if err != nil {
		log.Error("CreateSignedUrl failed:", err.Error())
		return "", err
	}

	return output.SignedUrl, nil
}

func ObsGetPreSignedUrl(uuid, fileName string) (string, error) {
	input := &obs.CreateSignedUrlInput{}
	input.Method = obs.HttpMethodGet
	input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/")
	input.Bucket = setting.Bucket
	input.Expires = 60 * 60

	reqParams := make(map[string]string)
	reqParams["response-content-disposition"] = "attachment; filename=\"" + fileName + "\""
	input.QueryParams = reqParams
	output, err := ObsCli.CreateSignedUrl(input)
	if err != nil {
		log.Error("CreateSignedUrl failed:", err.Error())
		return "", err
	}

	return output.SignedUrl, nil
}

func ObsCreateObject(path string) error {
	input := &obs.PutObjectInput{}
	input.Bucket = setting.Bucket
	input.Key = path

	_, err := ObsCli.PutObject(input)
	if err != nil {
		log.Error("PutObject failed:", err.Error())
		return err
	}

	return nil
}