diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 9b7113ee9..a1a6e131a 100755 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -144,8 +144,8 @@ func (m *MinioStorage) HasObject(path string) (bool, error) { // Indicate to our routine to exit cleanly upon return. defer close(doneCh) - log.Info("minio object key=" + m.buildMinioPath(path)) - objectCh := m.client.ListObjects(m.bucket, m.buildMinioPath(path), false, doneCh) + //objectCh := m.client.ListObjects(m.bucket, m.buildMinioPath(path), false, doneCh) + objectCh := m.client.ListObjects(m.bucket, path, false, doneCh) for object := range objectCh { if object.Err != nil { return hasObject, object.Err diff --git a/modules/storage/minio_ext.go b/modules/storage/minio_ext.go index 4b738c068..0ed9a59a4 100755 --- a/modules/storage/minio_ext.go +++ b/modules/storage/minio_ext.go @@ -101,7 +101,7 @@ func getClients() (*minio_ext.Client, *miniov6.Core, error) { return client, core, nil } -func GenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, partSize int64) (string, error) { +func GenMultiPartSignedUrl(objectName string, uploadId string, partNumber int, partSize int64) (string, error) { minioClient, _, err := getClients() if err != nil { log.Error("getClients failed:", err.Error()) @@ -110,7 +110,7 @@ func GenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, partSiz minio := setting.Attachment.Minio bucketName := minio.Bucket - objectName := strings.TrimPrefix(path.Join(minio.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") + //objectName := strings.TrimPrefix(path.Join(minio.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") return minioClient.GenUploadPartSignedUrl(uploadId, bucketName, objectName, partNumber, partSize, PresignedUploadPartUrlExpireTime, setting.Attachment.Minio.Location) } @@ -301,7 +301,7 @@ func MinioPathCopy(bucketName string, srcPath string, destPath string) (int64, e return fileTotalSize, nil } -func NewMultiPartUpload(uuid string) (string, error) { +func NewMultiPartUpload(objectName string) (string, error) { _, core, err := getClients() if err != nil { log.Error("getClients failed:", err.Error()) @@ -310,7 +310,7 @@ func NewMultiPartUpload(uuid string) (string, error) { minio := setting.Attachment.Minio bucketName := minio.Bucket - objectName := strings.TrimPrefix(path.Join(minio.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") + //objectName := strings.TrimPrefix(path.Join(minio.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") return core.NewMultipartUpload(bucketName, objectName, miniov6.PutObjectOptions{}) } diff --git a/modules/storage/obs.go b/modules/storage/obs.go index 2cb3af927..2ea604c79 100755 --- a/modules/storage/obs.go +++ b/modules/storage/obs.go @@ -114,10 +114,11 @@ func GetObsPartInfos(uuid, uploadID, fileName string) (string, error) { return chunks, nil } -func NewObsMultiPartUpload(uuid, fileName string) (string, error) { +func NewObsMultiPartUpload(objectName string) (string, error) { input := &obs.InitiateMultipartUploadInput{} input.Bucket = setting.Bucket - input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") + input.Key = objectName + //strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") output, err := ObsCli.InitiateMultipartUpload(input) if err != nil { @@ -526,11 +527,12 @@ func GetObsListObject(jobName, outPutPath, parentDir, versionName string) ([]Fil } } -func ObsGenMultiPartSignedUrl(uuid string, uploadId string, partNumber int, fileName string) (string, error) { +func ObsGenMultiPartSignedUrl(objectName string, uploadId string, partNumber int) (string, error) { input := &obs.CreateSignedUrlInput{} input.Bucket = setting.Bucket - input.Key = strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") + input.Key = objectName + //strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, fileName)), "/") input.Expires = 60 * 60 input.Method = obs.HttpMethodPut diff --git a/routers/repo/ai_model_manage.go b/routers/repo/ai_model_manage.go index ee6a71304..b366d5436 100644 --- a/routers/repo/ai_model_manage.go +++ b/routers/repo/ai_model_manage.go @@ -22,6 +22,7 @@ import ( ) const ( + Attachment_model = "model" Model_prefix = "aimodels/" tplModelManageIndex = "repo/modelmanage/index" tplModelManageDownload = "repo/modelmanage/download" diff --git a/routers/repo/attachment.go b/routers/repo/attachment.go index dc2c417e4..d0eeed076 100755 --- a/routers/repo/attachment.go +++ b/routers/repo/attachment.go @@ -11,6 +11,7 @@ import ( "fmt" "mime/multipart" "net/http" + "path" "strconv" "strings" @@ -415,7 +416,7 @@ func AddAttachment(ctx *context.Context) { uuid := ctx.Query("uuid") has := false if typeCloudBrain == models.TypeCloudBrainOne { - has, err = storage.Attachments.HasObject(models.AttachmentRelativePath(uuid)) + has, err = storage.Attachments.HasObject(setting.Attachment.Minio.BasePath + models.AttachmentRelativePath(uuid)) if err != nil { ctx.ServerError("HasObject", err) return @@ -528,10 +529,28 @@ func UpdateAttachmentDecompressState(ctx *context.Context) { }) } +func getCloudOneMinioPrefix(scene string) string { + if scene == Attachment_model { + return Model_prefix + } else { + return setting.Attachment.Minio.BasePath + } +} + +func getCloudTwoOBSPrefix(scene string) string { + if scene == Attachment_model { + return Model_prefix + } else { + return setting.BasePath + } +} + func GetSuccessChunks(ctx *context.Context) { fileMD5 := ctx.Query("md5") typeCloudBrain := ctx.QueryInt("type") fileName := ctx.Query("file_name") + scene := ctx.Query("scene") + log.Info("scene=" + scene) var chunks string err := checkTypeCloudBrain(typeCloudBrain) @@ -557,7 +576,7 @@ func GetSuccessChunks(ctx *context.Context) { isExist := false if typeCloudBrain == models.TypeCloudBrainOne { - isExist, err = storage.Attachments.HasObject(models.AttachmentRelativePath(fileChunk.UUID)) + isExist, err = storage.Attachments.HasObject(getCloudOneMinioPrefix(scene) + models.AttachmentRelativePath(fileChunk.UUID)) if err != nil { ctx.ServerError("HasObject failed", err) return @@ -568,7 +587,7 @@ func GetSuccessChunks(ctx *context.Context) { if oldAttachment != nil { oldFileName = oldAttachment.Name } - isExist, err = storage.ObsHasObject(setting.BasePath + models.AttachmentRelativePath(fileChunk.UUID) + "/" + oldFileName) + isExist, err = storage.ObsHasObject(getCloudTwoOBSPrefix(scene) + models.AttachmentRelativePath(fileChunk.UUID) + "/" + oldFileName) if err != nil { ctx.ServerError("ObsHasObject failed", err) return @@ -642,24 +661,43 @@ func GetSuccessChunks(ctx *context.Context) { }) return } + if scene == Attachment_model { + //使用description存储模型信息 - dataset, err := models.GetDatasetByID(attach.DatasetID) - if err != nil { - ctx.ServerError("GetDatasetByID", err) - return + } else { + dataset, err := models.GetDatasetByID(attach.DatasetID) + if err != nil { + ctx.ServerError("GetDatasetByID", err) + return + } + + ctx.JSON(200, map[string]string{ + "uuid": fileChunk.UUID, + "uploaded": strconv.Itoa(fileChunk.IsUploaded), + "uploadID": fileChunk.UploadID, + "chunks": string(chunks), + "attachID": strconv.Itoa(int(attachID)), + "datasetID": strconv.Itoa(int(attach.DatasetID)), + "fileName": attach.Name, + "datasetName": dataset.Title, + }) } +} - ctx.JSON(200, map[string]string{ - "uuid": fileChunk.UUID, - "uploaded": strconv.Itoa(fileChunk.IsUploaded), - "uploadID": fileChunk.UploadID, - "chunks": string(chunks), - "attachID": strconv.Itoa(int(attachID)), - "datasetID": strconv.Itoa(int(attach.DatasetID)), - "fileName": attach.Name, - "datasetName": dataset.Title, - }) +func getCloudOneMinioObjectName(scene string, uuid string, filename string) string { + if scene == Attachment_model { + return strings.TrimPrefix(path.Join(Model_prefix, path.Join(uuid[0:1], uuid[1:2], uuid, filename)), "/") + } else { + return strings.TrimPrefix(path.Join(setting.Attachment.Minio.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid)), "/") + } +} +func getCloudTwoOBSObjectName(scene string, uuid string, filename string) string { + if scene == Attachment_model { + return strings.TrimPrefix(path.Join(Model_prefix, path.Join(uuid[0:1], uuid[1:2], uuid, filename)), "/") + } else { + return strings.TrimPrefix(path.Join(setting.BasePath, path.Join(uuid[0:1], uuid[1:2], uuid, filename)), "/") + } } func NewMultipart(ctx *context.Context) { @@ -682,6 +720,7 @@ func NewMultipart(ctx *context.Context) { } fileName := ctx.Query("file_name") + scene := ctx.Query("scene") if setting.Attachment.StoreType == storage.MinioStorageType { totalChunkCounts := ctx.QueryInt("totalChunkCounts") @@ -699,13 +738,13 @@ func NewMultipart(ctx *context.Context) { uuid := gouuid.NewV4().String() var uploadID string if typeCloudBrain == models.TypeCloudBrainOne { - uploadID, err = storage.NewMultiPartUpload(uuid) + uploadID, err = storage.NewMultiPartUpload(getCloudOneMinioObjectName(scene, uuid, fileName)) if err != nil { ctx.ServerError("NewMultipart", err) return } } else { - uploadID, err = storage.NewObsMultiPartUpload(uuid, fileName) + uploadID, err = storage.NewObsMultiPartUpload(getCloudTwoOBSObjectName(scene, uuid, fileName)) if err != nil { ctx.ServerError("NewObsMultiPartUpload", err) return @@ -790,7 +829,7 @@ func GetMultipartUploadUrl(ctx *context.Context) { partNumber := ctx.QueryInt("chunkNumber") size := ctx.QueryInt64("size") fileName := ctx.Query("file_name") - + scene := ctx.Query("scene") typeCloudBrain := ctx.QueryInt("type") err := checkTypeCloudBrain(typeCloudBrain) if err != nil { @@ -805,7 +844,7 @@ func GetMultipartUploadUrl(ctx *context.Context) { return } - url, err = storage.GenMultiPartSignedUrl(uuid, uploadID, partNumber, size) + url, err = storage.GenMultiPartSignedUrl(getCloudOneMinioObjectName(scene, uuid, fileName), uploadID, partNumber, size) if err != nil { ctx.Error(500, fmt.Sprintf("GenMultiPartSignedUrl failed: %v", err)) return @@ -815,7 +854,7 @@ func GetMultipartUploadUrl(ctx *context.Context) { url = setting.PROXYURL + "/obs_proxy_multipart?uuid=" + uuid + "&uploadId=" + uploadID + "&partNumber=" + fmt.Sprint(partNumber) + "&file_name=" + fileName log.Info("return url=" + url) } else { - url, err = storage.ObsGenMultiPartSignedUrl(uuid, uploadID, partNumber, fileName) + url, err = storage.ObsGenMultiPartSignedUrl(getCloudTwoOBSObjectName(scene, uuid, fileName), uploadID, partNumber) if err != nil { ctx.Error(500, fmt.Sprintf("ObsGenMultiPartSignedUrl failed: %v", err)) return @@ -1013,7 +1052,7 @@ func queryDatasets(ctx *context.Context, attachs []*models.AttachmentUsername) { } for _, attch := range attachs { - has, err := storage.Attachments.HasObject(models.AttachmentRelativePath(attch.UUID)) + has, err := storage.Attachments.HasObject(setting.Attachment.Minio.BasePath + models.AttachmentRelativePath(attch.UUID)) if err != nil || !has { continue }