JeshuaRen 2 years ago
parent
commit
cfb0efc16f
12 changed files with 567 additions and 112 deletions
  1. +15
    -0
      pkgs/iterator/empty.go
  2. +80
    -10
      sdks/storage/bucket.go
  3. +1
    -1
      sdks/storage/cache.go
  4. +1
    -1
      sdks/storage/models.go
  5. +1
    -1
      sdks/storage/node.go
  6. +169
    -19
      sdks/storage/object.go
  7. +110
    -43
      sdks/storage/package.go
  8. +2
    -2
      sdks/storage/storage.go
  9. +33
    -33
      sdks/storage/storage_test.go
  10. +32
    -1
      sdks/storage/utils.go
  11. +111
    -1
      utils/http/http.go
  12. +12
    -0
      utils/serder/serder.go

+ 15
- 0
pkgs/iterator/empty.go View File

@@ -0,0 +1,15 @@
package iterator

type emptyIterator[T any] struct{}

func (i *emptyIterator[T]) MoveNext() (T, error) {
var ret T
return ret, ErrNoMoreItem
}
func (i *emptyIterator[T]) Close() {

}

func Empty[T any]() Iterator[T] {
return &emptyIterator[T]{}
}

+ 80
- 10
sdks/storage/bucket.go View File

@@ -15,18 +15,53 @@ func (c *Client) Bucket() *BucketService {
return &BucketService{c}
}

const BucketGetByNamePath = "/bucket/getByName"

type BucketGetByName struct {
UserID UserID `json:"userID" form:"userID" binding:"required"`
Name string `json:"name" form:"name" binding:"required"`
}
type BucketGetByNameResp struct {
Bucket Bucket `json:"bucket"`
}

func (c *BucketService) GetByName(req BucketGetByName) (*BucketGetByNameResp, error) {
url, err := url.JoinPath(c.baseURL, BucketGetByNamePath)
if err != nil {
return nil, err
}

resp, err := myhttp.GetForm(url, myhttp.RequestParam{
Query: req,
})
if err != nil {
return nil, err
}

codeResp, err := ParseJSONResponse[response[BucketGetByNameResp]](resp)
if err != nil {
return nil, err
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}

return nil, codeResp.ToError()
}

const BucketCreatePath = "/bucket/create"

type BucketCreateReq struct {
UserID UserID `json:"userID" binding:"required"`
BucketName string `json:"bucketName" binding:"required"`
type BucketCreate struct {
UserID UserID `json:"userID" binding:"required"`
Name string `json:"name" binding:"required"`
}

type BucketCreateResp struct {
BucketID BucketID `json:"bucketID"`
Bucket Bucket `json:"bucket"`
}

func (c *BucketService) Create(req BucketCreateReq) (*BucketCreateResp, error) {
func (c *BucketService) Create(req BucketCreate) (*BucketCreateResp, error) {
url, err := url.JoinPath(c.baseURL, BucketCreatePath)
if err != nil {
return nil, err
@@ -39,7 +74,7 @@ func (c *BucketService) Create(req BucketCreateReq) (*BucketCreateResp, error) {
return nil, err
}

codeResp, err := myhttp.ParseJSONResponse[response[BucketCreateResp]](resp)
codeResp, err := ParseJSONResponse[response[BucketCreateResp]](resp)
if err != nil {
return nil, err
}
@@ -53,27 +88,62 @@ func (c *BucketService) Create(req BucketCreateReq) (*BucketCreateResp, error) {

const BucketDeletePath = "/bucket/delete"

type BucketDeleteReq struct {
type BucketDelete struct {
UserID UserID `json:"userID" binding:"required"`
BucketID BucketID `json:"bucketID" binding:"required"`
}

type BucketDeleteResp struct{}

func (c *BucketService) Delete(req BucketDeleteReq) (*BucketDeleteResp, error) {
func (c *BucketService) Delete(req BucketDelete) error {
url, err := url.JoinPath(c.baseURL, BucketDeletePath)
if err != nil {
return nil, err
return err
}

resp, err := myhttp.PostJSON(url, myhttp.RequestParam{
Body: req,
})
if err != nil {
return err
}

codeResp, err := ParseJSONResponse[response[BucketDeleteResp]](resp)
if err != nil {
return err
}

if codeResp.Code == errorcode.OK {
return nil
}

return codeResp.ToError()
}

const BucketListUserBucketsPath = "/bucket/listUserBuckets"

type BucketListUserBucketsReq struct {
UserID UserID `form:"userID" json:"userID" binding:"required"`
}

type BucketListUserBucketsResp struct {
Buckets []Bucket `json:"buckets"`
}

func (c *BucketService) ListUserBuckets(req BucketListUserBucketsReq) (*BucketListUserBucketsResp, error) {
url, err := url.JoinPath(c.baseURL, BucketListUserBucketsPath)
if err != nil {
return nil, err
}

resp, err := myhttp.GetForm(url, myhttp.RequestParam{
Query: req,
})
if err != nil {
return nil, err
}

codeResp, err := myhttp.ParseJSONResponse[response[BucketDeleteResp]](resp)
codeResp, err := ParseJSONResponse[response[BucketListUserBucketsResp]](resp)
if err != nil {
return nil, err
}


+ 1
- 1
sdks/storage/cache.go View File

@@ -29,7 +29,7 @@ func (c *Client) CacheMovePackage(req CacheMovePackageReq) (*CacheMovePackageRes
return nil, err
}

jsonResp, err := myhttp.ParseJSONResponse[response[CacheMovePackageResp]](resp)
jsonResp, err := ParseJSONResponse[response[CacheMovePackageResp]](resp)
if err != nil {
return nil, err
}


+ 1
- 1
sdks/storage/models.go View File

@@ -10,7 +10,7 @@ import (
)

const (
ObjectPathSeperator = "/"
ObjectPathSeparator = "/"
)

type NodeID int64


+ 1
- 1
sdks/storage/node.go View File

@@ -30,7 +30,7 @@ func (c *Client) NodeGetNodes(req NodeGetNodesReq) (*NodeGetNodesResp, error) {
return nil, err
}

jsonResp, err := myhttp.ParseJSONResponse[response[NodeGetNodesResp]](resp)
jsonResp, err := ParseJSONResponse[response[NodeGetNodesResp]](resp)
if err != nil {
return nil, err
}


+ 169
- 19
sdks/storage/object.go View File

@@ -5,6 +5,7 @@ import (
"io"
"net/url"
"strings"
"time"

"gitlink.org.cn/cloudream/common/consts/errorcode"
"gitlink.org.cn/cloudream/common/pkgs/iterator"
@@ -24,7 +25,7 @@ func (c *Client) Object() *ObjectService {

const ObjectUploadPath = "/object/upload"

type ObjectUploadReq struct {
type ObjectUpload struct {
ObjectUploadInfo
Files UploadObjectIterator `json:"-"`
}
@@ -35,16 +36,22 @@ type ObjectUploadInfo struct {
NodeAffinity *NodeID `json:"nodeAffinity"`
}

type IterObjectUpload struct {
type UploadingObject struct {
Path string
File io.ReadCloser
}

type UploadObjectIterator = iterator.Iterator[*IterObjectUpload]
type UploadObjectIterator = iterator.Iterator[*UploadingObject]

type ObjectUploadResp struct{}
type ObjectUploadResp struct {
Uploadeds []UploadedObject `json:"uploadeds"`
}
type UploadedObject struct {
Object *Object `json:"object"`
Error string `json:"error"`
}

func (c *ObjectService) Upload(req ObjectUploadReq) (*ObjectUploadResp, error) {
func (c *ObjectService) Upload(req ObjectUpload) (*ObjectUploadResp, error) {
url, err := url.JoinPath(c.baseURL, ObjectUploadPath)
if err != nil {
return nil, err
@@ -57,7 +64,7 @@ func (c *ObjectService) Upload(req ObjectUploadReq) (*ObjectUploadResp, error) {

resp, err := myhttp.PostMultiPart(url, myhttp.MultiPartRequestParam{
Form: map[string]string{"info": string(infoJSON)},
Files: iterator.Map(req.Files, func(src *IterObjectUpload) (*myhttp.IterMultiPartFile, error) {
Files: iterator.Map(req.Files, func(src *UploadingObject) (*myhttp.IterMultiPartFile, error) {
return &myhttp.IterMultiPartFile{
FieldName: "files",
FileName: src.Path,
@@ -71,8 +78,9 @@ func (c *ObjectService) Upload(req ObjectUploadReq) (*ObjectUploadResp, error) {

contType := resp.Header.Get("Content-Type")
if strings.Contains(contType, myhttp.ContentTypeJSON) {
var err error
var codeResp response[ObjectUploadResp]
if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil {
if codeResp, err = serder.JSONToObjectStreamEx[response[ObjectUploadResp]](resp.Body); err != nil {
return nil, fmt.Errorf("parsing response: %w", err)
}

@@ -89,12 +97,16 @@ func (c *ObjectService) Upload(req ObjectUploadReq) (*ObjectUploadResp, error) {

const ObjectDownloadPath = "/object/download"

type ObjectDownloadReq struct {
UserID int64 `json:"userID"`
ObjectID int64 `json:"objectID"`
type ObjectDownload struct {
UserID UserID `form:"userID" json:"userID" binding:"required"`
ObjectID ObjectID `form:"objectID" json:"objectID" binding:"required"`
}
type DownloadingObject struct {
Path string
File io.ReadCloser
}

func (c *ObjectService) Download(req ObjectDownloadReq) (io.ReadCloser, error) {
func (c *ObjectService) Download(req ObjectDownload) (*DownloadingObject, error) {
url, err := url.JoinPath(c.baseURL, ObjectDownloadPath)
if err != nil {
return nil, err
@@ -118,24 +130,162 @@ func (c *ObjectService) Download(req ObjectDownloadReq) (io.ReadCloser, error) {
return nil, codeResp.ToError()
}

if strings.Contains(contType, myhttp.ContentTypeOctetStream) {
return resp.Body, nil
_, files, err := myhttp.ParseMultiPartResponse(resp)
if err != nil {
return nil, err
}

return nil, fmt.Errorf("unknow response content type: %s", contType)
file, err := files.MoveNext()
if err == iterator.ErrNoMoreItem {
return nil, fmt.Errorf("no file found in response")
}
if err != nil {
return nil, err
}

return &DownloadingObject{
Path: file.FileName,
File: file.File,
}, nil
}

const ObjectUpdateInfoPath = "/object/updateInfo"

type UpdatingObject struct {
ObjectID ObjectID `json:"objectID" binding:"required"`
UpdateTime time.Time `json:"updateTime" binding:"required"`
}

func (u *UpdatingObject) ApplyTo(obj *Object) {
obj.UpdateTime = u.UpdateTime
}

type ObjectUpdateInfo struct {
UserID UserID `json:"userID" binding:"required"`
Updatings []UpdatingObject `json:"updatings" binding:"required"`
}

type ObjectUpdateInfoResp struct {
Successes []ObjectID `json:"successes"`
}

func (c *ObjectService) UpdateInfo(req ObjectUpdateInfo) (*ObjectUpdateInfoResp, error) {
url, err := url.JoinPath(c.baseURL, ObjectUpdateInfoPath)
if err != nil {
return nil, err
}

resp, err := myhttp.PostJSON(url, myhttp.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}

jsonResp, err := ParseJSONResponse[response[ObjectUpdateInfoResp]](resp)
if err != nil {
return nil, err
}

if jsonResp.Code == errorcode.OK {
return &jsonResp.Data, nil
}

return nil, jsonResp.ToError()
}

const ObjectMovePath = "/object/move"

type MovingObject struct {
ObjectID ObjectID `json:"objectID" binding:"required"`
PackageID PackageID `json:"packageID" binding:"required"`
Path string `json:"path" binding:"required"`
}

func (m *MovingObject) ApplyTo(obj *Object) {
obj.PackageID = m.PackageID
obj.Path = m.Path
}

type ObjectMove struct {
UserID UserID `json:"userID" binding:"required"`
Movings []MovingObject `json:"movings" binding:"required"`
}

type ObjectMoveResp struct {
Successes []ObjectID `json:"successes"`
}

func (c *ObjectService) Move(req ObjectMove) (*ObjectMoveResp, error) {
url, err := url.JoinPath(c.baseURL, ObjectMovePath)
if err != nil {
return nil, err
}

resp, err := myhttp.PostJSON(url, myhttp.RequestParam{
Body: req,
})
if err != nil {
return nil, err
}

jsonResp, err := ParseJSONResponse[response[ObjectMoveResp]](resp)
if err != nil {
return nil, err
}

if jsonResp.Code == errorcode.OK {
return &jsonResp.Data, nil
}

return nil, jsonResp.ToError()
}

const ObjectDeletePath = "/object/delete"

type ObjectDelete struct {
UserID UserID `json:"userID" binding:"required"`
ObjectIDs []ObjectID `json:"objectIDs" binding:"required"`
}

type ObjectDeleteResp struct{}

func (c *ObjectService) Delete(req ObjectDelete) error {
url, err := url.JoinPath(c.baseURL, ObjectDeletePath)
if err != nil {
return err
}

resp, err := myhttp.PostJSON(url, myhttp.RequestParam{
Body: req,
})
if err != nil {
return err
}

jsonResp, err := ParseJSONResponse[response[ObjectDeleteResp]](resp)
if err != nil {
return err
}

if jsonResp.Code == errorcode.OK {
return nil
}

return jsonResp.ToError()
}

const ObjectGetPackageObjectsPath = "/object/getPackageObjects"

type ObjectGetPackageObjectsReq struct {
UserID UserID `json:"userID"`
PackageID PackageID `json:"packageID"`
type ObjectGetPackageObjects struct {
UserID UserID `form:"userID" json:"userID" binding:"required"`
PackageID PackageID `form:"packageID" json:"packageID" binding:"required"`
}
type ObjectGetPackageObjectsResp struct {
Objects []Object `json:"objects"`
}

func (c *ObjectService) GetPackageObjects(req ObjectGetPackageObjectsReq) (*ObjectGetPackageObjectsResp, error) {
func (c *ObjectService) GetPackageObjects(req ObjectGetPackageObjects) (*ObjectGetPackageObjectsResp, error) {
url, err := url.JoinPath(c.baseURL, ObjectGetPackageObjectsPath)
if err != nil {
return nil, err
@@ -148,7 +298,7 @@ func (c *ObjectService) GetPackageObjects(req ObjectGetPackageObjectsReq) (*Obje
return nil, err
}

jsonResp, err := myhttp.ParseJSONResponse[response[ObjectGetPackageObjectsResp]](resp)
jsonResp, err := ParseJSONResponse[response[ObjectGetPackageObjectsResp]](resp)
if err != nil {
return nil, err
}


+ 110
- 43
sdks/storage/package.go View File

@@ -21,8 +21,8 @@ func (c *Client) Package() *PackageService {
const PackageGetPath = "/package/get"

type PackageGetReq struct {
UserID UserID `json:"userID"`
PackageID PackageID `json:"packageID"`
UserID UserID `form:"userID" json:"userID" binding:"required"`
PackageID PackageID `form:"packageID" json:"packageID" binding:"required"`
}
type PackageGetResp struct {
Package
@@ -41,7 +41,43 @@ func (c *PackageService) Get(req PackageGetReq) (*PackageGetResp, error) {
return nil, err
}

codeResp, err := myhttp.ParseJSONResponse[response[PackageGetResp]](resp)
codeResp, err := ParseJSONResponse[response[PackageGetResp]](resp)
if err != nil {
return nil, err
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}

return nil, codeResp.ToError()
}

const PackageGetByNamePath = "/package/getByName"

type PackageGetByName struct {
UserID UserID `form:"userID" json:"userID" binding:"required"`
BucketName string `form:"bucketName" json:"bucketName" binding:"required"`
PackageName string `form:"packageName" json:"packageName" binding:"required"`
}
type PackageGetByNameResp struct {
Package Package `json:"package"`
}

func (c *PackageService) GetByName(req PackageGetByName) (*PackageGetByNameResp, error) {
url, err := url.JoinPath(c.baseURL, PackageGetByNamePath)
if err != nil {
return nil, err
}

resp, err := myhttp.GetForm(url, myhttp.RequestParam{
Query: req,
})
if err != nil {
return nil, err
}

codeResp, err := ParseJSONResponse[response[PackageGetByNameResp]](resp)
if err != nil {
return nil, err
}
@@ -55,17 +91,17 @@ func (c *PackageService) Get(req PackageGetReq) (*PackageGetResp, error) {

const PackageCreatePath = "/package/create"

type PackageCreateReq struct {
type PackageCreate struct {
UserID UserID `json:"userID"`
BucketID BucketID `json:"bucketID"`
Name string `json:"name"`
}

type PackageCreateResp struct {
PackageID PackageID `json:"packageID,string"`
Package Package `json:"package"`
}

func (s *PackageService) Create(req PackageCreateReq) (*PackageCreateResp, error) {
func (s *PackageService) Create(req PackageCreate) (*PackageCreateResp, error) {
url, err := url.JoinPath(s.baseURL, PackageCreatePath)
if err != nil {
return nil, err
@@ -78,7 +114,7 @@ func (s *PackageService) Create(req PackageCreateReq) (*PackageCreateResp, error
return nil, err
}

codeResp, err := myhttp.ParseJSONResponse[response[PackageCreateResp]](resp)
codeResp, err := ParseJSONResponse[response[PackageCreateResp]](resp)
if err != nil {
return nil, err
}
@@ -90,13 +126,15 @@ func (s *PackageService) Create(req PackageCreateReq) (*PackageCreateResp, error
return nil, codeResp.ToError()
}

type PackageDeleteReq struct {
UserID UserID `json:"userID"`
PackageID PackageID `json:"packageID"`
const PackageDeletePath = "/package/delete"

type PackageDelete struct {
UserID UserID `json:"userID" binding:"required"`
PackageID PackageID `json:"packageID" binding:"required"`
}

func (c *PackageService) Delete(req PackageDeleteReq) error {
url, err := url.JoinPath(c.baseURL, "/package/delete")
func (c *PackageService) Delete(req PackageDelete) error {
url, err := url.JoinPath(c.baseURL, PackageDeletePath)
if err != nil {
return err
}
@@ -126,9 +164,47 @@ func (c *PackageService) Delete(req PackageDeleteReq) error {
return fmt.Errorf("unknow response content type: %s", contType)
}

const PackageListBucketPackagesPath = "/package/listBucketPackages"

type PackageListBucketPackages struct {
UserID UserID `form:"userID" json:"userID" binding:"required"`
BucketID BucketID `form:"bucketID" json:"bucketID" binding:"required"`
}

type PackageListBucketPackagesResp struct {
Packages []Package `json:"packages"`
}

func (c *PackageService) ListBucketPackages(req PackageListBucketPackages) (*PackageListBucketPackagesResp, error) {
url, err := url.JoinPath(c.baseURL, PackageListBucketPackagesPath)
if err != nil {
return nil, err
}

resp, err := myhttp.GetForm(url, myhttp.RequestParam{
Query: req,
})
if err != nil {
return nil, err
}

codeResp, err := ParseJSONResponse[response[PackageListBucketPackagesResp]](resp)
if err != nil {
return nil, err
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}

return nil, codeResp.ToError()
}

const PackageGetCachedNodesPath = "/package/getCachedNodes"

type PackageGetCachedNodesReq struct {
PackageID PackageID `json:"packageID"`
UserID UserID `json:"userID"`
PackageID PackageID `form:"packageID" json:"packageID" binding:"required"`
UserID UserID `form:"userID" json:"userID" binding:"required"`
}

type PackageGetCachedNodesResp struct {
@@ -136,7 +212,7 @@ type PackageGetCachedNodesResp struct {
}

func (c *PackageService) GetCachedNodes(req PackageGetCachedNodesReq) (*PackageGetCachedNodesResp, error) {
url, err := url.JoinPath(c.baseURL, "/package/getCachedNodes")
url, err := url.JoinPath(c.baseURL, PackageGetCachedNodesPath)
if err != nil {
return nil, err
}
@@ -146,27 +222,24 @@ func (c *PackageService) GetCachedNodes(req PackageGetCachedNodesReq) (*PackageG
if err != nil {
return nil, err
}
contType := resp.Header.Get("Content-Type")
if strings.Contains(contType, myhttp.ContentTypeJSON) {

var codeResp response[PackageGetCachedNodesResp]
if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil {
return nil, fmt.Errorf("parsing response: %w", err)
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}
codeResp, err := ParseJSONResponse[response[PackageGetCachedNodesResp]](resp)
if err != nil {
return nil, err
}

return nil, codeResp.ToError()
if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}

return nil, fmt.Errorf("unknow response content type: %s", contType)
return nil, codeResp.ToError()
}

const PackageGetLoadedNodesPath = "/package/getLoadedNodes"

type PackageGetLoadedNodesReq struct {
PackageID PackageID `json:"packageID"`
UserID UserID `json:"userID"`
PackageID PackageID `form:"packageID" json:"packageID" binding:"required"`
UserID UserID `form:"userID" json:"userID" binding:"required"`
}

type PackageGetLoadedNodesResp struct {
@@ -174,7 +247,7 @@ type PackageGetLoadedNodesResp struct {
}

func (c *PackageService) GetLoadedNodes(req PackageGetLoadedNodesReq) (*PackageGetLoadedNodesResp, error) {
url, err := url.JoinPath(c.baseURL, "/package/getLoadedNodes")
url, err := url.JoinPath(c.baseURL, PackageGetLoadedNodesPath)
if err != nil {
return nil, err
}
@@ -185,20 +258,14 @@ func (c *PackageService) GetLoadedNodes(req PackageGetLoadedNodesReq) (*PackageG
return nil, err
}

contType := resp.Header.Get("Content-Type")
if strings.Contains(contType, myhttp.ContentTypeJSON) {

var codeResp response[PackageGetLoadedNodesResp]
if err := serder.JSONToObjectStream(resp.Body, &codeResp); err != nil {
return nil, fmt.Errorf("parsing response: %w", err)
}

if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}
codeResp, err := ParseJSONResponse[response[PackageGetLoadedNodesResp]](resp)
if err != nil {
return nil, err
}

return nil, codeResp.ToError()
if codeResp.Code == errorcode.OK {
return &codeResp.Data, nil
}

return nil, fmt.Errorf("unknow response content type: %s", contType)
return nil, codeResp.ToError()
}

+ 2
- 2
sdks/storage/storage.go View File

@@ -32,7 +32,7 @@ func (c *Client) StorageLoadPackage(req StorageLoadPackageReq) (*StorageLoadPack
return nil, err
}

codeResp, err := myhttp.ParseJSONResponse[response[StorageLoadPackageResp]](resp)
codeResp, err := ParseJSONResponse[response[StorageLoadPackageResp]](resp)
if err != nil {
return nil, err
}
@@ -109,7 +109,7 @@ func (c *Client) StorageGetInfo(req StorageGetInfoReq) (*StorageGetInfoResp, err
return nil, err
}

codeResp, err := myhttp.ParseJSONResponse[response[StorageGetInfoResp]](resp)
codeResp, err := ParseJSONResponse[response[StorageGetInfoResp]](resp)
if err != nil {
return nil, err
}


+ 33
- 33
sdks/storage/storage_test.go View File

@@ -23,24 +23,24 @@ func Test_PackageGet(t *testing.T) {
}

pkgName := uuid.NewString()
createResp, err := cli.Package().Create(PackageCreateReq{
createResp, err := cli.Package().Create(PackageCreate{
UserID: 1,
BucketID: 1,
Name: pkgName,
})
So(err, ShouldBeNil)

_, err = cli.Object().Upload(ObjectUploadReq{
_, err = cli.Object().Upload(ObjectUpload{
ObjectUploadInfo: ObjectUploadInfo{
UserID: 1,
PackageID: createResp.PackageID,
PackageID: createResp.Package.PackageID,
},
Files: iterator.Array(
&IterObjectUpload{
Path: "test",
&UploadingObject{
Path: "abc/test",
File: io.NopCloser(bytes.NewBuffer(fileData)),
},
&IterObjectUpload{
&UploadingObject{
Path: "test2",
File: io.NopCloser(bytes.NewBuffer(fileData)),
},
@@ -50,16 +50,16 @@ func Test_PackageGet(t *testing.T) {

getResp, err := cli.Package().Get(PackageGetReq{
UserID: 1,
PackageID: createResp.PackageID,
PackageID: createResp.Package.PackageID,
})
So(err, ShouldBeNil)

So(getResp.PackageID, ShouldEqual, createResp.PackageID)
So(getResp.PackageID, ShouldEqual, createResp.Package.PackageID)
So(getResp.Package.Name, ShouldEqual, pkgName)

err = cli.Package().Delete(PackageDeleteReq{
err = cli.Package().Delete(PackageDelete{
UserID: 1,
PackageID: createResp.PackageID,
PackageID: createResp.Package.PackageID,
})
So(err, ShouldBeNil)
})
@@ -79,25 +79,25 @@ func Test_Object(t *testing.T) {
nodeAff := NodeID(2)

pkgName := uuid.NewString()
createResp, err := cli.Package().Create(PackageCreateReq{
createResp, err := cli.Package().Create(PackageCreate{
UserID: 1,
BucketID: 1,
Name: pkgName,
})
So(err, ShouldBeNil)

_, err = cli.Object().Upload(ObjectUploadReq{
_, err = cli.Object().Upload(ObjectUpload{
ObjectUploadInfo: ObjectUploadInfo{
UserID: 1,
PackageID: createResp.PackageID,
PackageID: createResp.Package.PackageID,
NodeAffinity: &nodeAff,
},
Files: iterator.Array(
&IterObjectUpload{
&UploadingObject{
Path: "test",
File: io.NopCloser(bytes.NewBuffer(fileData)),
},
&IterObjectUpload{
&UploadingObject{
Path: "test2",
File: io.NopCloser(bytes.NewBuffer(fileData)),
},
@@ -116,9 +116,9 @@ func Test_Object(t *testing.T) {
// So(downFileData, ShouldResemble, fileData)
// downFs.Close()

err = cli.Package().Delete(PackageDeleteReq{
err = cli.Package().Delete(PackageDelete{
UserID: 1,
PackageID: createResp.PackageID,
PackageID: createResp.Package.PackageID,
})
So(err, ShouldBeNil)
})
@@ -136,24 +136,24 @@ func Test_Storage(t *testing.T) {
}

pkgName := uuid.NewString()
createResp, err := cli.Package().Create(PackageCreateReq{
createResp, err := cli.Package().Create(PackageCreate{
UserID: 1,
BucketID: 1,
Name: pkgName,
})
So(err, ShouldBeNil)

_, err = cli.Object().Upload(ObjectUploadReq{
_, err = cli.Object().Upload(ObjectUpload{
ObjectUploadInfo: ObjectUploadInfo{
UserID: 1,
PackageID: createResp.PackageID,
PackageID: createResp.Package.PackageID,
},
Files: iterator.Array(
&IterObjectUpload{
&UploadingObject{
Path: "test",
File: io.NopCloser(bytes.NewBuffer(fileData)),
},
&IterObjectUpload{
&UploadingObject{
Path: "test2",
File: io.NopCloser(bytes.NewBuffer(fileData)),
},
@@ -163,14 +163,14 @@ func Test_Storage(t *testing.T) {

_, err = cli.StorageLoadPackage(StorageLoadPackageReq{
UserID: 1,
PackageID: createResp.PackageID,
PackageID: createResp.Package.PackageID,
StorageID: 1,
})
So(err, ShouldBeNil)

err = cli.Package().Delete(PackageDeleteReq{
err = cli.Package().Delete(PackageDelete{
UserID: 1,
PackageID: createResp.PackageID,
PackageID: createResp.Package.PackageID,
})
So(err, ShouldBeNil)
})
@@ -188,24 +188,24 @@ func Test_Cache(t *testing.T) {
}

pkgName := uuid.NewString()
createResp, err := cli.Package().Create(PackageCreateReq{
createResp, err := cli.Package().Create(PackageCreate{
UserID: 1,
BucketID: 1,
Name: pkgName,
})
So(err, ShouldBeNil)

_, err = cli.Object().Upload(ObjectUploadReq{
_, err = cli.Object().Upload(ObjectUpload{
ObjectUploadInfo: ObjectUploadInfo{
UserID: 1,
PackageID: createResp.PackageID,
PackageID: createResp.Package.PackageID,
},
Files: iterator.Array(
&IterObjectUpload{
&UploadingObject{
Path: "test.txt",
File: io.NopCloser(bytes.NewBuffer(fileData)),
},
&IterObjectUpload{
&UploadingObject{
Path: "test2.txt",
File: io.NopCloser(bytes.NewBuffer(fileData)),
},
@@ -215,14 +215,14 @@ func Test_Cache(t *testing.T) {

_, err = cli.CacheMovePackage(CacheMovePackageReq{
UserID: 1,
PackageID: createResp.PackageID,
PackageID: createResp.Package.PackageID,
NodeID: 1,
})
So(err, ShouldBeNil)

err = cli.Package().Delete(PackageDeleteReq{
err = cli.Package().Delete(PackageDelete{
UserID: 1,
PackageID: createResp.PackageID,
PackageID: createResp.Package.PackageID,
})
So(err, ShouldBeNil)
})


+ 32
- 1
sdks/storage/utils.go View File

@@ -1,7 +1,38 @@
package cdssdk

import "path/filepath"
import (
"fmt"
"io"
"net/http"
"path/filepath"
"strings"

myhttp "gitlink.org.cn/cloudream/common/utils/http"
"gitlink.org.cn/cloudream/common/utils/math"
"gitlink.org.cn/cloudream/common/utils/serder"
)

func MakeIPFSFilePath(fileHash string) string {
return filepath.Join("ipfs", fileHash)
}

func ParseJSONResponse[TBody any](resp *http.Response) (TBody, error) {
var ret TBody
contType := resp.Header.Get("Content-Type")
if strings.Contains(contType, myhttp.ContentTypeJSON) {
var err error
if ret, err = serder.JSONToObjectStreamEx[TBody](resp.Body); err != nil {
return ret, fmt.Errorf("parsing response: %w", err)
}

return ret, nil
}

cont, err := io.ReadAll(resp.Body)
if err != nil {
return ret, fmt.Errorf("unknow response content type: %s, status: %d", contType, resp.StatusCode)
}
strCont := string(cont)

return ret, fmt.Errorf("unknow response content type: %s, status: %d, body(prefix): %s", contType, resp.StatusCode, strCont[:math.Min(len(strCont), 200)])
}

+ 111
- 1
utils/http/http.go View File

@@ -4,6 +4,7 @@ import (
"bytes"
"fmt"
"io"
"mime"
"mime/multipart"
"net/http"
ul "net/url"
@@ -131,6 +132,115 @@ func ParseJSONResponse[TBody any](resp *http.Response) (TBody, error) {
return ret, fmt.Errorf("unknow response content type: %s, status: %d, body(prefix): %s", contType, resp.StatusCode, strCont[:math.Min(len(strCont), 200)])
}

type MultiPartFile struct {
FieldName string
FileName string
File io.ReadCloser
}

type multiPartFileIterator struct {
mr *multipart.Reader
firstFile *multipart.Part
}

func (m *multiPartFileIterator) MoveNext() (*MultiPartFile, error) {
if m.firstFile != nil {
f := m.firstFile
m.firstFile = nil

fileName, err := ul.PathUnescape(f.FileName())
if err != nil {
return nil, fmt.Errorf("unescape file name: %w", err)
}

return &MultiPartFile{
FieldName: f.FormName(),
FileName: fileName,
File: f,
}, nil
}

for {
part, err := m.mr.NextPart()
if err == io.EOF {
return nil, iterator.ErrNoMoreItem
}
if err != nil {
return nil, err
}

fileName, err := ul.PathUnescape(part.FileName())
if err != nil {
return nil, fmt.Errorf("unescape file name: %w", err)
}

if part.FileName() != "" {
return &MultiPartFile{
FieldName: part.FormName(),
FileName: fileName,
File: part,
}, nil
}
}
}

func (m *multiPartFileIterator) Close() {}

// 解析multipart/form-data响应,只支持form参数在头部的情况
func ParseMultiPartResponse(resp *http.Response) (map[string]string, iterator.Iterator[*MultiPartFile], error) {
mtype, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
if err != nil {
return nil, nil, fmt.Errorf("parse content type: %w", err)
}

if mtype != ContentTypeMultiPart {
return nil, nil, fmt.Errorf("unknow content type: %s", mtype)
}

boundary := params["boundary"]
if boundary == "" {
return nil, nil, fmt.Errorf("no boundary in content type: %s", mtype)
}

formValues := make(map[string]string)
rd := multipart.NewReader(resp.Body, boundary)

var firstFile *multipart.Part
for {
part, err := rd.NextPart()
if err == io.EOF {
return formValues, iterator.Empty[*MultiPartFile](), nil
}
if err != nil {
return nil, nil, err
}

formName := part.FormName()
fileName := part.FileName()

if formName == "" {
continue
}

if fileName != "" {
firstFile = part
break
}

data, err := io.ReadAll(part)
if err != nil {
return nil, nil, err
}

formValues[formName] = string(data)
}

return formValues, &multiPartFileIterator{
mr: rd,
firstFile: firstFile,
}, nil
}

type MultiPartRequestParam struct {
Header any
Query any
@@ -196,7 +306,7 @@ func PostMultiPart(url string, param MultiPartRequestParam) (*http.Response, err
err = func() error {
defer file.File.Close()

w, err := muWriter.CreateFormFile(file.FieldName, file.FileName)
w, err := muWriter.CreateFormFile(file.FieldName, ul.PathEscape(file.FileName))
if err != nil {
return fmt.Errorf("create form file failed, err: %w", err)
}


+ 12
- 0
utils/serder/serder.go View File

@@ -53,6 +53,18 @@ func JSONToObjectEx[T any](data []byte) (T, error) {
return ret, nil
}

// 将JSON字符串转为对象。支持TypeUnion。
func JSONToObjectStreamEx[T any](stream io.Reader) (T, error) {
var ret T
dec := defaultAPI.NewDecoder(stream)
err := dec.Decode(&ret)
if err != nil {
return ret, err
}

return ret, nil
}

// 将对象转为JSON字符串。如果需要支持解析TypeUnion类型,则使用"Ex"结尾的同名函数。
func ObjectToJSON(obj any) ([]byte, error) {
return json.Marshal(obj)


Loading…
Cancel
Save