diff --git a/api/storage/package.go b/api/storage/package.go index 82cf02e..c07f32d 100644 --- a/api/storage/package.go +++ b/api/storage/package.go @@ -8,6 +8,7 @@ import ( "gitlink.org.cn/cloudream/common/consts/errorcode" "gitlink.org.cn/cloudream/common/models" + "gitlink.org.cn/cloudream/common/pkgs/iterator" myhttp "gitlink.org.cn/cloudream/common/utils/http" "gitlink.org.cn/cloudream/common/utils/serder" ) @@ -17,12 +18,16 @@ type PackageUploadReq struct { BucketID int64 `json:"bucketID"` Name string `json:"name"` Redundancy models.TypedRedundancyInfo `json:"redundancy"` - Files []PackageUploadFile `json:"-"` + Files PackageUploadFileIterator `json:"-"` } -type PackageUploadFile struct { + +type IterPackageUploadFile struct { Path string - File io.Reader + File io.ReadCloser } + +type PackageUploadFileIterator = iterator.Iterator[*IterPackageUploadFile] + type PackageUploadResp struct { PackageID int64 `json:"packageID,string"` } @@ -38,18 +43,15 @@ func (c *Client) PackageUpload(req PackageUploadReq) (*PackageUploadResp, error) return nil, fmt.Errorf("package info to json: %w", err) } - var uploadFiles []myhttp.MultiPartRequestFile - for _, file := range req.Files { - uploadFiles = append(uploadFiles, myhttp.MultiPartRequestFile{ - FieldName: "files", - FileName: file.Path, - File: file.File, - }) - } - resp, err := myhttp.PostMultiPart(url, myhttp.MultiPartRequestParam{ - Form: map[string]string{"info": string(infoJSON)}, - Files: uploadFiles, + Form: map[string]string{"info": string(infoJSON)}, + Files: iterator.Map(req.Files, func(src *IterPackageUploadFile) (*myhttp.IterMultiPartFile, error) { + return &myhttp.IterMultiPartFile{ + FieldName: "files", + FileName: src.Path, + File: src.File, + }, nil + }), }) if err != nil { return nil, err diff --git a/api/storage/storage_test.go b/api/storage/storage_test.go index 6e7a8b2..53c352f 100644 --- a/api/storage/storage_test.go +++ b/api/storage/storage_test.go @@ -3,11 +3,13 @@ package storage import ( "bytes" "fmt" + "io" "testing" "github.com/google/uuid" . "github.com/smartystreets/goconvey/convey" "gitlink.org.cn/cloudream/common/models" + "gitlink.org.cn/cloudream/common/pkgs/iterator" ) func Test_Object(t *testing.T) { @@ -29,16 +31,16 @@ func Test_Object(t *testing.T) { Type: models.RedundancyRep, Info: models.NewRepRedundancyInfo(1), }, - Files: []PackageUploadFile{ - { + Files: iterator.Array( + &IterPackageUploadFile{ Path: "test", - File: bytes.NewBuffer(fileData), + File: io.NopCloser(bytes.NewBuffer(fileData)), }, - { + &IterPackageUploadFile{ Path: "test2", - File: bytes.NewBuffer(fileData), + File: io.NopCloser(bytes.NewBuffer(fileData)), }, - }, + ), }) So(err, ShouldBeNil) @@ -80,16 +82,16 @@ func Test_Storage(t *testing.T) { Type: models.RedundancyRep, Info: models.NewRepRedundancyInfo(1), }, - Files: []PackageUploadFile{ - { + Files: iterator.Array( + &IterPackageUploadFile{ Path: "test", - File: bytes.NewBuffer(fileData), + File: io.NopCloser(bytes.NewBuffer(fileData)), }, - { + &IterPackageUploadFile{ Path: "test2", - File: bytes.NewBuffer(fileData), + File: io.NopCloser(bytes.NewBuffer(fileData)), }, - }, + ), }) So(err, ShouldBeNil) @@ -127,16 +129,16 @@ func Test_Cache(t *testing.T) { Type: models.RedundancyRep, Info: models.NewRepRedundancyInfo(1), }, - Files: []PackageUploadFile{ - { - Path: "test", - File: bytes.NewBuffer(fileData), + Files: iterator.Array( + &IterPackageUploadFile{ + Path: "test.txt", + File: io.NopCloser(bytes.NewBuffer(fileData)), }, - { - Path: "test3", - File: bytes.NewBuffer(fileData), + &IterPackageUploadFile{ + Path: "test2.txt", + File: io.NopCloser(bytes.NewBuffer(fileData)), }, - }, + ), }) So(err, ShouldBeNil) diff --git a/pkgs/iterator/array.go b/pkgs/iterator/array.go new file mode 100644 index 0000000..65ff4e2 --- /dev/null +++ b/pkgs/iterator/array.go @@ -0,0 +1,27 @@ +package iterator + +type ArrayIterator[T any] struct { + arr []T + index int +} + +func (i *ArrayIterator[T]) MoveNext() (T, error) { + if i.index >= len(i.arr) { + var ret T + return ret, ErrNoMoreItem + } + + item := i.arr[i.index] + i.index++ + + return item, nil +} + +func (i *ArrayIterator[T]) Close() { +} + +func Array[T any](eles ...T) *ArrayIterator[T] { + return &ArrayIterator[T]{ + arr: eles, + } +} diff --git a/pkgs/iterator/iterator.go b/pkgs/iterator/iterator.go new file mode 100644 index 0000000..3ced512 --- /dev/null +++ b/pkgs/iterator/iterator.go @@ -0,0 +1,12 @@ +package iterator + +import ( + "errors" +) + +var ErrNoMoreItem = errors.New("no more item") + +type Iterator[T any] interface { + MoveNext() (T, error) + Close() +} diff --git a/pkgs/iterator/mapper.go b/pkgs/iterator/mapper.go new file mode 100644 index 0000000..6b43851 --- /dev/null +++ b/pkgs/iterator/mapper.go @@ -0,0 +1,27 @@ +package iterator + +type Mapper[TSrc any, TDst any] struct { + srcIter Iterator[TSrc] + mapper func(TSrc) (TDst, error) +} + +func (i *Mapper[TSrc, TDst]) MoveNext() (TDst, error) { + src, err := i.srcIter.MoveNext() + if err != nil { + var ret TDst + return ret, err + } + + return i.mapper(src) +} + +func (i *Mapper[TSrc, TDst]) Close() { + i.srcIter.Close() +} + +func Map[TSrc any, TDst any](srcIter Iterator[TSrc], mapper func(src TSrc) (TDst, error)) *Mapper[TSrc, TDst] { + return &Mapper[TSrc, TDst]{ + srcIter: srcIter, + mapper: mapper, + } +} diff --git a/utils/http/http.go b/utils/http/http.go index db73873..ff3dc77 100644 --- a/utils/http/http.go +++ b/utils/http/http.go @@ -8,6 +8,7 @@ import ( ul "net/url" "strings" + "gitlink.org.cn/cloudream/common/pkgs/iterator" "gitlink.org.cn/cloudream/common/utils/serder" ) @@ -112,12 +113,14 @@ type MultiPartRequestParam struct { Header any Query any Form any - Files []MultiPartRequestFile + Files MultiPartFileIterator } -type MultiPartRequestFile struct { + +type MultiPartFileIterator = iterator.Iterator[*IterMultiPartFile] +type IterMultiPartFile struct { FieldName string // 这个文件所属的form字段 FileName string // 文件名 - File io.Reader + File io.ReadCloser } func PostMultiPart(url string, param MultiPartRequestParam) (*http.Response, error) { @@ -159,17 +162,34 @@ func PostMultiPart(url string, param MultiPartRequestParam) (*http.Response, err } } - for _, file := range param.Files { - w, err := muWriter.CreateFormFile(file.FieldName, file.FileName) + for { + file, err := param.Files.MoveNext() + if err == iterator.ErrNoMoreItem { + break + } if err != nil { - return fmt.Errorf("create form file failed, err: %w", err) + return fmt.Errorf("opening file: %w", err) } - _, err = io.Copy(w, file.File) + err = func() error { + defer file.File.Close() + + w, err := muWriter.CreateFormFile(file.FieldName, file.FileName) + if err != nil { + return fmt.Errorf("create form file failed, err: %w", err) + } + + _, err = io.Copy(w, file.File) + if err != nil { + return err + } + return nil + }() if err != nil { return err } } + return nil }() }() diff --git a/utils/io/io.go b/utils/io/io.go index 577947a..e5ee9c7 100644 --- a/utils/io/io.go +++ b/utils/io/io.go @@ -81,3 +81,34 @@ func WithCloser(reader io.Reader, closer func(reader io.Reader) error) io.ReadCl closer: closer, } } + +type LazyReadCloser struct { + open func() (io.ReadCloser, error) + stream io.ReadCloser +} + +func (r *LazyReadCloser) Read(buf []byte) (n int, err error) { + if r.stream == nil { + var err error + r.stream, err = r.open() + if err != nil { + return 0, err + } + } + + return r.stream.Read(buf) +} + +func (r *LazyReadCloser) Close() error { + if r.stream == nil { + return nil + } + + return r.stream.Close() +} + +func Lazy(open func() (io.ReadCloser, error)) *LazyReadCloser { + return &LazyReadCloser{ + open: open, + } +} diff --git a/utils/os/dir_iterator.go b/utils/os/dir_iterator.go new file mode 100644 index 0000000..53631f4 --- /dev/null +++ b/utils/os/dir_iterator.go @@ -0,0 +1,66 @@ +package os + +import ( + "os" + "path/filepath" + + "gitlink.org.cn/cloudream/common/pkgs/iterator" +) + +type DirIterator struct { + rootPath string + walked bool + walkedInfos []FileInfo + index int +} + +type FileInfo struct { + Path string + Info os.FileInfo +} + +func (i *DirIterator) MoveNext() (*FileInfo, error) { + if !i.walked { + i.walked = true + // TODO 可以考虑优化成MoveNext一次就产生一个FileInfo的形式 + err := filepath.WalkDir(i.rootPath, func(fname string, fi os.DirEntry, err error) error { + if err != nil { + return nil + } + + if !fi.IsDir() { + info, err := fi.Info() + if err != nil { + return err + } + + i.walkedInfos = append(i.walkedInfos, FileInfo{ + Path: fname, + Info: info, + }) + } + return nil + }) + if err != nil { + return nil, err + } + } + + if i.index >= len(i.walkedInfos) { + return nil, iterator.ErrNoMoreItem + } + + item := i.walkedInfos[i.index] + i.index++ + return &item, nil +} + +func (i *DirIterator) Close() { + +} + +func WalkDir(rootPath string) *DirIterator { + return &DirIterator{ + rootPath: rootPath, + } +}