增加iterator

api/storage/package.go

@@ -8,6 +8,7 @@ import (
 
 	"gitlink.org.cn/cloudream/common/consts/errorcode"
 	"gitlink.org.cn/cloudream/common/models"
+	"gitlink.org.cn/cloudream/common/pkgs/iterator"
 	myhttp "gitlink.org.cn/cloudream/common/utils/http"
 	"gitlink.org.cn/cloudream/common/utils/serder"
 )
@@ -17,12 +18,16 @@ type PackageUploadReq struct {
 	BucketID   int64                      `json:"bucketID"`
 	Name       string                     `json:"name"`
 	Redundancy models.TypedRedundancyInfo `json:"redundancy"`
-	Files      []PackageUploadFile        `json:"-"`
+	Files      PackageUploadFileIterator  `json:"-"`
 }
-type PackageUploadFile struct {
+
+type IterPackageUploadFile struct {
 	Path string
-	File io.Reader
+	File io.ReadCloser
 }
+
+type PackageUploadFileIterator = iterator.Iterator[*IterPackageUploadFile]
+
 type PackageUploadResp struct {
 	PackageID int64 `json:"packageID,string"`
 }
@@ -38,18 +43,15 @@ func (c *Client) PackageUpload(req PackageUploadReq) (*PackageUploadResp, error)
 		return nil, fmt.Errorf("package info to json: %w", err)
 	}
 
-	var uploadFiles []myhttp.MultiPartRequestFile
-	for _, file := range req.Files {
-		uploadFiles = append(uploadFiles, myhttp.MultiPartRequestFile{
-			FieldName: "files",
-			FileName:  file.Path,
-			File:      file.File,
-		})
-	}
-
 	resp, err := myhttp.PostMultiPart(url, myhttp.MultiPartRequestParam{
-		Form:  map[string]string{"info": string(infoJSON)},
-		Files: uploadFiles,
+		Form: map[string]string{"info": string(infoJSON)},
+		Files: iterator.Map(req.Files, func(src *IterPackageUploadFile) (*myhttp.IterMultiPartFile, error) {
+			return &myhttp.IterMultiPartFile{
+				FieldName: "files",
+				FileName:  src.Path,
+				File:      src.File,
+			}, nil
+		}),
 	})
 	if err != nil {
 		return nil, err

api/storage/storage_test.go

@@ -3,11 +3,13 @@ package storage
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"testing"
 
 	"github.com/google/uuid"
 	. "github.com/smartystreets/goconvey/convey"
 	"gitlink.org.cn/cloudream/common/models"
+	"gitlink.org.cn/cloudream/common/pkgs/iterator"
 )
 
 func Test_Object(t *testing.T) {
@@ -29,16 +31,16 @@ func Test_Object(t *testing.T) {
 				Type: models.RedundancyRep,
 				Info: models.NewRepRedundancyInfo(1),
 			},
-			Files: []PackageUploadFile{
-				{
+			Files: iterator.Array(
+				&IterPackageUploadFile{
 					Path: "test",
-					File: bytes.NewBuffer(fileData),
+					File: io.NopCloser(bytes.NewBuffer(fileData)),
 				},
-				{
+				&IterPackageUploadFile{
 					Path: "test2",
-					File: bytes.NewBuffer(fileData),
+					File: io.NopCloser(bytes.NewBuffer(fileData)),
 				},
-			},
+			),
 		})
 		So(err, ShouldBeNil)
 
@@ -80,16 +82,16 @@ func Test_Storage(t *testing.T) {
 				Type: models.RedundancyRep,
 				Info: models.NewRepRedundancyInfo(1),
 			},
-			Files: []PackageUploadFile{
-				{
+			Files: iterator.Array(
+				&IterPackageUploadFile{
 					Path: "test",
-					File: bytes.NewBuffer(fileData),
+					File: io.NopCloser(bytes.NewBuffer(fileData)),
 				},
-				{
+				&IterPackageUploadFile{
 					Path: "test2",
-					File: bytes.NewBuffer(fileData),
+					File: io.NopCloser(bytes.NewBuffer(fileData)),
 				},
-			},
+			),
 		})
 		So(err, ShouldBeNil)
 
@@ -127,16 +129,16 @@ func Test_Cache(t *testing.T) {
 				Type: models.RedundancyRep,
 				Info: models.NewRepRedundancyInfo(1),
 			},
-			Files: []PackageUploadFile{
-				{
-					Path: "test",
-					File: bytes.NewBuffer(fileData),
+			Files: iterator.Array(
+				&IterPackageUploadFile{
+					Path: "test.txt",
+					File: io.NopCloser(bytes.NewBuffer(fileData)),
 				},
-				{
-					Path: "test3",
-					File: bytes.NewBuffer(fileData),
+				&IterPackageUploadFile{
+					Path: "test2.txt",
+					File: io.NopCloser(bytes.NewBuffer(fileData)),
 				},
-			},
+			),
 		})
 		So(err, ShouldBeNil)
 

pkgs/iterator/array.go

@@ -0,0 +1,27 @@
+package iterator
+
+type ArrayIterator[T any] struct {
+	arr   []T
+	index int
+}
+
+func (i *ArrayIterator[T]) MoveNext() (T, error) {
+	if i.index >= len(i.arr) {
+		var ret T
+		return ret, ErrNoMoreItem
+	}
+
+	item := i.arr[i.index]
+	i.index++
+
+	return item, nil
+}
+
+func (i *ArrayIterator[T]) Close() {
+}
+
+func Array[T any](eles ...T) *ArrayIterator[T] {
+	return &ArrayIterator[T]{
+		arr: eles,
+	}
+}

pkgs/iterator/iterator.go

@@ -0,0 +1,12 @@
+package iterator
+
+import (
+	"errors"
+)
+
+var ErrNoMoreItem = errors.New("no more item")
+
+type Iterator[T any] interface {
+	MoveNext() (T, error)
+	Close()
+}

pkgs/iterator/mapper.go

@@ -0,0 +1,27 @@
+package iterator
+
+type Mapper[TSrc any, TDst any] struct {
+	srcIter Iterator[TSrc]
+	mapper  func(TSrc) (TDst, error)
+}
+
+func (i *Mapper[TSrc, TDst]) MoveNext() (TDst, error) {
+	src, err := i.srcIter.MoveNext()
+	if err != nil {
+		var ret TDst
+		return ret, err
+	}
+
+	return i.mapper(src)
+}
+
+func (i *Mapper[TSrc, TDst]) Close() {
+	i.srcIter.Close()
+}
+
+func Map[TSrc any, TDst any](srcIter Iterator[TSrc], mapper func(src TSrc) (TDst, error)) *Mapper[TSrc, TDst] {
+	return &Mapper[TSrc, TDst]{
+		srcIter: srcIter,
+		mapper:  mapper,
+	}
+}

utils/http/http.go

@@ -8,6 +8,7 @@ import (
 	ul "net/url"
 	"strings"
 
+	"gitlink.org.cn/cloudream/common/pkgs/iterator"
 	"gitlink.org.cn/cloudream/common/utils/serder"
 )
 
@@ -112,12 +113,14 @@ type MultiPartRequestParam struct {
 	Header any
 	Query  any
 	Form   any
-	Files  []MultiPartRequestFile
+	Files  MultiPartFileIterator
 }
-type MultiPartRequestFile struct {
+
+type MultiPartFileIterator = iterator.Iterator[*IterMultiPartFile]
+type IterMultiPartFile struct {
 	FieldName string // 这个文件所属的form字段
 	FileName  string // 文件名
-	File      io.Reader
+	File      io.ReadCloser
 }
 
 func PostMultiPart(url string, param MultiPartRequestParam) (*http.Response, error) {
@@ -159,17 +162,34 @@ func PostMultiPart(url string, param MultiPartRequestParam) (*http.Response, err
 				}
 			}
 
-			for _, file := range param.Files {
-				w, err := muWriter.CreateFormFile(file.FieldName, file.FileName)
+			for {
+				file, err := param.Files.MoveNext()
+				if err == iterator.ErrNoMoreItem {
+					break
+				}
 				if err != nil {
-					return fmt.Errorf("create form file failed, err: %w", err)
+					return fmt.Errorf("opening file: %w", err)
 				}
 
-				_, err = io.Copy(w, file.File)
+				err = func() error {
+					defer file.File.Close()
+
+					w, err := muWriter.CreateFormFile(file.FieldName, file.FileName)
+					if err != nil {
+						return fmt.Errorf("create form file failed, err: %w", err)
+					}
+
+					_, err = io.Copy(w, file.File)
+					if err != nil {
+						return err
+					}
+					return nil
+				}()
 				if err != nil {
 					return err
 				}
 			}
+
 			return nil
 		}()
 	}()

utils/io/io.go

@@ -81,3 +81,34 @@ func WithCloser(reader io.Reader, closer func(reader io.Reader) error) io.ReadCl
 		closer: closer,
 	}
 }
+
+type LazyReadCloser struct {
+	open   func() (io.ReadCloser, error)
+	stream io.ReadCloser
+}
+
+func (r *LazyReadCloser) Read(buf []byte) (n int, err error) {
+	if r.stream == nil {
+		var err error
+		r.stream, err = r.open()
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	return r.stream.Read(buf)
+}
+
+func (r *LazyReadCloser) Close() error {
+	if r.stream == nil {
+		return nil
+	}
+
+	return r.stream.Close()
+}
+
+func Lazy(open func() (io.ReadCloser, error)) *LazyReadCloser {
+	return &LazyReadCloser{
+		open: open,
+	}
+}

utils/os/dir_iterator.go

@@ -0,0 +1,66 @@
+package os
+
+import (
+	"os"
+	"path/filepath"
+
+	"gitlink.org.cn/cloudream/common/pkgs/iterator"
+)
+
+type DirIterator struct {
+	rootPath    string
+	walked      bool
+	walkedInfos []FileInfo
+	index       int
+}
+
+type FileInfo struct {
+	Path string
+	Info os.FileInfo
+}
+
+func (i *DirIterator) MoveNext() (*FileInfo, error) {
+	if !i.walked {
+		i.walked = true
+		// TODO 可以考虑优化成MoveNext一次就产生一个FileInfo的形式
+		err := filepath.WalkDir(i.rootPath, func(fname string, fi os.DirEntry, err error) error {
+			if err != nil {
+				return nil
+			}
+
+			if !fi.IsDir() {
+				info, err := fi.Info()
+				if err != nil {
+					return err
+				}
+
+				i.walkedInfos = append(i.walkedInfos, FileInfo{
+					Path: fname,
+					Info: info,
+				})
+			}
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if i.index >= len(i.walkedInfos) {
+		return nil, iterator.ErrNoMoreItem
+	}
+
+	item := i.walkedInfos[i.index]
+	i.index++
+	return &item, nil
+}
+
+func (i *DirIterator) Close() {
+
+}
+
+func WalkDir(rootPath string) *DirIterator {
+	return &DirIterator{
+		rootPath: rootPath,
+	}
+}