package downloader import ( "fmt" "io" lru "github.com/hashicorp/golang-lru/v2" "gitlink.org.cn/cloudream/common/pkgs/iterator" "gitlink.org.cn/cloudream/storage2/client/internal/db" "gitlink.org.cn/cloudream/storage2/client/types" stgglb "gitlink.org.cn/cloudream/storage2/common/globals" "gitlink.org.cn/cloudream/storage2/common/pkgs/connectivity" "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader/strategy" coormq "gitlink.org.cn/cloudream/storage2/common/pkgs/mq/coordinator" "gitlink.org.cn/cloudream/storage2/common/pkgs/storage/agtpool" ) const ( DefaultMaxStripCacheCount = 128 ) type DownloadIterator = iterator.Iterator[*Downloading] type DownloadReqeust struct { ObjectID types.ObjectID Offset int64 Length int64 } type downloadReqeust2 struct { Detail *types.ObjectDetail Raw DownloadReqeust } type Downloading struct { Object *types.Object File io.ReadCloser // 文件流,如果文件不存在,那么为nil Request DownloadReqeust } type Downloader struct { strips *StripCache cfg Config conn *connectivity.Collector stgAgts *agtpool.AgentPool selector *strategy.Selector db *db.DB } func NewDownloader(cfg Config, conn *connectivity.Collector, stgAgts *agtpool.AgentPool, sel *strategy.Selector, db *db.DB) Downloader { if cfg.MaxStripCacheCount == 0 { cfg.MaxStripCacheCount = DefaultMaxStripCacheCount } ch, _ := lru.New[ECStripKey, ObjectECStrip](cfg.MaxStripCacheCount) return Downloader{ strips: ch, cfg: cfg, conn: conn, stgAgts: stgAgts, selector: sel, db: db, } } func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator { coorCli, err := stgglb.CoordinatorMQPool.Acquire() if err != nil { return iterator.FuseError[*Downloading](fmt.Errorf("new coordinator client: %w", err)) } defer stgglb.CoordinatorMQPool.Release(coorCli) objIDs := make([]types.ObjectID, len(reqs)) for i, req := range reqs { objIDs[i] = req.ObjectID } if len(objIDs) == 0 { return iterator.Empty[*Downloading]() } // objDetails, err := coorCli.GetObjectDetails(coormq.ReqGetObjectDetails(objIDs)) // if err != nil { // return iterator.FuseError[*Downloading](fmt.Errorf("request to coordinator: %w", err)) // } objDetails, err := d.db.GetObjectDetails(objIDs) if err != nil { return iterator.FuseError[*Downloading](fmt.Errorf("request to db: %w", err)) } req2s := make([]downloadReqeust2, len(reqs)) for i, req := range reqs { req2s[i] = downloadReqeust2{ Detail: objDetails.Objects[i], Raw: req, } } return NewDownloadObjectIterator(d, req2s) } func (d *Downloader) DownloadObjectByDetail(detail types.ObjectDetail, off int64, length int64) (*Downloading, error) { req2s := []downloadReqeust2{{ Detail: &detail, Raw: DownloadReqeust{ ObjectID: detail.Object.ObjectID, Offset: off, Length: length, }, }} iter := NewDownloadObjectIterator(d, req2s) return iter.MoveNext() } func (d *Downloader) DownloadPackage(pkgID types.PackageID) DownloadIterator { coorCli, err := stgglb.CoordinatorMQPool.Acquire() if err != nil { return iterator.FuseError[*Downloading](fmt.Errorf("new coordinator client: %w", err)) } defer stgglb.CoordinatorMQPool.Release(coorCli) pkgDetail, err := coorCli.GetPackageObjectDetails(coormq.ReqGetPackageObjectDetails(pkgID)) if err != nil { return iterator.FuseError[*Downloading](fmt.Errorf("request to coordinator: %w", err)) } req2s := make([]downloadReqeust2, len(pkgDetail.Objects)) for i, objDetail := range pkgDetail.Objects { dt := objDetail req2s[i] = downloadReqeust2{ Detail: &dt, Raw: DownloadReqeust{ ObjectID: objDetail.Object.ObjectID, Offset: 0, Length: objDetail.Object.Size, }, } } return NewDownloadObjectIterator(d, req2s) } type ObjectECStrip struct { Data []byte ObjectFileHash types.FileHash // 添加这条缓存时,Object的FileHash } type ECStripKey struct { ObjectID types.ObjectID StripIndex int64 } type StripCache = lru.Cache[ECStripKey, ObjectECStrip]