ShardStore只负责管理元数据，读写功能统一交给BaseStore

client/internal/services/user_space.go

@@ -160,7 +160,7 @@ func (svc *UserSpaceService) SpaceToSpace(srcSpaceID clitypes.UserSpaceID, srcPa
 	srcPathComps := clitypes.SplitObjectPath(srcPath)
 	srcDirCompLen := len(srcPathComps) - 1
 
-	entryTree := trie.NewTrie[*types.BaseStoreEntry]()
+	entryTree := trie.NewTrie[*types.ListEntry]()
 	for _, e := range listAllResp.Entries {
 		pa, ok := strings.CutSuffix(e.Path, clitypes.ObjectPathSeparator)
 		comps := clitypes.SplitObjectPath(pa)
@@ -171,7 +171,7 @@ func (svc *UserSpaceService) SpaceToSpace(srcSpaceID clitypes.UserSpaceID, srcPa
 		e2.IsDir = e2.IsDir || ok
 	}
 
-	entryTree.Iterate(func(path []string, node *trie.Node[*types.BaseStoreEntry], isWordNode bool) trie.VisitCtrl {
+	entryTree.Iterate(func(path []string, node *trie.Node[*types.ListEntry], isWordNode bool) trie.VisitCtrl {
 		if node.Value == nil {
 			return trie.VisitContinue
 		}
@@ -190,7 +190,7 @@ func (svc *UserSpaceService) SpaceToSpace(srcSpaceID clitypes.UserSpaceID, srcPa
 
 	var filePathes []string
 	var dirPathes []string
-	entryTree.Iterate(func(path []string, node *trie.Node[*types.BaseStoreEntry], isWordNode bool) trie.VisitCtrl {
+	entryTree.Iterate(func(path []string, node *trie.Node[*types.ListEntry], isWordNode bool) trie.VisitCtrl {
 		if node.Value == nil {
 			return trie.VisitContinue
 		}

client/internal/uploader/user_space_upload.go

@@ -122,7 +122,7 @@ func (u *Uploader) UserSpaceUpload(userSpaceID clitypes.UserSpaceID, rootPath st
 	return &pkg, nil
 }
 
-func (u *Uploader) uploadFromBaseStore(srcSpace *clitypes.UserSpaceDetail, targetSpace *clitypes.UserSpaceDetail, entries []types.BaseStoreEntry, rootPath string) ([]db.AddObjectEntry, error) {
+func (u *Uploader) uploadFromBaseStore(srcSpace *clitypes.UserSpaceDetail, targetSpace *clitypes.UserSpaceDetail, entries []types.ListEntry, rootPath string) ([]db.AddObjectEntry, error) {
 	ft := ioswitch2.FromTo{}
 
 	for _, e := range entries {

common/pkgs/ioswitch2/fromto.go

@@ -108,21 +108,21 @@ func (f *FromDriver) GetStreamIndex() StreamIndex {
 	return f.StreamIndex
 }
 
-type FromShardstore struct {
+type FromShardStore struct {
 	FileHash    clitypes.FileHash
 	UserSpace   clitypes.UserSpaceDetail
 	StreamIndex StreamIndex
 }
 
-func NewFromShardstore(fileHash clitypes.FileHash, space clitypes.UserSpaceDetail, strIdx StreamIndex) *FromShardstore {
-	return &FromShardstore{
+func NewFromShardstore(fileHash clitypes.FileHash, space clitypes.UserSpaceDetail, strIdx StreamIndex) *FromShardStore {
+	return &FromShardStore{
 		FileHash:    fileHash,
 		UserSpace:   space,
 		StreamIndex: strIdx,
 	}
 }
 
-func (f *FromShardstore) GetStreamIndex() StreamIndex {
+func (f *FromShardStore) GetStreamIndex() StreamIndex {
 	return f.StreamIndex
 }
 
@@ -176,26 +176,26 @@ func (t *ToDriver) GetRange() math2.Range {
 }
 
 type ToShardStore struct {
-	Space            clitypes.UserSpaceDetail
-	StreamIndex      StreamIndex
-	Range            math2.Range
-	FileHashStoreKey string
+	Space          clitypes.UserSpaceDetail
+	StreamIndex    StreamIndex
+	Range          math2.Range
+	ResultStoreKey string
 }
 
-func NewToShardStore(space clitypes.UserSpaceDetail, strIdx StreamIndex, fileHashStoreKey string) *ToShardStore {
+func NewToShardStore(space clitypes.UserSpaceDetail, strIdx StreamIndex, retStoreKey string) *ToShardStore {
 	return &ToShardStore{
-		Space:            space,
-		StreamIndex:      strIdx,
-		FileHashStoreKey: fileHashStoreKey,
+		Space:          space,
+		StreamIndex:    strIdx,
+		ResultStoreKey: retStoreKey,
 	}
 }
 
-func NewToShardStoreWithRange(space clitypes.UserSpaceDetail, streamIndex StreamIndex, fileHashStoreKey string, rng math2.Range) *ToShardStore {
+func NewToShardStoreWithRange(space clitypes.UserSpaceDetail, streamIndex StreamIndex, retStoreKey string, rng math2.Range) *ToShardStore {
 	return &ToShardStore{
-		Space:            space,
-		StreamIndex:      streamIndex,
-		FileHashStoreKey: fileHashStoreKey,
-		Range:            rng,
+		Space:          space,
+		StreamIndex:    streamIndex,
+		ResultStoreKey: retStoreKey,
+		Range:          rng,
 	}
 }
 

common/pkgs/ioswitch2/ops2/base_store.go

@@ -12,27 +12,19 @@ import (
 	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
+	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
 )
 
 func init() {
 	exec.UseOp[*BaseWrite]()
 	exec.UseOp[*BaseRead]()
-	exec.UseVarValue[*FileInfoValue]()
-}
-
-type FileInfoValue struct {
-	Hash clitypes.FileHash `json:"hash"`
-	Size int64             `json:"size"`
-}
-
-func (v *FileInfoValue) Clone() exec.VarValue {
-	return &FileInfoValue{Hash: v.Hash, Size: v.Size}
 }
 
 type BaseRead struct {
 	Output    exec.VarID
 	UserSpace clitypes.UserSpaceDetail
 	Path      string
+	Option    types.OpenOption
 }
 
 func (o *BaseRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
@@ -53,7 +45,7 @@ func (o *BaseRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
 		return fmt.Errorf("getting base store of storage %v: %w", o.UserSpace, err)
 	}
 
-	stream, err := store.Read(o.Path)
+	stream, err := store.Read(o.Path, o.Option)
 	if err != nil {
 		return fmt.Errorf("reading object %v: %w", o.Path, err)
 	}
@@ -73,6 +65,56 @@ func (o *BaseRead) String() string {
 	return fmt.Sprintf("PublicRead %v:%v -> %v", o.UserSpace, o.Path, o.Output)
 }
 
+type BaseReadDyn struct {
+	UserSpace clitypes.UserSpaceDetail
+	Output    exec.VarID
+	FileInfo  exec.VarID
+	Option    types.OpenOption
+}
+
+func (o *BaseReadDyn) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
+	logger.
+		WithField("Output", o.Output).
+		WithField("UserSpace", o.UserSpace).
+		WithField("Path", o.FileInfo).
+		Debug("base read dynamic")
+	defer logger.Debug("base read dynamic end")
+
+	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
+	if err != nil {
+		return fmt.Errorf("getting storage pool: %w", err)
+	}
+
+	info, err := exec.BindVar[*FileInfoValue](e, ctx.Context, o.FileInfo)
+	if err != nil {
+		return err
+	}
+
+	store, err := stgPool.GetBaseStore(&o.UserSpace)
+	if err != nil {
+		return fmt.Errorf("getting base store of storage %v: %w", o.UserSpace, err)
+	}
+
+	stream, err := store.Read(info.Path, o.Option)
+	if err != nil {
+		return fmt.Errorf("reading object %v: %w", o.FileInfo, err)
+	}
+
+	fut := future.NewSetVoid()
+	output := &exec.StreamValue{
+		Stream: io2.AfterReadClosed(stream, func(closer io.ReadCloser) {
+			fut.SetVoid()
+		}),
+	}
+
+	e.PutVar(o.Output, output)
+	return fut.Wait(ctx.Context)
+}
+
+func (o *BaseReadDyn) String() string {
+	return fmt.Sprintf("BaseReadDyn %v:%v -> %v", o.UserSpace, o.FileInfo, o.Output)
+}
+
 type BaseWrite struct {
 	Input     exec.VarID
 	UserSpace clitypes.UserSpaceDetail
@@ -102,14 +144,13 @@ func (o *BaseWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
 	}
 	defer input.Stream.Close()
 
-	info, err := store.Write(o.Path, input.Stream)
+	ret, err := store.Write(o.Path, input.Stream)
 	if err != nil {
 		return err
 	}
 
 	e.PutVar(o.FileInfo, &FileInfoValue{
-		Hash: info.Hash,
-		Size: info.Size,
+		FileInfo: ret,
 	})
 	return nil
 }
@@ -123,13 +164,15 @@ type BaseReadNode struct {
 	From      ioswitch2.From
 	UserSpace clitypes.UserSpaceDetail
 	Path      string
+	Option    types.OpenOption
 }
 
-func (b *GraphNodeBuilder) NewPublicRead(from ioswitch2.From, userSpace clitypes.UserSpaceDetail, path string) *BaseReadNode {
+func (b *GraphNodeBuilder) NewBaseRead(from ioswitch2.From, userSpace clitypes.UserSpaceDetail, path string, opt types.OpenOption) *BaseReadNode {
 	node := &BaseReadNode{
 		From:      from,
 		UserSpace: userSpace,
 		Path:      path,
+		Option:    opt,
 	}
 	b.AddNode(node)
 
@@ -153,18 +196,65 @@ func (t *BaseReadNode) GenerateOp() (exec.Op, error) {
 		Output:    t.Output().Var().VarID,
 		UserSpace: t.UserSpace,
 		Path:      t.Path,
+		Option:    t.Option,
+	}, nil
+}
+
+type BaseReadDynNode struct {
+	dag.NodeBase
+	From      ioswitch2.From
+	UserSpace clitypes.UserSpaceDetail
+	Option    types.OpenOption
+}
+
+func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitch2.From, userSpace clitypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
+	node := &BaseReadDynNode{
+		From:      from,
+		UserSpace: userSpace,
+		Option:    opt,
+	}
+	b.AddNode(node)
+
+	node.OutputStreams().Init(node, 1)
+	node.InputValues().Init(1)
+	return node
+}
+
+func (t *BaseReadDynNode) GetFrom() ioswitch2.From {
+	return t.From
+}
+
+func (t *BaseReadDynNode) FileInfoSlot() dag.ValueInputSlot {
+	return dag.ValueInputSlot{
+		Node:  t,
+		Index: 0,
+	}
+}
+
+func (t *BaseReadDynNode) Output() dag.StreamOutputSlot {
+	return dag.StreamOutputSlot{
+		Node:  t,
+		Index: 0,
+	}
+}
+
+func (t *BaseReadDynNode) GenerateOp() (exec.Op, error) {
+	return &BaseReadDyn{
+		UserSpace: t.UserSpace,
+		Output:    t.Output().Var().VarID,
+		FileInfo:  t.FileInfoSlot().Var().VarID,
+		Option:    t.Option,
 	}, nil
 }
 
 type BaseWriteNode struct {
 	dag.NodeBase
-	To               ioswitch2.To
-	UserSpace        clitypes.UserSpaceDetail
-	Path             string
-	FileInfoStoreKey string
+	To        ioswitch2.To
+	UserSpace clitypes.UserSpaceDetail
+	Path      string
 }
 
-func (b *GraphNodeBuilder) NewPublicWrite(to ioswitch2.To, userSpace clitypes.UserSpaceDetail, path string) *BaseWriteNode {
+func (b *GraphNodeBuilder) NewBaseWrite(to ioswitch2.To, userSpace clitypes.UserSpaceDetail, path string) *BaseWriteNode {
 	node := &BaseWriteNode{
 		To:        to,
 		UserSpace: userSpace,
@@ -173,6 +263,7 @@ func (b *GraphNodeBuilder) NewPublicWrite(to ioswitch2.To, userSpace clitypes.Us
 	b.AddNode(node)
 
 	node.InputStreams().Init(1)
+	node.OutputValues().Init(node, 1)
 	return node
 }
 
@@ -180,10 +271,6 @@ func (t *BaseWriteNode) GetTo() ioswitch2.To {
 	return t.To
 }
 
-func (t *BaseWriteNode) SetInput(input *dag.StreamVar) {
-	input.To(t, 0)
-}
-
 func (t *BaseWriteNode) Input() dag.StreamInputSlot {
 	return dag.StreamInputSlot{
 		Node:  t,
@@ -191,8 +278,11 @@ func (t *BaseWriteNode) Input() dag.StreamInputSlot {
 	}
 }
 
-func (t *BaseWriteNode) FileInfoVar() *dag.ValueVar {
-	return t.OutputValues().Get(0)
+func (t *BaseWriteNode) FileInfoVar() dag.ValueOutputSlot {
+	return dag.ValueOutputSlot{
+		Node:  t,
+		Index: 0,
+	}
 }
 
 func (t *BaseWriteNode) GenerateOp() (exec.Op, error) {
@@ -200,6 +290,6 @@ func (t *BaseWriteNode) GenerateOp() (exec.Op, error) {
 		Input:     t.InputStreams().Get(0).VarID,
 		UserSpace: t.UserSpace,
 		Path:      t.Path,
-		FileInfo:  t.FileInfoVar().VarID,
+		FileInfo:  t.FileInfoVar().Var().VarID,
 	}, nil
 }

common/pkgs/ioswitch2/ops2/bypass.go

@@ -11,205 +11,12 @@ import (
 )
 
 func init() {
-	exec.UseOp[*BypassToShardStore]()
-	exec.UseOp[*BypassToBaseStore]()
-
-	exec.UseVarValue[*BypassedFileInfoValue]()
-	exec.UseVarValue[*BypassHandleResultValue]()
-
-	exec.UseOp[*BypassFromShardStore]()
-	exec.UseOp[*BypassFromBaseStore]()
-	exec.UseVarValue[*BypassFilePathValue]()
-
-	exec.UseOp[*BypassFromShardStoreHTTP]()
+	exec.UseOp[*GetShardHTTPRequest]()
 	exec.UseVarValue[*HTTPRequestValue]()
 }
 
-type BypassedFileInfoValue struct {
-	types.BypassedFileInfo
-}
-
-func (v *BypassedFileInfoValue) Clone() exec.VarValue {
-	return &BypassedFileInfoValue{
-		BypassedFileInfo: v.BypassedFileInfo,
-	}
-}
-
-type BypassHandleResultValue struct {
-	Commited bool
-}
-
-func (r *BypassHandleResultValue) Clone() exec.VarValue {
-	return &BypassHandleResultValue{
-		Commited: r.Commited,
-	}
-}
-
-type BypassToShardStore struct {
-	UserSpace      clitypes.UserSpaceDetail
-	BypassFileInfo exec.VarID
-	BypassCallback exec.VarID
-	FileInfo       exec.VarID
-}
-
-func (o *BypassToShardStore) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
-	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
-	if err != nil {
-		return err
-	}
-
-	shardStore, err := stgPool.GetShardStore(&o.UserSpace)
-	if err != nil {
-		return err
-	}
-
-	br, ok := shardStore.(types.BypassShardWrite)
-	if !ok {
-		return fmt.Errorf("shard store %v not support bypass write", o.UserSpace)
-	}
-
-	fileInfo, err := exec.BindVar[*BypassedFileInfoValue](e, ctx.Context, o.BypassFileInfo)
-	if err != nil {
-		return err
-	}
-
-	err = br.BypassedShard(fileInfo.BypassedFileInfo)
-	if err != nil {
-		return err
-	}
-
-	e.PutVar(o.BypassCallback, &BypassHandleResultValue{Commited: true})
-	e.PutVar(o.FileInfo, &FileInfoValue{Hash: fileInfo.Hash, Size: fileInfo.Size})
-	return nil
-}
-
-func (o *BypassToShardStore) String() string {
-	return fmt.Sprintf("BypassToShardStore[UserSpace:%v] Info: %v, Callback: %v", o.UserSpace, o.BypassFileInfo, o.BypassCallback)
-}
-
-type BypassToBaseStore struct {
-	UserSpace      clitypes.UserSpaceDetail
-	BypassFileInfo exec.VarID
-	BypassCallback exec.VarID
-	DestPath       string
-}
-
-func (o *BypassToBaseStore) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
-	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
-	if err != nil {
-		return err
-	}
-
-	store, err := stgPool.GetBaseStore(&o.UserSpace)
-	if err != nil {
-		return err
-	}
-
-	br, ok := store.(types.BypassPublicWrite)
-	if !ok {
-		return fmt.Errorf("base store %v not support bypass write", o.UserSpace)
-	}
-
-	fileInfo, err := exec.BindVar[*BypassedFileInfoValue](e, ctx.Context, o.BypassFileInfo)
-	if err != nil {
-		return err
-	}
-
-	err = br.BypassedPublic(fileInfo.BypassedFileInfo, o.DestPath)
-	if err != nil {
-		return err
-	}
-
-	e.PutVar(o.BypassCallback, &BypassHandleResultValue{Commited: true})
-	return nil
-}
-
-func (o *BypassToBaseStore) String() string {
-	return fmt.Sprintf("BypassToBaseStore[UserSpace:%v] Info: %v, Callback: %v", o.UserSpace, o.BypassFileInfo, o.BypassCallback)
-}
-
-type BypassFilePathValue struct {
-	types.BypassFilePath
-}
-
-func (v *BypassFilePathValue) Clone() exec.VarValue {
-	return &BypassFilePathValue{
-		BypassFilePath: v.BypassFilePath,
-	}
-}
-
-type BypassFromShardStore struct {
-	UserSpace clitypes.UserSpaceDetail
-	FileHash  clitypes.FileHash
-	Output    exec.VarID
-}
-
-func (o *BypassFromShardStore) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
-	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
-	if err != nil {
-		return err
-	}
-
-	shardStore, err := stgPool.GetShardStore(&o.UserSpace)
-	if err != nil {
-		return err
-	}
-
-	br, ok := shardStore.(types.BypassShardRead)
-	if !ok {
-		return fmt.Errorf("shard store %v not support bypass read", o.UserSpace)
-	}
-
-	path, err := br.BypassShardRead(o.FileHash)
-	if err != nil {
-		return err
-	}
-
-	e.PutVar(o.Output, &BypassFilePathValue{BypassFilePath: path})
-	return nil
-}
-
-func (o *BypassFromShardStore) String() string {
-	return fmt.Sprintf("BypassFromShardStore[UserSpace:%v] FileHash: %v, Output: %v", o.UserSpace, o.FileHash, o.Output)
-}
-
-type BypassFromBaseStore struct {
-	UserSpace clitypes.UserSpaceDetail
-	Path      string
-	Output    exec.VarID
-}
-
-func (o *BypassFromBaseStore) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
-	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
-	if err != nil {
-		return err
-	}
-
-	store, err := stgPool.GetBaseStore(&o.UserSpace)
-	if err != nil {
-		return err
-	}
-
-	br, ok := store.(types.BypassPublicRead)
-	if !ok {
-		return fmt.Errorf("base store %v not support bypass read", o.UserSpace)
-	}
-
-	path, err := br.BypassPublicRead(o.Path)
-	if err != nil {
-		return err
-	}
-
-	e.PutVar(o.Output, &BypassFilePathValue{BypassFilePath: path})
-	return nil
-}
-
-func (o *BypassFromBaseStore) String() string {
-	return fmt.Sprintf("BypassFromBaseStore[UserSpace:%v] Path: %v, Output: %v", o.UserSpace, o.Path, o.Output)
-}
-
 // 旁路Http读取
-type BypassFromShardStoreHTTP struct {
+type GetShardHTTPRequest struct {
 	UserSpace clitypes.UserSpaceDetail
 	FileHash  clitypes.FileHash
 	Output    exec.VarID
@@ -225,7 +32,7 @@ func (v *HTTPRequestValue) Clone() exec.VarValue {
 	}
 }
 
-func (o *BypassFromShardStoreHTTP) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
+func (o *GetShardHTTPRequest) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
 	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
 	if err != nil {
 		return err
@@ -236,12 +43,12 @@ func (o *BypassFromShardStoreHTTP) Execute(ctx *exec.ExecContext, e *exec.Execut
 		return err
 	}
 
-	br, ok := shardStore.(types.HTTPBypassShardRead)
+	br, ok := shardStore.(types.HTTPShardRead)
 	if !ok {
 		return fmt.Errorf("shard store %v not support bypass read", o.UserSpace)
 	}
 
-	req, err := br.HTTPBypassRead(o.FileHash)
+	req, err := br.MakeHTTPReadRequest(o.FileHash)
 	if err != nil {
 		return err
 	}
@@ -250,174 +57,19 @@ func (o *BypassFromShardStoreHTTP) Execute(ctx *exec.ExecContext, e *exec.Execut
 	return nil
 }
 
-func (o *BypassFromShardStoreHTTP) String() string {
-	return fmt.Sprintf("BypassFromShardStoreHTTP[UserSpace:%v] FileHash: %v, Output: %v", o.UserSpace, o.FileHash, o.Output)
-}
-
-// 旁路写入
-type BypassToShardStoreNode struct {
-	dag.NodeBase
-	UserSpace        clitypes.UserSpaceDetail
-	FileHashStoreKey string
-}
-
-func (b *GraphNodeBuilder) NewBypassToShardStore(userSpace clitypes.UserSpaceDetail, fileHashStoreKey string) *BypassToShardStoreNode {
-	node := &BypassToShardStoreNode{
-		UserSpace:        userSpace,
-		FileHashStoreKey: fileHashStoreKey,
-	}
-	b.AddNode(node)
-
-	node.InputValues().Init(1)
-	node.OutputValues().Init(node, 2)
-	return node
-}
-
-func (n *BypassToShardStoreNode) BypassFileInfoSlot() dag.ValueInputSlot {
-	return dag.ValueInputSlot{
-		Node:  n,
-		Index: 0,
-	}
-}
-
-func (n *BypassToShardStoreNode) BypassCallbackVar() dag.ValueOutputSlot {
-	return dag.ValueOutputSlot{
-		Node:  n,
-		Index: 0,
-	}
-}
-
-func (n *BypassToShardStoreNode) FileHashVar() dag.ValueOutputSlot {
-	return dag.ValueOutputSlot{
-		Node:  n,
-		Index: 1,
-	}
-}
-
-func (t *BypassToShardStoreNode) GenerateOp() (exec.Op, error) {
-	return &BypassToShardStore{
-		UserSpace:      t.UserSpace,
-		BypassFileInfo: t.BypassFileInfoSlot().Var().VarID,
-		BypassCallback: t.BypassCallbackVar().Var().VarID,
-		FileInfo:       t.FileHashVar().Var().VarID,
-	}, nil
-}
-
-type BypassToBaseStoreNode struct {
-	dag.NodeBase
-	UserSpace clitypes.UserSpaceDetail
-	DestPath  string
-}
-
-func (b *GraphNodeBuilder) NewBypassToBaseStore(userSpace clitypes.UserSpaceDetail, dstPath string) *BypassToBaseStoreNode {
-	node := &BypassToBaseStoreNode{
-		UserSpace: userSpace,
-		DestPath:  dstPath,
-	}
-	b.AddNode(node)
-
-	node.InputValues().Init(1)
-	node.OutputValues().Init(node, 1)
-	return node
-}
-
-func (n *BypassToBaseStoreNode) BypassFileInfoSlot() dag.ValueInputSlot {
-	return dag.ValueInputSlot{
-		Node:  n,
-		Index: 0,
-	}
-}
-
-func (n *BypassToBaseStoreNode) BypassCallbackVar() dag.ValueOutputSlot {
-	return dag.ValueOutputSlot{
-		Node:  n,
-		Index: 0,
-	}
-}
-
-func (t *BypassToBaseStoreNode) GenerateOp() (exec.Op, error) {
-	return &BypassToBaseStore{
-		UserSpace:      t.UserSpace,
-		BypassFileInfo: t.BypassFileInfoSlot().Var().VarID,
-		BypassCallback: t.BypassCallbackVar().Var().VarID,
-		DestPath:       t.DestPath,
-	}, nil
-}
-
-// 旁路读取
-type BypassFromShardStoreNode struct {
-	dag.NodeBase
-	UserSpace clitypes.UserSpaceDetail
-	FileHash  clitypes.FileHash
-}
-
-func (b *GraphNodeBuilder) NewBypassFromShardStore(userSpace clitypes.UserSpaceDetail, fileHash clitypes.FileHash) *BypassFromShardStoreNode {
-	node := &BypassFromShardStoreNode{
-		UserSpace: userSpace,
-		FileHash:  fileHash,
-	}
-	b.AddNode(node)
-
-	node.OutputValues().Init(node, 1)
-	return node
-}
-
-func (n *BypassFromShardStoreNode) FilePathVar() dag.ValueOutputSlot {
-	return dag.ValueOutputSlot{
-		Node:  n,
-		Index: 0,
-	}
-}
-
-func (n *BypassFromShardStoreNode) GenerateOp() (exec.Op, error) {
-	return &BypassFromShardStore{
-		UserSpace: n.UserSpace,
-		FileHash:  n.FileHash,
-		Output:    n.FilePathVar().Var().VarID,
-	}, nil
-}
-
-type BypassFromBaseStoreNode struct {
-	dag.NodeBase
-	UserSpace clitypes.UserSpaceDetail
-	Path      string
-}
-
-func (b *GraphNodeBuilder) NewBypassFromBaseStore(userSpace clitypes.UserSpaceDetail, path string) *BypassFromBaseStoreNode {
-	node := &BypassFromBaseStoreNode{
-		UserSpace: userSpace,
-		Path:      path,
-	}
-	b.AddNode(node)
-
-	node.OutputValues().Init(node, 1)
-	return node
-}
-
-func (n *BypassFromBaseStoreNode) FilePathVar() dag.ValueOutputSlot {
-	return dag.ValueOutputSlot{
-		Node:  n,
-		Index: 0,
-	}
-}
-
-func (n *BypassFromBaseStoreNode) GenerateOp() (exec.Op, error) {
-	return &BypassFromBaseStore{
-		UserSpace: n.UserSpace,
-		Path:      n.Path,
-		Output:    n.FilePathVar().Var().VarID,
-	}, nil
+func (o *GetShardHTTPRequest) String() string {
+	return fmt.Sprintf("GetShardHTTPRequest[UserSpace:%v] FileHash: %v, Output: %v", o.UserSpace, o.FileHash, o.Output)
 }
 
 // 旁路Http读取
-type BypassFromShardStoreHTTPNode struct {
+type GetShardHTTPRequestNode struct {
 	dag.NodeBase
 	UserSpace clitypes.UserSpaceDetail
 	FileHash  clitypes.FileHash
 }
 
-func (b *GraphNodeBuilder) NewBypassFromShardStoreHTTP(userSpace clitypes.UserSpaceDetail, fileHash clitypes.FileHash) *BypassFromShardStoreHTTPNode {
-	node := &BypassFromShardStoreHTTPNode{
+func (b *GraphNodeBuilder) NewGetShardHTTPRequest(userSpace clitypes.UserSpaceDetail, fileHash clitypes.FileHash) *GetShardHTTPRequestNode {
+	node := &GetShardHTTPRequestNode{
 		UserSpace: userSpace,
 		FileHash:  fileHash,
 	}
@@ -427,15 +79,15 @@ func (b *GraphNodeBuilder) NewBypassFromShardStoreHTTP(userSpace clitypes.UserSp
 	return node
 }
 
-func (n *BypassFromShardStoreHTTPNode) HTTPRequestVar() dag.ValueOutputSlot {
+func (n *GetShardHTTPRequestNode) HTTPRequestVar() dag.ValueOutputSlot {
 	return dag.ValueOutputSlot{
 		Node:  n,
 		Index: 0,
 	}
 }
 
-func (n *BypassFromShardStoreHTTPNode) GenerateOp() (exec.Op, error) {
-	return &BypassFromShardStoreHTTP{
+func (n *GetShardHTTPRequestNode) GenerateOp() (exec.Op, error) {
+	return &GetShardHTTPRequest{
 		UserSpace: n.UserSpace,
 		FileHash:  n.FileHash,
 		Output:    n.HTTPRequestVar().Var().VarID,

common/pkgs/ioswitch2/ops2/ec.go

@@ -158,12 +158,11 @@ func (o *ECMultiply) String() string {
 }
 
 type CallECMultiplier struct {
-	UserSpace       clitypes.UserSpaceDetail
-	Coef            [][]byte
-	Inputs          []exec.VarID
-	Outputs         []exec.VarID
-	BypassCallbacks []exec.VarID
-	ChunkSize       int
+	UserSpace clitypes.UserSpaceDetail
+	Coef      [][]byte
+	Inputs    []exec.VarID
+	Outputs   []exec.VarID
+	ChunkSize int
 }
 
 func (o *CallECMultiplier) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
@@ -176,6 +175,7 @@ func (o *CallECMultiplier) Execute(ctx *exec.ExecContext, e *exec.Executor) erro
 	if err != nil {
 		return err
 	}
+	defer ecMul.Close()
 
 	inputs, err := exec.BindArray[*HTTPRequestValue](e, ctx.Context, o.Inputs)
 	if err != nil {
@@ -191,32 +191,15 @@ func (o *CallECMultiplier) Execute(ctx *exec.ExecContext, e *exec.Executor) erro
 	if err != nil {
 		return err
 	}
-	defer ecMul.Abort()
 
-	outputVals := make([]*BypassedFileInfoValue, 0, len(outputs))
+	outputVals := make([]*FileInfoValue, 0, len(outputs))
 	for _, output := range outputs {
-		outputVals = append(outputVals, &BypassedFileInfoValue{
-			BypassedFileInfo: output,
+		outputVals = append(outputVals, &FileInfoValue{
+			FileInfo: output,
 		})
 	}
 	exec.PutArray(e, o.Outputs, outputVals)
 
-	callbacks, err := exec.BindArray[*BypassHandleResultValue](e, ctx.Context, o.BypassCallbacks)
-	if err != nil {
-		return err
-	}
-
-	allSuc := true
-	for _, callback := range callbacks {
-		if !callback.Commited {
-			allSuc = false
-		}
-	}
-
-	if allSuc {
-		ecMul.Complete()
-	}
-
 	return nil
 }
 
@@ -305,31 +288,24 @@ func (t *CallECMultiplierNode) InitFrom(node *ECMultiplyNode) {
 	t.InputIndexes = node.InputIndexes
 	t.OutputIndexes = node.OutputIndexes
 
-	t.InputValues().Init(len(t.InputIndexes) + len(t.OutputIndexes)) // 流的输出+回调的输入
+	t.InputValues().Init(len(t.InputIndexes))
 	t.OutputValues().Init(t, len(t.OutputIndexes))
 }
 
-func (t *CallECMultiplierNode) InputSlot(idx int) dag.ValueInputSlot {
+func (t *CallECMultiplierNode) HTTPRequestSlot(idx int) dag.ValueInputSlot {
 	return dag.ValueInputSlot{
 		Node:  t,
 		Index: idx,
 	}
 }
 
-func (t *CallECMultiplierNode) OutputVar(idx int) dag.ValueOutputSlot {
+func (t *CallECMultiplierNode) FileInfoVar(idx int) dag.ValueOutputSlot {
 	return dag.ValueOutputSlot{
 		Node:  t,
 		Index: idx,
 	}
 }
 
-func (t *CallECMultiplierNode) BypassCallbackSlot(idx int) dag.ValueInputSlot {
-	return dag.ValueInputSlot{
-		Node:  t,
-		Index: idx + len(t.InputIndexes),
-	}
-}
-
 func (t *CallECMultiplierNode) GenerateOp() (exec.Op, error) {
 	rs, err := ec.NewRs(t.EC.K, t.EC.N)
 	if err != nil {
@@ -341,11 +317,10 @@ func (t *CallECMultiplierNode) GenerateOp() (exec.Op, error) {
 	}
 
 	return &CallECMultiplier{
-		UserSpace:       t.UserSpace,
-		Coef:            coef,
-		Inputs:          t.InputValues().GetVarIDsRanged(0, len(t.InputIndexes)),
-		Outputs:         t.OutputValues().GetVarIDs(),
-		BypassCallbacks: t.InputValues().GetVarIDsStart(len(t.InputIndexes)),
-		ChunkSize:       t.EC.ChunkSize,
+		UserSpace: t.UserSpace,
+		Coef:      coef,
+		Inputs:    t.InputValues().GetVarIDs(),
+		Outputs:   t.OutputValues().GetVarIDs(),
+		ChunkSize: t.EC.ChunkSize,
 	}, nil
 }

common/pkgs/ioswitch2/ops2/multipart.go

@@ -40,11 +40,10 @@ func (v *UploadedPartInfoValue) Clone() exec.VarValue {
 }
 
 type MultipartInitiator struct {
-	UserSpace        clitypes.UserSpaceDetail
-	UploadArgs       exec.VarID
-	UploadedParts    []exec.VarID
-	BypassFileOutput exec.VarID // 分片上传之后的临时文件的路径
-	BypassCallback   exec.VarID // 临时文件使用结果，用于告知Initiator如何处理临时文件
+	UserSpace     clitypes.UserSpaceDetail
+	UploadArgs    exec.VarID
+	UploadedParts []exec.VarID
+	FileOutput    exec.VarID // 分片上传之后的临时文件的路径
 }
 
 func (o *MultipartInitiator) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
@@ -63,7 +62,7 @@ func (o *MultipartInitiator) Execute(ctx *exec.ExecContext, e *exec.Executor) er
 	if err != nil {
 		return err
 	}
-	defer multiTask.Abort()
+	defer multiTask.Close()
 
 	// 分发上传参数
 	e.PutVar(o.UploadArgs, &MultipartUploadArgsValue{
@@ -88,25 +87,15 @@ func (o *MultipartInitiator) Execute(ctx *exec.ExecContext, e *exec.Executor) er
 	}
 
 	// 告知后续Op临时文件的路径
-	e.PutVar(o.BypassFileOutput, &BypassedFileInfoValue{
-		BypassedFileInfo: fileInfo,
+	e.PutVar(o.FileOutput, &FileInfoValue{
+		FileInfo: fileInfo,
 	})
 
-	// 等待后续Op处理临时文件
-	cb, err := exec.BindVar[*BypassHandleResultValue](e, ctx.Context, o.BypassCallback)
-	if err != nil {
-		return fmt.Errorf("getting temp file callback: %v", err)
-	}
-
-	if cb.Commited {
-		multiTask.Complete()
-	}
-
 	return nil
 }
 
 func (o *MultipartInitiator) String() string {
-	return fmt.Sprintf("MultipartInitiator Args: %v, Parts: %v, BypassFileOutput: %v, BypassCallback: %v", o.UploadArgs, o.UploadedParts, o.BypassFileOutput, o.BypassCallback)
+	return fmt.Sprintf("MultipartInitiator Args: %v, Parts: %v, BypassFileOutput: %v", o.UploadArgs, o.UploadedParts, o.FileOutput)
 }
 
 type MultipartUpload struct {
@@ -170,7 +159,6 @@ func (b *GraphNodeBuilder) NewMultipartInitiator(userSpace clitypes.UserSpaceDet
 	b.AddNode(node)
 
 	node.OutputValues().Init(node, 2)
-	node.InputValues().Init(1)
 	return node
 }
 
@@ -181,21 +169,14 @@ func (n *MultipartInitiatorNode) UploadArgsVar() dag.ValueOutputSlot {
 	}
 }
 
-func (n *MultipartInitiatorNode) BypassFileInfoVar() dag.ValueOutputSlot {
+func (n *MultipartInitiatorNode) FileInfoVar() dag.ValueOutputSlot {
 	return dag.ValueOutputSlot{
 		Node:  n,
 		Index: 1,
 	}
 }
 
-func (n *MultipartInitiatorNode) BypassCallbackSlot() dag.ValueInputSlot {
-	return dag.ValueInputSlot{
-		Node:  n,
-		Index: 0,
-	}
-}
-
-func (n *MultipartInitiatorNode) AppendPartInfoSlot() dag.ValueInputSlot {
+func (n *MultipartInitiatorNode) PartInfoSlot() dag.ValueInputSlot {
 	return dag.ValueInputSlot{
 		Node:  n,
 		Index: n.InputValues().EnlargeOne(),
@@ -204,11 +185,10 @@ func (n *MultipartInitiatorNode) AppendPartInfoSlot() dag.ValueInputSlot {
 
 func (n *MultipartInitiatorNode) GenerateOp() (exec.Op, error) {
 	return &MultipartInitiator{
-		UserSpace:        n.UserSpace,
-		UploadArgs:       n.UploadArgsVar().Var().VarID,
-		UploadedParts:    n.InputValues().GetVarIDsStart(1),
-		BypassFileOutput: n.BypassFileInfoVar().Var().VarID,
-		BypassCallback:   n.BypassCallbackSlot().Var().VarID,
+		UserSpace:     n.UserSpace,
+		UploadArgs:    n.UploadArgsVar().Var().VarID,
+		UploadedParts: n.InputValues().GetVarIDs(),
+		FileOutput:    n.FileInfoVar().Var().VarID,
 	}, nil
 }
 

common/pkgs/ioswitch2/ops2/ops.go

@@ -2,8 +2,10 @@ package ops2
 
 import (
 	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
+	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
 	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/plan/ops"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
+	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
 )
 
 type GraphNodeBuilder struct {
@@ -24,5 +26,12 @@ type ToNode interface {
 	dag.Node
 	GetTo() ioswitch2.To
 	Input() dag.StreamInputSlot
-	SetInput(input *dag.StreamVar)
+}
+
+type FileInfoValue struct {
+	types.FileInfo
+}
+
+func (v *FileInfoValue) Clone() exec.VarValue {
+	return &FileInfoValue{v.FileInfo}
 }

common/pkgs/ioswitch2/ops2/s2s.go

@@ -7,7 +7,6 @@ import (
 	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
 	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
-	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
 )
 
 func init() {
@@ -15,110 +14,158 @@ func init() {
 }
 
 type S2STransfer struct {
-	Src            clitypes.UserSpaceDetail
-	SrcPath        exec.VarID
-	Dst            clitypes.UserSpaceDetail
-	Output         exec.VarID
-	BypassCallback exec.VarID
-	S2SOption      types.S2SOption
+	SrcSpace clitypes.UserSpaceDetail
+	SrcPath  string
+	DstSpace clitypes.UserSpaceDetail
+	DstPath  string
+	Output   exec.VarID
 }
 
 func (o *S2STransfer) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
-	srcPath, err := exec.BindVar[*BypassFilePathValue](e, ctx.Context, o.SrcPath)
-	if err != nil {
-		return err
-	}
-
 	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
 	if err != nil {
 		return fmt.Errorf("getting storage pool: %w", err)
 	}
 
-	s2s, err := stgPool.GetS2STransfer(&o.Dst)
+	s2s, err := stgPool.GetS2STransfer(&o.DstSpace)
 	if err != nil {
 		return err
 	}
 
 	// 传输文件
-	dstPath, err := s2s.Transfer(ctx.Context, &o.Src, srcPath.Path, o.S2SOption)
+	fileInfo, err := s2s.Transfer(ctx.Context, &o.SrcSpace, o.SrcPath, o.DstPath)
 	if err != nil {
 		return err
 	}
-	defer s2s.Abort()
+	defer s2s.Close()
 
 	// 告知后续Op处理临时文件
-	e.PutVar(o.Output, &BypassedFileInfoValue{BypassedFileInfo: types.BypassedFileInfo{
-		Path: dstPath,
-		Hash: srcPath.Info.Hash,
-		Size: srcPath.Info.Size,
-	}})
-
-	// 等待后续Op处理临时文件
-	cb, err := exec.BindVar[*BypassHandleResultValue](e, ctx.Context, o.BypassCallback)
+	e.PutVar(o.Output, &FileInfoValue{FileInfo: fileInfo})
+	return nil
+}
+
+func (o *S2STransfer) String() string {
+	return fmt.Sprintf("S2STransfer %v:%v -> %v:%v", o.SrcSpace.UserSpace.Storage.String(), o.SrcPath, o.DstSpace.UserSpace.Storage.String(), o.Output)
+}
+
+type S2STransferDyn struct {
+	SrcSpace    clitypes.UserSpaceDetail
+	SrcFileInfo exec.VarID
+	DstSpace    clitypes.UserSpaceDetail
+	DstPath     string
+	Output      exec.VarID
+}
+
+func (o *S2STransferDyn) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
+	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
+	if err != nil {
+		return fmt.Errorf("getting storage pool: %w", err)
+	}
+
+	srcInfo, err := exec.BindVar[*FileInfoValue](e, ctx.Context, o.SrcFileInfo)
+	if err != nil {
+		return err
+	}
+
+	s2s, err := stgPool.GetS2STransfer(&o.DstSpace)
 	if err != nil {
-		return fmt.Errorf("getting temp file callback: %v", err)
+		return err
 	}
 
-	if cb.Commited {
-		s2s.Complete()
+	// 传输文件
+	fileInfo, err := s2s.Transfer(ctx.Context, &o.SrcSpace, srcInfo.Path, o.DstPath)
+	if err != nil {
+		return err
 	}
+	defer s2s.Close()
 
+	// 告知后续Op处理临时文件
+	e.PutVar(o.Output, &FileInfoValue{FileInfo: fileInfo})
 	return nil
 }
 
-func (o *S2STransfer) String() string {
-	return fmt.Sprintf("S2STransfer %v:%v -> %v:%v, Callback: %v", o.Src.UserSpace.Storage.String(), o.SrcPath, o.Dst.UserSpace.Storage.String(), o.Output, o.BypassCallback)
+func (o *S2STransferDyn) String() string {
+	return fmt.Sprintf("S2STransferDyn %v:%v -> %v:%v", o.SrcSpace.UserSpace.Storage.String(), o.SrcFileInfo, o.DstSpace.UserSpace.Storage.String(), o.Output)
 }
 
 type S2STransferNode struct {
 	dag.NodeBase
-	Src clitypes.UserSpaceDetail
-	Dst clitypes.UserSpaceDetail
-	Opt types.S2SOption
+	SrcSpace clitypes.UserSpaceDetail
+	SrcPath  string
+	DstSpace clitypes.UserSpaceDetail
+	DstPath  string
 }
 
-func (b *GraphNodeBuilder) NewS2STransfer(src, dst clitypes.UserSpaceDetail, opt types.S2SOption) *S2STransferNode {
+func (b *GraphNodeBuilder) NewS2STransfer(srcSpace clitypes.UserSpaceDetail, srcPath string, dstSpace clitypes.UserSpaceDetail, dstPath string) *S2STransferNode {
 	n := &S2STransferNode{
-		Src: src,
-		Dst: dst,
-		Opt: opt,
+		SrcSpace: srcSpace,
+		SrcPath:  srcPath,
+		DstSpace: dstSpace,
+		DstPath:  dstPath,
 	}
 	b.AddNode(n)
 
 	n.OutputValues().Init(n, 1)
-	n.InputValues().Init(2)
-
 	return n
 }
 
-func (n *S2STransferNode) SrcPathSlot() dag.ValueInputSlot {
-	return dag.ValueInputSlot{
+func (n *S2STransferNode) FileInfoVar() dag.ValueOutputSlot {
+	return dag.ValueOutputSlot{
 		Node:  n,
 		Index: 0,
 	}
 }
 
-func (n *S2STransferNode) BypassCallbackSlot() dag.ValueInputSlot {
+func (n *S2STransferNode) GenerateOp() (exec.Op, error) {
+	return &S2STransfer{
+		SrcSpace: n.SrcSpace,
+		SrcPath:  n.SrcPath,
+		DstSpace: n.DstSpace,
+		DstPath:  n.DstPath,
+		Output:   n.FileInfoVar().Var().VarID,
+	}, nil
+}
+
+type S2STransferDynNode struct {
+	dag.NodeBase
+	SrcSpace clitypes.UserSpaceDetail
+	DstSpace clitypes.UserSpaceDetail
+	DstPath  string
+}
+
+func (b *GraphNodeBuilder) NewS2STransferDyn(srcSpace clitypes.UserSpaceDetail, dstSpace clitypes.UserSpaceDetail, dstPath string) *S2STransferDynNode {
+	n := &S2STransferDynNode{
+		SrcSpace: srcSpace,
+		DstSpace: dstSpace,
+		DstPath:  dstPath,
+	}
+	b.AddNode(n)
+
+	n.InputValues().Init(1)
+	n.OutputValues().Init(n, 1)
+	return n
+}
+
+func (n *S2STransferDynNode) SrcFileInfoSlot() dag.ValueInputSlot {
 	return dag.ValueInputSlot{
 		Node:  n,
-		Index: 1,
+		Index: 0,
 	}
 }
 
-func (n *S2STransferNode) BypassFileInfoVar() dag.ValueOutputSlot {
+func (n *S2STransferDynNode) FileInfoVar() dag.ValueOutputSlot {
 	return dag.ValueOutputSlot{
 		Node:  n,
 		Index: 0,
 	}
 }
 
-func (n *S2STransferNode) GenerateOp() (exec.Op, error) {
-	return &S2STransfer{
-		Src:            n.Src,
-		SrcPath:        n.SrcPathSlot().Var().VarID,
-		Dst:            n.Dst,
-		Output:         n.BypassFileInfoVar().Var().VarID,
-		BypassCallback: n.BypassCallbackSlot().Var().VarID,
-		S2SOption:      n.Opt,
+func (n *S2STransferDynNode) GenerateOp() (exec.Op, error) {
+	return &S2STransferDyn{
+		SrcSpace:    n.SrcSpace,
+		SrcFileInfo: n.SrcFileInfoSlot().Var().VarID,
+		DstSpace:    n.DstSpace,
+		DstPath:     n.DstPath,
+		Output:      n.FileInfoVar().Var().VarID,
 	}, nil
 }

common/pkgs/ioswitch2/ops2/shard_store.go

@@ -2,78 +2,56 @@ package ops2
 
 import (
 	"fmt"
-	"io"
 
-	"gitlink.org.cn/cloudream/common/pkgs/future"
 	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
 	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
-	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	"gitlink.org.cn/cloudream/common/utils/io2"
 	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
-	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
-	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
 )
 
 func init() {
-	exec.UseOp[*ShardRead]()
-	exec.UseOp[*ShardWrite]()
+	exec.UseOp[*GetShardInfo]()
+	exec.UseOp[*StoreShard]()
 }
 
-type ShardRead struct {
-	Output    exec.VarID
+type GetShardInfo struct {
 	UserSpace clitypes.UserSpaceDetail
-	Open      types.OpenOption
+	FileHash  clitypes.FileHash
+	ShardInfo exec.VarID
 }
 
-func (o *ShardRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
-	logger.
-		WithField("Open", o.Open.String()).
-		Debugf("reading from shard store")
-	defer logger.Debugf("reading from shard store finished")
-
+func (o *GetShardInfo) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
 	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
 	if err != nil {
-		return fmt.Errorf("getting storage pool: %w", err)
+		return fmt.Errorf("getting shard store: %w", err)
 	}
 
 	store, err := stgPool.GetShardStore(&o.UserSpace)
 	if err != nil {
-		return fmt.Errorf("getting shard store of user space %v: %w", o.UserSpace, err)
+		return fmt.Errorf("getting shard store: %w", err)
 	}
 
-	file, err := store.Open(o.Open)
+	info, err := store.Info(o.FileHash)
 	if err != nil {
-		return fmt.Errorf("opening shard store file: %w", err)
+		return fmt.Errorf("getting shard info: %w", err)
 	}
-
-	fut := future.NewSetVoid()
-	e.PutVar(o.Output, &exec.StreamValue{
-		Stream: io2.AfterReadClosedOnce(file, func(closer io.ReadCloser) {
-			fut.SetVoid()
-		}),
+	e.PutVar(o.ShardInfo, &FileInfoValue{
+		FileInfo: info,
 	})
-
-	return fut.Wait(ctx.Context)
+	return nil
 }
 
-func (o *ShardRead) String() string {
-	return fmt.Sprintf("ShardRead %v -> %v", o.Open.String(), o.Output)
+func (o *GetShardInfo) String() string {
+	return fmt.Sprintf("GetShardInfo(%v)", o.FileHash)
 }
 
-type ShardWrite struct {
-	Input       exec.VarID
-	FileHashVar exec.VarID
-	UserSpace   clitypes.UserSpaceDetail
+type StoreShard struct {
+	UserSpace clitypes.UserSpaceDetail
+	FileInfo  exec.VarID
+	ShardInfo exec.VarID
 }
 
-func (o *ShardWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
-	logger.
-		WithField("Input", o.Input).
-		WithField("FileHash", o.FileHashVar).
-		Debugf("writting file to shard store")
-	defer logger.Debugf("write to shard store finished")
-
+func (o *StoreShard) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
 	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
 	if err != nil {
 		return fmt.Errorf("getting storage pool: %w", err)
@@ -81,120 +59,97 @@ func (o *ShardWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
 
 	store, err := stgPool.GetShardStore(&o.UserSpace)
 	if err != nil {
-		return fmt.Errorf("getting shard store of user space %v: %w", o.UserSpace, err)
+		return fmt.Errorf("getting shard store: %w", err)
 	}
 
-	input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
+	info, err := exec.BindVar[*FileInfoValue](e, ctx.Context, o.FileInfo)
 	if err != nil {
 		return err
 	}
-	defer input.Stream.Close()
 
-	fileInfo, err := store.Create(input.Stream)
+	stored, err := store.Store(info.Path, info.Hash, info.Size)
 	if err != nil {
-		return fmt.Errorf("writing file to shard store: %w", err)
+		return fmt.Errorf("adding shard: %w", err)
 	}
 
-	e.PutVar(o.FileHashVar, &FileInfoValue{
-		Hash: fileInfo.Hash,
-		Size: fileInfo.Size,
+	e.PutVar(o.ShardInfo, &FileInfoValue{
+		FileInfo: stored,
 	})
 	return nil
 }
 
-func (o *ShardWrite) String() string {
-	return fmt.Sprintf("ShardWrite %v -> %v", o.Input, o.FileHashVar)
+func (o *StoreShard) String() string {
+	return fmt.Sprintf("StoreShard: addInfo=%v, shardInfo=%v", o.FileInfo, o.ShardInfo)
 }
 
-type ShardReadNode struct {
+type GetShardInfoNode struct {
 	dag.NodeBase
-	From      *ioswitch2.FromShardstore
 	UserSpace clitypes.UserSpaceDetail
-	Open      types.OpenOption
+	FileHash  clitypes.FileHash
 }
 
-func (b *GraphNodeBuilder) NewShardRead(fr *ioswitch2.FromShardstore, userSpace clitypes.UserSpaceDetail, open types.OpenOption) *ShardReadNode {
-	node := &ShardReadNode{
-		From:      fr,
+func (b *GraphNodeBuilder) NewGetShardInfo(userSpace clitypes.UserSpaceDetail, fileHash clitypes.FileHash) *GetShardInfoNode {
+	node := &GetShardInfoNode{
 		UserSpace: userSpace,
-		Open:      open,
+		FileHash:  fileHash,
 	}
 	b.AddNode(node)
 
-	node.OutputStreams().Init(node, 1)
+	node.OutputValues().Init(node, 1)
 	return node
 }
 
-func (t *ShardReadNode) GetFrom() ioswitch2.From {
-	return t.From
-}
-
-func (t *ShardReadNode) Output() dag.StreamOutputSlot {
-	return dag.StreamOutputSlot{
-		Node:  t,
+func (n *GetShardInfoNode) FileInfoVar() dag.ValueOutputSlot {
+	return dag.ValueOutputSlot{
+		Node:  n,
 		Index: 0,
 	}
 }
 
-func (t *ShardReadNode) GenerateOp() (exec.Op, error) {
-	return &ShardRead{
-		Output:    t.OutputStreams().Get(0).VarID,
-		UserSpace: t.UserSpace,
-		Open:      t.Open,
+func (n *GetShardInfoNode) GenerateOp() (exec.Op, error) {
+	return &GetShardInfo{
+		UserSpace: n.UserSpace,
+		FileHash:  n.FileHash,
+		ShardInfo: n.FileInfoVar().Var().VarID,
 	}, nil
 }
 
-// func (t *IPFSReadType) String() string {
-// 	return fmt.Sprintf("IPFSRead[%s,%v+%v]%v%v", t.FileHash, t.Option.Offset, t.Option.Length, formatStreamIO(node), formatValueIO(node))
-// }
-
-type ShardWriteNode struct {
+type StoreShardNode struct {
 	dag.NodeBase
-	To               *ioswitch2.ToShardStore
-	UserSpace        clitypes.UserSpaceDetail
-	FileHashStoreKey string
+	UserSpace    clitypes.UserSpaceDetail
+	ShardInfoKey string
 }
 
-func (b *GraphNodeBuilder) NewShardWrite(to *ioswitch2.ToShardStore, userSpace clitypes.UserSpaceDetail, fileHashStoreKey string) *ShardWriteNode {
-	node := &ShardWriteNode{
-		To:               to,
-		UserSpace:        userSpace,
-		FileHashStoreKey: fileHashStoreKey,
+func (b *GraphNodeBuilder) NewStoreShard(userSpace clitypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
+	node := &StoreShardNode{
+		UserSpace:    userSpace,
+		ShardInfoKey: shardInfoKey,
 	}
 	b.AddNode(node)
 
-	node.InputStreams().Init(1)
+	node.InputValues().Init(1)
 	node.OutputValues().Init(node, 1)
 	return node
 }
 
-func (t *ShardWriteNode) GetTo() ioswitch2.To {
-	return t.To
-}
-
-func (t *ShardWriteNode) SetInput(input *dag.StreamVar) {
-	input.To(t, 0)
-}
-
-func (t *ShardWriteNode) Input() dag.StreamInputSlot {
-	return dag.StreamInputSlot{
+func (t *StoreShardNode) FileInfoSlot() dag.ValueInputSlot {
+	return dag.ValueInputSlot{
 		Node:  t,
 		Index: 0,
 	}
 }
 
-func (t *ShardWriteNode) FileHashVar() *dag.ValueVar {
-	return t.OutputValues().Get(0)
+func (t *StoreShardNode) ShardInfoVar() dag.ValueOutputSlot {
+	return dag.ValueOutputSlot{
+		Node:  t,
+		Index: 0,
+	}
 }
 
-func (t *ShardWriteNode) GenerateOp() (exec.Op, error) {
-	return &ShardWrite{
-		Input:       t.InputStreams().Get(0).VarID,
-		FileHashVar: t.OutputValues().Get(0).VarID,
-		UserSpace:   t.UserSpace,
+func (t *StoreShardNode) GenerateOp() (exec.Op, error) {
+	return &StoreShard{
+		UserSpace: t.UserSpace,
+		FileInfo:  t.FileInfoSlot().Var().VarID,
+		ShardInfo: t.ShardInfoVar().Var().VarID,
 	}, nil
 }
-
-// func (t *IPFSWriteType) String() string {
-// 	return fmt.Sprintf("IPFSWrite[%s,%v+%v]%v%v", t.FileHashStoreKey, t.Range.Offset, t.Range.Length, formatStreamIO(node), formatValueIO(node))
-// }

common/pkgs/ioswitch2/parser/gen/generator.go

@@ -7,6 +7,7 @@ import (
 	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
 	"gitlink.org.cn/cloudream/common/utils/lo2"
 	"gitlink.org.cn/cloudream/common/utils/math2"
+	"gitlink.org.cn/cloudream/common/utils/os2"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser/state"
@@ -234,7 +235,7 @@ func Extend(ctx *state.GenerateState) error {
 			return fmt.Errorf("no output stream found for data index %d", to.GetStreamIndex())
 		}
 
-		toNode.SetInput(str)
+		str.ToSlot(toNode.Input())
 	}
 
 	return nil
@@ -258,13 +259,17 @@ func buildFromNode(ctx *state.GenerateState, f ioswitch2.From) (ops2.FromNode, e
 	}
 
 	switch f := f.(type) {
-	case *ioswitch2.FromShardstore:
-		t := ctx.DAG.NewShardRead(f, f.UserSpace, types.NewOpen(f.FileHash))
+	case *ioswitch2.FromShardStore:
+		getShard := ctx.DAG.NewGetShardInfo(f.UserSpace, f.FileHash)
+		getShard.Env().ToEnvDriver()
+		getShard.Env().Pinned = true
+
+		read := ctx.DAG.NewBaseReadDyn(f, f.UserSpace, types.DefaultOpen())
 
 		if f.StreamIndex.IsRaw() {
-			t.Open.WithNullableLength(repRange.Offset, repRange.Length)
+			read.Option.WithNullableLength(repRange.Offset, repRange.Length)
 		} else if f.StreamIndex.IsEC() {
-			t.Open.WithNullableLength(blkRange.Offset, blkRange.Length)
+			read.Option.WithNullableLength(blkRange.Offset, blkRange.Length)
 		} else if f.StreamIndex.IsSegment() {
 			segStart := ctx.Ft.SegmentParam.CalcSegmentStart(f.StreamIndex.Index)
 			segLen := ctx.Ft.SegmentParam.Segments[f.StreamIndex.Index]
@@ -283,23 +288,23 @@ func buildFromNode(ctx *state.GenerateState, f ioswitch2.From) (ops2.FromNode, e
 				openLen = openEnd - segStart - openOff
 			}
 
-			t.Open.WithNullableLength(openOff, &openLen)
+			read.Option.WithNullableLength(openOff, &openLen)
 		}
 
 		switch addr := f.UserSpace.RecommendHub.Address.(type) {
 		case *cortypes.HttpAddressInfo:
-			t.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: f.UserSpace.RecommendHub})
-			t.Env().Pinned = true
+			read.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: f.UserSpace.RecommendHub})
+			read.Env().Pinned = true
 
 		case *cortypes.GRPCAddressInfo:
-			t.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: f.UserSpace.RecommendHub, Address: *addr})
-			t.Env().Pinned = true
+			read.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: f.UserSpace.RecommendHub, Address: *addr})
+			read.Env().Pinned = true
 
 		default:
 			return nil, fmt.Errorf("unsupported node address type %T", addr)
 		}
 
-		return t, nil
+		return read, nil
 
 	case *ioswitch2.FromDriver:
 		n := ctx.DAG.NewFromDriver(f, f.Handle)
@@ -338,18 +343,9 @@ func buildFromNode(ctx *state.GenerateState, f ioswitch2.From) (ops2.FromNode, e
 
 	case *ioswitch2.FromBaseStore:
 		// TODO 可以考虑支持设置读取范围
-		n := ctx.DAG.NewPublicRead(f, f.UserSpace, f.Path)
-		switch addr := f.UserSpace.RecommendHub.Address.(type) {
-		case *cortypes.HttpAddressInfo:
-			n.Env().ToEnvWorker(&ioswitch2.HttpHubWorker{Hub: f.UserSpace.RecommendHub})
-			n.Env().Pinned = true
-
-		case *cortypes.GRPCAddressInfo:
-			n.Env().ToEnvWorker(&ioswitch2.HubWorker{Hub: f.UserSpace.RecommendHub, Address: *addr})
-			n.Env().Pinned = true
-
-		default:
-			return nil, fmt.Errorf("unsupported node address type %T", addr)
+		n := ctx.DAG.NewBaseRead(f, f.UserSpace, f.Path, types.DefaultOpen())
+		if err := setEnvByAddress(n, f.UserSpace.RecommendHub, f.UserSpace.RecommendHub.Address); err != nil {
+			return nil, err
 		}
 
 		return n, nil
@@ -362,15 +358,21 @@ func buildFromNode(ctx *state.GenerateState, f ioswitch2.From) (ops2.FromNode, e
 func buildToNode(ctx *state.GenerateState, t ioswitch2.To) (ops2.ToNode, error) {
 	switch t := t.(type) {
 	case *ioswitch2.ToShardStore:
-		n := ctx.DAG.NewShardWrite(t, t.Space, t.FileHashStoreKey)
+		tempFileName := os2.GenerateRandomFileName(20)
 
-		if err := setEnvByAddress(n, t.Space.RecommendHub, t.Space.RecommendHub.Address); err != nil {
+		write := ctx.DAG.NewBaseWrite(t, t.Space, tempFileName)
+		if err := setEnvByAddress(write, t.Space.RecommendHub, t.Space.RecommendHub.Address); err != nil {
 			return nil, err
 		}
+		write.Env().Pinned = true
 
-		n.Env().Pinned = true
+		add := ctx.DAG.NewStoreShard(t.Space, t.ResultStoreKey)
+		add.Env().ToEnvDriver()
+		add.Env().Pinned = true
 
-		return n, nil
+		write.FileInfoVar().ToSlot(add.FileInfoSlot())
+
+		return write, nil
 
 	case *ioswitch2.ToDriver:
 		n := ctx.DAG.NewToDriver(t, t.Handle)
@@ -380,7 +382,7 @@ func buildToNode(ctx *state.GenerateState, t ioswitch2.To) (ops2.ToNode, error)
 		return n, nil
 
 	case *ioswitch2.ToBaseStore:
-		n := ctx.DAG.NewPublicWrite(t, t.Space, t.ObjectPath)
+		n := ctx.DAG.NewBaseWrite(t, t.Space, t.ObjectPath)
 
 		if err := setEnvByAddress(n, t.Space.RecommendHub, t.Space.RecommendHub.Address); err != nil {
 			return nil, err

common/pkgs/ioswitch2/parser/opt/ec.go

@@ -53,7 +53,7 @@ func UseECMultiplier(ctx *state.GenerateState) {
 		}
 
 		var to *ioswitch2.ToShardStore
-		var swNodes []*ops2.ShardWriteNode
+		var bwNodes []*ops2.BaseWriteNode
 		// 所有的输出流必须有且只有一个相同的目的地
 		// 暂时只支持分片存储
 		for i := 0; i < mulNode.OutputStreams().Len(); i++ {
@@ -62,17 +62,22 @@ func UseECMultiplier(ctx *state.GenerateState) {
 			}
 
 			dstNode := mulNode.OutputStreams().Get(i).Dst.Get(0)
-			swNode, ok := dstNode.(*ops2.ShardWriteNode)
+			swNode, ok := dstNode.(*ops2.BaseWriteNode)
+			if !ok {
+				return true
+			}
+
+			toShard, ok := swNode.To.(*ioswitch2.ToShardStore)
 			if !ok {
 				return true
 			}
 
 			if to == nil {
-				to = swNode.To
-			} else if to.Space.UserSpace.Storage.Equals(swNode.UserSpace.UserSpace.Storage) {
+				to = toShard
+			} else if to.Space.UserSpace.UserSpaceID != swNode.UserSpace.UserSpace.UserSpaceID {
 				return true
 			}
-			swNodes = append(swNodes, swNode)
+			bwNodes = append(bwNodes, swNode)
 		}
 		_, err := factory.GetBuilder(&to.Space).CreateECMultiplier()
 		if err != nil {
@@ -80,19 +85,25 @@ func UseECMultiplier(ctx *state.GenerateState) {
 		}
 
 		// 每一个输入流都必须直接来自于存储服务，而且要支持通过HTTP读取文件
-		var srNodes []*ops2.ShardReadNode
+		var brNodes []*ops2.BaseReadDynNode
+		var fromShards []*ioswitch2.FromShardStore
 		for i := 0; i < mulNode.InputStreams().Len(); i++ {
 			inNode := mulNode.InputStreams().Get(i).Src
-			srNode, ok := inNode.(*ops2.ShardReadNode)
+			brNode, ok := inNode.(*ops2.BaseReadDynNode)
+			if !ok {
+				return true
+			}
+			fromShard, ok := brNode.From.(*ioswitch2.FromShardStore)
 			if !ok {
 				return true
 			}
 
-			if !factory.GetBuilder(&srNode.From.UserSpace).FeatureDesc().HasBypassHTTPRead {
+			if !factory.GetBuilder(&brNode.UserSpace).FeatureDesc().HasBypassHTTPRead {
 				return true
 			}
 
-			srNodes = append(srNodes, srNode)
+			brNodes = append(brNodes, brNode)
+			fromShards = append(fromShards, fromShard)
 		}
 
 		// 检查满足条件后，替换ECMultiply指令
@@ -111,28 +122,26 @@ func UseECMultiplier(ctx *state.GenerateState) {
 		}
 
 		callMul.InitFrom(mulNode)
-		for i, srNode := range srNodes {
-			srNode.Output().Var().NotTo(mulNode)
+		for i, brNode := range brNodes {
+			brNode.Output().Var().NotTo(mulNode)
 			// 只有完全没有输出的ShardReadNode才可以被删除
-			if srNode.Output().Var().Dst.Len() == 0 {
-				ctx.DAG.RemoveNode(srNode)
-				delete(ctx.FromNodes, srNode.From)
+			if brNode.Output().Var().Dst.Len() == 0 {
+				ctx.DAG.RemoveNode(brNode)
+				delete(ctx.FromNodes, brNode.From)
 			}
 
-			hbr := ctx.DAG.NewBypassFromShardStoreHTTP(srNode.UserSpace, srNode.From.FileHash)
-			hbr.Env().CopyFrom(srNode.Env())
-			hbr.HTTPRequestVar().ToSlot(callMul.InputSlot(i))
+			hbr := ctx.DAG.NewGetShardHTTPRequest(brNode.UserSpace, fromShards[i].FileHash)
+			hbr.Env().CopyFrom(brNode.Env())
+			hbr.HTTPRequestVar().ToSlot(callMul.HTTPRequestSlot(i))
 		}
 
-		for i, swNode := range swNodes {
-			ctx.DAG.RemoveNode(swNode)
-			delete(ctx.ToNodes, swNode.To)
+		for i, bwNode := range bwNodes {
+			ctx.DAG.RemoveNode(bwNode)
+			delete(ctx.ToNodes, bwNode.To)
 
-			bs := ctx.DAG.NewBypassToShardStore(to.Space, swNode.FileHashStoreKey)
-			bs.Env().CopyFrom(swNode.Env())
-
-			callMul.OutputVar(i).ToSlot(bs.BypassFileInfoSlot())
-			bs.BypassCallbackVar().ToSlot(callMul.BypassCallbackSlot(i))
+			for _, dstSlot := range bwNode.FileInfoVar().ListDstSlots() {
+				callMul.FileInfoVar(i).ToSlot(dstSlot)
+			}
 		}
 
 		ctx.DAG.RemoveNode(mulNode)

common/pkgs/ioswitch2/parser/opt/misc.go

@@ -37,27 +37,15 @@ func DropUnused(ctx *state.GenerateState) {
 
 // 为IPFS写入指令存储结果
 func StoreShardWriteResult(ctx *state.GenerateState) {
-	dag.WalkOnlyType[*ops2.ShardWriteNode](ctx.DAG.Graph, func(n *ops2.ShardWriteNode) bool {
-		if n.FileHashStoreKey == "" {
+	dag.WalkOnlyType[*ops2.StoreShardNode](ctx.DAG.Graph, func(n *ops2.StoreShardNode) bool {
+		if n.ShardInfoKey == "" {
 			return true
 		}
 
 		storeNode := ctx.DAG.NewStore()
 		storeNode.Env().ToEnvDriver()
 
-		storeNode.Store(n.FileHashStoreKey, n.FileHashVar())
-		return true
-	})
-
-	dag.WalkOnlyType[*ops2.BypassToShardStoreNode](ctx.DAG.Graph, func(n *ops2.BypassToShardStoreNode) bool {
-		if n.FileHashStoreKey == "" {
-			return true
-		}
-
-		storeNode := ctx.DAG.NewStore()
-		storeNode.Env().ToEnvDriver()
-
-		storeNode.Store(n.FileHashStoreKey, n.FileHashVar().Var())
+		storeNode.Store(n.ShardInfoKey, n.ShardInfoVar().Var())
 		return true
 	})
 }
@@ -77,7 +65,7 @@ func GenerateRange(ctx *state.GenerateState) {
 				Length: toRng.Length,
 			})
 			toInput.Var().NotTo(toNode)
-			toNode.SetInput(rnged)
+			rnged.ToSlot(toInput)
 
 		} else if toStrIdx.IsEC() {
 			stripSize := int64(ctx.Ft.ECParam.ChunkSize * ctx.Ft.ECParam.K)
@@ -93,7 +81,7 @@ func GenerateRange(ctx *state.GenerateState) {
 				Length: toRng.Length,
 			})
 			toInput.Var().NotTo(toNode)
-			toNode.SetInput(rnged)
+			rnged.ToSlot(toInput)
 		} else if toStrIdx.IsSegment() {
 			// if frNode, ok := toNode.Input().Var().From().Node.(ops2.FromNode); ok {
 			// 	// 目前只有To也是分段时，才可能对接一个提供分段的From，此时不需要再生成Range指令

common/pkgs/ioswitch2/parser/opt/multipart.go

@@ -9,22 +9,22 @@ import (
 )
 
 // 将SegmentJoin指令替换成分片上传指令
-func UseMultipartUpcopyToShardStore(ctx *state.GenerateState) {
+func UseMultipartUploadToShardStore(ctx *state.GenerateState) {
 	dag.WalkOnlyType[*ops2.SegmentJoinNode](ctx.DAG.Graph, func(joinNode *ops2.SegmentJoinNode) bool {
 		if joinNode.Joined().Dst.Len() != 1 {
 			return true
 		}
 
 		joinDst := joinNode.Joined().Dst.Get(0)
-		shardNode, ok := joinDst.(*ops2.ShardWriteNode)
+		bwNode, ok := joinDst.(*ops2.BaseWriteNode)
 		if !ok {
 			return true
 		}
 
 		// SegmentJoin的输出流的范围必须与ToShardStore的输入流的范围相同，
 		// 虽然可以通过调整SegmentJoin的输入流来调整范围，但太复杂，暂不支持
-		toStrIdx := shardNode.GetTo().GetStreamIndex()
-		toStrRng := shardNode.GetTo().GetRange()
+		toStrIdx := bwNode.GetTo().GetStreamIndex()
+		toStrRng := bwNode.GetTo().GetRange()
 		if toStrIdx.IsRaw() {
 			if !toStrRng.Equals(ctx.StreamRange) {
 				return true
@@ -34,7 +34,7 @@ func UseMultipartUpcopyToShardStore(ctx *state.GenerateState) {
 		}
 
 		// Join的目的地必须支持MultipartUpload功能才能替换成分片上传
-		multiUpload, err := factory.GetBuilder(&shardNode.UserSpace).CreateMultiparter()
+		multiUpload, err := factory.GetBuilder(&bwNode.UserSpace).CreateMultiparter()
 		if err != nil {
 			return true
 		}
@@ -47,8 +47,8 @@ func UseMultipartUpcopyToShardStore(ctx *state.GenerateState) {
 			}
 		}
 
-		initNode := ctx.DAG.NewMultipartInitiator(shardNode.UserSpace)
-		initNode.Env().CopyFrom(shardNode.Env())
+		initNode := ctx.DAG.NewMultipartInitiator(bwNode.UserSpace)
+		initNode.Env().CopyFrom(bwNode.Env())
 
 		partNumber := 1
 		for i, size := range joinNode.Segments {
@@ -64,24 +64,24 @@ func UseMultipartUpcopyToShardStore(ctx *state.GenerateState) {
 				joinInput.Var().ToSlot(splitNode.InputSlot())
 
 				for i2 := 0; i2 < len(splits); i2++ {
-					uploadNode := ctx.DAG.NewMultipartUpload(shardNode.UserSpace, partNumber, splits[i2])
+					uploadNode := ctx.DAG.NewMultipartUpload(bwNode.UserSpace, partNumber, splits[i2])
 					uploadNode.Env().CopyFrom(joinInput.Var().Src.Env())
 
 					initNode.UploadArgsVar().ToSlot(uploadNode.UploadArgsSlot())
 					splitNode.SegmentVar(i2).ToSlot(uploadNode.PartStreamSlot())
-					uploadNode.UploadResultVar().ToSlot(initNode.AppendPartInfoSlot())
+					uploadNode.UploadResultVar().ToSlot(initNode.PartInfoSlot())
 
 					partNumber++
 				}
 			} else {
 				// 否则直接上传整个分段
-				uploadNode := ctx.DAG.NewMultipartUpload(shardNode.UserSpace, partNumber, size)
+				uploadNode := ctx.DAG.NewMultipartUpload(bwNode.UserSpace, partNumber, size)
 				// 上传指令直接在流的产生节点执行
 				uploadNode.Env().CopyFrom(joinInput.Var().Src.Env())
 
 				initNode.UploadArgsVar().ToSlot(uploadNode.UploadArgsSlot())
 				joinInput.Var().ToSlot(uploadNode.PartStreamSlot())
-				uploadNode.UploadResultVar().ToSlot(initNode.AppendPartInfoSlot())
+				uploadNode.UploadResultVar().ToSlot(initNode.PartInfoSlot())
 
 				partNumber++
 			}
@@ -89,17 +89,15 @@ func UseMultipartUpcopyToShardStore(ctx *state.GenerateState) {
 			joinInput.Var().NotTo(joinNode)
 		}
 
-		bypassNode := ctx.DAG.NewBypassToShardStore(shardNode.UserSpace, shardNode.FileHashStoreKey)
-		bypassNode.Env().CopyFrom(shardNode.Env())
-
-		// 分片上传Node产生的结果送到bypassNode，bypassNode将处理结果再送回分片上传Node
-		initNode.BypassFileInfoVar().ToSlot(bypassNode.BypassFileInfoSlot())
-		bypassNode.BypassCallbackVar().ToSlot(initNode.BypassCallbackSlot())
+		// BaseWriteNode的FileInfoVar替换为MultipartInitiatorNode的FileInfoVar
+		for _, dstSlot := range bwNode.FileInfoVar().ListDstSlots() {
+			initNode.FileInfoVar().ToSlot(dstSlot)
+		}
 
 		// 最后删除Join指令和ToShardStore指令
 		ctx.DAG.RemoveNode(joinNode)
-		ctx.DAG.RemoveNode(shardNode)
-		delete(ctx.ToNodes, shardNode.GetTo())
+		ctx.DAG.RemoveNode(bwNode)
+		delete(ctx.ToNodes, bwNode.GetTo())
 		return true
 	})
 }

common/pkgs/ioswitch2/parser/opt/s2s.go

@@ -5,7 +5,6 @@ import (
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser/state"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory"
-	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
 )
 
 // 将直接从一个存储服务传到另一个存储服务的过程换成S2S传输
@@ -17,7 +16,7 @@ func UseS2STransfer(ctx *state.GenerateState) {
 
 	for fr, frNode := range ctx.FromNodes {
 		switch fr := fr.(type) {
-		case *ioswitch2.FromShardstore:
+		case *ioswitch2.FromShardStore:
 			s2sFromShardStore(ctx, fr, frNode)
 		case *ioswitch2.FromBaseStore:
 			s2sFromBaseStore(ctx, fr, frNode)
@@ -25,7 +24,7 @@ func UseS2STransfer(ctx *state.GenerateState) {
 	}
 }
 
-func s2sFromShardStore(ctx *state.GenerateState, fromShard *ioswitch2.FromShardstore, frNode ops2.FromNode) {
+func s2sFromShardStore(ctx *state.GenerateState, fromShard *ioswitch2.FromShardStore, frNode ops2.FromNode) {
 	fromStgBld := factory.GetBuilder(&fromShard.UserSpace)
 	if !fromStgBld.FeatureDesc().HasBypassShardRead {
 		return
@@ -43,31 +42,16 @@ func s2sFromShardStore(ctx *state.GenerateState, fromShard *ioswitch2.FromShards
 	}
 
 	failed := false
-	var toShards []*ops2.ShardWriteNode
-	var toPublics []*ops2.BaseWriteNode
+	var toBases []*ops2.BaseWriteNode
 
 loop:
 	for i := 0; i < outVar.Dst.Len(); i++ {
 		dstNode := outVar.Dst.Get(i)
 
 		switch dstNode := dstNode.(type) {
-		case *ops2.ShardWriteNode:
-			dstStgBld := factory.GetBuilder(&dstNode.UserSpace)
-			if !dstStgBld.FeatureDesc().HasBypassShardWrite {
-				failed = true
-				break
-			}
-
-			if !s2s.CanTransfer(&dstNode.UserSpace) {
-				failed = true
-				break
-			}
-
-			toShards = append(toShards, dstNode)
-
 		case *ops2.BaseWriteNode:
 			dstStgBld := factory.GetBuilder(&dstNode.UserSpace)
-			if !dstStgBld.FeatureDesc().HasBypassPublicWrite {
+			if !dstStgBld.FeatureDesc().HasBypassBaseWrite {
 				failed = true
 				break
 			}
@@ -77,7 +61,7 @@ loop:
 				break
 			}
 
-			toPublics = append(toPublics, dstNode)
+			toBases = append(toBases, dstNode)
 
 		default:
 			failed = true
@@ -88,50 +72,24 @@ loop:
 		return
 	}
 
-	for _, toShard := range toShards {
-		s2sNode := ctx.DAG.NewS2STransfer(fromShard.UserSpace, toShard.UserSpace, types.S2SOption{})
-		// 直传指令在目的地Hub上执行
-		s2sNode.Env().CopyFrom(toShard.Env())
-
-		// 先获取文件路径，送到S2S节点
-		brNode := ctx.DAG.NewBypassFromShardStore(fromShard.UserSpace, fromShard.FileHash)
-		brNode.Env().CopyFrom(frNode.Env())
-		brNode.FilePathVar().ToSlot(s2sNode.SrcPathSlot())
-
-		// 传输结果通知目的节点
-		bwNode := ctx.DAG.NewBypassToShardStore(toShard.UserSpace, toShard.To.FileHashStoreKey)
-		bwNode.Env().CopyFrom(toShard.Env())
-
-		s2sNode.BypassFileInfoVar().ToSlot(bwNode.BypassFileInfoSlot())
-		bwNode.BypassCallbackVar().ToSlot(s2sNode.BypassCallbackSlot())
-
-		// 从计划中删除目标节点
-		ctx.DAG.RemoveNode(toShard)
-		delete(ctx.ToNodes, toShard.To)
-	}
-
-	for _, toPub := range toPublics {
-		s2sNode := ctx.DAG.NewS2STransfer(fromShard.UserSpace, toPub.UserSpace, types.S2SOption{
-			DestPathHint: toPub.Path,
-		})
+	for _, toBase := range toBases {
+		s2sNode := ctx.DAG.NewS2STransferDyn(fromShard.UserSpace, toBase.UserSpace, toBase.Path)
 		// 直传指令在目的地Hub上执行
-		s2sNode.Env().CopyFrom(toPub.Env())
+		s2sNode.Env().CopyFrom(toBase.Env())
 
 		// 先获取文件路径，送到S2S节点
-		brNode := ctx.DAG.NewBypassFromShardStore(fromShard.UserSpace, fromShard.FileHash)
-		brNode.Env().CopyFrom(toPub.Env())
-		brNode.FilePathVar().ToSlot(s2sNode.SrcPathSlot())
+		gsNode := ctx.DAG.NewGetShardInfo(fromShard.UserSpace, fromShard.FileHash)
+		gsNode.Env().CopyFrom(toBase.Env())
+		gsNode.FileInfoVar().ToSlot(s2sNode.SrcFileInfoSlot())
 
-		// 传输结果通知目的节点
-		bwNode := ctx.DAG.NewBypassToBaseStore(toPub.UserSpace, toPub.Path)
-		bwNode.Env().CopyFrom(toPub.Env())
-
-		s2sNode.BypassFileInfoVar().ToSlot(bwNode.BypassFileInfoSlot())
-		bwNode.BypassCallbackVar().ToSlot(s2sNode.BypassCallbackSlot())
+		// 原本BaseWriteNode的FileInfoVar被替换成S2SNode的FileInfoVar
+		for _, dstSlot := range toBase.FileInfoVar().ListDstSlots() {
+			s2sNode.FileInfoVar().ToSlot(dstSlot)
+		}
 
 		// 从计划中删除目标节点
-		ctx.DAG.RemoveNode(toPub)
-		delete(ctx.ToNodes, toPub.To)
+		ctx.DAG.RemoveNode(toBase)
+		delete(ctx.ToNodes, toBase.To)
 	}
 
 	// 从计划中删除源节点
@@ -139,9 +97,9 @@ loop:
 	delete(ctx.FromNodes, frNode.GetFrom())
 }
 
-func s2sFromBaseStore(ctx *state.GenerateState, fromPub *ioswitch2.FromBaseStore, frNode ops2.FromNode) {
-	fromStgBld := factory.GetBuilder(&fromPub.UserSpace)
-	if !fromStgBld.FeatureDesc().HasBypassPublicRead {
+func s2sFromBaseStore(ctx *state.GenerateState, fromBase *ioswitch2.FromBaseStore, frNode ops2.FromNode) {
+	fromStgBld := factory.GetBuilder(&fromBase.UserSpace)
+	if !fromStgBld.FeatureDesc().HasBypassBaseRead {
 		return
 	}
 
@@ -157,7 +115,7 @@ func s2sFromBaseStore(ctx *state.GenerateState, fromPub *ioswitch2.FromBaseStore
 	}
 
 	failed := false
-	var toPublics []*ops2.BaseWriteNode
+	var toBases []*ops2.BaseWriteNode
 
 loop:
 	for i := 0; i < outVar.Dst.Len(); i++ {
@@ -166,7 +124,7 @@ loop:
 		switch dstNode := dstNode.(type) {
 		case *ops2.BaseWriteNode:
 			dstStgBld := factory.GetBuilder(&dstNode.UserSpace)
-			if !dstStgBld.FeatureDesc().HasBypassPublicWrite {
+			if !dstStgBld.FeatureDesc().HasBypassBaseWrite {
 				failed = true
 				break
 			}
@@ -176,7 +134,7 @@ loop:
 				break
 			}
 
-			toPublics = append(toPublics, dstNode)
+			toBases = append(toBases, dstNode)
 
 		default:
 			failed = true
@@ -187,28 +145,19 @@ loop:
 		return
 	}
 
-	for _, toPub := range toPublics {
-		s2sNode := ctx.DAG.NewS2STransfer(fromPub.UserSpace, toPub.UserSpace, types.S2SOption{
-			DestPathHint: toPub.Path,
-		})
+	for _, toBase := range toBases {
+		s2sNode := ctx.DAG.NewS2STransfer(fromBase.UserSpace, fromBase.Path, toBase.UserSpace, toBase.Path)
 		// 直传指令在目的地Hub上执行
-		s2sNode.Env().CopyFrom(toPub.Env())
-
-		// 先获取文件路径，送到S2S节点
-		brNode := ctx.DAG.NewBypassFromBaseStore(fromPub.UserSpace, fromPub.Path)
-		brNode.Env().CopyFrom(toPub.Env())
-		brNode.FilePathVar().ToSlot(s2sNode.SrcPathSlot())
-
-		// 传输结果通知目的节点
-		bwNode := ctx.DAG.NewBypassToBaseStore(toPub.UserSpace, toPub.Path)
-		bwNode.Env().CopyFrom(toPub.Env())
+		s2sNode.Env().CopyFrom(toBase.Env())
 
-		s2sNode.BypassFileInfoVar().ToSlot(bwNode.BypassFileInfoSlot())
-		bwNode.BypassCallbackVar().ToSlot(s2sNode.BypassCallbackSlot())
+		// 原本BaseWriteNode的FileInfoVar被替换成S2SNode的FileInfoVar
+		for _, dstSlot := range toBase.FileInfoVar().ListDstSlots() {
+			s2sNode.FileInfoVar().ToSlot(dstSlot)
+		}
 
 		// 从计划中删除目标节点
-		ctx.DAG.RemoveNode(toPub)
-		delete(ctx.ToNodes, toPub.To)
+		ctx.DAG.RemoveNode(toBase)
+		delete(ctx.ToNodes, toBase.To)
 	}
 
 	// 从计划中删除源节点

common/pkgs/ioswitch2/parser/parser.go

@@ -79,7 +79,7 @@ func Parse(ft ioswitch2.FromTo, blder *exec.PlanBuilder) error {
 	opt.RemoveUnusedFromNode(state)
 	opt.UseECMultiplier(state)
 	opt.UseS2STransfer(state)
-	opt.UseMultipartUpcopyToShardStore(state)
+	opt.UseMultipartUploadToShardStore(state)
 	opt.DropUnused(state)
 	opt.StoreShardWriteResult(state)
 	opt.GenerateRange(state)

common/pkgs/ioswitch2/plans/complete_multipart.go

@@ -3,6 +3,7 @@ package plans
 import (
 	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
 	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/plan"
+	"gitlink.org.cn/cloudream/common/utils/os2"
 	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
@@ -20,24 +21,34 @@ func CompleteMultipart(blocks []clitypes.ObjectBlock, blockSpaces []clitypes.Use
 	joinNode.Env().Pinned = true
 
 	for i, blk := range blocks {
-		rd := da.NewShardRead(nil, blockSpaces[i], types.NewOpen(blk.FileHash))
-		rd.Env().ToEnvWorker(getWorkerInfo(blockSpaces[i].RecommendHub))
-		rd.Env().Pinned = true
+		gs := da.NewGetShardInfo(blockSpaces[i], blk.FileHash)
+		gs.Env().ToEnvDriver()
+		gs.Env().Pinned = true
 
-		rd.Output().ToSlot(joinNode.InputSlot(i))
+		br := da.NewBaseReadDyn(nil, blockSpaces[i], types.DefaultOpen())
+		br.Env().ToEnvWorker(getWorkerInfo(blockSpaces[i].RecommendHub))
+		br.Env().Pinned = true
+
+		gs.FileInfoVar().ToSlot(br.FileInfoSlot())
+
+		br.Output().ToSlot(joinNode.InputSlot(i))
 	}
 
 	// TODO 应该采取更合理的方式同时支持Parser和直接生成DAG
-	wr := da.NewShardWrite(nil, targetSpace, shardInfoKey)
-	wr.Env().ToEnvWorker(getWorkerInfo(targetSpace.RecommendHub))
-	wr.Env().Pinned = true
+	br := da.NewBaseWrite(nil, targetSpace, os2.GenerateRandomFileName(20))
+	br.Env().ToEnvWorker(getWorkerInfo(targetSpace.RecommendHub))
+	br.Env().Pinned = true
+
+	as := da.NewStoreShard(targetSpace, shardInfoKey)
+	as.Env().ToEnvDriver()
+	as.Env().Pinned = true
 
-	joinNode.Joined().ToSlot(wr.Input())
+	joinNode.Joined().ToSlot(br.Input())
 
 	if shardInfoKey != "" {
 		store := da.NewStore()
 		store.Env().ToEnvDriver()
-		store.Store(shardInfoKey, wr.FileHashVar())
+		store.Store(shardInfoKey, as.ShardInfoVar().Var())
 	}
 
 	err := plan.Compile(da.Graph, blder)

common/pkgs/ioswitchlrc/ops2/base_store.go

@@ -0,0 +1,400 @@
+package ops2
+
+import (
+	"fmt"
+	"io"
+
+	"gitlink.org.cn/cloudream/common/pkgs/future"
+	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
+	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
+	"gitlink.org.cn/cloudream/common/pkgs/logger"
+	"gitlink.org.cn/cloudream/common/utils/io2"
+	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
+	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
+	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
+	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
+)
+
+func init() {
+	exec.UseOp[*BaseWrite]()
+	exec.UseOp[*BaseRead]()
+}
+
+type BaseRead struct {
+	Output    exec.VarID
+	UserSpace clitypes.UserSpaceDetail
+	Path      string
+	Option    types.OpenOption
+}
+
+func (o *BaseRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
+	logger.
+		WithField("Output", o.Output).
+		WithField("UserSpace", o.UserSpace).
+		WithField("Path", o.Path).
+		Debug("base read")
+	defer logger.Debug("base read end")
+
+	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
+	if err != nil {
+		return fmt.Errorf("getting storage pool: %w", err)
+	}
+
+	store, err := stgPool.GetBaseStore(&o.UserSpace)
+	if err != nil {
+		return fmt.Errorf("getting base store of storage %v: %w", o.UserSpace, err)
+	}
+
+	stream, err := store.Read(o.Path, o.Option)
+	if err != nil {
+		return fmt.Errorf("reading object %v: %w", o.Path, err)
+	}
+
+	fut := future.NewSetVoid()
+	output := &exec.StreamValue{
+		Stream: io2.AfterReadClosed(stream, func(closer io.ReadCloser) {
+			fut.SetVoid()
+		}),
+	}
+
+	e.PutVar(o.Output, output)
+	return fut.Wait(ctx.Context)
+}
+
+func (o *BaseRead) String() string {
+	return fmt.Sprintf("PublicRead %v:%v -> %v", o.UserSpace, o.Path, o.Output)
+}
+
+type BaseReadPathVar struct {
+	UserSpace clitypes.UserSpaceDetail
+	Output    exec.VarID
+	Path      exec.VarID
+	Option    types.OpenOption
+}
+
+func (o *BaseReadPathVar) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
+	logger.
+		WithField("Output", o.Output).
+		WithField("UserSpace", o.UserSpace).
+		WithField("Path", o.Path).
+		Debug("base read")
+	defer logger.Debug("base read end")
+
+	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
+	if err != nil {
+		return fmt.Errorf("getting storage pool: %w", err)
+	}
+
+	path, err := exec.BindVar[*FileInfoValue](e, ctx.Context, o.Path)
+	if err != nil {
+		return err
+	}
+
+	store, err := stgPool.GetBaseStore(&o.UserSpace)
+	if err != nil {
+		return fmt.Errorf("getting base store of storage %v: %w", o.UserSpace, err)
+	}
+
+	stream, err := store.Read(path.Path, o.Option)
+	if err != nil {
+		return fmt.Errorf("reading object %v: %w", o.Path, err)
+	}
+
+	fut := future.NewSetVoid()
+	output := &exec.StreamValue{
+		Stream: io2.AfterReadClosed(stream, func(closer io.ReadCloser) {
+			fut.SetVoid()
+		}),
+	}
+
+	e.PutVar(o.Output, output)
+	return fut.Wait(ctx.Context)
+}
+
+func (o *BaseReadPathVar) String() string {
+	return fmt.Sprintf("BaseReadPathVar %v:%v -> %v", o.UserSpace, o.Path, o.Output)
+}
+
+type BaseWrite struct {
+	Input       exec.VarID
+	UserSpace   clitypes.UserSpaceDetail
+	Path        string
+	WriteResult exec.VarID
+}
+
+func (o *BaseWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
+	logger.
+		WithField("Input", o.Input).
+		Debugf("write file to base store")
+	defer logger.Debugf("write file to base store finished")
+
+	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
+	if err != nil {
+		return fmt.Errorf("getting storage pool: %w", err)
+	}
+
+	store, err := stgPool.GetBaseStore(&o.UserSpace)
+	if err != nil {
+		return fmt.Errorf("getting base store of storage %v: %w", o.UserSpace, err)
+	}
+
+	input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
+	if err != nil {
+		return err
+	}
+	defer input.Stream.Close()
+
+	ret, err := store.Write(o.Path, input.Stream)
+	if err != nil {
+		return err
+	}
+
+	e.PutVar(o.WriteResult, &FileInfoValue{
+		FileInfo: ret,
+	})
+	return nil
+}
+
+func (o *BaseWrite) String() string {
+	return fmt.Sprintf("PublicWrite %v -> %v:%v", o.Input, o.UserSpace, o.Path)
+}
+
+// type BaseWriteTemp struct {
+// 	Input       exec.VarID
+// 	UserSpace   clitypes.UserSpaceDetail
+// 	Path        string
+// 	Signal      exec.VarID
+// 	WriteResult exec.VarID
+// }
+
+// func (o *BaseWriteTemp) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
+// 	logger.
+// 		WithField("Input", o.Input).
+// 		Debugf("write file to base store")
+// 	defer logger.Debugf("write file to base store finished")
+
+// 	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
+// 	if err != nil {
+// 		return fmt.Errorf("getting storage pool: %w", err)
+// 	}
+
+// 	_, err = exec.BindVar[*exec.SignalValue](e, ctx.Context, o.Signal)
+// 	if err != nil {
+// 		return err
+// 	}
+
+// 	store, err := stgPool.GetBaseStore(&o.UserSpace)
+// 	if err != nil {
+// 		return fmt.Errorf("getting base store of storage %v: %w", o.UserSpace, err)
+// 	}
+
+// 	input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
+// 	if err != nil {
+// 		return err
+// 	}
+// 	defer input.Stream.Close()
+
+// 	ret, err := store.Write(o.Path, input.Stream)
+// 	if err != nil {
+// 		return err
+// 	}
+
+// 	e.PutVar(o.WriteResult, &WriteResultValue{
+// 		WriteResult: ret,
+// 	})
+// 	return nil
+// }
+
+// func (o *BaseWriteTemp) String() string {
+// 	return fmt.Sprintf("PublicWriteTemp[signal: %v] %v -> %v:%v", o.Signal, o.Input, o.UserSpace, o.Path)
+// }
+
+type BaseReadNode struct {
+	dag.NodeBase
+	From      ioswitchlrc.From
+	UserSpace clitypes.UserSpaceDetail
+	Path      string
+	Option    types.OpenOption
+}
+
+func (b *GraphNodeBuilder) NewBaseRead(from ioswitchlrc.From, userSpace clitypes.UserSpaceDetail, path string, opt types.OpenOption) *BaseReadNode {
+	node := &BaseReadNode{
+		From:      from,
+		UserSpace: userSpace,
+		Path:      path,
+		Option:    opt,
+	}
+	b.AddNode(node)
+
+	node.OutputStreams().Init(node, 1)
+	return node
+}
+
+func (t *BaseReadNode) GetFrom() ioswitchlrc.From {
+	return t.From
+}
+
+func (t *BaseReadNode) Output() dag.StreamOutputSlot {
+	return dag.StreamOutputSlot{
+		Node:  t,
+		Index: 0,
+	}
+}
+
+func (t *BaseReadNode) GenerateOp() (exec.Op, error) {
+	return &BaseRead{
+		Output:    t.Output().Var().VarID,
+		UserSpace: t.UserSpace,
+		Path:      t.Path,
+		Option:    t.Option,
+	}, nil
+}
+
+type BaseReadPathVarNode struct {
+	dag.NodeBase
+	From      ioswitchlrc.From
+	UserSpace clitypes.UserSpaceDetail
+	Option    types.OpenOption
+}
+
+func (b *GraphNodeBuilder) NewBaseReadPathVar(from ioswitchlrc.From, userSpace clitypes.UserSpaceDetail, opt types.OpenOption) *BaseReadPathVarNode {
+	node := &BaseReadPathVarNode{
+		From:      from,
+		UserSpace: userSpace,
+		Option:    opt,
+	}
+	b.AddNode(node)
+
+	node.OutputStreams().Init(node, 1)
+	node.InputValues().Init(1)
+	return node
+}
+
+func (t *BaseReadPathVarNode) GetFrom() ioswitchlrc.From {
+	return t.From
+}
+
+func (t *BaseReadPathVarNode) Output() dag.StreamOutputSlot {
+	return dag.StreamOutputSlot{
+		Node:  t,
+		Index: 0,
+	}
+}
+
+func (t *BaseReadPathVarNode) PathVar() dag.ValueInputSlot {
+	return dag.ValueInputSlot{
+		Node:  t,
+		Index: 0,
+	}
+}
+
+func (t *BaseReadPathVarNode) GenerateOp() (exec.Op, error) {
+	return &BaseReadPathVar{
+		UserSpace: t.UserSpace,
+		Output:    t.Output().Var().VarID,
+		Path:      t.PathVar().Var().VarID,
+		Option:    t.Option,
+	}, nil
+}
+
+type BaseWriteNode struct {
+	dag.NodeBase
+	To        ioswitchlrc.To
+	UserSpace clitypes.UserSpaceDetail
+	Path      string
+}
+
+func (b *GraphNodeBuilder) NewBaseWrite(to ioswitchlrc.To, userSpace clitypes.UserSpaceDetail, path string) *BaseWriteNode {
+	node := &BaseWriteNode{
+		To:        to,
+		UserSpace: userSpace,
+		Path:      path,
+	}
+	b.AddNode(node)
+
+	node.InputStreams().Init(1)
+	node.OutputValues().Init(node, 1)
+	return node
+}
+
+func (t *BaseWriteNode) GetTo() ioswitchlrc.To {
+	return t.To
+}
+
+func (t *BaseWriteNode) Input() dag.StreamInputSlot {
+	return dag.StreamInputSlot{
+		Node:  t,
+		Index: 0,
+	}
+}
+
+func (t *BaseWriteNode) FileInfoVar() dag.ValueOutputSlot {
+	return dag.ValueOutputSlot{
+		Node:  t,
+		Index: 0,
+	}
+}
+
+func (t *BaseWriteNode) GenerateOp() (exec.Op, error) {
+	return &BaseWrite{
+		Input:       t.InputStreams().Get(0).VarID,
+		UserSpace:   t.UserSpace,
+		Path:        t.Path,
+		WriteResult: t.FileInfoVar().Var().VarID,
+	}, nil
+}
+
+// type BaseWriteTempNode struct {
+// 	dag.NodeBase
+// 	To        ioswitchlrc.To
+// 	UserSpace clitypes.UserSpaceDetail
+// 	Path      string
+// }
+
+// func (b *GraphNodeBuilder) NewBaseWriteTemp(to ioswitchlrc.To, userSpace clitypes.UserSpaceDetail, path string) *BaseWriteTempNode {
+// 	node := &BaseWriteTempNode{
+// 		To:        to,
+// 		UserSpace: userSpace,
+// 		Path:      path,
+// 	}
+// 	b.AddNode(node)
+
+// 	node.InputStreams().Init(1)
+// 	node.InputValues().Init(1)
+// 	return node
+// }
+
+// func (t *BaseWriteTempNode) GetTo() ioswitchlrc.To {
+// 	return t.To
+// }
+
+// func (t *BaseWriteTempNode) Input() dag.StreamInputSlot {
+// 	return dag.StreamInputSlot{
+// 		Node:  t,
+// 		Index: 0,
+// 	}
+// }
+
+// func (t *BaseWriteTempNode) TargetSignal() dag.ValueInputSlot {
+// 	return dag.ValueInputSlot{
+// 		Node:  t,
+// 		Index: 0,
+// 	}
+// }
+
+// func (t *BaseWriteTempNode) FileInfoVar() dag.ValueOutputSlot {
+// 	return dag.ValueOutputSlot{
+// 		Node:  t,
+// 		Index: 0,
+// 	}
+// }
+
+// func (t *BaseWriteTempNode) GenerateOp() (exec.Op, error) {
+// 	return &BaseWriteTemp{
+// 		Input:       t.Input().Var().VarID,
+// 		UserSpace:   t.UserSpace,
+// 		Path:        t.Path,
+// 		Signal:      t.TargetSignal().Var().VarID,
+// 		WriteResult: t.FileInfoVar().Var().VarID,
+// 	}, nil
+// }

common/pkgs/ioswitchlrc/ops2/ops.go

@@ -2,7 +2,9 @@ package ops2
 
 import (
 	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
+	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
 	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/plan/ops"
+	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
 )
 
 type GraphNodeBuilder struct {
@@ -21,73 +23,12 @@ type FromNode interface {
 type ToNode interface {
 	dag.Node
 	Input() dag.StreamInputSlot
-	SetInput(input *dag.StreamVar)
 }
 
-// func formatStreamIO(node *dag.Node) string {
-// 	is := ""
-// 	for i, in := range node.InputStreams {
-// 		if i > 0 {
-// 			is += ","
-// 		}
-
-// 		if in == nil {
-// 			is += "."
-// 		} else {
-// 			is += fmt.Sprintf("%v", in.ID)
-// 		}
-// 	}
-
-// 	os := ""
-// 	for i, out := range node.OutputStreams {
-// 		if i > ops
-// 			os += ","
-// 		}
-
-// 		if out == nil {
-// 			os += "."
-// 		} else {
-// 			os += fmt.Sprintf("%v", out.ID)
-// 		}
-// 	}
-
-// 	if is == "" && os == "" {
-// 		return ""
-// 	}
-
-// 	return fmt.Sprintf("S{%s>%s}", is, os)
-// }
-
-// func formatValueIO(node *dag.Node) string {
-// 	is := ""
-// 	for i, in := range node.InputValues {
-// 		if i > 0 {
-// 			is += ","
-// 		}
-
-// 		if in == nil {
-// 			is += "."
-// 		} else {
-// 			is += fmt.Sprintf("%v", in.ID)
-// 		}
-// 	}
-
-// 	os := ""
-// 	for i, out := range node.OutputValues {
-// 		if i > 0 {
-// 			os += ","
-// 		}
-
-// 		if out == nil {
-// 			os += "."
-// 		} else {
-// 			os += fmt.Sprintf("%v", out.ID)
-// 		}
-// 	}
-
-// 	if is == "" && os == "" {
-// 		return ""
-// 	}
+type FileInfoValue struct {
+	types.FileInfo
+}
 
-// 	return fmt.Sprintf("V{%s>%s}", is, os)
-// }
+func (v *FileInfoValue) Clone() exec.VarValue {
+	return &FileInfoValue{v.FileInfo}
+}

common/pkgs/ioswitchlrc/ops2/shard_store.go

@@ -2,88 +2,56 @@ package ops2
 
 import (
 	"fmt"
-	"io"
 
-	"gitlink.org.cn/cloudream/common/pkgs/future"
 	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
 	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/exec"
-	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	"gitlink.org.cn/cloudream/common/utils/io2"
 	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
-	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
-	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
 )
 
 func init() {
-	exec.UseOp[*ShardRead]()
-	exec.UseOp[*ShardWrite]()
-	exec.UseVarValue[*ShardInfoValue]()
+	exec.UseOp[*GetShardInfo]()
+	exec.UseOp[*StoreShard]()
 }
 
-type ShardInfoValue struct {
-	Hash clitypes.FileHash `json:"hash"`
-	Size int64             `json:"size"`
-}
-
-func (v *ShardInfoValue) Clone() exec.VarValue {
-	return &ShardInfoValue{Hash: v.Hash, Size: v.Size}
-}
-
-type ShardRead struct {
-	Output    exec.VarID
+type GetShardInfo struct {
 	UserSpace clitypes.UserSpaceDetail
-	Open      types.OpenOption
+	FileHash  clitypes.FileHash
+	ShardInfo exec.VarID
 }
 
-func (o *ShardRead) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
-	logger.
-		WithField("Open", o.Open.String()).
-		Debugf("reading from shard store")
-	defer logger.Debugf("reading from shard store finished")
-
+func (o *GetShardInfo) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
 	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
 	if err != nil {
-		return fmt.Errorf("getting storage pool: %w", err)
+		return fmt.Errorf("getting shard store: %w", err)
 	}
 
 	store, err := stgPool.GetShardStore(&o.UserSpace)
 	if err != nil {
-		return fmt.Errorf("getting shard store of user space %v: %w", o.UserSpace, err)
+		return fmt.Errorf("getting shard store: %w", err)
 	}
 
-	file, err := store.Open(o.Open)
+	info, err := store.Info(o.FileHash)
 	if err != nil {
-		return fmt.Errorf("opening shard store file: %w", err)
+		return fmt.Errorf("getting shard info: %w", err)
 	}
-
-	fut := future.NewSetVoid()
-	e.PutVar(o.Output, &exec.StreamValue{
-		Stream: io2.AfterReadClosedOnce(file, func(closer io.ReadCloser) {
-			fut.SetVoid()
-		}),
+	e.PutVar(o.ShardInfo, &FileInfoValue{
+		FileInfo: info,
 	})
-
-	return fut.Wait(ctx.Context)
+	return nil
 }
 
-func (o *ShardRead) String() string {
-	return fmt.Sprintf("ShardRead %v -> %v", o.Open.String(), o.Output)
+func (o *GetShardInfo) String() string {
+	return fmt.Sprintf("GetShardInfo(%v)", o.FileHash)
 }
 
-type ShardWrite struct {
-	Input       exec.VarID
-	FileHashVar exec.VarID
-	UserSpace   clitypes.UserSpaceDetail
+type StoreShard struct {
+	UserSpace clitypes.UserSpaceDetail
+	FileInfo  exec.VarID
+	ShardInfo exec.VarID
 }
 
-func (o *ShardWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
-	logger.
-		WithField("Input", o.Input).
-		WithField("FileHash", o.FileHashVar).
-		Debugf("writting file to shard store")
-	defer logger.Debugf("write to shard store finished")
-
+func (o *StoreShard) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
 	stgPool, err := exec.GetValueByType[*pool.Pool](ctx)
 	if err != nil {
 		return fmt.Errorf("getting storage pool: %w", err)
@@ -91,120 +59,97 @@ func (o *ShardWrite) Execute(ctx *exec.ExecContext, e *exec.Executor) error {
 
 	store, err := stgPool.GetShardStore(&o.UserSpace)
 	if err != nil {
-		return fmt.Errorf("getting shard store of user space %v: %w", o.UserSpace, err)
+		return fmt.Errorf("getting shard store: %w", err)
 	}
 
-	input, err := exec.BindVar[*exec.StreamValue](e, ctx.Context, o.Input)
+	info, err := exec.BindVar[*FileInfoValue](e, ctx.Context, o.FileInfo)
 	if err != nil {
 		return err
 	}
-	defer input.Stream.Close()
 
-	fileInfo, err := store.Create(input.Stream)
+	stored, err := store.Store(info.Path, info.Hash, info.Size)
 	if err != nil {
-		return fmt.Errorf("writing file to shard store: %w", err)
+		return fmt.Errorf("adding shard: %w", err)
 	}
 
-	e.PutVar(o.FileHashVar, &ShardInfoValue{
-		Hash: fileInfo.Hash,
-		Size: fileInfo.Size,
+	e.PutVar(o.ShardInfo, &FileInfoValue{
+		FileInfo: stored,
 	})
 	return nil
 }
 
-func (o *ShardWrite) String() string {
-	return fmt.Sprintf("ShardWrite %v -> %v", o.Input, o.FileHashVar)
+func (o *StoreShard) String() string {
+	return fmt.Sprintf("StoreShard: addInfo=%v, shardInfo=%v", o.FileInfo, o.ShardInfo)
 }
 
-type ShardReadNode struct {
+type GetShardInfoNode struct {
 	dag.NodeBase
-	From      *ioswitchlrc.FromNode
 	UserSpace clitypes.UserSpaceDetail
-	Open      types.OpenOption
+	FileHash  clitypes.FileHash
 }
 
-func (b *GraphNodeBuilder) NewShardRead(fr *ioswitchlrc.FromNode, userSpace clitypes.UserSpaceDetail, open types.OpenOption) *ShardReadNode {
-	node := &ShardReadNode{
-		From:      fr,
+func (b *GraphNodeBuilder) NewGetShardInfo(userSpace clitypes.UserSpaceDetail, fileHash clitypes.FileHash) *GetShardInfoNode {
+	node := &GetShardInfoNode{
 		UserSpace: userSpace,
-		Open:      open,
+		FileHash:  fileHash,
 	}
 	b.AddNode(node)
 
-	node.OutputStreams().Init(node, 1)
+	node.OutputValues().Init(node, 1)
 	return node
 }
 
-func (t *ShardReadNode) GetFrom() ioswitchlrc.From {
-	return t.From
-}
-
-func (t *ShardReadNode) Output() dag.StreamOutputSlot {
-	return dag.StreamOutputSlot{
-		Node:  t,
+func (n *GetShardInfoNode) FileInfoVar() dag.ValueOutputSlot {
+	return dag.ValueOutputSlot{
+		Node:  n,
 		Index: 0,
 	}
 }
 
-func (t *ShardReadNode) GenerateOp() (exec.Op, error) {
-	return &ShardRead{
-		Output:    t.OutputStreams().Get(0).VarID,
-		UserSpace: t.UserSpace,
-		Open:      t.Open,
+func (n *GetShardInfoNode) GenerateOp() (exec.Op, error) {
+	return &GetShardInfo{
+		UserSpace: n.UserSpace,
+		FileHash:  n.FileHash,
+		ShardInfo: n.FileInfoVar().Var().VarID,
 	}, nil
 }
 
-// func (t *IPFSReadType) String() string {
-// 	return fmt.Sprintf("IPFSRead[%s,%v+%v]%v%v", t.FileHash, t.Option.Offset, t.Option.Length, formatStreamIO(node), formatValueIO(node))
-// }
-
-type ShardWriteNode struct {
+type StoreShardNode struct {
 	dag.NodeBase
-	To               *ioswitchlrc.ToNode
-	UserSpace        clitypes.UserSpaceDetail
-	FileHashStoreKey string
+	UserSpace    clitypes.UserSpaceDetail
+	ShardInfoKey string
 }
 
-func (b *GraphNodeBuilder) NewShardWrite(to *ioswitchlrc.ToNode, userSpace clitypes.UserSpaceDetail, fileHashStoreKey string) *ShardWriteNode {
-	node := &ShardWriteNode{
-		To:               to,
-		UserSpace:        userSpace,
-		FileHashStoreKey: fileHashStoreKey,
+func (b *GraphNodeBuilder) NewStoreShard(userSpace clitypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
+	node := &StoreShardNode{
+		UserSpace:    userSpace,
+		ShardInfoKey: shardInfoKey,
 	}
 	b.AddNode(node)
 
-	node.InputStreams().Init(1)
+	node.InputValues().Init(1)
 	node.OutputValues().Init(node, 1)
 	return node
 }
 
-func (t *ShardWriteNode) GetTo() ioswitchlrc.To {
-	return t.To
-}
-
-func (t *ShardWriteNode) SetInput(input *dag.StreamVar) {
-	input.To(t, 0)
-}
-
-func (t *ShardWriteNode) Input() dag.StreamInputSlot {
-	return dag.StreamInputSlot{
+func (t *StoreShardNode) FileInfoSlot() dag.ValueInputSlot {
+	return dag.ValueInputSlot{
 		Node:  t,
 		Index: 0,
 	}
 }
 
-func (t *ShardWriteNode) FileHashVar() *dag.ValueVar {
-	return t.OutputValues().Get(0)
+func (t *StoreShardNode) ShardInfoVar() dag.ValueOutputSlot {
+	return dag.ValueOutputSlot{
+		Node:  t,
+		Index: 0,
+	}
 }
 
-func (t *ShardWriteNode) GenerateOp() (exec.Op, error) {
-	return &ShardWrite{
-		Input:       t.InputStreams().Get(0).VarID,
-		FileHashVar: t.OutputValues().Get(0).VarID,
-		UserSpace:   t.UserSpace,
+func (t *StoreShardNode) GenerateOp() (exec.Op, error) {
+	return &StoreShard{
+		UserSpace: t.UserSpace,
+		FileInfo:  t.FileInfoSlot().Var().VarID,
+		ShardInfo: t.ShardInfoVar().Var().VarID,
 	}, nil
 }
-
-// func (t *IPFSWriteType) String() string {
-// 	return fmt.Sprintf("IPFSWrite[%s,%v+%v]%v%v", t.FileHashStoreKey, t.Range.Offset, t.Range.Length, formatStreamIO(node), formatValueIO(node))
-// }

common/pkgs/ioswitchlrc/parser/generator.go

@@ -71,7 +71,7 @@ func buildDAGEncode(ctx *GenerateContext, fr ioswitchlrc.From, toes []ioswitchlr
 			}
 			ctx.ToNodes[to] = toNode
 
-			toNode.SetInput(frNode.Output().Var())
+			frNode.Output().ToSlot(toNode.Input())
 		} else if idx < ctx.LRC.K {
 			dataToes = append(dataToes, to)
 		} else {
@@ -94,7 +94,7 @@ func buildDAGEncode(ctx *GenerateContext, fr ioswitchlrc.From, toes []ioswitchlr
 		}
 		ctx.ToNodes[to] = toNode
 
-		toNode.SetInput(splitNode.SubStream(to.GetDataIndex()))
+		splitNode.SubStream(to.GetDataIndex()).ToSlot(toNode.Input())
 	}
 
 	if len(parityToes) == 0 {
@@ -116,7 +116,7 @@ func buildDAGEncode(ctx *GenerateContext, fr ioswitchlrc.From, toes []ioswitchlr
 		}
 		ctx.ToNodes[to] = toNode
 
-		toNode.SetInput(conType.NewOutput(to.GetDataIndex()))
+		conType.NewOutput(to.GetDataIndex()).ToSlot(toNode.Input())
 	}
 	return nil
 }
@@ -174,7 +174,7 @@ func buildDAGReconstructAny(ctx *GenerateContext, frs []ioswitchlrc.From, toes [
 			}
 			ctx.ToNodes[to] = toNode
 
-			toNode.SetInput(fr.Output().Var())
+			fr.Output().ToSlot(toNode.Input())
 			continue
 		}
 
@@ -203,7 +203,7 @@ func buildDAGReconstructAny(ctx *GenerateContext, frs []ioswitchlrc.From, toes [
 		}
 		ctx.ToNodes[to] = toNode
 
-		toNode.SetInput(conNode.NewOutput(to.GetDataIndex()))
+		conNode.NewOutput(to.GetDataIndex()).ToSlot(toNode.Input())
 	}
 
 	if len(completeToes) == 0 {
@@ -230,7 +230,7 @@ func buildDAGReconstructAny(ctx *GenerateContext, frs []ioswitchlrc.From, toes [
 		}
 		ctx.ToNodes[to] = toNode
 
-		toNode.SetInput(joinNode.Joined())
+		joinNode.Joined().ToSlot(toNode.Input())
 	}
 
 	// 如果不需要Construct任何块，则删除这个节点
@@ -293,7 +293,7 @@ func buildDAGReconstructGroup(ctx *GenerateContext, frs []ioswitchlrc.From, toes
 		}
 		ctx.ToNodes[to] = toNode
 
-		toNode.SetInput(missedBlk)
+		missedBlk.ToSlot(toNode.Input())
 	}
 
 	return nil

common/pkgs/ioswitchlrc/parser/passes.go

@@ -6,6 +6,7 @@ import (
 
 	"gitlink.org.cn/cloudream/common/pkgs/ioswitch/dag"
 	"gitlink.org.cn/cloudream/common/utils/math2"
+	"gitlink.org.cn/cloudream/common/utils/os2"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/ops2"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
@@ -63,19 +64,23 @@ func buildFromNode(ctx *GenerateContext, f ioswitchlrc.From) (ops2.FromNode, err
 
 	switch f := f.(type) {
 	case *ioswitchlrc.FromNode:
-		t := ctx.DAG.NewShardRead(f, f.Space, types.NewOpen(f.FileHash))
+		getShard := ctx.DAG.NewGetShardInfo(f.Space, f.FileHash)
+		getShard.Env().ToEnvDriver()
+		getShard.Env().Pinned = true
+
+		read := ctx.DAG.NewBaseReadPathVar(f, f.Space, types.DefaultOpen())
 
 		if f.DataIndex == -1 {
-			t.Open.WithNullableLength(repRange.Offset, repRange.Length)
+			read.Option.WithNullableLength(repRange.Offset, repRange.Length)
 		} else {
-			t.Open.WithNullableLength(blkRange.Offset, blkRange.Length)
+			read.Option.WithNullableLength(blkRange.Offset, blkRange.Length)
 		}
 
 		// TODO2 支持HTTP协议
-		t.Env().ToEnvWorker(&ioswitchlrc.HubWorker{Hub: f.Space.RecommendHub, Address: *f.Space.RecommendHub.Address.(*cortypes.GRPCAddressInfo)})
-		t.Env().Pinned = true
+		getShard.Env().ToEnvWorker(&ioswitchlrc.HubWorker{Hub: f.Space.RecommendHub, Address: *f.Space.RecommendHub.Address.(*cortypes.GRPCAddressInfo)})
+		getShard.Env().Pinned = true
 
-		return t, nil
+		return read, nil
 
 	case *ioswitchlrc.FromDriver:
 		n := ctx.DAG.NewFromDriver(f.Handle)
@@ -100,21 +105,29 @@ func buildFromNode(ctx *GenerateContext, f ioswitchlrc.From) (ops2.FromNode, err
 func buildToNode(ctx *GenerateContext, t ioswitchlrc.To) (ops2.ToNode, error) {
 	switch t := t.(type) {
 	case *ioswitchlrc.ToNode:
-		n := ctx.DAG.NewShardWrite(t, t.Space, t.FileHashStoreKey)
+		tempFileName := os2.GenerateRandomFileName(20)
+		write := ctx.DAG.NewBaseWrite(t, t.Space, tempFileName)
+
 		switch addr := t.Space.RecommendHub.Address.(type) {
 		// case *cdssdk.HttpAddressInfo:
 		// n.Env().ToEnvWorker(&ioswitchlrc.HttpHubWorker{Node: t.Hub})
 		// TODO2 支持HTTP协议
 		case *cortypes.GRPCAddressInfo:
-			n.Env().ToEnvWorker(&ioswitchlrc.HubWorker{Hub: t.Space.RecommendHub, Address: *addr})
+			write.Env().ToEnvWorker(&ioswitchlrc.HubWorker{Hub: t.Space.RecommendHub, Address: *addr})
 
 		default:
 			return nil, fmt.Errorf("unsupported node address type %T", addr)
 		}
 
-		n.Env().Pinned = true
+		write.Env().Pinned = true
 
-		return n, nil
+		add := ctx.DAG.NewStoreShard(t.Space, t.FileHashStoreKey)
+		add.Env().ToEnvDriver()
+		add.Env().Pinned = true
+
+		write.FileInfoVar().ToSlot(add.FileInfoSlot())
+
+		return write, nil
 
 	case *ioswitchlrc.ToDriver:
 		n := ctx.DAG.NewToDriver(t.Handle)
@@ -207,15 +220,15 @@ func dropUnused(ctx *GenerateContext) {
 
 // 为IPFS写入指令存储结果
 func storeIPFSWriteResult(ctx *GenerateContext) {
-	dag.WalkOnlyType[*ops2.ShardWriteNode](ctx.DAG.Graph, func(n *ops2.ShardWriteNode) bool {
-		if n.FileHashStoreKey == "" {
+	dag.WalkOnlyType[*ops2.StoreShardNode](ctx.DAG.Graph, func(n *ops2.StoreShardNode) bool {
+		if n.ShardInfoKey == "" {
 			return true
 		}
 
 		storeNode := ctx.DAG.NewStore()
 		storeNode.Env().ToEnvDriver()
 
-		storeNode.Store(n.FileHashStoreKey, n.FileHashVar())
+		storeNode.Store(n.ShardInfoKey, n.ShardInfoVar().Var())
 		return true
 	})
 }
@@ -238,7 +251,7 @@ func generateRange(ctx *GenerateContext) {
 				Length: toRng.Length,
 			})
 			toInput.Var().NotTo(toNode)
-			toNode.SetInput(rnged)
+			rnged.ToSlot(toInput)
 
 		} else {
 			stripSize := int64(ctx.LRC.ChunkSize * ctx.LRC.K)
@@ -254,7 +267,7 @@ func generateRange(ctx *GenerateContext) {
 				Length: toRng.Length,
 			})
 			toInput.Var().NotTo(toNode)
-			toNode.SetInput(rnged)
+			rnged.ToSlot(toInput)
 		}
 	}
 }

common/pkgs/rpc/hub/user_space.go

@@ -19,7 +19,7 @@ type BaseStoreListAll struct {
 	Path      string
 }
 type BaseStoreListAllResp struct {
-	Entries []stgtypes.BaseStoreEntry
+	Entries []stgtypes.ListEntry
 }
 
 func (c *Client) BaseStoreListAll(ctx context.Context, req *BaseStoreListAll) (*BaseStoreListAllResp, *rpc.CodeError) {

common/pkgs/storage/efile/ec_multiplier.go

@@ -15,16 +15,15 @@ import (
 )
 
 type ECMultiplier struct {
-	blder     *builder
-	url       string
-	feat      *cortypes.ECMultiplierFeature
-	outputs   []string
-	completed bool
+	blder   *builder
+	url     string
+	feat    *cortypes.ECMultiplierFeature
+	outputs []string
 }
 
 // 进行EC运算，coef * inputs。coef为编码矩阵，inputs为待编码数据，chunkSize为分块大小。
 // 输出为每一个块文件的路径，数组长度 = len(coef)
-func (m *ECMultiplier) Multiply(coef [][]byte, inputs []types.HTTPRequest, chunkSize int) ([]types.BypassedFileInfo, error) {
+func (m *ECMultiplier) Multiply(coef [][]byte, inputs []types.HTTPRequest, chunkSize int) ([]types.FileInfo, error) {
 	type Request struct {
 		Inputs    []types.HTTPRequest `json:"inputs"`
 		Outputs   []string            `json:"outputs"`
@@ -94,9 +93,9 @@ func (m *ECMultiplier) Multiply(coef [][]byte, inputs []types.HTTPRequest, chunk
 		return nil, fmt.Errorf("data length not match outputs length")
 	}
 
-	ret := make([]types.BypassedFileInfo, len(r.Data))
+	ret := make([]types.FileInfo, len(r.Data))
 	for i, data := range r.Data {
-		ret[i] = types.BypassedFileInfo{
+		ret[i] = types.FileInfo{
 			Path: m.outputs[i],
 			Size: data.Size,
 			Hash: clitypes.NewFullHashFromString(data.Sha256),
@@ -106,29 +105,4 @@ func (m *ECMultiplier) Multiply(coef [][]byte, inputs []types.HTTPRequest, chunk
 	return ret, nil
 }
 
-// 完成计算
-func (m *ECMultiplier) Complete() {
-	m.completed = true
-}
-
-// 取消计算。如果已经调用了Complete，则应该无任何影响
-func (m *ECMultiplier) Abort() {
-	if !m.completed {
-		u, err := url.JoinPath(m.url, "efile/openapi/v2/file/remove")
-		if err != nil {
-			return
-		}
-
-		token, err := m.blder.getToken()
-		if err != nil {
-			return
-		}
-
-		for _, output := range m.outputs {
-			http2.PostJSON(u, http2.RequestParam{
-				Header: map[string]string{"token": token},
-				Query:  map[string]string{"paths": output},
-			})
-		}
-	}
-}
+func (m *ECMultiplier) Close() {}

common/pkgs/storage/local/base_store.go

@@ -48,19 +48,34 @@ func (s *BaseStore) Write(objPath string, stream io.Reader) (types.FileInfo, err
 	}
 
 	return types.FileInfo{
-		Hash: clitypes.NewFullHash(hasher.Sum()),
+		Path: objPath,
 		Size: counter.Count(),
+		Hash: clitypes.NewFullHash(hasher.Sum()),
 	}, nil
 }
 
-func (s *BaseStore) Read(objPath string) (io.ReadCloser, error) {
+func (s *BaseStore) Read(objPath string, opt types.OpenOption) (io.ReadCloser, error) {
 	absObjPath := filepath.Join(s.root, objPath)
-	f, err := os.Open(absObjPath)
+	file, err := os.Open(absObjPath)
 	if err != nil {
 		return nil, err
 	}
 
-	return f, nil
+	if opt.Offset > 0 {
+		_, err = file.Seek(opt.Offset, io.SeekStart)
+		if err != nil {
+			file.Close()
+			return nil, err
+		}
+	}
+
+	var ret io.ReadCloser = file
+
+	if opt.Length >= 0 {
+		ret = io2.Length(ret, opt.Length)
+	}
+
+	return ret, nil
 }
 
 func (s *BaseStore) Mkdir(path string) error {
@@ -73,10 +88,10 @@ func (s *BaseStore) Mkdir(path string) error {
 	return nil
 }
 
-func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
+func (s *BaseStore) ListAll(path string) ([]types.ListEntry, error) {
 	absObjPath := filepath.Join(s.root, path)
 
-	var es []types.BaseStoreEntry
+	var es []types.ListEntry
 	err := filepath.WalkDir(absObjPath, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
@@ -88,7 +103,7 @@ func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
 		}
 
 		if d.IsDir() {
-			es = append(es, types.BaseStoreEntry{
+			es = append(es, types.ListEntry{
 				Path:  filepath.ToSlash(relaPath),
 				Size:  0,
 				IsDir: true,
@@ -100,7 +115,7 @@ func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
 			return err
 		}
 
-		es = append(es, types.BaseStoreEntry{
+		es = append(es, types.ListEntry{
 			Path:  filepath.ToSlash(relaPath),
 			Size:  info.Size(),
 			IsDir: false,
@@ -117,6 +132,36 @@ func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
 	return es, nil
 }
 
+func (s *BaseStore) CleanTemps() {
+	log := s.getLogger()
+
+	entries, err := os.ReadDir(filepath.Join(s.root, TempDir))
+	if err != nil {
+		log.Warnf("read temp dir: %v", err)
+		return
+	}
+
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+
+		info, err := entry.Info()
+		if err != nil {
+			log.Warnf("get temp file %v info: %v", entry.Name(), err)
+			continue
+		}
+
+		path := filepath.Join(s.root, TempDir, entry.Name())
+		err = os.Remove(path)
+		if err != nil {
+			log.Warnf("remove temp file %v: %v", path, err)
+		} else {
+			log.Infof("remove unused temp file %v, size: %v, last mod time: %v", path, info.Size(), info.ModTime())
+		}
+	}
+}
+
 func (s *BaseStore) getLogger() logger.Logger {
 	return logger.WithField("BaseStore", "Local").WithField("UserSpace", s.detail.UserSpace)
 }

common/pkgs/storage/local/multipart_upload.go

@@ -86,14 +86,14 @@ func (i *MultipartTask) InitState() types.MultipartInitState {
 	}
 }
 
-func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPartInfo) (types.BypassedFileInfo, error) {
+func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPartInfo) (types.FileInfo, error) {
 	parts = sort2.Sort(parts, func(l, r types.UploadedPartInfo) int {
 		return l.PartNumber - r.PartNumber
 	})
 
 	joined, err := os.Create(i.joinedFilePath)
 	if err != nil {
-		return types.BypassedFileInfo{}, err
+		return types.FileInfo{}, err
 	}
 	defer joined.Close()
 
@@ -103,14 +103,14 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
 	for _, part := range parts {
 		partSize, err := i.writePart(part, joined, hasher)
 		if err != nil {
-			return types.BypassedFileInfo{}, err
+			return types.FileInfo{}, err
 		}
 
 		size += partSize
 	}
 
 	h := hasher.Sum(nil)
-	return types.BypassedFileInfo{
+	return types.FileInfo{
 		Path: joined.Name(),
 		Size: size,
 		Hash: clitypes.NewFullHash(h),
@@ -147,11 +147,6 @@ func (i *MultipartTask) writePart(partInfo types.UploadedPartInfo, joined *os.Fi
 	return size, nil
 }
 
-func (i *MultipartTask) Complete() {
-	i.Abort()
-}
-
-func (i *MultipartTask) Abort() {
-	os.Remove(i.joinedFilePath)
+func (i *MultipartTask) Close() {
 	os.RemoveAll(i.tempPartsDir)
 }

common/pkgs/storage/local/s2s.go

@@ -2,12 +2,9 @@ package local
 
 import (
 	"context"
-	"fmt"
 	"io"
 	"os"
-	"path/filepath"
 
-	"gitlink.org.cn/cloudream/common/utils/os2"
 	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
 	cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
@@ -34,43 +31,33 @@ func (s *S2STransfer) CanTransfer(src *clitypes.UserSpaceDetail) bool {
 }
 
 // 执行数据直传
-func (s *S2STransfer) Transfer(ctx context.Context, src *clitypes.UserSpaceDetail, srcPath string, opt types.S2SOption) (string, error) {
-	absTempDir, err := filepath.Abs(s.feat.TempDir)
-	if err != nil {
-		return "", fmt.Errorf("get abs temp dir %v: %v", s.feat.TempDir, err)
-	}
-
-	s.dstPath = opt.DestPathHint
-	if s.dstPath == "" {
-		s.dstPath = filepath.Join(absTempDir, os2.GenerateRandomFileName(10))
-	}
+func (s *S2STransfer) Transfer(ctx context.Context, src *clitypes.UserSpaceDetail, srcPath string, dstPath string) (types.FileInfo, error) {
+	s.dstPath = dstPath
 
 	copy, err := os.OpenFile(s.dstPath, os.O_WRONLY|os.O_CREATE, 0644)
 	if err != nil {
-		return "", err
+		return types.FileInfo{}, err
 	}
 	defer copy.Close()
 
 	srcFile, err := os.Open(srcPath)
 	if err != nil {
-		return "", err
+		return types.FileInfo{}, err
 	}
 	defer srcFile.Close()
 
-	_, err = io.Copy(copy, srcFile)
+	n, err := io.Copy(copy, srcFile)
 	if err != nil {
-		return "", err
+		return types.FileInfo{}, err
 	}
 
-	return s.dstPath, nil
+	return types.FileInfo{
+		Path: dstPath,
+		Size: n,
+		Hash: "",
+	}, nil
 }
 
-func (s *S2STransfer) Complete() {
+func (s *S2STransfer) Close() {
 
 }
-
-func (s *S2STransfer) Abort() {
-	if s.dstPath != "" {
-		os.Remove(s.dstPath)
-	}
-}

common/pkgs/storage/local/shard_store.go

@@ -1,18 +1,14 @@
 package local
 
 import (
-	"crypto/sha256"
 	"errors"
 	"fmt"
-	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
 	"sync"
-	"time"
 
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	"gitlink.org.cn/cloudream/common/utils/io2"
 	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
 )
@@ -23,11 +19,10 @@ const (
 )
 
 type ShardStore struct {
-	detail           *clitypes.UserSpaceDetail
-	absRoot          string
-	lock             sync.Mutex
-	workingTempFiles map[string]bool
-	done             chan any
+	detail  *clitypes.UserSpaceDetail
+	absRoot string
+	lock    sync.Mutex
+	done    chan any
 }
 
 func NewShardStore(root string, detail *clitypes.UserSpaceDetail) (*ShardStore, error) {
@@ -37,163 +32,31 @@ func NewShardStore(root string, detail *clitypes.UserSpaceDetail) (*ShardStore,
 	}
 
 	return &ShardStore{
-		detail:           detail,
-		absRoot:          absRoot,
-		workingTempFiles: make(map[string]bool),
-		done:             make(chan any, 1),
+		detail:  detail,
+		absRoot: absRoot,
+		done:    make(chan any, 1),
 	}, nil
 }
 
 func (s *ShardStore) Start(ch *types.StorageEventChan) {
 	s.getLogger().Infof("component start, root: %v, max size: %v", s.absRoot, s.detail.UserSpace.ShardStore.MaxSize)
-
-	go func() {
-		removeTempTicker := time.NewTicker(time.Minute * 10)
-		defer removeTempTicker.Stop()
-
-		for {
-			select {
-			case <-removeTempTicker.C:
-				s.removeUnusedTempFiles()
-			case <-s.done:
-				return
-			}
-		}
-	}()
-}
-
-func (s *ShardStore) removeUnusedTempFiles() {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	log := s.getLogger()
-
-	entries, err := os.ReadDir(filepath.Join(s.absRoot, TempDir))
-	if err != nil {
-		log.Warnf("read temp dir: %v", err)
-		return
-	}
-
-	for _, entry := range entries {
-		if entry.IsDir() {
-			continue
-		}
-
-		if s.workingTempFiles[entry.Name()] {
-			continue
-		}
-
-		info, err := entry.Info()
-		if err != nil {
-			log.Warnf("get temp file %v info: %v", entry.Name(), err)
-			continue
-		}
-
-		path := filepath.Join(s.absRoot, TempDir, entry.Name())
-		err = os.Remove(path)
-		if err != nil {
-			log.Warnf("remove temp file %v: %v", path, err)
-		} else {
-			log.Infof("remove unused temp file %v, size: %v, last mod time: %v", path, info.Size(), info.ModTime())
-		}
-	}
 }
 
 func (s *ShardStore) Stop() {
 	s.getLogger().Infof("component stop")
-
-	select {
-	case s.done <- nil:
-	default:
-	}
-}
-
-func (s *ShardStore) Create(stream io.Reader) (types.FileInfo, error) {
-	file, err := s.createTempFile()
-	if err != nil {
-		return types.FileInfo{}, err
-	}
-
-	counter := io2.Counter(stream)
-	size, hash, err := s.writeTempFile(file, counter)
-	// TODO2
-	// if stgglb.Stats.HubStorageTransfer != nil {
-	// 	stgglb.Stats.HubStorageTransfer.RecordUpload(s.detail.Storage.StorageID, counter.Count(), err == nil)
-	// }
-	if err != nil {
-		// Name是文件完整路径
-		s.onCreateFailed(file.Name())
-		return types.FileInfo{}, err
-	}
-
-	return s.onCreateFinished(file.Name(), size, hash)
-}
-
-func (s *ShardStore) createTempFile() (*os.File, error) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	tmpDir := filepath.Join(s.absRoot, TempDir)
-
-	err := os.MkdirAll(tmpDir, 0755)
-	if err != nil {
-		s.lock.Unlock()
-		return nil, err
-	}
-
-	file, err := os.CreateTemp(tmpDir, "tmp-*")
-	if err != nil {
-		s.lock.Unlock()
-		return nil, err
-	}
-
-	s.workingTempFiles[filepath.Base(file.Name())] = true
-
-	return file, nil
-}
-
-func (s *ShardStore) writeTempFile(file *os.File, stream io.Reader) (int64, clitypes.FileHash, error) {
-	defer file.Close()
-
-	buf := make([]byte, 32*1024)
-	size := int64(0)
-
-	hasher := sha256.New()
-	for {
-		n, err := stream.Read(buf)
-		if n > 0 {
-			size += int64(n)
-			io2.WriteAll(hasher, buf[:n])
-			err := io2.WriteAll(file, buf[:n])
-			if err != nil {
-				return 0, "", err
-			}
-		}
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return 0, "", err
-		}
-	}
-
-	h := hasher.Sum(nil)
-	return size, clitypes.NewFullHash(h), nil
 }
 
-func (s *ShardStore) onCreateFinished(tempFilePath string, size int64, hash clitypes.FileHash) (types.FileInfo, error) {
+func (s *ShardStore) Store(path string, hash clitypes.FileHash, size int64) (types.FileInfo, error) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
-	defer delete(s.workingTempFiles, filepath.Base(tempFilePath))
 
 	log := s.getLogger()
 
-	log.Debugf("write file %v finished, size: %v, hash: %v", tempFilePath, size, hash)
+	log.Debugf("%v bypass uploaded, size: %v, hash: %v", path, size, hash)
 
 	blockDir := s.getFileDirFromHash(hash)
 	err := os.MkdirAll(blockDir, 0755)
 	if err != nil {
-		s.removeTempFile(tempFilePath)
 		log.Warnf("make block dir %v: %v", blockDir, err)
 		return types.FileInfo{}, fmt.Errorf("making block dir: %w", err)
 	}
@@ -201,77 +64,24 @@ func (s *ShardStore) onCreateFinished(tempFilePath string, size int64, hash clit
 	newPath := filepath.Join(blockDir, string(hash))
 	_, err = os.Stat(newPath)
 	if os.IsNotExist(err) {
-		err = os.Rename(tempFilePath, newPath)
+		err = os.Rename(path, newPath)
 		if err != nil {
-			s.removeTempFile(tempFilePath)
-			log.Warnf("rename %v to %v: %v", tempFilePath, newPath, err)
+			log.Warnf("rename %v to %v: %v", path, newPath, err)
 			return types.FileInfo{}, fmt.Errorf("rename file: %w", err)
 		}
 
 	} else if err != nil {
-		s.removeTempFile(tempFilePath)
 		log.Warnf("get file %v stat: %v", newPath, err)
 		return types.FileInfo{}, fmt.Errorf("get file stat: %w", err)
-	} else {
-		s.removeTempFile(tempFilePath)
 	}
 
 	return types.FileInfo{
-		Hash:        hash,
-		Size:        size,
-		Description: newPath,
+		Hash: hash,
+		Size: size,
+		Path: newPath,
 	}, nil
 }
 
-func (s *ShardStore) onCreateFailed(tempFilePath string) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	s.getLogger().Debugf("writting file %v aborted", tempFilePath)
-	s.removeTempFile(tempFilePath)
-	delete(s.workingTempFiles, filepath.Base(tempFilePath))
-}
-
-func (s *ShardStore) removeTempFile(path string) {
-	err := os.Remove(path)
-	if err != nil {
-		s.getLogger().Warnf("removing temp file %v: %v", path, err)
-	}
-}
-
-// 使用NewOpen函数创建Option对象
-func (s *ShardStore) Open(opt types.OpenOption) (io.ReadCloser, error) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	filePath := s.getFilePathFromHash(opt.FileHash)
-	file, err := os.Open(filePath)
-	if err != nil {
-		return nil, err
-	}
-
-	if opt.Offset > 0 {
-		_, err = file.Seek(opt.Offset, io.SeekStart)
-		if err != nil {
-			file.Close()
-			return nil, err
-		}
-	}
-
-	var ret io.ReadCloser = file
-
-	if opt.Length >= 0 {
-		ret = io2.Length(ret, opt.Length)
-	}
-
-	return io2.CounterCloser(ret, func(cnt int64, err error) {
-		// TODO2
-		// if stgglb.Stats.HubStorageTransfer != nil {
-		// 	stgglb.Stats.HubStorageTransfer.RecordDownload(s.detail.Storage.StorageID, cnt, err == nil || err == io.EOF)
-		// }
-	}), nil
-}
-
 func (s *ShardStore) Info(hash clitypes.FileHash) (types.FileInfo, error) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
@@ -283,9 +93,9 @@ func (s *ShardStore) Info(hash clitypes.FileHash) (types.FileInfo, error) {
 	}
 
 	return types.FileInfo{
-		Hash:        hash,
-		Size:        info.Size(),
-		Description: filePath,
+		Hash: hash,
+		Size: info.Size(),
+		Path: filePath,
 	}, nil
 }
 
@@ -316,9 +126,9 @@ func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
 		}
 
 		infos = append(infos, types.FileInfo{
-			Hash:        fileHash,
-			Size:        info.Size(),
-			Description: filepath.Join(blockDir, path),
+			Hash: fileHash,
+			Size: info.Size(),
+			Path: filepath.Join(blockDir, path),
 		})
 		return nil
 	})
@@ -399,59 +209,3 @@ func (s *ShardStore) getFileDirFromHash(hash clitypes.FileHash) string {
 func (s *ShardStore) getFilePathFromHash(hash clitypes.FileHash) string {
 	return filepath.Join(s.absRoot, BlocksDir, hash.GetHashPrefix(2), string(hash))
 }
-
-var _ types.BypassShardWrite = (*ShardStore)(nil)
-
-func (s *ShardStore) BypassedShard(info types.BypassedFileInfo) error {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	log := s.getLogger()
-
-	log.Debugf("%v bypass uploaded, size: %v, hash: %v", info.Path, info.Size, info.Hash)
-
-	blockDir := s.getFileDirFromHash(info.Hash)
-	err := os.MkdirAll(blockDir, 0755)
-	if err != nil {
-		log.Warnf("make block dir %v: %v", blockDir, err)
-		return fmt.Errorf("making block dir: %w", err)
-	}
-
-	newPath := filepath.Join(blockDir, string(info.Hash))
-	_, err = os.Stat(newPath)
-	if os.IsNotExist(err) {
-		err = os.Rename(info.Path, newPath)
-		if err != nil {
-			log.Warnf("rename %v to %v: %v", info.Path, newPath, err)
-			return fmt.Errorf("rename file: %w", err)
-		}
-
-	} else if err != nil {
-		log.Warnf("get file %v stat: %v", newPath, err)
-		return fmt.Errorf("get file stat: %w", err)
-	}
-
-	return nil
-}
-
-var _ types.BypassShardRead = (*ShardStore)(nil)
-
-func (s *ShardStore) BypassShardRead(fileHash clitypes.FileHash) (types.BypassFilePath, error) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	filePath := s.getFilePathFromHash(fileHash)
-	stat, err := os.Stat(filePath)
-	if err != nil {
-		return types.BypassFilePath{}, err
-	}
-
-	return types.BypassFilePath{
-		Path: filePath,
-		Info: types.FileInfo{
-			Hash:        fileHash,
-			Size:        stat.Size(),
-			Description: filePath,
-		},
-	}, nil
-}

common/pkgs/storage/obs/obs.go

@@ -31,11 +31,11 @@ func newBuilder(detail *clitypes.UserSpaceDetail) types.StorageBuilder {
 
 func (b *builder) FeatureDesc() types.FeatureDesc {
 	return types.FeatureDesc{
-		HasBypassShardWrite:  true,
-		HasBypassPublicWrite: true,
-		HasBypassShardRead:   true,
-		HasBypassPublicRead:  true,
-		HasBypassHTTPRead:    true,
+		HasBypassShardWrite: true,
+		HasBypassBaseWrite:  true,
+		HasBypassShardRead:  true,
+		HasBypassBaseRead:   true,
+		HasBypassHTTPRead:   true,
 	}
 }
 

common/pkgs/storage/obs/obs_test.go

@@ -6,7 +6,6 @@ import (
 
 	. "github.com/smartystreets/goconvey/convey"
 	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
-	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
 	cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
 )
 
@@ -28,7 +27,7 @@ func Test_S2S(t *testing.T) {
 			},
 		}
 
-		newPath, err := s2s.Transfer(context.TODO(), &clitypes.UserSpaceDetail{
+		_, err := s2s.Transfer(context.TODO(), &clitypes.UserSpaceDetail{
 			UserSpace: clitypes.UserSpace{
 				Storage: &cortypes.OBSType{
 					Region:    "cn-north-4",
@@ -41,10 +40,9 @@ func Test_S2S(t *testing.T) {
 					SK: "",
 				},
 			},
-		}, "test_data/test03.txt", types.S2SOption{})
-		defer s2s.Abort()
+		}, "test_data/test03.txt", "atest.txt")
+		defer s2s.Close()
 
 		So(err, ShouldEqual, nil)
-		t.Logf("newPath: %s", newPath)
 	})
 }

common/pkgs/storage/obs/s2s.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/auth/basic"
 	oms "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/oms/v2"
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/oms/v2/model"
@@ -40,10 +42,10 @@ func (s *S2STransfer) CanTransfer(src *clitypes.UserSpaceDetail) bool {
 }
 
 // 执行数据直传。返回传输后的文件路径
-func (s *S2STransfer) Transfer(ctx context.Context, src *clitypes.UserSpaceDetail, srcPath string, opt types.S2SOption) (string, error) {
+func (s *S2STransfer) Transfer(ctx context.Context, src *clitypes.UserSpaceDetail, srcPath string, dstPath string) (types.FileInfo, error) {
 	req := s.makeRequest(src, srcPath)
 	if req == nil {
-		return "", fmt.Errorf("unsupported source storage type: %T", src.UserSpace.Storage)
+		return types.FileInfo{}, fmt.Errorf("unsupported source storage type: %T", src.UserSpace.Storage)
 	}
 
 	auth, err := basic.NewCredentialsBuilder().
@@ -52,12 +54,12 @@ func (s *S2STransfer) Transfer(ctx context.Context, src *clitypes.UserSpaceDetai
 		WithProjectId(s.stgType.ProjectID).
 		SafeBuild()
 	if err != nil {
-		return "", err
+		return types.FileInfo{}, err
 	}
 
 	region, err := omsregion.SafeValueOf(s.stgType.Region)
 	if err != nil {
-		return "", err
+		return types.FileInfo{}, err
 	}
 
 	cli, err := oms.OmsClientBuilder().
@@ -65,9 +67,10 @@ func (s *S2STransfer) Transfer(ctx context.Context, src *clitypes.UserSpaceDetai
 		WithCredential(auth).
 		SafeBuild()
 	if err != nil {
-		return "", err
+		return types.FileInfo{}, err
 	}
 
+	// 先上传成一个临时文件
 	tempPrefix := s3.JoinKey(s.feat.TempDir, os2.GenerateRandomFileName(10)) + "/"
 
 	taskType := model.GetCreateTaskReqTaskTypeEnum().OBJECT
@@ -86,17 +89,37 @@ func (s *S2STransfer) Transfer(ctx context.Context, src *clitypes.UserSpaceDetai
 		},
 	})
 	if err != nil {
-		return "", fmt.Errorf("create task: %w", err)
+		return types.FileInfo{}, fmt.Errorf("create task: %w", err)
 	}
 
 	s.taskID = resp.Id
 
-	err = s.waitTask(ctx, *resp.Id)
+	// 轮询任务状态，直到完成
+	size, err := s.waitTask(ctx, *resp.Id)
 	if err != nil {
-		return "", fmt.Errorf("wait task: %w", err)
+		return types.FileInfo{}, fmt.Errorf("wait task: %w", err)
 	}
 
-	return s3.JoinKey(tempPrefix, srcPath), nil
+	// 传输完成后，将文件名改成目标路径
+	obsCli, bkt, err := createClient(s.stgType, s.cred)
+	if err != nil {
+		return types.FileInfo{}, err
+	}
+
+	_, err = obsCli.CopyObject(ctx, &awss3.CopyObjectInput{
+		Bucket:     aws.String(bkt),
+		CopySource: aws.String(s3.JoinKey(bkt, tempPrefix, srcPath)),
+		Key:        aws.String(dstPath),
+	})
+	if err != nil {
+		return types.FileInfo{}, fmt.Errorf("copy object: %w", err)
+	}
+
+	return types.FileInfo{
+		Path: dstPath,
+		Size: size,
+		Hash: "",
+	}, nil
 }
 
 func (s *S2STransfer) makeRequest(srcStg *clitypes.UserSpaceDetail, srcPath string) *model.SrcNodeReq {
@@ -123,7 +146,7 @@ func (s *S2STransfer) makeRequest(srcStg *clitypes.UserSpaceDetail, srcPath stri
 	}
 }
 
-func (s *S2STransfer) waitTask(ctx context.Context, taskId int64) error {
+func (s *S2STransfer) waitTask(ctx context.Context, taskId int64) (int64, error) {
 	ticker := time.NewTicker(time.Second * 5)
 	defer ticker.Stop()
 
@@ -139,38 +162,32 @@ func (s *S2STransfer) waitTask(ctx context.Context, taskId int64) error {
 				continue
 			}
 
-			return fmt.Errorf("show task failed too many times: %w", err)
+			return 0, fmt.Errorf("show task failed too many times: %w", err)
 		}
 		failures = 0
 
 		if *resp.Status == 3 {
-			return fmt.Errorf("task stopped")
+			return 0, fmt.Errorf("task stopped")
 		}
 
 		if *resp.Status == 4 {
-			return errors.New(resp.ErrorReason.String())
+			return 0, errors.New(resp.ErrorReason.String())
 		}
 
 		if *resp.Status == 5 {
-			return nil
+			return *resp.CompleteSize, nil
 		}
 
 		select {
 		case <-ticker.C:
 			continue
 		case <-ctx.Done():
-			return ctx.Err()
+			return 0, ctx.Err()
 		}
 	}
 }
 
-// 完成传输
-func (s *S2STransfer) Complete() {
-
-}
-
-// 取消传输。如果已经调用了Complete，则这个方法应该无效果
-func (s *S2STransfer) Abort() {
+func (s *S2STransfer) Close() {
 	if s.taskID != nil {
 		s.omsCli.StopTask(&model.StopTaskRequest{
 			TaskId: fmt.Sprintf("%v", *s.taskID),
@@ -179,7 +196,5 @@ func (s *S2STransfer) Abort() {
 		s.omsCli.DeleteTask(&model.DeleteTaskRequest{
 			TaskId: fmt.Sprintf("%v", *s.taskID),
 		})
-
-		// TODO 清理临时文件
 	}
 }

common/pkgs/storage/obs/shard_store.go

@@ -32,7 +32,7 @@ func NewShardStore(detail *clitypes.UserSpaceDetail, stgType *cortypes.OBSType,
 	return &sd, nil
 }
 
-func (s *ShardStore) HTTPBypassRead(fileHash clitypes.FileHash) (types.HTTPRequest, error) {
+func (s *ShardStore) MakeHTTPReadRequest(fileHash clitypes.FileHash) (types.HTTPRequest, error) {
 	cli, err := obs.New(s.cred.AK, s.cred.SK, s.stgType.Endpoint)
 	if err != nil {
 		return types.HTTPRequest{}, err

common/pkgs/storage/s3/base_store.go

@@ -62,6 +62,7 @@ func (s *BaseStore) Write(objPath string, stream io.Reader) (types.FileInfo, err
 		}
 
 		return types.FileInfo{
+			Path: key,
 			Hash: clitypes.NewFullHash(hash),
 			Size: counter.Count(),
 		}, nil
@@ -78,17 +79,24 @@ func (s *BaseStore) Write(objPath string, stream io.Reader) (types.FileInfo, err
 	}
 
 	return types.FileInfo{
+		Path: key,
 		Hash: clitypes.NewFullHash(hashStr.Sum()),
 		Size: counter.Count(),
 	}, nil
 }
 
-func (s *BaseStore) Read(objPath string) (io.ReadCloser, error) {
+func (s *BaseStore) Read(objPath string, opt types.OpenOption) (io.ReadCloser, error) {
 	key := objPath
 
+	rngStr := fmt.Sprintf("bytes=%d-", opt.Offset)
+	if opt.Length >= 0 {
+		rngStr += fmt.Sprintf("%d", opt.Offset+opt.Length-1)
+	}
+
 	resp, err := s.cli.GetObject(context.TODO(), &s3.GetObjectInput{
 		Bucket: aws.String(s.Bucket),
 		Key:    aws.String(key),
+		Range:  aws.String(rngStr),
 	})
 
 	if err != nil {
@@ -107,7 +115,7 @@ func (s *BaseStore) Mkdir(path string) error {
 	return err
 }
 
-func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
+func (s *BaseStore) ListAll(path string) ([]types.ListEntry, error) {
 	key := path
 	// TODO 待测试
 
@@ -117,7 +125,7 @@ func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
 		Delimiter: aws.String("/"),
 	}
 
-	var objs []types.BaseStoreEntry
+	var objs []types.ListEntry
 
 	var marker *string
 	for {
@@ -128,7 +136,7 @@ func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
 		}
 
 		for _, obj := range resp.Contents {
-			objs = append(objs, types.BaseStoreEntry{
+			objs = append(objs, types.ListEntry{
 				Path:  *obj.Key,
 				Size:  *obj.Size,
 				IsDir: false,
@@ -145,43 +153,59 @@ func (s *BaseStore) ListAll(path string) ([]types.BaseStoreEntry, error) {
 	return objs, nil
 }
 
-func (s *BaseStore) getLogger() logger.Logger {
-	return logger.WithField("BaseStore", "S3").WithField("Storage", s.Detail.UserSpace.Storage.String())
-}
+func (s *BaseStore) CleanTemps() {
+	log := s.getLogger()
 
-var _ types.BypassPublicRead = (*BaseStore)(nil)
+	var deletes []s3types.ObjectIdentifier
+	deleteObjs := make(map[string]s3types.Object)
+	var marker *string
+	for {
+		resp, err := s.cli.ListObjects(context.Background(), &s3.ListObjectsInput{
+			Bucket: aws.String(s.Bucket),
+			Prefix: aws.String(JoinKey(s.Detail.UserSpace.ShardStore.BaseDir, TempDir, "/")),
+			Marker: marker,
+		})
 
-func (s *BaseStore) BypassPublicRead(pa string) (types.BypassFilePath, error) {
-	info, err := s.cli.HeadObject(context.TODO(), &s3.HeadObjectInput{
-		Bucket: aws.String(s.Bucket),
-		Key:    aws.String(pa),
-	})
-	if err != nil {
-		s.getLogger().Warnf("get file %v: %v", pa, err)
-		return types.BypassFilePath{}, err
-	}
+		if err != nil {
+			log.Warnf("read temp dir: %v", err)
+			return
+		}
 
-	return types.BypassFilePath{
-		Path: pa,
-		Info: types.FileInfo{
-			Size:        *info.ContentLength,
-			Description: pa,
-		},
-	}, nil
-}
+		for _, obj := range resp.Contents {
+			deletes = append(deletes, s3types.ObjectIdentifier{
+				Key: obj.Key,
+			})
+			deleteObjs[*obj.Key] = obj
+		}
 
-var _ types.BypassPublicWrite = (*BaseStore)(nil)
+		if !*resp.IsTruncated {
+			break
+		}
 
-func (s *BaseStore) BypassedPublic(info types.BypassedFileInfo, dstPath string) error {
-	_, err := s.cli.CopyObject(context.TODO(), &s3.CopyObjectInput{
-		Bucket:     aws.String(s.Bucket),
-		CopySource: aws.String(s.Bucket + "/" + info.Path),
-		Key:        aws.String(dstPath),
+		marker = resp.NextMarker
+	}
+
+	if len(deletes) == 0 {
+		return
+	}
+
+	resp, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{
+		Bucket: aws.String(s.Bucket),
+		Delete: &s3types.Delete{
+			Objects: deletes,
+		},
 	})
 	if err != nil {
-		s.getLogger().Warnf("bypass public write %v to %v: %v", info.Path, dstPath, err)
-		return err
+		log.Warnf("delete temp files: %v", err)
+		return
+	}
+
+	for _, del := range resp.Deleted {
+		obj := deleteObjs[*del.Key]
+		log.Infof("remove unused temp file %v, size: %v, last mod time: %v", *obj.Key, *obj.Size, *obj.LastModified)
 	}
+}
 
-	return nil
+func (s *BaseStore) getLogger() logger.Logger {
+	return logger.WithField("BaseStore", "S3").WithField("Storage", s.Detail.UserSpace.Storage.String())
 }

common/pkgs/storage/s3/multipart_upload.go

@@ -101,7 +101,7 @@ func (i *MultipartTask) InitState() types.MultipartInitState {
 	}
 }
 
-func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPartInfo) (types.BypassedFileInfo, error) {
+func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPartInfo) (types.FileInfo, error) {
 	parts = sort2.Sort(parts, func(l, r types.UploadedPartInfo) int {
 		return l.PartNumber - r.PartNumber
 	})
@@ -127,7 +127,7 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
 		},
 	})
 	if err != nil {
-		return types.BypassedFileInfo{}, err
+		return types.FileInfo{}, err
 	}
 
 	headResp, err := i.cli.HeadObject(ctx, &s3.HeadObjectInput{
@@ -135,12 +135,12 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
 		Key:    aws.String(i.tempFilePath),
 	})
 	if err != nil {
-		return types.BypassedFileInfo{}, err
+		return types.FileInfo{}, err
 	}
 
 	hash := clitypes.CalculateCompositeHash(partHashes)
 
-	return types.BypassedFileInfo{
+	return types.FileInfo{
 		Path: i.tempFilePath,
 		Size: *headResp.ContentLength,
 		Hash: hash,
@@ -148,20 +148,10 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
 
 }
 
-func (i *MultipartTask) Complete() {
-
-}
-
-func (i *MultipartTask) Abort() {
-	// TODO2 根据注释描述，Abort不能停止正在上传的分片，需要等待其上传完成才能彻底删除，
-	// 考虑增加定时任务去定时清理
+func (i *MultipartTask) Close() {
 	i.cli.AbortMultipartUpload(context.Background(), &s3.AbortMultipartUploadInput{
 		Bucket:   aws.String(i.bucket),
 		Key:      aws.String(i.tempFilePath),
 		UploadId: aws.String(i.uploadID),
 	})
-	i.cli.DeleteObject(context.Background(), &s3.DeleteObjectInput{
-		Bucket: aws.String(i.bucket),
-		Key:    aws.String(i.tempFilePath),
-	})
 }

common/pkgs/storage/s3/s3.go

@@ -30,11 +30,11 @@ func newBuilder(detail *clitypes.UserSpaceDetail) types.StorageBuilder {
 
 func (b *builder) FeatureDesc() types.FeatureDesc {
 	return types.FeatureDesc{
-		HasBypassShardWrite:  true,
-		HasBypassPublicWrite: true,
-		HasBypassShardRead:   true,
-		HasBypassPublicRead:  true,
-		HasBypassHTTPRead:    false,
+		HasBypassShardWrite: true,
+		HasBypassBaseWrite:  true,
+		HasBypassShardRead:  true,
+		HasBypassBaseRead:   true,
+		HasBypassHTTPRead:   false,
 	}
 }
 

common/pkgs/storage/s3/shard_store.go

@@ -2,20 +2,12 @@ package s3
 
 import (
 	"context"
-	"crypto/sha256"
-	"errors"
-	"fmt"
-	"io"
-	"path/filepath"
 	"sync"
-	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"gitlink.org.cn/cloudream/common/pkgs/logger"
-	"gitlink.org.cn/cloudream/common/utils/io2"
-	"gitlink.org.cn/cloudream/common/utils/os2"
 	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
 	"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
 )
@@ -30,286 +22,58 @@ type ShardStoreOption struct {
 }
 
 type ShardStore struct {
-	Detail           *clitypes.UserSpaceDetail
-	Bucket           string
-	cli              *s3.Client
-	opt              ShardStoreOption
-	lock             sync.Mutex
-	workingTempFiles map[string]bool
-	done             chan any
+	Detail *clitypes.UserSpaceDetail
+	Bucket string
+	cli    *s3.Client
+	opt    ShardStoreOption
+	lock   sync.Mutex
 }
 
 func NewShardStore(detail *clitypes.UserSpaceDetail, cli *s3.Client, bkt string, opt ShardStoreOption) (*ShardStore, error) {
 	return &ShardStore{
-		Detail:           detail,
-		Bucket:           bkt,
-		cli:              cli,
-		opt:              opt,
-		workingTempFiles: make(map[string]bool),
-		done:             make(chan any, 1),
+		Detail: detail,
+		Bucket: bkt,
+		cli:    cli,
+		opt:    opt,
 	}, nil
 }
 
 func (s *ShardStore) Start(ch *types.StorageEventChan) {
 	s.getLogger().Infof("start, root: %v", s.Detail.UserSpace.ShardStore.BaseDir)
-
-	go func() {
-		removeTempTicker := time.NewTicker(time.Minute * 10)
-		defer removeTempTicker.Stop()
-
-		for {
-			select {
-			case <-removeTempTicker.C:
-				s.removeUnusedTempFiles()
-			case <-s.done:
-				return
-			}
-		}
-	}()
-}
-
-func (s *ShardStore) removeUnusedTempFiles() {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	log := s.getLogger()
-
-	var deletes []s3types.ObjectIdentifier
-	deleteObjs := make(map[string]s3types.Object)
-	var marker *string
-	for {
-		resp, err := s.cli.ListObjects(context.Background(), &s3.ListObjectsInput{
-			Bucket: aws.String(s.Bucket),
-			Prefix: aws.String(JoinKey(s.Detail.UserSpace.ShardStore.BaseDir, TempDir, "/")),
-			Marker: marker,
-		})
-
-		if err != nil {
-			log.Warnf("read temp dir: %v", err)
-			return
-		}
-
-		for _, obj := range resp.Contents {
-			objName := BaseKey(*obj.Key)
-
-			if s.workingTempFiles[objName] {
-				continue
-			}
-
-			deletes = append(deletes, s3types.ObjectIdentifier{
-				Key: obj.Key,
-			})
-			deleteObjs[*obj.Key] = obj
-		}
-
-		if !*resp.IsTruncated {
-			break
-		}
-
-		marker = resp.NextMarker
-	}
-
-	if len(deletes) == 0 {
-		return
-	}
-
-	resp, err := s.cli.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{
-		Bucket: aws.String(s.Bucket),
-		Delete: &s3types.Delete{
-			Objects: deletes,
-		},
-	})
-	if err != nil {
-		log.Warnf("delete temp files: %v", err)
-		return
-	}
-
-	for _, del := range resp.Deleted {
-		obj := deleteObjs[*del.Key]
-		log.Infof("remove unused temp file %v, size: %v, last mod time: %v", *obj.Key, *obj.Size, *obj.LastModified)
-	}
 }
 
 func (s *ShardStore) Stop() {
 	s.getLogger().Infof("component stop")
-
-	select {
-	case s.done <- nil:
-	default:
-	}
-}
-
-func (s *ShardStore) Create(stream io.Reader) (types.FileInfo, error) {
-	if s.opt.UseAWSSha256 {
-		return s.createWithAwsSha256(stream)
-	} else {
-		return s.createWithCalcSha256(stream)
-	}
-}
-
-func (s *ShardStore) createWithAwsSha256(stream io.Reader) (types.FileInfo, error) {
-	log := s.getLogger()
-
-	key, fileName := s.createTempFile()
-
-	counter := io2.Counter(stream)
-
-	resp, err := s.cli.PutObject(context.TODO(), &s3.PutObjectInput{
-		Bucket:            aws.String(s.Bucket),
-		Key:               aws.String(key),
-		Body:              counter,
-		ChecksumAlgorithm: s3types.ChecksumAlgorithmSha256,
-	})
-	// TODO2
-	// if stgglb.Stats.HubStorageTransfer != nil {
-	// 	stgglb.Stats.HubStorageTransfer.RecordUpload(s.Detail.Storage.StorageID, counter.Count(), err == nil)
-	// }
-	if err != nil {
-		log.Warnf("uploading file %v: %v", key, err)
-
-		s.lock.Lock()
-		defer s.lock.Unlock()
-
-		delete(s.workingTempFiles, fileName)
-		return types.FileInfo{}, err
-	}
-
-	if resp.ChecksumSHA256 == nil {
-		log.Warnf("SHA256 checksum not found in response of uploaded file %v", key)
-		s.onCreateFailed(key, fileName)
-		return types.FileInfo{}, errors.New("SHA256 checksum not found in response")
-	}
-
-	hash, err := DecodeBase64Hash(*resp.ChecksumSHA256)
-	if err != nil {
-		log.Warnf("decode SHA256 checksum %v: %v", *resp.ChecksumSHA256, err)
-		s.onCreateFailed(key, fileName)
-		return types.FileInfo{}, fmt.Errorf("decode SHA256 checksum: %v", err)
-	}
-
-	return s.onCreateFinished(key, counter.Count(), clitypes.NewFullHash(hash))
-}
-
-func (s *ShardStore) createWithCalcSha256(stream io.Reader) (types.FileInfo, error) {
-	log := s.getLogger()
-
-	key, fileName := s.createTempFile()
-
-	hashStr := io2.NewReadHasher(sha256.New(), stream)
-	counter := io2.Counter(hashStr)
-
-	_, err := s.cli.PutObject(context.TODO(), &s3.PutObjectInput{
-		Bucket: aws.String(s.Bucket),
-		Key:    aws.String(key),
-		Body:   counter,
-	})
-	// TODO2
-	// if stgglb.Stats.HubStorageTransfer != nil {
-	// 	stgglb.Stats.HubStorageTransfer.RecordUpload(s.Detail.Storage.StorageID, counter.Count(), err == nil)
-	// }
-	if err != nil {
-		log.Warnf("uploading file %v: %v", key, err)
-
-		s.lock.Lock()
-		defer s.lock.Unlock()
-
-		delete(s.workingTempFiles, fileName)
-		return types.FileInfo{}, err
-	}
-
-	return s.onCreateFinished(key, counter.Count(), clitypes.NewFullHash(hashStr.Sum()))
 }
 
-func (s *ShardStore) createTempFile() (string, string) {
+func (s *ShardStore) Store(path string, hash clitypes.FileHash, size int64) (types.FileInfo, error) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
-	tmpDir := JoinKey(s.Detail.UserSpace.ShardStore.BaseDir, TempDir)
-	tmpName := os2.GenerateRandomFileName(20)
-
-	s.workingTempFiles[tmpName] = true
-	return JoinKey(tmpDir, tmpName), tmpName
-}
-
-func (s *ShardStore) onCreateFinished(tempFilePath string, size int64, hash clitypes.FileHash) (types.FileInfo, error) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	defer delete(s.workingTempFiles, filepath.Base(tempFilePath))
-	defer func() {
-		// 不管是否成功。即使失败了也有定时清理机制去兜底
-		s.cli.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
-			Bucket: aws.String(s.Bucket),
-			Key:    aws.String(tempFilePath),
-		})
-	}()
-
 	log := s.getLogger()
 
-	log.Debugf("write file %v finished, size: %v, hash: %v", tempFilePath, size, hash)
+	log.Debugf("write file %v finished, size: %v, hash: %v", path, size, hash)
 
 	blockDir := s.GetFileDirFromHash(hash)
 	newPath := JoinKey(blockDir, string(hash))
 
 	_, err := s.cli.CopyObject(context.Background(), &s3.CopyObjectInput{
 		Bucket:     aws.String(s.Bucket),
-		CopySource: aws.String(JoinKey(s.Bucket, tempFilePath)),
+		CopySource: aws.String(JoinKey(s.Bucket, path)),
 		Key:        aws.String(newPath),
 	})
 	if err != nil {
-		log.Warnf("copy file %v to %v: %v", tempFilePath, newPath, err)
+		log.Warnf("copy file %v to %v: %v", path, newPath, err)
 		return types.FileInfo{}, err
 	}
 
 	return types.FileInfo{
-		Hash:        hash,
-		Size:        size,
-		Description: newPath,
+		Hash: hash,
+		Size: size,
+		Path: newPath,
 	}, nil
 }
 
-func (s *ShardStore) onCreateFailed(key string, fileName string) {
-	// 不管是否成功。即使失败了也有定时清理机制去兜底
-	s.cli.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
-		Bucket: aws.String(s.Bucket),
-		Key:    aws.String(key),
-	})
-
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	delete(s.workingTempFiles, fileName)
-}
-
-// 使用NewOpen函数创建Option对象
-func (s *ShardStore) Open(opt types.OpenOption) (io.ReadCloser, error) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	filePath := s.GetFilePathFromHash(opt.FileHash)
-
-	rngStr := fmt.Sprintf("bytes=%d-", opt.Offset)
-	if opt.Length >= 0 {
-		rngStr += fmt.Sprintf("%d", opt.Offset+opt.Length-1)
-	}
-
-	resp, err := s.cli.GetObject(context.TODO(), &s3.GetObjectInput{
-		Bucket: aws.String(s.Bucket),
-		Key:    aws.String(filePath),
-		Range:  aws.String(rngStr),
-	})
-	if err != nil {
-		s.getLogger().Warnf("get file %v: %v", filePath, err)
-		return nil, err
-	}
-
-	return io2.CounterCloser(resp.Body, func(cnt int64, err error) {
-		// TODO2
-		// if stgglb.Stats.HubStorageTransfer != nil {
-		// 	stgglb.Stats.HubStorageTransfer.RecordDownload(s.Detail.Storage.StorageID, cnt, err == nil || err == io.EOF)
-		// }
-	}), nil
-}
-
 func (s *ShardStore) Info(hash clitypes.FileHash) (types.FileInfo, error) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
@@ -325,9 +89,9 @@ func (s *ShardStore) Info(hash clitypes.FileHash) (types.FileInfo, error) {
 	}
 
 	return types.FileInfo{
-		Hash:        hash,
-		Size:        *info.ContentLength,
-		Description: filePath,
+		Hash: hash,
+		Size: *info.ContentLength,
+		Path: filePath,
 	}, nil
 }
 
@@ -361,9 +125,9 @@ func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
 			}
 
 			infos = append(infos, types.FileInfo{
-				Hash:        fileHash,
-				Size:        *obj.Size,
-				Description: *obj.Key,
+				Hash: fileHash,
+				Size: *obj.Size,
+				Path: *obj.Key,
 			})
 		}
 
@@ -462,66 +226,3 @@ func (s *ShardStore) GetFileDirFromHash(hash clitypes.FileHash) string {
 func (s *ShardStore) GetFilePathFromHash(hash clitypes.FileHash) string {
 	return JoinKey(s.Detail.UserSpace.ShardStore.BaseDir, BlocksDir, hash.GetHashPrefix(2), string(hash))
 }
-
-var _ types.BypassShardWrite = (*ShardStore)(nil)
-
-func (s *ShardStore) BypassedShard(info types.BypassedFileInfo) error {
-	if info.Hash == "" {
-		return fmt.Errorf("empty file hash is not allowed by this shard store")
-	}
-
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	defer func() {
-		// 不管是否成功。即使失败了也有定时清理机制去兜底
-		s.cli.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
-			Bucket: aws.String(s.Bucket),
-			Key:    aws.String(info.Path),
-		})
-	}()
-
-	log := s.getLogger()
-
-	log.Debugf("%v bypass uploaded, size: %v, hash: %v", info.Path, info.Size, info.Hash)
-
-	blockDir := s.GetFileDirFromHash(info.Hash)
-	newPath := JoinKey(blockDir, string(info.Hash))
-
-	_, err := s.cli.CopyObject(context.Background(), &s3.CopyObjectInput{
-		CopySource: aws.String(JoinKey(s.Bucket, info.Path)),
-		Bucket:     aws.String(s.Bucket),
-		Key:        aws.String(newPath),
-	})
-	if err != nil {
-		log.Warnf("copy file %v to %v: %v", info.Path, newPath, err)
-		return fmt.Errorf("copy file: %w", err)
-	}
-
-	return nil
-}
-
-var _ types.BypassShardRead = (*ShardStore)(nil)
-
-func (s *ShardStore) BypassShardRead(fileHash clitypes.FileHash) (types.BypassFilePath, error) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	filePath := s.GetFilePathFromHash(fileHash)
-	info, err := s.cli.HeadObject(context.TODO(), &s3.HeadObjectInput{
-		Bucket: aws.String(s.Bucket),
-		Key:    aws.String(filePath),
-	})
-	if err != nil {
-		s.getLogger().Warnf("get file %v: %v", filePath, err)
-		return types.BypassFilePath{}, err
-	}
-
-	return types.BypassFilePath{
-		Path: filePath,
-		Info: types.FileInfo{
-			Hash:        fileHash,
-			Size:        *info.ContentLength,
-			Description: filePath,
-		},
-	}, nil
-}

common/pkgs/storage/types/base_store.go

@@ -1,10 +1,11 @@
 package types
 
 import (
+	"fmt"
 	"io"
 )
 
-type BaseStoreEntry struct {
+type ListEntry struct {
 	Path  string
 	Size  int64
 	IsDir bool
@@ -12,11 +13,55 @@ type BaseStoreEntry struct {
 
 type BaseStore interface {
 	Write(path string, stream io.Reader) (FileInfo, error)
-	Read(path string) (io.ReadCloser, error)
+	Read(path string, opt OpenOption) (io.ReadCloser, error)
 	// 创建指定路径的文件夹。对于不支持空文件夹的存储系统来说，可以采用创建以/结尾的对象的方式来模拟文件夹。
 	Mkdir(path string) error
 	// 返回指定路径下的所有文件，文件路径是包含path在内的完整路径。返回结果的第一条一定是路径本身，可能是文件，也可能是目录。
 	// 如果路径不存在，那么不会返回错误，而是返回一个空列表。
 	// 返回的内容严格按照存储系统的原始结果来，比如当存储系统是一个对象存储时，那么就可能不会包含目录，或者包含用于模拟的以“/”结尾的对象。
-	ListAll(path string) ([]BaseStoreEntry, error)
+	ListAll(path string) ([]ListEntry, error)
+	// 清空临时目录。只应该在此存储服务未被使用时调用
+	CleanTemps()
+}
+
+type OpenOption struct {
+	Offset int64
+	Length int64
+}
+
+func DefaultOpen() OpenOption {
+	return OpenOption{
+		Offset: 0,
+		Length: -1,
+	}
+}
+
+func (o *OpenOption) WithLength(len int64) OpenOption {
+	o.Length = len
+	return *o
+}
+
+// [start, end)，不包含end
+func (o *OpenOption) WithRange(start int64, end int64) OpenOption {
+	o.Offset = start
+	o.Length = end - start
+	return *o
+}
+
+func (o *OpenOption) WithNullableLength(offset int64, length *int64) {
+	o.Offset = offset
+	if length != nil {
+		o.Length = *length
+	}
+}
+
+func (o *OpenOption) String() string {
+	rangeStart := fmt.Sprintf("%d", o.Offset)
+
+	rangeEnd := ""
+	if o.Length >= 0 {
+		rangeEnd = fmt.Sprintf("%d", o.Offset+o.Length)
+	}
+
+	return fmt.Sprintf("%s:%s", rangeStart, rangeEnd)
 }

common/pkgs/storage/types/bypass.go

@@ -4,44 +4,10 @@ import (
 	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
 )
 
-// 通过旁路上传后的文件的信息
-type BypassedFileInfo struct {
-	Path string
-	Hash clitypes.FileHash
-	Size int64
-}
-
-// 不通过ShardStore上传文件，但上传完成后需要通知ShardStore。
-type BypassShardWrite interface {
-	BypassedShard(info BypassedFileInfo) error
-}
-
-// 不通过BaseStore上传文件。
-type BypassPublicWrite interface {
-	BypassedPublic(info BypassedFileInfo, dstPath string) error
-}
-
-// 描述指定文件在分片存储中的路径。可以考虑设计成interface。
-type BypassFilePath struct {
-	Path string
-	// 文件信息。此结构体中的字段不一定都会存在，比如如果是从BypassPublicRead接口获取的，那么Hash字段就为空。
-	Info FileInfo
-}
-
-// 不通过ShardStore读取文件，但需要它返回文件的路径。
-type BypassShardRead interface {
-	BypassShardRead(fileHash clitypes.FileHash) (BypassFilePath, error)
-}
-
-// 不通过BaseStore读取文件。虽然仅使用path就已经可以读取到文件，但还是增加了此接口用于获取更详细的文件信息。
-type BypassPublicRead interface {
-	BypassPublicRead(path string) (BypassFilePath, error)
-}
-
 // 能通过一个Http请求直接访问文件
 // 仅用于分片存储。
-type HTTPBypassShardRead interface {
-	HTTPBypassRead(fileHash clitypes.FileHash) (HTTPRequest, error)
+type HTTPShardRead interface {
+	MakeHTTPReadRequest(fileHash clitypes.FileHash) (HTTPRequest, error)
 }
 
 type HTTPRequest struct {

common/pkgs/storage/types/ec_multiplier.go

@@ -3,9 +3,6 @@ package types
 type ECMultiplier interface {
 	// 进行EC运算，coef * inputs。coef为编码矩阵，inputs为待编码数据，chunkSize为分块大小。
 	// 输出为每一个块文件的路径，数组长度 = len(coef)
-	Multiply(coef [][]byte, inputs []HTTPRequest, chunkSize int) ([]BypassedFileInfo, error)
-	// 完成计算
-	Complete()
-	// 取消计算。如果已经调用了Complete，则应该无任何影响
-	Abort()
+	Multiply(coef [][]byte, inputs []HTTPRequest, chunkSize int) ([]FileInfo, error)
+	Close()
 }

common/pkgs/storage/types/empty_builder.go

@@ -13,6 +13,7 @@ type EmptyBuilder struct {
 func (b *EmptyBuilder) FeatureDesc() FeatureDesc {
 	return FeatureDesc{}
 }
+
 func (b *EmptyBuilder) CreateShardStore() (ShardStore, error) {
 	return nil, fmt.Errorf("create shard store for %T: %w", b.Detail.UserSpace.Storage, ErrUnsupported)
 }

common/pkgs/storage/types/s2s.go

@@ -6,19 +6,10 @@ import (
 	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
 )
 
-type S2SOption struct {
-	// 传输目的地路径。如果为空，则自动生成一个临时路径。
-	// 只是一个建议，非强制，如果存储服务不支持直接指定目的路径，则可以忽略这个参数。
-	DestPathHint string
-}
-
 type S2STransfer interface {
 	// 判断是否能从指定的源存储中直传到当前存储的目的路径。仅在生成计划时使用
 	CanTransfer(src *clitypes.UserSpaceDetail) bool
-	// 执行数据直传。返回传输后的文件路径
-	Transfer(ctx context.Context, src *clitypes.UserSpaceDetail, srcPath string, opt S2SOption) (string, error)
-	// 完成传输
-	Complete()
-	// 取消传输。如果已经调用了Complete，则这个方法应该无效果
-	Abort()
+	// 从远端获取文件并保存到本地路径
+	Transfer(ctx context.Context, src *clitypes.UserSpaceDetail, srcPath string, dstPath string) (FileInfo, error)
+	Close()
 }

common/pkgs/storage/types/s3_client.go

@@ -17,11 +17,8 @@ type Multiparter interface {
 type MultipartTask interface {
 	InitState() MultipartInitState
 	// 所有分片上传完成后，合并分片
-	JoinParts(ctx context.Context, parts []UploadedPartInfo) (BypassedFileInfo, error)
-	// 合成之后的文件已被使用
-	Complete()
-	// 取消上传。如果在调用Complete之前调用，则应该删除合并后的文件。如果已经调用Complete，则应该不做任何事情。
-	Abort()
+	JoinParts(ctx context.Context, parts []UploadedPartInfo) (FileInfo, error)
+	Close()
 }
 
 // TODO 可以考虑重构成一个接口，支持不同的类型的分片有不同内容的实现

common/pkgs/storage/types/shard_store.go

@@ -1,9 +1,6 @@
 package types
 
 import (
-	"fmt"
-	"io"
-
 	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
 )
 
@@ -11,7 +8,9 @@ type Status interface {
 	String() string
 }
 
-type OKStatus struct{}
+type OKStatus struct {
+	StoreEvent
+}
 
 func (s *OKStatus) String() string {
 	return "OK"
@@ -20,15 +19,14 @@ func (s *OKStatus) String() string {
 var StatusOK = &OKStatus{}
 
 type StoreEvent interface {
+	IsStoreEvent()
 }
 
 type ShardStore interface {
 	Start(ch *StorageEventChan)
 	Stop()
-	// 写入一个新文件，写入后获得FileHash
-	Create(stream io.Reader) (FileInfo, error)
-	// 使用F函数创建Option对象
-	Open(opt OpenOption) (io.ReadCloser, error)
+	// 将存储系统中已有的文件作为分片纳入管理范围
+	Store(path string, hash clitypes.FileHash, size int64) (FileInfo, error)
 	// 获得指定文件信息
 	Info(fileHash clitypes.FileHash) (FileInfo, error)
 	// 获取所有文件信息，尽量保证操作是原子的
@@ -39,14 +37,6 @@ type ShardStore interface {
 	Stats() Stats
 }
 
-type FileInfo struct {
-	// 文件的SHA256哈希值，全大写的16进制字符串格式
-	Hash clitypes.FileHash
-	Size int64
-	// 文件描述信息，比如文件名，用于调试
-	Description string
-}
-
 type Stats struct {
 	// 存储服务状态，如果状态正常，此值应该是StatusOK
 	Status Status
@@ -59,54 +49,3 @@ type Stats struct {
 	// 描述信息，用于调试
 	Description string
 }
-
-type OpenOption struct {
-	FileHash clitypes.FileHash
-	Offset   int64
-	Length   int64
-}
-
-func NewOpen(fileHash clitypes.FileHash) OpenOption {
-	return OpenOption{
-		FileHash: fileHash,
-		Offset:   0,
-		Length:   -1,
-	}
-}
-
-func (o *OpenOption) WithLength(len int64) OpenOption {
-	o.Length = len
-	return *o
-}
-
-// [start, end)，不包含end
-func (o *OpenOption) WithRange(start int64, end int64) OpenOption {
-	o.Offset = start
-	o.Length = end - start
-	return *o
-}
-
-func (o *OpenOption) WithNullableLength(offset int64, length *int64) {
-	o.Offset = offset
-	if length != nil {
-		o.Length = *length
-	}
-}
-
-func (o *OpenOption) String() string {
-	rangeStart := ""
-	if o.Offset > 0 {
-		rangeStart = fmt.Sprintf("%d", o.Offset)
-	}
-
-	rangeEnd := ""
-	if o.Length >= 0 {
-		rangeEnd = fmt.Sprintf("%d", o.Offset+o.Length)
-	}
-
-	if rangeStart == "" && rangeEnd == "" {
-		return string(o.FileHash)
-	}
-
-	return fmt.Sprintf("%s[%s:%s]", string(o.FileHash), rangeStart, rangeEnd)
-}

common/pkgs/storage/types/types.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 
 	"gitlink.org.cn/cloudream/common/pkgs/async"
+	clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
 )
 
 var ErrStorageNotFound = errors.New("storage not found")
@@ -35,11 +36,20 @@ type FeatureDesc struct {
 	// 是否能旁路上传分片
 	HasBypassShardWrite bool
 	// 是否能旁路上传公共存储
-	HasBypassPublicWrite bool
+	HasBypassBaseWrite bool
 	// 是否能旁路读取分片
 	HasBypassShardRead bool
 	// 公共存储是否支持旁路读取
-	HasBypassPublicRead bool
+	HasBypassBaseRead bool
 	// 是否能通过HTTP读取
 	HasBypassHTTPRead bool
 }
+
+type FileInfo struct {
+	// 分片在存储系统中的路径，可以通过BaseStore读取的
+	Path string
+	// 文件大小
+	Size int64
+	// 分片的哈希值，不一定有值，根据来源不同，可能为空
+	Hash clitypes.FileHash
+}