From 2954df83a7be2337a44b2421f77eca3200414d2b Mon Sep 17 00:00:00 2001 From: Sydonian <794346190@qq.com> Date: Tue, 22 Oct 2024 17:43:24 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0ShardStore=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/pkgs/shardstore/pool/pool.go | 13 ++++ .../pkgs/shardstore/storages/local/config.go | 12 ++++ .../pkgs/shardstore/storages/local/local.go | 33 ++++++++++ .../pkgs/shardstore/storages/local/writer.go | 20 ++++++ common/pkgs/shardstore/types/option.go | 54 ++++++++++++++++ common/pkgs/shardstore/types/shardstore.go | 64 +++++++++++++++++++ 6 files changed, 196 insertions(+) create mode 100644 common/pkgs/shardstore/pool/pool.go create mode 100644 common/pkgs/shardstore/storages/local/config.go create mode 100644 common/pkgs/shardstore/storages/local/local.go create mode 100644 common/pkgs/shardstore/storages/local/writer.go create mode 100644 common/pkgs/shardstore/types/option.go create mode 100644 common/pkgs/shardstore/types/shardstore.go diff --git a/common/pkgs/shardstore/pool/pool.go b/common/pkgs/shardstore/pool/pool.go new file mode 100644 index 0000000..65fa3ba --- /dev/null +++ b/common/pkgs/shardstore/pool/pool.go @@ -0,0 +1,13 @@ +package pool + +import ( + cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" + "gitlink.org.cn/cloudream/storage/common/pkgs/shardstore/types" +) + +type ShardStorePool struct { +} + +func (p *ShardStorePool) Get(stgID cdssdk.StorageID) (types.ShardStore, error) { + +} diff --git a/common/pkgs/shardstore/storages/local/config.go b/common/pkgs/shardstore/storages/local/config.go new file mode 100644 index 0000000..256f326 --- /dev/null +++ b/common/pkgs/shardstore/storages/local/config.go @@ -0,0 +1,12 @@ +package local + +import "gitlink.org.cn/cloudream/storage/common/pkgs/shardstore/types" + +type Config struct { + RootPath string + MaxSize int64 +} + +func (c *Config) Build() types.ShardStore { + return &Local{cfg: *c} +} diff --git a/common/pkgs/shardstore/storages/local/local.go b/common/pkgs/shardstore/storages/local/local.go new file mode 100644 index 0000000..d35af7f --- /dev/null +++ b/common/pkgs/shardstore/storages/local/local.go @@ -0,0 +1,33 @@ +package local + +import ( + "io" + + "gitlink.org.cn/cloudream/storage/common/pkgs/shardstore/types" +) + +type Local struct { + cfg Config +} + +func (s *Local) New() io.Writer { + +} + +// 使用F函数创建Option对象 +func (s *Local) Open(opt types.OpenOption) (io.ReadCloser, error) { + +} + +func (s *Local) Remove(hash types.FileHash) error { + +} + +// 遍历所有文件，callback返回false则停止遍历 +func (s *Local) Walk(callback func(info types.FileInfo) bool) error { + +} + +func (s *Local) Stats() (types.Stats, error) { + +} diff --git a/common/pkgs/shardstore/storages/local/writer.go b/common/pkgs/shardstore/storages/local/writer.go new file mode 100644 index 0000000..b93ccaf --- /dev/null +++ b/common/pkgs/shardstore/storages/local/writer.go @@ -0,0 +1,20 @@ +package local + +import "gitlink.org.cn/cloudream/storage/common/pkgs/shardstore/types" + +type Writer struct { +} + +func (w *Writer) Write(data []byte) error { + +} + +// 取消写入 +func (w *Writer) Abort() error { + +} + +// 结束写入，获得文件哈希值 +func (w *Writer) Finish() (types.FileInfo, error) { + +} diff --git a/common/pkgs/shardstore/types/option.go b/common/pkgs/shardstore/types/option.go new file mode 100644 index 0000000..da315b7 --- /dev/null +++ b/common/pkgs/shardstore/types/option.go @@ -0,0 +1,54 @@ +package types + +import "fmt" + +type OpenOption struct { + FileHash FileHash + Offset int64 + Length int64 +} + +func NewOpen(fileHash FileHash) OpenOption { + return OpenOption{ + FileHash: fileHash, + Offset: 0, + Length: -1, + } +} + +func (o *OpenOption) WithLength(len int64) OpenOption { + o.Length = len + return *o +} + +// [start, end]，即包含end +func (o *OpenOption) WithRange(start int64, end int64) OpenOption { + o.Offset = start + o.Length = end - start + 1 + return *o +} + +func (o *OpenOption) WithNullableLength(offset int64, length *int64) { + o.Offset = offset + if length != nil { + o.Length = *length + } +} + +func (o *OpenOption) String() string { + rangeStart := "" + if o.Offset > 0 { + rangeStart = fmt.Sprintf("%d", o.Offset) + } + + rangeEnd := "" + if o.Length >= 0 { + rangeEnd = fmt.Sprintf("%d", o.Offset+o.Length-1) + } + + if rangeStart == "" && rangeEnd == "" { + return string(o.FileHash) + } + + return fmt.Sprintf("%s[%s:%s]", string(o.FileHash), rangeStart, rangeEnd) +} diff --git a/common/pkgs/shardstore/types/shardstore.go b/common/pkgs/shardstore/types/shardstore.go new file mode 100644 index 0000000..44c8402 --- /dev/null +++ b/common/pkgs/shardstore/types/shardstore.go @@ -0,0 +1,64 @@ +package types + +import "io" + +type FileHash string + +type Status interface { + String() string +} + +type OKStatus struct{} + +func (s *OKStatus) String() string { + return "OK" +} + +var StatusOK = &OKStatus{} + +type ShardStore interface { + // 准备写入一个新文件，写入后获得FileHash + New() Writer + // 使用F函数创建Option对象 + Open(opt OpenOption) (io.ReadCloser, error) + // 删除文件 + Remove(hash FileHash) error + // 遍历所有文件，callback返回false则停止遍历 + Walk(callback func(info FileInfo) bool) error + // 获得存储系统信息 + Stats() Stats +} + +type Config interface { + Build() (ShardStore, error) +} + +type FileInfo struct { + // 文件的SHA256哈希值，全大写的16进制字符串格式 + Hash FileHash + Size int64 + // 文件描述信息，比如文件名，用于调试 + Description string +} + +type Stats struct { + // 存储服务状态，如果状态正常，此值应该是StatusOK + Status Status + // 文件总数 + FileCount int + // 存储空间总大小 + TotalSize int64 + // 已使用的存储空间大小，可以超过存储空间总大小 + UsedSize int64 + // 描述信息，用于调试 + Description string +} + +type Writer interface { + io.Writer + // 取消写入。要求允许在调用了Finish之后再调用此函数，且此时不应该有任何影响。 + // 方便defer机制 + Abort() error + // 结束写入，获得文件哈希值 + Finish() (FileInfo, error) +}