Browse Source

Client的types移动到common中

master
Sydonian 4 months ago
parent
commit
e9200bc78c
100 changed files with 673 additions and 673 deletions
  1. +5
    -5
      client/internal/accessstat/access_stat.go
  2. +10
    -10
      client/internal/cmdline/migrate.go
  3. +1
    -1
      client/internal/cmdline/serve.go
  4. +3
    -3
      client/internal/cmdline/test.go
  5. +1
    -1
      client/internal/cmdline/vfstest.go
  6. +1
    -1
      client/internal/db/bucket.go
  7. +1
    -1
      client/internal/db/object.go
  8. +1
    -1
      client/internal/db/object_access_stat.go
  9. +1
    -1
      client/internal/db/object_block.go
  10. +1
    -1
      client/internal/db/package.go
  11. +1
    -1
      client/internal/db/package_access_stat.go
  12. +1
    -1
      client/internal/db/pinned_object.go
  13. +1
    -1
      client/internal/db/space_sync_task.go
  14. +1
    -1
      client/internal/db/user_space.go
  15. +23
    -23
      client/internal/downloader/downloader.go
  16. +1
    -1
      client/internal/downloader/iterator.go
  17. +4
    -4
      client/internal/downloader/lrc_strip_iterator.go
  18. +1
    -1
      client/internal/downloader/strategy/selector.go
  19. +1
    -1
      client/internal/downloader/strip_iterator.go
  20. +1
    -1
      client/internal/http/types/config.go
  21. +6
    -6
      client/internal/http/v1/object.go
  22. +7
    -7
      client/internal/http/v1/package.go
  23. +4
    -4
      client/internal/http/v1/presigned.go
  24. +7
    -7
      client/internal/http/v1/space_syncer.go
  25. +2
    -2
      client/internal/http/v1/user_space.go
  26. +1
    -1
      client/internal/metacache/user_space_meta.go
  27. +4
    -4
      client/internal/mount/mount_linux.go
  28. +4
    -4
      client/internal/mount/mount_win.go
  29. +7
    -7
      client/internal/mount/vfs/cache/cache.go
  30. +3
    -3
      client/internal/mount/vfs/cache/file.go
  31. +17
    -17
      client/internal/mount/vfs/fuse.go
  32. +2
    -2
      client/internal/mount/vfs/fuse_bucket.go
  33. +2
    -2
      client/internal/mount/vfs/fuse_dir.go
  34. +3
    -3
      client/internal/mount/vfs/fuse_file.go
  35. +2
    -2
      client/internal/mount/vfs/fuse_root.go
  36. +1
    -1
      client/internal/repl/getp.go
  37. +4
    -4
      client/internal/repl/load.go
  38. +4
    -4
      client/internal/repl/lsp.go
  39. +1
    -1
      client/internal/repl/newloadp.go
  40. +1
    -1
      client/internal/repl/put.go
  41. +1
    -1
      client/internal/services/bucket.go
  42. +2
    -2
      client/internal/services/object.go
  43. +2
    -2
      client/internal/services/package.go
  44. +26
    -26
      client/internal/services/user_space.go
  45. +7
    -7
      client/internal/services/utils.go
  46. +1
    -1
      client/internal/spacesyncer/execute.go
  47. +7
    -7
      client/internal/spacesyncer/execute_diff.go
  48. +1
    -1
      client/internal/spacesyncer/execute_full.go
  49. +3
    -3
      client/internal/spacesyncer/filter.go
  50. +16
    -16
      client/internal/spacesyncer/space_syncer.go
  51. +3
    -3
      client/internal/spacesyncer/trigger.go
  52. +6
    -6
      client/internal/speedstats/speedstats.go
  53. +10
    -10
      client/internal/ticktock/change_redundancy.go
  54. +8
    -8
      client/internal/ticktock/check_shardstore.go
  55. +88
    -88
      client/internal/ticktock/redundancy_recover.go
  56. +53
    -53
      client/internal/ticktock/redundancy_shrink.go
  57. +1
    -1
      client/internal/ticktock/redundancy_shrink_test.go
  58. +1
    -1
      client/internal/ticktock/user_space_gc.go
  59. +1
    -1
      client/internal/uploader/create_load.go
  60. +1
    -1
      client/internal/uploader/update.go
  61. +13
    -13
      client/internal/uploader/uploader.go
  62. +7
    -7
      client/internal/uploader/user_space_upload.go
  63. +7
    -7
      client/sdk/api/v1/bucket.go
  64. +1
    -1
      client/sdk/api/v1/object.go
  65. +24
    -24
      client/sdk/api/v1/package.go
  66. +22
    -22
      client/sdk/api/v1/presigned.go
  67. +13
    -13
      client/sdk/api/v1/space_syncer.go
  68. +2
    -2
      client/sdk/api/v1/storage_test.go
  69. +2
    -2
      client/sdk/api/v1/system.go
  70. +18
    -18
      client/sdk/api/v1/user_space.go
  71. +15
    -15
      common/pkgs/ioswitch2/fromto.go
  72. +14
    -14
      common/pkgs/ioswitch2/ops2/base_store.go
  73. +6
    -6
      common/pkgs/ioswitch2/ops2/bypass.go
  74. +7
    -7
      common/pkgs/ioswitch2/ops2/ec.go
  75. +7
    -7
      common/pkgs/ioswitch2/ops2/multipart.go
  76. +17
    -17
      common/pkgs/ioswitch2/ops2/s2s.go
  77. +9
    -9
      common/pkgs/ioswitch2/ops2/shard_store.go
  78. +2
    -2
      common/pkgs/ioswitch2/parser/gen/generator.go
  79. +2
    -2
      common/pkgs/ioswitch2/parser/opt/utils.go
  80. +2
    -2
      common/pkgs/ioswitch2/plans/complete_multipart.go
  81. +2
    -2
      common/pkgs/ioswitch2/plans/utils.go
  82. +7
    -7
      common/pkgs/ioswitchlrc/fromto.go
  83. +14
    -14
      common/pkgs/ioswitchlrc/ops2/base_store.go
  84. +5
    -5
      common/pkgs/ioswitchlrc/ops2/ec.go
  85. +9
    -9
      common/pkgs/ioswitchlrc/ops2/shard_store.go
  86. +5
    -5
      common/pkgs/ioswitchlrc/parser/generator.go
  87. +2
    -2
      common/pkgs/ioswitchlrc/parser/utils.go
  88. +4
    -4
      common/pkgs/publock/reqbuilder/user_space.go
  89. +5
    -5
      common/pkgs/rpc/hub/cache.go
  90. +3
    -3
      common/pkgs/rpc/hub/user_space.go
  91. +3
    -3
      common/pkgs/storage/efile/ec_multiplier.go
  92. +3
    -3
      common/pkgs/storage/efile/efile.go
  93. +2
    -2
      common/pkgs/storage/factory/factory.go
  94. +3
    -3
      common/pkgs/storage/factory/reg/reg.go
  95. +8
    -8
      common/pkgs/storage/local/base_store.go
  96. +4
    -4
      common/pkgs/storage/local/dir_reader.go
  97. +3
    -3
      common/pkgs/storage/local/local.go
  98. +4
    -4
      common/pkgs/storage/local/multipart_upload.go
  99. +5
    -5
      common/pkgs/storage/local/s2s.go
  100. +12
    -12
      common/pkgs/storage/local/shard_store.go

+ 5
- 5
client/internal/accessstat/access_stat.go View File

@@ -7,7 +7,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/async" "gitlink.org.cn/cloudream/common/pkgs/async"
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type AccessStatEventChan = async.UnboundChannel[AccessStatEvent] type AccessStatEventChan = async.UnboundChannel[AccessStatEvent]
@@ -30,9 +30,9 @@ type AccessStat struct {
} }


type entryKey struct { type entryKey struct {
objID clitypes.ObjectID
pkgID clitypes.PackageID
spaceID clitypes.UserSpaceID
objID jcsypes.ObjectID
pkgID jcsypes.PackageID
spaceID jcsypes.UserSpaceID
} }


func NewAccessStat(cfg Config, db *db.DB) *AccessStat { func NewAccessStat(cfg Config, db *db.DB) *AccessStat {
@@ -44,7 +44,7 @@ func NewAccessStat(cfg Config, db *db.DB) *AccessStat {
} }
} }


func (p *AccessStat) AddAccessCounter(objID clitypes.ObjectID, pkgID clitypes.PackageID, spaceID clitypes.UserSpaceID, value float64) {
func (p *AccessStat) AddAccessCounter(objID jcsypes.ObjectID, pkgID jcsypes.PackageID, spaceID jcsypes.UserSpaceID, value float64) {
p.lock.Lock() p.lock.Lock()
defer p.lock.Unlock() defer p.lock.Unlock()




+ 10
- 10
client/internal/cmdline/migrate.go View File

@@ -6,7 +6,7 @@ import (


"github.com/spf13/cobra" "github.com/spf13/cobra"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/config" "gitlink.org.cn/cloudream/jcs-pub/client/internal/config"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/driver/mysql" "gorm.io/driver/mysql"
"gorm.io/gorm" "gorm.io/gorm"
) )
@@ -38,15 +38,15 @@ func migrate(configPath string) {
} }
db = db.Set("gorm:table_options", "CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci") db = db.Set("gorm:table_options", "CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci")


migrateOne(db, clitypes.Bucket{})
migrateOne(db, clitypes.ObjectAccessStat{})
migrateOne(db, clitypes.ObjectBlock{})
migrateOne(db, clitypes.Object{})
migrateOne(db, clitypes.PackageAccessStat{})
migrateOne(db, clitypes.Package{})
migrateOne(db, clitypes.PinnedObject{})
migrateOne(db, clitypes.UserSpace{})
migrateOne(db, clitypes.SpaceSyncTask{})
migrateOne(db, jcsypes.Bucket{})
migrateOne(db, jcsypes.ObjectAccessStat{})
migrateOne(db, jcsypes.ObjectBlock{})
migrateOne(db, jcsypes.Object{})
migrateOne(db, jcsypes.PackageAccessStat{})
migrateOne(db, jcsypes.Package{})
migrateOne(db, jcsypes.PinnedObject{})
migrateOne(db, jcsypes.UserSpace{})
migrateOne(db, jcsypes.SpaceSyncTask{})


fmt.Println("migrate success") fmt.Println("migrate success")
} }


+ 1
- 1
client/internal/cmdline/serve.go View File

@@ -22,11 +22,11 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/ticktock" "gitlink.org.cn/cloudream/jcs-pub/client/internal/ticktock"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader" "gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
"gitlink.org.cn/cloudream/jcs-pub/common/models/datamap"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/sysevent" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/sysevent"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
) )


// 初始化函数,将ServeHTTP命令注册到命令列表中。 // 初始化函数,将ServeHTTP命令注册到命令列表中。


+ 3
- 3
client/internal/cmdline/test.go View File

@@ -19,9 +19,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/spacesyncer" "gitlink.org.cn/cloudream/jcs-pub/client/internal/spacesyncer"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/speedstats" "gitlink.org.cn/cloudream/jcs-pub/client/internal/speedstats"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader" "gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
"gitlink.org.cn/cloudream/jcs-pub/common/models/datamap"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
@@ -29,6 +27,8 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/sysevent" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/sysevent"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
) )


func init() { func init() {
@@ -61,7 +61,7 @@ func doTest(svc *services.Service) {


ft = ioswitch2.NewFromTo() ft = ioswitch2.NewFromTo()
ft.AddFrom(ioswitch2.NewFromShardstore("Full1AE5436AF72D8EF93923486E0E167315CEF0C91898064DADFAC22216FFBC5E3D", *space1, ioswitch2.RawStream())) ft.AddFrom(ioswitch2.NewFromShardstore("Full1AE5436AF72D8EF93923486E0E167315CEF0C91898064DADFAC22216FFBC5E3D", *space1, ioswitch2.RawStream()))
ft.AddTo(ioswitch2.NewToBaseStore(*space2, clitypes.PathFromComps("test3.txt")))
ft.AddTo(ioswitch2.NewToBaseStore(*space2, jcsypes.PathFromComps("test3.txt")))
plans := exec.NewPlanBuilder() plans := exec.NewPlanBuilder()
parser.Parse(ft, plans) parser.Parse(ft, plans)
fmt.Println(plans) fmt.Println(plans)


+ 1
- 1
client/internal/cmdline/vfstest.go View File

@@ -23,11 +23,11 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/speedstats" "gitlink.org.cn/cloudream/jcs-pub/client/internal/speedstats"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader" "gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
"gitlink.org.cn/cloudream/jcs-pub/common/models/datamap"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/sysevent" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/sysevent"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
) )


// 初始化函数,将ServeHTTP命令注册到命令列表中。 // 初始化函数,将ServeHTTP命令注册到命令列表中。


+ 1
- 1
client/internal/db/bucket.go View File

@@ -4,7 +4,7 @@ import (
"fmt" "fmt"
"time" "time"


"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
) )




+ 1
- 1
client/internal/db/object.go View File

@@ -10,7 +10,7 @@ import (
"gorm.io/gorm/clause" "gorm.io/gorm/clause"


"gitlink.org.cn/cloudream/common/utils/sort2" "gitlink.org.cn/cloudream/common/utils/sort2"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type ObjectDB struct { type ObjectDB struct {


+ 1
- 1
client/internal/db/object_access_stat.go View File

@@ -1,7 +1,7 @@
package db package db


import ( import (
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
"gorm.io/gorm/clause" "gorm.io/gorm/clause"
) )


+ 1
- 1
client/internal/db/object_block.go View File

@@ -1,7 +1,7 @@
package db package db


import ( import (
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm/clause" "gorm.io/gorm/clause"
) )




+ 1
- 1
client/internal/db/package.go View File

@@ -5,7 +5,7 @@ import (
"time" "time"


"github.com/samber/lo" "github.com/samber/lo"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
) )




+ 1
- 1
client/internal/db/package_access_stat.go View File

@@ -1,7 +1,7 @@
package db package db


import ( import (
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
"gorm.io/gorm/clause" "gorm.io/gorm/clause"
) )


+ 1
- 1
client/internal/db/pinned_object.go View File

@@ -3,7 +3,7 @@ package db
import ( import (
"time" "time"


"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm/clause" "gorm.io/gorm/clause"
) )




+ 1
- 1
client/internal/db/space_sync_task.go View File

@@ -1,6 +1,6 @@
package db package db


import "gitlink.org.cn/cloudream/jcs-pub/client/types"
import "gitlink.org.cn/cloudream/jcs-pub/common/types"


type SpaceSyncTaskDB struct { type SpaceSyncTaskDB struct {
*DB *DB


+ 1
- 1
client/internal/db/user_space.go View File

@@ -1,7 +1,7 @@
package db package db


import ( import (
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm/clause" "gorm.io/gorm/clause"
) )




+ 23
- 23
client/internal/downloader/downloader.go View File

@@ -9,9 +9,9 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader/strategy" "gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader/strategy"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/speedstats" "gitlink.org.cn/cloudream/jcs-pub/client/internal/speedstats"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


const ( const (
@@ -21,18 +21,18 @@ const (
type DownloadIterator = iterator.Iterator[*Downloading] type DownloadIterator = iterator.Iterator[*Downloading]


type DownloadReqeust struct { type DownloadReqeust struct {
ObjectID clitypes.ObjectID
ObjectID jcsypes.ObjectID
Offset int64 Offset int64
Length int64 Length int64
} }


type downloadReqeust2 struct { type downloadReqeust2 struct {
Detail *clitypes.ObjectDetail
Detail *jcsypes.ObjectDetail
Raw DownloadReqeust Raw DownloadReqeust
} }


type Downloading struct { type Downloading struct {
Object *clitypes.Object
Object *jcsypes.Object
File io.ReadCloser // 文件流,如果文件不存在,那么为nil File io.ReadCloser // 文件流,如果文件不存在,那么为nil
Request DownloadReqeust Request DownloadReqeust
} }
@@ -65,7 +65,7 @@ func NewDownloader(cfg Config, conn *connectivity.Collector, stgPool *pool.Pool,
} }


func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator { func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator {
objIDs := make([]clitypes.ObjectID, len(reqs))
objIDs := make([]jcsypes.ObjectID, len(reqs))
for i, req := range reqs { for i, req := range reqs {
objIDs[i] = req.ObjectID objIDs[i] = req.ObjectID
} }
@@ -79,7 +79,7 @@ func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator {
return iterator.FuseError[*Downloading](fmt.Errorf("request to db: %w", err)) return iterator.FuseError[*Downloading](fmt.Errorf("request to db: %w", err))
} }


detailsMap := make(map[clitypes.ObjectID]*clitypes.ObjectDetail)
detailsMap := make(map[jcsypes.ObjectID]*jcsypes.ObjectDetail)
for _, detail := range objDetails { for _, detail := range objDetails {
d := detail d := detail
detailsMap[detail.Object.ObjectID] = &d detailsMap[detail.Object.ObjectID] = &d
@@ -96,7 +96,7 @@ func (d *Downloader) DownloadObjects(reqs []DownloadReqeust) DownloadIterator {
return NewDownloadObjectIterator(d, req2s) return NewDownloadObjectIterator(d, req2s)
} }


func (d *Downloader) DownloadObjectByDetail(detail clitypes.ObjectDetail, off int64, length int64) (*Downloading, error) {
func (d *Downloader) DownloadObjectByDetail(detail jcsypes.ObjectDetail, off int64, length int64) (*Downloading, error) {
req2s := []downloadReqeust2{{ req2s := []downloadReqeust2{{
Detail: &detail, Detail: &detail,
Raw: DownloadReqeust{ Raw: DownloadReqeust{
@@ -110,56 +110,56 @@ func (d *Downloader) DownloadObjectByDetail(detail clitypes.ObjectDetail, off in
return iter.MoveNext() return iter.MoveNext()
} }


func (d *Downloader) DownloadPackage(pkgID clitypes.PackageID, prefix string) (clitypes.Package, DownloadIterator, error) {
pkg, details, err := db.DoTx02(d.db, func(tx db.SQLContext) (clitypes.Package, []clitypes.ObjectDetail, error) {
func (d *Downloader) DownloadPackage(pkgID jcsypes.PackageID, prefix string) (jcsypes.Package, DownloadIterator, error) {
pkg, details, err := db.DoTx02(d.db, func(tx db.SQLContext) (jcsypes.Package, []jcsypes.ObjectDetail, error) {
pkg, err := d.db.Package().GetByID(tx, pkgID) pkg, err := d.db.Package().GetByID(tx, pkgID)
if err != nil { if err != nil {
return clitypes.Package{}, nil, err
return jcsypes.Package{}, nil, err
} }


var details []clitypes.ObjectDetail
var details []jcsypes.ObjectDetail
if prefix != "" { if prefix != "" {
objs, err := d.db.Object().GetWithPathPrefix(tx, pkgID, prefix) objs, err := d.db.Object().GetWithPathPrefix(tx, pkgID, prefix)
if err != nil { if err != nil {
return clitypes.Package{}, nil, err
return jcsypes.Package{}, nil, err
} }


objIDs := make([]clitypes.ObjectID, len(objs))
objIDs := make([]jcsypes.ObjectID, len(objs))
for i, obj := range objs { for i, obj := range objs {
objIDs[i] = obj.ObjectID objIDs[i] = obj.ObjectID
} }


allBlocks, err := d.db.ObjectBlock().BatchGetByObjectID(tx, objIDs) allBlocks, err := d.db.ObjectBlock().BatchGetByObjectID(tx, objIDs)
if err != nil { if err != nil {
return clitypes.Package{}, nil, err
return jcsypes.Package{}, nil, err
} }


allPinnedObjs, err := d.db.PinnedObject().BatchGetByObjectID(tx, objIDs) allPinnedObjs, err := d.db.PinnedObject().BatchGetByObjectID(tx, objIDs)
if err != nil { if err != nil {
return clitypes.Package{}, nil, err
return jcsypes.Package{}, nil, err


} }
details = make([]clitypes.ObjectDetail, 0, len(objs))
details = make([]jcsypes.ObjectDetail, 0, len(objs))
for _, obj := range objs { for _, obj := range objs {
detail := clitypes.ObjectDetail{
detail := jcsypes.ObjectDetail{
Object: obj, Object: obj,
} }
details = append(details, detail) details = append(details, detail)
} }


clitypes.DetailsFillObjectBlocks(details, allBlocks)
clitypes.DetailsFillPinnedAt(details, allPinnedObjs)
jcsypes.DetailsFillObjectBlocks(details, allBlocks)
jcsypes.DetailsFillPinnedAt(details, allPinnedObjs)
} else { } else {
details, err = d.db.Object().GetPackageObjectDetails(tx, pkgID) details, err = d.db.Object().GetPackageObjectDetails(tx, pkgID)
if err != nil { if err != nil {
return clitypes.Package{}, nil, err
return jcsypes.Package{}, nil, err
} }
} }


return pkg, details, nil return pkg, details, nil
}) })
if err != nil { if err != nil {
return clitypes.Package{}, nil, err
return jcsypes.Package{}, nil, err
} }


req2s := make([]downloadReqeust2, len(details)) req2s := make([]downloadReqeust2, len(details))
@@ -180,11 +180,11 @@ func (d *Downloader) DownloadPackage(pkgID clitypes.PackageID, prefix string) (c


type ObjectECStrip struct { type ObjectECStrip struct {
Data []byte Data []byte
ObjectFileHash clitypes.FileHash // 添加这条缓存时,Object的FileHash
ObjectFileHash jcsypes.FileHash // 添加这条缓存时,Object的FileHash
} }


type ECStripKey struct { type ECStripKey struct {
ObjectID clitypes.ObjectID
ObjectID jcsypes.ObjectID
StripIndex int64 StripIndex int64
} }




+ 1
- 1
client/internal/downloader/iterator.go View File

@@ -12,13 +12,13 @@ import (
"gitlink.org.cn/cloudream/common/utils/io2" "gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader/strategy" "gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader/strategy"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/iterator" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/iterator"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type downloadSpaceInfo struct { type downloadSpaceInfo struct {


+ 4
- 4
client/internal/downloader/lrc_strip_iterator.go View File

@@ -8,18 +8,18 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/iterator" "gitlink.org.cn/cloudream/common/pkgs/iterator"
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/parser" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/parser"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type LRCStripIterator struct { type LRCStripIterator struct {
downloader *Downloader downloader *Downloader
object clitypes.Object
object jcsypes.Object
blocks []downloadBlock blocks []downloadBlock
red clitypes.LRCRedundancy
red jcsypes.LRCRedundancy
curStripIndex int64 curStripIndex int64
cache *StripCache cache *StripCache
dataChan chan dataChanEntry dataChan chan dataChanEntry
@@ -28,7 +28,7 @@ type LRCStripIterator struct {
inited bool inited bool
} }


func NewLRCStripIterator(downloder *Downloader, object clitypes.Object, blocks []downloadBlock, red clitypes.LRCRedundancy, beginStripIndex int64, cache *StripCache, maxPrefetch int) *LRCStripIterator {
func NewLRCStripIterator(downloder *Downloader, object jcsypes.Object, blocks []downloadBlock, red jcsypes.LRCRedundancy, beginStripIndex int64, cache *StripCache, maxPrefetch int) *LRCStripIterator {
if maxPrefetch <= 0 { if maxPrefetch <= 0 {
maxPrefetch = 1 maxPrefetch = 1
} }


+ 1
- 1
client/internal/downloader/strategy/selector.go View File

@@ -10,8 +10,8 @@ import (
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/common/utils/sort2" "gitlink.org.cn/cloudream/common/utils/sort2"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/metacache" "gitlink.org.cn/cloudream/jcs-pub/client/internal/metacache"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/consts" "gitlink.org.cn/cloudream/jcs-pub/common/consts"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )




+ 1
- 1
client/internal/downloader/strip_iterator.go View File

@@ -8,11 +8,11 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/iterator" "gitlink.org.cn/cloudream/common/pkgs/iterator"
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type downloadBlock struct { type downloadBlock struct {


+ 1
- 1
client/internal/http/types/config.go View File

@@ -9,7 +9,7 @@ import (
"os" "os"


"gitlink.org.cn/cloudream/jcs-pub/client/sdk/signer" "gitlink.org.cn/cloudream/jcs-pub/client/sdk/signer"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type ConfigJSON struct { type ConfigJSON struct {


+ 6
- 6
client/internal/http/v1/object.go View File

@@ -18,8 +18,8 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader" "gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types" "gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode" "gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type ObjectService struct { type ObjectService struct {
@@ -113,9 +113,9 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
return return
} }


copyToPath := make([]clitypes.JPath, 0, len(info.CopyToPath))
copyToPath := make([]jcsypes.JPath, 0, len(info.CopyToPath))
for _, p := range info.CopyToPath { for _, p := range info.CopyToPath {
copyToPath = append(copyToPath, clitypes.PathFromJcsPathString(p))
copyToPath = append(copyToPath, jcsypes.PathFromJcsPathString(p))
} }


up, err := s.svc.Uploader.BeginUpdate(info.PackageID, info.Affinity, info.CopyTo, copyToPath) up, err := s.svc.Uploader.BeginUpdate(info.PackageID, info.Affinity, info.CopyTo, copyToPath)
@@ -147,7 +147,7 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
} }
path = filepath.ToSlash(path) path = filepath.ToSlash(path)


err = up.Upload(clitypes.PathFromJcsPathString(path), file)
err = up.Upload(jcsypes.PathFromJcsPathString(path), file)
if err != nil { if err != nil {
log.Warnf("uploading file: %s", err.Error()) log.Warnf("uploading file: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.FileName(), err))) ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.FileName(), err)))
@@ -163,7 +163,7 @@ func (s *ObjectService) Upload(ctx *gin.Context) {
return return
} }


uploadeds := make([]clitypes.Object, len(pathes))
uploadeds := make([]jcsypes.Object, len(pathes))
for i := range pathes { for i := range pathes {
uploadeds[i] = ret.Objects[pathes[i]] uploadeds[i] = ret.Objects[pathes[i]]
} }
@@ -398,7 +398,7 @@ func (s *ObjectService) DeleteByPath(ctx *gin.Context) {
return return
} }


err = s.svc.ObjectSvc().Delete([]clitypes.ObjectID{resp.Objects[0].ObjectID})
err = s.svc.ObjectSvc().Delete([]jcsypes.ObjectID{resp.Objects[0].ObjectID})
if err != nil { if err != nil {
log.Warnf("deleting objects: %s", err.Error()) log.Warnf("deleting objects: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "delete objects failed")) ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, "delete objects failed"))


+ 7
- 7
client/internal/http/v1/package.go View File

@@ -19,9 +19,9 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader" "gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types" "gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode" "gitlink.org.cn/cloudream/jcs-pub/common/ecode"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/iterator" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/iterator"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
) )


@@ -151,9 +151,9 @@ func (s *PackageService) CreateLoad(ctx *gin.Context) {
return return
} }


copyToPath := make([]clitypes.JPath, 0, len(info.CopyToPath))
copyToPath := make([]jcsypes.JPath, 0, len(info.CopyToPath))
for _, p := range info.CopyToPath { for _, p := range info.CopyToPath {
copyToPath = append(copyToPath, clitypes.PathFromJcsPathString(p))
copyToPath = append(copyToPath, jcsypes.PathFromJcsPathString(p))
} }


up, err := s.svc.Uploader.BeginCreateUpload(info.BucketID, info.Name, info.CopyTo, copyToPath) up, err := s.svc.Uploader.BeginCreateUpload(info.BucketID, info.Name, info.CopyTo, copyToPath)
@@ -184,7 +184,7 @@ func (s *PackageService) CreateLoad(ctx *gin.Context) {
} }
path = filepath.ToSlash(path) path = filepath.ToSlash(path)


err = up.Upload(clitypes.PathFromJcsPathString(path), file)
err = up.Upload(jcsypes.PathFromJcsPathString(path), file)
if err != nil { if err != nil {
log.Warnf("uploading file: %s", err.Error()) log.Warnf("uploading file: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.FileName(), err))) ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", file.FileName(), err)))
@@ -200,7 +200,7 @@ func (s *PackageService) CreateLoad(ctx *gin.Context) {
return return
} }


objs := make([]clitypes.Object, len(pathes))
objs := make([]jcsypes.Object, len(pathes))
for i := range pathes { for i := range pathes {
objs[i] = ret.Objects[pathes[i]] objs[i] = ret.Objects[pathes[i]]
} }
@@ -233,7 +233,7 @@ func (s *PackageService) Download(ctx *gin.Context) {
} }
} }


func (s *PackageService) downloadZip(ctx *gin.Context, req cliapi.PackageDownload, pkg clitypes.Package, iter downloader.DownloadIterator) {
func (s *PackageService) downloadZip(ctx *gin.Context, req cliapi.PackageDownload, pkg jcsypes.Package, iter downloader.DownloadIterator) {
log := logger.WithField("HTTP", "Package.Download") log := logger.WithField("HTTP", "Package.Download")


ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".zip") ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".zip")
@@ -276,7 +276,7 @@ func (s *PackageService) downloadZip(ctx *gin.Context, req cliapi.PackageDownloa
} }
} }


func (s *PackageService) downloadTar(ctx *gin.Context, req cliapi.PackageDownload, pkg clitypes.Package, iter downloader.DownloadIterator) {
func (s *PackageService) downloadTar(ctx *gin.Context, req cliapi.PackageDownload, pkg jcsypes.Package, iter downloader.DownloadIterator) {
log := logger.WithField("HTTP", "Package.Download") log := logger.WithField("HTTP", "Package.Download")


ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".tar") ctx.Header("Content-Disposition", "attachment; filename="+url.PathEscape(pkg.Name)+".tar")


+ 4
- 4
client/internal/http/v1/presigned.go View File

@@ -14,8 +14,8 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader" "gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types" "gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode" "gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type PresignedService struct { type PresignedService struct {
@@ -156,9 +156,9 @@ func (s *PresignedService) ObjectUpload(ctx *gin.Context) {
return return
} }


copyToPath := make([]clitypes.JPath, 0, len(req.CopyToPath))
copyToPath := make([]jcsypes.JPath, 0, len(req.CopyToPath))
for _, p := range req.CopyToPath { for _, p := range req.CopyToPath {
copyToPath = append(copyToPath, clitypes.PathFromJcsPathString(p))
copyToPath = append(copyToPath, jcsypes.PathFromJcsPathString(p))
} }


up, err := s.svc.Uploader.BeginUpdate(req.PackageID, req.Affinity, req.CopyTo, copyToPath) up, err := s.svc.Uploader.BeginUpdate(req.PackageID, req.Affinity, req.CopyTo, copyToPath)
@@ -171,7 +171,7 @@ func (s *PresignedService) ObjectUpload(ctx *gin.Context) {


path := filepath.ToSlash(req.Path) path := filepath.ToSlash(req.Path)


err = up.Upload(clitypes.PathFromJcsPathString(path), ctx.Request.Body)
err = up.Upload(jcsypes.PathFromJcsPathString(path), ctx.Request.Body)
if err != nil { if err != nil {
log.Warnf("uploading file: %s", err.Error()) log.Warnf("uploading file: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", req.Path, err))) ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("uploading file %v: %v", req.Path, err)))


+ 7
- 7
client/internal/http/v1/space_syncer.go View File

@@ -7,8 +7,8 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types" "gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode" "gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type SpaceSyncerService struct { type SpaceSyncerService struct {
@@ -41,21 +41,21 @@ func (s *SpaceSyncerService) CreateTask(ctx *gin.Context) {
return return
} }


dests := make([]clitypes.SpaceSyncDest, 0, len(req.DestUserSpaceIDs))
dests := make([]jcsypes.SpaceSyncDest, 0, len(req.DestUserSpaceIDs))
for _, id := range req.DestUserSpaceIDs { for _, id := range req.DestUserSpaceIDs {
dests = append(dests, clitypes.SpaceSyncDest{
DestUserSpaceID: clitypes.UserSpaceID(id),
DestPath: clitypes.PathFromJcsPathString(req.DestPathes[0]),
dests = append(dests, jcsypes.SpaceSyncDest{
DestUserSpaceID: jcsypes.UserSpaceID(id),
DestPath: jcsypes.PathFromJcsPathString(req.DestPathes[0]),
}) })
} }


info, err := s.svc.SpaceSyncer.CreateTask(clitypes.SpaceSyncTask{
info, err := s.svc.SpaceSyncer.CreateTask(jcsypes.SpaceSyncTask{
Trigger: req.Trigger, Trigger: req.Trigger,
Mode: req.Mode, Mode: req.Mode,
Filters: req.Filters, Filters: req.Filters,
Options: req.Options, Options: req.Options,
SrcUserSpaceID: req.SrcUserSpaceID, SrcUserSpaceID: req.SrcUserSpaceID,
SrcPath: clitypes.PathFromJcsPathString(req.SrcPath),
SrcPath: jcsypes.PathFromJcsPathString(req.SrcPath),
Dests: dests, Dests: dests,
}) })
if err != nil { if err != nil {


+ 2
- 2
client/internal/http/v1/user_space.go View File

@@ -8,8 +8,8 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types" "gitlink.org.cn/cloudream/jcs-pub/client/internal/http/types"
cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" cliapi "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/ecode" "gitlink.org.cn/cloudream/jcs-pub/common/ecode"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type UserSpaceService struct { type UserSpaceService struct {
@@ -52,7 +52,7 @@ func (s *UserSpaceService) CreatePackage(ctx *gin.Context) {
return return
} }


pkg, err := s.svc.Uploader.UserSpaceUpload(req.UserSpaceID, clitypes.PathFromJcsPathString(req.Path), req.BucketID, req.Name, req.SpaceAffinity)
pkg, err := s.svc.Uploader.UserSpaceUpload(req.UserSpaceID, jcsypes.PathFromJcsPathString(req.Path), req.BucketID, req.Name, req.SpaceAffinity)
if err != nil { if err != nil {
log.Warnf("userspace create package: %s", err.Error()) log.Warnf("userspace create package: %s", err.Error())
ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("userspace create package: %v", err))) ctx.JSON(http.StatusOK, types.Failed(ecode.OperationFailed, fmt.Sprintf("userspace create package: %v", err)))


+ 1
- 1
client/internal/metacache/user_space_meta.go View File

@@ -5,9 +5,9 @@ import (
"time" "time"


"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator" corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )




+ 4
- 4
client/internal/mount/mount_linux.go View File

@@ -12,7 +12,7 @@ import (
fuse2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" fuse2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader" "gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type Mount struct { type Mount struct {
@@ -107,14 +107,14 @@ func (m *Mount) StartReclaimSpace() {
m.vfs.ReclaimSpace() m.vfs.ReclaimSpace()
} }


func (m *Mount) NotifyObjectInvalid(obj clitypes.Object) {
func (m *Mount) NotifyObjectInvalid(obj jcsypes.Object) {


} }


func (m *Mount) NotifyPackageInvalid(pkg clitypes.Package) {
func (m *Mount) NotifyPackageInvalid(pkg jcsypes.Package) {


} }


func (m *Mount) NotifyBucketInvalid(bkt clitypes.Bucket) {
func (m *Mount) NotifyBucketInvalid(bkt jcsypes.Bucket) {


} }

+ 4
- 4
client/internal/mount/mount_win.go View File

@@ -8,7 +8,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader" "gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/config" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/config"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader" "gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type Mount struct { type Mount struct {
@@ -43,14 +43,14 @@ func (m *Mount) StartReclaimSpace() {


} }


func (m *Mount) NotifyObjectInvalid(obj clitypes.Object) {
func (m *Mount) NotifyObjectInvalid(obj jcsypes.Object) {


} }


func (m *Mount) NotifyPackageInvalid(pkg clitypes.Package) {
func (m *Mount) NotifyPackageInvalid(pkg jcsypes.Package) {


} }


func (m *Mount) NotifyBucketInvalid(bkt clitypes.Bucket) {
func (m *Mount) NotifyBucketInvalid(bkt jcsypes.Bucket) {


} }

+ 7
- 7
client/internal/mount/vfs/cache/cache.go View File

@@ -21,7 +21,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/config" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/config"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader" "gitlink.org.cn/cloudream/jcs-pub/client/internal/uploader"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type CacheEntry interface { type CacheEntry interface {
@@ -196,7 +196,7 @@ func (c *Cache) CreateFile(pathComps []string) *CacheFile {
// 尝试加载缓存文件,如果文件不存在,则使用obj的信息创建一个新缓存文件,而如果obj为nil,那么会返回nil。 // 尝试加载缓存文件,如果文件不存在,则使用obj的信息创建一个新缓存文件,而如果obj为nil,那么会返回nil。
// //
// 记得使用Release减少引用计数 // 记得使用Release减少引用计数
func (c *Cache) LoadFile(pathComps []string, obj *clitypes.Object) *CacheFile {
func (c *Cache) LoadFile(pathComps []string, obj *jcsypes.Object) *CacheFile {
c.lock.Lock() c.lock.Lock()
defer c.lock.Unlock() defer c.lock.Unlock()


@@ -490,7 +490,7 @@ func (c *Cache) Move(pathComps []string, newPathComps []string) error {
type syncPackage struct { type syncPackage struct {
bktName string bktName string
pkgName string pkgName string
pkg clitypes.Package
pkg jcsypes.Package
upObjs []*uploadingObject upObjs []*uploadingObject
} }


@@ -917,7 +917,7 @@ func (c *Cache) doUpdatingOnly(pkgs []*syncPackage) {
pathes := make([]string, 0, len(p.upObjs)) pathes := make([]string, 0, len(p.upObjs))
modTimes := make([]time.Time, 0, len(p.upObjs)) modTimes := make([]time.Time, 0, len(p.upObjs))
for _, obj := range p.upObjs { for _, obj := range p.upObjs {
pathes = append(pathes, clitypes.JoinObjectPath(obj.pathComps[2:]...))
pathes = append(pathes, jcsypes.JoinObjectPath(obj.pathComps[2:]...))
modTimes = append(modTimes, obj.modTime) modTimes = append(modTimes, obj.modTime)
} }


@@ -1008,7 +1008,7 @@ func (c *Cache) doUploading(pkgs []*syncPackage) {


counter := io2.Counter(&rd) counter := io2.Counter(&rd)


err = upder.Upload(clitypes.PathFromComps(o.pathComps[2:]...), counter, uploader.UploadOption{
err = upder.Upload(jcsypes.PathFromComps(o.pathComps[2:]...), counter, uploader.UploadOption{
CreateTime: o.modTime, CreateTime: o.modTime,
}) })
if err != nil { if err != nil {
@@ -1036,8 +1036,8 @@ func (c *Cache) doUploading(pkgs []*syncPackage) {
continue continue
} }


oldPath := clitypes.JoinObjectPath(o.pathComps[2:]...)
newPath := clitypes.JoinObjectPath(o.cache.pathComps[2:]...)
oldPath := jcsypes.JoinObjectPath(o.pathComps[2:]...)
newPath := jcsypes.JoinObjectPath(o.cache.pathComps[2:]...)


if o.isDeleted { if o.isDeleted {
upder.CancelObject(oldPath) upder.CancelObject(oldPath)


+ 3
- 3
client/internal/mount/vfs/cache/file.go View File

@@ -14,7 +14,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/common/utils/serder" "gitlink.org.cn/cloudream/common/utils/serder"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type CacheLevel int type CacheLevel int
@@ -102,7 +102,7 @@ type CacheFile struct {
cache *Cache cache *Cache
pathComps []string pathComps []string
info FileInfo info FileInfo
remoteObj *clitypes.Object
remoteObj *jcsypes.Object
rwLock *sync.RWMutex rwLock *sync.RWMutex
readers []*CacheFileHandle readers []*CacheFileHandle
writers []*CacheFileHandle writers []*CacheFileHandle
@@ -262,7 +262,7 @@ func loadCacheFile(cache *Cache, pathComps []string) (*CacheFile, error) {
return ch, nil return ch, nil
} }


func newCacheFileFromObject(cache *Cache, pathComps []string, obj *clitypes.Object) (*CacheFile, error) {
func newCacheFileFromObject(cache *Cache, pathComps []string, obj *jcsypes.Object) (*CacheFile, error) {
metaPath := cache.GetCacheMetaPath(pathComps...) metaPath := cache.GetCacheMetaPath(pathComps...)
dataPath := cache.GetCacheDataPath(pathComps...) dataPath := cache.GetCacheDataPath(pathComps...)




+ 17
- 17
client/internal/mount/vfs/fuse.go View File

@@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
) )


@@ -35,7 +35,7 @@ func child(vfs *Vfs, ctx context.Context, parent FuseNode, name string) (fuse.Fs
return nil return nil
} }


objPath := clitypes.JoinObjectPath(childPathComps[2:]...)
objPath := jcsypes.JoinObjectPath(childPathComps[2:]...)
obj, err := d.Object().GetByPath(tx, pkg.PackageID, objPath) obj, err := d.Object().GetByPath(tx, pkg.PackageID, objPath)
if err == nil { if err == nil {
ret = newFileFromObject(vfs, childPathComps, obj) ret = newFileFromObject(vfs, childPathComps, obj)
@@ -45,7 +45,7 @@ func child(vfs *Vfs, ctx context.Context, parent FuseNode, name string) (fuse.Fs
return err return err
} }


err = d.Object().HasObjectWithPrefix(tx, pkg.PackageID, objPath+clitypes.ObjectPathSeparator)
err = d.Object().HasObjectWithPrefix(tx, pkg.PackageID, objPath+jcsypes.ObjectPathSeparator)
if err == nil { if err == nil {
dir := vfs.cache.LoadDir(childPathComps, &cache.CreateDirOption{ dir := vfs.cache.LoadDir(childPathComps, &cache.CreateDirOption{
ModTime: time.Now(), ModTime: time.Now(),
@@ -98,10 +98,10 @@ func listChildren(vfs *Vfs, ctx context.Context, parent FuseNode) ([]fuse.FsEntr
return err return err
} }


objPath := clitypes.JoinObjectPath(myPathComps[2:]...)
objPath := jcsypes.JoinObjectPath(myPathComps[2:]...)
objPrefix := objPath objPrefix := objPath
if objPath != "" { if objPath != "" {
objPrefix += clitypes.ObjectPathSeparator
objPrefix += jcsypes.ObjectPathSeparator
} }


objs, coms, err := d.Object().GetByPrefixGrouped(tx, pkg.PackageID, objPrefix) objs, coms, err := d.Object().GetByPrefixGrouped(tx, pkg.PackageID, objPrefix)
@@ -110,8 +110,8 @@ func listChildren(vfs *Vfs, ctx context.Context, parent FuseNode) ([]fuse.FsEntr
} }


for _, dir := range coms { for _, dir := range coms {
dir = strings.TrimSuffix(dir, clitypes.ObjectPathSeparator)
pathComps := lo2.AppendNew(myPathComps, clitypes.BaseName(dir))
dir = strings.TrimSuffix(dir, jcsypes.ObjectPathSeparator)
pathComps := lo2.AppendNew(myPathComps, jcsypes.BaseName(dir))


cd := vfs.cache.LoadDir(pathComps, &cache.CreateDirOption{ cd := vfs.cache.LoadDir(pathComps, &cache.CreateDirOption{
ModTime: time.Now(), ModTime: time.Now(),
@@ -124,7 +124,7 @@ func listChildren(vfs *Vfs, ctx context.Context, parent FuseNode) ([]fuse.FsEntr
} }


for _, obj := range objs { for _, obj := range objs {
pathComps := lo2.AppendNew(myPathComps, clitypes.BaseName(obj.Path))
pathComps := lo2.AppendNew(myPathComps, jcsypes.BaseName(obj.Path))
file := newFileFromObject(vfs, pathComps, obj) file := newFileFromObject(vfs, pathComps, obj)
dbEntries[file.Name()] = file dbEntries[file.Name()] = file
} }
@@ -179,14 +179,14 @@ func newFile(vfs *Vfs, ctx context.Context, name string, parent FuseNode, flags


func removeChild(vfs *Vfs, ctx context.Context, name string, parent FuseNode) error { func removeChild(vfs *Vfs, ctx context.Context, name string, parent FuseNode) error {
pathComps := lo2.AppendNew(parent.PathComps(), name) pathComps := lo2.AppendNew(parent.PathComps(), name)
joinedPath := clitypes.JoinObjectPath(pathComps[2:]...)
joinedPath := jcsypes.JoinObjectPath(pathComps[2:]...)
d := vfs.db d := vfs.db


// TODO 生成系统事件 // TODO 生成系统事件
return vfs.db.DoTx(func(tx db.SQLContext) error { return vfs.db.DoTx(func(tx db.SQLContext) error {
pkg, err := d.Package().GetByFullName(tx, pathComps[0], pathComps[1]) pkg, err := d.Package().GetByFullName(tx, pathComps[0], pathComps[1])
if err == nil { if err == nil {
err := d.Object().HasObjectWithPrefix(tx, pkg.PackageID, joinedPath+clitypes.ObjectPathSeparator)
err := d.Object().HasObjectWithPrefix(tx, pkg.PackageID, joinedPath+jcsypes.ObjectPathSeparator)
if err == nil { if err == nil {
return fuse.ErrNotEmpty return fuse.ErrNotEmpty
} }
@@ -211,7 +211,7 @@ func removeChild(vfs *Vfs, ctx context.Context, name string, parent FuseNode) er
func moveChild(vfs *Vfs, ctx context.Context, oldName string, oldParent FuseNode, newName string, newParent FuseNode) error { func moveChild(vfs *Vfs, ctx context.Context, oldName string, oldParent FuseNode, newName string, newParent FuseNode) error {
newParentPath := newParent.PathComps() newParentPath := newParent.PathComps()
newChildPath := lo2.AppendNew(newParentPath, newName) newChildPath := lo2.AppendNew(newParentPath, newName)
newChildPathJoined := clitypes.JoinObjectPath(newChildPath[2:]...)
newChildPathJoined := jcsypes.JoinObjectPath(newChildPath[2:]...)


// 不允许移动任何内容到Package层级以上 // 不允许移动任何内容到Package层级以上
if len(newParentPath) < 2 { if len(newParentPath) < 2 {
@@ -219,7 +219,7 @@ func moveChild(vfs *Vfs, ctx context.Context, oldName string, oldParent FuseNode
} }


oldChildPath := lo2.AppendNew(oldParent.PathComps(), oldName) oldChildPath := lo2.AppendNew(oldParent.PathComps(), oldName)
oldChildPathJoined := clitypes.JoinObjectPath(oldChildPath[2:]...)
oldChildPathJoined := jcsypes.JoinObjectPath(oldChildPath[2:]...)


// 先更新远程,再更新本地,因为远程使用事务更新,可以回滚,而本地不行 // 先更新远程,再更新本地,因为远程使用事务更新,可以回滚,而本地不行
return vfs.db.DoTx(func(tx db.SQLContext) error { return vfs.db.DoTx(func(tx db.SQLContext) error {
@@ -259,7 +259,7 @@ func moveRemote(vfs *Vfs, tx db.SQLContext, oldChildPath []string, newParentPath
return fuse.ErrExists return fuse.ErrExists
} }


err = d.Object().HasObjectWithPrefix(tx, newPkg.PackageID, newChildPathJoined+clitypes.ObjectPathSeparator)
err = d.Object().HasObjectWithPrefix(tx, newPkg.PackageID, newChildPathJoined+jcsypes.ObjectPathSeparator)
if err == nil { if err == nil {
return fuse.ErrExists return fuse.ErrExists
} }
@@ -283,17 +283,17 @@ func moveRemote(vfs *Vfs, tx db.SQLContext, oldChildPath []string, newParentPath
oldObj.PackageID = newPkg.PackageID oldObj.PackageID = newPkg.PackageID
oldObj.Path = newChildPathJoined oldObj.Path = newChildPathJoined


return d.Object().BatchUpdate(tx, []clitypes.Object{oldObj})
return d.Object().BatchUpdate(tx, []jcsypes.Object{oldObj})
} }
if err != gorm.ErrRecordNotFound { if err != gorm.ErrRecordNotFound {
return err return err
} }


err = d.Object().HasObjectWithPrefix(tx, oldPkg.PackageID, oldChildPathJoined+clitypes.ObjectPathSeparator)
err = d.Object().HasObjectWithPrefix(tx, oldPkg.PackageID, oldChildPathJoined+jcsypes.ObjectPathSeparator)
if err == nil { if err == nil {
return d.Object().MoveByPrefix(tx, return d.Object().MoveByPrefix(tx,
oldPkg.PackageID, oldChildPathJoined+clitypes.ObjectPathSeparator,
newPkg.PackageID, newChildPathJoined+clitypes.ObjectPathSeparator,
oldPkg.PackageID, oldChildPathJoined+jcsypes.ObjectPathSeparator,
newPkg.PackageID, newChildPathJoined+jcsypes.ObjectPathSeparator,
) )
} }
if err == gorm.ErrRecordNotFound { if err == gorm.ErrRecordNotFound {


+ 2
- 2
client/internal/mount/vfs/fuse_bucket.go View File

@@ -9,7 +9,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
) )


@@ -117,7 +117,7 @@ func (r *FuseBucket) listChildren() ([]fuse.FsEntry, error) {
return nil, err return nil, err
} }


pkgMap := make(map[string]*clitypes.Package)
pkgMap := make(map[string]*jcsypes.Package)
for _, pkg := range pkgs { for _, pkg := range pkgs {
p := pkg p := pkg
pkgMap[pkg.Name] = &p pkgMap[pkg.Name] = &p


+ 2
- 2
client/internal/mount/vfs/fuse_dir.go View File

@@ -8,7 +8,7 @@ import (
db2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/db" db2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
) )


@@ -108,7 +108,7 @@ func (r *FuseDir) loadCacheDir() *cache.CacheDir {
return err return err
} }


err = r.vfs.db.Object().HasObjectWithPrefix(tx, pkg.PackageID, clitypes.JoinObjectPath(r.pathComps[2:]...))
err = r.vfs.db.Object().HasObjectWithPrefix(tx, pkg.PackageID, jcsypes.JoinObjectPath(r.pathComps[2:]...))
if err == nil { if err == nil {
createOpt = &cache.CreateDirOption{ createOpt = &cache.CreateDirOption{
ModTime: time.Now(), ModTime: time.Now(),


+ 3
- 3
client/internal/mount/vfs/fuse_file.go View File

@@ -6,7 +6,7 @@ import (


"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
) )


@@ -28,7 +28,7 @@ func newFileFromCache(info cache.CacheEntryInfo, vfs *Vfs) *FuseFileNode {
} }
} }


func newFileFromObject(vfs *Vfs, pathComps []string, obj clitypes.Object) *FuseFileNode {
func newFileFromObject(vfs *Vfs, pathComps []string, obj jcsypes.Object) *FuseFileNode {
return &FuseFileNode{ return &FuseFileNode{
vfs: vfs, vfs: vfs,
pathComps: pathComps, pathComps: pathComps,
@@ -117,7 +117,7 @@ func (n *FuseFileNode) loadCacheFile() *cache.CacheFile {
return n.vfs.cache.LoadFile(n.pathComps, nil) return n.vfs.cache.LoadFile(n.pathComps, nil)
} }


cdsObj, err := n.vfs.db.Object().GetByFullPath(n.vfs.db.DefCtx(), n.pathComps[0], n.pathComps[1], clitypes.JoinObjectPath(n.pathComps[2:]...))
cdsObj, err := n.vfs.db.Object().GetByFullPath(n.vfs.db.DefCtx(), n.pathComps[0], n.pathComps[1], jcsypes.JoinObjectPath(n.pathComps[2:]...))
if err == nil { if err == nil {
file := n.vfs.cache.LoadFile(n.pathComps, &cdsObj) file := n.vfs.cache.LoadFile(n.pathComps, &cdsObj)
if file == nil { if file == nil {


+ 2
- 2
client/internal/mount/vfs/fuse_root.go View File

@@ -8,7 +8,7 @@ import (
db2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/db" db2 "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache" "gitlink.org.cn/cloudream/jcs-pub/client/internal/mount/vfs/cache"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"
) )


@@ -104,7 +104,7 @@ func (r *FuseRoot) listChildren() ([]fuse.FsEntry, error) {
return nil, err return nil, err
} }


bktMap := make(map[string]*clitypes.Bucket)
bktMap := make(map[string]*jcsypes.Bucket)
for _, bkt := range bkts { for _, bkt := range bkts {
b := bkt b := bkt
bktMap[bkt.Name] = &b bktMap[bkt.Name] = &b


+ 1
- 1
client/internal/repl/getp.go View File

@@ -13,7 +13,7 @@ import (
"github.com/inhies/go-bytesize" "github.com/inhies/go-bytesize"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/config" "gitlink.org.cn/cloudream/jcs-pub/client/internal/config"
cdssdk "gitlink.org.cn/cloudream/jcs-pub/client/types"
cdssdk "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/iterator" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/iterator"
) )




+ 4
- 4
client/internal/repl/load.go View File

@@ -8,7 +8,7 @@ import (
"time" "time"


"github.com/spf13/cobra" "github.com/spf13/cobra"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -31,7 +31,7 @@ func init() {
fmt.Printf("Invalid user space ID: %s\n", args[1]) fmt.Printf("Invalid user space ID: %s\n", args[1])
} }


loadByID(cmdCtx, clitypes.PackageID(pkgID), clitypes.UserSpaceID(userSpaceID), args[2])
loadByID(cmdCtx, jcsypes.PackageID(pkgID), jcsypes.UserSpaceID(userSpaceID), args[2])
} else { } else {
loadByPath(cmdCtx, args[0], args[1], args[2]) loadByPath(cmdCtx, args[0], args[1], args[2])
} }
@@ -42,7 +42,7 @@ func init() {
} }


func loadByPath(cmdCtx *CommandContext, pkgPath string, stgName string, rootPath string) { func loadByPath(cmdCtx *CommandContext, pkgPath string, stgName string, rootPath string) {
comps := strings.Split(strings.Trim(pkgPath, clitypes.ObjectPathSeparator), clitypes.ObjectPathSeparator)
comps := strings.Split(strings.Trim(pkgPath, jcsypes.ObjectPathSeparator), jcsypes.ObjectPathSeparator)
if len(comps) != 2 { if len(comps) != 2 {
fmt.Printf("Package path must be in format of <bucket>/<package>") fmt.Printf("Package path must be in format of <bucket>/<package>")
return return
@@ -63,7 +63,7 @@ func loadByPath(cmdCtx *CommandContext, pkgPath string, stgName string, rootPath
loadByID(cmdCtx, pkg.PackageID, stg.StorageID, rootPath) loadByID(cmdCtx, pkg.PackageID, stg.StorageID, rootPath)
} }


func loadByID(cmdCtx *CommandContext, pkgID clitypes.PackageID, stgID clitypes.StorageID, rootPath string) {
func loadByID(cmdCtx *CommandContext, pkgID jcsypes.PackageID, stgID jcsypes.StorageID, rootPath string) {
startTime := time.Now() startTime := time.Now()


err := cmdCtx.Cmdline.Svc.StorageSvc().LoadPackage(pkgID, stgID, rootPath) err := cmdCtx.Cmdline.Svc.StorageSvc().LoadPackage(pkgID, stgID, rootPath)


+ 4
- 4
client/internal/repl/lsp.go View File

@@ -7,7 +7,7 @@ import (


"github.com/jedib0t/go-pretty/v6/table" "github.com/jedib0t/go-pretty/v6/table"
"github.com/spf13/cobra" "github.com/spf13/cobra"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -26,7 +26,7 @@ func init() {
return return
} }


lspOneByID(cmdCtx, clitypes.PackageID(id))
lspOneByID(cmdCtx, jcsypes.PackageID(id))
} else { } else {
lspByPath(cmdCtx, args[0]) lspByPath(cmdCtx, args[0])
} }
@@ -40,7 +40,7 @@ func init() {
func lspByPath(cmdCtx *CommandContext, path string) { func lspByPath(cmdCtx *CommandContext, path string) {
db2 := cmdCtx.repl.db db2 := cmdCtx.repl.db


comps := strings.Split(strings.Trim(path, clitypes.ObjectPathSeparator), clitypes.ObjectPathSeparator)
comps := strings.Split(strings.Trim(path, jcsypes.ObjectPathSeparator), jcsypes.ObjectPathSeparator)
if len(comps) != 2 { if len(comps) != 2 {
fmt.Printf("Package path must be in format of <bucket>/<package>") fmt.Printf("Package path must be in format of <bucket>/<package>")
return return
@@ -58,7 +58,7 @@ func lspByPath(cmdCtx *CommandContext, path string) {
fmt.Println(wr.Render()) fmt.Println(wr.Render())
} }


func lspOneByID(cmdCtx *CommandContext, id clitypes.PackageID) {
func lspOneByID(cmdCtx *CommandContext, id jcsypes.PackageID) {
db2 := cmdCtx.repl.db db2 := cmdCtx.repl.db


pkg, err := db2.Package().GetByID(db2.DefCtx(), id) pkg, err := db2.Package().GetByID(db2.DefCtx(), id)


+ 1
- 1
client/internal/repl/newloadp.go View File

@@ -10,7 +10,7 @@ import (


"github.com/jedib0t/go-pretty/v6/table" "github.com/jedib0t/go-pretty/v6/table"
"github.com/spf13/cobra" "github.com/spf13/cobra"
cdssdk "gitlink.org.cn/cloudream/jcs-pub/client/types"
cdssdk "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {


+ 1
- 1
client/internal/repl/put.go View File

@@ -12,7 +12,7 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"gitlink.org.cn/cloudream/common/consts/errorcode" "gitlink.org.cn/cloudream/common/consts/errorcode"
"gitlink.org.cn/cloudream/common/pkgs/mq" "gitlink.org.cn/cloudream/common/pkgs/mq"
cdssdk "gitlink.org.cn/cloudream/jcs-pub/client/types"
cdssdk "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {


+ 1
- 1
client/internal/services/bucket.go View File

@@ -3,7 +3,7 @@ package services
import ( import (
"time" "time"


"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


// BucketService 是对存储桶进行操作的服务类 // BucketService 是对存储桶进行操作的服务类


+ 2
- 2
client/internal/services/object.go View File

@@ -12,12 +12,12 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader" "gitlink.org.cn/cloudream/jcs-pub/client/internal/downloader"
"gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1" "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api/v1"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/models/datamap"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/plans" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/plans"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
"gorm.io/gorm" "gorm.io/gorm"
) )




+ 2
- 2
client/internal/services/package.go View File

@@ -7,8 +7,8 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"


"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/models/datamap"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
) )


// PackageService 提供对包相关操作的服务接口 // PackageService 提供对包相关操作的服务接口


+ 26
- 26
client/internal/services/user_space.go View File

@@ -7,8 +7,8 @@ import (


"github.com/samber/lo" "github.com/samber/lo"
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gorm.io/gorm" "gorm.io/gorm"


"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
@@ -31,41 +31,41 @@ func (svc *Service) UserSpaceSvc() *UserSpaceService {
return &UserSpaceService{Service: svc} return &UserSpaceService{Service: svc}
} }


func (svc *UserSpaceService) Get(userspaceID clitypes.UserSpaceID) (clitypes.UserSpace, error) {
func (svc *UserSpaceService) Get(userspaceID jcsypes.UserSpaceID) (jcsypes.UserSpace, error) {
return svc.DB.UserSpace().GetByID(svc.DB.DefCtx(), userspaceID) return svc.DB.UserSpace().GetByID(svc.DB.DefCtx(), userspaceID)
} }


func (svc *UserSpaceService) GetByName(name string) (clitypes.UserSpace, error) {
func (svc *UserSpaceService) GetByName(name string) (jcsypes.UserSpace, error) {
return svc.DB.UserSpace().GetByName(svc.DB.DefCtx(), name) return svc.DB.UserSpace().GetByName(svc.DB.DefCtx(), name)
} }


func (svc *UserSpaceService) GetAll() ([]clitypes.UserSpace, error) {
func (svc *UserSpaceService) GetAll() ([]jcsypes.UserSpace, error) {
return svc.DB.UserSpace().GetAll(svc.DB.DefCtx()) return svc.DB.UserSpace().GetAll(svc.DB.DefCtx())
} }


func (svc *UserSpaceService) Create(req cliapi.UserSpaceCreate) (*cliapi.UserSpaceCreateResp, *ecode.CodeError) { func (svc *UserSpaceService) Create(req cliapi.UserSpaceCreate) (*cliapi.UserSpaceCreateResp, *ecode.CodeError) {
db2 := svc.DB db2 := svc.DB
space, err := db.DoTx01(db2, func(tx db.SQLContext) (clitypes.UserSpace, error) {
space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcsypes.UserSpace, error) {
space, err := db2.UserSpace().GetByName(tx, req.Name) space, err := db2.UserSpace().GetByName(tx, req.Name)
if err == nil { if err == nil {
return clitypes.UserSpace{}, gorm.ErrDuplicatedKey
return jcsypes.UserSpace{}, gorm.ErrDuplicatedKey
} }
if err != gorm.ErrRecordNotFound { if err != gorm.ErrRecordNotFound {
return clitypes.UserSpace{}, err
return jcsypes.UserSpace{}, err
} }


space = clitypes.UserSpace{
space = jcsypes.UserSpace{
Name: req.Name, Name: req.Name,
Storage: req.Storage, Storage: req.Storage,
Credential: req.Credential, Credential: req.Credential,
ShardStore: req.ShardStore, ShardStore: req.ShardStore,
Features: req.Features, Features: req.Features,
WorkingDir: clitypes.PathFromJcsPathString(req.WorkingDir),
WorkingDir: jcsypes.PathFromJcsPathString(req.WorkingDir),
Revision: 0, Revision: 0,
} }
err = db2.UserSpace().Create(tx, &space) err = db2.UserSpace().Create(tx, &space)
if err != nil { if err != nil {
return clitypes.UserSpace{}, err
return jcsypes.UserSpace{}, err
} }
return space, nil return space, nil
}) })
@@ -80,19 +80,19 @@ func (svc *UserSpaceService) Create(req cliapi.UserSpaceCreate) (*cliapi.UserSpa


func (svc *UserSpaceService) Update(req cliapi.UserSpaceUpdate) (*cliapi.UserSpaceUpdateResp, *ecode.CodeError) { func (svc *UserSpaceService) Update(req cliapi.UserSpaceUpdate) (*cliapi.UserSpaceUpdateResp, *ecode.CodeError) {
db2 := svc.DB db2 := svc.DB
space, err := db.DoTx01(db2, func(tx db.SQLContext) (clitypes.UserSpace, error) {
space, err := db.DoTx01(db2, func(tx db.SQLContext) (jcsypes.UserSpace, error) {
space, err := db2.UserSpace().GetByID(tx, req.UserSpaceID) space, err := db2.UserSpace().GetByID(tx, req.UserSpaceID)
if err != nil { if err != nil {
return clitypes.UserSpace{}, err
return jcsypes.UserSpace{}, err
} }


if space.Name != req.Name { if space.Name != req.Name {
_, err = db2.UserSpace().GetByName(tx, req.Name) _, err = db2.UserSpace().GetByName(tx, req.Name)
if err == nil { if err == nil {
return clitypes.UserSpace{}, gorm.ErrDuplicatedKey
return jcsypes.UserSpace{}, gorm.ErrDuplicatedKey
} }
if err != gorm.ErrRecordNotFound { if err != gorm.ErrRecordNotFound {
return clitypes.UserSpace{}, err
return jcsypes.UserSpace{}, err
} }
} }


@@ -110,7 +110,7 @@ func (svc *UserSpaceService) Update(req cliapi.UserSpaceUpdate) (*cliapi.UserSpa
} }


// 通知元数据缓存无效 // 通知元数据缓存无效
svc.UserSpaceMeta.Drop([]clitypes.UserSpaceID{req.UserSpaceID})
svc.UserSpaceMeta.Drop([]jcsypes.UserSpaceID{req.UserSpaceID})


// 通知存储服务组件池停止组件。TODO 对于在Hub上运行的组件,需要一个机制去定时清理 // 通知存储服务组件池停止组件。TODO 对于在Hub上运行的组件,需要一个机制去定时清理
svc.StgPool.Drop(stgglb.UserID, space.UserSpaceID) svc.StgPool.Drop(stgglb.UserID, space.UserSpaceID)
@@ -155,7 +155,7 @@ func (svc *UserSpaceService) Delete(req cliapi.UserSpaceDelete) (*cliapi.UserSpa
} }


// 通知元数据缓存无效 // 通知元数据缓存无效
svc.UserSpaceMeta.Drop([]clitypes.UserSpaceID{req.UserSpaceID})
svc.UserSpaceMeta.Drop([]jcsypes.UserSpaceID{req.UserSpaceID})


// 通知存储服务组件池停止组件。TODO 对于在Hub上运行的组件,需要一个机制去定时清理 // 通知存储服务组件池停止组件。TODO 对于在Hub上运行的组件,需要一个机制去定时清理
svc.StgPool.Drop(stgglb.UserID, req.UserSpaceID) svc.StgPool.Drop(stgglb.UserID, req.UserSpaceID)
@@ -166,13 +166,13 @@ func (svc *UserSpaceService) Delete(req cliapi.UserSpaceDelete) (*cliapi.UserSpa
} }


func (svc *UserSpaceService) Test(req cliapi.UserSpaceTest) (*cliapi.UserSpaceTestResp, *ecode.CodeError) { func (svc *UserSpaceService) Test(req cliapi.UserSpaceTest) (*cliapi.UserSpaceTestResp, *ecode.CodeError) {
detail := clitypes.UserSpaceDetail{
detail := jcsypes.UserSpaceDetail{
UserID: stgglb.UserID, UserID: stgglb.UserID,
UserSpace: clitypes.UserSpace{
UserSpace: jcsypes.UserSpace{
Name: "test", Name: "test",
Storage: req.Storage, Storage: req.Storage,
Credential: req.Credential, Credential: req.Credential,
WorkingDir: clitypes.PathFromJcsPathString(req.WorikingDir),
WorkingDir: jcsypes.PathFromJcsPathString(req.WorikingDir),
}, },
} }
blder := factory.GetBuilder(&detail) blder := factory.GetBuilder(&detail)
@@ -189,7 +189,7 @@ func (svc *UserSpaceService) Test(req cliapi.UserSpaceTest) (*cliapi.UserSpaceTe
return &cliapi.UserSpaceTestResp{}, nil return &cliapi.UserSpaceTestResp{}, nil
} }


func (svc *UserSpaceService) DownloadPackage(packageID clitypes.PackageID, userspaceID clitypes.UserSpaceID, rootPath string) error {
func (svc *UserSpaceService) DownloadPackage(packageID jcsypes.PackageID, userspaceID jcsypes.UserSpaceID, rootPath string) error {
destSpace := svc.UserSpaceMeta.Get(userspaceID) destSpace := svc.UserSpaceMeta.Get(userspaceID)
if destSpace == nil { if destSpace == nil {
return fmt.Errorf("userspace not found: %d", userspaceID) return fmt.Errorf("userspace not found: %d", userspaceID)
@@ -208,10 +208,10 @@ func (svc *UserSpaceService) DownloadPackage(packageID clitypes.PackageID, users
} }
defer mutex.Unlock() defer mutex.Unlock()


rootJPath := clitypes.PathFromJcsPathString(rootPath)
rootJPath := jcsypes.PathFromJcsPathString(rootPath)


dIndex := 0 dIndex := 0
var pinned []clitypes.PinnedObject
var pinned []jcsypes.PinnedObject
for dIndex < len(details) { for dIndex < len(details) {
plans := exec.NewPlanBuilder() plans := exec.NewPlanBuilder()
for i := 0; i < 10 && dIndex < len(details); i++ { for i := 0; i < 10 && dIndex < len(details); i++ {
@@ -252,7 +252,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID clitypes.PackageID, users
return fmt.Errorf("unsupported download strategy: %T", strg) return fmt.Errorf("unsupported download strategy: %T", strg)
} }


objPath := clitypes.PathFromJcsPathString(details[dIndex].Object.Path)
objPath := jcsypes.PathFromJcsPathString(details[dIndex].Object.Path)
dstPath := rootJPath.ConcatNew(objPath) dstPath := rootJPath.ConcatNew(objPath)


newDstSpace := *destSpace newDstSpace := *destSpace
@@ -265,7 +265,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID clitypes.PackageID, users
// 顺便保存到同存储服务的分片存储中 // 顺便保存到同存储服务的分片存储中
if destSpace.UserSpace.ShardStore != nil { if destSpace.UserSpace.ShardStore != nil {
ft.AddTo(ioswitch2.NewToShardStore(newDstSpace, ioswitch2.RawStream(), "")) ft.AddTo(ioswitch2.NewToShardStore(newDstSpace, ioswitch2.RawStream(), ""))
pinned = append(pinned, clitypes.PinnedObject{
pinned = append(pinned, jcsypes.PinnedObject{
ObjectID: details[dIndex].Object.ObjectID, ObjectID: details[dIndex].Object.ObjectID,
UserSpaceID: destSpace.UserSpace.UserSpaceID, UserSpaceID: destSpace.UserSpace.UserSpaceID,
CreateTime: time.Now(), CreateTime: time.Now(),
@@ -297,7 +297,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID clitypes.PackageID, users
} }


err = svc.DB.DoTx(func(tx db.SQLContext) error { err = svc.DB.DoTx(func(tx db.SQLContext) error {
objIDs := make([]clitypes.ObjectID, len(pinned))
objIDs := make([]jcsypes.ObjectID, len(pinned))
for i, obj := range pinned { for i, obj := range pinned {
objIDs[i] = obj.ObjectID objIDs[i] = obj.ObjectID
} }
@@ -307,7 +307,7 @@ func (svc *UserSpaceService) DownloadPackage(packageID clitypes.PackageID, users
return err return err
} }


pinned = lo.Filter(pinned, func(p clitypes.PinnedObject, idx int) bool { return avaiIDs[p.ObjectID] })
pinned = lo.Filter(pinned, func(p jcsypes.PinnedObject, idx int) bool { return avaiIDs[p.ObjectID] })
return svc.DB.PinnedObject().BatchTryCreate(svc.DB.DefCtx(), pinned) return svc.DB.PinnedObject().BatchTryCreate(svc.DB.DefCtx(), pinned)
}) })
if err != nil { if err != nil {


+ 7
- 7
client/internal/services/utils.go View File

@@ -1,22 +1,22 @@
package services package services


import ( import (
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/models/datamap"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
) )


func getBlockTypeFromRed(red clitypes.Redundancy) string {
func getBlockTypeFromRed(red jcsypes.Redundancy) string {
switch red.(type) { switch red.(type) {
case *clitypes.NoneRedundancy:
case *jcsypes.NoneRedundancy:
return datamap.BlockTypeRaw return datamap.BlockTypeRaw


case *clitypes.ECRedundancy:
case *jcsypes.ECRedundancy:
return datamap.BlockTypeEC return datamap.BlockTypeEC


case *clitypes.LRCRedundancy:
case *jcsypes.LRCRedundancy:
return datamap.BlockTypeEC return datamap.BlockTypeEC


case *clitypes.SegmentRedundancy:
case *jcsypes.SegmentRedundancy:
return datamap.BlockTypeSegment return datamap.BlockTypeSegment
} }
return "" return ""


+ 1
- 1
client/internal/spacesyncer/execute.go View File

@@ -2,8 +2,8 @@ package spacesyncer


import ( import (
"gitlink.org.cn/cloudream/common/pkgs/trie" "gitlink.org.cn/cloudream/common/pkgs/trie"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func execute(syncer *SpaceSyncer, task *task) { func execute(syncer *SpaceSyncer, task *task) {


+ 7
- 7
client/internal/spacesyncer/execute_diff.go View File

@@ -8,14 +8,14 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/pkgs/trie" "gitlink.org.cn/cloudream/common/pkgs/trie"
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func executeDiff(syncer *SpaceSyncer, task *task, mode *clitypes.SpaceSyncModeDiff) {
func executeDiff(syncer *SpaceSyncer, task *task, mode *jcsypes.SpaceSyncModeDiff) {
log := logger.WithField("Mod", logMod).WithField("TaskID", task.Task.TaskID) log := logger.WithField("Mod", logMod).WithField("TaskID", task.Task.TaskID)


startTime := time.Now() startTime := time.Now()
@@ -112,7 +112,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *clitypes.SpaceSyncModeDi
} }


var willSync []stgtypes.DirEntry var willSync []stgtypes.DirEntry
var willMkdirs []clitypes.JPath
var willMkdirs []jcsypes.JPath


dirTree.Iterate(func(path []string, node *trie.Node[srcDstDirEntry], isWordNode bool) trie.VisitCtrl { dirTree.Iterate(func(path []string, node *trie.Node[srcDstDirEntry], isWordNode bool) trie.VisitCtrl {
if node.Value.src == nil { if node.Value.src == nil {
@@ -123,7 +123,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *clitypes.SpaceSyncModeDi
if node.Value.src.IsDir { if node.Value.src.IsDir {
if node.Value.dst == nil { if node.Value.dst == nil {
if node.IsEmpty() { if node.IsEmpty() {
willMkdirs = append(willMkdirs, clitypes.PathFromComps(path...))
willMkdirs = append(willMkdirs, jcsypes.PathFromComps(path...))
} }
} }
} else { } else {
@@ -193,7 +193,7 @@ func executeDiff(syncer *SpaceSyncer, task *task, mode *clitypes.SpaceSyncModeDi
} }
} }


func diffCreateSrcNode(tree *trie.Trie[srcDstDirEntry], path clitypes.JPath, e *stgtypes.DirEntry) {
func diffCreateSrcNode(tree *trie.Trie[srcDstDirEntry], path jcsypes.JPath, e *stgtypes.DirEntry) {
var ptr = &tree.Root var ptr = &tree.Root
for _, c := range path.Comps() { for _, c := range path.Comps() {
if ptr.Value.src != nil && ptr.Value.src.IsDir { if ptr.Value.src != nil && ptr.Value.src.IsDir {
@@ -205,7 +205,7 @@ func diffCreateSrcNode(tree *trie.Trie[srcDstDirEntry], path clitypes.JPath, e *
ptr.Value.src = e ptr.Value.src = e
} }


func diffCreateDstNode(tree *trie.Trie[srcDstDirEntry], path clitypes.JPath, e *stgtypes.DirEntry) {
func diffCreateDstNode(tree *trie.Trie[srcDstDirEntry], path jcsypes.JPath, e *stgtypes.DirEntry) {
var ptr = &tree.Root var ptr = &tree.Root
for _, c := range path.Comps() { for _, c := range path.Comps() {
if ptr.Value.src != nil && ptr.Value.src.IsDir { if ptr.Value.src != nil && ptr.Value.src.IsDir {
@@ -227,7 +227,7 @@ type srcDstDirEntry struct {
dst *stgtypes.DirEntry dst *stgtypes.DirEntry
} }


func cmpFile(diff *clitypes.SpaceSyncModeDiff, src, dst *stgtypes.DirEntry) bool {
func cmpFile(diff *jcsypes.SpaceSyncModeDiff, src, dst *stgtypes.DirEntry) bool {
if diff.IncludeSize && src.Size != dst.Size { if diff.IncludeSize && src.Size != dst.Size {
return false return false
} }


+ 1
- 1
client/internal/spacesyncer/execute_full.go View File

@@ -8,11 +8,11 @@ import (


"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/pkgs/trie" "gitlink.org.cn/cloudream/common/pkgs/trie"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func executeFull(syncer *SpaceSyncer, task *task) { func executeFull(syncer *SpaceSyncer, task *task) {


+ 3
- 3
client/internal/spacesyncer/filter.go View File

@@ -1,8 +1,8 @@
package spacesyncer package spacesyncer


import ( import (
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type FilterFn func(info stgtypes.DirEntry) bool type FilterFn func(info stgtypes.DirEntry) bool
@@ -11,7 +11,7 @@ func buildFilter(task *task) FilterFn {
var fns []FilterFn var fns []FilterFn
for _, f := range task.Task.Filters { for _, f := range task.Task.Filters {
switch f := f.(type) { switch f := f.(type) {
case *clitypes.SpaceSyncFilterSize:
case *jcsypes.SpaceSyncFilterSize:
fns = append(fns, filterSize(f)) fns = append(fns, filterSize(f))
} }
} }
@@ -26,7 +26,7 @@ func buildFilter(task *task) FilterFn {
} }
} }


func filterSize(filter *clitypes.SpaceSyncFilterSize) FilterFn {
func filterSize(filter *jcsypes.SpaceSyncFilterSize) FilterFn {
return func(info stgtypes.DirEntry) bool { return func(info stgtypes.DirEntry) bool {
if filter.MinSize > 0 && info.Size < filter.MinSize { if filter.MinSize > 0 && info.Size < filter.MinSize {
return false return false


+ 16
- 16
client/internal/spacesyncer/space_syncer.go View File

@@ -9,8 +9,8 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/metacache" "gitlink.org.cn/cloudream/jcs-pub/client/internal/metacache"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
stgpool "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" stgpool "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


const ( const (
@@ -26,7 +26,7 @@ type SpaceSyncer struct {
stgPool *stgpool.Pool stgPool *stgpool.Pool
spaceMeta *metacache.UserSpaceMeta spaceMeta *metacache.UserSpaceMeta
lock sync.Mutex lock sync.Mutex
tasks map[clitypes.SpaceSyncTaskID]*task
tasks map[jcsypes.SpaceSyncTaskID]*task
} }


func New(db *db.DB, stgPool *stgpool.Pool, spaceMeta *metacache.UserSpaceMeta) *SpaceSyncer { func New(db *db.DB, stgPool *stgpool.Pool, spaceMeta *metacache.UserSpaceMeta) *SpaceSyncer {
@@ -34,7 +34,7 @@ func New(db *db.DB, stgPool *stgpool.Pool, spaceMeta *metacache.UserSpaceMeta) *
db: db, db: db,
stgPool: stgPool, stgPool: stgPool,
spaceMeta: spaceMeta, spaceMeta: spaceMeta,
tasks: make(map[clitypes.SpaceSyncTaskID]*task),
tasks: make(map[jcsypes.SpaceSyncTaskID]*task),
} }
} }


@@ -50,7 +50,7 @@ func (s *SpaceSyncer) Start() *async.UnboundChannel[SpaceSyncerEvent] {
if err != nil { if err != nil {
log.Warnf("load task from db: %v", err) log.Warnf("load task from db: %v", err)
} else { } else {
var rms []clitypes.SpaceSyncTaskID
var rms []jcsypes.SpaceSyncTaskID
for _, t := range allTask { for _, t := range allTask {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
tsk := task{ tsk := task{
@@ -60,14 +60,14 @@ func (s *SpaceSyncer) Start() *async.UnboundChannel[SpaceSyncerEvent] {
} }


switch tr := t.Trigger.(type) { switch tr := t.Trigger.(type) {
case *clitypes.SpaceSyncTriggerOnce:
case *jcsypes.SpaceSyncTriggerOnce:
// Once类型的任务没有执行完也不执行了 // Once类型的任务没有执行完也不执行了
rms = append(rms, t.TaskID) rms = append(rms, t.TaskID)


case *clitypes.SpaceSyncTriggerInterval:
case *jcsypes.SpaceSyncTriggerInterval:
triggerInterval(s, &tsk, tr) triggerInterval(s, &tsk, tr)


case *clitypes.SpaceSyncTriggerAt:
case *jcsypes.SpaceSyncTriggerAt:
triggerAt(s, &tsk, tr) triggerAt(s, &tsk, tr)
} }


@@ -95,10 +95,10 @@ func (s *SpaceSyncer) Stop() {
t.CancelFn() t.CancelFn()
} }


s.tasks = make(map[clitypes.SpaceSyncTaskID]*task)
s.tasks = make(map[jcsypes.SpaceSyncTaskID]*task)
} }


func (s *SpaceSyncer) CreateTask(t clitypes.SpaceSyncTask) (*TaskInfo, error) {
func (s *SpaceSyncer) CreateTask(t jcsypes.SpaceSyncTask) (*TaskInfo, error) {
log := logger.WithField("Mod", logMod) log := logger.WithField("Mod", logMod)


d := s.db d := s.db
@@ -126,13 +126,13 @@ func (s *SpaceSyncer) CreateTask(t clitypes.SpaceSyncTask) (*TaskInfo, error) {
s.lock.Unlock() s.lock.Unlock()


switch tr := t.Trigger.(type) { switch tr := t.Trigger.(type) {
case *clitypes.SpaceSyncTriggerOnce:
case *jcsypes.SpaceSyncTriggerOnce:
triggerOnce(s, &tsk) triggerOnce(s, &tsk)


case *clitypes.SpaceSyncTriggerInterval:
case *jcsypes.SpaceSyncTriggerInterval:
triggerInterval(s, &tsk, tr) triggerInterval(s, &tsk, tr)


case *clitypes.SpaceSyncTriggerAt:
case *jcsypes.SpaceSyncTriggerAt:
triggerAt(s, &tsk, tr) triggerAt(s, &tsk, tr)
} }


@@ -143,7 +143,7 @@ func (s *SpaceSyncer) CreateTask(t clitypes.SpaceSyncTask) (*TaskInfo, error) {
}, nil }, nil
} }


func (s *SpaceSyncer) CancelTask(taskID clitypes.SpaceSyncTaskID) {
func (s *SpaceSyncer) CancelTask(taskID jcsypes.SpaceSyncTaskID) {
log := logger.WithField("Mod", logMod) log := logger.WithField("Mod", logMod)


s.lock.Lock() s.lock.Lock()
@@ -166,7 +166,7 @@ func (s *SpaceSyncer) CancelTask(taskID clitypes.SpaceSyncTaskID) {
log.Infof("task %v canceled", taskID) log.Infof("task %v canceled", taskID)
} }


func (s *SpaceSyncer) GetTask(taskID clitypes.SpaceSyncTaskID) *clitypes.SpaceSyncTask {
func (s *SpaceSyncer) GetTask(taskID jcsypes.SpaceSyncTaskID) *jcsypes.SpaceSyncTask {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()


@@ -181,11 +181,11 @@ func (s *SpaceSyncer) GetTask(taskID clitypes.SpaceSyncTaskID) *clitypes.SpaceSy
} }


type TaskInfo struct { type TaskInfo struct {
Task clitypes.SpaceSyncTask
Task jcsypes.SpaceSyncTask
} }


type task struct { type task struct {
Task clitypes.SpaceSyncTask
Task jcsypes.SpaceSyncTask
Context context.Context Context context.Context
CancelFn func() CancelFn func()
} }

+ 3
- 3
client/internal/spacesyncer/trigger.go View File

@@ -5,7 +5,7 @@ import (


"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/sort2" "gitlink.org.cn/cloudream/common/utils/sort2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func triggerOnce(syncer *SpaceSyncer, task *task) { func triggerOnce(syncer *SpaceSyncer, task *task) {
@@ -31,7 +31,7 @@ func triggerOnce(syncer *SpaceSyncer, task *task) {
}() }()
} }


func triggerInterval(syncer *SpaceSyncer, task *task, trigger *clitypes.SpaceSyncTriggerInterval) {
func triggerInterval(syncer *SpaceSyncer, task *task, trigger *jcsypes.SpaceSyncTriggerInterval) {
go func() { go func() {
log := logger.WithField("Mod", logMod) log := logger.WithField("Mod", logMod)


@@ -66,7 +66,7 @@ func triggerInterval(syncer *SpaceSyncer, task *task, trigger *clitypes.SpaceSyn
}() }()
} }


func triggerAt(syncer *SpaceSyncer, task *task, trigger *clitypes.SpaceSyncTriggerAt) {
func triggerAt(syncer *SpaceSyncer, task *task, trigger *jcsypes.SpaceSyncTriggerAt) {
go func() { go func() {
log := logger.WithField("Mod", logMod) log := logger.WithField("Mod", logMod)




+ 6
- 6
client/internal/speedstats/speedstats.go View File

@@ -5,7 +5,7 @@ import (
"sync" "sync"
"time" "time"


clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


const ( const (
@@ -121,9 +121,9 @@ func (p *SpeedStats) ShouldAtClient(size int64) bool {
return v < prob return v < prob
} }


func (p *SpeedStats) DumpStatus() clitypes.SpeedStatsStatus {
return clitypes.SpeedStatsStatus{
Below100M: []clitypes.SpeedStatsStatusEntry{
func (p *SpeedStats) DumpStatus() jcsypes.SpeedStatsStatus {
return jcsypes.SpeedStatsStatus{
Below100M: []jcsypes.SpeedStatsStatusEntry{
{ {
TotalSize: p.stats100M[0].TotalSize, TotalSize: p.stats100M[0].TotalSize,
TotalTime: p.stats100M[0].TotalTime, TotalTime: p.stats100M[0].TotalTime,
@@ -137,7 +137,7 @@ func (p *SpeedStats) DumpStatus() clitypes.SpeedStatsStatus {
LastSpeed: p.stats100M[1].LastSpeed, LastSpeed: p.stats100M[1].LastSpeed,
}, },
}, },
Below1G: []clitypes.SpeedStatsStatusEntry{
Below1G: []jcsypes.SpeedStatsStatusEntry{
{ {
TotalSize: p.stats1G[0].TotalSize, TotalSize: p.stats1G[0].TotalSize,
TotalTime: p.stats1G[0].TotalTime, TotalTime: p.stats1G[0].TotalTime,
@@ -151,7 +151,7 @@ func (p *SpeedStats) DumpStatus() clitypes.SpeedStatsStatus {
LastSpeed: p.stats1G[1].LastSpeed, LastSpeed: p.stats1G[1].LastSpeed,
}, },
}, },
Above1G: []clitypes.SpeedStatsStatusEntry{
Above1G: []jcsypes.SpeedStatsStatusEntry{
{ {
TotalSize: p.statsAbove1G[0].TotalSize, TotalSize: p.statsAbove1G[0].TotalSize,
TotalTime: p.statsAbove1G[0].TotalTime, TotalTime: p.statsAbove1G[0].TotalTime,


+ 10
- 10
client/internal/ticktock/change_redundancy.go View File

@@ -7,9 +7,9 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/reflect2" "gitlink.org.cn/cloudream/common/utils/reflect2"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/models/datamap"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
) )


const ( const (
@@ -34,7 +34,7 @@ func (j *ChangeRedundancy) Execute(t *TickTock) {


ctx := &changeRedundancyContext{ ctx := &changeRedundancyContext{
ticktock: t, ticktock: t,
allUserSpaces: make(map[clitypes.UserSpaceID]*userSpaceUsageInfo),
allUserSpaces: make(map[jcsypes.UserSpaceID]*userSpaceUsageInfo),
} }


spaceIDs, err := t.db.UserSpace().GetAllIDs(t.db.DefCtx()) spaceIDs, err := t.db.UserSpace().GetAllIDs(t.db.DefCtx())
@@ -58,7 +58,7 @@ func (j *ChangeRedundancy) Execute(t *TickTock) {
return return
} }


lastPkgID := clitypes.PackageID(0)
lastPkgID := jcsypes.PackageID(0)


loop: loop:
for { for {
@@ -90,16 +90,16 @@ loop:


type changeRedundancyContext struct { type changeRedundancyContext struct {
ticktock *TickTock ticktock *TickTock
allUserSpaces map[clitypes.UserSpaceID]*userSpaceUsageInfo
mostBlockStgIDs []clitypes.UserSpaceID
allUserSpaces map[jcsypes.UserSpaceID]*userSpaceUsageInfo
mostBlockStgIDs []jcsypes.UserSpaceID
} }


type userSpaceUsageInfo struct { type userSpaceUsageInfo struct {
UserSpace *clitypes.UserSpaceDetail
UserSpace *jcsypes.UserSpaceDetail
AccessAmount float64 AccessAmount float64
} }


func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg clitypes.PackageDetail) error {
func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg jcsypes.PackageDetail) error {
log := logger.WithType[ChangeRedundancy]("TickTock") log := logger.WithType[ChangeRedundancy]("TickTock")
db2 := ctx.ticktock.db db2 := ctx.ticktock.db


@@ -121,7 +121,7 @@ func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg clitypes.
info.AccessAmount = stat.Amount info.AccessAmount = stat.Amount
} }


lastObjID := clitypes.ObjectID(0)
lastObjID := jcsypes.ObjectID(0)
for { for {
objs, err := db.DoTx31(db2, db2.Object().BatchGetDetailsPaged, pkg.Package.PackageID, lastObjID, BatchGetObjectDetailCount) objs, err := db.DoTx31(db2, db2.Object().BatchGetDetailsPaged, pkg.Package.PackageID, lastObjID, BatchGetObjectDetailCount)
if err != nil { if err != nil {
@@ -139,7 +139,7 @@ func (j *ChangeRedundancy) changeOne(ctx *changeRedundancyContext, pkg clitypes.


ctx.mostBlockStgIDs = j.summaryRepObjectBlockUserSpaces(ctx, objs, 2) ctx.mostBlockStgIDs = j.summaryRepObjectBlockUserSpaces(ctx, objs, 2)


var willShrinks []clitypes.ObjectDetail
var willShrinks []jcsypes.ObjectDetail


for _, obj := range objs { for _, obj := range objs {
newRed, selectedSpaces := j.chooseRedundancy(ctx, obj) newRed, selectedSpaces := j.chooseRedundancy(ctx, obj)


+ 8
- 8
client/internal/ticktock/check_shardstore.go View File

@@ -9,8 +9,8 @@ import (
"gitlink.org.cn/cloudream/common/utils/reflect2" "gitlink.org.cn/cloudream/common/utils/reflect2"


"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


// CheckShardStore 代表一个用于处理代理缓存检查事件的结构体 // CheckShardStore 代表一个用于处理代理缓存检查事件的结构体
@@ -52,7 +52,7 @@ func (j *CheckShardStore) Execute(t *TickTock) {
} }
} }


func (j *CheckShardStore) checkOne(t *TickTock, space *clitypes.UserSpaceDetail) error {
func (j *CheckShardStore) checkOne(t *TickTock, space *jcsypes.UserSpaceDetail) error {
// addr, ok := space.RecommendHub.Address.(*cortypes.GRPCAddressInfo) // addr, ok := space.RecommendHub.Address.(*cortypes.GRPCAddressInfo)
// if !ok { // if !ok {
// return fmt.Errorf("master of user space %v has no grpc address", space.UserSpace) // return fmt.Errorf("master of user space %v has no grpc address", space.UserSpace)
@@ -80,9 +80,9 @@ func (j *CheckShardStore) checkOne(t *TickTock, space *clitypes.UserSpaceDetail)
return fmt.Errorf("listing all files: %w", err) return fmt.Errorf("listing all files: %w", err)
} }


fileHashes := lo.Map(infos, func(info stgtypes.FileInfo, _ int) clitypes.FileHash { return info.Hash })
fileHashes := lo.Map(infos, func(info stgtypes.FileInfo, _ int) jcsypes.FileHash { return info.Hash })


realFileHashes := lo.SliceToMap(fileHashes, func(hash clitypes.FileHash) (clitypes.FileHash, bool) { return hash, true })
realFileHashes := lo.SliceToMap(fileHashes, func(hash jcsypes.FileHash) (jcsypes.FileHash, bool) { return hash, true })


// 在事务中执行缓存更新操作 // 在事务中执行缓存更新操作
t.db.DoTx(func(tx db.SQLContext) error { t.db.DoTx(func(tx db.SQLContext) error {
@@ -95,7 +95,7 @@ func (j *CheckShardStore) checkOne(t *TickTock, space *clitypes.UserSpaceDetail)
} }


// checkPinnedObject 对比PinnedObject表,若实际文件不存在,则进行删除操作 // checkPinnedObject 对比PinnedObject表,若实际文件不存在,则进行删除操作
func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *clitypes.UserSpaceDetail, realFileHashes map[clitypes.FileHash]bool) {
func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *jcsypes.UserSpaceDetail, realFileHashes map[jcsypes.FileHash]bool) {
log := logger.WithType[CheckShardStore]("TickTock") log := logger.WithType[CheckShardStore]("TickTock")


objs, err := t.db.PinnedObject().GetObjectsByUserSpaceID(tx, space.UserSpace.UserSpaceID) objs, err := t.db.PinnedObject().GetObjectsByUserSpaceID(tx, space.UserSpace.UserSpaceID)
@@ -104,7 +104,7 @@ func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *
return return
} }


var rms []clitypes.ObjectID
var rms []jcsypes.ObjectID
for _, c := range objs { for _, c := range objs {
if realFileHashes[c.FileHash] { if realFileHashes[c.FileHash] {
continue continue
@@ -121,7 +121,7 @@ func (*CheckShardStore) checkPinnedObject(t *TickTock, tx db.SQLContext, space *
} }


// checkObjectBlock 对比ObjectBlock表,若实际文件不存在,则进行删除操作 // checkObjectBlock 对比ObjectBlock表,若实际文件不存在,则进行删除操作
func (*CheckShardStore) checkObjectBlock(t *TickTock, tx db.SQLContext, space *clitypes.UserSpaceDetail, realFileHashes map[clitypes.FileHash]bool) {
func (*CheckShardStore) checkObjectBlock(t *TickTock, tx db.SQLContext, space *jcsypes.UserSpaceDetail, realFileHashes map[jcsypes.FileHash]bool) {
log := logger.WithType[CheckShardStore]("TickTock") log := logger.WithType[CheckShardStore]("TickTock")


blocks, err := t.db.ObjectBlock().GetByUserSpaceID(tx, space.UserSpace.UserSpaceID) blocks, err := t.db.ObjectBlock().GetByUserSpaceID(tx, space.UserSpace.UserSpaceID)
@@ -130,7 +130,7 @@ func (*CheckShardStore) checkObjectBlock(t *TickTock, tx db.SQLContext, space *c
return return
} }


var rms []clitypes.FileHash
var rms []jcsypes.FileHash
for _, b := range blocks { for _, b := range blocks {
if realFileHashes[b.FileHash] { if realFileHashes[b.FileHash] {
continue continue


+ 88
- 88
client/internal/ticktock/redundancy_recover.go View File

@@ -10,62 +10,62 @@ import (
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/common/utils/sort2" "gitlink.org.cn/cloudream/common/utils/sort2"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/models/datamap"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
lrcparser "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/parser" lrcparser "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/parser"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )


func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj clitypes.ObjectDetail) (clitypes.Redundancy, []*userSpaceUsageInfo) {
func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail) (jcsypes.Redundancy, []*userSpaceUsageInfo) {
switch obj.Object.Redundancy.(type) { switch obj.Object.Redundancy.(type) {
case *clitypes.NoneRedundancy:
case *jcsypes.NoneRedundancy:
if obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold { if obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold {
newStgs := t.chooseNewUserSpacesForEC(ctx, &clitypes.DefaultECRedundancy)
return &clitypes.DefaultECRedundancy, newStgs
newStgs := t.chooseNewUserSpacesForEC(ctx, &jcsypes.DefaultECRedundancy)
return &jcsypes.DefaultECRedundancy, newStgs
} }


return &clitypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &clitypes.DefaultRepRedundancy)
return &jcsypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcsypes.DefaultRepRedundancy)


case *clitypes.RepRedundancy:
case *jcsypes.RepRedundancy:
if obj.Object.Size >= ctx.ticktock.cfg.ECFileSizeThreshold { if obj.Object.Size >= ctx.ticktock.cfg.ECFileSizeThreshold {
newStgs := t.chooseNewUserSpacesForEC(ctx, &clitypes.DefaultECRedundancy)
return &clitypes.DefaultECRedundancy, newStgs
newStgs := t.chooseNewUserSpacesForEC(ctx, &jcsypes.DefaultECRedundancy)
return &jcsypes.DefaultECRedundancy, newStgs
} }


newSpaces := t.rechooseUserSpacesForRep(ctx, &clitypes.DefaultRepRedundancy)
newSpaces := t.rechooseUserSpacesForRep(ctx, &jcsypes.DefaultRepRedundancy)
for _, s := range newSpaces { for _, s := range newSpaces {
if !obj.ContainsBlock(0, s.UserSpace.UserSpace.UserSpaceID) && !obj.ContainsPinned(s.UserSpace.UserSpace.UserSpaceID) { if !obj.ContainsBlock(0, s.UserSpace.UserSpace.UserSpaceID) && !obj.ContainsPinned(s.UserSpace.UserSpace.UserSpaceID) {
return &clitypes.DefaultRepRedundancy, newSpaces
return &jcsypes.DefaultRepRedundancy, newSpaces
} }
} }


return nil, nil return nil, nil


case *clitypes.ECRedundancy:
case *jcsypes.ECRedundancy:
if obj.Object.Size < ctx.ticktock.cfg.ECFileSizeThreshold { if obj.Object.Size < ctx.ticktock.cfg.ECFileSizeThreshold {
return &clitypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &clitypes.DefaultRepRedundancy)
return &jcsypes.DefaultRepRedundancy, t.chooseNewUserSpacesForRep(ctx, &jcsypes.DefaultRepRedundancy)
} }


newSpaces := t.rechooseUserSpacesForEC(ctx, obj, &clitypes.DefaultECRedundancy)
newSpaces := t.rechooseUserSpacesForEC(ctx, obj, &jcsypes.DefaultECRedundancy)
for i, s := range newSpaces { for i, s := range newSpaces {
if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) { if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) {
return &clitypes.DefaultECRedundancy, newSpaces
return &jcsypes.DefaultECRedundancy, newSpaces
} }
} }


return nil, nil return nil, nil


case *clitypes.LRCRedundancy:
newLRCStgs := t.rechooseUserSpacesForLRC(ctx, obj, &clitypes.DefaultLRCRedundancy)
case *jcsypes.LRCRedundancy:
newLRCStgs := t.rechooseUserSpacesForLRC(ctx, obj, &jcsypes.DefaultLRCRedundancy)


for i, s := range newLRCStgs { for i, s := range newLRCStgs {
if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) { if !obj.ContainsBlock(i, s.UserSpace.UserSpace.UserSpaceID) {
return &clitypes.DefaultLRCRedundancy, newLRCStgs
return &jcsypes.DefaultLRCRedundancy, newLRCStgs
} }
} }


@@ -74,7 +74,7 @@ func (t *ChangeRedundancy) chooseRedundancy(ctx *changeRedundancyContext, obj cl
return nil, nil return nil, nil
} }


func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj clitypes.ObjectDetail, newRed clitypes.Redundancy, selectedUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, newRed jcsypes.Redundancy, selectedUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
log := logger.WithType[ChangeRedundancy]("TickTock") log := logger.WithType[ChangeRedundancy]("TickTock")


var updating *db.UpdatingObjectRedundancy var updating *db.UpdatingObjectRedundancy
@@ -82,48 +82,48 @@ func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj
var err error var err error


switch srcRed := obj.Object.Redundancy.(type) { switch srcRed := obj.Object.Redundancy.(type) {
case *clitypes.NoneRedundancy:
case *jcsypes.NoneRedundancy:
switch newRed := newRed.(type) { switch newRed := newRed.(type) {
case *clitypes.RepRedundancy:
case *jcsypes.RepRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> rep") log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> rep")
updating, evt, err = t.noneToRep(ctx, obj, newRed, selectedUserSpaces) updating, evt, err = t.noneToRep(ctx, obj, newRed, selectedUserSpaces)


case *clitypes.ECRedundancy:
case *jcsypes.ECRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> ec") log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> ec")
updating, evt, err = t.noneToEC(ctx, obj, newRed, selectedUserSpaces) updating, evt, err = t.noneToEC(ctx, obj, newRed, selectedUserSpaces)


case *clitypes.LRCRedundancy:
case *jcsypes.LRCRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> lrc") log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> lrc")
updating, evt, err = t.noneToLRC(ctx, obj, newRed, selectedUserSpaces) updating, evt, err = t.noneToLRC(ctx, obj, newRed, selectedUserSpaces)


case *clitypes.SegmentRedundancy:
case *jcsypes.SegmentRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> segment") log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: none -> segment")
updating, evt, err = t.noneToSeg(ctx, obj, newRed, selectedUserSpaces) updating, evt, err = t.noneToSeg(ctx, obj, newRed, selectedUserSpaces)
} }


case *clitypes.RepRedundancy:
case *jcsypes.RepRedundancy:
switch newRed := newRed.(type) { switch newRed := newRed.(type) {
case *clitypes.RepRedundancy:
case *jcsypes.RepRedundancy:
updating, evt, err = t.repToRep(ctx, obj, srcRed, selectedUserSpaces) updating, evt, err = t.repToRep(ctx, obj, srcRed, selectedUserSpaces)


case *clitypes.ECRedundancy:
case *jcsypes.ECRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: rep -> ec") log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: rep -> ec")
updating, evt, err = t.repToEC(ctx, obj, newRed, selectedUserSpaces) updating, evt, err = t.repToEC(ctx, obj, newRed, selectedUserSpaces)
} }


case *clitypes.ECRedundancy:
case *jcsypes.ECRedundancy:
switch newRed := newRed.(type) { switch newRed := newRed.(type) {
case *clitypes.RepRedundancy:
case *jcsypes.RepRedundancy:
log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: ec -> rep") log.WithField("ObjectID", obj.Object.ObjectID).Debugf("redundancy: ec -> rep")
updating, evt, err = t.ecToRep(ctx, obj, srcRed, newRed, selectedUserSpaces) updating, evt, err = t.ecToRep(ctx, obj, srcRed, newRed, selectedUserSpaces)


case *clitypes.ECRedundancy:
case *jcsypes.ECRedundancy:
updating, evt, err = t.ecToEC(ctx, obj, srcRed, newRed, selectedUserSpaces) updating, evt, err = t.ecToEC(ctx, obj, srcRed, newRed, selectedUserSpaces)
} }


case *clitypes.LRCRedundancy:
case *jcsypes.LRCRedundancy:
switch newRed := newRed.(type) { switch newRed := newRed.(type) {
case *clitypes.LRCRedundancy:
case *jcsypes.LRCRedundancy:
updating, evt, err = t.lrcToLRC(ctx, obj, srcRed, newRed, selectedUserSpaces) updating, evt, err = t.lrcToLRC(ctx, obj, srcRed, newRed, selectedUserSpaces)
} }
} }
@@ -132,16 +132,16 @@ func (t *ChangeRedundancy) doChangeRedundancy(ctx *changeRedundancyContext, obj
} }


// 统计每个对象块所在的节点,选出块最多的不超过userspaceCnt个节点 // 统计每个对象块所在的节点,选出块最多的不超过userspaceCnt个节点
func (t *ChangeRedundancy) summaryRepObjectBlockUserSpaces(ctx *changeRedundancyContext, objs []clitypes.ObjectDetail, userspaceCnt int) []clitypes.UserSpaceID {
func (t *ChangeRedundancy) summaryRepObjectBlockUserSpaces(ctx *changeRedundancyContext, objs []jcsypes.ObjectDetail, userspaceCnt int) []jcsypes.UserSpaceID {
type stgBlocks struct { type stgBlocks struct {
UserSpaceID clitypes.UserSpaceID
UserSpaceID jcsypes.UserSpaceID
Count int Count int
} }


stgBlocksMap := make(map[clitypes.UserSpaceID]*stgBlocks)
stgBlocksMap := make(map[jcsypes.UserSpaceID]*stgBlocks)
for _, obj := range objs { for _, obj := range objs {
shouldUseEC := obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold shouldUseEC := obj.Object.Size > ctx.ticktock.cfg.ECFileSizeThreshold
if _, ok := obj.Object.Redundancy.(*clitypes.RepRedundancy); ok && !shouldUseEC {
if _, ok := obj.Object.Redundancy.(*jcsypes.RepRedundancy); ok && !shouldUseEC {
for _, block := range obj.Blocks { for _, block := range obj.Blocks {
if _, ok := stgBlocksMap[block.UserSpaceID]; !ok { if _, ok := stgBlocksMap[block.UserSpaceID]; !ok {
stgBlocksMap[block.UserSpaceID] = &stgBlocks{ stgBlocksMap[block.UserSpaceID] = &stgBlocks{
@@ -159,14 +159,14 @@ func (t *ChangeRedundancy) summaryRepObjectBlockUserSpaces(ctx *changeRedundancy
return right.Count - left.Count return right.Count - left.Count
}) })


ids := lo.Map(userspaces, func(item *stgBlocks, idx int) clitypes.UserSpaceID { return item.UserSpaceID })
ids := lo.Map(userspaces, func(item *stgBlocks, idx int) jcsypes.UserSpaceID { return item.UserSpaceID })
if len(ids) > userspaceCnt { if len(ids) > userspaceCnt {
ids = ids[:userspaceCnt] ids = ids[:userspaceCnt]
} }
return ids return ids
} }


func (t *ChangeRedundancy) chooseNewUserSpacesForRep(ctx *changeRedundancyContext, red *clitypes.RepRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) chooseNewUserSpacesForRep(ctx *changeRedundancyContext, red *jcsypes.RepRedundancy) []*userSpaceUsageInfo {
sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int { sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
return sort2.Cmp(right.AccessAmount, left.AccessAmount) return sort2.Cmp(right.AccessAmount, left.AccessAmount)
}) })
@@ -174,7 +174,7 @@ func (t *ChangeRedundancy) chooseNewUserSpacesForRep(ctx *changeRedundancyContex
return t.chooseSoManyUserSpaces(red.RepCount, sortedUserSpaces) return t.chooseSoManyUserSpaces(red.RepCount, sortedUserSpaces)
} }


func (t *ChangeRedundancy) chooseNewUserSpacesForEC(ctx *changeRedundancyContext, red *clitypes.ECRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) chooseNewUserSpacesForEC(ctx *changeRedundancyContext, red *jcsypes.ECRedundancy) []*userSpaceUsageInfo {
sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int { sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
return sort2.Cmp(right.AccessAmount, left.AccessAmount) return sort2.Cmp(right.AccessAmount, left.AccessAmount)
}) })
@@ -182,7 +182,7 @@ func (t *ChangeRedundancy) chooseNewUserSpacesForEC(ctx *changeRedundancyContext
return t.chooseSoManyUserSpaces(red.N, sortedUserSpaces) return t.chooseSoManyUserSpaces(red.N, sortedUserSpaces)
} }


func (t *ChangeRedundancy) chooseNewUserSpacesForLRC(ctx *changeRedundancyContext, red *clitypes.LRCRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) chooseNewUserSpacesForLRC(ctx *changeRedundancyContext, red *jcsypes.LRCRedundancy) []*userSpaceUsageInfo {
sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int { sortedUserSpaces := sort2.Sort(lo.Values(ctx.allUserSpaces), func(left *userSpaceUsageInfo, right *userSpaceUsageInfo) int {
return sort2.Cmp(right.AccessAmount, left.AccessAmount) return sort2.Cmp(right.AccessAmount, left.AccessAmount)
}) })
@@ -198,7 +198,7 @@ func (t *ChangeRedundancy) chooseNewUserSpacesForSeg(ctx *changeRedundancyContex
return t.chooseSoManyUserSpaces(segCount, sortedUserSpaces) return t.chooseSoManyUserSpaces(segCount, sortedUserSpaces)
} }


func (t *ChangeRedundancy) rechooseUserSpacesForRep(ctx *changeRedundancyContext, red *clitypes.RepRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) rechooseUserSpacesForRep(ctx *changeRedundancyContext, red *jcsypes.RepRedundancy) []*userSpaceUsageInfo {
type rechooseUserSpace struct { type rechooseUserSpace struct {
*userSpaceUsageInfo *userSpaceUsageInfo
HasBlock bool HasBlock bool
@@ -233,7 +233,7 @@ func (t *ChangeRedundancy) rechooseUserSpacesForRep(ctx *changeRedundancyContext
return t.chooseSoManyUserSpaces(red.RepCount, lo.Map(sortedStgs, func(userspace *rechooseUserSpace, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo })) return t.chooseSoManyUserSpaces(red.RepCount, lo.Map(sortedStgs, func(userspace *rechooseUserSpace, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo }))
} }


func (t *ChangeRedundancy) rechooseUserSpacesForEC(ctx *changeRedundancyContext, obj clitypes.ObjectDetail, red *clitypes.ECRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) rechooseUserSpacesForEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.ECRedundancy) []*userSpaceUsageInfo {
type rechooseStg struct { type rechooseStg struct {
*userSpaceUsageInfo *userSpaceUsageInfo
CachedBlockIndex int CachedBlockIndex int
@@ -269,7 +269,7 @@ func (t *ChangeRedundancy) rechooseUserSpacesForEC(ctx *changeRedundancyContext,
return t.chooseSoManyUserSpaces(red.N, lo.Map(sortedStgs, func(userspace *rechooseStg, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo })) return t.chooseSoManyUserSpaces(red.N, lo.Map(sortedStgs, func(userspace *rechooseStg, idx int) *userSpaceUsageInfo { return userspace.userSpaceUsageInfo }))
} }


func (t *ChangeRedundancy) rechooseUserSpacesForLRC(ctx *changeRedundancyContext, obj clitypes.ObjectDetail, red *clitypes.LRCRedundancy) []*userSpaceUsageInfo {
func (t *ChangeRedundancy) rechooseUserSpacesForLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.LRCRedundancy) []*userSpaceUsageInfo {
type rechooseStg struct { type rechooseStg struct {
*userSpaceUsageInfo *userSpaceUsageInfo
CachedBlockIndex int CachedBlockIndex int
@@ -341,7 +341,7 @@ func (t *ChangeRedundancy) chooseSoManyUserSpaces(count int, stgs []*userSpaceUs
return chosen return chosen
} }


func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj clitypes.ObjectDetail, red *clitypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 { if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep") return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
} }
@@ -352,7 +352,7 @@ func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj clitypes.
} }


// 如果选择的备份节点都是同一个,那么就只要上传一次 // 如果选择的备份节点都是同一个,那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) clitypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })


ft := ioswitch2.NewFromTo() ft := ioswitch2.NewFromTo()
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream())) ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
@@ -374,11 +374,11 @@ func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj clitypes.
return nil, nil, fmt.Errorf("executing io plan: %w", err) return nil, nil, fmt.Errorf("executing io plan: %w", err)
} }


var blocks []clitypes.ObjectBlock
var blocks []jcsypes.ObjectBlock
var blockChgs []datamap.BlockChange var blockChgs []datamap.BlockChange
for i, stg := range uploadStgs { for i, stg := range uploadStgs {
r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue) r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, clitypes.ObjectBlock{
blocks = append(blocks, jcsypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: 0, Index: 0,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID, UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
@@ -412,7 +412,7 @@ func (t *ChangeRedundancy) noneToRep(ctx *changeRedundancyContext, obj clitypes.
}, nil }, nil
} }


func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj clitypes.ObjectDetail, red *clitypes.ECRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.ECRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 { if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec") return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec")
} }
@@ -441,12 +441,12 @@ func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj clitypes.O
return nil, nil, fmt.Errorf("executing io plan: %w", err) return nil, nil, fmt.Errorf("executing io plan: %w", err)
} }


var blocks []clitypes.ObjectBlock
var blocks []jcsypes.ObjectBlock
var evtTargetBlocks []datamap.Block var evtTargetBlocks []datamap.Block
var evtBlockTrans []datamap.DataTransfer var evtBlockTrans []datamap.DataTransfer
for i := 0; i < red.N; i++ { for i := 0; i < red.N; i++ {
r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue) r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, clitypes.ObjectBlock{
blocks = append(blocks, jcsypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: i, Index: i,
UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID, UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
@@ -494,7 +494,7 @@ func (t *ChangeRedundancy) noneToEC(ctx *changeRedundancyContext, obj clitypes.O
}, nil }, nil
} }


func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj clitypes.ObjectDetail, red *clitypes.LRCRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.LRCRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 { if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec") return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to ec")
} }
@@ -522,12 +522,12 @@ func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj clitypes.
return nil, nil, fmt.Errorf("executing io plan: %w", err) return nil, nil, fmt.Errorf("executing io plan: %w", err)
} }


var blocks []clitypes.ObjectBlock
var blocks []jcsypes.ObjectBlock
var evtTargetBlocks []datamap.Block var evtTargetBlocks []datamap.Block
var evtBlockTrans []datamap.DataTransfer var evtBlockTrans []datamap.DataTransfer
for i := 0; i < red.N; i++ { for i := 0; i < red.N; i++ {
r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue) r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, clitypes.ObjectBlock{
blocks = append(blocks, jcsypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: i, Index: i,
UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID, UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
@@ -576,7 +576,7 @@ func (t *ChangeRedundancy) noneToLRC(ctx *changeRedundancyContext, obj clitypes.
nil nil
} }


func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj clitypes.ObjectDetail, red *clitypes.SegmentRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.SegmentRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 { if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep") return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
} }
@@ -587,7 +587,7 @@ func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj clitypes.
} }


// 如果选择的备份节点都是同一个,那么就只要上传一次 // 如果选择的备份节点都是同一个,那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) clitypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })


ft := ioswitch2.NewFromTo() ft := ioswitch2.NewFromTo()
ft.SegmentParam = red ft.SegmentParam = red
@@ -610,12 +610,12 @@ func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj clitypes.
return nil, nil, fmt.Errorf("executing io plan: %w", err) return nil, nil, fmt.Errorf("executing io plan: %w", err)
} }


var blocks []clitypes.ObjectBlock
var blocks []jcsypes.ObjectBlock
var evtTargetBlocks []datamap.Block var evtTargetBlocks []datamap.Block
var evtBlockTrans []datamap.DataTransfer var evtBlockTrans []datamap.DataTransfer
for i, stg := range uploadStgs { for i, stg := range uploadStgs {
r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue) r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, clitypes.ObjectBlock{
blocks = append(blocks, jcsypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: i, Index: i,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID, UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
@@ -664,7 +664,7 @@ func (t *ChangeRedundancy) noneToSeg(ctx *changeRedundancyContext, obj clitypes.
nil nil
} }


func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj clitypes.ObjectDetail, red *clitypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
if len(obj.Blocks) == 0 { if len(obj.Blocks) == 0 {
return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep") return nil, nil, fmt.Errorf("object is not cached on any userspaces, cannot change its redundancy to rep")
} }
@@ -675,7 +675,7 @@ func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj clitypes.O
} }


// 如果选择的备份节点都是同一个,那么就只要上传一次 // 如果选择的备份节点都是同一个,那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) clitypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })


ft := ioswitch2.NewFromTo() ft := ioswitch2.NewFromTo()
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream())) ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *srcStg.UserSpace, ioswitch2.RawStream()))
@@ -697,11 +697,11 @@ func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj clitypes.O
return nil, nil, fmt.Errorf("executing io plan: %w", err) return nil, nil, fmt.Errorf("executing io plan: %w", err)
} }


var blocks []clitypes.ObjectBlock
var blocks []jcsypes.ObjectBlock
var blockChgs []datamap.BlockChange var blockChgs []datamap.BlockChange
for i, stg := range uploadStgs { for i, stg := range uploadStgs {
r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue) r := ret.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, clitypes.ObjectBlock{
blocks = append(blocks, jcsypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: 0, Index: 0,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID, UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
@@ -737,14 +737,14 @@ func (t *ChangeRedundancy) repToRep(ctx *changeRedundancyContext, obj clitypes.O
nil nil
} }


func (t *ChangeRedundancy) repToEC(ctx *changeRedundancyContext, obj clitypes.ObjectDetail, red *clitypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) repToEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, red *jcsypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
return t.noneToEC(ctx, obj, red, uploadUserSpaces) return t.noneToEC(ctx, obj, red, uploadUserSpaces)
} }


func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj clitypes.ObjectDetail, srcRed *clitypes.ECRedundancy, tarRed *clitypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []clitypes.GrouppedObjectBlock
func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, srcRed *jcsypes.ECRedundancy, tarRed *jcsypes.RepRedundancy, uploadStgs []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []jcsypes.GrouppedObjectBlock
var chosenBlockIndexes []int var chosenBlockIndexes []int
var chosenBlockStg []clitypes.UserSpaceDetail
var chosenBlockStg []jcsypes.UserSpaceDetail
for _, block := range obj.GroupBlocks() { for _, block := range obj.GroupBlocks() {
if len(block.UserSpaceIDs) > 0 { if len(block.UserSpaceIDs) > 0 {
// TODO 考虑选择最优的节点 // TODO 考虑选择最优的节点
@@ -768,7 +768,7 @@ func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj clitypes.Ob
} }


// 如果选择的备份节点都是同一个,那么就只要上传一次 // 如果选择的备份节点都是同一个,那么就只要上传一次
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) clitypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })
uploadStgs = lo.UniqBy(uploadStgs, func(item *userSpaceUsageInfo) jcsypes.UserSpaceID { return item.UserSpace.UserSpace.UserSpaceID })


planBlder := exec.NewPlanBuilder() planBlder := exec.NewPlanBuilder()
ft := ioswitch2.NewFromTo() ft := ioswitch2.NewFromTo()
@@ -795,11 +795,11 @@ func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj clitypes.Ob
return nil, nil, fmt.Errorf("executing io plan: %w", err) return nil, nil, fmt.Errorf("executing io plan: %w", err)
} }


var blocks []clitypes.ObjectBlock
var blocks []jcsypes.ObjectBlock


for i := range uploadStgs { for i := range uploadStgs {
r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue) r := ioRet.Get(fmt.Sprintf("%d", i)).(*ops2.FileInfoValue)
blocks = append(blocks, clitypes.ObjectBlock{
blocks = append(blocks, jcsypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: 0, Index: 0,
UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID, UserSpaceID: uploadStgs[i].UserSpace.UserSpace.UserSpaceID,
@@ -866,11 +866,11 @@ func (t *ChangeRedundancy) ecToRep(ctx *changeRedundancyContext, obj clitypes.Ob
nil nil
} }


func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj clitypes.ObjectDetail, srcRed *clitypes.ECRedundancy, tarRed *clitypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, srcRed *jcsypes.ECRedundancy, tarRed *jcsypes.ECRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
grpBlocks := obj.GroupBlocks() grpBlocks := obj.GroupBlocks()


var chosenBlocks []clitypes.GrouppedObjectBlock
var chosenBlockStg []clitypes.UserSpaceDetail
var chosenBlocks []jcsypes.GrouppedObjectBlock
var chosenBlockStg []jcsypes.UserSpaceDetail
for _, block := range grpBlocks { for _, block := range grpBlocks {
if len(block.UserSpaceIDs) > 0 { if len(block.UserSpaceIDs) > 0 {
stg, ok := ctx.allUserSpaces[block.UserSpaceIDs[0]] stg, ok := ctx.allUserSpaces[block.UserSpaceIDs[0]]
@@ -909,16 +909,16 @@ func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj clitypes.Obj
}) })
} }


var newBlocks []clitypes.ObjectBlock
var newBlocks []jcsypes.ObjectBlock
shouldUpdateBlocks := false shouldUpdateBlocks := false
for i, stg := range uploadUserSpaces { for i, stg := range uploadUserSpaces {
newBlock := clitypes.ObjectBlock{
newBlock := jcsypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: i, Index: i,
UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID, UserSpaceID: stg.UserSpace.UserSpace.UserSpaceID,
} }


grp, ok := lo.Find(grpBlocks, func(grp clitypes.GrouppedObjectBlock) bool { return grp.Index == i })
grp, ok := lo.Find(grpBlocks, func(grp jcsypes.GrouppedObjectBlock) bool { return grp.Index == i })


// 如果新选中的节点已经记录在Block表中,那么就不需要任何变更 // 如果新选中的节点已经记录在Block表中,那么就不需要任何变更
if ok && lo.Contains(grp.UserSpaceIDs, stg.UserSpace.UserSpace.UserSpaceID) { if ok && lo.Contains(grp.UserSpaceIDs, stg.UserSpace.UserSpace.UserSpaceID) {
@@ -988,7 +988,7 @@ func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj clitypes.Obj


var blockChgs []datamap.BlockChange var blockChgs []datamap.BlockChange
for _, block := range obj.Blocks { for _, block := range obj.Blocks {
keep := lo.ContainsBy(newBlocks, func(newBlock clitypes.ObjectBlock) bool {
keep := lo.ContainsBy(newBlocks, func(newBlock jcsypes.ObjectBlock) bool {
return newBlock.Index == block.Index && newBlock.UserSpaceID == block.UserSpaceID return newBlock.Index == block.Index && newBlock.UserSpaceID == block.UserSpaceID
}) })
if !keep { if !keep {
@@ -1019,7 +1019,7 @@ func (t *ChangeRedundancy) ecToEC(ctx *changeRedundancyContext, obj clitypes.Obj
nil nil
} }


func (t *ChangeRedundancy) lrcToLRC(ctx *changeRedundancyContext, obj clitypes.ObjectDetail, srcRed *clitypes.LRCRedundancy, tarRed *clitypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
func (t *ChangeRedundancy) lrcToLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, srcRed *jcsypes.LRCRedundancy, tarRed *jcsypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {


blocksGrpByIndex := obj.GroupBlocks() blocksGrpByIndex := obj.GroupBlocks()


@@ -1062,8 +1062,8 @@ func (t *ChangeRedundancy) lrcToLRC(ctx *changeRedundancyContext, obj clitypes.O
/* /*
TODO2 修复这一块的代码 TODO2 修复这一块的代码


func (t *ChangeRedundancy) groupReconstructLRC(obj clitypes.ObjectDetail, lostBlocks []int, lostBlockGrps []int, grpedBlocks []clitypes.GrouppedObjectBlock, red *clitypes.LRCRedundancy, uploadUserSpaces []*UserSpaceLoadInfo) (*db.UpdatingObjectRedundancy, error) {
grped := make(map[int]clitypes.GrouppedObjectBlock)
func (t *ChangeRedundancy) groupReconstructLRC(obj jcsypes.ObjectDetail, lostBlocks []int, lostBlockGrps []int, grpedBlocks []jcsypes.GrouppedObjectBlock, red *jcsypes.LRCRedundancy, uploadUserSpaces []*UserSpaceLoadInfo) (*db.UpdatingObjectRedundancy, error) {
grped := make(map[int]jcsypes.GrouppedObjectBlock)
for _, b := range grpedBlocks { for _, b := range grpedBlocks {
grped[b.Index] = b grped[b.Index] = b
} }
@@ -1098,9 +1098,9 @@ TODO2 修复这一块的代码
return nil, fmt.Errorf("executing io plan: %w", err) return nil, fmt.Errorf("executing io plan: %w", err)
} }


var newBlocks []clitypes.ObjectBlock
var newBlocks []jcsypes.ObjectBlock
for _, i := range lostBlocks { for _, i := range lostBlocks {
newBlocks = append(newBlocks, clitypes.ObjectBlock{
newBlocks = append(newBlocks, jcsypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: i, Index: i,
UserSpaceID: uploadUserSpaces[i].UserSpace.UserSpace.UserSpaceID, UserSpaceID: uploadUserSpaces[i].UserSpace.UserSpace.UserSpaceID,
@@ -1109,7 +1109,7 @@ TODO2 修复这一块的代码
} }
for _, b := range grpedBlocks { for _, b := range grpedBlocks {
for _, hubID := range b.UserSpaceIDs { for _, hubID := range b.UserSpaceIDs {
newBlocks = append(newBlocks, clitypes.ObjectBlock{
newBlocks = append(newBlocks, jcsypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: b.Index, Index: b.Index,
UserSpaceID: hubID, UserSpaceID: hubID,
@@ -1125,9 +1125,9 @@ TODO2 修复这一块的代码
}, nil }, nil
} }
*/ */
func (t *ChangeRedundancy) reconstructLRC(ctx *changeRedundancyContext, obj clitypes.ObjectDetail, grpBlocks []clitypes.GrouppedObjectBlock, red *clitypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []clitypes.GrouppedObjectBlock
var chosenBlockStg []clitypes.UserSpaceDetail
func (t *ChangeRedundancy) reconstructLRC(ctx *changeRedundancyContext, obj jcsypes.ObjectDetail, grpBlocks []jcsypes.GrouppedObjectBlock, red *jcsypes.LRCRedundancy, uploadUserSpaces []*userSpaceUsageInfo) (*db.UpdatingObjectRedundancy, datamap.SysEventBody, error) {
var chosenBlocks []jcsypes.GrouppedObjectBlock
var chosenBlockStg []jcsypes.UserSpaceDetail


for _, block := range grpBlocks { for _, block := range grpBlocks {
if len(block.UserSpaceIDs) > 0 && block.Index < red.M() { if len(block.UserSpaceIDs) > 0 && block.Index < red.M() {
@@ -1154,16 +1154,16 @@ func (t *ChangeRedundancy) reconstructLRC(ctx *changeRedundancyContext, obj clit


var froms []ioswitchlrc.From var froms []ioswitchlrc.From
var toes []ioswitchlrc.To var toes []ioswitchlrc.To
var newBlocks []clitypes.ObjectBlock
var newBlocks []jcsypes.ObjectBlock
shouldUpdateBlocks := false shouldUpdateBlocks := false
for i, userspace := range uploadUserSpaces { for i, userspace := range uploadUserSpaces {
newBlock := clitypes.ObjectBlock{
newBlock := jcsypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: i, Index: i,
UserSpaceID: userspace.UserSpace.UserSpace.UserSpaceID, UserSpaceID: userspace.UserSpace.UserSpace.UserSpaceID,
} }


grp, ok := lo.Find(grpBlocks, func(grp clitypes.GrouppedObjectBlock) bool { return grp.Index == i })
grp, ok := lo.Find(grpBlocks, func(grp jcsypes.GrouppedObjectBlock) bool { return grp.Index == i })


// 如果新选中的节点已经记录在Block表中,那么就不需要任何变更 // 如果新选中的节点已经记录在Block表中,那么就不需要任何变更
if ok && lo.Contains(grp.UserSpaceIDs, userspace.UserSpace.UserSpace.UserSpaceID) { if ok && lo.Contains(grp.UserSpaceIDs, userspace.UserSpace.UserSpace.UserSpaceID) {


+ 53
- 53
client/internal/ticktock/redundancy_shrink.go View File

@@ -14,21 +14,21 @@ import (
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/common/utils/sort2" "gitlink.org.cn/cloudream/common/utils/sort2"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/consts" "gitlink.org.cn/cloudream/jcs-pub/common/consts"
"gitlink.org.cn/cloudream/jcs-pub/common/models/datamap"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types/datamap"
) )


func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext, pkg clitypes.PackageDetail, objs []clitypes.ObjectDetail, reen *publock.Reentrant) ([]db.UpdatingObjectRedundancy, []datamap.SysEventBody, error) {
func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext, pkg jcsypes.PackageDetail, objs []jcsypes.ObjectDetail, reen *publock.Reentrant) ([]db.UpdatingObjectRedundancy, []datamap.SysEventBody, error) {
log := logger.WithType[ChangeRedundancy]("TickTock") log := logger.WithType[ChangeRedundancy]("TickTock")


var readerStgIDs []clitypes.UserSpaceID
var readerStgIDs []jcsypes.UserSpaceID
for _, space := range execCtx.allUserSpaces { for _, space := range execCtx.allUserSpaces {
// TODO 可以考虑做成配置 // TODO 可以考虑做成配置
if space.AccessAmount >= float64(pkg.ObjectCount/2) { if space.AccessAmount >= float64(pkg.ObjectCount/2) {
@@ -37,17 +37,17 @@ func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext,
} }


// 只对ec和rep对象进行处理 // 只对ec和rep对象进行处理
var ecObjects []clitypes.ObjectDetail
var repObjects []clitypes.ObjectDetail
var ecObjects []jcsypes.ObjectDetail
var repObjects []jcsypes.ObjectDetail
for _, obj := range objs { for _, obj := range objs {
if _, ok := obj.Object.Redundancy.(*clitypes.ECRedundancy); ok {
if _, ok := obj.Object.Redundancy.(*jcsypes.ECRedundancy); ok {
ecObjects = append(ecObjects, obj) ecObjects = append(ecObjects, obj)
} else if _, ok := obj.Object.Redundancy.(*clitypes.RepRedundancy); ok {
} else if _, ok := obj.Object.Redundancy.(*jcsypes.RepRedundancy); ok {
repObjects = append(repObjects, obj) repObjects = append(repObjects, obj)
} }
} }


planningStgIDs := make(map[clitypes.UserSpaceID]bool)
planningStgIDs := make(map[jcsypes.UserSpaceID]bool)


var sysEvents []datamap.SysEventBody var sysEvents []datamap.SysEventBody


@@ -80,7 +80,7 @@ func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext,
// 对于ec对象,则每个对象单独进行退火算法 // 对于ec对象,则每个对象单独进行退火算法
var ecObjectsUpdating []db.UpdatingObjectRedundancy var ecObjectsUpdating []db.UpdatingObjectRedundancy
for i, obj := range ecObjects { for i, obj := range ecObjects {
ecRed := obj.Object.Redundancy.(*clitypes.ECRedundancy)
ecRed := obj.Object.Redundancy.(*jcsypes.ECRedundancy)
solu := t.startAnnealing(execCtx, readerStgIDs, annealingObject{ solu := t.startAnnealing(execCtx, readerStgIDs, annealingObject{
totalBlockCount: ecRed.N, totalBlockCount: ecRed.N,
minBlockCnt: ecRed.K, minBlockCnt: ecRed.K,
@@ -105,15 +105,15 @@ func (t *ChangeRedundancy) doRedundancyShrink(execCtx *changeRedundancyContext,
return append(repObjectsUpdating, ecObjectsUpdating...), sysEvents, nil return append(repObjectsUpdating, ecObjectsUpdating...), sysEvents, nil
} }


func (t *ChangeRedundancy) summaryRepObjectBlockNodes(objs []clitypes.ObjectDetail) []clitypes.UserSpaceID {
func (t *ChangeRedundancy) summaryRepObjectBlockNodes(objs []jcsypes.ObjectDetail) []jcsypes.UserSpaceID {
type stgBlocks struct { type stgBlocks struct {
UserSpaceID clitypes.UserSpaceID
UserSpaceID jcsypes.UserSpaceID
Count int Count int
} }


stgBlocksMap := make(map[clitypes.UserSpaceID]*stgBlocks)
stgBlocksMap := make(map[jcsypes.UserSpaceID]*stgBlocks)
for _, obj := range objs { for _, obj := range objs {
cacheBlockStgs := make(map[clitypes.UserSpaceID]bool)
cacheBlockStgs := make(map[jcsypes.UserSpaceID]bool)
for _, block := range obj.Blocks { for _, block := range obj.Blocks {
if _, ok := stgBlocksMap[block.UserSpaceID]; !ok { if _, ok := stgBlocksMap[block.UserSpaceID]; !ok {
stgBlocksMap[block.UserSpaceID] = &stgBlocks{ stgBlocksMap[block.UserSpaceID] = &stgBlocks{
@@ -153,17 +153,17 @@ func (t *ChangeRedundancy) summaryRepObjectBlockNodes(objs []clitypes.ObjectDeta
} }
} }


return lo.Map(stgs, func(item *stgBlocks, idx int) clitypes.UserSpaceID { return item.UserSpaceID })
return lo.Map(stgs, func(item *stgBlocks, idx int) jcsypes.UserSpaceID { return item.UserSpaceID })
} }


type annealingState struct { type annealingState struct {
ctx *changeRedundancyContext ctx *changeRedundancyContext
readerStgIDs []clitypes.UserSpaceID // 近期可能访问此对象的节点
stgsSortedByReader map[clitypes.UserSpaceID][]stgDist // 拥有数据的节点到每个可能访问对象的节点按距离排序
object annealingObject // 进行退火的对象
blockList []objectBlock // 排序后的块分布情况
stgBlockBitmaps map[clitypes.UserSpaceID]*bitmap.Bitmap64 // 用位图的形式表示每一个节点上有哪些块
stgCombTree combinatorialTree // 节点组合树,用于加速计算容灾度
readerStgIDs []jcsypes.UserSpaceID // 近期可能访问此对象的节点
stgsSortedByReader map[jcsypes.UserSpaceID][]stgDist // 拥有数据的节点到每个可能访问对象的节点按距离排序
object annealingObject // 进行退火的对象
blockList []objectBlock // 排序后的块分布情况
stgBlockBitmaps map[jcsypes.UserSpaceID]*bitmap.Bitmap64 // 用位图的形式表示每一个节点上有哪些块
stgCombTree combinatorialTree // 节点组合树,用于加速计算容灾度


maxScore float64 // 搜索过程中得到过的最大分数 maxScore float64 // 搜索过程中得到过的最大分数
maxScoreRmBlocks []bool // 最大分数对应的删除方案 maxScoreRmBlocks []bool // 最大分数对应的删除方案
@@ -178,30 +178,30 @@ type annealingState struct {


type objectBlock struct { type objectBlock struct {
Index int Index int
UserSpaceID clitypes.UserSpaceID
HasEntity bool // 节点拥有实际的文件数据块
HasShadow bool // 如果节点拥有完整文件数据,那么认为这个节点拥有所有块,这些块被称为影子块
FileHash clitypes.FileHash // 只有在拥有实际文件数据块时,这个字段才有值
Size int64 // 块大小
UserSpaceID jcsypes.UserSpaceID
HasEntity bool // 节点拥有实际的文件数据块
HasShadow bool // 如果节点拥有完整文件数据,那么认为这个节点拥有所有块,这些块被称为影子块
FileHash jcsypes.FileHash // 只有在拥有实际文件数据块时,这个字段才有值
Size int64 // 块大小
} }


type stgDist struct { type stgDist struct {
UserSpaceID clitypes.UserSpaceID
UserSpaceID jcsypes.UserSpaceID
Distance float64 Distance float64
} }


type combinatorialTree struct { type combinatorialTree struct {
nodes []combinatorialTreeNode nodes []combinatorialTreeNode
blocksMaps map[int]bitmap.Bitmap64 blocksMaps map[int]bitmap.Bitmap64
stgIDToLocalStgID map[clitypes.UserSpaceID]int
localStgIDToStgID []clitypes.UserSpaceID
stgIDToLocalStgID map[jcsypes.UserSpaceID]int
localStgIDToStgID []jcsypes.UserSpaceID
} }


type annealingObject struct { type annealingObject struct {
totalBlockCount int totalBlockCount int
minBlockCnt int minBlockCnt int
pinnedAt []clitypes.UserSpaceID
blocks []clitypes.ObjectBlock
pinnedAt []jcsypes.UserSpaceID
blocks []jcsypes.ObjectBlock
} }


const ( const (
@@ -210,10 +210,10 @@ const (
iterActionBreak = 2 iterActionBreak = 2
) )


func newCombinatorialTree(stgBlocksMaps map[clitypes.UserSpaceID]*bitmap.Bitmap64) combinatorialTree {
func newCombinatorialTree(stgBlocksMaps map[jcsypes.UserSpaceID]*bitmap.Bitmap64) combinatorialTree {
tree := combinatorialTree{ tree := combinatorialTree{
blocksMaps: make(map[int]bitmap.Bitmap64), blocksMaps: make(map[int]bitmap.Bitmap64),
stgIDToLocalStgID: make(map[clitypes.UserSpaceID]int),
stgIDToLocalStgID: make(map[jcsypes.UserSpaceID]int),
} }


tree.nodes = make([]combinatorialTreeNode, (1 << len(stgBlocksMaps))) tree.nodes = make([]combinatorialTreeNode, (1 << len(stgBlocksMaps)))
@@ -271,7 +271,7 @@ func (t *combinatorialTree) GetDepth(index int) int {


// 更新某一个算力中心节点的块分布位图,同时更新它对应组合树节点的所有子节点。 // 更新某一个算力中心节点的块分布位图,同时更新它对应组合树节点的所有子节点。
// 如果更新到某个节点时,已有K个块,那么就不会再更新它的子节点 // 如果更新到某个节点时,已有K个块,那么就不会再更新它的子节点
func (t *combinatorialTree) UpdateBitmap(stgID clitypes.UserSpaceID, mp bitmap.Bitmap64, k int) {
func (t *combinatorialTree) UpdateBitmap(stgID jcsypes.UserSpaceID, mp bitmap.Bitmap64, k int) {
t.blocksMaps[t.stgIDToLocalStgID[stgID]] = mp t.blocksMaps[t.stgIDToLocalStgID[stgID]] = mp
// 首先定义两种遍历树节点时的移动方式: // 首先定义两种遍历树节点时的移动方式:
// 1. 竖直移动(深度增加):从一个节点移动到它最左边的子节点。每移动一步,index+1 // 1. 竖直移动(深度增加):从一个节点移动到它最左边的子节点。每移动一步,index+1
@@ -416,13 +416,13 @@ type annealingSolution struct {
minAccessCost float64 // 本方案的最小访问费用 minAccessCost float64 // 本方案的最小访问费用
} }


func (t *ChangeRedundancy) startAnnealing(ctx *changeRedundancyContext, readerStgIDs []clitypes.UserSpaceID, object annealingObject) annealingSolution {
func (t *ChangeRedundancy) startAnnealing(ctx *changeRedundancyContext, readerStgIDs []jcsypes.UserSpaceID, object annealingObject) annealingSolution {
state := &annealingState{ state := &annealingState{
ctx: ctx, ctx: ctx,
readerStgIDs: readerStgIDs, readerStgIDs: readerStgIDs,
stgsSortedByReader: make(map[clitypes.UserSpaceID][]stgDist),
stgsSortedByReader: make(map[jcsypes.UserSpaceID][]stgDist),
object: object, object: object,
stgBlockBitmaps: make(map[clitypes.UserSpaceID]*bitmap.Bitmap64),
stgBlockBitmaps: make(map[jcsypes.UserSpaceID]*bitmap.Bitmap64),
} }


t.initBlockList(state) t.initBlockList(state)
@@ -486,7 +486,7 @@ func (t *ChangeRedundancy) startAnnealing(ctx *changeRedundancyContext, readerSt
} }


func (t *ChangeRedundancy) initBlockList(ctx *annealingState) { func (t *ChangeRedundancy) initBlockList(ctx *annealingState) {
blocksMap := make(map[clitypes.UserSpaceID][]objectBlock)
blocksMap := make(map[jcsypes.UserSpaceID][]objectBlock)


// 先生成所有的影子块 // 先生成所有的影子块
for _, pinned := range ctx.object.pinnedAt { for _, pinned := range ctx.object.pinnedAt {
@@ -680,7 +680,7 @@ func (t *ChangeRedundancy) alwaysAccept(curTemp float64, dScore float64, cooling
return v > rand.Float64() return v > rand.Float64()
} }


func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, solu annealingSolution, obj clitypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[clitypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, solu annealingSolution, obj jcsypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[jcsypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
entry := db.UpdatingObjectRedundancy{ entry := db.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
FileHash: obj.Object.FileHash, FileHash: obj.Object.FileHash,
@@ -694,8 +694,8 @@ func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, s
ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *fromStg, ioswitch2.RawStream())) ft.AddFrom(ioswitch2.NewFromShardstore(obj.Object.FileHash, *fromStg, ioswitch2.RawStream()))


for i, f := range solu.rmBlocks { for i, f := range solu.rmBlocks {
hasCache := lo.ContainsBy(obj.Blocks, func(b clitypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n clitypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
hasCache := lo.ContainsBy(obj.Blocks, func(b jcsypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n jcsypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
willRm := f willRm := f


if !willRm { if !willRm {
@@ -706,7 +706,7 @@ func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, s


planningHubIDs[solu.blockList[i].UserSpaceID] = true planningHubIDs[solu.blockList[i].UserSpaceID] = true
} }
entry.Blocks = append(entry.Blocks, clitypes.ObjectBlock{
entry.Blocks = append(entry.Blocks, jcsypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: solu.blockList[i].Index, Index: solu.blockList[i].Index,
UserSpaceID: solu.blockList[i].UserSpaceID, UserSpaceID: solu.blockList[i].UserSpaceID,
@@ -724,12 +724,12 @@ func (t *ChangeRedundancy) makePlansForRepObject(ctx *changeRedundancyContext, s
return entry return entry
} }


func (t *ChangeRedundancy) generateSysEventForRepObject(solu annealingSolution, obj clitypes.ObjectDetail) []datamap.SysEventBody {
func (t *ChangeRedundancy) generateSysEventForRepObject(solu annealingSolution, obj jcsypes.ObjectDetail) []datamap.SysEventBody {
var blockChgs []datamap.BlockChange var blockChgs []datamap.BlockChange


for i, f := range solu.rmBlocks { for i, f := range solu.rmBlocks {
hasCache := lo.ContainsBy(obj.Blocks, func(b clitypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n clitypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
hasCache := lo.ContainsBy(obj.Blocks, func(b jcsypes.ObjectBlock) bool { return b.UserSpaceID == solu.blockList[i].UserSpaceID }) ||
lo.ContainsBy(obj.PinnedAt, func(n jcsypes.UserSpaceID) bool { return n == solu.blockList[i].UserSpaceID })
willRm := f willRm := f


if !willRm { if !willRm {
@@ -782,7 +782,7 @@ func (t *ChangeRedundancy) generateSysEventForRepObject(solu annealingSolution,
return []datamap.SysEventBody{transEvt, distEvt} return []datamap.SysEventBody{transEvt, distEvt}
} }


func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, solu annealingSolution, obj clitypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[clitypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, solu annealingSolution, obj jcsypes.ObjectDetail, planBld *exec.PlanBuilder, planningHubIDs map[jcsypes.UserSpaceID]bool) db.UpdatingObjectRedundancy {
entry := db.UpdatingObjectRedundancy{ entry := db.UpdatingObjectRedundancy{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
FileHash: obj.Object.FileHash, FileHash: obj.Object.FileHash,
@@ -790,11 +790,11 @@ func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, so
Redundancy: obj.Object.Redundancy, Redundancy: obj.Object.Redundancy,
} }


reconstrct := make(map[clitypes.UserSpaceID]*[]int)
reconstrct := make(map[jcsypes.UserSpaceID]*[]int)
for i, f := range solu.rmBlocks { for i, f := range solu.rmBlocks {
block := solu.blockList[i] block := solu.blockList[i]
if !f { if !f {
entry.Blocks = append(entry.Blocks, clitypes.ObjectBlock{
entry.Blocks = append(entry.Blocks, jcsypes.ObjectBlock{
ObjectID: obj.Object.ObjectID, ObjectID: obj.Object.ObjectID,
Index: block.Index, Index: block.Index,
UserSpaceID: block.UserSpaceID, UserSpaceID: block.UserSpaceID,
@@ -815,7 +815,7 @@ func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, so
} }
} }


ecRed := obj.Object.Redundancy.(*clitypes.ECRedundancy)
ecRed := obj.Object.Redundancy.(*jcsypes.ECRedundancy)


for id, idxs := range reconstrct { for id, idxs := range reconstrct {
// 依次生成每个节点上的执行计划,因为如果放到一个计划里一起生成,不能保证每个节点上的块用的都是本节点上的副本 // 依次生成每个节点上的执行计划,因为如果放到一个计划里一起生成,不能保证每个节点上的块用的都是本节点上的副本
@@ -838,10 +838,10 @@ func (t *ChangeRedundancy) makePlansForECObject(ctx *changeRedundancyContext, so
return entry return entry
} }


func (t *ChangeRedundancy) generateSysEventForECObject(solu annealingSolution, obj clitypes.ObjectDetail) []datamap.SysEventBody {
func (t *ChangeRedundancy) generateSysEventForECObject(solu annealingSolution, obj jcsypes.ObjectDetail) []datamap.SysEventBody {
var blockChgs []datamap.BlockChange var blockChgs []datamap.BlockChange


reconstrct := make(map[clitypes.UserSpaceID]*[]int)
reconstrct := make(map[jcsypes.UserSpaceID]*[]int)
for i, f := range solu.rmBlocks { for i, f := range solu.rmBlocks {
block := solu.blockList[i] block := solu.blockList[i]
if !f { if !f {
@@ -917,7 +917,7 @@ func (t *ChangeRedundancy) generateSysEventForECObject(solu annealingSolution, o
return []datamap.SysEventBody{transEvt, distEvt} return []datamap.SysEventBody{transEvt, distEvt}
} }


func (t *ChangeRedundancy) executePlans(ctx *changeRedundancyContext, planBld *exec.PlanBuilder, planningSpaceIDs map[clitypes.UserSpaceID]bool, reen *publock.Reentrant) (exec.PlanResult, error) {
func (t *ChangeRedundancy) executePlans(ctx *changeRedundancyContext, planBld *exec.PlanBuilder, planningSpaceIDs map[jcsypes.UserSpaceID]bool, reen *publock.Reentrant) (exec.PlanResult, error) {
reqBlder := reqbuilder.NewBuilder() reqBlder := reqbuilder.NewBuilder()
for id, _ := range planningSpaceIDs { for id, _ := range planningSpaceIDs {
reqBlder.UserSpace().Buzy(id) reqBlder.UserSpace().Buzy(id)
@@ -955,7 +955,7 @@ func (t *ChangeRedundancy) executePlans(ctx *changeRedundancyContext, planBld *e
return ioSwRets, nil return ioSwRets, nil
} }


func (t *ChangeRedundancy) populateECObjectEntry(entry *db.UpdatingObjectRedundancy, obj clitypes.ObjectDetail, ioRets exec.PlanResult) {
func (t *ChangeRedundancy) populateECObjectEntry(entry *db.UpdatingObjectRedundancy, obj jcsypes.ObjectDetail, ioRets exec.PlanResult) {
for i := range entry.Blocks { for i := range entry.Blocks {
if entry.Blocks[i].FileHash != "" { if entry.Blocks[i].FileHash != "" {
continue continue


+ 1
- 1
client/internal/ticktock/redundancy_shrink_test.go View File

@@ -5,7 +5,7 @@ import (


. "github.com/smartystreets/goconvey/convey" . "github.com/smartystreets/goconvey/convey"
"gitlink.org.cn/cloudream/common/pkgs/bitmap" "gitlink.org.cn/cloudream/common/pkgs/bitmap"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func newTreeTest(nodeBlocksMap []bitmap.Bitmap64) combinatorialTree { func newTreeTest(nodeBlocksMap []bitmap.Bitmap64) combinatorialTree {


+ 1
- 1
client/internal/ticktock/user_space_gc.go View File

@@ -7,7 +7,7 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/reflect2" "gitlink.org.cn/cloudream/common/utils/reflect2"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"


"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
) )


+ 1
- 1
client/internal/uploader/create_load.go View File

@@ -8,12 +8,12 @@ import (
"time" "time"


"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type CreateUploader struct { type CreateUploader struct {


+ 1
- 1
client/internal/uploader/update.go View File

@@ -9,12 +9,12 @@ import (


"github.com/samber/lo" "github.com/samber/lo"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type UpdateUploader struct { type UpdateUploader struct {


+ 13
- 13
client/internal/uploader/uploader.go View File

@@ -13,7 +13,6 @@ import (
"gitlink.org.cn/cloudream/common/utils/sort2" "gitlink.org.cn/cloudream/common/utils/sort2"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/metacache" "gitlink.org.cn/cloudream/jcs-pub/client/internal/metacache"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/connectivity"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
@@ -23,6 +22,7 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type Uploader struct { type Uploader struct {
@@ -43,7 +43,7 @@ func NewUploader(pubLock *publock.Service, connectivity *connectivity.Collector,
} }
} }


func (u *Uploader) BeginUpdate(pkgID clitypes.PackageID, affinity clitypes.UserSpaceID, copyTo []clitypes.UserSpaceID, copyToPath []clitypes.JPath) (*UpdateUploader, error) {
func (u *Uploader) BeginUpdate(pkgID jcsypes.PackageID, affinity jcsypes.UserSpaceID, copyTo []jcsypes.UserSpaceID, copyToPath []jcsypes.JPath) (*UpdateUploader, error) {
spaceIDs, err := u.db.UserSpace().GetAllIDs(u.db.DefCtx()) spaceIDs, err := u.db.UserSpace().GetAllIDs(u.db.DefCtx())
if err != nil { if err != nil {
return nil, fmt.Errorf("getting user space ids: %w", err) return nil, fmt.Errorf("getting user space ids: %w", err)
@@ -84,9 +84,9 @@ func (u *Uploader) BeginUpdate(pkgID clitypes.PackageID, affinity clitypes.UserS
return nil, fmt.Errorf("user no available userspaces") return nil, fmt.Errorf("user no available userspaces")
} }


copyToSpaces := make([]clitypes.UserSpaceDetail, len(copyTo))
copyToSpaces := make([]jcsypes.UserSpaceDetail, len(copyTo))
for i, spaceID := range copyTo { for i, spaceID := range copyTo {
space, ok := lo.Find(spaceDetails, func(space *clitypes.UserSpaceDetail) bool {
space, ok := lo.Find(spaceDetails, func(space *jcsypes.UserSpaceDetail) bool {
return space.UserSpace.UserSpaceID == spaceID return space.UserSpace.UserSpaceID == spaceID
}) })
if !ok { if !ok {
@@ -118,7 +118,7 @@ func (u *Uploader) BeginUpdate(pkgID clitypes.PackageID, affinity clitypes.UserS
// 1. 选择设置了亲和性的节点 // 1. 选择设置了亲和性的节点
// 2. 从与当前客户端相同地域的节点中随机选一个 // 2. 从与当前客户端相同地域的节点中随机选一个
// 3. 没有的话从所有节点选择延迟最低的节点 // 3. 没有的话从所有节点选择延迟最低的节点
func (w *Uploader) chooseUploadStorage(spaces []UploadSpaceInfo, spaceAffinity clitypes.UserSpaceID) UploadSpaceInfo {
func (w *Uploader) chooseUploadStorage(spaces []UploadSpaceInfo, spaceAffinity jcsypes.UserSpaceID) UploadSpaceInfo {
if spaceAffinity > 0 { if spaceAffinity > 0 {
aff, ok := lo.Find(spaces, func(space UploadSpaceInfo) bool { return space.Space.UserSpace.UserSpaceID == spaceAffinity }) aff, ok := lo.Find(spaces, func(space UploadSpaceInfo) bool { return space.Space.UserSpace.UserSpaceID == spaceAffinity })
if ok { if ok {
@@ -137,10 +137,10 @@ func (w *Uploader) chooseUploadStorage(spaces []UploadSpaceInfo, spaceAffinity c
return spaces[0] return spaces[0]
} }


func (u *Uploader) BeginCreateUpload(bktID clitypes.BucketID, pkgName string, copyTo []clitypes.UserSpaceID, copyToPath []clitypes.JPath) (*CreateUploader, error) {
func (u *Uploader) BeginCreateUpload(bktID jcsypes.BucketID, pkgName string, copyTo []jcsypes.UserSpaceID, copyToPath []jcsypes.JPath) (*CreateUploader, error) {
getSpaces := u.spaceMeta.GetMany(copyTo) getSpaces := u.spaceMeta.GetMany(copyTo)


spacesStgs := make([]clitypes.UserSpaceDetail, len(copyTo))
spacesStgs := make([]jcsypes.UserSpaceDetail, len(copyTo))
for i, stg := range getSpaces { for i, stg := range getSpaces {
if stg == nil { if stg == nil {
return nil, fmt.Errorf("storage %v not found", copyTo[i]) return nil, fmt.Errorf("storage %v not found", copyTo[i])
@@ -148,10 +148,10 @@ func (u *Uploader) BeginCreateUpload(bktID clitypes.BucketID, pkgName string, co
spacesStgs[i] = *stg spacesStgs[i] = *stg
} }


pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (clitypes.Package, error) {
pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (jcsypes.Package, error) {
_, err := u.db.Bucket().GetByID(tx, bktID) _, err := u.db.Bucket().GetByID(tx, bktID)
if err != nil { if err != nil {
return clitypes.Package{}, err
return jcsypes.Package{}, err
} }


return u.db.Package().Create(u.db.DefCtx(), bktID, pkgName, time.Now()) return u.db.Package().Create(u.db.DefCtx(), bktID, pkgName, time.Now())
@@ -178,19 +178,19 @@ func (u *Uploader) BeginCreateUpload(bktID clitypes.BucketID, pkgName string, co
}, nil }, nil
} }


func (u *Uploader) UploadPart(objID clitypes.ObjectID, index int, stream io.Reader) error {
func (u *Uploader) UploadPart(objID jcsypes.ObjectID, index int, stream io.Reader) error {
detail, err := u.db.Object().GetDetail(u.db.DefCtx(), objID) detail, err := u.db.Object().GetDetail(u.db.DefCtx(), objID)
if err != nil { if err != nil {
return fmt.Errorf("getting object detail: %w", err) return fmt.Errorf("getting object detail: %w", err)
} }


objDe := detail objDe := detail
_, ok := objDe.Object.Redundancy.(*clitypes.MultipartUploadRedundancy)
_, ok := objDe.Object.Redundancy.(*jcsypes.MultipartUploadRedundancy)
if !ok { if !ok {
return fmt.Errorf("object %v is not a multipart upload", objID) return fmt.Errorf("object %v is not a multipart upload", objID)
} }


var space clitypes.UserSpaceDetail
var space jcsypes.UserSpaceDetail
if len(objDe.Blocks) > 0 { if len(objDe.Blocks) > 0 {
cstg := u.spaceMeta.Get(objDe.Blocks[0].UserSpaceID) cstg := u.spaceMeta.Get(objDe.Blocks[0].UserSpaceID)
if cstg == nil { if cstg == nil {
@@ -272,7 +272,7 @@ func (u *Uploader) UploadPart(objID clitypes.ObjectID, index int, stream io.Read


shardInfo := ret.Get("shard").(*ops2.FileInfoValue) shardInfo := ret.Get("shard").(*ops2.FileInfoValue)
err = u.db.DoTx(func(tx db.SQLContext) error { err = u.db.DoTx(func(tx db.SQLContext) error {
return u.db.Object().AppendPart(tx, clitypes.ObjectBlock{
return u.db.Object().AppendPart(tx, jcsypes.ObjectBlock{
ObjectID: objID, ObjectID: objID,
Index: index, Index: index,
UserSpaceID: space.UserSpace.UserSpaceID, UserSpaceID: space.UserSpace.UserSpaceID,


+ 7
- 7
client/internal/uploader/user_space_upload.go View File

@@ -9,7 +9,6 @@ import (


"github.com/samber/lo" "github.com/samber/lo"
"gitlink.org.cn/cloudream/jcs-pub/client/internal/db" "gitlink.org.cn/cloudream/jcs-pub/client/internal/db"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals" stgglb "gitlink.org.cn/cloudream/jcs-pub/common/globals"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
@@ -18,19 +17,20 @@ import (
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/reqbuilder"
corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator" corrpc "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc/coordinator"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )


func (u *Uploader) UserSpaceUpload(userSpaceID clitypes.UserSpaceID, rootPath clitypes.JPath, targetBktID clitypes.BucketID, newPkgName string, uploadAffinity clitypes.UserSpaceID) (*clitypes.Package, error) {
func (u *Uploader) UserSpaceUpload(userSpaceID jcsypes.UserSpaceID, rootPath jcsypes.JPath, targetBktID jcsypes.BucketID, newPkgName string, uploadAffinity jcsypes.UserSpaceID) (*jcsypes.Package, error) {
srcSpace := u.spaceMeta.Get(userSpaceID) srcSpace := u.spaceMeta.Get(userSpaceID)
if srcSpace == nil { if srcSpace == nil {
return nil, fmt.Errorf("user space %d not found", userSpaceID) return nil, fmt.Errorf("user space %d not found", userSpaceID)
} }


pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (clitypes.Package, error) {
pkg, err := db.DoTx01(u.db, func(tx db.SQLContext) (jcsypes.Package, error) {
_, err := u.db.Bucket().GetByID(tx, targetBktID) _, err := u.db.Bucket().GetByID(tx, targetBktID)
if err != nil { if err != nil {
return clitypes.Package{}, err
return jcsypes.Package{}, err
} }


return u.db.Package().Create(tx, targetBktID, newPkgName, time.Now()) return u.db.Package().Create(tx, targetBktID, newPkgName, time.Now())
@@ -49,7 +49,7 @@ func (u *Uploader) UserSpaceUpload(userSpaceID clitypes.UserSpaceID, rootPath cl
} }


spaceDetails := u.spaceMeta.GetMany(spaceIDs) spaceDetails := u.spaceMeta.GetMany(spaceIDs)
spaceDetails = lo.Filter(spaceDetails, func(e *clitypes.UserSpaceDetail, i int) bool {
spaceDetails = lo.Filter(spaceDetails, func(e *jcsypes.UserSpaceDetail, i int) bool {
return e != nil && e.UserSpace.ShardStore != nil return e != nil && e.UserSpace.ShardStore != nil
}) })


@@ -153,7 +153,7 @@ func (u *Uploader) UserSpaceUpload(userSpaceID clitypes.UserSpaceID, rootPath cl
return &pkg, nil return &pkg, nil
} }


func (u *Uploader) uploadFromBaseStore(srcSpace *clitypes.UserSpaceDetail, targetSpace *clitypes.UserSpaceDetail, entries []types.DirEntry, rootPath clitypes.JPath) ([]db.AddObjectEntry, error) {
func (u *Uploader) uploadFromBaseStore(srcSpace *jcsypes.UserSpaceDetail, targetSpace *jcsypes.UserSpaceDetail, entries []types.DirEntry, rootPath jcsypes.JPath) ([]db.AddObjectEntry, error) {
ft := ioswitch2.FromTo{} ft := ioswitch2.FromTo{}


for _, e := range entries { for _, e := range entries {
@@ -198,7 +198,7 @@ func (u *Uploader) uploadFromBaseStore(srcSpace *clitypes.UserSpaceDetail, targe
Size: info.Size, Size: info.Size,
FileHash: info.Hash, FileHash: info.Hash,
CreateTime: time.Now(), CreateTime: time.Now(),
UserSpaceIDs: []clitypes.UserSpaceID{targetSpace.UserSpace.UserSpaceID},
UserSpaceIDs: []jcsypes.UserSpaceID{targetSpace.UserSpace.UserSpaceID},
}) })
} }




+ 7
- 7
client/sdk/api/v1/bucket.go View File

@@ -4,7 +4,7 @@ import (
"net/http" "net/http"


"gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/sdks"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type BucketService struct { type BucketService struct {
@@ -18,7 +18,7 @@ func (c *Client) Bucket() *BucketService {
const BucketGetPath = "/bucket/get" const BucketGetPath = "/bucket/get"


type BucketGet struct { type BucketGet struct {
BucketID clitypes.BucketID `json:"bucketID" binding:"required"`
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
} }


func (r *BucketGet) MakeParam() *sdks.RequestParam { func (r *BucketGet) MakeParam() *sdks.RequestParam {
@@ -26,7 +26,7 @@ func (r *BucketGet) MakeParam() *sdks.RequestParam {
} }


type BucketGetResp struct { type BucketGetResp struct {
Bucket clitypes.Bucket `json:"bucket"`
Bucket jcsypes.Bucket `json:"bucket"`
} }


func (r *BucketGetResp) ParseResponse(resp *http.Response) error { func (r *BucketGetResp) ParseResponse(resp *http.Response) error {
@@ -48,7 +48,7 @@ func (r *BucketGetByName) MakeParam() *sdks.RequestParam {
} }


type BucketGetByNameResp struct { type BucketGetByNameResp struct {
Bucket clitypes.Bucket `json:"bucket"`
Bucket jcsypes.Bucket `json:"bucket"`
} }


func (r *BucketGetByNameResp) ParseResponse(resp *http.Response) error { func (r *BucketGetByNameResp) ParseResponse(resp *http.Response) error {
@@ -70,7 +70,7 @@ func (r *BucketCreate) MakeParam() *sdks.RequestParam {
} }


type BucketCreateResp struct { type BucketCreateResp struct {
Bucket clitypes.Bucket `json:"bucket"`
Bucket jcsypes.Bucket `json:"bucket"`
} }


func (r *BucketCreateResp) ParseResponse(resp *http.Response) error { func (r *BucketCreateResp) ParseResponse(resp *http.Response) error {
@@ -84,7 +84,7 @@ func (c *BucketService) Create(req BucketCreate) (*BucketCreateResp, error) {
const BucketDeletePath = "/bucket/delete" const BucketDeletePath = "/bucket/delete"


type BucketDelete struct { type BucketDelete struct {
BucketID clitypes.BucketID `json:"bucketID" binding:"required"`
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
} }


func (r *BucketDelete) MakeParam() *sdks.RequestParam { func (r *BucketDelete) MakeParam() *sdks.RequestParam {
@@ -111,7 +111,7 @@ func (r *BucketListAll) MakeParam() *sdks.RequestParam {
} }


type BucketListAllResp struct { type BucketListAllResp struct {
Buckets []clitypes.Bucket `json:"buckets"`
Buckets []jcsypes.Bucket `json:"buckets"`
} }


func (r *BucketListAllResp) ParseResponse(resp *http.Response) error { func (r *BucketListAllResp) ParseResponse(resp *http.Response) error {


+ 1
- 1
client/sdk/api/v1/object.go View File

@@ -14,7 +14,7 @@ import (
"gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/sdks"
"gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/http2"
"gitlink.org.cn/cloudream/common/utils/serder" "gitlink.org.cn/cloudream/common/utils/serder"
"gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type ObjectService struct { type ObjectService struct {


+ 24
- 24
client/sdk/api/v1/package.go View File

@@ -13,7 +13,7 @@ import (
"gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/sdks"
"gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/http2"
"gitlink.org.cn/cloudream/common/utils/serder" "gitlink.org.cn/cloudream/common/utils/serder"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type PackageService struct { type PackageService struct {
@@ -27,7 +27,7 @@ func (c *Client) Package() *PackageService {
const PackageGetPath = "/package/get" const PackageGetPath = "/package/get"


type PackageGet struct { type PackageGet struct {
PackageID clitypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
PackageID jcsypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
} }


func (r *PackageGet) MakeParam() *sdks.RequestParam { func (r *PackageGet) MakeParam() *sdks.RequestParam {
@@ -35,7 +35,7 @@ func (r *PackageGet) MakeParam() *sdks.RequestParam {
} }


type PackageGetResp struct { type PackageGetResp struct {
Package clitypes.Package `json:"package"`
Package jcsypes.Package `json:"package"`
} }


func (r *PackageGetResp) ParseResponse(resp *http.Response) error { func (r *PackageGetResp) ParseResponse(resp *http.Response) error {
@@ -58,7 +58,7 @@ func (r *PackageGetByFullName) MakeParam() *sdks.RequestParam {
} }


type PackageGetByFullNameResp struct { type PackageGetByFullNameResp struct {
Package clitypes.Package `json:"package"`
Package jcsypes.Package `json:"package"`
} }


func (r *PackageGetByFullNameResp) ParseResponse(resp *http.Response) error { func (r *PackageGetByFullNameResp) ParseResponse(resp *http.Response) error {
@@ -72,8 +72,8 @@ func (c *PackageService) GetByFullName(req PackageGetByFullName) (*PackageGetByF
const PackageCreatePath = "/package/create" const PackageCreatePath = "/package/create"


type PackageCreate struct { type PackageCreate struct {
BucketID clitypes.BucketID `json:"bucketID"`
Name string `json:"name"`
BucketID jcsypes.BucketID `json:"bucketID"`
Name string `json:"name"`
} }


func (r *PackageCreate) MakeParam() *sdks.RequestParam { func (r *PackageCreate) MakeParam() *sdks.RequestParam {
@@ -81,7 +81,7 @@ func (r *PackageCreate) MakeParam() *sdks.RequestParam {
} }


type PackageCreateResp struct { type PackageCreateResp struct {
Package clitypes.Package `json:"package"`
Package jcsypes.Package `json:"package"`
} }


func (r *PackageCreateResp) ParseResponse(resp *http.Response) error { func (r *PackageCreateResp) ParseResponse(resp *http.Response) error {
@@ -99,14 +99,14 @@ type PackageCreateUpload struct {
Files UploadObjectIterator `json:"-"` Files UploadObjectIterator `json:"-"`
} }
type PackageCreateUploadInfo struct { type PackageCreateUploadInfo struct {
BucketID clitypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
CopyTo []clitypes.UserSpaceID `json:"copyTo"`
CopyToPath []string `json:"copyToPath"`
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
CopyTo []jcsypes.UserSpaceID `json:"copyTo"`
CopyToPath []string `json:"copyToPath"`
} }
type PackageCreateUploadResp struct { type PackageCreateUploadResp struct {
Package clitypes.Package `json:"package"`
Objects []clitypes.Object `json:"objects"`
Package jcsypes.Package `json:"package"`
Objects []jcsypes.Object `json:"objects"`
} }


func (c *PackageService) CreateUpload(req PackageCreateUpload) (*PackageCreateUploadResp, error) { func (c *PackageService) CreateUpload(req PackageCreateUpload) (*PackageCreateUploadResp, error) {
@@ -148,10 +148,10 @@ func (c *PackageService) CreateUpload(req PackageCreateUpload) (*PackageCreateUp
const PackageDownloadPath = "/package/download" const PackageDownloadPath = "/package/download"


type PackageDownload struct { type PackageDownload struct {
PackageID clitypes.PackageID `url:"packageID" form:"packageID" binding:"required"`
Prefix string `url:"prefix" form:"prefix"`
NewPrefix *string `url:"newPrefix,omitempty" form:"newPrefix"`
Zip bool `url:"zip,omitempty" form:"zip"`
PackageID jcsypes.PackageID `url:"packageID" form:"packageID" binding:"required"`
Prefix string `url:"prefix" form:"prefix"`
NewPrefix *string `url:"newPrefix,omitempty" form:"newPrefix"`
Zip bool `url:"zip,omitempty" form:"zip"`
} }


func (r *PackageDownload) MakeParam() *sdks.RequestParam { func (r *PackageDownload) MakeParam() *sdks.RequestParam {
@@ -208,7 +208,7 @@ func (c *PackageService) Download(req PackageDownload) (*DownloadingPackage, err
const PackageDeletePath = "/package/delete" const PackageDeletePath = "/package/delete"


type PackageDelete struct { type PackageDelete struct {
PackageID clitypes.PackageID `json:"packageID" binding:"required"`
PackageID jcsypes.PackageID `json:"packageID" binding:"required"`
} }


func (r *PackageDelete) MakeParam() *sdks.RequestParam { func (r *PackageDelete) MakeParam() *sdks.RequestParam {
@@ -228,9 +228,9 @@ func (c *PackageService) Delete(req PackageDelete) error {
const PackageClonePath = "/package/clone" const PackageClonePath = "/package/clone"


type PackageClone struct { type PackageClone struct {
PackageID clitypes.PackageID `json:"packageID" binding:"required"`
BucketID clitypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
PackageID jcsypes.PackageID `json:"packageID" binding:"required"`
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
} }


func (r *PackageClone) MakeParam() *sdks.RequestParam { func (r *PackageClone) MakeParam() *sdks.RequestParam {
@@ -238,7 +238,7 @@ func (r *PackageClone) MakeParam() *sdks.RequestParam {
} }


type PackageCloneResp struct { type PackageCloneResp struct {
Package clitypes.Package `json:"package"`
Package jcsypes.Package `json:"package"`
} }


func (r *PackageCloneResp) ParseResponse(resp *http.Response) error { func (r *PackageCloneResp) ParseResponse(resp *http.Response) error {
@@ -252,7 +252,7 @@ func (c *PackageService) Clone(req PackageClone) (*PackageCloneResp, error) {
const PackageListBucketPackagesPath = "/package/listBucketPackages" const PackageListBucketPackagesPath = "/package/listBucketPackages"


type PackageListBucketPackages struct { type PackageListBucketPackages struct {
BucketID clitypes.BucketID `form:"bucketID" url:"bucketID" binding:"required"`
BucketID jcsypes.BucketID `form:"bucketID" url:"bucketID" binding:"required"`
} }


func (r *PackageListBucketPackages) MakeParam() *sdks.RequestParam { func (r *PackageListBucketPackages) MakeParam() *sdks.RequestParam {
@@ -260,7 +260,7 @@ func (r *PackageListBucketPackages) MakeParam() *sdks.RequestParam {
} }


type PackageListBucketPackagesResp struct { type PackageListBucketPackagesResp struct {
Packages []clitypes.Package `json:"packages"`
Packages []jcsypes.Package `json:"packages"`
} }


func (r *PackageListBucketPackagesResp) ParseResponse(resp *http.Response) error { func (r *PackageListBucketPackagesResp) ParseResponse(resp *http.Response) error {


+ 22
- 22
client/sdk/api/v1/presigned.go View File

@@ -7,7 +7,7 @@ import (


"github.com/google/go-querystring/query" "github.com/google/go-querystring/query"
"gitlink.org.cn/cloudream/jcs-pub/client/sdk/signer" "gitlink.org.cn/cloudream/jcs-pub/client/sdk/signer"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type PresignedService struct { type PresignedService struct {
@@ -33,10 +33,10 @@ func (c *PresignedService) ObjectListByPath(req PresignedObjectListByPath, expir
const PresignedObjectDownloadByPathPath = "/presigned/object/downloadByPath" const PresignedObjectDownloadByPathPath = "/presigned/object/downloadByPath"


type PresignedObjectDownloadByPath struct { type PresignedObjectDownloadByPath struct {
PackageID clitypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
Path string `form:"path" url:"path" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
PackageID jcsypes.PackageID `form:"packageID" url:"packageID" binding:"required"`
Path string `form:"path" url:"path" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
} }


func (c *PresignedService) ObjectDownloadByPath(req PresignedObjectDownloadByPath, expireIn int) (string, error) { func (c *PresignedService) ObjectDownloadByPath(req PresignedObjectDownloadByPath, expireIn int) (string, error) {
@@ -46,9 +46,9 @@ func (c *PresignedService) ObjectDownloadByPath(req PresignedObjectDownloadByPat
const PresignedObjectDownloadPath = "/presigned/object/download" const PresignedObjectDownloadPath = "/presigned/object/download"


type PresignedObjectDownload struct { type PresignedObjectDownload struct {
ObjectID clitypes.ObjectID `form:"objectID" url:"objectID" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
ObjectID jcsypes.ObjectID `form:"objectID" url:"objectID" binding:"required"`
Offset int64 `form:"offset" url:"offset,omitempty"`
Length *int64 `form:"length" url:"length,omitempty"`
} }


func (c *PresignedService) ObjectDownload(req PresignedObjectDownload, expireIn int) (string, error) { func (c *PresignedService) ObjectDownload(req PresignedObjectDownload, expireIn int) (string, error) {
@@ -58,15 +58,15 @@ func (c *PresignedService) ObjectDownload(req PresignedObjectDownload, expireIn
const PresignedObjectUploadPath = "/presigned/object/upload" const PresignedObjectUploadPath = "/presigned/object/upload"


type PresignedObjectUpload struct { type PresignedObjectUpload struct {
PackageID clitypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
Affinity clitypes.UserSpaceID `form:"affinity" url:"affinity,omitempty"`
CopyTo []clitypes.UserSpaceID `form:"copyTo" url:"copyTo,omitempty"`
CopyToPath []string `form:"copyToPath" url:"copyToPath,omitempty"`
PackageID jcsypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
Affinity jcsypes.UserSpaceID `form:"affinity" url:"affinity,omitempty"`
CopyTo []jcsypes.UserSpaceID `form:"copyTo" url:"copyTo,omitempty"`
CopyToPath []string `form:"copyToPath" url:"copyToPath,omitempty"`
} }


type PresignedObjectUploadResp struct { type PresignedObjectUploadResp struct {
Object clitypes.Object `json:"object"`
Object jcsypes.Object `json:"object"`
} }


func (c *PresignedService) ObjectUpload(req PresignedObjectUpload, expireIn int) (string, error) { func (c *PresignedService) ObjectUpload(req PresignedObjectUpload, expireIn int) (string, error) {
@@ -76,12 +76,12 @@ func (c *PresignedService) ObjectUpload(req PresignedObjectUpload, expireIn int)
const PresignedObjectNewMultipartUploadPath = "/presigned/object/newMultipartUpload" const PresignedObjectNewMultipartUploadPath = "/presigned/object/newMultipartUpload"


type PresignedObjectNewMultipartUpload struct { type PresignedObjectNewMultipartUpload struct {
PackageID clitypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
PackageID jcsypes.PackageID `form:"packageID" binding:"required" url:"packageID"`
Path string `form:"path" binding:"required" url:"path"`
} }


type PresignedObjectNewMultipartUploadResp struct { type PresignedObjectNewMultipartUploadResp struct {
Object clitypes.Object `json:"object"`
Object jcsypes.Object `json:"object"`
} }


func (c *PresignedService) ObjectNewMultipartUpload(req PresignedObjectNewMultipartUpload, expireIn int) (string, error) { func (c *PresignedService) ObjectNewMultipartUpload(req PresignedObjectNewMultipartUpload, expireIn int) (string, error) {
@@ -91,8 +91,8 @@ func (c *PresignedService) ObjectNewMultipartUpload(req PresignedObjectNewMultip
const PresignedObjectUploadPartPath = "/presigned/object/uploadPart" const PresignedObjectUploadPartPath = "/presigned/object/uploadPart"


type PresignedObjectUploadPart struct { type PresignedObjectUploadPart struct {
ObjectID clitypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Index int `form:"index" binding:"required" url:"index"`
ObjectID jcsypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Index int `form:"index" binding:"required" url:"index"`
} }


type PresignedUploadPartResp struct{} type PresignedUploadPartResp struct{}
@@ -104,12 +104,12 @@ func (c *PresignedService) ObjectUploadPart(req PresignedObjectUploadPart, expir
const PresignedObjectCompleteMultipartUploadPath = "/presigned/object/completeMultipartUpload" const PresignedObjectCompleteMultipartUploadPath = "/presigned/object/completeMultipartUpload"


type PresignedObjectCompleteMultipartUpload struct { type PresignedObjectCompleteMultipartUpload struct {
ObjectID clitypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Indexes []int `form:"indexes" binding:"required" url:"indexes"`
ObjectID jcsypes.ObjectID `form:"objectID" binding:"required" url:"objectID"`
Indexes []int `form:"indexes" binding:"required" url:"indexes"`
} }


type PresignedObjectCompleteMultipartUploadResp struct { type PresignedObjectCompleteMultipartUploadResp struct {
Object clitypes.Object `json:"object"`
Object jcsypes.Object `json:"object"`
} }


func (c *PresignedService) ObjectCompleteMultipartUpload(req PresignedObjectCompleteMultipartUpload, expireIn int) (string, error) { func (c *PresignedService) ObjectCompleteMultipartUpload(req PresignedObjectCompleteMultipartUpload, expireIn int) (string, error) {


+ 13
- 13
client/sdk/api/v1/space_syncer.go View File

@@ -4,7 +4,7 @@ import (
"net/http" "net/http"


"gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/sdks"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type SpaceSyncerService struct { type SpaceSyncerService struct {
@@ -20,14 +20,14 @@ func (c *Client) SpaceSyncer() *SpaceSyncerService {
const SpaceSyncerCreateTaskPath = "/spaceSyncer/createTask" const SpaceSyncerCreateTaskPath = "/spaceSyncer/createTask"


type SpaceSyncerCreateTask struct { type SpaceSyncerCreateTask struct {
Trigger clitypes.SpaceSyncTrigger `json:"trigger" binding:"required"`
Mode clitypes.SpaceSyncMode `json:"mode" binding:"required"`
Filters []clitypes.SpaceSyncFilter `json:"filters"`
Options clitypes.SpaceSyncOptions `json:"options" binding:"required"`
SrcUserSpaceID clitypes.UserSpaceID `json:"srcUserSpaceID" binding:"required"`
SrcPath string `json:"srcPath"`
DestUserSpaceIDs []clitypes.UserSpaceID `json:"destUserSpaceIDs" binding:"required"`
DestPathes []string `json:"destPathes" binding:"required"`
Trigger jcsypes.SpaceSyncTrigger `json:"trigger" binding:"required"`
Mode jcsypes.SpaceSyncMode `json:"mode" binding:"required"`
Filters []jcsypes.SpaceSyncFilter `json:"filters"`
Options jcsypes.SpaceSyncOptions `json:"options" binding:"required"`
SrcUserSpaceID jcsypes.UserSpaceID `json:"srcUserSpaceID" binding:"required"`
SrcPath string `json:"srcPath"`
DestUserSpaceIDs []jcsypes.UserSpaceID `json:"destUserSpaceIDs" binding:"required"`
DestPathes []string `json:"destPathes" binding:"required"`
} }


func (r *SpaceSyncerCreateTask) MakeParam() *sdks.RequestParam { func (r *SpaceSyncerCreateTask) MakeParam() *sdks.RequestParam {
@@ -35,7 +35,7 @@ func (r *SpaceSyncerCreateTask) MakeParam() *sdks.RequestParam {
} }


type SpaceSyncerCreateTaskResp struct { type SpaceSyncerCreateTaskResp struct {
Task clitypes.SpaceSyncTask `json:"task"`
Task jcsypes.SpaceSyncTask `json:"task"`
} }


func (r *SpaceSyncerCreateTaskResp) ParseResponse(resp *http.Response) error { func (r *SpaceSyncerCreateTaskResp) ParseResponse(resp *http.Response) error {
@@ -49,7 +49,7 @@ func (c *SpaceSyncerService) CreateTask(req SpaceSyncerCreateTask) (*SpaceSyncer
const SpaceSyncerGetTaskPath = "/spaceSyncer/getTask" const SpaceSyncerGetTaskPath = "/spaceSyncer/getTask"


type SpaceSyncerGetTask struct { type SpaceSyncerGetTask struct {
TaskID clitypes.SpaceSyncTaskID `url:"taskID" binding:"required"`
TaskID jcsypes.SpaceSyncTaskID `url:"taskID" binding:"required"`
} }


func (r *SpaceSyncerGetTask) MakeParam() *sdks.RequestParam { func (r *SpaceSyncerGetTask) MakeParam() *sdks.RequestParam {
@@ -57,7 +57,7 @@ func (r *SpaceSyncerGetTask) MakeParam() *sdks.RequestParam {
} }


type SpaceSyncerGetTaskResp struct { type SpaceSyncerGetTaskResp struct {
Task clitypes.SpaceSyncTask `json:"task"`
Task jcsypes.SpaceSyncTask `json:"task"`
} }


func (r *SpaceSyncerGetTaskResp) ParseResponse(resp *http.Response) error { func (r *SpaceSyncerGetTaskResp) ParseResponse(resp *http.Response) error {
@@ -71,7 +71,7 @@ func (c *SpaceSyncerService) GetTask(req SpaceSyncerGetTask) (*SpaceSyncerGetTas
const SpaceSyncerCancelTaskPath = "/spaceSyncer/cancelTask" const SpaceSyncerCancelTaskPath = "/spaceSyncer/cancelTask"


type SpaceSyncerCancelTask struct { type SpaceSyncerCancelTask struct {
TaskID clitypes.SpaceSyncTaskID `json:"taskID" binding:"required"`
TaskID jcsypes.SpaceSyncTaskID `json:"taskID" binding:"required"`
} }


func (r *SpaceSyncerCancelTask) MakeParam() *sdks.RequestParam { func (r *SpaceSyncerCancelTask) MakeParam() *sdks.RequestParam {


+ 2
- 2
client/sdk/api/v1/storage_test.go View File

@@ -10,7 +10,7 @@ import (
. "github.com/smartystreets/goconvey/convey" . "github.com/smartystreets/goconvey/convey"
"gitlink.org.cn/cloudream/common/pkgs/iterator" "gitlink.org.cn/cloudream/common/pkgs/iterator"
"gitlink.org.cn/cloudream/jcs-pub/client/sdk/api" "gitlink.org.cn/cloudream/jcs-pub/client/sdk/api"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func Test_PackageGet(t *testing.T) { func Test_PackageGet(t *testing.T) {
@@ -74,7 +74,7 @@ func Test_Object(t *testing.T) {
fileData[i] = byte(i) fileData[i] = byte(i)
} }


stgAff := clitypes.UserSpaceID(2)
stgAff := jcsypes.UserSpaceID(2)


pkgName := uuid.NewString() pkgName := uuid.NewString()
createResp, err := cli.Package().Create(PackageCreate{ createResp, err := cli.Package().Create(PackageCreate{


+ 2
- 2
client/sdk/api/v1/system.go View File

@@ -4,7 +4,7 @@ import (
"net/http" "net/http"


"gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/sdks"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type SystemService struct { type SystemService struct {
@@ -25,7 +25,7 @@ func (r *SystemStatus) MakeParam() *sdks.RequestParam {
} }


type SystemStatusResp struct { type SystemStatusResp struct {
SpeedStats clitypes.SpeedStatsStatus `json:"speedStats"`
SpeedStats jcsypes.SpeedStatsStatus `json:"speedStats"`
} }


func (r *SystemStatusResp) ParseResponse(resp *http.Response) error { func (r *SystemStatusResp) ParseResponse(resp *http.Response) error {


+ 18
- 18
client/sdk/api/v1/user_space.go View File

@@ -4,7 +4,7 @@ import (
"net/http" "net/http"


"gitlink.org.cn/cloudream/common/sdks" "gitlink.org.cn/cloudream/common/sdks"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )


@@ -19,9 +19,9 @@ func (c *Client) UserSpace() *UserSpaceService {
const UserSpaceDownloadPackagePath = "/userSpace/downloadPackage" const UserSpaceDownloadPackagePath = "/userSpace/downloadPackage"


type UserSpaceDownloadPackageReq struct { type UserSpaceDownloadPackageReq struct {
PackageID clitypes.PackageID `json:"packageID" binding:"required"`
UserSpaceID clitypes.UserSpaceID `json:"userSpaceID" binding:"required"`
RootPath string `json:"rootPath"`
PackageID jcsypes.PackageID `json:"packageID" binding:"required"`
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
RootPath string `json:"rootPath"`
} }


func (r *UserSpaceDownloadPackageReq) MakeParam() *sdks.RequestParam { func (r *UserSpaceDownloadPackageReq) MakeParam() *sdks.RequestParam {
@@ -41,11 +41,11 @@ func (c *UserSpaceService) DownloadPackage(req UserSpaceDownloadPackageReq) (*Us
const UserSpaceCreatePackagePath = "/userSpace/createPackage" const UserSpaceCreatePackagePath = "/userSpace/createPackage"


type UserSpaceCreatePackageReq struct { type UserSpaceCreatePackageReq struct {
UserSpaceID clitypes.UserSpaceID `json:"userSpaceID" binding:"required"`
Path string `json:"path" binding:"required"`
BucketID clitypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
SpaceAffinity clitypes.UserSpaceID `json:"spaceAffinity"`
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
Path string `json:"path" binding:"required"`
BucketID jcsypes.BucketID `json:"bucketID" binding:"required"`
Name string `json:"name" binding:"required"`
SpaceAffinity jcsypes.UserSpaceID `json:"spaceAffinity"`
} }


func (r *UserSpaceCreatePackageReq) MakeParam() *sdks.RequestParam { func (r *UserSpaceCreatePackageReq) MakeParam() *sdks.RequestParam {
@@ -53,7 +53,7 @@ func (r *UserSpaceCreatePackageReq) MakeParam() *sdks.RequestParam {
} }


type UserSpaceCreatePackageResp struct { type UserSpaceCreatePackageResp struct {
Package clitypes.Package `json:"package"`
Package jcsypes.Package `json:"package"`
} }


func (r *UserSpaceCreatePackageResp) ParseResponse(resp *http.Response) error { func (r *UserSpaceCreatePackageResp) ParseResponse(resp *http.Response) error {
@@ -67,7 +67,7 @@ func (c *UserSpaceService) CreatePackage(req UserSpaceCreatePackageReq) (*UserSp
const UserSpaceGetPath = "/userSpace/get" const UserSpaceGetPath = "/userSpace/get"


type UserSpaceGet struct { type UserSpaceGet struct {
UserSpaceID clitypes.UserSpaceID `form:"userSpaceID" url:"userSpaceID" binding:"required"`
UserSpaceID jcsypes.UserSpaceID `form:"userSpaceID" url:"userSpaceID" binding:"required"`
} }


func (r *UserSpaceGet) MakeParam() *sdks.RequestParam { func (r *UserSpaceGet) MakeParam() *sdks.RequestParam {
@@ -75,7 +75,7 @@ func (r *UserSpaceGet) MakeParam() *sdks.RequestParam {
} }


type UserSpaceGetResp struct { type UserSpaceGetResp struct {
UserSpace clitypes.UserSpace `json:"userSpace"`
UserSpace jcsypes.UserSpace `json:"userSpace"`
} }


func (r *UserSpaceGetResp) ParseResponse(resp *http.Response) error { func (r *UserSpaceGetResp) ParseResponse(resp *http.Response) error {
@@ -97,7 +97,7 @@ func (r *UserSpaceGetByName) MakeParam() *sdks.RequestParam {
} }


type UserSpaceGetByNameResp struct { type UserSpaceGetByNameResp struct {
UserSpace clitypes.UserSpace `json:"userSpace"`
UserSpace jcsypes.UserSpace `json:"userSpace"`
} }


func (r *UserSpaceGetByNameResp) ParseResponse(resp *http.Response) error { func (r *UserSpaceGetByNameResp) ParseResponse(resp *http.Response) error {
@@ -117,7 +117,7 @@ func (r *UserSpaceGetAll) MakeParam() *sdks.RequestParam {
} }


type UserSpaceGetAllResp struct { type UserSpaceGetAllResp struct {
UserSpaces []clitypes.UserSpace `json:"userSpaces"`
UserSpaces []jcsypes.UserSpace `json:"userSpaces"`
} }


func (r *UserSpaceGetAllResp) ParseResponse(resp *http.Response) error { func (r *UserSpaceGetAllResp) ParseResponse(resp *http.Response) error {
@@ -146,7 +146,7 @@ func (r *UserSpaceCreate) MakeParam() *sdks.RequestParam {
} }


type UserSpaceCreateResp struct { type UserSpaceCreateResp struct {
UserSpace clitypes.UserSpace `json:"userSpace"`
UserSpace jcsypes.UserSpace `json:"userSpace"`
} }


func (r *UserSpaceCreateResp) ParseResponse(resp *http.Response) error { func (r *UserSpaceCreateResp) ParseResponse(resp *http.Response) error {
@@ -161,7 +161,7 @@ func (c *UserSpaceService) Create(req UserSpaceCreate) (*UserSpaceCreateResp, er
const UserSpaceUpdatePath = "/userSpace/update" const UserSpaceUpdatePath = "/userSpace/update"


type UserSpaceUpdate struct { type UserSpaceUpdate struct {
UserSpaceID clitypes.UserSpaceID `json:"userSpaceID" binding:"required"`
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
Name string `json:"name" binding:"required"` Name string `json:"name" binding:"required"`
Credential cortypes.StorageCredential `json:"credential" binding:"required"` Credential cortypes.StorageCredential `json:"credential" binding:"required"`
Features []cortypes.StorageFeature `json:"features"` Features []cortypes.StorageFeature `json:"features"`
@@ -172,7 +172,7 @@ func (r *UserSpaceUpdate) MakeParam() *sdks.RequestParam {
} }


type UserSpaceUpdateResp struct { type UserSpaceUpdateResp struct {
UserSpace clitypes.UserSpace `json:"userSpace"`
UserSpace jcsypes.UserSpace `json:"userSpace"`
} }


func (r *UserSpaceUpdateResp) ParseResponse(resp *http.Response) error { func (r *UserSpaceUpdateResp) ParseResponse(resp *http.Response) error {
@@ -187,7 +187,7 @@ func (c *UserSpaceService) Update(req UserSpaceUpdate) (*UserSpaceUpdateResp, er
const UserSpaceDeletePath = "/userSpace/delete" const UserSpaceDeletePath = "/userSpace/delete"


type UserSpaceDelete struct { type UserSpaceDelete struct {
UserSpaceID clitypes.UserSpaceID `json:"userSpaceID" binding:"required"`
UserSpaceID jcsypes.UserSpaceID `json:"userSpaceID" binding:"required"`
} }


func (r *UserSpaceDelete) MakeParam() *sdks.RequestParam { func (r *UserSpaceDelete) MakeParam() *sdks.RequestParam {


+ 15
- 15
common/pkgs/ioswitch2/fromto.go View File

@@ -2,9 +2,9 @@ package ioswitch2


import ( import (
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type From interface { type From interface {
@@ -69,9 +69,9 @@ type FromTos []FromTo


type FromTo struct { type FromTo struct {
// 如果输入或者输出用到了EC编码的流,则需要提供EC参数。 // 如果输入或者输出用到了EC编码的流,则需要提供EC参数。
ECParam *clitypes.ECRedundancy
ECParam *jcsypes.ECRedundancy
// 同上 // 同上
SegmentParam *clitypes.SegmentRedundancy
SegmentParam *jcsypes.SegmentRedundancy
Froms []From Froms []From
Toes []To Toes []To
} }
@@ -110,12 +110,12 @@ func (f *FromDriver) GetStreamIndex() StreamIndex {
} }


type FromShardStore struct { type FromShardStore struct {
FileHash clitypes.FileHash
UserSpace clitypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcsypes.UserSpaceDetail
StreamIndex StreamIndex StreamIndex StreamIndex
} }


func NewFromShardstore(fileHash clitypes.FileHash, space clitypes.UserSpaceDetail, strIdx StreamIndex) *FromShardStore {
func NewFromShardstore(fileHash jcsypes.FileHash, space jcsypes.UserSpaceDetail, strIdx StreamIndex) *FromShardStore {
return &FromShardStore{ return &FromShardStore{
FileHash: fileHash, FileHash: fileHash,
UserSpace: space, UserSpace: space,
@@ -128,11 +128,11 @@ func (f *FromShardStore) GetStreamIndex() StreamIndex {
} }


type FromBaseStore struct { type FromBaseStore struct {
UserSpace clitypes.UserSpaceDetail
Path clitypes.JPath
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
} }


func NewFromBaseStore(space clitypes.UserSpaceDetail, path clitypes.JPath) *FromBaseStore {
func NewFromBaseStore(space jcsypes.UserSpaceDetail, path jcsypes.JPath) *FromBaseStore {
return &FromBaseStore{ return &FromBaseStore{
UserSpace: space, UserSpace: space,
Path: path, Path: path,
@@ -177,13 +177,13 @@ func (t *ToDriver) GetRange() math2.Range {
} }


type ToShardStore struct { type ToShardStore struct {
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
StreamIndex StreamIndex StreamIndex StreamIndex
Range math2.Range Range math2.Range
ResultStoreKey string ResultStoreKey string
} }


func NewToShardStore(space clitypes.UserSpaceDetail, strIdx StreamIndex, retStoreKey string) *ToShardStore {
func NewToShardStore(space jcsypes.UserSpaceDetail, strIdx StreamIndex, retStoreKey string) *ToShardStore {
return &ToShardStore{ return &ToShardStore{
UserSpace: space, UserSpace: space,
StreamIndex: strIdx, StreamIndex: strIdx,
@@ -191,7 +191,7 @@ func NewToShardStore(space clitypes.UserSpaceDetail, strIdx StreamIndex, retStor
} }
} }


func NewToShardStoreWithRange(space clitypes.UserSpaceDetail, streamIndex StreamIndex, retStoreKey string, rng math2.Range) *ToShardStore {
func NewToShardStoreWithRange(space jcsypes.UserSpaceDetail, streamIndex StreamIndex, retStoreKey string, rng math2.Range) *ToShardStore {
return &ToShardStore{ return &ToShardStore{
UserSpace: space, UserSpace: space,
StreamIndex: streamIndex, StreamIndex: streamIndex,
@@ -209,12 +209,12 @@ func (t *ToShardStore) GetRange() math2.Range {
} }


type ToBaseStore struct { type ToBaseStore struct {
UserSpace clitypes.UserSpaceDetail
ObjectPath clitypes.JPath
UserSpace jcsypes.UserSpaceDetail
ObjectPath jcsypes.JPath
Option types.WriteOption Option types.WriteOption
} }


func NewToBaseStore(space clitypes.UserSpaceDetail, objectPath clitypes.JPath) *ToBaseStore {
func NewToBaseStore(space jcsypes.UserSpaceDetail, objectPath jcsypes.JPath) *ToBaseStore {
return &ToBaseStore{ return &ToBaseStore{
UserSpace: space, UserSpace: space,
ObjectPath: objectPath, ObjectPath: objectPath,


+ 14
- 14
common/pkgs/ioswitch2/ops2/base_store.go View File

@@ -8,12 +8,12 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/future" "gitlink.org.cn/cloudream/common/pkgs/future"
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/io2" "gitlink.org.cn/cloudream/common/utils/io2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


const ( const (
@@ -43,8 +43,8 @@ func (v *BaseReadStatsValue) Clone() exec.VarValue {


type BaseRead struct { type BaseRead struct {
Output exec.VarID Output exec.VarID
UserSpace clitypes.UserSpaceDetail
Path clitypes.JPath
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
Option types.OpenOption Option types.OpenOption
} }


@@ -101,7 +101,7 @@ func (o *BaseRead) String() string {
} }


type BaseReadDyn struct { type BaseReadDyn struct {
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
Output exec.VarID Output exec.VarID
FileInfo exec.VarID FileInfo exec.VarID
Option types.OpenOption Option types.OpenOption
@@ -167,8 +167,8 @@ func (o *BaseReadDyn) String() string {


type BaseWrite struct { type BaseWrite struct {
Input exec.VarID Input exec.VarID
UserSpace clitypes.UserSpaceDetail
Path clitypes.JPath
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
FileInfo exec.VarID FileInfo exec.VarID
Option types.WriteOption Option types.WriteOption
} }
@@ -213,12 +213,12 @@ func (o *BaseWrite) String() string {
type BaseReadNode struct { type BaseReadNode struct {
dag.NodeBase dag.NodeBase
From ioswitch2.From From ioswitch2.From
UserSpace clitypes.UserSpaceDetail
Path clitypes.JPath
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
Option types.OpenOption Option types.OpenOption
} }


func (b *GraphNodeBuilder) NewBaseRead(from ioswitch2.From, userSpace clitypes.UserSpaceDetail, path clitypes.JPath, opt types.OpenOption) *BaseReadNode {
func (b *GraphNodeBuilder) NewBaseRead(from ioswitch2.From, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.OpenOption) *BaseReadNode {
node := &BaseReadNode{ node := &BaseReadNode{
From: from, From: from,
UserSpace: userSpace, UserSpace: userSpace,
@@ -254,11 +254,11 @@ func (t *BaseReadNode) GenerateOp() (exec.Op, error) {
type BaseReadDynNode struct { type BaseReadDynNode struct {
dag.NodeBase dag.NodeBase
From ioswitch2.From From ioswitch2.From
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
Option types.OpenOption Option types.OpenOption
} }


func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitch2.From, userSpace clitypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitch2.From, userSpace jcsypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
node := &BaseReadDynNode{ node := &BaseReadDynNode{
From: from, From: from,
UserSpace: userSpace, UserSpace: userSpace,
@@ -301,12 +301,12 @@ func (t *BaseReadDynNode) GenerateOp() (exec.Op, error) {
type BaseWriteNode struct { type BaseWriteNode struct {
dag.NodeBase dag.NodeBase
To ioswitch2.To To ioswitch2.To
UserSpace clitypes.UserSpaceDetail
Path clitypes.JPath
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
Option types.WriteOption Option types.WriteOption
} }


func (b *GraphNodeBuilder) NewBaseWrite(to ioswitch2.To, userSpace clitypes.UserSpaceDetail, path clitypes.JPath, opt types.WriteOption) *BaseWriteNode {
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitch2.To, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.WriteOption) *BaseWriteNode {
node := &BaseWriteNode{ node := &BaseWriteNode{
To: to, To: to,
UserSpace: userSpace, UserSpace: userSpace,


+ 6
- 6
common/pkgs/ioswitch2/ops2/bypass.go View File

@@ -3,11 +3,11 @@ package ops2
import ( import (
"fmt" "fmt"


clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -17,8 +17,8 @@ func init() {


// 旁路Http读取 // 旁路Http读取
type GetShardHTTPRequest struct { type GetShardHTTPRequest struct {
UserSpace clitypes.UserSpaceDetail
FileHash clitypes.FileHash
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
Output exec.VarID Output exec.VarID
} }


@@ -64,11 +64,11 @@ func (o *GetShardHTTPRequest) String() string {
// 旁路Http读取 // 旁路Http读取
type GetShardHTTPRequestNode struct { type GetShardHTTPRequestNode struct {
dag.NodeBase dag.NodeBase
UserSpace clitypes.UserSpaceDetail
FileHash clitypes.FileHash
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
} }


func (b *GraphNodeBuilder) NewGetShardHTTPRequest(userSpace clitypes.UserSpaceDetail, fileHash clitypes.FileHash) *GetShardHTTPRequestNode {
func (b *GraphNodeBuilder) NewGetShardHTTPRequest(userSpace jcsypes.UserSpaceDetail, fileHash jcsypes.FileHash) *GetShardHTTPRequestNode {
node := &GetShardHTTPRequestNode{ node := &GetShardHTTPRequestNode{
UserSpace: userSpace, UserSpace: userSpace,
FileHash: fileHash, FileHash: fileHash,


+ 7
- 7
common/pkgs/ioswitch2/ops2/ec.go View File

@@ -7,13 +7,13 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/future" "gitlink.org.cn/cloudream/common/pkgs/future"
"gitlink.org.cn/cloudream/common/utils/io2" "gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/common/utils/sync2" "gitlink.org.cn/cloudream/common/utils/sync2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/utils" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/utils"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -146,7 +146,7 @@ func (o *ECMultiply) String() string {
} }


type CallECMultiplier struct { type CallECMultiplier struct {
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
Coef [][]byte Coef [][]byte
Inputs []exec.VarID Inputs []exec.VarID
Outputs []exec.VarID Outputs []exec.VarID
@@ -203,12 +203,12 @@ func (o *CallECMultiplier) String() string {


type ECMultiplyNode struct { type ECMultiplyNode struct {
dag.NodeBase dag.NodeBase
EC clitypes.ECRedundancy
EC jcsypes.ECRedundancy
InputIndexes []int InputIndexes []int
OutputIndexes []int OutputIndexes []int
} }


func (b *GraphNodeBuilder) NewECMultiply(ec clitypes.ECRedundancy) *ECMultiplyNode {
func (b *GraphNodeBuilder) NewECMultiply(ec jcsypes.ECRedundancy) *ECMultiplyNode {
node := &ECMultiplyNode{ node := &ECMultiplyNode{
EC: ec, EC: ec,
} }
@@ -257,13 +257,13 @@ func (t *ECMultiplyNode) GenerateOp() (exec.Op, error) {


type CallECMultiplierNode struct { type CallECMultiplierNode struct {
dag.NodeBase dag.NodeBase
UserSpace clitypes.UserSpaceDetail
EC clitypes.ECRedundancy
UserSpace jcsypes.UserSpaceDetail
EC jcsypes.ECRedundancy
InputIndexes []int InputIndexes []int
OutputIndexes []int OutputIndexes []int
} }


func (b *GraphNodeBuilder) NewCallECMultiplier(userSpace clitypes.UserSpaceDetail) *CallECMultiplierNode {
func (b *GraphNodeBuilder) NewCallECMultiplier(userSpace jcsypes.UserSpaceDetail) *CallECMultiplierNode {
node := &CallECMultiplierNode{ node := &CallECMultiplierNode{
UserSpace: userSpace, UserSpace: userSpace,
} }


+ 7
- 7
common/pkgs/ioswitch2/ops2/multipart.go View File

@@ -5,11 +5,11 @@ import (
"time" "time"


log "gitlink.org.cn/cloudream/common/pkgs/logger" log "gitlink.org.cn/cloudream/common/pkgs/logger"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -40,7 +40,7 @@ func (v *UploadedPartInfoValue) Clone() exec.VarValue {
} }


type MultipartInitiator struct { type MultipartInitiator struct {
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
UploadArgs exec.VarID UploadArgs exec.VarID
UploadedParts []exec.VarID UploadedParts []exec.VarID
FileOutput exec.VarID // 分片上传之后的临时文件的路径 FileOutput exec.VarID // 分片上传之后的临时文件的路径
@@ -99,7 +99,7 @@ func (o *MultipartInitiator) String() string {
} }


type MultipartUpload struct { type MultipartUpload struct {
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
UploadArgs exec.VarID UploadArgs exec.VarID
UploadResult exec.VarID UploadResult exec.VarID
PartStream exec.VarID PartStream exec.VarID
@@ -149,10 +149,10 @@ func (o *MultipartUpload) String() string {


type MultipartInitiatorNode struct { type MultipartInitiatorNode struct {
dag.NodeBase dag.NodeBase
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
} }


func (b *GraphNodeBuilder) NewMultipartInitiator(userSpace clitypes.UserSpaceDetail) *MultipartInitiatorNode {
func (b *GraphNodeBuilder) NewMultipartInitiator(userSpace jcsypes.UserSpaceDetail) *MultipartInitiatorNode {
node := &MultipartInitiatorNode{ node := &MultipartInitiatorNode{
UserSpace: userSpace, UserSpace: userSpace,
} }
@@ -194,12 +194,12 @@ func (n *MultipartInitiatorNode) GenerateOp() (exec.Op, error) {


type MultipartUploadNode struct { type MultipartUploadNode struct {
dag.NodeBase dag.NodeBase
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
PartNumber int PartNumber int
PartSize int64 PartSize int64
} }


func (b *GraphNodeBuilder) NewMultipartUpload(userSpace clitypes.UserSpaceDetail, partNumber int, partSize int64) *MultipartUploadNode {
func (b *GraphNodeBuilder) NewMultipartUpload(userSpace jcsypes.UserSpaceDetail, partNumber int, partSize int64) *MultipartUploadNode {
node := &MultipartUploadNode{ node := &MultipartUploadNode{
UserSpace: userSpace, UserSpace: userSpace,
PartNumber: partNumber, PartNumber: partNumber,


+ 17
- 17
common/pkgs/ioswitch2/ops2/s2s.go View File

@@ -3,10 +3,10 @@ package ops2
import ( import (
"fmt" "fmt"


clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -15,10 +15,10 @@ func init() {
} }


type S2STransfer struct { type S2STransfer struct {
SrcSpace clitypes.UserSpaceDetail
SrcPath clitypes.JPath
DstSpace clitypes.UserSpaceDetail
DstPath clitypes.JPath
SrcSpace jcsypes.UserSpaceDetail
SrcPath jcsypes.JPath
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
Output exec.VarID Output exec.VarID
} }


@@ -55,10 +55,10 @@ func (o *S2STransfer) String() string {
} }


type S2STransferDyn struct { type S2STransferDyn struct {
SrcSpace clitypes.UserSpaceDetail
SrcSpace jcsypes.UserSpaceDetail
SrcFileInfo exec.VarID SrcFileInfo exec.VarID
DstSpace clitypes.UserSpaceDetail
DstPath clitypes.JPath
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
Output exec.VarID Output exec.VarID
} }


@@ -101,13 +101,13 @@ func (o *S2STransferDyn) String() string {


type S2STransferNode struct { type S2STransferNode struct {
dag.NodeBase dag.NodeBase
SrcSpace clitypes.UserSpaceDetail
SrcPath clitypes.JPath
DstSpace clitypes.UserSpaceDetail
DstPath clitypes.JPath
SrcSpace jcsypes.UserSpaceDetail
SrcPath jcsypes.JPath
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
} }


func (b *GraphNodeBuilder) NewS2STransfer(srcSpace clitypes.UserSpaceDetail, srcPath clitypes.JPath, dstSpace clitypes.UserSpaceDetail, dstPath clitypes.JPath) *S2STransferNode {
func (b *GraphNodeBuilder) NewS2STransfer(srcSpace jcsypes.UserSpaceDetail, srcPath jcsypes.JPath, dstSpace jcsypes.UserSpaceDetail, dstPath jcsypes.JPath) *S2STransferNode {
n := &S2STransferNode{ n := &S2STransferNode{
SrcSpace: srcSpace, SrcSpace: srcSpace,
SrcPath: srcPath, SrcPath: srcPath,
@@ -139,12 +139,12 @@ func (n *S2STransferNode) GenerateOp() (exec.Op, error) {


type S2STransferDynNode struct { type S2STransferDynNode struct {
dag.NodeBase dag.NodeBase
SrcSpace clitypes.UserSpaceDetail
DstSpace clitypes.UserSpaceDetail
DstPath clitypes.JPath
SrcSpace jcsypes.UserSpaceDetail
DstSpace jcsypes.UserSpaceDetail
DstPath jcsypes.JPath
} }


func (b *GraphNodeBuilder) NewS2STransferDyn(srcSpace clitypes.UserSpaceDetail, dstSpace clitypes.UserSpaceDetail, dstPath clitypes.JPath) *S2STransferDynNode {
func (b *GraphNodeBuilder) NewS2STransferDyn(srcSpace jcsypes.UserSpaceDetail, dstSpace jcsypes.UserSpaceDetail, dstPath jcsypes.JPath) *S2STransferDynNode {
n := &S2STransferDynNode{ n := &S2STransferDynNode{
SrcSpace: srcSpace, SrcSpace: srcSpace,
DstSpace: dstSpace, DstSpace: dstSpace,


+ 9
- 9
common/pkgs/ioswitch2/ops2/shard_store.go View File

@@ -3,10 +3,10 @@ package ops2
import ( import (
"fmt" "fmt"


clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -15,8 +15,8 @@ func init() {
} }


type GetShardInfo struct { type GetShardInfo struct {
UserSpace clitypes.UserSpaceDetail
FileHash clitypes.FileHash
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
ShardInfo exec.VarID ShardInfo exec.VarID
} }


@@ -46,7 +46,7 @@ func (o *GetShardInfo) String() string {
} }


type StoreShard struct { type StoreShard struct {
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
FileInfo exec.VarID FileInfo exec.VarID
ShardInfo exec.VarID ShardInfo exec.VarID
} }
@@ -84,11 +84,11 @@ func (o *StoreShard) String() string {


type GetShardInfoNode struct { type GetShardInfoNode struct {
dag.NodeBase dag.NodeBase
UserSpace clitypes.UserSpaceDetail
FileHash clitypes.FileHash
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
} }


func (b *GraphNodeBuilder) NewGetShardInfo(userSpace clitypes.UserSpaceDetail, fileHash clitypes.FileHash) *GetShardInfoNode {
func (b *GraphNodeBuilder) NewGetShardInfo(userSpace jcsypes.UserSpaceDetail, fileHash jcsypes.FileHash) *GetShardInfoNode {
node := &GetShardInfoNode{ node := &GetShardInfoNode{
UserSpace: userSpace, UserSpace: userSpace,
FileHash: fileHash, FileHash: fileHash,
@@ -116,11 +116,11 @@ func (n *GetShardInfoNode) GenerateOp() (exec.Op, error) {


type StoreShardNode struct { type StoreShardNode struct {
dag.NodeBase dag.NodeBase
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
ShardInfoKey string ShardInfoKey string
} }


func (b *GraphNodeBuilder) NewStoreShard(userSpace clitypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
func (b *GraphNodeBuilder) NewStoreShard(userSpace jcsypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
node := &StoreShardNode{ node := &StoreShardNode{
UserSpace: userSpace, UserSpace: userSpace,
ShardInfoKey: shardInfoKey, ShardInfoKey: shardInfoKey,


+ 2
- 2
common/pkgs/ioswitch2/parser/gen/generator.go View File

@@ -7,12 +7,12 @@ import (
"gitlink.org.cn/cloudream/common/utils/lo2" "gitlink.org.cn/cloudream/common/utils/lo2"
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/common/utils/os2" "gitlink.org.cn/cloudream/common/utils/os2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser/state" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/parser/state"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )


@@ -385,7 +385,7 @@ func buildToNode(ctx *state.GenerateState, t ioswitch2.To) (ops2.ToNode, error)
} }
} }


func setEnvBySpace(n dag.Node, space *clitypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
if space.RecommendHub == nil { if space.RecommendHub == nil {
n.Env().ToEnvDriver(true) n.Env().ToEnvDriver(true)
return nil return nil


+ 2
- 2
common/pkgs/ioswitch2/parser/opt/utils.go View File

@@ -3,13 +3,13 @@ package opt
import ( import (
"fmt" "fmt"


clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )


func setEnvBySpace(n dag.Node, space *clitypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
if space.RecommendHub == nil { if space.RecommendHub == nil {
n.Env().ToEnvDriver(true) n.Env().ToEnvDriver(true)
return nil return nil


+ 2
- 2
common/pkgs/ioswitch2/plans/complete_multipart.go View File

@@ -4,14 +4,14 @@ import (
"fmt" "fmt"


"gitlink.org.cn/cloudream/common/utils/os2" "gitlink.org.cn/cloudream/common/utils/os2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2/ops2"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func CompleteMultipart(blocks []clitypes.ObjectBlock, blockSpaces []clitypes.UserSpaceDetail, targetSpace clitypes.UserSpaceDetail, shardInfoKey string, blder *exec.PlanBuilder) error {
func CompleteMultipart(blocks []jcsypes.ObjectBlock, blockSpaces []jcsypes.UserSpaceDetail, targetSpace jcsypes.UserSpaceDetail, shardInfoKey string, blder *exec.PlanBuilder) error {
da := ops2.NewGraphNodeBuilder() da := ops2.NewGraphNodeBuilder()


sizes := make([]int64, len(blocks)) sizes := make([]int64, len(blocks))


+ 2
- 2
common/pkgs/ioswitch2/plans/utils.go View File

@@ -3,13 +3,13 @@ package plans
import ( import (
"fmt" "fmt"


clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )


func setEnvBySpace(n dag.Node, space *clitypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
if space.RecommendHub == nil { if space.RecommendHub == nil {
n.Env().ToEnvDriver(true) n.Env().ToEnvDriver(true)
return nil return nil


+ 7
- 7
common/pkgs/ioswitchlrc/fromto.go View File

@@ -2,9 +2,9 @@ package ioswitchlrc


import ( import (
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type From interface { type From interface {
@@ -39,12 +39,12 @@ func (f *FromDriver) GetDataIndex() int {
} }


type FromNode struct { type FromNode struct {
FileHash clitypes.FileHash
UserSpace clitypes.UserSpaceDetail
FileHash jcsypes.FileHash
UserSpace jcsypes.UserSpaceDetail
DataIndex int DataIndex int
} }


func NewFromStorage(fileHash clitypes.FileHash, space clitypes.UserSpaceDetail, dataIndex int) *FromNode {
func NewFromStorage(fileHash jcsypes.FileHash, space jcsypes.UserSpaceDetail, dataIndex int) *FromNode {
return &FromNode{ return &FromNode{
FileHash: fileHash, FileHash: fileHash,
DataIndex: dataIndex, DataIndex: dataIndex,
@@ -88,14 +88,14 @@ func (t *ToDriver) GetRange() math2.Range {
} }


type ToNode struct { type ToNode struct {
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
DataIndex int DataIndex int
Range math2.Range Range math2.Range
FileHashStoreKey string FileHashStoreKey string
Option types.WriteOption Option types.WriteOption
} }


func NewToStorage(space clitypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string) *ToNode {
func NewToStorage(space jcsypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string) *ToNode {
return &ToNode{ return &ToNode{
UserSpace: space, UserSpace: space,
DataIndex: dataIndex, DataIndex: dataIndex,
@@ -103,7 +103,7 @@ func NewToStorage(space clitypes.UserSpaceDetail, dataIndex int, fileHashStoreKe
} }
} }


func NewToStorageWithRange(space clitypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string, rng math2.Range) *ToNode {
func NewToStorageWithRange(space jcsypes.UserSpaceDetail, dataIndex int, fileHashStoreKey string, rng math2.Range) *ToNode {
return &ToNode{ return &ToNode{
UserSpace: space, UserSpace: space,
DataIndex: dataIndex, DataIndex: dataIndex,


+ 14
- 14
common/pkgs/ioswitchlrc/ops2/base_store.go View File

@@ -7,12 +7,12 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/future" "gitlink.org.cn/cloudream/common/pkgs/future"
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/io2" "gitlink.org.cn/cloudream/common/utils/io2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -23,8 +23,8 @@ func init() {


type BaseRead struct { type BaseRead struct {
Output exec.VarID Output exec.VarID
UserSpace clitypes.UserSpaceDetail
Path clitypes.JPath
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
Option types.OpenOption Option types.OpenOption
} }


@@ -67,7 +67,7 @@ func (o *BaseRead) String() string {
} }


type BaseReadDyn struct { type BaseReadDyn struct {
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
Output exec.VarID Output exec.VarID
Path exec.VarID Path exec.VarID
Option types.OpenOption Option types.OpenOption
@@ -118,8 +118,8 @@ func (o *BaseReadDyn) String() string {


type BaseWrite struct { type BaseWrite struct {
Input exec.VarID Input exec.VarID
UserSpace clitypes.UserSpaceDetail
Path clitypes.JPath
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
WriteResult exec.VarID WriteResult exec.VarID
Option types.WriteOption Option types.WriteOption
} }
@@ -164,12 +164,12 @@ func (o *BaseWrite) String() string {
type BaseReadNode struct { type BaseReadNode struct {
dag.NodeBase dag.NodeBase
From ioswitchlrc.From From ioswitchlrc.From
UserSpace clitypes.UserSpaceDetail
Path clitypes.JPath
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
Option types.OpenOption Option types.OpenOption
} }


func (b *GraphNodeBuilder) NewBaseRead(from ioswitchlrc.From, userSpace clitypes.UserSpaceDetail, path clitypes.JPath, opt types.OpenOption) *BaseReadNode {
func (b *GraphNodeBuilder) NewBaseRead(from ioswitchlrc.From, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.OpenOption) *BaseReadNode {
node := &BaseReadNode{ node := &BaseReadNode{
From: from, From: from,
UserSpace: userSpace, UserSpace: userSpace,
@@ -205,11 +205,11 @@ func (t *BaseReadNode) GenerateOp() (exec.Op, error) {
type BaseReadDynNode struct { type BaseReadDynNode struct {
dag.NodeBase dag.NodeBase
From ioswitchlrc.From From ioswitchlrc.From
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
Option types.OpenOption Option types.OpenOption
} }


func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitchlrc.From, userSpace clitypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
func (b *GraphNodeBuilder) NewBaseReadDyn(from ioswitchlrc.From, userSpace jcsypes.UserSpaceDetail, opt types.OpenOption) *BaseReadDynNode {
node := &BaseReadDynNode{ node := &BaseReadDynNode{
From: from, From: from,
UserSpace: userSpace, UserSpace: userSpace,
@@ -252,12 +252,12 @@ func (t *BaseReadDynNode) GenerateOp() (exec.Op, error) {
type BaseWriteNode struct { type BaseWriteNode struct {
dag.NodeBase dag.NodeBase
To ioswitchlrc.To To ioswitchlrc.To
UserSpace clitypes.UserSpaceDetail
Path clitypes.JPath
UserSpace jcsypes.UserSpaceDetail
Path jcsypes.JPath
Option types.WriteOption Option types.WriteOption
} }


func (b *GraphNodeBuilder) NewBaseWrite(to ioswitchlrc.To, userSpace clitypes.UserSpaceDetail, path clitypes.JPath, opt types.WriteOption) *BaseWriteNode {
func (b *GraphNodeBuilder) NewBaseWrite(to ioswitchlrc.To, userSpace jcsypes.UserSpaceDetail, path jcsypes.JPath, opt types.WriteOption) *BaseWriteNode {
node := &BaseWriteNode{ node := &BaseWriteNode{
To: to, To: to,
UserSpace: userSpace, UserSpace: userSpace,


+ 5
- 5
common/pkgs/ioswitchlrc/ops2/ec.go View File

@@ -7,12 +7,12 @@ import (
"gitlink.org.cn/cloudream/common/pkgs/future" "gitlink.org.cn/cloudream/common/pkgs/future"
"gitlink.org.cn/cloudream/common/utils/io2" "gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/common/utils/sync2" "gitlink.org.cn/cloudream/common/utils/sync2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ec/lrc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ec/lrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/utils" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/utils"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -114,12 +114,12 @@ func (o *GalMultiply) String() string {


type LRCConstructAnyNode struct { type LRCConstructAnyNode struct {
dag.NodeBase dag.NodeBase
LRC clitypes.LRCRedundancy
LRC jcsypes.LRCRedundancy
InputIndexes []int InputIndexes []int
OutputIndexes []int OutputIndexes []int
} }


func (b *GraphNodeBuilder) NewLRCConstructAny(lrc clitypes.LRCRedundancy) *LRCConstructAnyNode {
func (b *GraphNodeBuilder) NewLRCConstructAny(lrc jcsypes.LRCRedundancy) *LRCConstructAnyNode {
node := &LRCConstructAnyNode{ node := &LRCConstructAnyNode{
LRC: lrc, LRC: lrc,
} }
@@ -168,11 +168,11 @@ func (t *LRCConstructAnyNode) GenerateOp() (exec.Op, error) {


type LRCConstructGroupNode struct { type LRCConstructGroupNode struct {
dag.NodeBase dag.NodeBase
LRC clitypes.LRCRedundancy
LRC jcsypes.LRCRedundancy
TargetBlockIndex int TargetBlockIndex int
} }


func (b *GraphNodeBuilder) NewLRCConstructGroup(lrc clitypes.LRCRedundancy) *LRCConstructGroupNode {
func (b *GraphNodeBuilder) NewLRCConstructGroup(lrc jcsypes.LRCRedundancy) *LRCConstructGroupNode {
node := &LRCConstructGroupNode{ node := &LRCConstructGroupNode{
LRC: lrc, LRC: lrc,
} }


+ 9
- 9
common/pkgs/ioswitchlrc/ops2/shard_store.go View File

@@ -3,10 +3,10 @@ package ops2
import ( import (
"fmt" "fmt"


clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/pool"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


func init() { func init() {
@@ -15,8 +15,8 @@ func init() {
} }


type GetShardInfo struct { type GetShardInfo struct {
UserSpace clitypes.UserSpaceDetail
FileHash clitypes.FileHash
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
ShardInfo exec.VarID ShardInfo exec.VarID
} }


@@ -46,7 +46,7 @@ func (o *GetShardInfo) String() string {
} }


type StoreShard struct { type StoreShard struct {
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
FileInfo exec.VarID FileInfo exec.VarID
ShardInfo exec.VarID ShardInfo exec.VarID
} }
@@ -84,11 +84,11 @@ func (o *StoreShard) String() string {


type GetShardInfoNode struct { type GetShardInfoNode struct {
dag.NodeBase dag.NodeBase
UserSpace clitypes.UserSpaceDetail
FileHash clitypes.FileHash
UserSpace jcsypes.UserSpaceDetail
FileHash jcsypes.FileHash
} }


func (b *GraphNodeBuilder) NewGetShardInfo(userSpace clitypes.UserSpaceDetail, fileHash clitypes.FileHash) *GetShardInfoNode {
func (b *GraphNodeBuilder) NewGetShardInfo(userSpace jcsypes.UserSpaceDetail, fileHash jcsypes.FileHash) *GetShardInfoNode {
node := &GetShardInfoNode{ node := &GetShardInfoNode{
UserSpace: userSpace, UserSpace: userSpace,
FileHash: fileHash, FileHash: fileHash,
@@ -116,11 +116,11 @@ func (n *GetShardInfoNode) GenerateOp() (exec.Op, error) {


type StoreShardNode struct { type StoreShardNode struct {
dag.NodeBase dag.NodeBase
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
ShardInfoKey string ShardInfoKey string
} }


func (b *GraphNodeBuilder) NewStoreShard(userSpace clitypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
func (b *GraphNodeBuilder) NewStoreShard(userSpace jcsypes.UserSpaceDetail, shardInfoKey string) *StoreShardNode {
node := &StoreShardNode{ node := &StoreShardNode{
UserSpace: userSpace, UserSpace: userSpace,
ShardInfoKey: shardInfoKey, ShardInfoKey: shardInfoKey,


+ 5
- 5
common/pkgs/ioswitchlrc/parser/generator.go View File

@@ -4,16 +4,16 @@ import (
"fmt" "fmt"


"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/exec"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/plan"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/ops2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitchlrc/ops2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type GenerateContext struct { type GenerateContext struct {
LRC clitypes.LRCRedundancy
LRC jcsypes.LRCRedundancy
DAG *ops2.GraphNodeBuilder DAG *ops2.GraphNodeBuilder
To []ioswitchlrc.To To []ioswitchlrc.To
ToNodes map[ioswitchlrc.To]ops2.ToNode ToNodes map[ioswitchlrc.To]ops2.ToNode
@@ -27,7 +27,7 @@ func Encode(fr ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder)
} }


ctx := GenerateContext{ ctx := GenerateContext{
LRC: clitypes.DefaultLRCRedundancy,
LRC: jcsypes.DefaultLRCRedundancy,
DAG: ops2.NewGraphNodeBuilder(), DAG: ops2.NewGraphNodeBuilder(),
To: toes, To: toes,
ToNodes: make(map[ioswitchlrc.To]ops2.ToNode), ToNodes: make(map[ioswitchlrc.To]ops2.ToNode),
@@ -124,7 +124,7 @@ func buildDAGEncode(ctx *GenerateContext, fr ioswitchlrc.From, toes []ioswitchlr
// 提供数据块+编码块中的k个块,重建任意块,包括完整文件。 // 提供数据块+编码块中的k个块,重建任意块,包括完整文件。
func ReconstructAny(frs []ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder) error { func ReconstructAny(frs []ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder) error {
ctx := GenerateContext{ ctx := GenerateContext{
LRC: clitypes.DefaultLRCRedundancy,
LRC: jcsypes.DefaultLRCRedundancy,
DAG: ops2.NewGraphNodeBuilder(), DAG: ops2.NewGraphNodeBuilder(),
To: toes, To: toes,
ToNodes: make(map[ioswitchlrc.To]ops2.ToNode), ToNodes: make(map[ioswitchlrc.To]ops2.ToNode),
@@ -245,7 +245,7 @@ func buildDAGReconstructAny(ctx *GenerateContext, frs []ioswitchlrc.From, toes [
// 输入同一组的多个块,恢复出剩下缺少的一个块。 // 输入同一组的多个块,恢复出剩下缺少的一个块。
func ReconstructGroup(frs []ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder) error { func ReconstructGroup(frs []ioswitchlrc.From, toes []ioswitchlrc.To, blder *exec.PlanBuilder) error {
ctx := GenerateContext{ ctx := GenerateContext{
LRC: clitypes.DefaultLRCRedundancy,
LRC: jcsypes.DefaultLRCRedundancy,
DAG: ops2.NewGraphNodeBuilder(), DAG: ops2.NewGraphNodeBuilder(),
To: toes, To: toes,
ToNodes: make(map[ioswitchlrc.To]ops2.ToNode), ToNodes: make(map[ioswitchlrc.To]ops2.ToNode),


+ 2
- 2
common/pkgs/ioswitchlrc/parser/utils.go View File

@@ -3,13 +3,13 @@ package parser
import ( import (
"fmt" "fmt"


clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch/dag"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/ioswitch2"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )


func setEnvBySpace(n dag.Node, space *clitypes.UserSpaceDetail) error {
func setEnvBySpace(n dag.Node, space *jcsypes.UserSpaceDetail) error {
if space.RecommendHub == nil { if space.RecommendHub == nil {
n.Env().ToEnvDriver(true) n.Env().ToEnvDriver(true)
return nil return nil


+ 4
- 4
common/pkgs/publock/reqbuilder/user_space.go View File

@@ -3,9 +3,9 @@ package reqbuilder
import ( import (
"strconv" "strconv"


clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/lockprovider" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/lockprovider"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/publock/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type UserSpaceLockReqBuilder struct { type UserSpaceLockReqBuilder struct {
@@ -15,7 +15,7 @@ type UserSpaceLockReqBuilder struct {
func (b *LockRequestBuilder) UserSpace() *UserSpaceLockReqBuilder { func (b *LockRequestBuilder) UserSpace() *UserSpaceLockReqBuilder {
return &UserSpaceLockReqBuilder{LockRequestBuilder: b} return &UserSpaceLockReqBuilder{LockRequestBuilder: b}
} }
func (b *UserSpaceLockReqBuilder) Buzy(spaceID clitypes.UserSpaceID) *UserSpaceLockReqBuilder {
func (b *UserSpaceLockReqBuilder) Buzy(spaceID jcsypes.UserSpaceID) *UserSpaceLockReqBuilder {
b.locks = append(b.locks, types.Lock{ b.locks = append(b.locks, types.Lock{
Path: b.makePath(spaceID), Path: b.makePath(spaceID),
Name: lockprovider.UserSpaceBuzyLock, Name: lockprovider.UserSpaceBuzyLock,
@@ -24,7 +24,7 @@ func (b *UserSpaceLockReqBuilder) Buzy(spaceID clitypes.UserSpaceID) *UserSpaceL
return b return b
} }


func (b *UserSpaceLockReqBuilder) GC(spaceID clitypes.UserSpaceID) *UserSpaceLockReqBuilder {
func (b *UserSpaceLockReqBuilder) GC(spaceID jcsypes.UserSpaceID) *UserSpaceLockReqBuilder {
b.locks = append(b.locks, types.Lock{ b.locks = append(b.locks, types.Lock{
Path: b.makePath(spaceID), Path: b.makePath(spaceID),
Name: lockprovider.UserSpaceGCLock, Name: lockprovider.UserSpaceGCLock,
@@ -33,6 +33,6 @@ func (b *UserSpaceLockReqBuilder) GC(spaceID clitypes.UserSpaceID) *UserSpaceLoc
return b return b
} }


func (b *UserSpaceLockReqBuilder) makePath(hubID clitypes.UserSpaceID) []string {
func (b *UserSpaceLockReqBuilder) makePath(hubID jcsypes.UserSpaceID) []string {
return []string{lockprovider.UserSpaceLockPathPrefix, strconv.FormatInt(int64(hubID), 10)} return []string{lockprovider.UserSpaceLockPathPrefix, strconv.FormatInt(int64(hubID), 10)}
} }

+ 5
- 5
common/pkgs/rpc/hub/cache.go View File

@@ -4,7 +4,7 @@ package hubrpc
import ( import (
"context" "context"


clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
) )


@@ -15,10 +15,10 @@ type CacheSvc interface {


// 获取Cache中文件列表 // 获取Cache中文件列表
type CheckCache struct { type CheckCache struct {
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
} }
type CheckCacheResp struct { type CheckCacheResp struct {
FileHashes []clitypes.FileHash
FileHashes []jcsypes.FileHash
} }


func (c *Client) CheckCache(ctx context.Context, req *CheckCache) (*CheckCacheResp, *rpc.CodeError) { func (c *Client) CheckCache(ctx context.Context, req *CheckCache) (*CheckCacheResp, *rpc.CodeError) {
@@ -33,8 +33,8 @@ func (s *Server) CheckCache(ctx context.Context, req *rpc.Request) (*rpc.Respons


// 清理Cache中不用的文件 // 清理Cache中不用的文件
type CacheGC struct { type CacheGC struct {
UserSpace clitypes.UserSpaceDetail
Availables []clitypes.FileHash
UserSpace jcsypes.UserSpaceDetail
Availables []jcsypes.FileHash
} }
type CacheGCResp struct{} type CacheGCResp struct{}




+ 3
- 3
common/pkgs/rpc/hub/user_space.go View File

@@ -4,7 +4,7 @@ package hubrpc
import ( import (
"context" "context"


clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/rpc"
stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" stgtypes "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
) )
@@ -16,7 +16,7 @@ type UserSpaceSvc interface {


// 列出指定BaseStore的指定位置内的所有文件 // 列出指定BaseStore的指定位置内的所有文件
type BaseStoreListAll struct { type BaseStoreListAll struct {
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
Path string Path string
} }
type BaseStoreListAllResp struct { type BaseStoreListAllResp struct {
@@ -35,7 +35,7 @@ func (s *Server) BaseStoreListAll(ctx context.Context, req *rpc.Request) (*rpc.R


// 批量在指定BaseStore中创建文件夹 // 批量在指定BaseStore中创建文件夹
type BaseStoreMkdirs struct { type BaseStoreMkdirs struct {
UserSpace clitypes.UserSpaceDetail
UserSpace jcsypes.UserSpaceDetail
Pathes []string Pathes []string
} }




+ 3
- 3
common/pkgs/storage/efile/ec_multiplier.go View File

@@ -9,8 +9,8 @@ import (
"gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/http2"
"gitlink.org.cn/cloudream/common/utils/os2" "gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/common/utils/serder" "gitlink.org.cn/cloudream/common/utils/serder"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )


@@ -98,9 +98,9 @@ func (m *ECMultiplier) Multiply(coef [][]byte, inputs []types.HTTPRequest, chunk
for i, data := range r.Data { for i, data := range r.Data {
ret[i] = types.FileInfo{ ret[i] = types.FileInfo{
// TODO 要确认一下output的格式 // TODO 要确认一下output的格式
Path: clitypes.PathFromJcsPathString(m.outputs[i]),
Path: jcsypes.PathFromJcsPathString(m.outputs[i]),
Size: data.Size, Size: data.Size,
Hash: clitypes.NewFullHashFromString(data.Sha256),
Hash: jcsypes.NewFullHashFromString(data.Sha256),
} }
} }




+ 3
- 3
common/pkgs/storage/efile/efile.go View File

@@ -8,14 +8,14 @@ import (


"gitlink.org.cn/cloudream/common/utils/http2" "gitlink.org.cn/cloudream/common/utils/http2"
"gitlink.org.cn/cloudream/common/utils/serder" "gitlink.org.cn/cloudream/common/utils/serder"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )


func init() { func init() {
reg.RegisterBuilder[*cortypes.EFileType](func(detail *clitypes.UserSpaceDetail) types.StorageBuilder {
reg.RegisterBuilder[*cortypes.EFileType](func(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
return &builder{ return &builder{
detail: detail, detail: detail,
} }
@@ -24,7 +24,7 @@ func init() {


type builder struct { type builder struct {
types.EmptyBuilder types.EmptyBuilder
detail *clitypes.UserSpaceDetail
detail *jcsypes.UserSpaceDetail
token string token string
tokenLock sync.Mutex tokenLock sync.Mutex
getTokenTime time.Time getTokenTime time.Time


+ 2
- 2
common/pkgs/storage/factory/factory.go View File

@@ -3,15 +3,15 @@ package factory
import ( import (
"reflect" "reflect"


clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
_ "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage" _ "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


// 此函数永远不会返回nil。如果找不到对应的Builder,则会返回EmptyBuilder, // 此函数永远不会返回nil。如果找不到对应的Builder,则会返回EmptyBuilder,
// 此Builder的所有函数都会返回否定值或者封装后的ErrUnsupported错误(需要使用errors.Is检查) // 此Builder的所有函数都会返回否定值或者封装后的ErrUnsupported错误(需要使用errors.Is检查)
func GetBuilder(detail *clitypes.UserSpaceDetail) types.StorageBuilder {
func GetBuilder(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
typ := reflect.TypeOf(detail.UserSpace.Storage) typ := reflect.TypeOf(detail.UserSpace.Storage)


ctor, ok := reg.StorageBuilders[typ] ctor, ok := reg.StorageBuilders[typ]


+ 3
- 3
common/pkgs/storage/factory/reg/reg.go View File

@@ -4,12 +4,12 @@ import (
"reflect" "reflect"


"gitlink.org.cn/cloudream/common/utils/reflect2" "gitlink.org.cn/cloudream/common/utils/reflect2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )


type BuilderCtor func(detail *clitypes.UserSpaceDetail) types.StorageBuilder
type BuilderCtor func(detail *jcsypes.UserSpaceDetail) types.StorageBuilder


var StorageBuilders = make(map[reflect.Type]BuilderCtor) var StorageBuilders = make(map[reflect.Type]BuilderCtor)


@@ -21,7 +21,7 @@ func RegisterBuilder[T cortypes.StorageType](ctor BuilderCtor) {
// 注:此函数只给storage包内部使用,外部包请使用外层的factory.GetBuilder // 注:此函数只给storage包内部使用,外部包请使用外层的factory.GetBuilder
// 此函数永远不会返回nil。如果找不到对应的Builder,则会返回EmptyBuilder, // 此函数永远不会返回nil。如果找不到对应的Builder,则会返回EmptyBuilder,
// 此Builder的所有函数都会返回否定值或者封装后的ErrUnsupported错误(需要使用errors.Is检查) // 此Builder的所有函数都会返回否定值或者封装后的ErrUnsupported错误(需要使用errors.Is检查)
func GetBuilderInternal(detail *clitypes.UserSpaceDetail) types.StorageBuilder {
func GetBuilderInternal(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
typ := reflect.TypeOf(detail.UserSpace.Storage) typ := reflect.TypeOf(detail.UserSpace.Storage)


ctor, ok := StorageBuilders[typ] ctor, ok := StorageBuilders[typ]


+ 8
- 8
common/pkgs/storage/local/base_store.go View File

@@ -8,23 +8,23 @@ import (


"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/utils/io2" "gitlink.org.cn/cloudream/common/utils/io2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type BaseStore struct { type BaseStore struct {
root string root string
detail *clitypes.UserSpaceDetail
detail *jcsypes.UserSpaceDetail
} }


func NewBaseStore(root string, detail *clitypes.UserSpaceDetail) (*BaseStore, error) {
func NewBaseStore(root string, detail *jcsypes.UserSpaceDetail) (*BaseStore, error) {
return &BaseStore{ return &BaseStore{
root: root, root: root,
detail: detail, detail: detail,
}, nil }, nil
} }


func (s *BaseStore) Write(pat clitypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
func (s *BaseStore) Write(pat jcsypes.JPath, stream io.Reader, opt types.WriteOption) (types.FileInfo, error) {
log := s.getLogger() log := s.getLogger()


absObjPath := filepath.Join(s.root, pat.String()) absObjPath := filepath.Join(s.root, pat.String())
@@ -58,11 +58,11 @@ func (s *BaseStore) Write(pat clitypes.JPath, stream io.Reader, opt types.WriteO
return types.FileInfo{ return types.FileInfo{
Path: pat, Path: pat,
Size: counter.Count(), Size: counter.Count(),
Hash: clitypes.NewFullHash(hasher.Sum()),
Hash: jcsypes.NewFullHash(hasher.Sum()),
}, nil }, nil
} }


func (s *BaseStore) Read(objPath clitypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
func (s *BaseStore) Read(objPath jcsypes.JPath, opt types.OpenOption) (io.ReadCloser, error) {
absObjPath := filepath.Join(s.root, objPath.JoinOSPath()) absObjPath := filepath.Join(s.root, objPath.JoinOSPath())
file, err := os.Open(absObjPath) file, err := os.Open(absObjPath)
if err != nil { if err != nil {
@@ -86,7 +86,7 @@ func (s *BaseStore) Read(objPath clitypes.JPath, opt types.OpenOption) (io.ReadC
return ret, nil return ret, nil
} }


func (s *BaseStore) Mkdir(path clitypes.JPath) error {
func (s *BaseStore) Mkdir(path jcsypes.JPath) error {
absObjPath := filepath.Join(s.root, path.JoinOSPath()) absObjPath := filepath.Join(s.root, path.JoinOSPath())
err := os.MkdirAll(absObjPath, 0755) err := os.MkdirAll(absObjPath, 0755)
if err != nil { if err != nil {
@@ -96,7 +96,7 @@ func (s *BaseStore) Mkdir(path clitypes.JPath) error {
return nil return nil
} }


func (s *BaseStore) ReadDir(pat clitypes.JPath) types.DirReader {
func (s *BaseStore) ReadDir(pat jcsypes.JPath) types.DirReader {
return &DirReader{ return &DirReader{
absRootPath: filepath.Join(s.root, pat.JoinOSPath()), absRootPath: filepath.Join(s.root, pat.JoinOSPath()),
rootJPath: pat.Clone(), rootJPath: pat.Clone(),


+ 4
- 4
common/pkgs/storage/local/dir_reader.go View File

@@ -5,15 +5,15 @@ import (
"os" "os"
"path/filepath" "path/filepath"


clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type DirReader struct { type DirReader struct {
// 完整的根路径(包括ReadDir的path参数),比如包括了盘符 // 完整的根路径(包括ReadDir的path参数),比如包括了盘符
absRootPath string absRootPath string
// ReadDir函数传递进来的path参数 // ReadDir函数传递进来的path参数
rootJPath clitypes.JPath
rootJPath jcsypes.JPath
init bool init bool
curEntries []dirEntry curEntries []dirEntry
} }
@@ -42,7 +42,7 @@ func (r *DirReader) Next() (types.DirEntry, error) {


for _, e := range es { for _, e := range es {
r.curEntries = append(r.curEntries, dirEntry{ r.curEntries = append(r.curEntries, dirEntry{
dir: clitypes.JPath{},
dir: jcsypes.JPath{},
entry: e, entry: e,
}) })
} }
@@ -103,7 +103,7 @@ func (r *DirReader) Close() {
} }


type dirEntry struct { type dirEntry struct {
dir clitypes.JPath
dir jcsypes.JPath
entry os.DirEntry entry os.DirEntry
} }




+ 3
- 3
common/pkgs/storage/local/local.go View File

@@ -3,14 +3,14 @@ package local
import ( import (
"fmt" "fmt"


clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/factory/reg"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )


func init() { func init() {
reg.RegisterBuilder[*cortypes.LocalType](func(detail *clitypes.UserSpaceDetail) types.StorageBuilder {
reg.RegisterBuilder[*cortypes.LocalType](func(detail *jcsypes.UserSpaceDetail) types.StorageBuilder {
return &builder{ return &builder{
detail: detail, detail: detail,
} }
@@ -19,7 +19,7 @@ func init() {


type builder struct { type builder struct {
types.EmptyBuilder types.EmptyBuilder
detail *clitypes.UserSpaceDetail
detail *jcsypes.UserSpaceDetail
} }


func (b *builder) FeatureDesc() types.FeatureDesc { func (b *builder) FeatureDesc() types.FeatureDesc {


+ 4
- 4
common/pkgs/storage/local/multipart_upload.go View File

@@ -12,13 +12,13 @@ import (
"gitlink.org.cn/cloudream/common/utils/io2" "gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/common/utils/os2" "gitlink.org.cn/cloudream/common/utils/os2"
"gitlink.org.cn/cloudream/common/utils/sort2" "gitlink.org.cn/cloudream/common/utils/sort2"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )


type Multiparter struct { type Multiparter struct {
detail *clitypes.UserSpaceDetail
detail *jcsypes.UserSpaceDetail
localStg *cortypes.LocalCred localStg *cortypes.LocalCred
feat *cortypes.MultipartUploadFeature feat *cortypes.MultipartUploadFeature
} }
@@ -80,7 +80,7 @@ type MultipartTask struct {
absTempDir string // 应该要是绝对路径 absTempDir string // 应该要是绝对路径
tempFileName string tempFileName string
tempPartsDir string tempPartsDir string
joinedFileJPath clitypes.JPath
joinedFileJPath jcsypes.JPath
absJoinedFilePath string absJoinedFilePath string
uploadID string uploadID string
} }
@@ -118,7 +118,7 @@ func (i *MultipartTask) JoinParts(ctx context.Context, parts []types.UploadedPar
return types.FileInfo{ return types.FileInfo{
Path: i.joinedFileJPath, Path: i.joinedFileJPath,
Size: size, Size: size,
Hash: clitypes.NewFullHash(h),
Hash: jcsypes.NewFullHash(h),
}, nil }, nil
} }




+ 5
- 5
common/pkgs/storage/local/s2s.go View File

@@ -6,20 +6,20 @@ import (
"os" "os"
"path/filepath" "path/filepath"


clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types" cortypes "gitlink.org.cn/cloudream/jcs-pub/coordinator/types"
) )


type S2STransfer struct { type S2STransfer struct {
feat *cortypes.S2STransferFeature feat *cortypes.S2STransferFeature
detail *clitypes.UserSpaceDetail
detail *jcsypes.UserSpaceDetail
localStg *cortypes.LocalCred localStg *cortypes.LocalCred
dstPath clitypes.JPath
dstPath jcsypes.JPath
} }


// 只有同一个机器的存储之间才可以进行数据直传 // 只有同一个机器的存储之间才可以进行数据直传
func (*S2STransfer) CanTransfer(src, dst *clitypes.UserSpaceDetail) bool {
func (*S2STransfer) CanTransfer(src, dst *jcsypes.UserSpaceDetail) bool {
if types.FindFeature[*cortypes.S2STransferFeature](dst) == nil { if types.FindFeature[*cortypes.S2STransferFeature](dst) == nil {
return false return false
} }
@@ -37,7 +37,7 @@ func (*S2STransfer) CanTransfer(src, dst *clitypes.UserSpaceDetail) bool {
} }


// 执行数据直传 // 执行数据直传
func (s *S2STransfer) Transfer(ctx context.Context, src *clitypes.UserSpaceDetail, srcPath clitypes.JPath, dstPath clitypes.JPath) (types.FileInfo, error) {
func (s *S2STransfer) Transfer(ctx context.Context, src *jcsypes.UserSpaceDetail, srcPath jcsypes.JPath, dstPath jcsypes.JPath) (types.FileInfo, error) {
s.dstPath = dstPath s.dstPath = dstPath


copy, err := os.OpenFile(filepath.Join(s.localStg.RootDir, s.dstPath.JoinOSPath()), os.O_WRONLY|os.O_CREATE, 0644) copy, err := os.OpenFile(filepath.Join(s.localStg.RootDir, s.dstPath.JoinOSPath()), os.O_WRONLY|os.O_CREATE, 0644)


+ 12
- 12
common/pkgs/storage/local/shard_store.go View File

@@ -9,19 +9,19 @@ import (
"sync" "sync"


"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
clitypes "gitlink.org.cn/cloudream/jcs-pub/client/types"
"gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types" "gitlink.org.cn/cloudream/jcs-pub/common/pkgs/storage/types"
jcsypes "gitlink.org.cn/cloudream/jcs-pub/common/types"
) )


type ShardStore struct { type ShardStore struct {
detail *clitypes.UserSpaceDetail
detail *jcsypes.UserSpaceDetail
stgRoot string stgRoot string
storeAbsRoot string storeAbsRoot string
lock sync.Mutex lock sync.Mutex
done chan any done chan any
} }


func NewShardStore(root string, detail *clitypes.UserSpaceDetail) (*ShardStore, error) {
func NewShardStore(root string, detail *jcsypes.UserSpaceDetail) (*ShardStore, error) {
storeAbsRoot, err := filepath.Abs(filepath.Join(root, detail.UserSpace.WorkingDir.JoinOSPath(), types.ShardStoreWorkingDir)) storeAbsRoot, err := filepath.Abs(filepath.Join(root, detail.UserSpace.WorkingDir.JoinOSPath(), types.ShardStoreWorkingDir))
if err != nil { if err != nil {
return nil, fmt.Errorf("get abs root: %w", err) return nil, fmt.Errorf("get abs root: %w", err)
@@ -43,7 +43,7 @@ func (s *ShardStore) Stop() {
s.getLogger().Infof("component stop") s.getLogger().Infof("component stop")
} }


func (s *ShardStore) Store(path clitypes.JPath, hash clitypes.FileHash, size int64) (types.FileInfo, error) {
func (s *ShardStore) Store(path jcsypes.JPath, hash jcsypes.FileHash, size int64) (types.FileInfo, error) {
fullTempPath := filepath.Join(s.stgRoot, path.JoinOSPath()) fullTempPath := filepath.Join(s.stgRoot, path.JoinOSPath())


s.lock.Lock() s.lock.Lock()
@@ -81,7 +81,7 @@ func (s *ShardStore) Store(path clitypes.JPath, hash clitypes.FileHash, size int
}, nil }, nil
} }


func (s *ShardStore) Info(hash clitypes.FileHash) (types.FileInfo, error) {
func (s *ShardStore) Info(hash jcsypes.FileHash) (types.FileInfo, error) {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()


@@ -118,7 +118,7 @@ func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
return err return err
} }


fileHash, err := clitypes.ParseHash(filepath.Base(info.Name()))
fileHash, err := jcsypes.ParseHash(filepath.Base(info.Name()))
if err != nil { if err != nil {
return nil return nil
} }
@@ -137,11 +137,11 @@ func (s *ShardStore) ListAll() ([]types.FileInfo, error) {
return infos, nil return infos, nil
} }


func (s *ShardStore) GC(avaiables []clitypes.FileHash) error {
func (s *ShardStore) GC(avaiables []jcsypes.FileHash) error {
s.lock.Lock() s.lock.Lock()
defer s.lock.Unlock() defer s.lock.Unlock()


avais := make(map[clitypes.FileHash]bool)
avais := make(map[jcsypes.FileHash]bool)
for _, hash := range avaiables { for _, hash := range avaiables {
avais[hash] = true avais[hash] = true
} }
@@ -162,7 +162,7 @@ func (s *ShardStore) GC(avaiables []clitypes.FileHash) error {
return err return err
} }


fileHash, err := clitypes.ParseHash(filepath.Base(info.Name()))
fileHash, err := jcsypes.ParseHash(filepath.Base(info.Name()))
if err != nil { if err != nil {
return nil return nil
} }
@@ -199,14 +199,14 @@ func (s *ShardStore) getLogger() logger.Logger {
return logger.WithField("ShardStore", "Local").WithField("Storage", s.detail.UserSpace.Storage.String()) return logger.WithField("ShardStore", "Local").WithField("Storage", s.detail.UserSpace.Storage.String())
} }


func (s *ShardStore) getFileDirFromHash(hash clitypes.FileHash) string {
func (s *ShardStore) getFileDirFromHash(hash jcsypes.FileHash) string {
return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2)) return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2))
} }


func (s *ShardStore) getFilePathFromHash(hash clitypes.FileHash) string {
func (s *ShardStore) getFilePathFromHash(hash jcsypes.FileHash) string {
return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2), string(hash)) return filepath.Join(s.storeAbsRoot, hash.GetHashPrefix(2), string(hash))
} }


func (s *ShardStore) getJPathFromHash(hash clitypes.FileHash) clitypes.JPath {
func (s *ShardStore) getJPathFromHash(hash jcsypes.FileHash) jcsypes.JPath {
return s.detail.UserSpace.WorkingDir.ConcatCompsNew(types.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash)) return s.detail.UserSpace.WorkingDir.ConcatCompsNew(types.ShardStoreWorkingDir, hash.GetHashPrefix(2), string(hash))
} }

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save