Browse Source

将client2与client合并

gitlink
Sydonian 8 months ago
parent
commit
43e0e1dbe4
40 changed files with 217 additions and 293 deletions
  1. +2
    -2
      client/internal/cmdline/commandline.go
  2. +1
    -1
      client/internal/cmdline/getp.go
  3. +1
    -1
      client/internal/cmdline/load.go
  4. +1
    -1
      client/internal/cmdline/lsp.go
  5. +4
    -19
      client/internal/cmdline/mount.go
  6. +1
    -1
      client/internal/cmdline/newloadp.go
  7. +1
    -1
      client/internal/cmdline/put.go
  8. +170
    -16
      client/internal/cmdline/serve.go
  9. +1
    -1
      client/internal/cmdline/sysevent.go
  10. +5
    -5
      client/internal/cmdline/test.go
  11. +7
    -2
      client/internal/config/config.go
  12. +0
    -0
      client/internal/mount/config/config.go
  13. +0
    -0
      client/internal/mount/fuse/dir_handle.go
  14. +0
    -0
      client/internal/mount/fuse/dir_node.go
  15. +0
    -0
      client/internal/mount/fuse/file_handle.go
  16. +0
    -0
      client/internal/mount/fuse/file_node.go
  17. +1
    -1
      client/internal/mount/fuse/fuse.go
  18. +0
    -0
      client/internal/mount/fuse/node.go
  19. +0
    -0
      client/internal/mount/fuse/types.go
  20. +3
    -3
      client/internal/mount/mount.go
  21. +1
    -1
      client/internal/mount/mount_win.go
  22. +2
    -2
      client/internal/mount/vfs/cache/cache.go
  23. +0
    -0
      client/internal/mount/vfs/cache/dir.go
  24. +1
    -1
      client/internal/mount/vfs/cache/file.go
  25. +0
    -0
      client/internal/mount/vfs/cache/range_test.go
  26. +0
    -0
      client/internal/mount/vfs/cache/remote.go
  27. +0
    -0
      client/internal/mount/vfs/cache/utils.go
  28. +1
    -1
      client/internal/mount/vfs/dir_reader.go
  29. +0
    -0
      client/internal/mount/vfs/fuse.go
  30. +2
    -2
      client/internal/mount/vfs/fuse_bucket.go
  31. +2
    -2
      client/internal/mount/vfs/fuse_dir.go
  32. +2
    -2
      client/internal/mount/vfs/fuse_file.go
  33. +2
    -2
      client/internal/mount/vfs/fuse_package.go
  34. +2
    -2
      client/internal/mount/vfs/fuse_root.go
  35. +3
    -3
      client/internal/mount/vfs/vfs.go
  36. +1
    -157
      client/main.go
  37. +0
    -5
      client2/internal/cmd/cmd.go
  38. +0
    -43
      client2/internal/config/config.go
  39. +0
    -7
      client2/main.go
  40. +0
    -9
      magefiles/main.go

+ 2
- 2
client/internal/cmdline/commandline.go View File

@@ -17,7 +17,7 @@ type CommandContext struct {
// TODO 逐步使用cobra代替cmdtrie // TODO 逐步使用cobra代替cmdtrie
var commands cmdtrie.CommandTrie[CommandContext, error] = cmdtrie.NewCommandTrie[CommandContext, error]() var commands cmdtrie.CommandTrie[CommandContext, error] = cmdtrie.NewCommandTrie[CommandContext, error]()


var rootCmd = cobra.Command{}
var RootCmd = cobra.Command{}


type Commandline struct { type Commandline struct {
Svc *services.Service Svc *services.Service
@@ -37,7 +37,7 @@ func (c *Commandline) DispatchCommand(allArgs []string) {
if err != nil { if err != nil {
if err == cmdtrie.ErrCommandNotFound { if err == cmdtrie.ErrCommandNotFound {
ctx := context.WithValue(context.Background(), "cmdCtx", &cmdCtx) ctx := context.WithValue(context.Background(), "cmdCtx", &cmdCtx)
err = rootCmd.ExecuteContext(ctx)
err = RootCmd.ExecuteContext(ctx)
if err != nil { if err != nil {
fmt.Println(err) fmt.Println(err)
os.Exit(1) os.Exit(1)


+ 1
- 1
client/internal/cmdline/getp.go View File

@@ -40,7 +40,7 @@ func init() {
} }
cmd.Flags().BoolVarP(&usePkgID, "id", "i", false, "Download with package id instead of path") cmd.Flags().BoolVarP(&usePkgID, "id", "i", false, "Download with package id instead of path")


rootCmd.AddCommand(cmd)
RootCmd.AddCommand(cmd)
} }


func getpByPath(cmdCtx *CommandContext, path string, output string) { func getpByPath(cmdCtx *CommandContext, path string, output string) {


+ 1
- 1
client/internal/cmdline/load.go View File

@@ -37,7 +37,7 @@ func init() {
}, },
} }
cmd.Flags().BoolVarP(&useID, "id", "i", false, "Use ID for both package and storage service instead of their name or path") cmd.Flags().BoolVarP(&useID, "id", "i", false, "Use ID for both package and storage service instead of their name or path")
rootCmd.AddCommand(&cmd)
RootCmd.AddCommand(&cmd)
} }


func loadByPath(cmdCtx *CommandContext, pkgPath string, stgName string, rootPath string) { func loadByPath(cmdCtx *CommandContext, pkgPath string, stgName string, rootPath string) {


+ 1
- 1
client/internal/cmdline/lsp.go View File

@@ -34,7 +34,7 @@ func init() {
} }
usePkgID = cmd.Flags().BoolP("id", "i", false, "List with package id instead of path") usePkgID = cmd.Flags().BoolP("id", "i", false, "List with package id instead of path")


rootCmd.AddCommand(cmd)
RootCmd.AddCommand(cmd)
} }


func lspByPath(cmdCtx *CommandContext, path string) { func lspByPath(cmdCtx *CommandContext, path string) {


client2/internal/cmd/mount.go → client/internal/cmdline/mount.go View File

@@ -1,4 +1,4 @@
package cmd
package cmdline


import ( import (
"fmt" "fmt"
@@ -7,9 +7,9 @@ import (


"github.com/spf13/cobra" "github.com/spf13/cobra"
"gitlink.org.cn/cloudream/common/pkgs/logger" "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/storage2/client2/internal/config"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount"
mntcfg "gitlink.org.cn/cloudream/storage2/client2/internal/mount/config"
"gitlink.org.cn/cloudream/storage2/client/internal/config"
"gitlink.org.cn/cloudream/storage2/client/internal/mount"
mntcfg "gitlink.org.cn/cloudream/storage2/client/internal/mount/config"
stgglb "gitlink.org.cn/cloudream/storage2/common/globals" stgglb "gitlink.org.cn/cloudream/storage2/common/globals"
"gitlink.org.cn/cloudream/storage2/common/pkgs/connectivity" "gitlink.org.cn/cloudream/storage2/common/pkgs/connectivity"
"gitlink.org.cn/cloudream/storage2/common/pkgs/db2" "gitlink.org.cn/cloudream/storage2/common/pkgs/db2"
@@ -119,18 +119,3 @@ func mountCmd(mountPoint string, configPath string) {
} }
} }
} }

func serveDistLock(svc *distlock.Service) {
logger.Info("start serving distlock")

err := svc.Serve()

if err != nil {
logger.Errorf("distlock stopped with error: %s", err.Error())
}

logger.Info("distlock stopped")

// TODO 仅简单结束了程序
os.Exit(1)
}

+ 1
- 1
client/internal/cmdline/newloadp.go View File

@@ -50,7 +50,7 @@ func init() {
}, },
} }


rootCmd.AddCommand(cmd)
RootCmd.AddCommand(cmd)
} }


func newloadp(cmdCtx *CommandContext, path string, bucketID cdssdk.BucketID, packageName string, storageIDs []cdssdk.StorageID, rootPathes []string) { func newloadp(cmdCtx *CommandContext, path string, bucketID cdssdk.BucketID, packageName string, storageIDs []cdssdk.StorageID, rootPathes []string) {


+ 1
- 1
client/internal/cmdline/put.go View File

@@ -118,5 +118,5 @@ func init() {
} }
cmd.Flags().Int64VarP(&stgID, "storage", "s", 0, "storage affinity") cmd.Flags().Int64VarP(&stgID, "storage", "s", 0, "storage affinity")


rootCmd.AddCommand(cmd)
RootCmd.AddCommand(cmd)
} }

+ 170
- 16
client/internal/cmdline/serve.go View File

@@ -2,43 +2,197 @@ package cmdline


import ( import (
"fmt" "fmt"
"os"
"time"


"github.com/spf13/cobra"
"gitlink.org.cn/cloudream/common/pkgs/logger"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/storage2/client/internal/config" "gitlink.org.cn/cloudream/storage2/client/internal/config"
"gitlink.org.cn/cloudream/storage2/client/internal/http" "gitlink.org.cn/cloudream/storage2/client/internal/http"
"gitlink.org.cn/cloudream/storage2/client/internal/services"
"gitlink.org.cn/cloudream/storage2/client/internal/task"
stgglb "gitlink.org.cn/cloudream/storage2/common/globals"
"gitlink.org.cn/cloudream/storage2/common/pkgs/accessstat"
"gitlink.org.cn/cloudream/storage2/common/pkgs/connectivity"
"gitlink.org.cn/cloudream/storage2/common/pkgs/distlock"
"gitlink.org.cn/cloudream/storage2/common/pkgs/downloader"
"gitlink.org.cn/cloudream/storage2/common/pkgs/downloader/strategy"
"gitlink.org.cn/cloudream/storage2/common/pkgs/metacache"
coormq "gitlink.org.cn/cloudream/storage2/common/pkgs/mq/coordinator"
"gitlink.org.cn/cloudream/storage2/common/pkgs/storage/agtpool"
"gitlink.org.cn/cloudream/storage2/common/pkgs/uploader"
) )


// ServeHTTP 启动HTTP服务。
// ctx: 命令行上下文,包含服务配置等信息。
// args: 命令行参数,第一个参数可选地指定HTTP服务器监听地址。
// 返回值: 如果启动过程中遇到错误,返回错误信息;否则返回nil。
func ServeHTTP(ctx CommandContext, args []string) error {
// 初始化函数,将ServeHTTP命令注册到命令列表中。
func init() {
var configPath, listenAddr string
cmd := cobra.Command{
Use: "serve",
Short: "start serving storage service",
Run: func(cmd *cobra.Command, args []string) {
serveHTTP(configPath, listenAddr)
},
}
cmd.Flags().StringVarP(&configPath, "config", "c", "", "config file path")
cmd.Flags().StringVarP(&listenAddr, "listen", "l", "", "listen address")
RootCmd.AddCommand(&cmd)
}

func serveHTTP(configPath string, listenAddr string) {
err := config.Init(configPath)
if err != nil {
fmt.Printf("init config failed, err: %s", err.Error())
os.Exit(1)
}

err = logger.Init(&config.Cfg().Logger)
if err != nil {
fmt.Printf("init logger failed, err: %s", err.Error())
os.Exit(1)
}

stgglb.InitLocal(&config.Cfg().Local)
stgglb.InitMQPool(config.Cfg().RabbitMQ)
stgglb.InitAgentRPCPool(&config.Cfg().AgentGRPC)

// 连接性信息收集
var conCol connectivity.Collector
if config.Cfg().Local.HubID != nil {
//如果client与某个hub处于同一台机器,则使用这个hub的连通性信息
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
logger.Warnf("acquire coordinator mq failed, err: %s", err.Error())
os.Exit(1)
}
getCons, err := coorCli.GetHubConnectivities(coormq.ReqGetHubConnectivities([]cdssdk.HubID{*config.Cfg().Local.HubID}))
if err != nil {
logger.Warnf("get hub connectivities failed, err: %s", err.Error())
os.Exit(1)
}
consMap := make(map[cdssdk.HubID]connectivity.Connectivity)
for _, con := range getCons.Connectivities {
var delay *time.Duration
if con.Latency != nil {
d := time.Duration(*con.Latency * float32(time.Millisecond))
delay = &d
}
consMap[con.FromHubID] = connectivity.Connectivity{
ToHubID: con.ToHubID,
Latency: delay,
}
}
conCol = connectivity.NewCollectorWithInitData(&config.Cfg().Connectivity, nil, consMap)
logger.Info("use local hub connectivities")

} else {
// 否则需要就地收集连通性信息
conCol = connectivity.NewCollector(&config.Cfg().Connectivity, nil)
conCol.CollectInPlace()
}

metaCacheHost := metacache.NewHost()
go metaCacheHost.Serve()
stgMeta := metaCacheHost.AddStorageMeta()
hubMeta := metaCacheHost.AddHubMeta()
conMeta := metaCacheHost.AddConnectivity()

// 分布式锁
distlockSvc, err := distlock.NewService(&config.Cfg().DistLock)
if err != nil {
logger.Warnf("new distlock service failed, err: %s", err.Error())
os.Exit(1)
}
go serveDistLock(distlockSvc)

// 访问统计
acStat := accessstat.NewAccessStat(accessstat.Config{
// TODO 考虑放到配置里
ReportInterval: time.Second * 10,
})
go serveAccessStat(acStat)

// 存储管理器
stgAgts := agtpool.NewPool()

// 任务管理器
taskMgr := task.NewManager(distlockSvc, &conCol, stgAgts)

strgSel := strategy.NewSelector(config.Cfg().DownloadStrategy, stgMeta, hubMeta, conMeta)

// 下载器
dlder := downloader.NewDownloader(config.Cfg().Downloader, &conCol, stgAgts, strgSel)

// 上传器
uploader := uploader.NewUploader(distlockSvc, &conCol, stgAgts, stgMeta)

svc, err := services.NewService(distlockSvc, &taskMgr, &dlder, acStat, uploader, strgSel, stgMeta)
if err != nil {
logger.Warnf("new services failed, err: %s", err.Error())
os.Exit(1)
}

// 默认监听地址为":7890",如果提供了命令行参数,则使用参数指定的地址。 // 默认监听地址为":7890",如果提供了命令行参数,则使用参数指定的地址。
listenAddr := ":7890"
if len(args) > 0 {
listenAddr = args[0]
if listenAddr == "" {
listenAddr = ":7890"
} }


awsAuth, err := http.NewAWSAuth(config.Cfg().AuthAccessKey, config.Cfg().AuthSecretKey) awsAuth, err := http.NewAWSAuth(config.Cfg().AuthAccessKey, config.Cfg().AuthSecretKey)
if err != nil { if err != nil {
return fmt.Errorf("new aws auth: %w", err)
logger.Warnf("new aws auth: %v", err)
os.Exit(1)
} }


// 创建一个新的HTTP服务器实例。 // 创建一个新的HTTP服务器实例。
httpSvr, err := http.NewServer(listenAddr, ctx.Cmdline.Svc, awsAuth)
httpSvr, err := http.NewServer(listenAddr, svc, awsAuth)
if err != nil { if err != nil {
return fmt.Errorf("new http server: %w", err)
logger.Warnf("new http server: %v", err)
os.Exit(1)
} }


// 启动HTTP服务。 // 启动HTTP服务。
err = httpSvr.Serve() err = httpSvr.Serve()
if err != nil { if err != nil {
return fmt.Errorf("serving http: %w", err)
logger.Warnf("serving http: %v", err)
os.Exit(1)
}
}

func serveDistLock(svc *distlock.Service) {
logger.Info("start serving distlock")

err := svc.Serve()

if err != nil {
logger.Errorf("distlock stopped with error: %s", err.Error())
} }


return nil
logger.Info("distlock stopped")

// TODO 仅简单结束了程序
os.Exit(1)
} }


// 初始化函数,将ServeHTTP命令注册到命令列表中。
func init() {
commands.MustAdd(ServeHTTP, "serve", "http")
func serveAccessStat(svc *accessstat.AccessStat) {
logger.Info("start serving access stat")

ch := svc.Start()
loop:
for {
val, err := ch.Receive()
if err != nil {
logger.Errorf("access stat stopped with error: %v", err)
break
}

switch val := val.(type) {
case error:
logger.Errorf("access stat stopped with error: %v", val)
break loop
}
}
logger.Info("access stat stopped")

// TODO 仅简单结束了程序
os.Exit(1)
} }

+ 1
- 1
client/internal/cmdline/sysevent.go View File

@@ -16,7 +16,7 @@ func init() {
Use: "sysevent", Use: "sysevent",
} }


rootCmd.AddCommand(cmd)
RootCmd.AddCommand(cmd)


outputJSON := false outputJSON := false
watchCmd := &cobra.Command{ watchCmd := &cobra.Command{


+ 5
- 5
client/internal/cmdline/test.go View File

@@ -17,7 +17,7 @@ import (
) )


func init() { func init() {
rootCmd.AddCommand(&cobra.Command{
RootCmd.AddCommand(&cobra.Command{
Use: "test", Use: "test",
Short: "test", Short: "test",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
@@ -72,7 +72,7 @@ func init() {
}, },
}) })


rootCmd.AddCommand(&cobra.Command{
RootCmd.AddCommand(&cobra.Command{
Use: "test32", Use: "test32",
Short: "test32", Short: "test32",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
@@ -118,7 +118,7 @@ func init() {
fut.Wait(context.TODO()) fut.Wait(context.TODO())
}, },
}) })
rootCmd.AddCommand(&cobra.Command{
RootCmd.AddCommand(&cobra.Command{
Use: "test1", Use: "test1",
Short: "test1", Short: "test1",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
@@ -188,7 +188,7 @@ func init() {
}, },
}) })


rootCmd.AddCommand(&cobra.Command{
RootCmd.AddCommand(&cobra.Command{
Use: "test4", Use: "test4",
Short: "test4", Short: "test4",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
@@ -238,7 +238,7 @@ func init() {
}, },
}) })


rootCmd.AddCommand(&cobra.Command{
RootCmd.AddCommand(&cobra.Command{
Use: "test11", Use: "test11",
Short: "test11", Short: "test11",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {


+ 7
- 2
client/internal/config/config.go View File

@@ -8,6 +8,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/config" "gitlink.org.cn/cloudream/common/utils/config"
stgmodels "gitlink.org.cn/cloudream/storage2/common/models" stgmodels "gitlink.org.cn/cloudream/storage2/common/models"
"gitlink.org.cn/cloudream/storage2/common/pkgs/connectivity" "gitlink.org.cn/cloudream/storage2/common/pkgs/connectivity"
db "gitlink.org.cn/cloudream/storage2/common/pkgs/db2/config"
"gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader"
"gitlink.org.cn/cloudream/storage2/common/pkgs/downloader/strategy" "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader/strategy"
agtrpc "gitlink.org.cn/cloudream/storage2/common/pkgs/grpc/agent" agtrpc "gitlink.org.cn/cloudream/storage2/common/pkgs/grpc/agent"
@@ -26,14 +27,18 @@ type Config struct {
AuthAccessKey string `json:"authAccessKey"` // TODO 临时办法 AuthAccessKey string `json:"authAccessKey"` // TODO 临时办法
AuthSecretKey string `json:"authSecretKey"` AuthSecretKey string `json:"authSecretKey"`
MaxHTTPBodySize int64 `json:"maxHttpBodySize"` MaxHTTPBodySize int64 `json:"maxHttpBodySize"`
DB db.Config `json:"db"`
} }


var cfg Config var cfg Config


// Init 初始化client // Init 初始化client
// TODO 这里的modeulName参数弄成可配置的更好 // TODO 这里的modeulName参数弄成可配置的更好
func Init() error {
return config.DefaultLoad("client", &cfg)
func Init(configPath string) error {
if configPath == "" {
return config.DefaultLoad("client", &cfg)
}
return config.Load(configPath, &cfg)
} }


func Cfg() *Config { func Cfg() *Config {


client2/internal/mount/config/config.go → client/internal/mount/config/config.go View File


client2/internal/mount/fuse/dir_handle.go → client/internal/mount/fuse/dir_handle.go View File


client2/internal/mount/fuse/dir_node.go → client/internal/mount/fuse/dir_node.go View File


client2/internal/mount/fuse/file_handle.go → client/internal/mount/fuse/file_handle.go View File


client2/internal/mount/fuse/file_node.go → client/internal/mount/fuse/file_node.go View File


client2/internal/mount/fuse/fuse.go → client/internal/mount/fuse/fuse.go View File

@@ -8,7 +8,7 @@ import (


fusefs "github.com/hanwen/go-fuse/v2/fs" fusefs "github.com/hanwen/go-fuse/v2/fs"
"github.com/hanwen/go-fuse/v2/fuse" "github.com/hanwen/go-fuse/v2/fuse"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/config"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/config"
) )


type Fuse struct { type Fuse struct {

client2/internal/mount/fuse/node.go → client/internal/mount/fuse/node.go View File


client2/internal/mount/fuse/types.go → client/internal/mount/fuse/types.go View File


client2/internal/mount/mount.go → client/internal/mount/mount.go View File

@@ -7,9 +7,9 @@ import (
"github.com/hanwen/go-fuse/v2/fuse" "github.com/hanwen/go-fuse/v2/fuse"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/utils/sync2" "gitlink.org.cn/cloudream/common/utils/sync2"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/config"
fuse2 "gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/vfs"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/config"
fuse2 "gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/vfs"
"gitlink.org.cn/cloudream/storage2/common/pkgs/db2" "gitlink.org.cn/cloudream/storage2/common/pkgs/db2"
"gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader"
"gitlink.org.cn/cloudream/storage2/common/pkgs/uploader" "gitlink.org.cn/cloudream/storage2/common/pkgs/uploader"

client2/internal/mount/mount_win.go → client/internal/mount/mount_win.go View File

@@ -7,7 +7,7 @@ import (


cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/utils/sync2" "gitlink.org.cn/cloudream/common/utils/sync2"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/config"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/config"
"gitlink.org.cn/cloudream/storage2/common/pkgs/db2" "gitlink.org.cn/cloudream/storage2/common/pkgs/db2"
"gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader"
"gitlink.org.cn/cloudream/storage2/common/pkgs/uploader" "gitlink.org.cn/cloudream/storage2/common/pkgs/uploader"

client2/internal/mount/vfs/cache/cache.go → client/internal/mount/vfs/cache/cache.go View File

@@ -16,8 +16,8 @@ import (
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/utils/io2" "gitlink.org.cn/cloudream/common/utils/io2"
"gitlink.org.cn/cloudream/common/utils/lo2" "gitlink.org.cn/cloudream/common/utils/lo2"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/config"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/config"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/common/pkgs/db2" "gitlink.org.cn/cloudream/storage2/common/pkgs/db2"
"gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader"
"gitlink.org.cn/cloudream/storage2/common/pkgs/uploader" "gitlink.org.cn/cloudream/storage2/common/pkgs/uploader"

client2/internal/mount/vfs/cache/dir.go → client/internal/mount/vfs/cache/dir.go View File


client2/internal/mount/vfs/cache/file.go → client/internal/mount/vfs/cache/file.go View File

@@ -13,7 +13,7 @@ import (
"gitlink.org.cn/cloudream/common/utils/lo2" "gitlink.org.cn/cloudream/common/utils/lo2"
"gitlink.org.cn/cloudream/common/utils/math2" "gitlink.org.cn/cloudream/common/utils/math2"
"gitlink.org.cn/cloudream/common/utils/serder" "gitlink.org.cn/cloudream/common/utils/serder"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse"
) )


type FileInfo struct { type FileInfo struct {

client2/internal/mount/vfs/cache/range_test.go → client/internal/mount/vfs/cache/range_test.go View File


client2/internal/mount/vfs/cache/remote.go → client/internal/mount/vfs/cache/remote.go View File


client2/internal/mount/vfs/cache/utils.go → client/internal/mount/vfs/cache/utils.go View File


client2/internal/mount/vfs/dir_reader.go → client/internal/mount/vfs/dir_reader.go View File

@@ -1,6 +1,6 @@
package vfs package vfs


import "gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse"
import "gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse"


type FuseDirReader struct { type FuseDirReader struct {
allEntries []fuse.FsEntry allEntries []fuse.FsEntry

client2/internal/mount/vfs/fuse.go → client/internal/mount/vfs/fuse.go View File


client2/internal/mount/vfs/fuse_bucket.go → client/internal/mount/vfs/fuse_bucket.go View File

@@ -7,8 +7,8 @@ import (
"time" "time"


cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/vfs/cache"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/vfs/cache"
"gitlink.org.cn/cloudream/storage2/common/pkgs/db2" "gitlink.org.cn/cloudream/storage2/common/pkgs/db2"
"gorm.io/gorm" "gorm.io/gorm"
) )

client2/internal/mount/vfs/fuse_dir.go → client/internal/mount/vfs/fuse_dir.go View File

@@ -8,8 +8,8 @@ import (


cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/utils/lo2" "gitlink.org.cn/cloudream/common/utils/lo2"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/vfs/cache"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/vfs/cache"
"gitlink.org.cn/cloudream/storage2/common/pkgs/db2" "gitlink.org.cn/cloudream/storage2/common/pkgs/db2"
"gorm.io/gorm" "gorm.io/gorm"
) )

client2/internal/mount/vfs/fuse_file.go → client/internal/mount/vfs/fuse_file.go View File

@@ -5,8 +5,8 @@ import (
"time" "time"


cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/vfs/cache"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/vfs/cache"
"gorm.io/gorm" "gorm.io/gorm"
) )



client2/internal/mount/vfs/fuse_package.go → client/internal/mount/vfs/fuse_package.go View File

@@ -8,8 +8,8 @@ import (


cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/common/utils/lo2" "gitlink.org.cn/cloudream/common/utils/lo2"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/vfs/cache"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/vfs/cache"
"gitlink.org.cn/cloudream/storage2/common/pkgs/db2" "gitlink.org.cn/cloudream/storage2/common/pkgs/db2"
"gorm.io/gorm" "gorm.io/gorm"
) )

client2/internal/mount/vfs/fuse_root.go → client/internal/mount/vfs/fuse_root.go View File

@@ -6,8 +6,8 @@ import (
"time" "time"


cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/vfs/cache"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/vfs/cache"
"gitlink.org.cn/cloudream/storage2/common/pkgs/db2" "gitlink.org.cn/cloudream/storage2/common/pkgs/db2"
"gorm.io/gorm" "gorm.io/gorm"
) )

client2/internal/mount/vfs/vfs.go → client/internal/mount/vfs/vfs.go View File

@@ -1,9 +1,9 @@
package vfs package vfs


import ( import (
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/config"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client2/internal/mount/vfs/cache"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/config"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse"
"gitlink.org.cn/cloudream/storage2/client/internal/mount/vfs/cache"
"gitlink.org.cn/cloudream/storage2/common/pkgs/db2" "gitlink.org.cn/cloudream/storage2/common/pkgs/db2"
"gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader"
"gitlink.org.cn/cloudream/storage2/common/pkgs/uploader" "gitlink.org.cn/cloudream/storage2/common/pkgs/uploader"

+ 1
- 157
client/main.go View File

@@ -1,167 +1,11 @@
package main package main


import ( import (
"fmt"
"os"
"time"

_ "google.golang.org/grpc/balancer/grpclb" _ "google.golang.org/grpc/balancer/grpclb"


"gitlink.org.cn/cloudream/common/pkgs/logger"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
"gitlink.org.cn/cloudream/storage2/client/internal/cmdline" "gitlink.org.cn/cloudream/storage2/client/internal/cmdline"
"gitlink.org.cn/cloudream/storage2/client/internal/config"
"gitlink.org.cn/cloudream/storage2/client/internal/services"
"gitlink.org.cn/cloudream/storage2/client/internal/task"
stgglb "gitlink.org.cn/cloudream/storage2/common/globals"
"gitlink.org.cn/cloudream/storage2/common/pkgs/accessstat"
"gitlink.org.cn/cloudream/storage2/common/pkgs/connectivity"
"gitlink.org.cn/cloudream/storage2/common/pkgs/distlock"
"gitlink.org.cn/cloudream/storage2/common/pkgs/downloader"
"gitlink.org.cn/cloudream/storage2/common/pkgs/downloader/strategy"
"gitlink.org.cn/cloudream/storage2/common/pkgs/metacache"
coormq "gitlink.org.cn/cloudream/storage2/common/pkgs/mq/coordinator"
"gitlink.org.cn/cloudream/storage2/common/pkgs/storage/agtpool"
"gitlink.org.cn/cloudream/storage2/common/pkgs/uploader"
) )


func main() { func main() {
err := config.Init()
if err != nil {
fmt.Printf("init config failed, err: %s", err.Error())
os.Exit(1)
}

err = logger.Init(&config.Cfg().Logger)
if err != nil {
fmt.Printf("init logger failed, err: %s", err.Error())
os.Exit(1)
}

stgglb.InitLocal(&config.Cfg().Local)
stgglb.InitMQPool(config.Cfg().RabbitMQ)
stgglb.InitAgentRPCPool(&config.Cfg().AgentGRPC)

// 连接性信息收集
var conCol connectivity.Collector
if config.Cfg().Local.HubID != nil {
//如果client与某个hub处于同一台机器,则使用这个hub的连通性信息
coorCli, err := stgglb.CoordinatorMQPool.Acquire()
if err != nil {
logger.Warnf("acquire coordinator mq failed, err: %s", err.Error())
os.Exit(1)
}
getCons, err := coorCli.GetHubConnectivities(coormq.ReqGetHubConnectivities([]cdssdk.HubID{*config.Cfg().Local.HubID}))
if err != nil {
logger.Warnf("get hub connectivities failed, err: %s", err.Error())
os.Exit(1)
}
consMap := make(map[cdssdk.HubID]connectivity.Connectivity)
for _, con := range getCons.Connectivities {
var delay *time.Duration
if con.Latency != nil {
d := time.Duration(*con.Latency * float32(time.Millisecond))
delay = &d
}
consMap[con.FromHubID] = connectivity.Connectivity{
ToHubID: con.ToHubID,
Latency: delay,
}
}
conCol = connectivity.NewCollectorWithInitData(&config.Cfg().Connectivity, nil, consMap)
logger.Info("use local hub connectivities")

} else {
// 否则需要就地收集连通性信息
conCol = connectivity.NewCollector(&config.Cfg().Connectivity, nil)
conCol.CollectInPlace()
}

metaCacheHost := metacache.NewHost()
go metaCacheHost.Serve()
stgMeta := metaCacheHost.AddStorageMeta()
hubMeta := metaCacheHost.AddHubMeta()
conMeta := metaCacheHost.AddConnectivity()

// 分布式锁
distlockSvc, err := distlock.NewService(&config.Cfg().DistLock)
if err != nil {
logger.Warnf("new distlock service failed, err: %s", err.Error())
os.Exit(1)
}
go serveDistLock(distlockSvc)

// 访问统计
acStat := accessstat.NewAccessStat(accessstat.Config{
// TODO 考虑放到配置里
ReportInterval: time.Second * 10,
})
go serveAccessStat(acStat)

// 存储管理器
stgAgts := agtpool.NewPool()

// 任务管理器
taskMgr := task.NewManager(distlockSvc, &conCol, stgAgts)

strgSel := strategy.NewSelector(config.Cfg().DownloadStrategy, stgMeta, hubMeta, conMeta)

// 下载器
dlder := downloader.NewDownloader(config.Cfg().Downloader, &conCol, stgAgts, strgSel)

// 上传器
uploader := uploader.NewUploader(distlockSvc, &conCol, stgAgts, stgMeta)

svc, err := services.NewService(distlockSvc, &taskMgr, &dlder, acStat, uploader, strgSel, stgMeta)
if err != nil {
logger.Warnf("new services failed, err: %s", err.Error())
os.Exit(1)
}

cmds, err := cmdline.NewCommandline(svc)
if err != nil {
logger.Warnf("new command line failed, err: %s", err.Error())
os.Exit(1)
}

cmds.DispatchCommand(os.Args[1:])
}

func serveDistLock(svc *distlock.Service) {
logger.Info("start serving distlock")

err := svc.Serve()

if err != nil {
logger.Errorf("distlock stopped with error: %s", err.Error())
}

logger.Info("distlock stopped")

// TODO 仅简单结束了程序
os.Exit(1)
}

func serveAccessStat(svc *accessstat.AccessStat) {
logger.Info("start serving access stat")

ch := svc.Start()
loop:
for {
val, err := ch.Receive()
if err != nil {
logger.Errorf("access stat stopped with error: %v", err)
break
}

switch val := val.(type) {
case error:
logger.Errorf("access stat stopped with error: %v", val)
break loop
}
}
logger.Info("access stat stopped")

// TODO 仅简单结束了程序
os.Exit(1)
cmdline.RootCmd.Execute()
} }

+ 0
- 5
client2/internal/cmd/cmd.go View File

@@ -1,5 +0,0 @@
package cmd

import "github.com/spf13/cobra"

var RootCmd = cobra.Command{}

+ 0
- 43
client2/internal/config/config.go View File

@@ -1,43 +0,0 @@
package config

import (
"gitlink.org.cn/cloudream/common/pkgs/distlock"
log "gitlink.org.cn/cloudream/common/pkgs/logger"
"gitlink.org.cn/cloudream/common/pkgs/mq"
cdssdk "gitlink.org.cn/cloudream/common/sdks/storage"
c "gitlink.org.cn/cloudream/common/utils/config"
stgmodels "gitlink.org.cn/cloudream/storage2/common/models"
"gitlink.org.cn/cloudream/storage2/common/pkgs/connectivity"
db "gitlink.org.cn/cloudream/storage2/common/pkgs/db2/config"
"gitlink.org.cn/cloudream/storage2/common/pkgs/downloader"
"gitlink.org.cn/cloudream/storage2/common/pkgs/downloader/strategy"
"gitlink.org.cn/cloudream/storage2/common/pkgs/grpc"
)

type Config struct {
ID cdssdk.HubID `json:"id"`
ListenAddr string `json:"listenAddr"`
Local stgmodels.LocalMachineInfo `json:"local"`
GRPC *grpc.Config `json:"grpc"`
Logger log.Config `json:"logger"`
RabbitMQ mq.Config `json:"rabbitMQ"`
DistLock distlock.Config `json:"distlock"`
Connectivity connectivity.Config `json:"connectivity"`
Downloader downloader.Config `json:"downloader"`
DownloadStrategy strategy.Config `json:"downloadStrategy"`
DB db.Config `json:"db"`
}

var cfg Config

func Init(path string) error {
if path == "" {
return c.DefaultLoad("client2", &cfg)
}

return c.Load(path, &cfg)
}

func Cfg() *Config {
return &cfg
}

+ 0
- 7
client2/main.go View File

@@ -1,7 +0,0 @@
package main

import "gitlink.org.cn/cloudream/storage2/client2/internal/cmd"

func main() {
cmd.RootCmd.Execute()
}

+ 0
- 9
magefiles/main.go View File

@@ -117,15 +117,6 @@ func Client() error {
}) })
} }


func Client2() error {
return magefiles.Build(magefiles.BuildArgs{
OutputName: "client2",
OutputDir: "client2",
AssetsDir: "assets",
EntryFile: "client2/main.go",
})
}

func Coordinator() error { func Coordinator() error {
return magefiles.Build(magefiles.BuildArgs{ return magefiles.Build(magefiles.BuildArgs{
OutputName: "coordinator", OutputName: "coordinator",


Loading…
Cancel
Save