| @@ -17,7 +17,7 @@ type CommandContext struct { | |||||
| // TODO 逐步使用cobra代替cmdtrie | // TODO 逐步使用cobra代替cmdtrie | ||||
| var commands cmdtrie.CommandTrie[CommandContext, error] = cmdtrie.NewCommandTrie[CommandContext, error]() | var commands cmdtrie.CommandTrie[CommandContext, error] = cmdtrie.NewCommandTrie[CommandContext, error]() | ||||
| var rootCmd = cobra.Command{} | |||||
| var RootCmd = cobra.Command{} | |||||
| type Commandline struct { | type Commandline struct { | ||||
| Svc *services.Service | Svc *services.Service | ||||
| @@ -37,7 +37,7 @@ func (c *Commandline) DispatchCommand(allArgs []string) { | |||||
| if err != nil { | if err != nil { | ||||
| if err == cmdtrie.ErrCommandNotFound { | if err == cmdtrie.ErrCommandNotFound { | ||||
| ctx := context.WithValue(context.Background(), "cmdCtx", &cmdCtx) | ctx := context.WithValue(context.Background(), "cmdCtx", &cmdCtx) | ||||
| err = rootCmd.ExecuteContext(ctx) | |||||
| err = RootCmd.ExecuteContext(ctx) | |||||
| if err != nil { | if err != nil { | ||||
| fmt.Println(err) | fmt.Println(err) | ||||
| os.Exit(1) | os.Exit(1) | ||||
| @@ -40,7 +40,7 @@ func init() { | |||||
| } | } | ||||
| cmd.Flags().BoolVarP(&usePkgID, "id", "i", false, "Download with package id instead of path") | cmd.Flags().BoolVarP(&usePkgID, "id", "i", false, "Download with package id instead of path") | ||||
| rootCmd.AddCommand(cmd) | |||||
| RootCmd.AddCommand(cmd) | |||||
| } | } | ||||
| func getpByPath(cmdCtx *CommandContext, path string, output string) { | func getpByPath(cmdCtx *CommandContext, path string, output string) { | ||||
| @@ -37,7 +37,7 @@ func init() { | |||||
| }, | }, | ||||
| } | } | ||||
| cmd.Flags().BoolVarP(&useID, "id", "i", false, "Use ID for both package and storage service instead of their name or path") | cmd.Flags().BoolVarP(&useID, "id", "i", false, "Use ID for both package and storage service instead of their name or path") | ||||
| rootCmd.AddCommand(&cmd) | |||||
| RootCmd.AddCommand(&cmd) | |||||
| } | } | ||||
| func loadByPath(cmdCtx *CommandContext, pkgPath string, stgName string, rootPath string) { | func loadByPath(cmdCtx *CommandContext, pkgPath string, stgName string, rootPath string) { | ||||
| @@ -34,7 +34,7 @@ func init() { | |||||
| } | } | ||||
| usePkgID = cmd.Flags().BoolP("id", "i", false, "List with package id instead of path") | usePkgID = cmd.Flags().BoolP("id", "i", false, "List with package id instead of path") | ||||
| rootCmd.AddCommand(cmd) | |||||
| RootCmd.AddCommand(cmd) | |||||
| } | } | ||||
| func lspByPath(cmdCtx *CommandContext, path string) { | func lspByPath(cmdCtx *CommandContext, path string) { | ||||
| @@ -1,4 +1,4 @@ | |||||
| package cmd | |||||
| package cmdline | |||||
| import ( | import ( | ||||
| "fmt" | "fmt" | ||||
| @@ -7,9 +7,9 @@ import ( | |||||
| "github.com/spf13/cobra" | "github.com/spf13/cobra" | ||||
| "gitlink.org.cn/cloudream/common/pkgs/logger" | "gitlink.org.cn/cloudream/common/pkgs/logger" | ||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/config" | |||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount" | |||||
| mntcfg "gitlink.org.cn/cloudream/storage2/client2/internal/mount/config" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/config" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount" | |||||
| mntcfg "gitlink.org.cn/cloudream/storage2/client/internal/mount/config" | |||||
| stgglb "gitlink.org.cn/cloudream/storage2/common/globals" | stgglb "gitlink.org.cn/cloudream/storage2/common/globals" | ||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/connectivity" | "gitlink.org.cn/cloudream/storage2/common/pkgs/connectivity" | ||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | ||||
| @@ -119,18 +119,3 @@ func mountCmd(mountPoint string, configPath string) { | |||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| func serveDistLock(svc *distlock.Service) { | |||||
| logger.Info("start serving distlock") | |||||
| err := svc.Serve() | |||||
| if err != nil { | |||||
| logger.Errorf("distlock stopped with error: %s", err.Error()) | |||||
| } | |||||
| logger.Info("distlock stopped") | |||||
| // TODO 仅简单结束了程序 | |||||
| os.Exit(1) | |||||
| } | |||||
| @@ -50,7 +50,7 @@ func init() { | |||||
| }, | }, | ||||
| } | } | ||||
| rootCmd.AddCommand(cmd) | |||||
| RootCmd.AddCommand(cmd) | |||||
| } | } | ||||
| func newloadp(cmdCtx *CommandContext, path string, bucketID cdssdk.BucketID, packageName string, storageIDs []cdssdk.StorageID, rootPathes []string) { | func newloadp(cmdCtx *CommandContext, path string, bucketID cdssdk.BucketID, packageName string, storageIDs []cdssdk.StorageID, rootPathes []string) { | ||||
| @@ -118,5 +118,5 @@ func init() { | |||||
| } | } | ||||
| cmd.Flags().Int64VarP(&stgID, "storage", "s", 0, "storage affinity") | cmd.Flags().Int64VarP(&stgID, "storage", "s", 0, "storage affinity") | ||||
| rootCmd.AddCommand(cmd) | |||||
| RootCmd.AddCommand(cmd) | |||||
| } | } | ||||
| @@ -2,43 +2,197 @@ package cmdline | |||||
| import ( | import ( | ||||
| "fmt" | "fmt" | ||||
| "os" | |||||
| "time" | |||||
| "github.com/spf13/cobra" | |||||
| "gitlink.org.cn/cloudream/common/pkgs/logger" | |||||
| cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/config" | "gitlink.org.cn/cloudream/storage2/client/internal/config" | ||||
| "gitlink.org.cn/cloudream/storage2/client/internal/http" | "gitlink.org.cn/cloudream/storage2/client/internal/http" | ||||
| "gitlink.org.cn/cloudream/storage2/client/internal/services" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/task" | |||||
| stgglb "gitlink.org.cn/cloudream/storage2/common/globals" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/accessstat" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/connectivity" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/distlock" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader/strategy" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/metacache" | |||||
| coormq "gitlink.org.cn/cloudream/storage2/common/pkgs/mq/coordinator" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/storage/agtpool" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/uploader" | |||||
| ) | ) | ||||
| // ServeHTTP 启动HTTP服务。 | |||||
| // ctx: 命令行上下文,包含服务配置等信息。 | |||||
| // args: 命令行参数,第一个参数可选地指定HTTP服务器监听地址。 | |||||
| // 返回值: 如果启动过程中遇到错误,返回错误信息;否则返回nil。 | |||||
| func ServeHTTP(ctx CommandContext, args []string) error { | |||||
| // 初始化函数,将ServeHTTP命令注册到命令列表中。 | |||||
| func init() { | |||||
| var configPath, listenAddr string | |||||
| cmd := cobra.Command{ | |||||
| Use: "serve", | |||||
| Short: "start serving storage service", | |||||
| Run: func(cmd *cobra.Command, args []string) { | |||||
| serveHTTP(configPath, listenAddr) | |||||
| }, | |||||
| } | |||||
| cmd.Flags().StringVarP(&configPath, "config", "c", "", "config file path") | |||||
| cmd.Flags().StringVarP(&listenAddr, "listen", "l", "", "listen address") | |||||
| RootCmd.AddCommand(&cmd) | |||||
| } | |||||
| func serveHTTP(configPath string, listenAddr string) { | |||||
| err := config.Init(configPath) | |||||
| if err != nil { | |||||
| fmt.Printf("init config failed, err: %s", err.Error()) | |||||
| os.Exit(1) | |||||
| } | |||||
| err = logger.Init(&config.Cfg().Logger) | |||||
| if err != nil { | |||||
| fmt.Printf("init logger failed, err: %s", err.Error()) | |||||
| os.Exit(1) | |||||
| } | |||||
| stgglb.InitLocal(&config.Cfg().Local) | |||||
| stgglb.InitMQPool(config.Cfg().RabbitMQ) | |||||
| stgglb.InitAgentRPCPool(&config.Cfg().AgentGRPC) | |||||
| // 连接性信息收集 | |||||
| var conCol connectivity.Collector | |||||
| if config.Cfg().Local.HubID != nil { | |||||
| //如果client与某个hub处于同一台机器,则使用这个hub的连通性信息 | |||||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||||
| if err != nil { | |||||
| logger.Warnf("acquire coordinator mq failed, err: %s", err.Error()) | |||||
| os.Exit(1) | |||||
| } | |||||
| getCons, err := coorCli.GetHubConnectivities(coormq.ReqGetHubConnectivities([]cdssdk.HubID{*config.Cfg().Local.HubID})) | |||||
| if err != nil { | |||||
| logger.Warnf("get hub connectivities failed, err: %s", err.Error()) | |||||
| os.Exit(1) | |||||
| } | |||||
| consMap := make(map[cdssdk.HubID]connectivity.Connectivity) | |||||
| for _, con := range getCons.Connectivities { | |||||
| var delay *time.Duration | |||||
| if con.Latency != nil { | |||||
| d := time.Duration(*con.Latency * float32(time.Millisecond)) | |||||
| delay = &d | |||||
| } | |||||
| consMap[con.FromHubID] = connectivity.Connectivity{ | |||||
| ToHubID: con.ToHubID, | |||||
| Latency: delay, | |||||
| } | |||||
| } | |||||
| conCol = connectivity.NewCollectorWithInitData(&config.Cfg().Connectivity, nil, consMap) | |||||
| logger.Info("use local hub connectivities") | |||||
| } else { | |||||
| // 否则需要就地收集连通性信息 | |||||
| conCol = connectivity.NewCollector(&config.Cfg().Connectivity, nil) | |||||
| conCol.CollectInPlace() | |||||
| } | |||||
| metaCacheHost := metacache.NewHost() | |||||
| go metaCacheHost.Serve() | |||||
| stgMeta := metaCacheHost.AddStorageMeta() | |||||
| hubMeta := metaCacheHost.AddHubMeta() | |||||
| conMeta := metaCacheHost.AddConnectivity() | |||||
| // 分布式锁 | |||||
| distlockSvc, err := distlock.NewService(&config.Cfg().DistLock) | |||||
| if err != nil { | |||||
| logger.Warnf("new distlock service failed, err: %s", err.Error()) | |||||
| os.Exit(1) | |||||
| } | |||||
| go serveDistLock(distlockSvc) | |||||
| // 访问统计 | |||||
| acStat := accessstat.NewAccessStat(accessstat.Config{ | |||||
| // TODO 考虑放到配置里 | |||||
| ReportInterval: time.Second * 10, | |||||
| }) | |||||
| go serveAccessStat(acStat) | |||||
| // 存储管理器 | |||||
| stgAgts := agtpool.NewPool() | |||||
| // 任务管理器 | |||||
| taskMgr := task.NewManager(distlockSvc, &conCol, stgAgts) | |||||
| strgSel := strategy.NewSelector(config.Cfg().DownloadStrategy, stgMeta, hubMeta, conMeta) | |||||
| // 下载器 | |||||
| dlder := downloader.NewDownloader(config.Cfg().Downloader, &conCol, stgAgts, strgSel) | |||||
| // 上传器 | |||||
| uploader := uploader.NewUploader(distlockSvc, &conCol, stgAgts, stgMeta) | |||||
| svc, err := services.NewService(distlockSvc, &taskMgr, &dlder, acStat, uploader, strgSel, stgMeta) | |||||
| if err != nil { | |||||
| logger.Warnf("new services failed, err: %s", err.Error()) | |||||
| os.Exit(1) | |||||
| } | |||||
| // 默认监听地址为":7890",如果提供了命令行参数,则使用参数指定的地址。 | // 默认监听地址为":7890",如果提供了命令行参数,则使用参数指定的地址。 | ||||
| listenAddr := ":7890" | |||||
| if len(args) > 0 { | |||||
| listenAddr = args[0] | |||||
| if listenAddr == "" { | |||||
| listenAddr = ":7890" | |||||
| } | } | ||||
| awsAuth, err := http.NewAWSAuth(config.Cfg().AuthAccessKey, config.Cfg().AuthSecretKey) | awsAuth, err := http.NewAWSAuth(config.Cfg().AuthAccessKey, config.Cfg().AuthSecretKey) | ||||
| if err != nil { | if err != nil { | ||||
| return fmt.Errorf("new aws auth: %w", err) | |||||
| logger.Warnf("new aws auth: %v", err) | |||||
| os.Exit(1) | |||||
| } | } | ||||
| // 创建一个新的HTTP服务器实例。 | // 创建一个新的HTTP服务器实例。 | ||||
| httpSvr, err := http.NewServer(listenAddr, ctx.Cmdline.Svc, awsAuth) | |||||
| httpSvr, err := http.NewServer(listenAddr, svc, awsAuth) | |||||
| if err != nil { | if err != nil { | ||||
| return fmt.Errorf("new http server: %w", err) | |||||
| logger.Warnf("new http server: %v", err) | |||||
| os.Exit(1) | |||||
| } | } | ||||
| // 启动HTTP服务。 | // 启动HTTP服务。 | ||||
| err = httpSvr.Serve() | err = httpSvr.Serve() | ||||
| if err != nil { | if err != nil { | ||||
| return fmt.Errorf("serving http: %w", err) | |||||
| logger.Warnf("serving http: %v", err) | |||||
| os.Exit(1) | |||||
| } | |||||
| } | |||||
| func serveDistLock(svc *distlock.Service) { | |||||
| logger.Info("start serving distlock") | |||||
| err := svc.Serve() | |||||
| if err != nil { | |||||
| logger.Errorf("distlock stopped with error: %s", err.Error()) | |||||
| } | } | ||||
| return nil | |||||
| logger.Info("distlock stopped") | |||||
| // TODO 仅简单结束了程序 | |||||
| os.Exit(1) | |||||
| } | } | ||||
| // 初始化函数,将ServeHTTP命令注册到命令列表中。 | |||||
| func init() { | |||||
| commands.MustAdd(ServeHTTP, "serve", "http") | |||||
| func serveAccessStat(svc *accessstat.AccessStat) { | |||||
| logger.Info("start serving access stat") | |||||
| ch := svc.Start() | |||||
| loop: | |||||
| for { | |||||
| val, err := ch.Receive() | |||||
| if err != nil { | |||||
| logger.Errorf("access stat stopped with error: %v", err) | |||||
| break | |||||
| } | |||||
| switch val := val.(type) { | |||||
| case error: | |||||
| logger.Errorf("access stat stopped with error: %v", val) | |||||
| break loop | |||||
| } | |||||
| } | |||||
| logger.Info("access stat stopped") | |||||
| // TODO 仅简单结束了程序 | |||||
| os.Exit(1) | |||||
| } | } | ||||
| @@ -16,7 +16,7 @@ func init() { | |||||
| Use: "sysevent", | Use: "sysevent", | ||||
| } | } | ||||
| rootCmd.AddCommand(cmd) | |||||
| RootCmd.AddCommand(cmd) | |||||
| outputJSON := false | outputJSON := false | ||||
| watchCmd := &cobra.Command{ | watchCmd := &cobra.Command{ | ||||
| @@ -17,7 +17,7 @@ import ( | |||||
| ) | ) | ||||
| func init() { | func init() { | ||||
| rootCmd.AddCommand(&cobra.Command{ | |||||
| RootCmd.AddCommand(&cobra.Command{ | |||||
| Use: "test", | Use: "test", | ||||
| Short: "test", | Short: "test", | ||||
| Run: func(cmd *cobra.Command, args []string) { | Run: func(cmd *cobra.Command, args []string) { | ||||
| @@ -72,7 +72,7 @@ func init() { | |||||
| }, | }, | ||||
| }) | }) | ||||
| rootCmd.AddCommand(&cobra.Command{ | |||||
| RootCmd.AddCommand(&cobra.Command{ | |||||
| Use: "test32", | Use: "test32", | ||||
| Short: "test32", | Short: "test32", | ||||
| Run: func(cmd *cobra.Command, args []string) { | Run: func(cmd *cobra.Command, args []string) { | ||||
| @@ -118,7 +118,7 @@ func init() { | |||||
| fut.Wait(context.TODO()) | fut.Wait(context.TODO()) | ||||
| }, | }, | ||||
| }) | }) | ||||
| rootCmd.AddCommand(&cobra.Command{ | |||||
| RootCmd.AddCommand(&cobra.Command{ | |||||
| Use: "test1", | Use: "test1", | ||||
| Short: "test1", | Short: "test1", | ||||
| Run: func(cmd *cobra.Command, args []string) { | Run: func(cmd *cobra.Command, args []string) { | ||||
| @@ -188,7 +188,7 @@ func init() { | |||||
| }, | }, | ||||
| }) | }) | ||||
| rootCmd.AddCommand(&cobra.Command{ | |||||
| RootCmd.AddCommand(&cobra.Command{ | |||||
| Use: "test4", | Use: "test4", | ||||
| Short: "test4", | Short: "test4", | ||||
| Run: func(cmd *cobra.Command, args []string) { | Run: func(cmd *cobra.Command, args []string) { | ||||
| @@ -238,7 +238,7 @@ func init() { | |||||
| }, | }, | ||||
| }) | }) | ||||
| rootCmd.AddCommand(&cobra.Command{ | |||||
| RootCmd.AddCommand(&cobra.Command{ | |||||
| Use: "test11", | Use: "test11", | ||||
| Short: "test11", | Short: "test11", | ||||
| Run: func(cmd *cobra.Command, args []string) { | Run: func(cmd *cobra.Command, args []string) { | ||||
| @@ -8,6 +8,7 @@ import ( | |||||
| "gitlink.org.cn/cloudream/common/utils/config" | "gitlink.org.cn/cloudream/common/utils/config" | ||||
| stgmodels "gitlink.org.cn/cloudream/storage2/common/models" | stgmodels "gitlink.org.cn/cloudream/storage2/common/models" | ||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/connectivity" | "gitlink.org.cn/cloudream/storage2/common/pkgs/connectivity" | ||||
| db "gitlink.org.cn/cloudream/storage2/common/pkgs/db2/config" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" | "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" | ||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader/strategy" | "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader/strategy" | ||||
| agtrpc "gitlink.org.cn/cloudream/storage2/common/pkgs/grpc/agent" | agtrpc "gitlink.org.cn/cloudream/storage2/common/pkgs/grpc/agent" | ||||
| @@ -26,14 +27,18 @@ type Config struct { | |||||
| AuthAccessKey string `json:"authAccessKey"` // TODO 临时办法 | AuthAccessKey string `json:"authAccessKey"` // TODO 临时办法 | ||||
| AuthSecretKey string `json:"authSecretKey"` | AuthSecretKey string `json:"authSecretKey"` | ||||
| MaxHTTPBodySize int64 `json:"maxHttpBodySize"` | MaxHTTPBodySize int64 `json:"maxHttpBodySize"` | ||||
| DB db.Config `json:"db"` | |||||
| } | } | ||||
| var cfg Config | var cfg Config | ||||
| // Init 初始化client | // Init 初始化client | ||||
| // TODO 这里的modeulName参数弄成可配置的更好 | // TODO 这里的modeulName参数弄成可配置的更好 | ||||
| func Init() error { | |||||
| return config.DefaultLoad("client", &cfg) | |||||
| func Init(configPath string) error { | |||||
| if configPath == "" { | |||||
| return config.DefaultLoad("client", &cfg) | |||||
| } | |||||
| return config.Load(configPath, &cfg) | |||||
| } | } | ||||
| func Cfg() *Config { | func Cfg() *Config { | ||||
| @@ -8,7 +8,7 @@ import ( | |||||
| fusefs "github.com/hanwen/go-fuse/v2/fs" | fusefs "github.com/hanwen/go-fuse/v2/fs" | ||||
| "github.com/hanwen/go-fuse/v2/fuse" | "github.com/hanwen/go-fuse/v2/fuse" | ||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/config" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/config" | |||||
| ) | ) | ||||
| type Fuse struct { | type Fuse struct { | ||||
| @@ -7,9 +7,9 @@ import ( | |||||
| "github.com/hanwen/go-fuse/v2/fuse" | "github.com/hanwen/go-fuse/v2/fuse" | ||||
| cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | ||||
| "gitlink.org.cn/cloudream/common/utils/sync2" | "gitlink.org.cn/cloudream/common/utils/sync2" | ||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/config" | |||||
| fuse2 "gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/vfs" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/config" | |||||
| fuse2 "gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/vfs" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | ||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" | "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" | ||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/uploader" | "gitlink.org.cn/cloudream/storage2/common/pkgs/uploader" | ||||
| @@ -7,7 +7,7 @@ import ( | |||||
| cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | ||||
| "gitlink.org.cn/cloudream/common/utils/sync2" | "gitlink.org.cn/cloudream/common/utils/sync2" | ||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/config" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/config" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | ||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" | "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" | ||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/uploader" | "gitlink.org.cn/cloudream/storage2/common/pkgs/uploader" | ||||
| @@ -16,8 +16,8 @@ import ( | |||||
| cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | ||||
| "gitlink.org.cn/cloudream/common/utils/io2" | "gitlink.org.cn/cloudream/common/utils/io2" | ||||
| "gitlink.org.cn/cloudream/common/utils/lo2" | "gitlink.org.cn/cloudream/common/utils/lo2" | ||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/config" | |||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/config" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | ||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" | "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" | ||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/uploader" | "gitlink.org.cn/cloudream/storage2/common/pkgs/uploader" | ||||
| @@ -13,7 +13,7 @@ import ( | |||||
| "gitlink.org.cn/cloudream/common/utils/lo2" | "gitlink.org.cn/cloudream/common/utils/lo2" | ||||
| "gitlink.org.cn/cloudream/common/utils/math2" | "gitlink.org.cn/cloudream/common/utils/math2" | ||||
| "gitlink.org.cn/cloudream/common/utils/serder" | "gitlink.org.cn/cloudream/common/utils/serder" | ||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse" | |||||
| ) | ) | ||||
| type FileInfo struct { | type FileInfo struct { | ||||
| @@ -1,6 +1,6 @@ | |||||
| package vfs | package vfs | ||||
| import "gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse" | |||||
| import "gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse" | |||||
| type FuseDirReader struct { | type FuseDirReader struct { | ||||
| allEntries []fuse.FsEntry | allEntries []fuse.FsEntry | ||||
| @@ -7,8 +7,8 @@ import ( | |||||
| "time" | "time" | ||||
| cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | ||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/vfs/cache" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/vfs/cache" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | ||||
| "gorm.io/gorm" | "gorm.io/gorm" | ||||
| ) | ) | ||||
| @@ -8,8 +8,8 @@ import ( | |||||
| cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | ||||
| "gitlink.org.cn/cloudream/common/utils/lo2" | "gitlink.org.cn/cloudream/common/utils/lo2" | ||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/vfs/cache" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/vfs/cache" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | ||||
| "gorm.io/gorm" | "gorm.io/gorm" | ||||
| ) | ) | ||||
| @@ -5,8 +5,8 @@ import ( | |||||
| "time" | "time" | ||||
| cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | ||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/vfs/cache" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/vfs/cache" | |||||
| "gorm.io/gorm" | "gorm.io/gorm" | ||||
| ) | ) | ||||
| @@ -8,8 +8,8 @@ import ( | |||||
| cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | ||||
| "gitlink.org.cn/cloudream/common/utils/lo2" | "gitlink.org.cn/cloudream/common/utils/lo2" | ||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/vfs/cache" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/vfs/cache" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | ||||
| "gorm.io/gorm" | "gorm.io/gorm" | ||||
| ) | ) | ||||
| @@ -6,8 +6,8 @@ import ( | |||||
| "time" | "time" | ||||
| cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | ||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/vfs/cache" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/vfs/cache" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | ||||
| "gorm.io/gorm" | "gorm.io/gorm" | ||||
| ) | ) | ||||
| @@ -1,9 +1,9 @@ | |||||
| package vfs | package vfs | ||||
| import ( | import ( | ||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/config" | |||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client2/internal/mount/vfs/cache" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/config" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/fuse" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/mount/vfs/cache" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | "gitlink.org.cn/cloudream/storage2/common/pkgs/db2" | ||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" | "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" | ||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/uploader" | "gitlink.org.cn/cloudream/storage2/common/pkgs/uploader" | ||||
| @@ -1,167 +1,11 @@ | |||||
| package main | package main | ||||
| import ( | import ( | ||||
| "fmt" | |||||
| "os" | |||||
| "time" | |||||
| _ "google.golang.org/grpc/balancer/grpclb" | _ "google.golang.org/grpc/balancer/grpclb" | ||||
| "gitlink.org.cn/cloudream/common/pkgs/logger" | |||||
| cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/cmdline" | "gitlink.org.cn/cloudream/storage2/client/internal/cmdline" | ||||
| "gitlink.org.cn/cloudream/storage2/client/internal/config" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/services" | |||||
| "gitlink.org.cn/cloudream/storage2/client/internal/task" | |||||
| stgglb "gitlink.org.cn/cloudream/storage2/common/globals" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/accessstat" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/connectivity" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/distlock" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader/strategy" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/metacache" | |||||
| coormq "gitlink.org.cn/cloudream/storage2/common/pkgs/mq/coordinator" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/storage/agtpool" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/uploader" | |||||
| ) | ) | ||||
| func main() { | func main() { | ||||
| err := config.Init() | |||||
| if err != nil { | |||||
| fmt.Printf("init config failed, err: %s", err.Error()) | |||||
| os.Exit(1) | |||||
| } | |||||
| err = logger.Init(&config.Cfg().Logger) | |||||
| if err != nil { | |||||
| fmt.Printf("init logger failed, err: %s", err.Error()) | |||||
| os.Exit(1) | |||||
| } | |||||
| stgglb.InitLocal(&config.Cfg().Local) | |||||
| stgglb.InitMQPool(config.Cfg().RabbitMQ) | |||||
| stgglb.InitAgentRPCPool(&config.Cfg().AgentGRPC) | |||||
| // 连接性信息收集 | |||||
| var conCol connectivity.Collector | |||||
| if config.Cfg().Local.HubID != nil { | |||||
| //如果client与某个hub处于同一台机器,则使用这个hub的连通性信息 | |||||
| coorCli, err := stgglb.CoordinatorMQPool.Acquire() | |||||
| if err != nil { | |||||
| logger.Warnf("acquire coordinator mq failed, err: %s", err.Error()) | |||||
| os.Exit(1) | |||||
| } | |||||
| getCons, err := coorCli.GetHubConnectivities(coormq.ReqGetHubConnectivities([]cdssdk.HubID{*config.Cfg().Local.HubID})) | |||||
| if err != nil { | |||||
| logger.Warnf("get hub connectivities failed, err: %s", err.Error()) | |||||
| os.Exit(1) | |||||
| } | |||||
| consMap := make(map[cdssdk.HubID]connectivity.Connectivity) | |||||
| for _, con := range getCons.Connectivities { | |||||
| var delay *time.Duration | |||||
| if con.Latency != nil { | |||||
| d := time.Duration(*con.Latency * float32(time.Millisecond)) | |||||
| delay = &d | |||||
| } | |||||
| consMap[con.FromHubID] = connectivity.Connectivity{ | |||||
| ToHubID: con.ToHubID, | |||||
| Latency: delay, | |||||
| } | |||||
| } | |||||
| conCol = connectivity.NewCollectorWithInitData(&config.Cfg().Connectivity, nil, consMap) | |||||
| logger.Info("use local hub connectivities") | |||||
| } else { | |||||
| // 否则需要就地收集连通性信息 | |||||
| conCol = connectivity.NewCollector(&config.Cfg().Connectivity, nil) | |||||
| conCol.CollectInPlace() | |||||
| } | |||||
| metaCacheHost := metacache.NewHost() | |||||
| go metaCacheHost.Serve() | |||||
| stgMeta := metaCacheHost.AddStorageMeta() | |||||
| hubMeta := metaCacheHost.AddHubMeta() | |||||
| conMeta := metaCacheHost.AddConnectivity() | |||||
| // 分布式锁 | |||||
| distlockSvc, err := distlock.NewService(&config.Cfg().DistLock) | |||||
| if err != nil { | |||||
| logger.Warnf("new distlock service failed, err: %s", err.Error()) | |||||
| os.Exit(1) | |||||
| } | |||||
| go serveDistLock(distlockSvc) | |||||
| // 访问统计 | |||||
| acStat := accessstat.NewAccessStat(accessstat.Config{ | |||||
| // TODO 考虑放到配置里 | |||||
| ReportInterval: time.Second * 10, | |||||
| }) | |||||
| go serveAccessStat(acStat) | |||||
| // 存储管理器 | |||||
| stgAgts := agtpool.NewPool() | |||||
| // 任务管理器 | |||||
| taskMgr := task.NewManager(distlockSvc, &conCol, stgAgts) | |||||
| strgSel := strategy.NewSelector(config.Cfg().DownloadStrategy, stgMeta, hubMeta, conMeta) | |||||
| // 下载器 | |||||
| dlder := downloader.NewDownloader(config.Cfg().Downloader, &conCol, stgAgts, strgSel) | |||||
| // 上传器 | |||||
| uploader := uploader.NewUploader(distlockSvc, &conCol, stgAgts, stgMeta) | |||||
| svc, err := services.NewService(distlockSvc, &taskMgr, &dlder, acStat, uploader, strgSel, stgMeta) | |||||
| if err != nil { | |||||
| logger.Warnf("new services failed, err: %s", err.Error()) | |||||
| os.Exit(1) | |||||
| } | |||||
| cmds, err := cmdline.NewCommandline(svc) | |||||
| if err != nil { | |||||
| logger.Warnf("new command line failed, err: %s", err.Error()) | |||||
| os.Exit(1) | |||||
| } | |||||
| cmds.DispatchCommand(os.Args[1:]) | |||||
| } | |||||
| func serveDistLock(svc *distlock.Service) { | |||||
| logger.Info("start serving distlock") | |||||
| err := svc.Serve() | |||||
| if err != nil { | |||||
| logger.Errorf("distlock stopped with error: %s", err.Error()) | |||||
| } | |||||
| logger.Info("distlock stopped") | |||||
| // TODO 仅简单结束了程序 | |||||
| os.Exit(1) | |||||
| } | |||||
| func serveAccessStat(svc *accessstat.AccessStat) { | |||||
| logger.Info("start serving access stat") | |||||
| ch := svc.Start() | |||||
| loop: | |||||
| for { | |||||
| val, err := ch.Receive() | |||||
| if err != nil { | |||||
| logger.Errorf("access stat stopped with error: %v", err) | |||||
| break | |||||
| } | |||||
| switch val := val.(type) { | |||||
| case error: | |||||
| logger.Errorf("access stat stopped with error: %v", val) | |||||
| break loop | |||||
| } | |||||
| } | |||||
| logger.Info("access stat stopped") | |||||
| // TODO 仅简单结束了程序 | |||||
| os.Exit(1) | |||||
| cmdline.RootCmd.Execute() | |||||
| } | } | ||||
| @@ -1,5 +0,0 @@ | |||||
| package cmd | |||||
| import "github.com/spf13/cobra" | |||||
| var RootCmd = cobra.Command{} | |||||
| @@ -1,43 +0,0 @@ | |||||
| package config | |||||
| import ( | |||||
| "gitlink.org.cn/cloudream/common/pkgs/distlock" | |||||
| log "gitlink.org.cn/cloudream/common/pkgs/logger" | |||||
| "gitlink.org.cn/cloudream/common/pkgs/mq" | |||||
| cdssdk "gitlink.org.cn/cloudream/common/sdks/storage" | |||||
| c "gitlink.org.cn/cloudream/common/utils/config" | |||||
| stgmodels "gitlink.org.cn/cloudream/storage2/common/models" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/connectivity" | |||||
| db "gitlink.org.cn/cloudream/storage2/common/pkgs/db2/config" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/downloader/strategy" | |||||
| "gitlink.org.cn/cloudream/storage2/common/pkgs/grpc" | |||||
| ) | |||||
| type Config struct { | |||||
| ID cdssdk.HubID `json:"id"` | |||||
| ListenAddr string `json:"listenAddr"` | |||||
| Local stgmodels.LocalMachineInfo `json:"local"` | |||||
| GRPC *grpc.Config `json:"grpc"` | |||||
| Logger log.Config `json:"logger"` | |||||
| RabbitMQ mq.Config `json:"rabbitMQ"` | |||||
| DistLock distlock.Config `json:"distlock"` | |||||
| Connectivity connectivity.Config `json:"connectivity"` | |||||
| Downloader downloader.Config `json:"downloader"` | |||||
| DownloadStrategy strategy.Config `json:"downloadStrategy"` | |||||
| DB db.Config `json:"db"` | |||||
| } | |||||
| var cfg Config | |||||
| func Init(path string) error { | |||||
| if path == "" { | |||||
| return c.DefaultLoad("client2", &cfg) | |||||
| } | |||||
| return c.Load(path, &cfg) | |||||
| } | |||||
| func Cfg() *Config { | |||||
| return &cfg | |||||
| } | |||||
| @@ -1,7 +0,0 @@ | |||||
| package main | |||||
| import "gitlink.org.cn/cloudream/storage2/client2/internal/cmd" | |||||
| func main() { | |||||
| cmd.RootCmd.Execute() | |||||
| } | |||||
| @@ -117,15 +117,6 @@ func Client() error { | |||||
| }) | }) | ||||
| } | } | ||||
| func Client2() error { | |||||
| return magefiles.Build(magefiles.BuildArgs{ | |||||
| OutputName: "client2", | |||||
| OutputDir: "client2", | |||||
| AssetsDir: "assets", | |||||
| EntryFile: "client2/main.go", | |||||
| }) | |||||
| } | |||||
| func Coordinator() error { | func Coordinator() error { | ||||
| return magefiles.Build(magefiles.BuildArgs{ | return magefiles.Build(magefiles.BuildArgs{ | ||||
| OutputName: "coordinator", | OutputName: "coordinator", | ||||